From 4b4c9c902c5774c64806982676fa5cd22ab2206f Mon Sep 17 00:00:00 2001 From: Joshua Park Date: Tue, 11 Aug 2026 13:52:37 -0700 Subject: [PATCH 1/9] 1 --- benchmark/linear_attention/Dockerfile | 6 +- benchmark/linear_attention/README.md | 40 +- .../benchmark_single_linear_attention.py | 271 +- benchmark/linear_attention/plot_results.py | 31 +- python/cudnn/_pygraph.py | 43 +- python/cudnn/engines/manifest.py | 3 + python/cudnn/frost/buffers.py | 32 +- python/cudnn/frost/tile_dsl/barrier.py | 31 +- python/cudnn/frost/tile_dsl/mma.py | 49 + python/cudnn/frost/tile_dsl/swizzle.py | 6 +- .../linear_attention/cutile/gdn_engine.py | 318 +- .../linear_attention/cutile/kda_engine.py | 387 +- .../linear_attention/cutile/kernels/common.py | 66 +- .../cutile/kernels/gdn_chunk_cutile.py | 463 +- .../cutile/kernels/kda_chunk_cutile.py | 853 ++-- python/cudnn/linear_attention/engine_utils.py | 134 - .../cudnn/linear_attention/frost/__init__.py | 12 +- .../linear_attention/frost/common/downcast.py | 98 + .../frost/common/head_reduce.py | 79 +- .../linear_attention/frost/common/host.py | 26 + .../linear_attention/frost/common/split_k.py | 188 +- .../linear_attention/frost/common/thd.py | 285 +- .../linear_attention/frost/gdn2_engine.py | 472 +- .../linear_attention/frost/gdn_engine.py | 568 ++- .../linear_attention/frost/kda_engine.py | 490 +- .../frost/kernel/gdn2_bprop_config.py | 57 +- .../frost/kernel/gdn2_bprop_f16.py | 4007 ++++++++++++++++- .../frost/kernel/gdn2_prefill_config.py | 2 +- .../frost/kernel/gdn2_prefill_f16.py | 1969 ++++---- .../frost/kernel/gdn2_recompute_config.py | 66 + .../frost/kernel/gdn2_recompute_f16.py | 2683 +++++++++++ .../frost/kernel/gdn_bprop_config.py | 5 +- .../frost/kernel/gdn_bprop_f16.py | 3884 +++++++--------- .../frost/kernel/gdn_prefill_config.py | 18 +- .../frost/kernel/gdn_prefill_f16.py | 2807 +++++------- .../frost/kernel/gdn_recompute_config.py | 71 + .../frost/kernel/gdn_recompute_f16.py | 2660 +++++++++++ .../frost/kernel/kda_bprop_config.py | 49 +- .../frost/kernel/kda_bprop_f16.py | 3849 +++++++++++++++- .../frost/kernel/kda_prefill_config.py | 2 +- .../frost/kernel/kda_prefill_f16.py | 1991 ++++---- .../frost/kernel/kda_recompute_config.py | 66 + .../frost/kernel/kda_recompute_f16.py | 2579 +++++++++++ .../cudnn/linear_attention/graph_analyzer.py | 363 ++ python/cudnn/linear_attention/ops/gdn.py | 446 +- python/cudnn/linear_attention/ops/gdn2.py | 710 ++- python/cudnn/linear_attention/ops/kda.py | 657 ++- test/python/linear_attention/common.py | 208 - test/python/linear_attention/conftest.py | 44 +- .../linear_attention/cutile/__init__.py | 2 - .../linear_attention/cutile/conftest.py | 32 - .../linear_attention/cutile/test_gdn_bprop.py | 272 -- .../linear_attention/cutile/test_gdn_fprop.py | 236 - .../linear_attention/cutile/test_kda_bprop.py | 273 -- .../linear_attention/cutile/test_kda_fprop.py | 236 - .../python/linear_attention/frost/__init__.py | 2 - .../python/linear_attention/frost/conftest.py | 23 - .../frost/examples/01_gdn_prefill.py | 106 + .../frost/examples/02_gdn_backward.py | 118 + .../frost/examples/03_kda_prefill.py | 115 + .../frost/examples/04_kda_backward.py | 109 + .../frost/examples/05_gdn2_prefill.py | 119 + .../frost/examples/06_gdn2_backward.py | 120 + .../frost/test_gdn2_bprop_kernel.py | 44 - .../frost/test_gdn2_prefill_kernel.py | 676 --- .../frost/test_gdn_bprop_kernel.py | 964 ---- .../frost/test_gdn_prefill_kernel.py | 644 --- .../frost/test_kda_bprop_kernel.py | 42 - .../frost/test_kda_prefill_kernel.py | 757 ---- test/python/linear_attention/ops/__init__.py | 2 - .../linear_attention/ops/test_gdn2_op.py | 136 - .../linear_attention/ops/test_gdn_op.py | 205 - .../linear_attention/ops/test_kda_op.py | 234 - test/python/linear_attention/reference_gdn.py | 38 +- .../python/linear_attention/reference_gdn2.py | 38 +- test/python/linear_attention/reference_kda.py | 42 +- test/python/linear_attention/test_la.py | 911 ++++ 77 files changed, 26880 insertions(+), 13760 deletions(-) delete mode 100644 python/cudnn/linear_attention/engine_utils.py create mode 100644 python/cudnn/linear_attention/frost/common/downcast.py create mode 100644 python/cudnn/linear_attention/frost/common/host.py create mode 100644 python/cudnn/linear_attention/frost/kernel/gdn2_recompute_config.py create mode 100644 python/cudnn/linear_attention/frost/kernel/gdn2_recompute_f16.py create mode 100644 python/cudnn/linear_attention/frost/kernel/gdn_recompute_config.py create mode 100644 python/cudnn/linear_attention/frost/kernel/gdn_recompute_f16.py create mode 100644 python/cudnn/linear_attention/frost/kernel/kda_recompute_config.py create mode 100644 python/cudnn/linear_attention/frost/kernel/kda_recompute_f16.py create mode 100644 python/cudnn/linear_attention/graph_analyzer.py delete mode 100644 test/python/linear_attention/common.py delete mode 100644 test/python/linear_attention/cutile/__init__.py delete mode 100644 test/python/linear_attention/cutile/conftest.py delete mode 100644 test/python/linear_attention/cutile/test_gdn_bprop.py delete mode 100644 test/python/linear_attention/cutile/test_gdn_fprop.py delete mode 100644 test/python/linear_attention/cutile/test_kda_bprop.py delete mode 100644 test/python/linear_attention/cutile/test_kda_fprop.py delete mode 100644 test/python/linear_attention/frost/__init__.py delete mode 100644 test/python/linear_attention/frost/conftest.py create mode 100644 test/python/linear_attention/frost/examples/01_gdn_prefill.py create mode 100644 test/python/linear_attention/frost/examples/02_gdn_backward.py create mode 100644 test/python/linear_attention/frost/examples/03_kda_prefill.py create mode 100644 test/python/linear_attention/frost/examples/04_kda_backward.py create mode 100644 test/python/linear_attention/frost/examples/05_gdn2_prefill.py create mode 100644 test/python/linear_attention/frost/examples/06_gdn2_backward.py delete mode 100644 test/python/linear_attention/frost/test_gdn2_bprop_kernel.py delete mode 100644 test/python/linear_attention/frost/test_gdn2_prefill_kernel.py delete mode 100644 test/python/linear_attention/frost/test_gdn_bprop_kernel.py delete mode 100644 test/python/linear_attention/frost/test_gdn_prefill_kernel.py delete mode 100644 test/python/linear_attention/frost/test_kda_bprop_kernel.py delete mode 100644 test/python/linear_attention/frost/test_kda_prefill_kernel.py delete mode 100644 test/python/linear_attention/ops/__init__.py delete mode 100644 test/python/linear_attention/ops/test_gdn2_op.py delete mode 100644 test/python/linear_attention/ops/test_gdn_op.py delete mode 100644 test/python/linear_attention/ops/test_kda_op.py create mode 100644 test/python/linear_attention/test_la.py diff --git a/benchmark/linear_attention/Dockerfile b/benchmark/linear_attention/Dockerfile index 16f4889b8..04edf7e1b 100644 --- a/benchmark/linear_attention/Dockerfile +++ b/benchmark/linear_attention/Dockerfile @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -FROM nvcr.io/nvidia/pytorch:26.03-py3 +FROM nvcr.io/nvidia/pytorch:26.07-py3 # Set working directory WORKDIR /workspace @@ -26,5 +26,9 @@ RUN pip install nvidia-cutlass-dsl[cu13]==4.7.0 apache-tvm-ffi flash-linear-atte RUN git clone https://github.com/QwenLM/FlashQLA.git RUN pip install -v /workspace/FlashQLA +# Install FlashKDA from source. +RUN git clone https://github.com/MoonshotAI/FlashKDA.git +RUN pip install -v /workspace/FlashKDA + # Install the chart dependencies for plot_results.py RUN pip install pandas matplotlib seaborn diff --git a/benchmark/linear_attention/README.md b/benchmark/linear_attention/README.md index 676dc6177..7c2dac8c2 100644 --- a/benchmark/linear_attention/README.md +++ b/benchmark/linear_attention/README.md @@ -2,7 +2,7 @@ ## Introduction -This directory contains benchmarking tools for linear attention operations (Gated DeltaNet and its variants) across various backends. The benchmarks target training use cases with support for forward and backward passes, grouped-value attention (GVA), and the per-sequence recurrent state ports (initial state in, final state out). +This directory contains benchmarking tools for linear attention operations (GDN/KDA/GDN-2) across various backends. The benchmarks target training use cases with support for forward and backward passes. ## Contents @@ -36,7 +36,7 @@ python benchmark_single_linear_attention.py \ --la_backend cudnn --variant kda --data_type bfloat16 \ --skip_ref --profile_pass bwd -# cuDNN Frontend (GDN-2, forward only) +# cuDNN Frontend (GDN-2, forward pass) python benchmark_single_linear_attention.py \ --batch_size 1 --seqlen 8192 \ --num_q_heads 16 --num_kv_heads 16 --head_dim 128 \ @@ -64,8 +64,14 @@ python benchmark_single_linear_attention.py \ --la_backend flash_qla --variant gdn --data_type bfloat16 \ --skip_ref --fwd_bwd -# Recurrent state ports: seed with an initial state and request the final -# state (its gradient feeds the backward pass) +# FlashKDA comparison point (kda variant only, forward only, bf16) +python benchmark_single_linear_attention.py \ + --batch_size 1 --seqlen 8192 \ + --num_q_heads 32 --num_kv_heads 32 --head_dim 128 \ + --la_backend flash_kda --variant kda --data_type bfloat16 \ + --skip_ref + +# Input initial state and dump state for every chunk python benchmark_single_linear_attention.py \ --batch_size 1 --seqlen 8192 \ --num_q_heads 8 --num_kv_heads 64 --head_dim 128 \ @@ -75,33 +81,15 @@ python benchmark_single_linear_attention.py \ Run `python benchmark_single_linear_attention.py --help` for all options. -Dropping `--skip_ref` validates the forward output against FLA (the same way the SDPA benchmark validates against FlashAttention 4). +The `kda` and `gdn2` variants fuse q/k L2 normalization in-kernel on every backend; `gdn` runs unfused. ## Supported Backends | Backend | Description | |---------|-------------| | `cudnn` | cuDNN (native, via the cuDNN Frontend torch custom ops) | -| `fla` | FLA (flash-linear-attention, Triton) | +| `fla` | FLA (flash-linear-attention, Triton; `gdn`, `kda`, and `gdn2`) | | `flash_qla` | FlashQLA (TileLang fused GDN kernels, `gdn` variant only) | +| `flash_kda` | FlashKDA (`kda` forward variant only) | -The cuDNN backend routes through the pygraph engines: FROST (Cutlass DSL) on SM100-class devices, the cuTile engines elsewhere. Both passes run through autograd, exactly like a training step. - -## Supported Variants - -| Variant | Description | -|---------|-------------| -| `gdn` | Gated DeltaNet: scalar per-token decay and write strength | -| `kda` | Kimi Delta Attention: per-key-channel decay | -| `gdn2` | Gated DeltaNet v2: channel-wise decay/erase/write gates (forward only, cuDNN only) | - -The benchmark runs `kda` and `gdn2` with the in-kernel q/k L2 normalization off (`use_qk_l2norm_in_kernel=False`) on every backend, for an apples-to-apples comparison. - -Recent `fla` releases dispatch `chunk_gated_delta_rule` to FlashQLA whenever `flash_qla` is importable; the benchmark sets `FLA_DISABLE_BACKEND_DISPATCH=1` (unless already set) so the `fla` backend always measures FLA's own Triton kernels and the two backends stay distinct. - -## Notes - -- Head convention: `--num_q_heads` counts the query/key heads and `--num_kv_heads` counts the value heads; the gates, output, and recurrent state live at `max(num_q_heads, num_kv_heads)` heads. Both grouping directions are supported for `gdn`: grouped-value attention (`num_kv_heads > num_q_heads`, v-heads grouped over q-heads) and GQA (`num_q_heads > num_kv_heads`, q-heads grouped over v-heads, e.g. `--num_q_heads 64 --num_kv_heads 8`). The two counts must be equal or one a multiple of the other; `kda` and `gdn2` support the GVA direction only, and so does the `flash_qla` backend. -- The cuDNN ops use the THD (token-packed) layout internally; the benchmark expresses the dense batch as `cu_seqlens = [0, T, 2T, ...]`. -- `--initial_state` provides a per-sequence fp32 recurrent state (its gradient is produced in the backward pass); `--store_on` requests the per-sequence final state from the forward pass and feeds its gradient in the backward pass. Both are once-per-kernel I/O ports (one `[head_dim_qk, head_dim_vo]` tile per sequence per state head). -- Performance is measured with the torch profiler (device time of the matched kernels), with a 256 MB L2 flush before each timed iteration and the median reported. TFLOPS use the chunked-BMM FLOPs model documented in the script's `flops()`. +The cuDNN backend routes through the pygraph engines: FROST (Cutlass DSL) on SM100-class devices, the cuTile engines elsewhere. \ No newline at end of file diff --git a/benchmark/linear_attention/benchmark_single_linear_attention.py b/benchmark/linear_attention/benchmark_single_linear_attention.py index bd64dd231..f4926d9d6 100644 --- a/benchmark/linear_attention/benchmark_single_linear_attention.py +++ b/benchmark/linear_attention/benchmark_single_linear_attention.py @@ -36,9 +36,12 @@ "float16": 8192, } -# Chunk size of the chunked linear attention algorithms (both backends tile -# the sequence into 64-token chunks; the FLOPs model below depends on it). -_CHUNK_SIZE = 64 +# Chunk size of the chunked linear attention algorithms per variant (the +# FLOPs model below depends on it). +_CHUNK_SIZE = {"gdn": 64, "kda": 16, "gdn2": 16} + +# Safe-gate lower bound for kda. +_KDA_GATE_LOWER_BOUND = -5.0 def _peak_flops_per_clock_per_sm(dtype_str): @@ -177,13 +180,13 @@ def parse_args(): "--variant", default="gdn", type=str, - help="Linear attention variant to use. Can be 'gdn' (scalar decay), 'kda' (per-key-channel decay), or 'gdn2' (channel-wise decay/erase/write gates, forward only).", + help="Linear attention variant to use. Can be 'gdn' (scalar decay), 'kda' (per-key-channel decay), or 'gdn2' (channel-wise decay/erase/write gates).", choices=["gdn", "kda", "gdn2"], ) parser.add_argument( "--store_on", action="store_true", - help="Request the per-sequence final recurrent state from the forward pass (and feed its gradient in the backward pass)", + help="Dump the recurrent state every chunk plus the per-sequence final state from the forward pass (backends without a per-chunk state output are rejected; with backward, the final state's gradient feeds the backward pass)", ) parser.add_argument( "--initial_state", @@ -198,6 +201,7 @@ def parse_args(): choices=[ "fla", "flash_qla", + "flash_kda", "cudnn", ], ) @@ -250,13 +254,13 @@ def run_benchmark( head_dim_qk: Head dimension for Q/K (optional, for asymmetric) head_dim_vo: Head dimension for V/O (optional, for asymmetric) data_type: Data type ("bfloat16", "float16") - backend: Backend name ("cudnn", "fla", "flash_qla") + backend: Backend name ("cudnn", "fla", "flash_qla", "flash_kda") variant: Linear attention variant ("gdn", "kda", "gdn2") profile_pass: Which pass to profile ("fwd", "bwd", "both") num_iterations: Number of benchmark iterations num_warmup_iterations: Warmup iterations before measurement skip_ref: Skip reference validation - store_on: Request the final recurrent state from the forward pass + store_on: Dump per-chunk states plus the final recurrent state from the forward pass initial_state: Provide an initial recurrent state verbose: Print verbose output @@ -431,17 +435,43 @@ def run_benchmark( # The gates, output, and recurrent state live at HO = max(q, v) heads: # GVA groups v-heads over q-heads, GQA (gdn only) the reverse. num_o_heads = max(num_q_heads, num_kv_heads) - if num_q_heads > num_kv_heads and args.variant != "gdn": - raise ValueError("GQA (num_q_heads > num_kv_heads) is only supported with the 'gdn' variant") - if args.variant == "gdn2" and run_bwd: - raise ValueError("gdn2 is forward only (the backward kernel is a stub); use --profile_pass fwd") - if args.variant == "gdn2" and args.la_backend == "fla": - raise ValueError("gdn2 is only supported with the 'cudnn' backend") + if num_q_heads > num_kv_heads and args.variant != "gdn" and args.la_backend != "cudnn": + raise ValueError(f"GQA (num_q_heads > num_kv_heads) with kda/gdn2 is only supported by the cudnn backend, not {args.la_backend}") + if args.variant == "gdn2" and args.la_backend == "fla" and num_q_heads != num_kv_heads: + raise ValueError("gdn2 with the 'fla' backend requires equal q/kv head counts") if args.la_backend == "flash_qla": if args.variant != "gdn": raise ValueError("flash_qla only supports the 'gdn' variant") if num_q_heads > num_kv_heads: raise ValueError("flash_qla does not support GQA (num_q_heads > num_kv_heads)") + if args.la_backend == "flash_kda": + if args.variant != "kda": + raise ValueError("flash_kda only supports the 'kda' variant") + if num_q_heads != num_kv_heads: + raise ValueError("flash_kda requires equal q/kv head counts") + if head_dim_qk != 128 or head_dim_vo != 128: + raise ValueError("flash_kda requires head_dim 128 for both qk and vo") + if args.data_type != "bfloat16": + raise ValueError("flash_kda only supports bfloat16") + if run_bwd: + raise ValueError("flash_kda is forward only; use --profile_pass fwd") + + if args.store_on: + if args.la_backend in ("flash_kda", "flash_qla"): + raise ValueError(f"--store_on dumps the state every chunk; {args.la_backend} only outputs the final state") + if args.la_backend == "fla" and args.variant == "gdn": + raise ValueError("--store_on dumps the state every chunk; fla's chunk_gated_delta_rule has no intermediate-state output") + if args.la_backend == "fla" and run_bwd: + raise ValueError("--store_on dumps the state every chunk; fla dumps intermediate states in inference mode only, use --profile_pass fwd") + + # FlashKDA always fuses q/k l2norm in-kernel, so kda/gdn2 fuse it on + # every backend for apples-to-apples comparisons; gdn runs unfused. + use_qk_l2norm = args.variant in ("kda", "gdn2") + + # Forward-only kda runs feed raw safe-gate/beta logits to every backend + # (the FlashKDA ABI); training runs keep post-activation gates (the + # cudnn/fla backwards then differentiate the same math). + kda_raw_gates = args.variant == "kda" and not run_bwd l2_flush_size_mb = 256 l2_flush_size = l2_flush_size_mb * 1024 * 1024 @@ -472,9 +502,22 @@ def run_benchmark( ## boundaries. A dense batch is cu_seqlens = [0, T, 2T, ...]. cu_seqlens = torch.arange(0, batch_size + 1, dtype=torch.int32, device=device) * seqlen - def cudnn_linear_attention(query, key, value, gate, beta, write_gate, s0): + ## --store_on dumps the per-chunk state series alongside the final + ## state (the state_checkpoints output; one entry per kernel chunk). + ckpt_tokens = _CHUNK_SIZE[args.variant] if args.store_on else 0 + kda_safe_gate_kwargs = {} + if kda_raw_gates: + kda_safe_gate_kwargs = dict( + use_beta_sigmoid_in_kernel=True, + safe_gate=True, + gate_lower_bound=_KDA_GATE_LOWER_BOUND, + a_log=torch.zeros(num_o_heads, dtype=torch.float32, device=device), + dt_bias=torch.zeros(num_o_heads, head_dim_qk, dtype=torch.float32, device=device), + ) + + def cudnn_linear_attention(query, key, value, gate, beta, write_gate, state0): if args.variant == "gdn": - return gated_delta_net( + out = gated_delta_net( query, key, value, @@ -482,11 +525,13 @@ def cudnn_linear_attention(query, key, value, gate, beta, write_gate, s0): beta, cu_seqlens, scale=attn_scale, - initial_state=s0, + initial_state=state0, output_final_state=args.store_on, + use_qk_l2norm_in_kernel=use_qk_l2norm, + checkpoint_every_n_tokens=ckpt_tokens, ) elif args.variant == "kda": - return kimi_delta_attention( + out = kimi_delta_attention( query, key, value, @@ -494,12 +539,14 @@ def cudnn_linear_attention(query, key, value, gate, beta, write_gate, s0): beta, cu_seqlens, scale=attn_scale, - initial_state=s0, + initial_state=state0, output_final_state=args.store_on, - use_qk_l2norm_in_kernel=False, + use_qk_l2norm_in_kernel=use_qk_l2norm, + checkpoint_every_n_tokens=ckpt_tokens, + **kda_safe_gate_kwargs, ) else: # gdn2 - return gated_delta_net_v2( + out = gated_delta_net_v2( query, key, value, @@ -508,10 +555,12 @@ def cudnn_linear_attention(query, key, value, gate, beta, write_gate, s0): write_gate, cu_seqlens, scale=attn_scale, - initial_state=s0, + initial_state=state0, output_final_state=args.store_on, - use_qk_l2norm_in_kernel=False, + use_qk_l2norm_in_kernel=use_qk_l2norm, + checkpoint_every_n_tokens=ckpt_tokens, ) + return out[0], out[1] if args.la_backend == "flash_qla": attn_scale = head_dim_qk ** (-0.5) @@ -523,7 +572,7 @@ def cudnn_linear_attention(query, key, value, gate, beta, write_gate, s0): print(f"[INFO] FlashQLA Version: {getattr(flash_qla, '__version__', 'unknown')}") - def flash_qla_linear_attention(query, key, value, gate, beta, write_gate, s0): + def flash_qla_linear_attention(query, key, value, gate, beta, write_gate, state0): return fqla_chunk_gated_delta_rule( query, key, @@ -531,10 +580,39 @@ def flash_qla_linear_attention(query, key, value, gate, beta, write_gate, s0): gate, beta, scale=attn_scale, - initial_state=s0, + initial_state=state0, output_final_state=args.store_on, ) + if args.la_backend == "flash_kda": + attn_scale = head_dim_qk ** (-0.5) + + import flash_kda + + if args.verbose: + print(f"[INFO] FlashKDA Version: {getattr(flash_kda, '__version__', 'unknown')}") + + fkda_a_log = torch.zeros(num_o_heads, dtype=torch.float32, device=device) + fkda_dt_bias = torch.zeros(num_o_heads, head_dim_qk, dtype=torch.float32, device=device) + + def flash_kda_linear_attention(query, key, value, gate, beta, write_gate, state0): + out = torch.empty_like(value) + flash_kda.fwd( + query, + key, + value, + gate, + beta, + attn_scale, + out, + A_log=fkda_a_log, + dt_bias=fkda_dt_bias, + lower_bound=_KDA_GATE_LOWER_BOUND, + initial_state=state0, + final_state=None, + ) + return out, None + if args.la_backend == "fla" or (not args.skip_ref): attn_scale = head_dim_qk ** (-0.5) @@ -548,10 +626,26 @@ def flash_qla_linear_attention(query, key, value, gate, beta, write_gate, s0): except ImportError as e: raise RuntimeError(f"The installed fla does not provide KDA (fla.ops.kda): {e}") else: # gdn2 - raise ValueError("gdn2 is only supported with the 'cudnn' backend (no fla implementation, so no reference either); use --skip_ref") + try: + from fla.ops.gdn2 import chunk_gdn2 + except ImportError as e: + raise RuntimeError(f"The installed fla does not provide GDN2 (fla.ops.gdn2): {e}") + + ## FLA takes dense (B, T, H, D) tensors; g is the log-space decay + ## (raw safe-gate logits in forward-only kda runs). --store_on adds + ## the per-chunk state dump (fla supports it in inference mode only). + fla_kda_kwargs = {} + if args.variant == "kda" and kda_raw_gates: + fla_kda_kwargs = dict( + use_gate_in_kernel=True, + safe_gate=True, + lower_bound=_KDA_GATE_LOWER_BOUND, + use_beta_sigmoid_in_kernel=True, + A_log=torch.zeros(num_o_heads, dtype=torch.float32, device=device), + dt_bias=torch.zeros(num_o_heads * head_dim_qk, dtype=torch.float32, device=device), + ) - ## FLA takes dense (B, T, H, D) tensors; g is the log-space decay. - def fla_linear_attention(query, key, value, gate, beta, write_gate, s0): + def fla_linear_attention(query, key, value, gate, beta, write_gate, state0): if args.variant == "gdn": return chunk_gated_delta_rule( query, @@ -560,10 +654,27 @@ def fla_linear_attention(query, key, value, gate, beta, write_gate, s0): gate, beta, scale=attn_scale, - initial_state=s0, + initial_state=state0, output_final_state=args.store_on, + use_qk_l2norm_in_kernel=use_qk_l2norm, ) - else: # kda + elif args.variant == "kda": + if args.store_on: + with torch.inference_mode(): + o, fs, _state_checkpoints = chunk_kda( + query, + key, + value, + gate, + beta, + scale=attn_scale, + initial_state=state0, + output_final_state=True, + use_qk_l2norm_in_kernel=use_qk_l2norm, + return_intermediate_states=True, + **fla_kda_kwargs, + ) + return o, fs return chunk_kda( query, key, @@ -571,9 +682,39 @@ def fla_linear_attention(query, key, value, gate, beta, write_gate, s0): gate, beta, scale=attn_scale, - initial_state=s0, + initial_state=state0, + output_final_state=args.store_on, + use_qk_l2norm_in_kernel=use_qk_l2norm, + **fla_kda_kwargs, + ) + else: # gdn2 + if args.store_on: + with torch.inference_mode(): + o, fs, _state_checkpoints = chunk_gdn2( + query, + key, + value, + gate, + beta, + write_gate, + scale=attn_scale, + initial_state=state0, + output_final_state=True, + use_qk_l2norm_in_kernel=use_qk_l2norm, + return_intermediate_states=True, + ) + return o, fs + return chunk_gdn2( + query, + key, + value, + gate, + beta, + write_gate, + scale=attn_scale, + initial_state=state0, output_final_state=args.store_on, - use_qk_l2norm_in_kernel=False, + use_qk_l2norm_in_kernel=use_qk_l2norm, ) def get_linear_attention_function(backend): @@ -581,6 +722,8 @@ def get_linear_attention_function(backend): return fla_linear_attention elif backend == "flash_qla": return flash_qla_linear_attention + elif backend == "flash_kda": + return flash_kda_linear_attention elif backend == "cudnn": return cudnn_linear_attention else: @@ -595,20 +738,25 @@ def preprocess_qkv(query, key, value, backend): key.reshape(batch_size * seqlen, *key.shape[2:]), value.reshape(batch_size * seqlen, *value.shape[2:]), ) - elif backend in ("fla", "flash_qla"): + elif backend in ("fla", "flash_qla", "flash_kda"): return query, key, value else: raise ValueError(f"Invalid backend: {backend}") def preprocess_gates(gate, beta, write_gate, backend): if backend == "cudnn": + beta_t = beta.reshape(batch_size * seqlen, *beta.shape[2:]) + if kda_raw_gates: + beta_t = beta_t.to(target_dtype) return ( gate.reshape(batch_size * seqlen, *gate.shape[2:]), - beta.reshape(batch_size * seqlen, *beta.shape[2:]), + beta_t, write_gate.reshape(batch_size * seqlen, *write_gate.shape[2:]) if write_gate is not None else None, ) elif backend in ("fla", "flash_qla"): return gate, beta, write_gate + elif backend == "flash_kda": + return gate.to(target_dtype).contiguous(), beta.to(target_dtype).contiguous(), None else: raise ValueError(f"Invalid backend: {backend}") @@ -616,7 +764,7 @@ def preprocess_gates(gate, beta, write_gate, backend): def postprocess_o(output, backend): if backend == "cudnn": return output.reshape(batch_size, seqlen, num_o_heads, head_dim_vo) - elif backend in ("fla", "flash_qla"): + elif backend in ("fla", "flash_qla", "flash_kda"): return output else: raise ValueError(f"Invalid backend: {backend}") @@ -632,13 +780,13 @@ def flops( ): assert mode in ["fwd", "bwd", "fwd_bwd"] - # Chunked linear attention BMM FLOPs per 64-token chunk per (batch, + # Chunked linear attention BMM FLOPs per chunk per (batch, # state head), chunk size C, dims K (qk) and V (vo): # Forward: 5 BMM classes => # intra scores + WY prep (2 x C*C*K), WY apply (C*C*K + C*C*V), # intra output (C*C*V), inter output + state update (2 x C*K*V) # Backward: recompute + gradient chains, ~3x forward. - C = _CHUNK_SIZE + C = _CHUNK_SIZE[args.variant] num_chunks = ceil_div(seqlen, C) per_chunk = 2 * (3 * C * C * head_dim_qk + 2 * C * C * head_dim_vo + 2 * C * head_dim_qk * head_dim_vo) base = batch_size * num_kv_heads * num_chunks * per_chunk @@ -679,9 +827,15 @@ def generate_gates(io_dtype): beta = torch.rand(batch_size, seqlen, num_o_heads, device=device) write_gate = None elif args.variant == "kda": - # per-key-channel decay [B, T, HO, K] fp32 + post-sigmoid scalar beta + # per-key-channel decay [B, T, HO, K] fp32 + post-sigmoid scalar + # beta; forward-only runs feed raw logits with the same effective + # distributions (the in-kernel activations invert them) gate = torch.empty(batch_size, seqlen, num_o_heads, head_dim_qk, device=device).uniform_(0.5, 1.0).log() - beta = torch.rand(batch_size, seqlen, num_o_heads, device=device).sigmoid() + beta = torch.rand(batch_size, seqlen, num_o_heads, device=device) + if kda_raw_gates: + gate = torch.special.logit((gate / _KDA_GATE_LOWER_BOUND).clamp(1e-7, 1 - 1e-7)) + else: + beta = beta.sigmoid() write_gate = None else: # gdn2 # per-key decay/erase [B, T, HO, K] + per-value write gate [B, T, HO, V] @@ -732,15 +886,20 @@ def generate_gates(io_dtype): value.requires_grad_(True) gate.requires_grad_(True) beta.requires_grad_(True) + if write_gate is not None: + write_gate.requires_grad_(True) - # Per-sequence recurrent state ports (once-per-kernel I/O): the - # initial state seeds the recurrence; the final state is requested - # with --store_on and its gradient feeds the backward pass. - s0 = None + # Recurrent state ports: the initial state seeds the recurrence; + # --store_on dumps the per-chunk states plus the final state (whose + # gradient feeds the backward pass). + state0 = None if args.initial_state: - s0 = torch.randn(batch_size, num_o_heads, head_dim_qk, head_dim_vo, dtype=torch.float32, device=device) * 0.05 + state0 = torch.randn(batch_size, num_o_heads, head_dim_qk, head_dim_vo, dtype=torch.float32, device=device) * 0.05 + if args.la_backend == "flash_kda": + # FlashKDA state ports are V-major [B, H, V, K] + state0 = state0.transpose(-1, -2).contiguous() if run_bwd: - s0.requires_grad_(True) + state0.requires_grad_(True) if args.la_backend == "cudnn": dOutput = torch.randn(batch_size * seqlen, num_o_heads, head_dim_vo, dtype=target_dtype, device=device) else: @@ -755,7 +914,7 @@ def generate_gates(io_dtype): if run_fwd: with profile(activities=[ProfilerActivity.CUDA], record_shapes=True) as prof: with record_function("linear_attention.forward"): # Custom marker - output, final_state = la_function(query, key, value, gate, beta, write_gate, s0) + output, final_state = la_function(query, key, value, gate, beta, write_gate, state0) torch.cuda.synchronize() # Ensure all kernels finish # Filter profiler results by kernel name prefix @@ -766,6 +925,7 @@ def generate_gates(io_dtype): or item.key.startswith("kernel_cutlass") or item.key.startswith("triton_") or "chunk_" in item.key + or "_flash_kda_" in item.key or "l2norm" in item.key or "cutile" in item.key or "_kernel" in item.key @@ -777,7 +937,7 @@ def generate_gates(io_dtype): if i >= dry_run_iters: forward_times.append(fwd_time) else: - output, final_state = la_function(query, key, value, gate, beta, write_gate, s0) + output, final_state = la_function(query, key, value, gate, beta, write_gate, state0) torch.cuda.synchronize() if run_bwd: @@ -815,6 +975,7 @@ def generate_gates(io_dtype): or item.key.startswith("kernel_cutlass") or item.key.startswith("triton_") or "chunk_" in item.key + or "_flash_kda_" in item.key or "l2norm" in item.key or "cutile" in item.key or "_kernel" in item.key @@ -835,11 +996,23 @@ def generate_gates(io_dtype): if args.la_backend == "cudnn": gate_ref = gate.detach().reshape(batch_size, seqlen, *gate.shape[1:]) beta_ref = beta.detach().reshape(batch_size, seqlen, *beta.shape[1:]) + elif args.la_backend == "flash_kda": + # raw logits pass through; the fla reference applies the + # same in-kernel activations + gate_ref = gate.detach().float() + beta_ref = beta.detach().float() else: gate_ref = gate.detach() beta_ref = beta.detach() - s0_ref = s0.detach() if s0 is not None else None - output_ref, _ = fla_linear_attention(query_ref, key_ref, value_ref, gate_ref, beta_ref, None, s0_ref) + state0_ref = state0.detach() if state0 is not None else None + if state0_ref is not None and args.la_backend == "flash_kda": + state0_ref = state0_ref.transpose(-1, -2).contiguous() + wg_ref = None + if write_gate is not None: + wg_ref = write_gate.detach() + if args.la_backend == "cudnn": + wg_ref = wg_ref.reshape(batch_size, seqlen, *write_gate.shape[1:]) + output_ref, _ = fla_linear_attention(query_ref, key_ref, value_ref, gate_ref, beta_ref, wg_ref, state0_ref) torch.testing.assert_close(output.detach(), output_ref, rtol=1e-2, atol=1e-2) forward_diffs.append(torch.max(torch.abs(output.detach() - output_ref.detach())).item()) @@ -854,7 +1027,7 @@ def generate_gates(io_dtype): else: forward_diffs.append(0.0) - del query, key, value, gate, beta, write_gate, output, final_state, s0, dOutput, dFinal + del query, key, value, gate, beta, write_gate, output, final_state, state0, dOutput, dFinal _clock_sampler.stop() diff --git a/benchmark/linear_attention/plot_results.py b/benchmark/linear_attention/plot_results.py index 654a93e90..d8277e704 100644 --- a/benchmark/linear_attention/plot_results.py +++ b/benchmark/linear_attention/plot_results.py @@ -30,6 +30,9 @@ "cudnn": {"name": "cuDNN", "color": "#76b900", "order": 2}, } +# Backends dropped from every chart (rows may still exist in older CSVs). +UNAVAILABLE_BACKENDS = ("flash_kda",) + LABEL_FONT_SIZE = 10 LEGEND_FONT_SIZE = 8 TITLE_FONT_SIZE = 12 @@ -60,10 +63,13 @@ def get_backend_display_name(backend: str, cudnn_version: str = None) -> str: return base_name -def generate_charts(df: pd.DataFrame, output_dir: Path, gpu_name: str = "", cudnn_version: str = None, variant: str = "gdn", batch_sizes: list = None) -> list: +def generate_charts( + df: pd.DataFrame, output_dir: Path, gpu_name: str = "", cudnn_version: str = None, variant: str = "gdn", batch_sizes: list = None, x_axis: str = "seqlen" +) -> list: output_dir.mkdir(parents=True, exist_ok=True) df = df[df["variant"] == variant].copy() + df = df[~df["backend"].isin(UNAVAILABLE_BACKENDS)].copy() if batch_sizes: df = df[df["batch_size"].isin(batch_sizes)].copy() if df.empty: @@ -76,10 +82,13 @@ def generate_charts(df: pd.DataFrame, output_dir: Path, gpu_name: str = "", cudn for _, row in df[["backend", "backend_display"]].drop_duplicates().iterrows(): palette[row["backend_display"]] = BACKEND_CONFIG.get(row["backend"], {}).get("color", "gray") + x_col, group_col = ("seqlen", "batch_size") if x_axis == "seqlen" else ("batch_size", "seqlen") + x_label = "Sequence Length" if x_axis == "seqlen" else "Batch Size" + saved_paths = [] - for batch_size in sorted(df["batch_size"].unique()): - sub = df[df["batch_size"] == batch_size].copy() - sub.sort_values(["seqlen", "backend_order"], inplace=True) + for group_val in sorted(df[group_col].unique()): + sub = df[df[group_col] == group_val].copy() + sub.sort_values([x_col, "backend_order"], inplace=True) hue_order = list(sub.sort_values("backend_order")["backend_display"].drop_duplicates()) fwd_df = sub[sub["fwd_tflops"] > 0] @@ -99,8 +108,9 @@ def generate_charts(df: pd.DataFrame, output_dir: Path, gpu_name: str = "", cudn heads = sub["num_q_heads"].iloc[0] head_dim = sub["head_dim"].iloc[0] gpu_info = f" ({gpu_name})" if gpu_name else "" + group_label = f"Batch = {group_val}" if x_axis == "seqlen" else f"SeqLen = {group_val}" fig.suptitle( - f"{variant.upper()} Linear Attention (BF16) — Batch = {batch_size}, Heads = {heads}, d = {head_dim}{gpu_info}", + f"{variant.upper()} Linear Attention (BF16) — {group_label}, Heads = {heads}, d = {head_dim}{gpu_info}", fontsize=TITLE_FONT_SIZE, ) @@ -112,7 +122,7 @@ def generate_charts(df: pd.DataFrame, output_dir: Path, gpu_name: str = "", cudn continue sns.barplot( data=pass_df, - x="seqlen", + x=x_col, y=y_col, hue="backend_display", hue_order=hue_order, @@ -122,7 +132,7 @@ def generate_charts(df: pd.DataFrame, output_dir: Path, gpu_name: str = "", cudn linewidth=0.5, errorbar=None, ) - ax.set_xlabel("Sequence Length", fontsize=LABEL_FONT_SIZE) + ax.set_xlabel(x_label, fontsize=LABEL_FONT_SIZE) ax.set_ylabel("TFLOPS", fontsize=LABEL_FONT_SIZE) ax.set_title(pass_name, fontsize=TITLE_FONT_SIZE) ax.legend(title="Backend", fontsize=LEGEND_FONT_SIZE) @@ -131,7 +141,9 @@ def generate_charts(df: pd.DataFrame, output_dir: Path, gpu_name: str = "", cudn ax.bar_label(container, fmt="%.0f", fontsize=BAR_LABEL_FONT_SIZE) plt.tight_layout() - output_path = output_dir / f"{variant}_b{batch_size}.png" + gv = int(group_val) + stem = f"{variant}_b{gv}" if x_axis == "seqlen" else f"{variant}_t{gv}_bsweep" + output_path = output_dir / f"{stem}.png" plt.savefig(output_path, dpi=150, bbox_inches="tight") plt.close() saved_paths.append(output_path) @@ -148,6 +160,7 @@ def main(): parser.add_argument("--cudnn-version", default=None, help="cuDNN backend version for the legend (e.g. 9.24.0)") parser.add_argument("--variant", default="gdn", help="Linear attention variant to plot") parser.add_argument("--batch-sizes", default=None, help="Comma-separated batch sizes to plot (default: all in the CSV)") + parser.add_argument("--x-axis", default="seqlen", choices=("seqlen", "batch"), help="Bar-group axis: seqlen (one chart per batch) or batch (one chart per seqlen)") args = parser.parse_args() batch_sizes = [int(b) for b in args.batch_sizes.split(",")] if args.batch_sizes else None @@ -157,7 +170,7 @@ def main(): raise ValueError(f"CSV is missing expected columns: {missing}") output_dir = args.output_dir if args.output_dir is not None else args.csv.parent - generate_charts(df, output_dir, gpu_name=args.gpu_name, cudnn_version=args.cudnn_version, variant=args.variant, batch_sizes=batch_sizes) + generate_charts(df, output_dir, gpu_name=args.gpu_name, cudnn_version=args.cudnn_version, variant=args.variant, batch_sizes=batch_sizes, x_axis=args.x_axis) if __name__ == "__main__": diff --git a/python/cudnn/_pygraph.py b/python/cudnn/_pygraph.py index a24c45d73..96af2a43b 100644 --- a/python/cudnn/_pygraph.py +++ b/python/cudnn/_pygraph.py @@ -2447,13 +2447,14 @@ def _linear_attention_final_state_dims(node): return [cu.dim[0] - 1, max(q[1], v[1]), q[2], v[2]] -def _linear_attention_h_dims(node): - # [total_h, HO, K, V] at capacity: sum_b (sl_b - 1) // N <= total_T // N +def _linear_attention_state_checkpoints_dims(node): n = int(node.params.get("checkpoint_every_n_tokens", 0) or 0) q, v = node.inputs["q"].dim, node.inputs["v"].dim - if not n or not q or not v: + cu = node.inputs["cu_seqlens"].dim if node.inputs.get("cu_seqlens") is not None else None + if not n or not q or not v or not cu: return None - return [max(v[0] // n, 1), max(q[1], v[1]), q[2], v[2]] + b = int(cu[0]) - 1 + return [max((v[0] - b) // n, 1), max(q[1], v[1]), q[2], v[2]] def _linear_attention_o_dims(node): @@ -2620,19 +2621,19 @@ def _training_phase(node): # norm stats exist only in TRAINING forward phase # ---- linear attention ---------------------------------------------------- "gdn": dict( node_type=NodeType.GDN, - inputs=("q", "k", "v", "g", "beta", "cu_seqlens", "initial_state"), - attrs=("scale", "output_final_state", "use_qk_l2norm", "checkpoint_every_n_tokens"), - outputs=("O", "final_state", "H"), + inputs=("q", "k", "v", "g", "beta", "cu_seqlens", "initial_state", "a_log", "dt_bias"), + attrs=("scale", "output_final_state", "use_qk_l2norm", "checkpoint_every_n_tokens", "safe_gate"), + outputs=("O", "final_state", "state_checkpoints"), maybe={ "final_state": lambda n: bool(n.params.get("output_final_state", False)), - "H": lambda n: bool(n.params.get("checkpoint_every_n_tokens") or 0), + "state_checkpoints": lambda n: bool(n.params.get("checkpoint_every_n_tokens") or 0), }, - infer={"O": _linear_attention_o_dims, "final_state": _linear_attention_final_state_dims, "H": _linear_attention_h_dims}, + infer={"O": _linear_attention_o_dims, "final_state": _linear_attention_final_state_dims, "state_checkpoints": _linear_attention_state_checkpoints_dims}, python_only=True, ), "gdn_bwd": dict( node_type=NodeType.GDN_BWD, - inputs=("q", "k", "v", "g", "beta", "cu_seqlens", "dO", "h", "initial_state", "d_final_state"), + inputs=("q", "k", "v", "g", "beta", "cu_seqlens", "dO", "state_checkpoints", "initial_state", "d_final_state"), attrs=("scale", "use_qk_l2norm"), outputs=("dQ", "dK", "dV", "dG", "dBeta", "d_initial_state"), maybe={"d_initial_state": lambda n: "initial_state" in n.inputs}, @@ -2643,17 +2644,17 @@ def _training_phase(node): # norm stats exist only in TRAINING forward phase node_type=NodeType.KDA, inputs=("q", "k", "v", "g", "beta", "cu_seqlens", "initial_state", "a_log", "dt_bias"), attrs=("scale", "output_final_state", "use_qk_l2norm", "checkpoint_every_n_tokens", "use_beta_sigmoid", "safe_gate", "gate_lower_bound"), - outputs=("O", "final_state", "H"), + outputs=("O", "final_state", "state_checkpoints"), maybe={ "final_state": lambda n: bool(n.params.get("output_final_state", False)), - "H": lambda n: bool(n.params.get("checkpoint_every_n_tokens") or 0), + "state_checkpoints": lambda n: bool(n.params.get("checkpoint_every_n_tokens") or 0), }, - infer={"O": _like("v"), "final_state": _linear_attention_final_state_dims, "H": _linear_attention_h_dims}, + infer={"O": _linear_attention_o_dims, "final_state": _linear_attention_final_state_dims, "state_checkpoints": _linear_attention_state_checkpoints_dims}, python_only=True, ), "kda_bwd": dict( node_type=NodeType.KDA_BWD, - inputs=("q", "k", "v", "g", "beta", "cu_seqlens", "dO", "h", "initial_state", "d_final_state"), + inputs=("q", "k", "v", "g", "beta", "cu_seqlens", "dO", "state_checkpoints", "initial_state", "d_final_state"), attrs=("scale", "use_qk_l2norm"), outputs=("dQ", "dK", "dV", "dG", "dBeta", "d_initial_state"), maybe={"d_initial_state": lambda n: "initial_state" in n.inputs}, @@ -2662,20 +2663,20 @@ def _training_phase(node): # norm stats exist only in TRAINING forward phase ), "gdn2": dict( node_type=NodeType.GDN2, - inputs=("q", "k", "v", "g", "beta", "w", "cu_seqlens", "initial_state"), - attrs=("scale", "output_final_state", "use_qk_l2norm", "checkpoint_every_n_tokens", "use_beta_w_sigmoid"), - outputs=("O", "final_state", "H"), + inputs=("q", "k", "v", "g", "beta", "w", "cu_seqlens", "initial_state", "a_log", "dt_bias"), + attrs=("scale", "output_final_state", "use_qk_l2norm", "checkpoint_every_n_tokens", "safe_gate", "gate_lower_bound"), + outputs=("O", "final_state", "state_checkpoints"), maybe={ "final_state": lambda n: bool(n.params.get("output_final_state", False)), - "H": lambda n: bool(n.params.get("checkpoint_every_n_tokens") or 0), + "state_checkpoints": lambda n: bool(n.params.get("checkpoint_every_n_tokens") or 0), }, - infer={"O": _like("v"), "final_state": _linear_attention_final_state_dims, "H": _linear_attention_h_dims}, + infer={"O": _linear_attention_o_dims, "final_state": _linear_attention_final_state_dims, "state_checkpoints": _linear_attention_state_checkpoints_dims}, python_only=True, ), "gdn2_bwd": dict( node_type=NodeType.GDN2_BWD, - inputs=("q", "k", "v", "g", "beta", "w", "cu_seqlens", "dO", "h", "initial_state", "d_final_state"), - attrs=("scale",), + inputs=("q", "k", "v", "g", "beta", "w", "cu_seqlens", "dO", "state_checkpoints", "initial_state", "d_final_state"), + attrs=("scale", "use_qk_l2norm"), outputs=("dQ", "dK", "dV", "dG", "dBeta", "dW", "d_initial_state"), maybe={"d_initial_state": lambda n: "initial_state" in n.inputs}, infer={ diff --git a/python/cudnn/engines/manifest.py b/python/cudnn/engines/manifest.py index fabdc4756..bb68f1db3 100644 --- a/python/cudnn/engines/manifest.py +++ b/python/cudnn/engines/manifest.py @@ -143,6 +143,7 @@ def offered_ids(self) -> Dict[str, int]: "cudnn.linear_attention", "GdnEngines", slots={"gdn_frost": EngineSlot(0), "gdn_cutile": EngineSlot(1)}, + analyzer=("cudnn.linear_attention.graph_analyzer", "analyze"), ), EngineFamily( KDA_ID_BASE, @@ -150,6 +151,7 @@ def offered_ids(self) -> Dict[str, int]: "cudnn.linear_attention", "KdaEngines", slots={"kda_frost": EngineSlot(0), "kda_cutile": EngineSlot(1)}, + analyzer=("cudnn.linear_attention.graph_analyzer", "analyze"), ), EngineFamily( GDN2_ID_BASE, @@ -157,6 +159,7 @@ def offered_ids(self) -> Dict[str, int]: "cudnn.linear_attention", "Gdn2Engines", slots={"gdn2_frost": EngineSlot(0)}, + analyzer=("cudnn.linear_attention.graph_analyzer", "analyze"), ), EngineFamily( FROST_GEMM_ID_BASE, diff --git a/python/cudnn/frost/buffers.py b/python/cudnn/frost/buffers.py index 014d12631..416aea873 100644 --- a/python/cudnn/frost/buffers.py +++ b/python/cudnn/frost/buffers.py @@ -99,6 +99,14 @@ def dtype_name(buf) -> str: return str(buf.dtype).split(".")[-1] +def data_ptr(buf) -> int: + """Device address of a tensor-like (``data_ptr()`` or the CUDA array interface).""" + fn = getattr(buf, "data_ptr", None) + if fn is not None: + return fn() + return buf.__cuda_array_interface__["data"][0] + + class DeviceView: """Zero-copy DLPack view over a raw CUDA pointer. @@ -288,6 +296,23 @@ def is_contiguous(shape, strides) -> bool: return True +def dense_layout_ok(shape, strides) -> bool: + """Relaxed layout soundness (rank-agnostic, size-1 dims wildcarded): + innermost dim stride-1, no zero stride on a size>1 dim, non-overlapping + strides (padded / permuted outer strides allowed).""" + if strides is None: + return True + if shape[-1] != 1 and strides[-1] != 1: + return False + active = sorted((int(st), int(sz)) for st, sz in zip(strides, shape) if sz != 1) + span = 1 + for st, sz in active: + if st < span: + return False + span = st * sz + return True + + def memset_zero_async(ptr: int, nbytes: int, stream) -> None: """Stream-ordered zero-fill of a device range via ``cuda.bindings``.""" from cuda.bindings import runtime as _rt @@ -455,7 +480,10 @@ def cutedsl_state(): ``version`` is ``(distribution, version)`` or None: the public wheel is nvidia-cutlass-dsl, internal RCs ship as nvidia-cutlass-dsl-internal, and the two number themselves differently. Presence is what gates; the pair only - refines the message and feeds :func:`cutedsl_too_old`. + refines the message and feeds :func:`cutedsl_too_old`. The internal dist is + checked first: when both are installed the RC's packages shadow the public + wheel on sys.path, and a stale public version string must not veto a + machine that works. """ global _DSL_STATE if _DSL_STATE is None: @@ -467,7 +495,7 @@ def cutedsl_state(): except (ImportError, ValueError): installed = False version = None - for dist in ("nvidia-cutlass-dsl", "nvidia-cutlass-dsl-internal"): + for dist in ("nvidia-cutlass-dsl-internal", "nvidia-cutlass-dsl"): try: version = (dist, importlib.metadata.version(dist)) break diff --git a/python/cudnn/frost/tile_dsl/barrier.py b/python/cudnn/frost/tile_dsl/barrier.py index 553313ffd..64a7d1967 100644 --- a/python/cudnn/frost/tile_dsl/barrier.py +++ b/python/cudnn/frost/tile_dsl/barrier.py @@ -23,14 +23,23 @@ def start(cls, phase: int = 0): return cls(idx=cutlass.Int32(0), phase=cutlass.Int32(phase)) -def advance(state, stages): - if stages < 1: +@cute.jit +def advance(state, stages, step: int = 1): + if cutlass.const_expr(stages < 1): raise ValueError(f"PipelineState.advance requires stages >= 1, got {stages}") - incr = state.idx + cutlass.Int32(1) - stages_i = cutlass.Int32(stages) - new_idx = incr % stages_i - flip = incr // stages_i - new_phase = state.phase ^ flip + incr = state.idx + cutlass.Int32(step) + if cutlass.const_expr(stages & (stages - 1) == 0 or step > stages): + # pow2 folds to mask+shift; >1-wrap steps need the divmod + stages_i = cutlass.Int32(stages) + new_idx = incr % stages_i + flip = (incr // stages_i) & cutlass.Int32(1) + new_phase = state.phase ^ flip + else: + # non-pow2, at most one wrap: compare-select beats the magic-multiply + wrapped = incr >= cutlass.Int32(stages) + new_idx = incr - cutlass.Int32(stages) if wrapped else incr + flip = cutlass.Int32(1) if wrapped else cutlass.Int32(0) + new_phase = state.phase ^ flip return PipelineState(idx=new_idx, phase=new_phase) @@ -210,8 +219,14 @@ def arrive( arrive_expect_tx(self.smem_ptr, n_bytes, pred=pred) elif cutlass.const_expr(self.producer == int(Producer.MMA_COMMIT)): + # commit barriers take tcgen05 commits ONLY; mixing thread arrives + # onto a commit barrier is banned (split into a THREAD sibling) if cta_group is None: - raise TypeError("MBarrier(producer=MMA_COMMIT).arrive() requires " "cta_group= (Python compile-time int).") + raise TypeError( + "MBarrier(producer=MMA_COMMIT) only accepts the commit form " + "(.arrive(cta_group=...)). Thread arrivals must go to a " + "separate MBarrier(producer=THREAD) sibling; waiters wait both." + ) commit_mma(self.smem_ptr, mcast_mask, cta_group, pred=pred) else: diff --git a/python/cudnn/frost/tile_dsl/mma.py b/python/cudnn/frost/tile_dsl/mma.py index b2bf2da36..5f3d37e33 100644 --- a/python/cudnn/frost/tile_dsl/mma.py +++ b/python/cudnn/frost/tile_dsl/mma.py @@ -479,6 +479,55 @@ def mma_step( acc[s_off + 3] = c3 +@cute.jit +def mma_step_k8( + acc, + a_frag, + b_frag, + *, + k_step: cutlass.Constexpr[int], + M: cutlass.Constexpr[int], + N: cutlass.Constexpr[int], + ab_dtype: cutlass.Constexpr[Type[cutlass.Numeric]] = cutlass.Float16, +): + if cutlass.const_expr(M % 16 != 0): + raise ValueError(f"mma_step_k8: M must be a multiple of 16, got M={M}") + if cutlass.const_expr(ab_dtype != cutlass.Float16 and ab_dtype != cutlass.BFloat16): + raise TypeError(f"mma_step_k8: ab_dtype must be Float16 or BFloat16, got {ab_dtype}") + M_BLOCKS = M // 16 + N_FRAGS = N // 8 + a_stride = len(a_frag) // M_BLOCKS + + ab_tag = "f16" if cutlass.const_expr(ab_dtype == cutlass.Float16) else "bf16" + mma_ptx = f"mma.sync.aligned.m16n8k8.row.col.f32.{ab_tag}.{ab_tag}.f32" " {$0,$1,$2,$3}, {$4,$5}, {$6}, {$7,$8,$9,$10};" + + for m_block in cutlass.range_constexpr(M_BLOCKS): + a_off = m_block * a_stride + k_step * 2 + a0 = a_frag[a_off + 0] + a1 = a_frag[a_off + 1] + acc_base = m_block * N_FRAGS * 4 + for n_frag in cutlass.range_constexpr(N_FRAGS): + b0 = b_frag[n_frag] + s_off = acc_base + n_frag * 4 + c0, c1, c2, c3 = inline_ptx( + mma_ptx, + write_only_types=[cutlass.Float32, cutlass.Float32, cutlass.Float32, cutlass.Float32], + read_only_args=[ + a0, + a1, + b0, + acc[s_off + 0], + acc[s_off + 1], + acc[s_off + 2], + acc[s_off + 3], + ], + ) + acc[s_off + 0] = c0 + acc[s_off + 1] = c1 + acc[s_off + 2] = c2 + acc[s_off + 3] = c3 + + @cute.jit def mma( acc, diff --git a/python/cudnn/frost/tile_dsl/swizzle.py b/python/cudnn/frost/tile_dsl/swizzle.py index e8b16adea..b660b95f5 100644 --- a/python/cudnn/frost/tile_dsl/swizzle.py +++ b/python/cudnn/frost/tile_dsl/swizzle.py @@ -8,11 +8,7 @@ @cute.jit def swizzle_xor_128b(row, col_elem, *, elem_bytes: cutlass.Constexpr[int] = 2): - chunk_elems = 16 // elem_bytes - chunk_idx = col_elem // chunk_elems - in_chunk = col_elem % chunk_elems - swz_chunk = chunk_idx ^ (row & 7) - return swz_chunk * chunk_elems + in_chunk + return col_elem ^ ((row & 7) * cutlass.const_expr(16 // elem_bytes)) @cute.jit diff --git a/python/cudnn/linear_attention/cutile/gdn_engine.py b/python/cudnn/linear_attention/cutile/gdn_engine.py index 6dae5642e..cce58bec2 100644 --- a/python/cudnn/linear_attention/cutile/gdn_engine.py +++ b/python/cudnn/linear_attention/cutile/gdn_engine.py @@ -1,51 +1,50 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""GDN (Gated DeltaNet) execution backend: GDN / GDN_BWD nodes on the -chunked cuTile kernels (``kernels/gdn_chunk_cutile``).""" +"""cuTile GDN engine: GDN / GDN_BWD nodes on the chunked cuTile kernels +(``kernels/gdn_chunk_cutile``).""" -from typing import TYPE_CHECKING, Any, Dict +from typing import TYPE_CHECKING from cudnn import behavior_note from cudnn.engines.base import BaseEngine, CompiledPlan, resolve_node_buffers from cudnn.graph_types import NodeType + from cudnn.frost import buffers from cudnn.frost.workspace import Workspace -from cudnn.linear_attention.engine_utils import _dtype_name +from ..graph_analyzer import check_layouts_compact, analyze, expect_table, to_buffer_dtype -if TYPE_CHECKING: - from cudnn.pygraph import pygraph +# entry base-alignment expectations; ports not listed assume 16 +CUTILE_ALIGN = {"cu_seqlens": 4, "a_log": 4, "beta": 4} -_REQUIRED_PORTS = { - NodeType.GDN: ("q", "k", "v", "g", "beta", "cu_seqlens"), - NodeType.GDN_BWD: ("q", "k", "v", "g", "beta", "cu_seqlens", "dO"), -} +if TYPE_CHECKING: + from cudnn._pygraph import pygraph -def _node_ws_layout(node): +def node_ws_layout(node): """Static carve plan for one node's pipeline intermediates: name -> (offset, dtype-name, shape). The chunk count is data-dependent (varlen), so chunk-indexed entries are sized and SHAPED at the bound ``cdiv(total, 64) + N`` — the device-built table's sentinel tail keeps bound-gridded launches inert past the real count. Terminal pipeline buffers (``o``/``final_state``; the backward's ``dq``/``dk`` finals, - ``wy_dv``, ``dg_cum``, ``db``, ``dh0``) are NOT carved — execute plants + ``wy_dv``, ``dg_cum``, ``db``, ``dstate0``) are NOT carved — execute plants the caller's output buffers under those names.""" - from .kernels.gdn_chunk_cutile import _BT, _cdiv, _next_power_of_2 + from .kernels.gdn_chunk_cutile import BT_CHUNK, cdiv, next_power_of_2 q, v, cu = (node.inputs[p] for p in ("q", "v", "cu_seqlens")) total, H, K = q.dim HV, V = v.dim[1], v.dim[2] N = cu.dim[0] - 1 - io = _dtype_name(q.get_data_type()) + io = to_buffer_dtype(q.get_data_type()) f32 = "float32" - NT_bound = _cdiv(total, _BT) + N + NT_bound = cdiv(total, BT_CHUNK) + N l2norm = bool(node.params.get("use_qk_l2norm", False)) size = 0 table = {} - def add(name, dtype, shape): + def carve(name, dtype, shape): nonlocal size nbytes = buffers.DTYPE_ITEMSIZE[dtype] for s in shape: @@ -53,161 +52,79 @@ def add(name, dtype, shape): table[name] = (size, dtype, tuple(int(s) for s in shape)) size += (nbytes + 127) & ~127 # 128B-aligned sequential carve - add("chunk_table", "int32", (NT_bound, 2)) - add("chunk_count", "int32", (1,)) - add("chunk_offsets", "int32", (N + 1,)) - add("dummy", "int32", (4,)) # inert stub backing for absent optional kernel args - add("g_cum", f32, (total, HV)) - add("A", io, (total, HV, _BT)) - add("w", io, (total, HV, K)) - add("u", io, (total, HV, V)) - add("h", io, (NT_bound, HV, K, V)) - add("v_new", io, (total, HV, V)) + carve("chunk_table", "int32", (NT_bound, 2)) + carve("chunk_count", "int32", (1,)) + carve("chunk_offsets", "int32", (N + 1,)) + carve("dummy", "int32", (4,)) # inert stub backing for absent optional kernel args + carve("g_cum", f32, (total, HV)) + carve("A", io, (total, HV, BT_CHUNK)) + carve("w", io, (total, HV, K)) + carve("u", io, (total, HV, V)) + carve("state_checkpoints", io, (NT_bound, HV, K, V)) + carve("v_new", io, (total, HV, V)) if l2norm: - add("q_norm", io, (total * H, K)) - add("q_rstd", f32, (total * H,)) - add("k_norm", io, (total * H, K)) - add("k_rstd", f32, (total * H,)) + carve("q_norm", io, (total * H, K)) + carve("q_rstd", f32, (total * H,)) + carve("k_norm", io, (total * H, K)) + carve("k_rstd", f32, (total * H,)) if node.node_type == NodeType.GDN_BWD: - add("dv", io, (total, HV, V)) - add("dh", io, (NT_bound, HV, K, V)) - add("dv2", io, (total, HV, V)) - NK = _cdiv(K, min(max(_next_power_of_2(K), 16), 64)) - add("dg_nk", f32, (NK, total, HV)) - add("dw", io, (total, HV, K)) + carve("dv", io, (total, HV, V)) + carve("dstate", io, (NT_bound, HV, K, V)) + carve("dv2", io, (total, HV, V)) + NK = cdiv(K, min(max(next_power_of_2(K), 16), 64)) + carve("dg_nk", f32, (NK, total, HV)) + carve("dw", io, (total, HV, K)) if HV != H or l2norm: - # dq/dk are finals only without l2norm on an MHA config; every + # dQ/dK are finals only without l2norm on an MHA config; every # other combination keeps them (or their head-reduced pair) as # pipeline intermediates - add("dq", io, (total, HV, K)) - add("dk", io, (total, HV, K)) + carve("dq", io, (total, HV, K)) + carve("dk", io, (total, HV, K)) if HV != H: - add("wy_dk_hred", io, (total, H, K)) + carve("wy_dk_hred", io, (total, H, K)) if l2norm: - add("dq_hred", io, (total, H, K)) - add("dk_hred", io, (total, H, K)) - add("dg", f32, (total, HV)) - add("wy_dk", io, (total, HV, K)) - add("wy_dg", f32, (total, HV)) + carve("dq_hred", io, (total, H, K)) + carve("dk_hred", io, (total, H, K)) + carve("dg", f32, (total, HV)) + carve("wy_dk", io, (total, HV, K)) + carve("wy_dg", f32, (total, HV)) return size, table -class _CuTilePlan(CompiledPlan): +class GdnCuTilePlan(CompiledPlan): """Carve plan over the caller's workspace: the layout is static per node; - the buffer arrives with every execute (the explicit-workspace convention).""" + the buffer arrives with every execute.""" - def __init__(self, engine, graph): - self._engine = engine - self._layouts = [(node, *_node_ws_layout(node)) for node in graph.nodes] + def __init__(self, graph): + self.layouts = [(node, *node_ws_layout(node)) for node in graph.nodes] + self.expects = {node: expect_table(node, CUTILE_ALIGN) for node in graph.nodes} # nodes execute sequentially, each re-carving the same buffer - self._ws_bytes = max(nbytes for _, nbytes, _ in self._layouts) + self.ws_bytes = max(nbytes for _, nbytes, _ in self.layouts) def get_workspace_size(self) -> int: - return self._ws_bytes + return self.ws_bytes def execute(self, graph, uid_to_data, ctx) -> None: - self._engine._execute(resolve_node_buffers(graph, uid_to_data), self, ctx) - - -class GdnCuTileEngine(BaseEngine): - """cuTile chunked-kernel backend for single-node GDN graphs (THD layout).""" - - name = "gdn_cutile" - behavior_notes = (behavior_note.RUNTIME_COMPILATION,) # JIT-compiled at build_plans() - - def check_support(self, graph: "pygraph") -> None: - if buffers.current_sm() is None: - raise NotImplementedError("GdnCuTileEngine requires a CUDA device") - try: - from cuda.bindings import runtime as _rt - - err, _cudart_version = _rt.cudaRuntimeGetVersion() - if int(err) != 0: - raise NotImplementedError(f"GdnCuTileEngine: cudaRuntimeGetVersion failed ({err})") - except ImportError as e: - raise NotImplementedError(f"GdnCuTileEngine requires cuda.bindings: {e}") - if _cudart_version < 13030: - raise NotImplementedError(f"GdnCuTileEngine requires CUDA 13.3+ (found {_cudart_version})") - try: - from .kernels.gdn_chunk_cutile import ( # noqa: F401 - chunk_gated_delta_rule, - ) - except ImportError as e: - raise NotImplementedError(f"GdnCuTileEngine requires the cuda.tile runtime: {e}") - - import cudnn - - supported_dtypes = (cudnn.data_type.HALF, cudnn.data_type.BFLOAT16, None) - if not graph.nodes: - raise NotImplementedError("GdnCuTileEngine: empty graph") - for node in graph.nodes: - required = _REQUIRED_PORTS.get(node.node_type) - if required is None: - raise NotImplementedError(f"GdnCuTileEngine only supports GDN/GDN_BWD nodes, got {node.node_type.name}") - for port in required: - if port not in node.inputs: - raise NotImplementedError(f"GdnCuTileEngine: {node.node_type.name} node '{node.name}' is missing input '{port}'") - if int(node.params.get("checkpoint_every_n_tokens", 0) or 0) > 0 or "H" in node.outputs: - raise NotImplementedError("GdnCuTileEngine: per-chunk H output is not supported") - q, k, v = (node.inputs[p] for p in ("q", "k", "v")) - for p in ("q", "k", "v"): - t = node.inputs[p] - if t.get_data_type() not in supported_dtypes: - raise NotImplementedError(f"GdnCuTileEngine: '{p}' must be fp16/bf16, got {t.get_data_type()}") - if t.dim and len(t.dim) != 3: - raise NotImplementedError(f"GdnCuTileEngine: '{p}' must be THD [total_T, heads, dim], got rank {len(t.dim)}") - if q.dim and k.dim and q.dim[1] != k.dim[1]: - raise NotImplementedError(f"GdnCuTileEngine: q and k head counts differ ({q.dim[1]} vs {k.dim[1]})") - if q.dim and v.dim and v.dim[1] % q.dim[1] != 0: - raise NotImplementedError( - f"GdnCuTileEngine: v heads ({v.dim[1]}) must be a multiple of q heads ({q.dim[1]}; GQA-style v broadcast is FROST-only)" - ) - if q.dim and q.dim[-1] > 256: - raise NotImplementedError(f"GdnCuTileEngine: head dim K must be <= 256, got {q.dim[-1]}") - if node.inputs["cu_seqlens"].get_data_type() not in (cudnn.data_type.INT32, None): - raise NotImplementedError("GdnCuTileEngine: cu_seqlens must be int32 (the device-side table builder reads it directly)") - io = q.get_data_type() - f32 = cudnn.data_type.FLOAT - if node.node_type == NodeType.GDN: - out_dtypes = {"O": io, "final_state": f32} - required_out = ("O",) - else: - beta_dt = node.inputs["beta"].get_data_type() - out_dtypes = {"dQ": io, "dK": io, "dV": io, "dG": f32, "dBeta": beta_dt, "d_initial_state": f32} - required_out = ("dQ", "dK", "dV", "dG", "dBeta") - if ("initial_state" in node.inputs) != ("d_initial_state" in node.outputs): - raise NotImplementedError("GdnCuTileEngine: d_initial_state output must be requested iff initial_state is given") - for port in required_out: - if port not in node.outputs: - raise NotImplementedError(f"GdnCuTileEngine: {node.node_type.name} node '{node.name}' is missing output '{port}'") - for port, want in out_dtypes.items(): - t = node.outputs.get(port) - if t is not None and t.get_data_type() not in (want, None): - raise NotImplementedError(f"GdnCuTileEngine: output '{port}' must be {want} (written in place), got {t.get_data_type()}") - - def build_plan(self, graph, plan, ctx=None) -> CompiledPlan: - return _CuTilePlan(self, graph) - - def _execute(self, node_buffers, plan, ctx) -> None: from .kernels.common import build_chunk_table - from .kernels.gdn_chunk_cutile import _BT + from .kernels.gdn_chunk_cutile import BT_CHUNK - stream = getattr(ctx, "stream", None) - stream = 0 if stream is None else stream - ws = Workspace(getattr(ctx, "workspace", None), plan._ws_bytes, "GdnCuTileEngine") - for node, _nbytes, table in plan._layouts: + node_buffers = resolve_node_buffers(graph, uid_to_data) + stream = ctx.stream if ctx.stream is not None else 0 + ws = Workspace(ctx.workspace, self.ws_bytes, "GdnCuTileEngine") + for node, _nbytes, table in self.layouts: nb = node_buffers[node] + check_layouts_compact("GdnCuTileEngine", self.expects[node], nb) cu_seqlens = nb.inputs["cu_seqlens"] N = node.inputs["cu_seqlens"].dim[0] - 1 bufs = {name: ws.view(off, dt, shape) for name, (off, dt, shape) in table.items()} bound = bufs["chunk_table"].shape[0] - build_chunk_table(bufs["chunk_table"], bufs["chunk_count"], bufs["chunk_offsets"], cu_seqlens, N, _BT, bound, stream=stream) + build_chunk_table(bufs["chunk_table"], bufs["chunk_count"], bufs["chunk_offsets"], cu_seqlens, N, BT_CHUNK, bound, stream=stream) if node.node_type == NodeType.GDN: - self._execute_fwd(node, nb, bufs, stream) + self.execute_fwd(node, nb, bufs, stream) else: - self._execute_bwd(node, nb, bufs, stream) + self.execute_bwd(node, nb, bufs, stream) - def _execute_fwd(self, node, nb, bufs, stream) -> None: + def execute_fwd(self, node, nb, bufs, stream) -> None: from .kernels.gdn_chunk_cutile import chunk_gated_delta_rule_fwd, l2norm_fwd want_state = "final_state" in node.outputs @@ -222,6 +139,9 @@ def _execute_fwd(self, node, nb, bufs, stream) -> None: bufs["o"] = nb.outputs["O"] if want_state: bufs["final_state"] = nb.outputs["final_state"] + gate_kwargs = {} + if node.params.get("safe_gate", False): + gate_kwargs = dict(use_gate_in_kernel=True, A_log=nb.inputs["a_log"], dt_bias=nb.inputs["dt_bias"]) chunk_gated_delta_rule_fwd( q=q, k=k, @@ -230,6 +150,7 @@ def _execute_fwd(self, node, nb, bufs, stream) -> None: beta=beta, scale=scale, initial_state=nb.inputs.get("initial_state"), + **gate_kwargs, output_final_state=want_state, cu_seqlens=nb.inputs["cu_seqlens"], chunk_indices=bufs["chunk_table"], @@ -237,17 +158,17 @@ def _execute_fwd(self, node, nb, bufs, stream) -> None: stream=stream, ) - def _execute_bwd(self, node, nb, bufs, stream) -> None: + def execute_bwd(self, node, nb, bufs, stream) -> None: + from .kernels.common import add_inplace, reshaped from .kernels.gdn_chunk_cutile import ( RCP_LN2, - _BT, + BT_CHUNK, chunk_gated_delta_rule_bwd, chunk_gated_delta_rule_fwd_intra, chunk_local_cumsum, l2norm_bwd, l2norm_fwd, ) - from .kernels.common import add_inplace, reshaped H, K = node.inputs["q"].dim[1], node.inputs["q"].dim[-1] HV = node.inputs["v"].dim[1] @@ -255,7 +176,7 @@ def _execute_bwd(self, node, nb, bufs, stream) -> None: g, beta, do = nb.inputs["g"], nb.inputs["beta"], nb.inputs["dO"] cu_seqlens = nb.inputs["cu_seqlens"] initial_state = nb.inputs.get("initial_state") - dht = nb.inputs.get("d_final_state") + dstate_in = nb.inputs.get("d_final_state") chunk_indices = bufs["chunk_table"] scale = node.params.get("scale") or K**-0.5 l2norm = bool(node.params.get("use_qk_l2norm", False)) @@ -263,8 +184,7 @@ def _execute_bwd(self, node, nb, bufs, stream) -> None: q, q_rstd = l2norm_fwd(q, out=bufs["q_norm"], rstd_out=bufs["q_rstd"], stream=stream) k, k_rstd = l2norm_fwd(k, out=bufs["k_norm"], rstd_out=bufs["k_rstd"], stream=stream) - # terminal pipeline buffers = the caller's output buffers (with - # l2norm, dq/dk stay carves and l2norm_bwd writes the caller's) + # terminal pipeline buffers = the caller's output buffers if not l2norm: bufs["dq" if HV == H else "dq_hred"] = nb.outputs["dQ"] bufs["dk" if HV == H else "dk_hred"] = nb.outputs["dK"] @@ -272,11 +192,10 @@ def _execute_bwd(self, node, nb, bufs, stream) -> None: bufs["dg_cum"] = nb.outputs["dG"] bufs["db"] = nb.outputs["dBeta"] if initial_state is not None: - bufs["dh0"] = nb.outputs["d_initial_state"] + bufs["dstate0"] = nb.outputs["d_initial_state"] # recompute the forward's cumulative gate and intra-chunk WY matrix - # (the backward entry expects them) - g_cum = chunk_local_cumsum(g, chunk_size=_BT, scale=RCP_LN2, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices, out=bufs["g_cum"], stream=stream) + g_cum = chunk_local_cumsum(g, chunk_size=BT_CHUNK, scale=RCP_LN2, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices, out=bufs["g_cum"], stream=stream) _, _, A = chunk_gated_delta_rule_fwd_intra( k=k, v=v, g=g_cum, beta=beta, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices, bufs=bufs, compute_wu=False, stream=stream ) @@ -290,7 +209,7 @@ def _execute_bwd(self, node, nb, bufs, stream) -> None: scale=scale, initial_state=initial_state, do=do, - dht=dht, + dstate_in=dstate_in, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices, bufs=bufs, @@ -300,8 +219,93 @@ def _execute_bwd(self, node, nb, bufs, stream) -> None: l2norm_bwd(q, q_rstd, dq, out=nb.outputs["dQ"], bufs=bufs, stream=stream) l2norm_bwd(k, k_rstd, dk, dy2=dk2, out=nb.outputs["dK"], bufs=bufs, stream=stream) else: - # dk/dk2 are the head-reduced finals for GVA, HV-head for MHA + # dK/dK2 are the head-reduced finals for GVA, HV-head for MHA n_dk = 1 - for s_ in dk.shape: - n_dk *= int(s_) + for s in dk.shape: + n_dk *= int(s) add_inplace(reshaped(dk, (n_dk,)), reshaped(dk2, (n_dk,)), n_dk, stream=stream) + + +class GdnCuTileEngine(BaseEngine): + """cuTile chunked-kernel backend for single-node GDN graphs (THD layout).""" + + name = "gdn_cutile" + behavior_notes = (behavior_note.RUNTIME_COMPILATION,) # JIT-compiled + autotuned on first execute per shape + + def check_support(self, graph: "pygraph") -> None: + import cudnn + + if buffers.current_sm() is None: + raise NotImplementedError("GdnCuTileEngine requires a CUDA device") + try: + from cuda.bindings import runtime + + err, cudart_version = runtime.cudaRuntimeGetVersion() + if int(err) != 0: + raise NotImplementedError(f"GdnCuTileEngine: cudaRuntimeGetVersion failed ({err})") + except ImportError as exc: + raise NotImplementedError(f"GdnCuTileEngine requires cuda.bindings: {exc}") from exc + if cudart_version < 13030: + raise NotImplementedError(f"GdnCuTileEngine requires CUDA 13.3+ (found {cudart_version})") + try: + from .kernels.gdn_chunk_cutile import chunk_gated_delta_rule # noqa: F401 — availability probe: ImportError = decline + except ImportError as exc: + raise NotImplementedError(f"GdnCuTileEngine requires the cuda.tile runtime: {exc}") from exc + + facts = graph._facts_for(analyze) + if facts is None or facts.op != "GDN": + raise NotImplementedError("GdnCuTileEngine supports exactly one GDN/GDN_BWD node") + if facts.invalid: + raise NotImplementedError(f"GdnCuTileEngine: {facts.invalid}") + if facts.checkpoint_every_n_tokens > 0 or facts.wants_state_checkpoints: + raise NotImplementedError("GdnCuTileEngine: per-chunk state_checkpoints output is not supported") + if facts.is_bwd and facts.safe_gate: + raise NotImplementedError("GdnCuTileEngine: safe_gate is forward-only") + f32 = cudnn.data_type.FLOAT + for port, got in ( + ("initial_state", facts.state_dtype), + ("final_state", facts.final_state_dtype), + ("d_final_state", facts.d_final_state_dtype), + ("d_initial_state", facts.d_initial_state_dtype), + ("a_log", facts.a_log_dtype), + ("dt_bias", facts.dt_bias_dtype), + ): + if got not in (f32, None): + raise NotImplementedError(f"GdnCuTileEngine: '{port}' must be fp32 (callers convert), got {got}") + if not facts.uniform_io: + raise NotImplementedError("GdnCuTileEngine: q/k/v dtypes must match") + if facts.io_dtype not in (cudnn.data_type.HALF, cudnn.data_type.BFLOAT16, None): + raise NotImplementedError(f"GdnCuTileEngine: q/k/v must be fp16/bf16, got {facts.io_dtype}") + if not facts.thd_layout: + raise NotImplementedError("GdnCuTileEngine: q/k/v must be THD [total_T, heads, dim]") + if facts.h_k != facts.h_q: + raise NotImplementedError(f"GdnCuTileEngine: q and k head counts differ ({facts.h_q} vs {facts.h_k})") + if facts.h_q and facts.h_v % facts.h_q != 0: + raise NotImplementedError( + f"GdnCuTileEngine: v heads ({facts.h_v}) must be a multiple of q heads ({facts.h_q}; GQA-style v broadcast is FROST-only)" + ) + if facts.d_qk > 256: + raise NotImplementedError(f"GdnCuTileEngine: head dim K must be <= 256, got {facts.d_qk}") + if facts.cu_dtype not in (cudnn.data_type.INT32, None): + raise NotImplementedError("GdnCuTileEngine: cu_seqlens must be int32 (the device-side table builder reads it directly)") + io = facts.io_dtype + f32 = cudnn.data_type.FLOAT + if not facts.is_bwd: + out_dtypes = {"O": (facts.o_dtype, io), "final_state": (facts.final_state_dtype, f32)} + else: + out_dtypes = { + "dQ": (facts.dq_dtype, io), + "dK": (facts.dk_dtype, io), + "dV": (facts.dv_dtype, io), + "dG": (facts.dg_dtype, f32), + "dBeta": (facts.dbeta_dtype, facts.beta_dtype), + "d_initial_state": (facts.d_initial_state_dtype, f32), + } + if facts.has_initial_state != facts.wants_d_initial_state: + raise NotImplementedError("GdnCuTileEngine: d_initial_state output must be requested iff initial_state is given") + for port, (got, want) in out_dtypes.items(): + if got is not None and got not in (want, None): + raise NotImplementedError(f"GdnCuTileEngine: output '{port}' must be {want} (written in place), got {got}") + + def build_plan(self, graph, plan, ctx=None) -> CompiledPlan: + return GdnCuTilePlan(graph) diff --git a/python/cudnn/linear_attention/cutile/kda_engine.py b/python/cudnn/linear_attention/cutile/kda_engine.py index 94d96886d..1373be1e3 100644 --- a/python/cudnn/linear_attention/cutile/kda_engine.py +++ b/python/cudnn/linear_attention/cutile/kda_engine.py @@ -1,269 +1,188 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""KDA (Kimi Delta Attention) execution backend: KDA / KDA_BWD nodes on the -chunked cuTile kernels (``kernels/kda_chunk_cutile``).""" +"""cuTile KDA engine: KDA / KDA_BWD nodes on the chunked cuTile kernels +(``kernels/kda_chunk_cutile``).""" -from typing import TYPE_CHECKING, Any, Dict +from typing import TYPE_CHECKING from cudnn import behavior_note from cudnn.engines.base import BaseEngine, CompiledPlan, resolve_node_buffers from cudnn.graph_types import NodeType + from cudnn.frost import buffers from cudnn.frost.workspace import Workspace -from cudnn.linear_attention.engine_utils import _dtype_name +from ..graph_analyzer import check_layouts_compact, analyze, expect_table, to_buffer_dtype -if TYPE_CHECKING: - from cudnn.pygraph import pygraph +# entry base-alignment expectations; ports not listed assume 16 +CUTILE_ALIGN = {"cu_seqlens": 4, "a_log": 4, "beta": 4} -_REQUIRED_PORTS = { - NodeType.KDA: ("q", "k", "v", "g", "beta", "cu_seqlens"), - NodeType.KDA_BWD: ("q", "k", "v", "g", "beta", "cu_seqlens", "dO"), -} +if TYPE_CHECKING: + from cudnn._pygraph import pygraph -def _node_ws_layout(node): +def node_ws_layout(node): """Static carve plan for one node's ``chunk_kda`` pipeline intermediates: - name -> (offset, dtype, shape). The chunk count is data-dependent - (varlen), so the 'h'/'dh' entries are SIZED with the upper bound - cdiv(total,64)+N and their shape carries None in the NT slot, substituted - at execute. A KDA_BWD node carries the union of the forward re-run's - intermediates and the backward temporaries, so both live disjointly in - one carve. Terminal pipeline buffers (the forward's ``o``/``fs``; the - backward's boundary casts / l2norm outputs, ``dv2`` and ``dh0``) are NOT - carved — execute plants the caller's output buffers under those names.""" - from .kernels.kda_chunk_cutile import _BT as BT, _cdiv, _next_power_of_2 + name -> (offset, dtype-name, shape). The chunk count is data-dependent + (varlen), so chunk-indexed entries are sized and SHAPED at the bound + ``cdiv(total, 64) + N`` — the device-built table's sentinel tail keeps + bound-gridded launches inert past the real count. A KDA_BWD node carries + the union of the forward re-run's intermediates and the backward + temporaries, so both live disjointly in one carve. Terminal pipeline + buffers (the forward's ``o``/``final_state``; the backward's boundary casts / + l2norm outputs, ``dv2`` and ``dstate0``) are NOT carved — execute plants the + caller's output buffers under those names.""" + from .kernels.kda_chunk_cutile import BT_CHUNK, cdiv, next_power_of_2 q, v, g, cu = (node.inputs[p] for p in ("q", "v", "g", "cu_seqlens")) total, H, K = q.dim HV, V = v.dim[1], v.dim[2] N = cu.dim[0] - 1 - io = _dtype_name(q.get_data_type()) + io = to_buffer_dtype(q.get_data_type()) f32 = "float32" BC = 32 if K >= 64 else 16 # fwd_intra sub-chunk (see chunk_kda_fwd_intra) - NK = _cdiv(K, min(64, _next_power_of_2(K))) # bwd_intra K-split (see chunk_kda_bwd_intra) - NT_bound = _cdiv(total, BT) + N + NT_bound = cdiv(total, BT_CHUNK) + N l2norm = bool(node.params.get("use_qk_l2norm", False)) size = 0 table = {} - def add(name, dtype, shape): + def carve(name, dtype, shape): nonlocal size nbytes = buffers.DTYPE_ITEMSIZE[dtype] - shape = tuple(NT_bound if s is None else int(s) for s in shape) for s in shape: - nbytes *= s - table[name] = (size, dtype, shape) + nbytes *= int(s) + table[name] = (size, dtype, tuple(int(s) for s in shape)) size += (nbytes + 127) & ~127 # 128B-aligned sequential carve - add("chunk_table", "int32", (NT_bound, 2)) - add("chunk_count", "int32", (1,)) - add("chunk_offsets", "int32", (N + 1,)) - add("dummy", "int32", (4,)) # inert stub backing for absent optional kernel args + carve("chunk_table", "int32", (NT_bound, 2)) + carve("chunk_count", "int32", (1,)) + carve("chunk_offsets", "int32", (N + 1,)) + carve("dummy", "int32", (4,)) # inert stub backing for absent optional kernel args # chunk_kda forward (also re-run inside KDA_BWD) - add("g_cum", f32, (total, HV, K)) - add("Aqk", io, (total, HV, BT)) - add("Akk", io, (total, HV, BT)) - add("Akkd", f32, (total, HV, BC)) - add("w", io, (total, HV, K)) - add("u", io, (total, HV, V)) - add("qg", io, (total, HV, K)) - add("kg", io, (total, HV, K)) - add("h", io, (None, HV, K, V)) - add("v_new", io, (total, HV, V)) + carve("g_cum", f32, (total, HV, K)) + carve("Aqk", io, (total, HV, BT_CHUNK)) + carve("Akk", io, (total, HV, BT_CHUNK)) + carve("Akkd", f32, (total, HV, BC)) + carve("w", io, (total, HV, K)) + carve("u", io, (total, HV, V)) + carve("qg", io, (total, HV, K)) + carve("kg", io, (total, HV, K)) + carve("state_checkpoints", io, (NT_bound, HV, K, V)) + carve("v_new", io, (total, HV, V)) + if node.params.get("use_beta_sigmoid", False): + carve("beta_sig", f32, (total, HV)) if node.node_type == NodeType.KDA_BWD: - add("o", io, (total, HV, V)) # discarded output of the forward re-run + carve("o", io, (total, HV, V)) # discarded output of the forward re-run if l2norm: - add("q_norm", io, (total * H, K)) - add("q_rstd", f32, (total * H,)) - add("k_norm", io, (total * H, K)) - add("k_rstd", f32, (total * H,)) + carve("q_norm", io, (total * H, K)) + carve("q_rstd", f32, (total * H,)) + carve("k_norm", io, (total * H, K)) + carve("k_rstd", f32, (total * H,)) if node.node_type == NodeType.KDA_BWD: - add("dAqk", f32, (total, HV, BT)) - add("dv_dAv", io, (total, HV, V)) - add("dh", io, (None, HV, K, V)) - add("dv_dhu", io, (total, HV, V)) - add("dq", f32, (total, HV, K)) - add("dk", f32, (total, HV, K)) - add("dg", f32, (total, HV, K)) - if _dtype_name(node.inputs["beta"].get_data_type()) != f32: - add("db", f32, (total, HV)) - add("dAkk", f32, (total, HV, BT)) - add("dq2", f32, (total, HV, K)) - add("dk2", f32, (total, HV, K)) + carve("dAqk", f32, (total, HV, BT_CHUNK)) + carve("dv_dAv", io, (total, HV, V)) + carve("dstate", io, (NT_bound, HV, K, V)) + carve("dv_dstate_u", io, (total, HV, V)) + carve("dq", f32, (total, HV, K)) + carve("dk", f32, (total, HV, K)) + carve("dg", f32, (total, HV, K)) + if to_buffer_dtype(node.inputs["beta"].get_data_type()) != f32: + carve("db", f32, (total, HV)) + carve("dAkk", f32, (total, HV, BT_CHUNK)) + carve("dq2", f32, (total, HV, K)) + carve("dk2", f32, (total, HV, K)) if HV != H: - add("dq_hred", f32, (total, H, K)) - add("dk_hred", f32, (total, H, K)) - add("db2", f32, (NK, total, HV)) - add("dg2", f32, (total, HV, K)) - if _dtype_name(g.get_data_type()) != f32: - add("dg_cum", f32, (total, HV, K)) + carve("dq_hred", f32, (total, H, K)) + carve("dk_hred", f32, (total, H, K)) + NK = cdiv(K, min(64, next_power_of_2(K))) # bwd_intra K-split (see chunk_kda_bwd_intra) + carve("db2", f32, (NK, total, HV)) + carve("dg2", f32, (total, HV, K)) + if to_buffer_dtype(g.get_data_type()) != f32: + carve("dg_cum", f32, (total, HV, K)) return size, table -class _KdaCuTilePlan(CompiledPlan): - """Carve plan over the caller's workspace (see GdnCuTileEngine's - ``_CuTilePlan``): static per-node layout; the buffer arrives with every - execute (the explicit-workspace convention).""" +class KdaCuTilePlan(CompiledPlan): + """Carve plan over the caller's workspace: the layout is static per node; + the buffer arrives with every execute.""" - def __init__(self, engine, graph): - self._engine = engine - self._layouts = [(node, *_node_ws_layout(node)) for node in graph.nodes] + def __init__(self, graph): + self.layouts = [(node, *node_ws_layout(node)) for node in graph.nodes] + self.expects = {node: expect_table(node, CUTILE_ALIGN) for node in graph.nodes} # nodes execute sequentially, each re-carving the same buffer - self._ws_bytes = max(nbytes for _, nbytes, _ in self._layouts) + self.ws_bytes = max(nbytes for _, nbytes, _ in self.layouts) def get_workspace_size(self) -> int: - return self._ws_bytes + return self.ws_bytes def execute(self, graph, uid_to_data, ctx) -> None: - self._engine._execute(resolve_node_buffers(graph, uid_to_data), self, ctx) - - -class KdaCuTileEngine(BaseEngine): - """cuTile chunked-kernel backend for single-node KDA graphs (THD layout).""" - - name = "kda_cutile" - behavior_notes = (behavior_note.RUNTIME_COMPILATION,) # JIT-compiled at build_plans() - - def check_support(self, graph: "pygraph") -> None: - if buffers.current_sm() is None: - raise NotImplementedError("KdaCuTileEngine requires a CUDA device") - try: - from cuda.bindings import runtime as _rt - - err, _cudart_version = _rt.cudaRuntimeGetVersion() - if int(err) != 0: - raise NotImplementedError(f"KdaCuTileEngine: cudaRuntimeGetVersion failed ({err})") - except ImportError as e: - raise NotImplementedError(f"KdaCuTileEngine requires cuda.bindings: {e}") - if _cudart_version < 13030: - raise NotImplementedError(f"KdaCuTileEngine requires CUDA 13.3+ (found {_cudart_version})") - try: - from .kernels.kda_chunk_cutile import ( # noqa: F401 - chunk_kda, - ) - except ImportError as e: - raise NotImplementedError(f"KdaCuTileEngine requires the cuda.tile runtime: {e}") - - import cudnn - - supported_dtypes = (cudnn.data_type.HALF, cudnn.data_type.BFLOAT16, None) - if not graph.nodes: - raise NotImplementedError("KdaCuTileEngine: empty graph") - for node in graph.nodes: - required = _REQUIRED_PORTS.get(node.node_type) - if required is None: - raise NotImplementedError(f"KdaCuTileEngine only supports KDA/KDA_BWD nodes, got {node.node_type.name}") - for port in required: - if port not in node.inputs: - raise NotImplementedError(f"KdaCuTileEngine: {node.node_type.name} node '{node.name}' is missing input '{port}'") - if int(node.params.get("checkpoint_every_n_tokens", 0) or 0) > 0 or "H" in node.outputs: - raise NotImplementedError("KdaCuTileEngine: per-chunk H output is not supported") - q, k, v = (node.inputs[p] for p in ("q", "k", "v")) - for p in ("q", "k", "v"): - t = node.inputs[p] - if t.get_data_type() not in supported_dtypes: - raise NotImplementedError(f"KdaCuTileEngine: '{p}' must be fp16/bf16, got {t.get_data_type()}") - if t.dim and len(t.dim) != 3: - raise NotImplementedError(f"KdaCuTileEngine: '{p}' must be THD [total_T, heads, dim], got rank {len(t.dim)}") - if q.dim and k.dim and q.dim[1] != k.dim[1]: - raise NotImplementedError(f"KdaCuTileEngine: q and k head counts differ ({q.dim[1]} vs {k.dim[1]})") - if q.dim and v.dim and v.dim[1] % q.dim[1] != 0: - raise NotImplementedError( - f"KdaCuTileEngine: v heads ({v.dim[1]}) must be a multiple of q heads ({q.dim[1]}; GQA-style v broadcast is FROST-only)" - ) - if q.dim and q.dim[-1] > 256: - raise NotImplementedError(f"KdaCuTileEngine: head dim K must be <= 256, got {q.dim[-1]}") - if node.inputs["cu_seqlens"].get_data_type() not in (cudnn.data_type.INT32, None): - raise NotImplementedError("KdaCuTileEngine: cu_seqlens must be int32 (the device-side table builder reads it directly)") - io = q.get_data_type() - f32 = cudnn.data_type.FLOAT - if node.node_type == NodeType.KDA: - out_dtypes = {"O": io, "final_state": f32} - required_out = ("O",) - else: - out_dtypes = { - "dQ": io, - "dK": io, - "dV": io, - "dG": node.inputs["g"].get_data_type(), - "dBeta": node.inputs["beta"].get_data_type(), - "d_initial_state": f32, - } - required_out = ("dQ", "dK", "dV", "dG", "dBeta") - if ("initial_state" in node.inputs) != ("d_initial_state" in node.outputs): - raise NotImplementedError("KdaCuTileEngine: d_initial_state output must be requested iff initial_state is given") - for port in required_out: - if port not in node.outputs: - raise NotImplementedError(f"KdaCuTileEngine: {node.node_type.name} node '{node.name}' is missing output '{port}'") - for port, want in out_dtypes.items(): - t = node.outputs.get(port) - if t is not None and t.get_data_type() not in (want, None): - raise NotImplementedError(f"KdaCuTileEngine: output '{port}' must be {want} (written in place), got {t.get_data_type()}") - - def build_plan(self, graph, plan, ctx=None) -> CompiledPlan: - return _KdaCuTilePlan(self, graph) - - def _execute(self, node_buffers, plan, ctx) -> None: from .kernels.common import build_chunk_table - from .kernels.kda_chunk_cutile import _BT + from .kernels.kda_chunk_cutile import BT_CHUNK - stream = getattr(ctx, "stream", None) - stream = 0 if stream is None else stream - ws = Workspace(getattr(ctx, "workspace", None), plan._ws_bytes, "KdaCuTileEngine") - for node, _nbytes, table in plan._layouts: + node_buffers = resolve_node_buffers(graph, uid_to_data) + stream = ctx.stream if ctx.stream is not None else 0 + ws = Workspace(ctx.workspace, self.ws_bytes, "KdaCuTileEngine") + for node, _nbytes, table in self.layouts: nb = node_buffers[node] + check_layouts_compact("KdaCuTileEngine", self.expects[node], nb) cu_seqlens = nb.inputs["cu_seqlens"] N = node.inputs["cu_seqlens"].dim[0] - 1 bufs = {name: ws.view(off, dt, shape) for name, (off, dt, shape) in table.items()} bound = bufs["chunk_table"].shape[0] - build_chunk_table(bufs["chunk_table"], bufs["chunk_count"], bufs["chunk_offsets"], cu_seqlens, N, _BT, bound, stream=stream) + build_chunk_table(bufs["chunk_table"], bufs["chunk_count"], bufs["chunk_offsets"], cu_seqlens, N, BT_CHUNK, bound, stream=stream) if node.node_type == NodeType.KDA: - self._execute_fwd(node, nb, bufs, stream) + self.execute_fwd(node, nb, bufs, stream) else: - self._execute_bwd(node, nb, bufs, stream) - - @staticmethod - def _state_f32(s0): - # the kernel wants the recurrent state in fp32; callers convert - if s0 is not None and not str(s0.dtype).endswith("float32"): - raise ValueError("KdaCuTileEngine: state ports must be fp32 (callers convert)") - return s0 + self.execute_bwd(node, nb, bufs, stream) - def _execute_fwd(self, node, nb, bufs, stream) -> None: + def execute_fwd(self, node, nb, bufs, stream) -> None: from .kernels.kda_chunk_cutile import chunk_kda want_state = "final_state" in node.outputs + q, k, v = nb.inputs["q"], nb.inputs["k"], nb.inputs["v"] + g, beta = nb.inputs["g"], nb.inputs["beta"] # terminal pipeline buffers = the caller's output buffers bufs["o"] = nb.outputs["O"] if want_state: - bufs["fs"] = nb.outputs["final_state"] + bufs["final_state"] = nb.outputs["final_state"] + raw_gate_kwargs = {} + if node.params.get("use_beta_sigmoid", False): + raw_gate_kwargs["use_beta_sigmoid_in_kernel"] = True + if node.params.get("safe_gate", False): + raw_gate_kwargs.update( + safe_gate=True, + use_gate_in_kernel=True, + lower_bound=float(node.params.get("gate_lower_bound") or -5.0), + A_log=nb.inputs["a_log"], + dt_bias=nb.inputs["dt_bias"], + ) chunk_kda( - nb.inputs["q"], - nb.inputs["k"], - nb.inputs["v"], - nb.inputs["g"], - nb.inputs["beta"], + q, + k, + v, + g, + beta, scale=node.params.get("scale"), - initial_state=self._state_f32(nb.inputs.get("initial_state")), + initial_state=nb.inputs.get("initial_state"), output_final_state=want_state, use_qk_l2norm_in_kernel=bool(node.params.get("use_qk_l2norm", False)), cu_seqlens=nb.inputs["cu_seqlens"], chunk_indices=bufs["chunk_table"], bufs=bufs, stream=stream, + **raw_gate_kwargs, ) - def _execute_bwd(self, node, nb, bufs, stream) -> None: + def execute_bwd(self, node, nb, bufs, stream) -> None: from .kernels.common import reshaped from .kernels.kda_chunk_cutile import chunk_kda_grad total, H, K = node.inputs["q"].dim cu_seqlens = nb.inputs["cu_seqlens"] initial_state = nb.inputs.get("initial_state") - dht = nb.inputs.get("d_final_state") + dstate_in = nb.inputs.get("d_final_state") do, q, k, v = nb.inputs["dO"], nb.inputs["q"], nb.inputs["k"], nb.inputs["v"] g, beta = nb.inputs["g"], nb.inputs["beta"] scale = node.params.get("scale") or K**-0.5 @@ -280,7 +199,7 @@ def _execute_bwd(self, node, nb, bufs, stream) -> None: bufs["dg_cast" if "dg_cum" in bufs else "dg_cum"] = nb.outputs["dG"] bufs["db_cast" if "db" in bufs else "db"] = nb.outputs["dBeta"] if initial_state is not None: - bufs["dh0"] = nb.outputs["d_initial_state"] + bufs["dstate0"] = nb.outputs["d_initial_state"] chunk_kda_grad( q, @@ -289,12 +208,100 @@ def _execute_bwd(self, node, nb, bufs, stream) -> None: g, beta, do, - dht=self._state_f32(dht), + dstate_in=dstate_in, scale=scale, - initial_state=self._state_f32(initial_state), + initial_state=initial_state, use_qk_l2norm_in_kernel=bool(node.params.get("use_qk_l2norm", False)), cu_seqlens=cu_seqlens, chunk_indices=bufs["chunk_table"], bufs=bufs, stream=stream, ) + + +class KdaCuTileEngine(BaseEngine): + """cuTile chunked-kernel backend for single-node KDA graphs (THD layout).""" + + name = "kda_cutile" + behavior_notes = (behavior_note.RUNTIME_COMPILATION,) # JIT-compiled + autotuned on first execute per shape + + def check_support(self, graph: "pygraph") -> None: + import cudnn + + if buffers.current_sm() is None: + raise NotImplementedError("KdaCuTileEngine requires a CUDA device") + try: + from cuda.bindings import runtime + + err, cudart_version = runtime.cudaRuntimeGetVersion() + if int(err) != 0: + raise NotImplementedError(f"KdaCuTileEngine: cudaRuntimeGetVersion failed ({err})") + except ImportError as exc: + raise NotImplementedError(f"KdaCuTileEngine requires cuda.bindings: {exc}") from exc + if cudart_version < 13030: + raise NotImplementedError(f"KdaCuTileEngine requires CUDA 13.3+ (found {cudart_version})") + try: + from .kernels.kda_chunk_cutile import chunk_kda # noqa: F401 — availability probe: ImportError = decline + except ImportError as exc: + raise NotImplementedError(f"KdaCuTileEngine requires the cuda.tile runtime: {exc}") from exc + + facts = graph._facts_for(analyze) + if facts is None or facts.op != "KDA": + raise NotImplementedError("KdaCuTileEngine supports exactly one KDA/KDA_BWD node") + if facts.invalid: + raise NotImplementedError(f"KdaCuTileEngine: {facts.invalid}") + if facts.checkpoint_every_n_tokens > 0 or facts.wants_state_checkpoints: + raise NotImplementedError("KdaCuTileEngine: per-chunk state_checkpoints output is not supported") + if facts.is_bwd and (facts.safe_gate or facts.use_beta_sigmoid): + raise NotImplementedError("KdaCuTileEngine: raw-logit gate modes (safe_gate / use_beta_sigmoid) are forward-only") + f32 = cudnn.data_type.FLOAT + for port, got in ( + ("initial_state", facts.state_dtype), + ("final_state", facts.final_state_dtype), + ("d_final_state", facts.d_final_state_dtype), + ("d_initial_state", facts.d_initial_state_dtype), + ("a_log", facts.a_log_dtype), + ("dt_bias", facts.dt_bias_dtype), + ): + if got not in (f32, None): + raise NotImplementedError(f"KdaCuTileEngine: '{port}' must be fp32 (callers convert), got {got}") + glb = graph.nodes and list(graph.nodes)[0].params.get("gate_lower_bound") + if glb is not None and glb is not False and float(glb) >= 0: + raise NotImplementedError(f"KdaCuTileEngine: gate_lower_bound must be negative (log-gate floor), got {glb}") + if not facts.uniform_io: + raise NotImplementedError("KdaCuTileEngine: q/k/v dtypes must match") + if facts.io_dtype not in (cudnn.data_type.HALF, cudnn.data_type.BFLOAT16, None): + raise NotImplementedError(f"KdaCuTileEngine: q/k/v must be fp16/bf16, got {facts.io_dtype}") + if not facts.thd_layout: + raise NotImplementedError("KdaCuTileEngine: q/k/v must be THD [total_T, heads, dim]") + if facts.h_k != facts.h_q: + raise NotImplementedError(f"KdaCuTileEngine: q and k head counts differ ({facts.h_q} vs {facts.h_k})") + if facts.h_q and facts.h_v % facts.h_q != 0: + raise NotImplementedError( + f"KdaCuTileEngine: v heads ({facts.h_v}) must be a multiple of q heads ({facts.h_q}; GQA-style v broadcast is FROST-only)" + ) + if facts.d_qk > 256: + raise NotImplementedError(f"KdaCuTileEngine: head dim K must be <= 256, got {facts.d_qk}") + if facts.cu_dtype not in (cudnn.data_type.INT32, None): + raise NotImplementedError("KdaCuTileEngine: cu_seqlens must be int32 (the device-side table builder reads it directly)") + io = facts.io_dtype + f32 = cudnn.data_type.FLOAT + if not facts.is_bwd: + out_dtypes = {"O": (facts.o_dtype, io), "final_state": (facts.final_state_dtype, f32)} + else: + out_dtypes = { + "dQ": (facts.dq_dtype, io), + "dK": (facts.dk_dtype, io), + "dV": (facts.dv_dtype, io), + "dG": (facts.dg_dtype, facts.g_dtype), + "dBeta": (facts.dbeta_dtype, facts.beta_dtype), + "d_initial_state": (facts.d_initial_state_dtype, f32), + } + if facts.has_initial_state != facts.wants_d_initial_state: + raise NotImplementedError("KdaCuTileEngine: d_initial_state output must be requested iff initial_state is given") + for port, (got, want) in out_dtypes.items(): + if got is not None and got not in (want, None): + raise NotImplementedError(f"KdaCuTileEngine: output '{port}' must be {want} (written in place), got {got}") + + def build_plan(self, graph, plan, ctx=None) -> CompiledPlan: + return KdaCuTilePlan(graph) diff --git a/python/cudnn/linear_attention/cutile/kernels/common.py b/python/cudnn/linear_attention/cutile/kernels/common.py index 0b1edb32f..83c33ecd3 100644 --- a/python/cudnn/linear_attention/cutile/kernels/common.py +++ b/python/cudnn/linear_attention/cutile/kernels/common.py @@ -33,10 +33,10 @@ ConstInt = ct.Constant[int] -_TILE = 2048 +TILE = 2048 -def _cdiv(a: int, b: int) -> int: +def cdiv(a: int, b: int) -> int: return (a + b - 1) // b @@ -51,20 +51,46 @@ def zero_fill(buf, *, stream) -> None: memset_zero_async(ptr, n, stream) -def reshaped(buf, shape): - """A ``shape``-d view of a contiguous device buffer: native ``reshape`` - when the object has one (torch & co.), else a fresh DLPack view over the - same pointer (the kernels derive their index rank from the array rank).""" - if hasattr(buf, "reshape"): - return buf.reshape(*shape) +def reshaped(buf, target_shape): + """A ``target_shape``-d DeviceView over the same pointer (contiguous by + the engine gate's contract; the kernels derive their index rank from the + array rank).""" from cudnn.frost.buffers import DeviceView, probe - ptr, _shape, _strides, dtype, dev = probe(buf) - return DeviceView(ptr, shape, dtype, dev) + ptr, shape, _strides, dtype, dev = probe(buf) + return DeviceView(ptr, shape, dtype, dev).reshape(tuple(target_shape)) + + +def dummy(dtype_name: str, bufs): + """Inert typed view over the workspace's 16-byte ``dummy`` carve, for + ABSENT optional kernel args (always paired with a flag==0, never + dereferenced). Dtype-bound so the compiled signature stays stable; the + library allocates nothing.""" + from cudnn.frost.buffers import DTYPE_ITEMSIZE, DeviceView + + d = bufs["dummy"] + return DeviceView(d.data_ptr(), (16 // DTYPE_ITEMSIZE[dtype_name],), dtype_name, d.__dlpack_device__()[1]) + + +def opt(t, bufs, dtype_name: str = "float32"): + """Resolve an optional tensor argument to a non-null cuTile launch arg: + the buffer if present (contiguous by the engine contract), else an inert + dummy (paired with a USE_*/HAS_* integer flag). cuTile never accepts None + in launch args, so this is the required dummy-tensor-plus-flag pattern.""" + if t is None: + return dummy(dtype_name, bufs) + return t + + +def dev_id(buf) -> int: + """Device ordinal of a DLPack/CAI buffer.""" + from cudnn.frost.buffers import probe + + return probe(buf)[4] @ct.kernel -def _add_inplace_kernel(dst, src, TILE: ConstInt): +def add_inplace_kernel(dst, src, TILE: ConstInt): pid = ct.bid(0) a = ct.load(dst, index=(pid,), shape=(TILE,), padding_mode=ct.PaddingMode.ZERO) b = ct.load(src, index=(pid,), shape=(TILE,), padding_mode=ct.PaddingMode.ZERO) @@ -73,11 +99,11 @@ def _add_inplace_kernel(dst, src, TILE: ConstInt): def add_inplace(dst, src, numel: int, *, stream) -> None: """``dst += src`` over ``numel`` flat elements (same dtype, contiguous).""" - ct.launch(stream, (_cdiv(numel, _TILE),), _add_inplace_kernel, (dst, src, _TILE)) + ct.launch(stream, (cdiv(numel, TILE),), add_inplace_kernel, (dst, src, TILE)) @ct.kernel -def _cast_copy_kernel(dst, src, TILE: ConstInt): +def cast_copy_kernel(dst, src, TILE: ConstInt): pid = ct.bid(0) t = ct.load(src, index=(pid,), shape=(TILE,), padding_mode=ct.PaddingMode.ZERO) ct.store(dst, index=(pid,), tile=ct.astype(t, dst.dtype)) @@ -86,11 +112,11 @@ def _cast_copy_kernel(dst, src, TILE: ConstInt): def cast_copy(dst, src, numel: int, *, stream) -> None: """``dst[:] = src`` over ``numel`` flat elements, converting to ``dst``'s dtype (a plain copy when the dtypes already match).""" - ct.launch(stream, (_cdiv(numel, _TILE),), _cast_copy_kernel, (dst, src, _TILE)) + ct.launch(stream, (cdiv(numel, TILE),), cast_copy_kernel, (dst, src, TILE)) @ct.kernel -def _sum_leading_kernel(dst, src, R: ConstInt, ACC: ConstInt, TILE: ConstInt): +def sum_leading_kernel(dst, src, R: ConstInt, ACC: ConstInt, TILE: ConstInt): pid = ct.bid(0) acc = ct.astype(ct.load(src, index=(0, pid), shape=(1, TILE), padding_mode=ct.PaddingMode.ZERO), ct.float32) for r in range(1, R): @@ -106,11 +132,11 @@ def sum_leading(dst, src, r: int, m: int, *, stream, accumulate: bool = False) - axis into ``dst`` (flat ``[m]``), accumulating in fp32. ``r`` is a compile-time constant (small fan-ins: split partials, head groups). ``accumulate`` adds the reduction onto ``dst`` instead of overwriting.""" - ct.launch(stream, (_cdiv(m, _TILE),), _sum_leading_kernel, (dst, src, r, int(accumulate), _TILE)) + ct.launch(stream, (cdiv(m, TILE),), sum_leading_kernel, (dst, src, r, int(accumulate), TILE)) @ct.kernel -def _build_chunk_table_kernel(cu_seqlens, table, count, offsets, N: ConstInt, CS: ConstInt, BOUND: ConstInt): +def build_chunk_table_kernel(cu_seqlens, table, count, offsets, N: ConstInt, CS: ConstInt, BOUND: ConstInt): run = 0 last = N - 1 ct.store(offsets, (0,), 0) @@ -147,11 +173,11 @@ def build_chunk_table(table, count, offsets, cu_seqlens, n_seqs: int, chunk_size chunk grids at ``bound`` unchanged. ``count`` (one int32) receives the real chunk count; ``offsets`` (int32 ``[n_seqs + 1]``) receives the per-sequence chunk prefix (``prepare_chunk_offsets`` semantics).""" - ct.launch(stream, (1,), _build_chunk_table_kernel, (cu_seqlens, reshaped(table, (2 * bound,)), count, offsets, n_seqs, chunk_size, bound)) + ct.launch(stream, (1,), build_chunk_table_kernel, (cu_seqlens, reshaped(table, (2 * bound,)), count, offsets, n_seqs, chunk_size, bound)) @ct.kernel -def _head_group_sum_kernel(dst, src, G: ConstInt, BT: ConstInt, BK: ConstInt): +def head_group_sum_kernel(dst, src, G: ConstInt, BT: ConstInt, BK: ConstInt): t = ct.bid(0) h = ct.bid(1) k = ct.bid(2) @@ -166,4 +192,4 @@ def head_group_sum(dst, src, t: int, h: int, g: int, k: int, *, stream) -> None: ``src`` is a 3-D ``[t, h*g, k]`` buffer, ``dst`` 3-D ``[t, h, k]``; fp32 accumulation, ``g`` consecutive heads per group (compile-time).""" BT, BK = 64, 128 # padded loads + clipped stores absorb ragged t/k - ct.launch(stream, (_cdiv(t, BT), h, _cdiv(k, BK)), _head_group_sum_kernel, (dst, src, g, BT, BK)) + ct.launch(stream, (cdiv(t, BT), h, cdiv(k, BK)), head_group_sum_kernel, (dst, src, g, BT, BK)) diff --git a/python/cudnn/linear_attention/cutile/kernels/gdn_chunk_cutile.py b/python/cudnn/linear_attention/cutile/kernels/gdn_chunk_cutile.py index 827006aa5..0828a7f51 100644 --- a/python/cudnn/linear_attention/cutile/kernels/gdn_chunk_cutile.py +++ b/python/cudnn/linear_attention/cutile/kernels/gdn_chunk_cutile.py @@ -20,9 +20,10 @@ from types import SimpleNamespace import cuda.tile as ct +from cuda.tile.tune import exhaustive_search -from .common import add_inplace, head_group_sum, reshaped, sum_leading, zero_fill -from cudnn.frost.buffers import dtype_name as _dtname +from .common import add_inplace, dev_id, dummy, head_group_sum, opt, reshaped, sum_leading, zero_fill +from cudnn.frost.buffers import dtype_name as dtname logger = logging.getLogger(__name__) @@ -31,36 +32,31 @@ RCP_LN2 = 1.4426950216 # 1/ln(2) # chunk size (BT tile) of these kernels; the engine's carve layout imports it -_BT = 64 +BT_CHUNK = 64 # Host-side utilities -def _cdiv(a: int, b: int) -> int: +def cdiv(a: int, b: int) -> int: return (a + b - 1) // b -def _next_power_of_2(n: int) -> int: +def next_power_of_2(n: int) -> int: return 1 << (n - 1).bit_length() # Launch-hint autotuning (occupancy x num_worker_warps) for hot kernels. -# These large unrolled tiles emit 255-reg kernels defaulting to occupancy=1, which -# starves the SM; a higher occupancy hint budgets fewer regs/thread -> more blocks/SM. -# Tuned once per (kernel, grid-shape) key and cached. -from cuda.tile.tune import exhaustive_search # noqa: E402 -_LAUNCH_HINT_CACHE: dict = {} +LAUNCH_HINT_CACHE: dict = {} -# occ=4/nww=4 = consistent winner; nww=8 and default kept as fallbacks. # NOTE: the upstream occupancy=2 config is EXCLUDED: on tileiras 13.2 (which # ignores num_worker_warps hints) a freshly compiled occ=2 # chunk_bwd_kernel_dqkwg deadlocks on its first launch (deterministic; # occ=1/4 are fine, and occ=2 is fine on the dhu/kkt kernels). Restore the # SimpleNamespace(occupancy=2, num_worker_warps=4) entry once the runtime is # on tileiras >= 13.3. -_LAUNCH_HINT_CONFIGS = [ +LAUNCH_HINT_CONFIGS = [ SimpleNamespace(occupancy=4, num_worker_warps=4), SimpleNamespace(occupancy=4, num_worker_warps=8), SimpleNamespace(occupancy=1, num_worker_warps=8), @@ -68,47 +64,47 @@ def _next_power_of_2(n: int) -> int: ] -def _ensure_cuda_context(): +def ensure_cuda_context(): """Make the calling thread's CUDA driver context current. exhaustive_search times configs through the driver API, which fails on threads with an empty driver context stack (e.g. autograd backward worker threads). Retain + set-current the device primary context. Best-effort: never fatal.""" try: - from cuda.bindings import driver as _drv + from cuda.bindings import driver as drv - err, cur = _drv.cuCtxGetCurrent() - if err == _drv.CUresult.CUDA_SUCCESS and int(cur) != 0: + err, cur = drv.cuCtxGetCurrent() + if err == drv.CUresult.CUDA_SUCCESS and int(cur) != 0: return - from cuda.bindings import runtime as _rt + from cuda.bindings import runtime as rt - err_d, dev = _rt.cudaGetDevice() + err_d, dev = rt.cudaGetDevice() if int(err_d) != 0: return - err, ctx = _drv.cuDevicePrimaryCtxRetain(dev) - if err == _drv.CUresult.CUDA_SUCCESS: - _drv.cuCtxSetCurrent(ctx) + err, ctx = drv.cuDevicePrimaryCtxRetain(dev) + if err == drv.CUresult.CUDA_SUCCESS: + drv.cuCtxSetCurrent(ctx) except Exception: # noqa: BLE001 pass -def _device_attrs(): +def device_attrs(): """(sm_count, cc_major) of the current device via the runtime API (which auto-inits the primary context, so this works before any framework call).""" try: - from cuda.bindings import runtime as _rt + from cuda.bindings import runtime as rt - err, dev = _rt.cudaGetDevice() + err, dev = rt.cudaGetDevice() if int(err) != 0: return 0, 0 - err, sm = _rt.cudaDeviceGetAttribute(_rt.cudaDeviceAttr.cudaDevAttrMultiProcessorCount, dev) - err2, major = _rt.cudaDeviceGetAttribute(_rt.cudaDeviceAttr.cudaDevAttrComputeCapabilityMajor, dev) + err, sm = rt.cudaDeviceGetAttribute(rt.cudaDeviceAttr.cudaDevAttrMultiProcessorCount, dev) + err2, major = rt.cudaDeviceGetAttribute(rt.cudaDeviceAttr.cudaDevAttrComputeCapabilityMajor, dev) return (int(sm) if int(err) == 0 else 0), (int(major) if int(err2) == 0 else 0) except Exception: # noqa: BLE001 return 0, 0 -def _tuned_launch(kernel, stream, grid, args, cache_key, configs=None): +def tuned_launch(kernel, stream, grid, args, cache_key, configs=None): """Launch ``kernel`` with autotuned (occupancy, num_worker_warps) hints. Tunes once per ``cache_key`` via exhaustive_search, caches the specialized @@ -118,13 +114,13 @@ def _tuned_launch(kernel, stream, grid, args, cache_key, configs=None): """ kname = getattr(getattr(kernel, "_pyfunc", None), "__name__", repr(kernel)) key = (kname, cache_key) - if key not in _LAUNCH_HINT_CACHE: + if key not in LAUNCH_HINT_CACHE: tuned = None - _ensure_cuda_context() + ensure_cuda_context() try: with ct.compiler_timeout(20): result = exhaustive_search( - list(configs if configs is not None else _LAUNCH_HINT_CONFIGS), + list(configs if configs is not None else LAUNCH_HINT_CONFIGS), stream, lambda cfg: grid, kernel, @@ -136,8 +132,8 @@ def _tuned_launch(kernel, stream, grid, args, cache_key, configs=None): except Exception as e: # noqa: BLE001 logger.warning("launch-hint autotune failed for %s: %s; using default hints", kname, e) tuned = kernel - _LAUNCH_HINT_CACHE[key] = tuned - ct.launch(stream, grid, _LAUNCH_HINT_CACHE[key], args) + LAUNCH_HINT_CACHE[key] = tuned + ct.launch(stream, grid, LAUNCH_HINT_CACHE[key], args) def exp(x): @@ -153,21 +149,21 @@ def softplus(x): return ct.where(x < 20.0, ct.log(1.0 + ct.exp(x)), x) -def _tf32(a): +def tf32(a): """ct.mma/ct.matmul do not auto-cast fp32 operands to tf32; cast explicitly (allow-tf32 matmul semantics).""" return ct.astype(a, ct.tfloat32) if a.dtype == ct.float32 else a def safe_matmul(a, b): - return ct.matmul(_tf32(a), _tf32(b)) + return ct.matmul(tf32(a), tf32(b)) safe_dot = safe_matmul # alias: non-accumulating dot with fp32->tf32 guard def safe_mma(a, b, acc): - return ct.mma(_tf32(a), _tf32(b), acc) + return ct.mma(tf32(a), tf32(b), acc) def gather_flat(raw, off, *, mask=None, padding_value=0, check_bounds=False): @@ -178,7 +174,7 @@ def scatter_flat(raw, off, value, *, mask=None, check_bounds=False): raw.store_offset(off, value, mask=mask) -def _array_numel(arr): +def array_numel(arr): # Total element count; rank statically unrolled (<=5) since a Python for over # arr.shape is captured as a device loop and rejected. K<=256 bounds the rank. s = arr.shape @@ -196,7 +192,7 @@ def _array_numel(arr): # Bounds-checked flat access: 1-D bounds mask `0 <= off < numel` AND-ed with any -# caller mask. Pass `numel = _array_numel(arr)`. +# caller mask. Pass `numel = array_numel(arr)`. def gather_flat_cb(raw, off, numel, *, mask=None, padding_value=0): inb = (off >= 0) & (off < numel) m = inb if mask is None else (mask & inb) @@ -247,7 +243,7 @@ def l2norm_bwd_kernel1(y, rstd, dy, dy2, dx, eps, D, BD: ConstInt, HAS_DY2: Cons @ct.kernel def l2norm_fwd_kernel(x, y, rstd, eps, T, D: ConstInt, BD: ConstInt, BT: ConstInt): # D <= 512 path: BT rows per block, BD power-of-2 cols. Block-aligned -> - # ct.load with block index + ZERO padding (faithful to make_block_ptr). + # ct.load with block index + ZERO padding. i_t = ct.bid(0) b_x = ct.astype(ct.load(x, index=(i_t, 0), shape=(BT, BD), padding_mode=ct.PaddingMode.ZERO), ct.float32) b_rstd = ct.rsqrt(ct.sum(b_x * b_x, axis=1) + eps) @@ -413,7 +409,7 @@ def gdn_gate_bwd_kernel(g, A_log, dt_bias, dyg, dg, dA, db, T, H: ConstInt, BT: ct.scatter(db, i_t * H + i_h, b_db) -# chunk_gated_delta_rule_fwd_kkt_solve_kernel (fused kkt + solve, 4 sub-chunks) +# chunk_gated_delta_rule_fwd_kkt_solve_kernel (fused KK^T + solve, 4 sub-chunks) @ct.kernel def chunk_gated_delta_rule_fwd_kkt_solve_kernel( k, @@ -430,7 +426,7 @@ def chunk_gated_delta_rule_fwd_kkt_solve_kernel( BK: ConstInt, USE_G: ConstInt, ): - """Fused: beta * K @ K^T (lower triangular) + solve_tril (I+A)^{-1} in one pass.""" + """Fused: Beta * K @ K^T (lower triangular) + solve_tril (I+A)^{-1} in one pass.""" i_t = ct.bid(0) i_h = ct.bid(1) @@ -568,7 +564,7 @@ def chunk_gated_delta_rule_fwd_kkt_solve_kernel( b_A31 = ct.mma(b_k3, ct.transpose(b_k1), b_A31) b_A32 = ct.mma(b_k3, ct.transpose(b_k2), b_A32) - # -- Step 2: gate + beta scaling -- + # -- Step 2: Gate + Beta scaling -- m_d = ct.expand_dims(o_i, 1) > ct.expand_dims(o_i, 0) m_I = ct.expand_dims(o_i, 1) == ct.expand_dims(o_i, 0) @@ -821,13 +817,12 @@ def recompute_w_u_fwd_kernel( beta_seg = beta.slice(axis=0, start=bos, stop=eos) Z = ct.PaddingMode.ZERO - # Keep beta native bf16: scaling in bf16 avoids 2 extra ftof + # Keep Beta native bf16: scaling in bf16 avoids 2 extra ftof # converts/MMA-input that an f32 cast would force. b_b = ct.load(beta_seg, index=(i_t_loc, i_h), shape=(BT, 1), padding_mode=Z).reshape((BT,)) b_A = ct.load(A_seg, index=(i_t_loc, i_h, 0), shape=(BT, 1, BT), padding_mode=Z).reshape((BT, BT)) - # u = A @ (v * beta). latency=3 -> deeper pipeline of the per-tile TMA - # loads; small win over latency=2, no spill. + # U = A @ (V * Beta). latency=3 -> deeper pipeline of the per-tile TMA loads. for i_v in range(ct.cdiv(V, BV)): b_v = ct.load(v_seg, index=(i_t_loc, i_h, i_v), shape=(BT, 1, BV), padding_mode=Z, latency=3).reshape((BT, BV)) # bf16 * bf16 -> bf16. @@ -844,7 +839,7 @@ def recompute_w_u_fwd_kernel( ) b_g = exp2(b_g_val) - # w = A @ (k * beta * g) + # W = A @ (K * Beta * Gate) for i_k in range(ct.cdiv(K, BK)): b_k = ct.load(k_seg, index=(i_t_loc, i_kh, i_k), shape=(BT, 1, BK), padding_mode=Z, latency=3).reshape((BT, BK)) # bf16 * bf16 -> bf16; only the g-decay path promotes to f32. @@ -1122,7 +1117,7 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( o_bv = ct.arange(BV, dtype=ct.int32) m_v_lane = (i_v * BV + o_bv) < V - # --- Load initial state h0 -> b_h1..b_h4 --- + # --- Load initial state H0 -> b_h1..b_h4 --- if USE_INITIAL_STATE: if STATE_V_FIRST: row = (i_v * BV + o_bv)[:, None] @@ -1255,7 +1250,7 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( check_bounds=True, ) - # --- b_v = b_w @ b_h (accumulated over K blocks) --- + # --- V = W @ H (accumulated over K blocks) --- w_time = (t_base + i_t * BT + o_bt)[:, None] wmask_r = (i_t * BT + o_bt) < T bw1 = ct.gather( @@ -1409,7 +1404,7 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( b_v = ct.astype(b_v, k.dtype) - # --- b_h += b_k @ b_v (b_k is (64, BT) transposed load of k) --- + # --- H += K^T @ V (K loaded transposed as (64, BT)) --- k_time = (t_base + i_t * BT + o_bt)[None, :] kmask = ((0 + o64)[:, None] < K) & ((i_t * BT + o_bt)[None, :] < T) bk1 = ct.gather( @@ -1476,7 +1471,7 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( else: b_h4 = b_h4 + prod4 - # --- Store final state ht <- b_h* (ht (N, HV, *), per-sequence state) --- + # --- Store final state Ht <- b_h* (Ht (N, HV, *), per-sequence state) --- if STORE_FINAL_STATE: if STATE_V_FIRST: row = (i_v * BV + o_bv)[:, None] @@ -1513,7 +1508,7 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( w, g, gk, - dht, + dstate_in, dh0, do, dh, @@ -1570,34 +1565,34 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( dht_base = i_nh * K * V # Raw flat handles + element counts for 1-D flat gather/scatter (cuTile 1.4.0). - dhtf = dht.get_raw_memory() - dht_n = _array_numel(dht) + dhtf = dstate_in.get_raw_memory() + dht_n = array_numel(dstate_in) dhf = dh.get_raw_memory() - dh_n = _array_numel(dh) + dh_n = array_numel(dh) dh0f = dh0.get_raw_memory() - dh0_n = _array_numel(dh0) + dh0_n = array_numel(dh0) gf = g.get_raw_memory() - g_n = _array_numel(g) + g_n = array_numel(g) gkf = gk.get_raw_memory() - gk_n = _array_numel(gk) + gk_n = array_numel(gk) dof = do.get_raw_memory() - do_n = _array_numel(do) + do_n = array_numel(do) kf = k.get_raw_memory() - k_n = _array_numel(k) + k_n = array_numel(k) qf = q.get_raw_memory() - q_n = _array_numel(q) + q_n = array_numel(q) wf = w.get_raw_memory() - w_n = _array_numel(w) + w_n = array_numel(w) dvf = dv.get_raw_memory() - dv_n = _array_numel(dv) + dv_n = array_numel(dv) dv2f = dv2.get_raw_memory() - dv2_n = _array_numel(dv2) + dv2_n = array_numel(dv2) o64 = ct.arange(64, dtype=ct.int32) o_bt = ct.arange(BT, dtype=ct.int32) o_bv = ct.arange(BV, dtype=ct.int32) - # --- Load final-state gradient dht -> b_dh* --- + # --- Load final-state gradient dHt -> b_dh* --- if USE_FINAL_STATE_GRADIENT: if STATE_V_FIRST: row = (i_v * BV + o_bv)[:, None] @@ -1716,7 +1711,7 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( b_do = gather_flat_cb(dof, do_off, do_n, padding_value=0.0) b_do = ct.where(do_full, b_do, ct.zeros((BT, BV), dtype=b_do.dtype)) - # b_dv = sum_n b_k_n @ b_dh_n + # dV = sum_n K_n @ dH_n k_row = (i_t * BT + o_bt)[:, None] kmask_r = (i_t * BT + o_bt) < T bk1 = gather_flat_cb(kf, k_base + k_row * (H * K) + (0 + o64)[None, :], k_n, padding_value=0.0) @@ -1796,7 +1791,7 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( dv2_off = ct.where(do_full, dv2_off, dv2_oob) scatter_flat_cb(dv2f, dv2_off, ct.astype(b_dv, dv2.dtype), dv2_n) - # --- b_dh += trans?(q@do*scale - w@dv) per K block --- + # --- dH += trans?(Q @ dO * scale - W @ dV) per K block --- time = (i_t * BT + o_bt)[None, :] tmask = (i_t * BT + o_bt)[None, :] < T b_dv_cast = ct.astype(b_dv, ct.float32) @@ -1901,7 +1896,7 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( else: b_dh4 = b_dh4 + upd4 - # --- Store initial-state gradient dh0 <- b_dh* --- + # --- Store initial-state gradient dH0 <- b_dh* --- if USE_INITIAL_STATE: # Mask both the 64-wide K-block rows and BV-wide V cols; flat numel check # alone would let an over-range K row spill into the next head's dh0 slot. @@ -2028,16 +2023,15 @@ def chunk_fwd_kernel_o( b_A = ct.zeros((BT, BT), dtype=ct.float32) for i_k in range(ct.cdiv(K, BK)): - # latency=2 pipelining on this K-loop's TMA loads. b_q = ct.load(q_seg, index=(i_t, i_kh, i_k), shape=(BT, 1, BK), padding_mode=Z, latency=2).reshape((BT, BK)) b_k = ct.load(k_seg, index=(i_t, i_kh, i_k), shape=(BT, 1, BK), padding_mode=Z, latency=2).reshape((BT, BK)) if STATE_V_FIRST: - # b_h (BV, BK), o += q @ h^T + # b_h (BV, BK), O += Q @ H^T b_h = ct.load(h, index=(i_hc, i_v, i_k), shape=(1, BV, BK), padding_mode=Z, latency=2).reshape((BV, BK)) b_o = safe_mma(b_q, ct.transpose(b_h), b_o) else: - # b_h (BK, BV), o += q @ h + # b_h (BK, BV), O += Q @ H b_h = ct.load(h, index=(i_hc, i_k, i_v), shape=(1, BK, BV), padding_mode=Z, latency=2).reshape((BK, BV)) b_o = safe_mma(b_q, b_h, b_o) b_A = safe_mma(b_q, ct.transpose(b_k), b_A) @@ -2094,7 +2088,7 @@ def chunk_bwd_kernel_dqkwg( NV, # runtime cdiv(V,BV) -> rolled V-loop (avoids unroll reg-spill at K=256) ): # q/k/v/do/dv/dq/dk are (T,*,D) token-packed slabs, h/dh (NT_total*HV,*,*), - # dg (NK,T,HV); block-indexed ct.load -> TMA. latency=2. + # dg (NK,T,HV); block-indexed ct.load -> TMA. i_k = ct.bid(0) i_t = ct.bid(1) i_h = ct.bid(2) @@ -2331,30 +2325,6 @@ def chunk_bwd_kernel_dv_local( # an integer flag (USE_*/HAS_*). -def _dummy(dtype_name: str, bufs): - """Inert typed view over the workspace's 16-byte ``dummy`` carve, for - ABSENT optional kernel args (always paired with a flag==0, never - dereferenced). Dtype-bound so the compiled signature stays stable; the - library allocates nothing.""" - from cudnn.frost.buffers import DTYPE_ITEMSIZE, DeviceView - - d = bufs["dummy"] - return DeviceView(d.data_ptr(), (16 // DTYPE_ITEMSIZE[dtype_name],), dtype_name, d.__dlpack_device__()[1]) - - -def _i32(t): - if not str(t.dtype).endswith("int32"): - raise TypeError(f"index/boundary buffers must be int32 (callers convert), got {t.dtype}") - return t - - -def _opt(t, bufs, dtype_name: str = "float32"): - """The buffer if present (contiguous by the engine contract), else an inert dummy (paired with a flag).""" - if t is None: - return _dummy(dtype_name, bufs) - return t - - # explicit workspace: all device memory is the caller's — the engine plan (or a # standalone harness) pre-carves every pipeline intermediate as a named view in # ``bufs`` and passes outputs via ``out=``; nothing here allocates. @@ -2363,24 +2333,17 @@ def _opt(t, bufs, dtype_name: str = "float32"): # l2norm def l2norm_fwd(x, eps: float = 1e-6, out=None, rstd_out=None, stream=None): stream = 0 if stream is None else stream - if out is None or rstd_out is None: - raise ValueError("l2norm_fwd requires pre-allocated out= and rstd_out= buffers") x_shape_og = x.shape - x = x.reshape(-1, x.shape[-1]).contiguous() + x = reshaped(x, (-1, x.shape[-1])) y = reshaped(out, tuple(x.shape)) T, D = x.shape[0], x.shape[-1] MAX_FUSED_SIZE = 65536 // x.element_size() - BD = min(MAX_FUSED_SIZE, _next_power_of_2(D)) - if D > BD: - raise RuntimeError("This layer doesn't support feature dim >= 64KB.") + BD = min(MAX_FUSED_SIZE, next_power_of_2(D)) rstd = reshaped(rstd_out, (T,)) if D <= 512: BT = 32 - grid = (_cdiv(T, BT),) - # Memory-bound row reduction: default occupancy=1 starves the SM (the - # grid is huge). Autotune (occupancy, num_worker_warps) so the scheduler - # packs enough blocks/SM to hide DRAM latency. - _tuned_launch( + grid = (cdiv(T, BT),) + tuned_launch( l2norm_fwd_kernel, stream, grid, @@ -2404,26 +2367,18 @@ def l2norm_bwd( ): stream = 0 if stream is None else stream y_shape_og = y.shape - y = y.reshape(-1, dy.shape[-1]).contiguous() - dy = dy.reshape(-1, dy.shape[-1]).contiguous() - dy2_arg = dy2.reshape(-1, dy.shape[-1]).contiguous() if dy2 is not None else _dummy(_dtname(dy), bufs) - if out is None: - raise ValueError("l2norm_bwd requires a pre-allocated out= buffer") + y = y.reshape(-1, dy.shape[-1]) + dy = dy.reshape(-1, dy.shape[-1]) + dy2_arg = dy2.reshape(-1, dy.shape[-1]) if dy2 is not None else dummy(dtname(dy), bufs) dx = reshaped(out, tuple(y.shape)) T, D = y.shape[0], y.shape[-1] MAX_FUSED_SIZE = 65536 // y.element_size() - BD = min(MAX_FUSED_SIZE, _next_power_of_2(D)) - if D > BD: - raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") - rstd_flat = rstd.reshape(-1).contiguous() + BD = min(MAX_FUSED_SIZE, next_power_of_2(D)) + rstd_flat = rstd.reshape(-1) if D <= 512: BT = 32 - grid = (_cdiv(T, BT),) - # Memory-bound row reduction (mirrors l2norm_fwd): default occupancy=1 - # starves the SM on the huge row grid. Autotune (occupancy, nww) so the - # scheduler packs enough blocks/SM to hide DRAM latency. This kernel was - # the largest non-dot backward kernel. - _tuned_launch( + grid = (cdiv(T, BT),) + tuned_launch( l2norm_bwd_kernel, stream, grid, @@ -2441,37 +2396,33 @@ def l2norm_bwd( # fused_beta_sigmoid -_BETA_SIGMOID_BLOCK_SIZE = 2048 +BETA_SIGMOID_BLOCK_SIZE = 2048 def fused_beta_sigmoid_fwd(x, scale: float = 1.0, out=None, stream=None): stream = 0 if stream is None else stream - if out is None: - raise ValueError("fused_beta_sigmoid_fwd requires a pre-allocated out= buffer (fp32, x-shaped)") y = reshaped(out, tuple(x.shape)) n = x.numel() - grid = (_cdiv(n, _BETA_SIGMOID_BLOCK_SIZE),) + grid = (cdiv(n, BETA_SIGMOID_BLOCK_SIZE),) ct.launch( stream, grid, fused_beta_sigmoid_fwd_kernel, - (x.reshape(-1), y.reshape(-1), float(scale), n, _BETA_SIGMOID_BLOCK_SIZE), + (reshaped(x, (-1,)), y.reshape(-1), float(scale), n, BETA_SIGMOID_BLOCK_SIZE), ) return y def fused_beta_sigmoid_bwd(x, dy, scale: float = 1.0, out=None, stream=None): stream = 0 if stream is None else stream - if out is None: - raise ValueError("fused_beta_sigmoid_bwd requires a pre-allocated out= buffer (x-shaped)") dx = reshaped(out, tuple(x.shape)) n = x.numel() - grid = (_cdiv(n, _BETA_SIGMOID_BLOCK_SIZE),) + grid = (cdiv(n, BETA_SIGMOID_BLOCK_SIZE),) ct.launch( stream, grid, fused_beta_sigmoid_bwd_kernel, - (x.reshape(-1), dy.reshape(-1), dx.reshape(-1), float(scale), n, _BETA_SIGMOID_BLOCK_SIZE), + (reshaped(x, (-1,)), reshaped(dy, (-1,)), dx.reshape(-1), float(scale), n, BETA_SIGMOID_BLOCK_SIZE), ) return dx @@ -2496,16 +2447,12 @@ def chunk_local_cumsum_scalar( stream = 0 if stream is None else stream T, H = g.shape BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") - if out is None: - raise ValueError("chunk_local_cumsum requires a pre-allocated out= buffer (g-shaped)") NT = len(chunk_indices) g_out = reshaped(out, (T, H)) scale_val = float(scale) if scale is not None else 0.0 has_scale = int(scale is not None) - cu_arg = _i32(cu_seqlens) - ci_arg = _i32(chunk_indices) + cu_arg = cu_seqlens + ci_arg = chunk_indices ct.launch( stream, (NT, H), @@ -2536,7 +2483,6 @@ def chunk_local_cumsum( stream=None, ): stream = 0 if stream is None else stream - assert len(g.shape) == 2, f"Unsupported input shape {g.shape}, expected (T, H)." return chunk_local_cumsum_scalar( g=g, chunk_size=chunk_size, @@ -2564,23 +2510,19 @@ def gdn_gate_chunk_cumsum( stream = 0 if stream is None else stream T, H = g.shape BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") - if out is None: - raise ValueError("gdn_gate_chunk_cumsum requires a pre-allocated out= buffer (fp32, g-shaped)") NT = len(chunk_indices) o = reshaped(out, (T, H)) - dt_arg = _opt(dt_bias, bufs, _dtname(A_log)).reshape(-1) + dt_arg = reshaped(opt(dt_bias, bufs, dtname(A_log)), (-1,)) scale_val = float(scale) if scale is not None else 0.0 - cu_arg = _i32(cu_seqlens) - ci_arg = _i32(chunk_indices) + cu_arg = cu_seqlens + ci_arg = chunk_indices ct.launch( stream, (NT, H), gdn_gate_chunk_cumsum_scalar_kernel, ( g, - A_log.reshape(-1), + reshaped(A_log, (-1,)), dt_arg, o, scale_val, @@ -2604,23 +2546,21 @@ def gdn_gate_bwd(g, A_log, dt_bias, dyg, dg_out=None, dA_out=None, dbias_out=Non H = g.shape[-1] T = g.numel() // H BT = 32 - NT = _cdiv(T, BT) - if dg_out is None or dA_out is None or (dt_bias is not None and dbias_out is None): - raise ValueError("gdn_gate_bwd requires pre-allocated dg_out=, dA_out= (and dbias_out= with dt_bias)") + NT = cdiv(T, BT) dg = reshaped(dg_out, tuple(g.shape)) dA_nt = reshaped(bufs["dA_gate"], (NT, H)) db_nt = reshaped(bufs["db_gate"], (NT, H)) if dt_bias is not None else None - dt_arg = _opt(dt_bias, bufs, _dtname(A_log)).reshape(-1) - db_arg = db_nt.reshape(-1) if db_nt is not None else _dummy("float32", bufs) + dt_arg = reshaped(opt(dt_bias, bufs, dtname(A_log)), (-1,)) + db_arg = db_nt.reshape(-1) if db_nt is not None else dummy("float32", bufs) ct.launch( stream, (NT, H), gdn_gate_bwd_kernel, ( - g.reshape(-1), - A_log.reshape(-1), + reshaped(g, (-1,)), + reshaped(A_log, (-1,)), dt_arg, - dyg.reshape(-1), + reshaped(dyg, (-1,)), dg.reshape(-1), dA_nt.reshape(-1), db_arg, @@ -2643,17 +2583,15 @@ def recompute_w_u_fwd(k, v, beta, A, g=None, cu_seqlens=None, chunk_indices=None BT = A.shape[-1] BK = 64 BV = 64 - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") NT = len(chunk_indices) w = reshaped(bufs["w"], (T, HV, K)) u = reshaped(bufs["u"], (T, HV, V)) - beta2 = beta.reshape(T, HV) + beta2 = reshaped(beta, (T, HV)) A3 = A.reshape(T, HV, BT) - g_arg = g.reshape(T, HV) if g is not None else _dummy("float32", bufs) - cu_arg = _i32(cu_seqlens) - ci_arg = _i32(chunk_indices) - _tuned_launch( + g_arg = g.reshape(T, HV) if g is not None else dummy("float32", bufs) + cu_arg = cu_seqlens + ci_arg = chunk_indices + tuned_launch( recompute_w_u_fwd_kernel, stream, (NT, HV), @@ -2676,7 +2614,7 @@ def recompute_w_u_fwd(k, v, beta, A, g=None, cu_seqlens=None, chunk_indices=None BV, int(g is not None), ), - cache_key=((NT, HV), K, V, BT, BK, BV, int(g is not None), str(k.dtype)), + cache_key=(H, HV, K, V, BT, BK, BV, int(g is not None), str(k.dtype), dev_id(k)), ) return w, u @@ -2685,21 +2623,19 @@ def prepare_wy_repr_bwd(k, v, beta, A, dw, du, g=None, cu_seqlens=None, chunk_in stream = 0 if stream is None else stream T, H, K, V, HV = *k.shape, v.shape[-1], v.shape[1] BT = 64 - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") NT = len(chunk_indices) CONST_TILING = 64 - BK = min(max(_next_power_of_2(K), 16), CONST_TILING) - BV = min(max(_next_power_of_2(V), 16), CONST_TILING) + BK = min(max(next_power_of_2(K), 16), CONST_TILING) + BV = min(max(next_power_of_2(V), 16), CONST_TILING) dk = reshaped(bufs["wy_dk"], (T, HV, K)) dv = reshaped(bufs["wy_dv"], (T, HV, V)) dg = reshaped(bufs["wy_dg"], (T, HV)) if g is not None else None db = reshaped(bufs["db"], (T, HV)) - g_arg = _opt(g, bufs) - dg_arg = dg if dg is not None else _dummy("float32", bufs) - cu_arg = _i32(cu_seqlens) - ci_arg = _i32(chunk_indices) - _tuned_launch( + g_arg = opt(g, bufs) + dg_arg = dg if dg is not None else dummy("float32", bufs) + cu_arg = cu_seqlens + ci_arg = chunk_indices + tuned_launch( prepare_wy_repr_bwd_kernel, stream, (NT, HV), @@ -2725,10 +2661,10 @@ def prepare_wy_repr_bwd(k, v, beta, A, dw, du, g=None, cu_seqlens=None, chunk_in BK, BV, int(g is not None), - _cdiv(K, BK), - _cdiv(V, BV), + cdiv(K, BK), + cdiv(V, BV), ), - cache_key=((NT, HV), K, V, BT, BK, BV, int(g is not None), str(k.dtype)), + cache_key=(H, HV, K, V, BT, BK, BV, int(g is not None), str(k.dtype), dev_id(k)), ) if H != HV: dk_r = reshaped(bufs["wy_dk_hred"], (T, H, K)) @@ -2758,54 +2694,44 @@ def chunk_gated_delta_rule_fwd_h( stream = 0 if stream is None else stream T, H, K, V, HV = *k.shape, u.shape[-1], u.shape[1] BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") N, NT = len(cu_seqlens) - 1, len(chunk_indices) chunk_offsets = bufs["chunk_offsets"] - assert K <= 256, "kernel does not support head dimension larger than 256." state_shape = (N, HV, V, K) if state_v_first else (N, HV, K, V) - h = reshaped(bufs["h"], (NT, HV) + state_shape[2:]) + h = reshaped(bufs["state_checkpoints"], (NT, HV) + state_shape[2:]) final_state = reshaped(bufs["final_state"], state_shape) if output_final_state else None if final_state is not None: zero_fill(final_state, stream=stream) v_new = reshaped(bufs["v_new"], (T, HV, V)) if save_new_value else None - vnew_arg = v_new if v_new is not None else _dummy(_dtname(u), bufs) - g_arg = _opt(g, bufs) - gk_arg = _opt(gk, bufs) - h0_arg = initial_state if initial_state is not None else _dummy("float32", bufs) - ht_arg = final_state if final_state is not None else _dummy("float32", bufs) - cu_arg = _i32(cu_seqlens) - co_arg = _i32(chunk_offsets) - - # BV = V-tile width. Shrinking BV widens the V grid (more parallel CTAs) without - # adding serial work; drop to 32 for low head counts where BV=64 grid-starves the - # GPU, else keep 64. Chosen up front (not swept) to avoid a tileiras compile stall. + vnew_arg = v_new if v_new is not None else dummy(dtname(u), bufs) + g_arg = opt(g, bufs) + gk_arg = opt(gk, bufs) + h0_arg = initial_state if initial_state is not None else dummy("float32", bufs) + ht_arg = final_state if final_state is not None else dummy("float32", bufs) + cu_arg = cu_seqlens + co_arg = chunk_offsets + + # BV = V-tile width; chosen up front (not swept) to avoid a tileiras compile stall. BV = 64 if V % 32 == 0: try: - sm_count, cc_major = _device_attrs() + sm_count, cc_major = device_attrs() except Exception: # noqa: BLE001 sm_count = 0 cc_major = 0 - grid_blocks = _cdiv(V, 64) * (N * HV) - # Split only when the doubled BV=32 grid still fits within one wave - # (grid_blocks*2 <= SMs); otherwise the halved MMA width is a net loss. + grid_blocks = cdiv(V, 64) * (N * HV) if sm_count and grid_blocks * 2 <= sm_count: - # Grid-saturation split (all arches): halve BV to double CTAs when the - # BV=64 grid under-occupies (V-lanes independent -> no extra serial work). + # Grid-saturation split (all arches). BV = 32 elif cc_major == 8 and K >= 192: # Register-spill split (sm_80 only): at K>=192 the 4 fp32 (64,BV) state # accumulators overflow the 255-reg budget at BV=64 -> heavy spill. - # BV=32 halves the state (~10x less spill), fwd_h 2853->2468us. sm_100 - # keeps BV=64 (BV=32 regresses its already-saturated grid). BV = 32 - grid = (_cdiv(V, BV), N * HV) + grid = (cdiv(V, BV), N * HV) # Multi-dim arrays passed as-is (per-dim indices + real strides). - _tuned_launch( + tuned_launch( chunk_gated_delta_rule_fwd_kernel_h_blockdim64, stream, grid, @@ -2835,7 +2761,8 @@ def chunk_gated_delta_rule_fwd_h( int(state_v_first), ), cache_key=( - grid, + H, + HV, K, V, BT, @@ -2847,6 +2774,7 @@ def chunk_gated_delta_rule_fwd_h( int(save_new_value), int(state_v_first), str(k.dtype), + dev_id(k), ), ) return h, v_new, final_state @@ -2861,7 +2789,7 @@ def chunk_gated_delta_rule_bwd_dhu( g=None, gk=None, h0=None, - dht=None, + dstate_in=None, scale=None, state_v_first=False, cu_seqlens=None, @@ -2873,28 +2801,25 @@ def chunk_gated_delta_rule_bwd_dhu( stream = 0 if stream is None else stream T, H, K, V, HV = *q.shape, do.shape[-1], do.shape[1] BT = chunk_size - assert K <= 256, "kernel does not support head dimension being larger than 256." - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") N, NT = len(cu_seqlens) - 1, len(chunk_indices) chunk_offsets = bufs["chunk_offsets"] - dh = reshaped(bufs["dh"], (NT, HV, V, K) if state_v_first else (NT, HV, K, V)) - dh0 = reshaped(bufs["dh0"], tuple(h0.shape)) if h0 is not None else None + dh = reshaped(bufs["dstate"], (NT, HV, V, K) if state_v_first else (NT, HV, K, V)) + dh0 = reshaped(bufs["dstate0"], tuple(h0.shape)) if h0 is not None else None dv2 = reshaped(bufs["dv2"], (T, HV, V)) - # For K>128 the blockdim64 kernel's 4 (64xBV) fp32 dh accumulators overflow + # For K>128 the blockdim64 kernel's 4 (64xBV) fp32 dH accumulators overflow # tileiras allocation at BV=64; shrink BV to keep the live footprint in range. BV = 16 if K > 128 else 64 - grid = (_cdiv(V, BV), N * HV) - g_arg = _opt(g, bufs) - gk_arg = _opt(gk, bufs) - dht_arg = _opt(dht, bufs) - dh0_arg = dh0 if dh0 is not None else _dummy("float32", bufs) - cu_arg = _i32(cu_seqlens) - co_arg = _i32(chunk_offsets) - - _tuned_launch( + grid = (cdiv(V, BV), N * HV) + g_arg = opt(g, bufs) + gk_arg = opt(gk, bufs) + dht_arg = opt(dstate_in, bufs) + dh0_arg = dh0 if dh0 is not None else dummy("float32", bufs) + cu_arg = cu_seqlens + co_arg = chunk_offsets + + tuned_launch( chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64, stream, grid, @@ -2922,12 +2847,13 @@ def chunk_gated_delta_rule_bwd_dhu( int(g is not None), int(gk is not None), int(h0 is not None), - int(dht is not None), + int(dstate_in is not None), int(state_v_first), 1, ), cache_key=( - grid, + H, + HV, K, V, BT, @@ -2936,6 +2862,7 @@ def chunk_gated_delta_rule_bwd_dhu( int(gk is not None), int(state_v_first), str(q.dtype), + dev_id(q), ), ) return dh, dh0, dv2 @@ -2960,25 +2887,23 @@ def chunk_fwd_o( stream = 0 if stream is None else stream T, H, K, V, HV = *q.shape, v.shape[-1], v.shape[1] BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") NT = len(chunk_indices) if scale is None: scale = k.shape[-1] ** -0.5 o = reshaped(bufs["o"], (T, HV, V)) - BK = min(max(_next_power_of_2(K), 16), 64) + BK = min(max(next_power_of_2(K), 16), 64) BV = 64 - grid = (_cdiv(V, BV), NT, HV) - g_arg = g.reshape(T, HV) if g is not None else _dummy("float32", bufs) - gg_arg = _opt(g_gamma, bufs) - cu_arg = _i32(cu_seqlens) - ci_arg = _i32(chunk_indices) + grid = (cdiv(V, BV), NT, HV) + g_arg = g.reshape(T, HV) if g is not None else dummy("float32", bufs) + gg_arg = opt(g_gamma, bufs) + cu_arg = cu_seqlens + ci_arg = chunk_indices # h flattened to (NT*HV,*,*) slabs for block-indexed TMA. if state_v_first: h3 = h.reshape(NT * HV, V, K) else: h3 = h.reshape(NT * HV, K, V) - _tuned_launch( + tuned_launch( chunk_fwd_kernel_o, stream, grid, @@ -3005,7 +2930,8 @@ def chunk_fwd_o( int(state_v_first), ), cache_key=( - grid, + H, + HV, K, V, BT, @@ -3015,6 +2941,7 @@ def chunk_fwd_o( int(g_gamma is not None), int(state_v_first), str(q.dtype), + dev_id(q), ), ) return o @@ -3042,13 +2969,11 @@ def chunk_bwd_dqkwg( stream = 0 if stream is None else stream T, H, K, V, HV = *k.shape, v.shape[-1], v.shape[1] BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") NT = len(chunk_indices) CONST_TILING = 64 - BK = min(max(_next_power_of_2(K), 16), CONST_TILING) - BV = min(max(_next_power_of_2(V), 16), CONST_TILING) - NK = _cdiv(K, BK) + BK = min(max(next_power_of_2(K), 16), CONST_TILING) + BV = min(max(next_power_of_2(V), 16), CONST_TILING) + NK = cdiv(K, BK) dq = reshaped(bufs["dq"], (T, HV, K)) dk = reshaped(bufs["dk"], (T, HV, K)) dg = reshaped(bufs["dg_nk"], (NK, T, HV)) if g is not None else None @@ -3061,15 +2986,15 @@ def chunk_bwd_dqkwg( else: h3 = h.reshape(NT * HV, K, V) dh3 = dh.reshape(NT * HV, K, V) - g_arg = g.reshape(T, HV) if g is not None else _dummy(_dtname(q), bufs) - gg_arg = _opt(g_gamma, bufs) - dw3 = dw if dw is not None else _dummy(_dtname(k), bufs) - dv3 = dv.reshape(T, HV, V) if dv is not None else _dummy(_dtname(k), bufs) - dg3 = dg if dg is not None else _dummy("float32", bufs) - cu_arg = _i32(cu_seqlens) - ci_arg = _i32(chunk_indices) + g_arg = g.reshape(T, HV) if g is not None else dummy(dtname(q), bufs) + gg_arg = opt(g_gamma, bufs) + dw3 = dw if dw is not None else dummy(dtname(k), bufs) + dv3 = dv.reshape(T, HV, V) if dv is not None else dummy(dtname(k), bufs) + dg3 = dg if dg is not None else dummy("float32", bufs) + cu_arg = cu_seqlens + ci_arg = chunk_indices use_dw = int(dw is not None and dv is not None) - _tuned_launch( + tuned_launch( chunk_bwd_kernel_dqkwg, stream, grid, @@ -3101,10 +3026,11 @@ def chunk_bwd_dqkwg( int(g_gamma is not None), use_dw, int(state_v_first), - _cdiv(V, BV), + cdiv(V, BV), ), cache_key=( - grid, + H, + HV, K, V, BT, @@ -3115,6 +3041,7 @@ def chunk_bwd_dqkwg( use_dw, int(state_v_first), str(q.dtype), + dev_id(q), ), ) if H != HV: @@ -3134,23 +3061,19 @@ def chunk_bwd_dv_local(q, k, do, g=None, g_gamma=None, A=None, scale=None, cu_se stream = 0 if stream is None else stream T, H, K, V, HV = *k.shape, do.shape[-1], do.shape[1] BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") CONST_TILING = 64 - BK = min(max(_next_power_of_2(K), 16), CONST_TILING) - BV = min(max(_next_power_of_2(V), 16), CONST_TILING) + BK = min(max(next_power_of_2(K), 16), CONST_TILING) + BV = min(max(next_power_of_2(V), 16), CONST_TILING) NT = len(chunk_indices) dv = reshaped(bufs["dv"], (T, HV, V)) grid = (NT, HV) - g_arg = _opt(g, bufs) - gg_arg = _opt(g_gamma, bufs) - A_arg = _opt(A, bufs) - cu_arg = _i32(cu_seqlens) - ci_arg = _i32(chunk_indices) + g_arg = opt(g, bufs) + gg_arg = opt(g_gamma, bufs) + A_arg = opt(A, bufs) + cu_arg = cu_seqlens + ci_arg = chunk_indices scale_val = float(scale) if scale is not None else 0.0 - # Autotune (occupancy, nww): an occ hint budgets - # fewer regs/thread -> more blocks/SM. - _tuned_launch( + tuned_launch( chunk_bwd_kernel_dv_local, stream, grid, @@ -3177,7 +3100,8 @@ def chunk_bwd_dv_local(q, k, do, g=None, g_gamma=None, A=None, scale=None, cu_se int(A is not None), ), cache_key=( - grid, + H, + HV, K, V, BT, @@ -3187,30 +3111,28 @@ def chunk_bwd_dv_local(q, k, do, g=None, g_gamma=None, A=None, scale=None, cu_se int(g_gamma is not None), int(A is not None), str(q.dtype), + dev_id(q), ), ) return dv -# fused intra (kkt + solve_tril + recompute_w_u) +# fused intra (KK^T + solve_tril + recompute_w_u) def chunk_gated_delta_rule_fwd_intra(k, v, g=None, beta=None, cu_seqlens=None, chunk_size=64, chunk_indices=None, bufs=None, compute_wu=True, stream=None): stream = 0 if stream is None else stream T, H, K, HV = *k.shape, beta.shape[1] BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") NT = len(chunk_indices) A = reshaped(bufs["A"], (T, HV, BT)) zero_fill(A, stream=stream) BK = 64 - g_arg = _opt(g, bufs) - cu_arg = _i32(cu_seqlens) - ci_arg = _i32(chunk_indices) - # Masked BC=16 4-sub-block kernel: masks the partial last chunk's tail - # (m_out / o_t < T). + g_arg = opt(g, bufs) + cu_arg = cu_seqlens + ci_arg = chunk_indices + # Masked BC=16 4-sub-block kernel: masks the partial last chunk's tail. BC = 16 - _tuned_launch( + tuned_launch( chunk_gated_delta_rule_fwd_kkt_solve_kernel, stream, (NT, HV), @@ -3229,7 +3151,7 @@ def chunk_gated_delta_rule_fwd_intra(k, v, g=None, beta=None, cu_seqlens=None, c BK, int(g is not None), ), - cache_key=(NT, HV, K, BT, BC, BK, int(g is not None), str(k.dtype)), + cache_key=(H, HV, K, BT, BC, BK, int(g is not None), str(k.dtype), dev_id(k)), ) if not compute_wu: return None, None, A @@ -3260,7 +3182,6 @@ def chunk_gated_delta_rule_fwd( stream=None, ): stream = 0 if stream is None else stream - assert cp_context is None, "context-parallel not supported in this port" g_input = g if use_gate_in_kernel else None if use_gate_in_kernel: g = gdn_gate_chunk_cumsum( @@ -3307,7 +3228,7 @@ def chunk_gated_delta_rule_bwd( scale, initial_state, do, - dht, + dstate_in, state_v_first=False, cu_seqlens=None, cp_context=None, @@ -3320,7 +3241,6 @@ def chunk_gated_delta_rule_bwd( stream=None, ): stream = 0 if stream is None else stream - assert cp_context is None, "context-parallel not supported in this port" w, u = recompute_w_u_fwd(k=k, v=v, beta=beta, A=A, g=g, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices, bufs=bufs, stream=stream) h, v_new, _ = chunk_gated_delta_rule_fwd_h( k=k, @@ -3342,7 +3262,7 @@ def chunk_gated_delta_rule_bwd( w=w, g=g, h0=initial_state, - dht=dht, + dstate_in=dstate_in, do=do, dv=dv, scale=scale, @@ -3400,7 +3320,7 @@ def chunk_gated_delta_rule_grad( g, beta, do, - dht=None, + dstate_in=None, scale=None, initial_state=None, state_v_first=False, @@ -3420,7 +3340,7 @@ def chunk_gated_delta_rule_grad( ``q``/``k``/``v``/``do`` are ``[total_T, H, D]``, ``g``/``beta`` are ``[total_T, H]``; ``cu_seqlens`` and ``chunk_indices`` are required. - Recomputes the forward's prep (L2-normalized q/k + rstd, cumulative gate, + Recomputes the forward's prep (L2-normalized Q/K + rstd, cumulative gate, intra-chunk WY matrix) from the ORIGINAL inputs, then runs the backward kernels. @@ -3469,7 +3389,7 @@ def chunk_gated_delta_rule_grad( scale=scale, initial_state=initial_state, do=do, - dht=dht, + dstate_in=dstate_in, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices, state_v_first=state_v_first, @@ -3521,7 +3441,6 @@ def chunk_gated_delta_rule( required. Returns ``(o, final_state)`` in THD layout, written into ``bufs['o']`` / ``bufs['final_state']``.""" stream = 0 if stream is None else stream - assert cp_context is None, "context-parallel not supported" if "transpose_state_layout" in kwargs: if state_v_first: diff --git a/python/cudnn/linear_attention/cutile/kernels/kda_chunk_cutile.py b/python/cudnn/linear_attention/cutile/kernels/kda_chunk_cutile.py index 8d297f05c..bd0189fc0 100644 --- a/python/cudnn/linear_attention/cutile/kernels/kda_chunk_cutile.py +++ b/python/cudnn/linear_attention/cutile/kernels/kda_chunk_cutile.py @@ -21,13 +21,11 @@ from types import SimpleNamespace import cuda.tile as ct - -from .common import add_inplace, head_group_sum, reshaped, sum_leading, zero_fill -from cudnn.frost.buffers import dtype_name as _dtname - -_F32_DEFAULT = "float32-default" from cuda.tile.tune import exhaustive_search +from .common import dev_id, dummy, head_group_sum, opt, reshaped, sum_leading, zero_fill +from cudnn.frost.buffers import dtype_name as dtname + logger = logging.getLogger(__name__) ConstInt = ct.Constant[int] @@ -35,22 +33,21 @@ RCP_LN2 = 1.4426950216 # 1/ln(2) # chunk size (BT tile) of these kernels; the engine's carve layout imports it -_BT = 64 +BT_CHUNK = 64 # --------------------------------------------------------------------------- -# Launch-hint autotune (mirrors the sibling GDN -# cuTile kernels). Only the @ct.kernel launch hints +# Launch-hint autotune. Only the @ct.kernel launch hints # (occupancy x num_worker_warps) are explored via kernel.replace_hints(...); # the grid / args / algorithm are kept UNCHANGED. The first config in every # grid equals the kernel default (occupancy=1, num_worker_warps=4) so any shape # that does not improve keeps the original behaviour (no regression). # --------------------------------------------------------------------------- -_DISABLE_TUNE = os.environ.get("DISABLE_TUNE", "") not in ("", "0") -_launch_hint_cache: dict = {} +DISABLE_TUNE = os.environ.get("DISABLE_TUNE", "") not in ("", "0") +launch_hint_cache: dict = {} -def _launch_hint_configs(occ_choices, nww_choices=(4, 8)): +def launch_hint_configs(occ_choices, nww_choices=(4, 8)): """occupancy x num_worker_warps grid; deduped, default (occ=1,nww=4) first.""" seen = set() cfgs = [] @@ -64,7 +61,7 @@ def _launch_hint_configs(occ_choices, nww_choices=(4, 8)): return cfgs -def _autotuned_launch(kernel, cache_key, grid, args, occ_choices=(1, 2, 3, 4), nww_choices=(4, 8), timeout=30, stream=None): +def autotuned_launch(kernel, cache_key, grid, args, occ_choices=(1, 2, 3, 4), nww_choices=(4, 8), timeout=30, stream=None): """Launch ``kernel`` with the best launch hints for ``cache_key``. Tune-once/cache/launch over launch hints only (grid, args and signature are @@ -73,14 +70,14 @@ def _autotuned_launch(kernel, cache_key, grid, args, occ_choices=(1, 2, 3, 4), n no-improvement shape keeps the base behaviour. """ stream = 0 if stream is None else stream - if _DISABLE_TUNE: + if DISABLE_TUNE: ct.launch(stream, grid, kernel, args) return - if cache_key not in _launch_hint_cache: + if cache_key not in launch_hint_cache: tuned = None try: - configs = _launch_hint_configs(occ_choices, nww_choices) + configs = launch_hint_configs(occ_choices, nww_choices) with ct.compiler_timeout(timeout): result = exhaustive_search( configs, @@ -94,16 +91,16 @@ def _autotuned_launch(kernel, cache_key, grid, args, occ_choices=(1, 2, 3, 4), n tuned = kernel.replace_hints(occupancy=best.occupancy, num_worker_warps=best.num_worker_warps) except Exception: tuned = None - _launch_hint_cache[cache_key] = tuned + launch_hint_cache[cache_key] = tuned - tuned = _launch_hint_cache[cache_key] + tuned = launch_hint_cache[cache_key] if tuned is None: ct.launch(stream, grid, kernel, args) else: ct.launch(stream, grid, tuned, args) -def _autotuned_launch_bv(kernel, cache_key, bv_choices, grid_fn, args_fn, timeout=40, stream=None): +def autotuned_launch_bv(kernel, cache_key, bv_choices, grid_fn, args_fn, timeout=40, stream=None): """Launch ``kernel`` autotuning over BV (the V-tile block width) only. BV is a kernel ConstInt that drives the grid V-fan-out and the V-tile shapes @@ -114,30 +111,21 @@ def _autotuned_launch_bv(kernel, cache_key, bv_choices, grid_fn, args_fn, timeou BV is cached and the BV-specialized kernel re-launched on every subsequent call. ``bv_choices[0]`` is the safe fallback on DISABLE_TUNE / tuning failure. - Why BV is the dominant fwd_h lever and why the sweep is BV-ONLY: the state - scan is register-bound (255 reg/thread -> ~1 block/SM) with a serial per-CTA - NT inter-chunk loop, so the only parallelism lever is the V-tile CTA count - (grid = (cdiv(V, BV), N*HV)). ncu on the dashboard shapes shows the optimum - is NOT monotone in BV: at (T=2048,V=128,N*HV=64) BV=64 (grid 128, warps_active - 14%) fwd_h=246us beats BV=32 (grid 256, 8%) 290us and BV=16 374us -- fewer, - fatter CTAs win once the grid already fills the SMs, because a smaller BV just - multiplies redundant k/w/g reloads; at (T=1024,V=64,N*HV=64) BV=32 (grid 128) - 65us beats BV=64 (grid 64, under-filled) 123us and BV=8 123us. Crucially the - occupancy/num_worker_warps hint is nearly inert here (per-BV best occ/nww is - within ~0.3% of default), so we do NOT cross occ x nww into this search: - doing so inflated it to 16 configs and the larger exhaustive_search do_bench - mis-ranked BV (it picked BV=32 for the 2048 shape even though a direct - BV=64/occ-tuned run is 15% faster). A clean 4-config BV-only sweep -- mirroring - the kda_chunk_fwd helper's ``_tuned_launch_fwdh_bv`` -- ranks BV correctly - (picks 64 for the 2048 shape, 32 for the 1024 shape). + Why the sweep is BV-ONLY: the state scan is register-bound (255 reg/thread + -> ~1 block/SM) with a serial per-CTA NT inter-chunk loop, so the only + parallelism lever is the V-tile CTA count (grid = (cdiv(V, BV), N*HV)). + The optimum is NOT monotone in BV (a smaller BV multiplies redundant K/W/G + reloads once the grid already fills the SMs), and the occupancy x + num_worker_warps hint is nearly inert per BV -- do NOT cross occ x nww into + this search (the larger sweep mis-ranks BV). """ stream = 0 if stream is None else stream - if _DISABLE_TUNE: + if DISABLE_TUNE: bv = bv_choices[0] ct.launch(stream, grid_fn(bv), kernel, args_fn(bv)) return - if cache_key not in _launch_hint_cache: + if cache_key not in launch_hint_cache: chosen = None try: configs = [SimpleNamespace(BV=bv) for bv in bv_choices] @@ -153,9 +141,9 @@ def _autotuned_launch_bv(kernel, cache_key, bv_choices, grid_fn, args_fn, timeou chosen = result.best.config.BV except Exception: chosen = None - _launch_hint_cache[cache_key] = chosen + launch_hint_cache[cache_key] = chosen - chosen = _launch_hint_cache[cache_key] + chosen = launch_hint_cache[cache_key] bv = bv_choices[0] if chosen is None else chosen ct.launch(stream, grid_fn(bv), kernel, args_fn(bv)) @@ -165,55 +153,20 @@ def _autotuned_launch_bv(kernel, cache_key, bv_choices, grid_fn, args_fn, timeou # =========================================================================== -def _cdiv(a: int, b: int) -> int: +def cdiv(a: int, b: int) -> int: return (a + b - 1) // b -def _next_power_of_2(n: int) -> int: +def next_power_of_2(n: int) -> int: return 1 << (n - 1).bit_length() -def _dev_id(buf) -> int: - """Device ordinal of a DLPack/CAI buffer.""" - from cudnn.frost.buffers import probe - - return probe(buf)[4] - - -def _dummy(dtype_name: str, bufs): - """Inert typed view over the workspace's 16-byte ``dummy`` carve, for - ABSENT optional kernel args (always paired with a flag==0, never - dereferenced). Dtype-bound so the compiled signature stays stable; the - library allocates nothing.""" - from cudnn.frost.buffers import DTYPE_ITEMSIZE, DeviceView - - d = bufs["dummy"] - return DeviceView(d.data_ptr(), (16 // DTYPE_ITEMSIZE[dtype_name],), dtype_name, d.__dlpack_device__()[1]) - - -def _i32(t): - if not str(t.dtype).endswith("int32"): - raise TypeError(f"index/boundary buffers must be int32 (callers convert), got {t.dtype}") - return t - - -def _opt(t, bufs, dtype_name: str = "float32"): - """Resolve an optional tensor argument to a non-null cuTile launch arg: - the buffer if present (contiguous by the engine contract), else an inert - dummy (paired with a USE_*/HAS_* integer flag). cuTile never accepts None - in launch args, so this is the required dummy-tensor-plus-flag pattern.""" - if t is None: - return _dummy(dtype_name, bufs) - return t - - -def _i32_flat(t): - """Required index buffer: int32 guard + flat 1-D view. Kernels index these +def i32_flat(t): + """Required index buffer as a flat 1-D view. Kernels index these via flat loads (e.g. chunk_indices[i_t*2]); chunk_indices is (NT, 2) and its row-major flat layout is [seg0, intra0, seg1, intra1, ...].""" from .common import reshaped - t = _i32(t) n = 1 for s_ in t.shape: n *= int(s_) @@ -225,7 +178,7 @@ def _i32_flat(t): # ``bufs`` and passes outputs via ``out=``; nothing here allocates. -def _cast(bufs, name, src, ref, stream=None): +def cast(bufs, name, src, ref, stream=None): """Dtype cast at the gradient boundary, through the ``bufs[name]`` carve.""" if str(src.dtype).split(".")[-1] == str(ref.dtype).split(".")[-1]: return src @@ -239,37 +192,6 @@ def _cast(bufs, name, src, ref, stream=None): return dst -def _i32(t): - if not str(t.dtype).endswith("int32"): - raise TypeError(f"index/boundary buffers must be int32 (callers convert), got {t.dtype}") - return t - - -def _opt(t, device_id, dtype_name: str = "float32"): - """Resolve an optional tensor argument to a non-null cuTile launch arg: - the buffer if present (contiguous by the engine contract), else an inert - dummy (paired with a USE_*/HAS_* integer flag). cuTile never accepts None - in launch args, so this is the required dummy-tensor-plus-flag pattern.""" - if t is None: - return _dummy(dtype_name, device_id) - return t - - -def _opt_i32(t, device_id): - if t is None: - return _dummy("int32", device_id) - # Kernels index these via flat 1-D loads (e.g. chunk_indices[i_t*2]); flatten - # so the cuTile load index rank (1) matches. chunk_indices is (NT, 2) and its - # row-major flat layout is [seg0, intra0, seg1, intra1, ...] as expected. - from .common import reshaped - - t = _i32(t) - n = 1 - for s_ in t.shape: - n *= int(s_) - return reshaped(t, (n,)) - - # =========================================================================== # Device helpers (plain `def` — NOT @ct.kernel) # =========================================================================== @@ -288,33 +210,33 @@ def softplus(x): return ct.where(x <= 20.0, ct.log(1.0 + ct.exp(x)), x) -def _tf32(a): +def tf32(a): """ct.mma/ct.matmul do not auto-cast fp32 operands to tf32; cast explicitly (allow-tf32 matmul semantics).""" return ct.astype(a, ct.tfloat32) if a.dtype == ct.float32 else a -def _mma_operands(a, b): +def mma_operands(a, b): # ct.mma/ct.matmul require matching operand dtypes. Forward callers always # pass matched dtypes (left unchanged here); some backward callers mix bf16 - # with fp32, so promote both to fp32 in that case. Then apply the tf32 guard. + # with fp32, so promote both to fp32 in that case. if a.dtype != b.dtype: a = ct.astype(a, ct.float32) b = ct.astype(b, ct.float32) - return _tf32(a), _tf32(b) + return tf32(a), tf32(b) def safe_matmul(a, b): # tf32 is only the multiply precision; ct.matmul returns a tf32 tile which # is a restricted dtype (no elementwise arithmetic). Cast the result back to # fp32 so downstream adds work (fp32 dot-accumulator semantics). - aa, bb = _mma_operands(a, b) + aa, bb = mma_operands(a, b) out = ct.matmul(aa, bb) return ct.astype(out, ct.float32) if out.dtype != ct.float32 else out def safe_mma(a, b, acc): - aa, bb = _mma_operands(a, b) + aa, bb = mma_operands(a, b) return ct.mma(aa, bb, acc) @@ -322,9 +244,7 @@ def bf16_mma(a, b, acc): # Like safe_mma but stages the MMA operands as bf16 (2 bytes) instead of # tf32 (4 bytes). cuTile stages tiny MMA operands into static SMEM; tf32 # staging doubles that footprint vs bf16, which on the SMEM-bound - # compute-pairs kernel pins it to 1 block/SM. The q/k inputs are bf16 - # upstream, so bf16 operands keep the multiply precision the inputs already - # carry; the fp32 accumulator is preserved. + # compute-pairs kernel pins it to 1 block/SM. return ct.mma(ct.astype(a, ct.bfloat16), ct.astype(b, ct.bfloat16), acc) @@ -334,10 +254,9 @@ def reg_matmul(a, b): # SMEM-staged HMMA path (~83KB static smem/block -> 1 block/SM, ~12% occ). # Keep these dots in registers instead, via a # broadcast-multiply-reduce over the contraction dim: out[i,j] = - # sum_k a[i,k]*b[k,j]. Inputs are rounded to tf32 first so the multiply - # precision matches the existing tf32-MMA path (preserves tolerance). - aa = _tf32(a) - bb = _tf32(b) + # sum_k a[i,k]*b[k,j]. Inputs are rounded to tf32 first. + aa = tf32(a) + bb = tf32(b) aa = ct.astype(aa, ct.float32) bb = ct.astype(bb, ct.float32) return ct.sum(aa[:, :, None] * bb[None, :, :], axis=1) @@ -356,7 +275,7 @@ def ct_min(a, b): # GATHER RATIONALE (l2norm): each row is loaded once, reduced once, written # once (single-pass streaming, no cooperative reuse) -> ct.gather/ct.scatter # avoids the smem staging tax of ct.load. Column mask handles BD-power-of-2 -# padding; partial-row handling via masks (replaces boundary_check). +# padding; partial-row handling via masks. @ct.kernel @@ -439,7 +358,7 @@ def fused_beta_sigmoid_bwd_kernel(x, dy, dx, scale, n_elements, BLOCK_SIZE: Cons # =========================================================================== -# vector chunk_local_cumsum (used for the vector gate g) +# vector chunk_local_cumsum (used for the vector gate G) # =========================================================================== # GATHER RATIONALE: head-interleaved [T, H, S] sub-view; loaded once, # cumsum'd along T, written back. s/o passed flattened (1-D) by the wrapper. @@ -494,7 +413,7 @@ def chunk_local_cumsum_vector_kernel( # =========================================================================== # kda_gate_chunk_cumsum_vector / kda_gate_bwd # =========================================================================== -# Vector gate: g is [B, T, H, S]; A_log is [H]; dt_bias is [H*S]. +# Vector gate: G is [B, T, H, S]; A_log is [H]; dt_bias is [H*S]. # GATHER RATIONALE: head-interleaved [B, T, H, S] sub-view streamed once. @@ -577,7 +496,7 @@ def kda_gate_bwd_kernel( HAS_BIAS: ConstInt, USE_LOWER_BOUND: ConstInt, ): - # g/dyg/dg layout [T, H, D] -> view (T, D) stride (H*D, 1) at base i_h*D. + # G/dYg/dG layout [T, H, D] -> view (T, D) stride (H*D, 1) at base i_h*D. i_t = ct.bid(0) i_h = ct.bid(1) @@ -623,11 +542,11 @@ def kda_gate_bwd_kernel( # =========================================================================== -# chunk_gla_fwd_kernel_o -- output projection o = qg @ h + A @ v +# chunk_gla_fwd_kernel_o -- output projection O = QG @ H + A @ V # =========================================================================== -# GATHER RATIONALE: q/g/v are reused across the i_k loop (K split), but with -# head-interleaved strides + a transposed STATE_V_FIRST h-view that TMA cannot -# express here, so the faithful non-TMA element-offset gather path is used. +# GATHER RATIONALE: Q/G/V are reused across the i_k loop (K split), but with +# head-interleaved strides + a transposed STATE_V_FIRST H-view that TMA cannot +# express here, so the non-TMA element-offset gather path is used. @ct.kernel @@ -655,8 +574,8 @@ def chunk_gla_fwd_kernel_o( i_hv = ct.bid(2) i_h = i_hv // (HV // H) - # grid dim-1 is the GLOBAL chunk index; h is laid out per global chunk - # (h-state kernel writes slot chunk_offsets[i_n] + local). Capture it + # grid dim-1 is the GLOBAL chunk index; H is laid out per global chunk + # (H-state kernel writes slot chunk_offsets[i_n] + local). Capture it # before i_t is reassigned to the per-sequence (local) chunk index. i_tg = i_t i_n = ct.load(chunk_indices, (i_t * 2,), shape=()).item() @@ -731,10 +650,9 @@ def chunk_gla_fwd_kernel_o( # =========================================================================== -# chunk_delta_h (state-h fwd + dhu bwd) -- SHARED with GDN +# chunk_delta_h (state-H fwd + dhu bwd) # =========================================================================== -# These two kernels are byte-for-byte identical (in math) to the GDN -# chunk_delta_h kernels. For KDA the gate carried is the per-key vector gate +# For KDA the gate carried is the per-key vector gate # `gk` (USE_GK=1, USE_G=0); the per-time scalar gate `g` path is inert here. @@ -766,12 +684,11 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( STATE_V_FIRST: ConstInt, ): # The KV state is carried as a SINGLE full-width tile ((BV, BK) when - # STATE_V_FIRST else (BK, BV), BK=next_pow2(K)) instead of a 4-way 64-wide - # sub-block unroll, so the kernel is general for ANY K (no K<=256 cap). Every - # K-axis load zero-pads cols [K:BK] and every K-axis store masks them off, so - # the tail is 0 on load, contributes 0 to every MMA, and stays 0 in the - # state; a padded gk tail loads as 0 so exp2(0)=1 leaves those zero cols - # unchanged. + # STATE_V_FIRST else (BK, BV), BK=next_pow2(K)), so the kernel is general + # for ANY K (no K<=256 cap). Every K-axis load zero-pads cols [K:BK] and + # every K-axis store masks them off, so the tail is 0 on load, contributes + # 0 to every MMA, and stays 0 in the state; a padded Gk tail loads as 0 so + # exp2(0)=1 leaves those zero cols unchanged. i_v = ct.bid(0) i_nh = ct.bid(1) i_n = i_nh // HV @@ -801,8 +718,8 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( # K-tile / V-tile validity masks. The kernel carries a single BK-wide K tile # (BK=next_pow2(K)) and BV-wide V tiles; when K or V is not a multiple of the - # tile width the extra lanes alias the neighbouring head's h slot. The matmul - # state rows are zeroed via K-masked k/gk loads, but the raw h/h0/ht + # tile width the extra lanes alias the neighbouring head's H slot. The matmul + # state rows are zeroed via K-masked K/Gk loads, but the raw H/H0/Ht # gather/scatter are also masked here -> no cross-head corruption when # K % BK != 0 (i.e. K < BK) or V % BV != 0. mkh = o_bk < K @@ -942,7 +859,7 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( w, g, gk, - dht, + dstate_in, dh0, do, dh, @@ -964,14 +881,12 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( USE_FINAL_STATE_GRADIENT: ConstInt, STATE_V_FIRST: ConstInt, ): - # - # dh state is a SINGLE full-width tile ((BV, BK) when STATE_V_FIRST else - # (BK, BV), BK=next_pow2(K)) instead of a 4-way 64-wide sub-block unroll, so - # the kernel is general for ANY K (no K<=256 cap). Every flat gather/scatter - # over the K axis uses oK=arange(BK) with a `(oK < K)` mask, so rows/cols - # [K:BK] are zero on load, contribute 0 to every MMA, and are masked out on - # store; a gk padding tail loads as 0 so exp2(0)=1 leaves those zero state - # cols unchanged. + # dH state is a SINGLE full-width tile ((BV, BK) when STATE_V_FIRST else + # (BK, BV), BK=next_pow2(K)), so the kernel is general for ANY K (no K<=256 + # cap). Every flat gather/scatter over the K axis uses oK=arange(BK) with a + # `(oK < K)` mask, so rows/cols [K:BK] are zero on load, contribute 0 to + # every MMA, and are masked out on store; a Gk padding tail loads as 0 so + # exp2(0)=1 leaves those zero state cols unchanged. i_v = ct.bid(0) i_nh = ct.bid(1) i_n = i_nh // HV @@ -1001,18 +916,18 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( o_bt = ct.arange(BT, dtype=ct.int32) o_bv = ct.arange(BV, dtype=ct.int32) - # V-boundary mask for the state-gradient (dh/dh0) scatters: a partial + # V-boundary mask for the state-gradient (dH/dH0) scatters: a partial # trailing V tile (V not a multiple of BV) must not write rows/cols >= V, # else the store spills into the neighbouring chunk/head's state-gradient # (boundary-checked store semantics). m_v = (i_v * BV + o_bv) < V - # --- Load final state gradient dht -> b_dh (single full-width tile; [K:BK] loads 0) --- + # --- Load final state gradient dHt -> b_dh (single full-width tile; [K:BK] loads 0) --- if USE_FINAL_STATE_GRADIENT: if STATE_V_FIRST: row = (i_v * BV + o_bv)[:, None] b_dh = b_dh + ct.gather( - dht, + dstate_in, dht_base + row * K + o_bk[None, :], mask=(o_bk < K)[None, :], check_bounds=True, @@ -1021,7 +936,7 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( else: col = (i_v * BV + o_bv)[None, :] b_dh = b_dh + ct.gather( - dht, + dstate_in, dht_base + o_bk[:, None] * V + col, mask=(o_bk < K)[:, None], check_bounds=True, @@ -1029,12 +944,12 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( ) # cuTile range() requires a positive step; iterate forward and reverse the - # index to preserve the original backward (NT-1 .. 0) chunk traversal. + # index to preserve the backward (NT-1 .. 0) chunk traversal. for _i_t in range(NT): i_t = NT - 1 - _i_t dh_chunk = dh_base + i_t * HV * K * V - # Store current b_dh to dh[i_t] (single full-width tile; [K:BK] masked out) + # Store current b_dh to dH[i_t] (single full-width tile; [K:BK] masked out) if STATE_V_FIRST: row = (i_v * BV + o_bv)[:, None] ct.scatter( @@ -1067,7 +982,7 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( bg_last_exp = ct.astype(0.0, ct.float32) b_g_exp = ct.zeros((BT,), dtype=ct.float32) - # do, dv, dv2 tiles + # dO, dV, dV2 tiles v_col = (i_v * BV + o_bv)[None, :] vmask_c = (i_v * BV + o_bv) < V v_full = (m_t[:, None]) & (vmask_c[None, :]) @@ -1086,7 +1001,7 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( ) bk = ct.where(m_t[:, None], bk, ct.zeros((BT, BK), dtype=bk.dtype)) if USE_GK: - # gk offset: base (bos*HV+i_h)*K ; then + last_idx*HV*K + o_k. + # Gk offset: base (bos*HV+i_h)*K ; then + last_idx*HV*K + o_k. # Padded tail [K:BK] loads as 0 -> exp2(0)=1 leaves those zero cols unchanged. gkl = (bos * HV + i_h) * K + last_idx * HV * K b_gk_last = ct.astype(ct.gather(gk, gkl + o_bk, mask=o_bk < K, check_bounds=True, padding_value=0.0), ct.float32) @@ -1097,7 +1012,7 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( decay = ct.where(m_t, exp2(bg_last - b_g), ct.zeros((BT,), dtype=ct.float32)) b_dv = b_dv * decay[:, None] - # b_dv += dv ; store to dv2 + # b_dv += dV ; store to dV2 dv_off = dv_base + (i_t * BT + o_bt)[:, None] * (HV * V) + v_col b_dv_load = ct.gather(dv, dv_off, check_bounds=True, padding_value=0.0) b_dv_load = ct.where(v_full, b_dv_load, ct.zeros((BT, BV), dtype=b_dv_load.dtype)) @@ -1157,7 +1072,7 @@ def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( # wy_fast (recompute_w_u_fwd) # =========================================================================== # 2D structured gather/scatter on reshaped flat views; head-interleaved -# strides folded into the (row, col) tuple indices. gk is the per-key gate. +# strides folded into the (row, col) tuple indices. Gk is the per-key gate. @ct.kernel @@ -1365,7 +1280,7 @@ def chunk_kda_fwd_kernel_intra_token_parallel( BH: ConstInt, BK: ConstInt, ): - # used (numerically identical). BK = next_power_of_2(K) is passed by host + # BK = next_power_of_2(K) is passed by host # (cuTile tile shapes must be compile-time constants). i_tg = ct.bid(0) i_hg = ct.bid(1) @@ -1406,7 +1321,7 @@ def chunk_kda_fwd_kernel_intra_token_parallel( # cuTile gather/scatter require the index tuple rank to match the array rank # (no raw pointer arithmetic). Arrays arrive pre-flattened from the - # host: q/k -> (B*T*H, K), g -> (B*T*HV, K), beta/Aqk/Akk -> 1-D. + # host: Q/K -> (B*T*H, K), G -> (B*T*HV, K), Beta/Aqk/Akk -> 1-D. o_hv = i_hg * BH + ct.arange(BH, dtype=ct.int32) o_h = o_hv // G o_k = ct.arange(BK, dtype=ct.int32) @@ -1416,12 +1331,12 @@ def chunk_kda_fwd_kernel_intra_token_parallel( col = ct.broadcast_to(o_k[None, :], (BH, BK)) - # q/k: row = (bos + token) * H + head; col = key + # Q/K: row = (bos + token) * H + head; col = key qk_row = ct.broadcast_to(((bos + i_t) * H + o_h)[:, None], (BH, BK)) b_q = ct.astype(ct.gather(q, (qk_row, col), mask=m_hk, check_bounds=False, padding_value=0.0), ct.float32) b_k = ct.astype(ct.gather(k, (qk_row, col), mask=m_hk, check_bounds=False, padding_value=0.0), ct.float32) - # g: row = (bos + token) * HV + head; beta: idx = (bos + token) * HV + head + # G: row = (bos + token) * HV + head; Beta: idx = (bos + token) * HV + head g_row = ct.broadcast_to(((bos + i_t) * HV + o_hv)[:, None], (BH, BK)) b_g = ct.astype(ct.gather(g, (g_row, col), mask=m_hk, check_bounds=False, padding_value=0.0), ct.float32) b_beta = ct.astype(ct.gather(beta, beta_base + i_t * HV + o_hv, mask=m_hv, check_bounds=False, padding_value=0.0), ct.float32) @@ -1430,8 +1345,7 @@ def chunk_kda_fwd_kernel_intra_token_parallel( # Counted loop over the (static) BC-wide sub-chunk window. A runtime-bounded # `for j in range(i_ts, j_hi)` lowers to per-iteration branches that tileiras # cannot unroll/predicate; a counted `for jj in range(BC)` with a runtime - # guard derives `j` from `jj` and stays fully unrolled (numerically identical: - # inactive iterations are masked out). See MEMORY counted-loop note. + # guard derives `j` from `jj` and stays fully unrolled. for jj in range(BC): j = i_ts + jj if j < i_t + 1 and j < T_eff and j < i_ts + BC: @@ -1465,7 +1379,7 @@ def chunk_kda_fwd_kernel_intra_token_parallel( # =========================================================================== # One BC sub-chunk per block: compute the diagonal Aqk/Akk blocks then do an # in-place forward-substitution triangular solve. GATHER RATIONALE: head- -# interleaved q/k/g sub-views + a transposed b_kgt; faithful gather path. +# interleaved Q/K/G sub-views + a transposed b_kgt. @ct.kernel @@ -1560,30 +1474,16 @@ def chunk_kda_fwd_kernel_inter_diag_compute_solve( m_aqk = m_c[:, None] & ((i_i * BC + o_bc) < BT)[None, :] ct.scatter(Aqk, aqk_off, ct.astype(b_Aqk, Aqk.dtype), mask=m_aqk, check_bounds=False) - # forward substitution on the diagonal Akk -> inverse, written to Akk(diag buf). + # diagonal Akk -> inverse via Neumann series by squaring, written to Akk(diag buf). # - # Akk here is strictly-lower-triangular and nilpotent (Akk^BC = 0), and the - # inverse we want is (I + Akk)^-1 = I - Akk + Akk^2 - ... (the serial sweep - # inits b_Ai = -Akk, so with N := -Akk this is (I - N)^-1 = I + N + N^2 + - # ... + N^(BC-1)). A per-row serial forward-substitution sweep - # (`for i in range(2, min(BC, ...))`) lowers to a chain of BC-1 - # serial SIMT rank-1 updates -- the dominant compute cost of this - # compute-bound kernel (ncu: 51% SM, HMMA only from the two upfront dots). + # Akk is strictly-lower-triangular and nilpotent (Akk^BC = 0); with + # N := -Akk, (I + Akk)^-1 = (I - N)^-1 = sum_{k=0}^{BC-1} N^k + # = prod_{j=0}^{log2(BC)-1} (I + N^(2^j)) -- log2(BC) block matmuls, no + # serial row dependency. Rows beyond T_eff have N = 0 (b_gq/b_gk were + # zeroed by m_c) and converge to the identity. # - # We replace the serial sweep with a Neumann-series-by-squaring block - # matmul: (I - N)^-1 = prod_{j=0}^{log2(BC)-1} (I + N^(2^j)). This is - # algebraically identical (telescopes to sum_{k=0}^{BC-1} N^k since - # N^BC = 0) and turns the BC-1 serial steps into log2(BC) block matmuls - # (the whole tile updates in parallel, no serial row dependency). Rows - # beyond T_eff already have N = 0 (b_gq/b_gk were zeroed by m_c), so those - # rows converge to the identity exactly as the guarded serial loop did. - # - # Precision: the squarings run at tf32 (SOLVE_TRIL_DOT_PRECISION on tf32- - # capable arch) via safe_matmul, giving real HMMA tensor-core ops at M = BC - # (>=16; fp32 operands would fall back to SIMT even at M=32). This matches - # the tolerance across the full fwd matrix (incl. raw non-L2-normed k in the - # dispatch suite, RMS ratio well under 2e-2) once the series is anchored on - # the correct sign (N = -Akk). + # Precision: the squarings run at tf32 via safe_matmul (fp32 operands + # would fall back to SIMT even at M=32). b_N = -b_Akk # N := -Akk, so (I + Akk)^-1 = (I - N)^-1 = sum_k N^k b_Ai = ct.astype(m_I, ct.float32) + b_N # (I + N) # Squaring stages: `range(2, BC)` traces as a compile-time-bounded loop with @@ -1607,7 +1507,7 @@ def chunk_kda_fwd_kernel_inter_diag_compute_solve( # --------------------------------------------------------------------------- # chunk_kda_fwd_kernel_inter_solve_fused -- off-diagonal Akk + merged solve # --------------------------------------------------------------------------- -def _load_bc_bk(arr, base, row0, col0, stride_row, BC: ConstInt, BK: ConstInt, T_eff, K: ConstInt): +def load_bc_bk(arr, base, row0, col0, stride_row, BC: ConstInt, BK: ConstInt, T_eff, K: ConstInt): # Boundary-checked (BC,BK) block load at (row0,col0) of the (T,K) view # (row stride stride_row), cast to fp32, on the flattened 1-D view. o_r = ct.arange(BC, dtype=ct.int32) @@ -1619,7 +1519,7 @@ def _load_bc_bk(arr, base, row0, col0, stride_row, BC: ConstInt, BK: ConstInt, T return ct.astype(ct.gather(arr, off, mask=mask, check_bounds=False, padding_value=0.0), ct.float32) -def _store_bc_bc(arr, base, row0, col0, stride_row, blk, BC: ConstInt, BT_or_BC: ConstInt, T_eff): +def store_bc_bc(arr, base, row0, col0, stride_row, blk, BC: ConstInt, BT_or_BC: ConstInt, T_eff): o_r = ct.arange(BC, dtype=ct.int32) o_c = ct.arange(BC, dtype=ct.int32) rows = row0 + o_r @@ -1629,37 +1529,6 @@ def _store_bc_bc(arr, base, row0, col0, stride_row, blk, BC: ConstInt, BT_or_BC: ct.scatter(arr, off, ct.astype(blk, arr.dtype), mask=mask, check_bounds=False) -def _cat_pow2(pieces, axis): - """Concatenate a power-of-two-length tuple of equal-shaped 16x16 leaf tiles - via a balanced binary tree (fully unrolled, no recursion). ct.cat takes at - most 2 operands and requires every intermediate shape to be a power of two, - so a balanced tree (16 -> 32 -> 64) is required (a left-fold would produce - illegal sizes like 48). Mirrors the proven helper in the sibling GDN - chunk kernels.""" - n = len(pieces) - if n == 1: - return pieces[0] - if n == 2: - return ct.cat((pieces[0], pieces[1]), axis) - # n == 4 - return ct.cat( - (ct.cat((pieces[0], pieces[1]), axis), ct.cat((pieces[2], pieces[3]), axis)), - axis, - ) - - -def _store_bt_bt(arr, base, row0, stride_row, blk, BT: ConstInt, T_eff): - # Single masked scatter of a full [BT, BT] tile to a head-interleaved flat - # view (row stride `stride_row` = HV*BT). Replaces the up-to-12 per-sub-block - # `_store_bc_bc` scatters, which serialise writes to the same Akk output. - o_r = ct.arange(BT, dtype=ct.int32) - o_c = ct.arange(BT, dtype=ct.int32) - rows = row0 + o_r - off = base + rows[:, None] * stride_row + o_c[None, :] - mask = ct.broadcast_to((rows < T_eff)[:, None], (BT, BT)) - ct.scatter(arr, off, ct.astype(blk, arr.dtype), mask=mask, check_bounds=False) - - @ct.kernel def chunk_kda_fwd_kernel_inter_solve_fused( q, @@ -1727,18 +1596,18 @@ def chunk_kda_fwd_kernel_inter_solve_fused( b_Aqk32 = z b_Akk32 = z - # ---- off-diagonal blocks ---- + # ---- off-diagonal blocks ----------------------------------------------------- num_k = (K + BK - 1) // BK for i_k in range(num_k): kk = i_k * BK + o_k m_k = kk < K - b_k0 = _load_bc_bk(k, k_base, i_tc0, i_k * BK, H * K, BC, BK, T_eff, K) - b_g0 = _load_bc_bk(g, g_base, i_tc0, i_k * BK, HV * K, BC, BK, T_eff, K) + b_k0 = load_bc_bk(k, k_base, i_tc0, i_k * BK, H * K, BC, BK, T_eff, K) + b_g0 = load_bc_bk(g, g_base, i_tc0, i_k * BK, HV * K, BC, BK, T_eff, K) if i_tc1 < T_eff: - b_q1 = _load_bc_bk(q, q_base, i_tc1, i_k * BK, H * K, BC, BK, T_eff, K) - b_k1 = _load_bc_bk(k, k_base, i_tc1, i_k * BK, H * K, BC, BK, T_eff, K) - b_g1 = _load_bc_bk(g, g_base, i_tc1, i_k * BK, HV * K, BC, BK, T_eff, K) + b_q1 = load_bc_bk(q, q_base, i_tc1, i_k * BK, H * K, BC, BK, T_eff, K) + b_k1 = load_bc_bk(k, k_base, i_tc1, i_k * BK, H * K, BC, BK, T_eff, K) + b_g1 = load_bc_bk(g, g_base, i_tc1, i_k * BK, HV * K, BC, BK, T_eff, K) b_gn1 = ct.astype( ct.gather(g, g_base + i_tc1 * (HV * K) + kk, mask=m_k, check_bounds=False, padding_value=0.0), ct.float32, @@ -1749,9 +1618,9 @@ def chunk_kda_fwd_kernel_inter_solve_fused( b_Akk10 = safe_mma(b_k1 * b_gqn, b_kgt, b_Akk10) if NC >= 3 and i_tc2 < T_eff: - b_q2 = _load_bc_bk(q, q_base, i_tc2, i_k * BK, H * K, BC, BK, T_eff, K) - b_k2 = _load_bc_bk(k, k_base, i_tc2, i_k * BK, H * K, BC, BK, T_eff, K) - b_g2 = _load_bc_bk(g, g_base, i_tc2, i_k * BK, HV * K, BC, BK, T_eff, K) + b_q2 = load_bc_bk(q, q_base, i_tc2, i_k * BK, H * K, BC, BK, T_eff, K) + b_k2 = load_bc_bk(k, k_base, i_tc2, i_k * BK, H * K, BC, BK, T_eff, K) + b_g2 = load_bc_bk(g, g_base, i_tc2, i_k * BK, HV * K, BC, BK, T_eff, K) b_gn2 = ct.astype( ct.gather(g, g_base + i_tc2 * (HV * K) + kk, mask=m_k, check_bounds=False, padding_value=0.0), ct.float32, @@ -1767,9 +1636,9 @@ def chunk_kda_fwd_kernel_inter_solve_fused( b_Akk21 = safe_mma(b_kg2, b_kgt, b_Akk21) if NC >= 4 and i_tc3 < T_eff: - b_q3 = _load_bc_bk(q, q_base, i_tc3, i_k * BK, H * K, BC, BK, T_eff, K) - b_k3 = _load_bc_bk(k, k_base, i_tc3, i_k * BK, H * K, BC, BK, T_eff, K) - b_g3 = _load_bc_bk(g, g_base, i_tc3, i_k * BK, HV * K, BC, BK, T_eff, K) + b_q3 = load_bc_bk(q, q_base, i_tc3, i_k * BK, H * K, BC, BK, T_eff, K) + b_k3 = load_bc_bk(k, k_base, i_tc3, i_k * BK, H * K, BC, BK, T_eff, K) + b_g3 = load_bc_bk(g, g_base, i_tc3, i_k * BK, HV * K, BC, BK, T_eff, K) b_gn3 = ct.astype( ct.gather(g, g_base + i_tc3 * (HV * K) + kk, mask=m_k, check_bounds=False, padding_value=0.0), ct.float32, @@ -1787,17 +1656,17 @@ def chunk_kda_fwd_kernel_inter_solve_fused( b_Aqk32 = safe_mma(b_qg3, b_kgt, b_Aqk32) b_Akk32 = safe_mma(b_kg3, b_kgt, b_Akk32) - # ---- save off-diagonal Aqk blocks, scale Akk by beta ---- + # ---- save off-diagonal Aqk blocks, scale Akk by Beta ------------------------- if i_tc1 < T_eff: - _store_bc_bc(Aqk, Aqk_base, i_tc1, 0, HV * BT, b_Aqk10 * scale, BC, BT, T_eff) + store_bc_bc(Aqk, Aqk_base, i_tc1, 0, HV * BT, b_Aqk10 * scale, BC, BT, T_eff) b_b1 = ct.astype( ct.gather(beta, beta_base + (i_tc1 + o_i) * HV, mask=m_tc1, check_bounds=False, padding_value=0.0), ct.float32, ) b_Akk10 = b_Akk10 * b_b1[:, None] if NC >= 3 and i_tc2 < T_eff: - _store_bc_bc(Aqk, Aqk_base, i_tc2, 0, HV * BT, b_Aqk20 * scale, BC, BT, T_eff) - _store_bc_bc(Aqk, Aqk_base, i_tc2, BC, HV * BT, b_Aqk21 * scale, BC, BT, T_eff) + store_bc_bc(Aqk, Aqk_base, i_tc2, 0, HV * BT, b_Aqk20 * scale, BC, BT, T_eff) + store_bc_bc(Aqk, Aqk_base, i_tc2, BC, HV * BT, b_Aqk21 * scale, BC, BT, T_eff) b_b2 = ct.astype( ct.gather(beta, beta_base + (i_tc2 + o_i) * HV, mask=m_tc2, check_bounds=False, padding_value=0.0), ct.float32, @@ -1805,9 +1674,9 @@ def chunk_kda_fwd_kernel_inter_solve_fused( b_Akk20 = b_Akk20 * b_b2[:, None] b_Akk21 = b_Akk21 * b_b2[:, None] if NC >= 4 and i_tc3 < T_eff: - _store_bc_bc(Aqk, Aqk_base, i_tc3, 0, HV * BT, b_Aqk30 * scale, BC, BT, T_eff) - _store_bc_bc(Aqk, Aqk_base, i_tc3, BC, HV * BT, b_Aqk31 * scale, BC, BT, T_eff) - _store_bc_bc(Aqk, Aqk_base, i_tc3, 2 * BC, HV * BT, b_Aqk32 * scale, BC, BT, T_eff) + store_bc_bc(Aqk, Aqk_base, i_tc3, 0, HV * BT, b_Aqk30 * scale, BC, BT, T_eff) + store_bc_bc(Aqk, Aqk_base, i_tc3, BC, HV * BT, b_Aqk31 * scale, BC, BT, T_eff) + store_bc_bc(Aqk, Aqk_base, i_tc3, 2 * BC, HV * BT, b_Aqk32 * scale, BC, BT, T_eff) b_b3 = ct.astype( ct.gather(beta, beta_base + (i_tc3 + o_i) * HV, mask=m_tc3, check_bounds=False, padding_value=0.0), ct.float32, @@ -1816,13 +1685,13 @@ def chunk_kda_fwd_kernel_inter_solve_fused( b_Akk31 = b_Akk31 * b_b3[:, None] b_Akk32 = b_Akk32 * b_b3[:, None] - # ---- load diagonal inverse blocks from Akkd (fp32) ---- - b_Ai00 = _load_bc_bk(Akkd, Akkd_base, i_tc0, 0, HV * BC, BC, BC, T_eff, BC) - b_Ai11 = _load_bc_bk(Akkd, Akkd_base, i_tc1, 0, HV * BC, BC, BC, T_eff, BC) - b_Ai22 = _load_bc_bk(Akkd, Akkd_base, i_tc2, 0, HV * BC, BC, BC, T_eff, BC) if NC >= 3 else z - b_Ai33 = _load_bc_bk(Akkd, Akkd_base, i_tc3, 0, HV * BC, BC, BC, T_eff, BC) if NC >= 4 else z + # ---- load diagonal inverse blocks from Akkd (fp32) --------------------------- + b_Ai00 = load_bc_bk(Akkd, Akkd_base, i_tc0, 0, HV * BC, BC, BC, T_eff, BC) + b_Ai11 = load_bc_bk(Akkd, Akkd_base, i_tc1, 0, HV * BC, BC, BC, T_eff, BC) + b_Ai22 = load_bc_bk(Akkd, Akkd_base, i_tc2, 0, HV * BC, BC, BC, T_eff, BC) if NC >= 3 else z + b_Ai33 = load_bc_bk(Akkd, Akkd_base, i_tc3, 0, HV * BC, BC, BC, T_eff, BC) if NC >= 4 else z - # ---- forward substitution on diagonals (only when gate not pre-solved) ---- + # ---- forward substitution on diagonals (only when gate not pre-solved) ------- if not USE_SAFE_GATE: m_A = o_i[:, None] > o_i[None, :] m_I = o_i[:, None] == o_i[None, :] @@ -1836,9 +1705,7 @@ def chunk_kda_fwd_kernel_inter_solve_fused( # Counted loops over the static [BC] forward-substitution window with a # runtime guard (i < T_eff - i_tc0). A runtime upper bound (ct_min(..., # T_eff - i_tc0)) lowers to per-iteration branches tileiras can't - # unroll/predicate; the counted form stays fully unrolled and is - # numerically identical (inactive rows are skipped by the guard). See - # MEMORY counted-loop note. + # unroll/predicate; the counted form stays fully unrolled. for i in range(2, BC): if i < T_eff - i_tc0: b_a00 = -ct.astype( @@ -1885,7 +1752,7 @@ def chunk_kda_fwd_kernel_inter_solve_fused( if NC >= 4: b_Ai33 = b_Ai33 + ct.astype(m_I, ct.float32) - # ---- merged inverse using off-diagonals (tf32) ---- + # ---- merged inverse using off-diagonals (tf32) ------------------------------- b_Ai10 = -safe_matmul(safe_matmul(b_Ai11, b_Akk10), b_Ai00) b_Ai20 = z b_Ai21 = z @@ -1900,23 +1767,23 @@ def chunk_kda_fwd_kernel_inter_solve_fused( b_Ai31 = -safe_matmul(b_Ai33, safe_matmul(b_Akk31, b_Ai11) + safe_matmul(b_Akk32, b_Ai21)) b_Ai30 = -safe_matmul(b_Ai33, safe_matmul(b_Akk30, b_Ai00) + safe_matmul(b_Akk31, b_Ai10) + safe_matmul(b_Akk32, b_Ai20)) - # ---- store full Akk_inv to Akk ---- - _store_bc_bc(Akk, Akk_base, i_tc0, 0, HV * BT, b_Ai00, BC, BT, T_eff) - _store_bc_bc(Akk, Akk_base, i_tc1, 0, HV * BT, b_Ai10, BC, BT, T_eff) - _store_bc_bc(Akk, Akk_base, i_tc1, BC, HV * BT, b_Ai11, BC, BT, T_eff) + # ---- store full Akk_inv to Akk ----------------------------------------------- + store_bc_bc(Akk, Akk_base, i_tc0, 0, HV * BT, b_Ai00, BC, BT, T_eff) + store_bc_bc(Akk, Akk_base, i_tc1, 0, HV * BT, b_Ai10, BC, BT, T_eff) + store_bc_bc(Akk, Akk_base, i_tc1, BC, HV * BT, b_Ai11, BC, BT, T_eff) if NC >= 3: - _store_bc_bc(Akk, Akk_base, i_tc2, 0, HV * BT, b_Ai20, BC, BT, T_eff) - _store_bc_bc(Akk, Akk_base, i_tc2, BC, HV * BT, b_Ai21, BC, BT, T_eff) - _store_bc_bc(Akk, Akk_base, i_tc2, 2 * BC, HV * BT, b_Ai22, BC, BT, T_eff) + store_bc_bc(Akk, Akk_base, i_tc2, 0, HV * BT, b_Ai20, BC, BT, T_eff) + store_bc_bc(Akk, Akk_base, i_tc2, BC, HV * BT, b_Ai21, BC, BT, T_eff) + store_bc_bc(Akk, Akk_base, i_tc2, 2 * BC, HV * BT, b_Ai22, BC, BT, T_eff) if NC >= 4: - _store_bc_bc(Akk, Akk_base, i_tc3, 0, HV * BT, b_Ai30, BC, BT, T_eff) - _store_bc_bc(Akk, Akk_base, i_tc3, BC, HV * BT, b_Ai31, BC, BT, T_eff) - _store_bc_bc(Akk, Akk_base, i_tc3, 2 * BC, HV * BT, b_Ai32, BC, BT, T_eff) - _store_bc_bc(Akk, Akk_base, i_tc3, 3 * BC, HV * BT, b_Ai33, BC, BT, T_eff) + store_bc_bc(Akk, Akk_base, i_tc3, 0, HV * BT, b_Ai30, BC, BT, T_eff) + store_bc_bc(Akk, Akk_base, i_tc3, BC, HV * BT, b_Ai31, BC, BT, T_eff) + store_bc_bc(Akk, Akk_base, i_tc3, 2 * BC, HV * BT, b_Ai32, BC, BT, T_eff) + store_bc_bc(Akk, Akk_base, i_tc3, 3 * BC, HV * BT, b_Ai33, BC, BT, T_eff) # =========================================================================== -# chunk_bwd (dAv) -- dAqk = do @ v.T ; dv = A @ do +# chunk_bwd (dAv) -- dAqk = dO @ V^T ; dV = A @ dO # =========================================================================== @@ -2032,7 +1899,7 @@ def chunk_kda_bwd_kernel_wy_dqkg_fused( i_hv = ct.bid(1) i_h = i_hv // (HV // H) - # grid dim-0 is the GLOBAL chunk index; h/dh are laid out per global + # grid dim-0 is the GLOBAL chunk index; H/dH are laid out per global # chunk (written with chunk_offsets+local). Capture it before i_t is # reassigned to the per-sequence (local) chunk index. i_tg = i_t @@ -2068,7 +1935,7 @@ def chunk_kda_bwd_kernel_wy_dqkg_fused( b_beta = ct.gather(beta, beta_base + o_t * HV, mask=m_t, check_bounds=False, padding_value=0.0) - # p_A transposed: view (BT, T) stride (1, HV*BT), block (BT, BT) at (0, off_t) + # A transposed: view (BT, T) stride (1, HV*BT), block (BT, BT) at (0, off_t) A_row = o_bt[:, None] A_time = (off_t + o_bt)[None, :] A_m = ct.broadcast_to((o_bt < BT)[:, None], (BT, BT)) & ((off_t + o_bt)[None, :] < T) @@ -2112,13 +1979,13 @@ def chunk_kda_bwd_kernel_wy_dqkg_fused( off_v = i_v * BV v_cols = (off_v + o_bv)[None, :] if STATE_V_FIRST: - # h/dh: view (V, K) stride (K, 1), block (BV, BK) at (off_v, off_k) + # H/dH: view (V, K) stride (K, 1), block (BV, BK) at (off_v, off_k) h_rows = (off_v + o_bv)[:, None] h_m = ct.broadcast_to((off_v + o_bv)[:, None] < V, (BV, BK)) & ct.broadcast_to(m_k[None, :], (BV, BK)) b_h = ct.gather(h, h_base + h_rows * K + o_k[None, :], mask=h_m, check_bounds=False, padding_value=0.0) b_dh = ct.gather(dh, dh_base + h_rows * K + o_k[None, :], mask=h_m, check_bounds=False, padding_value=0.0) else: - # h/dh transposed: view (V, K) stride (1, V), block (BV, BK) at (off_v, off_k) + # H/dH transposed: view (V, K) stride (1, V), block (BV, BK) at (off_v, off_k) h_row = (off_v + o_bv)[:, None] h_col = o_k[None, :] h_m = ct.broadcast_to((off_v + o_bv)[:, None] < V, (BV, BK)) & ct.broadcast_to(m_k[None, :], (BV, BK)) @@ -2152,9 +2019,7 @@ def chunk_kda_bwd_kernel_wy_dqkg_fused( b_dq = safe_mma(b_do, ct.astype(b_h, b_do.dtype), b_dq) b_dk = safe_mma(b_v_new, ct.astype(b_dh, b_v_new.dtype), b_dk) b_dw = safe_mma(ct.astype(b_dv, b_v_new.dtype), ct.astype(b_h, b_v_new.dtype), b_dw) - # TODO(cutile): an explicit debug-barrier here was - # dropped -- no cuTile equivalent; the b_v reuse below relies on - # program order within the block. + # the b_v reuse below relies on program order within the block. if i_k == 0: b_v = ct.gather( v, @@ -2236,7 +2101,7 @@ def chunk_kda_bwd_kernel_wy_dqkg_fused( # =========================================================================== -# chunk_bwd (intra) -- dq/dk/dg/db from dAqk/dAkk +# chunk_bwd (intra) -- dQ/dK/dG/dBeta from dAqk/dAkk # =========================================================================== @@ -2340,7 +2205,7 @@ def chunk_kda_bwd_kernel_intra( b_dk2 = b_dk2 * b_gqn o_i = o_bc - # current block q, k + # current block Q, K b_q = ct.gather(q, q_base + cur_rows * (H * K) + cur_cols, mask=m_cur, check_bounds=False, padding_value=0.0) b_k = ct.gather(k, k_base + cur_rows * (H * K) + cur_cols, mask=m_cur, check_bounds=False, padding_value=0.0) @@ -2405,7 +2270,7 @@ def chunk_kda_bwd_kernel_intra( b_dg2 = ct.astype(b_q, ct.float32) * b_dq2 - # dq2 = b_dq2 + dq + # dQ2 = b_dq2 + dQ dq_off = dq_base + cur_rows * (HV * K) + cur_cols b_dq_prev = ct.astype(ct.gather(dq, dq_off, mask=m_cur, check_bounds=False, padding_value=0.0), ct.float32) b_dq2_out = b_dq2 + b_dq_prev @@ -2488,8 +2353,7 @@ def chunk_kda_bwd_kernel_intra( # A maskless gather still emits a default `other` (padding_value=0) # operand; without a matching `mask` the tileiras LDGSTS lowering # asserts (llOthers non-empty, llMasks empty). Supply an explicit - # row-validity mask. Values where the mask is false are zeroed but - # never used (m_i = o_i <= j gates the contribution below). + # row-validity mask. m_row = (i_ti + o_bc) < T_eff b_dAqk = ct.gather(dAqk, dAqk_base + base_o + j * (HV * BT), mask=m_row, check_bounds=False, padding_value=0.0) b_dAkk = ct.gather(dAkk, dAkk_base + base_o + j * (HV * BT), mask=m_row, check_bounds=False, padding_value=0.0) @@ -2673,28 +2537,21 @@ def l2norm_fwd( stream=None, ): stream = 0 if stream is None else stream - if out is None or rstd_out is None: - raise ValueError("l2norm_fwd requires pre-allocated out= and rstd_out= buffers") x_shape_og = x.shape - x = x.reshape(-1, x.shape[-1]).contiguous() + x = reshaped(x, (-1, x.shape[-1])) y = reshaped(out, tuple(x.shape)) T, D = x.shape[0], x.shape[-1] MAX_FUSED_SIZE = 65536 // x.element_size() - BD = min(MAX_FUSED_SIZE, _next_power_of_2(D)) - if D > BD: - raise RuntimeError("This layer doesn't support feature dim >= 64KB.") + BD = min(MAX_FUSED_SIZE, next_power_of_2(D)) rstd = reshaped(rstd_out, (T,)) if D <= 512: BT = 32 - grid = (_cdiv(T, BT),) + grid = (cdiv(T, BT),) # l2norm_fwd is a MEMORY-bound row reduction (each row loaded once, - # reduced, written once). Fixed occ=1/nww=4 ran ~4x slower than - # tuned. Sweep occupancy x num_worker_warps (launch hints only, single - # kernel unchanged); the default (occ=1,nww=4) is tried first so a - # non-improving shape keeps the base behaviour. Memory-bound => native/ - # higher occupancy hides DRAM latency (do NOT force occ=1 here). - _l2_key = ("l2norm_fwd_kernel", int(D), int(BD), int(BT), str(x.dtype), _dev_id(x)) - _autotuned_launch(l2norm_fwd_kernel, _l2_key, grid, (x, y, rstd, float(eps), T, D, BD, BT), occ_choices=(1, 2, 4, 8), nww_choices=(4,), stream=stream) + # reduced, written once); higher occupancy hides DRAM latency (do NOT + # force occ=1 here). + _l2_key = ("l2norm_fwd_kernel", int(D), int(BD), int(BT), str(x.dtype), dev_id(x)) + autotuned_launch(l2norm_fwd_kernel, _l2_key, grid, (x, y, rstd, float(eps), T, D, BD, BT), occ_choices=(1, 2, 4, 8), nww_choices=(4,), stream=stream) else: ct.launch(stream, (T,), l2norm_fwd_kernel1, (x, y, rstd, float(eps), D, BD)) return y.view(x_shape_og), rstd.view(x_shape_og[:-1]) @@ -2703,20 +2560,16 @@ def l2norm_fwd( def l2norm_bwd(y, rstd, dy, eps: float = 1e-6, out=None, stream=None): stream = 0 if stream is None else stream y_shape_og = y.shape - y = y.reshape(-1, dy.shape[-1]).contiguous() - dy = dy.reshape(-1, dy.shape[-1]).contiguous() - if out is None: - raise ValueError("l2norm_bwd requires a pre-allocated out= buffer") + y = y.reshape(-1, dy.shape[-1]) + dy = dy.reshape(-1, dy.shape[-1]) dx = reshaped(out, tuple(y.shape)) T, D = y.shape[0], y.shape[-1] MAX_FUSED_SIZE = 65536 // y.element_size() - BD = min(MAX_FUSED_SIZE, _next_power_of_2(D)) - if D > BD: - raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") - rstd_flat = rstd.reshape(-1).contiguous() + BD = min(MAX_FUSED_SIZE, next_power_of_2(D)) + rstd_flat = rstd.reshape(-1) if D <= 512: BT = 32 - grid = (_cdiv(T, BT),) + grid = (cdiv(T, BT),) ct.launch(stream, grid, l2norm_bwd_kernel, (y, rstd_flat, dy, dx, float(eps), T, D, BD, BT)) else: ct.launch(stream, (T,), l2norm_bwd_kernel1, (y, rstd_flat, dy, dx, float(eps), D, BD)) @@ -2726,37 +2579,33 @@ def l2norm_bwd(y, rstd, dy, eps: float = 1e-6, out=None, stream=None): # --------------------------------------------------------------------------- # fused_beta_sigmoid # --------------------------------------------------------------------------- -_BETA_SIGMOID_BLOCK_SIZE = 2048 +BETA_SIGMOID_BLOCK_SIZE = 2048 def fused_beta_sigmoid_fwd(x, scale: float = 1.0, out=None, stream=None): stream = 0 if stream is None else stream - if out is None: - raise ValueError("fused_beta_sigmoid_fwd requires a pre-allocated out= buffer (fp32, x-shaped)") y = reshaped(out, tuple(x.shape)) n = x.numel() - grid = (_cdiv(n, _BETA_SIGMOID_BLOCK_SIZE),) + grid = (cdiv(n, BETA_SIGMOID_BLOCK_SIZE),) ct.launch( stream, grid, fused_beta_sigmoid_fwd_kernel, - (x.reshape(-1), y.reshape(-1), float(scale), n, _BETA_SIGMOID_BLOCK_SIZE), + (reshaped(x, (-1,)), y.reshape(-1), float(scale), n, BETA_SIGMOID_BLOCK_SIZE), ) return y def fused_beta_sigmoid_bwd(x, dy, scale: float = 1.0, out=None, stream=None): stream = 0 if stream is None else stream - if out is None: - raise ValueError("fused_beta_sigmoid_bwd requires a pre-allocated out= buffer (x-shaped)") dx = reshaped(out, tuple(x.shape)) n = x.numel() - grid = (_cdiv(n, _BETA_SIGMOID_BLOCK_SIZE),) + grid = (cdiv(n, BETA_SIGMOID_BLOCK_SIZE),) ct.launch( stream, grid, fused_beta_sigmoid_bwd_kernel, - (x.reshape(-1), dy.reshape(-1), dx.reshape(-1), float(scale), n, _BETA_SIGMOID_BLOCK_SIZE), + (reshaped(x, (-1,)), dy.reshape(-1), dx.reshape(-1), float(scale), n, BETA_SIGMOID_BLOCK_SIZE), ) return dx @@ -2770,7 +2619,7 @@ def fused_beta_sigmoid(x, scale: float = 1.0, out=None, stream=None): # --------------------------------------------------------------------------- # chunk_local_cumsum (vector gate) / kda_gate_chunk_cumsum / kda_gate_bwd # --------------------------------------------------------------------------- -_BS_LIST_DEFAULT = 32 +BS_LIST_DEFAULT = 32 def chunk_local_cumsum_vector( @@ -2786,26 +2635,22 @@ def chunk_local_cumsum_vector( stream = 0 if stream is None else stream T, H, S = g.shape BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") - if out is None: - raise ValueError("chunk_local_cumsum requires a pre-allocated out= buffer (g-shaped)") NT = len(chunk_indices) assert chunk_size == 2 ** (chunk_size.bit_length() - 1), "chunk_size must be a power of 2" - BS = min(_BS_LIST_DEFAULT, _next_power_of_2(S)) + BS = min(BS_LIST_DEFAULT, next_power_of_2(S)) g_org = g g_out = reshaped(out, (T, H, S)) scale_val = float(scale) if scale is not None else 0.0 has_scale = int(scale is not None) - cu_arg = _i32_flat(cu_seqlens) - ci_arg = _i32_flat(chunk_indices) - grid = (_cdiv(S, BS), NT, H) + cu_arg = i32_flat(cu_seqlens) + ci_arg = i32_flat(chunk_indices) + grid = (cdiv(S, BS), NT, H) ct.launch( stream, grid, chunk_local_cumsum_vector_kernel, ( - g_org.reshape(-1), + reshaped(g_org, (-1,)), g_out.reshape(-1), scale_val, cu_arg, @@ -2832,7 +2677,6 @@ def chunk_local_cumsum( stream=None, ): stream = 0 if stream is None else stream - assert len(g.shape) == 3, f"KDA chunk_local_cumsum expects a (T, HV, K) vector gate, got shape {tuple(g.shape)}" return chunk_local_cumsum_vector( g=g, chunk_size=chunk_size, @@ -2861,27 +2705,23 @@ def kda_gate_chunk_cumsum( stream = 0 if stream is None else stream T, H, S = g.shape BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") - if out is None: - raise ValueError("kda_gate_chunk_cumsum requires a pre-allocated out= buffer (fp32, g-shaped)") NT = len(chunk_indices) assert chunk_size == 2 ** (chunk_size.bit_length() - 1), "chunk_size must be a power of 2" - BS = min(_BS_LIST_DEFAULT, _next_power_of_2(S)) + BS = min(BS_LIST_DEFAULT, next_power_of_2(S)) g_out = reshaped(out, (T, H, S)) - dt_arg = _opt(dt_bias, bufs, _dtname(A_log)).reshape(-1) + dt_arg = reshaped(opt(dt_bias, bufs, dtname(A_log)), (-1,)) scale_val = float(scale) if scale is not None else 0.0 lb_val = float(lower_bound) if lower_bound is not None else 0.0 - cu_arg = _i32_flat(cu_seqlens) - ci_arg = _i32_flat(chunk_indices) - grid = (_cdiv(S, BS), NT, H) + cu_arg = i32_flat(cu_seqlens) + ci_arg = i32_flat(chunk_indices) + grid = (cdiv(S, BS), NT, H) ct.launch( stream, grid, kda_gate_chunk_cumsum_vector_kernel, ( - g.reshape(-1), - A_log.reshape(-1), + reshaped(g, (-1,)), + reshaped(A_log, (-1,)), dt_arg, g_out.reshape(-1), scale_val, @@ -2909,15 +2749,13 @@ def kda_gate_bwd(g, A_log, dt_bias=None, dyg=None, lower_bound=None, dg_out=None H, K = g.shape[-2:] T = g.numel() // (H * K) BT = 32 - NT = _cdiv(T, BT) - BD = _next_power_of_2(K) - if dg_out is None or dA_out is None or (dt_bias is not None and dbias_out is None): - raise ValueError("kda_gate_bwd requires pre-allocated dg_out=, dA_out= (and dbias_out= with dt_bias)") + NT = cdiv(T, BT) + BD = next_power_of_2(K) dg = reshaped(dg_out, tuple(g.shape)) dA_nt = reshaped(bufs["dA_gate"], (NT, H)) db_nt = reshaped(bufs["db_gate"], (NT, H * K)) if dt_bias is not None else None - dt_arg = _opt(dt_bias, bufs, _dtname(A_log)).reshape(-1) - db_arg = db_nt.reshape(-1) if db_nt is not None else _dummy("float32", bufs) + dt_arg = reshaped(opt(dt_bias, bufs, dtname(A_log)), (-1,)) + db_arg = db_nt.reshape(-1) if db_nt is not None else dummy("float32", bufs) lb_val = float(lower_bound) if lower_bound is not None else 0.0 grid = (NT, H) ct.launch( @@ -2925,10 +2763,10 @@ def kda_gate_bwd(g, A_log, dt_bias=None, dyg=None, lower_bound=None, dg_out=None grid, kda_gate_bwd_kernel, ( - g.reshape(-1), - A_log.reshape(-1), + reshaped(g, (-1,)), + reshaped(A_log, (-1,)), dt_arg, - dyg.reshape(-1), + reshaped(dyg, (-1,)), dg.reshape(-1), dA_nt.reshape(-1), db_arg, @@ -2955,28 +2793,26 @@ def chunk_gla_fwd_o_gk(q, v, g, A, h, scale, state_v_first=False, cu_seqlens=Non stream = 0 if stream is None else stream T, H, K, HV, V = *q.shape, v.shape[1], v.shape[-1] BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") NT = len(chunk_indices) o = bufs["o"] zero_fill(o, stream=stream) - BK = min(max(_next_power_of_2(K), 16), 64) + BK = min(max(next_power_of_2(K), 16), 64) # BV=128 when V<=128 removes the V grid-split # (grid dim0 -> 1), doubling work/block but halving launched blocks. BV = 128 if V <= 128 else 64 - dev = _dev_id(q) - cu_arg = _i32_flat(cu_seqlens) - ci_arg = _i32_flat(chunk_indices) + dev = dev_id(q) + cu_arg = i32_flat(cu_seqlens) + ci_arg = i32_flat(chunk_indices) - grid = (_cdiv(V, BV), NT, HV) + grid = (cdiv(V, BV), NT, HV) # Kernel uses flat element-offset gather/scatter; pass 1-D views so the # cuTile index-tuple rank (1) matches the array rank. - _q_arg = q.reshape(-1) + _q_arg = reshaped(q, (-1,)) _v_arg = v.reshape(-1) _g_arg = g.reshape(-1) _h_arg = h.reshape(-1) - _o_arg = o.reshape(-1) + _o_arg = reshaped(o, (-1,)) _A_arg = A.reshape(-1) _o_args = ( _q_arg, @@ -2998,8 +2834,7 @@ def chunk_gla_fwd_o_gk(q, v, g, A, h, scale, state_v_first=False, cu_seqlens=Non int(state_v_first), ) # Launch-hint autotune (occupancy x num_worker_warps) on this - # output-projection kernel. Default config - # is tried first so non-improving shapes are unchanged. + # output-projection kernel. _o_key = ( "chunk_gla_fwd_kernel_o", int(H), @@ -3013,7 +2848,7 @@ def chunk_gla_fwd_o_gk(q, v, g, A, h, scale, state_v_first=False, cu_seqlens=Non str(q.dtype), str(dev), ) - _autotuned_launch(chunk_gla_fwd_kernel_o, _o_key, grid, _o_args, occ_choices=(1, 2, 4), nww_choices=(4,), stream=stream) + autotuned_launch(chunk_gla_fwd_kernel_o, _o_key, grid, _o_args, occ_choices=(1, 2, 4), nww_choices=(4,), stream=stream) return o @@ -3040,30 +2875,28 @@ def chunk_gated_delta_rule_fwd_h( stream = 0 if stream is None else stream T, H, K, V, HV = *k.shape, u.shape[-1], u.shape[1] BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") N, NT = len(cu_seqlens) - 1, len(chunk_indices) chunk_offsets = bufs["chunk_offsets"] # Full-width state/K tile: BK = next_pow2(K). The blockdim64 kernel carries the # KV state as a single (BV, BK)/(BK, BV) tile (no K<=256 cap); K-axis loads # zero-pad [K:BK] and stores drop it, so any K is supported. - BK = _next_power_of_2(K) + BK = next_power_of_2(K) state_shape = (N, HV, V, K) if state_v_first else (N, HV, K, V) - h = reshaped(bufs["h"], (NT, HV) + state_shape[2:]) - final_state = reshaped(bufs["fs"], state_shape) if output_final_state else None + h = reshaped(bufs["state_checkpoints"], (NT, HV) + state_shape[2:]) + final_state = reshaped(bufs["final_state"], state_shape) if output_final_state else None if final_state is not None: zero_fill(final_state, stream=stream) v_new = reshaped(bufs["v_new"], (T, HV, V)) if save_new_value else None - dev = _dev_id(k) - vnew_arg = v_new if v_new is not None else _dummy(_dtname(u), bufs) - g_arg = _opt(g, bufs) - gk_arg = _opt(gk, bufs) - h0_arg = initial_state if initial_state is not None else _dummy("float32", bufs) - ht_arg = final_state if final_state is not None else _dummy("float32", bufs) - cu_arg = _i32_flat(cu_seqlens) - co_arg = _i32_flat(chunk_offsets) + dev = dev_id(k) + vnew_arg = v_new if v_new is not None else dummy(dtname(u), bufs) + g_arg = opt(g, bufs) + gk_arg = opt(gk, bufs) + h0_arg = initial_state if initial_state is not None else dummy("float32", bufs) + ht_arg = final_state if final_state is not None else dummy("float32", bufs) + cu_arg = i32_flat(cu_seqlens) + co_arg = i32_flat(chunk_offsets) # Kernel uses flat element-offset gather/scatter; pass 1-D views so the # cuTile index-tuple rank (1) matches the array rank. @@ -3072,22 +2905,17 @@ def chunk_gated_delta_rule_fwd_h( _w_arg = w.reshape(-1) _vnew_arg = vnew_arg.reshape(-1) _h_arg = h.reshape(-1) - _h01d = h0_arg.reshape(-1) - _ht1d = ht_arg.reshape(-1) + _h01d = reshaped(h0_arg, (-1,)) + _ht1d = reshaped(ht_arg, (-1,)) _g1d = g_arg.reshape(-1) _gk1d = gk_arg.reshape(-1) # BV is a V-tiling block width (drives the grid V-fan-out and the V-tile # shapes only; the K axis is always split into fixed 64-wide blocks, so BV - # never changes the numerics). ncu shows this latency-bound state scan is - # grid-under-filled at the previous hard-coded BV=32 (only cdiv(V,32)*N*HV - # CTAs, 1 block/SM) -- shrinking BV multiplies the CTA count and was the - # dominant fwd_h speedup (BV 32->8: 8192/K128 492->260us). - # Sweep BV over {16, 8, 32} (16 first => safe, beats baseline everywhere); - # the tuner picks the per-shape grid fill that best hides the inter-chunk - # latency. + # never changes the numerics). The tuner picks the per-shape grid fill that + # best hides the inter-chunk latency. def _grid_fn(bv): - return (_cdiv(V, bv), N * HV) + return (cdiv(V, bv), N * HV) def _args_fn(bv): return ( @@ -3137,19 +2965,13 @@ def _args_fn(bv): # capped at <= V so a single tile is never larger than V. The grid-fill # tradeoff is shape-dependent, not monotone: shrinking BV multiplies the # V-tile CTA count but each CTA is register-bound (255 reg -> ~1 block/SM) - # and redundantly reloads k/w/g, so past the point where the grid already + # and redundantly reloads K/W/G, so past the point where the grid already # fills the SMs a *larger* BV wins (fewer, fatter CTAs, higher warp - # occupancy). ncu on the dashboard shapes: at (T=2048,V=128,N*HV=64) BV=64 - # (grid 128, warps_active 14%) is 246us vs BV=32 (grid 256, 8%) 289us vs - # BV=16 374us -- so 64 must be a candidate; at (T=1024,V=64,N*HV=64) BV=32 - # (grid 128) 65us beats BV=64 (grid 64) 123us and BV=8 123us. The tuner - # benchmarks every candidate and picks the per-shape winner, so we offer the - # full {16, 8, 32, 64} sweep (mirrors the fwd-helper); 16 stays first as the - # safe DISABLE_TUNE/failure fallback. + # occupancy); 16 stays first as the safe DISABLE_TUNE/failure fallback. _bv_choices = tuple(bv for bv in (16, 8, 32, 64) if bv <= max(V, 8)) if not _bv_choices: _bv_choices = (min(32, V),) - _autotuned_launch_bv(chunk_gated_delta_rule_fwd_kernel_h_blockdim64, _h_key, _bv_choices, _grid_fn, _args_fn, stream=stream) + autotuned_launch_bv(chunk_gated_delta_rule_fwd_kernel_h_blockdim64, _h_key, _bv_choices, _grid_fn, _args_fn, stream=stream) return h, v_new, final_state @@ -3162,7 +2984,7 @@ def chunk_gated_delta_rule_bwd_dhu( g=None, gk=None, h0=None, - dht=None, + dstate_in=None, scale: float | None = None, state_v_first: bool = False, cu_seqlens=None, @@ -3175,21 +2997,19 @@ def chunk_gated_delta_rule_bwd_dhu( T, H, K, V, HV = *q.shape, do.shape[-1], do.shape[1] BT = chunk_size # Full-width state/K tile: BK = next_pow2(K). The blockdim64 dhu kernel carries - # the dh state as a single (BV, BK)/(BK, BV) tile (no K<=256 cap); K-axis loads - # zero-pad/mask [K:BK] and stores drop it, so any K is supported. dh/dh0 + # the dH state as a single (BV, BK)/(BK, BV) tile (no K<=256 cap); K-axis loads + # zero-pad/mask [K:BK] and stores drop it, so any K is supported. dH/dH0 # carves stay K-wide. - BK = _next_power_of_2(K) - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") + BK = next_power_of_2(K) N, NT = len(cu_seqlens) - 1, len(chunk_indices) chunk_offsets = bufs["chunk_offsets"] - dh = reshaped(bufs["dh"], (NT, HV, V, K) if state_v_first else (NT, HV, K, V)) - dh0 = reshaped(bufs["dh0"], tuple(h0.shape)) if h0 is not None else None - dv2 = reshaped(bufs["dv_dhu"], (T, HV, V)) + dh = reshaped(bufs["dstate"], (NT, HV, V, K) if state_v_first else (NT, HV, K, V)) + dh0 = reshaped(bufs["dstate0"], tuple(h0.shape)) if h0 is not None else None + dv2 = reshaped(bufs["dv_dstate_u"], (T, HV, V)) BV = 64 - grid = (_cdiv(V, BV), N * HV) + grid = (cdiv(V, BV), N * HV) ct.launch( stream, @@ -3199,16 +3019,16 @@ def chunk_gated_delta_rule_bwd_dhu( q.reshape(-1), k.reshape(-1), w.reshape(-1), - _opt(g, bufs).reshape(-1), - _opt(gk, bufs).reshape(-1), - _opt(dht, bufs).reshape(-1), - (dh0 if dh0 is not None else _dummy("float32", bufs)).reshape(-1), - do.reshape(-1), + opt(g, bufs).reshape(-1), + opt(gk, bufs).reshape(-1), + reshaped(opt(dstate_in, bufs), (-1,)), + reshaped(dh0 if dh0 is not None else dummy("float32", bufs), (-1,)), + reshaped(do, (-1,)), dh.reshape(-1), dv.reshape(-1), dv2.reshape(-1), - _i32_flat(cu_seqlens), - _i32_flat(chunk_offsets), + i32_flat(cu_seqlens), + i32_flat(chunk_offsets), float(scale), H, HV, @@ -3220,7 +3040,7 @@ def chunk_gated_delta_rule_bwd_dhu( int(g is not None), int(gk is not None), int(h0 is not None), - int(dht is not None), + int(dstate_in is not None), int(state_v_first), ), ) @@ -3237,31 +3057,29 @@ def recompute_w_u_fwd(k, v, beta, A, gk, q=None, cu_seqlens=None, chunk_indices= BT = A.shape[-1] BK = 32 BV = 32 - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") NT = len(chunk_indices) - dev = _dev_id(k) + dev = dev_id(k) w = reshaped(bufs["w"], (T, HV, K)) u = reshaped(bufs["u"], (T, HV, V)) qg = reshaped(bufs["qg"], (T, HV, K)) if q is not None else None kg = reshaped(bufs["kg"], (T, HV, K)) - cu_arg = _i32_flat(cu_seqlens) - ci_arg = _i32_flat(chunk_indices) - q_arg = _opt(q, bufs, _dtname(k)) - qg_arg = qg if qg is not None else _dummy(_dtname(k), bufs) + cu_arg = i32_flat(cu_seqlens) + ci_arg = i32_flat(chunk_indices) + q_arg = opt(q, bufs, dtname(k)) + qg_arg = qg if qg is not None else dummy(dtname(k), bufs) # Kernel uses flat element-offset gather/scatter; pass 1-D views so the # cuTile index-tuple rank (1) matches the array rank. reshape(-1) on these - # contiguous tensors yields storage-aliasing views (outputs w/u/qg/kg too). + # contiguous tensors yields storage-aliasing views (outputs W/U/QG/KG too). _wu_grid = (NT, HV) _wu_args = ( - q_arg.reshape(-1), - k.reshape(-1), + reshaped(q_arg, (-1,)), + reshaped(k, (-1,)), qg_arg.reshape(-1), kg.reshape(-1), - v.reshape(-1), - beta.reshape(-1), + reshaped(v, (-1,)), + reshaped(beta, (-1,)), w.reshape(-1), u.reshape(-1), A.reshape(-1), @@ -3281,7 +3099,6 @@ def recompute_w_u_fwd(k, v, beta, A, gk, q=None, cu_seqlens=None, chunk_indices= ) # Launch-hint autotune (occupancy x num_worker_warps) on this wy_fast # recompute kernel. - # The default config is tried first so non-improving shapes are unchanged. _wu_key = ( "recompute_w_u_fwd_kda_kernel", int(H), @@ -3296,7 +3113,7 @@ def recompute_w_u_fwd(k, v, beta, A, gk, q=None, cu_seqlens=None, chunk_indices= str(k.dtype), str(dev), ) - _autotuned_launch(recompute_w_u_fwd_kda_kernel, _wu_key, _wu_grid, _wu_args, occ_choices=(1, 2, 4, 8), nww_choices=(4,), stream=stream) + autotuned_launch(recompute_w_u_fwd_kda_kernel, _wu_key, _wu_grid, _wu_args, occ_choices=(1, 2, 4, 8), nww_choices=(4,), stream=stream) return w, u, qg, kg @@ -3313,21 +3130,21 @@ def chunk_kda_fwd_intra_token_parallel(q, k, gk, beta, Aqk, Akk, scale, cu_seqle # BC-wide j-loop (gathers reused across heads share the same row indexing). # BH in {1,2,4,8}; HV must be divisible for the grid split. BH = 4 if (HV % 4 == 0) else (2 if (HV % 2 == 0) else 1) - BK = _next_power_of_2(K) - cu_arg = _i32_flat(cu_seqlens) - grid = (T, _cdiv(HV, BH)) + BK = next_power_of_2(K) + cu_arg = i32_flat(cu_seqlens) + grid = (T, cdiv(HV, BH)) # cuTile gather/scatter index-tuple rank must match the array rank, so pass - # pre-flattened views: q/k -> (T*H, K), gk -> (T*HV, K), beta/Aqk/Akk -> 1-D. + # pre-flattened views: Q/K -> (T*H, K), Gk -> (T*HV, K), Beta/Aqk/Akk -> 1-D. # Aqk/Akk are contiguous, so reshape(-1) is a view aliasing the original storage. ct.launch( stream, grid, chunk_kda_fwd_kernel_intra_token_parallel, ( - q.reshape(-1, K), - k.reshape(-1, K), + reshaped(q, (-1, K)), + reshaped(k, (-1, K)), gk.reshape(-1, K), - beta.reshape(-1), + reshaped(beta, (-1,)), Aqk.reshape(-1), Akk.reshape(-1), float(scale), @@ -3369,40 +3186,38 @@ def chunk_kda_fwd_intra( # BC=32 (NC=2) for BT=64,K>=64. With NC=2 there # is exactly ONE off-diagonal pair -> only 2 live [32,32] accumulators (vs 12 # [16,16] at NC=4), and its merged-inverse [32,32]@[32,32] ct.matmul lowers - # to HMMA (M=32) instead of SIMT scalar FADD/FMUL (M=16). See KDA fwd IR/SASS. + # to HMMA (M=32) instead of SIMT scalar FADD/FMUL (M=16). # K<64 falls back to BC=16/NC=4. BC = 32 if BT == 64 and K >= 64 else 16 - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") NT = len(chunk_indices) - NC = _cdiv(BT, BC) + NC = cdiv(BT, BC) # use_split_diag_compute_solve: pre-solve diagonals in # inter_diag_compute_solve so inter_solve_fused can SKIP forward-substitution. use_split_diag_compute_solve = (not safe_gate) and BT == 64 and K >= 64 use_solved_diagonal = safe_gate or use_split_diag_compute_solve - dev = _dev_id(k) + dev = dev_id(k) Aqk = reshaped(bufs["Aqk"], (T, HV, BT)) Akk = reshaped(bufs["Akk"], (T, HV, BT)) zero_fill(Akk, stream=stream) Akkd = reshaped(bufs["Akkd"], (T, HV, BC)) - cu_arg = _i32_flat(cu_seqlens) - ci_arg = _i32_flat(chunk_indices) + cu_arg = i32_flat(cu_seqlens) + ci_arg = i32_flat(chunk_indices) # Step 1: diagonal blocks into Akkd (fp32). When use_solved_diagonal is set # (safe_gate OR the split path) the diagonals are PRE-SOLVED # here so the inter-solve kernel can skip forward-substitution. if use_solved_diagonal: - BK = _next_power_of_2(K) + BK = next_power_of_2(K) # Kernel uses flat element-offset gather/scatter; pass 1-D views so the # cuTile index-tuple rank (1) matches the array rank. _diag_grid = (NT, NC, HV) _diag_args = ( - q.reshape(-1), - k.reshape(-1), + reshaped(q, (-1,)), + reshaped(k, (-1,)), gk.reshape(-1), - beta.reshape(-1), + reshaped(beta, (-1,)), Aqk.reshape(-1), Akkd.reshape(-1), float(scale), @@ -3416,8 +3231,7 @@ def chunk_kda_fwd_intra( BK, ) # Launch-hint autotune (occupancy x num_worker_warps) on this aux - # kernel; fixed default hints (occ=1, nww=4) ran ~2.9x slower. The - # default config is tried first so non-improving shapes are unchanged. + # kernel. _diag_key = ( "chunk_kda_fwd_kernel_inter_diag_compute_solve", int(H), @@ -3430,7 +3244,7 @@ def chunk_kda_fwd_intra( str(k.dtype), str(dev), ) - _autotuned_launch(chunk_kda_fwd_kernel_inter_diag_compute_solve, _diag_key, _diag_grid, _diag_args, stream=stream) + autotuned_launch(chunk_kda_fwd_kernel_inter_diag_compute_solve, _diag_key, _diag_grid, _diag_args, stream=stream) else: Aqk, Akkd = chunk_kda_fwd_intra_token_parallel( q=q, k=k, gk=gk, beta=beta, Aqk=Aqk, Akk=Akkd, scale=scale, cu_seqlens=cu_seqlens, chunk_size=BT, sub_chunk_size=BC, stream=stream @@ -3441,16 +3255,15 @@ def chunk_kda_fwd_intra( # kernel. The fused kernel handles NC>=3/NC>=4 internally (block-triangular # forward-substitution over all sub-chunk pairs). With use_solved_diagonal # the forward-substitution is skipped. - BKf = _next_power_of_2(K) + BKf = next_power_of_2(K) # Launch-hint autotune (occupancy x num_worker_warps) on this fused - # solve kernel. The - # default config is tried first so non-improving shapes are unchanged. + # solve kernel. _isf_grid = (NT, HV) _isf_args = ( - q.reshape(-1), - k.reshape(-1), + reshaped(q, (-1,)), + reshaped(k, (-1,)), gk.reshape(-1), - beta.reshape(-1), + reshaped(beta, (-1,)), Aqk.reshape(-1), Akkd.reshape(-1), Akk.reshape(-1), @@ -3479,7 +3292,7 @@ def chunk_kda_fwd_intra( str(k.dtype), str(dev), ) - _autotuned_launch(chunk_kda_fwd_kernel_inter_solve_fused, _isf_key, _isf_grid, _isf_args, occ_choices=(1, 2, 4), nww_choices=(4,), stream=stream) + autotuned_launch(chunk_kda_fwd_kernel_inter_solve_fused, _isf_key, _isf_grid, _isf_args, occ_choices=(1, 2, 4), nww_choices=(4,), stream=stream) w, u, qg, kg = recompute_w_u_fwd( k=k, v=v, beta=beta, A=Akk, q=q if disable_recompute else None, gk=gk, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices, bufs=bufs, stream=stream ) @@ -3493,25 +3306,22 @@ def chunk_kda_bwd_intra(q, k, g, beta, dAqk, dAkk, dq, dk, db, dg, cu_seqlens=No # Fast path: for BT >= 64 use larger # BC=32 / BK=64 sub-tiles and route through the SAFE_GATE matmul branch # instead of the BC-iteration scalar `for j` loops. The scalar path is the - # bwd bottleneck; the matmul branch is numerically - # equivalent (it is the same code reached when the user sets safe_gate=True). + # bwd bottleneck. use_fast_path = BT >= 64 BC = 32 if use_fast_path else min(16, BT) - BK = min(64 if use_fast_path else 32, _next_power_of_2(K)) + BK = min(64 if use_fast_path else 32, next_power_of_2(K)) safe_gate = safe_gate or use_fast_path - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") NT = len(chunk_indices) - NC = _cdiv(BT, BC) - NK = _cdiv(K, BK) + NC = cdiv(BT, BC) + NK = cdiv(K, BK) dq2 = reshaped(bufs["dq2"], (T, HV, K)) dk2 = reshaped(bufs["dk2"], (T, HV, K)) db2 = reshaped(bufs["db2"], (NK, T, HV)) dg2 = reshaped(bufs["dg2"], (T, HV, K)) - cu_arg = _i32_flat(cu_seqlens) - ci_arg = _i32_flat(chunk_indices) + cu_arg = i32_flat(cu_seqlens) + ci_arg = i32_flat(chunk_indices) grid = (NK * NC, NT, HV) # Kernel uses flat element-offset gather/scatter; pass 1-D views. ct.launch( @@ -3519,10 +3329,10 @@ def chunk_kda_bwd_intra(q, k, g, beta, dAqk, dAkk, dq, dk, db, dg, cu_seqlens=No grid, chunk_kda_bwd_kernel_intra, ( - q.reshape(-1), - k.reshape(-1), + reshaped(q, (-1,)), + reshaped(k, (-1,)), g.reshape(-1), - beta.reshape(-1), + reshaped(beta, (-1,)), dAqk.reshape(-1), dAkk.reshape(-1), dq.reshape(-1), @@ -3547,7 +3357,7 @@ def chunk_kda_bwd_intra(q, k, g, beta, dAqk, dAkk, dq, dk, db, dg, cu_seqlens=No ) dq = dq2 dk = dk2 - # db += sum_nk db2 (fp32 acc); the fan-in NK is a compile-time constant + # dBeta += sum_nk dBeta2 (fp32 acc); the fan-in NK is a compile-time constant sum_leading(reshaped(db, (T * HV,)), reshaped(db2, (NK, T * HV)), NK, T * HV, stream=stream, accumulate=True) dg = dg2 return dq, dk, db, dg @@ -3560,28 +3370,26 @@ def chunk_kda_bwd_dAv(q, k, v, do, A=None, scale=None, cu_seqlens=None, chunk_si stream = 0 if stream is None else stream T, H, K, HV, V = *k.shape, do.shape[1], do.shape[-1] BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") CONST_TILING = 64 - BK = min(max(_next_power_of_2(K), 16), CONST_TILING) - BV = min(max(_next_power_of_2(V), 16), CONST_TILING) + BK = min(max(next_power_of_2(K), 16), CONST_TILING) + BV = min(max(next_power_of_2(V), 16), CONST_TILING) NT = len(chunk_indices) dA = reshaped(bufs["dAqk"], (T, HV, BT)) dv = reshaped(bufs["dv_dAv"], (T, HV, V)) - cu_arg = _i32_flat(cu_seqlens) - ci_arg = _i32_flat(chunk_indices) + cu_arg = i32_flat(cu_seqlens) + ci_arg = i32_flat(chunk_indices) # Kernel uses flat element-offset gather/scatter; pass 1-D views. ct.launch( stream, (NT, HV), chunk_kda_bwd_kernel_dAv, ( - q.reshape(-1), - k.reshape(-1), + reshaped(q, (-1,)), + reshaped(k, (-1,)), v.reshape(-1), A.reshape(-1), - do.reshape(-1), + reshaped(do, (-1,)), dv.reshape(-1), dA.reshape(-1), cu_arg, @@ -3622,12 +3430,10 @@ def chunk_kda_bwd_wy_dqkg_fused( stream = 0 if stream is None else stream T, H, K, HV, V = *k.shape, v.shape[1], v.shape[-1] BT = chunk_size - if cu_seqlens is None or chunk_indices is None: - raise ValueError("cu_seqlens and chunk_indices are required (THD layout; callers build the (seq, intra) table)") NT = len(chunk_indices) CONST_TILING = 64 - BK = min(max(_next_power_of_2(K), 16), CONST_TILING) - BV = min(max(_next_power_of_2(V), 16), CONST_TILING) + BK = min(max(next_power_of_2(K), 16), CONST_TILING) + BV = min(max(next_power_of_2(V), 16), CONST_TILING) dq = reshaped(bufs["dq"], (T, HV, K)) dk = reshaped(bufs["dk"], (T, HV, K)) @@ -3643,15 +3449,15 @@ def chunk_kda_bwd_wy_dqkg_fused( grid, chunk_kda_bwd_kernel_wy_dqkg_fused, ( - q.reshape(-1), - k.reshape(-1), - v.reshape(-1), + reshaped(q, (-1,)), + reshaped(k, (-1,)), + reshaped(v, (-1,)), v_new.reshape(-1), g.reshape(-1), - beta.reshape(-1), + reshaped(beta, (-1,)), A.reshape(-1), h.reshape(-1), - do.reshape(-1), + reshaped(do, (-1,)), dh.reshape(-1), dq.reshape(-1), dk.reshape(-1), @@ -3660,8 +3466,8 @@ def chunk_kda_bwd_wy_dqkg_fused( dg.reshape(-1), db.reshape(-1), dA.reshape(-1), - _i32_flat(cu_seqlens), - _i32_flat(chunk_indices), + i32_flat(cu_seqlens), + i32_flat(chunk_indices), float(scale), H, HV, @@ -3703,12 +3509,12 @@ def chunk_kda_fwd( dt_bias=None, disable_recompute=False, return_intermediate_states=False, + compute_o=True, cp_context=None, bufs=None, stream=None, ): stream = 0 if stream is None else stream - assert cp_context is None, "context-parallel path is not supported in this self-contained port" g_org = None if use_gate_in_kernel: g_org = g @@ -3768,20 +3574,22 @@ def chunk_kda_fwd( stream=stream, ) - o = chunk_gla_fwd_o_gk( - q=q, - v=v_new, - g=g, - A=Aqk, - h=h, - scale=scale, - cu_seqlens=cu_seqlens, - chunk_size=chunk_size, - chunk_indices=chunk_indices, - state_v_first=state_v_first, - bufs=bufs, - stream=stream, - ) + o = None + if compute_o: + o = chunk_gla_fwd_o_gk( + q=q, + v=v_new, + g=g, + A=Aqk, + h=h, + scale=scale, + cu_seqlens=cu_seqlens, + chunk_size=chunk_size, + chunk_indices=chunk_indices, + state_v_first=state_v_first, + bufs=bufs, + stream=stream, + ) if disable_recompute is False: w, u, qg, kg, v_new = None, None, None, None, None if not return_intermediate_states: @@ -3801,7 +3609,7 @@ def chunk_kda_bwd( scale, initial_state, do, - dht, + dstate_in, g=None, g_org=None, state_v_first=False, @@ -3820,7 +3628,6 @@ def chunk_kda_bwd( **kwargs, ): stream = 0 if stream is None else stream - assert cp_context is None, "context-parallel path is not supported in this self-contained port" H, HV = q.shape[1], v.shape[1] G = HV // H @@ -3867,7 +3674,7 @@ def chunk_kda_bwd( w=w, gk=g, h0=initial_state, - dht=dht, + dstate_in=dstate_in, do=do, dv=dv, scale=scale, @@ -3919,7 +3726,7 @@ def chunk_kda_bwd( stream=stream, ) - # For GVA, reduce dq and dk from [T, HV, K] back to [T, H, K] + # For GVA, reduce dQ and dK from [T, HV, K] back to [T, H, K] if HV > H: T_, K_ = dq.shape[0], dq.shape[-1] dq_r = reshaped(bufs["dq_hred"], (T_, H, K_)) @@ -3959,7 +3766,7 @@ def chunk_kda_grad( g, beta, do, - dht=None, + dstate_in=None, scale=None, initial_state=None, use_qk_l2norm_in_kernel=False, @@ -3983,7 +3790,7 @@ def chunk_kda_grad( ``q``/``k``/``v``/``do`` are ``[total_T, H, D]``, ``g`` is ``[total_T, HV, K]``, ``beta`` is ``[total_T, HV]``; ``cu_seqlens`` and ``chunk_indices`` are required. Recomputes the forward's prep - (L2-normalized q/k + rstd, cumulative gate, WY factors, per-chunk + (L2-normalized Q/K + rstd, cumulative gate, WY factors, per-chunk states) from the inputs, then runs the backward kernels; intermediates live in the ``bufs`` carves when provided. @@ -3992,8 +3799,6 @@ def chunk_kda_grad( present). """ stream = 0 if stream is None else stream - if cu_seqlens is None: - raise ValueError("cu_seqlens is required (THD layout)") if scale is None: scale = q.shape[-1] ** -0.5 q_rstd, k_rstd = None, None @@ -4004,8 +3809,6 @@ def chunk_kda_grad( beta_raw = beta if use_beta_sigmoid_in_kernel: beta = fused_beta_sigmoid(beta_raw, scale=2.0 if allow_neg_eigval else 1.0, out=bufs["beta_sig"], stream=stream) - if cu_seqlens is not None and chunk_indices is None: - raise ValueError("varlen (cu_seqlens) requires chunk_indices — callers build the (seq, intra) table") _o, _fs, g_cumsum, Aqk, Akk, w, u, qg, kg, v_new, h, initial_state = chunk_kda_fwd( q=q_in, k=k_in, @@ -4025,6 +3828,8 @@ def chunk_kda_grad( dt_bias=dt_bias, chunk_size=chunk_size, state_v_first=state_v_first, + disable_recompute=True, + compute_o=False, bufs=bufs, stream=stream, ) @@ -4038,7 +3843,7 @@ def chunk_kda_grad( scale=scale, initial_state=initial_state, do=do, - dht=dht, + dstate_in=dstate_in, g=g_cumsum, g_org=g if use_gate_in_kernel else None, state_v_first=state_v_first, @@ -4050,6 +3855,7 @@ def chunk_kda_grad( use_gate_in_kernel=use_gate_in_kernel, A_log=A_log, dt_bias=dt_bias, + disable_recompute=True, bufs=bufs, w=w, u=u, @@ -4065,11 +3871,11 @@ def chunk_kda_grad( if use_beta_sigmoid_in_kernel: db = fused_beta_sigmoid_bwd(beta_raw, db, scale=2.0 if allow_neg_eigval else 1.0, out=db, stream=stream) return ( - _cast(bufs, "dq_cast", dq, q), - _cast(bufs, "dk_cast", dk, k), - _cast(bufs, "dv_cast", dv, v), - _cast(bufs, "dg_cast", dg, g), - _cast(bufs, "db_cast", db, beta_raw), + cast(bufs, "dq_cast", dq, q), + cast(bufs, "dk_cast", dk, k), + cast(bufs, "dv_cast", dv, v), + cast(bufs, "dg_cast", dg, g), + cast(bufs, "db_cast", db, beta_raw), dh0, dA, dbias, @@ -4110,48 +3916,19 @@ def chunk_kda( THD layout, written into ``bufs['o']`` / ``bufs['fs']``.""" stream = 0 if stream is None else stream if "transpose_state_layout" in kwargs: - if state_v_first: - raise ValueError("Cannot pass both `state_v_first` and the deprecated `transpose_state_layout`.") state_v_first = kwargs.pop("transpose_state_layout") - assert cp_context is None, "context-parallel path is not supported in this self-contained port" - - if cu_seqlens is None: - raise ValueError("cu_seqlens is required (THD layout)") - if initial_state is not None and initial_state.shape[0] != len(cu_seqlens) - 1: - raise ValueError( - f"The number of initial states is expected to equal the number of input sequences, " - f"i.e., {len(cu_seqlens) - 1} rather than {initial_state.shape[0]}.", - ) - if initial_state is not None: - assert str(initial_state.dtype).endswith("float32"), "initial_state must be in float32." - A_log, dt_bias = None, None if use_gate_in_kernel: - assert "A_log" in kwargs, "A_log must be provided when use_gate_in_kernel=True." A_log, dt_bias = kwargs["A_log"], kwargs.get("dt_bias") chunk_size = kwargs.pop("chunk_size", 64) - if chunk_size not in (32, 64): - raise ValueError(f"`chunk_size` must be either 32 or 64 for KDA, got {chunk_size}.") if safe_gate and use_gate_in_kernel: - if lower_bound is None: - raise ValueError("`lower_bound` must be specified when `safe_gate=True` and `use_gate_in_kernel=True`.") if not (-5 <= lower_bound < 0): raise ValueError(f"`lower_bound` must be in the safe range [-5, 0), got {lower_bound}.") - if allow_neg_eigval and not use_beta_sigmoid_in_kernel: - raise ValueError("`allow_neg_eigval=True` requires `use_beta_sigmoid_in_kernel=True`.") - T, H, K, HV = *q.shape, v.shape[1] - if k.shape[1] != H: - raise ValueError(f"q and k must have the same number of heads, got q.shape[1]={H} and k.shape[1]={k.shape[1]}") - if HV % H != 0: - raise ValueError(f"For GVA, HV ({HV}) must be divisible by H ({H}), got HV % H = {HV % H}") - assert q.shape == k.shape, f"q and k must have the same shape, got q={q.shape} vs k={k.shape}" - assert tuple(g.shape) == (T, HV, K), f"g must have shape [T, HV, K]={[T, HV, K]}, got {list(g.shape)}" - assert tuple(beta.shape) == (T, HV), f"beta must have shape [T, HV]={[T, HV]}, got {list(beta.shape)}" if scale is None: scale = K**-0.5 @@ -4162,8 +3939,6 @@ def chunk_kda( k_in, _k_rstd = l2norm_fwd(k, out=bufs["k_norm"], rstd_out=bufs["k_rstd"], stream=stream) if use_beta_sigmoid_in_kernel: beta = fused_beta_sigmoid(beta, scale=2.0 if allow_neg_eigval else 1.0, out=bufs["beta_sig"], stream=stream) - if cu_seqlens is not None and chunk_indices is None: - raise ValueError("varlen (cu_seqlens) requires chunk_indices — callers build the (seq, intra) table") o, final_state, _gc, _Aqk, _Akk, _w, _u, _qg, _kg, _vn, h, _s0 = chunk_kda_fwd( q=q_in, k=k_in, diff --git a/python/cudnn/linear_attention/engine_utils.py b/python/cudnn/linear_attention/engine_utils.py deleted file mode 100644 index 73be1d59c..000000000 --- a/python/cudnn/linear_attention/engine_utils.py +++ /dev/null @@ -1,134 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Engine-side helpers shared by the linear-attention backends: check_support -dtype gates, the explicit-workspace carver, and the compiled-plan wrapper.""" - -from __future__ import annotations - -from cudnn.engines.base import CompiledPlan, NodeBuffers, bind_ports - -from cudnn.frost import buffers -from cudnn.frost.workspace import Workspace - - -def _dtype_name(dt) -> str: - import cudnn - - return {cudnn.data_type.HALF: "float16", cudnn.data_type.BFLOAT16: "bfloat16", cudnn.data_type.FLOAT: "float32"}[dt] - - -def _require_dtype(engine: str, node, port: str, want, *, out: bool = False) -> None: - import cudnn # noqa: F401 — `want` members come from cudnn.data_type - - t = (node.outputs if out else node.inputs).get(port) - if t is None: - return - got = t.get_data_type() - if got is None: - return # unset (e.g. inferred outputs): the kernel validates the buffer - wanted = want if isinstance(want, tuple) else (want,) - if got not in wanted: - names = "/".join(w.name for w in wanted) - raise NotImplementedError(f"{engine}: '{port}' must be {names} (the kernel-native dtype; no staging), got {got}") - - -def _require_state_pair(engine: str, node) -> None: - """The kernels require matching initial/final state dtypes.""" - s0 = node.inputs.get("initial_state") - fs = node.outputs.get("final_state") - if s0 is None or fs is None: - return - a, b = s0.get_data_type(), fs.get_data_type() - if a is not None and b is not None and a != b: - raise NotImplementedError(f"{engine}: initial_state and final_state dtypes must match (got {a} vs {b})") - - -class _FrostPlan(CompiledPlan): - """A compiled linear-attention kernel, driven from the normalized pack. - - The port-to-slot join is a property of the graph, so it happens once and is - kept; only the addresses change between executes. What the kernel receives - is built from the pack, never the caller's object — the geometry it is - checked against and the geometry it runs on are then the same reading. - """ - - takes_variant_pack = True - - def __init__(self, compiled): - self._compiled = compiled - self._ports = None - self._name = type(compiled).__name__ - - def get_workspace_size(self) -> int: - return self._compiled.workspace_bytes() - - def execute(self, graph, variant_pack, ctx) -> None: - ports = self._ports - if ports is None: - ports = self._ports = bind_ports(graph, variant_pack) - _check_contiguous(variant_pack, ports) - node_buffers = {} - for node, slots in ports.items(): - names = list(slots.inputs) + list(slots.outputs) - views = variant_pack.operands(list(slots.inputs.values()) + list(slots.outputs.values())) - split = len(slots.inputs) - node_buffers[node] = NodeBuffers(dict(zip(names[:split], views[:split])), dict(zip(names[split:], views[split:]))) - required = self._compiled.workspace_bytes() - workspace = Workspace.over(variant_pack, required, self._name) if required else None - self._compiled(node_buffers, workspace=workspace, stream=ctx.stream) - - -def _check_contiguous(variant_pack, ports) -> None: - """Contiguity gate over the whole pack, decided from the strides it holds. - - The dim and stride were taken from the caller's object once, at - normalization; probing each buffer again cost 8.6 us apiece — nine per GDN - forward — to learn what the pack already knows. The scan itself is in the - native pack, 0.24 us for eight operands, so only naming the offender costs - anything and that happens once, on the way to raising. - - One gate for every kernel rather than a call per compiled callable naming - its own ports: the rule was the same list every time, and a port added to a - node but forgotten there would have gone unchecked. - """ - ok, offender = variant_pack.all_contiguous() - if ok: - return - for node, slots in ports.items(): - for direction in (slots.inputs, slots.outputs): - for port, slot in direction.items(): - if slot == offender: - raise ValueError(f"cudnn.frost {node.name!r}: buffer for {port!r} must be contiguous (buffers pass straight to the kernel)") - raise ValueError(f"cudnn.frost: the buffer at variant-pack slot {offender} must be contiguous") - - -_pinned_engines = None # e.g. ("gdn_cutile",) -- set by a suite, None => the manifest decides - - -def pin_engines(names): - """Force planning onto the named engines for this process, or None to stop. - - A suite that means to validate ONE implementation says which, by name. It - is not a way to add an engine: every engine is in the manifest, and this - only narrows which of them a plan may land on. - """ - global _pinned_engines - previous = _pinned_engines - _pinned_engines = tuple(names) if names else None - return previous - - -def apply_pin(graph): - """Select the pinned engine's plan, if a pin is in force. - - Runs after create_execution_plans(), so it selects from the ranked list the - heuristics produced rather than replacing them. - """ - if not _pinned_engines: - return - names = [graph.get_plan_name_at_index(i) for i in range(len(graph.plans))] - index = next((i for i, n in enumerate(names) if any(n == p or n.startswith(p + "[") for p in _pinned_engines)), None) - if index is None: - raise AssertionError(f"pinned engines {list(_pinned_engines)} produced no plan; plans={names}") - graph.select_plan(index) diff --git a/python/cudnn/linear_attention/frost/__init__.py b/python/cudnn/linear_attention/frost/__init__.py index b923f883e..85c2dd693 100644 --- a/python/cudnn/linear_attention/frost/__init__.py +++ b/python/cudnn/linear_attention/frost/__init__.py @@ -3,14 +3,10 @@ """cudnn.linear_attention.frost: the FROST linear-attention engines — Gated DeltaNet, Kimi Delta Attention, and Gated DeltaNet v2 on the SM100 -chunked kernels built on Cutlass primitives. ``GdnFrostEngine`` is the -default GDN engine on SM100/SM103; ``KdaFrostEngine`` and ``Gdn2FrostEngine`` -are forward-only (their backward kernels are stubs — KDA gradients run on -``KdaCuTileEngine``).""" - -# Lazy: importing one family's engine must not drag its neighbours in. The -# manifest's factories tolerate a missing optional dependency PER ENGINE, and -# eager imports here would have made one bad import cost all three. +chunked kernels built on Cutlass primitives. All three serve forward and +backward on SM100/SM103 and rank ahead of the cuTile fallbacks.""" + +# Lazy: importing one family's engine must not drag its neighbours in. import importlib from typing import Any diff --git a/python/cudnn/linear_attention/frost/common/downcast.py b/python/cudnn/linear_attention/frost/common/downcast.py new file mode 100644 index 000000000..0451ff59c --- /dev/null +++ b/python/cudnn/linear_attention/frost/common/downcast.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Initial-state staging for the FROST LA backward kernels: copy the caller's +``[N, HO, K, V]`` state (fp32 or io dtype, padded outer strides fine) into +the compact io-dtype buffer the per-(b,h) state descriptors read.""" + +import functools + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +from cutlass.cute.runtime import from_dlpack + + +@cute.kernel +def downcast_state_f16_kernel( + mState0: cute.Tensor, + mOut: cute.Tensor, + n_k: cutlass.Int32, + threads_per_row: cutlass.Int32, + rows_per_cta: cutlass.Int32, +) -> None: + """Row-chunk copy of the ``[N, HO, K, V]`` initial state into the io-dtype + buffer the backward's static state descriptor reads: grid (K-tiles, HO, N), + one 8-element V chunk per thread, source read through its (dynamic) + strides so padded outer layouts stage zero-copy.""" + bid = cute.arch.block_idx() + tidx = cutlass.Int32(cute.arch.thread_idx()[0]) + k_idx = cutlass.Int32(bid[0]) * rows_per_cta + tidx // threads_per_row + v0 = (tidx % threads_per_row) * cutlass.Int32(8) + n_idx = cutlass.Int32(bid[2]) + h_idx = cutlass.Int32(bid[1]) + if k_idx < n_k: + for i in cutlass.range_constexpr(8): + mOut[n_idx, h_idx, k_idx, v0 + i] = mState0[n_idx, h_idx, k_idx, v0 + i].to(mOut.element_type) + + +@cute.jit +def downcast_state_f16( + state0: cute.Tensor, + out: cute.Tensor, + n_k: cutlass.Int32, + threads_per_row: cutlass.Int32, + rows_per_cta: cutlass.Int32, + n_blocks: cutlass.Int32, + ho: cutlass.Int32, + n_seq: cutlass.Int32, + stream: cuda.CUstream, +): + downcast_state_f16_kernel( + state0, + out, + n_k, + threads_per_row, + rows_per_cta, + ).launch(grid=(n_blocks, ho, n_seq), block=(128, 1, 1), stream=stream) + + +@functools.cache +def downcast_state_cache(key): + return {} + + +def downcast_state(initial_state, out, *, stream): + """Copy the initial state ``[N, HO, K, V]`` (fp32 or io dtype, padded + outer strides fine) into ``out`` (io dtype, same shape, compact) — the + buffer the backward's per-(b,h) state descriptors read. Stride-aware: + the source is read through its own layout, never reshaped or copied + host-side.""" + if tuple(int(s_) for s_ in initial_state.shape) != tuple(int(s_) for s_ in out.shape): + raise ValueError(f"initial_state must match the io state buffer shape {tuple(out.shape)}; got {tuple(initial_state.shape)}") + n_seq, ho, k, v = (int(s_) for s_ in out.shape) + if v % 8 != 0: + raise ValueError(f"state V dim must be a multiple of 8 (8-element staging chunks); got {v}") + threads_per_row = v // 8 + rows_per_cta = max(128 // threads_per_row, 1) + n_blocks = (k + rows_per_cta - 1) // rows_per_cta + key = (str(initial_state.dtype), str(out.dtype)) + cache = downcast_state_cache(key) + cu_stream = cuda.CUstream(int(stream)) + if "compiled" not in cache: + state0_c = from_dlpack(initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) + out_c = from_dlpack(out, assumed_align=16).mark_layout_dynamic(leading_dim=3) + cache["compiled"] = cute.compile( + downcast_state_f16, + state0_c, + out_c, + cutlass.Int32(k), + cutlass.Int32(threads_per_row), + cutlass.Int32(rows_per_cta), + cutlass.Int32(n_blocks), + cutlass.Int32(ho), + cutlass.Int32(n_seq), + cu_stream, + options="--enable-tvm-ffi", + ) + cache["compiled"](initial_state, out, k, threads_per_row, rows_per_cta, n_blocks, ho, n_seq, cu_stream) diff --git a/python/cudnn/linear_attention/frost/common/head_reduce.py b/python/cudnn/linear_attention/frost/common/head_reduce.py index 3e363cbaf..bae195548 100644 --- a/python/cudnn/linear_attention/frost/common/head_reduce.py +++ b/python/cudnn/linear_attention/frost/common/head_reduce.py @@ -13,7 +13,7 @@ f16x2/bf16x2 pair, or one fp32 element), gathers it from all ``r`` group heads (coalesced, strided by ``inner_words``), accumulates in fp32, and stores one word back. Serves the f16/bf16 ``[total, HO, D]`` tensor grads -(dq/dk for GVA, dk/dv for GQA) and the fp32 ``[total, HO]`` gate/beta grads. +(dQ/dK for GVA, dK/dV for GQA) and the fp32 ``[total, HO]`` Gate/Beta grads. """ import cutlass @@ -22,16 +22,21 @@ from cutlass.cute.runtime import from_dlpack +from cudnn.frost import buffers +from .host import get_dtype from cudnn.frost.tile_dsl.pointwise import f16x2_to_f32, fp32_to_fp16 BLOCK = 256 @cute.kernel -def _head_reduce_kernel( +def head_reduce_kernel( mIn: cute.Tensor, mOut: cute.Tensor, total_words: cutlass.Int64, + out_row_words: cutlass.Int64, + out_head_words: cutlass.Int64, + h_count: cutlass.Constexpr[int], r: cutlass.Constexpr[int], inner_words: cutlass.Constexpr[int], io_dtype: cutlass.Constexpr, @@ -43,13 +48,16 @@ def _head_reduce_kernel( seg = gw // cutlass.Int64(inner_words) w_off = gw - seg * cutlass.Int64(inner_words) base = seg * cutlass.Int64(r * inner_words) + w_off + t_idx = seg // cutlass.Int64(h_count) + h_idx = seg - t_idx * cutlass.Int64(h_count) + out_off = t_idx * out_row_words + h_idx * out_head_words + w_off if cutlass.const_expr(io_dtype == cutlass.Float32): in_p = cute.recast_ptr(mIn.iterator, dtype=cutlass.Float32) out_p = cute.recast_ptr(mOut.iterator, dtype=cutlass.Float32) acc = (in_p + base).load() for i in cutlass.range_constexpr(r - 1): acc = acc + (in_p + (base + (i + 1) * inner_words)).load() - (out_p + gw).store(acc) + (out_p + out_off).store(acc) else: in_p = cute.recast_ptr(mIn.iterator, dtype=cutlass.Int32) out_p = cute.recast_ptr(mOut.iterator, dtype=cutlass.Int32) @@ -58,39 +66,31 @@ def _head_reduce_kernel( lo, hi = f16x2_to_f32((in_p + (base + (i + 1) * inner_words)).load(), dtype=io_dtype) acc_lo = acc_lo + lo acc_hi = acc_hi + hi - (out_p + gw).store(fp32_to_fp16(acc_lo, acc_hi, dtype=io_dtype)) + (out_p + out_off).store(fp32_to_fp16(acc_lo, acc_hi, dtype=io_dtype)) @cute.jit -def _launch( +def launch( mIn: cute.Tensor, mOut: cute.Tensor, total_words: cutlass.Int64, + out_row_words: cutlass.Int64, + out_head_words: cutlass.Int64, grid_x: cutlass.Int32, + h_count: cutlass.Constexpr[int], r: cutlass.Constexpr[int], inner_words: cutlass.Constexpr[int], io_dtype: cutlass.Constexpr, stream: cuda.CUstream, ) -> None: - _head_reduce_kernel(mIn, mOut, total_words, r, inner_words, io_dtype).launch( + head_reduce_kernel(mIn, mOut, total_words, out_row_words, out_head_words, h_count, r, inner_words, io_dtype).launch( grid=(grid_x, 1, 1), block=(BLOCK, 1, 1), stream=stream, ) -_compiled_cache = {} - - -def _cutlass_io_dtype(dtype) -> type: - name = str(dtype).split(".")[-1] - if name == "bfloat16": - return cutlass.BFloat16 - if name == "float16": - return cutlass.Float16 - if name == "float32": - return cutlass.Float32 - raise ValueError(f"head_group_reduce: unsupported dtype {dtype} (float16/bfloat16/float32 only)") +compiled_cache = {} def head_group_reduce(src, dst, *, stream) -> None: @@ -98,7 +98,9 @@ def head_group_reduce(src, dst, *, stream) -> None: rank-2 ``(total, HO)`` into ``(total, H)`` — by summing each group of ``r = HO // H`` consecutive heads (fp32 accumulation). - Both tensors are contiguous, same-dtype (f16/bf16, or fp32), + ``src`` is contiguous (kernel-internal wide buffer); ``dst`` needs a + stride-1 innermost dim with free outer strides (f16/bf16 outer strides + must be even — word-pair stores). Same-dtype (f16/bf16, or fp32), DLPack-compatible CUDA tensors; the f16/bf16 inner extent ``D`` must be even. Compile-cache-and-replay per ``(dtype, HO, H, D)``.""" if len(src.shape) == 2: @@ -114,7 +116,7 @@ def head_group_reduce(src, dst, *, stream) -> None: raise ValueError(f"head_group_reduce: shape mismatch {tuple(src.shape)} -> {tuple(dst.shape)}") if H <= 0 or HO % H != 0 or HO <= H: raise ValueError(f"head_group_reduce: bad head group HO={HO} H={H}") - io_dtype = _cutlass_io_dtype(src.dtype) + io_dtype = get_dtype(src.dtype) if str(src.dtype).split(".")[-1] != str(dst.dtype).split(".")[-1]: raise ValueError(f"head_group_reduce: dtype mismatch {src.dtype} vs {dst.dtype}") is_fp32 = io_dtype == cutlass.Float32 @@ -126,24 +128,37 @@ def head_group_reduce(src, dst, *, stream) -> None: grid_x = -(-total_words // BLOCK) cu_stream = cuda.CUstream(int(stream)) - key = (str(src.dtype).split(".")[-1], HO, H, D) - if key not in _compiled_cache: + _ptr, dst_shape, dst_strides, _dt, _dev = buffers.probe(dst) + if dst_strides is None: + dst_strides = [] + acc = 1 + for sz in reversed(dst_shape): + dst_strides.append(acc) + acc *= sz + dst_strides = tuple(reversed(dst_strides)) + if not is_fp32 and any(st % 2 != 0 for st, sz in zip(dst_strides[:-1], dst.shape[:-1]) if sz != 1): + raise ValueError(f"head_group_reduce: f16/bf16 dst outer strides must be even (word-pair stores), got {dst_strides}") + out_row_words = dst_strides[0] if is_fp32 else dst_strides[0] // 2 + out_head_words = (dst_strides[1] if is_fp32 else dst_strides[1] // 2) if len(dst.shape) == 3 else 1 - def _tok(t): - c = from_dlpack(t, assumed_align=4) - c.mark_compact_shape_dynamic(mode=0, stride_order=tuple(range(len(t.shape))), divisibility=1) - return c - - _compiled_cache[key] = cute.compile( - _launch, - _tok(src), - _tok(dst), + key = (str(src.dtype).split(".")[-1], HO, H, D) + if key not in compiled_cache: + + src_c = from_dlpack(src, assumed_align=4) + src_c.mark_compact_shape_dynamic(mode=0, stride_order=tuple(range(len(src.shape))), divisibility=1) + compiled_cache[key] = cute.compile( + launch, + src_c, + from_dlpack(dst, assumed_align=4).mark_layout_dynamic(leading_dim=len(dst.shape) - 1), cutlass.Int64(total_words), + cutlass.Int64(out_row_words), + cutlass.Int64(out_head_words), cutlass.Int32(grid_x), + H, r, inner_words, io_dtype, cu_stream, options="--enable-tvm-ffi", ) - _compiled_cache[key](src, dst, total_words, grid_x, cu_stream) + compiled_cache[key](src, dst, total_words, out_row_words, out_head_words, grid_x, cu_stream) diff --git a/python/cudnn/linear_attention/frost/common/host.py b/python/cudnn/linear_attention/frost/common/host.py new file mode 100644 index 000000000..7235ebdec --- /dev/null +++ b/python/cudnn/linear_attention/frost/common/host.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Host-side helpers shared by the FROST LA kernel modules (engine-invoked).""" + +import cutlass + +from .thd import TENSOR_MAP_QWORDS + + +def get_dtype(dtype): + """dtype string -> cutlass DSL type (bf16/fp16 io, fp32/bf16 states).""" + name = str(dtype) + if "bfloat16" in name: + return cutlass.BFloat16 + if "float16" in name or "half" in name: + return cutlass.Float16 + if "float32" in name: + return cutlass.Float32 + raise ValueError(f"Unsupported dtype {dtype}, expected bfloat16, float16, or float32") + + +def tensormap_workspace_bytes(mod, B: int) -> int: + """Runtime TMA-descriptor block for a kernel module: per-batch arrays + + static slots + 128 alignment slack.""" + return TENSOR_MAP_QWORDS * 8 * (mod.TENSORMAP_DESC_ARRAYS * B + mod.TENSORMAP_STATIC_SLOTS) + 128 diff --git a/python/cudnn/linear_attention/frost/common/split_k.py b/python/cudnn/linear_attention/frost/common/split_k.py index 3d4146bed..edeeaf8bb 100644 --- a/python/cudnn/linear_attention/frost/common/split_k.py +++ b/python/cudnn/linear_attention/frost/common/split_k.py @@ -16,15 +16,19 @@ Work-item table row (``WORK_ITEM_FIELDS`` x int32, chunk units):: - [batch_idx, head_idx, wstart, wend, cstart, cend] + [batch_idx, head_idx, wstart, wend, cstart, cend, batch_start, batch_end] + +``batch_start``/``batch_end`` are the token bounds ``cu_seqlens[b]`` / +``cu_seqlens[b+1]`` denormalized into the row: decode reads one 32-byte +vectorizable row instead of chasing a dependent ``cu_seqlens`` load pair. The item OWNS (writes outputs for) chunks ``[wstart, wend)``. The forward kernel COMPUTES chunks ``[cstart, wend)`` — ``[cstart, wstart)`` is the left warmup that rebuilds the incoming state from zero (accurate to ``2^log2_threshold`` because the gate decay over the window saturates). The backward kernel computes ``[wstart, cend)`` — ``[wend, cend)`` is the -right warmup for the reverse dH recurrence (the forward states come exactly -from the per-chunk H checkpoints). ``cstart == 0`` items seed the true +right warmup for the reverse dstate recurrence (the forward states come exactly +from the per-chunk state checkpoints). ``cstart == 0`` items seed the true initial state; ``cend == num_chunks`` items seed the true ``d_final_state`` — so the un-cut degenerate item ``(0, nc, 0, nc)`` reproduces the serial kernel exactly. @@ -68,7 +72,9 @@ from cutlass.cute.arch.nvvm_wrappers import inline_ptx from cutlass.cute.runtime import from_dlpack -WORK_ITEM_FIELDS = 6 +from cudnn.frost.buffers import data_ptr + +WORK_ITEM_FIELDS = 8 WARMUP_CAP_CHUNKS = 32 # hard warmup cap: a cut must saturate within one warp of chunks per side MAX_BLOCKS = 2048 # piece-count ceiling; host clamps ideal_chunks so the per-tile block count fits WARP_SIZE = 32 @@ -76,18 +82,18 @@ THREADS_PER_BLOCK = WARPS * WARP_SIZE SCAN_WARPS = 4 SCAN_THREADS = SCAN_WARPS * WARP_SIZE -SCAN_ROWS_PER_WARP = 4 # consecutive chunk rows per scan warp (amortizes the batch lookup + piece choice) -SCAN_TOKEN_STRIDE = 4 # sample every Nth token of a chunk: skipped tokens only RAISE the negative horizon sums (sound), and the safe-gate sigmoid is SFU-bound +SCAN_ROWS_PER_WARP = 4 # consecutive chunk rows per scan warp +SCAN_TOKEN_STRIDE = 4 # sample every Nth token of a chunk: skipped tokens only RAISE the negative horizon sums ORDER_THREADS = 1024 ORDER_ELEMS = 4 ORDER_CAPACITY = ( ORDER_THREADS * ORDER_ELEMS -) # sort capacity (32 KB SMEM); the kernel always launches — past this the device-side branch copies through unsorted (>25 items/SM, LPT is noise there) +) # sort capacity (32 KB SMEM); the kernel always launches — past this the device-side branch copies through unsorted OVERHEAD_TOKENS = 256 # per-item fixed cost for the piece model: state reseed + pipeline refill + typical warmup P_WINDOW = 16 # fill-regime piece-count search width P_BELOW = 8 # how far below the ideal-cap floor the fill-regime search may go -_DEFAULT_LOG2_THRESHOLD = -10.0 / math.log(2.0) # e^-10, in log2 units +DEFAULT_LOG2_THRESHOLD = -10.0 / math.log(2.0) # e^-10, in log2 units RCP_LN2 = 1.4426950408889634 # 1/ln(2): natural-log gates -> the scan's log2 domain @@ -133,15 +139,15 @@ def decode_work_item(cfg, tile_idx, cu_seqlens, mWorkItems): wend = mWorkItems[tile_idx, 3] cstart = mWorkItems[tile_idx, 4] cend = mWorkItems[tile_idx, 5] - batch_start = cu_seqlens[batch_idx] - batch_end = cu_seqlens[batch_idx + 1] + batch_start = mWorkItems[tile_idx, 6] + batch_end = mWorkItems[tile_idx, 7] seqlen_b = batch_end - batch_start num_chunks_b = cute.ceil_div(seqlen_b, cfg.b_t) else: batch_idx = tile_idx // cfg.n_heads_out head_idx = tile_idx % cfg.n_heads_out - batch_start = cu_seqlens[batch_idx] - batch_end = cu_seqlens[batch_idx + 1] + batch_start = cutlass.Int32(cu_seqlens[batch_idx]) + batch_end = cutlass.Int32(cu_seqlens[batch_idx + 1]) seqlen_b = batch_end - batch_start num_chunks_b = cute.ceil_div(seqlen_b, cfg.b_t) wstart = cutlass.Int32(0) @@ -152,7 +158,7 @@ def decode_work_item(cfg, tile_idx, cu_seqlens, mWorkItems): @cute.jit -def _emit_item(mWorkItems, mCount, batch_idx, head_idx, wstart, wend, cstart, cend): +def emit_item(mWorkItems, mCount, batch_idx, head_idx, wstart, wend, cstart, cend, batch_start, batch_end): count_addr = mCount.iterator.toint() wi = inline_ptx( "atom.global.add.s32 {$w0}, [{$r0}], 1;", @@ -165,10 +171,12 @@ def _emit_item(mWorkItems, mCount, batch_idx, head_idx, wstart, wend, cstart, ce mWorkItems[wi, 3] = wend mWorkItems[wi, 4] = cstart mWorkItems[wi, 5] = cend + mWorkItems[wi, 6] = batch_start + mWorkItems[wi, 7] = batch_end @cute.jit -def _clamped_log2(log_gate: cutlass.Constexpr[bool], gate_val: cutlass.Float32) -> cutlass.Float32: +def clamped_log2(log_gate: cutlass.Constexpr[bool], gate_val: cutlass.Float32) -> cutlass.Float32: """Log2-domain decay increment, clamped to <= 0 (a gate > 1 must not relax the horizon).""" if cutlass.const_expr(log_gate): @@ -179,7 +187,7 @@ def _clamped_log2(log_gate: cutlass.Constexpr[bool], gate_val: cutlass.Float32) @cute.jit -def _piece_choice(overhead_chunks: cutlass.Constexpr[int], num_chunks_b, n_tiles, num_sms, ideal_chunks): +def piece_choice(overhead_chunks: cutlass.Constexpr[int], num_chunks_b, n_tiles, num_sms, ideal_chunks): """Per-tile piece choice. Returns ``(span, num_blocks)``.""" # even spread: spans never exceed ideal_chunks (total work / SM count) p_hi = num_chunks_b if num_chunks_b < cutlass.Int32(MAX_BLOCKS) else cutlass.Int32(MAX_BLOCKS) @@ -189,8 +197,7 @@ def _piece_choice(overhead_chunks: cutlass.Constexpr[int], num_chunks_b, n_tiles p = p if p < p_hi else p_hi if n_tiles < cutlass.Int32(2) * num_sms: # fill regime: SMs to spare — search piece counts around the cap on - # the wave-quantized makespan estimate (the cap only binds in the - # many-tile regime, where its even spread kills varlen tails) + # the wave-quantized makespan estimate p_start = p - cutlass.Int32(P_BELOW) p_start = p_start if p_start > 0 else cutlass.Int32(1) best = cutlass.Int32(2147483647) @@ -203,8 +210,7 @@ def _piece_choice(overhead_chunks: cutlass.Constexpr[int], num_chunks_b, n_tiles hit = est < best best = est if hit else best p = cand if hit else p - # the estimate flatters marginal cuts (measured: <25% predicted gain - # loses to serial); cut only on a clear margin over uncut + # the estimate flatters marginal cuts; cut only on a clear margin over uncut est1 = ((n_tiles + num_sms - cutlass.Int32(1)) // num_sms) * (num_chunks_b + cutlass.Int32(overhead_chunks)) if cutlass.Int32(4) * best > cutlass.Int32(3) * est1: p = cutlass.Int32(1) @@ -217,23 +223,23 @@ def _piece_choice(overhead_chunks: cutlass.Constexpr[int], num_chunks_b, n_tiles @cute.jit -def _tile_spans(b_t: cutlass.Constexpr[int], overhead_chunks: cutlass.Constexpr[int], n_heads_out, n_tiles, num_sms, ideal_chunks, mCuSeqlens, tile): +def tile_spans(b_t: cutlass.Constexpr[int], overhead_chunks: cutlass.Constexpr[int], n_heads_out, n_tiles, num_sms, ideal_chunks, mCuSeqlens, tile): """Per-tile decode + piece choice. Returns ``(batch_idx, head_idx, batch_start, batch_end, num_chunks_b, cv_base, span, num_blocks)``; ``cv_base`` is the tile's row base in the GMEM chunk scratch.""" batch_idx = tile // n_heads_out head_idx = tile % n_heads_out - batch_start = mCuSeqlens[batch_idx] - batch_end = mCuSeqlens[batch_idx + 1] + batch_start = cutlass.Int32(mCuSeqlens[batch_idx]) + batch_end = cutlass.Int32(mCuSeqlens[batch_idx + 1]) seqlen_b = batch_end - batch_start num_chunks_b = cute.ceil_div(seqlen_b, b_t) cv_base = batch_start // cutlass.Int32(b_t) + batch_idx - span, num_blocks = _piece_choice(overhead_chunks, num_chunks_b, n_tiles, num_sms, ideal_chunks) + span, num_blocks = piece_choice(overhead_chunks, num_chunks_b, n_tiles, num_sms, ideal_chunks) return batch_idx, head_idx, batch_start, batch_end, num_chunks_b, cv_base, span, num_blocks @cute.jit -def _near_boundary(c, span, num_blocks): +def near_boundary(c, span, num_blocks): """True iff chunk ``c`` lies in the walk's read window of a candidate boundary: suffix ``[j*span - W, j*span)`` or prefix ``[j*span, j*span + W)`` for some ``j`` in ``[1, num_blocks)``.""" @@ -246,7 +252,7 @@ def _near_boundary(c, span, num_blocks): @cute.kernel -def _scan_kernel( +def scan_kernel( b_t: cutlass.Constexpr[int], log_gate: cutlass.Constexpr[bool], safe_gate: cutlass.Constexpr[bool], @@ -294,35 +300,35 @@ def _scan_kernel( hi = batch_size - cutlass.Int32(1) while lo < hi: mid = (lo + hi + cutlass.Int32(1)) // cutlass.Int32(2) - take = mCuSeqlens[mid] // cutlass.Int32(b_t) + mid <= row0 + take = cutlass.Int32(mCuSeqlens[mid]) // cutlass.Int32(b_t) + mid <= row0 lo = mid if take else lo hi = hi if take else mid - cutlass.Int32(1) batch_idx = lo - batch_start = mCuSeqlens[batch_idx] - batch_end = mCuSeqlens[batch_idx + 1] + batch_start = cutlass.Int32(mCuSeqlens[batch_idx]) + batch_end = cutlass.Int32(mCuSeqlens[batch_idx + 1]) num_chunks_b = cute.ceil_div(batch_end - batch_start, b_t) cv_base = batch_start // cutlass.Int32(b_t) + batch_idx - # the piece choice is per batch: computed once here and again only when - # a row crosses into the next batch — NOT per row (on uncut tiles this - # is the scan's entire cost) - span, num_blocks = _piece_choice(overhead_chunks, num_chunks_b, n_tiles, num_sms, ideal_chunks) + # the piece choice is computed per batch, NOT per row + span, num_blocks = piece_choice(overhead_chunks, num_chunks_b, n_tiles, num_sms, ideal_chunks) for rr in cutlass.range_constexpr(SCAN_ROWS_PER_WARP): row = row0 + cutlass.Int32(rr) - while (batch_idx + cutlass.Int32(1) < batch_size) and (mCuSeqlens[batch_idx + 1] // cutlass.Int32(b_t) + batch_idx + cutlass.Int32(1) <= row): + while (batch_idx + cutlass.Int32(1) < batch_size) and ( + cutlass.Int32(mCuSeqlens[batch_idx + 1]) // cutlass.Int32(b_t) + batch_idx + cutlass.Int32(1) <= row + ): batch_idx = batch_idx + cutlass.Int32(1) - batch_start = mCuSeqlens[batch_idx] - batch_end = mCuSeqlens[batch_idx + 1] + batch_start = cutlass.Int32(mCuSeqlens[batch_idx]) + batch_end = cutlass.Int32(mCuSeqlens[batch_idx + 1]) num_chunks_b = cute.ceil_div(batch_end - batch_start, b_t) cv_base = batch_start // cutlass.Int32(b_t) + batch_idx - span, num_blocks = _piece_choice(overhead_chunks, num_chunks_b, n_tiles, num_sms, ideal_chunks) + span, num_blocks = piece_choice(overhead_chunks, num_chunks_b, n_tiles, num_sms, ideal_chunks) c = row - cv_base if (c >= 0) and (c < num_chunks_b) and (num_blocks > 1): - if _near_boundary(c, span if span > 0 else cutlass.Int32(1), num_blocks): + if near_boundary(c, span if span > 0 else cutlass.Int32(1), num_blocks): # chunk value = max over channels of the per-channel # clamped-log2 sums (each lane owns a contiguous channel run) cpl = gate_channels // WARP_SIZE - row_elems = n_heads_out * cutlass.Int32(gate_channels) - lane_base = cutlass.Int64(head_idx * cutlass.Int32(gate_channels) + lidx * cutlass.Int32(cpl)) + row_elems = cutlass.Int32(mGate.stride[0]) + lane_base = cutlass.Int64(head_idx * cutlass.Int32(mGate.stride[1]) + lidx * cutlass.Int32(cpl)) gate_addr = mGate.iterator.toint() + lane_base * cutlass.Int64(4) gate_ptr = mGate.iterator + lane_base a_exp = cutlass.Float32(1.0) @@ -359,7 +365,7 @@ def _scan_kernel( contrib = contrib if inb else cutlass.Float32(0.0) else: gvq = gvq if inb else oob - contrib = _clamped_log2(log_gate, gvq) + contrib = clamped_log2(log_gate, gvq) ch_acc[q] = ch_acc[q] + contrib else: for q in cutlass.range_constexpr(cpl): @@ -371,7 +377,7 @@ def _scan_kernel( contrib = contrib if inb else cutlass.Float32(0.0) else: gv = gv if inb else oob - contrib = _clamped_log2(log_gate, gv) + contrib = clamped_log2(log_gate, gv) ch_acc[q] = ch_acc[q] + contrib m = ch_acc[0] for q in cutlass.range_constexpr(1, cpl): @@ -384,7 +390,7 @@ def _scan_kernel( @cute.kernel -def _scan_scalar_kernel( +def scan_scalar_kernel( b_t: cutlass.Constexpr[int], log_gate: cutlass.Constexpr[bool], overhead_chunks: cutlass.Constexpr[int], @@ -404,7 +410,7 @@ def _scan_scalar_kernel( for heads ``[hg*32, (hg+1)*32)``; lane ``l`` owns head ``hg*32 + l``, so gate reads and chunk-value writes are coalesced across lanes and every lane accumulates its own head — no reduction. CTA (0, 0) zeroes the - item count and the scheduler ring, as in the per-channel scan.""" + item count and the scheduler ring.""" tidx, _, _ = cute.arch.thread_idx() bidx = cute.arch.block_idx() tidx = cutlass.Int32(tidx) @@ -427,42 +433,44 @@ def _scan_scalar_kernel( hi = batch_size - cutlass.Int32(1) while lo < hi: mid = (lo + hi + cutlass.Int32(1)) // cutlass.Int32(2) - take = mCuSeqlens[mid] // cutlass.Int32(b_t) + mid <= row0 + take = cutlass.Int32(mCuSeqlens[mid]) // cutlass.Int32(b_t) + mid <= row0 lo = mid if take else lo hi = hi if take else mid - cutlass.Int32(1) batch_idx = lo - batch_start = mCuSeqlens[batch_idx] - batch_end = mCuSeqlens[batch_idx + 1] + batch_start = cutlass.Int32(mCuSeqlens[batch_idx]) + batch_end = cutlass.Int32(mCuSeqlens[batch_idx + 1]) num_chunks_b = cute.ceil_div(batch_end - batch_start, b_t) cv_base = batch_start // cutlass.Int32(b_t) + batch_idx - span, num_blocks = _piece_choice(overhead_chunks, num_chunks_b, n_tiles, num_sms, ideal_chunks) + span, num_blocks = piece_choice(overhead_chunks, num_chunks_b, n_tiles, num_sms, ideal_chunks) for rr in cutlass.range_constexpr(SCAN_ROWS_PER_WARP): row = row0 + cutlass.Int32(rr) - while (batch_idx + cutlass.Int32(1) < batch_size) and (mCuSeqlens[batch_idx + 1] // cutlass.Int32(b_t) + batch_idx + cutlass.Int32(1) <= row): + while (batch_idx + cutlass.Int32(1) < batch_size) and ( + cutlass.Int32(mCuSeqlens[batch_idx + 1]) // cutlass.Int32(b_t) + batch_idx + cutlass.Int32(1) <= row + ): batch_idx = batch_idx + cutlass.Int32(1) - batch_start = mCuSeqlens[batch_idx] - batch_end = mCuSeqlens[batch_idx + 1] + batch_start = cutlass.Int32(mCuSeqlens[batch_idx]) + batch_end = cutlass.Int32(mCuSeqlens[batch_idx + 1]) num_chunks_b = cute.ceil_div(batch_end - batch_start, b_t) cv_base = batch_start // cutlass.Int32(b_t) + batch_idx - span, num_blocks = _piece_choice(overhead_chunks, num_chunks_b, n_tiles, num_sms, ideal_chunks) + span, num_blocks = piece_choice(overhead_chunks, num_chunks_b, n_tiles, num_sms, ideal_chunks) c = row - cv_base if (c >= 0) and (c < num_chunks_b) and (num_blocks > 1): - if _near_boundary(c, span if span > 0 else cutlass.Int32(1), num_blocks): + if near_boundary(c, span if span > 0 else cutlass.Int32(1), num_blocks): oob = cutlass.Float32(0.0) if cutlass.const_expr(log_gate) else cutlass.Float32(1.0) acc = cutlass.Float32(0.0) for tt in cutlass.range(0, b_t, SCAN_TOKEN_STRIDE, unroll_full=True): pos = batch_start + c * cutlass.Int32(b_t) + cutlass.Int32(tt) inb = pos < batch_end pos_r = pos if inb else batch_start - gv = (mGate.iterator + cutlass.Int64(pos_r) * cutlass.Int64(n_heads_out) + h_r).load() + gv = (mGate.iterator + cutlass.Int64(pos_r) * cutlass.Int64(mGate.stride[0]) + h_r).load() gv = gv if inb else oob - acc = acc + _clamped_log2(log_gate, gv) + acc = acc + clamped_log2(log_gate, gv) if h_ok: mChunkVals[cv_base + c, h] = acc @cute.kernel -def _walk_kernel( +def walk_kernel( b_t: cutlass.Constexpr[int], overhead_chunks: cutlass.Constexpr[int], n_heads_out: cutlass.Int32, @@ -483,13 +491,13 @@ def _walk_kernel( bidx = cute.arch.block_idx()[0] tidx = cutlass.Int32(tidx) - batch_idx, head_idx, batch_start, batch_end, num_chunks_b, cv_base, span, num_blocks = _tile_spans( + batch_idx, head_idx, batch_start, batch_end, num_chunks_b, cv_base, span, num_blocks = tile_spans( b_t, overhead_chunks, n_heads_out, n_tiles, num_sms, ideal_chunks, mCuSeqlens, cutlass.Int32(bidx) ) if num_blocks <= 1: # single piece: no cuts, nothing scanned if tidx == 0: - _emit_item(mStaging, mCount, batch_idx, head_idx, cutlass.Int32(0), num_chunks_b, cutlass.Int32(0), num_chunks_b) + emit_item(mStaging, mCount, batch_idx, head_idx, cutlass.Int32(0), num_chunks_b, cutlass.Int32(0), num_chunks_b, batch_start, batch_end) else: # packed per-boundary probe results: warm_b | warm_f << 8, 0 = no cut @@ -545,15 +553,15 @@ def _walk_kernel( warm_f = r // cutlass.Int32(256) cend = wend + warm_f cend = cend if cend < num_chunks_b else num_chunks_b - _emit_item(mStaging, mCount, batch_idx, head_idx, prev_cut, wend, cur_cstart, cend) + emit_item(mStaging, mCount, batch_idx, head_idx, prev_cut, wend, cur_cstart, cend, batch_start, batch_end) cur_cstart = wend - warm_b prev_cut = wend jj = jj + cutlass.Int32(1) - _emit_item(mStaging, mCount, batch_idx, head_idx, prev_cut, num_chunks_b, cur_cstart, num_chunks_b) + emit_item(mStaging, mCount, batch_idx, head_idx, prev_cut, num_chunks_b, cur_cstart, num_chunks_b, batch_start, batch_end) @cute.kernel -def _order_kernel( +def order_kernel( mStaging: cute.Tensor, mCount: cute.Tensor, mWorkItems: cute.Tensor, @@ -565,7 +573,6 @@ def _order_kernel( tidx = cutlass.Int32(tidx) n = mCount[0] if n > cutlass.Int32(ORDER_CAPACITY): - # dozens of items per SM: LPT stops mattering, copy through i = tidx while i < n: for f in cutlass.range_constexpr(WORK_ITEM_FIELDS): @@ -600,7 +607,7 @@ def _order_kernel( nvvm.atomicrmw("max", sSpread.data_ptr(1), kmax, space=nvvm.SharedSpace.shared_cta) nvvm.barrier_cta_sync() if sSpread[0] == sSpread[1]: - # every key equal (uniform batches): any order is LPT, copy through + # every key equal (uniform batches): copy through i2 = tidx while i2 < n: for f in cutlass.range_constexpr(WORK_ITEM_FIELDS): @@ -639,7 +646,7 @@ def _order_kernel( @cute.jit -def _launch( +def launch( b_t: cutlass.Constexpr[int], log_gate: cutlass.Constexpr[bool], safe_gate: cutlass.Constexpr[bool], @@ -667,7 +674,7 @@ def _launch( stream: cuda.CUstream, ) -> None: if cutlass.const_expr(gate_channels > 0): - _scan_kernel( + scan_kernel( b_t, log_gate, safe_gate, @@ -693,7 +700,7 @@ def _launch( stream=stream, ) else: - _scan_scalar_kernel( + scan_scalar_kernel( b_t, log_gate, overhead_chunks, @@ -713,7 +720,7 @@ def _launch( block=(SCAN_THREADS, 1, 1), stream=stream, ) - _walk_kernel( + walk_kernel( b_t, overhead_chunks, n_heads_out, @@ -730,7 +737,7 @@ def _launch( block=(THREADS_PER_BLOCK, 1, 1), stream=stream, ) - _order_kernel( + order_kernel( mStaging, mCount, mWorkItems, @@ -741,7 +748,7 @@ def _launch( ) -_compiled_cache = {} +compiled_cache = {} def build_split_table( @@ -785,7 +792,7 @@ def build_split_table( B, b_t), HO)`` fp32 (contents managed here). Runs entirely on device — no host synchronization.""" if log2_threshold is None: - log2_threshold = _DEFAULT_LOG2_THRESHOLD + log2_threshold = DEFAULT_LOG2_THRESHOLD if len(gate.shape) not in (2, 3): raise ValueError(f"gate must be (total_tokens, HO) or (total_tokens, HO, DK), got {tuple(gate.shape)}") gate_channels = gate.shape[2] if len(gate.shape) == 3 else 0 @@ -798,7 +805,7 @@ def build_split_table( dt_bias = None if gate_channels and gate_channels % WARP_SIZE != 0: raise ValueError(f"per-channel gate dim must be a multiple of {WARP_SIZE}, got {gate_channels}") - if gate_channels and gate_channels % 128 == 0 and gate.data_ptr() % 16 != 0: + if gate_channels and gate_channels % 128 == 0 and data_ptr(gate) % 16 != 0: raise ValueError("per-channel gate base must be 16-byte aligned (vectorized scan loads)") gate_scale_log2 = float(gate_lower_bound) * RCP_LN2 if safe_gate else 0.0 n_heads_out = gate.shape[1] @@ -815,16 +822,23 @@ def build_split_table( overhead_chunks = max(1, OVERHEAD_TOKENS // b_t) cu_stream = cuda.CUstream(int(stream)) - key = (b_t, n_heads_out, bool(log_gate), bool(safe_gate), gate_channels, sched_ctr is not None) - if key not in _compiled_cache: - - def _dyn(t): - c = from_dlpack(t, assumed_align=4) - c.mark_compact_shape_dynamic(mode=0, stride_order=tuple(range(len(t.shape))), divisibility=1) - return c - - _compiled_cache[key] = cute.compile( - _launch, + key = (b_t, n_heads_out, bool(log_gate), bool(safe_gate), gate_channels, sched_ctr is not None, str(cu_seqlens.dtype)) + if key not in compiled_cache: + + dt_bias_c = None + if safe_gate: + dt_bias_c = from_dlpack(dt_bias, assumed_align=4) + dt_bias_c.mark_compact_shape_dynamic(mode=0, stride_order=tuple(range(len(dt_bias.shape))), divisibility=1) + chunk_scratch_c = from_dlpack(chunk_scratch, assumed_align=4) + chunk_scratch_c.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) + item_scratch_c = from_dlpack(item_scratch, assumed_align=4) + item_scratch_c.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) + work_items_c = from_dlpack(work_items, assumed_align=4) + work_items_c.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) + work_count_c = from_dlpack(work_count, assumed_align=4) + work_count_c.mark_compact_shape_dynamic(mode=0, stride_order=(0,), divisibility=1) + compiled_cache[key] = cute.compile( + launch, b_t, bool(log_gate), bool(safe_gate), @@ -838,21 +852,21 @@ def _dyn(t): cutlass.Int32(batch_size), cutlass.Float32(log2_threshold), cutlass.Float32(gate_scale_log2), - _dyn(gate), + from_dlpack(gate, assumed_align=4).mark_layout_dynamic(leading_dim=len(gate.shape) - 1), from_dlpack(a_log, assumed_align=4).mark_layout_dynamic() if safe_gate else None, - _dyn(dt_bias) if safe_gate else None, + dt_bias_c, from_dlpack(cu_seqlens, assumed_align=4).mark_layout_dynamic(), - _dyn(chunk_scratch), - _dyn(item_scratch), - _dyn(work_items), - _dyn(work_count), + chunk_scratch_c, + item_scratch_c, + work_items_c, + work_count_c, from_dlpack(sched_ctr, assumed_align=4).mark_layout_dynamic() if sched_ctr is not None else None, cutlass.Int32(n_scan_ctas), cutlass.Int32(n_walk_ctas), cu_stream, options="--enable-tvm-ffi", ) - _compiled_cache[key]( + compiled_cache[key]( n_heads_out, n_tiles, num_sms, diff --git a/python/cudnn/linear_attention/frost/common/thd.py b/python/cudnn/linear_attention/frost/common/thd.py index 70fb26991..c81f459be 100644 --- a/python/cudnn/linear_attention/frost/common/thd.py +++ b/python/cudnn/linear_attention/frost/common/thd.py @@ -4,214 +4,125 @@ """Shared THD / varlen (packed ``[T,H,D]`` + ``cu_seqlens``) device helpers. * :data:`TENSOR_MAP_QWORDS` — int64 words per 128-byte TMA descriptor. -* :func:`build_qkv_load_descs_kernel` — the separate device kernel that - builds a per-(batch x head) TMA-descriptor array in GMEM for varlen - loads/stores over a packed ``[T,H,D]`` tensor. -* :func:`build_h_descs_kernel` — its per-chunk-H sibling; derives the - per-sequence H offsets from the token ``cu_seqlens`` in place of a - caller-computed ``cu_h`` prefix array. -* :func:`build_state_descs_kernel` — per-(batch x head) descriptors over a - DENSE ``[N, HO, K, V]`` state tensor (one entry per slot). -* :func:`downcast_state_kernel` — elementwise fp32 -> io copy of the - initial state into the buffer the state descriptors read. +* :func:`emit_seq_descs` — device helper (one electing thread) that builds + a per-BATCH TMA-descriptor array in GMEM for varlen loads/stores over a + packed ``[T,H,D]`` tensor whose head axis is a load coordinate. Each op + calls it from a single combined ``build_all_descs_kernel`` launch (one + warp per array). +* :func:`emit_checkpoint_seq_descs` — its per-chunk-checkpoint sibling; derives the + per-sequence checkpoint offsets from the token ``cu_seqlens`` in place of a + caller-computed prefix array. +* :func:`emit_copy_desc` — verbatim single-slot copy of a fully static + descriptor (dense ``[N, HO, K, V]`` state; batch and head are both load + coordinates). """ import cutlass import cutlass.cute as cute import cutlass.experimental.primitives as nvvm -import cutlass.experimental.cuda.tensor_map as _tma from cutlass.base_dsl.typing import Pointer -import cuda.bindings.driver as _cuda_driver # noqa: F401 (cute.compile pulls cuda) TENSOR_MAP_QWORDS = 128 // 8 -@cute.kernel -def build_qkv_load_descs_kernel( - base_desc: cutlass.GridConstant[_tma.TensorMap], - desc_words: cute.Tensor, - cu_seqlens: cute.Tensor, - base_ptr: cute.Tensor, +@cute.jit +def emit_seq_descs( + base_desc, + desc_words, + cu_seqlens, + base_ptr, n_batch: cutlass.Int32, - n_heads: cutlass.Int32, - head_group: cutlass.Int32, - head_stride: cutlass.Int32, row_stride: cutlass.Int32, seq_ord: cutlass.Constexpr[int], ) -> None: - """Build a per-(batch x head) TMA-descriptor ARRAY for a VARLEN (THD) - Q/K/V/O tile over a packed ``[T,H,D]`` tensor + ``cu_seqlens``. It - re-points ``GLOBAL_ADDRESS`` per head so the flat output-head index - ``head_idx`` maps to its KV head (``head_idx // head_group``), - reproducing the GQA nested ``(h_r, h_v)`` head mode with stride-0 - replication. The cute-side ``tma_tensor[None, None, head_idx]`` head - indexing collapses to: - - * **identity (Q/O):** address head offset = ``head_idx * head_stride`` - (``head_group == 1``; ``head_idx // 1 == head_idx``). - * **grouped (K/V):** address head offset = - ``(head_idx // head_group) * head_stride`` — the stride-0 ``h_r`` - sub-mode means ``head_group`` consecutive Q heads share one KV head. - - Pass ``head_group == 1`` for identity and ``head_group == h_q // h_v`` - for K/V. The descriptor array is laid out ``[batch][head]`` (head-minor): - slot ``(b * n_heads + h)``. - - GLOBAL_ADDRESS folds BOTH the per-sequence token start - (``cu_seqlens[b] * row_stride``) AND the per-head offset - (``(head_idx // head_group) * head_stride``); GLOBAL_DIM[``seq_ord``] is - capped to the per-sequence token COUNT (``cu_seqlens[b+1] - cu_seqlens[b]``) - so a load box past the sequence end is OOB-clipped. Because the - GLOBAL_ADDRESS already points at the sequence's first token, the consumer - issues the TMA with a token coordinate of **0** (not the absolute packed - offset). - - Released via the GENERIC->TENSORMAP proxy fence; the consumer - (``tma_load_tile`` with ``gmem_slice.desc_ptr`` set) acquire-fences each - slot before the load. ``seq_ord`` is a compile-time ord - (``tensormap_replace`` ord must be a Python int).""" - if nvvm.elect_sync(): - desc_base = desc_words.iterator.raw_ptr() - src_words = Pointer(base_desc.get_ptr(), dtype=cutlass.Int64) - cu = cutlass.make_array_view(cu_seqlens) - base = base_ptr.iterator.raw_ptr() - for b in cutlass.range(0, n_batch, 1, unroll=1): - cu_b = cutlass.Int32(cu[b]) - s_b = cutlass.Int32(cu[b + cutlass.Int32(1)]) - cu_b - # Int64: the H descs cross 2^31 elements near cu[b] ~ 2k - # (row_stride = HO*DK*DV); inputs follow at larger B*T. - tok0 = cutlass.Int64(cu_b) * cutlass.Int64(row_stride) - for h in cutlass.range(0, n_heads, 1, unroll=1): - slot = b * n_heads + h - dptr = desc_base + slot * cutlass.Int32(TENSOR_MAP_QWORDS) - for i in cutlass.range_constexpr(TENSOR_MAP_QWORDS): - (dptr + i).store((src_words + i).load()) - kv_h = h // head_group - head_off = cutlass.Int64(kv_h) * cutlass.Int64(head_stride) - addr = base + (tok0 + head_off) - nvvm.tensormap_replace( - nvvm.TensormapField.GLOBAL_ADDRESS, - dptr, - new_value=addr.toint(cutlass.Int64), - ) - nvvm.tensormap_replace( - nvvm.TensormapField.GLOBAL_DIM, - dptr, - new_value=s_b, - ord=seq_ord, - ) - nvvm.fence_proxy_release( - nvvm.MemScope.GPU, - from_proxy=nvvm.Proxy.GENERIC, - to_proxy=nvvm.Proxy.TENSORMAP, + """Per-BATCH TMA-descriptor array for a VARLEN (THD) tensor whose base + descriptor carries the head axis as a real dimension (``(d, head, + token)``); the head index is a load COORDINATE, so only the sequence + base and length are patched per slot. GLOBAL_ADDRESS folds + ``cu_seqlens[b] * row_stride`` (Int64); GLOBAL_DIM[``seq_ord``] is + capped to the per-sequence token count so tail loads zero-fill and tail + stores clip in hardware. GQA/GVA head grouping happens at the issue + site (``head_idx // group`` with a static group), not here. Runs on + one electing thread; the calling warp elects and release-fences + (GENERIC->TENSORMAP).""" + desc_base = desc_words.iterator.raw_ptr() + src_words = Pointer(base_desc.get_ptr(), dtype=cutlass.Int64) + cu = cutlass.make_array_view(cu_seqlens) + base = base_ptr.iterator.raw_ptr() + for b in cutlass.range(0, n_batch, 1, unroll=1): + cu_b = cutlass.Int32(cu[b]) + s_b = cutlass.Int32(cu[b + cutlass.Int32(1)]) - cu_b + dptr = desc_base + b * cutlass.Int32(TENSOR_MAP_QWORDS) + for i in cutlass.range_constexpr(TENSOR_MAP_QWORDS): + (dptr + i).store((src_words + i).load()) + addr = base + cutlass.Int64(cu_b) * cutlass.Int64(row_stride) + nvvm.tensormap_replace( + nvvm.TensormapField.GLOBAL_ADDRESS, + dptr, + new_value=addr.toint(cutlass.Int64), ) - - -@cute.kernel -def downcast_state_kernel( - mS0: cute.Tensor, - mOut: cute.Tensor, - n: cutlass.Int32, -) -> None: - """Flat elementwise copy of the fp32 initial state into the io-dtype - buffer the backward's per-(b,h) state descriptors read (1-D views, one - element per thread).""" - idx = cutlass.Int32(cute.arch.block_idx()[0]) * cutlass.Int32(128) + cutlass.Int32(cute.arch.thread_idx()[0]) - if idx < n: - mOut[idx] = mS0[idx].to(mOut.element_type) - - -@cute.kernel -def build_state_descs_kernel( - base_desc: cutlass.GridConstant[_tma.TensorMap], - desc_words: cute.Tensor, - base_ptr: cute.Tensor, - n_batch: cutlass.Int32, - n_heads: cutlass.Int32, - ent_stride: cutlass.Int32, -) -> None: - """Per-(batch x head) TMA-descriptor array over a DENSE state tensor - ``[N, HO, K, V]``: slot ``(b * n_heads + h)`` gets GLOBAL_ADDRESS - pointing at its ``[K, V]`` tile (``slot * ent_stride`` elements in). - All dims come baked from the base descriptor (one entry per slot), so - only the address is patched.""" - if nvvm.elect_sync(): - desc_base = desc_words.iterator.raw_ptr() - src_words = Pointer(base_desc.get_ptr(), dtype=cutlass.Int64) - base = base_ptr.iterator.raw_ptr() - n_slots = n_batch * n_heads - for s in cutlass.range(0, n_slots, 1, unroll=1): - dptr = desc_base + s * cutlass.Int32(TENSOR_MAP_QWORDS) - for i in cutlass.range_constexpr(TENSOR_MAP_QWORDS): - (dptr + i).store((src_words + i).load()) - addr = base + cutlass.Int64(s) * cutlass.Int64(ent_stride) - nvvm.tensormap_replace( - nvvm.TensormapField.GLOBAL_ADDRESS, - dptr, - new_value=addr.toint(cutlass.Int64), - ) - nvvm.fence_proxy_release( - nvvm.MemScope.GPU, - from_proxy=nvvm.Proxy.GENERIC, - to_proxy=nvvm.Proxy.TENSORMAP, + nvvm.tensormap_replace( + nvvm.TensormapField.GLOBAL_DIM, + dptr, + new_value=s_b, + ord=seq_ord, ) -@cute.kernel -def build_h_descs_kernel( - base_desc: cutlass.GridConstant[_tma.TensorMap], - desc_words: cute.Tensor, - cu_seqlens: cute.Tensor, - base_ptr: cute.Tensor, +@cute.jit +def emit_checkpoint_seq_descs( + base_desc, + desc_words, + cu_seqlens, + base_ptr, n_batch: cutlass.Int32, - n_heads: cutlass.Int32, - head_stride: cutlass.Int32, row_stride: cutlass.Int32, every_n: cutlass.Int32, seq_ord: cutlass.Constexpr[int], ) -> None: - """Per-(batch x head) TMA-descriptor array for the per-chunk H tensor. - - Unlike :func:`build_qkv_load_descs_kernel` this derives the per-sequence - H offsets from the TOKEN ``cu_seqlens`` on the fly instead of taking a - caller-computed ``cu_h`` prefix array: ``count_b = (seqlen_b - 1) // - every_n`` (0 for empty sequences), running-prefix-summed. - GLOBAL_DIM[``seq_ord``] is capped to ``count_b`` exactly as the - cu_h-differences cap did. H heads are identity-mapped (no GQA - grouping).""" - if nvvm.elect_sync(): - desc_base = desc_words.iterator.raw_ptr() - src_words = Pointer(base_desc.get_ptr(), dtype=cutlass.Int64) - cu = cutlass.make_array_view(cu_seqlens) - base = base_ptr.iterator.raw_ptr() - run = cutlass.Int32(0) - for b in cutlass.range(0, n_batch, 1, unroll=1): - s_tok = cutlass.Int32(cu[b + cutlass.Int32(1)]) - cutlass.Int32(cu[b]) - cnt = (s_tok - cutlass.Int32(1)) // every_n - cnt = cnt if s_tok > 0 else cutlass.Int32(0) - h0 = run - run = run + cnt - ent0 = cutlass.Int64(h0) * cutlass.Int64(row_stride) - for h in cutlass.range(0, n_heads, 1, unroll=1): - slot = b * n_heads + h - dptr = desc_base + slot * cutlass.Int32(TENSOR_MAP_QWORDS) - for i in cutlass.range_constexpr(TENSOR_MAP_QWORDS): - (dptr + i).store((src_words + i).load()) - head_off = cutlass.Int64(h) * cutlass.Int64(head_stride) - addr = base + (ent0 + head_off) - nvvm.tensormap_replace( - nvvm.TensormapField.GLOBAL_ADDRESS, - dptr, - new_value=addr.toint(cutlass.Int64), - ) - nvvm.tensormap_replace( - nvvm.TensormapField.GLOBAL_DIM, - dptr, - new_value=cnt, - ord=seq_ord, - ) - nvvm.fence_proxy_release( - nvvm.MemScope.GPU, - from_proxy=nvvm.Proxy.GENERIC, - to_proxy=nvvm.Proxy.TENSORMAP, + """Per-BATCH descriptor array for the per-chunk checkpoint tensor with the head + axis as a descriptor dimension (``(dv, dk, chunk, head)``). Derives the + per-sequence checkpoint offsets from the TOKEN ``cu_seqlens`` on the fly + (``count_b = (seqlen_b - 1) // every_n``, running-prefix-summed) — an + address fold no coordinate transform can express — and caps + GLOBAL_DIM[``seq_ord``] to ``count_b``. The head index is a load + coordinate. Runs on one electing thread; the calling warp elects and + fences.""" + desc_base = desc_words.iterator.raw_ptr() + src_words = Pointer(base_desc.get_ptr(), dtype=cutlass.Int64) + cu = cutlass.make_array_view(cu_seqlens) + base = base_ptr.iterator.raw_ptr() + run = cutlass.Int32(0) + for b in cutlass.range(0, n_batch, 1, unroll=1): + s_tok = cutlass.Int32(cu[b + cutlass.Int32(1)]) - cutlass.Int32(cu[b]) + cnt = (s_tok - cutlass.Int32(1)) // every_n + cnt = cnt if s_tok > 0 else cutlass.Int32(0) + checkpoint_base = run + run = run + cnt + dptr = desc_base + b * cutlass.Int32(TENSOR_MAP_QWORDS) + for i in cutlass.range_constexpr(TENSOR_MAP_QWORDS): + (dptr + i).store((src_words + i).load()) + addr = base + cutlass.Int64(checkpoint_base) * cutlass.Int64(row_stride) + nvvm.tensormap_replace( + nvvm.TensormapField.GLOBAL_ADDRESS, + dptr, + new_value=addr.toint(cutlass.Int64), ) + nvvm.tensormap_replace( + nvvm.TensormapField.GLOBAL_DIM, + dptr, + new_value=cnt, + ord=seq_ord, + ) + + +@cute.jit +def emit_copy_desc(base_desc, desc_words) -> None: + """Verbatim single-slot copy of a fully static descriptor (e.g. the + dense ``[N, HO, K, V]`` initial state, whose batch and head are both + load coordinates). Runs on one electing thread; the calling warp + elects and fences.""" + desc_base = desc_words.iterator.raw_ptr() + src_words = Pointer(base_desc.get_ptr(), dtype=cutlass.Int64) + for i in cutlass.range_constexpr(TENSOR_MAP_QWORDS): + (desc_base + i).store((src_words + i).load()) diff --git a/python/cudnn/linear_attention/frost/gdn2_engine.py b/python/cudnn/linear_attention/frost/gdn2_engine.py index 3bcd0fa8a..b10a05333 100644 --- a/python/cudnn/linear_attention/frost/gdn2_engine.py +++ b/python/cudnn/linear_attention/frost/gdn2_engine.py @@ -3,7 +3,8 @@ """FROST GDN-2 engine: GDN2 nodes on the chunked prefill kernel (``kernel/gdn2_prefill_f16.py``, Blackwell SM100/SM103, bf16/fp16, BT=16). -Forward only (GDN2_BWD declines); the only GDN-2 engine — no cuTile +Forward + backward (GDN2_BWD on ``kernel/gdn2_bprop_f16.py`` with a +checkpoint regen on ``kernel/gdn2_recompute_f16.py``); the only GDN-2 engine — no cuTile fallback.""" from __future__ import annotations @@ -14,61 +15,28 @@ from cudnn import behavior_note from cudnn.engines.base import BaseEngine, CompiledPlan -from cudnn.frost import buffers -from cudnn.frost.workspace import Workspace, WorkspaceLayout, carve_plan -from ..engine_utils import _FrostPlan, _require_dtype, _require_state_pair +from cudnn.frost.buffers import current_device_id +from cudnn.frost.device import multiprocessor_count +from cudnn.frost.workspace import Workspace, WorkspaceLayout +from ..graph_analyzer import FrostLaPlan, check_layouts, frost_la_gate, require, analyze, expect_table - -def _the_gdn2_node(graph): - nodes = list(graph.nodes) - if len(nodes) != 1: - return None - node = nodes[0] - if getattr(node.node_type, "name", None) not in ("GDN2", "GDN2_BWD"): - return None - return node - - -def _check_common(node) -> None: - """Shape/dtype gates for the prefill kernel.""" - import cudnn - - q, k, v = (node.inputs[p] for p in ("q", "k", "v")) - io_dtypes = {q.get_data_type(), k.get_data_type(), v.get_data_type()} - {None} - if len(io_dtypes) > 1: - raise NotImplementedError(f"Gdn2FrostEngine: q/k/v dtypes must match, got {io_dtypes}") - for p, t in (("q", q), ("k", k), ("v", v)): - if t.get_data_type() not in (cudnn.data_type.BFLOAT16, cudnn.data_type.HALF, None): - raise NotImplementedError(f"Gdn2FrostEngine: '{p}' must be bf16 or fp16, got {t.get_data_type()}") - if not t.dim or len(t.dim) != 3: - raise NotImplementedError(f"Gdn2FrostEngine: '{p}' must be THD [total_T, heads, dim]") - if q.dim[-1] != 128 or v.dim[-1] != 128: - raise NotImplementedError(f"Gdn2FrostEngine: head dims must be 128 (the recurrent state is 128x128), got K={q.dim[-1]} V={v.dim[-1]}") - if k.dim[1] != q.dim[1]: - raise NotImplementedError(f"Gdn2FrostEngine: q and k head counts differ ({q.dim[1]} vs {k.dim[1]})") - if v.dim[1] % q.dim[1] != 0: - raise NotImplementedError(f"Gdn2FrostEngine: v heads ({v.dim[1]}) must be a multiple of q heads ({q.dim[1]})") - - # kernel-native operand dtypes: buffers pass through without staging - fp32 = cudnn.data_type.FLOAT - io = q.get_data_type() - _require_dtype("Gdn2FrostEngine", node, "g", fp32) - if io is not None: - _require_dtype("Gdn2FrostEngine", node, "beta", io) - _require_dtype("Gdn2FrostEngine", node, "w", io) - _require_dtype("Gdn2FrostEngine", node, "cu_seqlens", cudnn.data_type.INT32) - _require_dtype("Gdn2FrostEngine", node, "initial_state", (fp32, cudnn.data_type.BFLOAT16)) - _require_dtype("Gdn2FrostEngine", node, "final_state", (fp32, cudnn.data_type.BFLOAT16), out=True) - _require_state_pair("Gdn2FrostEngine", node) +GDN2_ALIGN = {"a_log": 4, "beta": 4, "cu_seqlens": 8} +GDN2_COMPACT = ("dt_bias",) def build_gdn2(graph): """The expensive step: import the kernel module (pulls in the Cutlass primitives; the cute.compile itself is cached inside the kernel per static config and runs on first execute, when the real buffers are known).""" - node = _the_gdn2_node(graph) - if node is None or node.node_type.name != "GDN2": - raise ValueError("build_gdn2: graph does not contain exactly one GDN2 node") + nodes = list(graph.nodes) + if len(nodes) != 1 or getattr(nodes[0].node_type, "name", None) not in ("GDN2", "GDN2_BWD"): + raise ValueError("build_gdn2: graph does not contain exactly one GDN2/GDN2_BWD node") + node = nodes[0] + if node.node_type.name == "GDN2_BWD": + from .kernel import gdn2_bprop_f16 as bwd_mod + from .kernel import gdn2_recompute_f16 as regen_mod + + return CompiledGdn2Bwd(node, bwd_mod, regen_mod) from .kernel import gdn2_prefill_f16 as kernel_mod return CompiledGdn2(node, kernel_mod) @@ -77,36 +45,50 @@ def build_gdn2(graph): class Gdn2FrostEngine(BaseEngine): """FROST chunked-kernel backend for single-node GDN-2 graphs (THD layout). - The only GDN-2 engine (SM100/SM103, forward only); declines ``GDN2_BWD`` - (the FROST backward kernel is a stub).""" + The only GDN-2 engine (SM100/SM103); GDN2_BWD runs on the FROST backward + kernel with a forward checkpoint recompute when the graph has no ``state_checkpoints`` input.""" name = "gdn2_frost" behavior_notes = (behavior_note.RUNTIME_COMPILATION,) # JIT-compiled at build_plans() def check_support(self, graph) -> None: - node = _the_gdn2_node(graph) - if node is None: - raise NotImplementedError("Gdn2FrostEngine supports exactly one GDN2 node") - if node.node_type.name == "GDN2_BWD": - raise NotImplementedError("Gdn2FrostEngine: the FROST GDN-2 backward kernel is a stub") - sm = buffers.current_sm() - if sm is None or not (100 <= sm <= 103): - raise NotImplementedError(f"Gdn2FrostEngine requires SM100-SM103 (found {sm})") - try: - import cutlass.experimental.primitives # noqa: F401 — availability probe: ImportError = decline - except ImportError as exc: - raise NotImplementedError(f"Gdn2FrostEngine requires the Cutlass DSL with cutlass.experimental.primitives: {exc}") from exc - for port in ("q", "k", "v", "g", "beta", "w", "cu_seqlens"): - if port not in node.inputs: - raise NotImplementedError(f"Gdn2FrostEngine: GDN2 node '{node.name}' is missing input '{port}'") - if int(node.params.get("checkpoint_every_n_tokens", 0) or 0) or "H" in node.outputs: - raise NotImplementedError("Gdn2FrostEngine: the per-chunk H output lands with the backward kernel (jopark/kda_gdn2_bprop)") - if node.node_type.name != "GDN2" and node.params.get("use_beta_w_sigmoid", False): - raise NotImplementedError("Gdn2FrostEngine: use_beta_w_sigmoid is a forward-node attribute") - _check_common(node) + import cudnn + + facts = graph._facts_for(analyze) + frost_la_gate("Gdn2FrostEngine", facts, "GDN2") + ckpt = facts.checkpoint_every_n_tokens + if ckpt and (facts.is_bwd or ckpt != 16): + raise NotImplementedError(f"Gdn2FrostEngine: checkpoint_every_n_tokens must be 16 (per-chunk) on the forward node (got {ckpt})") + if facts.is_bwd and facts.safe_gate: + raise NotImplementedError("Gdn2FrostEngine: safe_gate is a forward-node attribute") + fp32 = cudnn.data_type.FLOAT + if facts.io_dtype is not None: + require("Gdn2FrostEngine", "beta", facts.beta_dtype, facts.io_dtype) + require("Gdn2FrostEngine", "w", facts.w_dtype, facts.io_dtype) + require("Gdn2FrostEngine", "a_log", facts.a_log_dtype, fp32) + require("Gdn2FrostEngine", "dt_bias", facts.dt_bias_dtype, fp32) + if facts.is_bwd: + for port, got in (("dO", facts.do_dtype), ("state_checkpoints", facts.state_checkpoints_dtype)): + if got not in (facts.io_dtype, None): + raise NotImplementedError(f"Gdn2FrostEngine: '{port}' must match the io dtype") + require("Gdn2FrostEngine", "initial_state", facts.state_dtype, fp32) + require("Gdn2FrostEngine", "d_final_state", facts.d_final_state_dtype, fp32) + require("Gdn2FrostEngine", "d_initial_state", facts.d_initial_state_dtype, fp32) + require("Gdn2FrostEngine", "dG", facts.dg_dtype, fp32) + if facts.io_dtype is not None: + require("Gdn2FrostEngine", "dBeta", facts.dbeta_dtype, facts.io_dtype) + require("Gdn2FrostEngine", "dW", facts.dw_dtype, facts.io_dtype) + else: + state_dtypes = (fp32, cudnn.data_type.BFLOAT16) + require("Gdn2FrostEngine", "initial_state", facts.state_dtype, state_dtypes) + require("Gdn2FrostEngine", "final_state", facts.final_state_dtype, state_dtypes) + if facts.io_dtype is not None: + require("Gdn2FrostEngine", "state_checkpoints", facts.state_checkpoints_out_dtype, facts.io_dtype) + if not facts.state_pair_match: + raise NotImplementedError("Gdn2FrostEngine: initial_state and final_state dtypes must match") def build_plan(self, graph, plan, ctx=None) -> CompiledPlan: - return _FrostPlan(build_gdn2(graph)) + return FrostLaPlan(build_gdn2(graph)) class CompiledGdn2: @@ -115,52 +97,46 @@ class CompiledGdn2: def __init__(self, node, kernel_mod): from .common.split_k import WORK_ITEM_FIELDS, chunk_scratch_rows, compute_ideal_chunks, max_work_items - self._node = node - self._kernel = kernel_mod + self.node = node + self.kernel = kernel_mod + self.expect = expect_table(node, GDN2_ALIGN) scale = node.params.get("scale") - self._scale = float(scale) if scale is not None else 1.0 / math.sqrt(node.inputs["q"].dim[-1]) - self._use_qk_l2norm = bool(node.params.get("use_qk_l2norm", False)) - self._use_beta_w_sigmoid = bool(node.params.get("use_beta_w_sigmoid", False)) - self._has_fs = "final_state" in node.outputs + self.scale = float(scale) if scale is not None else 1.0 / math.sqrt(node.inputs["q"].dim[-1]) + self.use_qk_l2norm = bool(node.params.get("use_qk_l2norm", False)) + self.safe_gate = bool(node.params.get("safe_gate", False)) + glb = node.params.get("gate_lower_bound") + self.gate_lower_bound = float(glb) if glb is not None else kernel_mod.DEFAULT_GATE_LOWER_BOUND + self.has_final_state = "final_state" in node.outputs + self.has_state_checkpoints = "state_checkpoints" in node.outputs q, g = node.inputs["q"], node.inputs["g"] - self._b_t = kernel_mod.CFG.B_T + self.b_t = kernel_mod.CFG.B_T total = q.dim[0] HO = g.dim[1] B = node.inputs["cu_seqlens"].dim[0] - 1 layout = WorkspaceLayout() - self._off_sched = layout.add(8) # [ticket, done] for the dynamic scheduler - self._num_sm = kernel_mod._device_sm_count() - self._ideal = compute_ideal_chunks(total, HO, self._num_sm, self._b_t) - self._n_tiles = B * HO - self._work_item_rows = max_work_items(total, B, HO, self._ideal, self._b_t, self._num_sm) - self._n_heads_out = HO - self._off_work_items = layout.add(self._work_item_rows * WORK_ITEM_FIELDS * 4) - self._off_item_scratch = layout.add(self._work_item_rows * WORK_ITEM_FIELDS * 4) - self._off_work_count = layout.add(4) - self._chunk_scratch_rows = chunk_scratch_rows(total, B, self._b_t) - self._off_chunk_scratch = layout.add(self._chunk_scratch_rows * HO * 4) - self._tensormap_bytes = kernel_mod.get_workspace_size(B, HO, HO) - self._off_tensormaps = layout.add(self._tensormap_bytes, align=128) - self._ws_bytes = layout.size - - self._carve = carve_plan( - "gdn2", - [ - (self._off_sched, "int32", (2,)), - (self._off_work_items, "int32", (self._work_item_rows, WORK_ITEM_FIELDS)), - (self._off_item_scratch, "int32", (self._work_item_rows, WORK_ITEM_FIELDS)), - (self._off_work_count, "int32", (1,)), - (self._off_chunk_scratch, "float32", (self._chunk_scratch_rows, HO)), - (self._off_tensormaps, "int64", (self._tensormap_bytes // 8,)), - ], - ) + self.off_sched = layout.add(8) # [ticket, done] for the dynamic scheduler + self.num_sm = multiprocessor_count(current_device_id()) + self.ideal = compute_ideal_chunks(total, HO, self.num_sm, self.b_t) + self.n_tiles = B * HO + self.work_item_rows = max_work_items(total, B, HO, self.ideal, self.b_t, self.num_sm) + self.n_heads_out = HO + self.off_work_items = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_item_scratch = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_work_count = layout.add(4) + self.chunk_scratch_rows = chunk_scratch_rows(total, B, self.b_t) + self.off_chunk_scratch = layout.add(self.chunk_scratch_rows * HO * 4) + from .common.host import tensormap_workspace_bytes + + self.tensormap_bytes = tensormap_workspace_bytes(kernel_mod, B) + self.off_tensormaps = layout.add(self.tensormap_bytes, align=128) + self.ws_bytes = layout.size def workspace_bytes(self) -> int: - return self._ws_bytes + return self.ws_bytes def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: - nb = node_buffers[self._node] + nb = node_buffers[self.node] q = nb.inputs["q"] k = nb.inputs["k"] v = nb.inputs["v"] @@ -168,14 +144,41 @@ def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: beta = nb.inputs["beta"] w = nb.inputs["w"] cu = nb.inputs["cu_seqlens"] - s0 = nb.inputs.get("initial_state") + state0 = nb.inputs.get("initial_state") o = nb.outputs["O"] - fs = nb.outputs["final_state"] if self._has_fs else None + final_state = nb.outputs["final_state"] if self.has_final_state else None + state_checkpoints = nb.outputs["state_checkpoints"] if self.has_state_checkpoints else None + a_log = nb.inputs.get("a_log") + dt_bias = nb.inputs.get("dt_bias") + check_layouts( + "Gdn2FrostEngine (GDN2)", + expect=self.expect, + compact=GDN2_COMPACT, + q=q, + k=k, + v=v, + g=g, + beta=beta, + w=w, + cu_seqlens=cu, + initial_state=state0, + O=o, + final_state=final_state, + state_checkpoints=state_checkpoints, + a_log=a_log, + dt_bias=dt_bias, + ) stream = stream if stream is not None else 0 - ws = workspace - sched_ctr, work_items, item_scratch, work_count, chunk_scratch, tensormaps = ws.carve(self._carve) + ws = Workspace(workspace, self.ws_bytes, "Gdn2FrostEngine (GDN2)") + sched_ctr = ws.view(self.off_sched, "int32", (2,)) + from .common.split_k import WORK_ITEM_FIELDS + + work_items = ws.view(self.off_work_items, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + work_count = ws.view(self.off_work_count, "int32", (1,)) + item_scratch = ws.view(self.off_item_scratch, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + chunk_scratch = ws.view(self.off_chunk_scratch, "float32", (self.chunk_scratch_rows, self.n_heads_out)) from .common.split_k import build_split_table build_split_table( @@ -183,18 +186,26 @@ def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: cu, work_items, work_count, - ideal_chunks=self._ideal, - n_tiles=self._n_tiles, - num_sms=self._num_sm, - b_t=self._b_t, + ideal_chunks=self.ideal, + n_tiles=self.n_tiles, + num_sms=self.num_sm, + b_t=self.b_t, chunk_scratch=chunk_scratch, item_scratch=item_scratch, log_gate=True, + safe_gate=self.safe_gate, + a_log=a_log, + dt_bias=dt_bias, + gate_lower_bound=self.gate_lower_bound if self.safe_gate else None, sched_ctr=sched_ctr, stream=stream, ) - self._kernel.chunk_gdn2_sm100( + ckpt_kwargs = {} + if self.has_state_checkpoints: + # the kernel derives the per-sequence checkpoint entry offsets on device + ckpt_kwargs = dict(checkpoint_every_n_tokens=self.b_t, output_state_checkpoints=state_checkpoints) + self.kernel.chunk_gdn2_sm100( q, k, v, @@ -203,15 +214,232 @@ def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: w, o, cu, - s0, - fs, - self._scale, - use_qk_l2norm_in_kernel=self._use_qk_l2norm, - use_beta_w_sigmoid_in_kernel=self._use_beta_w_sigmoid, + state0, + final_state, + self.scale, + use_qk_l2norm_in_kernel=self.use_qk_l2norm, + safe_gate=self.safe_gate, + gate_lower_bound=self.gate_lower_bound, + a_log=a_log, + dt_bias=dt_bias, work_items=work_items, work_count=work_count, sched_ctr=sched_ctr, - tensormap_workspace=tensormaps, + tensormap_workspace=ws.view(self.off_tensormaps, "int64", (self.tensormap_bytes // 8,)), + **ckpt_kwargs, + stream=stream, + ) + return None + + +class CompiledGdn2Bwd: + """Compiled FROST GDN-2 backward plan: the workspace holds the + regenerated per-chunk checkpoint series when the graph carries no ``state_checkpoints`` input, + plus GVA head scratch for dQ/dK.""" + + def __init__(self, node, bwd_mod, regen_mod): + from .common.split_k import WORK_ITEM_FIELDS, chunk_scratch_rows, compute_ideal_chunks, max_work_items + + self.node = node + self.bwd = bwd_mod + self.regen = regen_mod + self.expect = expect_table(node, GDN2_ALIGN) + from .common.downcast import downcast_state + from .common.host import tensormap_workspace_bytes + + self.downcast_state = downcast_state + scale = node.params.get("scale") + self.scale = float(scale) if scale is not None else 1.0 / math.sqrt(node.inputs["q"].dim[-1]) + self.use_qk_l2norm = bool(node.params.get("use_qk_l2norm", False)) + self.has_state_checkpoints = "state_checkpoints" in node.inputs + self.has_state0 = "initial_state" in node.inputs + self.has_dstate0 = "d_initial_state" in node.outputs + + q, g, v = node.inputs["q"], node.inputs["g"], node.inputs["v"] + self.b_t = bwd_mod.CFG.B_T + total = q.dim[0] + HQ, HV = q.dim[1], v.dim[1] + HO = g.dim[1] + K, V = q.dim[-1], v.dim[-1] + B = node.inputs["cu_seqlens"].dim[0] - 1 + self.io_name = "float16" if node.inputs["q"].get_data_type().name == "HALF" else "bfloat16" + self.n_heads_out, self._total = HO, total + layout = WorkspaceLayout() + self.off_sched = layout.add(16) # one [ticket, done] ring each for the regen and bwd kernels + self.num_sm = multiprocessor_count(current_device_id()) + self.bwd_dyn_sched = B * HO <= self.num_sm + self.ideal = compute_ideal_chunks(total, HO, self.num_sm, self.b_t) + self.n_tiles = B * HO + self.work_item_rows = max_work_items(total, B, HO, self.ideal, self.b_t, self.num_sm) + self.off_work_items = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_item_scratch = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_work_count = layout.add(4) + self.chunk_scratch_rows = chunk_scratch_rows(total, B, self.b_t) + self.off_chunk_scratch = layout.add(self.chunk_scratch_rows * HO * 4) + # chunk-0 entering state, io dtype (downcast initial_state or zeros) + self.off_state0_io = layout.add(B * HO * K * V * 2) if self.has_state0 else None + if not self.has_state_checkpoints: + self.state_checkpoints_rows = max(total // self.b_t, 1) + self.off_state_checkpoints = layout.add(self.state_checkpoints_rows * HO * K * V * 2) + self.regen_tm_bytes = tensormap_workspace_bytes(regen_mod, B) + self.off_regen_tensormaps = layout.add(self.regen_tm_bytes, align=128) + HK = node.inputs["k"].dim[1] + self.fold_dq = HQ < HO + self.fold_dk = HK < HO + self.fold_dv = HV < HO + if self.fold_dq: + self.off_dq_ho = layout.add(total * HO * K * 2) + if self.fold_dk: + self.off_dk_ho = layout.add(total * HO * K * 2) + if self.fold_dv: + self.off_dv_ho = layout.add(total * HO * V * 2) + self.bwd_tm_bytes = tensormap_workspace_bytes(bwd_mod, B) + self.off_bwd_tensormaps = layout.add(self.bwd_tm_bytes, align=128) + self.ws_bytes = layout.size + + def workspace_bytes(self) -> int: + return self.ws_bytes + + def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: + nb = node_buffers[self.node] + q = nb.inputs["q"] + k = nb.inputs["k"] + v = nb.inputs["v"] + g = nb.inputs["g"] + beta = nb.inputs["beta"] + w = nb.inputs["w"] + cu = nb.inputs["cu_seqlens"] + do = nb.inputs["dO"] + state_checkpoints = nb.inputs.get("state_checkpoints") + state0 = nb.inputs.get("initial_state") + dstate_in = nb.inputs.get("d_final_state") + dq = nb.outputs["dQ"] + dk = nb.outputs["dK"] + dv = nb.outputs["dV"] + dg = nb.outputs["dG"] + db = nb.outputs["dBeta"] + dw = nb.outputs["dW"] + dstate0 = nb.outputs.get("d_initial_state") + check_layouts( + "Gdn2FrostEngine (GDN2_BWD)", + expect=self.expect, + q=q, + k=k, + v=v, + g=g, + beta=beta, + w=w, + cu_seqlens=cu, + dO=do, + state_checkpoints=state_checkpoints, + initial_state=state0, + d_final_state=dstate_in, + dQ=dq, + dK=dk, + dV=dv, + dG=dg, + dBeta=db, + dW=dw, + d_initial_state=dstate0, + ) + stream = stream if stream is not None else 0 + + HO, total = self.n_heads_out, self._total + K, V = q.shape[-1], v.shape[-1] + B = cu.shape[0] - 1 + ws = Workspace(workspace, self.ws_bytes, "Gdn2FrostEngine (GDN2_BWD)") + sched_regen = ws.view(self.off_sched, "int32", (2,)) + sched_bwd = ws.view(self.off_sched + 8, "int32", (2,)) + from .common.split_k import WORK_ITEM_FIELDS + + work_items = ws.view(self.off_work_items, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + work_count = ws.view(self.off_work_count, "int32", (1,)) + item_scratch = ws.view(self.off_item_scratch, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + chunk_scratch = ws.view(self.off_chunk_scratch, "float32", (self.chunk_scratch_rows, HO)) + from .common.split_k import build_split_table + + build_split_table( + g, + cu, + work_items, + work_count, + ideal_chunks=self.ideal, + n_tiles=self.n_tiles, + num_sms=self.num_sm, + b_t=self.b_t, + chunk_scratch=chunk_scratch, + item_scratch=item_scratch, + log_gate=True, + sched_ctr=ws.view(self.off_sched, "int32", (4,)), stream=stream, ) + + state0_io = None + if state0 is not None: + state0_io = ws.view(self.off_state0_io, self.io_name, (B, HO, K, V)) + self.downcast_state(state0, state0_io, stream=stream) + if self.has_state_checkpoints: + checkpoint_series = state_checkpoints + else: + checkpoint_series = ws.view(self.off_state_checkpoints, self.io_name, (self.state_checkpoints_rows, HO, K, V)) + self.regen.chunk_gdn2_recompute_sm100( + k, + v, + g, + beta, + w, + cu, + state0, + None, + checkpoint_every_n_tokens=self.b_t, + output_state_checkpoints=checkpoint_series, + use_qk_l2norm_in_kernel=self.use_qk_l2norm, + work_items=work_items, + work_count=work_count, + sched_ctr=sched_regen, + tensormap_workspace=ws.view(self.off_regen_tensormaps, "int64", (self.regen_tm_bytes // 8,)), + stream=stream, + ) + + dq_out, dk_out, dv_out = dq, dk, dv + if self.fold_dq: + dq_out = ws.view(self.off_dq_ho, self.io_name, (total, HO, K)) + if self.fold_dk: + dk_out = ws.view(self.off_dk_ho, self.io_name, (total, HO, K)) + if self.fold_dv: + dv_out = ws.view(self.off_dv_ho, self.io_name, (total, HO, V)) + + self.bwd.chunk_gdn2_bwd_sm100( + q, + k, + v, + g, + beta, + w, + do, + checkpoint_series, + dq_out, + dk_out, + dv_out, + dg, + db, + dw, + cu, + self.scale, + initial_state=state0_io, + d_initial_state=dstate0 if self.has_dstate0 else None, + d_final_state=dstate_in, + use_qk_l2norm_in_kernel=self.use_qk_l2norm, + work_items=work_items, + work_count=work_count, + sched_ctr=sched_bwd if self.bwd_dyn_sched else None, + tensormap_workspace=ws.view(self.off_bwd_tensormaps, "int64", (self.bwd_tm_bytes // 8,)), + stream=stream, + ) + if dq_out is not dq or dk_out is not dk or dv_out is not dv: + from .common.head_reduce import head_group_reduce + + for src_ho, dst in ((dq_out, dq), (dk_out, dk), (dv_out, dv)): + if src_ho is not dst: + head_group_reduce(src_ho, dst, stream=stream) return None diff --git a/python/cudnn/linear_attention/frost/gdn_engine.py b/python/cudnn/linear_attention/frost/gdn_engine.py index b5fd88d3b..a5b3d8587 100644 --- a/python/cudnn/linear_attention/frost/gdn_engine.py +++ b/python/cudnn/linear_attention/frost/gdn_engine.py @@ -1,9 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""FROST GDN engine: GDN / GDN_BWD nodes on the chunked prefill kernels -(``kernel/gdn_prefill_f16.py`` / ``kernel/gdn_bprop_f16.py``, Blackwell -SM100/SM103, bf16/fp16).""" +"""FROST GDN engine: GDN nodes on the chunked prefill kernel +(``kernel/gdn_prefill_f16.py``) and GDN_BWD nodes on the chunked backward +kernel (``kernel/gdn_bprop_f16.py``), Blackwell SM100/SM103, bf16/fp16. +The backward regenerates the per-chunk state checkpoints with the recompute kernel +(``kernel/gdn_recompute_f16.py``) when the graph does not provide one.""" from __future__ import annotations @@ -14,85 +16,29 @@ from cudnn.engines.base import BaseEngine, CompiledPlan from cudnn.frost import buffers -from cudnn.frost.workspace import Workspace, WorkspaceLayout, carve_plan -from ..engine_utils import _FrostPlan, _require_dtype, _require_state_pair +from cudnn.frost.buffers import current_device_id +from cudnn.frost.device import multiprocessor_count +from cudnn.frost.workspace import Workspace, WorkspaceLayout +from ..graph_analyzer import FrostLaPlan, check_layouts, frost_la_gate, require, analyze, expect_table - -def _the_gdn_node(graph): - nodes = list(graph.nodes) - if len(nodes) != 1: - return None - node = nodes[0] - if getattr(node.node_type, "name", None) not in ("GDN", "GDN_BWD"): - return None - return node - - -def _io_dtype_name(node) -> str: - """The io dtype's buffer-level name ('bfloat16' / 'float16').""" - import cudnn - - return "bfloat16" if node.inputs["q"].get_data_type() == cudnn.data_type.BFLOAT16 else "float16" - - -def _check_common(node) -> None: - """Shape/dtype gates shared by the fwd and bwd kernels.""" - import cudnn - - q, k, v = (node.inputs[p] for p in ("q", "k", "v")) - io_dtypes = {q.get_data_type(), k.get_data_type(), v.get_data_type()} - {None} - if len(io_dtypes) > 1: - raise NotImplementedError(f"GdnFrostEngine: q/k/v dtypes must match, got {io_dtypes}") - for p, t in (("q", q), ("k", k), ("v", v)): - if t.get_data_type() not in (cudnn.data_type.BFLOAT16, cudnn.data_type.HALF, None): - raise NotImplementedError(f"GdnFrostEngine: '{p}' must be bf16 or fp16, got {t.get_data_type()}") - if not t.dim or len(t.dim) != 3: - raise NotImplementedError(f"GdnFrostEngine: '{p}' must be THD [total_T, heads, dim]") - if q.dim[-1] != 128 or v.dim[-1] != 128: - raise NotImplementedError(f"GdnFrostEngine: head dims must be 128 (the recurrent state is 128x128), got K={q.dim[-1]} V={v.dim[-1]}") - hq, hk, hv = q.dim[1], k.dim[1], v.dim[1] - if hq != hk: - raise NotImplementedError(f"GdnFrostEngine: q and k head counts differ ({hq} vs {hk})") - if hv != hq and max(hq, hv) % min(hq, hv) != 0: - # GVA (v-heads grouped over q-heads) or GQA (q-heads grouped over - # v-heads, the kernel's opposite direction) - raise NotImplementedError(f"GdnFrostEngine: q heads ({hq}) and v heads ({hv}) must be equal or one a multiple of the other") - ho = hq if hq >= hv else hv - for p in ("g", "beta"): - t = node.inputs[p] - if t.dim and t.dim[1] != ho: - raise NotImplementedError(f"GdnFrostEngine: '{p}' must carry HO = max(q, v) heads ({ho}), got {t.dim[1]}") - - # kernel-native operand dtypes: buffers pass through without staging - fp32 = cudnn.data_type.FLOAT - io = (cudnn.data_type.BFLOAT16, cudnn.data_type.HALF) - state = (fp32, cudnn.data_type.BFLOAT16) - _require_dtype("GdnFrostEngine", node, "g", fp32) - _require_dtype("GdnFrostEngine", node, "beta", fp32) - _require_dtype("GdnFrostEngine", node, "cu_seqlens", cudnn.data_type.INT32) - _require_dtype("GdnFrostEngine", node, "initial_state", state) - _require_dtype("GdnFrostEngine", node, "final_state", state, out=True) - _require_state_pair("GdnFrostEngine", node) - if node.node_type.name == "GDN_BWD": - _require_dtype("GdnFrostEngine", node, "dO", io) - _require_dtype("GdnFrostEngine", node, "h", io) - _require_dtype("GdnFrostEngine", node, "d_final_state", fp32) - _require_dtype("GdnFrostEngine", node, "d_initial_state", fp32, out=True) - _require_dtype("GdnFrostEngine", node, "dG", fp32, out=True) - _require_dtype("GdnFrostEngine", node, "dBeta", fp32, out=True) +GDN_ALIGN = {"cu_seqlens": 4} +GDN_FWD_COMPACT = ("q", "k", "v", "g", "beta", "O", "state_checkpoints", "initial_state", "final_state") +GDN_BWD_RECOMPUTE_COMPACT = ("k", "v", "g", "beta", "initial_state") def build_gdn(graph): """The expensive step: import the kernel module (pulls in the Cutlass primitives; the cute.compile itself is cached inside the kernel per static config and runs on first execute, when the real buffers are known).""" - node = _the_gdn_node(graph) - if node is None: + nodes = list(graph.nodes) + if len(nodes) != 1 or getattr(nodes[0].node_type, "name", None) not in ("GDN", "GDN_BWD"): raise ValueError("build_gdn: graph does not contain exactly one GDN/GDN_BWD node") + node = nodes[0] if node.node_type.name == "GDN_BWD": - from .kernel import gdn_bprop_f16 as kernel_mod + from .kernel import gdn_bprop_f16 as bwd_mod + from .kernel import gdn_recompute_f16 as regen_mod - return CompiledGdnBwd(node, kernel_mod) + return CompiledGdnBwd(node, bwd_mod, regen_mod) from .kernel import gdn_prefill_f16 as kernel_mod return CompiledGdn(node, kernel_mod) @@ -102,60 +48,46 @@ class GdnFrostEngine(BaseEngine): """FROST chunked-kernel backend for single-node GDN graphs (THD layout). Default GDN engine on SM100/SM103 (lowest GDN engine_id); declines - elsewhere so the router falls back to ``GdnCuTileEngine``.""" + elsewhere so ranking falls back to ``GdnCuTileEngine``.""" name = "gdn_frost" behavior_notes = (behavior_note.RUNTIME_COMPILATION,) # JIT-compiled at build_plans() def check_support(self, graph) -> None: - node = _the_gdn_node(graph) - if node is None: - raise NotImplementedError("GdnFrostEngine supports exactly one GDN/GDN_BWD node") - sm = buffers.current_sm() - if sm is None or not (100 <= sm <= 103): - raise NotImplementedError(f"GdnFrostEngine requires SM100-SM103 (found {sm})") - try: - import cutlass.experimental.primitives # noqa: F401 — availability probe: ImportError = decline - except ImportError as exc: - raise NotImplementedError(f"GdnFrostEngine requires the Cutlass DSL with cutlass.experimental.primitives: {exc}") from exc - if node.params.get("use_qk_l2norm", False): + import cudnn + + facts = graph._facts_for(analyze) + frost_la_gate("GdnFrostEngine", facts, "GDN") + if facts.use_qk_l2norm: raise NotImplementedError("GdnFrostEngine: use_qk_l2norm is not supported (the kernel takes q/k as given)") - ports = ("q", "k", "v", "g", "beta", "cu_seqlens") - if node.node_type.name == "GDN_BWD": - ports += ("dO",) - for port in ports: - if port not in node.inputs: - raise NotImplementedError(f"GdnFrostEngine: {node.node_type.name} node '{node.name}' is missing input '{port}'") - ckpt = int(node.params.get("checkpoint_every_n_tokens", 0) or 0) - if ckpt and (node.node_type.name != "GDN" or ckpt % 64 != 0): + if facts.safe_gate: + raise NotImplementedError("GdnFrostEngine: safe_gate is not supported (no gate-activation path; the cuTile GDN engine serves it)") + ckpt = facts.checkpoint_every_n_tokens + if ckpt and (facts.is_bwd or ckpt % 64 != 0): raise NotImplementedError(f"GdnFrostEngine: checkpoint_every_n_tokens must be a positive multiple of 64 on the GDN node (got {ckpt})") - _check_common(node) + if not facts.gates_at_ho: + raise NotImplementedError(f"GdnFrostEngine: g/beta must carry HO = max(q, v) heads ({facts.h_o})") + fp32 = cudnn.data_type.FLOAT + io = (cudnn.data_type.BFLOAT16, cudnn.data_type.HALF) + state_dtypes = (fp32, cudnn.data_type.BFLOAT16) + require("GdnFrostEngine", "beta", facts.beta_dtype, fp32) + require("GdnFrostEngine", "initial_state", facts.state_dtype, state_dtypes) + require("GdnFrostEngine", "final_state", facts.final_state_dtype, state_dtypes) + if not facts.state_pair_match: + raise NotImplementedError("GdnFrostEngine: initial_state and final_state dtypes must match") + if facts.is_bwd: + require("GdnFrostEngine", "dO", facts.do_dtype, io) + if facts.io_dtype is not None: + require("GdnFrostEngine", "state_checkpoints", facts.state_checkpoints_dtype, facts.io_dtype) + require("GdnFrostEngine", "d_final_state", facts.d_final_state_dtype, fp32) + require("GdnFrostEngine", "d_initial_state", facts.d_initial_state_dtype, fp32) + require("GdnFrostEngine", "dG", facts.dg_dtype, fp32) + require("GdnFrostEngine", "dBeta", facts.dbeta_dtype, fp32) + elif facts.io_dtype is not None: + require("GdnFrostEngine", "state_checkpoints", facts.state_checkpoints_out_dtype, facts.io_dtype) def build_plan(self, graph, plan, ctx=None) -> CompiledPlan: - return _FrostPlan(build_gdn(graph)) - - -def _splits_for(gate, cu_i32, work_items, item_scratch, chunk_scratch, work_count, ideal_chunks, n_tiles, num_sms, b_t, stream, log_gate=False, sched_ctr=None): - """Launch the split pipeline (stream-ordered; the scan zeroes the count - and the scheduler ticket ring).""" - from .common.split_k import build_split_table - - build_split_table( - gate, - cu_i32, - work_items, - work_count, - ideal_chunks=ideal_chunks, - n_tiles=n_tiles, - num_sms=num_sms, - b_t=b_t, - chunk_scratch=chunk_scratch, - item_scratch=item_scratch, - log_gate=log_gate, - sched_ctr=sched_ctr, - stream=stream, - ) - return work_items, work_count + return FrostLaPlan(build_gdn(graph)) class CompiledGdn: @@ -164,94 +96,103 @@ class CompiledGdn: def __init__(self, node, kernel_mod): from .common.split_k import WORK_ITEM_FIELDS, chunk_scratch_rows, compute_ideal_chunks, max_work_items - self._node = node - self._kernel = kernel_mod + self.node = node + self.kernel = kernel_mod + self.expect = expect_table(node, GDN_ALIGN) scale = node.params.get("scale") - self._scale = float(scale) if scale is not None else 1.0 / math.sqrt(node.inputs["q"].dim[-1]) + self.scale = float(scale) if scale is not None else 1.0 / math.sqrt(node.inputs["q"].dim[-1]) q, v, g = node.inputs["q"], node.inputs["v"], node.inputs["g"] - self._b_t = kernel_mod.CFG.B_T - total, K, V = q.dim[0], q.dim[2], v.dim[2] + self.b_t = kernel_mod.CFG.B_T + total = q.dim[0] HO = g.dim[1] B = node.inputs["cu_seqlens"].dim[0] - 1 - self._has_fs = "final_state" in node.outputs - self._ckpt = int(node.params.get("checkpoint_every_n_tokens", 0) or 0) - self._has_h = "H" in node.outputs - # split work items assume chunk-granular state boundaries; other - # checkpoint cadences keep the serial per-(b,h) walk - self._split = self._ckpt in (0, self._b_t) + self.has_final_state = "final_state" in node.outputs + self.ckpt = int(node.params.get("checkpoint_every_n_tokens", 0) or 0) + self.has_state_checkpoints = "state_checkpoints" in node.outputs + # split table only for chunk-granular checkpoint cadences + self.split = self.ckpt in (0, self.b_t) layout = WorkspaceLayout() - self._off_tensormaps = layout.add(kernel_mod.get_workspace_size(B, q.dim[1], v.dim[1])) - self._tensormap_words = kernel_mod.get_workspace_size(B, q.dim[1], v.dim[1]) // 8 - self._off_sched = layout.add(8) # [ticket, done] for the dynamic scheduler - if self._split: - self._num_sm = kernel_mod._device_sm_count() - self._ideal = compute_ideal_chunks(total, HO, self._num_sm, self._b_t) - self._n_tiles = B * HO - self._n_heads_out = HO - self._work_item_rows = max_work_items(total, B, HO, self._ideal, self._b_t, self._num_sm) - self._off_work_items = layout.add(self._work_item_rows * WORK_ITEM_FIELDS * 4) - self._off_item_scratch = layout.add(self._work_item_rows * WORK_ITEM_FIELDS * 4) - self._off_work_count = layout.add(4) - self._chunk_scratch_rows = chunk_scratch_rows(total, B, self._b_t) - self._off_chunk_scratch = layout.add(self._chunk_scratch_rows * HO * 4) - self._ws_bytes = layout.size - - regions = [ - (self._off_tensormaps, "int64", (self._tensormap_words,)), - (self._off_sched, "int32", (2,)), - ] - if self._split: - regions += [ - (self._off_work_items, "int32", (self._work_item_rows, WORK_ITEM_FIELDS)), - (self._off_item_scratch, "int32", (self._work_item_rows, WORK_ITEM_FIELDS)), - (self._off_work_count, "int32", (1,)), - (self._off_chunk_scratch, "float32", (self._chunk_scratch_rows, HO)), - ] - self._carve = carve_plan("gdn", regions) + from .common.host import tensormap_workspace_bytes + + self.tensormap_words = tensormap_workspace_bytes(kernel_mod, B) // 8 + self.off_tensormaps = layout.add(self.tensormap_words * 8) + self.off_sched = layout.add(8) # [ticket, done] for the dynamic scheduler + if self.split: + self.num_sm = multiprocessor_count(current_device_id()) + self.ideal = compute_ideal_chunks(total, HO, self.num_sm, self.b_t) + self.n_tiles = B * HO + self.n_heads_out = HO + self.work_item_rows = max_work_items(total, B, HO, self.ideal, self.b_t, self.num_sm) + self.off_work_items = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_item_scratch = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_work_count = layout.add(4) + self.chunk_scratch_rows = chunk_scratch_rows(total, B, self.b_t) + self.off_chunk_scratch = layout.add(self.chunk_scratch_rows * HO * 4) + self.ws_bytes = layout.size def workspace_bytes(self) -> int: - return self._ws_bytes + return self.ws_bytes def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: - nb = node_buffers[self._node] + nb = node_buffers[self.node] q = nb.inputs["q"] k = nb.inputs["k"] v = nb.inputs["v"] g = nb.inputs["g"] beta = nb.inputs["beta"] cu = nb.inputs["cu_seqlens"] - s0 = nb.inputs.get("initial_state") + state0 = nb.inputs.get("initial_state") o = nb.outputs["O"] - fs = nb.outputs["final_state"] if self._has_fs else None - h = nb.outputs["H"] if self._has_h else None + final_state = nb.outputs["final_state"] if self.has_final_state else None + state_checkpoints = nb.outputs["state_checkpoints"] if self.has_state_checkpoints else None + check_layouts( + "GdnFrostEngine (GDN)", + expect=self.expect, + compact=GDN_FWD_COMPACT, + q=q, + k=k, + v=v, + g=g, + beta=beta, + cu_seqlens=cu, + initial_state=state0, + O=o, + final_state=final_state, + state_checkpoints=state_checkpoints, + ) - ws = workspace + ws = Workspace(workspace, self.ws_bytes, "GdnFrostEngine (GDN)") stream = stream if stream is not None else 0 + sched_ctr = ws.view(self.off_sched, "int32", (2,)) work_items = work_count = None - if self._split: - tensormaps, sched_ctr, work_items, item_scratch, work_count, chunk_scratch = ws.carve(self._carve) - _splits_for( + if self.split: + from .common.split_k import WORK_ITEM_FIELDS, build_split_table + + work_items = ws.view(self.off_work_items, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + item_scratch = ws.view(self.off_item_scratch, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + work_count = ws.view(self.off_work_count, "int32", (1,)) + chunk_scratch = ws.view(self.off_chunk_scratch, "float32", (self.chunk_scratch_rows, self.n_heads_out)) + build_split_table( g, cu, work_items, - item_scratch, - chunk_scratch, work_count, - self._ideal, - self._n_tiles, - self._num_sm, - self._b_t, - stream, + ideal_chunks=self.ideal, + n_tiles=self.n_tiles, + num_sms=self.num_sm, + b_t=self.b_t, + chunk_scratch=chunk_scratch, + item_scratch=item_scratch, log_gate=True, sched_ctr=sched_ctr, + stream=stream, ) else: - tensormaps, sched_ctr = ws.carve(self._carve) buffers.memset_zero_async(sched_ctr.data_ptr(), sched_ctr.nbytes, stream) - self._kernel.chunk_gdn_sm100( + self.kernel.chunk_gdn_sm100( q, k, v, @@ -259,16 +200,16 @@ def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: beta, o, cu, - s0, - fs, - self._scale, + state0, + final_state, + self.scale, work_items=work_items, work_count=work_count, sched_ctr=sched_ctr, - checkpoint_every_n_tokens=self._ckpt, - output_h=h, + checkpoint_every_n_tokens=self.ckpt, + output_state_checkpoints=state_checkpoints, log_gate=True, - workspace=tensormaps, + workspace=ws.view(self.off_tensormaps, "int64", (self.tensormap_words,)), stream=stream, ) return None @@ -278,93 +219,73 @@ class CompiledGdnBwd: """Compiled FROST GDN bprop plan: a callable over the resolved node buffers. Produces dQ/dK/dV/dG/dBeta; consumes the forward per-chunk states through - the node's ``h`` input, or regenerates them with a forward H-dump run when - the port is absent.""" + the node's ``state_checkpoints`` input, or regenerates them with the recompute + (checkpoint-only) kernel when the port is absent.""" - def __init__(self, node, kernel_mod): + def __init__(self, node, bwd_mod, regen_mod): from .common.split_k import WORK_ITEM_FIELDS, chunk_scratch_rows, compute_ideal_chunks, max_work_items - self._node = node - self._kernel = kernel_mod - scale = node.params.get("scale") - self._scale = float(scale) if scale is not None else 1.0 / math.sqrt(node.inputs["q"].dim[-1]) + self.node = node + self.bwd = bwd_mod + self.regen = regen_mod + self.expect = expect_table(node, GDN_ALIGN) + from .common.downcast import downcast_state + from .common.host import tensormap_workspace_bytes - from .kernel import gdn_prefill_f16 as fwd_mod + self.downcast_state = downcast_state + scale = node.params.get("scale") + self.scale = float(scale) if scale is not None else 1.0 / math.sqrt(node.inputs["q"].dim[-1]) - self._fwd = fwd_mod q, v, g = node.inputs["q"], node.inputs["v"], node.inputs["g"] - self._b_t = kernel_mod.CFG.B_T - total, K, V = q.dim[0], q.dim[2], v.dim[2] + self.b_t = bwd_mod.CFG.B_T + total = q.dim[0] + K, V = q.dim[-1], v.dim[-1] HQ, HV = q.dim[1], v.dim[1] HO = g.dim[1] B = node.inputs["cu_seqlens"].dim[0] - 1 - self._has_h = "h" in node.inputs - self._has_s0 = "initial_state" in node.inputs - self._has_dht = "d_final_state" in node.inputs - self._has_ds0 = "d_initial_state" in node.outputs - self._io_name = _io_dtype_name(node) - - # multi-wave grids pay ~2-3% for the ticket ring in the BACKWARD - # (its TMA-LDG publisher drives five load streams); static stride there - self._bwd_dyn_sched = B * HO <= kernel_mod._device_sm_count() + self.has_state_checkpoints = "state_checkpoints" in node.inputs + self.has_state0 = "initial_state" in node.inputs + self.io_name = "float16" if node.inputs["q"].get_data_type().name == "HALF" else "bfloat16" + + self.num_sm = multiprocessor_count(current_device_id()) + self.bwd_dyn_sched = B * HO <= self.num_sm layout = WorkspaceLayout() - self._off_sched = layout.add(16) # one [ticket, done] ring each for the regen and bwd kernels - self._tensormap_words = kernel_mod.get_workspace_size(B, HQ, HV) // 8 - self._off_tensormaps = layout.add(self._tensormap_words * 8) - # regenerated H series (io dtype [n, HO, K, V]); the entry count is - # bounded statically so no device sync sizes it - self._h_rows = 0 if self._has_h else max(total // self._b_t + B, 1) - self._off_h = layout.add(self._h_rows * HO * K * V * 2) if self._h_rows else None - # io-downcast initial state (chunk 0's S reads it via its own - # descriptor set) - self._off_s0io = layout.add(B * HO * K * V * 2) if self._has_s0 else None - if not self._has_h: - self._fwd_tensormap_words = fwd_mod.get_workspace_size(B, HO, HO) // 8 - self._off_fwd_tensormaps = layout.add(self._fwd_tensormap_words * 8) - self._is_gva = HV > HQ - self._is_gqa = HQ > HV - if self._is_gva: - self._off_dq_ho = layout.add(total * HV * K * 2) - self._off_dk_ho = layout.add(total * HV * K * 2) - if self._is_gqa: - self._off_dv_ho = layout.add(total * HQ * V * 2) - self._num_sm = kernel_mod._device_sm_count() - self._ideal = compute_ideal_chunks(total, HO, self._num_sm, self._b_t) - self._n_tiles = B * HO - self._work_item_rows = max_work_items(total, B, HO, self._ideal, self._b_t, self._num_sm) - self._off_work_items = layout.add(self._work_item_rows * WORK_ITEM_FIELDS * 4) - self._off_item_scratch = layout.add(self._work_item_rows * WORK_ITEM_FIELDS * 4) - self._off_work_count = layout.add(4) - self._chunk_scratch_rows = chunk_scratch_rows(total, B, self._b_t) - self._off_chunk_scratch = layout.add(self._chunk_scratch_rows * HO * 4) - self._shapes = (total, HQ, HV, HO, K, V, B) - self._ws_bytes = layout.size - - # The regions every backward carves. The rest (io state, regenerated H, - # head-group scratch) hang off build-time branches that are usually - # off, and stay on view() rather than turning this into index - # bookkeeping. The three scheduler views overlap on purpose: one ring - # each for the regen and bwd kernels, and both at once for the split - # pipeline that zeroes them. - self._carve = carve_plan( - "gdn_bwd", - [ - (self._off_sched, "int32", (2,)), - (self._off_sched + 8, "int32", (2,)), - (self._off_sched, "int32", (4,)), - (self._off_work_items, "int32", (self._work_item_rows, WORK_ITEM_FIELDS)), - (self._off_item_scratch, "int32", (self._work_item_rows, WORK_ITEM_FIELDS)), - (self._off_work_count, "int32", (1,)), - (self._off_chunk_scratch, "float32", (self._chunk_scratch_rows, HO)), - (self._off_tensormaps, "int64", (self._tensormap_words,)), - ], - ) + self.off_sched = layout.add(16) # one [ticket, done] ring each for the regen and bwd kernels + self.tensormap_words = tensormap_workspace_bytes(bwd_mod, B) // 8 + self.off_tensormaps = layout.add(self.tensormap_words * 8) + # chunk-0 entering state, io dtype (downcast initial_state; absent = in-kernel zeros) + self.off_state0_io = layout.add(B * HO * K * V * 2) if self.has_state0 else None + if not self.has_state_checkpoints: + self.state_checkpoints_rows = max(total // self.b_t + B, 1) + self.off_state_checkpoints = layout.add(self.state_checkpoints_rows * HO * K * V * 2) + self.regen_tensormap_words = tensormap_workspace_bytes(regen_mod, B) // 8 + self.off_regen_tensormaps = layout.add(self.regen_tensormap_words * 8) + HK = node.inputs["k"].dim[1] + self.fold_dq = HQ < HO + self.fold_dk = HK < HO + self.fold_dv = HV < HO + if self.fold_dq: + self.off_dq_ho = layout.add(total * HO * K * 2) + if self.fold_dk: + self.off_dk_ho = layout.add(total * HO * K * 2) + if self.fold_dv: + self.off_dv_ho = layout.add(total * HO * V * 2) + self.ideal = compute_ideal_chunks(total, HO, self.num_sm, self.b_t) + self.n_tiles = B * HO + self.work_item_rows = max_work_items(total, B, HO, self.ideal, self.b_t, self.num_sm) + self.off_work_items = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_item_scratch = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_work_count = layout.add(4) + self.chunk_scratch_rows = chunk_scratch_rows(total, B, self.b_t) + self.off_chunk_scratch = layout.add(self.chunk_scratch_rows * HO * 4) + self.n_heads_out, self._total = HO, total + self.ws_bytes = layout.size def workspace_bytes(self) -> int: - return self._ws_bytes + return self.ws_bytes - def __call__(self, node_buffers, *, workspace=None, stream=None): - nb = node_buffers[self._node] + def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: + nb = node_buffers[self.node] q = nb.inputs["q"] k = nb.inputs["k"] v = nb.inputs["v"] @@ -372,103 +293,134 @@ def __call__(self, node_buffers, *, workspace=None, stream=None): beta = nb.inputs["beta"] cu = nb.inputs["cu_seqlens"] do = nb.inputs["dO"] - h_in = nb.inputs.get("h") if self._has_h else None - s0 = nb.inputs.get("initial_state") if self._has_s0 else None - dht = nb.inputs.get("d_final_state") if self._has_dht else None - ds0 = nb.outputs.get("d_initial_state") + state_checkpoints = nb.inputs.get("state_checkpoints") + state0 = nb.inputs.get("initial_state") + dstate_in = nb.inputs.get("d_final_state") dq = nb.outputs["dQ"] dk = nb.outputs["dK"] dv = nb.outputs["dV"] dg = nb.outputs["dG"] - dbeta = nb.outputs["dBeta"] - ws = workspace - total, HQ, HV, HO, K, V, B = self._shapes + db = nb.outputs["dBeta"] + dstate0 = nb.outputs.get("d_initial_state") + check_layouts( + "GdnFrostEngine (GDN_BWD)", + expect=self.expect, + compact=() if self.has_state_checkpoints else GDN_BWD_RECOMPUTE_COMPACT, + q=q, + k=k, + v=v, + g=g, + beta=beta, + cu_seqlens=cu, + dO=do, + state_checkpoints=state_checkpoints, + initial_state=state0, + d_final_state=dstate_in, + d_initial_state=dstate0, + dQ=dq, + dK=dk, + dV=dv, + dG=dg, + dBeta=db, + ) + + ws = Workspace(workspace, self.ws_bytes, "GdnFrostEngine (GDN_BWD)") + HO, total = self.n_heads_out, self._total + K, V = q.shape[-1], v.shape[-1] + B = cu.shape[0] - 1 stream = stream if stream is not None else 0 - sched_fwd, sched_bwd, sched_all, work_items, item_scratch, work_count, chunk_scratch, tensormaps = ws.carve(self._carve) - _splits_for( + sched_regen = ws.view(self.off_sched, "int32", (2,)) + sched_bwd = ws.view(self.off_sched + 8, "int32", (2,)) + from .common.split_k import WORK_ITEM_FIELDS, build_split_table + + work_items = ws.view(self.off_work_items, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + item_scratch = ws.view(self.off_item_scratch, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + work_count = ws.view(self.off_work_count, "int32", (1,)) + chunk_scratch = ws.view(self.off_chunk_scratch, "float32", (self.chunk_scratch_rows, HO)) + build_split_table( g, cu, work_items, - item_scratch, - chunk_scratch, work_count, - self._ideal, - self._n_tiles, - self._num_sm, - self._b_t, - stream, + ideal_chunks=self.ideal, + n_tiles=self.n_tiles, + num_sms=self.num_sm, + b_t=self.b_t, + chunk_scratch=chunk_scratch, + item_scratch=item_scratch, log_gate=True, - sched_ctr=sched_all, + sched_ctr=ws.view(self.off_sched, "int32", (4,)), + stream=stream, ) - s0_io = None - if s0 is not None: - s0_io = ws.view(self._off_s0io, self._io_name, (B, HO, K, V)) - self._fwd.downcast_state(s0, s0_io, stream=stream) - if h_in is not None: - h = h_in + state0_io = None + if state0 is not None: + s_ptr, s_shape, s_strides, s_dtype, _sdev = buffers.probe(state0) + if s_dtype == self.io_name and buffers.is_contiguous(s_shape, s_strides) and s_ptr % 16 == 0: + # already the staged form: io dtype, compact, aligned — bind directly + state0_io = state0 + else: + state0_io = ws.view(self.off_state0_io, self.io_name, (B, HO, K, V)) + self.downcast_state(state0, state0_io, stream=stream) + if self.has_state_checkpoints: + checkpoint_series = state_checkpoints else: - h = ws.view(self._off_h, self._io_name, (self._h_rows, HO, K, V)) - self._fwd.chunk_gdn_sm100( - q, + checkpoint_series = ws.view(self.off_state_checkpoints, self.io_name, (self.state_checkpoints_rows, HO, K, V)) + self.regen.chunk_gdn_recompute_sm100( k, v, g, beta, - None, cu, - s0, + state0, None, - self._scale, - checkpoint_every_n_tokens=self._b_t, - output_h=h, + checkpoint_every_n_tokens=self.b_t, + output_state_checkpoints=checkpoint_series, work_items=work_items, work_count=work_count, - sched_ctr=sched_fwd, + sched_ctr=sched_regen, log_gate=True, - workspace=ws.view(self._off_fwd_tensormaps, "int64", (self._fwd_tensormap_words,)), + workspace=ws.view(self.off_regen_tensormaps, "int64", (self.regen_tensormap_words,)), stream=stream, ) dq_out, dk_out, dv_out = dq, dk, dv - if self._is_gva: - dq_out = ws.view(self._off_dq_ho, self._io_name, (total, HV, K)) - dk_out = ws.view(self._off_dk_ho, self._io_name, (total, HV, K)) - if self._is_gqa: - dv_out = ws.view(self._off_dv_ho, self._io_name, (total, HQ, V)) - self._kernel.chunk_gdn_bwd_sm100( + if self.fold_dq: + dq_out = ws.view(self.off_dq_ho, self.io_name, (total, HO, K)) + if self.fold_dk: + dk_out = ws.view(self.off_dk_ho, self.io_name, (total, HO, K)) + if self.fold_dv: + dv_out = ws.view(self.off_dv_ho, self.io_name, (total, HO, V)) + self.bwd.chunk_gdn_bwd_sm100( q, k, v, g, beta, do, - h, + checkpoint_series, dq_out, dk_out, dv_out, dg, - dbeta, + db, cu, - self._scale, - initial_state=s0_io, - d_initial_state=ds0, - d_final_state=dht, + self.scale, + initial_state=state0_io, + d_initial_state=dstate0, + d_final_state=dstate_in, work_items=work_items, work_count=work_count, - sched_ctr=sched_bwd if self._bwd_dyn_sched else None, + sched_ctr=sched_bwd if self.bwd_dyn_sched else None, log_gate=True, - workspace=tensormaps, + workspace=ws.view(self.off_tensormaps, "int64", (self.tensormap_words,)), stream=stream, ) - if self._is_gva: - from .common.head_reduce import head_group_reduce - - head_group_reduce(dq_out, dq, stream=stream) - head_group_reduce(dk_out, dk, stream=stream) - if self._is_gqa: + if dq_out is not dq or dk_out is not dk or dv_out is not dv: from .common.head_reduce import head_group_reduce - head_group_reduce(dv_out, dv, stream=stream) + for src_ho, dst in ((dq_out, dq), (dk_out, dk), (dv_out, dv)): + if src_ho is not dst: + head_group_reduce(src_ho, dst, stream=stream) return None diff --git a/python/cudnn/linear_attention/frost/kda_engine.py b/python/cudnn/linear_attention/frost/kda_engine.py index 85f58be11..e6405e1f7 100644 --- a/python/cudnn/linear_attention/frost/kda_engine.py +++ b/python/cudnn/linear_attention/frost/kda_engine.py @@ -2,8 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 """FROST KDA engine: KDA nodes on the chunked prefill kernel -(``kernel/kda_prefill_f16.py``, Blackwell SM100/SM103, bf16/fp16, BT=16). -Forward only: KDA_BWD declines and falls back to ``KdaCuTileEngine``.""" +(``kernel/kda_prefill_f16.py``) and KDA_BWD nodes on the chunked backward +kernel (``kernel/kda_bprop_f16.py``), Blackwell SM100/SM103, bf16/fp16, +BT=16. The backward regenerates the per-chunk state checkpoints with the recompute +kernel (``kernel/kda_recompute_f16.py``) when the graph does not provide +one.""" from __future__ import annotations @@ -13,65 +16,28 @@ from cudnn import behavior_note from cudnn.engines.base import BaseEngine, CompiledPlan -from cudnn.frost import buffers -from cudnn.frost.workspace import Workspace, WorkspaceLayout, carve_plan -from ..engine_utils import _FrostPlan, _require_dtype, _require_state_pair +from cudnn.frost.buffers import current_device_id +from cudnn.frost.device import multiprocessor_count +from cudnn.frost.workspace import Workspace, WorkspaceLayout +from ..graph_analyzer import FrostLaPlan, check_layouts, frost_la_gate, require, analyze, expect_table - -def _the_kda_node(graph): - nodes = list(graph.nodes) - if len(nodes) != 1: - return None - node = nodes[0] - if getattr(node.node_type, "name", None) not in ("KDA", "KDA_BWD"): - return None - return node - - -def _check_common(node) -> None: - """Shape/dtype gates for the prefill kernel.""" - import cudnn - - q, k, v = (node.inputs[p] for p in ("q", "k", "v")) - io_dtypes = {q.get_data_type(), k.get_data_type(), v.get_data_type()} - {None} - if len(io_dtypes) > 1: - raise NotImplementedError(f"KdaFrostEngine: q/k/v dtypes must match, got {io_dtypes}") - for p, t in (("q", q), ("k", k), ("v", v)): - if t.get_data_type() not in (cudnn.data_type.BFLOAT16, cudnn.data_type.HALF, None): - raise NotImplementedError(f"KdaFrostEngine: '{p}' must be bf16 or fp16, got {t.get_data_type()}") - if not t.dim or len(t.dim) != 3: - raise NotImplementedError(f"KdaFrostEngine: '{p}' must be THD [total_T, heads, dim]") - if q.dim[-1] != 128 or v.dim[-1] != 128: - raise NotImplementedError(f"KdaFrostEngine: head dims must be 128 (the recurrent state is 128x128), got K={q.dim[-1]} V={v.dim[-1]}") - if k.dim[1] != q.dim[1]: - raise NotImplementedError(f"KdaFrostEngine: q and k head counts differ ({q.dim[1]} vs {k.dim[1]})") - if v.dim[1] % q.dim[1] != 0: - raise NotImplementedError(f"KdaFrostEngine: v heads ({v.dim[1]}) must be a multiple of q heads ({q.dim[1]})") - - # kernel-native operand dtypes: buffers pass through without staging - fp32 = cudnn.data_type.FLOAT - _require_dtype("KdaFrostEngine", node, "g", fp32) - if node.node_type.name == "KDA" and node.params.get("use_beta_sigmoid", False): - # in-kernel sigmoid: beta arrives as io-dtype logits - if node.inputs["beta"].get_data_type() not in (q.get_data_type(), None): - raise NotImplementedError("KdaFrostEngine: use_beta_sigmoid takes io-dtype beta logits") - else: - _require_dtype("KdaFrostEngine", node, "beta", fp32) - _require_dtype("KdaFrostEngine", node, "a_log", fp32) - _require_dtype("KdaFrostEngine", node, "dt_bias", fp32) - _require_dtype("KdaFrostEngine", node, "cu_seqlens", cudnn.data_type.INT32) - _require_dtype("KdaFrostEngine", node, "initial_state", (fp32, cudnn.data_type.BFLOAT16)) - _require_dtype("KdaFrostEngine", node, "final_state", (fp32, cudnn.data_type.BFLOAT16), out=True) - _require_state_pair("KdaFrostEngine", node) +KDA_ALIGN = {"a_log": 4, "beta": 4, "cu_seqlens": 8} +KDA_COMPACT = ("dt_bias",) def build_kda(graph): """The expensive step: import the kernel module (pulls in the Cutlass primitives; the cute.compile itself is cached inside the kernel per static config and runs on first execute, when the real buffers are known).""" - node = _the_kda_node(graph) - if node is None or node.node_type.name != "KDA": - raise ValueError("build_kda: graph does not contain exactly one KDA node") + nodes = list(graph.nodes) + if len(nodes) != 1 or getattr(nodes[0].node_type, "name", None) not in ("KDA", "KDA_BWD"): + raise ValueError("build_kda: graph does not contain exactly one KDA/KDA_BWD node") + node = nodes[0] + if node.node_type.name == "KDA_BWD": + from .kernel import kda_bprop_f16 as bwd_mod + from .kernel import kda_recompute_f16 as regen_mod + + return CompiledKdaBwd(node, bwd_mod, regen_mod) from .kernel import kda_prefill_f16 as kernel_mod return CompiledKda(node, kernel_mod) @@ -80,42 +46,51 @@ def build_kda(graph): class KdaFrostEngine(BaseEngine): """FROST chunked-kernel backend for single-node KDA graphs (THD layout). - Default KDA forward engine on SM100/SM103 (lowest KDA engine_id); - declines ``KDA_BWD`` so the router falls back to ``KdaCuTileEngine``.""" + Default KDA engine on SM100/SM103 (lowest KDA engine_id); serves KDA + forward and KDA_BWD (with a forward checkpoint recompute when ``state_checkpoints`` is absent).""" name = "kda_frost" behavior_notes = (behavior_note.RUNTIME_COMPILATION,) # JIT-compiled at build_plans() def check_support(self, graph) -> None: - node = _the_kda_node(graph) - if node is None: - raise NotImplementedError("KdaFrostEngine supports exactly one KDA node") - if node.node_type.name == "KDA_BWD": - raise NotImplementedError("KdaFrostEngine: the FROST KDA backward kernel is a stub; KdaCuTileEngine covers gradients") - sm = buffers.current_sm() - if sm is None or not (100 <= sm <= 103): - raise NotImplementedError(f"KdaFrostEngine requires SM100-SM103 (found {sm})") - try: - import cutlass.experimental.primitives # noqa: F401 — availability probe: ImportError = decline - except ImportError as exc: - raise NotImplementedError(f"KdaFrostEngine requires the Cutlass DSL with cutlass.experimental.primitives: {exc}") from exc - for port in ("q", "k", "v", "g", "beta", "cu_seqlens"): - if port not in node.inputs: - raise NotImplementedError(f"KdaFrostEngine: KDA node '{node.name}' is missing input '{port}'") - if int(node.params.get("checkpoint_every_n_tokens", 0) or 0) or "H" in node.outputs: - raise NotImplementedError("KdaFrostEngine: the per-chunk H output lands with the backward kernel (jopark/kda_gdn2_bprop)") - if node.node_type.name != "KDA" and (node.params.get("use_beta_sigmoid", False) or node.params.get("safe_gate", False)): + import cudnn + + facts = graph._facts_for(analyze) + frost_la_gate("KdaFrostEngine", facts, "KDA") + ckpt = facts.checkpoint_every_n_tokens + if ckpt and (facts.is_bwd or ckpt != 16): + raise NotImplementedError(f"KdaFrostEngine: checkpoint_every_n_tokens must be 16 (per-chunk) on the forward node (got {ckpt})") + if facts.is_bwd and (facts.use_beta_sigmoid or facts.safe_gate): raise NotImplementedError("KdaFrostEngine: use_beta_sigmoid/safe_gate are forward-node attributes") - if node.params.get("safe_gate", False): - for p in ("a_log", "dt_bias"): - if p not in node.inputs: - raise NotImplementedError(f"KdaFrostEngine: safe_gate requires input '{p}'") - elif "a_log" in node.inputs or "dt_bias" in node.inputs: - raise NotImplementedError("KdaFrostEngine: a_log/dt_bias require safe_gate=True") - _check_common(node) + fp32 = cudnn.data_type.FLOAT + if not facts.is_bwd and facts.use_beta_sigmoid: + # in-kernel sigmoid: Beta arrives as io-dtype logits + if facts.beta_dtype not in (facts.io_dtype, None): + raise NotImplementedError("KdaFrostEngine: use_beta_sigmoid takes io-dtype beta logits") + else: + require("KdaFrostEngine", "beta", facts.beta_dtype, fp32) + require("KdaFrostEngine", "a_log", facts.a_log_dtype, fp32) + require("KdaFrostEngine", "dt_bias", facts.dt_bias_dtype, fp32) + if facts.is_bwd: + for port, got in (("dO", facts.do_dtype), ("state_checkpoints", facts.state_checkpoints_dtype)): + if got not in (facts.io_dtype, None): + raise NotImplementedError(f"KdaFrostEngine: '{port}' must match the io dtype") + require("KdaFrostEngine", "initial_state", facts.state_dtype, fp32) + require("KdaFrostEngine", "d_final_state", facts.d_final_state_dtype, fp32) + require("KdaFrostEngine", "d_initial_state", facts.d_initial_state_dtype, fp32) + require("KdaFrostEngine", "dG", facts.dg_dtype, fp32) + require("KdaFrostEngine", "dBeta", facts.dbeta_dtype, fp32) + else: + state_dtypes = (fp32, cudnn.data_type.BFLOAT16) + require("KdaFrostEngine", "initial_state", facts.state_dtype, state_dtypes) + if facts.io_dtype is not None: + require("KdaFrostEngine", "state_checkpoints", facts.state_checkpoints_out_dtype, facts.io_dtype) + require("KdaFrostEngine", "final_state", facts.final_state_dtype, state_dtypes) + if not facts.state_pair_match: + raise NotImplementedError("KdaFrostEngine: initial_state and final_state dtypes must match") def build_plan(self, graph, plan, ctx=None) -> CompiledPlan: - return _FrostPlan(build_kda(graph)) + return FrostLaPlan(build_kda(graph)) class CompiledKda: @@ -124,69 +99,87 @@ class CompiledKda: def __init__(self, node, kernel_mod): from .common.split_k import WORK_ITEM_FIELDS, chunk_scratch_rows, compute_ideal_chunks, max_work_items - self._node = node - self._kernel = kernel_mod + self.node = node + self.kernel = kernel_mod + self.expect = expect_table(node, KDA_ALIGN) scale = node.params.get("scale") - self._scale = float(scale) if scale is not None else 1.0 / math.sqrt(node.inputs["q"].dim[-1]) - self._use_qk_l2norm = bool(node.params.get("use_qk_l2norm", False)) - self._use_beta_sigmoid = bool(node.params.get("use_beta_sigmoid", False)) - self._safe_gate = bool(node.params.get("safe_gate", False)) + self.scale = float(scale) if scale is not None else 1.0 / math.sqrt(node.inputs["q"].dim[-1]) + self.use_qk_l2norm = bool(node.params.get("use_qk_l2norm", False)) + self.use_beta_sigmoid = bool(node.params.get("use_beta_sigmoid", False)) + self.safe_gate = bool(node.params.get("safe_gate", False)) glb = node.params.get("gate_lower_bound") - self._gate_lower_bound = float(glb) if glb is not None else kernel_mod.DEFAULT_GATE_LOWER_BOUND - self._has_fs = "final_state" in node.outputs + self.gate_lower_bound = float(glb) if glb is not None else kernel_mod.DEFAULT_GATE_LOWER_BOUND + self.has_final_state = "final_state" in node.outputs + self.has_state_checkpoints = "state_checkpoints" in node.outputs q, g = node.inputs["q"], node.inputs["g"] - self._b_t = kernel_mod.CFG.B_T + self.b_t = kernel_mod.CFG.B_T total = q.dim[0] HO = g.dim[1] B = node.inputs["cu_seqlens"].dim[0] - 1 layout = WorkspaceLayout() - self._off_sched = layout.add(8) # [ticket, done] for the dynamic scheduler - self._num_sm = kernel_mod._device_sm_count() - self._ideal = compute_ideal_chunks(total, HO, self._num_sm, self._b_t) - self._n_tiles = B * HO - self._work_item_rows = max_work_items(total, B, HO, self._ideal, self._b_t, self._num_sm) - self._n_heads_out = HO - self._off_work_items = layout.add(self._work_item_rows * WORK_ITEM_FIELDS * 4) - self._off_item_scratch = layout.add(self._work_item_rows * WORK_ITEM_FIELDS * 4) - self._off_work_count = layout.add(4) - self._chunk_scratch_rows = chunk_scratch_rows(total, B, self._b_t) - self._off_chunk_scratch = layout.add(self._chunk_scratch_rows * HO * 4) - self._tensormap_bytes = kernel_mod.get_workspace_size(B, HO, HO) - self._off_tensormaps = layout.add(self._tensormap_bytes, align=128) - self._ws_bytes = layout.size - - self._carve = carve_plan( - "kda", - [ - (self._off_sched, "int32", (2,)), - (self._off_work_items, "int32", (self._work_item_rows, WORK_ITEM_FIELDS)), - (self._off_item_scratch, "int32", (self._work_item_rows, WORK_ITEM_FIELDS)), - (self._off_work_count, "int32", (1,)), - (self._off_chunk_scratch, "float32", (self._chunk_scratch_rows, HO)), - (self._off_tensormaps, "int64", (self._tensormap_bytes // 8,)), - ], - ) + self.off_sched = layout.add(8) # [ticket, done] for the dynamic scheduler + self.num_sm = multiprocessor_count(current_device_id()) + self.ideal = compute_ideal_chunks(total, HO, self.num_sm, self.b_t) + self.n_tiles = B * HO + self.work_item_rows = max_work_items(total, B, HO, self.ideal, self.b_t, self.num_sm) + self.n_heads_out = HO + self.off_work_items = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_item_scratch = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_work_count = layout.add(4) + self.chunk_scratch_rows = chunk_scratch_rows(total, B, self.b_t) + self.off_chunk_scratch = layout.add(self.chunk_scratch_rows * HO * 4) + from .common.host import tensormap_workspace_bytes + + self.tensormap_bytes = tensormap_workspace_bytes(kernel_mod, B) + self.off_tensormaps = layout.add(self.tensormap_bytes, align=128) + self.ws_bytes = layout.size def workspace_bytes(self) -> int: - return self._ws_bytes + return self.ws_bytes def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: - nb = node_buffers[self._node] + nb = node_buffers[self.node] q = nb.inputs["q"] k = nb.inputs["k"] v = nb.inputs["v"] g = nb.inputs["g"] beta = nb.inputs["beta"] cu = nb.inputs["cu_seqlens"] - s0 = nb.inputs.get("initial_state") + state0 = nb.inputs.get("initial_state") o = nb.outputs["O"] - fs = nb.outputs["final_state"] if self._has_fs else None + final_state = nb.outputs["final_state"] if self.has_final_state else None + state_checkpoints = nb.outputs["state_checkpoints"] if self.has_state_checkpoints else None + a_log = nb.inputs.get("a_log") + dt_bias = nb.inputs.get("dt_bias") + check_layouts( + "KdaFrostEngine (KDA)", + expect=self.expect, + compact=KDA_COMPACT, + q=q, + k=k, + v=v, + g=g, + beta=beta, + cu_seqlens=cu, + initial_state=state0, + O=o, + final_state=final_state, + state_checkpoints=state_checkpoints, + a_log=a_log, + dt_bias=dt_bias, + ) stream = stream if stream is not None else 0 - ws = workspace - sched_ctr, work_items, item_scratch, work_count, chunk_scratch, tensormaps = ws.carve(self._carve) + ws = Workspace(workspace, self.ws_bytes, "KdaFrostEngine (KDA)") + sched_ctr = ws.view(self.off_sched, "int32", (2,)) + from .common.split_k import WORK_ITEM_FIELDS + + work_items = ws.view(self.off_work_items, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + work_count = ws.view(self.off_work_count, "int32", (1,)) + item_scratch = ws.view(self.off_item_scratch, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + chunk_scratch = ws.view(self.off_chunk_scratch, "float32", (self.chunk_scratch_rows, self.n_heads_out)) from .common.split_k import build_split_table build_split_table( @@ -194,22 +187,26 @@ def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: cu, work_items, work_count, - ideal_chunks=self._ideal, - n_tiles=self._n_tiles, - num_sms=self._num_sm, - b_t=self._b_t, + ideal_chunks=self.ideal, + n_tiles=self.n_tiles, + num_sms=self.num_sm, + b_t=self.b_t, chunk_scratch=chunk_scratch, item_scratch=item_scratch, log_gate=True, - safe_gate=self._safe_gate, - a_log=nb.inputs.get("a_log"), - dt_bias=nb.inputs.get("dt_bias"), - gate_lower_bound=self._gate_lower_bound if self._safe_gate else None, + safe_gate=self.safe_gate, + a_log=a_log, + dt_bias=dt_bias, + gate_lower_bound=self.gate_lower_bound if self.safe_gate else None, sched_ctr=sched_ctr, stream=stream, ) - self._kernel.chunk_kda_sm100( + ckpt_kwargs = {} + if self.has_state_checkpoints: + # the kernel derives the per-sequence checkpoint entry offsets on device + ckpt_kwargs = dict(checkpoint_every_n_tokens=self.b_t, output_state_checkpoints=state_checkpoints) + self.kernel.chunk_kda_sm100( q, k, v, @@ -217,19 +214,228 @@ def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: beta, o, cu, - s0, - fs, - self._scale, - use_qk_l2norm_in_kernel=self._use_qk_l2norm, - use_beta_sigmoid_in_kernel=self._use_beta_sigmoid, - safe_gate=self._safe_gate, - gate_lower_bound=self._gate_lower_bound, - a_log=nb.inputs.get("a_log"), - dt_bias=nb.inputs.get("dt_bias"), + state0, + final_state, + self.scale, + use_qk_l2norm_in_kernel=self.use_qk_l2norm, + use_beta_sigmoid_in_kernel=self.use_beta_sigmoid, + safe_gate=self.safe_gate, + gate_lower_bound=self.gate_lower_bound, + a_log=a_log, + dt_bias=dt_bias, work_items=work_items, work_count=work_count, sched_ctr=sched_ctr, - tensormap_workspace=tensormaps, + tensormap_workspace=ws.view(self.off_tensormaps, "int64", (self.tensormap_bytes // 8,)), + **ckpt_kwargs, + stream=stream, + ) + return None + + +class CompiledKdaBwd: + """Compiled FROST KDA backward plan. When the graph carries no ``state_checkpoints`` + input, the workspace holds a regenerated per-chunk checkpoint series (entry-0 + initial-state slots seeded on device, then the recompute kernel re-run + with shifted checkpoint bounds). GVA/GQA gradients land in HO-head + scratch and are reduced back to the native head counts.""" + + def __init__(self, node, bwd_mod, regen_mod): + from .common.split_k import WORK_ITEM_FIELDS, chunk_scratch_rows, compute_ideal_chunks, max_work_items + + self.node = node + self.bwd = bwd_mod + self.regen = regen_mod + self.expect = expect_table(node, KDA_ALIGN) + from .common.downcast import downcast_state + from .common.host import tensormap_workspace_bytes + + self.downcast_state = downcast_state + scale = node.params.get("scale") + self.scale = float(scale) if scale is not None else 1.0 / math.sqrt(node.inputs["q"].dim[-1]) + self.use_qk_l2norm = bool(node.params.get("use_qk_l2norm", False)) + self.has_state_checkpoints = "state_checkpoints" in node.inputs + self.has_state0 = "initial_state" in node.inputs + self.has_dstate0 = "d_initial_state" in node.outputs + + q, g, v = node.inputs["q"], node.inputs["g"], node.inputs["v"] + self.b_t = bwd_mod.CFG.B_T + total = q.dim[0] + HQ, HV = q.dim[1], v.dim[1] + HO = g.dim[1] + K, V = q.dim[-1], v.dim[-1] + B = node.inputs["cu_seqlens"].dim[0] - 1 + self.io_name = "float16" if node.inputs["q"].get_data_type().name == "HALF" else "bfloat16" + self.n_heads_out, self._total = HO, total + layout = WorkspaceLayout() + self.off_sched = layout.add(16) # one [ticket, done] ring each for the regen and bwd kernels + self.num_sm = multiprocessor_count(current_device_id()) + self.bwd_dyn_sched = B * HO <= self.num_sm + self.ideal = compute_ideal_chunks(total, HO, self.num_sm, self.b_t) + self.n_tiles = B * HO + self.work_item_rows = max_work_items(total, B, HO, self.ideal, self.b_t, self.num_sm) + self.off_work_items = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_item_scratch = layout.add(self.work_item_rows * WORK_ITEM_FIELDS * 4) + self.off_work_count = layout.add(4) + self.chunk_scratch_rows = chunk_scratch_rows(total, B, self.b_t) + self.off_chunk_scratch = layout.add(self.chunk_scratch_rows * HO * 4) + # chunk-0 entering state, io dtype (downcast initial_state; absent = in-kernel zeros) + self.off_state0_io = layout.add(B * HO * K * V * 2) if self.has_state0 else None + if not self.has_state_checkpoints: + self.state_checkpoints_rows = max(total // self.b_t, 1) + self.off_state_checkpoints = layout.add(self.state_checkpoints_rows * HO * K * V * 2) + self.regen_tm_bytes = tensormap_workspace_bytes(regen_mod, B) + self.off_regen_tensormaps = layout.add(self.regen_tm_bytes, align=128) + HK = node.inputs["k"].dim[1] + self.fold_dq = HQ < HO + self.fold_dk = HK < HO + self.fold_dv = HV < HO + if self.fold_dq: + self.off_dq_ho = layout.add(total * HO * K * 2) + if self.fold_dk: + self.off_dk_ho = layout.add(total * HO * K * 2) + if self.fold_dv: + self.off_dv_ho = layout.add(total * HO * V * 2) + self.bwd_tm_bytes = tensormap_workspace_bytes(bwd_mod, B) + self.off_bwd_tensormaps = layout.add(self.bwd_tm_bytes, align=128) + self.ws_bytes = layout.size + + def workspace_bytes(self) -> int: + return self.ws_bytes + + def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: + nb = node_buffers[self.node] + q = nb.inputs["q"] + k = nb.inputs["k"] + v = nb.inputs["v"] + g = nb.inputs["g"] + beta = nb.inputs["beta"] + cu = nb.inputs["cu_seqlens"] + do = nb.inputs["dO"] + state_checkpoints = nb.inputs.get("state_checkpoints") + state0 = nb.inputs.get("initial_state") + dstate_in = nb.inputs.get("d_final_state") + dq = nb.outputs["dQ"] + dk = nb.outputs["dK"] + dv = nb.outputs["dV"] + dg = nb.outputs["dG"] + db = nb.outputs["dBeta"] + dstate0 = nb.outputs.get("d_initial_state") + check_layouts( + "KdaFrostEngine (KDA_BWD)", + expect=self.expect, + q=q, + k=k, + v=v, + g=g, + beta=beta, + cu_seqlens=cu, + dO=do, + state_checkpoints=state_checkpoints, + initial_state=state0, + d_final_state=dstate_in, + dQ=dq, + dK=dk, + dV=dv, + dG=dg, + dBeta=db, + d_initial_state=dstate0, + ) + stream = stream if stream is not None else 0 + + HO, total = self.n_heads_out, self._total + K, V = q.shape[-1], v.shape[-1] + B = cu.shape[0] - 1 + ws = Workspace(workspace, self.ws_bytes, "KdaFrostEngine (KDA_BWD)") + sched_regen = ws.view(self.off_sched, "int32", (2,)) + sched_bwd = ws.view(self.off_sched + 8, "int32", (2,)) + from .common.split_k import WORK_ITEM_FIELDS + + work_items = ws.view(self.off_work_items, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + work_count = ws.view(self.off_work_count, "int32", (1,)) + item_scratch = ws.view(self.off_item_scratch, "int32", (self.work_item_rows, WORK_ITEM_FIELDS)) + chunk_scratch = ws.view(self.off_chunk_scratch, "float32", (self.chunk_scratch_rows, HO)) + from .common.split_k import build_split_table + + build_split_table( + g, + cu, + work_items, + work_count, + ideal_chunks=self.ideal, + n_tiles=self.n_tiles, + num_sms=self.num_sm, + b_t=self.b_t, + chunk_scratch=chunk_scratch, + item_scratch=item_scratch, + log_gate=True, + sched_ctr=ws.view(self.off_sched, "int32", (4,)), stream=stream, ) + + state0_io = None + if state0 is not None: + state0_io = ws.view(self.off_state0_io, self.io_name, (B, HO, K, V)) + self.downcast_state(state0, state0_io, stream=stream) + if self.has_state_checkpoints: + checkpoint_series = state_checkpoints + else: + checkpoint_series = ws.view(self.off_state_checkpoints, self.io_name, (self.state_checkpoints_rows, HO, K, V)) + self.regen.chunk_kda_recompute_sm100( + k, + v, + g, + beta, + cu, + state0, + None, + checkpoint_every_n_tokens=self.b_t, + output_state_checkpoints=checkpoint_series, + use_qk_l2norm_in_kernel=self.use_qk_l2norm, + work_items=work_items, + work_count=work_count, + sched_ctr=sched_regen, + tensormap_workspace=ws.view(self.off_regen_tensormaps, "int64", (self.regen_tm_bytes // 8,)), + stream=stream, + ) + + dq_out, dk_out, dv_out = dq, dk, dv + if self.fold_dq: + dq_out = ws.view(self.off_dq_ho, self.io_name, (total, HO, K)) + if self.fold_dk: + dk_out = ws.view(self.off_dk_ho, self.io_name, (total, HO, K)) + if self.fold_dv: + dv_out = ws.view(self.off_dv_ho, self.io_name, (total, HO, V)) + + self.bwd.chunk_kda_bwd_sm100( + q, + k, + v, + g, + beta, + do, + checkpoint_series, + dq_out, + dk_out, + dv_out, + dg, + db, + cu, + self.scale, + initial_state=state0_io, + d_initial_state=dstate0 if self.has_dstate0 else None, + d_final_state=dstate_in, + use_qk_l2norm_in_kernel=self.use_qk_l2norm, + work_items=work_items, + work_count=work_count, + sched_ctr=sched_bwd if self.bwd_dyn_sched else None, + tensormap_workspace=ws.view(self.off_bwd_tensormaps, "int64", (self.bwd_tm_bytes // 8,)), + stream=stream, + ) + if dq_out is not dq or dk_out is not dk or dv_out is not dv: + from .common.head_reduce import head_group_reduce + + for src_ho, dst in ((dq_out, dq), (dk_out, dk), (dv_out, dv)): + if src_ho is not dst: + head_group_reduce(src_ho, dst, stream=stream) return None diff --git a/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_config.py b/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_config.py index 549f14119..086d1ad49 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_config.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_config.py @@ -15,23 +15,64 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Gated DeltaNet v2 (GDN-2) Cutlass DSL backward kernel config — STUB. - -The FROST GDN-2 backward is not implemented yet (see ``gdn2_bprop_f16.py``); -these constants mirror the prefill config's tile shape for when the backward -kernel lands. +"""Gated DeltaNet v2 (GDN-2) Cutlass DSL backward kernel config (fixed +compile-time constants). The BT=16 backward (channel-wise erase gate beta + per-value write gate w) mirrors the prefill's 16-warp +(512-thread) specialization; the derived SMEM/TMEM sizes and offsets are +stamped by ``build_cfg`` in ``gdn2_bprop_f16.py``. Target arch: Blackwell SM100 (GB200) / SM103 (GB300). """ from dataclasses import dataclass +from typing import Tuple @dataclass(frozen=True) class Cfg: - B_T: int = 16 - D_K: int = 128 - D_V: int = 128 + # --- tile shape --- + B_T: int = 16 # chunk-inner token tile (BT=16 KDA schedule) + D_K: int = 128 # query/key head dim + D_V: int = 128 # value head dim + + # --- warp assignments (16 warps = 512 threads) --- + COMPUTE_GROUP_0_WARP_IDS: Tuple[int, ...] = (0, 1, 2, 3) # forward gate cumsum + decay-operand materialize + COMPUTE_GROUP_1_WARP_IDS: Tuple[int, ...] = (4, 5, 6, 7) # value-side TMEM staging / restages / dH capture + COMPUTE_GROUP_2_WARP_IDS: Tuple[int, ...] = (8, 9, 10, 11) # dq/dk-bank drain, dG assembly + reverse cumsum + SUPER_MMA_WARP_ID: int = 12 # register-MMA kk/qk/dA/dM + Neumann inverse + TCGEN05_MMA_WARP_ID: int = 13 # tcgen05 GEMM schedule + TMA_WARP_ID: int = 14 # q/k/v/gate/do/state(checkpoint) TMA loads + EPILOGUE_WARP_ID: int = 15 # dq/dk/dv TMA stores only + + # --- register split --- + # WG1's drain runs at the register ceiling (spilled at 152) while WG0 + # sits near ~100 live regs, so WG0 funds WG1. Constraints: compute + # groups can't go below the 128-reg launch base (setmaxregister is + # INCREASE-only there), and warps 12-15 are one warpgroup so they must + # share a single setmaxregister value (56; super/epilogue peak ~R49). + # dht configs keep ~15 in-loop WG2 spills at 136 (152 doesn't fit). + NUM_REGS_COMPUTE_GROUP_0: int = 128 + NUM_REGS_COMPUTE_GROUP_1: int = 192 + NUM_REGS_COMPUTE_GROUP_2: int = 136 + NUM_REGS_OTHER: int = 56 + + THREADS_PER_WARP: int = 32 + + BUFFER_ALIGN_BYTES: int = 1024 + + # --- SMEM / TMEM ring stage counts --- + SMEM_RAW_STAGES: int = 2 + SMEM_STATE_STAGES: int = 1 + SMEM_DECAY_STAGES: int = 2 + SMEM_QK_STAGES: int = 2 + SMEM_STATE_SCALE_DIAG_STAGES: int = 2 + SMEM_DQ_STAGES: int = 1 + SMEM_DK_STAGES: int = 1 + SMEM_DG_STAGES: int = 1 + SMEM_DB_STAGES: int = 1 + SMEM_DV_STAGES: int = 2 + SMEM_DWO_STAGES: int = 2 + + CLUSTER_SHAPE_MNK: Tuple[int, int, int] = (1, 1, 1) CFG = Cfg() diff --git a/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py index 012ba5e43..53d79d5f5 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py @@ -1,37 +1,3982 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# This kernel is derived from cuDNN, NVIDIA Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Gated DeltaNet v2 (GDN-2) Cutlass DSL backward kernel — STUB. - -The FROST GDN-2 backward is not implemented yet. Because the GDN-2 chunk -size is small, the design recomputes the forward per-chunk states inside the -backward (no H store in the prefill kernel). Until the backward kernel -lands, this module raises ``NotImplementedError``. - -Target arch: Blackwell SM100 (GB200) / SM103 (GB300). +"""Chunked Gated DeltaNet v2 (GDN-2) BPROP kernel for Blackwell SM100/SM103 +(Cutlass DSL), BT=16 tiling with a per-key-channel decay. Framework-neutral +entry ``chunk_gdn2_bwd_sm100``. + +Algorithm overview (per chunk c, iterated c = NT-1 .. 0; within-chunk +log2-domain gate cumsum G[t,d], eG = 2^G, eGl = 2^G[BT-1]): + Inputs : Q/K[BT,DK], V/dO[BT,DV], S = checkpoint[c-1] (state ENTERING chunk c, KV), + Gate[BT,DK], Beta[BT,DK] (per-key erase), w[BT,DV] (per-value write) + State : dH[DV,DK] (state gradient, fp32 TMEM, accumulated backward) + + Operands (WG0, prefill recompute): K_decay = eG.(Beta.K) (erase key, Beta + folded), K_inv = K/eG, K_restore = (eGl/eG).K, Q_decay = eG.Q, diag(eGl). + + Forward recompute: A_inv = (I + strict_tril(K_decay@K_inv^T))^-1 (register Neumann, + Beta pre-folded); A_qk = tril_incl(Q_decay@K_inv^T); W = w.V - S^T K_decay; U = A_inv@W. + + Backward math: + dU = K_restore@dH + A_qk^T@dO (Q_decay carries scale, so A_qk does too) + dY = A_inv^T@dU + dV = w.dY dW_out = V.dY (elementwise, WG1) + dA = tril_incl(dO@U^T) (unscaled) dM = dY@U^T dM_strict = +strict(dM) + dQ = eG.scale.(dO@S^T + dA@K_inv) + dK = Beta.eG.dK_decay + dK_inv/eG + (eGl/eG).dK_restore where (sign-flipped parts) + dK_decay part = dY@S^T + dM_strict@K_inv (= -dK_decay) + dK_inv part = dA^T@(scale.Q_decay) - dM_strict^T@K_decay (= dK_inv; one TMEM + acc, the minus rides the staged -dM_strict tile) + dK_restore part = U@dH^T (= +dK_restore) + dBeta[t,d] = k_n.eG.dK_decay = -k_n.eG.dK_decay part (per-channel, WG2) + dG[t,d] = q_n.dQ_pre + Beta.dBeta + k_n.(dK_inv_part/eG + - (eGl/eG).dK_restore_part) + dGlast[d] = eGl.sum_v(dH.S) + sum_t k_n.(eGl/eG).dK_restore_part + dGate = suffix-sum(dG + dGlast at row BT-1) (WG2 in-register reverse cumsum) + dH <- diag-GEMM(eGl).dH + (scale.Q_decay)^T@dO - K_decay^T@dY + +ABI: state_checkpoints `[total_checkpoints, HO, DK, DV]` (KV, v contiguous — the GDN checkpoint layout) io +dtype, the plain per-chunk series with NO initial-state slot (entry `c +- 1` = state entering chunk c >= 1; chunk 0 seeds from `initial_state`); beta `[T, HO, DK]` / w `[T, HO, DV]` io dtype +(post-sigmoid); dq/dk/dv io at HO heads; dg `[T, HO, DK]` fp32 (natural-log +gate domain); dbeta/dw io dtype like beta/w; d_initial_state / d_final_state +fp32 `[N, HO, DK, DV]` (K-major, matching the prefill states). + +Warp assignments (16 warps = 512 threads): + warps 0-3 : WG0 - Gate prefix scan + decay/restore operands (all chunks) + warps 4-7 : WG1 - value-side TMEM staging, restages, dstate capture, dV/dW_out + warps 8-11 : WG2 - dQ/dK part drain, dG/dBeta assembly, reverse cumsum + warp 12 : super-MMA - register KK/A_qk/dA/dM + Neumann inverse + warp 13 : tcgen05-MMA - the backward schedule + warp 14 : TMA load - Q/K/V/Gate/dO/state(checkpoint) loads + Beta scalars + warp 15 : epilogue - dQ/dK/dV/dGate/dBeta/dW_out TMA stores """ -from __future__ import annotations +from dataclasses import dataclass +from functools import lru_cache +from typing import NamedTuple, Optional, Type + +import cuda.bindings.driver as cuda_driver +import cutlass +import cutlass.experimental.cuda as cuda +import cutlass.experimental.primitives as nvvm +import cutlass.cute as cute +from cutlass.cute.runtime import from_dlpack + +from ..common.split_k import decode_work_item +from ..common.host import get_dtype +from cudnn.frost.buffers import current_device_id, data_ptr +from cudnn.frost.device import multiprocessor_count +from ..common.thd import TENSOR_MAP_QWORDS, emit_copy_desc, emit_checkpoint_seq_descs, emit_seq_descs +from .gdn2_bprop_config import CFG +from cudnn.frost.tile_dsl.barrier import ( + advance, + MBarrier, + PipelineState, + Producer, +) +from cudnn.frost.tile_dsl.handles import MmaDesc, SmemTile, tma_slice_runtime_desc +from cudnn.frost.tile_dsl.mma import mma_ss, mma_step, mma_ts_step +from cudnn.frost.tile_dsl.swizzle import swizzle_lin_S, swizzle_xor_128b +from cudnn.frost.tile_dsl.tma import tma_load_tile, tma_store_commit, tma_store_tile, tma_store_wait, tma_tensormap_acquire +from cudnn.frost.tile_dsl.pointwise import ( + opaque_f32_zero, + f16x2_to_f32, + fmul2, + ffma2, + movmatrix_16b, + mul_f16x2, + fp32_to_fp16, + sub_f16x2, +) + +LOG2_E: float = 1.4426950408889634 + +L2_NORM_EPS: float = 1.0e-12 + + +class Gdn2BwdBars(NamedTuple): + """Every inter-warp handoff as an ``MBarrier`` over its ring. Consumers + track ``(idx, phase)`` inline; the producer tag selects the arrive + lowering (``TMA_LOAD``/``MMA_COMMIT``/``THREAD``). + + Buffers read by both the MMA warp and a compute/warp group carry mixed + arrive counts (one MMA commit + N thread arrivers) so the producer only + reuses the slot once every reader is done.""" + + mb_q_ready: MBarrier + mb_q_done: MBarrier + mb_k_ready: MBarrier + mb_k_done: MBarrier + mb_gate_ready: MBarrier + mb_gate_done: MBarrier + mb_beta_ready: MBarrier + mb_beta_done: MBarrier + mb_do_ready: MBarrier + mb_do_done: MBarrier + mb_do_epi_done: MBarrier + mb_v_ready: MBarrier + mb_v_done: MBarrier + mb_w_ready: MBarrier + mb_w_done: MBarrier + mb_state_ready: MBarrier + mb_state_done: MBarrier + mb_state_cg0_done: MBarrier + mb_state_inp_ready: MBarrier + mb_state_inp_done: MBarrier + mb_state_inp_cg2_done: MBarrier + + mb_k_decay_inv_ready: MBarrier + mb_q_decay_k_restore_ready: MBarrier + mb_decay_done: MBarrier + mb_decay_super_done: MBarrier + + mb_a_ready: MBarrier + mb_aqk_ready: MBarrier + mb_da_ready: MBarrier + mb_dm_ready: MBarrier + mb_aqk_done: MBarrier + mb_a_done: MBarrier + mb_da_done: MBarrier + mb_dm_done: MBarrier + + mb_state_k_acc_ready: MBarrier + mb_rhs_ready: MBarrier + mb_update_acc_ready: MBarrier + mb_u_smem_ready: MBarrier + mb_u_done: MBarrier + mb_u_warps_done: MBarrier + mb_du_acc_ready: MBarrier + mb_du_inp_ready: MBarrier + mb_du_inp_done: MBarrier + mb_dy_acc_ready: MBarrier + mb_neg_dy_inp_ready: MBarrier + mb_neg_dy_inp_done: MBarrier + mb_sdy_ready: MBarrier + mb_sdy_done: MBarrier + mb_sdy_super_done: MBarrier + mb_dstate_acc_ready: MBarrier + mb_dstate_acc_done: MBarrier + mb_dstate_inp_ready: MBarrier + mb_dstate_inp_done: MBarrier + mb_dstate_smem_ready: MBarrier + mb_dstate_smem_done: MBarrier + mb_dstate_smem_cg2_done: MBarrier + + mb_dq_acc_ready: MBarrier + mb_dk_decay_part_ready: MBarrier + mb_dk_inv_part_ready: MBarrier + mb_dk_restore_part_ready: MBarrier + mb_parts_done: MBarrier + + mb_dq_tmastg_ready: MBarrier + mb_dq_tmastg_done: MBarrier + mb_dk_tmastg_ready: MBarrier + mb_dk_tmastg_done: MBarrier + mb_dv_tmastg_ready: MBarrier + mb_dv_tmastg_done: MBarrier + mb_dg_tmastg_ready: MBarrier + mb_dg_tmastg_done: MBarrier + mb_db_tmastg_ready: MBarrier + mb_db_tmastg_done: MBarrier + mb_dwo_tmastg_ready: MBarrier + mb_dwo_tmastg_done: MBarrier + mb_dstate0_stored: MBarrier + + mb_sched_ready: MBarrier + mb_sched_done: MBarrier + + +def make_gdn2_bwd_bars(cfg) -> Gdn2BwdBars: + """Bars factory. MUST be called from inside ``kernel`` (allocates the + mbarrier rings in SMEM ahead of the data buffers).""" + + def alloc(n): + return cutlass.Array(cutlass.Int64, n, space=cutlass.AddressSpace.smem, alignment=8) + + WARP = cfg.threads_per_warp + CG0 = len(cfg.compute_group_0_warp_ids) * WARP + CG2 = len(cfg.compute_group_2_warp_ids) * WARP + CG1 = len(cfg.compute_group_1_warp_ids) * WARP + MMA = 1 + + return Gdn2BwdBars( + mb_q_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_q_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0 + CG2, producer=Producer.THREAD), + mb_k_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_k_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0 + CG2, producer=Producer.THREAD), + mb_gate_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_gate_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0 + CG2, producer=Producer.THREAD), + mb_beta_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_beta_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0 + CG2, producer=Producer.THREAD), + mb_do_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_do_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_do_epi_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=WARP, producer=Producer.THREAD), + mb_v_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_v_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG1, producer=Producer.THREAD), + mb_w_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_w_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG1, producer=Producer.THREAD), + mb_state_ready=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_state_done=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_state_cg0_done=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=CG0, producer=Producer.THREAD), + mb_state_inp_ready=MBarrier(alloc(2), stages=2, init_count=CG0, producer=Producer.THREAD), + mb_state_inp_done=MBarrier(alloc(2), stages=2, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_state_inp_cg2_done=MBarrier(alloc(2), stages=2, init_count=CG2, producer=Producer.THREAD), + mb_k_decay_inv_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0, producer=Producer.THREAD), + mb_q_decay_k_restore_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0, producer=Producer.THREAD), + mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=2 * WARP, producer=Producer.THREAD), + mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_aqk_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_da_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_dm_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_aqk_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_a_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_da_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dm_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_state_k_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_rhs_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_update_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_u_smem_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_u_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_u_warps_done=MBarrier(alloc(1), stages=1, init_count=2 * WARP, producer=Producer.THREAD), + mb_du_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_du_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_du_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dy_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_neg_dy_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_neg_dy_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_sdy_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_sdy_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_sdy_super_done=MBarrier(alloc(1), stages=1, init_count=WARP, producer=Producer.THREAD), + mb_dstate_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dstate_acc_done=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_dstate_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_dstate_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dstate_smem_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_dstate_smem_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dstate_smem_cg2_done=MBarrier(alloc(1), stages=1, init_count=CG2, producer=Producer.THREAD), + mb_dq_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dk_decay_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dk_inv_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dk_restore_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_parts_done=MBarrier(alloc(1), stages=1, init_count=CG2, producer=Producer.THREAD), + mb_dq_tmastg_ready=MBarrier(alloc(cfg.smem_dq_stages), stages=cfg.smem_dq_stages, init_count=CG2, producer=Producer.THREAD), + mb_dq_tmastg_done=MBarrier(alloc(cfg.smem_dq_stages), stages=cfg.smem_dq_stages, init_count=WARP, producer=Producer.THREAD), + mb_dk_tmastg_ready=MBarrier(alloc(cfg.smem_dk_stages), stages=cfg.smem_dk_stages, init_count=CG2, producer=Producer.THREAD), + mb_dk_tmastg_done=MBarrier(alloc(cfg.smem_dk_stages), stages=cfg.smem_dk_stages, init_count=WARP, producer=Producer.THREAD), + mb_dv_tmastg_ready=MBarrier(alloc(cfg.smem_dv_stages), stages=cfg.smem_dv_stages, init_count=CG1, producer=Producer.THREAD), + mb_dv_tmastg_done=MBarrier(alloc(cfg.smem_dv_stages), stages=cfg.smem_dv_stages, init_count=WARP, producer=Producer.THREAD), + mb_dg_tmastg_ready=MBarrier(alloc(cfg.smem_dg_stages), stages=cfg.smem_dg_stages, init_count=CG2, producer=Producer.THREAD), + mb_dg_tmastg_done=MBarrier(alloc(cfg.smem_dg_stages), stages=cfg.smem_dg_stages, init_count=WARP, producer=Producer.THREAD), + mb_db_tmastg_ready=MBarrier(alloc(cfg.smem_db_stages), stages=cfg.smem_db_stages, init_count=CG2, producer=Producer.THREAD), + mb_db_tmastg_done=MBarrier(alloc(cfg.smem_db_stages), stages=cfg.smem_db_stages, init_count=WARP, producer=Producer.THREAD), + mb_dwo_tmastg_ready=MBarrier(alloc(cfg.smem_dwo_stages), stages=cfg.smem_dwo_stages, init_count=CG1, producer=Producer.THREAD), + mb_dwo_tmastg_done=MBarrier(alloc(cfg.smem_dwo_stages), stages=cfg.smem_dwo_stages, init_count=WARP, producer=Producer.THREAD), + mb_dstate0_stored=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_sched_ready=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=1, producer=Producer.THREAD), + mb_sched_done=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=15, producer=Producer.THREAD), + ) + + +# ---- Dynamic tile scheduler ------------------------------------------------------ + + +@cute.jit +def sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): + """TMA-warp side: pull the next tile off the global ticket, publish it.""" + if cutlass.const_expr(cfg.dyn_sched): + bars.mb_sched_done[sched_state.idx].wait(sched_state.phase) + if nvvm.elect_sync(): + fetched = cutlass.Int32(nvvm.atomicrmw("add", mSched.iterator, cutlass.Int32(1), mem_order="relaxed", syncscope="gpu")) + sSched[sched_state.idx] = num_ctas + fetched + nvvm.bar_warp_sync(cute.arch.FULL_MASK) + next_tile = sSched[sched_state.idx] + if nvvm.elect_sync(): + bars.mb_sched_ready[sched_state.idx].arrive() + return next_tile, advance(sched_state, cfg.sched_stages) + return tile_idx + num_ctas, sched_state + + +@cute.jit +def sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): + """Consumer side: read the TMA warp's published next tile.""" + if cutlass.const_expr(cfg.dyn_sched): + bars.mb_sched_ready[sched_state.idx].wait(sched_state.phase) + next_tile = sSched[sched_state.idx] + if nvvm.elect_sync(): + bars.mb_sched_done[sched_state.idx].arrive() + return next_tile, advance(sched_state, cfg.sched_stages) + return tile_idx + num_ctas, sched_state + + +# ---- Warp bodies ----------------------------------------------------------------- + + +@cute.jit +def epilogue_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_inv_raw, + sQ_decay_raw, + sDo_raw, + sU_raw, + sQk_raw, + sDq_raw, + sDk_raw, + sDv_raw, + sDg_raw, + sDb_raw, + sDwOut_raw, + desc_dq_base, + desc_dk_base, + desc_dv_base, + desc_dg_base, + desc_db_base, + desc_dwo_base, + bars, +) -> None: + """Epilogue warp role (warp 15): register-MMA A_qk/dA tiles and the + gradient TMA stores.""" + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + + # ---- ldmatrix/stmatrix lane decode ------------------------------------------- + rhs_row_coord = lane % 8 + (cutlass.Int32(8) if (lane // 16) else cutlass.Int32(0)) + rhs_col_offset = cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0) + lhs_row_coord = lane % 8 + (cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0)) + lhs_col_offset = cutlass.Int32(8) if ((lane // 8) // 2) else cutlass.Int32(0) + stsm_row_coord = lane & 7 + stsm_col_coord = cutlass.Int32(0) + if (lane // 8) & 1: + stsm_row_coord = stsm_row_coord + cutlass.Int32(8) + if lane // 8 >= 2: + stsm_col_coord = cutlass.Int32(8) + stsm_idx = swizzle_lin_S(stsm_row_coord * cfg.b_t + stsm_col_coord, bbits=1, mbase=3, sshift=3) + row_lo = lane // 4 + row_hi = row_lo + cutlass.Int32(8) + raw_index = PipelineState.start(phase=0) + u_index = PipelineState.start(phase=0) + gbase = cutlass.Int32(0) + + sDq_tma = SmemTile( + base=sDq_raw, + elems_per_stage=(cfg.b_t * cfg.d_k), + stages=cfg.smem_dq_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.b_t * 64, + ) + sDk_tma = SmemTile( + base=sDk_raw, + elems_per_stage=(cfg.b_t * cfg.d_k), + stages=cfg.smem_dk_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.b_t * 64, + ) + sDv_tma = SmemTile( + base=sDv_raw, + elems_per_stage=(cfg.b_t * cfg.d_v), + stages=cfg.smem_dv_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.b_t * 64, + ) + sDg_tma = SmemTile( + base=sDg_raw, + elems_per_stage=(cfg.b_t * cfg.d_k), + stages=cfg.smem_dg_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 32), + tma_granu_elems=32, + tma_subtile_stride_elems=cfg.b_t * 32, + ) + sDb_tma = SmemTile( + base=sDb_raw, + elems_per_stage=(cfg.b_t * cfg.d_k), + stages=cfg.smem_db_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.b_t * 64, + ) + sDwOut_tma = SmemTile( + base=sDwOut_raw, + elems_per_stage=(cfg.b_t * cfg.d_v), + stages=cfg.smem_dwo_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.b_t * 64, + ) + dq_index = PipelineState.start(phase=0) + dk_index = PipelineState.start(phase=0) + dv_index = PipelineState.start(phase=0) + dg_index = PipelineState.start(phase=0) + db_index = PipelineState.start(phase=0) + dwo_index = PipelineState.start(phase=0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + head_o = head_idx + slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_dq_slot = (desc_dq_base + slot).tospace(cutlass.AddressSpace.generic) + desc_dk_slot = (desc_dk_base + slot).tospace(cutlass.AddressSpace.generic) + desc_dv_slot = (desc_dv_base + slot).tospace(cutlass.AddressSpace.generic) + desc_dg_slot = (desc_dg_base + slot).tospace(cutlass.AddressSpace.generic) + desc_db_slot = (desc_db_base + slot).tospace(cutlass.AddressSpace.generic) + desc_dwo_slot = (desc_dwo_base + slot).tospace(cutlass.AddressSpace.generic) + if nvvm.elect_sync(): + tma_tensormap_acquire(desc_dq_slot) + tma_tensormap_acquire(desc_dk_slot) + tma_tensormap_acquire(desc_dv_slot) + tma_tensormap_acquire(desc_dg_slot) + tma_tensormap_acquire(desc_db_slot) + tma_tensormap_acquire(desc_dwo_slot) + sk_nt = cend - wstart + pend_start = cutlass.Int32(0) + pend_writes = cutlass.Boolean(False) + for rev_idx in cutlass.range(sk_nt, unroll=1): + chunk_idx = cend - cutlass.Int32(1) - rev_idx + chunk_start = chunk_idx * cfg.b_t + writes = chunk_idx < wend + gc = gbase + rev_idx + decay_stage = gc % cfg.smem_decay_stages + qk_stage = gc % cfg.smem_qk_stages + raw_stage = raw_index.idx + sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sQ_decay_ptr = sQ_decay_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sDo_ptr = sDo_raw.data_ptr() + raw_stage * (cfg.d_v * cfg.b_t) + sQk_ptr = sQk_raw.data_ptr() + qk_stage * (cfg.qk_tiles * cfg.b_t * cfg.b_t) + + # ---- A_qk = tril_incl(Q_decay @ K_inv^T) ----------------------------- + bars.mb_aqk_done[qk_stage].wait(((gc // cfg.smem_qk_stages) + 1) % 2) + bars.mb_q_decay_k_restore_ready[decay_stage].wait((gc // cfg.smem_decay_stages) % 2) + qk_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + qk_acc[accum_idx] = cutlass.Float32(0.0) + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + a_col = k_block * 16 + lhs_col_offset + a_seg = a_col // 64 + a_vec = nvvm.ldmatrix( + sQ_decay_ptr + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + b_col = k_block * 16 + rhs_col_offset + b_seg = b_col // 64 + b_vec = nvvm.ldmatrix( + sK_inv_ptr + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + mma_step( + qk_acc, + (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), + (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + qk_acc[accum_idx] = qk_acc[accum_idx] if row_coord >= col_coord else cutlass.Float32(0.0) + nvvm.stmatrix( + sQk_ptr + stsm_idx, + [ + fp32_to_fp16(qk_acc[0], qk_acc[1], dtype=cfg.io_dtype), + fp32_to_fp16(qk_acc[2], qk_acc[3], dtype=cfg.io_dtype), + fp32_to_fp16(qk_acc[4], qk_acc[5], dtype=cfg.io_dtype), + fp32_to_fp16(qk_acc[6], qk_acc[7], dtype=cfg.io_dtype), + ], + nvvm.MMALayout.ROW, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_aqk_ready[qk_stage].arrive() + bars.mb_decay_super_done[decay_stage].arrive() + + # ---- dA = tril_incl(dO @ U^T) ---------------------------------------- + bars.mb_da_done[qk_stage].wait(((gc // cfg.smem_qk_stages) + 1) % 2) + bars.mb_u_smem_ready.wait(u_index.phase) + u_index = advance(u_index, 1) + da_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + da_acc[accum_idx] = cutlass.Float32(0.0) + for k_block in cutlass.range_constexpr(cfg.d_v // 16): + a_col = k_block * 16 + lhs_col_offset + a_seg = a_col // 64 + a_vec = nvvm.ldmatrix( + sDo_ptr + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + b_col = k_block * 16 + rhs_col_offset + b_seg = b_col // 64 + b_vec = nvvm.ldmatrix( + sU_raw.data_ptr() + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + mma_step( + da_acc, + (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), + (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + bars.mb_u_warps_done.arrive() + bars.mb_do_epi_done[raw_stage].arrive() + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + da_acc[accum_idx] = da_acc[accum_idx] if row_coord >= col_coord else cutlass.Float32(0.0) + nvvm.stmatrix( + sQk_ptr + 2 * (cfg.b_t * cfg.b_t) + stsm_idx, + [ + fp32_to_fp16(da_acc[0], da_acc[1], dtype=cfg.io_dtype), + fp32_to_fp16(da_acc[2], da_acc[3], dtype=cfg.io_dtype), + fp32_to_fp16(da_acc[4], da_acc[5], dtype=cfg.io_dtype), + fp32_to_fp16(da_acc[6], da_acc[7], dtype=cfg.io_dtype), + ], + nvvm.MMALayout.ROW, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_da_ready[qk_stage].arrive() + raw_index = advance(raw_index, cfg.smem_raw_stages) + + # ---- dQ/dK/dGate/dBeta/dV/dW_out: previous chunk, one-behind store ladder ---- + if rev_idx > 0: + bars.mb_dq_tmastg_ready[dq_index.idx].wait(dq_index.phase) + bars.mb_dk_tmastg_ready[dk_index.idx].wait(dk_index.phase) + bars.mb_dg_tmastg_ready[dg_index.idx].wait(dg_index.phase) + bars.mb_db_tmastg_ready[db_index.idx].wait(db_index.phase) + bars.mb_dv_tmastg_ready[dv_index.idx].wait(dv_index.phase) + bars.mb_dwo_tmastg_ready[dwo_index.idx].wait(dwo_index.phase) + if pend_writes: + dq_slice = tma_slice_runtime_desc(desc_dq_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDq_tma[dq_index.idx], dq_slice, acquire=False) + tma_store_commit() + dk_slice = tma_slice_runtime_desc(desc_dk_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDk_tma[dk_index.idx], dk_slice, acquire=False) + tma_store_commit() + dg_slice = tma_slice_runtime_desc(desc_dg_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDg_tma[dg_index.idx], dg_slice, acquire=False) + tma_store_commit() + db_slice = tma_slice_runtime_desc(desc_db_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDb_tma[db_index.idx], db_slice, acquire=False) + tma_store_commit() + dv_slice = tma_slice_runtime_desc(desc_dv_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDv_tma[dv_index.idx], dv_slice, acquire=False) + tma_store_commit() + dwo_slice = tma_slice_runtime_desc(desc_dwo_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDwOut_tma[dwo_index.idx], dwo_slice, acquire=False) + tma_store_commit() + tma_store_wait(5) + bars.mb_dq_tmastg_done[dq_index.idx].arrive() + tma_store_wait(4) + bars.mb_dk_tmastg_done[dk_index.idx].arrive() + tma_store_wait(3) + bars.mb_dg_tmastg_done[dg_index.idx].arrive() + tma_store_wait(2) + bars.mb_db_tmastg_done[db_index.idx].arrive() + tma_store_wait(1) + bars.mb_dv_tmastg_done[dv_index.idx].arrive() + tma_store_wait(0) + bars.mb_dwo_tmastg_done[dwo_index.idx].arrive() + dq_index = advance(dq_index, cfg.smem_dq_stages) + dk_index = advance(dk_index, cfg.smem_dk_stages) + dg_index = advance(dg_index, cfg.smem_dg_stages) + db_index = advance(db_index, cfg.smem_db_stages) + dv_index = advance(dv_index, cfg.smem_dv_stages) + dwo_index = advance(dwo_index, cfg.smem_dwo_stages) + pend_start = chunk_start + pend_writes = writes + + # ---- tile tail: drain the last chunk's dQ/dK/dGate/dBeta/dV/dW_out ------- + if sk_nt > 0: + bars.mb_dq_tmastg_ready[dq_index.idx].wait(dq_index.phase) + bars.mb_dk_tmastg_ready[dk_index.idx].wait(dk_index.phase) + bars.mb_dg_tmastg_ready[dg_index.idx].wait(dg_index.phase) + bars.mb_db_tmastg_ready[db_index.idx].wait(db_index.phase) + bars.mb_dv_tmastg_ready[dv_index.idx].wait(dv_index.phase) + bars.mb_dwo_tmastg_ready[dwo_index.idx].wait(dwo_index.phase) + if pend_writes: + dq_slice = tma_slice_runtime_desc(desc_dq_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDq_tma[dq_index.idx], dq_slice, acquire=False) + tma_store_commit() + dk_slice = tma_slice_runtime_desc(desc_dk_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDk_tma[dk_index.idx], dk_slice, acquire=False) + tma_store_commit() + dg_slice = tma_slice_runtime_desc(desc_dg_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDg_tma[dg_index.idx], dg_slice, acquire=False) + tma_store_commit() + db_slice = tma_slice_runtime_desc(desc_db_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDb_tma[db_index.idx], db_slice, acquire=False) + tma_store_commit() + dv_slice = tma_slice_runtime_desc(desc_dv_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDv_tma[dv_index.idx], dv_slice, acquire=False) + tma_store_commit() + dwo_slice = tma_slice_runtime_desc(desc_dwo_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDwOut_tma[dwo_index.idx], dwo_slice, acquire=False) + tma_store_commit() + tma_store_wait(5) + bars.mb_dq_tmastg_done[dq_index.idx].arrive() + tma_store_wait(4) + bars.mb_dk_tmastg_done[dk_index.idx].arrive() + tma_store_wait(3) + bars.mb_dg_tmastg_done[dg_index.idx].arrive() + tma_store_wait(2) + bars.mb_db_tmastg_done[db_index.idx].arrive() + tma_store_wait(1) + bars.mb_dv_tmastg_done[dv_index.idx].arrive() + tma_store_wait(0) + bars.mb_dwo_tmastg_done[dwo_index.idx].arrive() + dq_index = advance(dq_index, cfg.smem_dq_stages) + dk_index = advance(dk_index, cfg.smem_dk_stages) + dg_index = advance(dg_index, cfg.smem_dg_stages) + db_index = advance(db_index, cfg.smem_db_stages) + dv_index = advance(dv_index, cfg.smem_dv_stages) + dwo_index = advance(dwo_index, cfg.smem_dwo_stages) + + gbase += sk_nt + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def super_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_decay_raw, + sK_inv_raw, + sU_raw, + sDy_raw, + sQk_raw, + bars, +) -> None: + """Super-MMA warp role (warp 12): builds the Neumann A_inv and + strict-tril dM staging tiles.""" + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + raw_index = PipelineState.start(phase=0) + sdy_index = PipelineState.start(phase=0) + + # ---- ldmatrix lane decode ---------------------------------------------------- + rhs_row_coord = lane % 8 + (cutlass.Int32(8) if (lane // 16) else cutlass.Int32(0)) + rhs_col_offset = cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0) + lhs_row_coord = lane % 8 + (cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0)) + lhs_col_offset = cutlass.Int32(8) if ((lane // 8) // 2) else cutlass.Int32(0) + stsm_row_coord = lane & 7 + stsm_col_coord = cutlass.Int32(0) + if (lane // 8) & 1: + stsm_row_coord = stsm_row_coord + cutlass.Int32(8) + if lane // 8 >= 2: + stsm_col_coord = cutlass.Int32(8) + stsm_idx = swizzle_lin_S(stsm_row_coord * cfg.b_t + stsm_col_coord, bbits=1, mbase=3, sshift=3) + row_lo = lane // 4 + row_hi = row_lo + cutlass.Int32(8) + + gbase = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + sk_nt = cend - wstart + for rev_idx in cutlass.range(sk_nt, unroll=1): + gc = gbase + rev_idx + decay_stage = gc % cfg.smem_decay_stages + qk_stage = gc % cfg.smem_qk_stages + raw_stage = raw_index.idx + sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sK_decay_ptr = sK_decay_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sQk_ptr = sQk_raw.data_ptr() + qk_stage * (cfg.qk_tiles * cfg.b_t * cfg.b_t) + + bars.mb_a_done[qk_stage].wait(((gc // cfg.smem_qk_stages) + 1) % 2) + + # ---- KK = K_decay @ K_inv^T ------------------------------------------ + bars.mb_k_decay_inv_ready[decay_stage].wait((gc // cfg.smem_decay_stages) % 2) + kk_lhs_row = lhs_row_coord + kk_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + kk_acc[accum_idx] = cutlass.Float32(0.0) + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + a_col = k_block * 16 + lhs_col_offset + a_seg = a_col // 64 + a_vec = nvvm.ldmatrix( + sK_decay_ptr + a_seg * (cfg.b_t * 64) + kk_lhs_row * 64 + swizzle_xor_128b(kk_lhs_row, a_col - a_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + b_col = k_block * 16 + rhs_col_offset + b_seg = b_col // 64 + b_vec = nvvm.ldmatrix( + sK_inv_ptr + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + mma_step( + kk_acc, + (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), + (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + + # ---- L = tril(KK, -1) ------------------------------------------------ + l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + lower = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + l_frag[accum_idx] = lower + l_a0 = fp32_to_fp16(l_frag[0], l_frag[1], dtype=cfg.io_dtype) + l_a1 = fp32_to_fp16(l_frag[2], l_frag[3], dtype=cfg.io_dtype) + l_a2 = fp32_to_fp16(l_frag[4], l_frag[5], dtype=cfg.io_dtype) + l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) + l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) + + ainv_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + eye = cutlass.Float32(1.0) if row_coord == col_coord else cutlass.Float32(0.0) + ainv_acc[accum_idx] = eye - l_values[accum_idx] + + lpow_a0, lpow_a1, lpow_a2, lpow_a3 = l_a0, l_a1, l_a2, l_a3 + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(l_a0), movmatrix_16b(l_a1), movmatrix_16b(l_a2), movmatrix_16b(l_a3) + for _round in cutlass.range_constexpr(3): + sq_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + sq_acc[accum_idx] = cutlass.Float32(0.0) + mma_step( + sq_acc, + (lpow_a0, lpow_a1, lpow_a2, lpow_a3), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + lpow_a0 = fp32_to_fp16(sq_acc[0], sq_acc[1], dtype=cfg.io_dtype) + lpow_a1 = fp32_to_fp16(sq_acc[2], sq_acc[3], dtype=cfg.io_dtype) + lpow_a2 = fp32_to_fp16(sq_acc[4], sq_acc[5], dtype=cfg.io_dtype) + lpow_a3 = fp32_to_fp16(sq_acc[6], sq_acc[7], dtype=cfg.io_dtype) + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(lpow_a0), movmatrix_16b(lpow_a1), movmatrix_16b(lpow_a2), movmatrix_16b(lpow_a3) + upd_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + upd_acc[accum_idx] = cutlass.Float32(0.0) + ainv_p0 = fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype) + ainv_p1 = fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype) + ainv_p2 = fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype) + ainv_p3 = fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype) + mma_step( + upd_acc, + (ainv_p0, ainv_p1, ainv_p2, ainv_p3), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + ainv_lo0, ainv_hi0 = f16x2_to_f32(ainv_p0, dtype=cfg.io_dtype) + ainv_lo1, ainv_hi1 = f16x2_to_f32(ainv_p1, dtype=cfg.io_dtype) + ainv_lo2, ainv_hi2 = f16x2_to_f32(ainv_p2, dtype=cfg.io_dtype) + ainv_lo3, ainv_hi3 = f16x2_to_f32(ainv_p3, dtype=cfg.io_dtype) + ainv_acc[0] = ainv_lo0 + upd_acc[0] + ainv_acc[1] = ainv_hi0 + upd_acc[1] + ainv_acc[2] = ainv_lo1 + upd_acc[2] + ainv_acc[3] = ainv_hi1 + upd_acc[3] + ainv_acc[4] = ainv_lo2 + upd_acc[4] + ainv_acc[5] = ainv_hi2 + upd_acc[5] + ainv_acc[6] = ainv_lo3 + upd_acc[6] + ainv_acc[7] = ainv_hi3 + upd_acc[7] + + nvvm.stmatrix( + sQk_ptr + 1 * (cfg.b_t * cfg.b_t) + stsm_idx, + [ + fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype), + ], + nvvm.MMALayout.ROW, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_a_ready[qk_stage].arrive() + bars.mb_decay_super_done[decay_stage].arrive() + + # ---- dM = dY @ U^T --------------------------------------------------- + bars.mb_dm_done[qk_stage].wait(((gc // cfg.smem_qk_stages) + 1) % 2) + bars.mb_sdy_ready.wait(sdy_index.phase) + sdy_index = advance(sdy_index, 1) + dm_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + dm_acc[accum_idx] = cutlass.Float32(0.0) + for k_block in cutlass.range_constexpr(cfg.d_v // 16): + a_col = k_block * 16 + lhs_col_offset + a_seg = a_col // 64 + a_vec = nvvm.ldmatrix( + sDy_raw.data_ptr() + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + b_col = k_block * 16 + rhs_col_offset + b_seg = b_col // 64 + b_vec = nvvm.ldmatrix( + sU_raw.data_ptr() + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + mma_step( + dm_acc, + (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), + (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + bars.mb_u_warps_done.arrive() + bars.mb_sdy_super_done.arrive() + + dm_strict = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + dm_strict[accum_idx] = dm_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + w0 = fp32_to_fp16(dm_strict[0], dm_strict[1], dtype=cfg.io_dtype) + w1 = fp32_to_fp16(dm_strict[2], dm_strict[3], dtype=cfg.io_dtype) + w2 = fp32_to_fp16(dm_strict[4], dm_strict[5], dtype=cfg.io_dtype) + w3 = fp32_to_fp16(dm_strict[6], dm_strict[7], dtype=cfg.io_dtype) + nvvm.stmatrix(sQk_ptr + 3 * (cfg.b_t * cfg.b_t) + stsm_idx, [w0, w1, w2, w3], nvvm.MMALayout.ROW, shape=nvvm.StoreShape.M8N8) + nw0 = fp32_to_fp16(-dm_strict[0], -dm_strict[1], dtype=cfg.io_dtype) + nw1 = fp32_to_fp16(-dm_strict[2], -dm_strict[3], dtype=cfg.io_dtype) + nw2 = fp32_to_fp16(-dm_strict[4], -dm_strict[5], dtype=cfg.io_dtype) + nw3 = fp32_to_fp16(-dm_strict[6], -dm_strict[7], dtype=cfg.io_dtype) + nvvm.stmatrix(sQk_ptr + 4 * (cfg.b_t * cfg.b_t) + stsm_idx, [nw0, nw1, nw2, nw3], nvvm.MMALayout.ROW, shape=nvvm.StoreShape.M8N8) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dm_ready[qk_stage].arrive() + raw_index = advance(raw_index, cfg.smem_raw_stages) + gbase += sk_nt + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def tcgen05_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + tmem_hold, + sState_alt, + sK_decay_lead16, + sK_inv_lead16, + sK_inv_amaj, + sK_restore_lead16, + sDo_lead16, + sDo_amaj, + sQ_decay_trans, + sK_decay_trans, + sU_lead16, + sDy_lead16, + sDstate_alt, + sQk, + sState_scale_diag, + bars, +) -> None: + """tcgen05-MMA warp role (warp 13): issues every tcgen05 GEMM and owns + the TMEM lifecycle.""" + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + tmem_base = tmem_hold.load() + bpe = cfg.io_dtype.width // 8 + + # ---- chunk-invariant GEMM descriptors ---------------------------------------- + idesc_mv_nt = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + ) + idesc_state_k_at = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + a_major=1, + ) + bmm_state_k_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.d_k, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + atranspose=True, + cta_group=1, + idesc=idesc_state_k_at, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bmm_dvinter_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.d_k, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_mv_nt, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_du_at = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + a_major=1, + b_major=1, + ) + bmm_du_at_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + atranspose=True, + cta_group=1, + idesc=idesc_du_at, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dstate_q_at = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.d_k, + m_dim=cfg.d_v, + a_major=1, + b_major=1, + ) + bmm_dstate_q_at_desc = MmaDesc( + M=cfg.d_v, + N=cfg.d_k, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + atranspose=True, + cta_group=1, + idesc=idesc_dstate_q_at, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bmm_qk_ts_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_mv_nt, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_mv_nt_t = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + b_major=1, + ) + bmm_qk_ts_t_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + cta_group=1, + idesc=idesc_mv_nt_t, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_diag = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=16, + m_dim=cfg.d_v, + ) + bmm_diag_desc = MmaDesc( + M=cfg.d_v, + N=16, + K=16, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_diag, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dstate_k = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.d_k, + m_dim=cfg.d_v, + b_major=1, + ) + bmm_dstate_k_desc = MmaDesc( + M=cfg.d_v, + N=cfg.d_k, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + cta_group=1, + idesc=idesc_dstate_k, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_state_ts = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_k, + ) + bmm_state_desc = MmaDesc( + M=cfg.d_k, + N=cfg.b_t, + K=cfg.d_v, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_state_ts, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bmm_state_ts_desc = MmaDesc( + M=cfg.d_k, + N=cfg.b_t, + K=cfg.d_v, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_state_ts, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dstate_at = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_k, + a_major=1, + ) + bmm_dstate_at_desc = MmaDesc( + M=cfg.d_k, + N=cfg.b_t, + K=cfg.d_v, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + atranspose=True, + cta_group=1, + idesc=idesc_dstate_at, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dgp = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_k, + ) + bmm_dgrad_ts_desc = MmaDesc( + M=cfg.d_k, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_dgp, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dgp_at = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_k, + a_major=1, + ) + bmm_dgrad_at_desc = MmaDesc( + M=cfg.d_k, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + atranspose=True, + cta_group=1, + idesc=idesc_dgp_at, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dgp_at_t = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_k, + a_major=1, + b_major=1, + ) + bmm_dgrad_at_t_desc = MmaDesc( + M=cfg.d_k, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + atranspose=True, + cta_group=1, + idesc=idesc_dgp_at_t, + kind=nvvm.Tcgen05MMAKind.F16, + ) + + state_index = PipelineState.start(phase=0) + rhs_index = PipelineState.start(phase=0) + dstate_inp_index = PipelineState.start(phase=0) + dstate_acc_done_index = PipelineState.start(phase=1) + du_inp_index = PipelineState.start(phase=0) + neg_dy_index = PipelineState.start(phase=0) + u_smem_index = PipelineState.start(phase=0) + dstate_smem_index = PipelineState.start(phase=0) + parts_done_index = PipelineState.start(phase=1) + + dstate0_index = PipelineState.start(phase=0) + + gbase = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + sk_nt = cend - wstart + for rev_idx in cutlass.range(sk_nt, unroll=1): + gc = gbase + rev_idx + decay_stage = gc % cfg.smem_decay_stages + qk_stage = gc % cfg.smem_qk_stages + decay_phase = (gc // cfg.smem_decay_stages) % 2 + qk_phase = (gc // cfg.smem_qk_stages) % 2 + has_dstate = cutlass.Boolean(rev_idx > 0) + if cutlass.const_expr(cfg.use_dstate_in): + has_dstate = cutlass.Boolean(True) + raw_stage_idx = gc % cfg.smem_raw_stages + + sQk_ptr = sQk[qk_stage] + chunk_idx = cend - cutlass.Int32(1) - rev_idx + + # ---- state_k = state(S) @ K_decay^T ---------------------------------- + bars.mb_k_decay_inv_ready[decay_stage].wait(decay_phase) + if chunk_idx >= FIRST_STATE_CHUNK: + bars.mb_state_ready[state_index.idx].wait(state_index.phase) + mma_ss( + bmm_state_k_desc, + sState_alt[state_index.idx].desc(), + sK_decay_lead16[decay_stage].desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_state_k_acc_offset), cutlass.Float32), + accumulate=False, + ) + if nvvm.elect_sync(): + bars.mb_state_k_acc_ready.arrive(cta_group=1) + bars.mb_state_done[state_index.idx].arrive(cta_group=1) + state_index = advance(state_index, cfg.smem_state_stages) + + # ---- dQ inter = state(T) @ dO^T -------------------------------------- + bars.mb_parts_done.wait(parts_done_index.phase) + parts_done_index = advance(parts_done_index, 1) + bars.mb_state_inp_ready[gc % 2].wait((gc // 2) % 2) + bars.mb_do_ready[raw_stage_idx].wait((gc // cfg.smem_raw_stages) % 2) + if chunk_idx >= FIRST_STATE_CHUNK: + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_state_inp_offset + (gc % 2) * (cfg.d_v // 2)), cutlass.Int8) + b_desc = sDo_lead16[raw_stage_idx].desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dq_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_state_ts_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_state_ts_desc.sps_B): + mma_ts_step( + bmm_state_ts_desc, + a_ptr.subview(sub * bmm_state_ts_desc.sps_B * bmm_state_ts_desc.tmem_advance_A), + b_desc + sub * (bmm_state_ts_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + + # ---- dU inter = dstate_inp(T) @ K_restore ---------------------------- + bars.mb_q_decay_k_restore_ready[decay_stage].wait(decay_phase) + if has_dstate: + bars.mb_dstate_inp_ready.wait(dstate_inp_index.phase) + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dstate_inp_offset), cutlass.Int8) + b_desc = sK_restore_lead16[decay_stage].desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_du_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_dvinter_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_dvinter_desc.sps_B): + mma_ts_step( + bmm_dvinter_desc, + a_ptr.subview(sub * bmm_dvinter_desc.sps_B * bmm_dvinter_desc.tmem_advance_A), + b_desc + sub * (bmm_dvinter_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + + # ---- dU intra += dO^T(S) @ A_qk -------------------------------------- + bars.mb_aqk_ready[qk_stage].wait(qk_phase) + mma_ss( + bmm_du_at_desc, + sDo_amaj[raw_stage_idx].desc(), + sQk_ptr.desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_du_acc_offset), cutlass.Float32), + accumulate=has_dstate, + ) + if nvvm.elect_sync(): + bars.mb_du_acc_ready.arrive(cta_group=1) + bars.mb_aqk_done[qk_stage].arrive(cta_group=1) + + # ---- dstate decay = dstate_inp(T) @ diag(eGl) ------------------------ + bars.mb_dstate_acc_done.wait(dstate_acc_done_index.phase) + dstate_acc_done_index = advance(dstate_acc_done_index, 1) + if has_dstate: + desc_diag = sState_scale_diag[decay_stage].desc() + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dstate_inp_offset) + k_block * 8, cutlass.Int8) + b_desc = desc_diag.advance_start_address(k_block * 256 * 2) + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dstate_acc_offset) + k_block * 16, cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_diag_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_diag_desc.sps_B): + mma_ts_step( + bmm_diag_desc, + a_ptr.subview(sub * bmm_diag_desc.sps_B * bmm_diag_desc.tmem_advance_A), + b_desc + sub * (bmm_diag_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + if nvvm.elect_sync(): + bars.mb_dstate_inp_done.arrive(cta_group=1) + dstate_inp_index = advance(dstate_inp_index, 1) + + # ---- dstate q-term += dO^T(S) @ Q_decay ------------------------------ + mma_ss( + bmm_dstate_q_at_desc, + sDo_amaj[raw_stage_idx].desc(), + sQ_decay_trans[decay_stage].desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dstate_acc_offset), cutlass.Float32), + accumulate=has_dstate, + ) + + # ---- U = rhs(T) @ A_inv ---------------------------------------------- + bars.mb_a_ready[qk_stage].wait(qk_phase) + bars.mb_rhs_ready.wait(rhs_index.phase) + rhs_index = advance(rhs_index, 1) + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_rhs_inp_offset), cutlass.Int8) + b_desc = sQk_ptr.shifted(1 * (cfg.b_t * cfg.b_t)).desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_update_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_qk_ts_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_qk_ts_desc.sps_B): + mma_ts_step( + bmm_qk_ts_desc, + a_ptr.subview(sub * bmm_qk_ts_desc.sps_B * bmm_qk_ts_desc.tmem_advance_A), + b_desc + sub * (bmm_qk_ts_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + if nvvm.elect_sync(): + bars.mb_update_acc_ready.arrive(cta_group=1) + bars.mb_do_done[raw_stage_idx].arrive(cta_group=1) + + # ---- dY = dU(T) @ A_inv ---------------------------------------------- + bars.mb_du_inp_ready.wait(du_inp_index.phase) + du_inp_index = advance(du_inp_index, 1) + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_du_inp_offset), cutlass.Int8) + b_desc = sQk_ptr.shifted(1 * (cfg.b_t * cfg.b_t)).desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dy_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_qk_ts_t_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_qk_ts_t_desc.sps_B): + mma_ts_step( + bmm_qk_ts_t_desc, + a_ptr.subview(sub * bmm_qk_ts_t_desc.sps_B * bmm_qk_ts_t_desc.tmem_advance_A), + b_desc + sub * (bmm_qk_ts_t_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + if nvvm.elect_sync(): + bars.mb_dy_acc_ready.arrive(cta_group=1) + bars.mb_du_inp_done.arrive(cta_group=1) + bars.mb_a_done[qk_stage].arrive(cta_group=1) + + # ---- dK_restore part = dstate(S) @ U^T ------------------------------- + bars.mb_u_smem_ready.wait(u_smem_index.phase) + u_smem_index = advance(u_smem_index, 1) + if has_dstate: + bars.mb_dstate_smem_ready.wait(dstate_smem_index.phase) + dstate_smem_index = advance(dstate_smem_index, 1) + mma_ss( + bmm_dstate_at_desc, + sDstate_alt[0].desc(), + sU_lead16[0].desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_restore_acc_offset), cutlass.Float32), + accumulate=False, + ) + if nvvm.elect_sync(): + bars.mb_dk_restore_part_ready.arrive(cta_group=1) + bars.mb_dstate_smem_done.arrive(cta_group=1) + if nvvm.elect_sync(): + bars.mb_u_done.arrive(cta_group=1) + + # ---- dstate k-term += -dY(T) @ K_decay ------------------------------- + bars.mb_neg_dy_inp_ready.wait(neg_dy_index.phase) + neg_dy_index = advance(neg_dy_index, 1) + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_neg_dy_inp_offset), cutlass.Int8) + b_desc = sK_decay_trans[decay_stage].desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dstate_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_dstate_k_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_dstate_k_desc.sps_B): + mma_ts_step( + bmm_dstate_k_desc, + a_ptr.subview(sub * bmm_dstate_k_desc.sps_B * bmm_dstate_k_desc.tmem_advance_A), + b_desc + sub * (bmm_dstate_k_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(True), + ) + if nvvm.elect_sync(): + bars.mb_dstate_acc_ready.arrive(cta_group=1) + bars.mb_neg_dy_inp_done.arrive(cta_group=1) + + # ---- dK_inv part = scale.Q_decay^T(S) @ dA --------------------------- + bars.mb_da_ready[qk_stage].wait(qk_phase) + mma_ss( + bmm_dgrad_at_t_desc, + sQ_decay_trans[decay_stage].desc(), + sQk_ptr.shifted(2 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), + accumulate=False, + ) + + # ---- dQ attn += K_inv^T(S) @ dA^T ------------------------------------ + if chunk_idx >= FIRST_STATE_CHUNK: + mma_ss( + bmm_dgrad_at_desc, + sK_inv_amaj[decay_stage].desc(), + sQk_ptr.shifted(2 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dq_acc_offset), cutlass.Float32), + accumulate=True, + ) + if chunk_idx < FIRST_STATE_CHUNK: + mma_ss( + bmm_dgrad_at_desc, + sK_inv_amaj[decay_stage].desc(), + sQk_ptr.shifted(2 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dq_acc_offset), cutlass.Float32), + accumulate=False, + ) + if nvvm.elect_sync(): + bars.mb_dq_acc_ready.arrive(cta_group=1) + bars.mb_da_done[qk_stage].arrive(cta_group=1) + + # ---- dK_decay part = state(T) @ dY^T --------------------------------- + if chunk_idx >= FIRST_STATE_CHUNK: + bars.mb_sdy_ready.wait(gc % 2) + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_state_inp_offset + (gc % 2) * (cfg.d_v // 2)), cutlass.Int8) + b_desc = sDy_lead16[0].desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_decay_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_state_ts_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_state_ts_desc.sps_B): + mma_ts_step( + bmm_state_ts_desc, + a_ptr.subview(sub * bmm_state_ts_desc.sps_B * bmm_state_ts_desc.tmem_advance_A), + b_desc + sub * (bmm_state_ts_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + if nvvm.elect_sync(): + bars.mb_sdy_done.arrive(cta_group=1) + bars.mb_state_inp_done[gc % 2].arrive(cta_group=1) + + # ---- dK_inv part += K_decay^T(S) @ -dM_strict ------------------------ + bars.mb_dm_ready[qk_stage].wait(qk_phase) + mma_ss( + bmm_dgrad_at_t_desc, + sK_decay_trans[decay_stage].desc(), + sQk_ptr.shifted(4 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), + accumulate=True, + ) + if nvvm.elect_sync(): + bars.mb_dk_inv_part_ready.arrive(cta_group=1) + + # ---- dK_decay part += K_inv^T(S) @ dM_strict^T ----------------------- + if chunk_idx >= FIRST_STATE_CHUNK: + mma_ss( + bmm_dgrad_at_desc, + sK_inv_amaj[decay_stage].desc(), + sQk_ptr.shifted(3 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), + accumulate=True, + ) + if chunk_idx < FIRST_STATE_CHUNK: + mma_ss( + bmm_dgrad_at_desc, + sK_inv_amaj[decay_stage].desc(), + sQk_ptr.shifted(3 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), + accumulate=False, + ) + if nvvm.elect_sync(): + bars.mb_dk_decay_part_ready.arrive(cta_group=1) + bars.mb_dm_done[qk_stage].arrive(cta_group=1) + bars.mb_decay_done[decay_stage].arrive(cta_group=1) + + # ---- tile end: WG1's dstate0 drain gates the next tile's dstate reuse ---- + bars.mb_dstate0_stored.wait(dstate0_index.phase) + dstate0_index = advance(dstate0_index, 1) + gbase += sk_nt + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + nvvm.tcgen05_dealloc( + nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), + cutlass.Int32(512), + group=nvvm.CTAGroup.CTA_1, + ) + + +@cute.jit +def tmaldg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + mSched, + sSched, + q_tx_bytes, + k_tx_bytes, + gate_tx_bytes, + beta_tx_bytes, + do_tx_bytes, + v_tx_bytes, + w_tx_bytes, + lane, + sQ_raw, + sK_raw, + sV_raw, + sGate_raw, + sDo_raw, + sBeta_raw, + sW_raw, + sState_raw, + desc_q_base, + desc_k_base, + desc_v_base, + desc_gate_base, + desc_do_base, + desc_beta_base, + desc_w_base, + desc_checkpoint_base, + desc_initial_state_base, + bars, +) -> None: + """TMA-LDG warp role (warp 14): persistent scheduler loop issuing every + G->S operand load.""" + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + + sQ_tma = SmemTile( + base=sQ_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sK_tma = SmemTile( + base=sK_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sV_tma = SmemTile( + base=sV_raw, + elems_per_stage=(cfg.d_v * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sGate_tma = SmemTile( + base=sGate_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 32), + tma_granu_elems=32, + tma_subtile_stride_elems=(cfg.b_t * 32), + ) + sDo_tma = SmemTile( + base=sDo_raw, + elems_per_stage=(cfg.d_v * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sBeta_tma = SmemTile( + base=sBeta_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sW_tma = SmemTile( + base=sW_raw, + elems_per_stage=(cfg.d_v * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sState_tma = SmemTile( + base=sState_raw, + elems_per_stage=(cfg.d_k * cfg.d_v), + stages=cfg.smem_state_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.d_k * 64, + ) + raw_index = PipelineState.start(phase=1) + state_index = PipelineState.start(phase=1) + sched_state = PipelineState.start(phase=1) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + next_tile, sched_state = sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) + head_o = head_idx + head_q = head_idx if cfg.q_ratio == 1 else head_idx // cutlass.Int32(cfg.q_ratio) + head_k = head_idx if cfg.k_ratio == 1 else head_idx // cutlass.Int32(cfg.k_ratio) + head_v = head_idx if cfg.v_ratio == 1 else head_idx // cutlass.Int32(cfg.v_ratio) + slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_q_slot = (desc_q_base + slot).tospace(cutlass.AddressSpace.generic) + desc_k_slot = (desc_k_base + slot).tospace(cutlass.AddressSpace.generic) + desc_v_slot = (desc_v_base + slot).tospace(cutlass.AddressSpace.generic) + desc_gate_slot = (desc_gate_base + slot).tospace(cutlass.AddressSpace.generic) + desc_do_slot = (desc_do_base + slot).tospace(cutlass.AddressSpace.generic) + desc_beta_slot = (desc_beta_base + slot).tospace(cutlass.AddressSpace.generic) + desc_w_slot = (desc_w_base + slot).tospace(cutlass.AddressSpace.generic) + desc_checkpoint_slot = (desc_checkpoint_base + slot).tospace(cutlass.AddressSpace.generic) + desc_initial_state_slot = (desc_initial_state_base + cutlass.Int32(0)).tospace(cutlass.AddressSpace.generic) + if nvvm.elect_sync(): + tma_tensormap_acquire(desc_q_slot) + tma_tensormap_acquire(desc_k_slot) + tma_tensormap_acquire(desc_v_slot) + tma_tensormap_acquire(desc_gate_slot) + tma_tensormap_acquire(desc_do_slot) + tma_tensormap_acquire(desc_beta_slot) + tma_tensormap_acquire(desc_w_slot) + tma_tensormap_acquire(desc_checkpoint_slot) + if cutlass.const_expr(cfg.use_initial_state): + tma_tensormap_acquire(desc_initial_state_slot) + sk_nt = cend - wstart + for rev_idx in cutlass.range(sk_nt, unroll=1): + chunk_idx = cend - cutlass.Int32(1) - rev_idx + chunk_start = chunk_idx * cfg.b_t + + # ---- Q load ---------------------------------------------------------- + bars.mb_q_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_q_ready[raw_index.idx].arrive(n_bytes=q_tx_bytes) + q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), head_q, chunk_start) + tma_load_tile(sQ_tma[raw_index.idx], q_slice, bars.mb_q_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- K load ---------------------------------------------------------- + bars.mb_k_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_k_ready[raw_index.idx].arrive(n_bytes=k_tx_bytes) + k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, chunk_start) + tma_load_tile(sK_tma[raw_index.idx], k_slice, bars.mb_k_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- Gate load ------------------------------------------------------- + bars.mb_gate_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_gate_ready[raw_index.idx].arrive(n_bytes=gate_tx_bytes) + gate_slice = tma_slice_runtime_desc(desc_gate_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sGate_tma[raw_index.idx], gate_slice, bars.mb_gate_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- Beta load ------------------------------------------------------- + bars.mb_beta_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_beta_ready[raw_index.idx].arrive(n_bytes=beta_tx_bytes) + beta_slice = tma_slice_runtime_desc(desc_beta_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sBeta_tma[raw_index.idx], beta_slice, bars.mb_beta_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- entering state: checkpoint[c - 1], or initial_state for chunk 0 when given -- + if chunk_idx >= FIRST_STATE_CHUNK: + state_idx = state_index.idx + bars.mb_state_done[state_idx].wait(state_index.phase) + bars.mb_state_cg0_done[state_idx].wait(state_index.phase) + state_index = advance(state_index, cfg.smem_state_stages) + if nvvm.elect_sync(): + bars.mb_state_ready[state_idx].arrive(n_bytes=cfg.tma_state_bytes) + if cutlass.const_expr(cfg.use_initial_state): + if chunk_idx == 0: + initial_state_slice = tma_slice_runtime_desc(desc_initial_state_slot, cutlass.Int32(0), cutlass.Int32(0), head_o, batch_idx) + tma_load_tile(sState_tma[state_idx], initial_state_slice, bars.mb_state_ready[state_idx].smem_ptr, acquire=False) + else: + state_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), chunk_idx - cutlass.Int32(1), head_o) + tma_load_tile(sState_tma[state_idx], state_slice, bars.mb_state_ready[state_idx].smem_ptr, acquire=False) + else: + state_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), chunk_idx - FIRST_STATE_CHUNK, head_o) + tma_load_tile(sState_tma[state_idx], state_slice, bars.mb_state_ready[state_idx].smem_ptr, acquire=False) + + # ---- dO load --------------------------------------------------------- + bars.mb_do_done[raw_index.idx].wait(raw_index.phase) + bars.mb_do_epi_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_do_ready[raw_index.idx].arrive(n_bytes=do_tx_bytes) + do_slice = tma_slice_runtime_desc(desc_do_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sDo_tma[raw_index.idx], do_slice, bars.mb_do_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- V load ---------------------------------------------------------- + bars.mb_v_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_v_ready[raw_index.idx].arrive(n_bytes=v_tx_bytes) + v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, chunk_start) + tma_load_tile(sV_tma[raw_index.idx], v_slice, bars.mb_v_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- W load ---------------------------------------------------------- + bars.mb_w_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_w_ready[raw_index.idx].arrive(n_bytes=w_tx_bytes) + w_slice = tma_slice_runtime_desc(desc_w_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sW_tma[raw_index.idx], w_slice, bars.mb_w_ready[raw_index.idx].smem_ptr, acquire=False) + raw_index = advance(raw_index, cfg.smem_raw_stages) + tile_idx = next_tile + + +@cute.jit +def compute0_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_hold, + warp_idx, + scale, + sK_inv_raw, + sGate_raw, + sK_raw, + sQ_raw, + sState_raw, + sV_raw, + sDo_raw, + sBeta_raw, + sW_raw, + sNorm_raw, + sK_decay_raw, + sK_restore_raw, + sQ_decay_raw, + sState_scale_diag_raw, + bars, +) -> None: + """WG0 warp role (warps 0-3): persistent tile-scheduler loop + gate prefix + scan and the decay/restore operand materialization into tcgen05 SMEM for + EVERY chunk (no ping-pong: the backward pipeline is drain-bound). Also + stashes the per-row q/k inverse norms for WG2's dG assembly and copies + H -> TMEM f16 at the chunk tail.""" + nvvm.setmaxregister(cfg.num_regs_compute_group_0, nvvm.SetMaxRegisterAction.INCREASE) + cg0_warp = warp_idx - cfg.compute_group_0_warp_ids[0] + tmem_base = tmem_hold.load() + tmem_col = tmem_base & 0xFFFF + tmem_row = tmem_base >> 16 + tmem_sp = warp_idx % (cfg.d_v // cfg.threads_per_warp) + value_dim = tmem_sp * cfg.threads_per_warp + lane + state_copy_addr = (tmem_row + tmem_sp * cfg.threads_per_warp) << 16 + state_index = PipelineState.start(phase=0) + gbase = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + sk_nt = cend - wstart + for rev_idx in cutlass.range(sk_nt, unroll=1): + chunk_idx = cend - cutlass.Int32(1) - rev_idx + gc = gbase + rev_idx + chunk_start = chunk_idx * cfg.b_t + decay_stage = gc % cfg.smem_decay_stages + raw_stage = gc % cfg.smem_raw_stages + sQ_ptr = sQ_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sK_ptr = sK_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sV_ptr = sV_raw.data_ptr() + raw_stage * (cfg.d_v * cfg.b_t) + sDo_ptr = sDo_raw.data_ptr() + raw_stage * (cfg.d_v * cfg.b_t) + sBetaP_ptr = sBeta_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sWP_ptr = sW_raw.data_ptr() + raw_stage * (cfg.d_v * cfg.b_t) + sGate_ptr = sGate_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sK_decay_ptr = sK_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sQ_decay_ptr = sQ_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sK_restore_ptr = sK_restore_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sState_scale_diag_ptr = sState_scale_diag_raw.data_ptr() + decay_stage * ((cfg.d_k // 16) * 256) + + bars.mb_gate_ready[raw_stage].wait((gc // cfg.smem_raw_stages) % 2) + bars.mb_q_ready[raw_stage].wait((gc // cfg.smem_raw_stages) % 2) + bars.mb_k_ready[raw_stage].wait((gc // cfg.smem_raw_stages) % 2) + bars.mb_beta_ready[raw_stage].wait((gc // cfg.smem_raw_stages) % 2) + + row_group_start = cg0_warp * (cfg.b_t // len(cfg.compute_group_0_warp_ids)) + lane_row_group = lane // 8 + lane_in_row_group = lane - lane_row_group * 8 + decay_row = row_group_start + lane_row_group + + g_prefix_ptr = sGate_ptr + prefix_dim = cg0_warp * cfg.threads_per_warp + lane + # ---- gate prefix scan: cumulative log-gate per key channel ----------- + gate_raw = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + for row in cutlass.range_constexpr(cfg.b_t): + f32_segment = prefix_dim // 32 + f32_segment_dim = prefix_dim - f32_segment * 32 + prefix_idx = f32_segment * (cfg.b_t * 32) + row * 32 + swizzle_xor_128b(row, f32_segment_dim, elem_bytes=4) + gate_raw[row] = (sGate_ptr + prefix_idx).load() + g_prefix_regs = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + for row in cutlass.range_constexpr(cfg.b_t): + gate = gate_raw[row] + token_idx = chunk_idx * cutlass.Int32(cfg.b_t) + cutlass.Int32(row) + if token_idx < seqlen_b: + gate = gate * cutlass.Float32(LOG2_E) + else: + gate = cutlass.Float32(0.0) + g_prefix_regs[row] = gate + + prefix_acc = cutlass.Float32(0.0) + for row_pair in cutlass.range_constexpr(cfg.b_t // 2): + row0 = row_pair * 2 + row1 = row0 + 1 + gate0 = g_prefix_regs[row0] + gate1 = g_prefix_regs[row1] + pair_vec = nvvm.add_packed_f32x2( + cutlass.Vector.from_elements((prefix_acc, gate0), cutlass.Float32), + cutlass.Vector.from_elements((gate0, gate1), cutlass.Float32), + ftz=False, + rnd="rn", + ) + prefix0, row_pair_sum = cutlass.Float32(pair_vec[0]), cutlass.Float32(pair_vec[1]) + prefix1 = prefix_acc + row_pair_sum + g_prefix_regs[row0] = prefix0 + g_prefix_regs[row1] = prefix1 + prefix_acc = prefix1 + + for row in cutlass.range_constexpr(cfg.b_t): + g_prefix_regs[row] = cute.math.exp2(g_prefix_regs[row], fastmath=True) + + exp_g_last = g_prefix_regs[cfg.b_t - 1] + # ---- decay-slot guard: previous use fully consumed ------------------- + operand_done_phase = ((gc // cfg.smem_decay_stages) + 1) % 2 + bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) + + for row in cutlass.range_constexpr(cfg.b_t): + f32_segment = prefix_dim // 32 + f32_segment_dim = prefix_dim - f32_segment * 32 + prefix_idx = f32_segment * (cfg.b_t * 32) + row * 32 + swizzle_xor_128b(row, f32_segment_dim, elem_bytes=4) + (sGate_ptr + prefix_idx).store(g_prefix_regs[row]) + + # ---- state-scale diag: stage exp2(g_last) decay blocks --------------- + block = prefix_dim // cutlass.Int32(16) + coord = prefix_dim - block * cutlass.Int32(16) + linear_idx = block * cutlass.Int32(256) + coord * cutlass.Int32(16) + coord + diag_idx = swizzle_lin_S(linear_idx, bbits=1, mbase=3, sshift=3) + sState_scale_diag_ptr[diag_idx] = exp_g_last.to(cfg.io_dtype) + + nvvm.barrier_cta_sync(cfg.cg0_sync_barrier_id, thread_count=cfg.cg0_threads) + + k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) + raw_q_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + raw_beta_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + # ---- optional q/k L2-norm + K_inv staging ---------------------------- + if cutlass.const_expr(cfg.l2norm): + qk0_lo = opaque_f32_zero() + qk0_hi = opaque_f32_zero() + qk1_lo = opaque_f32_zero() + qk1_hi = opaque_f32_zero() + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + raw_f16_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) + raw_q_vec = (sQ_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_k_vec = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_beta_vec = (sBetaP_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_q_vec_f32 = raw_q_vec.to(cutlass.Float32) + raw_k_vec_f32 = raw_k_vec.to(cutlass.Float32) + raw_beta_vec_f32 = raw_beta_vec.to(cutlass.Float32) + for dim_offset in cutlass.range_constexpr(8): + q_val = raw_q_vec_f32[dim_offset] + k_val = raw_k_vec_f32[dim_offset] + raw_q_regs[reg_base + dim_offset] = q_val + raw_k_regs[reg_base + dim_offset] = k_val + raw_beta_regs[reg_base + dim_offset] = raw_beta_vec_f32[dim_offset] + if cutlass.const_expr(cfg.l2norm): + if cutlass.const_expr(dim_offset % 2 == 0): + qk0_lo, qk0_hi = ffma2(q_val, k_val, q_val, k_val, qk0_lo, qk0_hi) + else: + qk1_lo, qk1_hi = ffma2(q_val, k_val, q_val, k_val, qk1_lo, qk1_hi) + + q_inv_norm = opaque_f32_zero() + cutlass.Float32(1.0) + k_inv_norm = opaque_f32_zero() + cutlass.Float32(1.0) + if cutlass.const_expr(cfg.l2norm): + q_sum_sq = qk0_lo + qk1_lo + k_sum_sq = qk0_hi + qk1_hi + q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 4, 31, kind=nvvm.Shfl.BFLY)) + q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 2, 31, kind=nvvm.Shfl.BFLY)) + q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 1, 31, kind=nvvm.Shfl.BFLY)) + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 4, 31, kind=nvvm.Shfl.BFLY)) + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 2, 31, kind=nvvm.Shfl.BFLY)) + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 1, 31, kind=nvvm.Shfl.BFLY)) + norm_floor_sq = cutlass.Float32(L2_NORM_EPS * L2_NORM_EPS) + q_inv_norm = cute.math.rsqrt(cute.math.max(q_sum_sq, norm_floor_sq), fastmath=True) + k_inv_norm = cute.math.rsqrt(cute.math.max(k_sum_sq, norm_floor_sq), fastmath=True) + if lane_in_row_group == 0: + sNorm_raw[raw_stage * (2 * cfg.b_t) + decay_row] = q_inv_norm + sNorm_raw[raw_stage * (2 * cfg.b_t) + cfg.b_t + decay_row] = k_inv_norm + q_stage_norm = q_inv_norm * scale + + # ---- decay/restore operands: exp2(+-g) applied per key channel ------- + exp_g_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + exp_g_last_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + for f32_group in cutlass.range_constexpr(2): + f32_dim_base = dim_base + f32_group * 4 + f32_segment = f32_dim_base // 32 + f32_segment_dim = f32_dim_base - f32_segment * 32 + g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) + exp_g_vec = (g_prefix_ptr + g_prefix_idx).load(count=4, alignment=16) + exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) + exp_g_last_vec = (g_prefix_ptr + exp_g_last_idx).load(count=4, alignment=16) + f32_reg_base = reg_base + f32_group * 4 + exp_g_regs[f32_reg_base] = exp_g_vec[0] + exp_g_regs[f32_reg_base + 1] = exp_g_vec[1] + exp_g_regs[f32_reg_base + 2] = exp_g_vec[2] + exp_g_regs[f32_reg_base + 3] = exp_g_vec[3] + exp_g_last_regs[f32_reg_base] = exp_g_last_vec[0] + exp_g_last_regs[f32_reg_base + 1] = exp_g_last_vec[1] + exp_g_last_regs[f32_reg_base + 2] = exp_g_last_vec[2] + exp_g_last_regs[f32_reg_base + 3] = exp_g_last_vec[3] + + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + # ---- k decay + k inv operands: exp2(g) * beta * k / exp2(-g) * k - + k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + for pair_idx in cutlass.range_constexpr(4): + dim0 = pair_idx * 2 + dim1 = dim0 + 1 + raw_reg_idx0 = reg_base + dim0 + raw_reg_idx1 = reg_base + dim1 + k_value0, k_value1 = fmul2(raw_k_regs[raw_reg_idx0], raw_k_regs[raw_reg_idx1], k_inv_norm, k_inv_norm) + k_beta0, k_beta1 = fmul2(k_value0, k_value1, raw_beta_regs[raw_reg_idx0], raw_beta_regs[raw_reg_idx1]) + k_pair = fp32_to_fp16(k_beta0, k_beta1, dtype=cfg.io_dtype) + exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) + k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) + exp_neg_g0 = cute.math.rcp(exp_g_regs[raw_reg_idx0], approx=True, ftz=True) + exp_neg_g1 = cute.math.rcp(exp_g_regs[raw_reg_idx1], approx=True, ftz=True) + exp_neg_pair = fp32_to_fp16(exp_neg_g0, exp_neg_g1, dtype=cfg.io_dtype) + k_norm_pair = fp32_to_fp16(k_value0, k_value1, dtype=cfg.io_dtype) + k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_norm_pair, exp_neg_pair, cfg.io_dtype) + + k_inv_vec = cutlass.Vector.from_elements( + ( + k_inv_words[dim_half * 4], + k_inv_words[dim_half * 4 + 1], + k_inv_words[dim_half * 4 + 2], + k_inv_words[dim_half * 4 + 3], + ), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + k_decay_vec = cutlass.Vector.from_elements( + (k_decay_words[0], k_decay_words[1], k_decay_words[2], k_decay_words[3]), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + op_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) + (sK_inv_ptr + op_idx).store(k_inv_vec, alignment=16) + (sK_decay_ptr + op_idx).store(k_decay_vec, alignment=16) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_k_decay_inv_ready[decay_stage].arrive() + + # ---- q decay + k_restore operands ------------------------------------ + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + q_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_restore_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + for pair_idx in cutlass.range_constexpr(4): + dim0 = pair_idx * 2 + dim1 = dim0 + 1 + raw_reg_idx0 = reg_base + dim0 + raw_reg_idx1 = reg_base + dim1 + q_value0, q_value1 = fmul2(raw_q_regs[raw_reg_idx0], raw_q_regs[raw_reg_idx1], q_stage_norm, q_stage_norm) + q_pair = fp32_to_fp16(q_value0, q_value1, dtype=cfg.io_dtype) + exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) + q_decay_words[pair_idx] = mul_f16x2(q_pair, exp_g_pair, cfg.io_dtype) + exp_g_last_pair = fp32_to_fp16(exp_g_last_regs[raw_reg_idx0], exp_g_last_regs[raw_reg_idx1], dtype=cfg.io_dtype) + k_restore_words[pair_idx] = mul_f16x2(k_inv_words[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) + + q_decay_vec = cutlass.Vector.from_elements( + (q_decay_words[0], q_decay_words[1], q_decay_words[2], q_decay_words[3]), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + k_restore_vec = cutlass.Vector.from_elements( + (k_restore_words[0], k_restore_words[1], k_restore_words[2], k_restore_words[3]), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + op_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) + (sQ_decay_ptr + op_idx).store(q_decay_vec, alignment=16) + (sK_restore_ptr + op_idx).store(k_restore_vec, alignment=16) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_q_decay_k_restore_ready[decay_stage].arrive() + bars.mb_q_done[raw_stage].arrive() + bars.mb_k_done[raw_stage].arrive() + bars.mb_gate_done[raw_stage].arrive() + bars.mb_beta_done[raw_stage].arrive() + + # ---- state copy: SMEM -> TMEM f16 ------------------------------------ + bars.mb_state_inp_done[gc % 2].wait(((gc // 2) + 1) % 2) + bars.mb_state_inp_cg2_done[gc % 2].wait(((gc // 2) + 1) % 2) + if chunk_idx >= FIRST_STATE_CHUNK: + bars.mb_state_ready[state_index.idx].wait(state_index.phase) + state_src = sState_raw.data_ptr() + state_index.idx * (cfg.d_k * cfg.d_v) + for pl in cutlass.range_constexpr(2): + for g8 in cutlass.range_constexpr(8): + state_vec = (state_src + pl * (cfg.d_k * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, g8 * 8, elem_bytes=2)).load( + count=8, alignment=16 + ) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr( + state_copy_addr + (tmem_col + cfg.tmem_state_inp_offset + (gc % 2) * (cfg.d_v // 2) + pl * 32 + g8 * 4), cutlass.Int8 + ), + state_vec.bitcast(cutlass.Int32), + ) + nvvm.tcgen05_wait("store") + bars.mb_state_cg0_done[state_index.idx].arrive() + state_index = advance(state_index, cfg.smem_state_stages) + bars.mb_state_inp_ready[gc % 2].arrive() + gbase += sk_nt + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def compute1_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_hold, + warp_idx, + mDstate0, + mDstate_in, + sV_raw, + sW_raw, + sDo_raw, + sU_raw, + sDy_raw, + sDv_raw, + sDwOut_raw, + sDstate_raw, + bars, +) -> None: + """WG1 warp role (warps 4-7): the value-side TMEM staging. Per chunk: + rhs W = w*v - state_k -> TMEM f16 (prefill's staging); + U readback -> sU; dU restage -> TMEM f16; dY + readback -> sdV(+beta.dY) / sdY(plain) / TMEM(-beta.dY) + the dbeta + v-term dot and store; end-of-chunk dH capture (dh acc -> dh_inp f16 + + sdH); tile edges: dht seeding and the dS0 drain.""" + nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) + tmem_base = tmem_hold.load() + tmem_col = tmem_base & 0xFFFF + tmem_row = tmem_base >> 16 + tmem_sp = warp_idx % (cfg.d_v // cfg.threads_per_warp) + ov_tok = (lane // 16) * 8 + (lane & 7) + ov_col = ((lane // 8) & 1) * 8 + value_dim = tmem_sp * cfg.threads_per_warp + lane + value_dim_base = tmem_sp * cfg.threads_per_warp + cg1_tidx = warp_idx % 4 * cfg.threads_per_warp + lane + + raw_index = PipelineState.start(phase=0) + state_k_index = PipelineState.start(phase=0) + update_index = PipelineState.start(phase=0) + du_acc_index = PipelineState.start(phase=0) + dy_acc_index = PipelineState.start(phase=0) + du_inp_done_index = PipelineState.start(phase=1) + neg_dy_done_index = PipelineState.start(phase=1) + u_done_index = PipelineState.start(phase=1) + sdy_done_index = PipelineState.start(phase=1) + dstate_ready_index = PipelineState.start(phase=0) + dstate_inp_done_index = PipelineState.start(phase=1) + dstate_smem_done_index = PipelineState.start(phase=1) + dv_done_index = PipelineState.start(phase=1) + dwo_done_index = PipelineState.start(phase=1) + + gbase = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + sk_nt = cend - wstart + + # ---- dht seeding: dh acc + dh_inp f16 + sdH ------------------------------ + if cutlass.const_expr(cfg.use_dstate_in): + if sk_nt > 0: + seed_true = cend == num_chunks_b + bars.mb_dstate_inp_done.wait(dstate_inp_done_index.phase) + dstate_inp_done_index = advance(dstate_inp_done_index, 1) + bars.mb_dstate_smem_done.wait(dstate_smem_done_index.phase) + bars.mb_dstate_smem_cg2_done.wait(dstate_smem_done_index.phase) + dstate_smem_done_index = advance(dstate_smem_done_index, 1) + row_addr = (tmem_row + tmem_sp * cfg.threads_per_warp) << 16 + for sub in cutlass.range_constexpr(cfg.d_k // 16): + seed_block = cutlass.Array(cutlass.Float32, 16, alignment=16) + for kk_i in cutlass.range_constexpr(16): + dval = mDstate_in[batch_idx, head_idx, sub * 16 + kk_i, value_dim].to(cutlass.Float32) + if cutlass.const_expr(cfg.split_k): + dval = dval if seed_true else cutlass.Float32(0.0) + seed_block[kk_i] = dval + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_acc_offset + sub * 16), cutlass.Float32), + seed_block[0:16], + ) + packed_seed = cutlass.Array(cutlass.Int32, 8, alignment=16) + for pc in cutlass.range_constexpr(8): + packed_seed[pc] = fp32_to_fp16(seed_block[2 * pc], seed_block[2 * pc + 1], dtype=cfg.io_dtype) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_inp_offset + sub * 8), cutlass.Int8), + packed_seed[0:8], + ) + for half in cutlass.range_constexpr(2): + d_base = sub * 16 + half * 8 + h_vec = cutlass.Vector.from_elements( + (packed_seed[half * 4], packed_seed[half * 4 + 1], packed_seed[half * 4 + 2], packed_seed[half * 4 + 3]), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + h_addr = (d_base // 64) * (cfg.d_v * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, d_base % 64, elem_bytes=2) + (sDstate_raw.data_ptr() + h_addr).store(h_vec, alignment=16) + nvvm.tcgen05_wait("store") + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dstate_inp_ready.arrive() + bars.mb_dstate_smem_ready.arrive() + + for rev_idx in cutlass.range(sk_nt, unroll=1): + chunk_idx = cend - cutlass.Int32(1) - rev_idx + gc = gbase + rev_idx + has_dstate = cutlass.Boolean(rev_idx > 0) + if cutlass.const_expr(cfg.use_dstate_in): + has_dstate = cutlass.Boolean(True) + + sV_ptr = sV_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) + sW_ptr = sW_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) + sDo_ptr = sDo_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) + row_addr_lo = tmem_row << 16 + row_addr_hi = (tmem_row + 16) << 16 + row_id0 = tmem_row + value_dim_base + row_id1 = row_id0 + 16 + + bars.mb_v_ready[raw_index.idx].wait((gc // cfg.smem_raw_stages) % 2) + bars.mb_w_ready[raw_index.idx].wait((gc // cfg.smem_raw_stages) % 2) + + # ---- rhs staging: W = w*V - state_k -> TMEM f16 ---------------------- + projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset + input_col_id = tmem_col + cfg.tmem_rhs_inp_offset + raw_v_regs0 = nvvm.ldmatrix( + sV_ptr + + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) + + ov_tok * 64 + + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + raw_v_regs1 = nvvm.ldmatrix( + sV_ptr + + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) + + ov_tok * 64 + + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + raw_w_regs0 = nvvm.ldmatrix( + sW_ptr + + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) + + ov_tok * 64 + + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + raw_w_regs1 = nvvm.ldmatrix( + sW_ptr + + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) + + ov_tok * 64 + + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + + packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + if chunk_idx >= FIRST_STATE_CHUNK: + bars.mb_state_k_acc_ready.wait(state_k_index.phase) + state_k_index = advance(state_k_index, 1) + state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_pair = fp32_to_fp16(state_k0[frag_pair], state_k0[frag_pair + 1], dtype=cfg.io_dtype) + wv_pair = mul_f16x2(raw_w_regs0[raw_matrix], raw_v_regs0[raw_matrix], cfg.io_dtype) + packed_rhs0[reg_idx ^ 2] = sub_f16x2(wv_pair, state_k_pair, cfg.io_dtype) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_pair = fp32_to_fp16(state_k1[frag_pair], state_k1[frag_pair + 1], dtype=cfg.io_dtype) + wv_pair = mul_f16x2(raw_w_regs1[raw_matrix], raw_v_regs1[raw_matrix], cfg.io_dtype) + packed_rhs1[reg_idx ^ 2] = sub_f16x2(wv_pair, state_k_pair, cfg.io_dtype) + if chunk_idx < FIRST_STATE_CHUNK: + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + packed_rhs0[reg_idx ^ 2] = mul_f16x2(raw_w_regs0[raw_matrix], raw_v_regs0[raw_matrix], cfg.io_dtype) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + packed_rhs1[reg_idx ^ 2] = mul_f16x2(raw_w_regs1[raw_matrix], raw_v_regs1[raw_matrix], cfg.io_dtype) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + input_col_id, cutlass.Int8), packed_rhs0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + input_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_wait("store") + bars.mb_rhs_ready.arrive() + + # ---- dU restage: dU acc -> TMEM f16 A operand ------------------------ + bars.mb_du_acc_ready.wait(du_acc_index.phase) + du_acc_index = advance(du_acc_index, 1) + bars.mb_du_inp_done.wait(du_inp_done_index.phase) + du_inp_done_index = advance(du_inp_done_index, 1) + du_col_id = tmem_col + cfg.tmem_du_acc_offset + du0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + du_col_id, cutlass.Float32), num=2) + du1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + du_col_id, cutlass.Float32), num=2) + + du_packed0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + du_packed1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + frag_pair = reg_idx * 2 + du_packed0[reg_idx] = fp32_to_fp16(du0[frag_pair], du0[frag_pair + 1], dtype=cfg.io_dtype) + du_packed1[reg_idx] = fp32_to_fp16(du1[frag_pair], du1[frag_pair + 1], dtype=cfg.io_dtype) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_packed0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_packed1[0:4]) + nvvm.tcgen05_wait("store") + bars.mb_du_inp_ready.arrive() + + # ---- U readback -> sU ------------------------------------------------ + bars.mb_update_acc_ready.wait(update_index.phase) + update_index = advance(update_index, 1) + bars.mb_u_done.wait(u_done_index.phase) + bars.mb_u_warps_done.wait(u_done_index.phase) + u_done_index = advance(u_done_index, 1) + u_col_id = tmem_col + cfg.tmem_update_acc_offset + u0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + u_col_id, cutlass.Float32), num=2) + u1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + u_col_id, cutlass.Float32), num=2) + + u_words0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + u_words1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + u_words0[reg_idx] = fp32_to_fp16(u0[2 * reg_idx], u0[2 * reg_idx + 1], dtype=cfg.io_dtype) + u_words1[reg_idx] = fp32_to_fp16(u1[2 * reg_idx], u1[2 * reg_idx + 1], dtype=cfg.io_dtype) + nvvm.stmatrix( + sU_raw.data_ptr() + + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) + + ov_tok * 64 + + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + u_words0.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.stmatrix( + sU_raw.data_ptr() + + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) + + ov_tok * 64 + + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + u_words1.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_u_smem_ready.arrive() + + # ---- dY readback ------------------------------------------------------- + bars.mb_dy_acc_ready.wait(dy_acc_index.phase) + dy_acc_index = advance(dy_acc_index, 1) + dy_col_id = tmem_col + cfg.tmem_dy_acc_offset + dy0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + dy_col_id, cutlass.Float32), num=2) + dy1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + dy_col_id, cutlass.Float32), num=2) + + # ---- dY -> sdY: pack + store + publish (super warp's dM operand) ------- + addr_lo0 = (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) + ov_tok * 64 + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2) + addr_lo1 = ( + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) + + ov_tok * 64 + + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2) + ) + dy_p0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + dy_p1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + dy_p0[reg_idx] = fp32_to_fp16(dy0[2 * reg_idx], dy0[2 * reg_idx + 1], dtype=cfg.io_dtype) + dy_p1[reg_idx] = fp32_to_fp16(dy1[2 * reg_idx], dy1[2 * reg_idx + 1], dtype=cfg.io_dtype) + bars.mb_sdy_done.wait(sdy_done_index.phase) + bars.mb_sdy_super_done.wait(sdy_done_index.phase) + sdy_done_index = advance(sdy_done_index, 1) + nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo0, dy_p0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) + nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo1, dy_p1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) + bars.mb_sdy_ready.arrive() + + # ---- -dY -> TMEM: A operand of the dH ds-term -------------------------- + neg_dy0 = cutlass.Array(cutlass.Float32, 8, alignment=16) + neg_dy1 = cutlass.Array(cutlass.Float32, 8, alignment=16) + for e in cutlass.range_constexpr(8): + neg_dy0[e] = -dy0[e] + neg_dy1[e] = -dy1[e] + neg_dy_p0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + neg_dy_p1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + frag_pair = reg_idx * 2 + neg_dy_p0[reg_idx] = fp32_to_fp16(neg_dy0[frag_pair], neg_dy0[frag_pair + 1], dtype=cfg.io_dtype) + neg_dy_p1[reg_idx] = fp32_to_fp16(neg_dy1[frag_pair], neg_dy1[frag_pair + 1], dtype=cfg.io_dtype) + bars.mb_neg_dy_inp_done.wait(neg_dy_done_index.phase) + neg_dy_done_index = advance(neg_dy_done_index, 1) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_neg_dy_inp_offset), cutlass.Int8), neg_dy_p0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_neg_dy_inp_offset), cutlass.Int8), neg_dy_p1[0:4]) + nvvm.tcgen05_wait("store") + bars.mb_neg_dy_inp_ready.arrive() + + # ---- scalar pass over own sdY: dV staging ---------------------------- + dv_stage = gc % cfg.smem_dv_stages + bars.mb_dv_tmastg_done[dv_stage].wait(dv_done_index.phase) + dv_done_index = advance(dv_done_index, cfg.smem_dv_stages) + dwo_stage = gc % cfg.smem_dwo_stages + bars.mb_dwo_tmastg_done[dwo_stage].wait(dwo_done_index.phase) + dwo_done_index = advance(dwo_done_index, cfg.smem_dwo_stages) + nvvm.barrier_cta_sync(cfg.cg1_sync_barrier_id, thread_count=cfg.cg1_threads) + sdv_stage_base = dv_stage * (cfg.b_t * cfg.d_v) + sdwo_stage_base = dwo_stage * (cfg.b_t * cfg.d_v) + c_seg = value_dim // 64 + c_dim = value_dim - c_seg * 64 + for t in cutlass.range_constexpr(cfg.b_t): + idx = c_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, c_dim, elem_bytes=2) + dy_v = (sDy_raw.data_ptr() + idx).load().to(cutlass.Float32) + w_v = (sW_ptr + idx).load().to(cutlass.Float32) + v_v = (sV_ptr + idx).load().to(cutlass.Float32) + (sDv_raw.data_ptr() + sdv_stage_base + idx).store((w_v * dy_v).to(cfg.io_dtype)) + (sDwOut_raw.data_ptr() + sdwo_stage_base + idx).store((v_v * dy_v).to(cfg.io_dtype)) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dv_tmastg_ready[dv_stage].arrive() + bars.mb_dwo_tmastg_ready[dwo_stage].arrive() + bars.mb_v_done[raw_index.idx].arrive() + bars.mb_w_done[raw_index.idx].arrive() + + # ---- dH capture for the next ----------------------------------------- + bars.mb_dstate_acc_ready.wait(dstate_ready_index.phase) + dstate_ready_index = advance(dstate_ready_index, 1) + if rev_idx + cutlass.Int32(1) < sk_nt: + bars.mb_dstate_inp_done.wait(dstate_inp_done_index.phase) + dstate_inp_done_index = advance(dstate_inp_done_index, 1) + bars.mb_dstate_smem_done.wait(dstate_smem_done_index.phase) + bars.mb_dstate_smem_cg2_done.wait(dstate_smem_done_index.phase) + dstate_smem_done_index = advance(dstate_smem_done_index, 1) + row_addr = (tmem_row + tmem_sp * cfg.threads_per_warp) << 16 + for sub in cutlass.range_constexpr(cfg.d_k // 32): + dstate_block = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_acc_offset + sub * 32), cutlass.Float32), num=32 + ) + packed_dstate = cutlass.Array(cutlass.Int32, 16, alignment=16) + for pc in cutlass.range_constexpr(16): + packed_dstate[pc] = fp32_to_fp16(dstate_block[2 * pc], dstate_block[2 * pc + 1], dtype=cfg.io_dtype) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_inp_offset + sub * 16), cutlass.Int8), + packed_dstate[0:16], + ) + for half in cutlass.range_constexpr(4): + d_base = sub * 32 + half * 8 + h_vec = cutlass.Vector.from_elements( + (packed_dstate[half * 4], packed_dstate[half * 4 + 1], packed_dstate[half * 4 + 2], packed_dstate[half * 4 + 3]), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + h_addr = (d_base // 64) * (cfg.d_v * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, d_base % 64, elem_bytes=2) + (sDstate_raw.data_ptr() + h_addr).store(h_vec, alignment=16) + nvvm.tcgen05_wait("store") + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dstate_inp_ready.arrive() + bars.mb_dstate_smem_ready.arrive() + bars.mb_dstate_acc_done.arrive() + raw_index = advance(raw_index, cfg.smem_raw_stages) + + # ---- tile end: dS0 drain / zero-length pass-through ---------------------- + if cutlass.const_expr(mDstate0 is not None): + if sk_nt > 0: + if wstart == 0: + row_addr = (tmem_row + tmem_sp * cfg.threads_per_warp) << 16 + for sub in cutlass.range_constexpr(cfg.d_k // 32): + dstate0_vec = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_acc_offset + sub * 32), cutlass.Float32), num=32 + ) + for kk_i in cutlass.range_constexpr(32): + mDstate0[batch_idx, head_idx, sub * 32 + kk_i, value_dim] = dstate0_vec[kk_i] + else: + for key_dim_base in cutlass.range_constexpr(0, cfg.d_k, 32): + for kk_i in cutlass.range_constexpr(32): + kd = key_dim_base + kk_i + if cutlass.const_expr(cfg.use_dstate_in): + mDstate0[batch_idx, head_idx, kd, value_dim] = mDstate_in[batch_idx, head_idx, kd, value_dim] + else: + mDstate0[batch_idx, head_idx, kd, value_dim] = cutlass.Float32(0.0) + bars.mb_dstate0_stored.arrive() + gbase += sk_nt + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def compute2_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_hold, + warp_idx, + sQ_raw, + sK_raw, + sBeta_raw, + sGate_raw, + sNorm_raw, + sDq_raw, + sDk_raw, + sRed1_raw, + sDstate_raw, + sDg_raw, + sDb_raw, + scale, + bars, +) -> None: + """WG2 warp role (warps 8-11): the gradient drain. Each thread owns one + key channel d for all 16 tokens: reads the dq accumulator and the four dk + parts from TMEM, assembles dq/dk with the per-channel gate factors, + applies the in-kernel L2-norm backward row projection, assembles the + per-channel dG including the g_last terms, reverse-cumsums it in + registers, stages dgate and db for the epilogue's TMA stores, and + stages dq/dk for the + epilogue's TMA stores.""" + nvvm.setmaxregister(cfg.num_regs_compute_group_2, nvvm.SetMaxRegisterAction.INCREASE) + tmem_base = tmem_hold.load() + tmem_col = tmem_base & 0xFFFF + tmem_row = tmem_base >> 16 + wg1_sp = warp_idx % 4 + channel = wg1_sp * cfg.threads_per_warp + lane + row_addr = (tmem_row + wg1_sp * cfg.threads_per_warp) << 16 + cg2_tidx = channel + + raw_index = PipelineState.start(phase=0) + dq_acc_index = PipelineState.start(phase=0) + dk_decay_part_index = PipelineState.start(phase=0) + dk_inv_part_index = PipelineState.start(phase=0) + dk_restore_part_index = PipelineState.start(phase=0) + dgl_dstate_smem_index = PipelineState.start(phase=0) + gbase = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + sk_nt = cend - wstart + for rev_idx in cutlass.range(sk_nt, unroll=1): + chunk_idx = cend - cutlass.Int32(1) - rev_idx + gc = gbase + rev_idx + chunk_start = chunk_idx * cfg.b_t + raw_stage = gc % cfg.smem_raw_stages + decay_stage = gc % cfg.smem_decay_stages + has_dstate = cutlass.Boolean(rev_idx > 0) + if cutlass.const_expr(cfg.use_dstate_in): + has_dstate = cutlass.Boolean(True) + sQ_ptr = sQ_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sK_ptr = sK_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sBetaP_ptr = sBeta_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sGate_ptr = sGate_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + writes = chunk_idx < wend + + bars.mb_k_decay_inv_ready[decay_stage].wait((gc // cfg.smem_decay_stages) % 2) + + # ---- per-channel gate factors ---------------------------------------- + f32_seg = channel // 32 + f32_dim = channel - f32_seg * 32 + f16_seg = channel // 64 + f16_dim = channel - f16_seg * 64 + eg = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + for t in cutlass.range_constexpr(cfg.b_t): + eg[t] = (sGate_ptr + f32_seg * (cfg.b_t * 32) + t * 32 + swizzle_xor_128b(t, f32_dim, elem_bytes=4)).load() + egl = eg[cfg.b_t - 1] + + # ---- dGlast hdot: sum_v sdH[v, c] * S0[c, v] ------------------------- + dgl_val = cutlass.Float32(0.0) + if has_dstate: + bars.mb_dstate_smem_ready.wait(dgl_dstate_smem_index.phase) + dgl_dstate_smem_index = advance(dgl_dstate_smem_index, 1) + bars.mb_state_inp_ready[gc % 2].wait((gc // 2) % 2) + for pl in cutlass.range_constexpr(2): + for row_half in cutlass.range_constexpr(2): + state_words = nvvm.tcgen05_ld( + "32x32b", + nvvm.make_tmem_ptr( + row_addr + (tmem_col + cfg.tmem_state_inp_offset + (gc % 2) * (cfg.d_v // 2) + pl * 32 + row_half * 16), cutlass.Float32 + ), + num=16, + ) + hacc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for i in cutlass.range_constexpr(8): + hacc[i] = opaque_f32_zero() + for j in cutlass.range_constexpr(16): + v0 = pl * 64 + row_half * 32 + 2 * j + state_pair = cutlass.Vector.from_elements((state_words[j],), cutlass.Float32).bitcast(cfg.io_dtype) + dstate_addr0 = (channel // 64) * (cfg.d_v * 64) + v0 * 64 + swizzle_xor_128b(v0, channel % 64, elem_bytes=2) + dstate_addr1 = (channel // 64) * (cfg.d_v * 64) + (v0 + 1) * 64 + swizzle_xor_128b(v0 + 1, channel % 64, elem_bytes=2) + hval0 = (sDstate_raw.data_ptr() + dstate_addr0).load().to(cutlass.Float32) + hval1 = (sDstate_raw.data_ptr() + dstate_addr1).load().to(cutlass.Float32) + hacc[(2 * j) % 8] = hacc[(2 * j) % 8] + hval0 * state_pair[0].to(cutlass.Float32) + hacc[(2 * j + 1) % 8] = hacc[(2 * j + 1) % 8] + hval1 * state_pair[1].to(cutlass.Float32) + part_a = (hacc[0] + hacc[4]) + (hacc[1] + hacc[5]) + part_b = (hacc[2] + hacc[6]) + (hacc[3] + hacc[7]) + dgl_val = dgl_val + (part_a + part_b) + bars.mb_dstate_smem_cg2_done.arrive() + bars.mb_state_inp_cg2_done[gc % 2].arrive() + + # ---- part-drain accumulators ------------------------------------------- + dq_n = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + dk_n = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + db_reg = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + dg_reg = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + dgl_acc = cutlass.Array(cutlass.Float32, 4, alignment=16) + for i in cutlass.range_constexpr(4): + dgl_acc[i] = opaque_f32_zero() + for t in cutlass.range_constexpr(cfg.b_t): + dk_n[t] = cutlass.Float32(0.0) + + # ---- dk_restore part drain: (eGl/eG) scale + dGlast k-dot -------------- + if has_dstate: + bars.mb_dk_restore_part_ready.wait(dk_restore_part_index.phase) + dk_restore_part_index = advance(dk_restore_part_index, 1) + dk_restore_part = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_restore_acc_offset), cutlass.Float32), num=cfg.b_t + ) + for t in cutlass.range_constexpr(cfg.b_t): + dk_hat = egl * cute.math.rcp(eg[t], approx=True, ftz=True) * dk_restore_part[t] + dk_n[t] = dk_hat + k_v = (sK_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) + if cutlass.const_expr(cfg.l2norm): + k_v = k_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + cfg.b_t + t] + dgl_acc[t % 4] = dgl_acc[t % 4] + k_v * dk_hat + + # ---- dq acc drain: eG.scale --------------------------------------------- + bars.mb_dq_acc_ready.wait(dq_acc_index.phase) + dq_acc_index = advance(dq_acc_index, 1) + dq_acc = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dq_acc_offset), cutlass.Float32), num=cfg.b_t) + for t2 in cutlass.range_constexpr(cfg.b_t // 2): + t = 2 * t2 + es_lo, es_hi = fmul2(eg[t], eg[t + 1], scale, scale) + dq_n[t], dq_n[t + 1] = fmul2(es_lo, es_hi, dq_acc[t], dq_acc[t + 1]) + + # ---- dk_inv part drain: (dA - dM) term, 1/eG scale ---------------------- + bars.mb_dk_inv_part_ready.wait(dk_inv_part_index.phase) + dk_inv_part_index = advance(dk_inv_part_index, 1) + dk_inv_part = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), num=cfg.b_t) + for t in cutlass.range_constexpr(cfg.b_t): + dk_n[t] = dk_n[t] + dk_inv_part[t] * cute.math.rcp(eg[t], approx=True, ftz=True) + + # ---- dk_decay part drain: -eG scale, seeds db and dG -------------------- + bars.mb_dk_decay_part_ready.wait(dk_decay_part_index.phase) + dk_decay_part_index = advance(dk_decay_part_index, 1) + dk_decay_part = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), num=cfg.b_t) + for t in cutlass.range_constexpr(cfg.b_t): + dk_decay = -eg[t] * dk_decay_part[t] + k_v = (sK_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) + if cutlass.const_expr(cfg.l2norm): + k_v = k_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + cfg.b_t + t] + db_reg[t] = k_v * dk_decay + dg_reg[t] = (sBetaP_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) * dk_decay + dk_n[t] = dk_n[t] + dg_reg[t] + + # ---- all parts drained: release the accs to the next chunk's MMAs ------ + bars.mb_parts_done.arrive() + + # ---- dG finalize ----------------------------------------------------- + for t in cutlass.range_constexpr(cfg.b_t): + q_v = (sQ_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) + k_v = (sK_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) + if cutlass.const_expr(cfg.l2norm): + q_v = q_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + t] + k_v = k_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + cfg.b_t + t] + dg_reg[t] = ( + q_v * dq_n[t] + + (sBetaP_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) * db_reg[t] + - k_v * (dk_n[t] - dg_reg[t]) + ) + dg_reg[cfg.b_t - 1] = dg_reg[cfg.b_t - 1] + ((dgl_acc[0] + dgl_acc[1]) + (dgl_acc[2] + dgl_acc[3])) + + # ---- L2-norm backward row projection --------------------------------- + if cutlass.const_expr(cfg.l2norm): + for grad, src_ptr, inv_off in ((dq_n, sQ_ptr, 0), (dk_n, sK_ptr, cfg.b_t)): + dots = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + for t in cutlass.range_constexpr(cfg.b_t): + qn_v = (src_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) + dots[t] = grad[t] * qn_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + inv_off + t] + for off in cutlass.range_constexpr(5): + step = cutlass.const_expr(1 << off) + for t in cutlass.range_constexpr(cfg.b_t): + dots[t] = dots[t] + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, dots[t], step, 31, kind=nvvm.Shfl.BFLY)) + if lane == 0: + for t in cutlass.range_constexpr(cfg.b_t): + sRed1_raw[wg1_sp * cfg.b_t + t] = dots[t] + nvvm.barrier_cta_sync(cfg.cg2_sync_barrier_id, thread_count=cfg.cg2_threads) + for t in cutlass.range_constexpr(cfg.b_t): + qn_v = (src_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) + total_dot = sRed1_raw[t] + sRed1_raw[cfg.b_t + t] + sRed1_raw[2 * cfg.b_t + t] + sRed1_raw[3 * cfg.b_t + t] + norm_t = sNorm_raw[raw_stage * (2 * cfg.b_t) + inv_off + t] + grad[t] = (grad[t] - qn_v * norm_t * total_dot) * norm_t + nvvm.barrier_cta_sync(cfg.cg2_sync_barrier_id, thread_count=cfg.cg2_threads) + + # ---- stage dq/dk for the epilogue TMA stores ------------------------- + dq_stage = gc % cfg.smem_dq_stages + dk_stage = gc % cfg.smem_dk_stages + bars.mb_dq_tmastg_done[dq_stage].wait(((gc // cfg.smem_dq_stages) + 1) % 2) + bars.mb_dk_tmastg_done[dk_stage].wait(((gc // cfg.smem_dk_stages) + 1) % 2) + dq_base = dq_stage * (cfg.b_t * cfg.d_k) + dk_base = dk_stage * (cfg.b_t * cfg.d_k) + for t in cutlass.range_constexpr(cfg.b_t): + out_idx = f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2) + (sDq_raw.data_ptr() + dq_base + out_idx).store(dq_n[t].to(cfg.io_dtype)) + (sDk_raw.data_ptr() + dk_base + out_idx).store(dk_n[t].to(cfg.io_dtype)) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dq_tmastg_ready[dq_stage].arrive() + bars.mb_dk_tmastg_ready[dk_stage].arrive() + + # ---- dGlast add ------------------------------------------------------ + if has_dstate: + if chunk_idx >= FIRST_STATE_CHUNK: + dg_reg[cfg.b_t - 1] = dg_reg[cfg.b_t - 1] + egl * dgl_val + + # ---- dG reverse cumsum ----------------------------------------------- + suffix = cutlass.Float32(0.0) + for rt in cutlass.range_constexpr(cfg.b_t): + t = cfg.b_t - 1 - rt + suffix = suffix + dg_reg[t] + dg_reg[t] = suffix + + # ---- stage dGate + db for the epilogue TMA stores -------------------- + dg_stage = gc % cfg.smem_dg_stages + bars.mb_dg_tmastg_done[dg_stage].wait(((gc // cfg.smem_dg_stages) + 1) % 2) + db_stage = gc % cfg.smem_db_stages + bars.mb_db_tmastg_done[db_stage].wait(((gc // cfg.smem_db_stages) + 1) % 2) + for t in cutlass.range_constexpr(cfg.b_t): + dg_idx = f32_seg * (cfg.b_t * 32) + t * 32 + swizzle_xor_128b(t, f32_dim, elem_bytes=4) + (sDg_raw.data_ptr() + dg_idx).store(dg_reg[t]) + db_idx = f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2) + (sDb_raw.data_ptr() + db_idx).store(db_reg[t].to(cfg.io_dtype)) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dg_tmastg_ready[dg_stage].arrive() + bars.mb_db_tmastg_ready[db_stage].arrive() + + bars.mb_q_done[raw_stage].arrive() + bars.mb_k_done[raw_stage].arrive() + bars.mb_gate_done[raw_stage].arrive() + bars.mb_beta_done[raw_stage].arrive() + raw_index = advance(raw_index, cfg.smem_raw_stages) + gbase += sk_nt + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +# --------------------------------------------------------------------------- +# Host-side assembly +# --------------------------------------------------------------------------- + + +@cute.kernel +def build_all_descs_kernel( + base_q: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_k: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_v: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_gate: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_do: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_beta: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_w: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_dq: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_dk: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_dv: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_dg: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_dwo: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_dbo: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_checkpoint: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_initial_state: cutlass.GridConstant[cuda.tensor_map.TensorMap], + desc_ws: cute.Tensor, + cu_seqlens: cute.Tensor, + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + gate: cute.Tensor, + do: cute.Tensor, + beta: cute.Tensor, + w: cute.Tensor, + dq: cute.Tensor, + dk: cute.Tensor, + dv: cute.Tensor, + dg: cute.Tensor, + dwo: cute.Tensor, + dbo: cute.Tensor, + state_checkpoints: cute.Tensor, + initial_state: cute.Tensor | None, + n_batch: cutlass.Int32, + q_rs: cutlass.Int32, + k_rs: cutlass.Int32, + v_rs: cutlass.Int32, + g_rs: cutlass.Int32, + do_rs: cutlass.Int32, + beta_rs: cutlass.Int32, + w_rs: cutlass.Int32, + dq_rs: cutlass.Int32, + dk_rs: cutlass.Int32, + dv_rs: cutlass.Int32, + dg_rs: cutlass.Int32, + dwo_rs: cutlass.Int32, + dbo_rs: cutlass.Int32, + checkpoint_rs: cutlass.Int32, + checkpoint_every_n: cutlass.Int32, +) -> None: + """Single-launch builder for the per-batch TMA-descriptor arrays (one + warp per array).""" + tidx, _, _ = cute.arch.thread_idx() + widx = cutlass.Int32(tidx) // cutlass.Int32(32) + arr_words = n_batch * cutlass.Int32(TENSOR_MAP_QWORDS) + sub0 = cute.make_tensor(desc_ws.iterator, cute.make_layout((arr_words,), stride=(1,))) + sub1 = cute.make_tensor(desc_ws.iterator + arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub2 = cute.make_tensor(desc_ws.iterator + 2 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub3 = cute.make_tensor(desc_ws.iterator + 3 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub4 = cute.make_tensor(desc_ws.iterator + 4 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub5 = cute.make_tensor(desc_ws.iterator + 5 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub6 = cute.make_tensor(desc_ws.iterator + 6 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub7 = cute.make_tensor(desc_ws.iterator + 7 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub8 = cute.make_tensor(desc_ws.iterator + 8 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub9 = cute.make_tensor(desc_ws.iterator + 9 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub10 = cute.make_tensor(desc_ws.iterator + 10 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub11 = cute.make_tensor(desc_ws.iterator + 11 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub12 = cute.make_tensor(desc_ws.iterator + 12 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub13 = cute.make_tensor(desc_ws.iterator + 13 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub14 = cute.make_tensor(desc_ws.iterator + 14 * arr_words, cute.make_layout((cutlass.Int32(TENSOR_MAP_QWORDS),), stride=(1,))) + + if widx == 0: + if nvvm.elect_sync(): + emit_seq_descs(base_q, sub0, cu_seqlens, q, n_batch, q_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 1: + if nvvm.elect_sync(): + emit_seq_descs(base_k, sub1, cu_seqlens, k, n_batch, k_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 2: + if nvvm.elect_sync(): + emit_seq_descs(base_v, sub2, cu_seqlens, v, n_batch, v_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 3: + if nvvm.elect_sync(): + emit_seq_descs(base_gate, sub3, cu_seqlens, gate, n_batch, g_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 4: + if nvvm.elect_sync(): + emit_seq_descs(base_do, sub4, cu_seqlens, do, n_batch, do_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 5: + if nvvm.elect_sync(): + emit_seq_descs(base_beta, sub5, cu_seqlens, beta, n_batch, beta_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 6: + if nvvm.elect_sync(): + emit_seq_descs(base_w, sub6, cu_seqlens, w, n_batch, w_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 7: + if nvvm.elect_sync(): + emit_seq_descs(base_dq, sub7, cu_seqlens, dq, n_batch, dq_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 8: + if nvvm.elect_sync(): + emit_seq_descs(base_dk, sub8, cu_seqlens, dk, n_batch, dk_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 9: + if nvvm.elect_sync(): + emit_seq_descs(base_dv, sub9, cu_seqlens, dv, n_batch, dv_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 10: + if nvvm.elect_sync(): + emit_seq_descs(base_dg, sub10, cu_seqlens, dg, n_batch, dg_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 11: + if nvvm.elect_sync(): + emit_seq_descs(base_dwo, sub11, cu_seqlens, dwo, n_batch, dwo_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 12: + if nvvm.elect_sync(): + emit_seq_descs(base_dbo, sub12, cu_seqlens, dbo, n_batch, dbo_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 13: + if nvvm.elect_sync(): + emit_checkpoint_seq_descs(base_checkpoint, sub13, cu_seqlens, state_checkpoints, n_batch, checkpoint_rs, checkpoint_every_n, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if cutlass.const_expr(initial_state is not None): + if widx == 14: + if nvvm.elect_sync(): + emit_copy_desc(base_initial_state, sub14) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + + +@cute.jit +def build_descs( + io_dtype: cutlass.Constexpr, + b_t: cutlass.Constexpr[int], + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + gate: cute.Tensor, + do: cute.Tensor, + beta: cute.Tensor, + w: cute.Tensor, + dq: cute.Tensor, + dk: cute.Tensor, + dv: cute.Tensor, + dg: cute.Tensor, + dwo: cute.Tensor, + dbo: cute.Tensor, + state_checkpoints: cute.Tensor, + initial_state: cute.Tensor | None, + cu_seqlens: cute.Tensor, + tensormap_workspace: cute.Tensor, + stream: cuda_driver.CUstream, +): + """Build the 15 per-(batch, head) TMA-descriptor arrays into + ``tensormap_workspace``.""" + h_q = q.shape[1] + h_k = k.shape[1] + h_v = v.shape[1] + ho = gate.shape[1] + batch_size = cu_seqlens.shape[0] - 1 + d_k = q.shape[2] + d_v = v.shape[2] + bpe = io_dtype.width // 8 + granu = 128 // bpe + seqlen = q.shape[0] + + def headed(t, dim, hn): + return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) + + swz = cuda.TensorMapSwizzle.s128b + base_q = cuda.create_tensor_map_tiled_from_view(headed(q, d_k, h_q), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_do = cuda.create_tensor_map_tiled_from_view(headed(do, d_v, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_beta = cuda.create_tensor_map_tiled_from_view(headed(beta, d_k, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_w = cuda.create_tensor_map_tiled_from_view(headed(w, d_v, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dq = cuda.create_tensor_map_tiled_from_view(headed(dq, d_k, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dk = cuda.create_tensor_map_tiled_from_view(headed(dk, d_k, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dv = cuda.create_tensor_map_tiled_from_view(headed(dv, d_v, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dg = cuda.create_tensor_map_tiled_from_view(headed(dg, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dwo = cuda.create_tensor_map_tiled_from_view(headed(dwo, d_v, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dbo = cuda.create_tensor_map_tiled_from_view(headed(dbo, d_k, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + + checkpoint_view = cute.make_tensor( + state_checkpoints.iterator, + cute.make_layout( + (d_v, d_k, state_checkpoints.shape[0], ho), + stride=(state_checkpoints.stride[3], state_checkpoints.stride[2], state_checkpoints.stride[0], state_checkpoints.stride[1]), + ), + ) + base_checkpoint = cuda.create_tensor_map_tiled_from_view(checkpoint_view, box_dims=(64, d_k, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz) + base_initial_state = base_checkpoint + if cutlass.const_expr(initial_state is not None): + initial_state_view = cute.make_tensor( + initial_state.iterator, + cute.make_layout( + (d_v, d_k, ho, batch_size), + stride=(initial_state.stride[3], initial_state.stride[2], initial_state.stride[1], initial_state.stride[0]), + ), + ) + base_initial_state = cuda.create_tensor_map_tiled_from_view(initial_state_view, box_dims=(64, d_k, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz) + + n_warps = 15 if initial_state is not None else 14 + build_all_descs_kernel( + base_q, + base_k, + base_v, + base_gate, + base_do, + base_beta, + base_w, + base_dq, + base_dk, + base_dv, + base_dg, + base_dwo, + base_dbo, + base_checkpoint, + base_initial_state, + tensormap_workspace, + cu_seqlens, + q, + k, + v, + gate, + do, + beta, + w, + dq, + dk, + dv, + dg, + dwo, + dbo, + state_checkpoints, + initial_state, + cutlass.Int32(batch_size), + cutlass.Int32(q.stride[0]), + cutlass.Int32(k.stride[0]), + cutlass.Int32(v.stride[0]), + cutlass.Int32(gate.stride[0]), + cutlass.Int32(do.stride[0]), + cutlass.Int32(beta.stride[0]), + cutlass.Int32(w.stride[0]), + cutlass.Int32(dq.stride[0]), + cutlass.Int32(dk.stride[0]), + cutlass.Int32(dv.stride[0]), + cutlass.Int32(dg.stride[0]), + cutlass.Int32(dwo.stride[0]), + cutlass.Int32(dbo.stride[0]), + cutlass.Int32(state_checkpoints.stride[0]), + cutlass.Int32(b_t), + ).launch(grid=(1, 1, 1), block=(32 * n_warps, 1, 1), stream=stream) + + +@cute.jit +def host( + cfg: cutlass.Constexpr, + state_checkpoints: cute.Tensor, + mState_init: cute.Tensor | None, + dg: cute.Tensor, + dbeta: cute.Tensor, + dw: cute.Tensor, + cu_seqlens: cute.Tensor, + d_initial_state: cute.Tensor | None, + d_final_state: cute.Tensor | None, + work_items: cute.Tensor | None, + work_count: cute.Tensor | None, + sched_ctr: cute.Tensor | None, + tensormap_workspace: cute.Tensor, + scale: cutlass.Float32, + stream, +) -> None: + num_sequences = cu_seqlens.shape[0] - 1 + ho = dg.shape[1] + + # ---- launch ------------------------------------------------------------------ + total_tiles = num_sequences * ho + n_desc = num_sequences + grid_shape = (cfg.max_active_clusters, 1, 1) + kernel( + cfg, + tensormap_workspace, + n_desc, + cu_seqlens, + dg, + dbeta, + dw, + d_initial_state, + d_final_state, + work_items, + work_count, + sched_ctr, + total_tiles, + scale, + ).launch( + grid=grid_shape, + block=(cfg.threads_per_cta, 1, 1), + stream=stream, + min_blocks_per_mp=1, + ) + + +@cute.kernel +def kernel( + cfg: cutlass.Constexpr, + tensormap_workspace: cute.Tensor, + n_desc: cutlass.Int32, + cu_seqlens: cute.Tensor, + mDg: cute.Tensor, + mDb: cute.Tensor, + mDw_out: cute.Tensor, + mDstate0: cute.Tensor | None, + mDstate_in: cute.Tensor | None, + mWorkItems: cute.Tensor | None, + mCount: cute.Tensor | None, + mSched: cute.Tensor | None, + total_tiles: cutlass.Int32, + scale: cutlass.Float32, +) -> None: + """BT=16 GDN-2 backward kernel (persistent, 16 warps).""" + tidx, _, _ = cute.arch.thread_idx() + bidx = cute.arch.block_idx()[0] + num_ctas = cute.arch.grid_dim()[0] + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + lane = tidx % cfg.threads_per_warp + + if cutlass.const_expr(cfg.split_k): + total_tiles = mCount[0] + assert cu_seqlens.element_type in (cutlass.Int32, cutlass.Int64) + assert mDg.element_type == cutlass.Float32 + assert mDb.element_type == cfg.io_dtype and mDw_out.element_type == cfg.io_dtype + + desc_base_words = tensormap_workspace.iterator.raw_ptr() + arr_words = n_desc * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_q_base = desc_base_words + desc_k_base = desc_base_words + arr_words + desc_v_base = desc_base_words + cutlass.Int32(2) * arr_words + desc_gate_base = desc_base_words + cutlass.Int32(3) * arr_words + desc_do_base = desc_base_words + cutlass.Int32(4) * arr_words + desc_beta_base = desc_base_words + cutlass.Int32(5) * arr_words + desc_w_base = desc_base_words + cutlass.Int32(6) * arr_words + desc_dq_base = desc_base_words + cutlass.Int32(7) * arr_words + desc_dk_base = desc_base_words + cutlass.Int32(8) * arr_words + desc_dv_base = desc_base_words + cutlass.Int32(9) * arr_words + desc_dg_base = desc_base_words + cutlass.Int32(10) * arr_words + desc_dwo_base = desc_base_words + cutlass.Int32(11) * arr_words + desc_db_base = desc_base_words + cutlass.Int32(12) * arr_words + desc_checkpoint_base = desc_base_words + cutlass.Int32(13) * arr_words + desc_initial_state_base = desc_base_words + cutlass.Int32(14) * arr_words + + SMEM = cutlass.AddressSpace.smem + bars = make_gdn2_bwd_bars(cfg) + tmem_hold = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=4) + sSched = cutlass.Array(cutlass.Int32, cfg.sched_stages, space=SMEM, alignment=16) + bpe = cfg.io_dtype.width // 8 + SWZ = 2 + LEAD = 16 + STRIDE = 8 * 128 + STATE_ALT_LEAD = cfg.d_v * 128 + + sQ_raw = cutlass.Array(cfg.io_dtype, cfg.raw_qk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_raw = cutlass.Array(cfg.io_dtype, cfg.raw_qk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sV_raw = cutlass.Array(cfg.io_dtype, cfg.raw_v_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sDo_raw = cutlass.Array(cfg.io_dtype, cfg.raw_v_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sGate_raw = cutlass.Array(cutlass.Float32, cfg.raw_gate_cosize, space=SMEM, alignment=1024) + sBeta_raw = cutlass.Array(cfg.io_dtype, cfg.raw_qk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sW_raw = cutlass.Array(cfg.io_dtype, cfg.raw_v_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_decay_raw = cutlass.Array(cfg.io_dtype, cfg.operand_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_inv_raw = cutlass.Array(cfg.io_dtype, cfg.operand_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_restore_raw = cutlass.Array(cfg.io_dtype, cfg.operand_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sQ_decay_raw = cutlass.Array(cfg.io_dtype, cfg.operand_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sState_scale_diag_raw = cutlass.Array(cfg.io_dtype, cfg.diag_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sQk_raw = cutlass.Array(cfg.io_dtype, cfg.qk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sState_raw = cutlass.Array(cfg.io_dtype, cfg.state_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sDstate_raw = cutlass.Array(cfg.io_dtype, cfg.d_k * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) + sU_raw = cutlass.Array(cfg.io_dtype, cfg.b_t * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) + sDy_raw = cutlass.Array(cfg.io_dtype, cfg.b_t * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) + sDq_raw = cutlass.Array(cfg.io_dtype, cfg.dq_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sDk_raw = cutlass.Array(cfg.io_dtype, cfg.dk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sDv_raw = cutlass.Array(cfg.io_dtype, cfg.dv_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sDg_raw = cutlass.Array(cutlass.Float32, cfg.dg_cosize, space=SMEM, alignment=1024) + sDwOut_raw = cutlass.Array(cfg.io_dtype, cfg.dwo_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sDb_raw = cutlass.Array(cfg.io_dtype, cfg.db_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sRed1_raw = cutlass.Array(cutlass.Float32, 4 * cfg.b_t, space=SMEM, alignment=64) + sNorm_raw = cutlass.Array(cutlass.Float32, cfg.smem_raw_stages * 2 * cfg.b_t, space=SMEM, alignment=64) + + sState_alt = SmemTile( + base=sState_raw.data_ptr().toint(), + elems_per_stage=((cfg.state_cosize) // (cfg.smem_state_stages)) * bpe, + stages=cfg.smem_state_stages, + leading_byte_offset=STATE_ALT_LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sK_decay_lead16 = SmemTile( + base=sK_decay_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sK_inv_lead16 = SmemTile( + base=sK_inv_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sK_restore_lead16 = SmemTile( + base=sK_restore_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sDo_lead16 = SmemTile( + base=sDo_raw.data_ptr().toint(), + elems_per_stage=((cfg.raw_v_cosize) // (cfg.smem_raw_stages)) * bpe, + stages=cfg.smem_raw_stages, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sDo_amaj = SmemTile( + base=sDo_raw.data_ptr().toint(), + elems_per_stage=((cfg.raw_v_cosize) // (cfg.smem_raw_stages)) * bpe, + stages=cfg.smem_raw_stages, + leading_byte_offset=cfg.b_t * 128, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sU_lead16 = SmemTile( + base=sU_raw.data_ptr().toint(), + elems_per_stage=((cfg.b_t * cfg.d_v) // (1)) * bpe, + stages=1, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sDy_lead16 = SmemTile( + base=sDy_raw.data_ptr().toint(), + elems_per_stage=((cfg.b_t * cfg.d_v) // (1)) * bpe, + stages=1, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sDstate_alt = SmemTile( + base=sDstate_raw.data_ptr().toint(), + elems_per_stage=((cfg.d_k * cfg.d_v) // (1)) * bpe, + stages=1, + leading_byte_offset=STATE_ALT_LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + + sQ_decay_trans = SmemTile( + base=sQ_decay_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=cfg.b_t * 128, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sK_inv_amaj = SmemTile( + base=sK_inv_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=cfg.b_t * 128, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sK_decay_trans = SmemTile( + base=sK_decay_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=cfg.b_t * 128, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sState_scale_diag = SmemTile( + base=sState_scale_diag_raw, + elems_per_stage=((cfg.d_k // 16) * 256), + stages=cfg.smem_decay_stages, + leading_byte_offset=16, + stride_byte_offset=(8 * 16 * 2), + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, + ) + sQk = SmemTile( + base=sQk_raw, + elems_per_stage=(cfg.qk_tiles * cfg.b_t * cfg.b_t), + stages=cfg.smem_qk_stages, + leading_byte_offset=16, + stride_byte_offset=(8 * cfg.b_t * 2), + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, + ) + q_tx_bytes = cutlass.const_expr(cfg.d_k * cfg.b_t * bpe) + k_tx_bytes = cutlass.const_expr(cfg.d_k * cfg.b_t * bpe) + gate_tx_bytes = cutlass.const_expr(cfg.d_k * cfg.b_t * 4) + beta_tx_bytes = cutlass.const_expr(cfg.d_k * cfg.b_t * bpe) + do_tx_bytes = cutlass.const_expr(cfg.d_v * cfg.b_t * bpe) + v_tx_bytes = cutlass.const_expr(cfg.d_v * cfg.b_t * bpe) + w_tx_bytes = cutlass.const_expr(cfg.d_v * cfg.b_t * bpe) + + if warp_idx == cfg.tma_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.smem_raw_stages): + bars.mb_q_ready[stage].init() + bars.mb_q_done[stage].init() + bars.mb_k_ready[stage].init() + bars.mb_k_done[stage].init() + bars.mb_gate_ready[stage].init() + bars.mb_gate_done[stage].init() + bars.mb_beta_ready[stage].init() + bars.mb_beta_done[stage].init() + bars.mb_do_ready[stage].init() + bars.mb_do_done[stage].init() + bars.mb_do_epi_done[stage].init() + bars.mb_v_ready[stage].init() + bars.mb_v_done[stage].init() + bars.mb_w_ready[stage].init() + bars.mb_w_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_state_stages): + bars.mb_state_ready[stage].init() + bars.mb_state_done[stage].init() + bars.mb_state_cg0_done[stage].init() + for stage in cutlass.range_constexpr(2): + bars.mb_state_inp_ready[stage].init() + bars.mb_state_inp_done[stage].init() + bars.mb_state_inp_cg2_done[stage].init() + elif warp_idx == cfg.tcgen05_mma_warp_id: + if nvvm.elect_sync(): + bars.mb_state_k_acc_ready.init() + bars.mb_rhs_ready.init() + bars.mb_update_acc_ready.init() + bars.mb_u_smem_ready.init() + bars.mb_u_done.init() + bars.mb_u_warps_done.init() + bars.mb_du_acc_ready.init() + bars.mb_du_inp_ready.init() + bars.mb_du_inp_done.init() + bars.mb_dy_acc_ready.init() + bars.mb_neg_dy_inp_ready.init() + bars.mb_neg_dy_inp_done.init() + bars.mb_sdy_ready.init() + bars.mb_sdy_done.init() + bars.mb_sdy_super_done.init() + bars.mb_dstate_acc_ready.init() + bars.mb_dstate_acc_done.init() + bars.mb_dstate_inp_ready.init() + bars.mb_dstate_inp_done.init() + bars.mb_dstate_smem_ready.init() + bars.mb_dstate_smem_done.init() + bars.mb_dstate_smem_cg2_done.init() + bars.mb_dq_acc_ready.init() + bars.mb_dk_decay_part_ready.init() + bars.mb_dk_inv_part_ready.init() + bars.mb_dk_restore_part_ready.init() + bars.mb_parts_done.init() + bars.mb_dstate0_stored.init() + elif warp_idx == cfg.super_mma_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.smem_decay_stages): + bars.mb_k_decay_inv_ready[stage].init() + bars.mb_q_decay_k_restore_ready[stage].init() + bars.mb_decay_done[stage].init() + bars.mb_decay_super_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_qk_stages): + bars.mb_a_ready[stage].init() + bars.mb_aqk_ready[stage].init() + bars.mb_da_ready[stage].init() + bars.mb_dm_ready[stage].init() + bars.mb_aqk_done[stage].init() + bars.mb_a_done[stage].init() + bars.mb_da_done[stage].init() + bars.mb_dm_done[stage].init() + elif warp_idx == cfg.epilogue_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.smem_dq_stages): + bars.mb_dq_tmastg_ready[stage].init() + bars.mb_dq_tmastg_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_dk_stages): + bars.mb_dk_tmastg_ready[stage].init() + bars.mb_dk_tmastg_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_dg_stages): + bars.mb_dg_tmastg_ready[stage].init() + bars.mb_dg_tmastg_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_db_stages): + bars.mb_db_tmastg_ready[stage].init() + bars.mb_db_tmastg_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_dv_stages): + bars.mb_dv_tmastg_ready[stage].init() + bars.mb_dv_tmastg_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_dwo_stages): + bars.mb_dwo_tmastg_ready[stage].init() + bars.mb_dwo_tmastg_done[stage].init() + for stage in cutlass.range_constexpr(cfg.sched_stages): + bars.mb_sched_ready[stage].init() + bars.mb_sched_done[stage].init() + diag_zero = cfg.io_dtype(0.0) + for diag_idx in cutlass.range(tidx, cfg.diag_cosize, cfg.threads_per_cta, unroll=1): + sState_scale_diag_raw[diag_idx] = diag_zero + nvvm.fence_mbarrier_init() + nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) + is_tmem_user = ( + (warp_idx >= cfg.compute_group_2_warp_ids[0] and warp_idx <= cfg.compute_group_2_warp_ids[-1]) + or (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) + or (warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]) + or warp_idx == cfg.tcgen05_mma_warp_id + ) + if is_tmem_user: + if warp_idx == cfg.tcgen05_mma_warp_id: + nvvm.tcgen05_alloc(tmem_hold, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) + if warp_idx == cfg.tcgen05_mma_warp_id: + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) + + if warp_idx == cfg.tma_warp_id: + tmaldg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + mSched, + sSched, + q_tx_bytes, + k_tx_bytes, + gate_tx_bytes, + beta_tx_bytes, + do_tx_bytes, + v_tx_bytes, + w_tx_bytes, + lane, + sQ_raw, + sK_raw, + sV_raw, + sGate_raw, + sDo_raw, + sBeta_raw, + sW_raw, + sState_raw, + desc_q_base, + desc_k_base, + desc_v_base, + desc_gate_base, + desc_do_base, + desc_beta_base, + desc_w_base, + desc_checkpoint_base, + desc_initial_state_base, + bars, + ) + elif warp_idx == cfg.super_mma_warp_id: + super_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_decay_raw, + sK_inv_raw, + sU_raw, + sDy_raw, + sQk_raw, + bars, + ) + elif warp_idx == cfg.tcgen05_mma_warp_id: + tcgen05_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + tmem_hold, + sState_alt, + sK_decay_lead16, + sK_inv_lead16, + sK_inv_amaj, + sK_restore_lead16, + sDo_lead16, + sDo_amaj, + sQ_decay_trans, + sK_decay_trans, + sU_lead16, + sDy_lead16, + sDstate_alt, + sQk, + sState_scale_diag, + bars, + ) + elif warp_idx == cfg.epilogue_warp_id: + epilogue_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_inv_raw, + sQ_decay_raw, + sDo_raw, + sU_raw, + sQk_raw, + sDq_raw, + sDk_raw, + sDv_raw, + sDg_raw, + sDb_raw, + sDwOut_raw, + desc_dq_base, + desc_dk_base, + desc_dv_base, + desc_dg_base, + desc_db_base, + desc_dwo_base, + bars, + ) + elif warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]: + compute0_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_hold, + warp_idx, + scale, + sK_inv_raw, + sGate_raw, + sK_raw, + sQ_raw, + sState_raw, + sV_raw, + sDo_raw, + sBeta_raw, + sW_raw, + sNorm_raw, + sK_decay_raw, + sK_restore_raw, + sQ_decay_raw, + sState_scale_diag_raw, + bars, + ) + elif warp_idx >= cfg.compute_group_2_warp_ids[0] and warp_idx <= cfg.compute_group_2_warp_ids[-1]: + compute2_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_hold, + warp_idx, + sQ_raw, + sK_raw, + sBeta_raw, + sGate_raw, + sNorm_raw, + sDq_raw, + sDk_raw, + sRed1_raw, + sDstate_raw, + sDg_raw, + sDb_raw, + scale, + bars, + ) + elif warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]: + compute1_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_hold, + warp_idx, + mDstate0, + mDstate_in, + sV_raw, + sW_raw, + sDo_raw, + sU_raw, + sDy_raw, + sDv_raw, + sDwOut_raw, + sDstate_raw, + bars, + ) + + +@dataclass +class Gdn2BwdCfg: + """Kernel cfg (fixed BT=16 schedule constants; derived TMEM column offsets + and SMEM buffer cosizes are stamped by ``build_cfg``).""" + + io_dtype: Type[cutlass.Numeric] + use_dstate_in: bool + use_dstate0: bool + l2norm: bool + use_initial_state: bool + q_ratio: int + k_ratio: int + v_ratio: int + n_heads_out: int + max_active_clusters: int + split_k: bool = False + dyn_sched: bool = False + sched_stages: int = 8 + + # ---- fixed constants stamped from CFG by build_cfg --------------------------- + compute_group_0_warp_ids: tuple = CFG.COMPUTE_GROUP_0_WARP_IDS + compute_group_2_warp_ids: tuple = CFG.COMPUTE_GROUP_2_WARP_IDS + compute_group_1_warp_ids: tuple = CFG.COMPUTE_GROUP_1_WARP_IDS + super_mma_warp_id: int = CFG.SUPER_MMA_WARP_ID + tcgen05_mma_warp_id: int = CFG.TCGEN05_MMA_WARP_ID + tma_warp_id: int = CFG.TMA_WARP_ID + epilogue_warp_id: int = CFG.EPILOGUE_WARP_ID + b_t: int = CFG.B_T + d_k: int = CFG.D_K + d_v: int = CFG.D_V + threads_per_warp: int = CFG.THREADS_PER_WARP + threads_per_cta: int = 0 + num_regs_compute_group_0: int = CFG.NUM_REGS_COMPUTE_GROUP_0 + num_regs_compute_group_1: int = CFG.NUM_REGS_COMPUTE_GROUP_1 + num_regs_compute_group_2: int = CFG.NUM_REGS_COMPUTE_GROUP_2 + num_regs_other: int = CFG.NUM_REGS_OTHER + + # ---- named barrier slots (ids 1-4; 0 is the CTA-wide sync) ------------------- + cg0_sync_barrier_id: int = 1 + cg0_threads: int = 0 + cg2_sync_barrier_id: int = 2 + cg2_threads: int = 0 + tmem_lifecycle_barrier_id: int = 3 + tmem_user_threads: int = 0 + cg1_sync_barrier_id: int = 4 + cg1_threads: int = 0 + + # ---- SMEM / TMEM stage counts + TMEM column offsets -------------------------- + smem_raw_stages: int = CFG.SMEM_RAW_STAGES + smem_state_stages: int = CFG.SMEM_STATE_STAGES + smem_decay_stages: int = CFG.SMEM_DECAY_STAGES + smem_qk_stages: int = CFG.SMEM_QK_STAGES + smem_dq_stages: int = CFG.SMEM_DQ_STAGES + smem_dk_stages: int = CFG.SMEM_DK_STAGES + smem_dg_stages: int = CFG.SMEM_DG_STAGES + smem_db_stages: int = CFG.SMEM_DB_STAGES + smem_dv_stages: int = CFG.SMEM_DV_STAGES + smem_dwo_stages: int = CFG.SMEM_DWO_STAGES + qk_tiles: int = 5 + tmem_dstate_acc_offset: int = 0 + tmem_dstate_inp_offset: int = 0 + tmem_state_k_acc_offset: int = 0 + tmem_update_acc_offset: int = 0 + tmem_du_acc_offset: int = 0 + tmem_dy_acc_offset: int = 0 + tmem_dq_acc_offset: int = 0 + tmem_dk_decay_acc_offset: int = 0 + tmem_dk_inv_acc_offset: int = 0 + tmem_dk_restore_acc_offset: int = 0 + tmem_q_decay_inp_offset: int = 0 + tmem_k_decay_inp_offset: int = 0 + tmem_k_inv_inp_offset: int = 0 + tmem_k_restore_inp_offset: int = 0 + tmem_rhs_inp_offset: int = 0 + tmem_du_inp_offset: int = 0 + tmem_neg_dy_inp_offset: int = 0 + tmem_state_inp_offset: int = 0 + buffer_align_bytes: int = CFG.BUFFER_ALIGN_BYTES + + # ---- buffer cosizes / TMA bytes stamped by build_cfg ------------------------- + raw_qk_cosize: int = 0 + raw_v_cosize: int = 0 + raw_gate_cosize: int = 0 + operand_cosize: int = 0 + diag_cosize: int = 0 + qk_cosize: int = 0 + state_cosize: int = 0 + dq_cosize: int = 0 + dk_cosize: int = 0 + dg_cosize: int = 0 + db_cosize: int = 0 + dv_cosize: int = 0 + dwo_cosize: int = 0 + tma_state_bytes: int = 0 + + +def build_cfg( + io_dtype: Type[cutlass.Numeric], + *, + use_dstate_in: bool, + use_dstate0: bool, + l2norm: bool, + use_initial_state: bool, + q_ratio: int, + k_ratio: int, + v_ratio: int, + n_heads_out: int, + max_active_clusters: int, + split_k: bool = False, + dyn_sched: bool = False, +) -> Gdn2BwdCfg: + if io_dtype not in (cutlass.Float16, cutlass.BFloat16): + raise ValueError(f"io_dtype={io_dtype} not supported; only Float16 and BFloat16 are supported") + cfg = Gdn2BwdCfg( + io_dtype=io_dtype, + use_dstate_in=use_dstate_in, + use_dstate0=use_dstate0, + l2norm=l2norm, + use_initial_state=use_initial_state, + q_ratio=q_ratio, + k_ratio=k_ratio, + v_ratio=v_ratio, + n_heads_out=n_heads_out, + max_active_clusters=max_active_clusters, + split_k=split_k, + dyn_sched=dyn_sched, + ) + cfg.threads_per_cta = 16 * cfg.threads_per_warp + cfg.cg0_threads = len(cfg.compute_group_0_warp_ids) * cfg.threads_per_warp + cfg.cg2_threads = len(cfg.compute_group_2_warp_ids) * cfg.threads_per_warp + cfg.cg1_threads = len(cfg.compute_group_1_warp_ids) * cfg.threads_per_warp + cfg.tmem_user_threads = ( + 1 + len(cfg.compute_group_2_warp_ids) + len(cfg.compute_group_1_warp_ids) + len(cfg.compute_group_0_warp_ids) + ) * cfg.threads_per_warp + + cfg.tmem_dstate_acc_offset = 0 + cfg.tmem_dstate_inp_offset = cfg.d_k + cfg.tmem_state_inp_offset = cfg.tmem_dstate_inp_offset + cfg.d_k // 2 + cfg.tmem_state_k_acc_offset = cfg.tmem_state_inp_offset + cfg.d_v + cfg.tmem_update_acc_offset = cfg.tmem_state_k_acc_offset + cfg.b_t + cfg.tmem_du_acc_offset = cfg.tmem_update_acc_offset + cfg.b_t + cfg.tmem_dy_acc_offset = cfg.tmem_du_acc_offset + cfg.b_t + cfg.tmem_dq_acc_offset = cfg.tmem_dy_acc_offset + cfg.b_t + cfg.tmem_dk_decay_acc_offset = cfg.tmem_dq_acc_offset + cfg.b_t + cfg.tmem_dk_inv_acc_offset = cfg.tmem_dk_decay_acc_offset + cfg.b_t + cfg.tmem_dk_restore_acc_offset = cfg.tmem_dk_inv_acc_offset + cfg.b_t + cfg.tmem_rhs_inp_offset = cfg.tmem_dk_restore_acc_offset + cfg.b_t + cfg.tmem_du_inp_offset = cfg.tmem_rhs_inp_offset + cfg.b_t // 2 + cfg.tmem_neg_dy_inp_offset = cfg.tmem_du_inp_offset + cfg.b_t // 2 + assert cfg.tmem_neg_dy_inp_offset + cfg.b_t // 2 <= 512 + + cfg.raw_qk_cosize = cfg.smem_raw_stages * cfg.d_k * cfg.b_t + cfg.raw_v_cosize = cfg.smem_raw_stages * cfg.d_v * cfg.b_t + cfg.raw_gate_cosize = cfg.smem_raw_stages * cfg.d_k * cfg.b_t + cfg.operand_cosize = cfg.smem_decay_stages * cfg.b_t * cfg.d_k + cfg.diag_cosize = cfg.smem_decay_stages * (cfg.d_k // 16) * 256 + cfg.qk_cosize = cfg.smem_qk_stages * cfg.qk_tiles * cfg.b_t * cfg.b_t + cfg.state_cosize = cfg.smem_state_stages * cfg.d_k * cfg.d_v + cfg.dq_cosize = cfg.smem_dq_stages * cfg.b_t * cfg.d_k + cfg.dk_cosize = cfg.smem_dk_stages * cfg.b_t * cfg.d_k + cfg.dg_cosize = cfg.smem_dg_stages * cfg.b_t * cfg.d_k + cfg.db_cosize = cfg.smem_db_stages * cfg.b_t * cfg.d_k + cfg.dv_cosize = cfg.smem_dv_stages * cfg.b_t * cfg.d_v + cfg.dwo_cosize = cfg.smem_dwo_stages * cfg.b_t * cfg.d_v + cfg.tma_state_bytes = cfg.d_k * cfg.d_v * (io_dtype.width // 8) + return cfg + + +TENSORMAP_DESC_ARRAYS = 14 # per-batch runtime TMA descriptors: Q, K, V, Gate, dO, Beta, W, Checkpoint, dQ, dK, dV, dG, dW_out, dB +TENSORMAP_STATIC_SLOTS = 1 # initial_state + + +# ---- Torch adapter / host-side compilation --------------------------------------- + + +@lru_cache(maxsize=None) +def get_compiled_cache( + io_dtype_str: str, + cu_dtype_str: str, + HQ: int, + HK: int, + HV: int, + use_dstate_in: bool, + use_dstate0: bool, + l2norm: bool, + use_initial_state: bool, + split_k: bool, + dyn_sched: bool, +): + return {} + + +def chunk_gdn2_bwd_sm100( + q, + k, + v, + gate, + beta, + w, + do, + state_checkpoints, + dq, + dk, + dv, + dg, + dbeta, + dw, + cu_seqlens, + scale: float, + *, + initial_state=None, + d_initial_state=None, + d_final_state=None, + use_qk_l2norm_in_kernel: bool = False, + work_items=None, + work_count=None, + sched_ctr=None, + tensormap_workspace, + stream, +) -> None: + """Execute the Blackwell BT=16 chunked GDN-2 backward kernel. + + All tensors must be contiguous and on the same CUDA device. + + Args: + q: ``(total_tokens, HQ, DK)`` float16/bfloat16 + k: ``(total_tokens, HK, DK)`` float16/bfloat16 + v: ``(total_tokens, HV, DV)`` float16/bfloat16 + gate: ``(total_tokens, HO, DK)`` fp32 natural-log per-channel decay + beta: ``(total_tokens, HO, DK)`` io dtype post-sigmoid per-key erase + w: ``(total_tokens, HO, DV)`` io dtype post-sigmoid per-value write + do: ``(total_tokens, HO, DV)`` io dtype + state_checkpoints: ``(total_checkpoints, HO, DK, DV)`` io dtype (KV, v contiguous — the GDN + checkpoint layout), the PLAIN per-chunk series with no initial-state + slot: sequence-local entry ``c - 1`` is the state ENTERING chunk c >= 1 + of sequence b; chunk 0 seeds from ``initial_state`` + dq/dk/dv: io dtype at ``HO = max(HQ, HV)`` heads, pre-allocated + dg: ``(total_tokens, HO, DK)`` fp32 (dL/d ln alpha), pre-allocated + dbeta: ``(total_tokens, HO, DK)`` io dtype, pre-allocated + dw: ``(total_tokens, HO, DV)`` io dtype, pre-allocated + cu_seqlens: ``(num_seqs + 1,)`` int32 + scale: attention scale factor + initial_state: ``(num_seqs, HO, DK, DV)`` io dtype (KV) — the state + entering chunk 0 (engine-provided zeros when the graph has none) + d_initial_state: fp32 ``(num_seqs, HO, DK, DV)`` OUT (dL/dS0), or None + d_final_state: fp32 ``(num_seqs, HO, DK, DV)`` IN (dL/d final state) + use_qk_l2norm_in_kernel: q/k arrive raw; the kernel normalizes for the + recompute math and chains the L2-norm backward into dq/dk + work_items/work_count: split-K table (``common/split_k.py``); + each item computes chunks ``[wstart, cend)`` backward and writes + gradients only for ``[wstart, wend)`` + sched_ctr: ``(2,)`` int32 zeroed scratch enabling the dynamic + (work-stealing) tile scheduler + tensormap_workspace: ``tensormap_workspace_bytes(module, B)`` bytes, + 128-byte aligned, for the per-(batch, head) TMA-descriptor + arrays (tail chunks clip/zero-fill in hardware) + """ + HQ = q.shape[1] + HK = k.shape[1] + HV = v.shape[1] + HO = max(HQ, HV) + use_dstate_in = d_final_state is not None + use_dstate0 = d_initial_state is not None + use_initial_state = initial_state is not None + split_k = work_items is not None + dyn_sched = sched_ctr is not None + for name, t in (("state_checkpoints", state_checkpoints), ("beta", beta), ("w", w), ("dbeta", dbeta), ("dw", dw)) + ( + (("initial_state", initial_state),) if use_initial_state else () + ): + if str(t.dtype).split(".")[-1] != str(q.dtype).split(".")[-1]: + raise ValueError(f"{name} dtype must match the io dtype: got {t.dtype} with io {q.dtype}") + for name, hh in (("HQ", HQ), ("HK", HK), ("HV", HV)): + if HO % hh != 0: + raise ValueError(f"{name}={hh} must divide {HO}") + B = cu_seqlens.shape[0] - 1 + + cu_stream = cuda_driver.CUstream(int(stream)) + cache = get_compiled_cache( + str(q.dtype), + str(cu_seqlens.dtype), + HQ, + HK, + HV, + use_dstate_in, + use_dstate0, + use_qk_l2norm_in_kernel, + use_initial_state, + split_k, + dyn_sched, + ) + + if "compiled" not in cache: + io_dtype = get_dtype(q.dtype) + cfg = build_cfg( + io_dtype, + use_dstate_in=use_dstate_in, + use_dstate0=use_dstate0, + l2norm=use_qk_l2norm_in_kernel, + use_initial_state=use_initial_state, + q_ratio=HO // HQ, + k_ratio=HO // HK, + v_ratio=HO // HV, + n_heads_out=HO, + max_active_clusters=multiprocessor_count(current_device_id()), + split_k=split_k, + dyn_sched=dyn_sched, + ) + + dstate0_cute = None + if use_dstate0: + dstate0_cute = from_dlpack(d_initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) + dstate_in_cute = None + if use_dstate_in: + dstate_in_cute = from_dlpack(d_final_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) + wi_cute = None + wc_cute = None + if split_k: + wi_cute = from_dlpack(work_items, assumed_align=16) + wi_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) + wc_cute = from_dlpack(work_count, assumed_align=4).mark_layout_dynamic() + sc_cute = None + if dyn_sched: + sc_cute = from_dlpack(sched_ctr, assumed_align=4).mark_layout_dynamic() + + tensormap_ws_cute = from_dlpack(tensormap_workspace, assumed_align=128).mark_layout_dynamic() + state_checkpoints_cute = from_dlpack(state_checkpoints, assumed_align=16).mark_layout_dynamic(leading_dim=len(state_checkpoints.shape) - 1) + initial_state_cute = ( + from_dlpack(initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=len(initial_state.shape) - 1) if use_initial_state else None + ) + dg_cute = from_dlpack(dg, assumed_align=16).mark_layout_dynamic(leading_dim=len(dg.shape) - 1) + dbeta_cute = from_dlpack(dbeta, assumed_align=16).mark_layout_dynamic(leading_dim=len(dbeta.shape) - 1) + dw_cute = from_dlpack(dw, assumed_align=16).mark_layout_dynamic(leading_dim=len(dw.shape) - 1) + cache["compiled"] = cute.compile( + host, + cfg, + state_checkpoints_cute, + initial_state_cute, + dg_cute, + dbeta_cute, + dw_cute, + from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic(), + dstate0_cute, + dstate_in_cute, + wi_cute, + wc_cute, + sc_cute, + tensormap_ws_cute, + scale, + cu_stream, + options="--enable-tvm-ffi --opt-level 2", + ) -def get_workspace_size(B: int, HQ: int, HV: int) -> int: - return 0 + # ---- per-(batch, head) descriptor arrays: rebuild on input change ------------ + # desc build runs every execute by contract (cu contents are data; + # buffer pointers may change) — capture-safe, single tiny launch + if "build_descs" not in cache: + io_dtype = get_dtype(q.dtype) + q_bd = from_dlpack(q, assumed_align=16).mark_layout_dynamic(leading_dim=2) + k_bd = from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2) + v_bd = from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2) + gate_bd = from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=2) + do_bd = from_dlpack(do, assumed_align=16).mark_layout_dynamic(leading_dim=2) + beta_bd = from_dlpack(beta, assumed_align=16).mark_layout_dynamic(leading_dim=2) + w_bd = from_dlpack(w, assumed_align=16).mark_layout_dynamic(leading_dim=2) + dq_bd = from_dlpack(dq, assumed_align=16).mark_layout_dynamic(leading_dim=2) + dk_bd = from_dlpack(dk, assumed_align=16).mark_layout_dynamic(leading_dim=2) + dv_bd = from_dlpack(dv, assumed_align=16).mark_layout_dynamic(leading_dim=2) + dg_bd = from_dlpack(dg, assumed_align=16).mark_layout_dynamic(leading_dim=2) + dwo_bd = from_dlpack(dw, assumed_align=16).mark_layout_dynamic(leading_dim=2) + dbo_bd = from_dlpack(dbeta, assumed_align=16).mark_layout_dynamic(leading_dim=2) + state_checkpoints_bd = from_dlpack(state_checkpoints, assumed_align=16).mark_layout_dynamic(leading_dim=3) + initial_state_bd = from_dlpack(initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) if use_initial_state else None -def chunk_gdn2_bwd_sm100(*args, **kwargs) -> None: - """Not implemented — the FROST GDN-2 backward kernel is a stub.""" - raise NotImplementedError("FROST GDN-2 backward is not implemented yet (recompute-in-bprop kernel is a stub).") + cu_bd = from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic() + ws_bd = from_dlpack(tensormap_workspace, assumed_align=128).mark_layout_dynamic() + cache["build_descs"] = cute.compile( + build_descs, + io_dtype, + CFG.B_T, + q_bd, + k_bd, + v_bd, + gate_bd, + do_bd, + beta_bd, + w_bd, + dq_bd, + dk_bd, + dv_bd, + dg_bd, + dwo_bd, + dbo_bd, + state_checkpoints_bd, + initial_state_bd, + cu_bd, + ws_bd, + cu_stream, + options="--enable-tvm-ffi", + ) + cache["build_descs"](q, k, v, gate, do, beta, w, dq, dk, dv, dg, dw, dbeta, state_checkpoints, initial_state, cu_seqlens, tensormap_workspace, cu_stream) + cache["compiled"]( + state_checkpoints, + initial_state, + dg, + dbeta, + dw, + cu_seqlens, + d_initial_state, + d_final_state, + work_items, + work_count, + sched_ctr, + tensormap_workspace, + scale, + cu_stream, + ) diff --git a/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_config.py b/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_config.py index 3749cb07c..ffc532ec8 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_config.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_config.py @@ -57,7 +57,7 @@ class Cfg: SMEM_SCHED_STAGES: int = 8 SMEM_O_STAGES: int = 2 SMEM_DECAY_STAGES: int = 2 - SMEM_PAIRWISE_STAGES: int = 2 + SMEM_QK_STAGES: int = 2 SMEM_STATE_SCALE_DIAG_STAGES: int = 3 QK_SCALE_READY_STAGES: int = 3 TMEM_Q_STATE_ACC_STAGES: int = 2 diff --git a/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_f16.py index 4203786d8..276b6cec0 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_f16.py @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Chunked Gated DeltaNet v2 (GDN-2) prefill kernel for Blackwell SM100 +"""Chunked Gated DeltaNet v2 (GDN-2) prefill kernel for Blackwell SM100/SM103 (Cutlass DSL), BT=16 tiling with per-key-channel decay + per-key erase gate (beta) + per-value write gate (w), using direct CUTLASS primitives. Framework-neutral entry ``chunk_gdn2_sm100``. @@ -40,43 +40,43 @@ checkpoints, and the head ratios are compile-time specializations. Warp assignments (16 warps = 512 threads): - warps 0-7 : compute group 0 - gate prefix scan + decay/restore operands - warps 8-11 : compute group 1 - TMEM value side, o drain, state stores - warp 12 : super-MMA - register-MMA kk^T + Neumann inverse + warps 0-7 : compute group 0 - Gate prefix scan + decay/restore operands + warps 8-11 : compute group 1 - TMEM value side, O drain, state stores + warp 12 : super-MMA - register-MMA KK^T + Neumann inverse warp 13 : tcgen05-MMA - the six state GEMMs + the TMEM lifecycle warp 14 : TMA load - per-chunk input G->S loads - warp 15 : epilogue - register-MMA qk + the O TMA store + warp 15 : epilogue - register-MMA QK + the O TMA store SMEM layout (~205 KB total): Buffer Bytes Stages - q / k / v raw 20480 5 <-- SW128 TMA ring (io dtype) - beta / w raw 2x 20480 5 <-- per-channel gates, same ring - gate raw 40960 5 <-- fp32 prefix-scan source + Q / K / V raw 20480 5 <-- SW128 TMA ring (io dtype) + Beta / W raw 2x 20480 5 <-- per-channel gates, same ring + Gate raw 40960 5 <-- fp32 prefix-scan source dt_bias (+a_log slot) 516 1 <-- SAFE_GATE only K_inv 8192 2 <-- token-major ldmatrix/tcgen05 B operand K decay / Q decay 2x 8192 2 <-- tcgen05 SW128 K-box-major A/B operands K restore 8192 2 <-- tcgen05 B operand for the state update state-scale diag 12288 3 <-- per-k-atom decay diagonal blocks - pairwise (A_inv / qk) 2048 2 <-- SW32 16x16 register-MMA tiles - o staging 8192 2 <-- W128 output drain + QK (A_inv / QK) 2048 2 <-- SW32 16x16 register-MMA tiles + O staging 8192 2 <-- W128 output drain TMEM layout (272 of 512 columns): Buffer Cols Purpose - state 0-127 S[DK,DV] fp32 recurrent state + state 0-127 state[DK,DV] fp32 recurrent state state inp 128-191 packed b16 A operand view of the state - q_state_acc 192-223 2-stage state*q -> o accumulator - state_k_acc 224-239 state*k fp32 accumulator + q_state_acc 192-223 2-stage state*Q -> O accumulator + state_k_acc 224-239 state*K fp32 accumulator update_acc 240-255 update fp32 accumulator - rhs input 256-263 packed b16 A operand: w*v - state*(beta*k) + rhs input 256-263 packed b16 A operand: W*V - state*(Beta*K) update input 264-271 packed b16 A operand: the update readback GEMM schedule (tcgen05-MMA warp, in issue order per chunk): - state*k -> state_k_acc - state*q -> q_state_acc (the o acc) + state*K -> state_k_acc + state*Q -> q_state_acc (the O acc) state decay (diag blocks) update = A_inv @ rhs -> update_acc - final_state += update @ k_restore - o += qk @ update -> q_state_acc + final_state += update @ K_restore + O += QK @ update -> q_state_acc Requires a cutlass DSL build providing `cutlass.experimental.*`; not available in the pip nvidia-cutlass-dsl releases. @@ -94,28 +94,30 @@ from cutlass.cute.runtime import from_dlpack from ..common.split_k import decode_work_item -from ..common.thd import TENSOR_MAP_QWORDS, build_h_descs_kernel, build_qkv_load_descs_kernel +from ..common.host import get_dtype +from cudnn.frost.buffers import current_device_id, data_ptr +from cudnn.frost.device import multiprocessor_count +from ..common.thd import TENSOR_MAP_QWORDS, emit_checkpoint_seq_descs, emit_seq_descs from .gdn2_prefill_config import CFG from cudnn.frost.tile_dsl.barrier import ( advance, - arrive, MBarrier, PipelineState, Producer, - wait, ) from cudnn.frost.tile_dsl.handles import GmemTileTma, MmaDesc, SmemTile, tma_slice_runtime_desc -from cudnn.frost.tile_dsl.mma import mma_step, mma_ts +from cudnn.frost.tile_dsl.mma import mma_step, mma_ts_step from cudnn.frost.tile_dsl.swizzle import swizzle_lin_128b, swizzle_lin_S, swizzle_xor_128b from cudnn.frost.tile_dsl.tma import tma_load_tile, tma_store_commit, tma_store_tile, tma_store_wait, tma_tensormap_acquire from cudnn.frost.tile_dsl.pointwise import ( + f16x2_to_f32, fadd2, fmul2, + ffma2, movmatrix_16b, mul_f16x2, opaque_f32_zero, fp32_to_fp16, - sigmoid_f16x2, sub_f16x2, ) @@ -132,39 +134,56 @@ class Gdn2Bars(NamedTuple): - """Every inter-warp handoff as an ``MBarrier`` over its ring (mirrors GDN's - ``GdnBars`` and KDA's ``KdaBars``).""" + """Every inter-warp handoff as an ``MBarrier`` over its ring.""" + + mb_q_ready: MBarrier + mb_q_done: MBarrier + mb_k_ready: MBarrier + mb_k_done: MBarrier + mb_v_ready: MBarrier + mb_v_done: MBarrier + mb_w_ready: MBarrier + mb_w_done: MBarrier + + mb_gate_ready: MBarrier + mb_gate_done: MBarrier + mb_beta_ready: MBarrier + mb_beta_done: MBarrier - mb_tma_done: MBarrier - mb_inputs_ready: MBarrier - mb_inputs_done: MBarrier mb_o_acc_ready: MBarrier mb_o_acc_done: MBarrier mb_state_k_acc_ready: MBarrier mb_update_acc_ready: MBarrier + mb_state_inp_ready: MBarrier - mb_state_scale_diag_done: MBarrier - mb_kk_qk_super_mma_done: MBarrier - mb_kk_qk_mma_done: MBarrier - mb_k_restore_done: MBarrier mb_rhs_ready: MBarrier mb_update_ready: MBarrier - mb_final_state_stored: MBarrier + mb_a_ready: MBarrier - mb_qk_acc_ready: MBarrier mb_a_done: MBarrier + mb_qk_acc_ready: MBarrier + mb_k_decay_inv_cg0_ready: MBarrier + mb_decay_done: MBarrier + mb_decay_super_done: MBarrier mb_qk_scale_ready: MBarrier - mb_k_decay_cg0_ready: MBarrier + mb_k_restore_done: MBarrier + mb_state_scale_diag_done: MBarrier + + mb_final_state_stored: MBarrier + mb_state_read_done: MBarrier + mb_o_tmastg_ready: MBarrier mb_o_tmastg_done: MBarrier + + mb_checkpoint_tmastg_ready: MBarrier + mb_checkpoint_tmastg_done: MBarrier + mb_sched_ready: MBarrier mb_sched_done: MBarrier - mb_h_tmastg_ready: MBarrier - mb_h_tmastg_done: MBarrier def make_gdn2_bars(cfg) -> Gdn2Bars: - """Bars factory. MUST be called from inside ``_kernel`` (allocates the + """Bars factory. MUST be called from inside ``kernel`` (allocates the mbarrier rings in SMEM ahead of the data buffers).""" def alloc(n): @@ -175,71 +194,62 @@ def alloc(n): CG1_THREADS = len(cfg.compute_group_1_warp_ids) * WARP return Gdn2Bars( - mb_tma_done=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.TMA_LOAD), - mb_inputs_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=WARP, producer=Producer.THREAD), - mb_inputs_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_q_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_q_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_k_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_k_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_v_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_v_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_w_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_w_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_gate_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_gate_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_beta_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_beta_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), mb_o_acc_ready=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), mb_o_acc_done=MBarrier(alloc(cfg.tmem_q_state_acc_stages), stages=cfg.tmem_q_state_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_state_k_acc_ready=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), mb_update_acc_ready=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), mb_state_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_state_scale_diag_done=MBarrier( - alloc(cfg.smem_state_scale_diag_stages), - stages=cfg.smem_state_scale_diag_stages, - init_count=1, - producer=Producer.MMA_COMMIT, - ), - mb_kk_qk_super_mma_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=2 * WARP, producer=Producer.THREAD), - mb_kk_qk_mma_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), - mb_k_restore_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_rhs_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_update_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_final_state_stored=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_a_ready=MBarrier(alloc(cfg.smem_pairwise_stages), stages=cfg.smem_pairwise_stages, init_count=WARP, producer=Producer.THREAD), - mb_qk_acc_ready=MBarrier(alloc(cfg.smem_pairwise_stages), stages=cfg.smem_pairwise_stages, init_count=WARP, producer=Producer.THREAD), - mb_a_done=MBarrier(alloc(cfg.smem_pairwise_stages), stages=cfg.smem_pairwise_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_a_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_qk_acc_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_k_decay_inv_cg0_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=2 * WARP, producer=Producer.THREAD), mb_qk_scale_ready=MBarrier( alloc(cfg.qk_scale_ready_stages), stages=cfg.qk_scale_ready_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD, ), - mb_k_decay_cg0_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_k_restore_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_state_scale_diag_done=MBarrier( + alloc(cfg.smem_state_scale_diag_stages), + stages=cfg.smem_state_scale_diag_stages, + init_count=1, + producer=Producer.MMA_COMMIT, + ), + mb_final_state_stored=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_state_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_tmastg_ready=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_tmastg_done=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=WARP, producer=Producer.THREAD), + mb_checkpoint_tmastg_ready=MBarrier( + alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=CG1_THREADS, producer=Producer.THREAD + ), + mb_checkpoint_tmastg_done=MBarrier(alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=WARP, producer=Producer.THREAD), mb_sched_ready=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=1, producer=Producer.THREAD), mb_sched_done=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=15, producer=Producer.THREAD), - # H staging handshake: CG1 fills sH (ready), the epilogue TMA-stores - # and frees it (done); single stage - mb_h_tmastg_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_h_tmastg_done=MBarrier(alloc(1), stages=1, init_count=WARP, producer=Producer.THREAD), ) -# --------------------------------------------------------------------------- -# Device-side helpers / warp bodies -# --------------------------------------------------------------------------- +# ---- Dynamic tile scheduler ------------------------------------------------------ @cute.jit -def _gate_log2(cfg, raw_gate: cutlass.Float32) -> cutlass.Float32: - """Map raw gate to the log2-domain decay increment used by KDA.""" - - if cutlass.const_expr(cfg.safe_gate): - half = cutlass.Float32(0.5) - sigmoid = cute.math.tanh(raw_gate * half, approx=True) * half + half - return cfg.gate_scale_log2 * sigmoid - # Default ABI: gate arrives in natural-log space - return raw_gate * cutlass.Float32(LOG2_E) - - -# --------------------------------------------------------------------------- -# Dynamic tile scheduler: global-ticket ring -# --------------------------------------------------------------------------- - - -@cute.jit -def _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): +def sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): """TMA-warp side: pull the next tile off the global ticket, publish it.""" if cutlass.const_expr(cfg.dyn_sched): bars.mb_sched_done[sched_state.idx].wait(sched_state.phase) @@ -255,7 +265,7 @@ def _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ct @cute.jit -def _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): +def sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): """Consumer side: read the TMA warp's published next tile.""" if cutlass.const_expr(cfg.dyn_sched): bars.mb_sched_ready[sched_state.idx].wait(sched_state.phase) @@ -267,15 +277,7 @@ def _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): @cute.jit -def _decay_key_dim(cfg, tok_coord, key_dim): - """Return the runtime key coordinate for tcgen05 SW128 decay operands.""" - - key_mask = cutlass.Int32(8) ^ ((tok_coord & cutlass.Int32(2)) * cutlass.Int32(16)) - return key_dim ^ key_mask - - -@cute.jit -def _diag_idx(cfg, key_dim): +def diag_idx(cfg, key_dim): """Return the SW32 index for one entry in the 8-block diagonal.""" block = key_dim // cutlass.Int32(16) @@ -286,7 +288,7 @@ def _diag_idx(cfg, key_dim): @cute.jit -def _tmaldg_warp( +def tmaldg_warp( cfg, total_tiles, bidx, @@ -296,7 +298,6 @@ def _tmaldg_warp( mSched, sSched, lane, - tma_tx_bytes, sBeta_raw, sGate_raw, sK_raw, @@ -311,12 +312,9 @@ def _tmaldg_warp( desc_w_base, bars, ) -> None: - """TMA-LDG warp role (warp 14): persistent tile-scheduler loop + - per-chunk q/k/v/gate/beta/w G->S loads on one shared tx-count mbarrier. - Loads go through the per-(batch, head) descriptor array: head grouping - and the sequence base live in each descriptor and the token extent is - capped per sequence, so coordinates are sequence-relative and tail - chunks zero-fill in hardware.""" + """TMA-LDG warp role (warp 14): persistent scheduler loop issuing the + per-chunk Q/K/V/Beta/W/Gate G->S loads.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) sQ_tma = SmemTile( base=sQ_raw, @@ -384,7 +382,6 @@ def _tmaldg_warp( tma_granu_elems=32, tma_subtile_stride_elems=(cfg.b_t * 32), ) - tma_index = PipelineState.start(phase=0) raw_index = PipelineState.start(phase=1) sched_state = PipelineState.start(phase=1) tile_idx = cutlass.Int32(bidx) @@ -392,14 +389,18 @@ def _tmaldg_warp( batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) - slot = (batch_idx * cutlass.Int32(cfg.n_heads_out) + head_idx) * cutlass.Int32(TENSOR_MAP_QWORDS) + head_o = head_idx + head_q = head_idx if cfg.q_ratio == 1 else head_idx // cutlass.Int32(cfg.q_ratio) + head_k = head_idx if cfg.k_ratio == 1 else head_idx // cutlass.Int32(cfg.k_ratio) + head_v = head_idx if cfg.v_ratio == 1 else head_idx // cutlass.Int32(cfg.v_ratio) + slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) desc_q_slot = (desc_q_base + slot).tospace(cutlass.AddressSpace.generic) desc_k_slot = (desc_k_base + slot).tospace(cutlass.AddressSpace.generic) desc_v_slot = (desc_v_base + slot).tospace(cutlass.AddressSpace.generic) desc_gate_slot = (desc_gate_base + slot).tospace(cutlass.AddressSpace.generic) desc_beta_slot = (desc_beta_base + slot).tospace(cutlass.AddressSpace.generic) desc_w_slot = (desc_w_base + slot).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): + if elect_one: tma_tensormap_acquire(desc_q_slot) tma_tensormap_acquire(desc_k_slot) tma_tensormap_acquire(desc_v_slot) @@ -408,32 +409,54 @@ def _tmaldg_warp( tma_tensormap_acquire(desc_w_slot) for chunk_idx in cutlass.range(cstart, wend, 1, unroll=1): chunk_start = chunk_idx * cfg.b_t - bars.mb_inputs_done[raw_index.idx].wait(raw_index.phase) - # ---- q/k/v/gate/beta/w TMA loads ------------------------------------ - if nvvm.elect_sync(): - bars.mb_tma_done.arrive(n_bytes=tma_tx_bytes) - q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), chunk_start) - tma_load_tile(sQ_tma[raw_index.idx], q_slice, bars.mb_tma_done.smem_ptr, acquire=False) - k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), chunk_start) - tma_load_tile(sK_tma[raw_index.idx], k_slice, bars.mb_tma_done.smem_ptr, acquire=False) - v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), chunk_start) - tma_load_tile(sV_tma[raw_index.idx], v_slice, bars.mb_tma_done.smem_ptr, acquire=False) - beta_slice = tma_slice_runtime_desc(desc_beta_slot, cutlass.Int32(0), chunk_start) - tma_load_tile(sBeta_tma[raw_index.idx], beta_slice, bars.mb_tma_done.smem_ptr, acquire=False) - w_slice = tma_slice_runtime_desc(desc_w_slot, cutlass.Int32(0), chunk_start) - tma_load_tile(sW_tma[raw_index.idx], w_slice, bars.mb_tma_done.smem_ptr, acquire=False) - gate_slice = tma_slice_runtime_desc(desc_gate_slot, cutlass.Int32(0), chunk_start) - tma_load_tile(sGate_tma[raw_index.idx], gate_slice, bars.mb_tma_done.smem_ptr, acquire=False) - - bars.mb_tma_done.wait(tma_index.phase) - tma_index = advance(tma_index, 1) - bars.mb_inputs_ready[raw_index.idx].arrive() + + # ---- Q load ---------------------------------------------------------- + bars.mb_q_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_q_ready[raw_index.idx].arrive(n_bytes=cfg.tma_q_bytes) + q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), head_q, chunk_start) + tma_load_tile(sQ_tma[raw_index.idx], q_slice, bars.mb_q_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- K load ---------------------------------------------------------- + bars.mb_k_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_k_ready[raw_index.idx].arrive(n_bytes=cfg.tma_k_bytes) + k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, chunk_start) + tma_load_tile(sK_tma[raw_index.idx], k_slice, bars.mb_k_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- V load ---------------------------------------------------------- + bars.mb_v_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_v_ready[raw_index.idx].arrive(n_bytes=cfg.tma_v_bytes) + v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, chunk_start) + tma_load_tile(sV_tma[raw_index.idx], v_slice, bars.mb_v_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- Beta load ------------------------------------------------------- + bars.mb_beta_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_beta_ready[raw_index.idx].arrive(n_bytes=cfg.tma_beta_bytes) + beta_slice = tma_slice_runtime_desc(desc_beta_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sBeta_tma[raw_index.idx], beta_slice, bars.mb_beta_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- W load ---------------------------------------------------------- + bars.mb_w_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_w_ready[raw_index.idx].arrive(n_bytes=cfg.tma_w_bytes) + w_slice = tma_slice_runtime_desc(desc_w_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sW_tma[raw_index.idx], w_slice, bars.mb_w_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- Gate load ------------------------------------------------------- + bars.mb_gate_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_gate_ready[raw_index.idx].arrive(n_bytes=cfg.tma_gate_bytes) + gate_slice = tma_slice_runtime_desc(desc_gate_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sGate_tma[raw_index.idx], gate_slice, bars.mb_gate_ready[raw_index.idx].smem_ptr, acquire=False) raw_index = advance(raw_index, cfg.smem_raw_stages) - tile_idx, sched_state = _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) + tile_idx, sched_state = sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) @cute.jit -def _super_mma_warp( +def super_mma_warp( cfg, total_tiles, bidx, @@ -443,21 +466,20 @@ def _super_mma_warp( sSched, lane, sK_inv_raw, - sPairwise_raw, + sQk_raw, sK_decay_raw, bars, ) -> None: - """Super-MMA warp role (warp 12): persistent tile-scheduler loop + - register-MMA kk^T, L = tril(kk, -1), and the Neumann-series A_inv, - staged to pairwise SMEM.""" + """Super-MMA warp role (warp 12): persistent scheduler loop computing the + Neumann-series A_inv by register MMA.""" nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) - # ---- ldmatrix/stmatrix lane decode --------------------------------- + + # ---- ldmatrix/stmatrix lane decode ------------------------------------------- rhs_row_coord = lane % 8 + (cutlass.Int32(8) if (lane // 16) else cutlass.Int32(0)) rhs_col_offset = cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0) lhs_row_coord = lane % 8 + (cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0)) lhs_col_offset = cutlass.Int32(8) if ((lane // 8) // 2) else cutlass.Int32(0) - decay_key_mask = cutlass.Int32(8) ^ ((lhs_row_coord & cutlass.Int32(2)) * cutlass.Int32(16)) - elems_per_128b = cutlass.Int32(64) + decay_key_mask = cutlass.Int32(8) stsm_row_coord = lane & 7 stsm_col_coord = cutlass.Int32(0) if (lane // 8) & 1: @@ -465,25 +487,26 @@ def _super_mma_warp( if lane // 8 >= 2: stsm_col_coord = cutlass.Int32(8) stsm_idx = swizzle_lin_S(stsm_row_coord * cfg.b_t + (stsm_col_coord ^ (cfg.b_t // 2)), bbits=1, mbase=3, sshift=3) - gbase = cutlass.Int32(0) + global_chunk_base = cutlass.Int32(0) sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) while tile_idx < total_tiles: batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) - sk_nt = wend - cstart # processed chunks; ring bookkeeping runs on gbase + li - for li in cutlass.range(sk_nt, unroll=1): - gc = gbase + li - decay_stage = gc % cfg.smem_decay_stages - pairwise_stage = gc % cfg.smem_pairwise_stages + num_tile_chunks = wend - cstart # processed chunks; ring bookkeeping runs on global_chunk_base + local_chunk + for local_chunk in cutlass.range(num_tile_chunks, unroll=1): + global_chunk = global_chunk_base + local_chunk + decay_stage = global_chunk % cfg.smem_decay_stages + qk_stage = global_chunk % cfg.smem_qk_stages sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) sK_decay_ptr = sK_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) - sPairwise_ptr = sPairwise_raw.data_ptr() + pairwise_stage * (2 * cfg.b_t * cfg.b_t) + sQk_ptr = sQk_raw.data_ptr() + qk_stage * (2 * cfg.b_t * cfg.b_t) + + bars.mb_a_done[qk_stage].wait(((global_chunk // cfg.smem_qk_stages) + 1) % 2) + bars.mb_k_decay_inv_cg0_ready[decay_stage].wait((global_chunk // cfg.smem_decay_stages) % 2) - bars.mb_a_done[pairwise_stage].wait(((gc // cfg.smem_pairwise_stages) + 1) % 2) - bars.mb_k_decay_cg0_ready[decay_stage].wait((gc // cfg.smem_decay_stages) % 2) - # ---- kk^T register MMA over the K blocks --------------------------- + # ---- KK = K_decay @ K_inv^T ------------------------------------------ kk_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): kk_acc[accum_idx] = cutlass.Float32(0.0) @@ -502,18 +525,11 @@ def _super_mma_warp( ) # Load A operand storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask - storage_slice = storage_key // elems_per_128b - key_in_slice = storage_key - storage_slice * elems_per_128b - storage_phase = key_in_slice // cutlass.Int32(16) - byte_in_slice = ( - lhs_row_coord * cutlass.Int32(128) - + storage_phase * cutlass.Int32(32) - + (key_in_slice - storage_phase * cutlass.Int32(16)) * cutlass.Int32(2) - ) + storage_slice = storage_key // 64 kk_lhs_vec = nvvm.ldmatrix( sK_decay_ptr - + storage_slice * cutlass.Int32(cfg.b_t) * elems_per_128b - + ((byte_in_slice ^ ((lhs_row_coord & cutlass.Int32(7)) << 4)) // cutlass.Int32(2)), + + storage_slice * (cfg.b_t * 64) + + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) @@ -527,7 +543,8 @@ def _super_mma_warp( N=16, ab_dtype=cfg.io_dtype, ) - # ---- L = tril(kk, -1) fragment -------------------------------------- + + # ---- L = tril(KK, -1) fragment --------------------------------------- row_lo = lane // 4 row_hi = row_lo + cutlass.Int32(8) l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) @@ -545,7 +562,7 @@ def _super_mma_warp( l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) - # ---- A_inv = I - L, then three Neumann doubling rounds ------------- + # ---- A_inv = I - L, then three Neumann doubling rounds --------------- ainv_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): row_coord = row_lo @@ -558,15 +575,16 @@ def _super_mma_warp( ainv_acc[accum_idx] = eye - l_values[accum_idx] lpow_a0, lpow_a1, lpow_a2, lpow_a3 = l_a0, l_a1, l_a2, l_a3 + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(l_a0), movmatrix_16b(l_a1), movmatrix_16b(l_a2), movmatrix_16b(l_a3) for _round in cutlass.range_constexpr(3): - # Lpow <- Lpow @ Lpow (packed A-layout fragments, B via movmatrix) + # ---- Lpow = Lpow @ Lpow ------------------------------------------ sq_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): sq_acc[accum_idx] = cutlass.Float32(0.0) mma_step( sq_acc, (lpow_a0, lpow_a1, lpow_a2, lpow_a3), - (movmatrix_16b(lpow_a0), movmatrix_16b(lpow_a1), movmatrix_16b(lpow_a2), movmatrix_16b(lpow_a3)), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), k_step=0, M=16, N=16, @@ -576,29 +594,39 @@ def _super_mma_warp( lpow_a1 = fp32_to_fp16(sq_acc[2], sq_acc[3], dtype=cfg.io_dtype) lpow_a2 = fp32_to_fp16(sq_acc[4], sq_acc[5], dtype=cfg.io_dtype) lpow_a3 = fp32_to_fp16(sq_acc[6], sq_acc[7], dtype=cfg.io_dtype) - # A_inv <- A_inv + A_inv @ Lpow, keeping A_inv in registers + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(lpow_a0), movmatrix_16b(lpow_a1), movmatrix_16b(lpow_a2), movmatrix_16b(lpow_a3) + # ---- A_inv += A_inv @ Lpow --------------------------------------- upd_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): upd_acc[accum_idx] = cutlass.Float32(0.0) + ainv_p0 = fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype) + ainv_p1 = fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype) + ainv_p2 = fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype) + ainv_p3 = fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype) mma_step( upd_acc, - ( - fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype), - fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype), - fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype), - fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype), - ), - (movmatrix_16b(lpow_a0), movmatrix_16b(lpow_a1), movmatrix_16b(lpow_a2), movmatrix_16b(lpow_a3)), + (ainv_p0, ainv_p1, ainv_p2, ainv_p3), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype, ) - for accum_idx in cutlass.range_constexpr(8): - ainv_acc[accum_idx] = ainv_acc[accum_idx].to(cfg.io_dtype).to(cutlass.Float32) + upd_acc[accum_idx] + ainv_lo0, ainv_hi0 = f16x2_to_f32(ainv_p0, dtype=cfg.io_dtype) + ainv_lo1, ainv_hi1 = f16x2_to_f32(ainv_p1, dtype=cfg.io_dtype) + ainv_lo2, ainv_hi2 = f16x2_to_f32(ainv_p2, dtype=cfg.io_dtype) + ainv_lo3, ainv_hi3 = f16x2_to_f32(ainv_p3, dtype=cfg.io_dtype) + ainv_acc[0] = ainv_lo0 + upd_acc[0] + ainv_acc[1] = ainv_hi0 + upd_acc[1] + ainv_acc[2] = ainv_lo1 + upd_acc[2] + ainv_acc[3] = ainv_hi1 + upd_acc[3] + ainv_acc[4] = ainv_lo2 + upd_acc[4] + ainv_acc[5] = ainv_hi2 + upd_acc[5] + ainv_acc[6] = ainv_lo3 + upd_acc[6] + ainv_acc[7] = ainv_hi3 + upd_acc[7] nvvm.stmatrix( - sPairwise_ptr + (cfg.b_t * cfg.b_t) + stsm_idx, + sQk_ptr + (cfg.b_t * cfg.b_t) + stsm_idx, [ fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype), fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype), @@ -609,14 +637,14 @@ def _super_mma_warp( shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_a_ready[pairwise_stage].arrive() - bars.mb_kk_qk_super_mma_done[decay_stage].arrive() - gbase += sk_nt - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_a_ready[qk_stage].arrive() + bars.mb_decay_super_done[decay_stage].arrive() + global_chunk_base += num_tile_chunks + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) @cute.jit -def _tcgen05_mma_warp( +def tcgen05_mma_warp( cfg, total_tiles, bidx, @@ -624,24 +652,34 @@ def _tcgen05_mma_warp( cu_seqlens, mWorkItems, sSched, - tmem_hold, - sPairwise, + sTmem_base, + sQk, sK_decay, sK_restore, sQ_decay, sState_scale_diag, bars, ) -> None: - """tcgen05-MMA warp role (warp 13): persistent tile-scheduler loop, issues - all six state GEMMs in dependency order and owns the TMEM lifecycle.""" + """tcgen05-MMA warp role (warp 13): persistent scheduler loop issuing + every tcgen05 GEMM.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) - tmem_base = tmem_hold.load() + tmem_base = sTmem_base.load() + state_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) + state_dsts = tuple(nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset + k * 16, cutlass.Float32) for k in range(cfg.d_k // 16)) + state_k_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_k_acc_offset, cutlass.Float32) + update_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_acc_offset, cutlass.Float32) + rhs_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_rhs_inp_offset, cutlass.Int8) + update_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_inp_offset, cutlass.Int8) + state_dst_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset, cutlass.Float32) state_inp_index = PipelineState.start(phase=0) + state_read_index = PipelineState.start(phase=0) rhs_index = PipelineState.start(phase=0) update_index = PipelineState.start(phase=0) final_state_index = PipelineState.start(phase=0) qk_scale_index = PipelineState.start(phase=0) - # ---- chunk-invariant GEMM descriptors ------------------------------ + + # ---- chunk-invariant GEMM descriptors ---------------------------------------- bpe = cfg.io_dtype.width // 8 idesc_acc = nvvm.Tcgen05InstrDesc.build( c_dtype=cutlass.Float32, @@ -691,7 +729,7 @@ def _tcgen05_mma_warp( idesc=idesc_diag, kind=nvvm.Tcgen05MMAKind.F16, ) - bmm_pairwise_desc = MmaDesc( + bmm_qk_desc = MmaDesc( M=cfg.d_v, N=cfg.b_t, K=cfg.b_t, @@ -715,128 +753,127 @@ def _tcgen05_mma_warp( idesc=idesc_final_state, kind=nvvm.Tcgen05MMAKind.F16, ) - gbase = cutlass.Int32(0) + STATE_A_SEG = bmm_state_desc.sps_B * bmm_state_desc.tmem_advance_A + STATE_B_SEG = bmm_state_desc.smem_subtile_B >> 4 + global_chunk_base = cutlass.Int32(0) sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) while tile_idx < total_tiles: batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) - sk_nt = wend - cstart - for li in cutlass.range(sk_nt, unroll=1): - gc = gbase + li - q_state_acc_stage = gc % cfg.tmem_q_state_acc_stages - decay_stage = gc % cfg.smem_decay_stages + num_tile_chunks = wend - cstart + for local_chunk in cutlass.range(num_tile_chunks, unroll=1): + global_chunk = global_chunk_base + local_chunk + q_state_acc_stage = global_chunk % cfg.tmem_q_state_acc_stages + decay_stage = global_chunk % cfg.smem_decay_stages state_scale_diag_stage = qk_scale_index.idx - pairwise_stage = gc % cfg.smem_pairwise_stages + qk_stage = global_chunk % cfg.smem_qk_stages sK_decay_stage = sK_decay[decay_stage] sQ_decay_stage = sQ_decay[decay_stage] sK_restore_stage = sK_restore[decay_stage] sState_scale_diag_stage = sState_scale_diag[state_scale_diag_stage] - sPairwise_stage = sPairwise[pairwise_stage] + sQk_stage = sQk[qk_stage] - # ---- state*k -> state_k_acc ---------------------------------------- - bars.mb_k_decay_cg0_ready[decay_stage].wait((gc // cfg.smem_decay_stages) % 2) + # ---- state_k = state(T) @ K_decay^T ---------------------------------- + bars.mb_k_decay_inv_cg0_ready[decay_stage].wait((global_chunk // cfg.smem_decay_stages) % 2) bars.mb_state_inp_ready.wait(state_inp_index.phase) state_inp_index = advance(state_inp_index, 1) desc_k_decay = sK_decay_stage.desc() - state_a_tmem_base = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) - mma_ts( - bmm_state_desc, - state_a_tmem_base, - desc_k_decay, - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_k_acc_offset, cutlass.Float32), - accumulate=False, - ) + for s in cutlass.range_constexpr(bmm_state_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_state_desc.sps_B): + mma_ts_step( + bmm_state_desc, + state_inp_ptr.subview(s * STATE_A_SEG), + desc_k_decay + s * STATE_B_SEG, + state_k_acc_ptr, + k, + cutlass.Boolean(s + k > 0), + ) - if nvvm.elect_sync(): + if elect_one: bars.mb_state_k_acc_ready.arrive(cta_group=1) - # ---- state*q -> q_state_acc (stays live until qk@update fuses into o) + # ---- q_state = state(T) @ Q_decay^T ---------------------------------- bars.mb_qk_scale_ready[qk_scale_index.idx].wait(qk_scale_index.phase) - bars.mb_o_acc_done[q_state_acc_stage].wait(((gc // cfg.tmem_q_state_acc_stages + cutlass.Int32(1)) % cutlass.Int32(2))) + bars.mb_o_acc_done[q_state_acc_stage].wait(((global_chunk // cfg.tmem_q_state_acc_stages + cutlass.Int32(1)) % cutlass.Int32(2))) desc_q_decay = sQ_decay_stage.desc() - state_a_tmem_base = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) - mma_ts( - bmm_state_desc, - state_a_tmem_base, - desc_q_decay, - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_q_state_acc_offset + q_state_acc_stage * cfg.b_t, cutlass.Float32), - accumulate=False, - ) + q_state_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_q_state_acc_offset + q_state_acc_stage * cfg.b_t, cutlass.Float32) + for s in cutlass.range_constexpr(bmm_state_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_state_desc.sps_B): + mma_ts_step( + bmm_state_desc, + state_inp_ptr.subview(s * STATE_A_SEG), + desc_q_decay + s * STATE_B_SEG, + q_state_acc_ptr, + k, + cutlass.Boolean(s + k > 0), + ) - if nvvm.elect_sync(): - bars.mb_kk_qk_mma_done[decay_stage].arrive(cta_group=1) + if elect_one: + bars.mb_decay_done[decay_stage].arrive(cta_group=1) + + # WAR: CG1's post-publish f32 checkpoint read must retire before the overwrite + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_read_done.wait(state_read_index.phase) + state_read_index = advance(state_read_index, 1) - # ---- state decay (per-k-atom diag blocks) ---------------------------- + # ---- state decay = state(T) @ diag(exp2(g_last)) (per-k-atom blocks) ---- desc_diag = sState_scale_diag_stage.desc() for k_block in cutlass.range_constexpr(cfg.d_k // 16): - mma_ts( + mma_ts_step( bmm_diag_desc, - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset + k_block * 8, cutlass.Int8), + state_inp_ptr.subview(k_block * bmm_diag_desc.tmem_advance_A), desc_diag.advance_start_address(k_block * 256 * 2), - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_offset + k_block * 16, cutlass.Float32), - accumulate=False, + state_dsts[k_block], + 0, + cutlass.Boolean(False), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_state_scale_diag_done[state_scale_diag_stage].arrive(cta_group=1) - # ---- update = A_inv @ rhs -> update_acc ------------------------------ - bars.mb_a_ready[pairwise_stage].wait((gc // cfg.smem_pairwise_stages) % 2) + # ---- update = rhs(T) @ A_inv ----------------------------------------- + bars.mb_a_ready[qk_stage].wait((global_chunk // cfg.smem_qk_stages) % 2) bars.mb_rhs_ready.wait(rhs_index.phase) rhs_index = advance(rhs_index, 1) - lhs_tmem = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_rhs_inp_offset, cutlass.Int8) - desc_pairwise = sPairwise_stage.shifted((cfg.b_t * cfg.b_t)).desc() - mma_ts( - bmm_pairwise_desc, - lhs_tmem, - desc_pairwise, - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_acc_offset, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): + desc_qk = sQk_stage.shifted((cfg.b_t * cfg.b_t)).desc() + mma_ts_step(bmm_qk_desc, rhs_inp_ptr, desc_qk, update_acc_ptr, 0, cutlass.Boolean(False)) + if elect_one: bars.mb_update_acc_ready.arrive(cta_group=1) - # ---- final_state += update @ k_restore ------------------------------- + # ---- final_state += update(T) @ K_restore ---------------------------- bars.mb_update_ready.wait(update_index.phase) update_index = advance(update_index, 1) - update_tmem = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_inp_offset, cutlass.Int8) desc_k_restore = sK_restore_stage.desc() - mma_ts( - bmm_final_state_desc, - update_tmem, - desc_k_restore, - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_offset, cutlass.Float32), - accumulate=True, - ) - if nvvm.elect_sync(): + mma_ts_step(bmm_final_state_desc, update_inp_ptr, desc_k_restore, state_dst_ptr, 0, cutlass.Boolean(True)) + if elect_one: bars.mb_k_restore_done[decay_stage].arrive(cta_group=1) - # ---- o += qk @ update -> q_state_acc ---------------------------------- - bars.mb_qk_acc_ready[pairwise_stage].wait((gc // cfg.smem_pairwise_stages) % 2) - lhs_tmem = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_inp_offset, cutlass.Int8) - desc_pairwise = sPairwise_stage.desc() - mma_ts( - bmm_pairwise_desc, - lhs_tmem, - desc_pairwise, + # ---- O += update(T) @ QK --------------------------------------------- + bars.mb_qk_acc_ready[qk_stage].wait((global_chunk // cfg.smem_qk_stages) % 2) + desc_qk = sQk_stage.desc() + mma_ts_step( + bmm_qk_desc, + update_inp_ptr, + desc_qk, nvvm.make_tmem_ptr(tmem_base + cfg.tmem_q_state_acc_offset + q_state_acc_stage * cfg.b_t, cutlass.Float32), - accumulate=True, + 0, + cutlass.Boolean(True), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_o_acc_ready.arrive(cta_group=1) - bars.mb_a_done[pairwise_stage].arrive(cta_group=1) + bars.mb_a_done[qk_stage].arrive(cta_group=1) qk_scale_index = advance(qk_scale_index, cfg.smem_state_scale_diag_stages) bars.mb_final_state_stored.wait(final_state_index.phase) final_state_index = advance(final_state_index, 1) - gbase += sk_nt - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + global_chunk_base += num_tile_chunks + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) nvvm.tcgen05_dealloc( nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), cutlass.Int32(512), @@ -845,7 +882,7 @@ def _tcgen05_mma_warp( @cute.jit -def _epilogue_warp( +def epilogue_warp( cfg, total_tiles, bidx, @@ -857,22 +894,23 @@ def _epilogue_warp( mO, sK_inv_raw, sO_raw, - sPairwise_raw, + sQk_raw, sQ_decay_raw, - sH_raw, + sCheckpoint_raw, desc_o_base, - desc_h_base, + desc_checkpoint_base, checkpoint_every_n_tokens, bars, ) -> None: - """Epilogue warp role (warp 15): register-MMA qk (causal), A_inv qk - staging, and the O TMA store drain.""" + """Epilogue warp role (warp 15): register-MMA QK (causal) and the O TMA + store drain.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) if cutlass.const_expr(cfg.enable_checkpoints): - sH_tma = SmemTile( - base=sH_raw, + sCheckpoint_tma = SmemTile( + base=sCheckpoint_raw, elems_per_stage=(cfg.d_k * cfg.d_v), - stages=1, + stages=cfg.smem_checkpoint_stages, leading_byte_offset=0, stride_byte_offset=0, layout=0, @@ -880,7 +918,7 @@ def _epilogue_warp( tma_granu_elems=64, tma_subtile_stride_elems=cfg.d_k * 64, ) - h_ready_index = PipelineState.start(phase=0) + checkpoint_ready_index = PipelineState.start(phase=0) sO_tma = SmemTile( base=sO_raw, elems_per_stage=(cfg.b_t * cfg.d_v), @@ -893,13 +931,13 @@ def _epilogue_warp( tma_subtile_stride_elems=cfg.b_t * 64, ) qk_scale_index = PipelineState.start(phase=0) - # ---- ldmatrix/stmatrix lane decode --------------------------------- + + # ---- ldmatrix/stmatrix lane decode ------------------------------------------- rhs_row_coord = lane % 8 + (cutlass.Int32(8) if (lane // 16) else cutlass.Int32(0)) rhs_col_offset = cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0) lhs_row_coord = lane % 8 + (cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0)) lhs_col_offset = cutlass.Int32(8) if ((lane // 8) // 2) else cutlass.Int32(0) - decay_key_mask = cutlass.Int32(8) ^ ((lhs_row_coord & cutlass.Int32(2)) * cutlass.Int32(16)) - elems_per_128b = cutlass.Int32(64) + decay_key_mask = cutlass.Int32(8) stsm_row_coord = lane & 7 stsm_col_coord = cutlass.Int32(0) if (lane // 8) & 1: @@ -907,7 +945,7 @@ def _epilogue_warp( if lane // 8 >= 2: stsm_col_coord = cutlass.Int32(8) stsm_idx = swizzle_lin_S(stsm_row_coord * cfg.b_t + (stsm_col_coord ^ (cfg.b_t // 2)), bbits=1, mbase=3, sshift=3) - gbase = cutlass.Int32(0) + global_chunk_base = cutlass.Int32(0) sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) while tile_idx < total_tiles: @@ -915,27 +953,31 @@ def _epilogue_warp( cfg, tile_idx, cu_seqlens, mWorkItems ) head_o = head_idx - o_slot = (batch_idx * cutlass.Int32(cfg.n_heads_out) + head_idx) * cutlass.Int32(TENSOR_MAP_QWORDS) + o_slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) desc_o_slot = (desc_o_base + o_slot).tospace(cutlass.AddressSpace.generic) if cutlass.const_expr(cfg.enable_checkpoints): - desc_h_slot = (desc_h_base + o_slot).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): - tma_tensormap_acquire(desc_h_slot) - if nvvm.elect_sync(): + desc_checkpoint_slot = (desc_checkpoint_base + o_slot).tospace(cutlass.AddressSpace.generic) + checkpoint_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) + checkpoint_quot = (cstart + cutlass.Int32(1)) // checkpoint_chunks + checkpoint_mod = (cstart + cutlass.Int32(1)) % checkpoint_chunks + if elect_one: + tma_tensormap_acquire(desc_checkpoint_slot) + if elect_one: tma_tensormap_acquire(desc_o_slot) - sk_nt = wend - cstart - for li in cutlass.range(sk_nt, unroll=1): - chunk_idx = cstart + li - gc = gbase + li - decay_stage = gc % cfg.smem_decay_stages - pairwise_stage = gc % cfg.smem_pairwise_stages + num_tile_chunks = wend - cstart + for local_chunk in cutlass.range(num_tile_chunks, unroll=1): + chunk_idx = cstart + local_chunk + global_chunk = global_chunk_base + local_chunk + decay_stage = global_chunk % cfg.smem_decay_stages + qk_stage = global_chunk % cfg.smem_qk_stages sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) sQ_decay_ptr = sQ_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) - sPairwise_ptr = sPairwise_raw.data_ptr() + pairwise_stage * (2 * cfg.b_t * cfg.b_t) + sQk_ptr = sQk_raw.data_ptr() + qk_stage * (2 * cfg.b_t * cfg.b_t) - bars.mb_a_done[pairwise_stage].wait(((gc // cfg.smem_pairwise_stages) + 1) % 2) + bars.mb_a_done[qk_stage].wait(((global_chunk // cfg.smem_qk_stages) + 1) % 2) bars.mb_qk_scale_ready[qk_scale_index.idx].wait(qk_scale_index.phase) - # ---- qk register MMA (inclusive-causal) ---------------------------- + + # ---- QK = Q_decay @ K_inv^T ----------------------------------------- qk_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): qk_acc[accum_idx] = cutlass.Float32(0.0) @@ -954,18 +996,11 @@ def _epilogue_warp( ) # Load A operand storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask - storage_slice = storage_key // elems_per_128b - key_in_slice = storage_key - storage_slice * elems_per_128b - storage_phase = key_in_slice // cutlass.Int32(16) - byte_in_slice = ( - lhs_row_coord * cutlass.Int32(128) - + storage_phase * cutlass.Int32(32) - + (key_in_slice - storage_phase * cutlass.Int32(16)) * cutlass.Int32(2) - ) + storage_slice = storage_key // 64 qk_lhs_vec = nvvm.ldmatrix( sQ_decay_ptr - + storage_slice * cutlass.Int32(cfg.b_t) * elems_per_128b - + ((byte_in_slice ^ ((lhs_row_coord & cutlass.Int32(7)) << 4)) // cutlass.Int32(2)), + + storage_slice * (cfg.b_t * 64) + + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) @@ -990,7 +1025,7 @@ def _epilogue_warp( qk_acc[accum_idx] = qk_acc[accum_idx] if row_coord >= col_coord else cutlass.Float32(0.0) nvvm.stmatrix( - sPairwise_ptr + stsm_idx, + sQk_ptr + stsm_idx, [ fp32_to_fp16(qk_acc[0], qk_acc[1], dtype=cfg.io_dtype), fp32_to_fp16(qk_acc[2], qk_acc[3], dtype=cfg.io_dtype), @@ -1001,64 +1036,95 @@ def _epilogue_warp( shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_qk_acc_ready[pairwise_stage].arrive() - bars.mb_kk_qk_super_mma_done[decay_stage].arrive() + bars.mb_qk_acc_ready[qk_stage].arrive() + bars.mb_decay_super_done[decay_stage].arrive() qk_scale_index = advance(qk_scale_index, cfg.qk_scale_ready_stages) - # ---- O drain: staged output tile -> GMEM TMA store ----------------- - if li > 0: + # ---- checkpoint + O drain: checkpoint stores first (CG1 stages checkpoint before O) -- + if local_chunk > 0: output_chunk = chunk_idx - cutlass.Int32(1) output_chunk_start = output_chunk * cfg.b_t - o_stage = (gc - cutlass.Int32(1)) % cfg.smem_o_stages - bars.mb_o_tmastg_ready[o_stage].wait(((gc - cutlass.Int32(1)) // cfg.smem_o_stages) % 2) - o_slice = tma_slice_runtime_desc(desc_o_slot, cutlass.Int32(0), output_chunk_start) + o_stage = (global_chunk - cutlass.Int32(1)) % cfg.smem_o_stages + did_checkpoint = cutlass.Int32(0) + checkpoint_stage = cutlass.Int32(0) + if cutlass.const_expr(cfg.enable_checkpoints): + # ---- checkpoint store ---------------------------------------- + do_checkpoint = checkpoint_mod == 0 + if cutlass.const_expr(cfg.split_k): + do_checkpoint = do_checkpoint and chunk_idx >= wstart + checkpoint_stage = checkpoint_ready_index.idx + if do_checkpoint: + bars.mb_checkpoint_tmastg_ready[checkpoint_ready_index.idx].wait(checkpoint_ready_index.phase) + checkpoint_ready_index = advance(checkpoint_ready_index, cfg.smem_checkpoint_stages) + checkpoint_entry = checkpoint_quot - cutlass.Int32(1) + checkpoint_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), checkpoint_entry, head_o) + tma_store_tile(sCheckpoint_tma[checkpoint_stage], checkpoint_slice, acquire=False) + tma_store_commit() + did_checkpoint = cutlass.Int32(1) + checkpoint_mod = checkpoint_mod + cutlass.Int32(1) + if checkpoint_mod == checkpoint_chunks: + checkpoint_mod = cutlass.Int32(0) + checkpoint_quot = checkpoint_quot + cutlass.Int32(1) + bars.mb_o_tmastg_ready[o_stage].wait(((global_chunk - cutlass.Int32(1)) // cfg.smem_o_stages) % 2) + o_slice = tma_slice_runtime_desc(desc_o_slot, cutlass.Int32(0), head_o, output_chunk_start) + did_o = cutlass.Int32(0) if cutlass.const_expr(cfg.split_k): - # warmup chunks stage O to SMEM but never store it if output_chunk >= wstart: tma_store_tile(sO_tma[o_stage], o_slice, acquire=False) tma_store_commit() + did_o = cutlass.Int32(1) else: tma_store_tile(sO_tma[o_stage], o_slice, acquire=False) tma_store_commit() - tma_store_wait(0) - bars.mb_o_tmastg_done[o_stage].arrive() - # ---- H store: CG1 staged the state entering chunk_idx ---------- - if cutlass.const_expr(cfg.enable_checkpoints): - if li > 0: - tokens_done = chunk_idx * cutlass.Int32(cfg.b_t) - do_h = tokens_done % checkpoint_every_n_tokens == 0 - if cutlass.const_expr(cfg.split_k): - do_h = do_h and chunk_idx >= wstart - if do_h: - bars.mb_h_tmastg_ready.wait(h_ready_index.phase) - h_ready_index = advance(h_ready_index, 1) - # sequence-local entry: the per-(b,h) descriptor folds the - # sequence base into GLOBAL_ADDRESS and caps the extent - h_entry = tokens_done // checkpoint_every_n_tokens - cutlass.Int32(1) - h_slice = tma_slice_runtime_desc(desc_h_slot, cutlass.Int32(0), cutlass.Int32(0), h_entry) - tma_store_tile(sH_tma[0], h_slice, acquire=False) - tma_store_commit() + did_o = cutlass.Int32(1) + if cutlass.const_expr(cfg.enable_checkpoints): + if did_checkpoint == 1 and did_o == 1: + tma_store_wait(1) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].arrive() tma_store_wait(0) - bars.mb_h_tmastg_done.arrive() - # ---- last computed chunk drain (always owned: it is wend - 1) ------ - if sk_nt > 0: + bars.mb_o_tmastg_done[o_stage].arrive() + if did_checkpoint == 1 and did_o == 0: + tma_store_wait(0) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].arrive() + bars.mb_o_tmastg_done[o_stage].arrive() + if did_checkpoint == 0: + if did_o == 1: + tma_store_wait(0) + bars.mb_o_tmastg_done[o_stage].arrive() + else: + tma_store_wait(0) + bars.mb_o_tmastg_done[o_stage].arrive() + + # ---- last computed chunk drain (always owned: it is wend - 1) ------------ + if num_tile_chunks > 0: output_chunk = wend - cutlass.Int32(1) - og = gbase + sk_nt - cutlass.Int32(1) + last_global_chunk = global_chunk_base + num_tile_chunks - cutlass.Int32(1) output_chunk_start = output_chunk * cfg.b_t - o_stage = og % cfg.smem_o_stages - bars.mb_o_tmastg_ready[o_stage].wait((og // cfg.smem_o_stages) % 2) - # a partial last chunk is clipped by the descriptor's token extent - o_slice = tma_slice_runtime_desc(desc_o_slot, cutlass.Int32(0), output_chunk_start) + o_stage = last_global_chunk % cfg.smem_o_stages + bars.mb_o_tmastg_ready[o_stage].wait((last_global_chunk // cfg.smem_o_stages) % 2) + o_slice = tma_slice_runtime_desc(desc_o_slot, cutlass.Int32(0), head_o, output_chunk_start) tma_store_tile(sO_tma[o_stage], o_slice, acquire=False) tma_store_commit() tma_store_wait(0) bars.mb_o_tmastg_done[o_stage].arrive() - gbase += sk_nt - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + global_chunk_base += num_tile_chunks + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def gate_scale(cfg, raw_gate: cutlass.Float32) -> cutlass.Float32: + """Map raw gate to the log2-domain decay increment.""" + + if cutlass.const_expr(cfg.safe_gate): + half = cutlass.Float32(0.5) + sigmoid = cute.math.tanh(raw_gate * half, approx=True) * half + half + return cfg.gate_scale_log2 * sigmoid + # Default ABI: gate arrives in natural-log space + return raw_gate * cutlass.Float32(LOG2_E) @cute.jit -def _compute0_warp_group( +def compute0_warp_group( cfg, total_tiles, bidx, @@ -1084,9 +1150,8 @@ def _compute0_warp_group( sState_scale_diag_raw, bars, ) -> None: - """CG0 warp role (warps 0-7, two ping-pong groups): persistent - tile-scheduler loop + gate prefix scan and the decay/restore operand - materialization into tcgen05 SMEM.""" + """CG0 warp role (warps 0-7, two ping-pong groups): Gate prefix scan and + the decay/restore operand materialization.""" nvvm.setmaxregister(cfg.num_regs_compute_group_0, nvvm.SetMaxRegisterAction.INCREASE) cg0_warp = warp_idx - cfg.compute_group_0_warp_ids[0] cg0_group_id = cg0_warp // cfg.cg0_warps_per_group @@ -1094,28 +1159,32 @@ def _compute0_warp_group( prefix_dim = cg0_local_warp * cfg.threads_per_warp + lane cg0_a_log_exp = cutlass.Float32(1.0) cg0_dt_bias_value = cutlass.Float32(0.0) - gbase = cutlass.Int32(0) + global_chunk_base = cutlass.Int32(0) sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) + opaque_one = opaque_f32_zero() + cutlass.Float32(1.0) while tile_idx < total_tiles: batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) head_o = head_idx - sk_nt = wend - cstart + num_tile_chunks = wend - cstart if cutlass.const_expr(cfg.safe_gate): - # per-head safe-gate constants straight from GMEM (the head - # changes per tile, so there is no SMEM staging to handshake) - if sk_nt > 0: + if num_tile_chunks > 0: cg0_a_log_exp = cute.math.exp2(mA_log[head_o].to(cutlass.Float32) * LOG2_E, fastmath=True) cg0_dt_bias_value = mDt_bias[head_o, prefix_dim].to(cutlass.Float32) - for li in cutlass.range(cg0_group_id, sk_nt, cfg.cg0_group_count, unroll=1): - chunk_idx = cstart + li - gc = gbase + li + cg0_first_global_chunk = global_chunk_base + cutlass.Int32(cg0_group_id) + diag_ring_stage = cg0_first_global_chunk % cutlass.Int32(cfg.smem_state_scale_diag_stages) + diag_ring_phase = (cg0_first_global_chunk // cutlass.Int32(cfg.smem_state_scale_diag_stages)) % cutlass.Int32(2) + raw_ring_stage = cg0_first_global_chunk % cutlass.Int32(cfg.smem_raw_stages) + raw_ring_phase = (cg0_first_global_chunk // cutlass.Int32(cfg.smem_raw_stages)) % cutlass.Int32(2) + for local_chunk in cutlass.range(cg0_group_id, num_tile_chunks, cfg.cg0_group_count, unroll=1): + chunk_idx = cstart + local_chunk + global_chunk = global_chunk_base + local_chunk chunk_start = chunk_idx * cfg.b_t - decay_stage = gc % cfg.smem_decay_stages - raw_stage = gc % cfg.smem_raw_stages - state_scale_diag_stage = gc % cfg.smem_state_scale_diag_stages + decay_stage = global_chunk % cfg.smem_decay_stages + raw_stage = raw_ring_stage + state_scale_diag_stage = diag_ring_stage qk_scale_ready_stage = state_scale_diag_stage sQ_ptr = sQ_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) sK_ptr = sK_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) @@ -1129,60 +1198,23 @@ def _compute0_warp_group( sK_restore_ptr = sK_restore_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) sState_scale_diag_ptr = sState_scale_diag_raw.data_ptr() + state_scale_diag_stage * ((cfg.d_k // 16) * 256) - bars.mb_inputs_ready[raw_stage].wait((gc // cfg.smem_raw_stages) % 2) - - # ---- tail chunk: zero-fill raw staging past seqlen ----------------- - if chunk_start + cutlass.Int32(cfg.b_t) > seqlen_b: - if cg0_local_warp == 0: - f16_zero = mQ.element_type(0.0) - f16_zero_vec = cutlass.Vector.from_elements( - ( - f16_zero, - f16_zero, - f16_zero, - f16_zero, - f16_zero, - f16_zero, - f16_zero, - f16_zero, - ), - mQ.element_type, - ) - f32_zero = cutlass.Float32(0.0) - f32_zero_vec = cutlass.Vector.from_elements( - (f32_zero, f32_zero, f32_zero, f32_zero), - cutlass.Float32, - ) - for row in cutlass.range_constexpr(cfg.b_t): - token_idx = chunk_start + cutlass.Int32(row) - if token_idx >= seqlen_b: - if lane < (cfg.d_k // 8): - f16_dim_base = lane * 8 - f16_segment = f16_dim_base // 64 - f16_segment_dim = f16_dim_base - f16_segment * 64 - f16_idx = f16_segment * (cfg.b_t * 64) + row * 64 + swizzle_xor_128b(row, f16_segment_dim, elem_bytes=2) - (sQ_ptr + f16_idx).store(f16_zero_vec, alignment=16) - (sK_ptr + f16_idx).store(f16_zero_vec, alignment=16) - (sV_ptr + f16_idx).store(f16_zero_vec, alignment=16) - (sBeta_ptr + f16_idx).store(f16_zero_vec, alignment=16) - (sW_ptr + f16_idx).store(f16_zero_vec, alignment=16) - if lane < (cfg.d_k // 4): - f32_dim_base = lane * 4 - f32_segment = f32_dim_base // 32 - f32_segment_dim = f32_dim_base - f32_segment * 32 - f32_idx = f32_segment * (cfg.b_t * 32) + row * 32 + swizzle_xor_128b(row, f32_segment_dim, elem_bytes=4) - (sGate_ptr + f32_idx).store(f32_zero_vec, alignment=16) - nvvm.barrier_cta_sync(cfg.nbar_cg0_group0_id + cg0_group_id, thread_count=cfg.cg0_threads_per_group) + bars.mb_gate_ready[raw_stage].wait(raw_ring_phase) + bars.mb_q_ready[raw_stage].wait(raw_ring_phase) + bars.mb_k_ready[raw_stage].wait(raw_ring_phase) + bars.mb_beta_ready[raw_stage].wait(raw_ring_phase) row_group_start = cg0_local_warp * (cfg.b_t // cfg.cg0_warps_per_group) lane_row_group = lane // 8 lane_in_row_group = lane - lane_row_group * 8 decay_row = row_group_start + lane_row_group - - g_prefix_ptr = sGate_ptr + decay_key_mask = cutlass.Int32(8) prefix_dim = cg0_local_warp * cfg.threads_per_warp + lane - # ---- gate prefix scan: cumulative log-gate per key channel -------- + + # ---- Gate prefix scan ----------------------------------------------- + f32_segment = prefix_dim // 32 + prefix_seg_base = f32_segment * (cfg.b_t * 32) + prefix_col = prefix_dim - f32_segment * 32 g_prefix_regs = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) if cutlass.const_expr(cfg.safe_gate): valid_rows = seqlen_b - chunk_idx * cutlass.Int32(cfg.b_t) @@ -1190,21 +1222,17 @@ def _compute0_warp_group( for row_pair in cutlass.range_constexpr(cfg.b_t // 2): row0 = row_pair * 2 row1 = row0 + 1 - f32_segment = prefix_dim // 32 - f32_segment_dim = prefix_dim - f32_segment * 32 - prefix_idx0 = f32_segment * (cfg.b_t * 32) + row0 * 32 + swizzle_xor_128b(row0, f32_segment_dim, elem_bytes=4) - f32_segment = prefix_dim // 32 - f32_segment_dim = prefix_dim - f32_segment * 32 - prefix_idx1 = f32_segment * (cfg.b_t * 32) + row1 * 32 + swizzle_xor_128b(row1, f32_segment_dim, elem_bytes=4) + prefix_idx0 = prefix_seg_base + swizzle_xor_128b(row0, row0 * 32 + prefix_col, elem_bytes=4) + prefix_idx1 = prefix_seg_base + swizzle_xor_128b(row1, row1 * 32 + prefix_col, elem_bytes=4) gate0 = (sGate_ptr + prefix_idx0).load() gate1 = (sGate_ptr + prefix_idx1).load() gate0 = cg0_a_log_exp * (gate0 + cg0_dt_bias_value) gate1 = cg0_a_log_exp * (gate1 + cg0_dt_bias_value) - gate0 = _gate_log2( + gate0 = gate_scale( cfg, gate0, ) - gate1 = _gate_log2( + gate1 = gate_scale( cfg, gate1, ) @@ -1214,13 +1242,11 @@ def _compute0_warp_group( g_prefix_regs[row1] = gate_pair[1] else: for row in cutlass.range_constexpr(cfg.b_t): - f32_segment = prefix_dim // 32 - f32_segment_dim = prefix_dim - f32_segment * 32 - prefix_idx = f32_segment * (cfg.b_t * 32) + row * 32 + swizzle_xor_128b(row, f32_segment_dim, elem_bytes=4) + prefix_idx = prefix_seg_base + swizzle_xor_128b(row, row * 32 + prefix_col, elem_bytes=4) gate = (sGate_ptr + prefix_idx).load() token_idx = chunk_idx * cutlass.Int32(cfg.b_t) + cutlass.Int32(row) if token_idx < seqlen_b: - gate = _gate_log2( + gate = gate_scale( cfg, gate, ) @@ -1240,31 +1266,32 @@ def _compute0_warp_group( g_prefix_regs[row1] = prefix1 prefix_acc = prefix1 + # ---- exp2(g): stage prefixes + final-token decay --------------------- for row in cutlass.range_constexpr(cfg.b_t): g_prefix_regs[row] = cute.math.exp2(g_prefix_regs[row], fastmath=True) - # ---- exp2(g): stage prefixes + final-token decay ------------------ exp_g_last = g_prefix_regs[cfg.b_t - 1] for row in cutlass.range_constexpr(cfg.b_t): - f32_segment = prefix_dim // 32 - f32_segment_dim = prefix_dim - f32_segment * 32 - prefix_idx = f32_segment * (cfg.b_t * 32) + row * 32 + swizzle_xor_128b(row, f32_segment_dim, elem_bytes=4) + prefix_idx = prefix_seg_base + swizzle_xor_128b(row, row * 32 + prefix_col, elem_bytes=4) (sGate_ptr + prefix_idx).store(g_prefix_regs[row]) - # ---- state-scale diag: stage exp2(g_last) decay blocks ------------- - bars.mb_state_scale_diag_done[state_scale_diag_stage].wait((gc // cfg.smem_state_scale_diag_stages + 1) % 2) - diag_idx = _diag_idx(cfg, prefix_dim) - sState_scale_diag_ptr[diag_idx] = exp_g_last.to(cfg.io_dtype) + # ---- state-scale diag: stage exp2(g_last) decay blocks --------------- + bars.mb_state_scale_diag_done[state_scale_diag_stage].wait(diag_ring_phase ^ cutlass.Int32(1)) + sState_scale_diag_ptr[diag_idx(cfg, prefix_dim)] = exp_g_last.to(cfg.io_dtype) - nvvm.barrier_cta_sync(cfg.nbar_cg0_group0_id + cg0_group_id, thread_count=cfg.cg0_threads_per_group) + nvvm.barrier_cta_sync(cfg.cg0_group_sync_barrier_base_id + cg0_group_id, thread_count=cfg.cg0_threads_per_group) k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) raw_q_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) raw_beta_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) - # ---- optional q/k L2-norm + K_inv staging -------------------------- - q_sum_sq = cutlass.Float32(0.0) - k_sum_sq = cutlass.Float32(0.0) + + # ---- optional Q/K L2-norm ------------------------------------------- + if cutlass.const_expr(cfg.l2norm): + qk0_lo = opaque_f32_zero() + qk0_hi = opaque_f32_zero() + qk1_lo = opaque_f32_zero() + qk1_hi = opaque_f32_zero() for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 @@ -1283,19 +1310,18 @@ def _compute0_warp_group( raw_q_regs[reg_base + dim_offset] = q_val raw_k_regs[reg_base + dim_offset] = k_val beta_val = raw_beta_vec_f32[dim_offset] - if cutlass.const_expr(cfg.beta_w_sigmoid): - # Roundtrip through the io dtype to match host-side beta.sigmoid() - half = cutlass.Float32(0.5) - beta_val = (cute.math.tanh(beta_val * half, approx=True) * half + half).to(cfg.io_dtype).to(cutlass.Float32) raw_beta_regs[reg_base + dim_offset] = beta_val - q_sum_sq = q_sum_sq + q_val * q_val - k_sum_sq = k_sum_sq + k_val * k_val + if cutlass.const_expr(cfg.l2norm): + if cutlass.const_expr(dim_offset % 2 == 0): + qk0_lo, qk0_hi = ffma2(q_val, k_val, q_val, k_val, qk0_lo, qk0_hi) + else: + qk1_lo, qk1_hi = ffma2(q_val, k_val, q_val, k_val, qk1_lo, qk1_hi) - # opaque 1.0: keeps the no-l2norm packed-mul operands out of libNVVM's - # constant folder (the documented inline_ptx "n"-constraint ICE) - q_inv_norm = opaque_f32_zero() + cutlass.Float32(1.0) - k_inv_norm = opaque_f32_zero() + cutlass.Float32(1.0) + q_inv_norm = opaque_one + k_inv_norm = opaque_one if cutlass.const_expr(cfg.l2norm): + q_sum_sq = qk0_lo + qk1_lo + k_sum_sq = qk0_hi + qk1_hi q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 4, 31, kind=nvvm.Shfl.BFLY)) q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 2, 31, kind=nvvm.Shfl.BFLY)) q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 1, 31, kind=nvvm.Shfl.BFLY)) @@ -1306,7 +1332,7 @@ def _compute0_warp_group( q_inv_norm = cute.math.rsqrt(cute.math.max(q_sum_sq, norm_floor_sq), fastmath=True) k_inv_norm = cute.math.rsqrt(cute.math.max(k_sum_sq, norm_floor_sq), fastmath=True) - # ---- decay/restore operands: exp2(+-g) applied per key channel ----- + # ---- decay/restore operands: exp2(+-g) applied per key channel ------- exp_g_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) exp_g_last_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) for dim_half in cutlass.range_constexpr(2): @@ -1318,11 +1344,11 @@ def _compute0_warp_group( f32_segment = f32_dim_base // 32 f32_segment_dim = f32_dim_base - f32_segment * 32 g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) - exp_g_vec = (g_prefix_ptr + g_prefix_idx).load(count=4, alignment=16) + exp_g_vec = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) f32_segment = f32_dim_base // 32 f32_segment_dim = f32_dim_base - f32_segment * 32 exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) - exp_g_last_vec = (g_prefix_ptr + exp_g_last_idx).load(count=4, alignment=16) + exp_g_last_vec = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) half_reg_base = f32_group * 4 f32_reg_base = reg_base + half_reg_base exp_g_regs[f32_reg_base] = exp_g_vec[0] @@ -1338,7 +1364,7 @@ def _compute0_warp_group( exp_g_last_regs[f32_reg_base + 2] = exp_g_last_vec[2] exp_g_last_regs[f32_reg_base + 3] = exp_g_last_vec[3] - # ---- k decay operand: exp2(g) * k ------------------------------ + # ---- K decay + K_inv operands: K * exp2(+g) and K * exp2(-g) ----- k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 @@ -1350,7 +1376,9 @@ def _compute0_warp_group( k_pair = fp32_to_fp16(k_beta0, k_beta1, dtype=cfg.io_dtype) exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) - k_inv_words[dim_half * 4 + pair_idx] = fp32_to_fp16(k_value0 * exp_neg_g_regs[dim0], k_value1 * exp_neg_g_regs[dim1], dtype=cfg.io_dtype) + exp_neg_pair = fp32_to_fp16(exp_neg_g_regs[dim0], exp_neg_g_regs[dim1], dtype=cfg.io_dtype) + k_norm_pair = fp32_to_fp16(k_value0, k_value1, dtype=cfg.io_dtype) + k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_norm_pair, exp_neg_pair, cfg.io_dtype) k_inv_vec = cutlass.Vector.from_elements( ( @@ -1371,33 +1399,27 @@ def _compute0_warp_group( cutlass.Int32, ).bitcast(cfg.io_dtype) if cutlass.const_expr(dim_half == 0): - operand_done_phase = ((gc // cfg.smem_decay_stages) + 1) % 2 - bars.mb_kk_qk_super_mma_done[decay_stage].wait(operand_done_phase) - bars.mb_kk_qk_mma_done[decay_stage].wait(operand_done_phase) + operand_done_phase = ((global_chunk // cfg.smem_decay_stages) + 1) % 2 + bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 k_inv_swizzled_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) (sK_inv_ptr + k_inv_swizzled_idx).store(k_inv_vec, alignment=16) - decay_storage_dim_base = _decay_key_dim( - cfg, - decay_row, - dim_base, + storage_key = dim_base ^ decay_key_mask + storage_slice = storage_key // 64 + decay_swizzled_idx = storage_slice * (cfg.b_t * 64) + swizzle_xor_128b( + decay_row, decay_row * 64 + storage_key - storage_slice * 64, elem_bytes=2 ) - decay_linear_idx_base = decay_row * cfg.d_k + decay_storage_dim_base - sw128_elems_per_128b = 128 // 2 - sw128_row = decay_linear_idx_base // cfg.d_k - sw128_col = decay_linear_idx_base - sw128_row * cfg.d_k - sw128_slice = sw128_col // sw128_elems_per_128b - sw128_col_in_slice = sw128_col - sw128_slice * sw128_elems_per_128b - sw128_slice_linear = sw128_row * sw128_elems_per_128b + sw128_col_in_slice - sw128_byte = sw128_slice_linear * 2 - sw128_mask = (sw128_byte >> 7 & 7) << 4 - decay_swizzled_idx_base = sw128_slice * cfg.b_t * sw128_elems_per_128b + (sw128_byte ^ sw128_mask) // 2 - (sK_decay_ptr + decay_swizzled_idx_base).store(k_decay_vec, alignment=16) + (sK_decay_ptr + decay_swizzled_idx).store(k_decay_vec, alignment=16) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_k_decay_cg0_ready[decay_stage].arrive() + bars.mb_k_decay_inv_cg0_ready[decay_stage].arrive() + bars.mb_q_done[raw_stage].arrive() + bars.mb_k_done[raw_stage].arrive() + bars.mb_gate_done[raw_stage].arrive() + bars.mb_beta_done[raw_stage].arrive() - # ---- q decay operand ----------------------------------------------- + # ---- Q_decay operand: Q * q_inv_norm -------------------------------- for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 @@ -1421,26 +1443,15 @@ def _compute0_warp_group( ), cutlass.Int32, ).bitcast(cfg.io_dtype) - decay_storage_dim_base = _decay_key_dim( - cfg, - decay_row, - dim_base, + storage_key = dim_base ^ decay_key_mask + storage_slice = storage_key // 64 + decay_swizzled_idx = storage_slice * (cfg.b_t * 64) + swizzle_xor_128b( + decay_row, decay_row * 64 + storage_key - storage_slice * 64, elem_bytes=2 ) - decay_linear_idx_base = decay_row * cfg.d_k + decay_storage_dim_base - sw128_elems_per_128b = 128 // 2 - sw128_row = decay_linear_idx_base // cfg.d_k - sw128_col = decay_linear_idx_base - sw128_row * cfg.d_k - sw128_slice = sw128_col // sw128_elems_per_128b - sw128_col_in_slice = sw128_col - sw128_slice * sw128_elems_per_128b - sw128_slice_linear = sw128_row * sw128_elems_per_128b + sw128_col_in_slice - sw128_byte = sw128_slice_linear * 2 - sw128_mask = (sw128_byte >> 7 & 7) << 4 - decay_swizzled_idx_base = sw128_slice * cfg.b_t * sw128_elems_per_128b + (sw128_byte ^ sw128_mask) // 2 - (sQ_decay_ptr + decay_swizzled_idx_base).store(q_decay_vec, alignment=16) - - bars.mb_k_restore_done[decay_stage].wait(((gc // cfg.smem_decay_stages + 1) % 2)) - - # ---- k_restore operand ---------------------------------------------- + (sQ_decay_ptr + decay_swizzled_idx).store(q_decay_vec, alignment=16) + + # ---- K_restore operand: K_inv * exp_g_last -------------------------- + bars.mb_k_restore_done[decay_stage].wait(((global_chunk // cfg.smem_decay_stages + 1) % 2)) for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 @@ -1466,12 +1477,20 @@ def _compute0_warp_group( (sK_restore_ptr + k_restore_idx).store(k_restore_vec, alignment=16) nvvm.fence_proxy("async.shared", space="cta") bars.mb_qk_scale_ready[qk_scale_ready_stage].arrive() - gbase += sk_nt - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + diag_ring_stage = diag_ring_stage + cutlass.Int32(cfg.cg0_group_count) + wrapped = diag_ring_stage >= cutlass.Int32(cfg.smem_state_scale_diag_stages) + diag_ring_stage = diag_ring_stage - cutlass.Int32(cfg.smem_state_scale_diag_stages) if wrapped else diag_ring_stage + diag_ring_phase = diag_ring_phase ^ (cutlass.Int32(1) if wrapped else cutlass.Int32(0)) + raw_ring_stage = raw_ring_stage + cutlass.Int32(cfg.cg0_group_count) + raw_ring_wrapped = raw_ring_stage >= cutlass.Int32(cfg.smem_raw_stages) + raw_ring_stage = raw_ring_stage - cutlass.Int32(cfg.smem_raw_stages) if raw_ring_wrapped else raw_ring_stage + raw_ring_phase = raw_ring_phase ^ (cutlass.Int32(1) if raw_ring_wrapped else cutlass.Int32(0)) + global_chunk_base += num_tile_chunks + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) @cute.jit -def _compute1_warp_group( +def compute1_warp_group( cfg, total_tiles, bidx, @@ -1480,41 +1499,39 @@ def _compute1_warp_group( mWorkItems, sSched, lane, - tmem_hold, + sTmem_base, warp_idx, - mS_out, - mS_init, + mState_out, + mState_init, mO, sO_raw, sV_raw, sW_raw, - sH_raw, + sCheckpoint_raw, checkpoint_every_n_tokens, scale, bars, ) -> None: - """CG1 warp role (warps 8-11): persistent tile-scheduler loop + value-side - TMEM staging (state input, rhs, update), output drain to SMEM, and - checkpoint/final-state stores (split-K: only owned entries).""" + """CG1 warp role (warps 8-11): persistent scheduler loop running the + value-side TMEM staging, O drain, and checkpoint/final-state stores.""" nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) sO_ptr = sO_raw.data_ptr() - sH_ptr = sH_raw.data_ptr() if cutlass.const_expr(cfg.enable_checkpoints) else sO_raw.data_ptr() - h_done_index = PipelineState.start(phase=1) # sH starts free - tmem_base = tmem_hold.load() + sCheckpoint_ptr = sCheckpoint_raw.data_ptr() if cutlass.const_expr(cfg.enable_checkpoints) else sO_raw.data_ptr() + checkpoint_done_index = PipelineState.start(phase=1) + tmem_base = sTmem_base.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 - tmem_sp = warp_idx % (cfg.d_v // cfg.threads_per_warp) - # ldmatrix.x4/stmatrix.x4 COL lane decode shared by the v loads and o stores - ov_tok = (lane // 16) * 8 + (lane & 7) - ov_col = ((lane // 8) & 1) * 8 - row_id = tmem_row + tmem_sp * cfg.threads_per_warp - value_dim = tmem_sp * cfg.threads_per_warp + lane + tmem_subpartition = warp_idx % (cfg.d_v // cfg.threads_per_warp) + frag_row_coord = (lane // 16) * 8 + (lane & 7) + frag_col_offset = ((lane // 8) & 1) * 8 + row_id = tmem_row + tmem_subpartition * cfg.threads_per_warp + value_dim = tmem_subpartition * cfg.threads_per_warp + lane state_k_acc_index = PipelineState.start(phase=0) update_acc_index = PipelineState.start(phase=0) o_acc_index = PipelineState.start(phase=0) - kr_index = PipelineState.start(phase=0) # CG1's per-chunk mb_k_restore_done wait slot + k_restore_index = PipelineState.start(phase=0) # CG1's per-chunk mb_k_restore_done wait slot raw_index = PipelineState.start(phase=0) # raw-ring slot for the sV/sW reads + inputs_done arrives - gbase = cutlass.Int32(0) + global_chunk_base = cutlass.Int32(0) sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) while tile_idx < total_tiles: @@ -1522,63 +1539,99 @@ def _compute1_warp_group( cfg, tile_idx, cu_seqlens, mWorkItems ) head_o = head_idx - sk_nt = wend - cstart - - if sk_nt > 0: - # ---- first chunk: seed state TMEM from mS_init (else zeros); - # split-K warmup items (cstart > 0) rebuild their state from zero - seed_from_s0 = cstart == 0 - for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): - state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) - for col in cutlass.range_constexpr(32): - key_dim = key_block_start + col - state_value = cutlass.Float32(0.0) - if cutlass.const_expr(mS_init is not None): - state_value = mS_init[batch_idx, head_o, key_dim, value_dim].to(cutlass.Float32) - if cutlass.const_expr(cfg.split_k): - state_value = state_value if seed_from_s0 else cutlass.Float32(0.0) - state_block[col] = state_value + num_tile_chunks = wend - cstart - nvvm.tcgen05_st( - "32x32b", - nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_offset + key_block_start), cutlass.Float32), - state_block[0:32], - ) + if num_tile_chunks > 0: + # ---- first chunk: seed state TMEM from mState_init (else zeros) ---------- + seed_from_initial_state = cstart == 0 + if cutlass.const_expr(mState_init is not None and cfg.split_k): + if seed_from_initial_state: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + state_block[col] = mState_init[batch_idx, head_o, key_dim, value_dim].to(cutlass.Float32) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + else: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + state_block[col] = cutlass.Float32(0.0) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + elif cutlass.const_expr(mState_init is not None): + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + state_block[col] = mState_init[batch_idx, head_o, key_dim, value_dim].to(cutlass.Float32) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + else: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + state_block[col] = cutlass.Float32(0.0) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) nvvm.tcgen05_wait("store") sV_ptr = sV_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) sW_ptr = sW_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) - row_addr = (tmem_row + tmem_sp * cfg.threads_per_warp) << 16 - state_col_id = tmem_col + cfg.tmem_state_offset - # ---- state -> packed b16 A operand (TMEM roundtrip) ---------------- + row_addr = (tmem_row + tmem_subpartition * cfg.threads_per_warp) << 16 + state_col_id = tmem_col + cfg.tmem_state_acc_offset + + # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- state_blocks = [] - for sub in cutlass.range_constexpr(cfg.d_k // 16): - state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) - nvvm.tcgen05_wait("load") + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) packed_col_id = tmem_col + cfg.tmem_state_inp_offset - for sub in cutlass.range_constexpr(cfg.d_k // 16): + for k_block in cutlass.range_constexpr(cfg.d_k // 16): packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) for packed_col in cutlass.range_constexpr(8): source_pair = packed_col ^ 4 - packed_state[packed_col] = fp32_to_fp16(state_blocks[sub][2 * source_pair], state_blocks[sub][2 * source_pair + 1], dtype=cfg.io_dtype) + packed_state[packed_col] = fp32_to_fp16( + state_blocks[k_block][2 * source_pair], state_blocks[k_block][2 * source_pair + 1], dtype=cfg.io_dtype + ) nvvm.tcgen05_st( "32x32b", - nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + sub * 8, cutlass.Int8), + nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + k_block * 8, cutlass.Int8), packed_state[0:8], ) nvvm.tcgen05_wait("store") bars.mb_state_inp_ready.arrive() + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_read_done.arrive() - # ---- rhs staging: rhs input = w*v - state*(beta*k) ----------------- + # ---- rhs staging: rhs input = W*V - state*(Beta*K) ------------------- bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) + bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) + bars.mb_w_ready[raw_index.idx].wait(raw_index.phase) projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset input_col_id = tmem_col + cfg.tmem_rhs_inp_offset - value_dim_base = tmem_sp * cfg.threads_per_warp + value_dim_base = tmem_subpartition * cfg.threads_per_warp - # ---- read back state*k acc + raw v fragments ----------------------- + # ---- read back state*K acc + raw V fragments ------------------------- row_id0 = tmem_row + value_dim_base state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) @@ -1587,37 +1640,36 @@ def _compute1_warp_group( raw_v_regs0 = nvvm.ldmatrix( sV_ptr - + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + frag_col_offset) % 64, elem_bytes=2), 4, nvvm.MMALayout.COL, ) raw_v_regs1 = nvvm.ldmatrix( sV_ptr - + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + 16 + frag_col_offset) % 64, elem_bytes=2), 4, nvvm.MMALayout.COL, ) raw_w_regs0 = nvvm.ldmatrix( sW_ptr - + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + frag_col_offset) % 64, elem_bytes=2), 4, nvvm.MMALayout.COL, ) raw_w_regs1 = nvvm.ldmatrix( sW_ptr - + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + 16 + frag_col_offset) % 64, elem_bytes=2), 4, nvvm.MMALayout.COL, ) - nvvm.tcgen05_wait("load") packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): @@ -1626,9 +1678,6 @@ def _compute1_warp_group( state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) w_pair = raw_w_regs0[raw_matrix] - if cutlass.const_expr(cfg.beta_w_sigmoid): - w_gate0, w_gate1 = sigmoid_f16x2(w_pair, cfg.io_dtype) - w_pair = fp32_to_fp16(w_gate0, w_gate1, dtype=cfg.io_dtype) wv_pair = mul_f16x2( w_pair, raw_v_regs0[raw_matrix], @@ -1647,9 +1696,6 @@ def _compute1_warp_group( state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) w_pair = raw_w_regs1[raw_matrix] - if cutlass.const_expr(cfg.beta_w_sigmoid): - w_gate0, w_gate1 = sigmoid_f16x2(w_pair, cfg.io_dtype) - w_pair = fp32_to_fp16(w_gate0, w_gate1, dtype=cfg.io_dtype) wv_pair = mul_f16x2( w_pair, raw_v_regs1[raw_matrix], @@ -1667,17 +1713,17 @@ def _compute1_warp_group( nvvm.tcgen05_wait("store") state_k_acc_index = advance(state_k_acc_index, 1) - bars.mb_inputs_done[raw_index.idx].arrive() + bars.mb_v_done[raw_index.idx].arrive() + bars.mb_w_done[raw_index.idx].arrive() bars.mb_rhs_ready.arrive() - # ---- update readback -> packed b16 A operand ----------------------- + # ---- update repack: acc TMEM -> packed b16 TMEM --------------------- bars.mb_update_acc_ready.wait(update_acc_index.phase) update = nvvm.tcgen05_ld( "32x32b", - nvvm.make_tmem_ptr((tmem_row + tmem_sp * cfg.threads_per_warp << 16) + (tmem_col + cfg.tmem_update_acc_offset), cutlass.Float32), + nvvm.make_tmem_ptr((tmem_row + tmem_subpartition * cfg.threads_per_warp << 16) + (tmem_col + cfg.tmem_update_acc_offset), cutlass.Float32), num=cfg.b_t, ) - nvvm.tcgen05_wait("load") packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): @@ -1695,85 +1741,116 @@ def _compute1_warp_group( update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() - bars.mb_k_restore_done[kr_index.idx].wait(kr_index.phase) - kr_index = advance(kr_index, cfg.smem_decay_stages) + bars.mb_k_restore_done[k_restore_index.idx].wait(k_restore_index.phase) + k_restore_index = advance(k_restore_index, cfg.smem_decay_stages) raw_index = advance(raw_index, cfg.smem_raw_stages) - # the first chunk is peeled above so this steady-state loop always - # drains the prior chunk's output - for li in cutlass.range(1, sk_nt, 1, unroll=1): - chunk_idx = cstart + li - gc = gbase + li + if cutlass.const_expr(cfg.enable_checkpoints): + cg1_checkpoint_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) + cg1_checkpoint_mod = (cstart + cutlass.Int32(1)) % cg1_checkpoint_chunks + for local_chunk in cutlass.range(1, num_tile_chunks, 1, unroll=1): + chunk_idx = cstart + local_chunk + global_chunk = global_chunk_base + local_chunk sV_ptr = sV_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) sW_ptr = sW_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) prev_output_chunk = chunk_idx - cutlass.Int32(1) - prev_og = gc - cutlass.Int32(1) - prev_o_stage = prev_og % cfg.smem_o_stages - prev_q_state_acc_stage = prev_og % cfg.tmem_q_state_acc_stages + prev_global_chunk = global_chunk - cutlass.Int32(1) + prev_o_stage = prev_global_chunk % cfg.smem_o_stages + prev_q_state_acc_stage = prev_global_chunk % cfg.tmem_q_state_acc_stages prev_o_stage_base = prev_o_stage * (cfg.b_t * cfg.d_v) - # H entry gate: the state read by this restage entered chunk_idx, - # i.e. the state after chunk_idx * b_t tokens (strictly before the - # sequence end -- the end state is only final_state); split-K - # warmup reconstructions below wstart belong to the previous item - do_h = False + do_checkpoint = False if cutlass.const_expr(cfg.enable_checkpoints): - do_h = (chunk_idx * cutlass.Int32(cfg.b_t)) % checkpoint_every_n_tokens == 0 + do_checkpoint = cg1_checkpoint_mod == 0 + cg1_checkpoint_mod = cg1_checkpoint_mod + cutlass.Int32(1) + cg1_checkpoint_mod = cutlass.Int32(0) if cg1_checkpoint_mod == cg1_checkpoint_chunks else cg1_checkpoint_mod if cutlass.const_expr(cfg.split_k): - do_h = do_h and chunk_idx >= wstart - if do_h: - # sH is free once the epilogue's previous TMA store retired - bars.mb_h_tmastg_done.wait(h_done_index.phase) - h_done_index = advance(h_done_index, 1) - row_addr = (tmem_row + tmem_sp * cfg.threads_per_warp) << 16 - state_col_id = tmem_col + cfg.tmem_state_offset - # ---- state -> packed b16 A operand (TMEM roundtrip) ---------------- + do_checkpoint = do_checkpoint and chunk_idx >= wstart + row_addr = (tmem_row + tmem_subpartition * cfg.threads_per_warp) << 16 + state_col_id = tmem_col + cfg.tmem_state_acc_offset + + # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- state_blocks = [] - for sub in cutlass.range_constexpr(cfg.d_k // 16): - state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) - bars.mb_o_tmastg_done[prev_o_stage].wait(((prev_og // cfg.smem_o_stages) + 1) % 2) - nvvm.tcgen05_wait("load") + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) + bars.mb_o_tmastg_done[prev_o_stage].wait(((prev_global_chunk // cfg.smem_o_stages) + 1) % 2) packed_col_id = tmem_col + cfg.tmem_state_inp_offset - for sub in cutlass.range_constexpr(cfg.d_k // 16): + for k_block in cutlass.range_constexpr(cfg.d_k // 16): packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) for packed_col in cutlass.range_constexpr(8): source_pair = packed_col ^ 4 - packed_state[packed_col] = fp32_to_fp16(state_blocks[sub][2 * source_pair], state_blocks[sub][2 * source_pair + 1], dtype=cfg.io_dtype) + packed_state[packed_col] = fp32_to_fp16( + state_blocks[k_block][2 * source_pair], state_blocks[k_block][2 * source_pair + 1], dtype=cfg.io_dtype + ) nvvm.tcgen05_st( "32x32b", - nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + sub * 8, cutlass.Int8), + nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + k_block * 8, cutlass.Int8), packed_state[0:8], ) - if cutlass.const_expr(cfg.enable_checkpoints): - # stage this sub's state to sH TRANSPOSED (KV: k rows, v - # contiguous in 64-v slabs, swizzled — the GDN H layout); - # each thread scatters its 16 k values down one v column - if do_h: - h_col = value_dim % cutlass.Int32(64) - h_seg = (value_dim // cutlass.Int32(64)) * (cfg.d_k * cutlass.Int32(64)) - k_base = sub * 16 - for j in cutlass.range_constexpr(16): - hv = state_blocks[sub][j].to(cfg.io_dtype) - (sH_ptr + h_seg + cutlass.Int32((k_base + j) * 64) + swizzle_xor_128b(cutlass.Int32(k_base + j), h_col, elem_bytes=2)).store(hv) - nvvm.tcgen05_wait("store") bars.mb_state_inp_ready.arrive() + + # ---- checkpoint: post-publish f32 fragment read; the decay GEMM's ------ + # overwrite is held by mb_state_read_done ------------------------------ if cutlass.const_expr(cfg.enable_checkpoints): - if do_h: + if do_checkpoint: + checkpoint_stage = checkpoint_done_index.idx + bars.mb_checkpoint_tmastg_done[checkpoint_stage].wait(checkpoint_done_index.phase) + checkpoint_done_index = advance(checkpoint_done_index, cfg.smem_checkpoint_stages) + checkpoint_stage_base = checkpoint_stage * (cfg.d_k * cfg.d_v) + row16_addr = ((tmem_row + tmem_subpartition * cfg.threads_per_warp) + 16) << 16 + checkpoint_vbase = tmem_subpartition * cfg.threads_per_warp + checkpoint_swz_off0 = (checkpoint_vbase + frag_col_offset) // 64 * (cfg.d_k * 64) + checkpoint_swz_col0 = (checkpoint_vbase + frag_col_offset) % 64 + checkpoint_swz_off = (checkpoint_vbase + 16 + frag_col_offset) // 64 * (cfg.d_k * 64) + checkpoint_swz_col = (checkpoint_vbase + 16 + frag_col_offset) % 64 + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + checkpoint_ld0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) + checkpoint_ld1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) + checkpoint_st0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_st1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + checkpoint_st0[reg_idx] = fp32_to_fp16(checkpoint_ld0[2 * reg_idx], checkpoint_ld0[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_st1[reg_idx] = fp32_to_fp16(checkpoint_ld1[2 * reg_idx], checkpoint_ld1[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_row = k_block * 16 + frag_row_coord + nvvm.stmatrix( + sCheckpoint_ptr + + checkpoint_stage_base + + checkpoint_swz_off0 + + checkpoint_row * 64 + + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col0, elem_bytes=2), + checkpoint_st0.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.stmatrix( + sCheckpoint_ptr + + checkpoint_stage_base + + checkpoint_swz_off + + checkpoint_row * 64 + + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col, elem_bytes=2), + checkpoint_st1.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.tcgen05_wait("load") + bars.mb_state_read_done.arrive() nvvm.fence_proxy("async.shared", space="cta") - bars.mb_h_tmastg_ready.arrive() + bars.mb_checkpoint_tmastg_ready[checkpoint_stage].arrive() + else: + bars.mb_state_read_done.arrive() bars.mb_o_acc_ready.wait(o_acc_index.phase) o_acc_index = advance(o_acc_index, 1) projection_col_id = tmem_col + cfg.tmem_q_state_acc_offset + prev_q_state_acc_stage * cfg.b_t - value_dim_base = tmem_sp * cfg.threads_per_warp + value_dim_base = tmem_subpartition * cfg.threads_per_warp row_id0 = tmem_row + value_dim_base row_id1 = row_id0 + 16 loaded0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) loaded1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) - # ---- output drain: q_state_acc -> scaled b16 -> SMEM stmatrix ------- + # ---- output drain: O acc TMEM -> scaled b16 SMEM -------------------- stsm_regs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) stsm_regs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): @@ -1785,9 +1862,9 @@ def _compute1_warp_group( nvvm.stmatrix( sO_ptr + prev_o_stage_base - + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + frag_col_offset) % 64, elem_bytes=2), stsm_regs0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, @@ -1795,26 +1872,27 @@ def _compute1_warp_group( nvvm.stmatrix( sO_ptr + prev_o_stage_base - + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + 16 + frag_col_offset) % 64, elem_bytes=2), stsm_regs1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) - # release only after the stmatrix pair: the STSM->F2FP->FMUL2->LDTM - # register chain pins the TMEM reads complete without a wait("load") bars.mb_o_acc_done[prev_q_state_acc_stage].arrive() nvvm.fence_proxy("async.shared", space="cta") bars.mb_o_tmastg_ready[prev_o_stage].arrive() bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) - # ---- rhs staging: rhs input = w*v - state*(beta*k) ----------------- + + # ---- rhs staging: rhs input = W*V - state*(Beta*K) ------------------- + bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) + bars.mb_w_ready[raw_index.idx].wait(raw_index.phase) projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset input_col_id = tmem_col + cfg.tmem_rhs_inp_offset - value_dim_base = tmem_sp * cfg.threads_per_warp + value_dim_base = tmem_subpartition * cfg.threads_per_warp - # ---- read back state*k acc + raw v fragments ----------------------- + # ---- read back state*K acc + raw V fragments ------------------------- row_id0 = tmem_row + value_dim_base state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) @@ -1823,37 +1901,36 @@ def _compute1_warp_group( raw_v_regs0 = nvvm.ldmatrix( sV_ptr - + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + frag_col_offset) % 64, elem_bytes=2), 4, nvvm.MMALayout.COL, ) raw_v_regs1 = nvvm.ldmatrix( sV_ptr - + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + 16 + frag_col_offset) % 64, elem_bytes=2), 4, nvvm.MMALayout.COL, ) raw_w_regs0 = nvvm.ldmatrix( sW_ptr - + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + frag_col_offset) % 64, elem_bytes=2), 4, nvvm.MMALayout.COL, ) raw_w_regs1 = nvvm.ldmatrix( sW_ptr - + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + 16 + frag_col_offset) % 64, elem_bytes=2), 4, nvvm.MMALayout.COL, ) - nvvm.tcgen05_wait("load") packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): @@ -1862,9 +1939,6 @@ def _compute1_warp_group( state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) w_pair = raw_w_regs0[raw_matrix] - if cutlass.const_expr(cfg.beta_w_sigmoid): - w_gate0, w_gate1 = sigmoid_f16x2(w_pair, cfg.io_dtype) - w_pair = fp32_to_fp16(w_gate0, w_gate1, dtype=cfg.io_dtype) wv_pair = mul_f16x2( w_pair, raw_v_regs0[raw_matrix], @@ -1883,9 +1957,6 @@ def _compute1_warp_group( state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) w_pair = raw_w_regs1[raw_matrix] - if cutlass.const_expr(cfg.beta_w_sigmoid): - w_gate0, w_gate1 = sigmoid_f16x2(w_pair, cfg.io_dtype) - w_pair = fp32_to_fp16(w_gate0, w_gate1, dtype=cfg.io_dtype) wv_pair = mul_f16x2( w_pair, raw_v_regs1[raw_matrix], @@ -1903,17 +1974,17 @@ def _compute1_warp_group( nvvm.tcgen05_wait("store") state_k_acc_index = advance(state_k_acc_index, 1) - bars.mb_inputs_done[raw_index.idx].arrive() + bars.mb_v_done[raw_index.idx].arrive() + bars.mb_w_done[raw_index.idx].arrive() bars.mb_rhs_ready.arrive() - # ---- update readback -> packed b16 A operand ----------------------- + # ---- update repack: acc TMEM -> packed b16 TMEM --------------------- bars.mb_update_acc_ready.wait(update_acc_index.phase) update = nvvm.tcgen05_ld( "32x32b", - nvvm.make_tmem_ptr((tmem_row + tmem_sp * cfg.threads_per_warp << 16) + (tmem_col + cfg.tmem_update_acc_offset), cutlass.Float32), + nvvm.make_tmem_ptr((tmem_row + tmem_subpartition * cfg.threads_per_warp << 16) + (tmem_col + cfg.tmem_update_acc_offset), cutlass.Float32), num=cfg.b_t, ) - nvvm.tcgen05_wait("load") packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): @@ -1931,29 +2002,29 @@ def _compute1_warp_group( update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() - bars.mb_k_restore_done[kr_index.idx].wait(kr_index.phase) - kr_index = advance(kr_index, cfg.smem_decay_stages) + bars.mb_k_restore_done[k_restore_index.idx].wait(k_restore_index.phase) + k_restore_index = advance(k_restore_index, cfg.smem_decay_stages) raw_index = advance(raw_index, cfg.smem_raw_stages) - if sk_nt > 0: - og = gbase + sk_nt - cutlass.Int32(1) + if num_tile_chunks > 0: + last_global_chunk = global_chunk_base + num_tile_chunks - cutlass.Int32(1) output_chunk = wend - cutlass.Int32(1) - final_o_stage = og % cfg.smem_o_stages - final_q_state_acc_stage = og % cfg.tmem_q_state_acc_stages + final_o_stage = last_global_chunk % cfg.smem_o_stages + final_q_state_acc_stage = last_global_chunk % cfg.tmem_q_state_acc_stages final_o_stage_base = final_o_stage * (cfg.b_t * cfg.d_v) - bars.mb_o_tmastg_done[final_o_stage].wait(((og // cfg.smem_o_stages) + 1) % 2) + bars.mb_o_tmastg_done[final_o_stage].wait(((last_global_chunk // cfg.smem_o_stages) + 1) % 2) bars.mb_o_acc_ready.wait(o_acc_index.phase) o_acc_index = advance(o_acc_index, 1) projection_col_id = tmem_col + cfg.tmem_q_state_acc_offset + final_q_state_acc_stage * cfg.b_t - value_dim_base = tmem_sp * cfg.threads_per_warp + value_dim_base = tmem_subpartition * cfg.threads_per_warp row_id0 = tmem_row + value_dim_base row_id1 = row_id0 + 16 loaded0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) loaded1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) - # ---- output drain: q_state_acc -> scaled b16 -> SMEM stmatrix ------- + # ---- output drain: O acc TMEM -> scaled b16 SMEM -------------------- stsm_regs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) stsm_regs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): @@ -1965,9 +2036,9 @@ def _compute1_warp_group( nvvm.stmatrix( sO_ptr + final_o_stage_base - + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + frag_col_offset) % 64, elem_bytes=2), stsm_regs0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, @@ -1975,54 +2046,48 @@ def _compute1_warp_group( nvvm.stmatrix( sO_ptr + final_o_stage_base - + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + 16 + frag_col_offset) % 64, elem_bytes=2), stsm_regs1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) - # release only after the stmatrix pair: the STSM->F2FP->FMUL2->LDTM - # register chain pins the TMEM reads complete without a wait("load") bars.mb_o_acc_done[final_q_state_acc_stage].arrive() nvvm.fence_proxy("async.shared", space="cta") bars.mb_o_tmastg_ready[final_o_stage].arrive() - # split-K: only the item owning the sequence's last chunk holds the - # true end-of-sequence state (legacy tiles always do: wend == nc) owns_final = wend == num_chunks_b - # ---- final-state drain: final_state acc -> GMEM -------------------- - if cutlass.const_expr(mS_out is not None): + + # ---- final-state drain: state acc TMEM -> GMEM --------------------------- + if cutlass.const_expr(mState_out is not None): if seqlen_b > 0: if owns_final: for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): loaded = nvvm.tcgen05_ld( "32x32b", - nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_offset + key_block_start), cutlass.Float32), + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), num=32, ) - nvvm.tcgen05_wait("load") for col in cutlass.range_constexpr(32): key_dim = key_block_start + col - mS_out[batch_idx, head_o, key_dim, value_dim] = loaded[col].to(mS_out.element_type) + mState_out[batch_idx, head_o, key_dim, value_dim] = loaded[col].to(mState_out.element_type) else: - # zero-length sequence: the state passes through untouched - # (S0 when seeded, zeros otherwise); pure GMEM, no TMEM for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): for col in cutlass.range_constexpr(32): key_dim = key_block_start + col - if cutlass.const_expr(mS_init is not None): - mS_out[batch_idx, head_o, key_dim, value_dim] = mS_init[batch_idx, head_o, key_dim, value_dim] + if cutlass.const_expr(mState_init is not None): + mState_out[batch_idx, head_o, key_dim, value_dim] = mState_init[batch_idx, head_o, key_dim, value_dim] else: - mS_out[batch_idx, head_o, key_dim, value_dim] = cutlass.Float32(0.0).to(mS_out.element_type) + mState_out[batch_idx, head_o, key_dim, value_dim] = cutlass.Float32(0.0).to(mState_out.element_type) bars.mb_final_state_stored.arrive() - gbase += sk_nt - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + global_chunk_base += num_tile_chunks + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) @cute.jit -def _host( +def host( cfg: cutlass.Constexpr, q: cute.Tensor, k: cute.Tensor, @@ -2045,15 +2110,13 @@ def _host( stream, ) -> None: num_sequences = cu_seqlens.shape[0] - 1 - ho = raw_gate.shape[1] - # ---- persistent launch: the grid only needs to cover the tiles ------ - total_tiles = num_sequences * ho - # CUDA-graph-stable launch: fixed SM-count grid + n_heads_out = raw_gate.shape[1] + total_tiles = num_sequences * n_heads_out grid_shape = (cfg.max_active_clusters, 1, 1) - _kernel( + kernel( cfg, tensormap_workspace, - cutlass.Int32(num_sequences * ho), + cutlass.Int32(num_sequences), q, k, v, @@ -2081,7 +2144,7 @@ def _host( @cute.kernel -def _kernel( +def kernel( cfg: cutlass.Constexpr, tensormap_workspace: cute.Tensor, n_desc: cutlass.Int32, @@ -2094,9 +2157,9 @@ def _kernel( mBeta: cute.Tensor, mW: cute.Tensor, cu_seqlens: cute.Tensor, - mS_init: cute.Tensor | None, + mState_init: cute.Tensor | None, mO: cute.Tensor, - mS_out: cute.Tensor | None, + mState_out: cute.Tensor | None, mWorkItems: cute.Tensor | None, mCount: cute.Tensor | None, mSched: cute.Tensor | None, @@ -2127,16 +2190,16 @@ def _kernel( assert mBeta.element_type == cfg.io_dtype and mW.element_type == cfg.io_dtype, "channel-wise beta/w must match the io dtype" assert cu_seqlens.element_type in (cutlass.Int32, cutlass.Int64) if cutlass.const_expr(cfg.use_initial_state): - assert mS_init is not None and mS_init.element_type in (cutlass.BFloat16, cutlass.Float32) + assert mState_init is not None and mState_init.element_type in (cutlass.BFloat16, cutlass.Float32) else: - assert mS_init is None, "mS_init must be None if use_initial_state is False" + assert mState_init is None, "mState_init must be None if use_initial_state is False" if cutlass.const_expr(cfg.store_final_state): - assert mS_out is not None and mS_out.element_type in (cutlass.BFloat16, cutlass.Float32) + assert mState_out is not None and mState_out.element_type in (cutlass.BFloat16, cutlass.Float32) else: - assert mS_out is None, "mS_out must be None if store_final_state is False" - if cutlass.const_expr(mS_init is not None and mS_out is not None): - assert mS_init.element_type == mS_out.element_type - # per-(batch, head) TMA-descriptor arrays: [q, k, v, gate, beta, w, o] + assert mState_out is None, "mState_out must be None if store_final_state is False" + if cutlass.const_expr(mState_init is not None and mState_out is not None): + assert mState_init.element_type == mState_out.element_type + # per-BATCH TMA-descriptor arrays (heads are load coordinates): [Q, K, V, Gate, Beta, W, O] desc_base_words = tensormap_workspace.iterator.raw_ptr() arr_words = n_desc * cutlass.Int32(TENSOR_MAP_QWORDS) desc_q_base = desc_base_words @@ -2146,19 +2209,17 @@ def _kernel( desc_beta_base = desc_base_words + cutlass.Int32(4) * arr_words desc_w_base = desc_base_words + cutlass.Int32(5) * arr_words desc_o_base = desc_base_words + cutlass.Int32(6) * arr_words - desc_h_base = desc_base_words + cutlass.Int32(7) * arr_words + desc_checkpoint_base = desc_base_words + cutlass.Int32(7) * arr_words # Buffers are declaration-ordered and intentionally non-aliased. SMEM = cutlass.AddressSpace.smem bars = make_gdn2_bars(cfg) - # The hand-written K-box-major SW128 mapping is normalized to phase 0, - # so both tcgen05 and ldmatrix can share 1KB-aligned operand buffers. - tmem_hold = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=4) + sTmem_base = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=4) sSched = cutlass.Array(cutlass.Int32, cfg.sched_stages, space=SMEM, alignment=16) sK_decay_raw = cutlass.Array(cfg.io_dtype, cfg.k_decay_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) sQ_decay_raw = cutlass.Array(cfg.io_dtype, cfg.q_decay_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) sK_restore_raw = cutlass.Array(cfg.io_dtype, cfg.k_restore_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) - sPairwise_raw = cutlass.Array(cfg.io_dtype, cfg.pairwise_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sQk_raw = cutlass.Array(cfg.io_dtype, cfg.qk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) sQ_raw = cutlass.Array(mQ.element_type, cfg.q_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) sK_raw = cutlass.Array(mK.element_type, cfg.k_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) sV_raw = cutlass.Array(mV.element_type, cfg.v_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) @@ -2169,18 +2230,15 @@ def _kernel( mO.element_type, cfg.o_cosize, space=SMEM, - # The scalar CG1 store computes W128 offsets relative to this buffer. - # Align to the full s128b period so absolute SMEM address bits do not - # add a hidden phase to the TMA store-side swizzle. alignment=cfg.buffer_align_bytes, ) sBeta_raw = cutlass.Array(mBeta.element_type, cfg.beta_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) sW_raw = cutlass.Array(mW.element_type, cfg.w_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) - sH_raw = ( - cutlass.Array(cfg.io_dtype, cfg.d_k * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) if cutlass.const_expr(cfg.enable_checkpoints) else sO_raw + sCheckpoint_raw = ( + cutlass.Array(cfg.io_dtype, cfg.smem_checkpoint_stages * cfg.d_k * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) + if cutlass.const_expr(cfg.enable_checkpoints) + else sO_raw ) - # K-box-major SW128 staging: 16B leading offset, 1KB stride; the decay - # stores apply a row-group key xor so tcgen05 B reads logical [DK, BT]. sK_decay = SmemTile( base=sK_decay_raw, elems_per_stage=(cfg.d_k * cfg.b_t), @@ -2213,29 +2271,30 @@ def _kernel( stride_byte_offset=(8 * 16 * 2), layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, ) - sPairwise = SmemTile( - base=sPairwise_raw, + sQk = SmemTile( + base=sQk_raw, elems_per_stage=(2 * cfg.b_t * cfg.b_t), - stages=cfg.smem_pairwise_stages, + stages=cfg.smem_qk_stages, leading_byte_offset=16, stride_byte_offset=(8 * cfg.b_t * 2), layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, ) - tma_tx_bytes = cutlass.const_expr( - cfg.d_k * cfg.b_t * mQ.element_type.width // 8 - + cfg.d_k * cfg.b_t * mK.element_type.width // 8 - + cfg.d_v * cfg.b_t * mV.element_type.width // 8 - + cfg.d_k * cfg.b_t * mGate.element_type.width // 8 - + cfg.d_k * cfg.b_t * mBeta.element_type.width // 8 - + cfg.d_v * cfg.b_t * mW.element_type.width // 8 - ) if warp_idx == cfg.tma_warp_id: if nvvm.elect_sync(): - bars.mb_tma_done.init() for stage in cutlass.range_constexpr(cfg.smem_raw_stages): - bars.mb_inputs_ready[stage].init() - bars.mb_inputs_done[stage].init() + bars.mb_q_ready[stage].init() + bars.mb_q_done[stage].init() + bars.mb_k_ready[stage].init() + bars.mb_k_done[stage].init() + bars.mb_gate_ready[stage].init() + bars.mb_gate_done[stage].init() + bars.mb_beta_ready[stage].init() + bars.mb_beta_done[stage].init() + bars.mb_v_ready[stage].init() + bars.mb_v_done[stage].init() + bars.mb_w_ready[stage].init() + bars.mb_w_done[stage].init() elif warp_idx == cfg.tcgen05_mma_warp_id: if nvvm.elect_sync(): bars.mb_o_acc_ready.init() @@ -2247,22 +2306,22 @@ def _kernel( for stage in cutlass.range_constexpr(cfg.smem_state_scale_diag_stages): bars.mb_state_scale_diag_done[stage].init() for stage in cutlass.range_constexpr(cfg.smem_decay_stages): - bars.mb_kk_qk_super_mma_done[stage].init() - bars.mb_kk_qk_mma_done[stage].init() + bars.mb_decay_done[stage].init() + bars.mb_decay_super_done[stage].init() bars.mb_k_restore_done[stage].init() bars.mb_rhs_ready.init() bars.mb_update_ready.init() bars.mb_final_state_stored.init() elif warp_idx == cfg.super_mma_warp_id: if nvvm.elect_sync(): - for stage in cutlass.range_constexpr(cfg.smem_pairwise_stages): + for stage in cutlass.range_constexpr(cfg.smem_qk_stages): bars.mb_a_ready[stage].init() bars.mb_qk_acc_ready[stage].init() bars.mb_a_done[stage].init() for stage in cutlass.range_constexpr(cfg.qk_scale_ready_stages): bars.mb_qk_scale_ready[stage].init() for stage in cutlass.range_constexpr(cfg.smem_decay_stages): - bars.mb_k_decay_cg0_ready[stage].init() + bars.mb_k_decay_inv_cg0_ready[stage].init() elif warp_idx == cfg.epilogue_warp_id: if nvvm.elect_sync(): for stage in cutlass.range_constexpr(cfg.smem_o_stages): @@ -2272,8 +2331,10 @@ def _kernel( bars.mb_sched_ready[stage].init() bars.mb_sched_done[stage].init() if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_h_tmastg_ready.init() - bars.mb_h_tmastg_done.init() + for stage in cutlass.range_constexpr(cfg.smem_checkpoint_stages): + bars.mb_checkpoint_tmastg_ready[stage].init() + bars.mb_checkpoint_tmastg_done[stage].init() + bars.mb_state_read_done.init() diag_zero = cfg.io_dtype(0.0) for diag_idx in cutlass.range(tidx, cfg.state_scale_diag_cosize, cfg.threads_per_cta, unroll=1): sState_scale_diag_raw[diag_idx] = diag_zero @@ -2281,30 +2342,26 @@ def _kernel( nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) if (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) or warp_idx == cfg.tcgen05_mma_warp_id: if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_alloc(tmem_hold, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.nbar_tmem_lifecycle_id, thread_count=cfg.tmem_user_threads) + nvvm.tcgen05_alloc(sTmem_base, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) if warp_idx == cfg.tcgen05_mma_warp_id: nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.nbar_tmem_lifecycle_id, thread_count=cfg.tmem_user_threads) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) # Actual SMEM/TMEM buffers for the BT=16 schedule: - # q/k/v : 16 x 128 each - # gate_log2 : 16 x 128 - # beta : 16 - # q/k inverse norm : 16 each, staged once per decay stage + # Q/K/V : 16 x 128 each + # Gate_log2 : 16 x 128 + # Beta : 16 + # Q/K inverse norm : 16 each, staged once per decay stage # exp_g_last : 128, CG0-local and staged once per decay stage # state-scale diag : 8 x 16 x 16 input dtype, zeroed once in the prologue - # q_decay/k_decay : tcgen05 SW128 operands shared with super-MMA - # k_restore : tcgen05 SW128 N-major final-state operand - # k_inv : 16 x 128 token-major for super-MMA RHS + # Q_decay/K_decay : tcgen05 SW128 operands shared with super-MMA + # K_restore : tcgen05 SW128 N-major final-state operand + # K_inv : 16 x 128 token-major for super-MMA RHS # A inverse/QK : 16 x 16 each, plus transposed tcgen05 operands - # state : external/kernel ABI is VK `[DV, DK]`; reference - # math can view it as KV `[DK, DV]` by transposing. - # The TS A-staging path keeps VK in TMEM so state*k - # is `[DV, DK] @ [DK, BT] -> [DV, BT]` with M=128. if warp_idx == cfg.tma_warp_id: - _tmaldg_warp( + tmaldg_warp( cfg, total_tiles, bidx, @@ -2314,7 +2371,6 @@ def _kernel( mSched, sSched, lane, - tma_tx_bytes, sBeta_raw, sGate_raw, sK_raw, @@ -2330,7 +2386,7 @@ def _kernel( bars, ) elif warp_idx == cfg.super_mma_warp_id: - _super_mma_warp( + super_mma_warp( cfg, total_tiles, bidx, @@ -2340,12 +2396,12 @@ def _kernel( sSched, lane, sK_inv_raw, - sPairwise_raw, + sQk_raw, sK_decay_raw, bars, ) elif warp_idx == cfg.tcgen05_mma_warp_id: - _tcgen05_mma_warp( + tcgen05_mma_warp( cfg, total_tiles, bidx, @@ -2353,8 +2409,8 @@ def _kernel( cu_seqlens, mWorkItems, sSched, - tmem_hold, - sPairwise, + sTmem_base, + sQk, sK_decay, sK_restore, sQ_decay, @@ -2362,7 +2418,7 @@ def _kernel( bars, ) elif warp_idx == cfg.epilogue_warp_id: - _epilogue_warp( + epilogue_warp( cfg, total_tiles, bidx, @@ -2374,16 +2430,16 @@ def _kernel( mO, sK_inv_raw, sO_raw, - sPairwise_raw, + sQk_raw, sQ_decay_raw, - sH_raw, + sCheckpoint_raw, desc_o_base, - desc_h_base, + desc_checkpoint_base, checkpoint_every_n_tokens, bars, ) elif warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]: - _compute0_warp_group( + compute0_warp_group( cfg, total_tiles, bidx, @@ -2410,7 +2466,7 @@ def _kernel( bars, ) elif warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]: - _compute1_warp_group( + compute1_warp_group( cfg, total_tiles, bidx, @@ -2419,15 +2475,15 @@ def _kernel( mWorkItems, sSched, lane, - tmem_hold, + sTmem_base, warp_idx, - mS_out, - mS_init, + mState_out, + mState_init, mO, sO_raw, sV_raw, sW_raw, - sH_raw, + sCheckpoint_raw, checkpoint_every_n_tokens, scale, bars, @@ -2439,8 +2495,7 @@ class Gdn2Cfg: """Kernel cfg (fixed BT=16 schedule constants; derived TMEM column offsets and SMEM buffer cosizes are stamped by ``build_cfg``; per-stage sizes are inlined at the use sites). Passed ``cfg``-first (a ``cutlass.Constexpr``) - into ``_host`` / ``_kernel`` and every warp body, mirroring GDN's - ``GdnCfg``.""" + into ``host`` / ``kernel`` and every warp body.""" io_dtype: Type[cutlass.Numeric] state_dtype: Type[cutlass.Numeric] @@ -2450,14 +2505,11 @@ class Gdn2Cfg: l2norm: bool safe_gate: bool gate_scale_log2: float - beta_w_sigmoid: bool q_ratio: int k_ratio: int v_ratio: int n_heads_out: int max_active_clusters: int - # split-K: tiles come from a work-item table (see common/split_k.py); - # each item computes chunks [cstart, wend) and writes only [wstart, wend) split_k: bool = False dyn_sched: bool = False sched_stages: int = CFG.SMEM_SCHED_STAGES @@ -2477,24 +2529,25 @@ class Gdn2Cfg: cg0_group_count: int = 2 cg0_warps_per_group: int = 4 cg0_threads_per_group: int = 0 - nbar_cg0_group0_id: int = 1 # CG0 group g syncs on nbar id 1 + g + cg0_group_sync_barrier_base_id: int = 1 # CG0 group g syncs on nbar id 1 + g tmem_user_threads: int = 0 - nbar_tmem_lifecycle_id: int = 3 + tmem_lifecycle_barrier_id: int = 3 num_regs_compute_group_0: int = CFG.NUM_REGS_COMPUTE_GROUP_0 num_regs_compute_group_1: int = CFG.NUM_REGS_COMPUTE_GROUP_1 num_regs_other: int = CFG.NUM_REGS_OTHER - # --- SMEM / TMEM ring stage counts --- + # ---- SMEM / TMEM ring stage counts ------------------------------------------- smem_raw_stages: int = CFG.SMEM_RAW_STAGES + smem_checkpoint_stages: int = 1 smem_o_stages: int = CFG.SMEM_O_STAGES smem_decay_stages: int = CFG.SMEM_DECAY_STAGES - smem_pairwise_stages: int = CFG.SMEM_PAIRWISE_STAGES + smem_qk_stages: int = CFG.SMEM_QK_STAGES smem_state_scale_diag_stages: int = CFG.SMEM_STATE_SCALE_DIAG_STAGES qk_scale_ready_stages: int = CFG.QK_SCALE_READY_STAGES tmem_q_state_acc_stages: int = CFG.TMEM_Q_STATE_ACC_STAGES - # --- TMEM column offsets (state doubles as the final_state acc) --- - tmem_state_offset: int = 0 + # ---- TMEM column offsets (state doubles as the final_state acc) -------------- + tmem_state_acc_offset: int = 0 tmem_state_inp_offset: int = 0 tmem_q_state_acc_offset: int = 0 tmem_state_k_acc_offset: int = 0 @@ -2502,7 +2555,7 @@ class Gdn2Cfg: tmem_rhs_inp_offset: int = 0 tmem_update_inp_offset: int = 0 - # --- SMEM buffer cosizes --- + # ---- SMEM buffer cosizes ----------------------------------------------------- q_cosize: int = 0 k_cosize: int = 0 v_cosize: int = 0 @@ -2515,7 +2568,15 @@ class Gdn2Cfg: k_restore_cosize: int = 0 state_scale_diag_cosize: int = 0 o_cosize: int = 0 - pairwise_cosize: int = 0 + + # TMA transaction bytes per stage + tma_q_bytes: int = 0 + tma_k_bytes: int = 0 + tma_gate_bytes: int = 0 + tma_beta_bytes: int = 0 + tma_v_bytes: int = 0 + tma_w_bytes: int = 0 + qk_cosize: int = 0 def build_cfg( @@ -2528,7 +2589,6 @@ def build_cfg( l2norm: bool, safe_gate: bool, gate_scale_log2: float, - beta_w_sigmoid: bool, q_ratio: int, k_ratio: int, v_ratio: int, @@ -2550,7 +2610,6 @@ def build_cfg( l2norm=l2norm, safe_gate=safe_gate, gate_scale_log2=gate_scale_log2, - beta_w_sigmoid=beta_w_sigmoid, q_ratio=q_ratio, k_ratio=k_ratio, v_ratio=v_ratio, @@ -2560,16 +2619,15 @@ def build_cfg( dyn_sched=dyn_sched, ) if enable_checkpoints: - # the 32 KB H staging buffer must fit next to the raw ring: trim the - # q/k/v/gate/beta/w TMA lookahead for H compiles - cfg.smem_raw_stages = 4 + cfg.smem_raw_stages = 3 + cfg.smem_checkpoint_stages = 2 cfg.threads_per_cta = 16 * cfg.threads_per_warp cfg.cg0_threads_per_group = cfg.cg0_warps_per_group * cfg.threads_per_warp cfg.tmem_user_threads = (1 + len(cfg.compute_group_1_warp_ids)) * cfg.threads_per_warp if cfg.smem_state_scale_diag_stages != cfg.qk_scale_ready_stages: raise ValueError("diag and qk-scale ready rings must share their rolling stage") - cfg.tmem_state_inp_offset = cfg.tmem_state_offset + cfg.d_k + cfg.tmem_state_inp_offset = cfg.tmem_state_acc_offset + cfg.d_k cfg.tmem_q_state_acc_offset = cfg.tmem_state_inp_offset + (cfg.d_k // 2) cfg.tmem_state_k_acc_offset = cfg.tmem_q_state_acc_offset + cfg.tmem_q_state_acc_stages * cfg.b_t cfg.tmem_update_acc_offset = cfg.tmem_state_k_acc_offset + cfg.b_t @@ -2589,19 +2647,104 @@ def build_cfg( cfg.k_restore_cosize = cfg.smem_decay_stages * cfg.d_k * cfg.b_t cfg.state_scale_diag_cosize = cfg.smem_state_scale_diag_stages * (cfg.d_k // 16) * 256 cfg.o_cosize = cfg.smem_o_stages * cfg.b_t * cfg.d_v - cfg.pairwise_cosize = cfg.smem_pairwise_stages * 2 * cfg.b_t * cfg.b_t + cfg.qk_cosize = cfg.smem_qk_stages * 2 * cfg.b_t * cfg.b_t + cfg.tma_q_bytes = cfg.d_k * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_k_bytes = cfg.d_k * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_gate_bytes = cfg.d_k * cfg.b_t * 4 + cfg.tma_beta_bytes = cfg.d_k * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_v_bytes = cfg.d_v * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_w_bytes = cfg.d_v * cfg.b_t * (cfg.io_dtype.width // 8) return cfg -def get_workspace_size(B: int, HQ: int, HV: int) -> int: - """Bytes for the per-(batch, head) TMA-descriptor arrays (q, k, v, gate, - beta, w, o, h) + 128 alignment slack.""" - HO = HQ if HQ >= HV else HV - return TENSOR_MAP_QWORDS * 8 * (8 * B * HO) + 128 +TENSORMAP_DESC_ARRAYS = 8 # per-batch runtime TMA descriptors: Q, K, V, Gate, Beta, W, O, Checkpoint +TENSORMAP_STATIC_SLOTS = 0 + + +@cute.kernel +def build_all_descs_kernel( + base_q: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_k: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_v: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_gate: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_beta: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_w: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_o: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_checkpoint: cutlass.GridConstant[cuda.tensor_map.TensorMap], + desc_ws: cute.Tensor, + cu_seqlens: cute.Tensor, + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + gate: cute.Tensor, + beta: cute.Tensor, + w: cute.Tensor, + o: cute.Tensor, + state_checkpoints: cute.Tensor | None, + n_batch: cutlass.Int32, + q_row_stride: cutlass.Int32, + k_row_stride: cutlass.Int32, + v_row_stride: cutlass.Int32, + gate_row_stride: cutlass.Int32, + beta_row_stride: cutlass.Int32, + w_row_stride: cutlass.Int32, + o_row_stride: cutlass.Int32, + checkpoint_row_stride: cutlass.Int32, + checkpoint_every_n: cutlass.Int32, +) -> None: + """Single-launch builder for the per-BATCH descriptor arrays (one warp + per array; warp ``i`` emits array ``i`` and release-fences its slots). + Heads are load coordinates, so only the sequence base and token extent + are patched per slot.""" + tidx, _, _ = cute.arch.thread_idx() + widx = cutlass.Int32(tidx) // cutlass.Int32(32) + arr_words = n_batch * cutlass.Int32(TENSOR_MAP_QWORDS) + sub0 = cute.make_tensor(desc_ws.iterator, cute.make_layout((arr_words,), stride=(1,))) + sub1 = cute.make_tensor(desc_ws.iterator + arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub2 = cute.make_tensor(desc_ws.iterator + 2 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub3 = cute.make_tensor(desc_ws.iterator + 3 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub4 = cute.make_tensor(desc_ws.iterator + 4 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub5 = cute.make_tensor(desc_ws.iterator + 5 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub6 = cute.make_tensor(desc_ws.iterator + 6 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub7 = cute.make_tensor(desc_ws.iterator + 7 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + + if widx == 0: + if nvvm.elect_sync(): + emit_seq_descs(base_q, sub0, cu_seqlens, q, n_batch, q_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 1: + if nvvm.elect_sync(): + emit_seq_descs(base_k, sub1, cu_seqlens, k, n_batch, k_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 2: + if nvvm.elect_sync(): + emit_seq_descs(base_v, sub2, cu_seqlens, v, n_batch, v_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 3: + if nvvm.elect_sync(): + emit_seq_descs(base_gate, sub3, cu_seqlens, gate, n_batch, gate_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 4: + if nvvm.elect_sync(): + emit_seq_descs(base_beta, sub4, cu_seqlens, beta, n_batch, beta_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 5: + if nvvm.elect_sync(): + emit_seq_descs(base_w, sub5, cu_seqlens, w, n_batch, w_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 6: + if nvvm.elect_sync(): + emit_seq_descs(base_o, sub6, cu_seqlens, o, n_batch, o_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if cutlass.const_expr(state_checkpoints is not None): + if widx == 7: + if nvvm.elect_sync(): + emit_checkpoint_seq_descs(base_checkpoint, sub7, cu_seqlens, state_checkpoints, n_batch, checkpoint_row_stride, checkpoint_every_n, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) @cute.jit -def _build_descs( +def build_descs( io_dtype: cutlass.Constexpr, b_t: cutlass.Constexpr[int], q: cute.Tensor, @@ -2611,148 +2754,96 @@ def _build_descs( beta: cute.Tensor, w: cute.Tensor, o: cute.Tensor, - h: cute.Tensor | None, + state_checkpoints: cute.Tensor | None, cu_seqlens: cute.Tensor, tensormap_workspace: cute.Tensor, - h_every_n: cutlass.Int32, + checkpoint_every_n: cutlass.Int32, stream: cuda_driver.CUstream, ): """Build the 8 per-(batch, head) TMA-descriptor arrays (q, k, v, gate, - beta, w, o, h) into ``tensormap_workspace``. + beta, w, o, state_checkpoints) into ``tensormap_workspace``. Launched on every execute: the descriptors fold cu_seqlens contents into GLOBAL_ADDRESS and GLOBAL_DIM, which the host cannot read without a D2H sync. Each descriptor folds the sequence base + head offset into GLOBAL_ADDRESS (Int64) and caps the token GLOBAL_DIM to the sequence length, so the main kernel's coordinates are sequence-relative and tail - chunks clip in hardware. The H descriptor is 3-D ``(dv, dk, entry)`` - over the packed ``[total_h, HO, DK, DV]`` series; its per-sequence entry + chunks clip in hardware. The checkpoint descriptor is 3-D ``(dv, dk, entry)`` + over the packed ``[total_checkpoints, HO, DK, DV]`` series; its per-sequence entry offsets ((seqlen-1)//N, prefix-summed) are derived on device and its - entry extent is capped per sequence, so H store coordinates are + entry extent is capped per sequence, so checkpoint store coordinates are sequence-local.""" h_q = q.shape[1] h_k = k.shape[1] h_v = v.shape[1] - ho = gate.shape[1] + n_heads_out = gate.shape[1] batch_size = cu_seqlens.shape[0] - 1 d_k = q.shape[2] d_v = v.shape[2] bpe = io_dtype.width // 8 - granu = 128 // bpe + tma_granu_elems = 128 // bpe seqlen = q.shape[0] - def _head0(t, dim, heads): - # 2-D (dim, token) head-0 view: box (granu, b_t) matches the main - # kernel's SMEM staging byte-for-byte - return cute.make_tensor(t.iterator, cute.make_layout((dim, seqlen), stride=(1, heads * dim))) + def headed(t, dim, hn): + return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) swz = cuda.TensorMapSwizzle.s128b - base_q = cuda.create_tensor_map_tiled_from_view(_head0(q, d_k, h_q), box_dims=(granu, b_t), stride_order=(0, 1), swizzle=swz) - base_k = cuda.create_tensor_map_tiled_from_view(_head0(k, d_k, h_k), box_dims=(granu, b_t), stride_order=(0, 1), swizzle=swz) - base_v = cuda.create_tensor_map_tiled_from_view(_head0(v, d_v, h_v), box_dims=(granu, b_t), stride_order=(0, 1), swizzle=swz) - base_gate = cuda.create_tensor_map_tiled_from_view(_head0(gate, d_k, ho), box_dims=(32, b_t), stride_order=(0, 1), swizzle=swz) - base_beta = cuda.create_tensor_map_tiled_from_view(_head0(beta, d_k, ho), box_dims=(granu, b_t), stride_order=(0, 1), swizzle=swz) - base_w = cuda.create_tensor_map_tiled_from_view(_head0(w, d_v, ho), box_dims=(granu, b_t), stride_order=(0, 1), swizzle=swz) - base_o = cuda.create_tensor_map_tiled_from_view(_head0(o, d_v, ho), box_dims=(granu, b_t), stride_order=(0, 1), swizzle=swz) - - arr_words = (batch_size * ho) * TENSOR_MAP_QWORDS - ws_iter = tensormap_workspace.iterator - - def _sub(i): - return cute.make_tensor(ws_iter + i * arr_words, cute.make_layout((arr_words,), stride=(1,))) - - build_qkv_load_descs_kernel( - base_q, _sub(0), cu_seqlens, q, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(ho // h_q), cutlass.Int32(d_k), cutlass.Int32(h_q * d_k), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_k, _sub(1), cu_seqlens, k, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(ho // h_k), cutlass.Int32(d_k), cutlass.Int32(h_k * d_k), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_v, _sub(2), cu_seqlens, v, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(ho // h_v), cutlass.Int32(d_v), cutlass.Int32(h_v * d_v), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_gate, _sub(3), cu_seqlens, gate, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(1), cutlass.Int32(d_k), cutlass.Int32(ho * d_k), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_beta, _sub(4), cu_seqlens, beta, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(1), cutlass.Int32(d_k), cutlass.Int32(ho * d_k), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_w, _sub(5), cu_seqlens, w, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(1), cutlass.Int32(d_v), cutlass.Int32(ho * d_v), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_o, _sub(6), cu_seqlens, o, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(1), cutlass.Int32(d_v), cutlass.Int32(ho * d_v), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - if cutlass.const_expr(h is not None): - h_view = cute.make_tensor( - h.iterator, + base_q = cuda.create_tensor_map_tiled_from_view(headed(q, d_k, h_q), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, n_heads_out), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_beta = cuda.create_tensor_map_tiled_from_view(headed(beta, d_k, n_heads_out), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_w = cuda.create_tensor_map_tiled_from_view(headed(w, d_v, n_heads_out), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_o = cuda.create_tensor_map_tiled_from_view(headed(o, d_v, n_heads_out), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + + base_checkpoint = base_o + if cutlass.const_expr(state_checkpoints is not None): + checkpoint_view = cute.make_tensor( + state_checkpoints.iterator, cute.make_layout( - (h.shape[3], h.shape[2], h.shape[0]), - stride=(h.stride[3], h.stride[2], h.stride[0]), + (state_checkpoints.shape[3], state_checkpoints.shape[2], state_checkpoints.shape[0], n_heads_out), + stride=(state_checkpoints.stride[3], state_checkpoints.stride[2], state_checkpoints.stride[0], state_checkpoints.stride[1]), ), ) - base_h = cuda.create_tensor_map_tiled_from_view(h_view, box_dims=(granu, d_k, 1), stride_order=(0, 1, 2), swizzle=swz) - build_h_descs_kernel( - base_h, - _sub(7), - cu_seqlens, - h, - cutlass.Int32(batch_size), - cutlass.Int32(ho), - cutlass.Int32(h.stride[1]), - cutlass.Int32(h.stride[0]), - h_every_n, - 2, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - - -# --------------------------------------------------------------------------- -# Torch adapter / host-side compilation -# --------------------------------------------------------------------------- - - -def _device_sm_count() -> int: - """Multiprocessor count of the current device (runtime API: auto-inits - the primary context, so this works before any other CUDA call).""" - from cuda.bindings import runtime as _rt - - err, dev = _rt.cudaGetDevice() - if int(err) != 0: - raise RuntimeError(f"cudaGetDevice failed: {err}") - err, count = _rt.cudaDeviceGetAttribute(_rt.cudaDeviceAttr.cudaDevAttrMultiProcessorCount, dev) - if int(err) != 0: - raise RuntimeError(f"cudaDeviceGetAttribute failed: {err}") - return count - - -def _data_ptr(t) -> int: - """Device address of a tensor-like (``data_ptr()`` or the CUDA array - interface).""" - fn = getattr(t, "data_ptr", None) - if fn is not None: - return fn() - return t.__cuda_array_interface__["data"][0] - - -def _cutlass_io_dtype(dtype): - name = str(dtype) - if "bfloat16" in name: - return cutlass.BFloat16 - if "float16" in name or "half" in name: - return cutlass.Float16 - raise ValueError(f"Unsupported dtype {dtype}, expected bfloat16 or float16") - - -def _cutlass_state_dtype(dtype): - name = str(dtype) - if "bfloat16" in name: - return cutlass.BFloat16 - if "float32" in name: - return cutlass.Float32 - raise ValueError(f"Unsupported state dtype {dtype}, expected float32 or bfloat16") + base_checkpoint = cuda.create_tensor_map_tiled_from_view(checkpoint_view, box_dims=(tma_granu_elems, d_k, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz) + n_warps = 8 if state_checkpoints is not None else 7 + build_all_descs_kernel( + base_q, + base_k, + base_v, + base_gate, + base_beta, + base_w, + base_o, + base_checkpoint, + tensormap_workspace, + cu_seqlens, + q, + k, + v, + gate, + beta, + w, + o, + state_checkpoints, + cutlass.Int32(batch_size), + cutlass.Int32(q.stride[0]), + cutlass.Int32(k.stride[0]), + cutlass.Int32(v.stride[0]), + cutlass.Int32(gate.stride[0]), + cutlass.Int32(beta.stride[0]), + cutlass.Int32(w.stride[0]), + cutlass.Int32(o.stride[0]), + cutlass.Int32(state_checkpoints.stride[0] if state_checkpoints is not None else 0), + checkpoint_every_n, + ).launch(grid=(1, 1, 1), block=(32 * n_warps, 1, 1), stream=stream) + + +# ---- Torch adapter / host-side compilation --------------------------------------- @lru_cache(maxsize=None) -def _get_compiled_cache( +def get_compiled_cache( io_dtype_str: str, state_dtype_str: str, cu_dtype_str: str, @@ -2765,7 +2856,6 @@ def _get_compiled_cache( l2norm: bool, safe_gate: bool, gate_lower_bound: float, - beta_w_sigmoid: bool, split_k: bool, dyn_sched: bool, ): @@ -2782,7 +2872,6 @@ def compile( l2norm: bool, safe_gate: bool, gate_scale_log2: float, - beta_w_sigmoid: bool, q_ratio: int, k_ratio: int, v_ratio: int, @@ -2800,9 +2889,9 @@ def compile( beta_cute, w_cute, cu_seqlens_cute, - s_in_cute, + state_in_cute, o_cute, - s_out_cute, + state_out_cute, work_items_cute=None, work_count_cute=None, sched_ctr_cute=None, @@ -2821,7 +2910,6 @@ def compile( l2norm=l2norm, safe_gate=safe_gate, gate_scale_log2=gate_scale_log2, - beta_w_sigmoid=beta_w_sigmoid, q_ratio=q_ratio, k_ratio=k_ratio, v_ratio=v_ratio, @@ -2832,7 +2920,7 @@ def compile( ) return cute.compile( - _host, + host, cfg, q_cute, k_cute, @@ -2843,9 +2931,9 @@ def compile( beta_cute, w_cute, cu_seqlens_cute, - s_in_cute, + state_in_cute, o_cute, - s_out_cute, + state_out_cute, work_items_cute, work_count_cute, sched_ctr_cute, @@ -2870,13 +2958,12 @@ def chunk_gdn2_sm100( output_state, scale: float, checkpoint_every_n_tokens: int = 0, - output_checkpoints=None, + output_state_checkpoints=None, use_qk_l2norm_in_kernel: bool = False, safe_gate: bool = False, gate_lower_bound: float = DEFAULT_GATE_LOWER_BOUND, a_log=None, dt_bias=None, - use_beta_w_sigmoid_in_kernel: bool = False, work_items=None, work_count=None, sched_ctr=None, @@ -2886,7 +2973,9 @@ def chunk_gdn2_sm100( ) -> None: """Execute the Blackwell BT=16 chunked GDN-2 prefill kernel. - All tensors must be contiguous and on the same CUDA device. + All tensors must be on the same CUDA device with a stride-1 innermost + dim; outer strides are free (padded / permuted views are read through + the TMA descriptors and dynamic layouts). Args: q: ``(total_tokens, HQ, DK)`` float16/bfloat16 @@ -2897,26 +2986,24 @@ def chunk_gdn2_sm100( ``lower_bound * sigmoid(exp(a_log) * (gate + dt_bias))``. beta: ``(total_tokens, HO, DK)`` io dtype, channel-wise erase gate w: ``(total_tokens, HO, DV)`` io dtype, channel-wise write gate - (both raw logits when ``use_beta_w_sigmoid_in_kernel``) output: ``(total_tokens, HO, DV)`` float16/bfloat16, pre-allocated cu_seqlens: ``(num_seqs + 1,)`` int32 initial_state: ``(num_seqs, HO, DK, DV)`` float32/bfloat16, or None output_state: ``(num_seqs, HO, DK, DV)`` float32/bfloat16, or None scale: attention scale factor (must not be 0) - checkpoint_every_n_tokens: emit an H entry every N tokens (0 = off). - H[j] is the state after ``(j + 1) * N`` tokens, STRICTLY BEFORE + checkpoint_every_n_tokens: emit a checkpoint entry every N tokens (0 = off). + checkpoint[j] is the state after ``(j + 1) * N`` tokens, STRICTLY BEFORE the sequence end — the end-of-sequence state is only ``output_state``. - output_checkpoints: ``(total_h, HO, DK, DV)`` io-dtype (KV, v - contiguous — the GDN H layout); the per-sequence entry offsets + output_state_checkpoints: ``(total_checkpoints, HO, DK, DV)`` io-dtype (KV, v + contiguous — the GDN checkpoint layout); the per-sequence entry offsets are derived on device from ``cu_seqlens`` ((seqlen-1)//N, prefix-summed), so there is no cu_checkpoints array use_qk_l2norm_in_kernel: L2-normalize q/k rows inside the kernel safe_gate: interpret ``gate`` through the safe-gate transform a_log: ``(HO,)`` float32, safe-gate per-head log-amplitude (None = 0) dt_bias: ``(HO, DK)`` float32, safe-gate channel bias (None = 0) - use_beta_w_sigmoid_in_kernel: ``beta``/``w`` hold logits; sigmoid in-kernel - work_items: ``(max_items, 6)`` int32 split-K work-item table from + work_items: ``(max_items, 8)`` int32 split-K work-item table from ``common/split_k.py``, or None for the one-tile-per-(b,h) layout. With a table, each item computes chunks ``[cstart, wend)`` and writes O/checkpoints only for @@ -2932,11 +3019,11 @@ def chunk_gdn2_sm100( store_final_state = output_state is not None enable_checkpoints = checkpoint_every_n_tokens > 0 if enable_checkpoints: - if output_checkpoints is None: - raise ValueError("checkpoint_every_n_tokens > 0 requires output_checkpoints") - if str(output_checkpoints.dtype).split(".")[-1] != str(q.dtype).split(".")[-1]: + if output_state_checkpoints is None: + raise ValueError("checkpoint_every_n_tokens > 0 requires output_state_checkpoints") + if str(output_state_checkpoints.dtype).split(".")[-1] != str(q.dtype).split(".")[-1]: raise ValueError( - f"output_checkpoints dtype must match the io dtype (fp32 state belongs to output_state): got {output_checkpoints.dtype} with io {q.dtype}" + f"output_state_checkpoints dtype must match the io dtype (fp32 state belongs to output_state): got {output_state_checkpoints.dtype} with io {q.dtype}" ) split_k = work_items is not None dyn_sched = sched_ctr is not None @@ -2968,11 +3055,9 @@ def chunk_gdn2_sm100( if not safe_gate: a_log = None dt_bias = None - if _data_ptr(tensormap_workspace) % 128 != 0: - raise ValueError("tensormap_workspace must be 128-byte aligned") cu_stream = cuda_driver.CUstream(int(stream)) - cache = _get_compiled_cache( + cache = get_compiled_cache( str(q.dtype), str(state_dtype_src), str(cu_seqlens.dtype), @@ -2985,46 +3070,36 @@ def chunk_gdn2_sm100( use_qk_l2norm_in_kernel, safe_gate, gate_lower_bound, - use_beta_w_sigmoid_in_kernel, split_k, dyn_sched, ) if "compiled" not in cache: - io_dtype = _cutlass_io_dtype(q.dtype) - state_dtype = _cutlass_state_dtype(state_dtype_src) - q_cute = from_dlpack(q, assumed_align=16) - q_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - k_cute = from_dlpack(k, assumed_align=16) - k_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - v_cute = from_dlpack(v, assumed_align=16) - v_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - gate_cute = from_dlpack(gate, assumed_align=16) - gate_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) + io_dtype = get_dtype(q.dtype) + state_dtype = get_dtype(state_dtype_src) + q_cute = from_dlpack(q, assumed_align=16).mark_layout_dynamic(leading_dim=2) + k_cute = from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2) + v_cute = from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2) + gate_cute = from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=2) a_log_cute = from_dlpack(a_log, assumed_align=4) if a_log is not None else None dt_bias_cute = from_dlpack(dt_bias, assumed_align=16) if dt_bias is not None else None - beta_cute = from_dlpack(beta, assumed_align=4) - beta_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - w_cute = from_dlpack(w, assumed_align=16) - w_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - o_cute = from_dlpack(output, assumed_align=16) - o_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) + beta_cute = from_dlpack(beta, assumed_align=4).mark_layout_dynamic(leading_dim=2) + w_cute = from_dlpack(w, assumed_align=16).mark_layout_dynamic(leading_dim=2) + o_cute = from_dlpack(output, assumed_align=16).mark_layout_dynamic(leading_dim=2) cu_seqlens_cute = from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic() - s_in_cute = None + state_in_cute = None if use_initial_state: - s_in_cute = from_dlpack(initial_state, assumed_align=16) - s_in_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=CFG.D_K) + state_in_cute = from_dlpack(initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) - s_out_cute = None + state_out_cute = None if store_final_state: - s_out_cute = from_dlpack(output_state, assumed_align=16) - s_out_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=CFG.D_K) + state_out_cute = from_dlpack(output_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) work_items_cute = None work_count_cute = None if split_k: - work_items_cute = from_dlpack(work_items, assumed_align=4) + work_items_cute = from_dlpack(work_items, assumed_align=16) work_items_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) work_count_cute = from_dlpack(work_count, assumed_align=4).mark_layout_dynamic() @@ -3043,14 +3118,13 @@ def chunk_gdn2_sm100( use_qk_l2norm_in_kernel, safe_gate, gate_scale_log2, - use_beta_w_sigmoid_in_kernel, q_ratio, k_ratio, v_ratio, HO, split_k, dyn_sched, - num_sm=_device_sm_count(), + num_sm=multiprocessor_count(current_device_id()), q_cute=q_cute, k_cute=k_cute, v_cute=v_cute, @@ -3060,9 +3134,9 @@ def chunk_gdn2_sm100( beta_cute=beta_cute, w_cute=w_cute, cu_seqlens_cute=cu_seqlens_cute, - s_in_cute=s_in_cute, + state_in_cute=state_in_cute, o_cute=o_cute, - s_out_cute=s_out_cute, + state_out_cute=state_out_cute, work_items_cute=work_items_cute, work_count_cute=work_count_cute, sched_ctr_cute=sched_ctr_cute, @@ -3073,43 +3147,29 @@ def chunk_gdn2_sm100( ) compiled = cache["compiled"] - - # The descriptors encode cu_seqlens' CONTENTS, which no key built from the - # buffers can track. The skip this replaces asked torch's _version counter, - # so it was sound for a torch caller and silently stale for every other - # producer. Rebuilding unconditionally measures free: 131 vs 135 us of host - # time, and 157 either way once the launches are waited on. - h_for_descs = output_checkpoints if enable_checkpoints else None - if cache.get("build_descs_has_h") != (h_for_descs is not None): + state_checkpoints_for_descs = output_state_checkpoints if enable_checkpoints else None + # desc build runs every execute by contract (cu contents are data; + # buffer pointers may change) — capture-safe, single tiny launch + if cache.get("build_descs_has_state_checkpoints") != (state_checkpoints_for_descs is not None): cache.pop("build_descs", None) - cache["build_descs_has_h"] = h_for_descs is not None + cache["build_descs_has_state_checkpoints"] = state_checkpoints_for_descs is not None if "build_descs" not in cache: - io_dtype = _cutlass_io_dtype(q.dtype) - - def _bd3(t): - c = from_dlpack(t, assumed_align=16) - c.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - return c - - def _bd4(t): - c = from_dlpack(t, assumed_align=16) - c.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2, 3), divisibility=1) - return c + io_dtype = get_dtype(q.dtype) cu_bd = from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic() ws_bd = from_dlpack(tensormap_workspace, assumed_align=128).mark_layout_dynamic() cache["build_descs"] = cute.compile( - _build_descs, + build_descs, io_dtype, CFG.B_T, - _bd3(q), - _bd3(k), - _bd3(v), - _bd3(gate), - _bd3(beta), - _bd3(w), - _bd3(output), - None if h_for_descs is None else _bd4(h_for_descs), + from_dlpack(q, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(beta, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(w, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(output, assumed_align=16).mark_layout_dynamic(leading_dim=2), + None if state_checkpoints_for_descs is None else from_dlpack(state_checkpoints_for_descs, assumed_align=16).mark_layout_dynamic(leading_dim=3), cu_bd, ws_bd, cutlass.Int32(checkpoint_every_n_tokens), @@ -3124,13 +3184,12 @@ def _bd4(t): beta, w, output, - h_for_descs, + state_checkpoints_for_descs, cu_seqlens, tensormap_workspace, checkpoint_every_n_tokens, cu_stream, ) - compiled( q, k, diff --git a/python/cudnn/linear_attention/frost/kernel/gdn2_recompute_config.py b/python/cudnn/linear_attention/frost/kernel/gdn2_recompute_config.py new file mode 100644 index 000000000..0f5fb5009 --- /dev/null +++ b/python/cudnn/linear_attention/frost/kernel/gdn2_recompute_config.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# This kernel is derived from cuDNN, NVIDIA Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gated DeltaNet v2 (GDN-2) Cutlass DSL recompute (state/checkpoint-only) kernel +config (fixed compile-time constants). A fork of the BT=16 KDA schedule +extended with the per-key erase gate (beta) and per-value write gate (w); the +derived SMEM/TMEM sizes and offsets are stamped by ``build_cfg`` in +``gdn2_recompute_f16.py``. + +Target arch: Blackwell SM100 (GB200) / SM103 (GB300). +""" + +from dataclasses import dataclass +from typing import Tuple + + +@dataclass(frozen=True) +class Cfg: + # --- tile shape --- + B_T: int = 16 # chunk-inner token tile (BT=16 schedule) + D_K: int = 128 # query/key head dim + D_V: int = 128 # value head dim + + # --- warp assignments (16 warps = 512 threads) --- + COMPUTE_GROUP_0_WARP_IDS: Tuple[int, ...] = (0, 1, 2, 3, 4, 5, 6, 7) # decay/beta-operand materialize + COMPUTE_GROUP_1_WARP_IDS: Tuple[int, ...] = (8, 9, 10, 11) # value-side TMEM (w*v - erase) + SUPER_MMA_WARP_ID: int = 12 # register-MMA kk + Neumann inverse + TCGEN05_MMA_WARP_ID: int = 13 # tcgen05 state GEMMs + TMA_WARP_ID: int = 14 # k/v/gate/beta/w TMA loads + EPILOGUE_WARP_ID: int = 15 # checkpoint TMA store + + # --- register split --- + NUM_REGS_COMPUTE_GROUP_0: int = 160 + NUM_REGS_COMPUTE_GROUP_1: int = 136 + NUM_REGS_OTHER: int = 56 + + THREADS_PER_WARP: int = 32 + + BUFFER_ALIGN_BYTES: int = 1024 + + # --- SMEM / TMEM ring stage counts --- + SMEM_RAW_STAGES: int = 5 + SMEM_SCHED_STAGES: int = 8 + SMEM_DECAY_STAGES: int = 2 + SMEM_QK_STAGES: int = 2 + SMEM_STATE_SCALE_DIAG_STAGES: int = 3 + QK_SCALE_READY_STAGES: int = 3 + + CLUSTER_SHAPE_MNK: Tuple[int, int, int] = (1, 1, 1) + + +CFG = Cfg() diff --git a/python/cudnn/linear_attention/frost/kernel/gdn2_recompute_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn2_recompute_f16.py new file mode 100644 index 000000000..db8b56d14 --- /dev/null +++ b/python/cudnn/linear_attention/frost/kernel/gdn2_recompute_f16.py @@ -0,0 +1,2683 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# This kernel is derived from cuDNN, NVIDIA Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Chunked Gated DeltaNet v2 (GDN-2) recompute (state/checkpoint-only) kernel for +Blackwell SM100/SM103 (Cutlass DSL), BT=16 tiling with per-key-channel decay + +per-key erase gate (beta) + per-value write gate (w), using direct CUTLASS +primitives. Framework-neutral entry ``chunk_gdn2_recompute_sm100``. + +A copy of the prefill (``gdn2_prefill_f16.py``) with the Q/O path removed: +it reproduces the per-chunk checkpoint series and the final state +byte-for-byte while skipping the attention output. + + S_t = S_{t-1} * diag(exp(g_t)) + v_new_t = w_t * v_t - (beta_t * k_t)^T S_t + S_t += k_t (x) v_new_t + +vs KDA: the erase gate Beta and write gate W are per-channel tensors. Beta +is folded into the K_decay operand (feeds KK^T and state*K), the strict-lower +tile loses its per-row Beta scale, and RHS becomes `W*V - state*K`; Beta/W +arrive by TMA alongside K/V. + +ABI: k `[T, HK, DK]`, v `[T, HV, DV]`, gate +`[T, HO, DK]` fp32 (natural-log decay unless SAFE_GATE), beta `[T, HO, DK]` +and w `[T, HO, DV]` in the io dtype, cu_seqlens int64, states/checkpoints +`[N, HO, DV, DK]` (VK). GQA/GVA head broadcast follows repeat_interleave: +source head = head_idx // (HO // H_x). State presence, L2NORM, SAFE_GATE, +checkpoints, and the head ratios are compile-time specializations. + +Warp assignments (16 warps = 512 threads): + warps 0-7 : compute group 0 - Gate prefix scan + decay/restore operands + warps 8-11 : compute group 1 - TMEM value side, state stores + warp 12 : super-MMA - register-MMA KK^T + Neumann inverse + warp 13 : tcgen05-MMA - the four state GEMMs + the TMEM lifecycle + warp 14 : TMA load - per-chunk input G->S loads + warp 15 : epilogue - the checkpoint TMA store + +SMEM layout: + Buffer Bytes Stages + K / V raw 20480 5 <-- SW128 TMA ring (io dtype) + Beta / W raw 2x 20480 5 <-- per-channel gates, same ring + Gate raw 40960 5 <-- fp32 prefix-scan source + dt_bias (+a_log slot) 516 1 <-- SAFE_GATE only + K_inv 8192 2 <-- token-major ldmatrix/tcgen05 B operand + K decay 8192 2 <-- tcgen05 SW128 K-box-major A/B operands + K restore 8192 2 <-- tcgen05 B operand for the state update + state-scale diag 12288 3 <-- per-k-atom decay diagonal blocks + qk (A_inv) 1024 2 <-- SW32 16x16 register-MMA tiles + +TMEM layout (240 of 512 columns): + Buffer Cols Purpose + state 0-127 state[DK,DV] fp32 recurrent state + state inp 128-191 packed b16 A operand view of the state + state_k_acc 192-207 state*K fp32 accumulator + update_acc 208-223 update fp32 accumulator + rhs input 224-231 packed b16 A operand: W*V - state*(Beta*K) + update input 232-239 packed b16 A operand: the update readback + +GEMM schedule (tcgen05-MMA warp, in issue order per chunk): + state*K -> state_k_acc + state decay (diag blocks) + update = A_inv @ rhs -> update_acc + final_state += update @ K_restore + +Requires a cutlass DSL build providing `cutlass.experimental.*`; not available +in the pip nvidia-cutlass-dsl releases. +""" + +from dataclasses import dataclass +from functools import lru_cache +from typing import Callable, NamedTuple, Optional, Type + +import cuda.bindings.driver as cuda_driver +import cutlass +import cutlass.experimental.cuda as cuda +import cutlass.experimental.primitives as nvvm +import cutlass.cute as cute +from cutlass.cute.runtime import from_dlpack + +from ..common.split_k import decode_work_item +from ..common.host import get_dtype +from cudnn.frost.buffers import current_device_id, data_ptr +from cudnn.frost.device import multiprocessor_count +from ..common.thd import TENSOR_MAP_QWORDS, emit_checkpoint_seq_descs, emit_seq_descs +from .gdn2_recompute_config import CFG +from cudnn.frost.tile_dsl.barrier import ( + advance, + MBarrier, + PipelineState, + Producer, +) +from cudnn.frost.tile_dsl.handles import GmemTileTma, MmaDesc, SmemTile, tma_slice_runtime_desc +from cudnn.frost.tile_dsl.mma import mma_step, mma_ts_step +from cudnn.frost.tile_dsl.swizzle import swizzle_lin_128b, swizzle_lin_S, swizzle_xor_128b +from cudnn.frost.tile_dsl.tma import tma_load_tile, tma_store_commit, tma_store_tile, tma_store_wait, tma_tensormap_acquire +from cudnn.frost.tile_dsl.pointwise import ( + f16x2_to_f32, + fadd2, + fmul2, + ffma2, + movmatrix_16b, + mul_f16x2, + opaque_f32_zero, + fp32_to_fp16, + sub_f16x2, +) + +LOG2_E: float = 1.4426950408889634 + + +DEFAULT_GATE_LOWER_BOUND: float = -5.0 + + +# Host-side API defaults. + + +L2_NORM_EPS: float = 1.0e-12 + + +class Gdn2Bars(NamedTuple): + """Every inter-warp handoff as an ``MBarrier`` over its ring.""" + + mb_k_ready: MBarrier + mb_k_done: MBarrier + mb_v_ready: MBarrier + mb_v_done: MBarrier + mb_w_ready: MBarrier + mb_w_done: MBarrier + + mb_gate_ready: MBarrier + mb_gate_done: MBarrier + mb_beta_ready: MBarrier + mb_beta_done: MBarrier + + mb_state_k_acc_ready: MBarrier + mb_update_acc_ready: MBarrier + + mb_state_inp_ready: MBarrier + mb_rhs_ready: MBarrier + mb_update_ready: MBarrier + + mb_k_decay_inv_cg0_ready: MBarrier + mb_decay_done: MBarrier + mb_decay_super_done: MBarrier + mb_k_restore_done: MBarrier + mb_qk_scale_ready: MBarrier + mb_state_scale_diag_done: MBarrier + mb_a_ready: MBarrier + mb_a_done: MBarrier + + mb_state_read_done: MBarrier + mb_final_state_stored: MBarrier + + mb_checkpoint_tmastg_ready: MBarrier + mb_checkpoint_tmastg_done: MBarrier + + mb_sched_ready: MBarrier + mb_sched_done: MBarrier + + +def make_gdn2_bars(cfg) -> Gdn2Bars: + """Bars factory. MUST be called from inside ``kernel`` (allocates the + mbarrier rings in SMEM ahead of the data buffers).""" + + def alloc(n): + return cutlass.Array(cutlass.Int64, n, space=cutlass.AddressSpace.smem, alignment=8) + + WARP = cfg.threads_per_warp + CG0_GROUP_THREADS = cfg.cg0_warps_per_group * WARP + CG1_THREADS = len(cfg.compute_group_1_warp_ids) * WARP + + return Gdn2Bars( + mb_k_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_k_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_v_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_v_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_w_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_w_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_gate_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_gate_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_beta_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_beta_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_state_k_acc_ready=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), + mb_update_acc_ready=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), + mb_state_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_rhs_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_update_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_k_decay_inv_cg0_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=WARP, producer=Producer.THREAD), + mb_k_restore_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_qk_scale_ready=MBarrier( + alloc(cfg.qk_scale_ready_stages), + stages=cfg.qk_scale_ready_stages, + init_count=CG0_GROUP_THREADS, + producer=Producer.THREAD, + ), + mb_state_scale_diag_done=MBarrier( + alloc(cfg.smem_state_scale_diag_stages), + stages=cfg.smem_state_scale_diag_stages, + init_count=1, + producer=Producer.MMA_COMMIT, + ), + mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_a_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_state_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_final_state_stored=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_checkpoint_tmastg_ready=MBarrier( + alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=CG1_THREADS, producer=Producer.THREAD + ), + mb_checkpoint_tmastg_done=MBarrier(alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=WARP, producer=Producer.THREAD), + mb_sched_ready=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=1, producer=Producer.THREAD), + mb_sched_done=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=15, producer=Producer.THREAD), + ) + + +# ---- Dynamic tile scheduler ------------------------------------------------------ + + +@cute.jit +def sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): + """TMA-warp side: pull the next tile off the global ticket, publish it.""" + if cutlass.const_expr(cfg.dyn_sched): + bars.mb_sched_done[sched_state.idx].wait(sched_state.phase) + if nvvm.elect_sync(): + fetched = cutlass.Int32(nvvm.atomicrmw("add", mSched.iterator, cutlass.Int32(1), mem_order="relaxed", syncscope="gpu")) + sSched[sched_state.idx] = num_ctas + fetched + nvvm.bar_warp_sync(cute.arch.FULL_MASK) + next_tile = sSched[sched_state.idx] + if nvvm.elect_sync(): + bars.mb_sched_ready[sched_state.idx].arrive() + return next_tile, advance(sched_state, cfg.sched_stages) + return tile_idx + num_ctas, sched_state + + +@cute.jit +def sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): + """Consumer side: read the TMA warp's published next tile.""" + if cutlass.const_expr(cfg.dyn_sched): + bars.mb_sched_ready[sched_state.idx].wait(sched_state.phase) + next_tile = sSched[sched_state.idx] + if nvvm.elect_sync(): + bars.mb_sched_done[sched_state.idx].arrive() + return next_tile, advance(sched_state, cfg.sched_stages) + return tile_idx + num_ctas, sched_state + + +@cute.jit +def diag_idx(cfg, key_dim): + """Return the SW32 index for one entry in the 8-block diagonal.""" + + block = key_dim // cutlass.Int32(16) + coord = key_dim - block * cutlass.Int32(16) + storage_col = coord ^ cutlass.Int32((cfg.b_t // 2)) + linear_idx = block * cutlass.Int32(256) + coord * cutlass.Int32(16) + storage_col + return swizzle_lin_S(linear_idx, bbits=1, mbase=3, sshift=3) + + +@cute.jit +def tmaldg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + mSched, + sSched, + lane, + sBeta_raw, + sGate_raw, + sK_raw, + sV_raw, + sW_raw, + desc_k_base, + desc_v_base, + desc_gate_base, + desc_beta_base, + desc_w_base, + bars, +) -> None: + """TMA-LDG warp role (warp 14): persistent scheduler loop issuing the + per-chunk k/v/beta/w/gate G->S loads.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + sK_tma = SmemTile( + base=sK_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sV_tma = SmemTile( + base=sV_raw, + elems_per_stage=(cfg.d_v * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sBeta_tma = SmemTile( + base=sBeta_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sW_tma = SmemTile( + base=sW_raw, + elems_per_stage=(cfg.d_v * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sGate_tma = SmemTile( + base=sGate_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 32), + tma_granu_elems=32, + tma_subtile_stride_elems=(cfg.b_t * 32), + ) + raw_index = PipelineState.start(phase=1) + sched_state = PipelineState.start(phase=1) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + head_o = head_idx + head_k = head_idx if cfg.k_ratio == 1 else head_idx // cutlass.Int32(cfg.k_ratio) + head_v = head_idx if cfg.v_ratio == 1 else head_idx // cutlass.Int32(cfg.v_ratio) + slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_k_slot = (desc_k_base + slot).tospace(cutlass.AddressSpace.generic) + desc_v_slot = (desc_v_base + slot).tospace(cutlass.AddressSpace.generic) + desc_gate_slot = (desc_gate_base + slot).tospace(cutlass.AddressSpace.generic) + desc_beta_slot = (desc_beta_base + slot).tospace(cutlass.AddressSpace.generic) + desc_w_slot = (desc_w_base + slot).tospace(cutlass.AddressSpace.generic) + if elect_one: + tma_tensormap_acquire(desc_k_slot) + tma_tensormap_acquire(desc_v_slot) + tma_tensormap_acquire(desc_gate_slot) + tma_tensormap_acquire(desc_beta_slot) + tma_tensormap_acquire(desc_w_slot) + for chunk_idx in cutlass.range(cstart, wend, 1, unroll=1): + chunk_start = chunk_idx * cfg.b_t + + # ---- K load ---------------------------------------------------------- + bars.mb_k_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_k_ready[raw_index.idx].arrive(n_bytes=cfg.tma_k_bytes) + k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, chunk_start) + tma_load_tile(sK_tma[raw_index.idx], k_slice, bars.mb_k_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- V load ---------------------------------------------------------- + bars.mb_v_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_v_ready[raw_index.idx].arrive(n_bytes=cfg.tma_v_bytes) + v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, chunk_start) + tma_load_tile(sV_tma[raw_index.idx], v_slice, bars.mb_v_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- Beta load ------------------------------------------------------- + bars.mb_beta_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_beta_ready[raw_index.idx].arrive(n_bytes=cfg.tma_beta_bytes) + beta_slice = tma_slice_runtime_desc(desc_beta_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sBeta_tma[raw_index.idx], beta_slice, bars.mb_beta_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- W load ---------------------------------------------------------- + bars.mb_w_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_w_ready[raw_index.idx].arrive(n_bytes=cfg.tma_w_bytes) + w_slice = tma_slice_runtime_desc(desc_w_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sW_tma[raw_index.idx], w_slice, bars.mb_w_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- Gate load ------------------------------------------------------- + bars.mb_gate_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_gate_ready[raw_index.idx].arrive(n_bytes=cfg.tma_gate_bytes) + gate_slice = tma_slice_runtime_desc(desc_gate_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sGate_tma[raw_index.idx], gate_slice, bars.mb_gate_ready[raw_index.idx].smem_ptr, acquire=False) + raw_index = advance(raw_index, cfg.smem_raw_stages) + tile_idx, sched_state = sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def super_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_inv_raw, + sQk_raw, + sK_decay_raw, + bars, +) -> None: + """Super-MMA warp role (warp 12): persistent scheduler loop computing the + register-MMA A_inv.""" + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + + # ---- ldmatrix/stmatrix lane decode ------------------------------------------- + rhs_row_coord = lane % 8 + (cutlass.Int32(8) if (lane // 16) else cutlass.Int32(0)) + rhs_col_offset = cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0) + lhs_row_coord = lane % 8 + (cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0)) + lhs_col_offset = cutlass.Int32(8) if ((lane // 8) // 2) else cutlass.Int32(0) + decay_key_mask = cutlass.Int32(8) + stsm_row_coord = lane & 7 + stsm_col_coord = cutlass.Int32(0) + if (lane // 8) & 1: + stsm_row_coord = stsm_row_coord + cutlass.Int32(8) + if lane // 8 >= 2: + stsm_col_coord = cutlass.Int32(8) + stsm_idx = swizzle_lin_S(stsm_row_coord * cfg.b_t + (stsm_col_coord ^ (cfg.b_t // 2)), bbits=1, mbase=3, sshift=3) + global_chunk_base = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + num_chunks_tile = wend - cstart # processed chunks; ring bookkeeping runs on global_chunk_base + local_chunk + for local_chunk in cutlass.range(num_chunks_tile, unroll=1): + global_chunk = global_chunk_base + local_chunk + decay_stage = global_chunk % cfg.smem_decay_stages + qk_stage = global_chunk % cfg.smem_qk_stages + sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sK_decay_ptr = sK_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sQk_ptr = sQk_raw.data_ptr() + qk_stage * (cfg.b_t * cfg.b_t) + + bars.mb_a_done[qk_stage].wait(((global_chunk // cfg.smem_qk_stages) + 1) % 2) + bars.mb_k_decay_inv_cg0_ready[decay_stage].wait((global_chunk // cfg.smem_decay_stages) % 2) + + # ---- KK = K_decay @ K_inv^T ------------------------------------------ + kk_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + kk_acc[accum_idx] = cutlass.Float32(0.0) + + for k_block in cutlass.range_constexpr((cfg.d_k // 16)): + # Load B operand + k_inv_col = k_block * 16 + rhs_col_offset + k_inv_segment = k_inv_col // 64 + rhs_vec = nvvm.ldmatrix( + sK_inv_ptr + + k_inv_segment * (cfg.b_t * 64) + + rhs_row_coord * 64 + + swizzle_xor_128b(rhs_row_coord, k_inv_col - k_inv_segment * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + # Load A operand + storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask + storage_slice = storage_key // 64 + kk_lhs_vec = nvvm.ldmatrix( + sK_decay_ptr + + storage_slice * (cfg.b_t * 64) + + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + + mma_step( + kk_acc, + (kk_lhs_vec[0], kk_lhs_vec[1], kk_lhs_vec[2], kk_lhs_vec[3]), + (rhs_vec[0], rhs_vec[1], rhs_vec[2], rhs_vec[3]), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + + # ---- L = tril(KK, -1) fragment --------------------------------------- + row_lo = lane // 4 + row_hi = row_lo + cutlass.Int32(8) + l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_lo + if cutlass.const_expr(accum_idx % 4 >= 2): + row_coord = row_hi + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + l_frag[accum_idx] = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + l_a0 = fp32_to_fp16(l_frag[0], l_frag[1], dtype=cfg.io_dtype) + l_a1 = fp32_to_fp16(l_frag[2], l_frag[3], dtype=cfg.io_dtype) + l_a2 = fp32_to_fp16(l_frag[4], l_frag[5], dtype=cfg.io_dtype) + l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) + l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) + + # ---- A_inv = I - L, then three Neumann doubling rounds --------------- + ainv_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_lo + if cutlass.const_expr(accum_idx % 4 >= 2): + row_coord = row_hi + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + eye = cutlass.Float32(1.0) if row_coord == col_coord else cutlass.Float32(0.0) + ainv_acc[accum_idx] = eye - l_values[accum_idx] + + lpow_a0, lpow_a1, lpow_a2, lpow_a3 = l_a0, l_a1, l_a2, l_a3 + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(l_a0), movmatrix_16b(l_a1), movmatrix_16b(l_a2), movmatrix_16b(l_a3) + for _round in cutlass.range_constexpr(3): + # ---- Lpow = Lpow @ Lpow ------------------------------------------ + sq_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + sq_acc[accum_idx] = cutlass.Float32(0.0) + mma_step( + sq_acc, + (lpow_a0, lpow_a1, lpow_a2, lpow_a3), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + lpow_a0 = fp32_to_fp16(sq_acc[0], sq_acc[1], dtype=cfg.io_dtype) + lpow_a1 = fp32_to_fp16(sq_acc[2], sq_acc[3], dtype=cfg.io_dtype) + lpow_a2 = fp32_to_fp16(sq_acc[4], sq_acc[5], dtype=cfg.io_dtype) + lpow_a3 = fp32_to_fp16(sq_acc[6], sq_acc[7], dtype=cfg.io_dtype) + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(lpow_a0), movmatrix_16b(lpow_a1), movmatrix_16b(lpow_a2), movmatrix_16b(lpow_a3) + # ---- A_inv += A_inv @ Lpow --------------------------------------- + upd_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + upd_acc[accum_idx] = cutlass.Float32(0.0) + ainv_p0 = fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype) + ainv_p1 = fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype) + ainv_p2 = fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype) + ainv_p3 = fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype) + mma_step( + upd_acc, + (ainv_p0, ainv_p1, ainv_p2, ainv_p3), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + ainv_lo0, ainv_hi0 = f16x2_to_f32(ainv_p0, dtype=cfg.io_dtype) + ainv_lo1, ainv_hi1 = f16x2_to_f32(ainv_p1, dtype=cfg.io_dtype) + ainv_lo2, ainv_hi2 = f16x2_to_f32(ainv_p2, dtype=cfg.io_dtype) + ainv_lo3, ainv_hi3 = f16x2_to_f32(ainv_p3, dtype=cfg.io_dtype) + ainv_acc[0] = ainv_lo0 + upd_acc[0] + ainv_acc[1] = ainv_hi0 + upd_acc[1] + ainv_acc[2] = ainv_lo1 + upd_acc[2] + ainv_acc[3] = ainv_hi1 + upd_acc[3] + ainv_acc[4] = ainv_lo2 + upd_acc[4] + ainv_acc[5] = ainv_hi2 + upd_acc[5] + ainv_acc[6] = ainv_lo3 + upd_acc[6] + ainv_acc[7] = ainv_hi3 + upd_acc[7] + + nvvm.stmatrix( + sQk_ptr + stsm_idx, + [ + fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype), + ], + nvvm.MMALayout.ROW, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_a_ready[qk_stage].arrive() + bars.mb_decay_super_done[decay_stage].arrive() + global_chunk_base += num_chunks_tile + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def tcgen05_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + tmem_base_holder, + sQk, + sK_decay, + sK_restore, + sState_scale_diag, + bars, +) -> None: + """tcgen05-MMA warp role (warp 13): persistent scheduler loop issuing every + tcgen05 GEMM.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + tmem_base = tmem_base_holder.load() + state_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) + state_dsts = tuple(nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset + k * 16, cutlass.Float32) for k in range(cfg.d_k // 16)) + state_k_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_k_acc_offset, cutlass.Float32) + update_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_acc_offset, cutlass.Float32) + rhs_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_rhs_inp_offset, cutlass.Int8) + update_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_inp_offset, cutlass.Int8) + state_dst_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset, cutlass.Float32) + state_inp_index = PipelineState.start(phase=0) + state_read_index = PipelineState.start(phase=0) + rhs_index = PipelineState.start(phase=0) + update_index = PipelineState.start(phase=0) + final_state_index = PipelineState.start(phase=0) + qk_scale_index = PipelineState.start(phase=0) + + # ---- chunk-invariant GEMM descriptors ---------------------------------------- + bpe = cfg.io_dtype.width // 8 + idesc_acc = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + b_major=0, + ) + idesc_diag = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=16, + m_dim=cfg.d_v, + b_major=0, + ) + idesc_final_state = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.d_k, + m_dim=cfg.d_v, + b_major=1, + ) + bmm_state_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.d_k, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_acc, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bmm_diag_desc = MmaDesc( + M=cfg.d_v, + N=16, + K=16, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_diag, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bmm_qk_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_acc, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bmm_final_state_desc = MmaDesc( + M=cfg.d_v, + N=cfg.d_k, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + cta_group=1, + idesc=idesc_final_state, + kind=nvvm.Tcgen05MMAKind.F16, + ) + STATE_A_SEG = bmm_state_desc.sps_B * bmm_state_desc.tmem_advance_A + STATE_B_SEG = bmm_state_desc.smem_subtile_B >> 4 + global_chunk_base = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + num_chunks_tile = wend - cstart + for local_chunk in cutlass.range(num_chunks_tile, unroll=1): + global_chunk = global_chunk_base + local_chunk + decay_stage = global_chunk % cfg.smem_decay_stages + state_scale_diag_stage = qk_scale_index.idx + qk_stage = global_chunk % cfg.smem_qk_stages + sK_decay_stage = sK_decay[decay_stage] + sK_restore_stage = sK_restore[decay_stage] + sState_scale_diag_stage = sState_scale_diag[state_scale_diag_stage] + sQk_stage = sQk[qk_stage] + + # ---- state_k_acc = state(T) @ K_decay^T ------------------------------ + bars.mb_k_decay_inv_cg0_ready[decay_stage].wait((global_chunk // cfg.smem_decay_stages) % 2) + bars.mb_state_inp_ready.wait(state_inp_index.phase) + state_inp_index = advance(state_inp_index, 1) + desc_k_decay = sK_decay_stage.desc() + + for s in cutlass.range_constexpr(bmm_state_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_state_desc.sps_B): + mma_ts_step( + bmm_state_desc, + state_inp_ptr.subview(s * STATE_A_SEG), + desc_k_decay + s * STATE_B_SEG, + state_k_acc_ptr, + k, + cutlass.Boolean(s + k > 0), + ) + + if elect_one: + bars.mb_state_k_acc_ready.arrive(cta_group=1) + bars.mb_decay_done[decay_stage].arrive(cta_group=1) + + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_read_done.wait(state_read_index.phase) + state_read_index = advance(state_read_index, 1) + + # ---- state decay = state(T) @ diag(exp2(g_last)) (per-k-atom blocks) -- + bars.mb_qk_scale_ready[qk_scale_index.idx].wait(qk_scale_index.phase) + desc_diag = sState_scale_diag_stage.desc() + + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + mma_ts_step( + bmm_diag_desc, + state_inp_ptr.subview(k_block * bmm_diag_desc.tmem_advance_A), + desc_diag.advance_start_address(k_block * 256 * 2), + state_dsts[k_block], + 0, + cutlass.Boolean(False), + ) + + if elect_one: + bars.mb_state_scale_diag_done[state_scale_diag_stage].arrive(cta_group=1) + + # ---- update_acc = rhs(T) @ A_inv^T ----------------------------------- + bars.mb_a_ready[qk_stage].wait((global_chunk // cfg.smem_qk_stages) % 2) + bars.mb_rhs_ready.wait(rhs_index.phase) + rhs_index = advance(rhs_index, 1) + desc_qk = sQk_stage.desc() + mma_ts_step(bmm_qk_desc, rhs_inp_ptr, desc_qk, update_acc_ptr, 0, cutlass.Boolean(False)) + if elect_one: + bars.mb_update_acc_ready.arrive(cta_group=1) + bars.mb_a_done[qk_stage].arrive(cta_group=1) + + # ---- final_state += U(T) @ K_restore --------------------------------- + bars.mb_update_ready.wait(update_index.phase) + update_index = advance(update_index, 1) + desc_k_restore = sK_restore_stage.desc() + + mma_ts_step(bmm_final_state_desc, update_inp_ptr, desc_k_restore, state_dst_ptr, 0, cutlass.Boolean(True)) + if elect_one: + bars.mb_k_restore_done[decay_stage].arrive(cta_group=1) + + qk_scale_index = advance(qk_scale_index, cfg.smem_state_scale_diag_stages) + + bars.mb_final_state_stored.wait(final_state_index.phase) + final_state_index = advance(final_state_index, 1) + global_chunk_base += num_chunks_tile + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + nvvm.tcgen05_dealloc( + nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), + cutlass.Int32(512), + group=nvvm.CTAGroup.CTA_1, + ) + + +@cute.jit +def epilogue_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + sCheckpoint_raw, + desc_checkpoint_base, + checkpoint_every_n_tokens, + bars, +) -> None: + """Epilogue warp role (warp 15): the checkpoint TMA store drain.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + if cutlass.const_expr(cfg.enable_checkpoints): + sCheckpoint_tma = SmemTile( + base=sCheckpoint_raw, + elems_per_stage=(cfg.d_k * cfg.d_v), + stages=cfg.smem_checkpoint_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.d_k * 64, + ) + checkpoint_ready_index = PipelineState.start(phase=0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + head_o = head_idx + num_chunks_tile = wend - cstart + if cutlass.const_expr(cfg.enable_checkpoints): + checkpoint_slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_checkpoint_slot = (desc_checkpoint_base + checkpoint_slot).tospace(cutlass.AddressSpace.generic) + checkpoint_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) + checkpoint_quot = (cstart + cutlass.Int32(1)) // checkpoint_chunks + checkpoint_mod = (cstart + cutlass.Int32(1)) % checkpoint_chunks + if elect_one: + tma_tensormap_acquire(desc_checkpoint_slot) + for local_chunk in cutlass.range(num_chunks_tile, unroll=1): + chunk_idx = cstart + local_chunk + if local_chunk > 0: + # ---- checkpoint store ---------------------------------------- + do_checkpoint = checkpoint_mod == 0 + if cutlass.const_expr(cfg.split_k): + do_checkpoint = do_checkpoint and chunk_idx >= wstart + if do_checkpoint: + checkpoint_stage = checkpoint_ready_index.idx + bars.mb_checkpoint_tmastg_ready[checkpoint_stage].wait(checkpoint_ready_index.phase) + checkpoint_ready_index = advance(checkpoint_ready_index, cfg.smem_checkpoint_stages) + checkpoint_entry = checkpoint_quot - cutlass.Int32(1) + checkpoint_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), checkpoint_entry, head_o) + tma_store_tile(sCheckpoint_tma[checkpoint_stage], checkpoint_slice, acquire=False) + tma_store_commit() + tma_store_wait(0) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].arrive() + checkpoint_mod = checkpoint_mod + cutlass.Int32(1) + if checkpoint_mod == checkpoint_chunks: + checkpoint_mod = cutlass.Int32(0) + checkpoint_quot = checkpoint_quot + cutlass.Int32(1) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def gate_scale(cfg, raw_gate: cutlass.Float32) -> cutlass.Float32: + """Map raw gate to the log2-domain decay increment.""" + + if cutlass.const_expr(cfg.safe_gate): + half = cutlass.Float32(0.5) + sigmoid = cute.math.tanh(raw_gate * half, approx=True) * half + half + return cfg.gate_scale_log2 * sigmoid + # Default ABI: Gate arrives in natural-log space + return raw_gate * cutlass.Float32(LOG2_E) + + +@cute.jit +def compute0_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + warp_idx, + mA_log, + mDt_bias, + sK_inv_raw, + sBeta_raw, + sGate_raw, + sK_raw, + sV_raw, + sW_raw, + sK_decay_raw, + sK_restore_raw, + sState_scale_diag_raw, + bars, +) -> None: + """CG0 warp-group role (warps 0-7): persistent scheduler loop computing the + Gate prefix scan and the decay/restore operands.""" + nvvm.setmaxregister(cfg.num_regs_compute_group_0, nvvm.SetMaxRegisterAction.INCREASE) + cg0_warp = warp_idx - cfg.compute_group_0_warp_ids[0] + cg0_group_id = cg0_warp // cfg.cg0_warps_per_group + cg0_local_warp = cg0_warp % cfg.cg0_warps_per_group + prefix_dim = cg0_local_warp * cfg.threads_per_warp + lane + cg0_a_log_exp = cutlass.Float32(1.0) + cg0_dt_bias_value = cutlass.Float32(0.0) + global_chunk_base = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + opaque_one = opaque_f32_zero() + cutlass.Float32(1.0) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + head_o = head_idx + num_chunks_tile = wend - cstart + if cutlass.const_expr(cfg.safe_gate): + if num_chunks_tile > 0: + cg0_a_log_exp = cute.math.exp2(mA_log[head_o].to(cutlass.Float32) * LOG2_E, fastmath=True) + cg0_dt_bias_value = mDt_bias[head_o, prefix_dim].to(cutlass.Float32) + # non-pow2 ring cursors: one divmod per tile each, +group-stride with wrap per chunk + group_chunk_base = global_chunk_base + cutlass.Int32(cg0_group_id) + diag_stage_cursor = group_chunk_base % cutlass.Int32(cfg.smem_state_scale_diag_stages) + diag_phase = (group_chunk_base // cutlass.Int32(cfg.smem_state_scale_diag_stages)) % cutlass.Int32(2) + raw_stage_cursor = group_chunk_base % cutlass.Int32(cfg.smem_raw_stages) + raw_phase = (group_chunk_base // cutlass.Int32(cfg.smem_raw_stages)) % cutlass.Int32(2) + for local_chunk in cutlass.range(cg0_group_id, num_chunks_tile, cfg.cg0_group_count, unroll=1): + chunk_idx = cstart + local_chunk + global_chunk = global_chunk_base + local_chunk + chunk_start = chunk_idx * cfg.b_t + decay_stage = global_chunk % cfg.smem_decay_stages + raw_stage = raw_stage_cursor + state_scale_diag_stage = diag_stage_cursor + qk_scale_ready_stage = state_scale_diag_stage + sK_ptr = sK_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sV_ptr = sV_raw.data_ptr() + raw_stage * (cfg.d_v * cfg.b_t) + sGate_ptr = sGate_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sBeta_ptr = sBeta_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sW_ptr = sW_raw.data_ptr() + raw_stage * (cfg.d_v * cfg.b_t) + sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sK_decay_ptr = sK_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sK_restore_ptr = sK_restore_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sState_scale_diag_ptr = sState_scale_diag_raw.data_ptr() + state_scale_diag_stage * ((cfg.d_k // 16) * 256) + + bars.mb_gate_ready[raw_stage].wait(raw_phase) + bars.mb_k_ready[raw_stage].wait(raw_phase) + bars.mb_beta_ready[raw_stage].wait(raw_phase) + + row_group_start = cg0_local_warp * (cfg.b_t // cfg.cg0_warps_per_group) + lane_row_group = lane // 8 + lane_in_row_group = lane - lane_row_group * 8 + decay_row = row_group_start + lane_row_group + decay_key_mask = cutlass.Int32(8) + + prefix_dim = cg0_local_warp * cfg.threads_per_warp + lane + + # ---- Gate prefix scan ----------------------------------------------- + f32_segment = prefix_dim // 32 + prefix_seg_base = f32_segment * (cfg.b_t * 32) + prefix_col = prefix_dim - f32_segment * 32 + g_prefix_regs = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + if cutlass.const_expr(cfg.safe_gate): + valid_rows = seqlen_b - chunk_idx * cutlass.Int32(cfg.b_t) + valid_mask = cutlass.vector.create_mask([cfg.b_t], [valid_rows]) + for row_pair in cutlass.range_constexpr(cfg.b_t // 2): + row0 = row_pair * 2 + row1 = row0 + 1 + prefix_idx0 = prefix_seg_base + swizzle_xor_128b(row0, row0 * 32 + prefix_col, elem_bytes=4) + prefix_idx1 = prefix_seg_base + swizzle_xor_128b(row1, row1 * 32 + prefix_col, elem_bytes=4) + gate0 = (sGate_ptr + prefix_idx0).load() + gate1 = (sGate_ptr + prefix_idx1).load() + gate0 = cg0_a_log_exp * (gate0 + cg0_dt_bias_value) + gate1 = cg0_a_log_exp * (gate1 + cg0_dt_bias_value) + gate0 = gate_scale( + cfg, + gate0, + ) + gate1 = gate_scale( + cfg, + gate1, + ) + gate_pair = cutlass.Vector.from_elements((gate0, gate1), cutlass.Float32) + gate_pair = cutlass.vector.where(valid_mask[row0 : row1 + 1], gate_pair, 0.0) + g_prefix_regs[row0] = gate_pair[0] + g_prefix_regs[row1] = gate_pair[1] + else: + for row in cutlass.range_constexpr(cfg.b_t): + prefix_idx = prefix_seg_base + swizzle_xor_128b(row, row * 32 + prefix_col, elem_bytes=4) + gate = (sGate_ptr + prefix_idx).load() + token_idx = chunk_idx * cutlass.Int32(cfg.b_t) + cutlass.Int32(row) + if token_idx < seqlen_b: + gate = gate_scale( + cfg, + gate, + ) + else: + gate = cutlass.Float32(0.0) + g_prefix_regs[row] = gate + + prefix_acc = cutlass.Float32(0.0) + for row_pair in cutlass.range_constexpr(cfg.b_t // 2): + row0 = row_pair * 2 + row1 = row0 + 1 + gate0 = g_prefix_regs[row0] + gate1 = g_prefix_regs[row1] + prefix0, row_pair_sum = fadd2(prefix_acc, gate0, gate0, gate1) + prefix1 = prefix_acc + row_pair_sum + g_prefix_regs[row0] = prefix0 + g_prefix_regs[row1] = prefix1 + prefix_acc = prefix1 + + # ---- exp2(g): stage prefixes + final-token decay --------------------- + for row in cutlass.range_constexpr(cfg.b_t): + g_prefix_regs[row] = cute.math.exp2(g_prefix_regs[row], fastmath=True) + + exp_g_last = g_prefix_regs[cfg.b_t - 1] + for row in cutlass.range_constexpr(cfg.b_t): + prefix_idx = prefix_seg_base + swizzle_xor_128b(row, row * 32 + prefix_col, elem_bytes=4) + (sGate_ptr + prefix_idx).store(g_prefix_regs[row]) + + # ---- state-scale diag: stage exp2(g_last) decay blocks --------------- + bars.mb_state_scale_diag_done[state_scale_diag_stage].wait(diag_phase ^ cutlass.Int32(1)) + sState_scale_diag_ptr[diag_idx(cfg, prefix_dim)] = exp_g_last.to(cfg.io_dtype) + + nvvm.barrier_cta_sync(cfg.cg0_group_sync_barrier_base_id + cg0_group_id, thread_count=cfg.cg0_threads_per_group) + + k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) + raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + raw_beta_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + + # ---- optional K L2-norm + K_inv staging ------------------------------ + if cutlass.const_expr(cfg.l2norm): + kk0_lo = opaque_f32_zero() + kk0_hi = opaque_f32_zero() + kk1_lo = opaque_f32_zero() + kk1_hi = opaque_f32_zero() + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + raw_f16_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) + raw_k_vec = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_beta_vec = (sBeta_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_k_vec_f32 = raw_k_vec.to(cutlass.Float32) + raw_beta_vec_f32 = raw_beta_vec.to(cutlass.Float32) + for dim_offset in cutlass.range_constexpr(8): + k_val = raw_k_vec_f32[dim_offset] + raw_k_regs[reg_base + dim_offset] = k_val + beta_val = raw_beta_vec_f32[dim_offset] + raw_beta_regs[reg_base + dim_offset] = beta_val + if cutlass.const_expr(cfg.l2norm): + if cutlass.const_expr(dim_offset % 2 == 0): + kk0_lo, kk0_hi = ffma2(k_val, k_val, k_val, k_val, kk0_lo, kk0_hi) + else: + kk1_lo, kk1_hi = ffma2(k_val, k_val, k_val, k_val, kk1_lo, kk1_hi) + + k_inv_norm = opaque_one + if cutlass.const_expr(cfg.l2norm): + k_sum_sq = kk0_hi + kk1_hi + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 4, 31, kind=nvvm.Shfl.BFLY)) + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 2, 31, kind=nvvm.Shfl.BFLY)) + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 1, 31, kind=nvvm.Shfl.BFLY)) + norm_floor_sq = cutlass.Float32(L2_NORM_EPS * L2_NORM_EPS) + k_inv_norm = cute.math.rsqrt(cute.math.max(k_sum_sq, norm_floor_sq), fastmath=True) + + # ---- decay/restore operands: exp2(+-g) applied per key channel ------- + exp_g_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + exp_g_last_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + exp_neg_g_regs = cutlass.Array(cutlass.Float32, 8, alignment=16) + for f32_group in cutlass.range_constexpr(2): + f32_dim_base = dim_base + f32_group * 4 + f32_segment = f32_dim_base // 32 + f32_segment_dim = f32_dim_base - f32_segment * 32 + g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) + exp_g_vec = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) + f32_segment = f32_dim_base // 32 + f32_segment_dim = f32_dim_base - f32_segment * 32 + exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) + exp_g_last_vec = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) + half_reg_base = f32_group * 4 + f32_reg_base = reg_base + half_reg_base + exp_g_regs[f32_reg_base] = exp_g_vec[0] + exp_g_regs[f32_reg_base + 1] = exp_g_vec[1] + exp_g_regs[f32_reg_base + 2] = exp_g_vec[2] + exp_g_regs[f32_reg_base + 3] = exp_g_vec[3] + exp_neg_g_regs[half_reg_base] = cute.math.rcp(exp_g_vec[0], approx=True, ftz=True) + exp_neg_g_regs[half_reg_base + 1] = cute.math.rcp(exp_g_vec[1], approx=True, ftz=True) + exp_neg_g_regs[half_reg_base + 2] = cute.math.rcp(exp_g_vec[2], approx=True, ftz=True) + exp_neg_g_regs[half_reg_base + 3] = cute.math.rcp(exp_g_vec[3], approx=True, ftz=True) + exp_g_last_regs[f32_reg_base] = exp_g_last_vec[0] + exp_g_last_regs[f32_reg_base + 1] = exp_g_last_vec[1] + exp_g_last_regs[f32_reg_base + 2] = exp_g_last_vec[2] + exp_g_last_regs[f32_reg_base + 3] = exp_g_last_vec[3] + + # ---- K_decay + K_inv operands: K * exp2(+g) and K * exp2(-g) ----- + k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + for pair_idx in cutlass.range_constexpr(4): + dim0 = pair_idx * 2 + dim1 = dim0 + 1 + raw_reg_idx0 = reg_base + dim0 + raw_reg_idx1 = reg_base + dim1 + k_value0, k_value1 = fmul2(raw_k_regs[raw_reg_idx0], raw_k_regs[raw_reg_idx1], k_inv_norm, k_inv_norm) + k_beta0, k_beta1 = fmul2(k_value0, k_value1, raw_beta_regs[raw_reg_idx0], raw_beta_regs[raw_reg_idx1]) + k_pair = fp32_to_fp16(k_beta0, k_beta1, dtype=cfg.io_dtype) + exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) + k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) + exp_neg_pair = fp32_to_fp16(exp_neg_g_regs[dim0], exp_neg_g_regs[dim1], dtype=cfg.io_dtype) + k_norm_pair = fp32_to_fp16(k_value0, k_value1, dtype=cfg.io_dtype) + k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_norm_pair, exp_neg_pair, cfg.io_dtype) + + k_inv_vec = cutlass.Vector.from_elements( + ( + k_inv_words[dim_half * 4], + k_inv_words[dim_half * 4 + 1], + k_inv_words[dim_half * 4 + 2], + k_inv_words[dim_half * 4 + 3], + ), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + k_decay_vec = cutlass.Vector.from_elements( + ( + k_decay_words[0], + k_decay_words[1], + k_decay_words[2], + k_decay_words[3], + ), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + if cutlass.const_expr(dim_half == 0): + operand_done_phase = ((global_chunk // cfg.smem_decay_stages) + 1) % 2 + bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + k_inv_swizzled_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) + (sK_inv_ptr + k_inv_swizzled_idx).store(k_inv_vec, alignment=16) + storage_key = dim_base ^ decay_key_mask + storage_slice = storage_key // 64 + decay_swizzled_idx = storage_slice * (cfg.b_t * 64) + swizzle_xor_128b( + decay_row, decay_row * 64 + storage_key - storage_slice * 64, elem_bytes=2 + ) + (sK_decay_ptr + decay_swizzled_idx).store(k_decay_vec, alignment=16) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_k_decay_inv_cg0_ready[decay_stage].arrive() + bars.mb_k_done[raw_stage].arrive() + bars.mb_gate_done[raw_stage].arrive() + bars.mb_beta_done[raw_stage].arrive() + + # ---- K_restore operand: K_inv * exp_g_last -------------------------- + bars.mb_k_restore_done[decay_stage].wait(((global_chunk // cfg.smem_decay_stages + 1) % 2)) + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + k_restore_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + for pair_idx in cutlass.range_constexpr(4): + dim0 = pair_idx * 2 + dim1 = dim0 + 1 + exp_g_last_pair = fp32_to_fp16(exp_g_last_regs[reg_base + dim0], exp_g_last_regs[reg_base + dim1], dtype=cfg.io_dtype) + k_restore_words[pair_idx] = mul_f16x2(k_inv_words[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) + storage_row = decay_row ^ (cfg.b_t // 2) + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + k_restore_idx = f16_segment * (cfg.b_t * 64) + storage_row * 64 + swizzle_xor_128b(storage_row, f16_segment_dim, elem_bytes=2) + k_restore_vec = cutlass.Vector.from_elements( + ( + k_restore_words[0], + k_restore_words[1], + k_restore_words[2], + k_restore_words[3], + ), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + (sK_restore_ptr + k_restore_idx).store(k_restore_vec, alignment=16) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_qk_scale_ready[qk_scale_ready_stage].arrive() + diag_stage_cursor = diag_stage_cursor + cutlass.Int32(cfg.cg0_group_count) + diag_wrapped = diag_stage_cursor >= cutlass.Int32(cfg.smem_state_scale_diag_stages) + diag_stage_cursor = diag_stage_cursor - cutlass.Int32(cfg.smem_state_scale_diag_stages) if diag_wrapped else diag_stage_cursor + diag_phase = diag_phase ^ (cutlass.Int32(1) if diag_wrapped else cutlass.Int32(0)) + raw_stage_cursor = raw_stage_cursor + cutlass.Int32(cfg.cg0_group_count) + raw_wrapped = raw_stage_cursor >= cutlass.Int32(cfg.smem_raw_stages) + raw_stage_cursor = raw_stage_cursor - cutlass.Int32(cfg.smem_raw_stages) if raw_wrapped else raw_stage_cursor + raw_phase = raw_phase ^ (cutlass.Int32(1) if raw_wrapped else cutlass.Int32(0)) + global_chunk_base += num_chunks_tile + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def compute1_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_base_holder, + warp_idx, + mState_out, + mState_init, + sV_raw, + sW_raw, + sCheckpoint_raw, + checkpoint_every_n_tokens, + bars, +) -> None: + """CG1 warp-group role (warps 8-11): persistent scheduler loop staging the + value-side TMEM operands and storing checkpoints/final state.""" + nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) + sCheckpoint_ptr = sCheckpoint_raw.data_ptr() + checkpoint_done_index = PipelineState.start(phase=1) + tmem_base = tmem_base_holder.load() + tmem_col = tmem_base & 0xFFFF + tmem_row = tmem_base >> 16 + tmem_subpartition = warp_idx % (cfg.d_v // cfg.threads_per_warp) + # ldmatrix.x4/stmatrix.x4 COL lane decode for the V/W loads + ldsm_row_coord = (lane // 16) * 8 + (lane & 7) + ldsm_col_offset = ((lane // 8) & 1) * 8 + row_id = tmem_row + tmem_subpartition * cfg.threads_per_warp + value_dim = tmem_subpartition * cfg.threads_per_warp + lane + state_k_acc_index = PipelineState.start(phase=0) + update_acc_index = PipelineState.start(phase=0) + k_restore_index = PipelineState.start(phase=0) # CG1's per-chunk mb_k_restore_done wait slot + raw_index = PipelineState.start(phase=0) # raw-ring slot for the sV/sW reads + inputs_done arrives + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + head_o = head_idx + num_chunks_tile = wend - cstart + + if num_chunks_tile > 0: + # ---- first chunk: seed state TMEM from mState_init (else zeros) ---------- + seed_from_initial_state = cstart == 0 + if cutlass.const_expr(mState_init is not None and cfg.split_k): + if seed_from_initial_state: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + state_block[col] = mState_init[batch_idx, head_o, key_dim, value_dim].to(cutlass.Float32) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + else: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + state_block[col] = cutlass.Float32(0.0) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + elif cutlass.const_expr(mState_init is not None): + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + state_block[col] = mState_init[batch_idx, head_o, key_dim, value_dim].to(cutlass.Float32) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + else: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + state_block[col] = cutlass.Float32(0.0) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + + nvvm.tcgen05_wait("store") + sV_ptr = sV_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) + sW_ptr = sW_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) + + row_addr = (tmem_row + tmem_subpartition * cfg.threads_per_warp) << 16 + state_col_id = tmem_col + cfg.tmem_state_acc_offset + + # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- + state_blocks = [] + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) + + packed_col_id = tmem_col + cfg.tmem_state_inp_offset + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) + for packed_col in cutlass.range_constexpr(8): + source_pair = packed_col ^ 4 + packed_state[packed_col] = fp32_to_fp16( + state_blocks[k_block][2 * source_pair], state_blocks[k_block][2 * source_pair + 1], dtype=cfg.io_dtype + ) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + k_block * 8, cutlass.Int8), + packed_state[0:8], + ) + + nvvm.tcgen05_wait("store") + bars.mb_state_inp_ready.arrive() + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_read_done.arrive() + + # ---- rhs staging: rhs input = W*V - state*(Beta*K) ------------------- + bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) + bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) + bars.mb_w_ready[raw_index.idx].wait(raw_index.phase) + projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset + input_col_id = tmem_col + cfg.tmem_rhs_inp_offset + value_dim_base = tmem_subpartition * cfg.threads_per_warp + + # ---- read back state*K acc + raw V fragments ------------------------- + row_id0 = tmem_row + value_dim_base + state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + + row_id1 = row_id0 + 16 + state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + + raw_v_regs0 = nvvm.ldmatrix( + sV_ptr + + (value_dim_base + ldsm_col_offset) // 64 * (cfg.b_t * 64) + + ldsm_row_coord * 64 + + swizzle_xor_128b(ldsm_row_coord, (value_dim_base + ldsm_col_offset) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + raw_v_regs1 = nvvm.ldmatrix( + sV_ptr + + (value_dim_base + 16 + ldsm_col_offset) // 64 * (cfg.b_t * 64) + + ldsm_row_coord * 64 + + swizzle_xor_128b(ldsm_row_coord, (value_dim_base + 16 + ldsm_col_offset) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + raw_w_regs0 = nvvm.ldmatrix( + sW_ptr + + (value_dim_base + ldsm_col_offset) // 64 * (cfg.b_t * 64) + + ldsm_row_coord * 64 + + swizzle_xor_128b(ldsm_row_coord, (value_dim_base + ldsm_col_offset) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + raw_w_regs1 = nvvm.ldmatrix( + sW_ptr + + (value_dim_base + 16 + ldsm_col_offset) // 64 * (cfg.b_t * 64) + + ldsm_row_coord * 64 + + swizzle_xor_128b(ldsm_row_coord, (value_dim_base + 16 + ldsm_col_offset) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + + packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] + state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) + w_pair = raw_w_regs0[raw_matrix] + wv_pair = mul_f16x2( + w_pair, + raw_v_regs0[raw_matrix], + cfg.io_dtype, + ) + packed_rhs0[reg_idx] = sub_f16x2( + wv_pair, + state_k_pair, + cfg.io_dtype, + ) + + packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] + state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) + w_pair = raw_w_regs1[raw_matrix] + wv_pair = mul_f16x2( + w_pair, + raw_v_regs1[raw_matrix], + cfg.io_dtype, + ) + packed_rhs1[reg_idx] = sub_f16x2( + wv_pair, + state_k_pair, + cfg.io_dtype, + ) + + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), packed_rhs0[0:4]) + + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), packed_rhs1[0:4]) + + nvvm.tcgen05_wait("store") + state_k_acc_index = advance(state_k_acc_index, 1) + bars.mb_v_done[raw_index.idx].arrive() + bars.mb_w_done[raw_index.idx].arrive() + bars.mb_rhs_ready.arrive() + + # ---- update repack: acc TMEM -> packed b16 TMEM --------------------- + bars.mb_update_acc_ready.wait(update_acc_index.phase) + update = nvvm.tcgen05_ld( + "32x32b", + nvvm.make_tmem_ptr((tmem_row + tmem_subpartition * cfg.threads_per_warp << 16) + (tmem_col + cfg.tmem_update_acc_offset), cutlass.Float32), + num=cfg.b_t, + ) + + packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): + source_pair = packed_col ^ 4 + token0 = source_pair * 2 + token1 = token0 + 1 + packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_row << 16) + (tmem_col + cfg.tmem_update_inp_offset), cutlass.Int8), + packed_update[0 : (cfg.b_t // 2)], + ) + nvvm.tcgen05_wait("store") + update_acc_index = advance(update_acc_index, 1) + bars.mb_update_ready.arrive() + + bars.mb_k_restore_done[k_restore_index.idx].wait(k_restore_index.phase) + k_restore_index = advance(k_restore_index, cfg.smem_decay_stages) + raw_index = advance(raw_index, cfg.smem_raw_stages) + + if cutlass.const_expr(cfg.enable_checkpoints): + cg1_checkpoint_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) + cg1_checkpoint_mod = (cstart + cutlass.Int32(1)) % cg1_checkpoint_chunks + for local_chunk in cutlass.range(1, num_chunks_tile, 1, unroll=1): + chunk_idx = cstart + local_chunk + sV_ptr = sV_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) + sW_ptr = sW_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) + + do_checkpoint = False + if cutlass.const_expr(cfg.enable_checkpoints): + do_checkpoint = cg1_checkpoint_mod == 0 + cg1_checkpoint_mod = cg1_checkpoint_mod + cutlass.Int32(1) + cg1_checkpoint_mod = cutlass.Int32(0) if cg1_checkpoint_mod == cg1_checkpoint_chunks else cg1_checkpoint_mod + if cutlass.const_expr(cfg.split_k): + do_checkpoint = do_checkpoint and chunk_idx >= wstart + row_addr = (tmem_row + tmem_subpartition * cfg.threads_per_warp) << 16 + state_col_id = tmem_col + cfg.tmem_state_acc_offset + + # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- + state_blocks = [] + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) + + packed_col_id = tmem_col + cfg.tmem_state_inp_offset + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) + for packed_col in cutlass.range_constexpr(8): + source_pair = packed_col ^ 4 + packed_state[packed_col] = fp32_to_fp16( + state_blocks[k_block][2 * source_pair], state_blocks[k_block][2 * source_pair + 1], dtype=cfg.io_dtype + ) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + k_block * 8, cutlass.Int8), + packed_state[0:8], + ) + nvvm.tcgen05_wait("store") + bars.mb_state_inp_ready.arrive() + + # ---- checkpoint: post-publish f32 fragment read; the decay GEMM's ------ + if cutlass.const_expr(cfg.enable_checkpoints): + if do_checkpoint: + checkpoint_stage = checkpoint_done_index.idx + bars.mb_checkpoint_tmastg_done[checkpoint_stage].wait(checkpoint_done_index.phase) + checkpoint_done_index = advance(checkpoint_done_index, cfg.smem_checkpoint_stages) + checkpoint_stage_base = checkpoint_stage * (cfg.d_k * cfg.d_v) + row16_addr = ((tmem_row + tmem_subpartition * cfg.threads_per_warp) + 16) << 16 + checkpoint_vbase = tmem_subpartition * cfg.threads_per_warp + checkpoint_swz_off0 = (checkpoint_vbase + ldsm_col_offset) // 64 * (cfg.d_k * 64) + checkpoint_swz_col0 = (checkpoint_vbase + ldsm_col_offset) % 64 + checkpoint_swz_off = (checkpoint_vbase + 16 + ldsm_col_offset) // 64 * (cfg.d_k * 64) + checkpoint_swz_col = (checkpoint_vbase + 16 + ldsm_col_offset) % 64 + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + checkpoint_ld0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) + checkpoint_ld1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) + checkpoint_st0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_st1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + checkpoint_st0[reg_idx] = fp32_to_fp16(checkpoint_ld0[2 * reg_idx], checkpoint_ld0[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_st1[reg_idx] = fp32_to_fp16(checkpoint_ld1[2 * reg_idx], checkpoint_ld1[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_row = k_block * 16 + ldsm_row_coord + nvvm.stmatrix( + sCheckpoint_ptr + + checkpoint_stage_base + + checkpoint_swz_off0 + + checkpoint_row * 64 + + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col0, elem_bytes=2), + checkpoint_st0.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.stmatrix( + sCheckpoint_ptr + + checkpoint_stage_base + + checkpoint_swz_off + + checkpoint_row * 64 + + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col, elem_bytes=2), + checkpoint_st1.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.tcgen05_wait("load") + bars.mb_state_read_done.arrive() + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_checkpoint_tmastg_ready[checkpoint_stage].arrive() + else: + bars.mb_state_read_done.arrive() + bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) + + # ---- rhs staging: rhs input = W*V - state*(Beta*K) ------------------- + bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) + bars.mb_w_ready[raw_index.idx].wait(raw_index.phase) + projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset + input_col_id = tmem_col + cfg.tmem_rhs_inp_offset + value_dim_base = tmem_subpartition * cfg.threads_per_warp + + # ---- read back state*K acc + raw V fragments ------------------------- + row_id0 = tmem_row + value_dim_base + state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + + row_id1 = row_id0 + 16 + state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + + raw_v_regs0 = nvvm.ldmatrix( + sV_ptr + + (value_dim_base + ldsm_col_offset) // 64 * (cfg.b_t * 64) + + ldsm_row_coord * 64 + + swizzle_xor_128b(ldsm_row_coord, (value_dim_base + ldsm_col_offset) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + raw_v_regs1 = nvvm.ldmatrix( + sV_ptr + + (value_dim_base + 16 + ldsm_col_offset) // 64 * (cfg.b_t * 64) + + ldsm_row_coord * 64 + + swizzle_xor_128b(ldsm_row_coord, (value_dim_base + 16 + ldsm_col_offset) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + raw_w_regs0 = nvvm.ldmatrix( + sW_ptr + + (value_dim_base + ldsm_col_offset) // 64 * (cfg.b_t * 64) + + ldsm_row_coord * 64 + + swizzle_xor_128b(ldsm_row_coord, (value_dim_base + ldsm_col_offset) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + raw_w_regs1 = nvvm.ldmatrix( + sW_ptr + + (value_dim_base + 16 + ldsm_col_offset) // 64 * (cfg.b_t * 64) + + ldsm_row_coord * 64 + + swizzle_xor_128b(ldsm_row_coord, (value_dim_base + 16 + ldsm_col_offset) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + + packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] + state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) + w_pair = raw_w_regs0[raw_matrix] + wv_pair = mul_f16x2( + w_pair, + raw_v_regs0[raw_matrix], + cfg.io_dtype, + ) + packed_rhs0[reg_idx] = sub_f16x2( + wv_pair, + state_k_pair, + cfg.io_dtype, + ) + + packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] + state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) + w_pair = raw_w_regs1[raw_matrix] + wv_pair = mul_f16x2( + w_pair, + raw_v_regs1[raw_matrix], + cfg.io_dtype, + ) + packed_rhs1[reg_idx] = sub_f16x2( + wv_pair, + state_k_pair, + cfg.io_dtype, + ) + + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), packed_rhs0[0:4]) + + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), packed_rhs1[0:4]) + + nvvm.tcgen05_wait("store") + state_k_acc_index = advance(state_k_acc_index, 1) + bars.mb_v_done[raw_index.idx].arrive() + bars.mb_w_done[raw_index.idx].arrive() + bars.mb_rhs_ready.arrive() + + # ---- update repack: acc TMEM -> packed b16 TMEM --------------------- + bars.mb_update_acc_ready.wait(update_acc_index.phase) + update = nvvm.tcgen05_ld( + "32x32b", + nvvm.make_tmem_ptr((tmem_row + tmem_subpartition * cfg.threads_per_warp << 16) + (tmem_col + cfg.tmem_update_acc_offset), cutlass.Float32), + num=cfg.b_t, + ) + + packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): + source_pair = packed_col ^ 4 + token0 = source_pair * 2 + token1 = token0 + 1 + packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_row << 16) + (tmem_col + cfg.tmem_update_inp_offset), cutlass.Int8), + packed_update[0 : (cfg.b_t // 2)], + ) + nvvm.tcgen05_wait("store") + update_acc_index = advance(update_acc_index, 1) + bars.mb_update_ready.arrive() + + bars.mb_k_restore_done[k_restore_index.idx].wait(k_restore_index.phase) + k_restore_index = advance(k_restore_index, cfg.smem_decay_stages) + raw_index = advance(raw_index, cfg.smem_raw_stages) + + owns_final = wend == num_chunks_b + + # ---- final-state drain: state acc TMEM -> GMEM --------------------------- + if cutlass.const_expr(mState_out is not None): + if seqlen_b > 0: + if owns_final: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + loaded = nvvm.tcgen05_ld( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + num=32, + ) + + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + mState_out[batch_idx, head_o, key_dim, value_dim] = loaded[col].to(mState_out.element_type) + else: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + if cutlass.const_expr(mState_init is not None): + mState_out[batch_idx, head_o, key_dim, value_dim] = mState_init[batch_idx, head_o, key_dim, value_dim] + else: + mState_out[batch_idx, head_o, key_dim, value_dim] = cutlass.Float32(0.0).to(mState_out.element_type) + bars.mb_final_state_stored.arrive() + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def host( + cfg: cutlass.Constexpr, + k: cute.Tensor, + v: cute.Tensor, + raw_gate: cute.Tensor, + a_log: cute.Tensor | None, + dt_bias: cute.Tensor | None, + beta: cute.Tensor, + w: cute.Tensor, + cu_seqlens: cute.Tensor, + initial_state: cute.Tensor | None, + final_state: cute.Tensor | None, + work_items: cute.Tensor | None, + work_count: cute.Tensor | None, + sched_ctr: cute.Tensor | None, + tensormap_workspace: cute.Tensor, + checkpoint_every_n_tokens: cutlass.Int32, + stream, +) -> None: + num_sequences = cu_seqlens.shape[0] - 1 + ho = raw_gate.shape[1] + total_tiles = num_sequences * ho + grid_shape = (cfg.max_active_clusters, 1, 1) + kernel( + cfg, + tensormap_workspace, + cutlass.Int32(num_sequences), + k, + v, + raw_gate, + a_log, + dt_bias, + beta, + w, + cu_seqlens, + initial_state, + final_state, + work_items, + work_count, + sched_ctr, + total_tiles, + checkpoint_every_n_tokens, + ).launch( + grid=grid_shape, + block=(cfg.threads_per_cta, 1, 1), + stream=stream, + min_blocks_per_mp=1, + ) + + +@cute.kernel +def kernel( + cfg: cutlass.Constexpr, + tensormap_workspace: cute.Tensor, + n_desc: cutlass.Int32, + mK: cute.Tensor, + mV: cute.Tensor, + mGate: cute.Tensor, + mA_log: cute.Tensor | None, + mDt_bias: cute.Tensor | None, + mBeta: cute.Tensor, + mW: cute.Tensor, + cu_seqlens: cute.Tensor, + mState_init: cute.Tensor | None, + mState_out: cute.Tensor | None, + mWorkItems: cute.Tensor | None, + mCount: cute.Tensor | None, + mSched: cute.Tensor | None, + total_tiles: cutlass.Int32, + checkpoint_every_n_tokens: cutlass.Int32, +) -> None: + """BT=16 GDN-2 recompute device kernel (persistent); grid + `(min(tiles, SM count), 1, 1)`.""" + + tidx, _, _ = cute.arch.thread_idx() + bidx = cute.arch.block_idx()[0] + num_ctas = cute.arch.grid_dim()[0] + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + lane = tidx % cfg.threads_per_warp + + if cutlass.const_expr(cfg.split_k): + total_tiles = mCount[0] + if cutlass.const_expr(cfg.dyn_sched): + assert mSched is not None and mSched.element_type == cutlass.Int32 + assert mK.element_type == cfg.io_dtype and mV.element_type == cfg.io_dtype + assert mGate.element_type == cutlass.Float32 + assert mBeta.element_type == cfg.io_dtype and mW.element_type == cfg.io_dtype, "channel-wise beta/w must match the io dtype" + assert cu_seqlens.element_type in (cutlass.Int32, cutlass.Int64) + if cutlass.const_expr(cfg.use_initial_state): + assert mState_init is not None and mState_init.element_type in (cutlass.BFloat16, cutlass.Float32) + else: + assert mState_init is None, "mState_init must be None if use_initial_state is False" + if cutlass.const_expr(cfg.store_final_state): + assert mState_out is not None and mState_out.element_type in (cutlass.BFloat16, cutlass.Float32) + else: + assert mState_out is None, "mState_out must be None if store_final_state is False" + if cutlass.const_expr(mState_init is not None and mState_out is not None): + assert mState_init.element_type == mState_out.element_type + # per-BATCH TMA-descriptor arrays (heads are load coordinates): [K, V, Gate, Beta, W, Checkpoint] + desc_base_words = tensormap_workspace.iterator.raw_ptr() + arr_words = n_desc * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_k_base = desc_base_words + desc_v_base = desc_base_words + arr_words + desc_gate_base = desc_base_words + cutlass.Int32(2) * arr_words + desc_beta_base = desc_base_words + cutlass.Int32(3) * arr_words + desc_w_base = desc_base_words + cutlass.Int32(4) * arr_words + desc_checkpoint_base = desc_base_words + cutlass.Int32(5) * arr_words + + # Buffers are declaration-ordered and intentionally non-aliased. + SMEM = cutlass.AddressSpace.smem + bars = make_gdn2_bars(cfg) + tmem_base_holder = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=4) + sSched = cutlass.Array(cutlass.Int32, cfg.sched_stages, space=SMEM, alignment=16) + sK_decay_raw = cutlass.Array(cfg.io_dtype, cfg.k_decay_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_restore_raw = cutlass.Array(cfg.io_dtype, cfg.k_restore_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sQk_raw = cutlass.Array(cfg.io_dtype, cfg.qk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_raw = cutlass.Array(mK.element_type, cfg.k_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sV_raw = cutlass.Array(mV.element_type, cfg.v_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sGate_raw = cutlass.Array(cutlass.Float32, cfg.gate_cosize, space=SMEM, alignment=1024) + sState_scale_diag_raw = cutlass.Array(cfg.io_dtype, cfg.state_scale_diag_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_inv_raw = cutlass.Array(cfg.io_dtype, cfg.k_inv_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sBeta_raw = cutlass.Array(mBeta.element_type, cfg.beta_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sW_raw = cutlass.Array(mW.element_type, cfg.w_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sCheckpoint_raw = ( + cutlass.Array(cfg.io_dtype, cfg.smem_checkpoint_stages * cfg.d_k * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) + if cutlass.const_expr(cfg.enable_checkpoints) + else sV_raw + ) + sK_decay = SmemTile( + base=sK_decay_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_decay_stages, + leading_byte_offset=16, + stride_byte_offset=1024, + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_128B, + ) + sK_restore = SmemTile( + base=sK_restore_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_decay_stages, + leading_byte_offset=(cfg.b_t * (cfg.d_v // 2) * 2), + stride_byte_offset=(8 * (cfg.d_v // 2) * 2), + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_128B, + ) + sState_scale_diag = SmemTile( + base=sState_scale_diag_raw, + elems_per_stage=((cfg.d_k // 16) * 256), + stages=cfg.smem_state_scale_diag_stages, + leading_byte_offset=16, + stride_byte_offset=(8 * 16 * 2), + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, + ) + sQk = SmemTile( + base=sQk_raw, + elems_per_stage=(cfg.b_t * cfg.b_t), + stages=cfg.smem_qk_stages, + leading_byte_offset=16, + stride_byte_offset=(8 * cfg.b_t * 2), + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, + ) + + if warp_idx == cfg.tma_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.smem_raw_stages): + bars.mb_k_ready[stage].init() + bars.mb_k_done[stage].init() + bars.mb_gate_ready[stage].init() + bars.mb_gate_done[stage].init() + bars.mb_beta_ready[stage].init() + bars.mb_beta_done[stage].init() + bars.mb_v_ready[stage].init() + bars.mb_v_done[stage].init() + bars.mb_w_ready[stage].init() + bars.mb_w_done[stage].init() + elif warp_idx == cfg.tcgen05_mma_warp_id: + if nvvm.elect_sync(): + bars.mb_state_k_acc_ready.init() + bars.mb_update_acc_ready.init() + bars.mb_state_inp_ready.init() + for stage in cutlass.range_constexpr(cfg.smem_state_scale_diag_stages): + bars.mb_state_scale_diag_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_decay_stages): + bars.mb_decay_done[stage].init() + bars.mb_decay_super_done[stage].init() + bars.mb_k_restore_done[stage].init() + bars.mb_rhs_ready.init() + bars.mb_update_ready.init() + bars.mb_final_state_stored.init() + elif warp_idx == cfg.super_mma_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.smem_qk_stages): + bars.mb_a_ready[stage].init() + bars.mb_a_done[stage].init() + for stage in cutlass.range_constexpr(cfg.qk_scale_ready_stages): + bars.mb_qk_scale_ready[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_decay_stages): + bars.mb_k_decay_inv_cg0_ready[stage].init() + elif warp_idx == cfg.epilogue_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.sched_stages): + bars.mb_sched_ready[stage].init() + bars.mb_sched_done[stage].init() + if cutlass.const_expr(cfg.enable_checkpoints): + for stage in cutlass.range_constexpr(cfg.smem_checkpoint_stages): + bars.mb_checkpoint_tmastg_ready[stage].init() + bars.mb_checkpoint_tmastg_done[stage].init() + bars.mb_state_read_done.init() + diag_zero = cfg.io_dtype(0.0) + for diag_idx in cutlass.range(tidx, cfg.state_scale_diag_cosize, cfg.threads_per_cta, unroll=1): + sState_scale_diag_raw[diag_idx] = diag_zero + nvvm.fence_mbarrier_init() + nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) + if (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) or warp_idx == cfg.tcgen05_mma_warp_id: + if warp_idx == cfg.tcgen05_mma_warp_id: + nvvm.tcgen05_alloc(tmem_base_holder, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) + if warp_idx == cfg.tcgen05_mma_warp_id: + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) + + # Actual SMEM/TMEM buffers for the BT=16 schedule: + # K/V : 16 x 128 each + # gate_log2 : 16 x 128 + # Beta : 16 + # K inverse norm : 16, staged once per decay stage + # exp_g_last : 128, CG0-local and staged once per decay stage + # state-scale diag : 8 x 16 x 16 input dtype, zeroed once in the prologue + # K_decay : tcgen05 SW128 operands shared with super-MMA + # K_restore : tcgen05 SW128 N-major final-state operand + # K_inv : 16 x 128 token-major for super-MMA RHS + # A inverse : 16 x 16, plus transposed tcgen05 operands + # state : external/kernel ABI is VK `[DV, DK]`; reference + # math can view it as KV `[DK, DV]` by transposing. + # The TS A-staging path keeps VK in TMEM so state*K + # is `[DV, DK] @ [DK, BT] -> [DV, BT]` with M=128. + + if warp_idx == cfg.tma_warp_id: + tmaldg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + mSched, + sSched, + lane, + sBeta_raw, + sGate_raw, + sK_raw, + sV_raw, + sW_raw, + desc_k_base, + desc_v_base, + desc_gate_base, + desc_beta_base, + desc_w_base, + bars, + ) + elif warp_idx == cfg.super_mma_warp_id: + super_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_inv_raw, + sQk_raw, + sK_decay_raw, + bars, + ) + elif warp_idx == cfg.tcgen05_mma_warp_id: + tcgen05_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + tmem_base_holder, + sQk, + sK_decay, + sK_restore, + sState_scale_diag, + bars, + ) + elif warp_idx == cfg.epilogue_warp_id: + epilogue_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + sCheckpoint_raw, + desc_checkpoint_base, + checkpoint_every_n_tokens, + bars, + ) + elif warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]: + compute0_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + warp_idx, + mA_log, + mDt_bias, + sK_inv_raw, + sBeta_raw, + sGate_raw, + sK_raw, + sV_raw, + sW_raw, + sK_decay_raw, + sK_restore_raw, + sState_scale_diag_raw, + bars, + ) + elif warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]: + compute1_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_base_holder, + warp_idx, + mState_out, + mState_init, + sV_raw, + sW_raw, + sCheckpoint_raw, + checkpoint_every_n_tokens, + bars, + ) + + +@dataclass +class Gdn2RecomputeCfg: + """Kernel cfg (fixed BT=16 schedule constants; derived TMEM column offsets + and SMEM buffer cosizes are stamped by ``build_cfg``; per-stage sizes are + inlined at the use sites). Passed ``cfg``-first (a ``cutlass.Constexpr``) + into ``host`` / ``kernel`` and every warp body.""" + + io_dtype: Type[cutlass.Numeric] + state_dtype: Type[cutlass.Numeric] + use_initial_state: bool + store_final_state: bool + enable_checkpoints: bool + l2norm: bool + safe_gate: bool + gate_scale_log2: float + k_ratio: int + v_ratio: int + n_heads_out: int + max_active_clusters: int + split_k: bool = False + dyn_sched: bool = False + sched_stages: int = CFG.SMEM_SCHED_STAGES + + compute_group_0_warp_ids: tuple[int, ...] = CFG.COMPUTE_GROUP_0_WARP_IDS + compute_group_1_warp_ids: tuple[int, ...] = CFG.COMPUTE_GROUP_1_WARP_IDS + super_mma_warp_id: int = CFG.SUPER_MMA_WARP_ID + tcgen05_mma_warp_id: int = CFG.TCGEN05_MMA_WARP_ID + tma_warp_id: int = CFG.TMA_WARP_ID + epilogue_warp_id: int = CFG.EPILOGUE_WARP_ID + b_t: int = CFG.B_T + d_k: int = CFG.D_K + d_v: int = CFG.D_V + threads_per_warp: int = CFG.THREADS_PER_WARP + buffer_align_bytes: int = CFG.BUFFER_ALIGN_BYTES + threads_per_cta: int = 0 + cg0_group_count: int = 2 + cg0_warps_per_group: int = 4 + cg0_threads_per_group: int = 0 + cg0_group_sync_barrier_base_id: int = 1 # CG0 group g syncs on nbar id 1 + g + tmem_user_threads: int = 0 + tmem_lifecycle_barrier_id: int = 3 + num_regs_compute_group_0: int = CFG.NUM_REGS_COMPUTE_GROUP_0 + num_regs_compute_group_1: int = CFG.NUM_REGS_COMPUTE_GROUP_1 + num_regs_other: int = CFG.NUM_REGS_OTHER + + # ---- SMEM / TMEM ring stage counts ------------------------------------------- + smem_raw_stages: int = CFG.SMEM_RAW_STAGES + smem_checkpoint_stages: int = 1 + smem_decay_stages: int = CFG.SMEM_DECAY_STAGES + smem_qk_stages: int = CFG.SMEM_QK_STAGES + smem_state_scale_diag_stages: int = CFG.SMEM_STATE_SCALE_DIAG_STAGES + qk_scale_ready_stages: int = CFG.QK_SCALE_READY_STAGES + + # ---- TMEM column offsets (state doubles as the final_state acc) -------------- + tmem_state_acc_offset: int = 0 + tmem_state_inp_offset: int = 0 + tmem_state_k_acc_offset: int = 0 + tmem_update_acc_offset: int = 0 + tmem_rhs_inp_offset: int = 0 + tmem_update_inp_offset: int = 0 + + # ---- SMEM buffer cosizes ----------------------------------------------------- + k_cosize: int = 0 + v_cosize: int = 0 + gate_cosize: int = 0 + beta_cosize: int = 0 + w_cosize: int = 0 + k_inv_cosize: int = 0 + k_decay_cosize: int = 0 + k_restore_cosize: int = 0 + state_scale_diag_cosize: int = 0 + + # TMA transaction bytes per stage + tma_k_bytes: int = 0 + tma_gate_bytes: int = 0 + tma_beta_bytes: int = 0 + tma_v_bytes: int = 0 + tma_w_bytes: int = 0 + qk_cosize: int = 0 + + +def build_cfg( + io_dtype: Type[cutlass.Numeric], + state_dtype: Type[cutlass.Numeric], + *, + use_initial_state: bool, + store_final_state: bool, + enable_checkpoints: bool, + l2norm: bool, + safe_gate: bool, + gate_scale_log2: float, + k_ratio: int, + v_ratio: int, + n_heads_out: int, + max_active_clusters: int, + split_k: bool = False, + dyn_sched: bool = False, +) -> Gdn2RecomputeCfg: + """Build the per-compile ``Gdn2RecomputeCfg`` (io_dtype in {Float16, BFloat16}); + fills the derived TMEM column offsets and SMEM buffer cosizes.""" + if io_dtype not in (cutlass.Float16, cutlass.BFloat16): + raise ValueError(f"io_dtype={io_dtype} not supported; only Float16 and BFloat16 are supported") + cfg = Gdn2RecomputeCfg( + io_dtype=io_dtype, + state_dtype=state_dtype, + use_initial_state=use_initial_state, + store_final_state=store_final_state, + enable_checkpoints=enable_checkpoints, + l2norm=l2norm, + safe_gate=safe_gate, + gate_scale_log2=gate_scale_log2, + k_ratio=k_ratio, + v_ratio=v_ratio, + n_heads_out=n_heads_out, + max_active_clusters=max_active_clusters, + split_k=split_k, + dyn_sched=dyn_sched, + ) + if enable_checkpoints: + cfg.smem_raw_stages = 4 + cfg.smem_checkpoint_stages = 2 + cfg.threads_per_cta = 16 * cfg.threads_per_warp + cfg.cg0_threads_per_group = cfg.cg0_warps_per_group * cfg.threads_per_warp + cfg.tmem_user_threads = (1 + len(cfg.compute_group_1_warp_ids)) * cfg.threads_per_warp + if cfg.smem_state_scale_diag_stages != cfg.qk_scale_ready_stages: + raise ValueError("diag and qk-scale ready rings must share their rolling stage") + + cfg.tmem_state_inp_offset = cfg.tmem_state_acc_offset + cfg.d_k + cfg.tmem_state_k_acc_offset = cfg.tmem_state_inp_offset + (cfg.d_k // 2) + cfg.tmem_update_acc_offset = cfg.tmem_state_k_acc_offset + cfg.b_t + cfg.tmem_rhs_inp_offset = cfg.tmem_update_acc_offset + cfg.b_t + cfg.tmem_update_inp_offset = cfg.tmem_rhs_inp_offset + (cfg.b_t // 2) + assert (cfg.tmem_update_inp_offset + (cfg.b_t // 2)) <= 512 + + cfg.k_cosize = cfg.smem_raw_stages * cfg.d_k * cfg.b_t + cfg.v_cosize = cfg.smem_raw_stages * cfg.d_v * cfg.b_t + cfg.gate_cosize = cfg.smem_raw_stages * cfg.d_k * cfg.b_t + cfg.beta_cosize = cfg.smem_raw_stages * cfg.d_k * cfg.b_t + cfg.w_cosize = cfg.smem_raw_stages * cfg.d_v * cfg.b_t + cfg.k_inv_cosize = cfg.smem_decay_stages * cfg.b_t * cfg.d_k + cfg.k_decay_cosize = cfg.smem_decay_stages * cfg.d_k * cfg.b_t + cfg.k_restore_cosize = cfg.smem_decay_stages * cfg.d_k * cfg.b_t + cfg.state_scale_diag_cosize = cfg.smem_state_scale_diag_stages * (cfg.d_k // 16) * 256 + cfg.qk_cosize = cfg.smem_qk_stages * cfg.b_t * cfg.b_t + cfg.tma_k_bytes = cfg.d_k * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_gate_bytes = cfg.d_k * cfg.b_t * 4 + cfg.tma_beta_bytes = cfg.d_k * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_v_bytes = cfg.d_v * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_w_bytes = cfg.d_v * cfg.b_t * (cfg.io_dtype.width // 8) + return cfg + + +TENSORMAP_DESC_ARRAYS = 6 # per-batch runtime TMA descriptors: K, V, Gate, Beta, W, Checkpoint +TENSORMAP_STATIC_SLOTS = 0 + + +@cute.kernel +def build_all_descs_kernel( + base_k: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_v: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_gate: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_beta: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_w: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_checkpoint: cutlass.GridConstant[cuda.tensor_map.TensorMap], + tensormap_workspace: cute.Tensor, + cu_seqlens: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + gate: cute.Tensor, + beta: cute.Tensor, + w: cute.Tensor, + state_checkpoints: cute.Tensor | None, + n_batch: cutlass.Int32, + k_row_stride: cutlass.Int32, + v_row_stride: cutlass.Int32, + gate_row_stride: cutlass.Int32, + beta_row_stride: cutlass.Int32, + w_row_stride: cutlass.Int32, + checkpoint_row_stride: cutlass.Int32, + checkpoint_every_n: cutlass.Int32, +) -> None: + """Single-launch builder for the per-batch TMA-descriptor arrays.""" + tidx, _, _ = cute.arch.thread_idx() + widx = cutlass.Int32(tidx) // cutlass.Int32(32) + arr_words = n_batch * cutlass.Int32(TENSOR_MAP_QWORDS) + sub0 = cute.make_tensor(tensormap_workspace.iterator, cute.make_layout((arr_words,), stride=(1,))) + sub1 = cute.make_tensor(tensormap_workspace.iterator + arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub2 = cute.make_tensor(tensormap_workspace.iterator + 2 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub3 = cute.make_tensor(tensormap_workspace.iterator + 3 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub4 = cute.make_tensor(tensormap_workspace.iterator + 4 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub5 = cute.make_tensor(tensormap_workspace.iterator + 5 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + + if widx == 0: + if nvvm.elect_sync(): + emit_seq_descs(base_k, sub0, cu_seqlens, k, n_batch, k_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 1: + if nvvm.elect_sync(): + emit_seq_descs(base_v, sub1, cu_seqlens, v, n_batch, v_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 2: + if nvvm.elect_sync(): + emit_seq_descs(base_gate, sub2, cu_seqlens, gate, n_batch, gate_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 3: + if nvvm.elect_sync(): + emit_seq_descs(base_beta, sub3, cu_seqlens, beta, n_batch, beta_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 4: + if nvvm.elect_sync(): + emit_seq_descs(base_w, sub4, cu_seqlens, w, n_batch, w_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if cutlass.const_expr(state_checkpoints is not None): + if widx == 5: + if nvvm.elect_sync(): + emit_checkpoint_seq_descs(base_checkpoint, sub5, cu_seqlens, state_checkpoints, n_batch, checkpoint_row_stride, checkpoint_every_n, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + + +@cute.jit +def build_descs( + io_dtype: cutlass.Constexpr, + b_t: cutlass.Constexpr[int], + k: cute.Tensor, + v: cute.Tensor, + gate: cute.Tensor, + beta: cute.Tensor, + w: cute.Tensor, + state_checkpoints: cute.Tensor | None, + cu_seqlens: cute.Tensor, + tensormap_workspace: cute.Tensor, + checkpoint_every_n: cutlass.Int32, + stream: cuda_driver.CUstream, +): + """Build the 6 per-batch TMA-descriptor arrays (k, v, gate, beta, w, state_checkpoints) + into ``tensormap_workspace``.""" + h_k = k.shape[1] + h_v = v.shape[1] + ho = gate.shape[1] + batch_size = cu_seqlens.shape[0] - 1 + d_k = k.shape[2] + d_v = v.shape[2] + bpe = io_dtype.width // 8 + tma_box_elems = 128 // bpe + seqlen = k.shape[0] + + def headed(t, dim, hn): + return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) + + swz = cuda.TensorMapSwizzle.s128b + base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_beta = cuda.create_tensor_map_tiled_from_view(headed(beta, d_k, ho), box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_w = cuda.create_tensor_map_tiled_from_view(headed(w, d_v, ho), box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + + base_checkpoint = base_k + if cutlass.const_expr(state_checkpoints is not None): + checkpoint_view = cute.make_tensor( + state_checkpoints.iterator, + cute.make_layout( + (state_checkpoints.shape[3], state_checkpoints.shape[2], state_checkpoints.shape[0], ho), + stride=(state_checkpoints.stride[3], state_checkpoints.stride[2], state_checkpoints.stride[0], state_checkpoints.stride[1]), + ), + ) + base_checkpoint = cuda.create_tensor_map_tiled_from_view(checkpoint_view, box_dims=(tma_box_elems, d_k, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz) + n_warps = 6 if state_checkpoints is not None else 5 + build_all_descs_kernel( + base_k, + base_v, + base_gate, + base_beta, + base_w, + base_checkpoint, + tensormap_workspace, + cu_seqlens, + k, + v, + gate, + beta, + w, + state_checkpoints, + cutlass.Int32(batch_size), + cutlass.Int32(k.stride[0]), + cutlass.Int32(v.stride[0]), + cutlass.Int32(gate.stride[0]), + cutlass.Int32(beta.stride[0]), + cutlass.Int32(w.stride[0]), + cutlass.Int32(state_checkpoints.stride[0] if state_checkpoints is not None else 0), + checkpoint_every_n, + ).launch(grid=(1, 1, 1), block=(32 * n_warps, 1, 1), stream=stream) + + +# ---- Torch adapter / host-side compilation --------------------------------------- + + +@lru_cache(maxsize=None) +def get_compiled_cache( + io_dtype_str: str, + state_dtype_str: str, + cu_dtype_str: str, + HO: int, + HK: int, + HV: int, + use_initial_state: bool, + store_final_state: bool, + enable_checkpoints: bool, + l2norm: bool, + safe_gate: bool, + gate_lower_bound: float, + split_k: bool, + dyn_sched: bool, +): + """Return a mutable dict that lazily stores the compiled kernel.""" + return {} + + +def compile( + io_dtype, + state_dtype, + use_initial_state: bool, + store_final_state: bool, + enable_checkpoints: bool, + l2norm: bool, + safe_gate: bool, + gate_scale_log2: float, + k_ratio: int, + v_ratio: int, + n_heads_out: int, + split_k: bool = False, + dyn_sched: bool = False, + *, + num_sm: int, + k_cute, + v_cute, + gate_cute, + a_log_cute, + dt_bias_cute, + beta_cute, + w_cute, + cu_seqlens_cute, + state_in_cute, + state_out_cute, + work_items_cute=None, + work_count_cute=None, + sched_ctr_cute=None, + tensormap_ws_cute, + checkpoint_every_n_tokens, + stream, +): + """JIT-compile the chunked GDN-2 recompute kernel for one static config.""" + cfg = build_cfg( + io_dtype, + state_dtype, + use_initial_state=use_initial_state, + store_final_state=store_final_state, + enable_checkpoints=enable_checkpoints, + l2norm=l2norm, + safe_gate=safe_gate, + gate_scale_log2=gate_scale_log2, + k_ratio=k_ratio, + v_ratio=v_ratio, + n_heads_out=n_heads_out, + max_active_clusters=num_sm, + split_k=split_k, + dyn_sched=dyn_sched, + ) + + return cute.compile( + host, + cfg, + k_cute, + v_cute, + gate_cute, + a_log_cute, + dt_bias_cute, + beta_cute, + w_cute, + cu_seqlens_cute, + state_in_cute, + state_out_cute, + work_items_cute, + work_count_cute, + sched_ctr_cute, + tensormap_ws_cute, + checkpoint_every_n_tokens, + stream, + options="--enable-tvm-ffi --opt-level 2", + ) + + +def chunk_gdn2_recompute_sm100( + k, + v, + gate, + beta, + w, + cu_seqlens, + initial_state, + output_state, + checkpoint_every_n_tokens: int = 0, + output_state_checkpoints=None, + use_qk_l2norm_in_kernel: bool = False, + safe_gate: bool = False, + gate_lower_bound: float = DEFAULT_GATE_LOWER_BOUND, + a_log=None, + dt_bias=None, + work_items=None, + work_count=None, + sched_ctr=None, + *, + tensormap_workspace, + stream, +) -> None: + """Execute the Blackwell BT=16 chunked GDN-2 recompute (state/checkpoint-only) kernel. + + All tensors must be on the same CUDA device with a stride-1 innermost + dim; outer strides are free (padded / permuted views are read through + the TMA descriptors and dynamic layouts). + + Args: + k: ``(total_tokens, HK, DK)`` float16/bfloat16 + v: ``(total_tokens, HV, DV)`` float16/bfloat16 + gate: ``(total_tokens, HO, DK)`` float32. Natural-log decay unless + ``safe_gate``, which applies the safe-gate transform + ``lower_bound * sigmoid(exp(a_log) * (gate + dt_bias))``. + beta: ``(total_tokens, HO, DK)`` io dtype, channel-wise erase gate + w: ``(total_tokens, HO, DV)`` io dtype, channel-wise write gate + cu_seqlens: ``(num_seqs + 1,)`` int32 + initial_state: ``(num_seqs, HO, DK, DV)`` float32/bfloat16, or None + output_state: ``(num_seqs, HO, DK, DV)`` float32/bfloat16, or None + checkpoint_every_n_tokens: emit a checkpoint entry every N tokens (0 = off). + checkpoint[j] is the state after ``(j + 1) * N`` tokens, STRICTLY BEFORE + the sequence end — the end-of-sequence state is only + ``output_state``. + output_state_checkpoints: ``(total_checkpoints, HO, DK, DV)`` io-dtype (KV, v + contiguous — the GDN checkpoint layout); the per-sequence entry offsets + are derived on device from ``cu_seqlens`` ((seqlen-1)//N, + prefix-summed), so there is no cu_checkpoints array + use_qk_l2norm_in_kernel: L2-normalize k rows inside the kernel + safe_gate: interpret ``gate`` through the safe-gate transform + a_log: ``(HO,)`` float32, safe-gate per-head log-amplitude (None = 0) + dt_bias: ``(HO, DK)`` float32, safe-gate channel bias (None = 0) + work_items: ``(max_items, 8)`` int32 split-K work-item table from + ``common/split_k.py``, or None for the one-tile-per-(b,h) + layout. With a table, each item computes chunks + ``[cstart, wend)`` and writes checkpoints only for + ``[wstart, wend)``. + work_count: ``(1,)`` int32 device-side item count (required with + work_items) + """ + HK = k.shape[1] + HV = v.shape[1] + HO = gate.shape[1] + use_initial_state = initial_state is not None + store_final_state = output_state is not None + enable_checkpoints = checkpoint_every_n_tokens > 0 + if enable_checkpoints: + if output_state_checkpoints is None: + raise ValueError("checkpoint_every_n_tokens > 0 requires output_state_checkpoints") + if str(output_state_checkpoints.dtype).split(".")[-1] != str(k.dtype).split(".")[-1]: + raise ValueError( + f"output_state_checkpoints dtype must match the io dtype (fp32 state belongs to output_state): got {output_state_checkpoints.dtype} with io {k.dtype}" + ) + split_k = work_items is not None + dyn_sched = sched_ctr is not None + if split_k: + if work_count is None: + raise ValueError("work_count is required with work_items") + if enable_checkpoints and checkpoint_every_n_tokens != CFG.B_T: + raise ValueError(f"split-K checkpoints require checkpoint_every_n_tokens == {CFG.B_T}, got {checkpoint_every_n_tokens}") + elif work_count is not None: + raise ValueError("work_count must be None without work_items") + + if initial_state is not None: + state_dtype_src = initial_state.dtype + elif output_state is not None: + state_dtype_src = output_state.dtype + else: + state_dtype_src = "float32" + + for name, h in (("HK", HK), ("HV", HV)): + if HO % h != 0: + raise ValueError(f"{name}={h} must divide sab heads {HO}") + k_ratio = HO // HK + v_ratio = HO // HV + gate_scale_log2 = gate_lower_bound * LOG2_E + + if safe_gate and (a_log is None or dt_bias is None): + raise ValueError("safe_gate requires a_log and dt_bias") + if not safe_gate: + a_log = None + dt_bias = None + cu_stream = cuda_driver.CUstream(int(stream)) + + cache = get_compiled_cache( + str(k.dtype), + str(state_dtype_src), + str(cu_seqlens.dtype), + HO, + HK, + HV, + use_initial_state, + store_final_state, + enable_checkpoints, + use_qk_l2norm_in_kernel, + safe_gate, + gate_lower_bound, + split_k, + dyn_sched, + ) + + if "compiled" not in cache: + io_dtype = get_dtype(k.dtype) + state_dtype = get_dtype(state_dtype_src) + k_cute = from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2) + v_cute = from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2) + gate_cute = from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=2) + a_log_cute = from_dlpack(a_log, assumed_align=4) if a_log is not None else None + dt_bias_cute = from_dlpack(dt_bias, assumed_align=16) if dt_bias is not None else None + beta_cute = from_dlpack(beta, assumed_align=4).mark_layout_dynamic(leading_dim=2) + w_cute = from_dlpack(w, assumed_align=16).mark_layout_dynamic(leading_dim=2) + cu_seqlens_cute = from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic() + + state_in_cute = None + if use_initial_state: + state_in_cute = from_dlpack(initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) + + state_out_cute = None + if store_final_state: + state_out_cute = from_dlpack(output_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) + + work_items_cute = None + work_count_cute = None + if split_k: + work_items_cute = from_dlpack(work_items, assumed_align=16) + work_items_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) + work_count_cute = from_dlpack(work_count, assumed_align=4).mark_layout_dynamic() + + sched_ctr_cute = None + if dyn_sched: + sched_ctr_cute = from_dlpack(sched_ctr, assumed_align=4).mark_layout_dynamic() + + tensormap_ws_cute = from_dlpack(tensormap_workspace, assumed_align=128).mark_layout_dynamic() + + cache["compiled"] = compile( + io_dtype, + state_dtype, + use_initial_state, + store_final_state, + enable_checkpoints, + use_qk_l2norm_in_kernel, + safe_gate, + gate_scale_log2, + k_ratio, + v_ratio, + HO, + split_k, + dyn_sched, + num_sm=multiprocessor_count(current_device_id()), + k_cute=k_cute, + v_cute=v_cute, + gate_cute=gate_cute, + a_log_cute=a_log_cute, + dt_bias_cute=dt_bias_cute, + beta_cute=beta_cute, + w_cute=w_cute, + cu_seqlens_cute=cu_seqlens_cute, + state_in_cute=state_in_cute, + state_out_cute=state_out_cute, + work_items_cute=work_items_cute, + work_count_cute=work_count_cute, + sched_ctr_cute=sched_ctr_cute, + tensormap_ws_cute=tensormap_ws_cute, + checkpoint_every_n_tokens=checkpoint_every_n_tokens, + stream=cu_stream, + ) + + compiled = cache["compiled"] + state_checkpoints_for_descs = output_state_checkpoints if enable_checkpoints else None + # desc build runs every execute by contract (cu contents are data; + # buffer pointers may change) — capture-safe, single tiny launch + if cache.get("build_descs_has_state_checkpoints") != (state_checkpoints_for_descs is not None): + cache.pop("build_descs", None) + cache["build_descs_has_state_checkpoints"] = state_checkpoints_for_descs is not None + if "build_descs" not in cache: + io_dtype = get_dtype(k.dtype) + + cu_bd = from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic() + ws_bd = from_dlpack(tensormap_workspace, assumed_align=128).mark_layout_dynamic() + cache["build_descs"] = cute.compile( + build_descs, + io_dtype, + CFG.B_T, + from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(beta, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(w, assumed_align=16).mark_layout_dynamic(leading_dim=2), + None if state_checkpoints_for_descs is None else from_dlpack(state_checkpoints_for_descs, assumed_align=16).mark_layout_dynamic(leading_dim=3), + cu_bd, + ws_bd, + cutlass.Int32(checkpoint_every_n_tokens), + cu_stream, + options="--enable-tvm-ffi", + ) + cache["build_descs"]( + k, + v, + gate, + beta, + w, + state_checkpoints_for_descs, + cu_seqlens, + tensormap_workspace, + checkpoint_every_n_tokens, + cu_stream, + ) + compiled( + k, + v, + gate, + a_log, + dt_bias, + beta, + w, + cu_seqlens, + initial_state if use_initial_state else None, + output_state if store_final_state else None, + work_items, + work_count, + sched_ctr, + tensormap_workspace, + checkpoint_every_n_tokens, + cu_stream, + ) diff --git a/python/cudnn/linear_attention/frost/kernel/gdn_bprop_config.py b/python/cudnn/linear_attention/frost/kernel/gdn_bprop_config.py index 0fcf93fda..c769e0847 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn_bprop_config.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn_bprop_config.py @@ -34,7 +34,6 @@ class Cfg: D_V: int = 128 # value head dim # --- TMA descriptor pool --- - BYTES_PER_TENSORMAP: int = 128 # --- warp assignments (12 warps total) --- COMPUTE_GROUP_0_WARP_IDS: Tuple[int, ...] = (0, 1, 2, 3) # T-pairwise / kk_epi / qk_epi / inverse / parts @@ -46,8 +45,8 @@ class Cfg: # --- register split --- NUM_REGS_COMPUTE_GROUP_0: int = 224 - NUM_REGS_COMPUTE_GROUP_1: int = 256 - NUM_REGS_OTHER: int = 24 + NUM_REGS_COMPUTE_GROUP_1: int = 248 + NUM_REGS_OTHER: int = 32 THREADS_PER_WARP: int = 32 diff --git a/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py index 0b36946fb..6df986764 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py @@ -19,22 +19,22 @@ Chunked Gated Delta Net (GDN) BPROP kernel for Blackwell SM100 (Cutlass primitives). Algorithm overview (per chunk c, iterated c = NT-1 .. 0): - Inputs : Q[BT,DK], K[BT,DK], V[BT,DV], dO[BT,DV], S_c[DK,DV] (= H[c-1], - the forward state ENTERING chunk c), gate[BT], beta[BT] - State : dH[DK,DV] (state gradient, held in TMEM, accumulated backward) + Inputs : Q[BT,DK], K[BT,DK], V[BT,DV], dO[BT,DV], S_c[DK,DV] (= checkpoint entry c-1, + the forward state ENTERING chunk c), Gate[BT], Beta[BT] + State : dstate[DK,DV] (state gradient, held in TMEM, accumulated backward) MMA order. A = the staged attention matrix - (CG0's qk epilogue, sQk); dO' = dO * gCumprod * scale (CG1 restage): - kk : W_kk[BT,BT] = K @ K^T -> shared acc (for T) - qk : W_qk[BT,BT] = Q @ K^T -> shared acc - dV inter : dV[DV,BT] = dH^T(TMEM) @ K^T -> the dV/dK slot - (acc=False; CG1 then scales it by gDecayScale IN PLACE) - ks : KS[BT,DV] = S^T(SMEM) @ K^T -> shared acc + (CG0's QK epilogue, sQk); dO' = dO * cumprod_vals * scale (CG1 restage): + KK : W_kk[BT,BT] = K @ K^T -> shared acc (for T) + QK : W_qk[BT,BT] = Q @ K^T -> shared acc + dV inter : dV[DV,BT] = dstate^T(TMEM) @ K^T -> the dV/dK slot + (acc=False; CG1 then scales it by decay_scale_vals IN PLACE) + KS : KS[BT,DV] = S^T(SMEM) @ K^T -> shared acc dU intra : dV/dK slot += dO^T(SMEM) @ A(SMEM) (waits CG1's in-place scale -> the accumulate is the reference's dv2 = dv_intra + exp2(g_last-g)*(k@dh)) - dH q-term : dH += dO'^T(TMEM) @ Q (acc=False on the - first backward chunk = dH init; the reference's + dstate q-term : dstate += dO'^T(TMEM) @ Q (acc=False on the + first backward chunk = dstate init; the reference's dh = dh*exp2(g_last) + (q*scale*exp2(g))^T @ do) dY : dY[DV,BT] = dU^T(TMEM f16) @ T(SMEM) -> shared acc (dY == dV == d(delta); T read TRANSPOSED vs prefill) @@ -42,64 +42,64 @@ (sV holds delta; CG0 masks it -> sDa for dQ/dK) dM core : dM[BT,BT] = dY(SMEM) @ U^T(SMEM) -- the WY inverse backward T^T dT T^T collapsed via - T^T dU = dY and T Y = U (beta folds through + T^T dU = dY and T Y = U (Beta folds through sAinv's column scale into both factors); CG0 applies -strict(2^{g_i-g_j}) -> sDm (the reference's dA22; both betas cancel against the K rows in the dM-terms) - dH ds-term : dH += dY'^T(TMEM f16) @ K (dY' = -gCumprod - * dY: the -(w^T @ d(delta)) term; ready commits HERE) + dstate update-term : dstate += dY'^T(TMEM f16) @ K (dY' = -cumprod_vals + * dY: the -(W^T @ d(delta)) term; ready commits HERE) dK dM-terms : dV/dK slot += K^T @ dM^T + K^T @ dM (the reference's dk += dA22^T@K_beta / dk_beta += dA22@K folded, the beta row scale cancelling with K_beta = beta*K) - dK s-path : SY[BT,DV] = S^T(SMEM) @ dY^T(SMEM) -> the shared f16 + dK state-path : SY[BT,DV] = S^T(SMEM) @ dY^T(SMEM) -> the shared f16 input columns (dO'/dU/dY' dead by then); CG1 reads it as - -gCumprod[t] * (dY @ S^T)[t,:] and adds it to the banked + -cumprod_vals[t] * (dY @ S^T)[t,:] and adds it to the banked inter+attn dK terms Per chunk CG1: restages dO' and dU and dY' -> the f16 input columns - (dY' overwrites dU), computes delta = V - gCumprod*(K @ S) in registers and stages - Y (= delta) and gks to their dedicated f16 TMEM slots (the u-GEMM and + (dY' overwrites dU), computes delta = V - cumprod_vals*(K @ state) in registers and stages + Y (= delta) and g_k_state to their dedicated f16 TMEM slots (the u-GEMM and the dV-pass read them), stages Q^T over the Y slot after the dV pass, loads dY and stages it plain to sdV (the dV output). SMEM layout (~226 KB of the 227 KB SM100 cap): Buffer Size (B) Stages - q 16384 1 - k 32768 2 <-- double-buffered (prefetch next chunk) - v 16384 1 <-- overwritten in place by u + Q 16384 1 + K 32768 2 <-- double-buffered (prefetch next chunk) + V 16384 1 <-- overwritten in place by U dO 16384 1 - S (forward state H[c-1]) 32768 1 <-- bf16 [DK,DV], TMA-loaded + state (checkpoint entry c-1) 32768 1 <-- bf16 [DK,DV], TMA-loaded A_inverse / T 8192 1 <-- inverse OUTPUT (upper tri = kernel-start zeros) - KK (pristine M_kk) 8192 1 <-- kk_epi's only store; inverse input + dG/dBeta + KK (pristine M_kk) 8192 1 <-- KK epi's only store; inverse input + dG/dBeta QK staging / sDa 8192 1 <-- ALIAS: A then the masked dA dM staging (sDm) 8192 1 <-- Step 8 -> dK dM-terms - dH_entry (sdH) 32768 1 <-- f16 restage, dK-inter's A + dstate_entry (sDstate) 32768 1 <-- f16 restage, dK-inter's A dQ store staging 16384 1 dK store staging 16384 1 dV store staging 16384 1 - cumsumlog / cumprod / beta 3 x 512 2 <-- in-place dG/dBeta staging + cumsumlog / cumprod / Beta 3 x 512 2 <-- in-place dG/dBeta staging -TMEM layout (512 cols; EXACTLY the prefill map, S->dH, O->dV): - cols 0-128 : dH accumulator (fp32) <-- prefill: state (S) +TMEM layout (512 cols; EXACTLY the prefill map, state->dstate, O->dV): + cols 0-128 : dstate accumulator (fp32) <-- prefill: state acc cols 128-192 : dV/dK accumulator (fp32) <-- one slot, five sequential per-chunk productions (dV inter -> dU intra -> dK inter -> dK attn -> dK dM-terms), each with its own mbar pair - cols 192-256 : dH input (f16 packed) <-- prefill: state_inp - cols 256-384 : shared accumulators x2 <-- kk / qk / ks / u / dY / dA + cols 192-256 : dstate input (f16 packed) <-- prefill: state_inp + cols 256-384 : shared accumulators x2 <-- KK / QK / k_state / U / dY / dA / dM core cols 384-448 : shared inputs x2 (f16 packed) <-- dO' / dU / dY'; the dK - s-path acc overwrites them after their last GEMM reads - (CG1's s-path readout precedes its next-chunk restages) - cols 448-512 : Y (448) + gks (480) f16 slots until the dV pass, then + state-path acc overwrites them after their last GEMM reads + (CG1's state-path readout precedes its next-chunk restages) + cols 448-512 : Y (448) + g_k_state (480) f16 slots until the dV pass, then Q^T (448) until CG1's dQ dot reads it Warp assignments (12 warps = 384 threads): - warps 0-3 : compute group 0 - T-pairwise, kk_epi, qk_epi, inverse - warps 4-7 : compute group 1 - dV epilogue, dH scale + f16 restage - (later: V-K*S -> SMEM, dY staging) + warps 0-3 : compute group 0 - T-pairwise, KK epi, QK epi, inverse + warps 4-7 : compute group 1 - dV epilogue, dstate scale + f16 restage + (later: V-K*state -> SMEM, dY staging) warp 8 : MMA warp - issues the GEMMs - warp 9 : TMA load warp - loads q, k (double-buf), v, dO, S(=H) - warp 10 : gate warp - loads gate/beta (double-buffered, + warp 9 : TMA load warp - loads Q, K (double-buf), V, dO, state(=checkpoint) + warp 10 : gate warp - loads Gate/Beta (double-buffered, backward order) + stores dG/dBeta warp 11 : epilogue warp - store dQ, dK, dV to global memory """ @@ -113,13 +113,14 @@ import cutlass import cutlass.cute as cute import cutlass.experimental.primitives as nvvm -import cutlass.experimental.cuda.tensor_map as _tma -from cutlass.cute.arch.nvvm_wrappers import inline_ptx +import cutlass.experimental.cuda.tensor_map as tma from cutlass.cute.runtime import from_dlpack -from cutlass.cutlass_dsl import min as _cutlass_min -from ..common.thd import build_h_descs_kernel, build_qkv_load_descs_kernel, build_state_descs_kernel, TENSOR_MAP_QWORDS +from ..common.thd import emit_copy_desc, emit_checkpoint_seq_descs, emit_seq_descs, TENSOR_MAP_QWORDS from ..common.split_k import decode_work_item +from ..common.host import get_dtype +from cudnn.frost.buffers import current_device_id, data_ptr +from cudnn.frost.device import multiprocessor_count RCP_LN2 = 1.4426950408889634 # 1/ln(2): natural-log gates -> the kernel's log2 domain from cudnn.frost.tile_dsl.barrier import ( @@ -127,10 +128,9 @@ Producer, PipelineState, advance, - arrive, ) from cudnn.frost.tile_dsl.handles import MmaDesc, SmemTile, tma_slice_runtime_desc -from cudnn.frost.tile_dsl.mma import mma_ss, mma_ts, mma_step +from cudnn.frost.tile_dsl.mma import mma_ss, mma_step_k8, mma_ts_step, mma_step from cudnn.frost.tile_dsl.pointwise import fp32_to_fp16, f16x2_to_f32, fmul2, fadd2, ffma2, opaque_f32_zero, sub_f16x2 from cudnn.frost.tile_dsl.swizzle import swizzle_lin_128b, swizzle_xor_128b from cudnn.frost.tile_dsl.tma import ( @@ -169,16 +169,17 @@ class GdnBwdBars(NamedTuple): mb_do_ready: MBarrier mb_do_mma_done: MBarrier mb_do_cg1_done: MBarrier - mb_s_ready: MBarrier - mb_s_done: MBarrier + mb_state_ready: MBarrier + mb_state_done: MBarrier + mb_state_cg0_done: MBarrier mb_gate_ready: MBarrier mb_gate_done: MBarrier mb_beta_ready: MBarrier mb_beta_done: MBarrier - mb_dh_acc_ready: MBarrier - mb_dh_acc_done: MBarrier + mb_dstate_acc_ready: MBarrier + mb_dstate_acc_done: MBarrier mb_du_scale_ready: MBarrier mb_du_scale_done: MBarrier mb_du_total_ready: MBarrier @@ -188,24 +189,42 @@ class GdnBwdBars(NamedTuple): mb_dk_attn_done: MBarrier mb_dk_total_ready: MBarrier mb_dk_total_done: MBarrier + mb_dq_acc_scale_ready: MBarrier + mb_dq_acc_scale_done: MBarrier + mb_dq_acc_total_ready: MBarrier + mb_dq_acc_total_done: MBarrier mb_kk_acc_ready: MBarrier mb_kk_acc_done: MBarrier mb_a_acc_ready: MBarrier - mb_ks_acc_ready: MBarrier + mb_k_state_acc_ready: MBarrier mb_u_acc_ready: MBarrier mb_dy_acc_ready: MBarrier + mb_da_acc_ready: MBarrier + mb_dm_ready: MBarrier + mb_dm_done: MBarrier + mb_dk_state_path_ready: MBarrier + + mb_dstate_inp_ready: MBarrier + mb_dstate_inp_done: MBarrier + mb_do_prime_inp_ready: MBarrier + mb_do_prime_inp_done: MBarrier + mb_du_inp_ready: MBarrier + mb_dyp_inp_ready: MBarrier + mb_dyp_inp_done: MBarrier + mb_y_ready: MBarrier mb_ainv_ready: MBarrier mb_ainv_done: MBarrier mb_qk_ready: MBarrier mb_qk_done: MBarrier - mb_dh_inp_ready: MBarrier - mb_dh_inp_done: MBarrier - mb_dop_inp_ready: MBarrier - mb_dop_inp_done: MBarrier - mb_du_inp_ready: MBarrier - mb_dyp_inp_ready: MBarrier - mb_dyp_inp_done: MBarrier + mb_u_ready: MBarrier + mb_dstate_smem_ready: MBarrier + mb_dstate_smem_done: MBarrier + mb_state_dot_dstate_done: MBarrier + mb_da_ready: MBarrier + + mb_dbeta_cg1_ready: MBarrier + mb_dgate_cg1_ready: MBarrier mb_dq_tmastg_ready: MBarrier mb_dq_tmastg_done: MBarrier @@ -213,31 +232,15 @@ class GdnBwdBars(NamedTuple): mb_dk_tmastg_done: MBarrier mb_dv_tmastg_ready: MBarrier mb_dv_tmastg_done: MBarrier - - mb_y_ready: MBarrier mb_sdv_done: MBarrier - mb_u_ready: MBarrier - mb_dhs_ready: MBarrier - mb_dhs_done: MBarrier - mb_da_ready: MBarrier - mb_dq_acc_scale_ready: MBarrier - mb_dq_acc_scale_done: MBarrier - mb_dq_acc_total_ready: MBarrier - mb_dq_acc_total_done: MBarrier - mb_da_acc_ready: MBarrier - mb_dm_ready: MBarrier - mb_dm_done: MBarrier - mb_dbeta_cg1_ready: MBarrier - mb_dgate_cg1_ready: MBarrier - mb_hdh_done: MBarrier - mb_dk_spath_ready: MBarrier + mb_tmem_done: MBarrier mb_sched_ready: MBarrier mb_sched_done: MBarrier def make_gdn_bars(cfg) -> GdnBwdBars: - """GdnBwdBars factory. MUST be called from inside ``_kernel`` (allocates SMEM; + """GdnBwdBars factory. MUST be called from inside ``kernel`` (allocates SMEM; the mbar rings sit ahead of the gate scalar arrays and data buffers).""" ONE_LANE = 1 MMA_ARRIVERS = len([cfg.mma_warp_id]) @@ -263,14 +266,17 @@ def alloc(n): mb_do_ready=MBarrier(alloc(cfg.smem_do_stages), stages=cfg.smem_do_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), mb_do_mma_done=MBarrier(alloc(cfg.smem_do_stages), stages=cfg.smem_do_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_do_cg1_done=MBarrier(alloc(cfg.smem_do_stages), stages=cfg.smem_do_stages, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_s_ready=MBarrier(alloc(cfg.smem_s_stages), stages=cfg.smem_s_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), - mb_s_done=MBarrier(alloc(cfg.smem_s_stages), stages=cfg.smem_s_stages, init_count=ONE_LANE + CG0_THREADS, producer=Producer.MMA_COMMIT), + mb_state_ready=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), + mb_state_done=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_state_cg0_done=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=CG0_THREADS, producer=Producer.THREAD), mb_gate_ready=MBarrier(alloc(cfg.smem_gate_stages), stages=cfg.smem_gate_stages, init_count=GATE_WARP, producer=Producer.THREAD), mb_gate_done=MBarrier(alloc(cfg.smem_gate_stages), stages=cfg.smem_gate_stages, init_count=CG0_PLUS_CG1, producer=Producer.THREAD), mb_beta_ready=MBarrier(alloc(cfg.smem_beta_stages), stages=cfg.smem_beta_stages, init_count=GATE_WARP, producer=Producer.THREAD), mb_beta_done=MBarrier(alloc(cfg.smem_beta_stages), stages=cfg.smem_beta_stages, init_count=CG0_PLUS_CG1, producer=Producer.THREAD), - mb_dh_acc_ready=MBarrier(alloc(cfg.tmem_dh_acc_stages), stages=cfg.tmem_dh_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dh_acc_done=MBarrier(alloc(cfg.tmem_dh_acc_stages), stages=cfg.tmem_dh_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_dstate_acc_ready=MBarrier( + alloc(cfg.tmem_dstate_acc_stages), stages=cfg.tmem_dstate_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT + ), + mb_dstate_acc_done=MBarrier(alloc(cfg.tmem_dstate_acc_stages), stages=cfg.tmem_dstate_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), # five sequential per-chunk productions share the dV/dK TMEM slot mb_du_scale_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_du_scale_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), @@ -281,62 +287,62 @@ def alloc(n): mb_dk_attn_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), mb_dk_total_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_dk_total_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_dq_acc_scale_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_dq_acc_scale_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_dq_acc_total_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_dq_acc_total_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), # shared accumulators at STATIC columns: group A holds - # kk -> ks -> dY -> dM core, group B holds A -> U -> dA. + # KK -> k_state -> dY -> dM core, group B holds A -> U -> dA. mb_kk_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_kk_acc_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), mb_a_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_ks_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_k_state_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_u_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_dy_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_ainv_ready=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_ainv_done=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_qk_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_qk_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dh_inp_ready=MBarrier(alloc(cfg.tmem_dh_inp_stages), stages=cfg.tmem_dh_inp_stages, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_dh_inp_done=MBarrier(alloc(cfg.tmem_dh_inp_stages), stages=cfg.tmem_dh_inp_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_da_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_dm_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_dm_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_dk_state_path_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_dstate_inp_ready=MBarrier(alloc(cfg.tmem_dstate_inp_stages), stages=cfg.tmem_dstate_inp_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_dstate_inp_done=MBarrier( + alloc(cfg.tmem_dstate_inp_stages), stages=cfg.tmem_dstate_inp_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT + ), # f16 input restages at STATIC columns: dO' alone; dU and dY' overlap - mb_dop_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_dop_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_do_prime_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_do_prime_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_du_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_dyp_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_dyp_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_y_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_ainv_ready=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_ainv_done=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_qk_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_qk_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_u_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_dstate_smem_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_dstate_smem_done=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_state_dot_dstate_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_da_ready=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_dbeta_cg1_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_dgate_cg1_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_dq_tmastg_ready=MBarrier(alloc(cfg.smem_dq_stages), stages=cfg.smem_dq_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_dq_tmastg_done=MBarrier(alloc(cfg.smem_dq_stages), stages=cfg.smem_dq_stages, init_count=EPI_WARP, producer=Producer.THREAD), mb_dk_tmastg_ready=MBarrier(alloc(cfg.smem_dk_stages), stages=cfg.smem_dk_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_dk_tmastg_done=MBarrier(alloc(cfg.smem_dk_stages), stages=cfg.smem_dk_stages, init_count=EPI_WARP, producer=Producer.THREAD), mb_dv_tmastg_ready=MBarrier(alloc(cfg.smem_dv_stages), stages=cfg.smem_dv_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_dv_tmastg_done=MBarrier(alloc(cfg.smem_dv_stages), stages=cfg.smem_dv_stages, init_count=EPI_WARP, producer=Producer.THREAD), - mb_y_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_sdv_done=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_u_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_dhs_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_dhs_done=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_da_ready=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_dq_acc_scale_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dq_acc_scale_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_dq_acc_total_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dq_acc_total_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_da_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dm_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dm_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_dbeta_cg1_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_dgate_cg1_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_hdh_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_dk_spath_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_tmem_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS + CG1_THREADS, producer=Producer.THREAD), mb_sched_ready=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=ONE_LANE, producer=Producer.THREAD), mb_sched_done=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=11, producer=Producer.THREAD), ) -# --------------------------------------------------------------------------- -# Dynamic tile scheduler: global-ticket work-stealing ring -# --------------------------------------------------------------------------- +# ---- Dynamic tile scheduler ------------------------------------------------------ @cute.jit -def _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): +def sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): """TMA-LDG-warp side: pull the next tile off the global ticket, publish it.""" if cutlass.const_expr(cfg.dyn_sched): bars.mb_sched_done[sched_state.idx].wait(sched_state.phase) @@ -352,7 +358,7 @@ def _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ct @cute.jit -def _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): +def sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): """Consumer side: read the TMA-LDG warp's published next tile.""" if cutlass.const_expr(cfg.dyn_sched): bars.mb_sched_ready[sched_state.idx].wait(sched_state.phase) @@ -363,20 +369,9 @@ def _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): return tile_idx + num_ctas, sched_state -# --------------------------------------------------------------------------- -# Device-side helpers / warp bodies -# --------------------------------------------------------------------------- - - @cute.jit -def _invert_diagonal_NxN(cfg, in_base, out_base, d, tidx, N: int = 8): - """Stage 1: Gauss-Jordan inversion of one diagonal NxN block, reading the - raw block from ``in_base`` and writing the inverse to ``out_base`` (same - swizzled offsets). - - The tile swizzle re-homes whole rows only, so a diagonal block's row stays - a contiguous N-element run at ``swz(row_lin_base)``. - """ +def invert_diagonal_NxN(cfg, in_base, out_base, d, tidx, N: int = 8): + """Gauss-Jordan inversion of one diagonal NxN block, ``in_base`` -> ``out_base`` (f16 SMEM).""" tidx_in_group = tidx % N BT = cfg.b_t @@ -416,23 +411,9 @@ def _invert_diagonal_NxN(cfg, in_base, out_base, d, tidx, N: int = 8): @cute.jit -def _mma_m16n8k8(a0, a1, b0, c_regs, dtype: cutlass.Constexpr): - """One m16n8k8 reg-reg mma (``mma_step`` only emits the k16 form), - accumulating into the ``c_regs`` buffer in place.""" - tag = "f16" if cutlass.const_expr(dtype == cutlass.Float16) else "bf16" - c_regs[0], c_regs[1], c_regs[2], c_regs[3] = inline_ptx( - f"mma.sync.aligned.m16n8k8.row.col.f32.{tag}.{tag}.f32" " {$0,$1,$2,$3}, {$4,$5}, {$6}, {$7,$8,$9,$10};", - write_only_types=[cutlass.Float32, cutlass.Float32, cutlass.Float32, cutlass.Float32], - read_only_args=[a0, a1, b0, c_regs[0], c_regs[1], c_regs[2], c_regs[3]], - ) - - -@cute.jit -def _warp_reduce_scatter_frag16(vals, lane_id): +def warp_reduce_scatter_frag_16_elems(vals, lane_id): """Reduce-scatter 16 fragment token-partials (tcol = (lane%4)*2 + - (j//2)*8 + (j%2)) over the 8 lane-groups: step k exchanges with - lane^(4< 16x16 (C <- -D^{-1} C A^{-1}). - Raw C blocks read from ``raw_base`` (sKK); corrected blocks and all - writes on ``base_int`` (sAinv). - - Keep the per-lane ldmatrix offset (``lds1``/``lds4``) a SEPARATE sum term - from the warp-uniform (row, col) origin in all the diagonal helpers — - folding it into the row term costs ~2% (per-lane address datapath). - """ +def blockwise_diagonal_8x8_to_16x16(cfg, base_int, raw_base, d0, lane_id): + """Off-diagonal correction 8x8 -> 16x16 (C <- -D^{-1} C A^{-1}); raw C from ``raw_base``, writes on ``base_int``.""" bpe = cfg.io_dtype.width // 8 - lds1 = (lane_id % 8) * 64 + ldsm_x1_off = (lane_id % 8) * 64 d = nvvm.ldmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + 8 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr( + base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + 8 + ldsm_x1_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), 1, nvvm.MMALayout.ROW, ) c = nvvm.ldmatrix( - cutlass.inttoptr(raw_base + swizzle_lin_128b((d0 + 8) * 64 + d0 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr(raw_base + swizzle_lin_128b((d0 + 8) * 64 + d0 + ldsm_x1_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), 1, nvvm.MMALayout.COL, ) + + # ---- T = -(D^{-1} @ C) ------------------------------------------------------- c_regs = cutlass.Array(cutlass.Float32, 4, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(4): c_regs[i] = cutlass.Float32(0.0) - _mma_m16n8k8(d, d, c, c_regs, cfg.io_dtype) + mma_step_k8(c_regs, [d, d], [c], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) for i in cutlass.range_constexpr(4): c_regs[i] = -c_regs[i] a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(2)] + + # ---- C = T @ A^{-1} ---------------------------------------------------------- ai = nvvm.ldmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b(d0 * 64 + d0 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr(base_int + swizzle_lin_128b(d0 * 64 + d0 + ldsm_x1_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), 1, nvvm.MMALayout.COL, ) o_regs = cutlass.Array(cutlass.Float32, 4, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(4): o_regs[i] = cutlass.Float32(0.0) - _mma_m16n8k8(a_f16[0], a_f16[1], ai, o_regs, cfg.io_dtype) + mma_step_k8(o_regs, a_f16, [ai], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) o_f16 = fp32_to_fp16(o_regs[0], o_regs[1], dtype=cfg.io_dtype) + + # ---- store corrected C ------------------------------------------------------- nvvm.stmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + ldsm_x1_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), o_f16, nvvm.MMALayout.ROW, ) @cute.jit -def _blockwise_diagonal_16x16_to_32x32(cfg, base_int, raw_base, d0, lane_id): - """Stage 3: off-diagonal correction 16x16 -> 32x32 (raw C from - ``raw_base``).""" +def blockwise_diagonal_16x16_to_32x32(cfg, base_int, raw_base, d0, lane_id): + """Off-diagonal correction 16x16 -> 32x32 (raw C from ``raw_base``).""" bpe = cfg.io_dtype.width // 8 - lds4 = (lane_id % 16) * 64 + (lane_id // 16) * 8 + ldsm_x4_off = (lane_id % 16) * 64 + (lane_id // 16) * 8 d = list( nvvm.ldmatrix( cutlass.inttoptr( - base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + 16 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + 16 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 ), 4, nvvm.MMALayout.ROW, @@ -513,11 +494,15 @@ def _blockwise_diagonal_16x16_to_32x32(cfg, base_int, raw_base, d0, lane_id): ) c = list( nvvm.ldmatrix( - cutlass.inttoptr(raw_base + swizzle_lin_128b((d0 + 16) * 64 + d0 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr( + raw_base + swizzle_lin_128b((d0 + 16) * 64 + d0 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), 4, nvvm.MMALayout.COL, ) ) + + # ---- T = -(D^{-1} @ C) ------------------------------------------------------- c_regs = cutlass.Array(cutlass.Float32, 8, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(8): c_regs[i] = cutlass.Float32(0.0) @@ -525,9 +510,11 @@ def _blockwise_diagonal_16x16_to_32x32(cfg, base_int, raw_base, d0, lane_id): for i in cutlass.range_constexpr(8): c_regs[i] = -c_regs[i] a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + + # ---- C = T @ A^{-1} ---------------------------------------------------------- ai = list( nvvm.ldmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b(d0 * 64 + d0 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr(base_int + swizzle_lin_128b(d0 * 64 + d0 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), 4, nvvm.MMALayout.COL, ) @@ -536,27 +523,28 @@ def _blockwise_diagonal_16x16_to_32x32(cfg, base_int, raw_base, d0, lane_id): for i in cutlass.range_constexpr(8): o_regs[i] = cutlass.Float32(0.0) mma_step(o_regs, a_f16, ai, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) - ow = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + + # ---- store corrected C ------------------------------------------------------- nvvm.stmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - ow, + cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + o_f16, nvvm.MMALayout.ROW, ) @cute.jit -def _blockwise_diagonal_32x32_to_64x64(cfg, base_int, raw_base, warp_id, lane_id): - """Stage 4: off-diagonal correction 32x32 -> 64x64 (2 warps, one 16-row - M-band each; raw C from ``raw_base``).""" +def blockwise_diagonal_32x32_to_64x64(cfg, base_int, raw_base, warp_id, lane_id): + """Off-diagonal correction 32x32 -> 64x64 (2 warps, one 16-row M-band each; raw C from ``raw_base``).""" band = warp_id % 2 bpe = cfg.io_dtype.width // 8 - lds4 = (lane_id % 16) * 64 + (lane_id // 16) * 8 + ldsm_x4_off = (lane_id % 16) * 64 + (lane_id // 16) * 8 a_regs = [] for vs in cutlass.range_constexpr(2): a_regs += list( nvvm.ldmatrix( cutlass.inttoptr( - base_int + swizzle_lin_128b((32 + band * 16) * 64 + 32 + vs * 16 + lds4, row_stride_log2=6) * bpe, + base_int + swizzle_lin_128b((32 + band * 16) * 64 + 32 + vs * 16 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16, ), @@ -569,7 +557,7 @@ def _blockwise_diagonal_32x32_to_64x64(cfg, base_int, raw_base, warp_id, lane_id b_regs += list( nvvm.ldmatrix( cutlass.inttoptr( - raw_base + swizzle_lin_128b((32 + (vs // 2) * 16) * 64 + (vs % 2) * 16 + lds4, row_stride_log2=6) * bpe, + raw_base + swizzle_lin_128b((32 + (vs // 2) * 16) * 64 + (vs % 2) * 16 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16, ), @@ -577,6 +565,8 @@ def _blockwise_diagonal_32x32_to_64x64(cfg, base_int, raw_base, warp_id, lane_id nvvm.MMALayout.COL, ) ) + + # ---- T = -(D^{-1} @ C) ------------------------------------------------------- c_regs = cutlass.Array(cutlass.Float32, 16, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(16): c_regs[i] = cutlass.Float32(0.0) @@ -585,12 +575,14 @@ def _blockwise_diagonal_32x32_to_64x64(cfg, base_int, raw_base, warp_id, lane_id for i in cutlass.range_constexpr(16): c_regs[i] = -c_regs[i] a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + + # ---- C = T @ A^{-1} ---------------------------------------------------------- ai_regs = [] for vs in cutlass.range_constexpr(4): ai_regs += list( nvvm.ldmatrix( cutlass.inttoptr( - base_int + swizzle_lin_128b(((vs // 2) * 16) * 64 + (vs % 2) * 16 + lds4, row_stride_log2=6) * bpe, + base_int + swizzle_lin_128b(((vs // 2) * 16) * 64 + (vs % 2) * 16 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16, ), @@ -603,25 +595,31 @@ def _blockwise_diagonal_32x32_to_64x64(cfg, base_int, raw_base, warp_id, lane_id o_regs[i] = cutlass.Float32(0.0) for ks in cutlass.range_constexpr(2): mma_step(o_regs, a_f16, ai_regs[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) - ow = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + + # ---- store corrected C ------------------------------------------------------- nvvm.barrier_cta_sync_aligned( cfg.inverse_inner_barrier_id, thread_count=cfg.inverse_inner_barrier_threads, ) nvvm.stmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((32 + band * 16) * 64 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - ow[0:4], + cutlass.inttoptr( + base_int + swizzle_lin_128b((32 + band * 16) * 64 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), + o_f16[0:4], nvvm.MMALayout.ROW, ) nvvm.stmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((32 + band * 16) * 64 + 16 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - ow[4:8], + cutlass.inttoptr( + base_int + swizzle_lin_128b((32 + band * 16) * 64 + 16 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), + o_f16[4:8], nvvm.MMALayout.ROW, ) @cute.jit -def _tmastg_warp( +def tmastg_warp( cfg, total_tiles, bidx, @@ -639,17 +637,18 @@ def _tmastg_warp( ): """TMA-STG warp role (warp 11): persistent tile-scheduler loop + per-chunk dQ/dK/dV TMA bulk-stores from the SMEM staging buffers to global memory.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) dq_index = PipelineState.start(phase=0) dk_index = PipelineState.start(phase=0) dv_index = PipelineState.start(phase=0) + sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) - S_MIN = 0 if cfg.use_initial_state else 1 - SFIRST_MIN = 1 if cfg.use_initial_state else 2 + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 bpe = cfg.io_dtype.width // 8 - granu = 128 // bpe + granule_elems = 128 // bpe sdQ_tma = SmemTile( base=sdQ_raw, elems_per_stage=(cfg.dq_cosize // cfg.smem_dq_stages), @@ -658,7 +657,7 @@ def _tmastg_warp( stride_byte_offset=0, layout=0, tma_loads_per_tile=2, - tma_granu_elems=granu, + tma_granu_elems=granule_elems, tma_subtile_stride_elems=4096, ) sdK_tma = SmemTile( @@ -669,7 +668,7 @@ def _tmastg_warp( stride_byte_offset=0, layout=0, tma_loads_per_tile=2, - tma_granu_elems=granu, + tma_granu_elems=granule_elems, tma_subtile_stride_elems=4096, ) sdV_tma = SmemTile( @@ -680,7 +679,7 @@ def _tmastg_warp( stride_byte_offset=0, layout=0, tma_loads_per_tile=2, - tma_granu_elems=granu, + tma_granu_elems=granule_elems, tma_subtile_stride_elems=4096, ) heads_out = cutlass.Int32(cfg.n_heads_out) @@ -691,11 +690,12 @@ def _tmastg_warp( cfg, tile_idx, cu_seqlens, mWorkItems ) - slot = (batch_idx * heads_out + head_idx) * desc_qwords + head_o = head_idx + slot = batch_idx * desc_qwords desc_dq_slot = (desc_dq_base + slot).tospace(cutlass.AddressSpace.generic) desc_dk_slot = (desc_dk_base + slot).tospace(cutlass.AddressSpace.generic) desc_dv_slot = (desc_dv_base + slot).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): + if elect_one: tma_tensormap_acquire(desc_dq_slot) tma_tensormap_acquire(desc_dk_slot) tma_tensormap_acquire(desc_dv_slot) @@ -707,9 +707,8 @@ def _tmastg_warp( dv_idx = dv_index.idx bars.mb_dv_tmastg_ready[dv_idx].wait(dv_index.phase) dv_index = advance(dv_index, cfg.smem_dv_stages) - dv_slice = tma_slice_runtime_desc(desc_dv_slot, cutlass.Int32(0), tok_coord) + dv_slice = tma_slice_runtime_desc(desc_dv_slot, cutlass.Int32(0), head_o, tok_coord) if cutlass.const_expr(cfg.split_k): - # right-warmup chunks stage grads to SMEM but never store them if chunk_idx < wend: tma_store_tile(sdV_tma[dv_idx], dv_slice, acquire=False) tma_store_commit() @@ -720,7 +719,7 @@ def _tmastg_warp( dq_idx = dq_index.idx bars.mb_dq_tmastg_ready[dq_idx].wait(dq_index.phase) dq_index = advance(dq_index, cfg.smem_dq_stages) - dq_slice = tma_slice_runtime_desc(desc_dq_slot, cutlass.Int32(0), tok_coord) + dq_slice = tma_slice_runtime_desc(desc_dq_slot, cutlass.Int32(0), head_o, tok_coord) if cutlass.const_expr(cfg.split_k): if chunk_idx < wend: tma_store_tile(sdQ_tma[dq_idx], dq_slice, acquire=False) @@ -732,7 +731,7 @@ def _tmastg_warp( dk_idx = dk_index.idx bars.mb_dk_tmastg_ready[dk_idx].wait(dk_index.phase) dk_index = advance(dk_index, cfg.smem_dk_stages) - dk_slice = tma_slice_runtime_desc(desc_dk_slot, cutlass.Int32(0), tok_coord) + dk_slice = tma_slice_runtime_desc(desc_dk_slot, cutlass.Int32(0), head_o, tok_coord) if cutlass.const_expr(cfg.split_k): if chunk_idx < wend: tma_store_tile(sdK_tma[dk_idx], dk_slice, acquire=False) @@ -747,11 +746,11 @@ def _tmastg_warp( bars.mb_dq_tmastg_done[dq_idx].arrive() tma_store_wait(0) bars.mb_dk_tmastg_done[dk_idx].arrive() - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) @cute.jit -def _gate_beta_warp( +def gate_beta_warp( cfg, total_tiles, bidx, @@ -769,8 +768,8 @@ def _gate_beta_warp( sSched, bars, ): - """Gate/beta LOAD + STORE warp role (warp 10): per-chunk gate/beta G->S - loads (BACKWARD order, one-chunk prefetch) and the dG/dBeta stores.""" + """Gate/beta LOAD + STORE warp role (warp 10): per-chunk Gate/Beta G->S + loads and the dG/dBeta stores.""" nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) gate_index = PipelineState.start(phase=1) beta_index = PipelineState.start(phase=1) @@ -780,21 +779,22 @@ def _gate_beta_warp( n_cols = cfg.b_t // cfg.threads_per_warp sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) - S_MIN = 0 if cfg.use_initial_state else 1 + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 SFIRST_MIN = 1 if cfg.use_initial_state else 2 while tile_idx < total_tiles: batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) - sk_nt = cend - wstart + num_item_chunks = cend - wstart if cutlass.const_expr(cfg.split_k): # dG/dBeta ownership: mask stores past the item's write range write_end = batch_start + wend * cfg.b_t write_end = write_end if write_end < batch_end else batch_end else: write_end = batch_end - # ---- prefetch: the FIRST backward chunk's gate/beta ------------ - if sk_nt > 0: + + # ---- prefetch: the FIRST backward chunk's Gate/Beta ---------------------- + if num_item_chunks > 0: chunk_offset = batch_start + (cend - 1) * cfg.b_t gGate = cute.domain_offset((chunk_offset,), mGate[None, head_idx]) gBeta = cute.domain_offset((chunk_offset,), mBeta[None, head_idx]) @@ -805,42 +805,42 @@ def _gate_beta_warp( pos = lidx + col * cfg.threads_per_warp pos_valid[col] = cute.elem_less(chunk_offset + pos, batch_end) - # --- Gate load (OOB neutral: 1.0 -> log2 = 0.0) --- - tGrGate = [cutlass.Float32(0.0)] * n_cols + # ---- Gate load: GMEM -> SMEM (OOB neutral: 1.0 -> log2 = 0.0) -------- + gate_vals = [cutlass.Float32(0.0)] * n_cols for col in cutlass.range_constexpr(n_cols): pos = lidx + col * cfg.threads_per_warp oob_neutral = cutlass.Float32(0.0) if cutlass.const_expr(cfg.log_gate) else cutlass.Float32(1.0) - tGrGate[col] = gGate[pos] if pos_valid[col] else oob_neutral + gate_vals[col] = gGate[pos] if pos_valid[col] else oob_neutral if cutlass.const_expr(cfg.log_gate): for col in cutlass.range_constexpr(n_cols): - tGrGate[col] = tGrGate[col] * cutlass.Float32(RCP_LN2) + gate_vals[col] = gate_vals[col] * cutlass.Float32(RCP_LN2) else: for col in cutlass.range_constexpr(n_cols): - tGrGate[col] = cute.math.log2(tGrGate[col] + 1e-10, fastmath=True) + gate_vals[col] = cute.math.log2(gate_vals[col] + 1e-10, fastmath=True) for offset in [1, 2, 4, 8, 16]: for col in cutlass.range_constexpr(n_cols): - n = nvvm.shfl_sync(0xFFFFFFFF, tGrGate[col], offset, 0, kind=nvvm.Shfl.UP) + n = nvvm.shfl_sync(0xFFFFFFFF, gate_vals[col], offset, 0, kind=nvvm.Shfl.UP) if lidx >= offset: - tGrGate[col] = tGrGate[col] + n + gate_vals[col] = gate_vals[col] + n for col in cutlass.range_constexpr(1, n_cols): last_v = nvvm.shfl_sync( 0xFFFFFFFF, - tGrGate[col - 1], + gate_vals[col - 1], cfg.threads_per_warp - 1, cfg.threads_per_warp - 1, kind=nvvm.Shfl.IDX, ) - tGrGate[col] += last_v + gate_vals[col] += last_v for col in cutlass.range_constexpr(n_cols): pos = lidx + col * cfg.threads_per_warp - sCumsumlog[pos, 0, gate_idx] = tGrGate[col] - sCumprod[pos, 0, gate_idx] = cute.math.exp2(tGrGate[col], fastmath=True) + sCumsumlog[pos, 0, gate_idx] = gate_vals[col] + sCumprod[pos, 0, gate_idx] = cute.math.exp2(gate_vals[col], fastmath=True) bars.mb_gate_ready[gate_idx].arrive() - # --- Beta load (per-element async G->S cp.async) --- + # ---- Beta load: GMEM -> SMEM (per-element cp.async) ------------------ beta_idx = beta_index.idx beta_index = advance(beta_index, cfg.smem_beta_stages) for col in cutlass.range_constexpr(n_cols): @@ -851,9 +851,9 @@ def _gate_beta_warp( nvvm.cp_async_shared_global(dst, src, 4, nvvm.LoadCacheModifier.CA, cp_size=cp_size) nvvm.cp_async_mbarrier_arrive(bars.mb_beta_ready[beta_idx].smem_ptr, noinc=True) - for rev_idx in cutlass.range(sk_nt): - # ---- prefetch the NEXT chunk's gate/beta ------------------- - if rev_idx + 1 < sk_nt: + for rev_idx in cutlass.range(num_item_chunks): + # ---- prefetch the NEXT chunk's Gate/Beta ----------------------------- + if rev_idx + 1 < num_item_chunks: chunk_offset = batch_start + (cend - 2 - rev_idx) * cfg.b_t gGate = cute.domain_offset((chunk_offset,), mGate[None, head_idx]) gBeta = cute.domain_offset((chunk_offset,), mBeta[None, head_idx]) @@ -864,37 +864,37 @@ def _gate_beta_warp( pos = lidx + col * cfg.threads_per_warp pos_valid[col] = cute.elem_less(chunk_offset + pos, batch_end) - tGrGate = [cutlass.Float32(0.0)] * n_cols + gate_vals = [cutlass.Float32(0.0)] * n_cols for col in cutlass.range_constexpr(n_cols): pos = lidx + col * cfg.threads_per_warp oob_neutral = cutlass.Float32(0.0) if cutlass.const_expr(cfg.log_gate) else cutlass.Float32(1.0) - tGrGate[col] = gGate[pos] if pos_valid[col] else oob_neutral + gate_vals[col] = gGate[pos] if pos_valid[col] else oob_neutral if cutlass.const_expr(cfg.log_gate): for col in cutlass.range_constexpr(n_cols): - tGrGate[col] = tGrGate[col] * cutlass.Float32(RCP_LN2) + gate_vals[col] = gate_vals[col] * cutlass.Float32(RCP_LN2) else: for col in cutlass.range_constexpr(n_cols): - tGrGate[col] = cute.math.log2(tGrGate[col] + 1e-10, fastmath=True) + gate_vals[col] = cute.math.log2(gate_vals[col] + 1e-10, fastmath=True) for offset in [1, 2, 4, 8, 16]: for col in cutlass.range_constexpr(n_cols): - n = nvvm.shfl_sync(0xFFFFFFFF, tGrGate[col], offset, 0, kind=nvvm.Shfl.UP) + n = nvvm.shfl_sync(0xFFFFFFFF, gate_vals[col], offset, 0, kind=nvvm.Shfl.UP) if lidx >= offset: - tGrGate[col] = tGrGate[col] + n + gate_vals[col] = gate_vals[col] + n for col in cutlass.range_constexpr(1, n_cols): last_v = nvvm.shfl_sync( 0xFFFFFFFF, - tGrGate[col - 1], + gate_vals[col - 1], cfg.threads_per_warp - 1, cfg.threads_per_warp - 1, kind=nvvm.Shfl.IDX, ) - tGrGate[col] += last_v + gate_vals[col] += last_v for col in cutlass.range_constexpr(n_cols): pos = lidx + col * cfg.threads_per_warp - sCumsumlog[pos, 0, gate_idx] = tGrGate[col] - sCumprod[pos, 0, gate_idx] = cute.math.exp2(tGrGate[col], fastmath=True) + sCumsumlog[pos, 0, gate_idx] = gate_vals[col] + sCumprod[pos, 0, gate_idx] = cute.math.exp2(gate_vals[col], fastmath=True) bars.mb_gate_ready[gate_idx].arrive() @@ -908,30 +908,30 @@ def _gate_beta_warp( nvvm.cp_async_shared_global(dst, src, 4, nvvm.LoadCacheModifier.CA, cp_size=cp_size) nvvm.cp_async_mbarrier_arrive(bars.mb_beta_ready[beta_idx].smem_ptr, noinc=True) - # ---- store-ready wait + in-place store back ---------------- + # ---- store-ready wait + in-place store back -------------------------- st_offset = batch_start + (cend - 1 - rev_idx) * cfg.b_t gGate_st = cute.domain_offset((st_offset,), mDg[None, head_idx]) gBeta_st = cute.domain_offset((st_offset,), mDbeta[None, head_idx]) g_st_idx = gate_store_index.idx bars.mb_gate_done[g_st_idx].wait(gate_store_index.phase) gate_store_index = advance(gate_store_index, cfg.smem_gate_stages) - tGrDg = [cutlass.Float32(0.0)] * n_cols + dg_vals = [cutlass.Float32(0.0)] * n_cols for col in cutlass.range_constexpr(n_cols): pos = lidx + col * cfg.threads_per_warp - tGrDg[col] = sCumsumlog[pos, 0, g_st_idx] + dg_vals[col] = sCumsumlog[pos, 0, g_st_idx] for offset in [1, 2, 4, 8, 16]: for col in cutlass.range_constexpr(n_cols): - n = nvvm.shfl_sync(0xFFFFFFFF, tGrDg[col], offset, 31, kind=nvvm.Shfl.DOWN) + n = nvvm.shfl_sync(0xFFFFFFFF, dg_vals[col], offset, 31, kind=nvvm.Shfl.DOWN) if lidx < cfg.threads_per_warp - offset: - tGrDg[col] = tGrDg[col] + n + dg_vals[col] = dg_vals[col] + n for col in cutlass.range_constexpr(n_cols - 1): - cc = cutlass.const_expr(n_cols - 2 - col) - later_total = nvvm.shfl_sync(0xFFFFFFFF, tGrDg[cc + 1], 0, 0, kind=nvvm.Shfl.IDX) - tGrDg[cc] = tGrDg[cc] + later_total + rev_col = cutlass.const_expr(n_cols - 2 - col) + later_total = nvvm.shfl_sync(0xFFFFFFFF, dg_vals[rev_col + 1], 0, 0, kind=nvvm.Shfl.IDX) + dg_vals[rev_col] = dg_vals[rev_col] + later_total for col in cutlass.range_constexpr(n_cols): pos = lidx + col * cfg.threads_per_warp if cute.elem_less(st_offset + pos, write_end): - gGate_st[pos] = tGrDg[col] + gGate_st[pos] = dg_vals[col] b_st_idx = beta_store_index.idx bars.mb_beta_done[b_st_idx].wait(beta_store_index.phase) beta_store_index = advance(beta_store_index, cfg.smem_beta_stages) @@ -939,18 +939,18 @@ def _gate_beta_warp( pos = lidx + col * cfg.threads_per_warp if cute.elem_less(st_offset + pos, write_end): gBeta_st[pos] = sBeta[pos, 0, b_st_idx] - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) @cute.jit -def _mma_warp( +def mma_warp( cfg, total_tiles, bidx, num_ctas, cu_seqlens, mWorkItems, - tmem_hold, + tmem_base_slot, sQ, sQ_trans, sK, @@ -959,62 +959,60 @@ def _mma_warp( sV_kmaj, sdO, sdO_kmaj, - sS, - sS_kmaj, + sState, + sState_kmaj, sAinv, sAinv_trans, sQk, sQk_trans, sDa, sDa_trans, - sdH, + sDstate, sDm, sDm_trans, - sdV, sdV_kmaj, sSched, bars, ): - """MMA (UMMA issuer) warp role (warp 8): persistent scheduler loop + - per-chunk (BACKWARD order) issue of the full GEMM pipeline (see the - module docstring for the per-chunk MMA order); owns the TMEM lifecycle - (alloc up front, dealloc once CG0 and CG1 both signal mb_tmem_done).""" + """MMA issuer role (warp 8): persistent scheduler loop issuing every + tcgen05 GEMM.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) - kk_read = PipelineState.start(phase=0) - dk_total_free = PipelineState.start(phase=1) - du_scale_read = PipelineState.start(phase=0) - dk_scale_read = PipelineState.start(phase=0) - dk_attn_read = PipelineState.start(phase=0) - dh_acc_index = PipelineState.start(phase=0 if cfg.use_dht else 1) + kk_acc_index = PipelineState.start(phase=0) + dk_total_index = PipelineState.start(phase=1) + du_scale_index = PipelineState.start(phase=0) + dk_scale_index = PipelineState.start(phase=0) + dk_attn_index = PipelineState.start(phase=0) + dstate_acc_index = PipelineState.start(phase=0 if cfg.use_dstate_in else 1) k_index = PipelineState.start(phase=0) q_index = PipelineState.start(phase=0) - s_index = PipelineState.start(phase=0) + state_index = PipelineState.start(phase=0) v_index = PipelineState.start(phase=0) ainv_index = PipelineState.start(phase=0) do_index = PipelineState.start(phase=0) - y_rdy_index = PipelineState.start(phase=0) + y_ready_index = PipelineState.start(phase=0) u_index = PipelineState.start(phase=0) - da_rdy_index = PipelineState.start(phase=0) - dv_rdy_index = PipelineState.start(phase=0) + da_ready_index = PipelineState.start(phase=0) + dv_ready_index = PipelineState.start(phase=0) dm_index = PipelineState.start(phase=0) - dhs_index = PipelineState.start(phase=0) + dstate_smem_index = PipelineState.start(phase=0) dq_scale_index = PipelineState.start(phase=0) dq_total_index = PipelineState.start(phase=1) qk_index = PipelineState.start(phase=0) - dop_inp_rdy = PipelineState.start(phase=0) - du_inp_rdy = PipelineState.start(phase=0) - dyp_inp_rdy = PipelineState.start(phase=0) - dh_inp_index = PipelineState.start(phase=0) + do_prime_inp_ready = PipelineState.start(phase=0) + du_inp_ready = PipelineState.start(phase=0) + dyp_inp_ready = PipelineState.start(phase=0) + dstate_inp_index = PipelineState.start(phase=0) - nvvm.tcgen05_alloc(tmem_hold, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.tcgen05_alloc(tmem_base_slot, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) nvvm.barrier_cta_sync_aligned( cfg.tmem_alloc_barrier_id, thread_count=cfg.tmem_alloc_barrier_threads, ) - tmem_base = tmem_hold.load() + tmem_base = tmem_base_slot.load() - # ---- chunk-invariant GEMM descriptors ------------------------------ + # ---- chunk-invariant GEMM descriptors ---------------------------------------- bpe = cfg.io_dtype.width // 8 idesc_qk = nvvm.Tcgen05InstrDesc.build( c_dtype=cutlass.Float32, @@ -1038,20 +1036,20 @@ def _mma_warp( tmem_shared_acc_col = tmem_base + cfg.tmem_shared_acc_offset tmem_shared_inp_col = tmem_base + cfg.tmem_shared_inp_offset SHARED_INP_STAGE_COLS = cfg.b_t // 2 - tmem_dop_col = tmem_shared_inp_col + tmem_do_prime_col = tmem_shared_inp_col tmem_du_col = tmem_shared_inp_col + SHARED_INP_STAGE_COLS tmem_dyp_col = tmem_du_col ACC_STAGE_COLS = cfg.b_t tmem_acc_a = tmem_shared_acc_col tmem_acc_b = tmem_shared_acc_col + ACC_STAGE_COLS tmem_kk_col = tmem_acc_a - tmem_ks_col = tmem_acc_a + tmem_k_state_col = tmem_acc_a tmem_dy_col = tmem_acc_a tmem_dm_core_col = tmem_acc_a tmem_a_col = tmem_acc_b tmem_u_col = tmem_acc_b tmem_da_col = tmem_acc_b - tmem_dk_spath_col = tmem_shared_inp_col + tmem_dk_state_path_col = tmem_shared_inp_col idesc_dv = nvvm.Tcgen05InstrDesc.build( c_dtype=cutlass.Float32, @@ -1073,11 +1071,11 @@ def _mma_warp( idesc=idesc_dv, kind=nvvm.Tcgen05MMAKind.F16, ) - tmem_dh_inp_col = tmem_base + cfg.tmem_dh_inp_offset - tmem_dvdk_col = tmem_base + cfg.tmem_dvdk_offset - DH_INP_STAGE_COLS = cfg.d_k // 2 + tmem_dstate_inp_col = tmem_base + cfg.tmem_dstate_inp_offset + tmem_dvdk_acc_col = tmem_base + cfg.tmem_dvdk_acc_offset + DSTATE_INP_STAGE_COLS = cfg.d_k // 2 - idesc_ks = nvvm.Tcgen05InstrDesc.build( + idesc_k_state = nvvm.Tcgen05InstrDesc.build( c_dtype=cutlass.Float32, a_dtype=cfg.io_dtype, b_dtype=cfg.io_dtype, @@ -1085,7 +1083,7 @@ def _mma_warp( m_dim=cfg.d_v, a_major=1, ) - bmm_ks_desc = MmaDesc( + bmm_k_state_desc = MmaDesc( M=cfg.d_v, N=cfg.b_t, K=cfg.d_k, @@ -1095,7 +1093,7 @@ def _mma_warp( btranspose=False, atranspose=True, cta_group=1, - idesc=idesc_ks, + idesc=idesc_k_state, kind=nvvm.Tcgen05MMAKind.F16, ) @@ -1121,7 +1119,7 @@ def _mma_warp( idesc=idesc_du, kind=nvvm.Tcgen05MMAKind.F16, ) - idesc_dh = nvvm.Tcgen05InstrDesc.build( + idesc_dstate_upd = nvvm.Tcgen05InstrDesc.build( c_dtype=cutlass.Float32, a_dtype=cfg.io_dtype, b_dtype=cfg.io_dtype, @@ -1129,7 +1127,7 @@ def _mma_warp( m_dim=cfg.d_k, b_major=1, ) - bmm_dh_desc = MmaDesc( + bmm_dstate_upd_desc = MmaDesc( M=cfg.d_k, N=cfg.d_v, K=cfg.b_t, @@ -1139,15 +1137,15 @@ def _mma_warp( btranspose=True, atranspose=False, cta_group=1, - idesc=idesc_dh, + idesc=idesc_dstate_upd, kind=nvvm.Tcgen05MMAKind.F16, ) SHARED_INP_STAGE_COLS = cfg.b_t // 2 - tmem_dop_col = tmem_shared_inp_col + tmem_do_prime_col = tmem_shared_inp_col tmem_du_col = tmem_shared_inp_col + SHARED_INP_STAGE_COLS tmem_dyp_col = tmem_du_col tmem_shared_inp_col = tmem_base + cfg.tmem_shared_inp_offset - tmem_dh_col = tmem_base + cfg.tmem_dh_offset + tmem_dstate_acc_col = tmem_base + cfg.tmem_dstate_acc_offset tmem_y_col = tmem_base + cfg.tmem_y_offset idesc_dy = nvvm.Tcgen05InstrDesc.build( @@ -1211,14 +1209,14 @@ def _mma_warp( idesc=idesc_u, kind=nvvm.Tcgen05MMAKind.F16, ) - idesc_dstate = nvvm.Tcgen05InstrDesc.build( + idesc_dqdk_inter = nvvm.Tcgen05InstrDesc.build( c_dtype=cutlass.Float32, a_dtype=cfg.io_dtype, b_dtype=cfg.io_dtype, n_dim=cfg.b_t, m_dim=cfg.d_k, ) - bmm_dstate_desc = MmaDesc( + bmm_dqdk_inter_desc = MmaDesc( M=cfg.d_k, N=cfg.b_t, K=cfg.d_v, @@ -1228,7 +1226,7 @@ def _mma_warp( btranspose=False, atranspose=False, cta_group=1, - idesc=idesc_dstate, + idesc=idesc_dqdk_inter, kind=nvvm.Tcgen05MMAKind.F16, ) idesc_dka = nvvm.Tcgen05InstrDesc.build( @@ -1275,673 +1273,400 @@ def _mma_warp( kind=nvvm.Tcgen05MMAKind.F16, ) + do_prime_inp_ptr = nvvm.make_tmem_ptr(tmem_do_prime_col, cutlass.Int8) + y_inp_ptr = nvvm.make_tmem_ptr(tmem_y_col, cutlass.Int8) + du_inp_ptr = nvvm.make_tmem_ptr(tmem_du_col, cutlass.Int8) + dyp_inp_ptr = nvvm.make_tmem_ptr(tmem_dyp_col, cutlass.Int8) + kk_acc_ptr = nvvm.make_tmem_ptr(tmem_kk_col, cutlass.Float32) + a_acc_ptr = nvvm.make_tmem_ptr(tmem_a_col, cutlass.Float32) + k_state_acc_ptr = nvvm.make_tmem_ptr(tmem_k_state_col, cutlass.Float32) + u_acc_ptr = nvvm.make_tmem_ptr(tmem_u_col, cutlass.Float32) + dy_acc_ptr = nvvm.make_tmem_ptr(tmem_dy_col, cutlass.Float32) + da_acc_ptr = nvvm.make_tmem_ptr(tmem_da_col, cutlass.Float32) + dm_core_acc_ptr = nvvm.make_tmem_ptr(tmem_dm_core_col, cutlass.Float32) + dk_state_path_acc_ptr = nvvm.make_tmem_ptr(tmem_dk_state_path_col, cutlass.Float32) + dstate_acc_ptr = nvvm.make_tmem_ptr(tmem_dstate_acc_col, cutlass.Float32) + dq_acc_ptr = nvvm.make_tmem_ptr(tmem_dstate_inp_col, cutlass.Float32) + dvdk_acc_ptr = nvvm.make_tmem_ptr(tmem_dvdk_acc_col, cutlass.Float32) + + # ---- warp-top descriptors (1-stage tiles are loop-constant; K advances) ---- + d_q0 = sQ[0].desc() + d_k0 = sK[0].desc() + d_k_trans0 = sK_trans[0].desc() + d_q_trans0 = sQ_trans[0].desc() + d_do0 = sdO[0].desc() + d_do_kmaj0 = sdO_kmaj[0].desc() + d_state0 = sState[0].desc() + d_state_kmaj0 = sState_kmaj[0].desc() + d_ainv0 = sAinv[0].desc() + d_ainv_trans0 = sAinv_trans[0].desc() + d_qk_trans0 = sQk_trans[0].desc() + d_v_kmaj0 = sV_kmaj[0].desc() + d_dv_kmaj0 = sdV_kmaj[0].desc() + d_da0 = sDa[0].desc() + d_da_trans0 = sDa_trans[0].desc() + d_dm0 = sDm[0].desc() + d_dm_trans0 = sDm_trans[0].desc() + d_dstate0 = sDstate[0].desc() + K_STAGE_BYTES = (cfg.k_cosize // cfg.smem_k_stages) * (cfg.io_dtype.width // 8) + sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) - S_MIN = 0 if cfg.use_initial_state else 1 - SFIRST_MIN = 1 if cfg.use_initial_state else 2 + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 while tile_idx < total_tiles: batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) - sk_nt = cend - wstart - - # ---- first backward chunk (c = NT-1): no dH yet; with a dht seed - # every chunk takes the steady path ---- - if cutlass.const_expr(not cfg.use_dht): - if sk_nt > 0: - k_idx = k_index.idx - bars.mb_k_ready[k_idx].wait(k_index.phase) - k_index = advance(k_index, cfg.smem_k_stages) - - desc_k = sK[k_idx].desc() - mma_ss( - bmm_qk_desc, - desc_k, - desc_k, - nvvm.make_tmem_ptr(tmem_kk_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_kk_acc_ready[0].arrive(cta_group=1) - - q_idx = q_index.idx - bars.mb_q_ready[q_idx].wait(q_index.phase) - q_index = advance(q_index, cfg.smem_q_stages) - - desc_q = sQ[q_idx].desc() - mma_ss( - bmm_qk_desc, - desc_q, - desc_k, - nvvm.make_tmem_ptr(tmem_a_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_a_acc_ready[0].arrive(cta_group=1) - - # ---- ks (K @ S) first, then dQ inter (S @ dO) ---------------------- - s_idx = s_index.idx - if cend >= SFIRST_MIN: - bars.mb_s_ready[s_idx].wait(s_index.phase) - s_index = advance(s_index, cfg.smem_s_stages) - bars.mb_kk_acc_done[0].wait(kk_read.phase) - kk_read = advance(kk_read, 1) - desc_s = sS[s_idx].desc() - mma_ss( - bmm_ks_desc, - desc_s, - desc_k, - nvvm.make_tmem_ptr(tmem_ks_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_ks_acc_ready[0].arrive(cta_group=1) - do_idx = do_index.idx - bars.mb_do_ready[do_idx].wait(do_index.phase) - do_index = advance(do_index, cfg.smem_do_stages) - if cend >= SFIRST_MIN: - bars.mb_dq_acc_total_done[0].wait(dq_total_index.phase) - dq_total_index = advance(dq_total_index, 1) - desc_s_k = sS_kmaj[s_idx].desc() - desc_do_k2 = sdO_kmaj[do_idx].desc() - mma_ss( - bmm_dstate_desc, - desc_s_k, - desc_do_k2, - nvvm.make_tmem_ptr(tmem_dh_inp_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_dq_acc_scale_ready[0].arrive(cta_group=1) - - # ---- dU intra: dO^T @ A -> the dV/dK slot ------------------ - du_qk_idx = qk_index.idx - bars.mb_qk_ready[du_qk_idx].wait(qk_index.phase) - qk_index = advance(qk_index, cfg.smem_qk_stages) - bars.mb_dk_total_done[0].wait(dk_total_free.phase) - dk_total_free = advance(dk_total_free, 1) - - desc_do_mn = sdO[do_idx].desc() - desc_qk_t = sQk_trans[du_qk_idx].desc() - mma_ss( - bmm_du_desc, - desc_do_mn, - desc_qk_t, - nvvm.make_tmem_ptr(tmem_dvdk_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_du_total_ready[0].arrive(cta_group=1) - - # ---- dH init: dO'^T @ Q ------------------------------------- - bars.mb_dop_inp_ready[0].wait(dop_inp_rdy.phase) - dop_inp_rdy = advance(dop_inp_rdy, 1) - dh_idx = dh_acc_index.idx - bars.mb_dh_acc_done[dh_idx].wait(dh_acc_index.phase) - dh_acc_index = advance(dh_acc_index, cfg.tmem_dh_acc_stages) - - do_a_ptr = nvvm.make_tmem_ptr( - tmem_dop_col, - cutlass.Int8, - ) - desc_q_t = sQ_trans[q_idx].desc() - mma_ts( - bmm_dh_desc, - do_a_ptr, - desc_q_t, - nvvm.make_tmem_ptr(tmem_dh_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_dop_inp_done[0].arrive(cta_group=1) - - # ---- Y operand acquire: y_ready = CG1's delta -------------- - bars.mb_y_ready[0].wait(y_rdy_index.phase) - y_rdy_index = advance(y_rdy_index, 1) - v_idx = v_index.idx - v_index = advance(v_index, cfg.smem_v_stages) - - # ---- U recompute: U^T = Y^T @ T^T -> shared acc ------------ - ainv_idx = ainv_index.idx - bars.mb_ainv_ready[ainv_idx].wait(ainv_index.phase) - ainv_index = advance(ainv_index, cfg.smem_ainv_stages) - - y_a_ptr = nvvm.make_tmem_ptr(tmem_y_col, cutlass.Int8) - desc_t_plain = sAinv[ainv_idx].desc() - mma_ts( - bmm_u_desc, - y_a_ptr, - desc_t_plain, - nvvm.make_tmem_ptr(tmem_u_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_u_acc_ready[0].arrive(cta_group=1) - - # ---- dY: dU^T(TMEM f16) @ T --------------------------------- - bars.mb_du_inp_ready[0].wait(du_inp_rdy.phase) - du_inp_rdy = advance(du_inp_rdy, 1) - - du_a_ptr = nvvm.make_tmem_ptr( - tmem_du_col, - cutlass.Int8, - ) - desc_ainv_t = sAinv_trans[ainv_idx].desc() - mma_ts( - bmm_dy_desc, - du_a_ptr, - desc_ainv_t, - nvvm.make_tmem_ptr(tmem_dy_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_dy_acc_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_ainv_done[ainv_idx].arrive(cta_group=1) - - # ---- dA_eff: dO @ U^T -> shared acc ------------------------- - bars.mb_u_ready[0].wait(u_index.phase) - u_index = advance(u_index, 1) + num_item_chunks = cend - wstart - desc_do_k = sdO_kmaj[do_idx].desc() - desc_u = sV_kmaj[v_idx].desc() - mma_ss( - bmm_da_desc, - desc_do_k, - desc_u, - nvvm.make_tmem_ptr(tmem_da_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_da_acc_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_do_mma_done[do_idx].arrive(cta_group=1) - - # ---- dM core: dY @ U^T -------------------------------------- - dv_stg_x_idx = dv_rdy_index.idx - bars.mb_dv_tmastg_ready[dv_stg_x_idx].wait(dv_rdy_index.phase) - dv_rdy_index = advance(dv_rdy_index, cfg.smem_dv_stages) - - desc_dy_dm = sdV_kmaj[0].desc() - desc_u_dm = sV_kmaj[v_idx].desc() - mma_ss( - bmm_da_desc, - desc_dy_dm, - desc_u_dm, - nvvm.make_tmem_ptr(tmem_dm_core_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_dm_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_v_mma_done[v_idx].arrive(cta_group=1) - - # ---- dH ds-term: dH += dY'^T @ K ---------------------------- - bars.mb_dyp_inp_ready[0].wait(dyp_inp_rdy.phase) - dyp_inp_rdy = advance(dyp_inp_rdy, 1) - - dyp_a_ptr = nvvm.make_tmem_ptr( - tmem_dyp_col, - cutlass.Int8, - ) - desc_k_t = sK_trans[k_idx].desc() - mma_ts( - bmm_dh_desc, - dyp_a_ptr, - desc_k_t, - nvvm.make_tmem_ptr(tmem_dh_col, cutlass.Float32), - accumulate=True, - ) - if nvvm.elect_sync(): - bars.mb_dh_acc_ready[dh_idx].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_dyp_inp_done[0].arrive(cta_group=1) - - # ---- dK attn: Q^T @ dA(sDa) -> the dV/dK slot -------------- - bars.mb_da_ready[0].wait(da_rdy_index.phase) - da_rdy_index = advance(da_rdy_index, 1) - - desc_q_mn = sQ_trans[q_idx].desc() - desc_da_t = sDa_trans[0].desc() - mma_ss( - bmm_dka_desc, - desc_q_mn, - desc_da_t, - nvvm.make_tmem_ptr(tmem_dvdk_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_dk_attn_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_q_mma_done[q_idx].arrive(cta_group=1) - - # ---- dQ attn: K^T @ dA onto the rescaled dQ acc ------------------- - if cend >= SFIRST_MIN: - bars.mb_dq_acc_scale_done[0].wait(dq_scale_index.phase) - dq_scale_index = advance(dq_scale_index, 1) - if cend < SFIRST_MIN: - bars.mb_dq_acc_total_done[0].wait(dq_total_index.phase) - dq_total_index = advance(dq_total_index, 1) - desc_k_mn = sK_trans[k_idx].desc() - desc_da_p = sDa[0].desc() - if cend >= SFIRST_MIN: - mma_ss( - bmm_dqa_desc, - desc_k_mn, - desc_da_p, - nvvm.make_tmem_ptr(tmem_dh_inp_col, cutlass.Float32), - accumulate=True, - ) - if cend < SFIRST_MIN: - mma_ss( - bmm_dqa_desc, - desc_k_mn, - desc_da_p, - nvvm.make_tmem_ptr(tmem_dh_inp_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_dq_acc_total_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_qk_done[du_qk_idx].arrive(cta_group=1) - - # ---- dK s-path: S @ dY^T -> shared acc, before the dM-terms -------- - if cend >= SFIRST_MIN: - desc_s_k5 = sS_kmaj[s_idx].desc() - desc_dy_k5 = sdV_kmaj[0].desc() - mma_ss( - bmm_dstate_desc, - desc_s_k5, - desc_dy_k5, - nvvm.make_tmem_ptr(tmem_dk_spath_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_dk_spath_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_s_done[s_idx].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_sdv_done[0].arrive(cta_group=1) - - # ---- dK dM-terms: K^T @ dM^T + K^T @ dM onto the slot ------ - bars.mb_dm_done[0].wait(dm_index.phase) - dm_index = advance(dm_index, 1) - bars.mb_dk_attn_done[0].wait(dk_attn_read.phase) - dk_attn_read = advance(dk_attn_read, 1) - desc_k_mn2 = sK_trans[k_idx].desc() - desc_dm_t = sDm_trans[0].desc() - mma_ss( - bmm_dka_desc, - desc_k_mn2, - desc_dm_t, - nvvm.make_tmem_ptr(tmem_dvdk_col, cutlass.Float32), - accumulate=True, - ) - desc_dm_p = sDm[0].desc() - mma_ss( - bmm_dqa_desc, - desc_k_mn2, - desc_dm_p, - nvvm.make_tmem_ptr(tmem_dvdk_col, cutlass.Float32), - accumulate=True, - ) - if nvvm.elect_sync(): - bars.mb_dk_total_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_k_mma_done[k_idx].arrive(cta_group=1) - - # ---- chunks NT-2 .. 0 (backward): full body ---------------------- - for rev_idx in cutlass.range(0 if cfg.use_dht else 1, sk_nt): + # ---- chunks NT-2 .. 0 (backward): full body ------------------------------ + for rev_idx in cutlass.range(num_item_chunks): chunk_idx = cend - 1 - rev_idx - # ---- kk: K @ K^T -> shared acc ------------------------------ + have_dstate = cutlass.Boolean(True) if cutlass.const_expr(cfg.use_dstate_in) else rev_idx > 0 + + # ---- KK = K(S) @ K^T ------------------------------------------------- k_idx = k_index.idx bars.mb_k_ready[k_idx].wait(k_index.phase) k_index = advance(k_index, cfg.smem_k_stages) - desc_k = sK[k_idx].desc() + desc_k = d_k0.advance_start_address(k_idx * K_STAGE_BYTES) mma_ss( bmm_qk_desc, desc_k, desc_k, - nvvm.make_tmem_ptr(tmem_kk_col, cutlass.Float32), + kk_acc_ptr, accumulate=False, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_kk_acc_ready[0].arrive(cta_group=1) - # ---- qk: Q @ K^T -> shared acc ------------------------------ + # ---- QK = Q(S) @ K^T ------------------------------------------------- q_idx = q_index.idx bars.mb_q_ready[q_idx].wait(q_index.phase) q_index = advance(q_index, cfg.smem_q_stages) - desc_q = sQ[q_idx].desc() + desc_q = d_q0 mma_ss( bmm_qk_desc, desc_q, desc_k, - nvvm.make_tmem_ptr(tmem_a_col, cutlass.Float32), + a_acc_ptr, accumulate=False, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_a_acc_ready[0].arrive(cta_group=1) - # ---- ks (K @ S): hoisted above dV inter so CG1's delta chain - # is not serialized behind the prev chunk's dK readout ---------- - s_idx = s_index.idx - if chunk_idx >= S_MIN: - bars.mb_s_ready[s_idx].wait(s_index.phase) - s_index = advance(s_index, cfg.smem_s_stages) - bars.mb_kk_acc_done[0].wait(kk_read.phase) - kk_read = advance(kk_read, 1) - desc_s = sS[s_idx].desc() + # ---- k_state = state^T(S) @ K^T ----------------------------------------------- + state_idx = state_index.idx + if chunk_idx >= FIRST_STATE_CHUNK: + bars.mb_state_ready[state_idx].wait(state_index.phase) + state_index = advance(state_index, cfg.smem_state_stages) + bars.mb_kk_acc_done[0].wait(kk_acc_index.phase) + kk_acc_index = advance(kk_acc_index, 1) + desc_state = d_state0 mma_ss( - bmm_ks_desc, - desc_s, + bmm_k_state_desc, + desc_state, desc_k, - nvvm.make_tmem_ptr(tmem_ks_col, cutlass.Float32), + k_state_acc_ptr, accumulate=False, ) - if nvvm.elect_sync(): - bars.mb_ks_acc_ready[0].arrive(cta_group=1) - - # ---- dV inter: dH^T @ K -> the dV/dK slot ------------------- - dh_inp_idx = dh_inp_index.idx - bars.mb_dh_inp_ready[dh_inp_idx].wait(dh_inp_index.phase) - dh_inp_index = advance(dh_inp_index, cfg.tmem_dh_inp_stages) - bars.mb_dk_total_done[0].wait(dk_total_free.phase) - dk_total_free = advance(dk_total_free, 1) - - dh_a_ptr = nvvm.make_tmem_ptr(tmem_dh_inp_col + dh_inp_idx * DH_INP_STAGE_COLS, cutlass.Int8) - mma_ts( - bmm_dv_desc, - dh_a_ptr, - desc_k, - nvvm.make_tmem_ptr(tmem_dvdk_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_du_scale_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_dh_inp_done[dh_inp_idx].arrive(cta_group=1) + if elect_one: + bars.mb_k_state_acc_ready[0].arrive(cta_group=1) + + # ---- dV inter = dstate^T(T) @ K ------------------------------------------ + dstate_inp_idx = dstate_inp_index.idx + if have_dstate: + bars.mb_dstate_inp_ready[dstate_inp_idx].wait(dstate_inp_index.phase) + dstate_inp_index = advance(dstate_inp_index, cfg.tmem_dstate_inp_stages) + bars.mb_dk_total_done[0].wait(dk_total_index.phase) + dk_total_index = advance(dk_total_index, 1) + + if have_dstate: + dstate_a_ptr = nvvm.make_tmem_ptr(tmem_dstate_inp_col + dstate_inp_idx * DSTATE_INP_STAGE_COLS, cutlass.Int8) + for sub in cutlass.range_constexpr(bmm_dv_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_dv_desc.sps_B): + mma_ts_step( + bmm_dv_desc, + dstate_a_ptr.subview(sub * bmm_dv_desc.sps_B * bmm_dv_desc.tmem_advance_A), + desc_k + sub * (bmm_dv_desc.smem_subtile_B >> 4), + dvdk_acc_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + if elect_one: + bars.mb_du_scale_ready[0].arrive(cta_group=1) + bars.mb_dstate_inp_done[dstate_inp_idx].arrive(cta_group=1) do_idx = do_index.idx bars.mb_do_ready[do_idx].wait(do_index.phase) do_index = advance(do_index, cfg.smem_do_stages) - if chunk_idx >= S_MIN: + if chunk_idx >= FIRST_STATE_CHUNK: bars.mb_dq_acc_total_done[0].wait(dq_total_index.phase) dq_total_index = advance(dq_total_index, 1) - desc_s_k = sS_kmaj[s_idx].desc() - desc_do_k2 = sdO_kmaj[do_idx].desc() + desc_state_kmaj_dq_inter = d_state_kmaj0 + desc_do_kmaj_dq_inter = d_do_kmaj0 mma_ss( - bmm_dstate_desc, - desc_s_k, - desc_do_k2, - nvvm.make_tmem_ptr(tmem_dh_inp_col, cutlass.Float32), + bmm_dqdk_inter_desc, + desc_state_kmaj_dq_inter, + desc_do_kmaj_dq_inter, + dq_acc_ptr, accumulate=False, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dq_acc_scale_ready[0].arrive(cta_group=1) - # ---- dU intra: dO^T @ A accumulated onto the scaled dV inter ---- + # ---- dU intra += dO^T(S) @ A ----------------------------------------- du_qk_idx = qk_index.idx bars.mb_qk_ready[du_qk_idx].wait(qk_index.phase) qk_index = advance(qk_index, cfg.smem_qk_stages) - bars.mb_du_scale_done[0].wait(du_scale_read.phase) - du_scale_read = advance(du_scale_read, 1) + if have_dstate: + bars.mb_du_scale_done[0].wait(du_scale_index.phase) + du_scale_index = advance(du_scale_index, 1) - desc_do_mn = sdO[do_idx].desc() - desc_qk_t = sQk_trans[du_qk_idx].desc() + desc_do_mnmaj = d_do0 + desc_qk_t = d_qk_trans0 mma_ss( bmm_du_desc, - desc_do_mn, + desc_do_mnmaj, desc_qk_t, - nvvm.make_tmem_ptr(tmem_dvdk_col, cutlass.Float32), - accumulate=True, + dvdk_acc_ptr, + accumulate=have_dstate, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_du_total_ready[0].arrive(cta_group=1) - # ---- dH update: dO'^T @ Q ----------------------------------- - bars.mb_dop_inp_ready[0].wait(dop_inp_rdy.phase) - dop_inp_rdy = advance(dop_inp_rdy, 1) - dh_idx = dh_acc_index.idx - bars.mb_dh_acc_done[dh_idx].wait(dh_acc_index.phase) - dh_acc_index = advance(dh_acc_index, cfg.tmem_dh_acc_stages) - - do_a_ptr = nvvm.make_tmem_ptr( - tmem_dop_col, - cutlass.Int8, - ) - desc_q_t = sQ_trans[q_idx].desc() - mma_ts( - bmm_dh_desc, - do_a_ptr, - desc_q_t, - nvvm.make_tmem_ptr(tmem_dh_col, cutlass.Float32), - accumulate=True, - ) - if nvvm.elect_sync(): - bars.mb_dop_inp_done[0].arrive(cta_group=1) - - # ---- Y operand acquire: y_ready = CG1's delta -------------- - bars.mb_y_ready[0].wait(y_rdy_index.phase) - y_rdy_index = advance(y_rdy_index, 1) - v_idx = v_index.idx - v_index = advance(v_index, cfg.smem_v_stages) + # ---- dstate update += dO'^T(T) @ Q --------------------------------------- + bars.mb_do_prime_inp_ready[0].wait(do_prime_inp_ready.phase) + do_prime_inp_ready = advance(do_prime_inp_ready, 1) + dstate_idx = dstate_acc_index.idx + bars.mb_dstate_acc_done[dstate_idx].wait(dstate_acc_index.phase) + dstate_acc_index = advance(dstate_acc_index, cfg.tmem_dstate_acc_stages) + + desc_q_t = d_q_trans0 + for sub in cutlass.range_constexpr(bmm_dstate_upd_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_dstate_upd_desc.sps_B): + mma_ts_step( + bmm_dstate_upd_desc, + do_prime_inp_ptr.subview(sub * bmm_dstate_upd_desc.sps_B * bmm_dstate_upd_desc.tmem_advance_A), + desc_q_t + sub * (bmm_dstate_upd_desc.smem_subtile_B >> 4), + dstate_acc_ptr, + k, + cutlass.Boolean(True) if cutlass.const_expr(sub + k > 0) else have_dstate, + ) + if elect_one: + bars.mb_do_prime_inp_done[0].arrive(cta_group=1) - # ---- U recompute: U^T = Y^T @ T^T -> shared acc ------------ + # ---- U^T recompute = Y^T(T) @ T^T ------------------------------------ ainv_idx = ainv_index.idx bars.mb_ainv_ready[ainv_idx].wait(ainv_index.phase) ainv_index = advance(ainv_index, cfg.smem_ainv_stages) + bars.mb_y_ready[0].wait(y_ready_index.phase) + y_ready_index = advance(y_ready_index, 1) + v_idx = v_index.idx + v_index = advance(v_index, cfg.smem_v_stages) - y_a_ptr = nvvm.make_tmem_ptr(tmem_y_col, cutlass.Int8) - desc_t_plain = sAinv[ainv_idx].desc() - mma_ts( - bmm_u_desc, - y_a_ptr, - desc_t_plain, - nvvm.make_tmem_ptr(tmem_u_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): + desc_ainv = d_ainv0 + for sub in cutlass.range_constexpr(bmm_u_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_u_desc.sps_B): + mma_ts_step( + bmm_u_desc, + y_inp_ptr.subview(sub * bmm_u_desc.sps_B * bmm_u_desc.tmem_advance_A), + desc_ainv + sub * (bmm_u_desc.smem_subtile_B >> 4), + u_acc_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + if elect_one: bars.mb_u_acc_ready[0].arrive(cta_group=1) - # ---- dY: dU^T(TMEM f16) @ T --------------------------------- - bars.mb_du_inp_ready[0].wait(du_inp_rdy.phase) - du_inp_rdy = advance(du_inp_rdy, 1) - - du_a_ptr = nvvm.make_tmem_ptr( - tmem_du_col, - cutlass.Int8, - ) - desc_ainv_t = sAinv_trans[ainv_idx].desc() - mma_ts( - bmm_dy_desc, - du_a_ptr, - desc_ainv_t, - nvvm.make_tmem_ptr(tmem_dy_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): + # ---- dY = dU^T(T) @ T ------------------------------------------------ + bars.mb_du_inp_ready[0].wait(du_inp_ready.phase) + du_inp_ready = advance(du_inp_ready, 1) + + desc_ainv_t = d_ainv_trans0 + for sub in cutlass.range_constexpr(bmm_dy_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_dy_desc.sps_B): + mma_ts_step( + bmm_dy_desc, + du_inp_ptr.subview(sub * bmm_dy_desc.sps_B * bmm_dy_desc.tmem_advance_A), + desc_ainv_t + sub * (bmm_dy_desc.smem_subtile_B >> 4), + dy_acc_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + if elect_one: bars.mb_dy_acc_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): bars.mb_ainv_done[ainv_idx].arrive(cta_group=1) - # ---- dK inter: dH_entry(SMEM) @ U^T -> the dV/dK slot ------- + # ---- dK_inter = dstate_entry(S) @ U^T ------------------------------------ bars.mb_u_ready[0].wait(u_index.phase) u_index = advance(u_index, 1) - bars.mb_dhs_ready[0].wait(dhs_index.phase) - dhs_index = advance(dhs_index, 1) + if have_dstate: + bars.mb_dstate_smem_ready[0].wait(dstate_smem_index.phase) + dstate_smem_index = advance(dstate_smem_index, 1) - desc_dh_k = sdH[0].desc() - desc_u_t = sV_kmaj[v_idx].desc() - mma_ss( - bmm_dstate_desc, - desc_dh_k, - desc_u_t, - nvvm.make_tmem_ptr(tmem_dvdk_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_dk_scale_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_dhs_done[0].arrive(cta_group=1) + desc_dstate_kmaj = d_dstate0 + desc_u_kmaj_dk_inter = d_v_kmaj0 + mma_ss( + bmm_dqdk_inter_desc, + desc_dstate_kmaj, + desc_u_kmaj_dk_inter, + dvdk_acc_ptr, + accumulate=False, + ) + if elect_one: + bars.mb_dk_scale_ready[0].arrive(cta_group=1) + bars.mb_dstate_smem_done[0].arrive(cta_group=1) - # ---- dA_eff: dO @ U^T -> shared acc (U gated at dK-inter) ---- - desc_do_k = sdO_kmaj[do_idx].desc() - desc_u = sV_kmaj[v_idx].desc() + # ---- dA_eff = dO(S) @ U^T -------------------------------------------- + desc_do_kmaj_da = d_do_kmaj0 + desc_u_kmaj_da = d_v_kmaj0 mma_ss( bmm_da_desc, - desc_do_k, - desc_u, - nvvm.make_tmem_ptr(tmem_da_col, cutlass.Float32), + desc_do_kmaj_da, + desc_u_kmaj_da, + da_acc_ptr, accumulate=False, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_da_acc_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): bars.mb_do_mma_done[do_idx].arrive(cta_group=1) - # ---- dM core: dY @ U^T -------------------------------------- - dv_stg_x_idx = dv_rdy_index.idx - bars.mb_dv_tmastg_ready[dv_stg_x_idx].wait(dv_rdy_index.phase) - dv_rdy_index = advance(dv_rdy_index, cfg.smem_dv_stages) + # ---- dM core = dY(S) @ U^T ------------------------------------------- + dv_ready_idx = dv_ready_index.idx + bars.mb_dv_tmastg_ready[dv_ready_idx].wait(dv_ready_index.phase) + dv_ready_index = advance(dv_ready_index, cfg.smem_dv_stages) - desc_dy_dm = sdV_kmaj[0].desc() - desc_u_dm = sV_kmaj[v_idx].desc() + desc_dy_kmaj_dm = d_dv_kmaj0 + desc_u_kmaj_dm = d_v_kmaj0 mma_ss( bmm_da_desc, - desc_dy_dm, - desc_u_dm, - nvvm.make_tmem_ptr(tmem_dm_core_col, cutlass.Float32), + desc_dy_kmaj_dm, + desc_u_kmaj_dm, + dm_core_acc_ptr, accumulate=False, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dm_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): bars.mb_v_mma_done[v_idx].arrive(cta_group=1) - # ---- dH ds-term: dH += dY'^T @ K ---------------------------- - bars.mb_dyp_inp_ready[0].wait(dyp_inp_rdy.phase) - dyp_inp_rdy = advance(dyp_inp_rdy, 1) - - dyp_a_ptr = nvvm.make_tmem_ptr( - tmem_dyp_col, - cutlass.Int8, - ) - desc_k_t = sK_trans[k_idx].desc() - mma_ts( - bmm_dh_desc, - dyp_a_ptr, - desc_k_t, - nvvm.make_tmem_ptr(tmem_dh_col, cutlass.Float32), - accumulate=True, - ) - if nvvm.elect_sync(): - bars.mb_dh_acc_ready[dh_idx].arrive(cta_group=1) - if nvvm.elect_sync(): + # ---- dstate update-term += dY'^T(T) @ K -------------------------------------- + bars.mb_dyp_inp_ready[0].wait(dyp_inp_ready.phase) + dyp_inp_ready = advance(dyp_inp_ready, 1) + + desc_k_t = d_k_trans0.advance_start_address(k_idx * K_STAGE_BYTES) + for sub in cutlass.range_constexpr(bmm_dstate_upd_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_dstate_upd_desc.sps_B): + mma_ts_step( + bmm_dstate_upd_desc, + dyp_inp_ptr.subview(sub * bmm_dstate_upd_desc.sps_B * bmm_dstate_upd_desc.tmem_advance_A), + desc_k_t + sub * (bmm_dstate_upd_desc.smem_subtile_B >> 4), + dstate_acc_ptr, + k, + cutlass.Boolean(True), + ) + if elect_one: + bars.mb_dstate_acc_ready[dstate_idx].arrive(cta_group=1) bars.mb_dyp_inp_done[0].arrive(cta_group=1) - # ---- dK attn: Q^T @ dA(sDa) onto the rescaled dK inter ------ - bars.mb_da_ready[0].wait(da_rdy_index.phase) - da_rdy_index = advance(da_rdy_index, 1) - bars.mb_dk_scale_done[0].wait(dk_scale_read.phase) - dk_scale_read = advance(dk_scale_read, 1) + # ---- dK attn += Q^T(S) @ dA ------------------------------------------ + bars.mb_da_ready[0].wait(da_ready_index.phase) + da_ready_index = advance(da_ready_index, 1) + if have_dstate: + bars.mb_dk_scale_done[0].wait(dk_scale_index.phase) + dk_scale_index = advance(dk_scale_index, 1) - desc_q_mn = sQ_trans[q_idx].desc() - desc_da_t = sDa_trans[0].desc() + desc_q_mnmaj_dk_attn = d_q_trans0 + desc_da_t = d_da_trans0 mma_ss( bmm_dka_desc, - desc_q_mn, + desc_q_mnmaj_dk_attn, desc_da_t, - nvvm.make_tmem_ptr(tmem_dvdk_col, cutlass.Float32), - accumulate=True, + dvdk_acc_ptr, + accumulate=have_dstate, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dk_attn_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): bars.mb_q_mma_done[q_idx].arrive(cta_group=1) - # ---- dQ attn: K^T @ dA onto the rescaled dQ acc ------------------- - if chunk_idx >= S_MIN: + # ---- dQ attn += K^T(S) @ dA ------------------------------------------ + if chunk_idx >= FIRST_STATE_CHUNK: bars.mb_dq_acc_scale_done[0].wait(dq_scale_index.phase) dq_scale_index = advance(dq_scale_index, 1) - if chunk_idx < S_MIN: + if chunk_idx < FIRST_STATE_CHUNK: bars.mb_dq_acc_total_done[0].wait(dq_total_index.phase) dq_total_index = advance(dq_total_index, 1) - desc_k_mn = sK_trans[k_idx].desc() - desc_da_p = sDa[0].desc() - if chunk_idx >= S_MIN: + desc_k_mnmaj_dq_attn = d_k_trans0.advance_start_address(k_idx * K_STAGE_BYTES) + desc_da = d_da0 + if chunk_idx >= FIRST_STATE_CHUNK: mma_ss( bmm_dqa_desc, - desc_k_mn, - desc_da_p, - nvvm.make_tmem_ptr(tmem_dh_inp_col, cutlass.Float32), + desc_k_mnmaj_dq_attn, + desc_da, + dq_acc_ptr, accumulate=True, ) - if chunk_idx < S_MIN: + if chunk_idx < FIRST_STATE_CHUNK: mma_ss( bmm_dqa_desc, - desc_k_mn, - desc_da_p, - nvvm.make_tmem_ptr(tmem_dh_inp_col, cutlass.Float32), + desc_k_mnmaj_dq_attn, + desc_da, + dq_acc_ptr, accumulate=False, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dq_acc_total_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): bars.mb_qk_done[du_qk_idx].arrive(cta_group=1) - # ---- dK s-path: S @ dY^T -> shared acc, before the dM-terms -------- - if chunk_idx >= S_MIN: - desc_s_k5 = sS_kmaj[s_idx].desc() - desc_dy_k5 = sdV_kmaj[0].desc() + # ---- dK state-path = state(S) @ dY^T ----------------------------------------- + if chunk_idx >= FIRST_STATE_CHUNK: + desc_state_kmaj_spath = d_state_kmaj0 + desc_dy_kmaj_spath = d_dv_kmaj0 mma_ss( - bmm_dstate_desc, - desc_s_k5, - desc_dy_k5, - nvvm.make_tmem_ptr(tmem_dk_spath_col, cutlass.Float32), + bmm_dqdk_inter_desc, + desc_state_kmaj_spath, + desc_dy_kmaj_spath, + dk_state_path_acc_ptr, accumulate=False, ) - if nvvm.elect_sync(): - bars.mb_dk_spath_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_s_done[s_idx].arrive(cta_group=1) - if nvvm.elect_sync(): + if elect_one: + bars.mb_dk_state_path_ready[0].arrive(cta_group=1) + bars.mb_state_done[state_idx].arrive(cta_group=1) + if elect_one: bars.mb_sdv_done[0].arrive(cta_group=1) - # ---- dK dM-terms: K^T @ dM^T + K^T @ dM onto the slot ------- + # ---- dK dM-terms += K^T(S) @ dM^T + K^T(S) @ dM ---------------------- bars.mb_dm_done[0].wait(dm_index.phase) dm_index = advance(dm_index, 1) - bars.mb_dk_attn_done[0].wait(dk_attn_read.phase) - dk_attn_read = advance(dk_attn_read, 1) - desc_k_mn2 = sK_trans[k_idx].desc() - desc_dm_t = sDm_trans[0].desc() + bars.mb_dk_attn_done[0].wait(dk_attn_index.phase) + dk_attn_index = advance(dk_attn_index, 1) + desc_k_mnmaj_dm_terms = d_k_trans0.advance_start_address(k_idx * K_STAGE_BYTES) + desc_dm_t = d_dm_trans0 mma_ss( bmm_dka_desc, - desc_k_mn2, + desc_k_mnmaj_dm_terms, desc_dm_t, - nvvm.make_tmem_ptr(tmem_dvdk_col, cutlass.Float32), + dvdk_acc_ptr, accumulate=True, ) - desc_dm_p = sDm[0].desc() + desc_dm = d_dm0 mma_ss( bmm_dqa_desc, - desc_k_mn2, - desc_dm_p, - nvvm.make_tmem_ptr(tmem_dvdk_col, cutlass.Float32), + desc_k_mnmaj_dm_terms, + desc_dm, + dvdk_acc_ptr, accumulate=True, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dk_total_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): bars.mb_k_mma_done[k_idx].arrive(cta_group=1) - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) - bars.mb_dk_total_done[0].wait(dk_total_free.phase) + bars.mb_dk_total_done[0].wait(dk_total_index.phase) bars.mb_tmem_done[0].wait(0) nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) @@ -1953,7 +1678,7 @@ def _mma_warp( @cute.jit -def _tmaldg_warp( +def tmaldg_warp( cfg, total_tiles, bidx, @@ -1964,36 +1689,33 @@ def _tmaldg_warp( sK_raw, sV_raw, sdO_raw, - sS_raw, + sState_raw, desc_q_base, desc_k_base, desc_v_base, desc_do_base, - desc_s_base, - desc_s0_base, + desc_checkpoint_base, + desc_initial_state_base, sSched, mSched, bars, ): - """TMA-LDG warp role (warp 9): persistent tile loop + per-chunk (BACKWARD - order) Q/K/V/dO loads, plus S = H[c-1] for chunks c >= S_MIN (with an - initial state, chunk 0's S comes from the io-dtype S0 descriptors - instead). The per-(b,h) runtime descriptors fold the sequence start and - head, so Q/K/V/dO load at token ``chunk_idx*BT`` and S at the - sequence-local index ``c - 1``.""" + """TMA-LDG warp role (warp 9): persistent tile loop issuing every + Q/K/V/dO/state TMA load.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) q_index = PipelineState.start(phase=1) k_index = PipelineState.start(phase=1) v_index = PipelineState.start(phase=1) do_index = PipelineState.start(phase=1) - s_index = PipelineState.start(phase=1) + state_index = PipelineState.start(phase=1) sched_state = PipelineState.start(phase=1) tile_idx = cutlass.Int32(bidx) - S_MIN = 0 if cfg.use_initial_state else 1 + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 SFIRST_MIN = 1 if cfg.use_initial_state else 2 bpe = cfg.io_dtype.width // 8 - granu = 128 // bpe + granule_elems = 128 // bpe bt = cfg.b_t q_stage_elems = cfg.q_cosize // cfg.smem_q_stages k_stage_elems = cfg.k_cosize // cfg.smem_k_stages @@ -2005,8 +1727,8 @@ def _tmaldg_warp( stride_byte_offset=0, layout=0, tma_loads_per_tile=2, - tma_granu_elems=granu, - tma_subtile_stride_elems=bt * granu, + tma_granu_elems=granule_elems, + tma_subtile_stride_elems=bt * granule_elems, ) sK_tma = SmemTile( base=sK_raw, @@ -2016,8 +1738,8 @@ def _tmaldg_warp( stride_byte_offset=0, layout=0, tma_loads_per_tile=2, - tma_granu_elems=granu, - tma_subtile_stride_elems=bt * granu, + tma_granu_elems=granule_elems, + tma_subtile_stride_elems=bt * granule_elems, ) sV_tma = SmemTile( base=sV_raw, @@ -2027,7 +1749,7 @@ def _tmaldg_warp( stride_byte_offset=0, layout=0, tma_loads_per_tile=2, - tma_granu_elems=granu, + tma_granu_elems=granule_elems, tma_subtile_stride_elems=4096, ) sdO_tma = SmemTile( @@ -2038,13 +1760,13 @@ def _tmaldg_warp( stride_byte_offset=0, layout=0, tma_loads_per_tile=2, - tma_granu_elems=granu, + tma_granu_elems=granule_elems, tma_subtile_stride_elems=4096, ) - sS_tma = SmemTile( - base=sS_raw, - elems_per_stage=(cfg.s_cosize // cfg.smem_s_stages), - stages=cfg.smem_s_stages, + sCheckpoint_tma = SmemTile( + base=sState_raw, + elems_per_stage=(cfg.state_cosize // cfg.smem_state_stages), + stages=cfg.smem_state_stages, leading_byte_offset=0, stride_byte_offset=0, layout=0, @@ -2060,97 +1782,91 @@ def _tmaldg_warp( cfg, tile_idx, cu_seqlens, mWorkItems ) - slot = (batch_idx * heads_out + head_idx) * desc_qwords + head_o = head_idx + head_q = head_idx if cfg.q_ratio == 1 else head_idx // cutlass.Int32(cfg.q_ratio) + head_k = head_idx if cfg.k_ratio == 1 else head_idx // cutlass.Int32(cfg.k_ratio) + head_v = head_idx if cfg.v_ratio == 1 else head_idx // cutlass.Int32(cfg.v_ratio) + slot = batch_idx * desc_qwords desc_q_slot = (desc_q_base + slot).tospace(cutlass.AddressSpace.generic) desc_k_slot = (desc_k_base + slot).tospace(cutlass.AddressSpace.generic) desc_v_slot = (desc_v_base + slot).tospace(cutlass.AddressSpace.generic) desc_do_slot = (desc_do_base + slot).tospace(cutlass.AddressSpace.generic) - desc_s_slot = (desc_s_base + slot).tospace(cutlass.AddressSpace.generic) + desc_checkpoint_slot = (desc_checkpoint_base + slot).tospace(cutlass.AddressSpace.generic) if cutlass.const_expr(cfg.use_initial_state): - desc_s0_slot = (desc_s0_base + slot).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): + desc_initial_state_slot = (desc_initial_state_base + cutlass.Int32(0)).tospace(cutlass.AddressSpace.generic) + if elect_one: tma_tensormap_acquire(desc_q_slot) tma_tensormap_acquire(desc_k_slot) tma_tensormap_acquire(desc_v_slot) tma_tensormap_acquire(desc_do_slot) - tma_tensormap_acquire(desc_s_slot) + tma_tensormap_acquire(desc_checkpoint_slot) if cutlass.const_expr(cfg.use_initial_state): - tma_tensormap_acquire(desc_s0_slot) + tma_tensormap_acquire(desc_initial_state_slot) for rev_idx in cutlass.range(cend - wstart): chunk_idx = cend - 1 - rev_idx tok_coord = chunk_idx * cutlass.Int32(cfg.b_t) - # ---------------------------------------------------------- - # K (B operand of GEMMs 1/2/3, double-buffered) - # ---------------------------------------------------------- + # ---- K load ---------------------------------------------------------- k_idx = k_index.idx bars.mb_k_mma_done[k_idx].wait(k_index.phase) bars.mb_k_cg0_done[k_idx].wait(k_index.phase) k_index = advance(k_index, cfg.smem_k_stages) - if nvvm.elect_sync(): + if elect_one: bars.mb_k_ready[k_idx].arrive(n_bytes=cfg.tma_k_bytes) - k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), tok_coord) + k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, tok_coord) tma_load_tile(sK_tma[k_idx], k_slice, bars.mb_k_ready[k_idx].smem_ptr, acquire=False) - # ---------------------------------------------------------- - # Q (A operand of the qk GEMM, single-buffered) - # ---------------------------------------------------------- + # ---- Q load ---------------------------------------------------------- q_idx = q_index.idx bars.mb_q_mma_done[q_idx].wait(q_index.phase) bars.mb_q_cg1_done[q_idx].wait(q_index.phase) q_index = advance(q_index, cfg.smem_q_stages) - if nvvm.elect_sync(): + if elect_one: bars.mb_q_ready[q_idx].arrive(n_bytes=cfg.tma_q_bytes) - q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), tok_coord) + q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), head_q, tok_coord) tma_load_tile(sQ_tma[q_idx], q_slice, bars.mb_q_ready[q_idx].smem_ptr, acquire=False) - # ---------------------------------------------------------- - # V (transposed [DV, T] descriptor) - # ---------------------------------------------------------- + # ---- V load ---------------------------------------------------------- v_idx = v_index.idx bars.mb_v_mma_done[v_idx].wait(v_index.phase) bars.mb_v_cg1_done[v_idx].wait(v_index.phase) v_index = advance(v_index, cfg.smem_v_stages) - if nvvm.elect_sync(): + if elect_one: bars.mb_v_ready[v_idx].arrive(n_bytes=cfg.tma_v_bytes) - v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), tok_coord) + v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, tok_coord) tma_load_tile(sV_tma[v_idx], v_slice, bars.mb_v_ready[v_idx].smem_ptr, acquire=False) - # ---------------------------------------------------------- - # dO (transposed [DV, T] descriptor, like V) - # ---------------------------------------------------------- + # ---- dO load --------------------------------------------------------- do_idx = do_index.idx bars.mb_do_mma_done[do_idx].wait(do_index.phase) bars.mb_do_cg1_done[do_idx].wait(do_index.phase) do_index = advance(do_index, cfg.smem_do_stages) - if nvvm.elect_sync(): + if elect_one: bars.mb_do_ready[do_idx].arrive(n_bytes=cfg.tma_do_bytes) - do_slice = tma_slice_runtime_desc(desc_do_slot, cutlass.Int32(0), tok_coord) + do_slice = tma_slice_runtime_desc(desc_do_slot, cutlass.Int32(0), head_o, tok_coord) tma_load_tile(sdO_tma[do_idx], do_slice, bars.mb_do_ready[do_idx].smem_ptr, acquire=False) - # ---------------------------------------------------------- - # S = H[c-1] (forward state entering chunk c; S0 for c = 0 - # when an initial state is given, none otherwise) - # ---------------------------------------------------------- - if chunk_idx >= S_MIN: - s_idx = s_index.idx - bars.mb_s_done[s_idx].wait(s_index.phase) - s_index = advance(s_index, cfg.smem_s_stages) - if nvvm.elect_sync(): - bars.mb_s_ready[s_idx].arrive(n_bytes=cfg.tma_s_bytes) + # ---- entering state: checkpoint c - 1, or initial_state for chunk 0 when given ---------- + if chunk_idx >= FIRST_STATE_CHUNK: + state_idx = state_index.idx + bars.mb_state_done[state_idx].wait(state_index.phase) + bars.mb_state_cg0_done[state_idx].wait(state_index.phase) + state_index = advance(state_index, cfg.smem_state_stages) + if elect_one: + bars.mb_state_ready[state_idx].arrive(n_bytes=cfg.tma_state_bytes) if cutlass.const_expr(cfg.use_initial_state): if chunk_idx == 0: - s0_slice = tma_slice_runtime_desc(desc_s0_slot, cutlass.Int32(0), cutlass.Int32(0), cutlass.Int32(0)) - tma_load_tile(sS_tma[s_idx], s0_slice, bars.mb_s_ready[s_idx].smem_ptr, acquire=False) + initial_state_slice = tma_slice_runtime_desc(desc_initial_state_slot, cutlass.Int32(0), cutlass.Int32(0), head_o, batch_idx) + tma_load_tile(sCheckpoint_tma[state_idx], initial_state_slice, bars.mb_state_ready[state_idx].smem_ptr, acquire=False) else: - s_slice = tma_slice_runtime_desc(desc_s_slot, cutlass.Int32(0), cutlass.Int32(0), chunk_idx - 1) - tma_load_tile(sS_tma[s_idx], s_slice, bars.mb_s_ready[s_idx].smem_ptr, acquire=False) + checkpoint_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), chunk_idx - 1, head_o) + tma_load_tile(sCheckpoint_tma[state_idx], checkpoint_slice, bars.mb_state_ready[state_idx].smem_ptr, acquire=False) else: - s_slice = tma_slice_runtime_desc(desc_s_slot, cutlass.Int32(0), cutlass.Int32(0), chunk_idx - S_MIN) - tma_load_tile(sS_tma[s_idx], s_slice, bars.mb_s_ready[s_idx].smem_ptr, acquire=False) + checkpoint_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), chunk_idx - FIRST_STATE_CHUNK, head_o) + tma_load_tile(sCheckpoint_tma[state_idx], checkpoint_slice, bars.mb_state_ready[state_idx].smem_ptr, acquire=False) - tile_idx, sched_state = _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) + tile_idx, sched_state = sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) for _ in range(cfg.smem_q_stages): bars.mb_q_mma_done[q_index.idx].wait(q_index.phase) @@ -2171,7 +1887,7 @@ def _tmaldg_warp( @cute.jit -def _compute0_warp( +def compute0_warp_group( cfg, total_tiles, bidx, @@ -2179,7 +1895,7 @@ def _compute0_warp( cu_seqlens, mWorkItems, tidx, - tmem_hold, + tmem_base_slot, scale, sCumsumlog, sCumprod, @@ -2191,14 +1907,14 @@ def _compute0_warp( sDm, sK, sdQ, - sdH, - ss_flat, - sdh_flat, + sDstate, + sstate_flat, + sdstate_flat, sSched, bars, ): - """Compute warp-group 0 role (warps 0-3): persistent scheduler loop + - per-chunk T-pairwise, kk_epi, hierarchical blockwise inverse, qk_epi.""" + """Compute warp-group 0 role (warps 0-3): persistent scheduler loop + building each chunk's blockwise-inverse T and attention matrices.""" nvvm.setmaxregister(cfg.num_regs_compute_group_0, nvvm.SetMaxRegisterAction.INCREASE) gate_index = PipelineState.start(phase=0) @@ -2206,14 +1922,14 @@ def _compute0_warp( qk_index = PipelineState.start(phase=1) ainv_index = PipelineState.start(phase=1) da_acc_index = PipelineState.start(phase=0) - dm_rdy_index = PipelineState.start(phase=0) + dm_ready_index = PipelineState.start(phase=0) cg0_dbeta_index = PipelineState.start(phase=0) nvvm.barrier_cta_sync_aligned( cfg.tmem_alloc_barrier_id, thread_count=cfg.tmem_alloc_barrier_threads, ) - tmem_base = tmem_hold.load() + tmem_base = tmem_base_slot.load() num_threads_cg0 = cfg.threads_per_warp * len(cfg.compute_group_0_warp_ids) cg0_tidx = tidx % num_threads_cg0 @@ -2229,44 +1945,42 @@ def _compute0_warp( tmem_shared_acc_col = tmem_base + cfg.tmem_shared_acc_offset tmem_shared_inp_col = tmem_base + cfg.tmem_shared_inp_offset SHARED_INP_STAGE_COLS = cfg.b_t // 2 - tmem_dop_col = tmem_shared_inp_col + tmem_do_prime_col = tmem_shared_inp_col tmem_du_col = tmem_shared_inp_col + SHARED_INP_STAGE_COLS tmem_dyp_col = tmem_du_col ACC_STAGE_COLS = cfg.b_t tmem_acc_a = tmem_shared_acc_col tmem_acc_b = tmem_shared_acc_col + ACC_STAGE_COLS tmem_kk_col = tmem_acc_a - tmem_ks_col = tmem_acc_a + tmem_k_state_col = tmem_acc_a tmem_dy_col = tmem_acc_a tmem_dm_core_col = tmem_acc_a tmem_a_col = tmem_acc_b tmem_u_col = tmem_acc_b tmem_da_col = tmem_acc_b - tmem_dk_spath_col = tmem_shared_inp_col + tmem_dk_state_path_col = tmem_shared_inp_col acc_zero = cfg.acc_dtype(0.0) mask_zero = opaque_f32_zero() - ov_tok = cg0_tidx % 8 + (cg0_tidx // 16 % 2) * 8 - ov_col = (cg0_tidx // 8 % 2) * 8 + (cg0_tidx // 32 % 2) * 32 - ov_slab = (cg0_tidx // 64) * 4096 + frag_row = cg0_tidx % 8 + (cg0_tidx // 16 % 2) * 8 + frag_col = (cg0_tidx // 8 % 2) * 8 + (cg0_tidx // 32 % 2) * 32 + frag_slab_off = (cg0_tidx // 64) * 4096 sK_base_p = cute.make_ptr(cfg.io_dtype, sK[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) k_stage_elems_cg0 = cfg.k_cosize // cfg.smem_k_stages sdQ_base = cute.make_ptr(cfg.io_dtype, sdQ[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) - sdH_parts_p = cute.make_ptr(cfg.io_dtype, sdH[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) - # dG fold scratch in sKK (NOT sAinv: its upper triangle is kernel-start - # zeros, never rewritten); kk_epi fully rewrites sKK every chunk and its - # readers (inverse, M-terms dot) precede the fold in CG0 program order + sdH_parts_p = cute.make_ptr(cfg.io_dtype, sDstate[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) + # dG fold scratch in sKK skk_red = cute.make_ptr(cutlass.Float32, sKK[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) cg0_k_index = PipelineState.start(phase=0) cg0_dgate_index = PipelineState.start(phase=0) - tmem_dvdk_col = tmem_base + cfg.tmem_dvdk_offset - tmem_dh_inp_col = tmem_base + cfg.tmem_dh_inp_offset - cg0_kk_rdy = PipelineState.start(phase=0) - cg0_a_rdy = PipelineState.start(phase=0) - cg0_dk_scale_rdy = PipelineState.start(phase=0) - cg0_dq_scale_rdy = PipelineState.start(phase=0) - cg0_dhs_rdy = PipelineState.start(phase=0) - cg0_dk_attn_rdy = PipelineState.start(phase=0) - DHT0 = 1 if cfg.use_dht else 0 + tmem_dvdk_acc_col = tmem_base + cfg.tmem_dvdk_acc_offset + tmem_dstate_inp_col = tmem_base + cfg.tmem_dstate_inp_offset + cg0_kk_ready = PipelineState.start(phase=0) + cg0_a_ready = PipelineState.start(phase=0) + cg0_dk_scale_ready = PipelineState.start(phase=0) + cg0_dq_scale_ready = PipelineState.start(phase=0) + cg0_dstate_smem_ready = PipelineState.start(phase=0) + cg0_dk_attn_ready = PipelineState.start(phase=0) + DSTATE_IN0 = 1 if cfg.use_dstate_in else 0 ainv_zero_ptr = cute.make_ptr(cutlass.Int32, sAinv[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) for z in cutlass.range_constexpr(cfg.ainv_cosize * bpe // 4 // num_threads_cg0): @@ -2274,31 +1988,39 @@ def _compute0_warp( sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) - S_MIN = 0 if cfg.use_initial_state else 1 + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 SFIRST_MIN = 1 if cfg.use_initial_state else 2 while tile_idx < total_tiles: batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) - sk_nt = cend - wstart + num_item_chunks = cend - wstart - for chunk_idx in cutlass.range(sk_nt): - # ---- Step 1: T-pairwise ------------------------------------ + for chunk_idx in cutlass.range(num_item_chunks): + # ---- T-pairwise ------------------------------------------------------ gate_idx = gate_index.idx bars.mb_gate_ready[gate_idx].wait(gate_index.phase) gate_index = advance(gate_index, cfg.smem_gate_stages) - gT = [] - gT_strict = [] + row_cs = [] + for r in cutlass.range_constexpr(2): + row_cs.append(sCumsumlog[warp_id * 16 + lane_id // 4 + r * 8, 0, gate_idx]) + col_cs = [] + for g in cutlass.range_constexpr(8): + for b in cutlass.range_constexpr(2): + col_cs.append(sCumsumlog[(lane_id % 4) * 2 + g * 8 + b, 0, gate_idx]) + decay_t = [] + decay_t_strict = [] for k in cutlass.range_constexpr(num_vals): crow = warp_id * 16 + lane_id // 4 + ((k // 2) % 2) * 8 ccol = (lane_id % 4) * 2 + ((k // 4) * 8 + k % 2) - gT.append(cute.math.exp2(sCumsumlog[crow, 0, gate_idx] - sCumsumlog[ccol, 0, gate_idx], fastmath=True) if crow >= ccol else mask_zero) - gT_strict.append(mask_zero if crow == ccol else gT[k]) - gDecayScale = [] + decay_t.append(cute.math.exp2(row_cs[(k // 2) % 2] - col_cs[(k // 4) * 2 + (k % 2)], fastmath=True) if crow >= ccol else mask_zero) + decay_t_strict.append(mask_zero if crow == ccol else decay_t[k]) last_cs = sCumsumlog[cfg.b_t - 1, 0, gate_idx] - for k in cutlass.range_constexpr(num_vals): - gDecayScale.append(cute.math.exp2(last_cs - sCumsumlog[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, gate_idx], fastmath=True)) + decay_scale_fp32 = [] + for i in cutlass.range_constexpr(16): + decay_scale_fp32.append(cute.math.exp2(last_cs - col_cs[i], fastmath=True)) + decay_scale_vals = [decay_scale_fp32[(k // 4) * 2 + (k % 2)] for k in range(num_vals)] cumprod_total = sCumprod[sCumprod.shape[0] - 1, 0, gate_idx] beta_idx = beta_index.idx @@ -2310,19 +2032,19 @@ def _compute0_warp( crow = warp_id * 16 + lane_id // 4 + ((k // 2) % 2) * 8 gBeta.append(sBeta[crow, 0, beta_idx]) - # ---- Step 2: kk_epi: M_kk[i,j] = W_kk[i,j] * T[i,j] * beta[i] ---- + # ---- KK epi: M_kk[i,j] = W_kk[i,j] * T[i,j] * Beta[i] --------------- ainv_idx = ainv_index.idx ainv_phase = ainv_index.phase ainv_index = advance(ainv_index, cfg.smem_ainv_stages) - bars.mb_kk_acc_ready[0].wait(cg0_kk_rdy.phase) - cg0_kk_rdy = advance(cg0_kk_rdy, 1) + bars.mb_kk_acc_ready[0].wait(cg0_kk_ready.phase) + cg0_kk_ready = advance(cg0_kk_ready, 1) ainv_base = sAinv[ainv_idx].base kk_base = sKK[ainv_idx].base kk_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_kk_col, cutlass.Float32), num=8) kk_f16 = [] for k in cutlass.range_constexpr(num_vals // 2): - p0, p1 = fmul2(kk_vec[2 * k], kk_vec[2 * k + 1], gT[2 * k], gT[2 * k + 1]) + p0, p1 = fmul2(kk_vec[2 * k], kk_vec[2 * k + 1], decay_t[2 * k], decay_t[2 * k + 1]) v0, v1 = fmul2(p0, p1, gBeta[2 * k], gBeta[2 * k + 1]) kk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) for c in cutlass.range_constexpr(ACC_N_FRAGS): @@ -2335,23 +2057,24 @@ def _compute0_warp( [kk_f16[c * 4 + 0], kk_f16[c * 4 + 1], kk_f16[c * 4 + 2], kk_f16[c * 4 + 3]], nvvm.MMALayout.ROW, ) - if chunk_idx < cend - S_MIN: + if chunk_idx < cend - FIRST_STATE_CHUNK: bars.mb_kk_acc_done[0].arrive() - # ---- Step 3: qk_epi: W_qkv[i,j] = W_qk[i,j] * T[i,j] * scale ---- + # ---- QK epi: W_qkv[i,j] = W_qk[i,j] * T[i,j] * scale ---------------- qk_idx = qk_index.idx - bars.mb_qk_done[qk_idx].wait(qk_index.phase) + qk_phase = qk_index.phase qk_index = advance(qk_index, cfg.smem_qk_stages) - bars.mb_a_acc_ready[0].wait(cg0_a_rdy.phase) - cg0_a_rdy = advance(cg0_a_rdy, 1) + bars.mb_a_acc_ready[0].wait(cg0_a_ready.phase) + cg0_a_ready = advance(cg0_a_ready, 1) qk_base = sQk[qk_idx].base qk_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_a_col, cutlass.Float32), num=8) qk_f16 = [] for k in cutlass.range_constexpr(num_vals // 2): - p0, p1 = fmul2(qk_vec[2 * k], qk_vec[2 * k + 1], gT[2 * k], gT[2 * k + 1]) + p0, p1 = fmul2(qk_vec[2 * k], qk_vec[2 * k + 1], decay_t[2 * k], decay_t[2 * k + 1]) v0, v1 = fmul2(p0, p1, scale, scale) qk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) + bars.mb_qk_done[qk_idx].wait(qk_phase) for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( cutlass.inttoptr( @@ -2365,40 +2088,40 @@ def _compute0_warp( nvvm.fence_proxy("async.shared", space="cta") bars.mb_qk_ready[qk_idx].arrive() - # ---- Step 4: blockwise inverse: A_inv = ------------------- + # ---- blockwise inverse: A_inv = ------------------------------------- bars.mb_ainv_done[ainv_idx].wait(ainv_phase) nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, ) if warp_id < 2: - _invert_diagonal_NxN(cfg, kk_base, ainv_base, cg0_tidx // 8, cg0_tidx, 8) + invert_diagonal_NxN(cfg, kk_base, ainv_base, cg0_tidx // 8, cg0_tidx, 8) nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, ) - _blockwise_diagonal_8x8_to_16x16(cfg, ainv_base, kk_base, warp_id * 16, lane_id) + blockwise_diagonal_8x8_to_16x16(cfg, ainv_base, kk_base, warp_id * 16, lane_id) nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, ) if warp_id < 2: - _blockwise_diagonal_16x16_to_32x32(cfg, ainv_base, kk_base, warp_id * 32, lane_id) + blockwise_diagonal_16x16_to_32x32(cfg, ainv_base, kk_base, warp_id * 32, lane_id) nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, ) if warp_id < 2: - _blockwise_diagonal_32x32_to_64x64(cfg, ainv_base, kk_base, warp_id, lane_id) + blockwise_diagonal_32x32_to_64x64(cfg, ainv_base, kk_base, warp_id, lane_id) nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, ) - # ---- beta column-scaling in place: A_inv[i,j] *= beta[j] ---- + # ---- Beta column-scaling in place: A_inv[i,j] *= Beta[j] ------------ beta_col = [] for k in cutlass.range_constexpr(num_vals): beta_col.append(sBeta[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, beta_idx]) @@ -2434,96 +2157,101 @@ def _compute0_warp( nvvm.fence_proxy("async.shared", space="cta") bars.mb_ainv_ready[ainv_idx].arrive() - # ---- dQ inter rescale: gCumprod * scale in place (CG0-owned; - # the scale_ready wait also covers the hdh sS read via ks) ---- - if chunk_idx < cend - S_MIN: - gCumprod = [] - for k in cutlass.range_constexpr(num_vals): - gCumprod.append(sCumprod[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, gate_idx]) - bars.mb_dq_acc_scale_ready[0].wait(cg0_dq_scale_rdy.phase) - cg0_dq_scale_rdy = advance(cg0_dq_scale_rdy, 1) - # all loads issue before the first store: a TMEM store between - # loads pins ptxas to one load latency per sub + # ---- dQ inter rescale ------------------------------------------------ + if chunk_idx < cend - FIRST_STATE_CHUNK: + cumprod_fp32 = [] + for g in cutlass.range_constexpr(8): + for b in cutlass.range_constexpr(2): + cumprod_fp32.append(sCumprod[(lane_id % 4) * 2 + g * 8 + b, 0, gate_idx]) + cumprod_vals = [cumprod_fp32[(k // 4) * 2 + (k % 2)] for k in range(num_vals)] + bars.mb_dq_acc_scale_ready[0].wait(cg0_dq_scale_ready.phase) + cg0_dq_scale_ready = advance(cg0_dq_scale_ready, 1) dqi_ptrs = [] dqi_vecs = [] for sub in cutlass.range_constexpr(2): - dqi_ptrs.append(nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dh_inp_col, cutlass.Float32)) + dqi_ptrs.append(nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dstate_inp_col, cutlass.Float32)) dqi_vecs.append(nvvm.tcgen05_ld("16x256b", dqi_ptrs[sub], num=8)) for sub in cutlass.range_constexpr(2): dqi_scaled = [] for j in cutlass.range_constexpr(16): - p0, p1 = fmul2(dqi_vecs[sub][2 * j], dqi_vecs[sub][2 * j + 1], gCumprod[2 * j], gCumprod[2 * j + 1]) + p0, p1 = fmul2(dqi_vecs[sub][2 * j], dqi_vecs[sub][2 * j + 1], cumprod_vals[2 * j], cumprod_vals[2 * j + 1]) s0, s1 = fmul2(p0, p1, scale, scale) dqi_scaled += [s0, s1] nvvm.tcgen05_st("16x256b", dqi_ptrs[sub], cutlass.Vector.from_elements(tuple(dqi_scaled), cutlass.Float32)) nvvm.tcgen05_wait("store") bars.mb_dq_acc_scale_done[0].arrive() - # ---- dg_last H ⊙ dH term (octet-vectorized LDS.128); the sS - # read is released via the mb_s_done arrive below. The sdH - # staging is gated by a dhs_ready mirror-wait (non-consuming) ---- - if chunk_idx + DHT0 >= 1: - bars.mb_dhs_ready[0].wait(cg0_dhs_rdy.phase) - cg0_dhs_rdy = advance(cg0_dhs_rdy, 1) - sdh_base = sdh_flat.iterator.toint() - ss_base = ss_flat.iterator.toint() - hdh_lo = [opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero()] - hdh_hi = [opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero()] + # ---- dG_last state dot dstate term ------------------------------------------- + if chunk_idx + DSTATE_IN0 >= 1: + bars.mb_dstate_smem_ready[0].wait(cg0_dstate_smem_ready.phase) + cg0_dstate_smem_ready = advance(cg0_dstate_smem_ready, 1) + sdstate_base = sdstate_flat.iterator.toint() + sstate_base = sstate_flat.iterator.toint() + state_dot_dstate_lo = [opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero()] + state_dot_dstate_hi = [opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero()] for oct_ in cutlass.range_constexpr(cfg.d_v // 8): - hs0 = (oct_ // 8) * (cfg.d_k * 64) + cg0_tidx * 64 + swizzle_xor_128b(cg0_tidx, (oct_ % 8) * 8) - dw = cute.make_tensor( - cute.make_ptr(cutlass.Int32, sdh_base + hs0 * bpe, mem_space=cute.AddressSpace.smem, assumed_align=16), + state_dot_off = (oct_ // 8) * (cfg.d_k * 64) + cg0_tidx * 64 + swizzle_xor_128b(cg0_tidx, (oct_ % 8) * 8) + dstate_w = cute.make_tensor( + cute.make_ptr(cutlass.Int32, sdstate_base + state_dot_off * bpe, mem_space=cute.AddressSpace.smem, assumed_align=16), cute.make_layout(4), ).load() - sw = cute.make_tensor( - cute.make_ptr(cutlass.Int32, ss_base + hs0 * bpe, mem_space=cute.AddressSpace.smem, assumed_align=16), + state_w = cute.make_tensor( + cute.make_ptr(cutlass.Int32, sstate_base + state_dot_off * bpe, mem_space=cute.AddressSpace.smem, assumed_align=16), cute.make_layout(4), ).load() for w in cutlass.range_constexpr(4): - d_lo, d_hi = f16x2_to_f32(dw[w], dtype=cfg.io_dtype) - s_lo, s_hi = f16x2_to_f32(sw[w], dtype=cfg.io_dtype) - hdh_lo[w], hdh_hi[w] = ffma2(d_lo, d_hi, s_lo, s_hi, hdh_lo[w], hdh_hi[w]) - hdh = ((hdh_lo[0] + hdh_lo[1]) + (hdh_lo[2] + hdh_lo[3])) + ((hdh_hi[0] + hdh_hi[1]) + (hdh_hi[2] + hdh_hi[3])) + dstate_lo, dstate_hi = f16x2_to_f32(dstate_w[w], dtype=cfg.io_dtype) + state_lo, state_hi = f16x2_to_f32(state_w[w], dtype=cfg.io_dtype) + state_dot_dstate_lo[w], state_dot_dstate_hi[w] = ffma2( + dstate_lo, dstate_hi, state_lo, state_hi, state_dot_dstate_lo[w], state_dot_dstate_hi[w] + ) + state_dot_dstate = ((state_dot_dstate_lo[0] + state_dot_dstate_lo[1]) + (state_dot_dstate_lo[2] + state_dot_dstate_lo[3])) + ( + (state_dot_dstate_hi[0] + state_dot_dstate_hi[1]) + (state_dot_dstate_hi[2] + state_dot_dstate_hi[3]) + ) for off in [1, 2, 4, 8, 16]: - hdh += nvvm.shfl_sync(0xFFFFFFFF, hdh, off, 31, kind=nvvm.Shfl.BFLY) - bars.mb_hdh_done[0].arrive() - if chunk_idx < cend - S_MIN: + state_dot_dstate += nvvm.shfl_sync(0xFFFFFFFF, state_dot_dstate, off, 31, kind=nvvm.Shfl.BFLY) + bars.mb_state_dot_dstate_done[0].arrive() + if chunk_idx < cend - FIRST_STATE_CHUNK: nvvm.fence_proxy("async.shared", space="cta") - nvvm.mbarrier_arrive(bars.mb_s_done[0].smem_ptr) + bars.mb_state_cg0_done[0].arrive() - # ---- dK inter rescale: gDecayScale in place + the skd dot ---- + # ---- dK inter rescale ------------------------------------------------ cg0_k_idx = cg0_k_index.idx cg0_k_index = advance(cg0_k_index, cfg.smem_k_stages) - skd = cutlass.Float32(0.0) - if chunk_idx + DHT0 >= 1: - bars.mb_dk_scale_ready[0].wait(cg0_dk_scale_rdy.phase) - cg0_dk_scale_rdy = advance(cg0_dk_scale_rdy, 1) - skd_lo = [opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero()] - skd_hi = [opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero()] + k_dot_dk_inter = cutlass.Float32(0.0) + if chunk_idx + DSTATE_IN0 >= 1: + bars.mb_dk_scale_ready[0].wait(cg0_dk_scale_ready.phase) + cg0_dk_scale_ready = advance(cg0_dk_scale_ready, 1) + k_dot_dk_inter_lo = [opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero()] + k_dot_dk_inter_hi = [opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero()] dki_ptrs = [] dki_vecs = [] for sub in cutlass.range_constexpr(2): - dki_ptrs.append(nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_col, cutlass.Float32)) + dki_ptrs.append(nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_acc_col, cutlass.Float32)) dki_vecs.append(nvvm.tcgen05_ld("16x256b", dki_ptrs[sub], num=8)) for sub in cutlass.range_constexpr(2): dki_scaled = [] for j in cutlass.range_constexpr(16): - s0, s1 = fmul2(dki_vecs[sub][2 * j], dki_vecs[sub][2 * j + 1], gDecayScale[2 * j], gDecayScale[2 * j + 1]) + s0, s1 = fmul2(dki_vecs[sub][2 * j], dki_vecs[sub][2 * j + 1], decay_scale_vals[2 * j], decay_scale_vals[2 * j + 1]) dki_scaled += [s0, s1] nvvm.tcgen05_st("16x256b", dki_ptrs[sub], cutlass.Vector.from_elements(tuple(dki_scaled), cutlass.Float32)) for m0 in cutlass.range_constexpr(4): - frag_addr = ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) + frag_addr = frag_slab_off + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) k_f16 = nvvm.ldmatrix((sK_base_p + cg0_k_idx * k_stage_elems_cg0 + frag_addr).raw_ptr(), 4, nvvm.MMALayout.COL) for i in cutlass.range_constexpr(4): k_lo, k_hi = f16x2_to_f32(k_f16[i], dtype=cfg.io_dtype) - skd_lo[i], skd_hi[i] = ffma2(dki_scaled[8 * m0 + 2 * i], dki_scaled[8 * m0 + 2 * i + 1], k_lo, k_hi, skd_lo[i], skd_hi[i]) + k_dot_dk_inter_lo[i], k_dot_dk_inter_hi[i] = ffma2( + dki_scaled[8 * m0 + 2 * i], dki_scaled[8 * m0 + 2 * i + 1], k_lo, k_hi, k_dot_dk_inter_lo[i], k_dot_dk_inter_hi[i] + ) nvvm.tcgen05_wait("store") bars.mb_dk_scale_done[0].arrive() - skd = ((skd_lo[0] + skd_lo[1]) + (skd_lo[2] + skd_lo[3])) + ((skd_hi[0] + skd_hi[1]) + (skd_hi[2] + skd_hi[3])) + k_dot_dk_inter = ((k_dot_dk_inter_lo[0] + k_dot_dk_inter_lo[1]) + (k_dot_dk_inter_lo[2] + k_dot_dk_inter_lo[3])) + ( + (k_dot_dk_inter_hi[0] + k_dot_dk_inter_hi[1]) + (k_dot_dk_inter_hi[2] + k_dot_dk_inter_hi[3]) + ) for off in [1, 2, 4, 8, 16]: - skd += nvvm.shfl_sync(0xFFFFFFFF, skd, off, 31, kind=nvvm.Shfl.BFLY) + k_dot_dk_inter += nvvm.shfl_sync(0xFFFFFFFF, k_dot_dk_inter, off, 31, kind=nvvm.Shfl.BFLY) - # ---- Step 6: dA epilogue ----------------------------------- + # ---- dA epilogue ----------------------------------------------------- bars.mb_da_acc_ready[0].wait(da_acc_index.phase) da_acc_index = advance(da_acc_index, 1) @@ -2531,7 +2259,7 @@ def _compute0_warp( da_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_da_col, cutlass.Float32), num=8) da_f16 = [] for k in cutlass.range_constexpr(num_vals // 2): - p0, p1 = fmul2(da_vec[2 * k], da_vec[2 * k + 1], gT[2 * k], gT[2 * k + 1]) + p0, p1 = fmul2(da_vec[2 * k], da_vec[2 * k + 1], decay_t[2 * k], decay_t[2 * k + 1]) v0, v1 = fmul2(p0, p1, scale, scale) da_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) for c in cutlass.range_constexpr(ACC_N_FRAGS): @@ -2547,15 +2275,15 @@ def _compute0_warp( nvvm.fence_proxy("async.shared", space="cta") bars.mb_da_ready[0].arrive() - # ---- Step 8: dM epilogue ----------------------------------- - bars.mb_dm_ready[0].wait(dm_rdy_index.phase) - dm_rdy_index = advance(dm_rdy_index, 1) + # ---- dM epilogue ----------------------------------------------------- + bars.mb_dm_ready[0].wait(dm_ready_index.phase) + dm_ready_index = advance(dm_ready_index, 1) dm_base = sDm[0].base dm_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dm_core_col, cutlass.Float32), num=8) dm_f16 = [] for k in cutlass.range_constexpr(num_vals // 2): - p0, p1 = fmul2(dm_vec[2 * k], dm_vec[2 * k + 1], gT_strict[2 * k], gT_strict[2 * k + 1]) + p0, p1 = fmul2(dm_vec[2 * k], dm_vec[2 * k + 1], decay_t_strict[2 * k], decay_t_strict[2 * k + 1]) v0 = cutlass.Float32(0.0) - p0 v1 = cutlass.Float32(0.0) - p1 dm_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) @@ -2572,21 +2300,21 @@ def _compute0_warp( nvvm.fence_proxy("async.shared", space="cta") bars.mb_dm_done[0].arrive() - # ---- dK attn read: 16x256b fragment view for the part_k dot ---- - bars.mb_dk_attn_ready[0].wait(cg0_dk_attn_rdy.phase) - cg0_dk_attn_rdy = advance(cg0_dk_attn_rdy, 1) + # ---- dK attn read ---------------------------------------------------- + bars.mb_dk_attn_ready[0].wait(cg0_dk_attn_ready.phase) + cg0_dk_attn_ready = advance(cg0_dk_attn_ready, 1) dks_frag = [] for sub in cutlass.range_constexpr(2): dks_vec = nvvm.tcgen05_ld( "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_col, cutlass.Float32), + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_acc_col, cutlass.Float32), num=8, ) dks_frag.append([dks_vec[k] for k in range(32)]) nvvm.tcgen05_wait("load") bars.mb_dk_attn_done[0].arrive() - # ---- dBeta/dGate M-terms: E = strict ⊙ dm_core ⊙ M_kk(sKK). ---- + # ---- dBeta/dGate M-terms: E = strict ⊙ dM_core ⊙ M_kk(sKK). ---------- kk_frag = [] for c in cutlass.range_constexpr(ACC_N_FRAGS): kk_frag += list( @@ -2625,40 +2353,42 @@ def _compute0_warp( for off in [1, 2]: for rp in cutlass.range_constexpr(2): row_part[rp] += nvvm.shfl_sync(0xFFFFFFFF, row_part[rp], off, 31, kind=nvvm.Shfl.BFLY) - # CG1's v-terms assign the dBeta base into sBeta; fold the k-term on top bars.mb_dbeta_cg1_ready[0].wait(cg0_dbeta_index.phase) cg0_dbeta_index = advance(cg0_dbeta_index, 1) if lane_id % 4 == 0: for rp in cutlass.range_constexpr(2): crow_r = warp_id * 16 + lane_id // 4 + rp * 8 sBeta[crow_r, 0, beta_idx] = sBeta[crow_r, 0, beta_idx] - row_part[rp] * binv_row[rp] - # ---- part reductions: fragment dot (K frags from sK, dks from - # the 16x256b r4 view), col_part folded in, reduce-scatter ---- - if chunk_idx + DHT0 >= S_MIN: + + # ---- part reductions ------------------------------------------------- + if chunk_idx + DSTATE_IN0 >= FIRST_STATE_CHUNK: part_k = [acc_zero] * 16 for sub in cutlass.range_constexpr(2): for m0 in cutlass.range_constexpr(4): - frag_addr = ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) + frag_addr = frag_slab_off + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) k_f16 = nvvm.ldmatrix((sK_base_p + cg0_k_idx * k_stage_elems_cg0 + frag_addr).raw_ptr(), 4, nvvm.MMALayout.COL) for i in cutlass.range_constexpr(4): k_lo, k_hi = f16x2_to_f32(k_f16[i], dtype=cfg.io_dtype) - kk0 = cutlass.const_expr(8 * m0 + 2 * i) - j0 = cutlass.const_expr((kk0 // 4) * 2 + (kk0 % 2)) + frag_e0 = cutlass.const_expr(8 * m0 + 2 * i) + part_e0 = cutlass.const_expr((frag_e0 // 4) * 2 + (frag_e0 % 2)) if cutlass.const_expr(sub == 0 and i % 2 == 0): - part_k[j0], part_k[j0 + 1] = fmul2(dks_frag[sub][kk0], dks_frag[sub][kk0 + 1], k_lo, k_hi) + part_k[part_e0], part_k[part_e0 + 1] = fmul2(dks_frag[sub][frag_e0], dks_frag[sub][frag_e0 + 1], k_lo, k_hi) else: - part_k[j0], part_k[j0 + 1] = ffma2(dks_frag[sub][kk0], dks_frag[sub][kk0 + 1], k_lo, k_hi, part_k[j0], part_k[j0 + 1]) + part_k[part_e0], part_k[part_e0 + 1] = ffma2( + dks_frag[sub][frag_e0], dks_frag[sub][frag_e0 + 1], k_lo, k_hi, part_k[part_e0], part_k[part_e0 + 1] + ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_k_cg0_done[cg0_k_idx].arrive() - am = [col_part[j] - part_k[j] for j in range(16)] - am_lo, am_hi = _warp_reduce_scatter_frag16(am, lane_id) - dg_last_w = skd + ((cumprod_total * hdh if chunk_idx + DHT0 >= 1 else acc_zero) if chunk_idx < cend - S_MIN else acc_zero) + dg_part = [col_part[j] - part_k[j] for j in range(16)] + dg_part_lo, dg_part_hi = warp_reduce_scatter_frag_16_elems(dg_part, lane_id) + dg_last_w = k_dot_dk_inter + ( + (cumprod_total * state_dot_dstate if chunk_idx + DSTATE_IN0 >= 1 else acc_zero) if chunk_idx < cend - FIRST_STATE_CHUNK else acc_zero + ) tok0 = (lane_id // 4) * 8 + (lane_id % 4) * 2 - (skk_red + warp_id * 64 + tok0).store(am_lo) - (skk_red + warp_id * 64 + tok0 + 1).store(am_hi + dg_last_w if lane_id == 31 else am_hi) + (skk_red + warp_id * 64 + tok0).store(dg_part_lo) + (skk_red + warp_id * 64 + tok0 + 1).store(dg_part_hi + dg_last_w if lane_id == 31 else dg_part_hi) - # CG1 adds part_q to dGate first; fold CG0's terms on top bars.mb_dgate_cg1_ready[0].wait(cg0_dgate_index.phase) cg0_dgate_index = advance(cg0_dgate_index, 1) if lane_id % 4 == 0: @@ -2672,30 +2402,31 @@ def _compute0_warp( ) sCumsumlog[cg0_tidx, 0, gate_idx] = sCumsumlog[cg0_tidx, 0, gate_idx] + dg_sum - if chunk_idx + DHT0 < S_MIN: + if chunk_idx + DSTATE_IN0 < FIRST_STATE_CHUNK: part_k = [acc_zero] * 16 for sub in cutlass.range_constexpr(2): for m0 in cutlass.range_constexpr(4): - frag_addr = ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) + frag_addr = frag_slab_off + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) k_f16 = nvvm.ldmatrix((sK_base_p + cg0_k_idx * k_stage_elems_cg0 + frag_addr).raw_ptr(), 4, nvvm.MMALayout.COL) for i in cutlass.range_constexpr(4): k_lo, k_hi = f16x2_to_f32(k_f16[i], dtype=cfg.io_dtype) - kk0 = cutlass.const_expr(8 * m0 + 2 * i) - j0 = cutlass.const_expr((kk0 // 4) * 2 + (kk0 % 2)) + frag_e0 = cutlass.const_expr(8 * m0 + 2 * i) + part_e0 = cutlass.const_expr((frag_e0 // 4) * 2 + (frag_e0 % 2)) if cutlass.const_expr(sub == 0 and i % 2 == 0): - part_k[j0], part_k[j0 + 1] = fmul2(dks_frag[sub][kk0], dks_frag[sub][kk0 + 1], k_lo, k_hi) + part_k[part_e0], part_k[part_e0 + 1] = fmul2(dks_frag[sub][frag_e0], dks_frag[sub][frag_e0 + 1], k_lo, k_hi) else: - part_k[j0], part_k[j0 + 1] = ffma2(dks_frag[sub][kk0], dks_frag[sub][kk0 + 1], k_lo, k_hi, part_k[j0], part_k[j0 + 1]) + part_k[part_e0], part_k[part_e0 + 1] = ffma2( + dks_frag[sub][frag_e0], dks_frag[sub][frag_e0 + 1], k_lo, k_hi, part_k[part_e0], part_k[part_e0 + 1] + ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_k_cg0_done[cg0_k_idx].arrive() - am = [col_part[j] - part_k[j] for j in range(16)] - am_lo, am_hi = _warp_reduce_scatter_frag16(am, lane_id) + dg_part = [col_part[j] - part_k[j] for j in range(16)] + dg_part_lo, dg_part_hi = warp_reduce_scatter_frag_16_elems(dg_part, lane_id) tok0 = (lane_id // 4) * 8 + (lane_id % 4) * 2 - (skk_red + warp_id * 64 + tok0).store(am_lo) - (skk_red + warp_id * 64 + tok0 + 1).store(am_hi) + (skk_red + warp_id * 64 + tok0).store(dg_part_lo) + (skk_red + warp_id * 64 + tok0 + 1).store(dg_part_hi) - # CG1 adds part_q to dGate first; fold CG0's terms on top bars.mb_dgate_cg1_ready[0].wait(cg0_dgate_index.phase) cg0_dgate_index = advance(cg0_dgate_index, 1) if lane_id % 4 == 0: @@ -2711,7 +2442,7 @@ def _compute0_warp( bars.mb_gate_done[gate_idx].arrive() bars.mb_beta_done[beta_idx].arrive() - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) for _ in range(cfg.smem_qk_stages): bars.mb_qk_done[qk_index.idx].wait(qk_index.phase) qk_index = advance(qk_index, cfg.smem_qk_stages) @@ -2719,23 +2450,22 @@ def _compute0_warp( bars.mb_ainv_done[ainv_index.idx].wait(ainv_index.phase) ainv_index = advance(ainv_index, cfg.smem_ainv_stages) - # CG0 done with TMEM: release the MMA warp's dealloc bars.mb_tmem_done[0].arrive() @cute.jit -def _compute1_warp( +def compute1_warp_group( cfg, total_tiles, bidx, num_ctas, cu_seqlens, mWorkItems, - mDs0_out, - mDht, + mDstate0_out, + mDstate_in, tidx, warp_idx, - tmem_hold, + tmem_base_slot, scale, sQ, sK, @@ -2747,38 +2477,35 @@ def _compute1_warp( sdQ, sdK, sdV, - sdH, + sDstate, sDa, sDm, sSched, bars, ): - """Compute warp-group 1 role (warps 4-7): persistent scheduler loop and - per-chunk (BACKWARD order): dV-acc / dQ-acc - in-place decay scales, dO'/dU/dY' TMEM restages, Y and gks staging, - u readout over sV, the dV-pass dG/dBeta v-terms, dV/dK staging with the - dK s-path fold, and the next-chunk dH prep at the chunk tail.""" + """Compute warp-group 1 role (warps 4-7): persistent scheduler loop + running each chunk's gradient epilogues and stagings.""" v_index = PipelineState.start(phase=0) do_index = PipelineState.start(phase=0) gate_index = PipelineState.start(phase=0) - cg1_ks_rdy = PipelineState.start(phase=0) - cg1_u_rdy = PipelineState.start(phase=0) - cg1_dy_rdy = PipelineState.start(phase=0) - cg1_du_scale_rdy = PipelineState.start(phase=0) - cg1_du_total_rdy = PipelineState.start(phase=0) - cg1_dk_total_rdy = PipelineState.start(phase=0) - dh_acc_index = PipelineState.start(phase=0) - dop_inp_free = PipelineState.start(phase=1) + cg1_k_state_ready = PipelineState.start(phase=0) + cg1_u_ready = PipelineState.start(phase=0) + cg1_dy_ready = PipelineState.start(phase=0) + cg1_du_scale_ready = PipelineState.start(phase=0) + cg1_du_total_ready = PipelineState.start(phase=0) + cg1_dk_total_ready = PipelineState.start(phase=0) + dstate_acc_index = PipelineState.start(phase=0) + do_prime_inp_free = PipelineState.start(phase=1) dyp_inp_free = PipelineState.start(phase=1) - dhs_index = PipelineState.start(phase=1) - cg1_hdh_index = PipelineState.start(phase=0) + dstate_smem_index = PipelineState.start(phase=1) + cg1_state_dot_dstate_index = PipelineState.start(phase=0) dq_index = PipelineState.start(phase=1) cg1_beta_index = PipelineState.start(phase=0) sdv_done_index = PipelineState.start(phase=1) - cg1_dk_spath_rdy = PipelineState.start(phase=0) - dq_total_rdy_index = PipelineState.start(phase=0) - dh_inp_index = PipelineState.start(phase=1) + cg1_dk_state_path_ready = PipelineState.start(phase=0) + dq_total_ready_index = PipelineState.start(phase=0) + dstate_inp_index = PipelineState.start(phase=1) dk_index = PipelineState.start(phase=1) dv_index = PipelineState.start(phase=1) @@ -2787,7 +2514,7 @@ def _compute1_warp( cfg.tmem_alloc_barrier_id, thread_count=cfg.tmem_alloc_barrier_threads, ) - tmem_base = tmem_hold.load() + tmem_base = tmem_base_slot.load() num_threads_cg1 = cfg.threads_per_warp * len(cfg.compute_group_1_warp_ids) cg1_tidx = tidx % num_threads_cg1 @@ -2796,31 +2523,31 @@ def _compute1_warp( ldtm_width = 32 sttm_width = ldtm_width // 2 num_state_subs = cutlass.const_expr(cfg.d_v // ldtm_width) - tmem_dh_col = tmem_base + cfg.tmem_dh_offset - tmem_dh_inp_col = tmem_base + cfg.tmem_dh_inp_offset - tmem_dvdk_col = tmem_base + cfg.tmem_dvdk_offset + tmem_dstate_acc_col = tmem_base + cfg.tmem_dstate_acc_offset + tmem_dstate_inp_col = tmem_base + cfg.tmem_dstate_inp_offset + tmem_dvdk_acc_col = tmem_base + cfg.tmem_dvdk_acc_offset tmem_shared_acc_col = tmem_base + cfg.tmem_shared_acc_offset tmem_shared_inp_col = tmem_base + cfg.tmem_shared_inp_offset SHARED_INP_STAGE_COLS = cfg.b_t // 2 - tmem_dop_col = tmem_shared_inp_col + tmem_do_prime_col = tmem_shared_inp_col tmem_du_col = tmem_shared_inp_col + SHARED_INP_STAGE_COLS tmem_dyp_col = tmem_du_col tmem_y_col = tmem_base + cfg.tmem_y_offset - tmem_gks_col = tmem_y_col + SHARED_INP_STAGE_COLS + tmem_g_k_state_col = tmem_y_col + SHARED_INP_STAGE_COLS ACC_STAGE_COLS = cfg.b_t tmem_acc_a = tmem_shared_acc_col tmem_acc_b = tmem_shared_acc_col + ACC_STAGE_COLS tmem_kk_col = tmem_acc_a - tmem_ks_col = tmem_acc_a + tmem_k_state_col = tmem_acc_a tmem_dy_col = tmem_acc_a tmem_dm_core_col = tmem_acc_a tmem_a_col = tmem_acc_b tmem_u_col = tmem_acc_b tmem_da_col = tmem_acc_b - tmem_dk_spath_col = tmem_shared_inp_col - ov_tok = cg1_tidx % 8 + (cg1_tidx // 16 % 2) * 8 - ov_col = (cg1_tidx // 8 % 2) * 8 + (cg1_tidx // 32 % 2) * 32 - ov_slab = (cg1_tidx // 64) * 4096 + tmem_dk_state_path_col = tmem_shared_inp_col + frag_row = cg1_tidx % 8 + (cg1_tidx // 16 % 2) * 8 + frag_col = (cg1_tidx // 8 % 2) * 8 + (cg1_tidx // 32 % 2) * 32 + frag_slab_off = (cg1_tidx // 64) * 4096 dv_stage_elems = cfg.dv_cosize // cfg.smem_dv_stages sdV_base = cute.make_ptr(cfg.io_dtype, sdV[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) v_stage_elems = cfg.v_cosize // cfg.smem_v_stages @@ -2832,601 +2559,119 @@ def _compute1_warp( dq_stage_elems = cfg.dq_cosize // cfg.smem_dq_stages sdQ_base = cute.make_ptr(cfg.io_dtype, sdQ[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) sdK_base = cute.make_ptr(cfg.io_dtype, sdK[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) - sdH_base_int = sdH[0].base - # dBeta/dGate reduction scratch: f32 view of the owned dQ stage (rebased per chunk) + sDstate_base_int = sDstate[0].base + # dBeta/dGate reduction scratch: f32 view of the owned dQ stage sred_base = cute.make_ptr(cutlass.Float32, sdQ[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) - # part_q scratch in sdH: consumed by dK-inter (covered by total_ready = - # the dQ-attn commit) and re-staged by CG1 itself later in the iteration - sdh_red = cute.make_ptr(cutlass.Float32, sdH[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) - dh_done_idx = cutlass.Int32(0) - cg1w = cg1_tidx // cfg.threads_per_warp + # part_q scratch in sDstate + sdstate_red = cute.make_ptr(cutlass.Float32, sDstate[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) + dstate_done_idx = cutlass.Int32(0) + cg1_warp_id = cg1_tidx // cfg.threads_per_warp sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) - S_MIN = 0 if cfg.use_initial_state else 1 + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 SFIRST_MIN = 1 if cfg.use_initial_state else 2 while tile_idx < total_tiles: batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) - sk_nt = cend - wstart - # ---- d_final_state prologue: seed the dH acc (f32) + the f16 - # entry staging (dh_inp + sdH) ---- - if cutlass.const_expr(cfg.use_dht): - if sk_nt > 0: - # split-K: only the item owning the sequence tail receives - # the true d_final_state; warmup items rebuild dH from zero - gDht = mDht[None, None, head_idx, batch_idx] - seed_from_dht = cend == num_chunks_b - dh_inp_idx = dh_inp_index.idx - bars.mb_dh_inp_done[dh_inp_idx].wait(dh_inp_index.phase) - dh_inp_index = advance(dh_inp_index, cfg.tmem_dh_inp_stages) + num_item_chunks = cend - wstart + + # ---- d_final_state prologue ---------------------------------------------- + if cutlass.const_expr(cfg.use_dstate_in): + if num_item_chunks > 0: + gDstate_in = mDstate_in[None, None, head_idx, batch_idx] + seed_from_dstate_in = cend == num_chunks_b + dstate_inp_idx = dstate_inp_index.idx + bars.mb_dstate_inp_done[dstate_inp_idx].wait(dstate_inp_index.phase) + dstate_inp_index = advance(dstate_inp_index, cfg.tmem_dstate_inp_stages) for sub in cutlass.range_constexpr(num_state_subs): - dht_vals = [] + dstate_in_vals = [] for kk in cutlass.range_constexpr(ldtm_width): - v = gDht[sub * ldtm_width + kk, cg1_tidx] + v = gDstate_in[sub * ldtm_width + kk, cg1_tidx] if cutlass.const_expr(cfg.split_k): - v = v if seed_from_dht else cutlass.Float32(0.0) - dht_vals.append(v) + v = v if seed_from_dstate_in else cutlass.Float32(0.0) + dstate_in_vals.append(v) nvvm.tcgen05_st( "32x32b", - nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dh_col + sub * ldtm_width, cutlass.Float32), - cutlass.Vector.from_elements(tuple(dht_vals), cutlass.Float32), + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dstate_acc_col + sub * ldtm_width, cutlass.Float32), + cutlass.Vector.from_elements(tuple(dstate_in_vals), cutlass.Float32), ) - dht_f16 = [fp32_to_fp16(dht_vals[2 * j], dht_vals[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] + dstate_in_f16 = [fp32_to_fp16(dstate_in_vals[2 * j], dstate_in_vals[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "32x32b", - nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dh_inp_col + sub * sttm_width, cutlass.Int32), - cutlass.Vector.from_elements(tuple(dht_f16), cutlass.Int32), + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dstate_inp_col + sub * sttm_width, cutlass.Int32), + cutlass.Vector.from_elements(tuple(dstate_in_f16), cutlass.Int32), ) nvvm.tcgen05_wait("store") - bars.mb_dh_inp_ready[dh_inp_idx].arrive() + bars.mb_dstate_inp_ready[dstate_inp_idx].arrive() - # sdH staging read back from the just-seeded acc - bars.mb_dhs_done[0].wait(dhs_index.phase) - dhs_index = advance(dhs_index, 1) - dhs_vecs = [] + bars.mb_dstate_smem_done[0].wait(dstate_smem_index.phase) + dstate_smem_index = advance(dstate_smem_index, 1) for b in cutlass.range_constexpr(2): - for hh in cutlass.range_constexpr(2): - dhs_vecs.append( - nvvm.tcgen05_ld( - "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + b * 16) << 16) + tmem_dh_col + hh * 64, cutlass.Float32), - num=8, - ) + for col_half in cutlass.range_constexpr(2): + dstate_smem_vec = nvvm.tcgen05_ld( + "16x256b", + nvvm.make_tmem_ptr(((tmem_warp_row + b * 16) << 16) + tmem_dstate_acc_col + col_half * 64, cutlass.Float32), + num=8, ) - for b in cutlass.range_constexpr(2): - for hh in cutlass.range_constexpr(2): - dhs_vec = dhs_vecs[b * 2 + hh] - dhs_f16 = [fp32_to_fp16(dhs_vec[2 * j], dhs_vec[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] + dstate_smem_f16 = [fp32_to_fp16(dstate_smem_vec[2 * j], dstate_smem_vec[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] for c in cutlass.range_constexpr(4): - dhs_row = hh * 64 + ov_tok + c * 16 + dstate_smem_row = col_half * 64 + frag_row + c * 16 nvvm.stmatrix( cutlass.inttoptr( - sdH_base_int + ((cg1_tidx // 64) * cfg.d_k * 64 + dhs_row * 64 + swizzle_xor_128b(dhs_row, ov_col + b * 16)) * 2, + sDstate_base_int + + ((cg1_tidx // 64) * cfg.d_k * 64 + dstate_smem_row * 64 + swizzle_xor_128b(dstate_smem_row, frag_col + b * 16)) * 2, cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [dhs_f16[c * 4 + 0], dhs_f16[c * 4 + 1], dhs_f16[c * 4 + 2], dhs_f16[c * 4 + 3]], + [dstate_smem_f16[c * 4 + 0], dstate_smem_f16[c * 4 + 1], dstate_smem_f16[c * 4 + 2], dstate_smem_f16[c * 4 + 3]], nvvm.MMALayout.COL, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_dhs_ready[0].arrive() + bars.mb_dstate_smem_ready[0].arrive() - if cutlass.const_expr(not cfg.use_dht): - if sk_nt > 0: - # ---- first backward chunk (c = NT-1): no dH yet ------------------- - gate_idx = gate_index.idx - bars.mb_gate_ready[gate_idx].wait(gate_index.phase) - gate_index = advance(gate_index, cfg.smem_gate_stages) - beta_idx = cg1_beta_index.idx - bars.mb_beta_ready[beta_idx].wait(cg1_beta_index.phase) - cg1_beta_index = advance(cg1_beta_index, cfg.smem_beta_stages) - num_vals = 32 - gCumprod = [] - for k in cutlass.range_constexpr(num_vals): - gCumprod.append(sCumprod[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, gate_idx]) - - # ---- dO' restage: dO * gCumprod * scale -> shared_inp TMEM ---- - bars.mb_dop_inp_done[0].wait(dop_inp_free.phase) - dop_inp_free = advance(dop_inp_free, 1) - do_idx = do_index.idx - bars.mb_do_ready[do_idx].wait(do_index.phase) - do_index = advance(do_index, cfg.smem_do_stages) - do_regs = [[cutlass.Float32(0.0), cutlass.Float32(0.0)] for _ in range(32)] - for c in cutlass.range_constexpr(8): - m0 = cutlass.const_expr(c % 4) - sub = cutlass.const_expr(c // 4) - do_f16 = nvvm.ldmatrix( - ( - sdO_base + do_idx * do_stage_elems + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) - ).raw_ptr(), - 4, - nvvm.MMALayout.COL, + # ---- chunks NT-1 .. 0 (backward) ------------------------------------------ + for rev_idx in cutlass.range(num_item_chunks): + chunk_idx = cend - 1 - rev_idx + have_dstate = cutlass.Boolean(True) if cutlass.const_expr(cfg.use_dstate_in) else rev_idx > 0 + gate_idx = gate_index.idx + bars.mb_gate_ready[gate_idx].wait(gate_index.phase) + gate_index = advance(gate_index, cfg.smem_gate_stages) + + beta_idx = cg1_beta_index.idx + bars.mb_beta_ready[beta_idx].wait(cg1_beta_index.phase) + cg1_beta_index = advance(cg1_beta_index, cfg.smem_beta_stages) + + # ---- dstate rescale: dstate *= this chunk's cumprod -------------------------- + if have_dstate: + cumprod_top = sCumprod[sCumprod.shape[0] - 1, 0, gate_idx] + for sub in cutlass.range_constexpr(num_state_subs): + dstate_rescale_vec = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dstate_acc_col + sub * ldtm_width, cutlass.Float32), num=32 ) - for i in cutlass.range_constexpr(4): - lo, hi = f16x2_to_f32(do_f16[i], dtype=cfg.io_dtype) - p0, p1 = fmul2(lo, hi, gCumprod[8 * m0 + 2 * i], gCumprod[8 * m0 + 2 * i + 1]) - do_regs[8 * m0 + 2 * i][sub], do_regs[8 * m0 + 2 * i + 1][sub] = fmul2(p0, p1, scale, scale) - for sub in cutlass.range_constexpr(2): - do_pack = [fp32_to_fp16(do_regs[2 * j][sub], do_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] + dstate_rescaled = [] + for j in cutlass.range_constexpr(16): + h0, h1 = fmul2(dstate_rescale_vec[2 * j], dstate_rescale_vec[2 * j + 1], cumprod_top, cumprod_top) + dstate_rescaled += [h0, h1] nvvm.tcgen05_st( - "16x128b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dop_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(do_pack), cutlass.Int32), - ) - nvvm.tcgen05_wait("store") - bars.mb_dop_inp_ready[0].arrive() - - # ---- v - k*state: delta = V - cumprod*(K @ S), in place ---- - v_idx = v_index.idx - bars.mb_v_ready[v_idx].wait(v_index.phase) - v_index = advance(v_index, cfg.smem_v_stages) - if cend >= SFIRST_MIN: - # V stays PACKED in the io dtype; the scaled KS is packed - # (needed for its own TMEM store anyway) and subtracted - # with packed 16-bit ops. - v_words = [[cutlass.Int32(0), cutlass.Int32(0)] for _ in range(16)] - for c in cutlass.range_constexpr(8): - m0 = cutlass.const_expr(c % 4) - sub = cutlass.const_expr(c // 4) - v_f16 = nvvm.ldmatrix( - ( - sV_base + v_idx * v_stage_elems + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) - ).raw_ptr(), - 4, - nvvm.MMALayout.COL, - ) - for i in cutlass.range_constexpr(4): - v_words[4 * m0 + i][sub] = v_f16[i] - bars.mb_ks_acc_ready[0].wait(cg1_ks_rdy.phase) - cg1_ks_rdy = advance(cg1_ks_rdy, 1) - ks_vecs = [] - for sub in cutlass.range_constexpr(2): - ks_vecs.append( - nvvm.tcgen05_ld( - "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_ks_col, cutlass.Float32), - num=8, - ) - ) - for sub in cutlass.range_constexpr(2): - gks_pack = [] - y_pack = [] - for j in cutlass.range_constexpr(16): - g0, g1 = fmul2(ks_vecs[sub][2 * j], ks_vecs[sub][2 * j + 1], gCumprod[2 * j], gCumprod[2 * j + 1]) - gks_word = fp32_to_fp16(g0, g1, dtype=cfg.io_dtype) - gks_pack.append(gks_word) - y_pack.append(sub_f16x2(v_words[j][sub], gks_word, cfg.io_dtype)) - nvvm.tcgen05_st( - "16x128b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(y_pack), cutlass.Int32), - ) - nvvm.tcgen05_st( - "16x128b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_gks_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(gks_pack), cutlass.Int32), - ) - if cend < SFIRST_MIN: - for sub in cutlass.range_constexpr(2): - v_pack = [] - for m0 in cutlass.range_constexpr(4): - v_f16 = nvvm.ldmatrix( - ( - sV_base + v_idx * v_stage_elems + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) - ).raw_ptr(), - 4, - nvvm.MMALayout.COL, - ) - for i in cutlass.range_constexpr(4): - v_pack.append(v_f16[i]) - nvvm.tcgen05_st( - "16x128b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(v_pack), cutlass.Int32), - ) - nvvm.tcgen05_wait("store") - bars.mb_y_ready[0].arrive() - - # ---- dU restage: dv acc ------------------------------------ - bars.mb_dyp_inp_done[0].wait(dyp_inp_free.phase) - dyp_inp_free = advance(dyp_inp_free, 1) - bars.mb_du_total_ready[0].wait(cg1_du_total_rdy.phase) - cg1_du_total_rdy = advance(cg1_du_total_rdy, 1) - du_vecs = [] - for sub in cutlass.range_constexpr(2): - du_vecs.append(nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_col, cutlass.Float32), num=8)) - for sub in cutlass.range_constexpr(2): - du_pack = [fp32_to_fp16(du_vecs[sub][2 * j], du_vecs[sub][2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] - nvvm.tcgen05_st( - "16x128b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_du_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(du_pack), cutlass.Int32), - ) - nvvm.tcgen05_wait("store") - bars.mb_du_inp_ready[0].arrive() - bars.mb_do_cg1_done[do_idx].arrive() - - # ---- U readout --------------------------------------------- - bars.mb_u_acc_ready[0].wait(cg1_u_rdy.phase) - cg1_u_rdy = advance(cg1_u_rdy, 1) - u_regs = [] - for sub in cutlass.range_constexpr(2): - u_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_u_col, cutlass.Float32), num=8) - u_regs.append([u_vec[k] for k in range(32)]) - for sub in cutlass.range_constexpr(2): - for m0 in cutlass.range_constexpr(4): - u_f16 = [fp32_to_fp16(u_regs[sub][8 * m0 + 2 * j], u_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] - nvvm.stmatrix( - ( - sV_base + v_idx * v_stage_elems + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) - ).raw_ptr(), - u_f16, - nvvm.MMALayout.COL, - ) - nvvm.fence_proxy("async.shared", space="cta") - bars.mb_u_ready[0].arrive() - bars.mb_v_cg1_done[v_idx].arrive() - - # ---- dY ---------------------------------------------------- - bars.mb_dy_acc_ready[0].wait(cg1_dy_rdy.phase) - cg1_dy_rdy = advance(cg1_dy_rdy, 1) - dy_regs = [] - for sub in cutlass.range_constexpr(2): - dy_vec = nvvm.tcgen05_ld( - "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dy_col, cutlass.Float32), - num=8, - ) - dy_regs.append([dy_vec[k] for k in range(32)]) - - # ---- dY' = -gCumprod * dY -> f16 shared_inp ---------------- - neg_one = cutlass.Float32(-1.0) - gCumprodNeg = [gCumprod[k] * neg_one for k in range(32)] - for sub in cutlass.range_constexpr(2): - dyp = [] - for j in cutlass.range_constexpr(16): - n0, n1 = fmul2(dy_regs[sub][2 * j], dy_regs[sub][2 * j + 1], gCumprodNeg[2 * j], gCumprodNeg[2 * j + 1]) - dyp += [n0, n1] - dyp_pack = [fp32_to_fp16(dyp[2 * j], dyp[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] - nvvm.tcgen05_st( - "16x128b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dyp_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(dyp_pack), cutlass.Int32), - ) - nvvm.tcgen05_wait("store") - bars.mb_dyp_inp_ready[0].arrive() - - # ---- dV staging: dV = dY -> the sdV slot ------------------- - dv_stg_idx = dv_index.idx - bars.mb_dv_tmastg_done[dv_stg_idx].wait(dv_index.phase) - dv_index = advance(dv_index, cfg.smem_dv_stages) - bars.mb_sdv_done[0].wait(sdv_done_index.phase) - sdv_done_index = advance(sdv_done_index, 1) - for sub in cutlass.range_constexpr(2): - for m0 in cutlass.range_constexpr(4): - dv_f16 = [fp32_to_fp16(dy_regs[sub][8 * m0 + 2 * j], dy_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] - nvvm.stmatrix( - ( - sdV_base - + dv_stg_idx * dv_stage_elems - + ov_slab - + (ov_tok + m0 * 16) * 64 - + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) - ).raw_ptr(), - dv_f16, - nvvm.MMALayout.COL, - ) - nvvm.fence_proxy("async.shared", space="cta") - bars.mb_dv_tmastg_ready[dv_stg_idx].arrive() - dk_stg_idx = dk_index.idx - bars.mb_dk_tmastg_done[dk_stg_idx].wait(dk_index.phase) - dk_index = advance(dk_index, cfg.smem_dk_stages) - dq_stg_idx = dq_index.idx - bars.mb_dq_tmastg_done[dq_stg_idx].wait(dq_index.phase) - dq_index = advance(dq_index, cfg.smem_dq_stages) - sred = sred_base + dq_stg_idx * (dq_stage_elems // 2) - - # ---- dBeta/dGate v-terms: dbeta_t += rowsum(dV ⊙ Y)_t / beta_t ---- - part_y = [cutlass.Float32(0.0)] * 16 - for sub in cutlass.range_constexpr(2): - y_pk = nvvm.tcgen05_ld("16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), num=8) - for j in cutlass.range_constexpr(16): - lo, hi = f16x2_to_f32(y_pk[j], dtype=cfg.io_dtype) - kk0 = cutlass.const_expr(2 * j) - j0 = cutlass.const_expr((kk0 // 4) * 2 + (kk0 % 2)) - if cutlass.const_expr(sub == 0 and j % 2 == 0): - part_y[j0], part_y[j0 + 1] = fmul2(dy_regs[sub][kk0], dy_regs[sub][kk0 + 1], lo, hi) - else: - part_y[j0], part_y[j0 + 1] = ffma2(dy_regs[sub][kk0], dy_regs[sub][kk0 + 1], lo, hi, part_y[j0], part_y[j0 + 1]) - py_lo, py_hi = _warp_reduce_scatter_frag16(part_y, lane_id) - vt_tok0 = (lane_id // 4) * 8 + (lane_id % 4) * 2 - if cend >= SFIRST_MIN: - part_g = [cutlass.Float32(0.0)] * 16 - for sub in cutlass.range_constexpr(2): - g_pk = nvvm.tcgen05_ld("16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_gks_col, cutlass.Int32), num=8) - for j in cutlass.range_constexpr(16): - lo, hi = f16x2_to_f32(g_pk[j], dtype=cfg.io_dtype) - kk0 = cutlass.const_expr(2 * j) - j0 = cutlass.const_expr((kk0 // 4) * 2 + (kk0 % 2)) - if cutlass.const_expr(sub == 0 and j % 2 == 0): - part_g[j0], part_g[j0 + 1] = fmul2(dy_regs[sub][kk0], dy_regs[sub][kk0 + 1], lo, hi) - else: - part_g[j0], part_g[j0 + 1] = ffma2(dy_regs[sub][kk0], dy_regs[sub][kk0 + 1], lo, hi, part_g[j0], part_g[j0 + 1]) - pg_lo, pg_hi = _warp_reduce_scatter_frag16(part_g, lane_id) - (sred + cg1w * 64 + vt_tok0).store(py_lo) - (sred + cg1w * 64 + vt_tok0 + 1).store(py_hi) - (sred + 256 + cg1w * 64 + vt_tok0).store(pg_lo) - (sred + 256 + cg1w * 64 + vt_tok0 + 1).store(pg_hi) - nvvm.barrier_cta_sync_aligned(cfg.cg1_barrier_id, thread_count=cfg.cg1_barrier_threads) - if cg1_tidx < 64: - binv_t = cute.math.rcp(sBeta[cg1_tidx, 0, beta_idx] + cutlass.Float32(1e-10), approx=True, ftz=True) - ysum = (sred + cg1_tidx).load() + (sred + 64 + cg1_tidx).load() + (sred + 128 + cg1_tidx).load() + (sred + 192 + cg1_tidx).load() - gsum = (sred + 256 + cg1_tidx).load() + (sred + 320 + cg1_tidx).load() + (sred + 384 + cg1_tidx).load() + (sred + 448 + cg1_tidx).load() - sBeta[cg1_tidx, 0, beta_idx] = ysum * binv_t - sCumsumlog[cg1_tidx, 0, gate_idx] = cutlass.Float32(0.0) - gsum - if cend < SFIRST_MIN: - (sred + cg1w * 64 + vt_tok0).store(py_lo) - (sred + cg1w * 64 + vt_tok0 + 1).store(py_hi) - nvvm.barrier_cta_sync_aligned(cfg.cg1_barrier_id, thread_count=cfg.cg1_barrier_threads) - if cg1_tidx < 64: - binv_t = cute.math.rcp(sBeta[cg1_tidx, 0, beta_idx] + cutlass.Float32(1e-10), approx=True, ftz=True) - ysum = (sred + cg1_tidx).load() + (sred + 64 + cg1_tidx).load() + (sred + 128 + cg1_tidx).load() + (sred + 192 + cg1_tidx).load() - sBeta[cg1_tidx, 0, beta_idx] = ysum * binv_t - sCumsumlog[cg1_tidx, 0, gate_idx] = cutlass.Float32(0.0) - bars.mb_beta_done[beta_idx].arrive() - bars.mb_dbeta_cg1_ready[0].arrive() - # sred reads must land before the dq stmatrix below reuses the stage - nvvm.barrier_cta_sync_aligned(cfg.cg1_barrier_id, thread_count=cfg.cg1_barrier_threads) - # ---- Q fragments held in registers over the dq stage (sQ - # free once read; the dQ dot consumes them -- no TMEM trip) ---- - q_frag = [] - for sub in cutlass.range_constexpr(2): - q_words = [] - for m0 in cutlass.range_constexpr(4): - q_f16 = nvvm.ldmatrix( - (sQ_base + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16)).raw_ptr(), - 4, - nvvm.MMALayout.COL, - ) - for i in cutlass.range_constexpr(4): - q_words.append(q_f16[i]) - q_frag.append(q_words) - # the store+wait order the sQ LDSM reads before the release arrive - # (a bare register consume can be hoisted past by ptxas); the dot - # below reads the registers, so the TMEM read-back is still gone - nvvm.tcgen05_st( - "16x128b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(q_words), cutlass.Int32), + "32x32b", + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dstate_acc_col + sub * ldtm_width, cutlass.Float32), + cutlass.Vector.from_elements(tuple(dstate_rescaled), cutlass.Float32), ) nvvm.tcgen05_wait("store") - bars.mb_q_cg1_done[0].arrive() - - # ---- dq final read -> sdQ (output staging; the fragments are - # held for the dQ dot below) --------- - bars.mb_dq_acc_total_ready[0].wait(dq_total_rdy_index.phase) - dq_total_rdy_index = advance(dq_total_rdy_index, 1) - dq_frag = [] - for sub in cutlass.range_constexpr(2): - dqv = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dh_inp_col, cutlass.Float32), num=8) - dq_frag.append([dqv[k] for k in range(32)]) - for sub in cutlass.range_constexpr(2): - for m0 in cutlass.range_constexpr(4): - frag_addr = ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) - dq_f16 = [fp32_to_fp16(dq_frag[sub][8 * m0 + 2 * j], dq_frag[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] - nvvm.stmatrix((sdQ_base + dq_stg_idx * dq_stage_elems + frag_addr).raw_ptr(), dq_f16, nvvm.MMALayout.COL) - nvvm.fence_proxy("async.shared", space="cta") - bars.mb_dq_acc_total_done[0].arrive() - bars.mb_dq_tmastg_ready[dq_stg_idx].arrive() - - # ---- dQ dot (part_q): fragment dot of the held dq acc with the - # staged Q^T, added to dGate FIRST; CG0 adds after parts_ready ---- - part_q = [cutlass.Float32(0.0)] * 16 - for sub in cutlass.range_constexpr(2): - for m0 in cutlass.range_constexpr(4): - for i in cutlass.range_constexpr(4): - q_lo, q_hi = f16x2_to_f32(q_frag[sub][4 * m0 + i], dtype=cfg.io_dtype) - kk0 = cutlass.const_expr(8 * m0 + 2 * i) - j0 = cutlass.const_expr((kk0 // 4) * 2 + (kk0 % 2)) - if cutlass.const_expr(sub == 0 and i % 2 == 0): - part_q[j0], part_q[j0 + 1] = fmul2(dq_frag[sub][kk0], dq_frag[sub][kk0 + 1], q_lo, q_hi) - else: - part_q[j0], part_q[j0 + 1] = ffma2(dq_frag[sub][kk0], dq_frag[sub][kk0 + 1], q_lo, q_hi, part_q[j0], part_q[j0 + 1]) - amq_lo, amq_hi = _warp_reduce_scatter_frag16(part_q, lane_id) - tok0 = (lane_id // 4) * 8 + (lane_id % 4) * 2 - (sdh_red + (cg1_tidx // 32) * 64 + tok0).store(amq_lo) - (sdh_red + (cg1_tidx // 32) * 64 + tok0 + 1).store(amq_hi) - nvvm.barrier_cta_sync_aligned(cfg.cg1_barrier_id, thread_count=cfg.cg1_barrier_threads) - if cg1_tidx < 64: - pq_sum = ( - (sdh_red + cg1_tidx).load() + (sdh_red + 64 + cg1_tidx).load() + (sdh_red + 128 + cg1_tidx).load() + (sdh_red + 192 + cg1_tidx).load() - ) - sCumsumlog[cg1_tidx, 0, gate_idx] = sCumsumlog[cg1_tidx, 0, gate_idx] + pq_sum - bars.mb_gate_done[gate_idx].arrive() - bars.mb_dgate_cg1_ready[0].arrive() - - # ---- NEXT-CHUNK dH prep (>= 2: single-chunk tiles have no - # consumer and the dh_acc_ready consume would starve the drain) ---- - if sk_nt >= 2: - dh_idx = dh_acc_index.idx - bars.mb_dh_acc_ready[dh_idx].wait(dh_acc_index.phase) - dh_acc_index = advance(dh_acc_index, cfg.tmem_dh_acc_stages) - dh_done_idx = dh_idx - dh_inp_idx = dh_inp_index.idx - bars.mb_dh_inp_done[dh_inp_idx].wait(dh_inp_index.phase) - dh_inp_index = advance(dh_inp_index, cfg.tmem_dh_inp_stages) - dh_regs = [[cutlass.Float32(0.0) for _ in range(num_state_subs)] for _ in range(32)] - # all loads issue before the first store: a TMEM store - # between loads pins ptxas to one load latency per sub - dh_vecs = [] - for sub in cutlass.range_constexpr(num_state_subs): - dh_vecs.append( - nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dh_col + sub * ldtm_width, cutlass.Float32), num=32) - ) - for sub in cutlass.range_constexpr(num_state_subs): - for k in cutlass.range_constexpr(32): - dh_regs[k][sub] = dh_vecs[sub][k] - - dh_f16 = [fp32_to_fp16(dh_regs[2 * j][sub], dh_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] - nvvm.tcgen05_st( - "32x32b", - nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dh_inp_col + sub * sttm_width, cutlass.Int32), - cutlass.Vector.from_elements(tuple(dh_f16), cutlass.Int32), - ) - nvvm.tcgen05_wait("store") - bars.mb_dh_inp_ready[dh_inp_idx].arrive() - - # ---- dK s-path fold: read while the dM-terms GEMMs run ------------ - if cend >= SFIRST_MIN: - bars.mb_dk_spath_ready[0].wait(cg1_dk_spath_rdy.phase) - cg1_dk_spath_rdy = advance(cg1_dk_spath_rdy, 1) - dk_spath_vecs = [] - for sub in cutlass.range_constexpr(2): - dk_spath_vecs.append( - nvvm.tcgen05_ld( - "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dk_spath_col, cutlass.Float32), - num=8, - ) - ) - nvvm.tcgen05_wait("load") - dk_spath_regs = [] - for sub in cutlass.range_constexpr(2): - dk_spath_row = [] - for j in cutlass.range_constexpr(16): - n0, n1 = fmul2(dk_spath_vecs[sub][2 * j], dk_spath_vecs[sub][2 * j + 1], gCumprodNeg[2 * j], gCumprodNeg[2 * j + 1]) - dk_spath_row += [n0, n1] - dk_spath_regs.append(dk_spath_row) - - bars.mb_dk_total_ready[0].wait(cg1_dk_total_rdy.phase) - cg1_dk_total_rdy = advance(cg1_dk_total_rdy, 1) - dmr_vecs = [] - for sub in cutlass.range_constexpr(2): - dmr_vecs.append( - nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_col, cutlass.Float32), num=8) - ) - nvvm.tcgen05_wait("load") - bars.mb_dk_total_done[0].arrive() - for sub in cutlass.range_constexpr(2): - dk_sum = [dmr_vecs[sub][k] + dk_spath_regs[sub][k] for k in range(32)] - for m0 in cutlass.range_constexpr(4): - dk_f16 = [fp32_to_fp16(dk_sum[8 * m0 + 2 * j], dk_sum[8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] - nvvm.stmatrix( - ( - sdK_base - + dk_stg_idx * dk_stage_elems - + ov_slab - + (ov_tok + m0 * 16) * 64 - + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) - ).raw_ptr(), - dk_f16, - nvvm.MMALayout.COL, - ) - nvvm.fence_proxy("async.shared", space="cta") - bars.mb_dk_tmastg_ready[dk_stg_idx].arrive() - if cend < SFIRST_MIN: - bars.mb_dk_total_ready[0].wait(cg1_dk_total_rdy.phase) - cg1_dk_total_rdy = advance(cg1_dk_total_rdy, 1) - dmr_vecs = [] - for sub in cutlass.range_constexpr(2): - dmr_vecs.append( - nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_col, cutlass.Float32), num=8) - ) - nvvm.tcgen05_wait("load") - bars.mb_dk_total_done[0].arrive() - for sub in cutlass.range_constexpr(2): - for m0 in cutlass.range_constexpr(4): - dk_f16 = [fp32_to_fp16(dmr_vecs[sub][8 * m0 + 2 * j], dmr_vecs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] - nvvm.stmatrix( - ( - sdK_base - + dk_stg_idx * dk_stage_elems - + ov_slab - + (ov_tok + m0 * 16) * 64 - + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) - ).raw_ptr(), - dk_f16, - nvvm.MMALayout.COL, - ) - nvvm.fence_proxy("async.shared", space="cta") - bars.mb_dk_tmastg_ready[dk_stg_idx].arrive() - - # ---- dH prep, sdH restage half (the dK readout above fires - # dk_total_done before this sweep) -------------------------- - if sk_nt >= 2: - # the sdH overwrite below waits only CG0's hdh read - bars.mb_hdh_done[0].wait(cg1_hdh_index.phase) - cg1_hdh_index = advance(cg1_hdh_index, 1) - bars.mb_dhs_done[0].wait(dhs_index.phase) - dhs_index = advance(dhs_index, 1) - for b in cutlass.range_constexpr(2): - for hh in cutlass.range_constexpr(2): - dhs_vec = nvvm.tcgen05_ld( - "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + b * 16) << 16) + tmem_dh_col + hh * 64, cutlass.Float32), - num=8, - ) - dhs_f16 = [fp32_to_fp16(dhs_vec[2 * j], dhs_vec[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] - for c in cutlass.range_constexpr(4): - dhs_row = hh * 64 + ov_tok + c * 16 - nvvm.stmatrix( - cutlass.inttoptr( - sdH_base_int + ((cg1_tidx // 64) * cfg.d_k * 64 + dhs_row * 64 + swizzle_xor_128b(dhs_row, ov_col + b * 16)) * 2, - cutlass.AddressSpace.smem, - cutlass.BFloat16, - ), - [dhs_f16[c * 4 + 0], dhs_f16[c * 4 + 1], dhs_f16[c * 4 + 2], dhs_f16[c * 4 + 3]], - nvvm.MMALayout.COL, - ) - nvvm.fence_proxy("async.shared", space="cta") - bars.mb_dhs_ready[0].arrive() - - if sk_nt < 2: - cg1_hdh_index = advance(cg1_hdh_index, 1) - - # ---- chunks NT-2 .. 0 (backward): full body ---------------------- - for rev_idx in cutlass.range(0 if cfg.use_dht else 1, sk_nt): - chunk_idx = cend - 1 - rev_idx - gate_idx = gate_index.idx - bars.mb_gate_ready[gate_idx].wait(gate_index.phase) - gate_index = advance(gate_index, cfg.smem_gate_stages) - - beta_idx = cg1_beta_index.idx - bars.mb_beta_ready[beta_idx].wait(cg1_beta_index.phase) - cg1_beta_index = advance(cg1_beta_index, cfg.smem_beta_stages) - - # ---- dH rescale: dH *= this chunk's cumprod ---- - cumprod_top = sCumprod[sCumprod.shape[0] - 1, 0, gate_idx] - dhr_vecs = [] - for sub in cutlass.range_constexpr(num_state_subs): - dhr_vecs.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dh_col + sub * ldtm_width, cutlass.Float32), num=32)) - for sub in cutlass.range_constexpr(num_state_subs): - dhr_scaled = [] - for j in cutlass.range_constexpr(16): - h0, h1 = fmul2(dhr_vecs[sub][2 * j], dhr_vecs[sub][2 * j + 1], cumprod_top, cumprod_top) - dhr_scaled += [h0, h1] - nvvm.tcgen05_st( - "32x32b", - nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dh_col + sub * ldtm_width, cutlass.Float32), - cutlass.Vector.from_elements(tuple(dhr_scaled), cutlass.Float32), - ) - nvvm.tcgen05_wait("store") - bars.mb_dh_acc_done[dh_done_idx].arrive() + bars.mb_dstate_acc_done[dstate_done_idx].arrive() num_vals = 32 - gDecayScale = [] - last_cumsumlog = sCumsumlog[cfg.b_t - 1, 0, gate_idx] - for k in cutlass.range_constexpr(num_vals): - gDecayScale.append(cute.math.exp2(last_cumsumlog - sCumsumlog[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, gate_idx], fastmath=True)) - gCumprod = [] - for k in cutlass.range_constexpr(num_vals): - gCumprod.append(sCumprod[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, gate_idx]) + cumprod_fp32 = [] + for g in cutlass.range_constexpr(8): + for b in cutlass.range_constexpr(2): + cumprod_fp32.append(sCumprod[(lane_id % 4) * 2 + g * 8 + b, 0, gate_idx]) + cumprod_vals = [cumprod_fp32[(k // 4) * 2 + (k % 2)] for k in range(num_vals)] - # ---- dO' restage: dO * gCumprod * scale -> shared_inp TMEM ---- - bars.mb_dop_inp_done[0].wait(dop_inp_free.phase) - dop_inp_free = advance(dop_inp_free, 1) + # ---- dO' restage: dO * cumprod_vals * scale -> shared_inp TMEM ----------- + bars.mb_do_prime_inp_done[0].wait(do_prime_inp_free.phase) + do_prime_inp_free = advance(do_prime_inp_free, 1) do_idx = do_index.idx bars.mb_do_ready[do_idx].wait(do_index.phase) do_index = advance(do_index, cfg.smem_do_stages) @@ -3435,101 +2680,121 @@ def _compute1_warp( m0 = cutlass.const_expr(c % 4) sub = cutlass.const_expr(c // 4) do_f16 = nvvm.ldmatrix( - (sdO_base + do_idx * do_stage_elems + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16)).raw_ptr(), + ( + sdO_base + + do_idx * do_stage_elems + + frag_slab_off + + (frag_row + m0 * 16) * 64 + + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) + ).raw_ptr(), 4, nvvm.MMALayout.COL, ) for i in cutlass.range_constexpr(4): lo, hi = f16x2_to_f32(do_f16[i], dtype=cfg.io_dtype) - p0, p1 = fmul2(lo, hi, gCumprod[8 * m0 + 2 * i], gCumprod[8 * m0 + 2 * i + 1]) + p0, p1 = fmul2(lo, hi, cumprod_vals[8 * m0 + 2 * i], cumprod_vals[8 * m0 + 2 * i + 1]) do_regs[8 * m0 + 2 * i][sub], do_regs[8 * m0 + 2 * i + 1][sub] = fmul2(p0, p1, scale, scale) for sub in cutlass.range_constexpr(2): do_pack = [fp32_to_fp16(do_regs[2 * j][sub], do_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "16x128b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dop_col, cutlass.Int32), + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_do_prime_col, cutlass.Int32), cutlass.Vector.from_elements(tuple(do_pack), cutlass.Int32), ) nvvm.tcgen05_wait("store") - bars.mb_dop_inp_ready[0].arrive() + bars.mb_do_prime_inp_ready[0].arrive() - # ---- dV inter: in-place decay scale ------------------------- - bars.mb_du_scale_ready[0].wait(cg1_du_scale_rdy.phase) - cg1_du_scale_rdy = advance(cg1_du_scale_rdy, 1) - dv_ptrs = [] - dv_vecs = [] - for sub in cutlass.range_constexpr(2): - dv_ptrs.append(nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_col, cutlass.Float32)) - dv_vecs.append(nvvm.tcgen05_ld("16x256b", dv_ptrs[sub], num=8)) - for sub in cutlass.range_constexpr(2): - dv_scaled = [] - for j in cutlass.range_constexpr(16): - s0, s1 = fmul2(dv_vecs[sub][2 * j], dv_vecs[sub][2 * j + 1], gDecayScale[2 * j], gDecayScale[2 * j + 1]) - dv_scaled += [s0, s1] - nvvm.tcgen05_st("16x256b", dv_ptrs[sub], cutlass.Vector.from_elements(tuple(dv_scaled), cutlass.Float32)) - nvvm.tcgen05_wait("store") - bars.mb_du_scale_done[0].arrive() + # ---- dV inter: in-place decay scale ---------------------------------- + if have_dstate: + last_cumsumlog = sCumsumlog[cfg.b_t - 1, 0, gate_idx] + col_cs_fp32 = [] + for g in cutlass.range_constexpr(8): + for b in cutlass.range_constexpr(2): + col_cs_fp32.append(sCumsumlog[(lane_id % 4) * 2 + g * 8 + b, 0, gate_idx]) + decay_scale_fp32 = [] + for i in cutlass.range_constexpr(16): + decay_scale_fp32.append(cute.math.exp2(last_cumsumlog - col_cs_fp32[i], fastmath=True)) + decay_scale_vals = [decay_scale_fp32[(k // 4) * 2 + (k % 2)] for k in range(num_vals)] + bars.mb_du_scale_ready[0].wait(cg1_du_scale_ready.phase) + cg1_du_scale_ready = advance(cg1_du_scale_ready, 1) + dv_ptrs = [] + dv_vecs = [] + for sub in cutlass.range_constexpr(2): + dv_ptrs.append(nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_acc_col, cutlass.Float32)) + dv_vecs.append(nvvm.tcgen05_ld("16x256b", dv_ptrs[sub], num=8)) + for sub in cutlass.range_constexpr(2): + dv_scaled = [] + for j in cutlass.range_constexpr(16): + s0, s1 = fmul2(dv_vecs[sub][2 * j], dv_vecs[sub][2 * j + 1], decay_scale_vals[2 * j], decay_scale_vals[2 * j + 1]) + dv_scaled += [s0, s1] + nvvm.tcgen05_st("16x256b", dv_ptrs[sub], cutlass.Vector.from_elements(tuple(dv_scaled), cutlass.Float32)) + nvvm.tcgen05_wait("store") + bars.mb_du_scale_done[0].arrive() - # ---- v - k*state: delta = V - cumprod*(K @ S), in place ---- + # ---- V - K*state: delta = V - cumprod*(K @ state), in place ------------- v_idx = v_index.idx bars.mb_v_ready[v_idx].wait(v_index.phase) v_index = advance(v_index, cfg.smem_v_stages) - if chunk_idx >= S_MIN: - v_regs = [[cutlass.Float32(0.0), cutlass.Float32(0.0)] for _ in range(32)] - gks_regs = [[cutlass.Float32(0.0), cutlass.Float32(0.0)] for _ in range(32)] + if chunk_idx >= FIRST_STATE_CHUNK: + v_words = [[cutlass.Int32(0), cutlass.Int32(0)] for _ in range(16)] for c in cutlass.range_constexpr(8): m0 = cutlass.const_expr(c % 4) sub = cutlass.const_expr(c // 4) v_f16 = nvvm.ldmatrix( - (sV_base + v_idx * v_stage_elems + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16)).raw_ptr(), + ( + sV_base + + v_idx * v_stage_elems + + frag_slab_off + + (frag_row + m0 * 16) * 64 + + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) + ).raw_ptr(), 4, nvvm.MMALayout.COL, ) for i in cutlass.range_constexpr(4): - lo, hi = f16x2_to_f32(v_f16[i], dtype=cfg.io_dtype) - v_regs[8 * m0 + 2 * i][sub] = lo - v_regs[8 * m0 + 2 * i + 1][sub] = hi - bars.mb_ks_acc_ready[0].wait(cg1_ks_rdy.phase) - cg1_ks_rdy = advance(cg1_ks_rdy, 1) + v_words[4 * m0 + i][sub] = v_f16[i] + bars.mb_k_state_acc_ready[0].wait(cg1_k_state_ready.phase) + cg1_k_state_ready = advance(cg1_k_state_ready, 1) for sub in cutlass.range_constexpr(2): - ks_vec = nvvm.tcgen05_ld( + k_state_vec = nvvm.tcgen05_ld( "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_ks_col, cutlass.Float32), + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_k_state_col, cutlass.Float32), num=8, ) + g_k_state_pack = [] + y_pack = [] for j in cutlass.range_constexpr(16): - g0, g1 = fmul2(ks_vec[2 * j], ks_vec[2 * j + 1], gCumprod[2 * j], gCumprod[2 * j + 1]) - gks_regs[2 * j][sub] = g0 - gks_regs[2 * j + 1][sub] = g1 - v_regs[2 * j][sub] = v_regs[2 * j][sub] - g0 - v_regs[2 * j + 1][sub] = v_regs[2 * j + 1][sub] - g1 - for sub in cutlass.range_constexpr(2): - y_pack = [fp32_to_fp16(v_regs[2 * j][sub], v_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] + g0, g1 = fmul2(k_state_vec[2 * j], k_state_vec[2 * j + 1], cumprod_vals[2 * j], cumprod_vals[2 * j + 1]) + g_k_state_word = fp32_to_fp16(g0, g1, dtype=cfg.io_dtype) + g_k_state_pack.append(g_k_state_word) + y_pack.append(sub_f16x2(v_words[j][sub], g_k_state_word, cfg.io_dtype)) nvvm.tcgen05_st( "16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), cutlass.Vector.from_elements(tuple(y_pack), cutlass.Int32), ) - gks_pack = [fp32_to_fp16(gks_regs[2 * j][sub], gks_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "16x128b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_gks_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(gks_pack), cutlass.Int32), + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_g_k_state_col, cutlass.Int32), + cutlass.Vector.from_elements(tuple(g_k_state_pack), cutlass.Int32), ) - if chunk_idx < S_MIN: + if chunk_idx < FIRST_STATE_CHUNK: for sub in cutlass.range_constexpr(2): v_pack = [] for m0 in cutlass.range_constexpr(4): v_f16 = nvvm.ldmatrix( ( - sV_base + v_idx * v_stage_elems + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) + sV_base + + v_idx * v_stage_elems + + frag_slab_off + + (frag_row + m0 * 16) * 64 + + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) ).raw_ptr(), 4, nvvm.MMALayout.COL, ) for i in cutlass.range_constexpr(4): - v_lo, v_hi = f16x2_to_f32(v_f16[i], dtype=cfg.io_dtype) - v_pack.append(fp32_to_fp16(v_lo, v_hi, dtype=cfg.io_dtype)) + v_pack.append(v_f16[i]) nvvm.tcgen05_st( "16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), @@ -3538,13 +2803,13 @@ def _compute1_warp( nvvm.tcgen05_wait("store") bars.mb_y_ready[0].arrive() - # ---- dU restage: dv acc ------------------------------------ + # ---- dU restage: dV acc ---------------------------------------------- bars.mb_dyp_inp_done[0].wait(dyp_inp_free.phase) dyp_inp_free = advance(dyp_inp_free, 1) - bars.mb_du_total_ready[0].wait(cg1_du_total_rdy.phase) - cg1_du_total_rdy = advance(cg1_du_total_rdy, 1) + bars.mb_du_total_ready[0].wait(cg1_du_total_ready.phase) + cg1_du_total_ready = advance(cg1_du_total_ready, 1) for sub in cutlass.range_constexpr(2): - du_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_col, cutlass.Float32), num=8) + du_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_acc_col, cutlass.Float32), num=8) du_pack = [fp32_to_fp16(du_vec[2 * j], du_vec[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "16x128b", @@ -3555,9 +2820,9 @@ def _compute1_warp( bars.mb_du_inp_ready[0].arrive() bars.mb_do_cg1_done[do_idx].arrive() - # ---- U readout --------------------------------------------- - bars.mb_u_acc_ready[0].wait(cg1_u_rdy.phase) - cg1_u_rdy = advance(cg1_u_rdy, 1) + # ---- U readout ------------------------------------------------------- + bars.mb_u_acc_ready[0].wait(cg1_u_ready.phase) + cg1_u_ready = advance(cg1_u_ready, 1) u_regs = [] for sub in cutlass.range_constexpr(2): u_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_u_col, cutlass.Float32), num=8) @@ -3566,7 +2831,13 @@ def _compute1_warp( for m0 in cutlass.range_constexpr(4): u_f16 = [fp32_to_fp16(u_regs[sub][8 * m0 + 2 * j], u_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] nvvm.stmatrix( - (sV_base + v_idx * v_stage_elems + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16)).raw_ptr(), + ( + sV_base + + v_idx * v_stage_elems + + frag_slab_off + + (frag_row + m0 * 16) * 64 + + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) + ).raw_ptr(), u_f16, nvvm.MMALayout.COL, ) @@ -3574,9 +2845,9 @@ def _compute1_warp( bars.mb_u_ready[0].arrive() bars.mb_v_cg1_done[v_idx].arrive() - # ---- dY ---------------------------------------------------- - bars.mb_dy_acc_ready[0].wait(cg1_dy_rdy.phase) - cg1_dy_rdy = advance(cg1_dy_rdy, 1) + # ---- dY -------------------------------------------------------------- + bars.mb_dy_acc_ready[0].wait(cg1_dy_ready.phase) + cg1_dy_ready = advance(cg1_dy_ready, 1) dy_regs = [] for sub in cutlass.range_constexpr(2): dy_vec = nvvm.tcgen05_ld( @@ -3586,13 +2857,13 @@ def _compute1_warp( ) dy_regs.append([dy_vec[k] for k in range(32)]) - # ---- dY' = -gCumprod * dY -> f16 shared_inp ---------------- + # ---- dY' = -cumprod_vals * dY -> f16 shared_inp -------------------------- neg_one = cutlass.Float32(-1.0) - gCumprodNeg = [gCumprod[k] * neg_one for k in range(32)] + cumprod_neg_vals = [cumprod_vals[k] * neg_one for k in range(32)] for sub in cutlass.range_constexpr(2): dyp = [] for j in cutlass.range_constexpr(16): - n0, n1 = fmul2(dy_regs[sub][2 * j], dy_regs[sub][2 * j + 1], gCumprodNeg[2 * j], gCumprodNeg[2 * j + 1]) + n0, n1 = fmul2(dy_regs[sub][2 * j], dy_regs[sub][2 * j + 1], cumprod_neg_vals[2 * j], cumprod_neg_vals[2 * j + 1]) dyp += [n0, n1] dyp_pack = [fp32_to_fp16(dyp[2 * j], dyp[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( @@ -3603,7 +2874,7 @@ def _compute1_warp( nvvm.tcgen05_wait("store") bars.mb_dyp_inp_ready[0].arrive() - # ---- dV staging: dV = dY -> the sdV slot ------------------- + # ---- dV staging: dV = dY -> the sdV slot ----------------------------- dv_stg_idx = dv_index.idx bars.mb_dv_tmastg_done[dv_stg_idx].wait(dv_index.phase) dv_index = advance(dv_index, cfg.smem_dv_stages) @@ -3614,7 +2885,11 @@ def _compute1_warp( dv_f16 = [fp32_to_fp16(dy_regs[sub][8 * m0 + 2 * j], dy_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] nvvm.stmatrix( ( - sdV_base + dv_stg_idx * dv_stage_elems + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) + sdV_base + + dv_stg_idx * dv_stage_elems + + frag_slab_off + + (frag_row + m0 * 16) * 64 + + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) ).raw_ptr(), dv_f16, nvvm.MMALayout.COL, @@ -3629,37 +2904,41 @@ def _compute1_warp( dq_index = advance(dq_index, cfg.smem_dq_stages) sred = sred_base + dq_stg_idx * (dq_stage_elems // 2) - # ---- dBeta/dGate v-terms: dbeta_t += rowsum(dV ⊙ Y)_t / beta_t ---- + # ---- dBeta/dGate v-terms: dBeta_t += rowsum(dV ⊙ Y)_t / Beta_t ------- part_y = [cutlass.Float32(0.0)] * 16 for sub in cutlass.range_constexpr(2): y_pk = nvvm.tcgen05_ld("16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), num=8) for j in cutlass.range_constexpr(16): lo, hi = f16x2_to_f32(y_pk[j], dtype=cfg.io_dtype) - kk0 = cutlass.const_expr(2 * j) - j0 = cutlass.const_expr((kk0 // 4) * 2 + (kk0 % 2)) + frag_e0 = cutlass.const_expr(2 * j) + part_e0 = cutlass.const_expr((frag_e0 // 4) * 2 + (frag_e0 % 2)) if cutlass.const_expr(sub == 0 and j % 2 == 0): - part_y[j0], part_y[j0 + 1] = fmul2(dy_regs[sub][kk0], dy_regs[sub][kk0 + 1], lo, hi) + part_y[part_e0], part_y[part_e0 + 1] = fmul2(dy_regs[sub][frag_e0], dy_regs[sub][frag_e0 + 1], lo, hi) else: - part_y[j0], part_y[j0 + 1] = ffma2(dy_regs[sub][kk0], dy_regs[sub][kk0 + 1], lo, hi, part_y[j0], part_y[j0 + 1]) - py_lo, py_hi = _warp_reduce_scatter_frag16(part_y, lane_id) + part_y[part_e0], part_y[part_e0 + 1] = ffma2( + dy_regs[sub][frag_e0], dy_regs[sub][frag_e0 + 1], lo, hi, part_y[part_e0], part_y[part_e0 + 1] + ) + py_lo, py_hi = warp_reduce_scatter_frag_16_elems(part_y, lane_id) vt_tok0 = (lane_id // 4) * 8 + (lane_id % 4) * 2 - if chunk_idx >= S_MIN: + if chunk_idx >= FIRST_STATE_CHUNK: part_g = [cutlass.Float32(0.0)] * 16 for sub in cutlass.range_constexpr(2): - g_pk = nvvm.tcgen05_ld("16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_gks_col, cutlass.Int32), num=8) + g_k_state_pk = nvvm.tcgen05_ld("16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_g_k_state_col, cutlass.Int32), num=8) for j in cutlass.range_constexpr(16): - lo, hi = f16x2_to_f32(g_pk[j], dtype=cfg.io_dtype) - kk0 = cutlass.const_expr(2 * j) - j0 = cutlass.const_expr((kk0 // 4) * 2 + (kk0 % 2)) + lo, hi = f16x2_to_f32(g_k_state_pk[j], dtype=cfg.io_dtype) + frag_e0 = cutlass.const_expr(2 * j) + part_e0 = cutlass.const_expr((frag_e0 // 4) * 2 + (frag_e0 % 2)) if cutlass.const_expr(sub == 0 and j % 2 == 0): - part_g[j0], part_g[j0 + 1] = fmul2(dy_regs[sub][kk0], dy_regs[sub][kk0 + 1], lo, hi) + part_g[part_e0], part_g[part_e0 + 1] = fmul2(dy_regs[sub][frag_e0], dy_regs[sub][frag_e0 + 1], lo, hi) else: - part_g[j0], part_g[j0 + 1] = ffma2(dy_regs[sub][kk0], dy_regs[sub][kk0 + 1], lo, hi, part_g[j0], part_g[j0 + 1]) - pg_lo, pg_hi = _warp_reduce_scatter_frag16(part_g, lane_id) - (sred + cg1w * 64 + vt_tok0).store(py_lo) - (sred + cg1w * 64 + vt_tok0 + 1).store(py_hi) - (sred + 256 + cg1w * 64 + vt_tok0).store(pg_lo) - (sred + 256 + cg1w * 64 + vt_tok0 + 1).store(pg_hi) + part_g[part_e0], part_g[part_e0 + 1] = ffma2( + dy_regs[sub][frag_e0], dy_regs[sub][frag_e0 + 1], lo, hi, part_g[part_e0], part_g[part_e0 + 1] + ) + pg_lo, pg_hi = warp_reduce_scatter_frag_16_elems(part_g, lane_id) + (sred + cg1_warp_id * 64 + vt_tok0).store(py_lo) + (sred + cg1_warp_id * 64 + vt_tok0 + 1).store(py_hi) + (sred + 256 + cg1_warp_id * 64 + vt_tok0).store(pg_lo) + (sred + 256 + cg1_warp_id * 64 + vt_tok0 + 1).store(pg_hi) nvvm.barrier_cta_sync_aligned(cfg.cg1_barrier_id, thread_count=cfg.cg1_barrier_threads) if cg1_tidx < 64: binv_t = cute.math.rcp(sBeta[cg1_tidx, 0, beta_idx] + cutlass.Float32(1e-10), approx=True, ftz=True) @@ -3667,9 +2946,9 @@ def _compute1_warp( gsum = (sred + 256 + cg1_tidx).load() + (sred + 320 + cg1_tidx).load() + (sred + 384 + cg1_tidx).load() + (sred + 448 + cg1_tidx).load() sBeta[cg1_tidx, 0, beta_idx] = ysum * binv_t sCumsumlog[cg1_tidx, 0, gate_idx] = cutlass.Float32(0.0) - gsum - if chunk_idx < S_MIN: - (sred + cg1w * 64 + vt_tok0).store(py_lo) - (sred + cg1w * 64 + vt_tok0 + 1).store(py_hi) + if chunk_idx < FIRST_STATE_CHUNK: + (sred + cg1_warp_id * 64 + vt_tok0).store(py_lo) + (sred + cg1_warp_id * 64 + vt_tok0 + 1).store(py_hi) nvvm.barrier_cta_sync_aligned(cfg.cg1_barrier_id, thread_count=cfg.cg1_barrier_threads) if cg1_tidx < 64: binv_t = cute.math.rcp(sBeta[cg1_tidx, 0, beta_idx] + cutlass.Float32(1e-10), approx=True, ftz=True) @@ -3678,26 +2957,21 @@ def _compute1_warp( sCumsumlog[cg1_tidx, 0, gate_idx] = cutlass.Float32(0.0) bars.mb_beta_done[beta_idx].arrive() bars.mb_dbeta_cg1_ready[0].arrive() - # sred reads must land before the dq stmatrix below reuses the stage nvvm.barrier_cta_sync_aligned(cfg.cg1_barrier_id, thread_count=cfg.cg1_barrier_threads) - # ---- Q fragments held in registers over the dq stage (sQ - # free once read; the dQ dot consumes them -- no TMEM trip) ---- + + # ---- Q fragments held in registers ----------------------------------- q_frag = [] for sub in cutlass.range_constexpr(2): q_words = [] for m0 in cutlass.range_constexpr(4): q_f16 = nvvm.ldmatrix( - (sQ_base + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16)).raw_ptr(), + (sQ_base + frag_slab_off + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16)).raw_ptr(), 4, nvvm.MMALayout.COL, ) for i in cutlass.range_constexpr(4): - q_lo, q_hi = f16x2_to_f32(q_f16[i], dtype=cfg.io_dtype) - q_words.append(fp32_to_fp16(q_lo, q_hi, dtype=cfg.io_dtype)) + q_words.append(q_f16[i]) q_frag.append(q_words) - # the store+wait order the sQ LDSM reads before the release arrive - # (a bare register consume can be hoisted past by ptxas); the dot - # below reads the registers, so the TMEM read-back is still gone nvvm.tcgen05_st( "16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), @@ -3706,122 +2980,133 @@ def _compute1_warp( nvvm.tcgen05_wait("store") bars.mb_q_cg1_done[0].arrive() - # ---- dq final read -> sdQ (output staging; the fragments are - # held for the dQ dot below) --------- - bars.mb_dq_acc_total_ready[0].wait(dq_total_rdy_index.phase) - dq_total_rdy_index = advance(dq_total_rdy_index, 1) + # ---- dQ final read -> sdQ -------------------------------------------- + bars.mb_dq_acc_total_ready[0].wait(dq_total_ready_index.phase) + dq_total_ready_index = advance(dq_total_ready_index, 1) dq_frag = [] for sub in cutlass.range_constexpr(2): - dqv = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dh_inp_col, cutlass.Float32), num=8) - dq_frag.append([dqv[k] for k in range(32)]) + dq_vals = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dstate_inp_col, cutlass.Float32), num=8) + dq_frag.append([dq_vals[k] for k in range(32)]) for m0 in cutlass.range_constexpr(4): - frag_addr = ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) - dq_f16 = [fp32_to_fp16(dqv[8 * m0 + 2 * j], dqv[8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + frag_addr = frag_slab_off + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) + dq_f16 = [fp32_to_fp16(dq_vals[8 * m0 + 2 * j], dq_vals[8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] nvvm.stmatrix((sdQ_base + dq_stg_idx * dq_stage_elems + frag_addr).raw_ptr(), dq_f16, nvvm.MMALayout.COL) nvvm.fence_proxy("async.shared", space="cta") bars.mb_dq_acc_total_done[0].arrive() bars.mb_dq_tmastg_ready[dq_stg_idx].arrive() - # ---- dQ dot (part_q): fragment dot of the held dq acc with the - # staged Q^T, added to dGate FIRST; CG0 adds after parts_ready ---- + # ---- dQ dot (part_q) ------------------------------------------------- part_q = [cutlass.Float32(0.0)] * 16 for sub in cutlass.range_constexpr(2): for m0 in cutlass.range_constexpr(4): for i in cutlass.range_constexpr(4): q_lo, q_hi = f16x2_to_f32(q_frag[sub][4 * m0 + i], dtype=cfg.io_dtype) - kk0 = cutlass.const_expr(8 * m0 + 2 * i) - j0 = cutlass.const_expr((kk0 // 4) * 2 + (kk0 % 2)) + frag_e0 = cutlass.const_expr(8 * m0 + 2 * i) + part_e0 = cutlass.const_expr((frag_e0 // 4) * 2 + (frag_e0 % 2)) if cutlass.const_expr(sub == 0 and i % 2 == 0): - part_q[j0], part_q[j0 + 1] = fmul2(dq_frag[sub][kk0], dq_frag[sub][kk0 + 1], q_lo, q_hi) + part_q[part_e0], part_q[part_e0 + 1] = fmul2(dq_frag[sub][frag_e0], dq_frag[sub][frag_e0 + 1], q_lo, q_hi) else: - part_q[j0], part_q[j0 + 1] = ffma2(dq_frag[sub][kk0], dq_frag[sub][kk0 + 1], q_lo, q_hi, part_q[j0], part_q[j0 + 1]) - amq_lo, amq_hi = _warp_reduce_scatter_frag16(part_q, lane_id) + part_q[part_e0], part_q[part_e0 + 1] = ffma2( + dq_frag[sub][frag_e0], dq_frag[sub][frag_e0 + 1], q_lo, q_hi, part_q[part_e0], part_q[part_e0 + 1] + ) + part_q_lo, part_q_hi = warp_reduce_scatter_frag_16_elems(part_q, lane_id) tok0 = (lane_id // 4) * 8 + (lane_id % 4) * 2 - (sdh_red + (cg1_tidx // 32) * 64 + tok0).store(amq_lo) - (sdh_red + (cg1_tidx // 32) * 64 + tok0 + 1).store(amq_hi) + (sdstate_red + (cg1_tidx // 32) * 64 + tok0).store(part_q_lo) + (sdstate_red + (cg1_tidx // 32) * 64 + tok0 + 1).store(part_q_hi) nvvm.barrier_cta_sync_aligned(cfg.cg1_barrier_id, thread_count=cfg.cg1_barrier_threads) if cg1_tidx < 64: - pq_sum = (sdh_red + cg1_tidx).load() + (sdh_red + 64 + cg1_tidx).load() + (sdh_red + 128 + cg1_tidx).load() + (sdh_red + 192 + cg1_tidx).load() + pq_sum = ( + (sdstate_red + cg1_tidx).load() + + (sdstate_red + 64 + cg1_tidx).load() + + (sdstate_red + 128 + cg1_tidx).load() + + (sdstate_red + 192 + cg1_tidx).load() + ) sCumsumlog[cg1_tidx, 0, gate_idx] = sCumsumlog[cg1_tidx, 0, gate_idx] + pq_sum bars.mb_gate_done[gate_idx].arrive() bars.mb_dgate_cg1_ready[0].arrive() - # ---- NEXT-CHUNK dH prep ------------------------------------ + # ---- NEXT-CHUNK dstate prep ---------------------------------------------- if chunk_idx >= wstart + 1: - dh_idx = dh_acc_index.idx - bars.mb_dh_acc_ready[dh_idx].wait(dh_acc_index.phase) - dh_acc_index = advance(dh_acc_index, cfg.tmem_dh_acc_stages) - dh_done_idx = dh_idx - dh_inp_idx = dh_inp_index.idx - bars.mb_dh_inp_done[dh_inp_idx].wait(dh_inp_index.phase) - dh_inp_index = advance(dh_inp_index, cfg.tmem_dh_inp_stages) - dh_regs = [[cutlass.Float32(0.0) for _ in range(num_state_subs)] for _ in range(32)] + dstate_idx = dstate_acc_index.idx + bars.mb_dstate_acc_ready[dstate_idx].wait(dstate_acc_index.phase) + dstate_acc_index = advance(dstate_acc_index, cfg.tmem_dstate_acc_stages) + dstate_done_idx = dstate_idx + dstate_inp_idx = dstate_inp_index.idx + bars.mb_dstate_inp_done[dstate_inp_idx].wait(dstate_inp_index.phase) + dstate_inp_index = advance(dstate_inp_index, cfg.tmem_dstate_inp_stages) + dstate_regs = [[cutlass.Float32(0.0) for _ in range(num_state_subs)] for _ in range(32)] for sub in cutlass.range_constexpr(num_state_subs): - dh_vec = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dh_col + sub * ldtm_width, cutlass.Float32), num=32) + dstate_vec = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dstate_acc_col + sub * ldtm_width, cutlass.Float32), num=32 + ) for k in cutlass.range_constexpr(32): - dh_regs[k][sub] = dh_vec[k] + dstate_regs[k][sub] = dstate_vec[k] - dh_f16 = [fp32_to_fp16(dh_regs[2 * j][sub], dh_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] + dstate_f16 = [fp32_to_fp16(dstate_regs[2 * j][sub], dstate_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "32x32b", - nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dh_inp_col + sub * sttm_width, cutlass.Int32), - cutlass.Vector.from_elements(tuple(dh_f16), cutlass.Int32), + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dstate_inp_col + sub * sttm_width, cutlass.Int32), + cutlass.Vector.from_elements(tuple(dstate_f16), cutlass.Int32), ) nvvm.tcgen05_wait("store") - bars.mb_dh_inp_ready[dh_inp_idx].arrive() + bars.mb_dstate_inp_ready[dstate_inp_idx].arrive() - # ---- dK s-path fold: read while the dM-terms GEMMs run ------------ - if chunk_idx >= S_MIN: - bars.mb_dk_spath_ready[0].wait(cg1_dk_spath_rdy.phase) - cg1_dk_spath_rdy = advance(cg1_dk_spath_rdy, 1) - dk_spath_vecs = [] + # ---- dK fold --------------------------------------------------------- + if chunk_idx >= FIRST_STATE_CHUNK: + bars.mb_dk_state_path_ready[0].wait(cg1_dk_state_path_ready.phase) + cg1_dk_state_path_ready = advance(cg1_dk_state_path_ready, 1) + dk_state_path_vecs = [] for sub in cutlass.range_constexpr(2): - dk_spath_vecs.append( + dk_state_path_vecs.append( nvvm.tcgen05_ld( "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dk_spath_col, cutlass.Float32), + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dk_state_path_col, cutlass.Float32), num=8, ) ) nvvm.tcgen05_wait("load") - dk_spath_regs = [] + dk_state_path_regs = [] for sub in cutlass.range_constexpr(2): - dk_spath_row = [] + dk_state_path_row = [] for j in cutlass.range_constexpr(16): - n0, n1 = fmul2(dk_spath_vecs[sub][2 * j], dk_spath_vecs[sub][2 * j + 1], gCumprodNeg[2 * j], gCumprodNeg[2 * j + 1]) - dk_spath_row += [n0, n1] - dk_spath_regs.append(dk_spath_row) + n0, n1 = fmul2(dk_state_path_vecs[sub][2 * j], dk_state_path_vecs[sub][2 * j + 1], cumprod_neg_vals[2 * j], cumprod_neg_vals[2 * j + 1]) + dk_state_path_row += [n0, n1] + dk_state_path_regs.append(dk_state_path_row) - bars.mb_dk_total_ready[0].wait(cg1_dk_total_rdy.phase) - cg1_dk_total_rdy = advance(cg1_dk_total_rdy, 1) + bars.mb_dk_total_ready[0].wait(cg1_dk_total_ready.phase) + cg1_dk_total_ready = advance(cg1_dk_total_ready, 1) dmr_vecs = [] for sub in cutlass.range_constexpr(2): - dmr_vecs.append(nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_col, cutlass.Float32), num=8)) + dmr_vecs.append( + nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_acc_col, cutlass.Float32), num=8) + ) nvvm.tcgen05_wait("load") bars.mb_dk_total_done[0].arrive() for sub in cutlass.range_constexpr(2): - dk_sum = [dmr_vecs[sub][k] + dk_spath_regs[sub][k] for k in range(32)] + dk_sum = [dmr_vecs[sub][k] + dk_state_path_regs[sub][k] for k in range(32)] for m0 in cutlass.range_constexpr(4): dk_f16 = [fp32_to_fp16(dk_sum[8 * m0 + 2 * j], dk_sum[8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] nvvm.stmatrix( ( sdK_base + dk_stg_idx * dk_stage_elems - + ov_slab - + (ov_tok + m0 * 16) * 64 - + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) + + frag_slab_off + + (frag_row + m0 * 16) * 64 + + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) ).raw_ptr(), dk_f16, nvvm.MMALayout.COL, ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_dk_tmastg_ready[dk_stg_idx].arrive() - if chunk_idx < S_MIN: - bars.mb_dk_total_ready[0].wait(cg1_dk_total_rdy.phase) - cg1_dk_total_rdy = advance(cg1_dk_total_rdy, 1) + if chunk_idx < FIRST_STATE_CHUNK: + bars.mb_dk_total_ready[0].wait(cg1_dk_total_ready.phase) + cg1_dk_total_ready = advance(cg1_dk_total_ready, 1) dmr_vecs = [] for sub in cutlass.range_constexpr(2): - dmr_vecs.append(nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_col, cutlass.Float32), num=8)) + dmr_vecs.append( + nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_acc_col, cutlass.Float32), num=8) + ) nvvm.tcgen05_wait("load") bars.mb_dk_total_done[0].arrive() for sub in cutlass.range_constexpr(2): @@ -3831,9 +3116,9 @@ def _compute1_warp( ( sdK_base + dk_stg_idx * dk_stage_elems - + ov_slab - + (ov_tok + m0 * 16) * 64 - + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) + + frag_slab_off + + (frag_row + m0 * 16) * 64 + + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) ).raw_ptr(), dk_f16, nvvm.MMALayout.COL, @@ -3841,96 +3126,87 @@ def _compute1_warp( nvvm.fence_proxy("async.shared", space="cta") bars.mb_dk_tmastg_ready[dk_stg_idx].arrive() - # ---- dH prep, sdH restage half (the dK readout above fires - # dk_total_done before this sweep) -------------------------- + # ---- dstate prep --------------------------------------------------------- if chunk_idx >= wstart + 1: - # the sdH overwrite below waits only CG0's hdh read - bars.mb_hdh_done[0].wait(cg1_hdh_index.phase) - cg1_hdh_index = advance(cg1_hdh_index, 1) - bars.mb_dhs_done[0].wait(dhs_index.phase) - dhs_index = advance(dhs_index, 1) - dhs_vecs = [] + bars.mb_state_dot_dstate_done[0].wait(cg1_state_dot_dstate_index.phase) + cg1_state_dot_dstate_index = advance(cg1_state_dot_dstate_index, 1) + bars.mb_dstate_smem_done[0].wait(dstate_smem_index.phase) + dstate_smem_index = advance(dstate_smem_index, 1) for b in cutlass.range_constexpr(2): - for hh in cutlass.range_constexpr(2): - dhs_vecs.append( - nvvm.tcgen05_ld( - "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + b * 16) << 16) + tmem_dh_col + hh * 64, cutlass.Float32), - num=8, - ) + for col_half in cutlass.range_constexpr(2): + dstate_smem_vec = nvvm.tcgen05_ld( + "16x256b", + nvvm.make_tmem_ptr(((tmem_warp_row + b * 16) << 16) + tmem_dstate_acc_col + col_half * 64, cutlass.Float32), + num=8, ) - for b in cutlass.range_constexpr(2): - for hh in cutlass.range_constexpr(2): - dhs_vec = dhs_vecs[b * 2 + hh] - dhs_f16 = [fp32_to_fp16(dhs_vec[2 * j], dhs_vec[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] + dstate_smem_f16 = [fp32_to_fp16(dstate_smem_vec[2 * j], dstate_smem_vec[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] for c in cutlass.range_constexpr(4): - dhs_row = hh * 64 + ov_tok + c * 16 + dstate_smem_row = col_half * 64 + frag_row + c * 16 nvvm.stmatrix( cutlass.inttoptr( - sdH_base_int + ((cg1_tidx // 64) * cfg.d_k * 64 + dhs_row * 64 + swizzle_xor_128b(dhs_row, ov_col + b * 16)) * 2, + sDstate_base_int + + ((cg1_tidx // 64) * cfg.d_k * 64 + dstate_smem_row * 64 + swizzle_xor_128b(dstate_smem_row, frag_col + b * 16)) * 2, cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [dhs_f16[c * 4 + 0], dhs_f16[c * 4 + 1], dhs_f16[c * 4 + 2], dhs_f16[c * 4 + 3]], + [dstate_smem_f16[c * 4 + 0], dstate_smem_f16[c * 4 + 1], dstate_smem_f16[c * 4 + 2], dstate_smem_f16[c * 4 + 3]], nvvm.MMALayout.COL, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_dhs_ready[0].arrive() + bars.mb_dstate_smem_ready[0].arrive() if chunk_idx < wstart + 1: - cg1_hdh_index = advance(cg1_hdh_index, 1) - - # ---- dH drain: with an initial state this is dL/dS0 ---- - if sk_nt > 0: - dh_idx = dh_acc_index.idx - bars.mb_dh_acc_ready[dh_idx].wait(dh_acc_index.phase) - dh_acc_index = advance(dh_acc_index, cfg.tmem_dh_acc_stages) - if cutlass.const_expr(cfg.use_initial_state): - # split-K: only the item owning chunk 0 drains dL/dS0 + cg1_state_dot_dstate_index = advance(cg1_state_dot_dstate_index, 1) + + # ---- dstate drain: with an initial state this is d_initial_state ---------------------- + if num_item_chunks > 0: + dstate_idx = dstate_acc_index.idx + bars.mb_dstate_acc_ready[dstate_idx].wait(dstate_acc_index.phase) + dstate_acc_index = advance(dstate_acc_index, cfg.tmem_dstate_acc_stages) + if cutlass.const_expr(cfg.use_dstate0): if cutlass.const_expr(cfg.split_k): if wstart == 0: - gDs0 = mDs0_out[None, None, head_idx, batch_idx] + gDstate0 = mDstate0_out[None, None, head_idx, batch_idx] for sub in cutlass.range_constexpr(num_state_subs): - ds0_vec = nvvm.tcgen05_ld( - "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dh_col + sub * ldtm_width, cutlass.Float32), num=32 + dstate0_vec = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dstate_acc_col + sub * ldtm_width, cutlass.Float32), num=32 ) for kk in cutlass.range_constexpr(32): - gDs0[sub * ldtm_width + kk, cg1_tidx] = ds0_vec[kk] + gDstate0[sub * ldtm_width + kk, cg1_tidx] = dstate0_vec[kk] else: - gDs0 = mDs0_out[None, None, head_idx, batch_idx] + gDstate0 = mDstate0_out[None, None, head_idx, batch_idx] for sub in cutlass.range_constexpr(num_state_subs): - ds0_vec = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dh_col + sub * ldtm_width, cutlass.Float32), num=32) + dstate0_vec = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dstate_acc_col + sub * ldtm_width, cutlass.Float32), num=32 + ) for kk in cutlass.range_constexpr(32): - gDs0[sub * ldtm_width + kk, cg1_tidx] = ds0_vec[kk] - if cutlass.const_expr(not cfg.use_dht): - bars.mb_dh_acc_done[dh_idx].arrive() + gDstate0[sub * ldtm_width + kk, cg1_tidx] = dstate0_vec[kk] + if cutlass.const_expr(not cfg.use_dstate_in): + bars.mb_dstate_acc_done[dstate_idx].arrive() else: - # zero-length sequence: the gradient passes straight through - # (dS0 = dHt when given, zeros otherwise); pure GMEM, no TMEM - if cutlass.const_expr(cfg.use_initial_state): + if cutlass.const_expr(cfg.use_dstate0): write_passthrough = True if cutlass.const_expr(cfg.split_k): write_passthrough = wstart == 0 if write_passthrough: - gDs0 = mDs0_out[None, None, head_idx, batch_idx] - if cutlass.const_expr(cfg.use_dht): - gDht = mDht[None, None, head_idx, batch_idx] + gDstate0 = mDstate0_out[None, None, head_idx, batch_idx] + if cutlass.const_expr(cfg.use_dstate_in): + gDstate_in = mDstate_in[None, None, head_idx, batch_idx] for sub in cutlass.range_constexpr(num_state_subs): for kk in cutlass.range_constexpr(32): - gDs0[sub * ldtm_width + kk, cg1_tidx] = gDht[sub * ldtm_width + kk, cg1_tidx] + gDstate0[sub * ldtm_width + kk, cg1_tidx] = gDstate_in[sub * ldtm_width + kk, cg1_tidx] else: for sub in cutlass.range_constexpr(num_state_subs): for kk in cutlass.range_constexpr(32): - gDs0[sub * ldtm_width + kk, cg1_tidx] = cutlass.Float32(0.0) + gDstate0[sub * ldtm_width + kk, cg1_tidx] = cutlass.Float32(0.0) - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) - # CG1 done with TMEM: release the MMA warp's dealloc bars.mb_tmem_done[0].arrive() - for _ in range(cfg.tmem_dh_inp_stages): - bars.mb_dh_inp_done[dh_inp_index.idx].wait(dh_inp_index.phase) - dh_inp_index = advance(dh_inp_index, cfg.tmem_dh_inp_stages) + for _ in range(cfg.tmem_dstate_inp_stages): + bars.mb_dstate_inp_done[dstate_inp_index.idx].wait(dstate_inp_index.phase) + dstate_inp_index = advance(dstate_inp_index, cfg.tmem_dstate_inp_stages) for _ in range(cfg.smem_dk_stages): bars.mb_dk_tmastg_done[dk_index.idx].wait(dk_index.phase) dk_index = advance(dk_index, cfg.smem_dk_stages) @@ -3939,8 +3215,94 @@ def _compute1_warp( dv_index = advance(dv_index, cfg.smem_dv_stages) +@cute.kernel +def build_all_descs_kernel( + base_q: cutlass.GridConstant[tma.TensorMap], + base_k: cutlass.GridConstant[tma.TensorMap], + base_v: cutlass.GridConstant[tma.TensorMap], + base_do: cutlass.GridConstant[tma.TensorMap], + base_checkpoint: cutlass.GridConstant[tma.TensorMap], + base_dq: cutlass.GridConstant[tma.TensorMap], + base_dk: cutlass.GridConstant[tma.TensorMap], + base_dv: cutlass.GridConstant[tma.TensorMap], + base_initial_state: cutlass.GridConstant[tma.TensorMap], + desc_ws: cute.Tensor, + cu_seqlens: cute.Tensor, + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + do_: cute.Tensor, + state_checkpoints: cute.Tensor, + dq: cute.Tensor, + dk: cute.Tensor, + dv: cute.Tensor, + state0: Optional[cute.Tensor], + n_batch: cutlass.Int32, + q_rs: cutlass.Int32, + k_rs: cutlass.Int32, + v_rs: cutlass.Int32, + do_rs: cutlass.Int32, + checkpoint_rs: cutlass.Int32, + checkpoint_every_n: cutlass.Int32, + dq_rs: cutlass.Int32, + dk_rs: cutlass.Int32, + dv_rs: cutlass.Int32, +) -> None: + """Single-launch builder for the per-BATCH descriptor arrays (one warp per array).""" + tidx, _, _ = cute.arch.thread_idx() + widx = cutlass.Int32(tidx) // cutlass.Int32(32) + arr_words = n_batch * cutlass.Int32(TENSOR_MAP_QWORDS) + sub0 = cute.make_tensor(desc_ws.iterator, cute.make_layout((arr_words,), stride=(1,))) + sub1 = cute.make_tensor(desc_ws.iterator + arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub2 = cute.make_tensor(desc_ws.iterator + 2 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub3 = cute.make_tensor(desc_ws.iterator + 3 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub4 = cute.make_tensor(desc_ws.iterator + 4 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub5 = cute.make_tensor(desc_ws.iterator + 5 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub6 = cute.make_tensor(desc_ws.iterator + 6 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub7 = cute.make_tensor(desc_ws.iterator + 7 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub8 = cute.make_tensor(desc_ws.iterator + 8 * arr_words, cute.make_layout((cutlass.Int32(TENSOR_MAP_QWORDS),), stride=(1,))) + + if widx == 0: + if nvvm.elect_sync(): + emit_seq_descs(base_q, sub0, cu_seqlens, q, n_batch, q_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 1: + if nvvm.elect_sync(): + emit_seq_descs(base_k, sub1, cu_seqlens, k, n_batch, k_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 2: + if nvvm.elect_sync(): + emit_seq_descs(base_v, sub2, cu_seqlens, v, n_batch, v_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 3: + if nvvm.elect_sync(): + emit_seq_descs(base_do, sub3, cu_seqlens, do_, n_batch, do_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 4: + if nvvm.elect_sync(): + emit_checkpoint_seq_descs(base_checkpoint, sub4, cu_seqlens, state_checkpoints, n_batch, checkpoint_rs, checkpoint_every_n, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 5: + if nvvm.elect_sync(): + emit_seq_descs(base_dq, sub5, cu_seqlens, dq, n_batch, dq_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 6: + if nvvm.elect_sync(): + emit_seq_descs(base_dk, sub6, cu_seqlens, dk, n_batch, dk_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 7: + if nvvm.elect_sync(): + emit_seq_descs(base_dv, sub7, cu_seqlens, dv, n_batch, dv_rs, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if cutlass.const_expr(state0 is not None): + if widx == 8: + if nvvm.elect_sync(): + emit_copy_desc(base_initial_state, sub8) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + + @cute.jit -def _build_descs( +def build_descs( io_dtype: cutlass.Constexpr, b_t: cutlass.Constexpr[int], q: cute.Tensor, @@ -3950,40 +3312,25 @@ def _build_descs( dq: cute.Tensor, dk: cute.Tensor, dv: cute.Tensor, - h: cute.Tensor, + state_checkpoints: cute.Tensor, cu_seqlens: cute.Tensor, - s0: Optional[cute.Tensor], + state0: Optional[cute.Tensor], tensormap_workspace: cute.Tensor, stream: cuda.CUstream, ): - """Build the per-(b,h) TMA-descriptor arrays (Q, K, V, dO, H loads; - dQ, dK, dV stores; the io-dtype S0 loads when ``s0`` is given) into - ``tensormap_workspace``. - - Launched on every execute: the descriptors fold cu_seqlens contents into - GLOBAL_ADDRESS and GLOBAL_DIM, which the host cannot read without a D2H sync. - - The H descriptor is 3-D ``(dv, dk, h)`` over the packed - ``[total_h, HO, DK, DV]`` H tensor; ``build_h_descs_kernel`` derives the - per-sequence H offset from the token ``cu_seqlens`` ((seqlen-1)//B_T, - prefix-summed), folds it and the head into GLOBAL_ADDRESS, and caps - GLOBAL_DIM[2] to the per-sequence H count, so the load coordinate is the - sequence-local H index. The S0 descriptors share the H descriptor - format (one ``[DK, DV]`` entry per slot over the dense - ``[N, HO, DK, DV]`` buffer), so the load path is interchangeable.""" + """Build the per-(b,h) TMA-descriptor arrays (Q, K, V, dO, checkpoint loads; + dQ, dK, dV stores; the io-dtype initial-state loads when ``state0`` is given) into + ``tensormap_workspace``.""" h_q = q.shape[1] h_k = k.shape[1] h_v = v.shape[1] batch_size = cu_seqlens.shape[0] - 1 heads_out = h_q if h_q >= h_v else h_v - q_group = heads_out // h_q - k_group = heads_out // h_k - v_group = heads_out // h_v d_v = v.shape[2] - d_k_h = h.shape[2] - d_v_h = h.shape[3] + d_k_state = state_checkpoints.shape[2] + d_v_state = state_checkpoints.shape[3] bpe = io_dtype.width // 8 - granu = 128 // bpe + granule_elems = 128 // bpe bt = b_t q_row_stride, q_head_stride = q.stride[0], q.stride[1] @@ -3994,161 +3341,87 @@ def _build_descs( dk_row_stride, dk_head_stride = dk.stride[0], dk.stride[1] dv_row_stride, dv_head_stride = dv.stride[0], dv.stride[1] - q_head0 = q[None, 0, None] - k_head0 = k[None, 0, None] + seqlen = q.shape[0] + d_k = q.shape[2] + q_headed = cute.make_tensor(q.iterator, cute.make_layout((seqlen, h_q, d_k), stride=(q_row_stride, q_head_stride, 1))) + k_headed = cute.make_tensor(k.iterator, cute.make_layout((seqlen, h_k, d_k), stride=(k_row_stride, k_head_stride, 1))) - def _trans_head0(t, d): - view = t[None, 0, None] + def trans_headed(t, d, hn): return cute.make_tensor( - view.iterator, - cute.make_layout((d, view.shape[0]), stride=(view.stride[1], view.stride[0])), + t.iterator, + cute.make_layout((d, hn, seqlen), stride=(1, t.stride[1], t.stride[0])), ) - v_head0 = _trans_head0(v, d_v) - do_head0 = _trans_head0(do_, d_v) - dq_head0 = _trans_head0(dq, dq.shape[2]) - dk_head0 = _trans_head0(dk, dk.shape[2]) - dv_head0 = _trans_head0(dv, d_v) - swz128 = _tma.TensorMapSwizzle.s128b - base_desc_q = _tma.create_tensor_map_tiled_from_view(q_head0, box_dims=(bt, granu), stride_order=(1, 0), swizzle=swz128) - base_desc_k = _tma.create_tensor_map_tiled_from_view(k_head0, box_dims=(bt, granu), stride_order=(1, 0), swizzle=swz128) - base_desc_v = _tma.create_tensor_map_tiled_from_view(v_head0, box_dims=(granu, bt), stride_order=(0, 1), swizzle=swz128) - base_desc_do = _tma.create_tensor_map_tiled_from_view(do_head0, box_dims=(granu, bt), stride_order=(0, 1), swizzle=swz128) - base_desc_dq = _tma.create_tensor_map_tiled_from_view(dq_head0, box_dims=(granu, bt), stride_order=(0, 1), swizzle=swz128) - base_desc_dk = _tma.create_tensor_map_tiled_from_view(dk_head0, box_dims=(granu, bt), stride_order=(0, 1), swizzle=swz128) - base_desc_dv = _tma.create_tensor_map_tiled_from_view(dv_head0, box_dims=(granu, bt), stride_order=(0, 1), swizzle=swz128) - h_view = cute.make_tensor( - h.iterator, + v_headed = trans_headed(v, d_v, h_v) + do_headed = trans_headed(do_, d_v, heads_out) + dq_headed = trans_headed(dq, dq.shape[2], heads_out) + dk_headed = trans_headed(dk, dk.shape[2], heads_out) + dv_headed = trans_headed(dv, d_v, heads_out) + swz128 = tma.TensorMapSwizzle.s128b + base_desc_q = tma.create_tensor_map_tiled_from_view(q_headed, box_dims=(bt, 1, granule_elems), stride_order=(2, 1, 0), swizzle=swz128) + base_desc_k = tma.create_tensor_map_tiled_from_view(k_headed, box_dims=(bt, 1, granule_elems), stride_order=(2, 1, 0), swizzle=swz128) + base_desc_v = tma.create_tensor_map_tiled_from_view(v_headed, box_dims=(granule_elems, 1, bt), stride_order=(0, 1, 2), swizzle=swz128) + base_desc_do = tma.create_tensor_map_tiled_from_view(do_headed, box_dims=(granule_elems, 1, bt), stride_order=(0, 1, 2), swizzle=swz128) + base_desc_dq = tma.create_tensor_map_tiled_from_view(dq_headed, box_dims=(granule_elems, 1, bt), stride_order=(0, 1, 2), swizzle=swz128) + base_desc_dk = tma.create_tensor_map_tiled_from_view(dk_headed, box_dims=(granule_elems, 1, bt), stride_order=(0, 1, 2), swizzle=swz128) + base_desc_dv = tma.create_tensor_map_tiled_from_view(dv_headed, box_dims=(granule_elems, 1, bt), stride_order=(0, 1, 2), swizzle=swz128) + checkpoint_view = cute.make_tensor( + state_checkpoints.iterator, cute.make_layout( - (d_v_h, d_k_h, h.shape[0]), - stride=(h.stride[3], h.stride[2], h.stride[0]), + (d_v_state, d_k_state, state_checkpoints.shape[0], heads_out), + stride=(state_checkpoints.stride[3], state_checkpoints.stride[2], state_checkpoints.stride[0], state_checkpoints.stride[1]), ), ) - base_desc_h = _tma.create_tensor_map_tiled_from_view(h_view, box_dims=(64, d_k_h, 1), stride_order=(0, 1, 2), swizzle=swz128) - - arr_words = (batch_size * heads_out) * TENSOR_MAP_QWORDS - ws_iter = tensormap_workspace.iterator + base_desc_checkpoint = tma.create_tensor_map_tiled_from_view(checkpoint_view, box_dims=(64, d_k_state, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz128) - def sub_array(i): - return cute.make_tensor(ws_iter + i * arr_words, cute.make_layout((arr_words,), stride=(1,))) + base_desc_state0 = base_desc_checkpoint + if cutlass.const_expr(state0 is not None): + initial_state_view = cute.make_tensor( + state0.iterator, + cute.make_layout( + (d_v_state, d_k_state, heads_out, batch_size), + stride=(state0.stride[3], state0.stride[2], state0.stride[1], state0.stride[0]), + ), + ) + base_desc_state0 = tma.create_tensor_map_tiled_from_view(initial_state_view, box_dims=(64, d_k_state, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz128) - build_qkv_load_descs_kernel( + n_warps = 9 if state0 is not None else 8 + build_all_descs_kernel( base_desc_q, - sub_array(0), - cu_seqlens, - q, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(q_group), - cutlass.Int32(q_head_stride), - cutlass.Int32(q_row_stride), - 1, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( base_desc_k, - sub_array(1), - cu_seqlens, - k, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(k_group), - cutlass.Int32(k_head_stride), - cutlass.Int32(k_row_stride), - 1, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( base_desc_v, - sub_array(2), - cu_seqlens, - v, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(v_group), - cutlass.Int32(v_head_stride), - cutlass.Int32(v_row_stride), - 1, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( base_desc_do, - sub_array(3), + base_desc_checkpoint, + base_desc_dq, + base_desc_dk, + base_desc_dv, + base_desc_state0, + tensormap_workspace, cu_seqlens, + q, + k, + v, do_, + state_checkpoints, + dq, + dk, + dv, + state0, cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(1), - cutlass.Int32(do_head_stride), + cutlass.Int32(q_row_stride), + cutlass.Int32(k_row_stride), + cutlass.Int32(v_row_stride), cutlass.Int32(do_row_stride), - 1, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_h_descs_kernel( - base_desc_h, - sub_array(4), - cu_seqlens, - h, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(h.stride[1]), - cutlass.Int32(h.stride[0]), + cutlass.Int32(state_checkpoints.stride[0]), cutlass.Int32(b_t), - 2, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_desc_dq, - sub_array(5), - cu_seqlens, - dq, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(1), - cutlass.Int32(dq_head_stride), cutlass.Int32(dq_row_stride), - 1, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_desc_dk, - sub_array(6), - cu_seqlens, - dk, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(1), - cutlass.Int32(dk_head_stride), cutlass.Int32(dk_row_stride), - 1, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_desc_dv, - sub_array(7), - cu_seqlens, - dv, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(1), - cutlass.Int32(dv_head_stride), cutlass.Int32(dv_row_stride), - 1, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - if cutlass.const_expr(s0 is not None): - s0_view = cute.make_tensor( - s0.iterator, - cute.make_layout( - (d_v_h, d_k_h, 1), - stride=(s0.stride[3], s0.stride[2], s0.stride[1]), - ), - ) - base_desc_s0 = _tma.create_tensor_map_tiled_from_view(s0_view, box_dims=(64, d_k_h, 1), stride_order=(0, 1, 2), swizzle=swz128) - build_state_descs_kernel( - base_desc_s0, - sub_array(8), - s0, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(s0.stride[1]), - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) + ).launch(grid=(1, 1, 1), block=(32 * n_warps, 1, 1), stream=stream) @cute.jit -def _host( +def host( cfg: cutlass.Constexpr, q: cute.Tensor, k: cute.Tensor, @@ -4162,8 +3435,8 @@ def _host( dk: cute.Tensor, dv: cute.Tensor, cu_seqlens: cute.Tensor, - ds0: Optional[cute.Tensor], - dht: Optional[cute.Tensor], + dstate0: Optional[cute.Tensor], + dstate_in: Optional[cute.Tensor], work_items: Optional[cute.Tensor], work_count: Optional[cute.Tensor], sched_ctr: Optional[cute.Tensor], @@ -4171,123 +3444,19 @@ def _host( tensormap_workspace: cute.Tensor, stream: cuda.CUstream, ): - h_q = q.shape[1] - h_v = v.shape[1] + h_q = cfg.h_q + h_k = cfg.h_k + h_v = cfg.h_v batch_size = cu_seqlens.shape[0] - 1 heads_out = h_q if h_q >= h_v else h_v - # ---- GQA reshapes: fold the head group into a ---------------------- - if cutlass.const_expr(cfg.is_GQA): - h_r = h_q // h_v - h_qv = h_v - q = cute.make_tensor( - q.iterator, - cute.make_layout( - (q.shape[0], q.shape[2], (h_r, h_v)), - stride=(q.stride[0], q.stride[2], (q.stride[1], h_r * q.stride[1])), - ), - ) - k = cute.make_tensor( - k.iterator, - cute.make_layout( - (k.shape[0], k.shape[2], (h_r, h_v)), - stride=(k.stride[0], k.stride[2], (0, k.stride[1])), - ), - ) - v = cute.make_tensor( - v.iterator, - cute.make_layout( - (v.shape[2], v.shape[0], (h_r, h_v)), - stride=(v.stride[2], v.stride[0], (0, v.stride[1])), - ), - ) - else: - h_r = h_v // h_q - h_qv = h_q - q = cute.make_tensor( - q.iterator, - cute.make_layout( - (q.shape[0], q.shape[2], (h_r, h_q)), - stride=(q.stride[0], q.stride[2], (0, q.stride[1])), - ), - ) - k = cute.make_tensor( - k.iterator, - cute.make_layout( - (k.shape[0], k.shape[2], (h_r, h_q)), - stride=(k.stride[0], k.stride[2], (0, k.stride[1])), - ), - ) - v = cute.make_tensor( - v.iterator, - cute.make_layout( - (v.shape[2], v.shape[0], (h_r, h_q)), - stride=(v.stride[2], v.stride[0], (v.stride[1], h_r * v.stride[1])), - ), - ) - - gate = cute.make_tensor( - gate.iterator, - cute.make_layout( - (gate.shape[0], (h_r, h_qv)), - stride=(gate.stride[0], (gate.stride[1], h_r * gate.stride[1])), - ), - ) - beta = cute.make_tensor( - beta.iterator, - cute.make_layout( - (beta.shape[0], (h_r, h_qv)), - stride=(beta.stride[0], (beta.stride[1], h_r * beta.stride[1])), - ), - ) - dg = cute.make_tensor( - dg.iterator, - cute.make_layout( - (dg.shape[0], (h_r, h_qv)), - stride=(dg.stride[0], (dg.stride[1], h_r * dg.stride[1])), - ), - ) - dbeta = cute.make_tensor( - dbeta.iterator, - cute.make_layout( - (dbeta.shape[0], (h_r, h_qv)), - stride=(dbeta.stride[0], (dbeta.stride[1], h_r * dbeta.stride[1])), - ), - ) - if cutlass.const_expr(ds0 is not None): - ds0 = cute.make_tensor( - ds0.iterator, - cute.make_layout( - (ds0.shape[2], ds0.shape[3], (h_r, h_qv), ds0.shape[0]), - stride=( - ds0.stride[2], - ds0.stride[3], - (ds0.stride[1], h_r * ds0.stride[1]), - ds0.stride[0], - ), - ), - ) - if cutlass.const_expr(dht is not None): - dht = cute.make_tensor( - dht.iterator, - cute.make_layout( - (dht.shape[2], dht.shape[3], (h_r, h_qv), dht.shape[0]), - stride=( - dht.stride[2], - dht.stride[3], - (dht.stride[1], h_r * dht.stride[1]), - dht.stride[0], - ), - ), - ) - - # ---- SMEM sizing: per-buffer element cosizes ----------------------- + # ---- SMEM sizing: per-buffer element cosizes --------------------------------- bpe = cfg.io_dtype.width // 8 q_tile_elems = cfg.b_t * cfg.d_k k_tile_elems = cfg.b_t * cfg.d_k v_tile_elems = cfg.d_v * cfg.b_t do_tile_elems = cfg.d_v * cfg.b_t - s_tile_elems = cfg.d_k * cfg.d_v + state_tile_elems = cfg.d_k * cfg.d_v ainv_tile_elems = cfg.b_t * cfg.b_t qk_tile_elems = cfg.b_t * cfg.b_t dq_tile_elems = cfg.b_t * cfg.d_k @@ -4297,7 +3466,7 @@ def _host( cfg.k_cosize = k_tile_elems * cfg.smem_k_stages cfg.v_cosize = v_tile_elems * cfg.smem_v_stages cfg.do_cosize = do_tile_elems * cfg.smem_do_stages - cfg.s_cosize = s_tile_elems * cfg.smem_s_stages + cfg.state_cosize = state_tile_elems * cfg.smem_state_stages cfg.ainv_cosize = ainv_tile_elems * cfg.smem_ainv_stages cfg.qk_cosize = qk_tile_elems * cfg.smem_qk_stages cfg.dq_cosize = dq_tile_elems * cfg.smem_dq_stages @@ -4311,17 +3480,19 @@ def _host( cfg.tma_k_bytes = k_tile_elems * bpe cfg.tma_v_bytes = v_tile_elems * bpe cfg.tma_do_bytes = do_tile_elems * bpe - cfg.tma_s_bytes = s_tile_elems * bpe + cfg.tma_state_bytes = state_tile_elems * bpe cfg.n_heads_out = heads_out - num_descs = batch_size * heads_out + cfg.q_ratio = heads_out // h_q + cfg.k_ratio = heads_out // h_k + cfg.v_ratio = heads_out // h_v + num_descs = batch_size - # ---- launch -------------------------------------------------------- + # ---- launch ------------------------------------------------------------------ total_tiles = batch_size * heads_out - # CUDA-graph-stable launch: fixed SM-count grid grid_shape = (cfg.max_active_clusters, 1, 1) - _kernel( + kernel( cfg, gate, beta, @@ -4339,8 +3510,8 @@ def _host( dq, dk, dv, - ds0, - dht, + dstate0, + dstate_in, work_items, work_count, sched_ctr, @@ -4356,7 +3527,7 @@ def _host( @cute.kernel -def _kernel( +def kernel( cfg: cutlass.Constexpr, mGate: cute.Tensor, mBeta: cute.Tensor, @@ -4374,21 +3545,15 @@ def _kernel( mdQ, mdK, mdV, - mDs0, - mDht, + mDstate0, + mDstate_in, mWorkItems: Optional[cute.Tensor], mCount: Optional[cute.Tensor], mSched: Optional[cute.Tensor], tensormap_workspace: cute.Tensor, n_desc: cutlass.Int32, ): - """ - Main GDN bprop chunked kernel. - - Warp specialization is the outermost control flow: each warp role owns - its own persistent tile-scheduler loop, iterating over (batch, head) - tiles and then over chunks within each tile in BACKWARD order. - """ + """Main GDN bprop chunked kernel (warp-specialized persistent body).""" tidx, _, _ = cute.arch.thread_idx() warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) bidx = cute.arch.block_idx()[0] @@ -4399,6 +3564,109 @@ def _kernel( if cutlass.const_expr(cfg.dyn_sched): assert mSched is not None, "mSched must be provided if dyn_sched is True" + if cutlass.const_expr(cfg.is_GQA): + h_r = cfg.h_q // cfg.h_v + h_qv = cfg.h_v + mQ = cute.make_tensor( + mQ.iterator, + cute.make_layout( + (mQ.shape[0], mQ.shape[2], (h_r, h_qv)), + stride=(mQ.stride[0], mQ.stride[2], (mQ.stride[1], h_r * mQ.stride[1])), + ), + ) + mK = cute.make_tensor( + mK.iterator, + cute.make_layout( + (mK.shape[0], mK.shape[2], (h_r, h_qv)), + stride=(mK.stride[0], mK.stride[2], (0, mK.stride[1])), + ), + ) + mV = cute.make_tensor( + mV.iterator, + cute.make_layout( + (mV.shape[2], mV.shape[0], (h_r, h_qv)), + stride=(mV.stride[2], mV.stride[0], (0, mV.stride[1])), + ), + ) + else: + h_r = cfg.h_v // cfg.h_q + h_qv = cfg.h_q + mQ = cute.make_tensor( + mQ.iterator, + cute.make_layout( + (mQ.shape[0], mQ.shape[2], (h_r, h_qv)), + stride=(mQ.stride[0], mQ.stride[2], (0, mQ.stride[1])), + ), + ) + mK = cute.make_tensor( + mK.iterator, + cute.make_layout( + (mK.shape[0], mK.shape[2], (h_r, h_qv)), + stride=(mK.stride[0], mK.stride[2], (0, mK.stride[1])), + ), + ) + mV = cute.make_tensor( + mV.iterator, + cute.make_layout( + (mV.shape[2], mV.shape[0], (h_r, h_qv)), + stride=(mV.stride[2], mV.stride[0], (mV.stride[1], h_r * mV.stride[1])), + ), + ) + mGate = cute.make_tensor( + mGate.iterator, + cute.make_layout( + (mGate.shape[0], (h_r, h_qv)), + stride=(mGate.stride[0], (mGate.stride[1], h_r * mGate.stride[1])), + ), + ) + mBeta = cute.make_tensor( + mBeta.iterator, + cute.make_layout( + (mBeta.shape[0], (h_r, h_qv)), + stride=(mBeta.stride[0], (mBeta.stride[1], h_r * mBeta.stride[1])), + ), + ) + mDg = cute.make_tensor( + mDg.iterator, + cute.make_layout( + (mDg.shape[0], (h_r, h_qv)), + stride=(mDg.stride[0], (mDg.stride[1], h_r * mDg.stride[1])), + ), + ) + mDbeta = cute.make_tensor( + mDbeta.iterator, + cute.make_layout( + (mDbeta.shape[0], (h_r, h_qv)), + stride=(mDbeta.stride[0], (mDbeta.stride[1], h_r * mDbeta.stride[1])), + ), + ) + if cutlass.const_expr(mDstate0 is not None): + mDstate0 = cute.make_tensor( + mDstate0.iterator, + cute.make_layout( + (mDstate0.shape[2], mDstate0.shape[3], (h_r, h_qv), mDstate0.shape[0]), + stride=( + mDstate0.stride[2], + mDstate0.stride[3], + (mDstate0.stride[1], h_r * mDstate0.stride[1]), + mDstate0.stride[0], + ), + ), + ) + if cutlass.const_expr(mDstate_in is not None): + mDstate_in = cute.make_tensor( + mDstate_in.iterator, + cute.make_layout( + (mDstate_in.shape[2], mDstate_in.shape[3], (h_r, h_qv), mDstate_in.shape[0]), + stride=( + mDstate_in.stride[2], + mDstate_in.stride[3], + (mDstate_in.stride[1], h_r * mDstate_in.stride[1]), + mDstate_in.stride[0], + ), + ), + ) + desc_base_words = tensormap_workspace.iterator.raw_ptr() desc_qwords = cutlass.Int32(TENSOR_MAP_QWORDS) arr_words = n_desc * desc_qwords @@ -4406,16 +3674,16 @@ def _kernel( desc_k_base = desc_base_words + arr_words desc_v_base = desc_base_words + cutlass.Int32(2) * arr_words desc_do_base = desc_base_words + cutlass.Int32(3) * arr_words - desc_s_base = desc_base_words + cutlass.Int32(4) * arr_words + desc_checkpoint_base = desc_base_words + cutlass.Int32(4) * arr_words desc_dq_base = desc_base_words + cutlass.Int32(5) * arr_words desc_dk_base = desc_base_words + cutlass.Int32(6) * arr_words desc_dv_base = desc_base_words + cutlass.Int32(7) * arr_words - desc_s0_base = desc_base_words + cutlass.Int32(8) * arr_words + desc_initial_state_base = desc_base_words + cutlass.Int32(8) * arr_words SMEM = cutlass.AddressSpace.smem bars = make_gdn_bars(cfg) sSched = cutlass.Array(cutlass.Int32, cfg.sched_stages, space=cutlass.AddressSpace.smem, alignment=16) - tmem_hold = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=16) + tmem_base_slot = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=16) cumsumlog_raw = cutlass.Array(cutlass.Float32, cute.cosize(cumsumlog_smem_layout_staged), space=SMEM, alignment=128) cumprod_raw = cutlass.Array(cutlass.Float32, cute.cosize(cumsumlog_smem_layout_staged), space=SMEM, alignment=128) beta_raw = cutlass.Array(cutlass.Float32, cute.cosize(beta_smem_layout_staged), space=SMEM, alignment=128) @@ -4426,7 +3694,7 @@ def _kernel( STRIDE = 8 * 128 KT_LEAD = (cfg.d_v // 2) * 128 V_LEAD = (cfg.d_v // 2) * 128 - S_LEAD = cfg.d_k * 128 + STATE_LEAD = cfg.d_k * 128 sQ_raw = cutlass.Array( cfg.io_dtype, cfg.q_cosize, @@ -4493,24 +3761,24 @@ def _kernel( stride_byte_offset=STRIDE, layout=SWZ, ) - sS_raw = cutlass.Array( + sState_raw = cutlass.Array( cfg.io_dtype, - cfg.s_cosize, + cfg.state_cosize, space=cutlass.AddressSpace.smem, alignment=cfg.buffer_align_bytes, ) - sS = SmemTile( - base=sS_raw.data_ptr().toint(), - elems_per_stage=(cfg.s_cosize // cfg.smem_s_stages) * bpe, - stages=cfg.smem_s_stages, - leading_byte_offset=S_LEAD, + sState = SmemTile( + base=sState_raw.data_ptr().toint(), + elems_per_stage=(cfg.state_cosize // cfg.smem_state_stages) * bpe, + stages=cfg.smem_state_stages, + leading_byte_offset=STATE_LEAD, stride_byte_offset=STRIDE, layout=SWZ, ) - sS_kmaj = SmemTile( - base=sS_raw.data_ptr().toint(), - elems_per_stage=(cfg.s_cosize // cfg.smem_s_stages) * bpe, - stages=cfg.smem_s_stages, + sState_kmaj = SmemTile( + base=sState_raw.data_ptr().toint(), + elems_per_stage=(cfg.state_cosize // cfg.smem_state_stages) * bpe, + stages=cfg.smem_state_stages, leading_byte_offset=LEAD, stride_byte_offset=STRIDE, layout=SWZ, @@ -4611,8 +3879,7 @@ def _kernel( stride_byte_offset=STRIDE, layout=SWZ, ) - # sub-bank split: V + sdH + dQ + dK + dV (the LDSM/LDS/STS/STSM-heavy - # set) allocated last -> all past the 128KB sub-bank boundary + # sub-bank split: V + sDstate + dQ + dK + dV allocated last sV_raw = cutlass.Array( cfg.io_dtype, cfg.v_cosize, @@ -4635,14 +3902,14 @@ def _kernel( stride_byte_offset=STRIDE, layout=SWZ, ) - sdH_raw = cutlass.Array( + sDstate_raw = cutlass.Array( cfg.io_dtype, cfg.d_k * cfg.d_v, space=cutlass.AddressSpace.smem, alignment=cfg.buffer_align_bytes, ) - sdH = SmemTile( - base=sdH_raw.data_ptr().toint(), + sDstate = SmemTile( + base=sDstate_raw.data_ptr().toint(), elems_per_stage=cfg.d_k * cfg.d_v * bpe, stages=1, leading_byte_offset=LEAD, @@ -4691,21 +3958,14 @@ def _kernel( stride_byte_offset=STRIDE, layout=SWZ, ) - sdV_kmaj = SmemTile( - base=sdV_raw.data_ptr().toint(), - elems_per_stage=(cfg.dv_cosize // cfg.smem_dv_stages) * bpe, - stages=cfg.smem_dv_stages, - leading_byte_offset=LEAD, - stride_byte_offset=STRIDE, - layout=SWZ, - ) - sdh_flat = cute.make_tensor( - cute.make_ptr(cfg.io_dtype, sdH_raw.data_ptr().toint(), mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes), + sdV_kmaj = sdV + sdstate_flat = cute.make_tensor( + cute.make_ptr(cfg.io_dtype, sDstate_raw.data_ptr().toint(), mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes), cute.make_layout(cfg.d_k * cfg.d_v), ) - ss_flat = cute.make_tensor( - cute.make_ptr(cfg.io_dtype, sS_raw.data_ptr().toint(), mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes), - cute.make_layout(cfg.s_cosize), + sstate_flat = cute.make_tensor( + cute.make_ptr(cfg.io_dtype, sState_raw.data_ptr().toint(), mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes), + cute.make_layout(cfg.state_cosize), ) sCumsumlog = cute.make_tensor( cute.make_ptr(cutlass.Float32, cumsumlog_raw.data_ptr().toint(), mem_space=cute.AddressSpace.smem, assumed_align=128), @@ -4720,9 +3980,7 @@ def _kernel( beta_smem_layout_staged, ) - # ------------------------------------------------------------------ - # mbarrier init (all threads) - # ------------------------------------------------------------------ + # ---- mbarrier init (all threads) --------------------------------------------- for s_ in range(cfg.sched_stages): bars.mb_sched_ready[s_].init() bars.mb_sched_done[s_].init() @@ -4742,18 +4000,19 @@ def _kernel( bars.mb_do_ready[s].init() bars.mb_do_mma_done[s].init() bars.mb_do_cg1_done[s].init() - for s in range(cfg.smem_s_stages): - bars.mb_s_ready[s].init() - bars.mb_s_done[s].init() + for s in range(cfg.smem_state_stages): + bars.mb_state_ready[s].init() + bars.mb_state_done[s].init() + bars.mb_state_cg0_done[s].init() for s in range(cfg.smem_gate_stages): bars.mb_gate_ready[s].init() bars.mb_gate_done[s].init() for s in range(cfg.smem_beta_stages): bars.mb_beta_ready[s].init() bars.mb_beta_done[s].init() - for s in range(cfg.tmem_dh_acc_stages): - bars.mb_dh_acc_ready[s].init() - bars.mb_dh_acc_done[s].init() + for s in range(cfg.tmem_dstate_acc_stages): + bars.mb_dstate_acc_ready[s].init() + bars.mb_dstate_acc_done[s].init() for b in ( bars.mb_du_scale_ready, bars.mb_du_scale_done, @@ -4770,7 +4029,7 @@ def _kernel( bars.mb_kk_acc_ready, bars.mb_kk_acc_done, bars.mb_a_acc_ready, - bars.mb_ks_acc_ready, + bars.mb_k_state_acc_ready, bars.mb_u_acc_ready, bars.mb_dy_acc_ready, ): @@ -4781,12 +4040,12 @@ def _kernel( for s in range(cfg.smem_qk_stages): bars.mb_qk_ready[s].init() bars.mb_qk_done[s].init() - for s in range(cfg.tmem_dh_inp_stages): - bars.mb_dh_inp_ready[s].init() - bars.mb_dh_inp_done[s].init() + for s in range(cfg.tmem_dstate_inp_stages): + bars.mb_dstate_inp_ready[s].init() + bars.mb_dstate_inp_done[s].init() for b in ( - bars.mb_dop_inp_ready, - bars.mb_dop_inp_done, + bars.mb_do_prime_inp_ready, + bars.mb_do_prime_inp_done, bars.mb_du_inp_ready, bars.mb_dyp_inp_ready, bars.mb_dyp_inp_done, @@ -4804,8 +4063,8 @@ def _kernel( bars.mb_y_ready[0].init() bars.mb_sdv_done[0].init() bars.mb_u_ready[0].init() - bars.mb_dhs_ready[0].init() - bars.mb_dhs_done[0].init() + bars.mb_dstate_smem_ready[0].init() + bars.mb_dstate_smem_done[0].init() bars.mb_da_ready[0].init() bars.mb_dq_acc_scale_ready[0].init() bars.mb_dq_acc_scale_done[0].init() @@ -4816,19 +4075,17 @@ def _kernel( bars.mb_dm_done[0].init() bars.mb_dbeta_cg1_ready[0].init() bars.mb_dgate_cg1_ready[0].init() - bars.mb_hdh_done[0].init() - bars.mb_dk_spath_ready[0].init() + bars.mb_state_dot_dstate_done[0].init() + bars.mb_dk_state_path_ready[0].init() bars.mb_tmem_done[0].init() nvvm.fence_mbarrier_init() nvvm.barrier_cta_sync() - # ------------------------------------------------------------------ - # 2. Warp specialization - each warp role owns its own scheduler loop - # ------------------------------------------------------------------ + # ---- warp specialization ----------------------------------------------------- if warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]: - _compute0_warp( + compute0_warp_group( cfg, total_tiles, bidx, @@ -4836,7 +4093,7 @@ def _kernel( cu_seqlens, mWorkItems, tidx, - tmem_hold=tmem_hold, + tmem_base_slot=tmem_base_slot, scale=scale, sCumsumlog=sCumsumlog, sCumprod=sCumprod, @@ -4848,26 +4105,26 @@ def _kernel( sDm=sDm, sK=sK, sdQ=sdQ, - sdH=sdH, - ss_flat=ss_flat, - sdh_flat=sdh_flat, + sDstate=sDstate, + sstate_flat=sstate_flat, + sdstate_flat=sdstate_flat, sSched=sSched, bars=bars, ) if warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]: - _compute1_warp( + compute1_warp_group( cfg, total_tiles, bidx, num_ctas, cu_seqlens, mWorkItems, - mDs0, - mDht, + mDstate0, + mDstate_in, tidx, warp_idx=warp_idx, - tmem_hold=tmem_hold, + tmem_base_slot=tmem_base_slot, scale=scale, sQ=sQ, sK=sK, @@ -4879,7 +4136,7 @@ def _kernel( sdQ=sdQ, sdK=sdK, sdV=sdV, - sdH=sdH, + sDstate=sDstate, sDa=sDa, sDm=sDm, sSched=sSched, @@ -4887,14 +4144,14 @@ def _kernel( ) elif warp_idx == cfg.mma_warp_id: - _mma_warp( + mma_warp( cfg, total_tiles, bidx, num_ctas, cu_seqlens, mWorkItems, - tmem_hold=tmem_hold, + tmem_base_slot=tmem_base_slot, sQ=sQ, sQ_trans=sQ_trans, sK=sK, @@ -4903,25 +4160,24 @@ def _kernel( sV_kmaj=sV_kmaj, sdO=sdO, sdO_kmaj=sdO_kmaj, - sS=sS, - sS_kmaj=sS_kmaj, + sState=sState, + sState_kmaj=sState_kmaj, sAinv=sAinv, sAinv_trans=sAinv_trans, sQk=sQk, sQk_trans=sQk_trans, sDa=sDa, sDa_trans=sDa_trans, - sdH=sdH, + sDstate=sDstate, sDm=sDm, sDm_trans=sDm_trans, - sdV=sdV, sdV_kmaj=sdV_kmaj, sSched=sSched, bars=bars, ) elif warp_idx == cfg.tma_qkv_warp_id: - _tmaldg_warp( + tmaldg_warp( cfg, total_tiles, bidx, @@ -4933,19 +4189,19 @@ def _kernel( sK_raw=sK_raw, sV_raw=sV_raw, sdO_raw=sdO_raw, - sS_raw=sS_raw, + sState_raw=sState_raw, desc_q_base=desc_q_base, desc_k_base=desc_k_base, desc_v_base=desc_v_base, desc_do_base=desc_do_base, - desc_s_base=desc_s_base, - desc_s0_base=desc_s0_base, + desc_checkpoint_base=desc_checkpoint_base, + desc_initial_state_base=desc_initial_state_base, sSched=sSched, bars=bars, ) if warp_idx == cfg.load_gate_beta_warp_id: - _gate_beta_warp( + gate_beta_warp( cfg, total_tiles, bidx, @@ -4964,7 +4220,7 @@ def _kernel( bars=bars, ) if warp_idx == cfg.epilogue_warp_id: - _tmastg_warp( + tmastg_warp( cfg, total_tiles, bidx, @@ -4988,23 +4244,20 @@ class GdnBwdCfg: The per-compile parameters (dtypes, GQA) are the ``cute.compile`` cache keys; the rest is derived from the module-global ``CFG`` constants. - ``_host`` stamps the shape-derived fields at trace time. + ``host`` stamps the shape-derived fields at trace time. """ use_initial_state: bool - use_dht: bool + use_dstate_in: bool + use_dstate0: bool io_dtype: Type[cutlass.Numeric] acc_dtype: Type[cutlass.Numeric] max_active_clusters: int is_GQA: bool - # split-K: tiles come from a work-item table (see common/split_k.py); - # each item computes chunks [wstart, cend) backward, writes [wstart, wend) split_k: bool = False - # gate input domain: natural-log decay instead of raw - # linear alpha; the gate warp then skips its log2 and rescales by 1/ln2 log_gate: bool = False - # --- fixed constants stamped from CFG by build_cfg --- + # ---- fixed constants stamped from CFG by build_cfg --------------------------- b_t: int = CFG.B_T d_k: int = CFG.D_K d_v: int = CFG.D_V @@ -5023,7 +4276,7 @@ class GdnBwdCfg: dyn_sched: bool = False sched_stages: int = CFG.SMEM_SCHED_STAGES - # --- named barrier slots (ids 1-6; 0 is the CTA-wide sync) --- + # ---- named barrier slots (ids 1-6; 0 is the CTA-wide sync) ------------------- tmem_alloc_barrier_id: int = 1 tmem_alloc_barrier_threads: int = 0 inverse_barrier_id: int = 2 @@ -5035,12 +4288,12 @@ class GdnBwdCfg: cg1_barrier_id: int = 5 cg1_barrier_threads: int = 0 - # --- SMEM / TMEM stage counts + TMEM column offsets --- + # ---- SMEM / TMEM stage counts + TMEM column offsets -------------------------- smem_q_stages: int = CFG.SMEM_Q_STAGES smem_k_stages: int = CFG.SMEM_K_STAGES smem_v_stages: int = CFG.SMEM_V_STAGES smem_do_stages: int = 1 - smem_s_stages: int = 1 + smem_state_stages: int = 1 smem_ainv_stages: int = CFG.SMEM_AINV_STAGES smem_qk_stages: int = CFG.SMEM_QK_STAGES smem_dq_stages: int = 1 @@ -5048,25 +4301,25 @@ class GdnBwdCfg: smem_dv_stages: int = 1 smem_gate_stages: int = 2 smem_beta_stages: int = 2 - tmem_dh_acc_stages: int = CFG.TMEM_DH_ACC_STAGES + tmem_dstate_acc_stages: int = CFG.TMEM_DH_ACC_STAGES tmem_dvdk_acc_stages: int = CFG.TMEM_DVDK_ACC_STAGES - tmem_dh_inp_stages: int = CFG.TMEM_DH_INP_STAGES + tmem_dstate_inp_stages: int = CFG.TMEM_DH_INP_STAGES tmem_shared_inp_stages: int = CFG.TMEM_SHARED_INP_STAGES tmem_shared_acc_stages: int = CFG.TMEM_SHARED_ACC_STAGES - tmem_dh_offset: int = 0 - tmem_dvdk_offset: int = 0 - tmem_dh_inp_offset: int = 0 + tmem_dstate_acc_offset: int = 0 + tmem_dvdk_acc_offset: int = 0 + tmem_dstate_inp_offset: int = 0 tmem_shared_acc_offset: int = 0 tmem_shared_inp_offset: int = 0 tmem_y_offset: int = 0 buffer_align_bytes: int = CFG.BUFFER_ALIGN_BYTES - # --- stamped by _host at trace time (shape-derived) --- + # ---- stamped by host at trace time (shape-derived) -------------------------- q_cosize: int = 0 k_cosize: int = 0 v_cosize: int = 0 do_cosize: int = 0 - s_cosize: int = 0 + state_cosize: int = 0 ainv_cosize: int = 0 qk_cosize: int = 0 dq_cosize: int = 0 @@ -5076,8 +4329,11 @@ class GdnBwdCfg: tma_k_bytes: int = 0 tma_v_bytes: int = 0 tma_do_bytes: int = 0 - tma_s_bytes: int = 0 + tma_state_bytes: int = 0 n_heads_out: int = 0 + q_ratio: int = 1 + k_ratio: int = 1 + v_ratio: int = 1 def build_cfg( @@ -5086,7 +4342,8 @@ def build_cfg( max_active_clusters: int, is_GQA: bool, use_initial_state: bool = False, - use_dht: bool = False, + use_dstate_in: bool = False, + use_dstate0: bool = False, split_k: bool = False, log_gate: bool = False, dyn_sched: bool = False, @@ -5097,7 +4354,8 @@ def build_cfg( raise ValueError(f"io_dtype={io_dtype} not supported; only Float16 and BFloat16 are supported") cfg = GdnBwdCfg( use_initial_state=use_initial_state, - use_dht=use_dht, + use_dstate_in=use_dstate_in, + use_dstate0=use_dstate0, io_dtype=io_dtype, acc_dtype=cutlass.Float32, max_active_clusters=max_active_clusters, @@ -5114,66 +4372,30 @@ def build_cfg( cfg.inverse_inner_barrier_threads = cfg.threads_per_warp * 2 cfg.init_state_store_barrier_threads = cfg.threads_per_warp * n_cg1 cfg.cg1_barrier_threads = cfg.threads_per_warp * n_cg1 - cfg.tmem_dh_offset = 0 - cfg.tmem_dvdk_offset = cfg.tmem_dh_offset + cfg.tmem_dh_acc_stages * 128 - cfg.tmem_dh_inp_offset = cfg.tmem_dvdk_offset + cfg.tmem_dvdk_acc_stages * 64 - cfg.tmem_shared_acc_offset = cfg.tmem_dh_inp_offset + cfg.tmem_dh_inp_stages * 64 + cfg.tmem_dstate_acc_offset = 0 + cfg.tmem_dvdk_acc_offset = cfg.tmem_dstate_acc_offset + cfg.tmem_dstate_acc_stages * 128 + cfg.tmem_dstate_inp_offset = cfg.tmem_dvdk_acc_offset + cfg.tmem_dvdk_acc_stages * 64 + cfg.tmem_shared_acc_offset = cfg.tmem_dstate_inp_offset + cfg.tmem_dstate_inp_stages * 64 cfg.tmem_shared_inp_offset = cfg.tmem_shared_acc_offset + cfg.tmem_shared_acc_stages * 64 cfg.tmem_y_offset = cfg.tmem_shared_inp_offset + cfg.tmem_shared_inp_stages * (cfg.b_t // 2) return cfg -def get_workspace_size(B: int, HQ: int, HV: int): - HO = HQ if HQ >= HV else HV - return CFG.BYTES_PER_TENSORMAP * (9 * B * HO) + 128 - - -def _check_cuda(err): - if err != cuda.CUresult.CUDA_SUCCESS: - raise RuntimeError(f"CUDA driver call failed: {err}") - - -def _data_ptr(t) -> int: - """Device address of a tensor-like (``data_ptr()`` or the CUDA array - interface).""" - fn = getattr(t, "data_ptr", None) - if fn is not None: - return fn() - return t.__cuda_array_interface__["data"][0] - - -def _device_sm_count() -> int: - """Multiprocessor count of the current device (runtime API: auto-inits - the primary context, so this works before any other CUDA call).""" - from cuda.bindings import runtime as _rt - - err, dev = _rt.cudaGetDevice() - if int(err) != 0: - raise RuntimeError(f"cudaGetDevice failed: {err}") - err, count = _rt.cudaDeviceGetAttribute(_rt.cudaDeviceAttr.cudaDevAttrMultiProcessorCount, dev) - if int(err) != 0: - raise RuntimeError(f"cudaDeviceGetAttribute failed: {err}") - return count - - -def _cutlass_io_dtype(dtype): - name = str(dtype) - if "bfloat16" in name: - return cutlass.BFloat16 - if "float16" in name or "half" in name: - return cutlass.Float16 - raise ValueError(f"Unsupported dtype {dtype}, expected bfloat16 or float16") +TENSORMAP_DESC_ARRAYS = 8 # per-batch runtime TMA descriptors: Q, K, V, dO, checkpoints, dQ, dK, dV +TENSORMAP_STATIC_SLOTS = 1 # initial_state @functools.cache -def _get_compiled_cache( +def get_compiled_cache( io_dtype_str: str, + cu_dtype_str: str, HQ: int, HK: int, HV: int, is_GQA: bool, use_initial_state: bool = False, - use_dht: bool = False, + use_dstate_in: bool = False, + use_dstate0: bool = False, split_k: bool = False, log_gate: bool = False, dyn_sched: bool = False, @@ -5186,12 +4408,16 @@ def compile( io_dtype, is_GQA: bool, use_initial_state: bool = False, - use_dht: bool = False, + use_dstate_in: bool = False, + use_dstate0: bool = False, split_k: bool = False, log_gate: bool = False, dyn_sched: bool = False, *, num_sm: int, + h_q: int, + h_k: int, + h_v: int, q_cute, k_cute, v_cute, @@ -5204,8 +4430,8 @@ def compile( dk_cute, dv_cute, cu_seqlens_cute, - ds0_cute=None, - dht_cute=None, + dstate0_cute=None, + dstate_in_cute=None, work_items_cute=None, work_count_cute=None, sched_ctr_cute=None, @@ -5219,14 +4445,18 @@ def compile( max_active_clusters=num_sm, is_GQA=is_GQA, use_initial_state=use_initial_state, - use_dht=use_dht, + use_dstate_in=use_dstate_in, + use_dstate0=use_dstate0, split_k=split_k, log_gate=log_gate, dyn_sched=dyn_sched, ) + cfg.h_q = h_q + cfg.h_k = h_k + cfg.h_v = h_v return cute.compile( - _host, + host, cfg, q_cute, k_cute, @@ -5240,8 +4470,8 @@ def compile( dk_cute, dv_cute, cu_seqlens_cute, - ds0_cute, - dht_cute, + dstate0_cute, + dstate_in_cute, work_items_cute, work_count_cute, sched_ctr_cute, @@ -5259,7 +4489,7 @@ def chunk_gdn_bwd_sm100( gate, beta, do, - h, + state_checkpoints, dq, dk, dv, @@ -5281,12 +4511,12 @@ def chunk_gdn_bwd_sm100( """Execute the Blackwell chunked GDN bprop kernel (THD / varlen entry). Produces dQ/dK/dV/dG/dBeta at ``HO = max(HQ, HV)`` heads (the caller - reduces over the head group; dgate = dL/d(ln alpha)). With + reduces over the head group; dGate = dL/d(ln alpha)). With ``initial_state`` (io dtype ``(num_seqs, HO, DK, DV)``, K-major — the caller downcasts its fp32 state), chunk 0's forward state loads from it through a dedicated per-(b,h) descriptor set; ``d_initial_state`` (fp32, - same shape) then also receives dL/dS0. The two go together. ``h`` is - always the PLAIN per-chunk series. All tensors are contiguous, + same shape) then also receives the initial-state gradient. The two go together. ``state_checkpoints`` is + always the PLAIN per-chunk checkpoint series. All tensors are contiguous, DLPack-compatible CUDA tensors on the same device. Compile-cache-and-replay. @@ -5298,84 +4528,73 @@ def chunk_gdn_bwd_sm100( alpha, or the natural-log decay when ``log_gate`` beta: ``(total_tokens, HO)`` float32, update gate do: ``(total_tokens, HO, DV)`` float16/bfloat16, output gradient - h: ``(total_h, HO, DK, DV)`` bfloat16, per-chunk forward states from - the prefill kernel's H output (``checkpoint_every_n_tokens=B_T``) + state_checkpoints: ``(total_checkpoints, HO, DK, DV)`` bfloat16, per-chunk + forward states from the prefill kernel's checkpoint output (``checkpoint_every_n_tokens=B_T``) dq/dk/dv: pre-allocated output gradients, shaped/typed like q/k/v at HO heads dg/dbeta: pre-allocated ``(total_tokens, HO)`` float32 gate/beta gradients cu_seqlens: ``(num_seqs + 1,)`` int32 - initial_state: ``(num_seqs, HO, DK, DV)`` io dtype (matching ``h``), + initial_state: ``(num_seqs, HO, DK, DV)`` io dtype (matching ``state_checkpoints``), or None scale: attention scale factor (must not be 0) - work_items: ``(max_items, 6)`` int32 split-K work-item table from + work_items: ``(max_items, 8)`` int32 split-K work-item table from ``common/split_k.py``, or None for the one-tile-per-(b,h) layout. With a table, each item computes chunks ``[wstart, cend)`` backward and writes gradients only for ``[wstart, wend)``. work_count: ``(1,)`` int32 device-side item count (required with work_items) - workspace: ``(>= get_workspace_size(B, HQ, HV) // 8,)`` int64, + workspace: ``(>= tensormap_workspace_bytes(module, B) // 8,)`` int64, 128-byte aligned; holds the per-(b,h) TMA descriptors stream: CUDA stream handle (``cudaStream_t`` as an int) """ HQ = q.shape[1] + HK = k.shape[1] HV = v.shape[1] + HO = max(HQ, HV) DK = q.shape[2] B = cu_seqlens.shape[0] - 1 is_GQA = HQ >= HV split_k = work_items is not None - io_dtype = _cutlass_io_dtype(q.dtype) - if str(h.dtype).split(".")[-1] != str(q.dtype).split(".")[-1]: - raise ValueError(f"h dtype must match the io dtype (the prefill H output): got {h.dtype} with io {q.dtype}") - if (initial_state is None) != (d_initial_state is None): - raise ValueError("initial_state and d_initial_state go together (chunk 0 reads S0; the drain produces dS0)") - if initial_state is not None and str(initial_state.dtype).split(".")[-1] != str(q.dtype).split(".")[-1]: - raise ValueError(f"initial_state must be io dtype (the caller downcasts): got {initial_state.dtype} with io {q.dtype}") - if split_k: - if work_count is None: - raise ValueError("work_count is required with work_items") - elif work_count is not None: - raise ValueError("work_count must be None without work_items") - - ws_words = get_workspace_size(B, HQ, HV) // 8 - if workspace.shape[0] < ws_words: - raise ValueError(f"workspace too small: need {ws_words} int64 words, " f"got {workspace.shape[0]}") - if _data_ptr(workspace) % 128 != 0: - raise ValueError("workspace must be 128-byte aligned") + io_dtype = get_dtype(q.dtype) + for name, hh in (("HQ", HQ), ("HK", HK), ("HV", HV)): + if HO % hh != 0: + raise ValueError(f"{name}={hh} must divide {HO}") + cu_stream = cuda.CUstream(int(stream)) dyn_sched = sched_ctr is not None - cache = _get_compiled_cache(str(q.dtype), HQ, k.shape[1], HV, is_GQA, d_initial_state is not None, d_final_state is not None, split_k, log_gate, dyn_sched) + cache = get_compiled_cache( + str(q.dtype), + str(cu_seqlens.dtype), + HQ, + HK, + HV, + is_GQA, + initial_state is not None, + d_final_state is not None, + d_initial_state is not None, + split_k, + log_gate, + dyn_sched, + ) if "compiled" not in cache: - def _tok3(t): - c = from_dlpack(t, assumed_align=16) - c.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - return c - - def _tok2(t): - c = from_dlpack(t, assumed_align=16) - c.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) - return c - cu_seqlens_cute = from_dlpack(cu_seqlens, assumed_align=4).mark_layout_dynamic() workspace_cute = from_dlpack(workspace, assumed_align=128).mark_layout_dynamic() - ds0_cute = None + dstate0_cute = None if d_initial_state is not None: - # prefill s_out marking (the drain reuses the fs-store indexing) - ds0_cute = from_dlpack(d_initial_state, assumed_align=16) - ds0_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=CFG.D_K) - dht_cute = None + dstate0_cute = from_dlpack(d_initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) + dstate_in_cute = None if d_final_state is not None: - dht_cute = from_dlpack(d_final_state, assumed_align=16) - dht_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=CFG.D_K) + dstate_in_cute = from_dlpack(d_final_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) work_items_cute = None work_count_cute = None if split_k: - work_items_cute = from_dlpack(work_items, assumed_align=4) + work_items_cute = from_dlpack(work_items, assumed_align=16) work_items_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) work_count_cute = from_dlpack(work_count, assumed_align=4).mark_layout_dynamic() sched_ctr_cute = None @@ -5384,26 +4603,30 @@ def _tok2(t): cache["compiled"] = compile( io_dtype, is_GQA, - use_initial_state=d_initial_state is not None, - use_dht=d_final_state is not None, + use_initial_state=initial_state is not None, + use_dstate_in=d_final_state is not None, + use_dstate0=d_initial_state is not None, split_k=split_k, log_gate=log_gate, dyn_sched=dyn_sched, - num_sm=_device_sm_count(), - q_cute=_tok3(q), - k_cute=_tok3(k), - v_cute=_tok3(v), - gate_cute=_tok2(gate), - beta_cute=_tok2(beta), - dg_cute=_tok2(dg), - dbeta_cute=_tok2(dbeta), - do_cute=_tok3(do), - dq_cute=_tok3(dq), - dk_cute=_tok3(dk), - dv_cute=_tok3(dv), + num_sm=multiprocessor_count(current_device_id()), + h_q=HQ, + h_k=HK, + h_v=HV, + q_cute=from_dlpack(q, assumed_align=16).mark_layout_dynamic(leading_dim=2), + k_cute=from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2), + v_cute=from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2), + gate_cute=from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=1), + beta_cute=from_dlpack(beta, assumed_align=16).mark_layout_dynamic(leading_dim=1), + dg_cute=from_dlpack(dg, assumed_align=16).mark_layout_dynamic(leading_dim=1), + dbeta_cute=from_dlpack(dbeta, assumed_align=16).mark_layout_dynamic(leading_dim=1), + do_cute=from_dlpack(do, assumed_align=16).mark_layout_dynamic(leading_dim=2), + dq_cute=from_dlpack(dq, assumed_align=16).mark_layout_dynamic(leading_dim=2), + dk_cute=from_dlpack(dk, assumed_align=16).mark_layout_dynamic(leading_dim=2), + dv_cute=from_dlpack(dv, assumed_align=16).mark_layout_dynamic(leading_dim=2), cu_seqlens_cute=cu_seqlens_cute, - ds0_cute=ds0_cute, - dht_cute=dht_cute, + dstate0_cute=dstate0_cute, + dstate_in_cute=dstate_in_cute, work_items_cute=work_items_cute, work_count_cute=work_count_cute, sched_ctr_cute=sched_ctr_cute, @@ -5414,47 +4637,36 @@ def _tok2(t): compiled = cache["compiled"] - # The descriptors encode cu_seqlens' CONTENTS, which no key built from the - # buffers can track. The skip this replaces asked torch's _version counter, - # so it was sound for a torch caller and silently stale for every other - # producer. Rebuilding unconditionally measures free: 131 vs 135 us of host - # time, and 157 either way once the launches are waited on. + # desc build runs every execute by contract (cu contents are data; + # buffer pointers may change) — capture-safe, single tiny launch if "build_descs" not in cache: - def _tok3_bc(t): - c = from_dlpack(t, assumed_align=16) - c.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - return c - - h_bc = from_dlpack(h, assumed_align=16) - h_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2, 3), divisibility=1) + checkpoints_bc = from_dlpack(state_checkpoints, assumed_align=16).mark_layout_dynamic(leading_dim=3) cu_bc = from_dlpack(cu_seqlens, assumed_align=4).mark_layout_dynamic() - s0_bc = None + state0_bc = None if initial_state is not None: - s0_bc = from_dlpack(initial_state, assumed_align=16) - s0_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2, 3), divisibility=1) + state0_bc = from_dlpack(initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) ws_bc = from_dlpack(workspace, assumed_align=128).mark_layout_dynamic() cache["build_descs"] = cute.compile( - _build_descs, + build_descs, io_dtype, CFG.B_T, - _tok3_bc(q), - _tok3_bc(k), - _tok3_bc(v), - _tok3_bc(do), - _tok3_bc(dq), - _tok3_bc(dk), - _tok3_bc(dv), - h_bc, + from_dlpack(q, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(do, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(dq, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(dk, assumed_align=16).mark_layout_dynamic(leading_dim=2), + from_dlpack(dv, assumed_align=16).mark_layout_dynamic(leading_dim=2), + checkpoints_bc, cu_bc, - s0_bc, + state0_bc, ws_bc, cu_stream, options="--enable-tvm-ffi", ) - cache["build_descs"](q, k, v, do, dq, dk, dv, h, cu_seqlens, initial_state, workspace, cu_stream) - + cache["build_descs"](q, k, v, do, dq, dk, dv, state_checkpoints, cu_seqlens, initial_state, workspace, cu_stream) compiled( q, k, diff --git a/python/cudnn/linear_attention/frost/kernel/gdn_prefill_config.py b/python/cudnn/linear_attention/frost/kernel/gdn_prefill_config.py index bce18e37c..2025dc6cf 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn_prefill_config.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn_prefill_config.py @@ -33,17 +33,14 @@ class Cfg: D_V: int = 128 # value head dim (M of GEMMs 3-6, N of GEMM 7) # --- TMA descriptor pool --- - BYTES_PER_TENSORMAP: int = 128 # --- warp assignments (12 warps total) --- COMPUTE_GROUP_0_WARP_IDS: Tuple[int, ...] = (0, 1, 2, 3) # T-pairwise / kk_epi / qk_epi / inverse COMPUTE_GROUP_1_WARP_IDS: Tuple[int, ...] = (4, 5, 6, 7) # kv_decay_v / v-k*state / epi ops - MMA_WARP_ID: int = 8 # CG0 issuer: KK/QK per pair + LOAD_GATE_BETA_WARP_ID: int = 8 # gate/beta chunk loads + TMEM lifecycle TMA_QKV_WARP_ID: int = 9 - MMA_CG1_WARP_ID: int = 10 # CG1 issuer: KS/QS/NV/QKV/KV per chunk - # The lightly loaded O/H epilogue warp also loads gate/beta. + MMA_WARP_ID: int = 10 # sole tcgen05 issuer: fused KK/QK pairs + KS/QS/NV/QKV/KV per chunk EPILOGUE_WARP_ID: int = 11 - LOAD_GATE_BETA_WARP_ID: int = 11 # --- register split --- NUM_REGS_COMPUTE_GROUP_0: int = 224 @@ -56,12 +53,11 @@ class Cfg: # --- SMEM stage counts --- SMEM_SCHED_STAGES: int = 2 - SMEM_Q_STAGES: int = 3 - SMEM_K_STAGES: int = 4 - SMEM_V_STAGES: int = 3 - SMEM_AINV_STAGES: int = 2 - SMEM_QK_STAGES: int = 2 - SMEM_O_STAGES: int = 2 + SMEM_KQ_STAGES: int = 4 + SMEM_V_STAGES: int = 2 + SMEM_AINV_STAGES: int = 3 + SMEM_QK_STAGES: int = 3 + SMEM_O_STAGES: int = 1 SMEM_GATE_STAGES: int = 3 SMEM_BETA_STAGES: int = 3 diff --git a/python/cudnn/linear_attention/frost/kernel/gdn_prefill_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn_prefill_f16.py index a19bc6864..16f670fc0 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn_prefill_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn_prefill_f16.py @@ -17,10 +17,10 @@ """ Chunked Gated Delta Net (GDN) prefill kernel for Blackwell SM100 (Cutlass primitives) -with optional per-chunk state (H) output. +with optional per-chunk state-checkpoint output. Algorithm overview (per chunk c, tokens [cC, (c+1)C)): - Inputs : Q[BT,DK], K[BT,DK], V[BT,DV], gate[BT] (scalar gate), beta[BT] (scalar LR) + Inputs : Q[BT,DK], K[BT,DK], V[BT,DV], Gate[BT] (scalar gate), Beta[BT] (scalar LR) State : S_prev[DK,DV] (recurrent state, held in TMEM) Preprocessing (compute warp group 0): @@ -29,56 +29,56 @@ T_pairwise[i,j] = cumprod[i] / cumprod[j] (i>=j) inter-token transfer weights (stored in registers; 128 regs/thread) - GEMM 1 - kk : W_kk[BT,BT] = K @ K^T (lower-triangular intra scores) - GEMM 2 - qk : W_qk[BT,BT] = Q @ K^T (output attention scores) - GEMM 3 - k*state : KS[BT,DV] = K @ S_prev (key applied to state) - GEMM 4 - q*state : QS[BT,DV] = Q @ S_prev (inter-chunk output, before T scaling) - GEMM 5 - new v : NV[BT,DV] = A_inv @ V (corrected value vectors) + GEMM 1 - KK : W_kk[BT,BT] = K @ K^T (lower-triangular intra scores) + GEMM 2 - QK : W_qk[BT,BT] = Q @ K^T (output attention scores) + GEMM 3 - K*state : KS[BT,DV] = K @ S_prev (key applied to state) + GEMM 4 - Q*state : QS[BT,DV] = Q @ S_prev (inter-chunk output, before T scaling) + GEMM 5 - new V : NV[BT,DV] = A_inv @ V (corrected value vectors) where A_inv = (I + M_kk)^{-1}, M_kk[i,j] = T[i,j]*beta[i]*W_kk[i,j] (lower-tri, hierarchical blockwise inverse) - GEMM 6 - qkv : O_intra[BT,DV] = W_qkv @ NV (intra-chunk output) + GEMM 6 - QKV : O_intra[BT,DV] = W_qkv @ NV (intra-chunk output) where W_qkv = T*beta*W_qk (scaled qk scores) - GEMM 7 - kv update : dS[DK,DV] = K^T @ delta (state update, BT contraction) + GEMM 7 - KV update : S_upd[DK,DV] = K^T @ delta (state update, BT contraction) where delta[BT,DV] = V - KS (delta rule residuals, after decay) Epilogue: O[BT,DV] = O_intra + T_col * QS (combine intra + inter) - S_next = cumprod[BT-1] * S_prev + dS (update state in TMEM) + S_next = cumprod[BT-1] * S_prev + S_upd (update state in TMEM) Chunks run in PAIRS (CG0 warp halves invert chunk 0 / chunk 1 in parallel); odd counts pad with a neutral zero-filled chunk. SMEM layout (227 KB = full; stage counts live in gdn_prefill_config.py; -enable_h compiles trim K/V stages to fit the H buffer): +enable_checkpoints compiles trim K/V stages to fit the checkpoint buffer): Buffer Size (B) Stages - q 16384 3 - k 16384 4 - v 16384 3 - A_inverse / new_v 8192 2 + Q 16384 3 + K 16384 4 + V 16384 3 + A_inverse / new_V 8192 2 QK output 8192 2 O store 16384 2 - H staging DK*DV*2 1 <-- enable_h only - cumsumlog / cumprod / beta 256 3 + checkpoint staging DK*DV*2 1 <-- enable_checkpoints only + cumsumlog / cumprod / Beta 256 3 sched ticket ring 4 2 <-- dyn_sched publish ring TMEM layout (512 columns): Buffer Cols - state (S) 128 <-- DKxDV fp32 = 128x128x4B - q*state / O acc 64 <-- BTxDV fp32 accumulator + state 128 <-- DKxDV fp32 = 128x128x4B + Q*state / O acc 64 <-- BTxDV fp32 accumulator state inp 64 <-- fp16 state staging (GEMMs 3/4 A operand) cg0 shared acc 128 <-- 2-stage ring: KK0/KK1 then QK0/QK1 cg1 shared acc 64 <-- 1-stage ring: KS then NV - vks+nv / decay_v inp 64 <-- slot 0 = VKS then NV, slot 1 = decay_v + v_k_state+NV / decay_V inp 64 <-- slot 0 = v_k_state then NV, slot 1 = decay_V Warp assignments (12 warps = 384 threads): - warps 0-3 : compute group 0 - T-pairwise x2, kk_epi x2, pair inverse, - qk_epi x2 - warps 4-7 : compute group 1 - state restage/rescale, v-k*state, - state*q_epi, new_v_epi, qkv_epilogue - warp 8 : CG0 MMA issuer - KK0/KK1/QK0/QK1 per pair; TMEM lifecycle - warp 9 : TMA load warp - loads q, k, v - warp 10 : CG1 MMA issuer - KS/QS/NV/QKV/KV per chunk - warp 11 : epilogue warp - gate/beta loads (4-chunk lookahead) + - O then H TMA stores + warps 0-3 : compute group 0 - T-pairwise x2, KK_epi x2, pair inverse, + QK_epi x2 + warps 4-7 : compute group 1 - state restage/rescale, V-K*state, + state*Q_epi, new_V_epi, QKV_epilogue + warp 8 : Gate/Beta loads + warp 9 : TMA load warp - loads Q, K, V + warp 10 : MMA warp - fused KK/QK pairs + k_state/q_state/NV/QKV/KV per chunk; + TMEM lifecycle + warp 11 : epilogue warp - O then checkpoint TMA stores """ import functools @@ -90,13 +90,15 @@ import cutlass import cutlass.cute as cute import cutlass.experimental.primitives as nvvm -import cutlass.experimental.cuda.tensor_map as _tma -from cutlass.cute.arch.nvvm_wrappers import inline_ptx +import cutlass.experimental.cuda.tensor_map as tma from cutlass.cute.runtime import from_dlpack -from cutlass.cutlass_dsl import min as _cutlass_min +from cutlass.cutlass_dsl import min -from ..common.thd import build_h_descs_kernel, build_qkv_load_descs_kernel, downcast_state_kernel, TENSOR_MAP_QWORDS +from ..common.thd import emit_checkpoint_seq_descs, emit_seq_descs, TENSOR_MAP_QWORDS from ..common.split_k import decode_work_item +from ..common.host import get_dtype +from cudnn.frost.buffers import current_device_id, data_ptr +from cudnn.frost.device import multiprocessor_count RCP_LN2 = 1.4426950408889634 # 1/ln(2): natural-log gates -> the kernel's log2 domain from cudnn.frost.tile_dsl.barrier import ( @@ -104,10 +106,9 @@ Producer, PipelineState, advance, - arrive, ) from cudnn.frost.tile_dsl.handles import MmaDesc, SmemTile, tma_slice_runtime_desc -from cudnn.frost.tile_dsl.mma import mma_ss, mma_ts, mma_step +from cudnn.frost.tile_dsl.mma import mma_ss, mma_step_k8, mma_ts_step, mma_step from cudnn.frost.tile_dsl.pointwise import fadd2, fp32_to_fp16, f16x2_to_f32, fmul2, opaque_f32_zero, sub_f16x2 from cudnn.frost.tile_dsl.swizzle import swizzle_lin_128b, swizzle_xor_128b from cudnn.frost.tile_dsl.tma import ( @@ -129,10 +130,8 @@ class GdnBars(NamedTuple): slot by arriving ``_done``. """ - mb_q_ready: MBarrier - mb_q_done: MBarrier - mb_k_ready: MBarrier - mb_k_done: MBarrier + mb_kq_ready: MBarrier + mb_kq_done: MBarrier mb_v_ready: MBarrier mb_v_done: MBarrier @@ -141,43 +140,45 @@ class GdnBars(NamedTuple): mb_beta_ready: MBarrier mb_beta_done: MBarrier - mb_kv_acc_ready: MBarrier - mb_kv_acc_scale_done: MBarrier + mb_state_acc_ready: MBarrier + mb_state_acc_scale_done: MBarrier mb_o_acc_ready: MBarrier mb_o_acc_done: MBarrier mb_o_state_scale_acc_ready: MBarrier mb_o_state_scale_acc_done: MBarrier mb_cg0_acc_ready: MBarrier mb_cg0_acc_done: MBarrier - mb_ks_ready: MBarrier - mb_nv_ready: MBarrier + + mb_state_inp_ready: MBarrier + mb_v_k_state_inp_ready: MBarrier + mb_nv_inp_ready: MBarrier + mb_decay_v_inp_ready: MBarrier mb_ainv_ready: MBarrier mb_ainv_done: MBarrier mb_qk_ready: MBarrier mb_qk_done: MBarrier - mb_state_inp_ready: MBarrier - mb_vks_inp_ready: MBarrier - mb_nv_inp_ready: MBarrier - mb_decay_v_inp_ready: MBarrier + + mb_k_state_ready: MBarrier + mb_nv_ready: MBarrier mb_o_tmastg_ready: MBarrier mb_o_tmastg_done: MBarrier - mb_h_tmastg_ready: MBarrier - mb_h_tmastg_done: MBarrier + mb_checkpoint_tmastg_ready: MBarrier + mb_checkpoint_tmastg_done: MBarrier mb_tmem_done: MBarrier + mb_sched_ready: MBarrier mb_sched_done: MBarrier def make_gdn_bars(cfg) -> GdnBars: - """GdnBars factory. MUST be called from inside ``_kernel`` (allocates SMEM; - the mbar rings sit ahead of the gate scalar arrays and data buffers).""" + """GdnBars factory. MUST be called from inside ``kernel`` (allocates SMEM).""" ONE_LANE = 1 MMA_ARRIVERS = len([cfg.mma_warp_id]) - BOTH_ISSUERS = len([cfg.mma_warp_id, cfg.mma_cg1_warp_id]) + KQ_RELEASE_SITES = 1 GATE_WARP = cfg.threads_per_warp * len([cfg.load_gate_beta_warp_id]) EPI_WARP = cfg.threads_per_warp * len([cfg.epilogue_warp_id]) CG0_THREADS = cfg.threads_per_warp * len(cfg.compute_group_0_warp_ids) @@ -188,18 +189,16 @@ def alloc(n): return cutlass.Array(cutlass.Int64, n, space=cutlass.AddressSpace.smem, alignment=16) return GdnBars( - mb_q_ready=MBarrier(alloc(cfg.smem_q_stages), stages=cfg.smem_q_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), - mb_q_done=MBarrier(alloc(cfg.smem_q_stages), stages=cfg.smem_q_stages, init_count=BOTH_ISSUERS, producer=Producer.MMA_COMMIT), - mb_k_ready=MBarrier(alloc(cfg.smem_k_stages), stages=cfg.smem_k_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), - mb_k_done=MBarrier(alloc(cfg.smem_k_stages), stages=cfg.smem_k_stages, init_count=BOTH_ISSUERS, producer=Producer.MMA_COMMIT), + mb_kq_ready=MBarrier(alloc(cfg.smem_kq_stages), stages=cfg.smem_kq_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), + mb_kq_done=MBarrier(alloc(cfg.smem_kq_stages), stages=cfg.smem_kq_stages, init_count=KQ_RELEASE_SITES, producer=Producer.MMA_COMMIT), mb_v_ready=MBarrier(alloc(cfg.smem_v_stages), stages=cfg.smem_v_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), mb_v_done=MBarrier(alloc(cfg.smem_v_stages), stages=cfg.smem_v_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_gate_ready=MBarrier(alloc(cfg.smem_gate_stages), stages=cfg.smem_gate_stages, init_count=GATE_WARP, producer=Producer.THREAD), mb_gate_done=MBarrier(alloc(cfg.smem_gate_stages), stages=cfg.smem_gate_stages, init_count=CG0_PLUS_CG1, producer=Producer.THREAD), mb_beta_ready=MBarrier(alloc(cfg.smem_beta_stages), stages=cfg.smem_beta_stages, init_count=GATE_WARP, producer=Producer.THREAD), mb_beta_done=MBarrier(alloc(cfg.smem_beta_stages), stages=cfg.smem_beta_stages, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_kv_acc_ready=MBarrier(alloc(cfg.tmem_kv_acc_stages), stages=cfg.tmem_kv_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_kv_acc_scale_done=MBarrier(alloc(cfg.tmem_kv_acc_stages), stages=cfg.tmem_kv_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_state_acc_ready=MBarrier(alloc(cfg.tmem_state_acc_stages), stages=cfg.tmem_state_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_state_acc_scale_done=MBarrier(alloc(cfg.tmem_state_acc_stages), stages=cfg.tmem_state_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_acc_ready=MBarrier(alloc(cfg.tmem_q_state_acc_stages), stages=cfg.tmem_q_state_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_o_acc_done=MBarrier(alloc(cfg.tmem_q_state_acc_stages), stages=cfg.tmem_q_state_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_state_scale_acc_ready=MBarrier( @@ -209,39 +208,32 @@ def alloc(n): alloc(cfg.tmem_q_state_acc_stages), stages=cfg.tmem_q_state_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD ), mb_cg0_acc_ready=MBarrier(alloc(cfg.tmem_cg0_acc_stages), stages=cfg.tmem_cg0_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_cg0_acc_done=MBarrier(alloc(cfg.tmem_cg0_acc_stages), stages=cfg.tmem_cg0_acc_stages, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_ks_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_nv_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_ainv_ready=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_ainv_done=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_qk_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_qk_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_cg0_acc_done=MBarrier(alloc(cfg.tmem_cg0_acc_stages), stages=cfg.tmem_cg0_acc_stages, init_count=CG0_THREADS // 2, producer=Producer.THREAD), mb_state_inp_ready=MBarrier(alloc(cfg.tmem_state_inp_stages), stages=cfg.tmem_state_inp_stages, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_vks_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_v_k_state_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_nv_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_decay_v_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_ainv_ready=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_ainv_done=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_qk_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=CG0_THREADS // 2, producer=Producer.THREAD), + mb_qk_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_k_state_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_nv_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_o_tmastg_ready=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_tmastg_done=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=EPI_WARP, producer=Producer.THREAD), - mb_h_tmastg_ready=MBarrier(alloc(cfg.smem_h_stages), stages=cfg.smem_h_stages, init_count=len(cfg.compute_group_1_warp_ids), producer=Producer.THREAD), - mb_h_tmastg_done=MBarrier(alloc(cfg.smem_h_stages), stages=cfg.smem_h_stages, init_count=EPI_WARP, producer=Producer.THREAD), + mb_checkpoint_tmastg_ready=MBarrier( + alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=len(cfg.compute_group_1_warp_ids), producer=Producer.THREAD + ), + mb_checkpoint_tmastg_done=MBarrier(alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=EPI_WARP, producer=Producer.THREAD), mb_tmem_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_sched_ready=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=1, producer=Producer.THREAD), mb_sched_done=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=11, producer=Producer.THREAD), ) -# --------------------------------------------------------------------------- -# Device-side helpers / warp bodies -# --------------------------------------------------------------------------- - - @cute.jit -def _invert_diagonal_NxN(cfg, base_int, d, tidx, N: int = 8): - """Stage 1: Gauss-Jordan inversion of one diagonal NxN block in-place (f16 SMEM). - - The tile swizzle re-homes whole rows only, so a diagonal block's row stays - a contiguous N-element run at ``swz(row_lin_base)``. - """ +def invert_diagonal_NxN(cfg, base_int, d, tidx, N: int = 8): + """Gauss-Jordan inversion of one diagonal NxN block in-place (f16 SMEM).""" tidx_in_group = tidx % N BT = cfg.b_t @@ -272,70 +264,65 @@ def _invert_diagonal_NxN(cfg, base_int, d, tidx, N: int = 8): @cute.jit -def _mma_m16n8k8(a0, a1, b0, c_regs, dtype: cutlass.Constexpr): - """One m16n8k8 reg-reg mma (``mma_step`` only emits the k16 form), - accumulating into the ``c_regs`` buffer in place.""" - tag = "f16" if cutlass.const_expr(dtype == cutlass.Float16) else "bf16" - c_regs[0], c_regs[1], c_regs[2], c_regs[3] = inline_ptx( - f"mma.sync.aligned.m16n8k8.row.col.f32.{tag}.{tag}.f32" " {$0,$1,$2,$3}, {$4,$5}, {$6}, {$7,$8,$9,$10};", - write_only_types=[cutlass.Float32, cutlass.Float32, cutlass.Float32, cutlass.Float32], - read_only_args=[a0, a1, b0, c_regs[0], c_regs[1], c_regs[2], c_regs[3]], - ) - - -@cute.jit -def _blockwise_diagonal_8x8_to_16x16(cfg, base_int, d0, lane_id): - """Stage 2: off-diagonal correction 8x8 -> 16x16 (C <- -D^{-1} C A^{-1}). - - Keep the per-lane ldmatrix offset (``lds1``/``lds4``) a SEPARATE sum term - from the warp-uniform (row, col) origin in all the diagonal helpers — - folding it into the row term costs ~2% (per-lane address datapath). - """ +def blockwise_diagonal_8x8_to_16x16(cfg, base_int, d0, lane_id): + """Off-diagonal correction 8x8 -> 16x16 (C <- -D^{-1} C A^{-1}).""" bpe = cfg.io_dtype.width // 8 - lds1 = (lane_id % 8) * 64 + ldsm_x1_lane_off = (lane_id % 8) * 64 d = nvvm.ldmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + 8 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr( + base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + 8 + ldsm_x1_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), 1, nvvm.MMALayout.ROW, ) c = nvvm.ldmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr( + base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + ldsm_x1_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), 1, nvvm.MMALayout.COL, ) + + # ---- T = -(D^{-1} @ C) ------------------------------------------------------- c_regs = cutlass.Array(cutlass.Float32, 4, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(4): c_regs[i] = cutlass.Float32(0.0) - _mma_m16n8k8(d, d, c, c_regs, cfg.io_dtype) + mma_step_k8(c_regs, [d, d], [c], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) for i in cutlass.range_constexpr(4): c_regs[i] = -c_regs[i] a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(2)] + + # ---- C = T @ A^{-1} ---------------------------------------------------------- ai = nvvm.ldmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b(d0 * 64 + d0 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr(base_int + swizzle_lin_128b(d0 * 64 + d0 + ldsm_x1_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), 1, nvvm.MMALayout.COL, ) o_regs = cutlass.Array(cutlass.Float32, 4, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(4): o_regs[i] = cutlass.Float32(0.0) - _mma_m16n8k8(a_f16[0], a_f16[1], ai, o_regs, cfg.io_dtype) + mma_step_k8(o_regs, a_f16, [ai], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) o_f16 = fp32_to_fp16(o_regs[0], o_regs[1], dtype=cfg.io_dtype) + + # ---- store corrected C ------------------------------------------------------- nvvm.stmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr( + base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + ldsm_x1_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), o_f16, nvvm.MMALayout.ROW, ) @cute.jit -def _blockwise_diagonal_16x16_to_32x32(cfg, base_int, d0, lane_id): - """Stage 3: off-diagonal correction 16x16 -> 32x32.""" +def blockwise_diagonal_16x16_to_32x32(cfg, base_int, d0, lane_id): + """Off-diagonal correction 16x16 -> 32x32.""" bpe = cfg.io_dtype.width // 8 - lds4 = (lane_id % 16) * 64 + (lane_id // 16) * 8 + ldsm_x4_lane_off = (lane_id % 16) * 64 + (lane_id // 16) * 8 d = list( nvvm.ldmatrix( cutlass.inttoptr( - base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + 16 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + 16 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 ), 4, nvvm.MMALayout.ROW, @@ -343,11 +330,15 @@ def _blockwise_diagonal_16x16_to_32x32(cfg, base_int, d0, lane_id): ) c = list( nvvm.ldmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr( + base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), 4, nvvm.MMALayout.COL, ) ) + + # ---- T = -(D^{-1} @ C) ------------------------------------------------------- c_regs = cutlass.Array(cutlass.Float32, 8, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(8): c_regs[i] = cutlass.Float32(0.0) @@ -355,9 +346,13 @@ def _blockwise_diagonal_16x16_to_32x32(cfg, base_int, d0, lane_id): for i in cutlass.range_constexpr(8): c_regs[i] = -c_regs[i] a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + + # ---- C = T @ A^{-1} ---------------------------------------------------------- ai = list( nvvm.ldmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b(d0 * 64 + d0 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + cutlass.inttoptr( + base_int + swizzle_lin_128b(d0 * 64 + d0 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), 4, nvvm.MMALayout.COL, ) @@ -366,27 +361,30 @@ def _blockwise_diagonal_16x16_to_32x32(cfg, base_int, d0, lane_id): for i in cutlass.range_constexpr(8): o_regs[i] = cutlass.Float32(0.0) mma_step(o_regs, a_f16, ai, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) - ow = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + + # ---- store corrected C ------------------------------------------------------- nvvm.stmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - ow, + cutlass.inttoptr( + base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), + o_f16, nvvm.MMALayout.ROW, ) @cute.jit -def _blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): - """Stage 4: off-diagonal correction 32x32 -> 64x64 (2 warps, one 16-row - M-band each).""" +def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): + """Off-diagonal correction 32x32 -> 64x64 (2 warps, one 16-row M-band each).""" band = warp_id % 2 bpe = cfg.io_dtype.width // 8 - lds4 = (lane_id % 16) * 64 + (lane_id // 16) * 8 + ldsm_x4_lane_off = (lane_id % 16) * 64 + (lane_id // 16) * 8 a_regs = [] for vs in cutlass.range_constexpr(2): a_regs += list( nvvm.ldmatrix( cutlass.inttoptr( - base_int + swizzle_lin_128b((32 + band * 16) * 64 + 32 + vs * 16 + lds4, row_stride_log2=6) * bpe, + base_int + swizzle_lin_128b((32 + band * 16) * 64 + 32 + vs * 16 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16, ), @@ -399,7 +397,7 @@ def _blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): b_regs += list( nvvm.ldmatrix( cutlass.inttoptr( - base_int + swizzle_lin_128b((32 + (vs // 2) * 16) * 64 + (vs % 2) * 16 + lds4, row_stride_log2=6) * bpe, + base_int + swizzle_lin_128b((32 + (vs // 2) * 16) * 64 + (vs % 2) * 16 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16, ), @@ -407,6 +405,8 @@ def _blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): nvvm.MMALayout.COL, ) ) + + # ---- T = -(D^{-1} @ C) ------------------------------------------------------- c_regs = cutlass.Array(cutlass.Float32, 16, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(16): c_regs[i] = cutlass.Float32(0.0) @@ -415,12 +415,14 @@ def _blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): for i in cutlass.range_constexpr(16): c_regs[i] = -c_regs[i] a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + + # ---- C = T @ A^{-1} ---------------------------------------------------------- ai_regs = [] for vs in cutlass.range_constexpr(4): ai_regs += list( nvvm.ldmatrix( cutlass.inttoptr( - base_int + swizzle_lin_128b(((vs // 2) * 16) * 64 + (vs % 2) * 16 + lds4, row_stride_log2=6) * bpe, + base_int + swizzle_lin_128b(((vs // 2) * 16) * 64 + (vs % 2) * 16 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16, ), @@ -433,114 +435,34 @@ def _blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): o_regs[i] = cutlass.Float32(0.0) for ks in cutlass.range_constexpr(2): mma_step(o_regs, a_f16, ai_regs[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) - ow = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + + # ---- store corrected C ------------------------------------------------------- nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, ) nvvm.stmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((32 + band * 16) * 64 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - ow[0:4], + cutlass.inttoptr( + base_int + swizzle_lin_128b((32 + band * 16) * 64 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), + o_f16[0:4], nvvm.MMALayout.ROW, ) nvvm.stmatrix( - cutlass.inttoptr(base_int + swizzle_lin_128b((32 + band * 16) * 64 + 16 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - ow[4:8], + cutlass.inttoptr( + base_int + swizzle_lin_128b((32 + band * 16) * 64 + 16 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), + o_f16[4:8], nvvm.MMALayout.ROW, ) -@cute.jit -def _load_gate_beta_chunk( - cfg, - lidx, - mGate, - mBeta, - sCumsumlog, - sCumprod, - sBeta, - gate_index, - beta_index, - head_idx, - batch_start, - batch_end, - chunk_idx, - bars, -): - """Load gate[BT]/beta[BT] for one chunk (epilogue warp). - - The OOB predicate is RUNTIME (covers the last valid chunk AND padded - pair chunks), keeping one body in SASS. OOB gate positions read a - clamped in-bounds address and select the neutral value (gate=1 -> - log 0); OOB beta lanes zero-fill via cp_size=0.""" - n_cols = cfg.b_t // cfg.threads_per_warp - chunk_offset = batch_start + chunk_idx * cfg.b_t - gGateSeq = mGate[None, head_idx] - gBeta = cute.domain_offset((chunk_offset,), mBeta[None, head_idx]) - - gate_idx = gate_index.idx - gate_phase = gate_index.phase - gate_index = advance(gate_index, cfg.smem_gate_stages) - - pos_valid = [None] * n_cols - tGrGate = [cutlass.Float32(0.0)] * n_cols - oob_neutral = cutlass.Float32(0.0) if cutlass.const_expr(cfg.log_gate) else cutlass.Float32(1.0) - for col in cutlass.range_constexpr(n_cols): - tok = chunk_offset + lidx + col * cfg.threads_per_warp - pos_valid[col] = cute.elem_less(tok, batch_end) - # batch_end >= 1 whenever chunks exist, so the clamp stays in bounds - tok_clamped = _cutlass_min(tok, batch_end - 1) - tGrGate[col] = gGateSeq[tok_clamped] if pos_valid[col] else oob_neutral - - if cutlass.const_expr(cfg.log_gate): - for col in cutlass.range_constexpr(n_cols): - tGrGate[col] = tGrGate[col] * cutlass.Float32(RCP_LN2) - else: - for col in cutlass.range_constexpr(n_cols): - tGrGate[col] = cute.math.log2(tGrGate[col] + 1e-10, fastmath=True) - for offset in [1, 2, 4, 8, 16]: - for col in cutlass.range_constexpr(n_cols): - n = nvvm.shfl_sync(0xFFFFFFFF, tGrGate[col], offset, 0, kind=nvvm.Shfl.UP) - if lidx >= offset: - tGrGate[col] = tGrGate[col] + n - for col in cutlass.range_constexpr(1, n_cols): - last_v = nvvm.shfl_sync( - 0xFFFFFFFF, - tGrGate[col - 1], - cfg.threads_per_warp - 1, - cfg.threads_per_warp - 1, - kind=nvvm.Shfl.IDX, - ) - tGrGate[col] += last_v - - bars.mb_gate_done[gate_idx].wait(gate_phase) - for col in cutlass.range_constexpr(n_cols): - pos = lidx + col * cfg.threads_per_warp - sCumsumlog[pos, 0, gate_idx] = tGrGate[col] - sCumprod[pos, 0, gate_idx] = cute.math.exp2(tGrGate[col], fastmath=True) - - bars.mb_gate_ready[gate_idx].arrive() - - # --- Beta load (per-element async G->S cp.async) --- - beta_idx = beta_index.idx - bars.mb_beta_done[beta_idx].wait(beta_index.phase) - beta_index = advance(beta_index, cfg.smem_beta_stages) - for col in cutlass.range_constexpr(n_cols): - pos = lidx + col * cfg.threads_per_warp - src = gBeta.iterator + gBeta.layout((pos,)) - dst = sBeta.iterator + sBeta.layout((pos, 0, beta_idx)) - cp_size = cutlass.Int32(4) * cutlass.Int32(pos_valid[col]) - nvvm.cp_async_shared_global(dst, src, 4, nvvm.LoadCacheModifier.CA, cp_size=cp_size) - nvvm.cp_async_mbarrier_arrive(bars.mb_beta_ready[beta_idx].smem_ptr, noinc=True) - return gate_index, beta_index - - -# --------------------------------------------------------------------------- -# Dynamic tile scheduler: global-ticket work-stealing ring +# ---- Dynamic tile scheduler ------------------------------------------------------ @cute.jit -def _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): +def sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): """TMA-LDG-warp side: pull the next tile off the global ticket, publish it.""" if cutlass.const_expr(cfg.dyn_sched): bars.mb_sched_done[sched_state.idx].wait(sched_state.phase) @@ -556,7 +478,7 @@ def _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ct @cute.jit -def _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): +def sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): """Consumer side: read the TMA-LDG warp's published next tile.""" if cutlass.const_expr(cfg.dyn_sched): bars.mb_sched_ready[sched_state.idx].wait(sched_state.phase) @@ -568,7 +490,7 @@ def _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): @cute.jit -def _tmastg_warp( +def tmastg_warp( cfg, total_tiles, bidx, @@ -577,31 +499,24 @@ def _tmastg_warp( mWorkItems, checkpoint_every_n_tokens, tidx, - mGate, - mBeta, - sCumsumlog, - sCumprod, - sBeta, sO_raw, - sH_raw, + sCheckpoint_raw, desc_o_base, - desc_h_base, + desc_checkpoint_base, sSched, bars, ): - """Epilogue warp role (warp 11): persistent tile-scheduler loop; loads - gate/beta with a four-chunk lookahead and issues the per-chunk O and - H-state TMA bulk-stores from SMEM staging to global memory.""" + """Epilogue warp role (warp 11): persistent scheduler loop issuing the + per-chunk O and state-checkpoint TMA stores.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) o_index = PipelineState.start(phase=0) - gate_index = PipelineState.start(phase=1) - beta_index = PipelineState.start(phase=1) lidx = tidx % cfg.threads_per_warp sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) bpe = cfg.io_dtype.width // 8 - granu = 128 // bpe + elems_per_128b = 128 // bpe sO_tma = SmemTile( base=sO_raw, elems_per_stage=(cfg.o_cosize // cfg.smem_o_stages), @@ -610,23 +525,24 @@ def _tmastg_warp( stride_byte_offset=0, layout=0, tma_loads_per_tile=2, - tma_granu_elems=granu, + tma_granu_elems=elems_per_128b, tma_subtile_stride_elems=4096, ) - if cutlass.const_expr(cfg.enable_h): - h_granu = 64 - sH_tma = SmemTile( - base=sH_raw, - elems_per_stage=(cfg.h_cosize // cfg.smem_h_stages), - stages=cfg.smem_h_stages, + if cutlass.const_expr(cfg.enable_checkpoints): + checkpoint_elems_per_128b = 64 + sCheckpoint_tma = SmemTile( + base=sCheckpoint_raw, + elems_per_stage=(cfg.checkpoint_cosize // cfg.smem_checkpoint_stages), + stages=cfg.smem_checkpoint_stages, leading_byte_offset=0, stride_byte_offset=0, layout=0, - tma_loads_per_tile=cfg.d_v // h_granu, - tma_granu_elems=h_granu, - tma_subtile_stride_elems=cfg.d_k * h_granu, + tma_loads_per_tile=cfg.d_v // checkpoint_elems_per_128b, + tma_granu_elems=checkpoint_elems_per_128b, + tma_subtile_stride_elems=cfg.d_k * checkpoint_elems_per_128b, ) - hr_cnt = cutlass.Int32(0) + checkpoint_store_cnt = cutlass.Int32(0) + ckpt_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) heads_out = cutlass.Int32(cfg.n_heads_out) desc_qwords = cutlass.Int32(TENSOR_MAP_QWORDS) @@ -637,179 +553,97 @@ def _tmastg_warp( n_local = wend - cstart n_padded = ((n_local + 1) // 2) * 2 - slot = (batch_idx * heads_out + head_idx) * desc_qwords - if cutlass.const_expr(cfg.enable_o): - desc_o_slot = (desc_o_base + slot).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): - tma_tensormap_acquire(desc_o_slot) - if cutlass.const_expr(cfg.enable_h): - desc_h_slot = (desc_h_base + slot).tospace(cutlass.AddressSpace.generic) - # sequence-local H index of this item's first owned entry - h_coord = wstart - 1 if wstart > 0 else cutlass.Int32(0) - if nvvm.elect_sync(): - tma_tensormap_acquire(desc_h_slot) + head_o = head_idx + slot = batch_idx * desc_qwords + desc_o_slot = (desc_o_base + slot).tospace(cutlass.AddressSpace.generic) + if elect_one: + tma_tensormap_acquire(desc_o_slot) + if cutlass.const_expr(cfg.enable_checkpoints): + desc_checkpoint_slot = (desc_checkpoint_base + slot).tospace(cutlass.AddressSpace.generic) + checkpoint_coord = wstart - 1 if wstart > 0 else cutlass.Int32(0) + checkpoint_mod = (cstart + cutlass.Int32(1)) % ckpt_chunks + if elect_one: + tma_tensormap_acquire(desc_checkpoint_slot) if n_local > 0: - # ---- gate/beta lookahead: chunk count is padded even, so the - # first two stages always exist and chunks 2/3 come together ---- - for pf in range(2): - gate_index, beta_index = _load_gate_beta_chunk( - cfg, lidx, mGate, mBeta, sCumsumlog, sCumprod, sBeta, gate_index, beta_index, head_idx, batch_start, batch_end, cstart + pf, bars - ) - if n_padded > 2: - for pf in range(2, 4): - gate_index, beta_index = _load_gate_beta_chunk( - cfg, - lidx, - mGate, - mBeta, - sCumsumlog, - sCumprod, - sBeta, - gate_index, - beta_index, - head_idx, - batch_start, - batch_end, - cstart + pf, - bars, - ) - for local_idx in cutlass.range(n_padded): - if local_idx + 4 < n_padded: - gate_index, beta_index = _load_gate_beta_chunk( - cfg, - lidx, - mGate, - mBeta, - sCumsumlog, - sCumprod, - sBeta, - gate_index, - beta_index, - head_idx, - batch_start, - batch_end, - cstart + local_idx + 4, - bars, - ) chunk_idx = cstart + local_idx did_o = cutlass.Int32(0) - if cutlass.const_expr(cfg.enable_o): - o_idx = o_index.idx - bars.mb_o_tmastg_ready[o_idx].wait(o_index.phase) - o_index = advance(o_index, cfg.smem_o_stages) - - # padded / warmup chunks stage O but never store it - if chunk_idx >= wstart and chunk_idx < wend: - tok_coord = chunk_idx * cutlass.Int32(cfg.b_t) - o_slice = tma_slice_runtime_desc(desc_o_slot, cutlass.Int32(0), tok_coord) - tma_store_tile(sO_tma[o_idx], o_slice, acquire=False) - tma_store_commit() - did_o = cutlass.Int32(1) - - did_h = cutlass.Int32(0) - if cutlass.const_expr(cfg.enable_h): + o_idx = o_index.idx + bars.mb_o_tmastg_ready[o_idx].wait(o_index.phase) + o_index = advance(o_index, cfg.smem_o_stages) + + if chunk_idx >= wstart and chunk_idx < wend: + tok_coord = chunk_idx * cutlass.Int32(cfg.b_t) + o_slice = tma_slice_runtime_desc(desc_o_slot, cutlass.Int32(0), head_o, tok_coord) + tma_store_tile(sO_tma[o_idx], o_slice, acquire=False) + tma_store_commit() + did_o = cutlass.Int32(1) + + did_checkpoint = cutlass.Int32(0) + if cutlass.const_expr(cfg.enable_checkpoints): + checkpoint_stage = checkpoint_store_cnt % cfg.smem_checkpoint_stages + checkpoint_phase = (checkpoint_store_cnt // cfg.smem_checkpoint_stages) & cutlass.Int32(1) if chunk_idx >= wstart - 1 and chunk_idx < wend - 1: - if (cfg.b_t * (chunk_idx + 1)) % checkpoint_every_n_tokens == 0: - bars.mb_h_tmastg_ready[hr_cnt % cfg.smem_h_stages].wait((hr_cnt // cfg.smem_h_stages) & cutlass.Int32(1)) - h_slice = tma_slice_runtime_desc(desc_h_slot, cutlass.Int32(0), cutlass.Int32(0), h_coord) - tma_store_tile(sH_tma[hr_cnt % cfg.smem_h_stages], h_slice, acquire=False) + if checkpoint_mod == 0: + bars.mb_checkpoint_tmastg_ready[checkpoint_stage].wait(checkpoint_phase) + checkpoint_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), checkpoint_coord, head_o) + tma_store_tile(sCheckpoint_tma[checkpoint_stage], checkpoint_slice, acquire=False) tma_store_commit() - h_coord += 1 - did_h = cutlass.Int32(1) - - if cutlass.const_expr(cfg.enable_o): - if cutlass.const_expr(cfg.enable_h): - if did_o == 1 and did_h == 1: - tma_store_wait(1) - bars.mb_o_tmastg_done[o_idx].arrive() - tma_store_wait(0) - bars.mb_h_tmastg_done[hr_cnt % cfg.smem_h_stages].arrive() - hr_cnt = hr_cnt + 1 - if did_o == 1 and did_h == 0: - tma_store_wait(0) - bars.mb_o_tmastg_done[o_idx].arrive() - if did_o == 0: - if did_h == 1: - tma_store_wait(0) - bars.mb_h_tmastg_done[hr_cnt % cfg.smem_h_stages].arrive() - hr_cnt = hr_cnt + 1 - bars.mb_o_tmastg_done[o_idx].arrive() - else: - if did_o == 1: + checkpoint_coord += 1 + did_checkpoint = cutlass.Int32(1) + checkpoint_mod = checkpoint_mod + cutlass.Int32(1) + checkpoint_mod = cutlass.Int32(0) if checkpoint_mod == ckpt_chunks else checkpoint_mod + + if cutlass.const_expr(cfg.enable_checkpoints): + if did_o == 1 and did_checkpoint == 1: + tma_store_wait(1) + bars.mb_o_tmastg_done[o_idx].arrive() + tma_store_wait(0) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].arrive() + checkpoint_store_cnt = checkpoint_store_cnt + 1 + if did_o == 1 and did_checkpoint == 0: + tma_store_wait(0) + bars.mb_o_tmastg_done[o_idx].arrive() + if did_o == 0: + if did_checkpoint == 1: tma_store_wait(0) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].arrive() + checkpoint_store_cnt = checkpoint_store_cnt + 1 bars.mb_o_tmastg_done[o_idx].arrive() else: - if cutlass.const_expr(cfg.enable_h): - if did_h == 1: - tma_store_wait(0) - bars.mb_h_tmastg_done[hr_cnt % cfg.smem_h_stages].arrive() - hr_cnt = hr_cnt + 1 - - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + if did_o == 1: + tma_store_wait(0) + bars.mb_o_tmastg_done[o_idx].arrive() - for _ in range(cfg.smem_gate_stages): - bars.mb_gate_done[gate_index.idx].wait(gate_index.phase) - gate_index = advance(gate_index, cfg.smem_gate_stages) - for _ in range(cfg.smem_beta_stages): - bars.mb_beta_done[beta_index.idx].wait(beta_index.phase) - beta_index = advance(beta_index, cfg.smem_beta_stages) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) @cute.jit -def _mma0_warp( +def gate_beta_warp( cfg, total_tiles, bidx, num_ctas, cu_seqlens, mWorkItems, - tmem_hold, - sQ, - sK, + tidx, + mGate, + mBeta, + sCumsumlog, + sCumprod, + sBeta, sSched, bars, ): - """CG0 MMA issuer role (warp 8): persistent scheduler loop + per-pair - KK0/KK1/QK0/QK1 issue into CG0's private accumulator ring; owns the TMEM - lifecycle (alloc up front, dealloc once CG1 signals mb_tmem_done).""" + """Gate/Beta producer (warp 8): persistent scheduler loop + the + cumsum/cumprod/Beta chunk loads.""" nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) - cg0_acc_index = PipelineState.start(phase=1) - k_index = PipelineState.start(phase=0) - q_index = PipelineState.start(phase=0) - - nvvm.tcgen05_alloc(tmem_hold, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync_aligned( - cfg.tmem_alloc_barrier_id, - thread_count=cfg.tmem_alloc_barrier_threads, - ) - tmem_base = tmem_hold.load() - - bpe = cfg.io_dtype.width // 8 - idesc_qk = nvvm.Tcgen05InstrDesc.build( - c_dtype=cutlass.Float32, - a_dtype=cfg.io_dtype, - b_dtype=cfg.io_dtype, - n_dim=cfg.b_t, - m_dim=cfg.b_t, - ) - bmm_qk_desc = MmaDesc( - M=cfg.b_t, - N=cfg.b_t, - K=cfg.d_k, - bpe_a=bpe, - bpe_b=bpe, - tile_k_hw=16, - btranspose=False, - cta_group=1, - idesc=idesc_qk, - kind=nvvm.Tcgen05MMAKind.F16, - ) - tmem_cg0_acc_col = tmem_base + cfg.tmem_cg0_acc_offset - ACC_STAGE_COLS = cfg.b_t + gate_index = PipelineState.start(phase=1) + beta_index = PipelineState.start(phase=1) + lidx = tidx % cfg.threads_per_warp sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) @@ -817,155 +651,148 @@ def _mma0_warp( batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) - n_pairs = (wend - cstart + 1) // 2 - for _pair in cutlass.range(n_pairs): # noqa: B007 - # ---- GEMM 1 (chunk 0 of pair): kk0 ------------------------- - kk0_acc_idx = cg0_acc_index.idx - bars.mb_cg0_acc_done[kk0_acc_idx].wait(cg0_acc_index.phase) - cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) - k0_idx = k_index.idx - bars.mb_k_ready[k0_idx].wait(k_index.phase) - k_index = advance(k_index, cfg.smem_k_stages) - - desc_k0 = sK[k0_idx].desc() - mma_ss( - bmm_qk_desc, - desc_k0, - desc_k0, - nvvm.make_tmem_ptr(tmem_cg0_acc_col + kk0_acc_idx * ACC_STAGE_COLS, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_cg0_acc_ready[kk0_acc_idx].arrive(cta_group=1) - - # ---- GEMM 1 (chunk 1 of pair): kk1 ------------------------- - kk1_acc_idx = cg0_acc_index.idx - bars.mb_cg0_acc_done[kk1_acc_idx].wait(cg0_acc_index.phase) - cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) - k1_idx = k_index.idx - bars.mb_k_ready[k1_idx].wait(k_index.phase) - k_index = advance(k_index, cfg.smem_k_stages) - - desc_k1 = sK[k1_idx].desc() - mma_ss( - bmm_qk_desc, - desc_k1, - desc_k1, - nvvm.make_tmem_ptr(tmem_cg0_acc_col + kk1_acc_idx * ACC_STAGE_COLS, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_cg0_acc_ready[kk1_acc_idx].arrive(cta_group=1) - - # ---- GEMM 2 (chunk 0 of pair): qk0 ------------------------- - q0_idx = q_index.idx - bars.mb_q_ready[q0_idx].wait(q_index.phase) - q_index = advance(q_index, cfg.smem_q_stages) - qk0_acc_idx = cg0_acc_index.idx - bars.mb_cg0_acc_done[qk0_acc_idx].wait(cg0_acc_index.phase) - cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) - - desc_q0 = sQ[q0_idx].desc() - mma_ss( - bmm_qk_desc, - desc_q0, - desc_k0, - nvvm.make_tmem_ptr(tmem_cg0_acc_col + qk0_acc_idx * ACC_STAGE_COLS, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_cg0_acc_ready[qk0_acc_idx].arrive(cta_group=1) - - # ---- GEMM 2 (chunk 1 of pair): qk1 ------------------------- - q1_idx = q_index.idx - bars.mb_q_ready[q1_idx].wait(q_index.phase) - q_index = advance(q_index, cfg.smem_q_stages) - qk1_acc_idx = cg0_acc_index.idx - bars.mb_cg0_acc_done[qk1_acc_idx].wait(cg0_acc_index.phase) - cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) - - desc_q1 = sQ[q1_idx].desc() - mma_ss( - bmm_qk_desc, - desc_q1, - desc_k1, - nvvm.make_tmem_ptr(tmem_cg0_acc_col + qk1_acc_idx * ACC_STAGE_COLS, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_cg0_acc_ready[qk1_acc_idx].arrive(cta_group=1) + n_local = wend - cstart + n_padded = ((n_local + 1) // 2) * 2 + if n_local > 0: + for local_idx in cutlass.range(n_padded): + # ---- Gate load: GMEM -> SMEM (OOB neutral) ----------------------- + chunk_idx = cstart + local_idx + n_cols = cfg.b_t // cfg.threads_per_warp + chunk_offset = batch_start + chunk_idx * cfg.b_t + gGateSeq = mGate[None, head_idx] + gBeta = cute.domain_offset((chunk_offset,), mBeta[None, head_idx]) - # this issuer's Q/K releases (the CG1 issuer commits its own) - if nvvm.elect_sync(): - bars.mb_q_done[q0_idx].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_q_done[q1_idx].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_k_done[k0_idx].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_k_done[k1_idx].arrive(cta_group=1) + gate_idx = gate_index.idx + gate_phase = gate_index.phase + gate_index = advance(gate_index, cfg.smem_gate_stages) - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + pos_valid = [None] * n_cols + gate_vals = [cutlass.Float32(0.0)] * n_cols + oob_neutral = cutlass.Float32(0.0) if cutlass.const_expr(cfg.log_gate) else cutlass.Float32(1.0) + for col in cutlass.range_constexpr(n_cols): + tok = chunk_offset + lidx + col * cfg.threads_per_warp + pos_valid[col] = cute.elem_less(tok, batch_end) + tok_clamped = min(tok, batch_end - 1) + gate_vals[col] = gGateSeq[tok_clamped] if pos_valid[col] else oob_neutral + + if cutlass.const_expr(cfg.log_gate): + for col in cutlass.range_constexpr(n_cols): + gate_vals[col] = gate_vals[col] * cutlass.Float32(RCP_LN2) + else: + for col in cutlass.range_constexpr(n_cols): + gate_vals[col] = cute.math.log2(gate_vals[col] + 1e-10, fastmath=True) + for offset in [1, 2, 4, 8, 16]: + for col in cutlass.range_constexpr(n_cols): + n = nvvm.shfl_sync(0xFFFFFFFF, gate_vals[col], offset, 0, kind=nvvm.Shfl.UP) + if lidx >= offset: + gate_vals[col] = gate_vals[col] + n + for col in cutlass.range_constexpr(1, n_cols): + last_v = nvvm.shfl_sync( + 0xFFFFFFFF, + gate_vals[col - 1], + cfg.threads_per_warp - 1, + cfg.threads_per_warp - 1, + kind=nvvm.Shfl.IDX, + ) + gate_vals[col] += last_v + + bars.mb_gate_done[gate_idx].wait(gate_phase) + for col in cutlass.range_constexpr(n_cols): + pos = lidx + col * cfg.threads_per_warp + sCumsumlog[pos, 0, gate_idx] = gate_vals[col] + sCumprod[pos, 0, gate_idx] = cute.math.exp2(gate_vals[col], fastmath=True) + bars.mb_gate_ready[gate_idx].arrive() + + # ---- Beta load: GMEM -> SMEM (per-element cp.async) -------------------------- + beta_idx = beta_index.idx + bars.mb_beta_done[beta_idx].wait(beta_index.phase) + beta_index = advance(beta_index, cfg.smem_beta_stages) + for col in cutlass.range_constexpr(n_cols): + pos = lidx + col * cfg.threads_per_warp + src = gBeta.iterator + gBeta.layout((pos,)) + dst = sBeta.iterator + sBeta.layout((pos, 0, beta_idx)) + cp_size = cutlass.Int32(4) * cutlass.Int32(pos_valid[col]) + nvvm.cp_async_shared_global(dst, src, 4, nvvm.LoadCacheModifier.CA, cp_size=cp_size) + nvvm.cp_async_mbarrier_arrive(bars.mb_beta_ready[beta_idx].smem_ptr, noinc=True) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) - bars.mb_tmem_done[0].wait(0) - nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) - nvvm.tcgen05_dealloc( - nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), - cutlass.Int32(512), - group=nvvm.CTAGroup.CTA_1, - ) + for _ in range(cfg.smem_gate_stages): + bars.mb_gate_done[gate_index.idx].wait(gate_index.phase) + gate_index = advance(gate_index, cfg.smem_gate_stages) + for _ in range(cfg.smem_beta_stages): + bars.mb_beta_done[beta_index.idx].wait(beta_index.phase) + beta_index = advance(beta_index, cfg.smem_beta_stages) @cute.jit -def _mma1_warp( +def mma_warp( cfg, total_tiles, bidx, num_ctas, cu_seqlens, mWorkItems, - tmem_hold, - sQ, - sK, - sK_trans, + tmem_base_slot, + sKQ, + sKQ_trans, sAinv, sQk, sSched, bars, ): - """CG1 MMA issuer role (warp 10): persistent scheduler loop + per-chunk - issue of the five state/output GEMMs (KS/QS/NV/QKV/KV) in dependency - order.""" + """MMA issuer role (warp 10): persistent scheduler loop issuing every + tcgen05 GEMM.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) o_acc_index = PipelineState.start(phase=1) o_state_scale_index = PipelineState.start(phase=0) kv_acc_index = PipelineState.start(phase=1) - k_index = PipelineState.start(phase=0) - q_index = PipelineState.start(phase=0) + kq_index = PipelineState.start(phase=0) + cg0_acc_index = PipelineState.start(phase=1) + kq_fused_index = PipelineState.start(phase=0) ainv_index = PipelineState.start(phase=0) qk_index = PipelineState.start(phase=0) state_inp_index = PipelineState.start(phase=0) - vks_inp_rdy = PipelineState.start(phase=0) - nv_inp_rdy = PipelineState.start(phase=0) - decay_v_inp_rdy = PipelineState.start(phase=0) + v_k_state_inp_ready = PipelineState.start(phase=0) + nv_inp_ready = PipelineState.start(phase=0) + decay_v_inp_ready = PipelineState.start(phase=0) + nvvm.tcgen05_alloc(tmem_base_slot, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) nvvm.barrier_cta_sync_aligned( cfg.tmem_alloc_barrier_id, thread_count=cfg.tmem_alloc_barrier_threads, ) - tmem_base = tmem_hold.load() - # ---- chunk-invariant GEMM descriptors ------------------------------ + # ---- chunk-invariant GEMM descriptors ---------------------------------------- + idesc_qk = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=2 * cfg.b_t, + ) + bmm_qk_desc = MmaDesc( + M=2 * cfg.b_t, + N=cfg.b_t, + K=cfg.d_k, + bpe_a=cfg.io_dtype.width // 8, + bpe_b=cfg.io_dtype.width // 8, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_qk, + kind=nvvm.Tcgen05MMAKind.F16, + ) bpe = cfg.io_dtype.width // 8 - idesc_qs = nvvm.Tcgen05InstrDesc.build( + idesc_q_state = nvvm.Tcgen05InstrDesc.build( c_dtype=cutlass.Float32, a_dtype=cfg.io_dtype, b_dtype=cfg.io_dtype, n_dim=cfg.b_t, m_dim=cfg.d_v, ) - bmm_qs_desc = MmaDesc( + bmm_q_state_desc = MmaDesc( M=cfg.d_v, N=cfg.b_t, K=cfg.d_k, @@ -975,7 +802,7 @@ def _mma1_warp( btranspose=False, atranspose=False, cta_group=1, - idesc=idesc_qs, + idesc=idesc_q_state, kind=nvvm.Tcgen05MMAKind.F16, ) idesc_qkv_ts = nvvm.Tcgen05InstrDesc.build( @@ -1019,21 +846,28 @@ def _mma1_warp( idesc=idesc_kv, kind=nvvm.Tcgen05MMAKind.F16, ) + KQ_SEG = (2 * cfg.b_t * 64 * bpe) >> 4 + KQ_BOX = (cfg.b_t * 64 * bpe) >> 4 + KQ_HALF_K = (cfg.d_k // 16) // 2 + KQ_A_HALF = KQ_HALF_K * bmm_q_state_desc.tmem_advance_A - tmem_state_col = tmem_base + cfg.tmem_state_offset - tmem_q_state_col = tmem_base + cfg.tmem_q_state_offset - tmem_state_inp_col = tmem_base + cfg.tmem_state_inp_offset - tmem_inp_col = tmem_base + cfg.tmem_inp_offset ACC_STAGE_COLS = cfg.b_t KV_ACC_STAGE_COLS = cfg.d_v STATE_INP_STAGE_COLS = cfg.d_k // 2 INP_SLOT_COLS = cfg.b_t // 2 - tmem_ks_col = tmem_base + cfg.tmem_cg1_acc_offset - tmem_nv_col = tmem_ks_col - tmem_vks_col = tmem_inp_col - # NV overwrites the VKS slot once GEMM 5 has consumed it - tmem_nv_inp_col = tmem_inp_col - tmem_decay_v_col = tmem_inp_col + INP_SLOT_COLS + + tmem_base = tmem_base_slot.load() + tmem_cg0_acc_col_f = tmem_base + cfg.tmem_cg0_acc_offset + tmem_state_col = tmem_base + cfg.tmem_state_acc_offset + tmem_q_state_col = tmem_base + cfg.tmem_q_state_acc_offset + tmem_state_inp_col = tmem_base + cfg.tmem_state_inp_offset + tmem_inp_col = tmem_base + cfg.tmem_v_k_state_decay_inp_offset + v_k_state_inp_ptr = nvvm.make_tmem_ptr(tmem_inp_col, cutlass.Int8) + delta_inp_ptr = nvvm.make_tmem_ptr(tmem_inp_col + INP_SLOT_COLS, cutlass.Int8) + k_state_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_cg1_acc_offset, cutlass.Float32) + nv_acc_ptr = k_state_acc_ptr + acc_cg0_0 = nvvm.make_tmem_ptr(tmem_cg0_acc_col_f, cutlass.Float32) + acc_cg0_1 = nvvm.make_tmem_ptr(tmem_cg0_acc_col_f + cfg.b_t, cutlass.Float32) sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) @@ -1043,217 +877,172 @@ def _mma1_warp( ) n_local = wend - cstart n_padded = ((n_local + 1) // 2) * 2 + n_pairs = n_padded // 2 - first_loop = 0 - if cutlass.const_expr(not cfg.use_initial_state): - # ---- peeled first chunk: S_prev = 0, GEMMs 3/4 skipped ----- - if n_local > 0: - k_idx = k_index.idx - bars.mb_k_ready[k_idx].wait(k_index.phase) - k_index = advance(k_index, cfg.smem_k_stages) - q_idx = q_index.idx - bars.mb_q_ready[q_idx].wait(q_index.phase) - q_index = advance(q_index, cfg.smem_q_stages) - if nvvm.elect_sync(): - bars.mb_q_done[q_idx].arrive(cta_group=1) - - # ---- GEMM 5: new_v --------------------------------- - bars.mb_vks_inp_ready[0].wait(vks_inp_rdy.phase) - vks_inp_rdy = advance(vks_inp_rdy, 1) - ainv_idx = ainv_index.idx - bars.mb_ainv_ready[ainv_idx].wait(ainv_index.phase) - ainv_index = advance(ainv_index, cfg.smem_ainv_stages) - - desc_ainv = sAinv[ainv_idx].desc() - vks_a_ptr = nvvm.make_tmem_ptr(tmem_vks_col, cutlass.Int8) - mma_ts( - bmm_qkv_ts_desc, - vks_a_ptr, - desc_ainv, - nvvm.make_tmem_ptr(tmem_nv_col, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_nv_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_ainv_done[ainv_idx].arrive(cta_group=1) - - # ---- GEMM 6: qkv ----------------------------------- - bars.mb_nv_inp_ready[0].wait(nv_inp_rdy.phase) - nv_inp_rdy = advance(nv_inp_rdy, 1) - qk_idx = qk_index.idx - bars.mb_qk_ready[qk_idx].wait(qk_index.phase) - qk_index = advance(qk_index, cfg.smem_qk_stages) - qs_idx = o_acc_index.idx - bars.mb_o_acc_done[qs_idx].wait(o_acc_index.phase) - o_acc_index = advance(o_acc_index, cfg.tmem_q_state_acc_stages) - - qkv_a_ptr = nvvm.make_tmem_ptr(tmem_nv_inp_col, cutlass.Int8) - desc_nv = sQk[qk_idx].desc() - mma_ts( - bmm_qkv_ts_desc, - qkv_a_ptr, - desc_nv, - nvvm.make_tmem_ptr(tmem_q_state_col + qs_idx * ACC_STAGE_COLS, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_qk_done[qk_idx].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_o_state_scale_acc_ready[qs_idx].arrive(cta_group=1) - - # ---- GEMM 7: kv_update --------------------------------------- - bars.mb_decay_v_inp_ready[0].wait(decay_v_inp_rdy.phase) - decay_v_inp_rdy = advance(decay_v_inp_rdy, 1) - kv_acc_idx = kv_acc_index.idx - bars.mb_kv_acc_scale_done[kv_acc_idx].wait(kv_acc_index.phase) - kv_acc_index = advance(kv_acc_index, cfg.tmem_kv_acc_stages) - - delta_a_ptr = nvvm.make_tmem_ptr(tmem_decay_v_col, cutlass.Int8) - desc_kt = sK_trans[k_idx].desc() - mma_ts( - bmm_kv_desc, - delta_a_ptr, - desc_kt, - nvvm.make_tmem_ptr(tmem_state_col + kv_acc_idx * KV_ACC_STAGE_COLS, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_kv_acc_ready[kv_acc_idx].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_k_done[k_idx].arrive(cta_group=1) - - first_loop = 1 - - for local_idx in cutlass.range(first_loop, n_padded): # noqa: B007 + # ---- fused KK^T/QK^T pair 0: both members issued ahead of the loop ------- + if n_pairs > 0: + f0_acc_idx = cg0_acc_index.idx + bars.mb_cg0_acc_done[f0_acc_idx].wait(cg0_acc_index.phase) + cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) + kqf_idx = kq_fused_index.idx + bars.mb_kq_ready[kqf_idx].wait(kq_fused_index.phase) + kq_fused_index = advance(kq_fused_index, cfg.smem_kq_stages) + desc_kqf = sKQ[kqf_idx].desc() + mma_ss(bmm_qk_desc, desc_kqf, desc_kqf, acc_cg0_0, accumulate=False, k_count=KQ_HALF_K) + mma_ss(bmm_qk_desc, desc_kqf + KQ_SEG, desc_kqf + KQ_SEG, acc_cg0_0, accumulate=True, k_count=KQ_HALF_K) + if elect_one: + bars.mb_cg0_acc_ready[f0_acc_idx].arrive(cta_group=1) + pend_acc_idx = cg0_acc_index.idx + bars.mb_cg0_acc_done[pend_acc_idx].wait(cg0_acc_index.phase) + cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) + kqf_idx = kq_fused_index.idx + bars.mb_kq_ready[kqf_idx].wait(kq_fused_index.phase) + kq_fused_index = advance(kq_fused_index, cfg.smem_kq_stages) + desc_kqf = sKQ[kqf_idx].desc() + desc_kqf_b = desc_kqf + KQ_BOX + mma_ss(bmm_qk_desc, desc_kqf, desc_kqf_b, acc_cg0_1, accumulate=False, k_count=KQ_HALF_K) + mma_ss(bmm_qk_desc, desc_kqf + KQ_SEG, desc_kqf_b + KQ_SEG, acc_cg0_1, accumulate=True, k_count=KQ_HALF_K) + if elect_one: + bars.mb_cg0_acc_ready[pend_acc_idx].arrive(cta_group=1) + + for local_idx in cutlass.range(n_padded): # noqa: B007 if cutlass.const_expr(cfg.use_initial_state): if local_idx == 0: - kv_acc_index = advance(kv_acc_index, cfg.tmem_kv_acc_stages) - - k_idx = k_index.idx - q_idx = q_index.idx - s_idx = state_inp_index.idx + if elect_one: + bars.mb_state_acc_ready[kv_acc_index.idx].arrive(cta_group=1) + kv_acc_index = advance(kv_acc_index, cfg.tmem_state_acc_stages) + have_state = cutlass.Boolean(True) if cutlass.const_expr(cfg.use_initial_state) else local_idx > 0 + + kq_idx = kq_index.idx + member = local_idx & 1 + state_inp_idx = state_inp_index.idx q_state_acc_idx = o_acc_index.idx ainv_idx = ainv_index.idx qk_idx = qk_index.idx - qs2_idx = o_state_scale_index.idx + q_state2_idx = o_state_scale_index.idx kv_acc_idx = kv_acc_index.idx - desc_k = sK[k_idx].desc() - desc_q = sQ[q_idx].desc() + kq_member_off = member * KQ_BOX + desc_k = sKQ[kq_idx].desc() + kq_member_off + desc_q = sKQ[kq_idx].desc() + (KQ_BOX - kq_member_off) desc_ainv = sAinv[ainv_idx].desc() desc_nv = sQk[qk_idx].desc() - desc_kt = sK_trans[k_idx].desc() - state_a_ptr = nvvm.make_tmem_ptr(tmem_state_inp_col + s_idx * STATE_INP_STAGE_COLS, cutlass.Int8) - vks_a_ptr = nvvm.make_tmem_ptr(tmem_vks_col, cutlass.Int8) - qkv_a_ptr = nvvm.make_tmem_ptr(tmem_nv_inp_col, cutlass.Int8) - delta_a_ptr = nvvm.make_tmem_ptr(tmem_decay_v_col, cutlass.Int8) - ks_acc_ptr = nvvm.make_tmem_ptr(tmem_ks_col, cutlass.Float32) - qs_acc_ptr = nvvm.make_tmem_ptr(tmem_q_state_col + q_state_acc_idx * ACC_STAGE_COLS, cutlass.Float32) - nv_acc_ptr = nvvm.make_tmem_ptr(tmem_nv_col, cutlass.Float32) - qkv_acc_ptr = nvvm.make_tmem_ptr(tmem_q_state_col + qs2_idx * ACC_STAGE_COLS, cutlass.Float32) - kv_acc_ptr = nvvm.make_tmem_ptr(tmem_state_col + kv_acc_idx * KV_ACC_STAGE_COLS, cutlass.Float32) - - bars.mb_k_ready[k_idx].wait(k_index.phase) - k_index = advance(k_index, cfg.smem_k_stages) - bars.mb_q_ready[q_idx].wait(q_index.phase) - q_index = advance(q_index, cfg.smem_q_stages) - - # ---- GEMM 3: k*state --------------------------------------- - bars.mb_state_inp_ready[s_idx].wait(state_inp_index.phase) - state_inp_index = advance(state_inp_index, cfg.tmem_state_inp_stages) - - mma_ts( - bmm_qs_desc, - state_a_ptr, - desc_k, - ks_acc_ptr, - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_ks_ready[0].arrive(cta_group=1) - - # ---- GEMM 4: q*state --------------------------------------- + desc_kt = sKQ_trans[kq_idx].desc() + kq_member_off + state_a_ptr = nvvm.make_tmem_ptr(tmem_state_inp_col + state_inp_idx * STATE_INP_STAGE_COLS, cutlass.Int8) + q_state_acc_ptr = nvvm.make_tmem_ptr(tmem_q_state_col + q_state_acc_idx * ACC_STAGE_COLS, cutlass.Float32) + qkv_acc_ptr = nvvm.make_tmem_ptr(tmem_q_state_col + q_state2_idx * ACC_STAGE_COLS, cutlass.Float32) + state_acc_ptr = nvvm.make_tmem_ptr(tmem_state_col + kv_acc_idx * KV_ACC_STAGE_COLS, cutlass.Float32) + + kq_index = advance(kq_index, cfg.smem_kq_stages) + + # ---- QK/KK lookahead (member 1) = [Q;K](S) @ K^T --------------------- + if member == 1: + if (local_idx >> 1) + 1 < n_pairs: + pend_acc_idx = cg0_acc_index.idx + bars.mb_cg0_acc_done[pend_acc_idx].wait(cg0_acc_index.phase) + cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) + kqf_idx = kq_fused_index.idx + bars.mb_kq_ready[kqf_idx].wait(kq_fused_index.phase) + kq_fused_index = advance(kq_fused_index, cfg.smem_kq_stages) + desc_kqf = sKQ[kqf_idx].desc() + desc_kqf_b = desc_kqf + KQ_BOX + mma_ss(bmm_qk_desc, desc_kqf, desc_kqf_b, acc_cg0_1, accumulate=False, k_count=KQ_HALF_K) + mma_ss(bmm_qk_desc, desc_kqf + KQ_SEG, desc_kqf_b + KQ_SEG, acc_cg0_1, accumulate=True, k_count=KQ_HALF_K) + if elect_one: + bars.mb_cg0_acc_ready[pend_acc_idx].arrive(cta_group=1) + + # ---- k_state^T (GEMM 3) = state^T(T) @ K^T ------------------------------------ + if have_state: + bars.mb_state_inp_ready[state_inp_idx].wait(state_inp_index.phase) + state_inp_index = advance(state_inp_index, cfg.tmem_state_inp_stages) + + for k in cutlass.range_constexpr(KQ_HALF_K): + mma_ts_step(bmm_q_state_desc, state_a_ptr, desc_k, k_state_acc_ptr, k, cutlass.Boolean(k > 0)) + for k in cutlass.range_constexpr(KQ_HALF_K): + mma_ts_step(bmm_q_state_desc, state_a_ptr.subview(KQ_A_HALF), desc_k + KQ_SEG, k_state_acc_ptr, k, cutlass.Boolean(True)) + if elect_one: + bars.mb_k_state_ready[0].arrive(cta_group=1) + + # ---- q_state^T (GEMM 4) = state^T(T) @ Q^T ------------------------------------ bars.mb_o_acc_done[q_state_acc_idx].wait(o_acc_index.phase) o_acc_index = advance(o_acc_index, cfg.tmem_q_state_acc_stages) - mma_ts( - bmm_qs_desc, - state_a_ptr, - desc_q, - qs_acc_ptr, - accumulate=False, - ) - if nvvm.elect_sync(): - bars.mb_o_acc_ready[q_state_acc_idx].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_q_done[q_idx].arrive(cta_group=1) - - # ---- GEMM 5: new_v ----------------------------------------- - # vks_inp_ready also proves CG1 read KS out of the shared column - bars.mb_vks_inp_ready[0].wait(vks_inp_rdy.phase) - vks_inp_rdy = advance(vks_inp_rdy, 1) + if have_state: + for k in cutlass.range_constexpr(KQ_HALF_K): + mma_ts_step(bmm_q_state_desc, state_a_ptr, desc_q, q_state_acc_ptr, k, cutlass.Boolean(k > 0)) + for k in cutlass.range_constexpr(KQ_HALF_K): + mma_ts_step(bmm_q_state_desc, state_a_ptr.subview(KQ_A_HALF), desc_q + KQ_SEG, q_state_acc_ptr, k, cutlass.Boolean(True)) + if elect_one: + bars.mb_o_acc_ready[q_state_acc_idx].arrive(cta_group=1) + + # ---- NV^T (GEMM 5) = v_k_state^T(T) @ A_inv^T ------------------------------ bars.mb_ainv_ready[ainv_idx].wait(ainv_index.phase) ainv_index = advance(ainv_index, cfg.smem_ainv_stages) - mma_ts( - bmm_qkv_ts_desc, - vks_a_ptr, - desc_ainv, - nv_acc_ptr, - accumulate=False, - ) - if nvvm.elect_sync(): + bars.mb_v_k_state_inp_ready[0].wait(v_k_state_inp_ready.phase) + v_k_state_inp_ready = advance(v_k_state_inp_ready, 1) + for k in cutlass.range_constexpr(cfg.b_t // 16): + mma_ts_step(bmm_qkv_ts_desc, v_k_state_inp_ptr, desc_ainv, nv_acc_ptr, k, cutlass.Boolean(k > 0)) + if elect_one: bars.mb_nv_ready[0].arrive(cta_group=1) - if nvvm.elect_sync(): bars.mb_ainv_done[ainv_idx].arrive(cta_group=1) - # ---- GEMM 6: qkv ------------------------------------------- - bars.mb_nv_inp_ready[0].wait(nv_inp_rdy.phase) - nv_inp_rdy = advance(nv_inp_rdy, 1) + # ---- KK/QK lookahead (member 0) = [K;Q](S) @ K^T --------------------- + if member == 0: + if (local_idx >> 1) + 1 < n_pairs: + f0_acc_idx = cg0_acc_index.idx + bars.mb_cg0_acc_done[f0_acc_idx].wait(cg0_acc_index.phase) + cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) + kqf_idx = kq_fused_index.idx + bars.mb_kq_ready[kqf_idx].wait(kq_fused_index.phase) + kq_fused_index = advance(kq_fused_index, cfg.smem_kq_stages) + desc_kqf = sKQ[kqf_idx].desc() + mma_ss(bmm_qk_desc, desc_kqf, desc_kqf, acc_cg0_0, accumulate=False, k_count=KQ_HALF_K) + mma_ss(bmm_qk_desc, desc_kqf + KQ_SEG, desc_kqf + KQ_SEG, acc_cg0_0, accumulate=True, k_count=KQ_HALF_K) + if elect_one: + bars.mb_cg0_acc_ready[f0_acc_idx].arrive(cta_group=1) + + # ---- O^T (GEMM 6) += NV^T(T) @ W_qkv^T ------------------------------- bars.mb_qk_ready[qk_idx].wait(qk_index.phase) qk_index = advance(qk_index, cfg.smem_qk_stages) - bars.mb_o_state_scale_acc_done[qs2_idx].wait(o_state_scale_index.phase) - o_state_scale_index = advance(o_state_scale_index, cfg.tmem_q_state_acc_stages) - mma_ts( - bmm_qkv_ts_desc, - qkv_a_ptr, - desc_nv, - qkv_acc_ptr, - accumulate=True, - ) - if nvvm.elect_sync(): + if have_state: + bars.mb_o_state_scale_acc_done[q_state2_idx].wait(o_state_scale_index.phase) + o_state_scale_index = advance(o_state_scale_index, cfg.tmem_q_state_acc_stages) + bars.mb_nv_inp_ready[0].wait(nv_inp_ready.phase) + nv_inp_ready = advance(nv_inp_ready, 1) + for k in cutlass.range_constexpr(cfg.b_t // 16): + mma_ts_step(bmm_qkv_ts_desc, v_k_state_inp_ptr, desc_nv, qkv_acc_ptr, k, cutlass.Boolean(True) if cutlass.const_expr(k > 0) else have_state) + if elect_one: bars.mb_qk_done[qk_idx].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_o_state_scale_acc_ready[qs2_idx].arrive(cta_group=1) - - # ---- GEMM 7: kv_update ------------------------------------------- - bars.mb_decay_v_inp_ready[0].wait(decay_v_inp_rdy.phase) - decay_v_inp_rdy = advance(decay_v_inp_rdy, 1) - bars.mb_kv_acc_scale_done[kv_acc_idx].wait(kv_acc_index.phase) - kv_acc_index = advance(kv_acc_index, cfg.tmem_kv_acc_stages) - mma_ts( - bmm_kv_desc, - delta_a_ptr, - desc_kt, - kv_acc_ptr, - accumulate=True, - ) - if nvvm.elect_sync(): - bars.mb_kv_acc_ready[kv_acc_idx].arrive(cta_group=1) - if nvvm.elect_sync(): - bars.mb_k_done[k_idx].arrive(cta_group=1) + bars.mb_o_state_scale_acc_ready[q_state2_idx].arrive(cta_group=1) - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + # ---- state^T (GEMM 7) += decay_V^T(T) @ K -------------------------------- + bars.mb_decay_v_inp_ready[0].wait(decay_v_inp_ready.phase) + decay_v_inp_ready = advance(decay_v_inp_ready, 1) + kv_acc_index = advance(kv_acc_index, cfg.tmem_state_acc_stages) + for k in cutlass.range_constexpr(cfg.b_t // 16): + mma_ts_step(bmm_kv_desc, delta_inp_ptr, desc_kt, state_acc_ptr, k, cutlass.Boolean(True) if cutlass.const_expr(k > 0) else have_state) + if elect_one: + bars.mb_state_acc_ready[kv_acc_idx].arrive(cta_group=1) + bars.mb_kq_done[kq_idx].arrive(cta_group=1) + + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + bars.mb_tmem_done[0].wait(0) + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) + nvvm.tcgen05_dealloc( + nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), + cutlass.Int32(512), + group=nvvm.CTAGroup.CTA_1, + ) @cute.jit -def _tmaldg_warp( +def tmaldg_warp( cfg, total_tiles, bidx, num_ctas, cu_seqlens, mWorkItems, - sQ_raw, - sK_raw, + sKQ_raw, sV_raw, desc_q_base, desc_k_base, @@ -1263,44 +1052,29 @@ def _tmaldg_warp( bars, ): """TMA-LDG warp role (warp 9): persistent scheduler loop + per-chunk - Q/K/V G->S TMA loads. - - Each load goes through a per-(b,h) runtime descriptor (built once on - host) whose GLOBAL_ADDRESS already folds the sequence start and head, so - the only load coordinate is token = ``chunk_idx*BT``.""" + Q/K/V G->S TMA loads.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) - q_index = PipelineState.start(phase=1) - k_index = PipelineState.start(phase=1) + kq_index = PipelineState.start(phase=1) v_index = PipelineState.start(phase=1) sched_state = PipelineState.start(phase=1) tile_idx = cutlass.Int32(bidx) bpe = cfg.io_dtype.width // 8 - granu = 128 // bpe + elems_per_128b = 128 // bpe bt = cfg.b_t - q_stage_elems = cfg.q_cosize // cfg.smem_q_stages - k_stage_elems = cfg.k_cosize // cfg.smem_k_stages - sQ_tma = SmemTile( - base=sQ_raw, - elems_per_stage=q_stage_elems, - stages=cfg.smem_q_stages, - leading_byte_offset=0, - stride_byte_offset=0, - layout=0, - tma_loads_per_tile=2, - tma_granu_elems=granu, - tma_subtile_stride_elems=bt * granu, - ) - sK_tma = SmemTile( - base=sK_raw, - elems_per_stage=k_stage_elems, - stages=cfg.smem_k_stages, + kq_stage_elems = cfg.kq_cosize // cfg.smem_kq_stages + kq_box_elems = kq_stage_elems // 4 + sKQ_lo_tma = SmemTile( + base=sKQ_raw, + elems_per_stage=kq_stage_elems, + stages=cfg.smem_kq_stages, leading_byte_offset=0, stride_byte_offset=0, layout=0, tma_loads_per_tile=2, - tma_granu_elems=granu, - tma_subtile_stride_elems=bt * granu, + tma_granu_elems=elems_per_128b, + tma_subtile_stride_elems=2 * bt * elems_per_128b, ) sV_tma = SmemTile( base=sV_raw, @@ -1310,7 +1084,7 @@ def _tmaldg_warp( stride_byte_offset=0, layout=0, tma_loads_per_tile=2, - tma_granu_elems=granu, + tma_granu_elems=elems_per_128b, tma_subtile_stride_elems=4096, ) heads_out = cutlass.Int32(cfg.n_heads_out) @@ -1321,68 +1095,82 @@ def _tmaldg_warp( cfg, tile_idx, cu_seqlens, mWorkItems ) - slot = (batch_idx * heads_out + head_idx) * desc_qwords + head_q = head_idx if cfg.q_ratio == 1 else head_idx // cutlass.Int32(cfg.q_ratio) + head_k = head_idx if cfg.k_ratio == 1 else head_idx // cutlass.Int32(cfg.k_ratio) + head_v = head_idx if cfg.v_ratio == 1 else head_idx // cutlass.Int32(cfg.v_ratio) + slot = batch_idx * desc_qwords desc_q_slot = (desc_q_base + slot).tospace(cutlass.AddressSpace.generic) desc_k_slot = (desc_k_base + slot).tospace(cutlass.AddressSpace.generic) desc_v_slot = (desc_v_base + slot).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): + if elect_one: tma_tensormap_acquire(desc_q_slot) tma_tensormap_acquire(desc_k_slot) tma_tensormap_acquire(desc_v_slot) - # padded loads zero-fill (descriptor token extent is capped) wend_padded = cstart + ((wend - cstart + 1) // 2) * 2 - for chunk_idx in cutlass.range(cstart, wend_padded): - tok_coord = chunk_idx * cutlass.Int32(cfg.b_t) - - # ---------------------------------------------------------- - # K (B operand of GEMM-kk / GEMM-qk, double-buffered) - # ---------------------------------------------------------- - k_idx = k_index.idx - bars.mb_k_done[k_idx].wait(k_index.phase) - k_index = advance(k_index, cfg.smem_k_stages) - if nvvm.elect_sync(): - bars.mb_k_ready[k_idx].arrive(n_bytes=cfg.tma_k_bytes) - k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), tok_coord) - tma_load_tile(sK_tma[k_idx], k_slice, bars.mb_k_ready[k_idx].smem_ptr, acquire=False) - - # ---------------------------------------------------------- - # Q (A operand of GEMM-qk, single-buffered) - # ---------------------------------------------------------- - q_idx = q_index.idx - bars.mb_q_done[q_idx].wait(q_index.phase) - q_index = advance(q_index, cfg.smem_q_stages) - if nvvm.elect_sync(): - bars.mb_q_ready[q_idx].arrive(n_bytes=cfg.tma_q_bytes) - q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), tok_coord) - tma_load_tile(sQ_tma[q_idx], q_slice, bars.mb_q_ready[q_idx].smem_ptr, acquire=False) + if wend_padded > cstart: + kq_idx = kq_index.idx + bars.mb_kq_done[kq_idx].wait(kq_index.phase) + kq_index = advance(kq_index, cfg.smem_kq_stages) + if elect_one: + bars.mb_kq_ready[kq_idx].arrive(n_bytes=cfg.tma_kq_bytes) + tok_coord = cstart * cutlass.Int32(cfg.b_t) + k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, tok_coord) + q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), head_q, tok_coord) + kq_tile = sKQ_lo_tma[kq_idx] + tma_load_tile(kq_tile, k_slice, bars.mb_kq_ready[kq_idx].smem_ptr, acquire=False) + tma_load_tile(kq_tile.shifted(kq_box_elems), q_slice, bars.mb_kq_ready[kq_idx].smem_ptr, acquire=False) + for chunk_idx in cutlass.range(cstart + 1, wend_padded): + tok_coord = chunk_idx * cutlass.Int32(cfg.b_t) + + # ---- K + Q interleaved ------------------------------------------- + kq_idx = kq_index.idx + bars.mb_kq_done[kq_idx].wait(kq_index.phase) + kq_index = advance(kq_index, cfg.smem_kq_stages) + if elect_one: + bars.mb_kq_ready[kq_idx].arrive(n_bytes=cfg.tma_kq_bytes) + member = (chunk_idx - cstart) & 1 + k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, tok_coord) + q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), head_q, tok_coord) + kq_tile = sKQ_lo_tma[kq_idx] + if member == 0: + tma_load_tile(kq_tile, k_slice, bars.mb_kq_ready[kq_idx].smem_ptr, acquire=False) + tma_load_tile(kq_tile.shifted(kq_box_elems), q_slice, bars.mb_kq_ready[kq_idx].smem_ptr, acquire=False) + else: + tma_load_tile(kq_tile, q_slice, bars.mb_kq_ready[kq_idx].smem_ptr, acquire=False) + tma_load_tile(kq_tile.shifted(kq_box_elems), k_slice, bars.mb_kq_ready[kq_idx].smem_ptr, acquire=False) + + # ---- V load ------------------------------------------------------ + v_idx = v_index.idx + bars.mb_v_done[v_idx].wait(v_index.phase) + v_index = advance(v_index, cfg.smem_v_stages) + if elect_one: + bars.mb_v_ready[v_idx].arrive(n_bytes=cfg.tma_v_bytes) + v_tok = (chunk_idx - 1) * cutlass.Int32(cfg.b_t) + v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, v_tok) + tma_load_tile(sV_tma[v_idx], v_slice, bars.mb_v_ready[v_idx].smem_ptr, acquire=False) - # ---------------------------------------------------------- - # V (A operand of GEMM-new_v; transposed [DV, T] descriptor) - # ---------------------------------------------------------- v_idx = v_index.idx bars.mb_v_done[v_idx].wait(v_index.phase) v_index = advance(v_index, cfg.smem_v_stages) - if nvvm.elect_sync(): + if elect_one: bars.mb_v_ready[v_idx].arrive(n_bytes=cfg.tma_v_bytes) - v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), tok_coord) + v_tok = (wend_padded - 1) * cutlass.Int32(cfg.b_t) + v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, v_tok) tma_load_tile(sV_tma[v_idx], v_slice, bars.mb_v_ready[v_idx].smem_ptr, acquire=False) - tile_idx, sched_state = _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) + tile_idx, sched_state = sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) - for _ in range(cfg.smem_q_stages): - bars.mb_q_done[q_index.idx].wait(q_index.phase) - q_index = advance(q_index, cfg.smem_q_stages) - for _ in range(cfg.smem_k_stages): - bars.mb_k_done[k_index.idx].wait(k_index.phase) - k_index = advance(k_index, cfg.smem_k_stages) + for _ in range(cfg.smem_kq_stages): + bars.mb_kq_done[kq_index.idx].wait(kq_index.phase) + kq_index = advance(kq_index, cfg.smem_kq_stages) for _ in range(cfg.smem_v_stages): bars.mb_v_done[v_index.idx].wait(v_index.phase) v_index = advance(v_index, cfg.smem_v_stages) @cute.jit -def _compute0_warp( +def compute0_warp_group( cfg, total_tiles, bidx, @@ -1390,26 +1178,24 @@ def _compute0_warp( cu_seqlens, mWorkItems, tidx, - tmem_hold, + tmem_base_slot, scale, sCumsumlog, sBeta, sAinv, sQk, - sH_raw, + sCheckpoint_raw, checkpoint_every_n_tokens, sSched, bars, ): - """Compute warp-group 0 role (warps 0-3): persistent scheduler loop + - per-PAIR T-pairwise x2, kk_epi x2, pair inverse (warps 0-1 invert chunk - 0's matrix while warps 2-3 invert chunk 1's), qk_epi x2, and (enable_h) - the H checkpoint readouts state TMEM -> sH woven around the QK epilogues.""" + """Compute warp-group 0 role (warps 0-3): persistent scheduler loop + computing each chunk pair's A^{-1} and QK epilogues.""" nvvm.setmaxregister(cfg.num_regs_compute_group_0, nvvm.SetMaxRegisterAction.INCREASE) gate_index = PipelineState.start(phase=0) beta_index = PipelineState.start(phase=0) - cg0_acc_rdy = PipelineState.start(phase=0) + cg0_acc_ready = PipelineState.start(phase=0) ainv_index = PipelineState.start(phase=1) qk_index = PipelineState.start(phase=1) @@ -1417,18 +1203,21 @@ def _compute0_warp( cfg.tmem_alloc_barrier_id, thread_count=cfg.tmem_alloc_barrier_threads, ) - tmem_base = tmem_hold.load() + tmem_base = tmem_base_slot.load() num_threads_cg0 = cfg.threads_per_warp * len(cfg.compute_group_0_warp_ids) cg0_tidx = tidx % num_threads_cg0 warp_id = cg0_tidx // cfg.threads_per_warp lane_id = cg0_tidx % cfg.threads_per_warp inverse_local_warp = warp_id % 2 + pair_half = warp_id // 2 + half_row_base = inverse_local_warp * 32 bpe = cfg.io_dtype.width // 8 num_vals = 32 FRAG_COLS = 16 ACC_N_FRAGS = cfg.b_t // FRAG_COLS store_row = warp_id * 16 + lane_id % 16 + store_row_frag = lane_id % 16 store_col = (lane_id // 16) * 8 tmem_warp_row = warp_id * cfg.threads_per_warp tmem_cg0_acc_col = tmem_base + cfg.tmem_cg0_acc_offset @@ -1448,32 +1237,49 @@ def _compute0_warp( n_padded = n_pairs * 2 for pair_i in cutlass.range(n_pairs): - chunk0 = cstart + pair_i * 2 - chunk1 = chunk0 + 1 - # ---- Step 1: T-pairwise, both chunks in one traversal -------- + # ---- Gate rows for this warp's KK / QK member roles ------------------ gate0_idx = gate_index.idx bars.mb_gate_ready[gate0_idx].wait(gate_index.phase) gate_index = advance(gate_index, cfg.smem_gate_stages) gate1_idx = gate_index.idx bars.mb_gate_ready[gate1_idx].wait(gate_index.phase) gate_index = advance(gate_index, cfg.smem_gate_stages) - - row_cs0_lo = sCumsumlog[crow_lo, 0, gate0_idx] - row_cs0_hi = sCumsumlog[crow_hi, 0, gate0_idx] - row_cs1_lo = sCumsumlog[crow_lo, 0, gate1_idx] - row_cs1_hi = sCumsumlog[crow_hi, 0, gate1_idx] - - gT0 = [] - gT1 = [] - for k in cutlass.range_constexpr(num_vals): - hi_row = cutlass.const_expr(((k // 2) % 2) == 1) - crow = crow_hi if cutlass.const_expr(hi_row) else crow_lo - ccol = (lane_id % 4) * 2 + ((k // 4) * 8 + k % 2) - is_lower = crow >= ccol - cs0 = row_cs0_hi if cutlass.const_expr(hi_row) else row_cs0_lo - cs1 = row_cs1_hi if cutlass.const_expr(hi_row) else row_cs1_lo - gT0.append(cute.math.exp2(cs0 - sCumsumlog[ccol, 0, gate0_idx], fastmath=True) if is_lower else mask_zero) - gT1.append(cute.math.exp2(cs1 - sCumsumlog[ccol, 0, gate1_idx], fastmath=True) if is_lower else mask_zero) + kk_gate_idx = gate1_idx if pair_half == 1 else gate0_idx + qk_gate_idx = gate0_idx if pair_half == 1 else gate1_idx + + row_u0_lo = half_row_base + lane_id // 4 + row_u0_hi = row_u0_lo + 8 + row_u1_lo = row_u0_lo + 16 + row_u1_hi = row_u0_lo + 24 + + kk_cumsumlog_rows = [] + qk_cumsumlog_rows = [] + for r in (row_u0_lo, row_u0_hi, row_u1_lo, row_u1_hi): + kk_cumsumlog_rows.append(sCumsumlog[r, 0, kk_gate_idx]) + qk_cumsumlog_rows.append(sCumsumlog[r, 0, qk_gate_idx]) + kk_cumsumlog_cols = [] + qk_cumsumlog_cols = [] + for g in cutlass.range_constexpr(8): + for b in cutlass.range_constexpr(2): + ccol = (lane_id % 4) * 2 + g * 8 + b + kk_cumsumlog_cols.append(sCumsumlog[ccol, 0, kk_gate_idx]) + qk_cumsumlog_cols.append(sCumsumlog[ccol, 0, qk_gate_idx]) + + decay_t_kk = [] + decay_t_qk = [] + for u in cutlass.range_constexpr(2): + for k in cutlass.range_constexpr(num_vals): + hi_row = ((k // 2) % 2) == 1 + crow_u0 = row_u0_hi if cutlass.const_expr(hi_row) else row_u0_lo + crow_u1 = row_u1_hi if cutlass.const_expr(hi_row) else row_u1_lo + crow = crow_u1 if cutlass.const_expr(u == 1) else crow_u0 + ccol = (lane_id % 4) * 2 + ((k // 4) * 8 + k % 2) + is_lower = crow >= ccol + kk_row_cumsumlog = kk_cumsumlog_rows[u * 2 + (1 if hi_row else 0)] + qk_row_cumsumlog = qk_cumsumlog_rows[u * 2 + (1 if hi_row else 0)] + col = (k // 4) * 2 + (k % 2) + decay_t_kk.append(cute.math.exp2(kk_row_cumsumlog - kk_cumsumlog_cols[col], fastmath=True) if is_lower else mask_zero) + decay_t_qk.append(cute.math.exp2(qk_row_cumsumlog - qk_cumsumlog_cols[col], fastmath=True) if is_lower else mask_zero) bars.mb_gate_done[gate0_idx].arrive() bars.mb_gate_done[gate1_idx].arrive() @@ -1483,110 +1289,101 @@ def _compute0_warp( beta1_idx = beta_index.idx bars.mb_beta_ready[beta1_idx].wait(beta_index.phase) beta_index = advance(beta_index, cfg.smem_beta_stages) - # beta row scaling only needs the two per-thread row scalars - beta0_lo = sBeta[crow_lo, 0, beta0_idx] - beta0_hi = sBeta[crow_hi, 0, beta0_idx] - beta1_lo = sBeta[crow_lo, 0, beta1_idx] - beta1_hi = sBeta[crow_hi, 0, beta1_idx] + kk_beta_idx = beta1_idx if pair_half == 1 else beta0_idx + kk_beta = [] + for r in (row_u0_lo, row_u0_hi, row_u1_lo, row_u1_hi): + kk_beta.append(sBeta[r, 0, kk_beta_idx]) + + # ---- KK_epi (each warp pair stages its own member) ------------------- + acc0_idx = cg0_acc_ready.idx + acc0_phase = cg0_acc_ready.phase + cg0_acc_ready = advance(cg0_acc_ready, cfg.tmem_cg0_acc_stages) + acc1_idx = cg0_acc_ready.idx + acc1_phase = cg0_acc_ready.phase + cg0_acc_ready = advance(cg0_acc_ready, cfg.tmem_cg0_acc_stages) + kk_acc_idx = acc1_idx if pair_half == 1 else acc0_idx + kk_acc_phase = acc1_phase if pair_half == 1 else acc0_phase + qk_acc_idx = acc0_idx if pair_half == 1 else acc1_idx + qk_acc_phase = acc0_phase if pair_half == 1 else acc1_phase - # ---- Step 2: kk_epi0 + kk_epi1 ------------------------------ ainv0_idx = ainv_index.idx - bars.mb_ainv_done[ainv0_idx].wait(ainv_index.phase) + ainv0_phase = ainv_index.phase ainv_index = advance(ainv_index, cfg.smem_ainv_stages) - kk0_acc_idx = cg0_acc_rdy.idx - bars.mb_cg0_acc_ready[kk0_acc_idx].wait(cg0_acc_rdy.phase) - cg0_acc_rdy = advance(cg0_acc_rdy, cfg.tmem_cg0_acc_stages) - - ainv0_base = sAinv[ainv0_idx].base - kk_vec = nvvm.tcgen05_ld( - "16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_cg0_acc_col + kk0_acc_idx * ACC_STAGE_COLS, cutlass.Float32), num=8 - ) - nvvm.tcgen05_wait("load") - bars.mb_cg0_acc_done[kk0_acc_idx].arrive() - kk_f16 = [] - for k in cutlass.range_constexpr(num_vals // 2): - b0 = beta0_hi if cutlass.const_expr((k % 2) == 1) else beta0_lo - p0, p1 = fmul2(kk_vec[2 * k], kk_vec[2 * k + 1], gT0[2 * k], gT0[2 * k + 1]) - v0, v1 = fmul2(p0, p1, b0, b0) - kk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) - for c in cutlass.range_constexpr(ACC_N_FRAGS): - nvvm.stmatrix( - cutlass.inttoptr( - ainv0_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, - cutlass.AddressSpace.smem, - cutlass.BFloat16, - ), - [kk_f16[c * 4 + 0], kk_f16[c * 4 + 1], kk_f16[c * 4 + 2], kk_f16[c * 4 + 3]], - nvvm.MMALayout.ROW, - ) - ainv1_idx = ainv_index.idx - bars.mb_ainv_done[ainv1_idx].wait(ainv_index.phase) + ainv1_phase = ainv_index.phase ainv_index = advance(ainv_index, cfg.smem_ainv_stages) - kk1_acc_idx = cg0_acc_rdy.idx - bars.mb_cg0_acc_ready[kk1_acc_idx].wait(cg0_acc_rdy.phase) - cg0_acc_rdy = advance(cg0_acc_rdy, cfg.tmem_cg0_acc_stages) + kk_ainv_idx = ainv1_idx if pair_half == 1 else ainv0_idx + kk_ainv_phase = ainv1_phase if pair_half == 1 else ainv0_phase + bars.mb_cg0_acc_ready[kk_acc_idx].wait(kk_acc_phase) + ainv0_base = sAinv[ainv0_idx].base ainv1_base = sAinv[ainv1_idx].base - kk_vec = nvvm.tcgen05_ld( - "16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_cg0_acc_col + kk1_acc_idx * ACC_STAGE_COLS, cutlass.Float32), num=8 + kk_base = ainv1_base if pair_half == 1 else ainv0_base + kk_vec0 = nvvm.tcgen05_ld( + "16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_cg0_acc_col + kk_acc_idx * ACC_STAGE_COLS, cutlass.Float32), num=8 + ) + kk_vec1 = nvvm.tcgen05_ld( + "16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + 16) << 16) + tmem_cg0_acc_col + kk_acc_idx * ACC_STAGE_COLS, cutlass.Float32), num=8 ) nvvm.tcgen05_wait("load") - bars.mb_cg0_acc_done[kk1_acc_idx].arrive() - kk_f16 = [] - for k in cutlass.range_constexpr(num_vals // 2): - b1 = beta1_hi if cutlass.const_expr((k % 2) == 1) else beta1_lo - p0, p1 = fmul2(kk_vec[2 * k], kk_vec[2 * k + 1], gT1[2 * k], gT1[2 * k + 1]) - v0, v1 = fmul2(p0, p1, b1, b1) - kk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) - for c in cutlass.range_constexpr(ACC_N_FRAGS): - nvvm.stmatrix( - cutlass.inttoptr( - ainv1_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, - cutlass.AddressSpace.smem, - cutlass.BFloat16, - ), - [kk_f16[c * 4 + 0], kk_f16[c * 4 + 1], kk_f16[c * 4 + 2], kk_f16[c * 4 + 3]], - nvvm.MMALayout.ROW, - ) + bars.mb_ainv_done[kk_ainv_idx].wait(kk_ainv_phase) + for u in cutlass.range_constexpr(2): + kk_vec = kk_vec1 if cutlass.const_expr(u == 1) else kk_vec0 + kk_f16 = [] + for k in cutlass.range_constexpr(num_vals // 2): + b0 = kk_beta[u * 2 + 1] if cutlass.const_expr((k % 2) == 1) else kk_beta[u * 2] + p0, p1 = fmul2(kk_vec[2 * k], kk_vec[2 * k + 1], decay_t_kk[u * num_vals + 2 * k], decay_t_kk[u * num_vals + 2 * k + 1]) + v0, v1 = fmul2(p0, p1, b0, b0) + kk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) + st_row = half_row_base + u * 16 + store_row_frag + for c in cutlass.range_constexpr(ACC_N_FRAGS): + nvvm.stmatrix( + cutlass.inttoptr( + kk_base + (st_row * cfg.b_t + swizzle_xor_128b(st_row, store_col + c * FRAG_COLS)) * bpe, + cutlass.AddressSpace.smem, + cutlass.BFloat16, + ), + [kk_f16[c * 4 + 0], kk_f16[c * 4 + 1], kk_f16[c * 4 + 2], kk_f16[c * 4 + 3]], + nvvm.MMALayout.ROW, + ) - # ---- pair inverse: warps 0-1 own matrix 0, warps 2-3 matrix 1 + # ---- pair inverse: warps 0-1 own matrix 0, warps 2-3 matrix 1 -------- inv_base = ainv1_base if warp_id >= 2 else ainv0_base - # Stage 1: diagonal 8x8 Gauss-Jordan, all four warps + # diagonal 8x8 Gauss-Jordan, all four warps nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, ) - _invert_diagonal_NxN(cfg, inv_base, (inverse_local_warp * cfg.threads_per_warp + lane_id) // 8, cg0_tidx, 8) + invert_diagonal_NxN(cfg, inv_base, (inverse_local_warp * cfg.threads_per_warp + lane_id) // 8, cg0_tidx, 8) nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, ) - # Stage 2: 8x8 -> 16x16 (both matrices per warp) - _blockwise_diagonal_8x8_to_16x16(cfg, ainv0_base, warp_id * 16, lane_id) - _blockwise_diagonal_8x8_to_16x16(cfg, ainv1_base, warp_id * 16, lane_id) + # 8x8 -> 16x16 (both matrices per warp) + blockwise_diagonal_8x8_to_16x16(cfg, ainv0_base, warp_id * 16, lane_id) + blockwise_diagonal_8x8_to_16x16(cfg, ainv1_base, warp_id * 16, lane_id) nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, ) - # Stage 3: 16x16 -> 32x32, one tile per warp within the group - _blockwise_diagonal_16x16_to_32x32(cfg, inv_base, inverse_local_warp * 32, lane_id) + # 16x16 -> 32x32, one tile per warp within the group + blockwise_diagonal_16x16_to_32x32(cfg, inv_base, inverse_local_warp * 32, lane_id) nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, ) - # Stage 4: 32x32 -> 64x64, two warps per matrix - _blockwise_diagonal_32x32_to_64x64(cfg, inv_base, inverse_local_warp, lane_id) + # 32x32 -> 64x64, two warps per matrix + blockwise_diagonal_32x32_to_64x64(cfg, inv_base, inverse_local_warp, lane_id) nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, ) - # ---- beta column-scaling + publish, stage 0 then stage 1 ---- + # ---- Beta column-scaling + publish, stage 0 -------------------------- beta_col = [] for k in cutlass.range_constexpr(num_vals): beta_col.append(sBeta[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, beta0_idx]) @@ -1622,6 +1419,7 @@ def _compute0_warp( bars.mb_ainv_ready[ainv0_idx].arrive() bars.mb_beta_done[beta0_idx].arrive() + # ---- Beta column-scaling + publish, stage 1 -------------------------- beta_col = [] for k in cutlass.range_constexpr(num_vals): beta_col.append(sBeta[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, beta1_idx]) @@ -1657,71 +1455,50 @@ def _compute0_warp( bars.mb_ainv_ready[ainv1_idx].arrive() bars.mb_beta_done[beta1_idx].arrive() - # ---- Step 3: qk_epi0 ---------------------------------------- + # ---- QK_epi (opposite member, both halves in parallel) --------------- qk0_idx = qk_index.idx - bars.mb_qk_done[qk0_idx].wait(qk_index.phase) + qk0_phase = qk_index.phase qk_index = advance(qk_index, cfg.smem_qk_stages) - qk0_acc_idx = cg0_acc_rdy.idx - bars.mb_cg0_acc_ready[qk0_acc_idx].wait(cg0_acc_rdy.phase) - cg0_acc_rdy = advance(cg0_acc_rdy, cfg.tmem_cg0_acc_stages) - - qk0_base = sQk[qk0_idx].base - qk_vec = nvvm.tcgen05_ld( - "16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_cg0_acc_col + qk0_acc_idx * ACC_STAGE_COLS, cutlass.Float32), num=8 - ) - nvvm.tcgen05_wait("load") - bars.mb_cg0_acc_done[qk0_acc_idx].arrive() - qk_f16 = [] - for k in cutlass.range_constexpr(num_vals // 2): - p0, p1 = fmul2(qk_vec[2 * k], qk_vec[2 * k + 1], gT0[2 * k], gT0[2 * k + 1]) - v0, v1 = fmul2(p0, p1, scale, scale) - qk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) - for c in cutlass.range_constexpr(ACC_N_FRAGS): - nvvm.stmatrix( - cutlass.inttoptr( - qk0_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, - cutlass.AddressSpace.smem, - cutlass.BFloat16, - ), - [qk_f16[c * 4 + 0], qk_f16[c * 4 + 1], qk_f16[c * 4 + 2], qk_f16[c * 4 + 3]], - nvvm.MMALayout.ROW, - ) - nvvm.fence_proxy("async.shared", space="cta") - bars.mb_qk_ready[qk0_idx].arrive() - - # ---- Step 4: qk_epi1 ---------------------------------------- qk1_idx = qk_index.idx - bars.mb_qk_done[qk1_idx].wait(qk_index.phase) + qk1_phase = qk_index.phase qk_index = advance(qk_index, cfg.smem_qk_stages) - qk1_acc_idx = cg0_acc_rdy.idx - bars.mb_cg0_acc_ready[qk1_acc_idx].wait(cg0_acc_rdy.phase) - cg0_acc_rdy = advance(cg0_acc_rdy, cfg.tmem_cg0_acc_stages) + my_qk_idx = qk0_idx if pair_half == 1 else qk1_idx + my_qk_phase = qk0_phase if pair_half == 1 else qk1_phase + + bars.mb_cg0_acc_ready[qk_acc_idx].wait(qk_acc_phase) - qk1_base = sQk[qk1_idx].base - qk_vec = nvvm.tcgen05_ld( - "16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_cg0_acc_col + qk1_acc_idx * ACC_STAGE_COLS, cutlass.Float32), num=8 + qk_base = sQk[my_qk_idx].base + qk_vec0 = nvvm.tcgen05_ld( + "16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_cg0_acc_col + qk_acc_idx * ACC_STAGE_COLS, cutlass.Float32), num=8 + ) + qk_vec1 = nvvm.tcgen05_ld( + "16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + 16) << 16) + tmem_cg0_acc_col + qk_acc_idx * ACC_STAGE_COLS, cutlass.Float32), num=8 ) nvvm.tcgen05_wait("load") - bars.mb_cg0_acc_done[qk1_acc_idx].arrive() - qk_f16 = [] - for k in cutlass.range_constexpr(num_vals // 2): - p0, p1 = fmul2(qk_vec[2 * k], qk_vec[2 * k + 1], gT1[2 * k], gT1[2 * k + 1]) - v0, v1 = fmul2(p0, p1, scale, scale) - qk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) - for c in cutlass.range_constexpr(ACC_N_FRAGS): - nvvm.stmatrix( - cutlass.inttoptr( - qk1_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, - cutlass.AddressSpace.smem, - cutlass.BFloat16, - ), - [qk_f16[c * 4 + 0], qk_f16[c * 4 + 1], qk_f16[c * 4 + 2], qk_f16[c * 4 + 3]], - nvvm.MMALayout.ROW, - ) + bars.mb_cg0_acc_done[qk_acc_idx].arrive() + bars.mb_qk_done[my_qk_idx].wait(my_qk_phase) + for u in cutlass.range_constexpr(2): + qk_vec = qk_vec1 if cutlass.const_expr(u == 1) else qk_vec0 + qk_f16 = [] + for k in cutlass.range_constexpr(num_vals // 2): + p0, p1 = fmul2(qk_vec[2 * k], qk_vec[2 * k + 1], decay_t_qk[u * num_vals + 2 * k], decay_t_qk[u * num_vals + 2 * k + 1]) + v0, v1 = fmul2(p0, p1, scale, scale) + qk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) + st_row = half_row_base + u * 16 + store_row_frag + for c in cutlass.range_constexpr(ACC_N_FRAGS): + nvvm.stmatrix( + cutlass.inttoptr( + qk_base + (st_row * cfg.b_t + swizzle_xor_128b(st_row, store_col + c * FRAG_COLS)) * bpe, + cutlass.AddressSpace.smem, + cutlass.BFloat16, + ), + [qk_f16[c * 4 + 0], qk_f16[c * 4 + 1], qk_f16[c * 4 + 2], qk_f16[c * 4 + 3]], + nvvm.MMALayout.ROW, + ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_qk_ready[qk1_idx].arrive() + bars.mb_qk_ready[my_qk_idx].arrive() - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) for _ in range(cfg.smem_ainv_stages): bars.mb_ainv_done[ainv_index.idx].wait(ainv_index.phase) ainv_index = advance(ainv_index, cfg.smem_ainv_stages) @@ -1731,7 +1508,7 @@ def _compute0_warp( @cute.jit -def _compute1_warp( +def compute1_warp_group( cfg, total_tiles, bidx, @@ -1740,35 +1517,34 @@ def _compute1_warp( mWorkItems, tidx, warp_idx, - tmem_hold, + tmem_base_slot, scale, sV, sCumsumlog, sCumprod, sBeta, sO, - sH_raw, - mS_init, - mS_out, + sCheckpoint_raw, + mState_init, + mState_out, checkpoint_every_n_tokens, sSched, bars, ): - """Compute warp-group 1 role (warps 4-7): persistent scheduler loop, - initial-state seed, then one uniform per-chunk body: fused state - restage+rescale, V-K*S, state*q_epi, new_v_epi + decay_v publish, qkv - epilogue; final-state store per item.""" + """Compute warp-group 1 role (warps 4-7): persistent scheduler loop + running the per-chunk state-update and output epilogues.""" + elect_one = nvvm.elect_sync() v_index = PipelineState.start(phase=0) gate_index = PipelineState.start(phase=0) kv_acc_index = PipelineState.start(phase=0) - o_acc_rdy_index = PipelineState.start(phase=0) - o_scale_rdy_index = PipelineState.start(phase=0) - ks_rdy_index = PipelineState.start(phase=0) - nv_rdy_index = PipelineState.start(phase=0) - kv_acc_seed_index = PipelineState.start(phase=1) + o_acc_ready_index = PipelineState.start(phase=0) + o_scale_ready_index = PipelineState.start(phase=0) + k_state_ready_index = PipelineState.start(phase=0) + nv_ready_index = PipelineState.start(phase=0) + state_acc_seed_index = PipelineState.start(phase=1) o_index = PipelineState.start(phase=1) - si_cnt = cutlass.Int32(0) + state_inp_cnt = cutlass.Int32(0) kv_done_idx = cutlass.Int32(0) nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) @@ -1776,7 +1552,7 @@ def _compute1_warp( cfg.tmem_alloc_barrier_id, thread_count=cfg.tmem_alloc_barrier_threads, ) - tmem_base = tmem_hold.load() + tmem_base = tmem_base_slot.load() num_threads_cg1 = cfg.threads_per_warp * len(cfg.compute_group_1_warp_ids) cg1_tidx = tidx % num_threads_cg1 @@ -1785,30 +1561,30 @@ def _compute1_warp( ldtm_width = 32 sttm_width = ldtm_width // 2 num_state_subs = cutlass.const_expr(cfg.d_v // ldtm_width) - tmem_state_col = tmem_base + cfg.tmem_state_offset + tmem_state_col = tmem_base + cfg.tmem_state_acc_offset tmem_state_inp_col = tmem_base + cfg.tmem_state_inp_offset - tmem_q_state_col = tmem_base + cfg.tmem_q_state_offset - tmem_inp_col = tmem_base + cfg.tmem_inp_offset + tmem_q_state_col = tmem_base + cfg.tmem_q_state_acc_offset + tmem_inp_col = tmem_base + cfg.tmem_v_k_state_decay_inp_offset ACC_STAGE_COLS = cfg.b_t INP_SLOT_COLS = cfg.b_t // 2 - tmem_ks_col = tmem_base + cfg.tmem_cg1_acc_offset - tmem_nv_col = tmem_ks_col - tmem_vks_col = tmem_inp_col + tmem_k_state_col = tmem_base + cfg.tmem_cg1_acc_offset + tmem_nv_col = tmem_k_state_col + tmem_v_k_state_col = tmem_inp_col tmem_nv_inp_col = tmem_inp_col tmem_decay_v_col = tmem_inp_col + INP_SLOT_COLS - ov_tok = cg1_tidx % 8 + (cg1_tidx // 16 % 2) * 8 - ov_col = (cg1_tidx // 8 % 2) * 8 + (cg1_tidx // 32 % 2) * 32 - ov_slab = (cg1_tidx // 64) * 4096 + v_o_smem_tok = cg1_tidx % 8 + (cg1_tidx // 16 % 2) * 8 + v_o_smem_col = (cg1_tidx // 8 % 2) * 8 + (cg1_tidx // 32 % 2) * 32 + v_o_smem_subtile_off = (cg1_tidx // 64) * 4096 v_stage_elems = cfg.v_cosize // cfg.smem_v_stages o_stage_elems = cfg.o_cosize // cfg.smem_o_stages sV_base = cute.make_ptr(cfg.io_dtype, sV[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) sO_base = cute.make_ptr(cfg.io_dtype, sO[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) num_vals = 32 - if cutlass.const_expr(cfg.enable_h): - sH_base_int = sH_raw.data_ptr().toint() - h_cnt = cutlass.Int32(0) - h_ov_tok = cg1_tidx % 8 + (cg1_tidx // 16 % 2) * 8 - h_ov_col = (cg1_tidx // 8 % 2) * 8 + (cg1_tidx // 32 % 2) * 32 + if cutlass.const_expr(cfg.enable_checkpoints): + sCheckpoint_base_int = sCheckpoint_raw.data_ptr().toint() + checkpoint_cnt = cutlass.Int32(0) + checkpoint_smem_row = cg1_tidx % 8 + (cg1_tidx // 16 % 2) * 8 + checkpoint_smem_col = (cg1_tidx // 8 % 2) * 8 + (cg1_tidx // 32 % 2) * 32 sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) @@ -1816,62 +1592,85 @@ def _compute1_warp( batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) - sk_nt = wend - cstart - n_padded = ((sk_nt + 1) // 2) * 2 - if sk_nt > 0: + n_local = wend - cstart + n_padded = ((n_local + 1) // 2) * 2 + if cutlass.const_expr(cfg.enable_checkpoints): + ckpt_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) + checkpoint_mod = cstart % ckpt_chunks + if n_local > 0: if cutlass.const_expr(cfg.use_initial_state): - # ---- initial-state seed: S_init GMEM -> state TMEM - # (split-K warmup items seed zeros through the same path) ---- - gS_init = mS_init[None, None, head_idx, batch_idx] - kv_init_idx = kv_acc_seed_index.idx - bars.mb_kv_acc_scale_done[kv_init_idx].wait(kv_acc_seed_index.phase) - kv_acc_seed_index = advance(kv_acc_seed_index, cfg.tmem_kv_acc_stages) - seed_from_s0 = cstart == 0 - for sub in cutlass.range_constexpr(num_state_subs): - words = [] - for k in cutlass.range_constexpr(32): - v = gS_init[sub * ldtm_width + k, cg1_tidx] - if cutlass.const_expr(cfg.state_dtype != cfg.acc_dtype): - v = v.to(cfg.acc_dtype) - if cutlass.const_expr(cfg.split_k): - v = v if seed_from_s0 else cutlass.Float32(0.0) - words.append(v) - nvvm.tcgen05_st( - "32x32b", - nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), - cutlass.Vector.from_elements(tuple(words), cutlass.Float32), - ) + # ---- initial-state seed: initial_state GMEM -> state TMEM --------------- + gState_init = mState_init[None, None, head_idx, batch_idx] + kv_init_idx = state_acc_seed_index.idx + bars.mb_state_acc_scale_done[kv_init_idx].wait(state_acc_seed_index.phase) + state_acc_seed_index = advance(state_acc_seed_index, cfg.tmem_state_acc_stages) + seed_from_initial_state = cstart == 0 + if cutlass.const_expr(cfg.split_k): + if seed_from_initial_state: + for sub in cutlass.range_constexpr(num_state_subs): + words = [] + for k in cutlass.range_constexpr(32): + v = gState_init[sub * ldtm_width + k, cg1_tidx] + if cutlass.const_expr(cfg.state_dtype != cfg.acc_dtype): + v = v.to(cfg.acc_dtype) + words.append(v) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), + cutlass.Vector.from_elements(tuple(words), cutlass.Float32), + ) + else: + for sub in cutlass.range_constexpr(num_state_subs): + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), + cutlass.Vector.from_elements(tuple(cutlass.Float32(0.0) for _ in range(32)), cutlass.Float32), + ) + else: + for sub in cutlass.range_constexpr(num_state_subs): + words = [] + for k in cutlass.range_constexpr(32): + v = gState_init[sub * ldtm_width + k, cg1_tidx] + if cutlass.const_expr(cfg.state_dtype != cfg.acc_dtype): + v = v.to(cfg.acc_dtype) + words.append(v) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), + cutlass.Vector.from_elements(tuple(words), cutlass.Float32), + ) nvvm.tcgen05_wait("store") nvvm.barrier_cta_sync_aligned( cfg.init_state_store_barrier_id, thread_count=cfg.init_state_store_barrier_threads, ) - if cg1_tidx == 0: - arrive(bars.mb_kv_acc_ready[kv_init_idx].smem_ptr) for local_idx in cutlass.range(n_padded): # noqa: B007 chunk_idx = cstart + local_idx + if cutlass.const_expr(cfg.enable_checkpoints): + do_checkpoint_now = checkpoint_mod == 0 + checkpoint_mod = checkpoint_mod + cutlass.Int32(1) + checkpoint_mod = cutlass.Int32(0) if checkpoint_mod == ckpt_chunks else checkpoint_mod valid_state = local_idx > 0 if cutlass.const_expr(cfg.use_initial_state): valid_state = cutlass.Boolean(True) - kv_acc_seed_index = advance(kv_acc_seed_index, cfg.tmem_kv_acc_stages) + state_acc_seed_index = advance(state_acc_seed_index, cfg.tmem_state_acc_stages) gate_idx = gate_index.idx bars.mb_gate_ready[gate_idx].wait(gate_index.phase) gate_index = advance(gate_index, cfg.smem_gate_stages) cumprod_total = sCumprod[sCumprod.shape[0] - 1, 0, gate_idx] - # ---- fused state restage + rescale (one read serves both) ---- + # ---- state restage + rescale ------------------------------------- if valid_state: kv_idx = kv_acc_index.idx - bars.mb_kv_acc_ready[kv_idx].wait(kv_acc_index.phase) - kv_acc_index = advance(kv_acc_index, cfg.tmem_kv_acc_stages) + bars.mb_state_acc_ready[kv_idx].wait(kv_acc_index.phase) + kv_acc_index = advance(kv_acc_index, cfg.tmem_state_acc_stages) kv_done_idx = kv_idx state_regs = [[cutlass.Float32(0.0) for _ in range(num_state_subs)] for _ in range(32)] - siu_idx = si_cnt % cfg.tmem_state_inp_stages - # loads before stores: a TMEM st blocks ld hoisting + state_inp_stage_idx = state_inp_cnt % cfg.tmem_state_inp_stages state_vecs = [] for sub in cutlass.range_constexpr(num_state_subs): state_vecs.append( @@ -1887,50 +1686,54 @@ def _compute1_warp( cutlass.Vector.from_elements(tuple(state_f16), cutlass.Int32), ) nvvm.tcgen05_wait("store") - bars.mb_state_inp_ready[siu_idx].arrive() - si_cnt = si_cnt + 1 + bars.mb_state_inp_ready[state_inp_stage_idx].arrive() + state_inp_cnt = state_inp_cnt + 1 - if cutlass.const_expr(cfg.enable_h): - # ---- H checkpoint: state TMEM -> sH before the decayed - # write-back overwrites it ---- - do_h = (cfg.b_t * chunk_idx - 1) % checkpoint_every_n_tokens == checkpoint_every_n_tokens - 1 and chunk_idx < wend + if cutlass.const_expr(cfg.enable_checkpoints): + # ---- state checkpoint ---------------------------------------- + do_checkpoint = do_checkpoint_now and chunk_idx > 0 and chunk_idx < wend if cutlass.const_expr(cfg.split_k): - do_h = do_h and chunk_idx >= wstart - if do_h: - h_regs = [[cutlass.Int32(0) for _ in range(16)] for _ in range(4)] + do_checkpoint = do_checkpoint and chunk_idx >= wstart + if do_checkpoint: + checkpoint_regs = [[cutlass.Int32(0) for _ in range(16)] for _ in range(4)] for b in cutlass.range_constexpr(2): - for hh in cutlass.range_constexpr(2): - h_vec = nvvm.tcgen05_ld( + for col_half in cutlass.range_constexpr(2): + checkpoint_vec = nvvm.tcgen05_ld( "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + b * 16) << 16) + tmem_state_col + hh * 64, cutlass.Float32), + nvvm.make_tmem_ptr(((tmem_warp_row + b * 16) << 16) + tmem_state_col + col_half * 64, cutlass.Float32), num=8, ) for j in cutlass.range_constexpr(16): - h_regs[b * 2 + hh][j] = fp32_to_fp16(h_vec[2 * j], h_vec[2 * j + 1], dtype=cfg.io_dtype) - bars.mb_h_tmastg_done[h_cnt % cfg.smem_h_stages].wait(cutlass.Int32(1) ^ ((h_cnt // cfg.smem_h_stages) & cutlass.Int32(1))) + checkpoint_regs[b * 2 + col_half][j] = fp32_to_fp16( + checkpoint_vec[2 * j], checkpoint_vec[2 * j + 1], dtype=cfg.io_dtype + ) + checkpoint_stage = checkpoint_cnt % cfg.smem_checkpoint_stages + checkpoint_phase_done = cutlass.Int32(1) ^ ((checkpoint_cnt // cfg.smem_checkpoint_stages) & cutlass.Int32(1)) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].wait(checkpoint_phase_done) for b in cutlass.range_constexpr(2): - for hh in cutlass.range_constexpr(2): - h_base = (h_cnt % cfg.smem_h_stages) * cfg.d_k * cfg.d_v + (cg1_tidx // 64) * cfg.d_k * 64 + for col_half in cutlass.range_constexpr(2): + checkpoint_base = checkpoint_stage * cfg.d_k * cfg.d_v + (cg1_tidx // 64) * cfg.d_k * 64 for c in cutlass.range_constexpr(4): - h_row = hh * 64 + h_ov_tok + c * 16 + checkpoint_row = col_half * 64 + checkpoint_smem_row + c * 16 nvvm.stmatrix( cutlass.inttoptr( - sH_base_int + (h_base + h_row * 64 + swizzle_xor_128b(h_row, h_ov_col + b * 16)) * 2, + sCheckpoint_base_int + + (checkpoint_base + checkpoint_row * 64 + swizzle_xor_128b(checkpoint_row, checkpoint_smem_col + b * 16)) * 2, cutlass.AddressSpace.smem, cfg.io_dtype, ), [ - h_regs[b * 2 + hh][c * 4 + 0], - h_regs[b * 2 + hh][c * 4 + 1], - h_regs[b * 2 + hh][c * 4 + 2], - h_regs[b * 2 + hh][c * 4 + 3], + checkpoint_regs[b * 2 + col_half][c * 4 + 0], + checkpoint_regs[b * 2 + col_half][c * 4 + 1], + checkpoint_regs[b * 2 + col_half][c * 4 + 2], + checkpoint_regs[b * 2 + col_half][c * 4 + 3], ], nvvm.MMALayout.COL, ) nvvm.fence_proxy("async.shared", space="cta") - if nvvm.elect_sync(): - bars.mb_h_tmastg_ready[h_cnt % cfg.smem_h_stages].arrive() - h_cnt = h_cnt + 1 + if elect_one: + bars.mb_checkpoint_tmastg_ready[checkpoint_stage].arrive() + checkpoint_cnt = checkpoint_cnt + 1 for sub in cutlass.range_constexpr(num_state_subs): state_scaled = [] @@ -1943,26 +1746,24 @@ def _compute1_warp( cutlass.Vector.from_elements(tuple(state_scaled), cutlass.Float32), ) nvvm.tcgen05_wait("store") - bars.mb_kv_acc_scale_done[kv_idx].arrive() + bars.mb_state_acc_scale_done[kv_idx].arrive() - # ---- deferred per-row gate register builds --------------- - gCumprod = [] + # ---- per-row Gate register builds -------------------------------- + cumprod_vals = [] for k in cutlass.range_constexpr(num_vals): - gCumprod.append(sCumprod[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, gate_idx]) + cumprod_vals.append(sCumprod[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, gate_idx]) last_cumsumlog = sCumsumlog[cfg.b_t - 1, 0, gate_idx] - # gathers before math: interleaving pins one LDS latency per pair - gCs = [] + cumsumlog_vals = [] for k in cutlass.range_constexpr(num_vals): - gCs.append(sCumsumlog[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, gate_idx]) - gDecayScale = [] + cumsumlog_vals.append(sCumsumlog[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, gate_idx]) + decay_scale_vals = [] for k in cutlass.range_constexpr(0, num_vals, 2): - d0, d1 = fadd2(last_cumsumlog, last_cumsumlog, -gCs[k], -gCs[k + 1]) - gDecayScale.append(cute.math.exp2(d0, fastmath=True)) - gDecayScale.append(cute.math.exp2(d1, fastmath=True)) + d0, d1 = fadd2(last_cumsumlog, last_cumsumlog, -cumsumlog_vals[k], -cumsumlog_vals[k + 1]) + decay_scale_vals.append(cute.math.exp2(d0, fastmath=True)) + decay_scale_vals.append(cute.math.exp2(d1, fastmath=True)) bars.mb_gate_done[gate_idx].arrive() - # ---- v - k*state (packed 16-bit; V ring cursor survives - # item boundaries, so no fixed SMEM stage assumption) ------- + # ---- V - K*state (packed 16-bit) --------------------------------- v_idx = v_index.idx bars.mb_v_ready[v_idx].wait(v_index.phase) v_index = advance(v_index, cfg.smem_v_stages) @@ -1972,64 +1773,71 @@ def _compute1_warp( m0 = cutlass.const_expr(c % 4) sub = cutlass.const_expr(c // 4) v_f16 = nvvm.ldmatrix( - (sV_base + v_idx * v_stage_elems + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16)).raw_ptr(), + ( + sV_base + + v_idx * v_stage_elems + + v_o_smem_subtile_off + + (v_o_smem_tok + m0 * 16) * 64 + + swizzle_xor_128b(v_o_smem_tok + m0 * 16, v_o_smem_col + sub * 16) + ).raw_ptr(), 4, nvvm.MMALayout.COL, ) for i in cutlass.range_constexpr(4): v_words[4 * m0 + i][sub] = v_f16[i] if valid_state: - bars.mb_ks_ready[0].wait(ks_rdy_index.phase) - ks_rdy_index = advance(ks_rdy_index, 1) + bars.mb_k_state_ready[0].wait(k_state_ready_index.phase) + k_state_ready_index = advance(k_state_ready_index, 1) for sub in cutlass.range_constexpr(2): - ks_vec = nvvm.tcgen05_ld( + k_state_vec = nvvm.tcgen05_ld( "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_ks_col, cutlass.Float32), + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_k_state_col, cutlass.Float32), num=8, ) for j in cutlass.range_constexpr(16): - s0, s1 = fmul2(ks_vec[2 * j], ks_vec[2 * j + 1], gCumprod[2 * j], gCumprod[2 * j + 1]) - ks_word = fp32_to_fp16(s0, s1, dtype=cfg.io_dtype) - v_words[j][sub] = sub_f16x2(v_words[j][sub], ks_word, cfg.io_dtype) + s0, s1 = fmul2(k_state_vec[2 * j], k_state_vec[2 * j + 1], cumprod_vals[2 * j], cumprod_vals[2 * j + 1]) + k_state_word = fp32_to_fp16(s0, s1, dtype=cfg.io_dtype) + v_words[j][sub] = sub_f16x2(v_words[j][sub], k_state_word, cfg.io_dtype) for sub in cutlass.range_constexpr(2): nvvm.tcgen05_st( "16x128b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_vks_col, cutlass.Int32), + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_v_k_state_col, cutlass.Int32), cutlass.Vector.from_elements(tuple(v_words[j][sub] for j in range(16)), cutlass.Int32), ) nvvm.tcgen05_wait("store") - bars.mb_vks_inp_ready[0].arrive() + bars.mb_v_k_state_inp_ready[0].arrive() - # ---- state*q_epi: QS *= cumprod * scale, in place ------- + # ---- state*Q_epi: q_state *= cumprod * scale ---------------- if valid_state: - qs_idx = o_acc_rdy_index.idx - bars.mb_o_acc_ready[qs_idx].wait(o_acc_rdy_index.phase) - o_acc_rdy_index = advance(o_acc_rdy_index, cfg.tmem_q_state_acc_stages) + q_state_idx = o_acc_ready_index.idx + bars.mb_o_acc_ready[q_state_idx].wait(o_acc_ready_index.phase) + o_acc_ready_index = advance(o_acc_ready_index, cfg.tmem_q_state_acc_stages) - qs_ptrs = [] - qs_vecs = [] + q_state_ptrs = [] + q_state_vecs = [] for sub in cutlass.range_constexpr(2): - qs_ptrs.append(nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_q_state_col + qs_idx * ACC_STAGE_COLS, cutlass.Float32)) - qs_vecs.append(nvvm.tcgen05_ld("16x256b", qs_ptrs[sub], num=8)) + q_state_ptrs.append( + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_q_state_col + q_state_idx * ACC_STAGE_COLS, cutlass.Float32) + ) + q_state_vecs.append(nvvm.tcgen05_ld("16x256b", q_state_ptrs[sub], num=8)) for sub in cutlass.range_constexpr(2): - qs_scaled = [] + q_state_scaled = [] for j in cutlass.range_constexpr(16): - p0, p1 = fmul2(qs_vecs[sub][2 * j], qs_vecs[sub][2 * j + 1], gCumprod[2 * j], gCumprod[2 * j + 1]) + p0, p1 = fmul2(q_state_vecs[sub][2 * j], q_state_vecs[sub][2 * j + 1], cumprod_vals[2 * j], cumprod_vals[2 * j + 1]) s0, s1 = fmul2(p0, p1, scale, scale) - qs_scaled += [s0, s1] - nvvm.tcgen05_st("16x256b", qs_ptrs[sub], cutlass.Vector.from_elements(tuple(qs_scaled), cutlass.Float32)) + q_state_scaled += [s0, s1] + nvvm.tcgen05_st("16x256b", q_state_ptrs[sub], cutlass.Vector.from_elements(tuple(q_state_scaled), cutlass.Float32)) nvvm.tcgen05_wait("store") - bars.mb_o_state_scale_acc_done[qs_idx].arrive() + bars.mb_o_state_scale_acc_done[q_state_idx].arrive() - # ---- new_v_epi + decay_v publish ------------------------- - bars.mb_nv_ready[0].wait(nv_rdy_index.phase) - nv_rdy_index = advance(nv_rdy_index, 1) + # ---- new_V_epi + decay_V publish --------------------------------- + bars.mb_nv_ready[0].wait(nv_ready_index.phase) + nv_ready_index = advance(nv_ready_index, 1) bars.mb_v_done[v_idx].arrive() nv_regs = [[cutlass.Float32(0.0), cutlass.Float32(0.0)] for _ in range(32)] - # NV reuses the VKS slot; both publishes share one fence nv_vecs = [] for sub in cutlass.range_constexpr(2): nv_vecs.append( @@ -2049,9 +1857,13 @@ def _compute1_warp( nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_nv_inp_col, cutlass.Int32), cutlass.Vector.from_elements(tuple(nv_f16), cutlass.Int32), ) + nvvm.tcgen05_wait("store") + bars.mb_nv_inp_ready[0].arrive() + + for sub in cutlass.range_constexpr(2): for j in cutlass.range_constexpr(16): nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub] = fmul2( - nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], gDecayScale[2 * j], gDecayScale[2 * j + 1] + nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], decay_scale_vals[2 * j], decay_scale_vals[2 * j + 1] ) decay_f16 = [fp32_to_fp16(nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( @@ -2060,56 +1872,52 @@ def _compute1_warp( cutlass.Vector.from_elements(tuple(decay_f16), cutlass.Int32), ) nvvm.tcgen05_wait("store") - bars.mb_nv_inp_ready[0].arrive() bars.mb_decay_v_inp_ready[0].arrive() - # ---- qkv_epilogue: O acc -> sO --------------------------- - if cutlass.const_expr(cfg.enable_o): - o_idx = o_index.idx - bars.mb_o_tmastg_done[o_idx].wait(o_index.phase) - o_index = advance(o_index, cfg.smem_o_stages) - qs2_idx = o_scale_rdy_index.idx - bars.mb_o_state_scale_acc_ready[qs2_idx].wait(o_scale_rdy_index.phase) - o_scale_rdy_index = advance(o_scale_rdy_index, cfg.tmem_q_state_acc_stages) - - if cutlass.const_expr(cfg.enable_o): - o_regs = [] - for sub in cutlass.range_constexpr(2): - o_vec = nvvm.tcgen05_ld( - "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_q_state_col + qs2_idx * ACC_STAGE_COLS, cutlass.Float32), - num=8, - ) - o_regs.append([o_vec[k] for k in range(32)]) - # REQUIRED: an arrive does not order in-flight tcgen05 loads - nvvm.tcgen05_wait("load") - bars.mb_o_acc_done[qs2_idx].arrive() - if cutlass.const_expr(cfg.enable_o): - for sub in cutlass.range_constexpr(2): - for m0 in cutlass.range_constexpr(4): - o_f16 = [fp32_to_fp16(o_regs[sub][8 * m0 + 2 * j], o_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] - nvvm.stmatrix( - ( - sO_base + o_idx * o_stage_elems + ov_slab + (ov_tok + m0 * 16) * 64 + swizzle_xor_128b(ov_tok + m0 * 16, ov_col + sub * 16) - ).raw_ptr(), - o_f16, - nvvm.MMALayout.COL, - ) - nvvm.fence_proxy("async.shared", space="cta") + # ---- QKV_epilogue: O acc TMEM -> sO SMEM ------------------------- + o_idx = o_index.idx + bars.mb_o_tmastg_done[o_idx].wait(o_index.phase) + o_index = advance(o_index, cfg.smem_o_stages) + q_state2_idx = o_scale_ready_index.idx + bars.mb_o_state_scale_acc_ready[q_state2_idx].wait(o_scale_ready_index.phase) + o_scale_ready_index = advance(o_scale_ready_index, cfg.tmem_q_state_acc_stages) - bars.mb_o_tmastg_ready[o_idx].arrive() + o_regs = [] + for sub in cutlass.range_constexpr(2): + o_vec = nvvm.tcgen05_ld( + "16x256b", + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_q_state_col + q_state2_idx * ACC_STAGE_COLS, cutlass.Float32), + num=8, + ) + o_regs.append([o_vec[k] for k in range(32)]) + nvvm.tcgen05_wait("load") + bars.mb_o_acc_done[q_state2_idx].arrive() + for sub in cutlass.range_constexpr(2): + for m0 in cutlass.range_constexpr(4): + o_f16 = [fp32_to_fp16(o_regs[sub][8 * m0 + 2 * j], o_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + nvvm.stmatrix( + ( + sO_base + + o_idx * o_stage_elems + + v_o_smem_subtile_off + + (v_o_smem_tok + m0 * 16) * 64 + + swizzle_xor_128b(v_o_smem_tok + m0 * 16, v_o_smem_col + sub * 16) + ).raw_ptr(), + o_f16, + nvvm.MMALayout.COL, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_o_tmastg_ready[o_idx].arrive() - # ---- final state S: state TMEM -> GMEM ------------------------- - if sk_nt > 0: - # required even unstored: next item's decay_v must not race GEMM 7 + # ---- final state: state TMEM -> GMEM ----------------------------------- + if n_local > 0: kv_last_idx = kv_acc_index.idx - bars.mb_kv_acc_ready[kv_last_idx].wait(kv_acc_index.phase) - kv_acc_index = advance(kv_acc_index, cfg.tmem_kv_acc_stages) + bars.mb_state_acc_ready[kv_last_idx].wait(kv_acc_index.phase) + kv_acc_index = advance(kv_acc_index, cfg.tmem_state_acc_stages) if cutlass.const_expr(cfg.store_final_state): - # split-K: only the last chunk's owner holds the final state if cutlass.const_expr(cfg.split_k): if wend == num_chunks_b: - gS_out = mS_out[None, None, head_idx, batch_idx] + gState_out = mState_out[None, None, head_idx, batch_idx] for sub in cutlass.range_constexpr(num_state_subs): state_vec = nvvm.tcgen05_ld( "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), num=32 @@ -2118,9 +1926,9 @@ def _compute1_warp( val = state_vec[k] if cutlass.const_expr(cfg.state_dtype != cfg.acc_dtype): val = val.to(cfg.state_dtype) - gS_out[sub * ldtm_width + k, cg1_tidx] = val + gState_out[sub * ldtm_width + k, cg1_tidx] = val else: - gS_out = mS_out[None, None, head_idx, batch_idx] + gState_out = mState_out[None, None, head_idx, batch_idx] for sub in cutlass.range_constexpr(num_state_subs): state_vec = nvvm.tcgen05_ld( "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), num=32 @@ -2129,45 +1937,101 @@ def _compute1_warp( val = state_vec[k] if cutlass.const_expr(cfg.state_dtype != cfg.acc_dtype): val = val.to(cfg.state_dtype) - gS_out[sub * ldtm_width + k, cg1_tidx] = val - bars.mb_kv_acc_scale_done[kv_last_idx].arrive() + gState_out[sub * ldtm_width + k, cg1_tidx] = val + bars.mb_state_acc_scale_done[kv_last_idx].arrive() else: - bars.mb_kv_acc_scale_done[kv_last_idx].arrive() + bars.mb_state_acc_scale_done[kv_last_idx].arrive() else: - # zero-length sequence: state passes through, pure GMEM if cutlass.const_expr(cfg.store_final_state): write_passthrough = True if cutlass.const_expr(cfg.split_k): write_passthrough = wend == num_chunks_b if write_passthrough: - gS_out = mS_out[None, None, head_idx, batch_idx] + gState_out = mState_out[None, None, head_idx, batch_idx] if cutlass.const_expr(cfg.use_initial_state): - gS_in = mS_init[None, None, head_idx, batch_idx] + gState_in = mState_init[None, None, head_idx, batch_idx] for sub in cutlass.range_constexpr(num_state_subs): for k in cutlass.range_constexpr(32): - gS_out[sub * ldtm_width + k, cg1_tidx] = gS_in[sub * ldtm_width + k, cg1_tidx] + gState_out[sub * ldtm_width + k, cg1_tidx] = gState_in[sub * ldtm_width + k, cg1_tidx] else: for sub in cutlass.range_constexpr(num_state_subs): for k in cutlass.range_constexpr(32): - gS_out[sub * ldtm_width + k, cg1_tidx] = cutlass.Float32(0.0).to(cfg.state_dtype) + gState_out[sub * ldtm_width + k, cg1_tidx] = cutlass.Float32(0.0).to(cfg.state_dtype) - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) - # CG1 done with TMEM: release the MMA warp's dealloc bars.mb_tmem_done[0].arrive() - if cutlass.const_expr(cfg.enable_o): - for _ in range(cfg.smem_o_stages): - bars.mb_o_tmastg_done[o_index.idx].wait(o_index.phase) - o_index = advance(o_index, cfg.smem_o_stages) - if cutlass.const_expr(cfg.enable_h): - for _ in range(cfg.smem_h_stages): - bars.mb_h_tmastg_done[h_cnt % cfg.smem_h_stages].wait(cutlass.Int32(1) ^ ((h_cnt // cfg.smem_h_stages) & cutlass.Int32(1))) - h_cnt = h_cnt + 1 + for _ in range(cfg.smem_o_stages): + bars.mb_o_tmastg_done[o_index.idx].wait(o_index.phase) + o_index = advance(o_index, cfg.smem_o_stages) + if cutlass.const_expr(cfg.enable_checkpoints): + for _ in range(cfg.smem_checkpoint_stages): + checkpoint_stage = checkpoint_cnt % cfg.smem_checkpoint_stages + checkpoint_phase_done = cutlass.Int32(1) ^ ((checkpoint_cnt // cfg.smem_checkpoint_stages) & cutlass.Int32(1)) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].wait(checkpoint_phase_done) + checkpoint_cnt = checkpoint_cnt + 1 + + +@cute.kernel +def build_all_descs_kernel( + base_q: cutlass.GridConstant[tma.TensorMap], + base_k: cutlass.GridConstant[tma.TensorMap], + base_v: cutlass.GridConstant[tma.TensorMap], + base_o: cutlass.GridConstant[tma.TensorMap], + base_checkpoint: cutlass.GridConstant[tma.TensorMap], + desc_ws: cute.Tensor, + cu_seqlens: cute.Tensor, + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + o: Optional[cute.Tensor], + state_checkpoints_out: Optional[cute.Tensor], + n_batch: cutlass.Int32, + q_row_stride: cutlass.Int32, + k_row_stride: cutlass.Int32, + v_row_stride: cutlass.Int32, + o_row_stride: cutlass.Int32, + checkpoint_row_stride: cutlass.Int32, + checkpoint_every_n: cutlass.Int32, +) -> None: + """Single-launch builder for the per-batch descriptor arrays (one warp + per array).""" + tidx, _, _ = cute.arch.thread_idx() + widx = cutlass.Int32(tidx) // cutlass.Int32(32) + arr_words = n_batch * cutlass.Int32(TENSOR_MAP_QWORDS) + sub0 = cute.make_tensor(desc_ws.iterator, cute.make_layout((arr_words,), stride=(1,))) + sub1 = cute.make_tensor(desc_ws.iterator + arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub2 = cute.make_tensor(desc_ws.iterator + 2 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub3 = cute.make_tensor(desc_ws.iterator + 3 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub4 = cute.make_tensor(desc_ws.iterator + 4 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + + if widx == 0: + if nvvm.elect_sync(): + emit_seq_descs(base_q, sub0, cu_seqlens, q, n_batch, q_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 1: + if nvvm.elect_sync(): + emit_seq_descs(base_k, sub1, cu_seqlens, k, n_batch, k_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 2: + if nvvm.elect_sync(): + emit_seq_descs(base_v, sub2, cu_seqlens, v, n_batch, v_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if cutlass.const_expr(o is not None): + if widx == 3: + if nvvm.elect_sync(): + emit_seq_descs(base_o, sub3, cu_seqlens, o, n_batch, o_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if cutlass.const_expr(state_checkpoints_out is not None): + if widx == 4: + if nvvm.elect_sync(): + emit_checkpoint_seq_descs(base_checkpoint, sub4, cu_seqlens, state_checkpoints_out, n_batch, checkpoint_row_stride, checkpoint_every_n, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) @cute.jit -def _build_descs( +def build_descs( io_dtype: cutlass.Constexpr, b_t: cutlass.Constexpr[int], q: cute.Tensor, @@ -2175,191 +2039,84 @@ def _build_descs( v: cute.Tensor, o: Optional[cute.Tensor], cu_seqlens: cute.Tensor, - h_out: Optional[cute.Tensor], - h_every_n: cutlass.Int32, + state_checkpoints_out: Optional[cute.Tensor], + checkpoint_every_n: cutlass.Int32, tensormap_workspace: cute.Tensor, stream: cuda.CUstream, ): - """Build the 5 per-(b,h) TMA-descriptor arrays (Q, K, V, O, S) into - ``tensormap_workspace``. Compiled and launched separately from the main - kernel, and launched on every execute: the descriptors fold ``cu_seqlens`` - contents into GLOBAL_ADDRESS and GLOBAL_DIM, which the host cannot read - without a D2H sync that CUDA-graph capture forbids. - - The H descriptor is 3-D ``(dv, dk, h)`` over the packed - ``[total_h, HO, DK, DV]`` H tensor; ``build_h_descs_kernel`` derives the - per-sequence H offset from the token ``cu_seqlens`` ((seqlen-1)//h_every_n, - prefix-summed), folds it and the head into GLOBAL_ADDRESS, and caps - GLOBAL_DIM[2] to the per-sequence H count, so the store coordinate is the - sequence-local H index.""" + """Build the 5 per-(b,h) TMA-descriptor arrays (Q, K, V, O, checkpoints) into + ``tensormap_workspace``.""" h_q = q.shape[1] h_k = k.shape[1] h_v = v.shape[1] batch_size = cu_seqlens.shape[0] - 1 heads_out = h_q if h_q >= h_v else h_v - q_group = heads_out // h_q - k_group = heads_out // h_k - v_group = heads_out // h_v d_v = v.shape[2] bpe = io_dtype.width // 8 - granu = 128 // bpe + elems_per_128b = 128 // bpe bt = b_t q_row_stride, q_head_stride = q.stride[0], q.stride[1] k_row_stride, k_head_stride = k.stride[0], k.stride[1] v_row_stride, v_head_stride = v.stride[0], v.stride[1] - q_head0 = q[None, 0, None] - k_head0 = k[None, 0, None] - v_view = v[None, 0, None] - v_head0 = cute.make_tensor( - v_view.iterator, - cute.make_layout((d_v, v_view.shape[0]), stride=(v_view.stride[1], v_view.stride[0])), - ) - swz128 = _tma.TensorMapSwizzle.s128b - base_desc_q = _tma.create_tensor_map_tiled_from_view(q_head0, box_dims=(bt, granu), stride_order=(1, 0), swizzle=swz128) - base_desc_k = _tma.create_tensor_map_tiled_from_view(k_head0, box_dims=(bt, granu), stride_order=(1, 0), swizzle=swz128) - base_desc_v = _tma.create_tensor_map_tiled_from_view(v_head0, box_dims=(granu, bt), stride_order=(0, 1), swizzle=swz128) - - arr_words = (batch_size * heads_out) * TENSOR_MAP_QWORDS - ws_iter = tensormap_workspace.iterator - - def sub_array(k): - return cute.make_tensor(ws_iter + k * arr_words, cute.make_layout((arr_words,), stride=(1,))) + seqlen = q.shape[0] + d_k = q.shape[2] + q_headed = cute.make_tensor(q.iterator, cute.make_layout((seqlen, h_q, d_k), stride=(q_row_stride, q_head_stride, 1))) + k_headed = cute.make_tensor(k.iterator, cute.make_layout((seqlen, h_k, d_k), stride=(k_row_stride, k_head_stride, 1))) + v_headed = cute.make_tensor(v.iterator, cute.make_layout((d_v, h_v, seqlen), stride=(1, v_head_stride, v_row_stride))) + swz128 = tma.TensorMapSwizzle.s128b + base_desc_q = tma.create_tensor_map_tiled_from_view(q_headed, box_dims=(bt, 1, elems_per_128b), stride_order=(2, 1, 0), swizzle=swz128) + base_desc_k = tma.create_tensor_map_tiled_from_view(k_headed, box_dims=(bt, 1, elems_per_128b), stride_order=(2, 1, 0), swizzle=swz128) + base_desc_v = tma.create_tensor_map_tiled_from_view(v_headed, box_dims=(elems_per_128b, 1, bt), stride_order=(0, 1, 2), swizzle=swz128) + + base_desc_o = base_desc_v + if cutlass.const_expr(o is not None): + o_headed = cute.make_tensor(o.iterator, cute.make_layout((d_v, heads_out, seqlen), stride=(1, o.stride[1], o.stride[0]))) + base_desc_o = tma.create_tensor_map_tiled_from_view(o_headed, box_dims=(elems_per_128b, 1, bt), stride_order=(0, 1, 2), swizzle=swz128) + + base_desc_checkpoint = base_desc_v + if cutlass.const_expr(state_checkpoints_out is not None): + d_k_state = state_checkpoints_out.shape[2] + d_v_state = state_checkpoints_out.shape[3] + checkpoint_elems_per_128b = 128 // (state_checkpoints_out.element_type.width // 8) + checkpoint_view = cute.make_tensor( + state_checkpoints_out.iterator, + cute.make_layout( + (d_v_state, d_k_state, state_checkpoints_out.shape[0], heads_out), + stride=(state_checkpoints_out.stride[3], state_checkpoints_out.stride[2], state_checkpoints_out.stride[0], state_checkpoints_out.stride[1]), + ), + ) + base_desc_checkpoint = tma.create_tensor_map_tiled_from_view( + checkpoint_view, box_dims=(checkpoint_elems_per_128b, d_k_state, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz128 + ) - build_qkv_load_descs_kernel( + n_warps = 5 if state_checkpoints_out is not None else (4 if o is not None else 3) + build_all_descs_kernel( base_desc_q, - sub_array(0), - cu_seqlens, - q, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(q_group), - cutlass.Int32(q_head_stride), - cutlass.Int32(q_row_stride), - 1, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( base_desc_k, - sub_array(1), - cu_seqlens, - k, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(k_group), - cutlass.Int32(k_head_stride), - cutlass.Int32(k_row_stride), - 1, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( base_desc_v, - sub_array(2), + base_desc_o, + base_desc_checkpoint, + tensormap_workspace, cu_seqlens, + q, + k, v, + o, + state_checkpoints_out, cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(v_group), - cutlass.Int32(v_head_stride), + cutlass.Int32(q_row_stride), + cutlass.Int32(k_row_stride), cutlass.Int32(v_row_stride), - 1, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - if cutlass.const_expr(o is not None): - o_row_stride, o_head_stride = o.stride[0], o.stride[1] - o_view = o[None, 0, None] - o_head0 = cute.make_tensor( - o_view.iterator, - cute.make_layout((d_v, o_view.shape[0]), stride=(o_view.stride[1], o_view.stride[0])), - ) - base_desc_o = _tma.create_tensor_map_tiled_from_view(o_head0, box_dims=(granu, bt), stride_order=(0, 1), swizzle=swz128) - build_qkv_load_descs_kernel( - base_desc_o, - sub_array(3), - cu_seqlens, - o, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(1), - cutlass.Int32(o_head_stride), - cutlass.Int32(o_row_stride), - 1, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - - if cutlass.const_expr(h_out is not None): - d_k_s = h_out.shape[2] - d_v_s = h_out.shape[3] - h_granu = 128 // (h_out.element_type.width // 8) - h_view = cute.make_tensor( - h_out.iterator, - cute.make_layout( - (d_v_s, d_k_s, h_out.shape[0]), - stride=(h_out.stride[3], h_out.stride[2], h_out.stride[0]), - ), - ) - base_desc_h = _tma.create_tensor_map_tiled_from_view(h_view, box_dims=(h_granu, d_k_s, 1), stride_order=(0, 1, 2), swizzle=swz128) - build_h_descs_kernel( - base_desc_h, - sub_array(4), - cu_seqlens, - h_out, - cutlass.Int32(batch_size), - cutlass.Int32(heads_out), - cutlass.Int32(h_out.stride[1]), - cutlass.Int32(h_out.stride[0]), - h_every_n, - 2, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) + cutlass.Int32(o.stride[0] if o is not None else 0), + cutlass.Int32(state_checkpoints_out.stride[0] if state_checkpoints_out is not None else 0), + checkpoint_every_n, + ).launch(grid=(1, 1, 1), block=(32 * n_warps, 1, 1), stream=stream) @cute.jit -def _downcast_state( - s0: cute.Tensor, - out: cute.Tensor, - n: cutlass.Int32, - n_blocks: cutlass.Int32, - stream: cuda.CUstream, -): - downcast_state_kernel( - s0, - out, - n, - ).launch(grid=(n_blocks, 1, 1), block=(128, 1, 1), stream=stream) - - -@functools.cache -def _downcast_state_cache(key): - return {} - - -def downcast_state(initial_state, out, *, stream): - """Copy the fp32 initial state ``[N, HO, K, V]`` into ``out`` (io dtype, - same shape) — the buffer the backward's per-(b,h) state descriptors - read.""" - n = 1 - for s_ in out.shape: - n *= int(s_) - key = (str(initial_state.dtype), str(out.dtype)) - cache = _downcast_state_cache(key) - cu_stream = cuda.CUstream(int(stream)) - n_blocks = (n + 127) // 128 - s0_flat = initial_state.reshape(n) - out_flat = out.reshape(n) - if "compiled" not in cache: - s0_c = from_dlpack(s0_flat, assumed_align=16).mark_layout_dynamic() - out_c = from_dlpack(out_flat, assumed_align=16).mark_layout_dynamic() - cache["compiled"] = cute.compile( - _downcast_state, - s0_c, - out_c, - cutlass.Int32(n), - cutlass.Int32(n_blocks), - cu_stream, - options="--enable-tvm-ffi", - ) - cache["compiled"](s0_flat, out_flat, n, n_blocks, cu_stream) - - -@cute.jit -def _host( +def host( cfg: cutlass.Constexpr, q: cute.Tensor, k: cute.Tensor, @@ -2368,8 +2125,8 @@ def _host( beta: cute.Tensor, o: Optional[cute.Tensor], cu_seqlens: cute.Tensor, - s_in: Optional[cute.Tensor], - s_out: Optional[cute.Tensor], + state_in: Optional[cute.Tensor], + state_out: Optional[cute.Tensor], work_items: Optional[cute.Tensor], work_count: Optional[cute.Tensor], sched_ctr: Optional[cute.Tensor], @@ -2379,11 +2136,12 @@ def _host( stream: cuda.CUstream, ): h_q = q.shape[1] + h_k = k.shape[1] h_v = v.shape[1] batch_size = cu_seqlens.shape[0] - 1 heads_out = h_q if h_q >= h_v else h_v - # ---- GQA reshapes: fold the head group into a ---------------------- + # ---- GQA reshapes: fold the head group into a -------------------------------- if cutlass.const_expr(cfg.is_GQA): h_r = h_q // h_v h_qv = h_v @@ -2455,71 +2213,72 @@ def _host( stride=(o.stride[2], o.stride[0], (o.stride[1], h_r * o.stride[1])), ), ) - if cutlass.const_expr(s_in is not None): - s_in = cute.make_tensor( - s_in.iterator, + if cutlass.const_expr(state_in is not None): + state_in = cute.make_tensor( + state_in.iterator, cute.make_layout( - (s_in.shape[2], s_in.shape[3], (h_r, h_qv), s_in.shape[0]), + (state_in.shape[2], state_in.shape[3], (h_r, h_qv), state_in.shape[0]), stride=( - s_in.stride[2], - s_in.stride[3], - (s_in.stride[1], h_r * s_in.stride[1]), - s_in.stride[0], + state_in.stride[2], + state_in.stride[3], + (state_in.stride[1], h_r * state_in.stride[1]), + state_in.stride[0], ), ), ) - if cutlass.const_expr(s_out is not None): - s_out = cute.make_tensor( - s_out.iterator, + if cutlass.const_expr(state_out is not None): + state_out = cute.make_tensor( + state_out.iterator, cute.make_layout( - (s_out.shape[2], s_out.shape[3], (h_r, h_qv), s_out.shape[0]), + (state_out.shape[2], state_out.shape[3], (h_r, h_qv), state_out.shape[0]), stride=( - s_out.stride[2], - s_out.stride[3], - (s_out.stride[1], h_r * s_out.stride[1]), - s_out.stride[0], + state_out.stride[2], + state_out.stride[3], + (state_out.stride[1], h_r * state_out.stride[1]), + state_out.stride[0], ), ), ) - # ---- SMEM sizing: per-buffer element cosizes ----------------------- + + # ---- SMEM sizing: per-buffer element cosizes --------------------------------- bpe = cfg.io_dtype.width // 8 - q_tile_elems = cfg.b_t * cfg.d_k - k_tile_elems = cfg.b_t * cfg.d_k + kq_tile_elems = 2 * cfg.b_t * cfg.d_k v_tile_elems = cfg.d_v * cfg.b_t ainv_tile_elems = cfg.b_t * cfg.b_t qk_tile_elems = cfg.b_t * cfg.b_t o_tile_elems = cfg.d_v * cfg.b_t - cfg.q_cosize = q_tile_elems * cfg.smem_q_stages - cfg.k_cosize = k_tile_elems * cfg.smem_k_stages + cfg.kq_cosize = kq_tile_elems * cfg.smem_kq_stages cfg.v_cosize = v_tile_elems * cfg.smem_v_stages cfg.ainv_cosize = ainv_tile_elems * cfg.smem_ainv_stages cfg.qk_cosize = qk_tile_elems * cfg.smem_qk_stages cfg.o_cosize = o_tile_elems * cfg.smem_o_stages - cfg.h_cosize = cfg.d_k * cfg.d_v * cfg.smem_h_stages + cfg.checkpoint_cosize = cfg.d_k * cfg.d_v * cfg.smem_checkpoint_stages cumsumlog_smem_layout_staged = cute.make_layout((cfg.b_t, 1, cfg.smem_gate_stages)) beta_smem_layout_staged = cute.make_layout((cfg.b_t, 1, cfg.smem_beta_stages)) - cfg.tma_q_bytes = q_tile_elems * bpe - cfg.tma_k_bytes = k_tile_elems * bpe + cfg.tma_kq_bytes = kq_tile_elems * bpe cfg.tma_v_bytes = v_tile_elems * bpe cfg.tma_o_bytes = o_tile_elems * bpe cfg.n_heads_out = heads_out - num_descs = batch_size * heads_out + cfg.q_ratio = heads_out // h_q + cfg.k_ratio = heads_out // h_k + cfg.v_ratio = heads_out // h_v + num_descs = batch_size - # ---- launch -------------------------------------------------------- + # ---- launch ------------------------------------------------------------------ total_tiles = batch_size * heads_out # CUDA-graph-stable launch: fixed SM-count grid; shapes ride on buffer contents grid_shape = (cfg.max_active_clusters, 1, 1) - _kernel( + kernel( cfg, gate, beta, cu_seqlens, - s_in, - s_out, + state_in, + state_out, work_items, work_count, sched_ctr, @@ -2544,13 +2303,13 @@ def _host( @cute.kernel -def _kernel( +def kernel( cfg: cutlass.Constexpr, mGate: cute.Tensor, mBeta: cute.Tensor, cu_seqlens: cute.Tensor, - mS_init: Optional[cute.Tensor], - mS_out: Optional[cute.Tensor], + mState_init: Optional[cute.Tensor], + mState_out: Optional[cute.Tensor], mWorkItems: Optional[cute.Tensor], mCount: Optional[cute.Tensor], mSched: Optional[cute.Tensor], @@ -2566,13 +2325,7 @@ def _kernel( tensormap_workspace: cute.Tensor, n_desc: cutlass.Int32, ): - """ - Main GDN chunked kernel. - - Warp specialization is the outermost control flow: each warp role owns - its own persistent tile-scheduler loop, iterating over (batch, head) - tiles and then over chunks within each tile. - """ + """Main GDN chunked kernel: warp-specialized persistent tile loop.""" tidx, _, _ = cute.arch.thread_idx() warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) bidx = cute.arch.block_idx()[0] @@ -2584,13 +2337,13 @@ def _kernel( assert mSched is not None, "mSched must be provided if dyn_sched is True" if cutlass.const_expr(cfg.use_initial_state): - assert mS_init is not None, "mS_init must be provided if use_initial_state is True" + assert mState_init is not None, "mState_init must be provided if use_initial_state is True" else: - assert mS_init is None, "mS_init must be None if use_initial_state is False" + assert mState_init is None, "mState_init must be None if use_initial_state is False" if cutlass.const_expr(cfg.store_final_state): - assert mS_out is not None, "mS_out must be provided if store_final_state is True" + assert mState_out is not None, "mState_out must be provided if store_final_state is True" else: - assert mS_out is None, "mS_out must be None if store_final_state is False" + assert mState_out is None, "mState_out must be None if store_final_state is False" desc_base_words = tensormap_workspace.iterator.raw_ptr() desc_qwords = cutlass.Int32(TENSOR_MAP_QWORDS) @@ -2599,15 +2352,9 @@ def _kernel( desc_k_base = desc_base_words + arr_words desc_v_base = desc_base_words + cutlass.Int32(2) * arr_words desc_o_base = desc_base_words + cutlass.Int32(3) * arr_words - desc_h_base = desc_base_words + cutlass.Int32(4) * arr_words + desc_checkpoint_base = desc_base_words + cutlass.Int32(4) * arr_words SMEM = cutlass.AddressSpace.smem - bars = make_gdn_bars(cfg) - tmem_hold = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=16) - sSched = cutlass.Array(cutlass.Int32, cfg.sched_stages, space=SMEM, alignment=16) - cumsumlog_raw = cutlass.Array(cutlass.Float32, cute.cosize(cumsumlog_smem_layout_staged), space=SMEM, alignment=128) - cumprod_raw = cutlass.Array(cutlass.Float32, cute.cosize(cumsumlog_smem_layout_staged), space=SMEM, alignment=128) - beta_raw = cutlass.Array(cutlass.Float32, cute.cosize(beta_smem_layout_staged), space=SMEM, alignment=128) bpe = cfg.io_dtype.width // 8 SWZ = 2 @@ -2615,42 +2362,57 @@ def _kernel( STRIDE = 8 * 128 KT_LEAD = (cfg.d_v // 2) * 128 V_LEAD = (cfg.d_v // 2) * 128 - sQ_raw = cutlass.Array( + sO_raw = cutlass.Array( cfg.io_dtype, - cfg.q_cosize, + cfg.o_cosize, space=cutlass.AddressSpace.smem, alignment=cfg.buffer_align_bytes, ) - sQ = SmemTile( - base=sQ_raw.data_ptr().toint(), - elems_per_stage=(cfg.q_cosize // cfg.smem_q_stages) * bpe, - stages=cfg.smem_q_stages, + sO = SmemTile( + base=sO_raw.data_ptr().toint(), + elems_per_stage=(cfg.o_cosize // cfg.smem_o_stages) * bpe, + stages=cfg.smem_o_stages, leading_byte_offset=LEAD, stride_byte_offset=STRIDE, layout=SWZ, ) - sK_raw = cutlass.Array( + if cutlass.const_expr(cfg.enable_checkpoints): + sCheckpoint_raw = cutlass.Array( + cfg.io_dtype, + cfg.checkpoint_cosize, + space=cutlass.AddressSpace.smem, + alignment=cfg.buffer_align_bytes, + ) + else: + sCheckpoint_raw = None + sKQ_raw = cutlass.Array( cfg.io_dtype, - cfg.k_cosize, + cfg.kq_cosize, space=cutlass.AddressSpace.smem, alignment=cfg.buffer_align_bytes, ) - sK = SmemTile( - base=sK_raw.data_ptr().toint(), - elems_per_stage=(cfg.k_cosize // cfg.smem_k_stages) * bpe, - stages=cfg.smem_k_stages, + sKQ = SmemTile( + base=sKQ_raw.data_ptr().toint(), + elems_per_stage=(cfg.kq_cosize // cfg.smem_kq_stages) * bpe, + stages=cfg.smem_kq_stages, leading_byte_offset=LEAD, stride_byte_offset=STRIDE, layout=SWZ, ) - sK_trans = SmemTile( - base=sK_raw.data_ptr().toint(), - elems_per_stage=(cfg.k_cosize // cfg.smem_k_stages) * bpe, - stages=cfg.smem_k_stages, - leading_byte_offset=KT_LEAD, + sKQ_trans = SmemTile( + base=sKQ_raw.data_ptr().toint(), + elems_per_stage=(cfg.kq_cosize // cfg.smem_kq_stages) * bpe, + stages=cfg.smem_kq_stages, + leading_byte_offset=2 * KT_LEAD, stride_byte_offset=STRIDE, layout=SWZ, ) + bars = make_gdn_bars(cfg) + tmem_base_slot = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=16) + sSched = cutlass.Array(cutlass.Int32, cfg.sched_stages, space=SMEM, alignment=16) + cumsumlog_raw = cutlass.Array(cutlass.Float32, cute.cosize(cumsumlog_smem_layout_staged), space=SMEM, alignment=128) + cumprod_raw = cutlass.Array(cutlass.Float32, cute.cosize(cumsumlog_smem_layout_staged), space=SMEM, alignment=128) + beta_raw = cutlass.Array(cutlass.Float32, cute.cosize(beta_smem_layout_staged), space=SMEM, alignment=128) sAinv_raw = cutlass.Array( cfg.io_dtype, cfg.ainv_cosize, @@ -2679,7 +2441,6 @@ def _kernel( stride_byte_offset=STRIDE, layout=SWZ, ) - # descriptor operands total 136 KB: V/O/H land past the sub-bank midpoint sV_raw = cutlass.Array( cfg.io_dtype, cfg.v_cosize, @@ -2694,29 +2455,6 @@ def _kernel( stride_byte_offset=STRIDE, layout=SWZ, ) - sO_raw = cutlass.Array( - cfg.io_dtype, - cfg.o_cosize, - space=cutlass.AddressSpace.smem, - alignment=cfg.buffer_align_bytes, - ) - sO = SmemTile( - base=sO_raw.data_ptr().toint(), - elems_per_stage=(cfg.o_cosize // cfg.smem_o_stages) * bpe, - stages=cfg.smem_o_stages, - leading_byte_offset=LEAD, - stride_byte_offset=STRIDE, - layout=SWZ, - ) - if cutlass.const_expr(cfg.enable_h): - sH_raw = cutlass.Array( - cfg.io_dtype, - cfg.h_cosize, - space=cutlass.AddressSpace.smem, - alignment=cfg.buffer_align_bytes, - ) - else: - sH_raw = None sCumsumlog = cute.make_tensor( cute.make_ptr(cutlass.Float32, cumsumlog_raw.data_ptr().toint(), mem_space=cute.AddressSpace.smem, assumed_align=128), cumsumlog_smem_layout_staged, @@ -2730,15 +2468,10 @@ def _kernel( beta_smem_layout_staged, ) - # ------------------------------------------------------------------ - # mbarrier init (all threads) - # ------------------------------------------------------------------ - for s in range(cfg.smem_q_stages): - bars.mb_q_ready[s].init() - bars.mb_q_done[s].init() - for s in range(cfg.smem_k_stages): - bars.mb_k_ready[s].init() - bars.mb_k_done[s].init() + # ---- mbarrier init (all threads) --------------------------------------------- + for s in range(cfg.smem_kq_stages): + bars.mb_kq_ready[s].init() + bars.mb_kq_done[s].init() for s in range(cfg.smem_v_stages): bars.mb_v_ready[s].init() bars.mb_v_done[s].init() @@ -2748,9 +2481,9 @@ def _kernel( for s in range(cfg.smem_beta_stages): bars.mb_beta_ready[s].init() bars.mb_beta_done[s].init() - for s in range(cfg.tmem_kv_acc_stages): - bars.mb_kv_acc_ready[s].init() - bars.mb_kv_acc_scale_done[s].init() + for s in range(cfg.tmem_state_acc_stages): + bars.mb_state_acc_ready[s].init() + bars.mb_state_acc_scale_done[s].init() for s in range(cfg.tmem_q_state_acc_stages): bars.mb_o_acc_ready[s].init() bars.mb_o_acc_done[s].init() @@ -2759,7 +2492,7 @@ def _kernel( for s in range(cfg.tmem_cg0_acc_stages): bars.mb_cg0_acc_ready[s].init() bars.mb_cg0_acc_done[s].init() - bars.mb_ks_ready[0].init() + bars.mb_k_state_ready[0].init() bars.mb_nv_ready[0].init() for s in range(cfg.smem_ainv_stages): bars.mb_ainv_ready[s].init() @@ -2769,14 +2502,14 @@ def _kernel( bars.mb_qk_done[s].init() for s in range(cfg.tmem_state_inp_stages): bars.mb_state_inp_ready[s].init() - for b in (bars.mb_vks_inp_ready, bars.mb_nv_inp_ready, bars.mb_decay_v_inp_ready): + for b in (bars.mb_v_k_state_inp_ready, bars.mb_nv_inp_ready, bars.mb_decay_v_inp_ready): b[0].init() for s in range(cfg.smem_o_stages): bars.mb_o_tmastg_ready[s].init() bars.mb_o_tmastg_done[s].init() - for s in range(cfg.smem_h_stages): - bars.mb_h_tmastg_ready[s].init() - bars.mb_h_tmastg_done[s].init() + for s in range(cfg.smem_checkpoint_stages): + bars.mb_checkpoint_tmastg_ready[s].init() + bars.mb_checkpoint_tmastg_done[s].init() for s_ in range(cfg.sched_stages): bars.mb_sched_ready[s_].init() bars.mb_sched_done[s_].init() @@ -2785,12 +2518,10 @@ def _kernel( nvvm.fence_mbarrier_init() nvvm.barrier_cta_sync() - # ------------------------------------------------------------------ - # 2. Warp specialization - each warp role owns its own scheduler loop - # ------------------------------------------------------------------ + # ---- warp specialization ----------------------------------------------------- if warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]: - _compute0_warp( + compute0_warp_group( cfg, total_tiles, bidx, @@ -2798,20 +2529,20 @@ def _kernel( cu_seqlens, mWorkItems, tidx, - tmem_hold=tmem_hold, + tmem_base_slot=tmem_base_slot, scale=scale, sCumsumlog=sCumsumlog, sBeta=sBeta, sAinv=sAinv, sQk=sQk, - sH_raw=sH_raw, + sCheckpoint_raw=sCheckpoint_raw, checkpoint_every_n_tokens=checkpoint_every_n_tokens, sSched=sSched, bars=bars, ) if warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]: - _compute1_warp( + compute1_warp_group( cfg, total_tiles, bidx, @@ -2820,48 +2551,50 @@ def _kernel( mWorkItems, tidx, warp_idx=warp_idx, - tmem_hold=tmem_hold, + tmem_base_slot=tmem_base_slot, scale=scale, sV=sV, sCumsumlog=sCumsumlog, sCumprod=sCumprod, sBeta=sBeta, sO=sO, - sH_raw=sH_raw, - mS_init=mS_init, - mS_out=mS_out, + sCheckpoint_raw=sCheckpoint_raw, + mState_init=mState_init, + mState_out=mState_out, checkpoint_every_n_tokens=checkpoint_every_n_tokens, sSched=sSched, bars=bars, ) - elif warp_idx == cfg.mma_warp_id: - _mma0_warp( + elif warp_idx == cfg.load_gate_beta_warp_id: + gate_beta_warp( cfg, total_tiles, bidx, num_ctas, cu_seqlens, mWorkItems, - tmem_hold=tmem_hold, - sQ=sQ, - sK=sK, + tidx=tidx, + mGate=mGate, + mBeta=mBeta, + sCumsumlog=sCumsumlog, + sCumprod=sCumprod, + sBeta=sBeta, sSched=sSched, bars=bars, ) - elif warp_idx == cfg.mma_cg1_warp_id: - _mma1_warp( + elif warp_idx == cfg.mma_warp_id: + mma_warp( cfg, total_tiles, bidx, num_ctas, cu_seqlens, mWorkItems, - tmem_hold=tmem_hold, - sQ=sQ, - sK=sK, - sK_trans=sK_trans, + tmem_base_slot=tmem_base_slot, + sKQ=sKQ, + sKQ_trans=sKQ_trans, sAinv=sAinv, sQk=sQk, sSched=sSched, @@ -2869,15 +2602,14 @@ def _kernel( ) elif warp_idx == cfg.tma_qkv_warp_id: - _tmaldg_warp( + tmaldg_warp( cfg, total_tiles, bidx, num_ctas, cu_seqlens, mWorkItems, - sQ_raw=sQ_raw, - sK_raw=sK_raw, + sKQ_raw=sKQ_raw, sV_raw=sV_raw, desc_q_base=desc_q_base, desc_k_base=desc_k_base, @@ -2888,7 +2620,7 @@ def _kernel( ) if warp_idx == cfg.epilogue_warp_id: - _tmastg_warp( + tmastg_warp( cfg, total_tiles, bidx, @@ -2897,15 +2629,10 @@ def _kernel( mWorkItems, checkpoint_every_n_tokens=checkpoint_every_n_tokens, tidx=tidx, - mGate=mGate, - mBeta=mBeta, - sCumsumlog=sCumsumlog, - sCumprod=sCumprod, - sBeta=sBeta, sO_raw=sO_raw, - sH_raw=sH_raw, + sCheckpoint_raw=sCheckpoint_raw, desc_o_base=desc_o_base, - desc_h_base=desc_h_base, + desc_checkpoint_base=desc_checkpoint_base, sSched=sSched, bars=bars, ) @@ -2917,9 +2644,9 @@ class GdnCfg: The per-compile parameters (dtypes, GQA, state flags) are the ``cute.compile`` cache keys; the rest is derived from the module-global - ``CFG`` constants. ``_host`` stamps the shape-derived fields at trace - time. Passed ``cfg``-first (a ``cutlass.Constexpr``) into ``_host`` / - ``_kernel`` and every warp body. + ``CFG`` constants. ``host`` stamps the shape-derived fields at trace + time. Passed ``cfg``-first (a ``cutlass.Constexpr``) into ``host`` / + ``kernel`` and every warp body. """ io_dtype: Type[cutlass.Numeric] @@ -2929,23 +2656,21 @@ class GdnCfg: is_GQA: bool use_initial_state: bool store_final_state: bool - enable_h: bool - enable_o: bool + enable_checkpoints: bool split_k: bool = False log_gate: bool = False dyn_sched: bool = False sched_stages: int = CFG.SMEM_SCHED_STAGES - # --- fixed constants stamped from CFG by build_cfg --- + # ---- fixed constants stamped from CFG by build_cfg --------------------------- b_t: int = CFG.B_T d_k: int = CFG.D_K d_v: int = CFG.D_V compute_group_0_warp_ids: Tuple[int, ...] = CFG.COMPUTE_GROUP_0_WARP_IDS compute_group_1_warp_ids: Tuple[int, ...] = CFG.COMPUTE_GROUP_1_WARP_IDS - mma_warp_id: int = CFG.MMA_WARP_ID - tma_qkv_warp_id: int = CFG.TMA_QKV_WARP_ID - mma_cg1_warp_id: int = CFG.MMA_CG1_WARP_ID load_gate_beta_warp_id: int = CFG.LOAD_GATE_BETA_WARP_ID + tma_qkv_warp_id: int = CFG.TMA_QKV_WARP_ID + mma_warp_id: int = CFG.MMA_WARP_ID epilogue_warp_id: int = CFG.EPILOGUE_WARP_ID num_regs_compute_group_0: int = CFG.NUM_REGS_COMPUTE_GROUP_0 num_regs_compute_group_1: int = CFG.NUM_REGS_COMPUTE_GROUP_1 @@ -2954,7 +2679,7 @@ class GdnCfg: threads_per_cta: int = 0 cluster_shape_mnk: Tuple[int, int, int] = CFG.CLUSTER_SHAPE_MNK - # --- named barrier slots (ids 1-4; 0 is the CTA-wide sync) --- + # ---- named barrier slots (ids 1-4; 0 is the CTA-wide sync) ------------------- tmem_alloc_barrier_id: int = 1 tmem_alloc_barrier_threads: int = 0 inverse_barrier_id: int = 2 @@ -2962,42 +2687,42 @@ class GdnCfg: init_state_store_barrier_id: int = 4 init_state_store_barrier_threads: int = 0 - # --- SMEM / TMEM stage counts + TMEM column offsets --- - smem_q_stages: int = CFG.SMEM_Q_STAGES - smem_k_stages: int = CFG.SMEM_K_STAGES + # ---- SMEM / TMEM stage counts + TMEM column offsets -------------------------- + smem_kq_stages: int = CFG.SMEM_KQ_STAGES smem_v_stages: int = CFG.SMEM_V_STAGES smem_ainv_stages: int = CFG.SMEM_AINV_STAGES smem_qk_stages: int = CFG.SMEM_QK_STAGES smem_o_stages: int = CFG.SMEM_O_STAGES - smem_h_stages: int = 1 + smem_checkpoint_stages: int = 1 smem_gate_stages: int = CFG.SMEM_GATE_STAGES smem_beta_stages: int = CFG.SMEM_BETA_STAGES - tmem_kv_acc_stages: int = CFG.TMEM_KV_ACC_STAGES + tmem_state_acc_stages: int = CFG.TMEM_KV_ACC_STAGES tmem_q_state_acc_stages: int = CFG.TMEM_Q_STATE_ACC_STAGES tmem_state_inp_stages: int = CFG.TMEM_STATE_INP_STAGES tmem_cg0_acc_stages: int = CFG.TMEM_CG0_ACC_STAGES tmem_cg1_acc_stages: int = CFG.TMEM_CG1_ACC_STAGES - tmem_state_offset: int = 0 - tmem_q_state_offset: int = 0 + tmem_state_acc_offset: int = 0 + tmem_q_state_acc_offset: int = 0 tmem_state_inp_offset: int = 0 tmem_cg0_acc_offset: int = 0 tmem_cg1_acc_offset: int = 0 - tmem_inp_offset: int = 0 + tmem_v_k_state_decay_inp_offset: int = 0 buffer_align_bytes: int = CFG.BUFFER_ALIGN_BYTES - # --- stamped by _host at trace time (shape-derived) --- - q_cosize: int = 0 - k_cosize: int = 0 + # ---- stamped by host at trace time (shape-derived) -------------------------- + kq_cosize: int = 0 v_cosize: int = 0 ainv_cosize: int = 0 qk_cosize: int = 0 o_cosize: int = 0 - h_cosize: int = 0 - tma_q_bytes: int = 0 - tma_k_bytes: int = 0 + checkpoint_cosize: int = 0 + tma_kq_bytes: int = 0 tma_v_bytes: int = 0 tma_o_bytes: int = 0 n_heads_out: int = 0 + q_ratio: int = 1 + k_ratio: int = 1 + v_ratio: int = 1 def build_cfg( @@ -3008,15 +2733,13 @@ def build_cfg( is_GQA: bool, use_initial_state: bool, store_final_state: bool = True, - enable_h: bool = False, - enable_o: bool = True, + enable_checkpoints: bool = False, split_k: bool = False, log_gate: bool = False, dyn_sched: bool = False, ) -> GdnCfg: """Build the per-compile ``GdnCfg`` (io_dtype ∈ {Float16, BFloat16}; - acc is always Float32). Fills the derived thread / barrier counts and - the TMEM column offsets.""" + acc is always Float32).""" if io_dtype not in (cutlass.Float16, cutlass.BFloat16): raise ValueError(f"io_dtype={io_dtype} not supported; only Float16 and BFloat16 are supported") cfg = GdnCfg( @@ -3027,103 +2750,51 @@ def build_cfg( is_GQA=is_GQA, use_initial_state=use_initial_state, store_final_state=store_final_state, - enable_h=enable_h, - enable_o=enable_o, + enable_checkpoints=enable_checkpoints, split_k=split_k, log_gate=log_gate, dyn_sched=dyn_sched, ) - cfg.smem_h_stages = 1 - if enable_h: - # trim K/V lookahead so the 32 KB H buffer fits - cfg.smem_k_stages = 3 - cfg.smem_v_stages = 2 + cfg.smem_checkpoint_stages = 1 + if enable_checkpoints: + cfg.smem_kq_stages = 3 if not use_initial_state: - # fund the lightweight warps from CG1 (peeled chunk carries more cursors) cfg.num_regs_compute_group_1 = 232 cfg.num_regs_other = 48 n_cg0 = len(cfg.compute_group_0_warp_ids) n_cg1 = len(cfg.compute_group_1_warp_ids) cfg.threads_per_cta = cfg.threads_per_warp * (4 + n_cg0 + n_cg1) - # both MMA issuer warps join the TMEM alloc barrier - cfg.tmem_alloc_barrier_threads = cfg.threads_per_warp * (2 + n_cg0 + n_cg1) + cfg.tmem_alloc_barrier_threads = cfg.threads_per_warp * (1 + n_cg0 + n_cg1) cfg.inverse_barrier_threads = cfg.threads_per_warp * n_cg0 cfg.init_state_store_barrier_threads = cfg.threads_per_warp * n_cg1 - cfg.tmem_state_offset = 0 - cfg.tmem_q_state_offset = cfg.tmem_state_offset + cfg.tmem_kv_acc_stages * 128 - cfg.tmem_state_inp_offset = cfg.tmem_q_state_offset + cfg.tmem_q_state_acc_stages * 64 + cfg.tmem_state_acc_offset = 0 + cfg.tmem_q_state_acc_offset = cfg.tmem_state_acc_offset + cfg.tmem_state_acc_stages * 128 + cfg.tmem_state_inp_offset = cfg.tmem_q_state_acc_offset + cfg.tmem_q_state_acc_stages * 64 cfg.tmem_cg0_acc_offset = cfg.tmem_state_inp_offset + cfg.tmem_state_inp_stages * 64 cfg.tmem_cg1_acc_offset = cfg.tmem_cg0_acc_offset + cfg.tmem_cg0_acc_stages * 64 - cfg.tmem_inp_offset = cfg.tmem_cg1_acc_offset + cfg.tmem_cg1_acc_stages * 64 + cfg.tmem_v_k_state_decay_inp_offset = cfg.tmem_cg1_acc_offset + cfg.tmem_cg1_acc_stages * 64 return cfg -def get_workspace_size(B: int, HQ: int, HV: int): - HO = HQ if HQ >= HV else HV - return CFG.BYTES_PER_TENSORMAP * (5 * B * HO) + 128 +TENSORMAP_DESC_ARRAYS = 5 # per-batch runtime TMA descriptors: Q, K, V, O, checkpoints +TENSORMAP_STATIC_SLOTS = 0 # --------------------------------------------------------------------------- -def _check_cuda(err): - if err != cuda.CUresult.CUDA_SUCCESS: - raise RuntimeError(f"CUDA driver call failed: {err}") - - -def _data_ptr(t) -> int: - """Device address of a tensor-like (``data_ptr()`` or the CUDA array - interface).""" - fn = getattr(t, "data_ptr", None) - if fn is not None: - return fn() - return t.__cuda_array_interface__["data"][0] - - -def _device_sm_count() -> int: - """Multiprocessor count of the current device (runtime API: auto-inits - the primary context, so this works before any other CUDA call).""" - from cuda.bindings import runtime as _rt - - err, dev = _rt.cudaGetDevice() - if int(err) != 0: - raise RuntimeError(f"cudaGetDevice failed: {err}") - err, count = _rt.cudaDeviceGetAttribute(_rt.cudaDeviceAttr.cudaDevAttrMultiProcessorCount, dev) - if int(err) != 0: - raise RuntimeError(f"cudaDeviceGetAttribute failed: {err}") - return count - - -def _cutlass_io_dtype(dtype): - name = str(dtype) - if "bfloat16" in name: - return cutlass.BFloat16 - if "float16" in name or "half" in name: - return cutlass.Float16 - raise ValueError(f"Unsupported dtype {dtype}, expected bfloat16 or float16") - - -def _cutlass_state_dtype(dtype): - name = str(dtype) - if "bfloat16" in name: - return cutlass.BFloat16 - if "float32" in name: - return cutlass.Float32 - raise ValueError(f"Unsupported state dtype {dtype}, expected float32 or bfloat16") - - @functools.cache -def _get_compiled_cache( +def get_compiled_cache( io_dtype_str: str, state_dtype_str: str, + cu_dtype_str: str, HQ: int, HK: int, HV: int, is_GQA: bool, use_initial_state: bool, store_final_state: bool, - enable_h: bool, - enable_o: bool, + enable_checkpoints: bool, split_k: bool, log_gate: bool, dyn_sched: bool, @@ -3138,8 +2809,7 @@ def compile( is_GQA: bool, use_initial_state: bool, store_final_state: bool, - enable_h: bool, - enable_o: bool, + enable_checkpoints: bool, split_k: bool = False, log_gate: bool = False, dyn_sched: bool = False, @@ -3152,8 +2822,8 @@ def compile( beta_cute, o_cute, cu_seqlens_cute, - s_in_cute, - s_out_cute, + state_in_cute, + state_out_cute, work_items_cute=None, work_count_cute=None, sched_ctr_cute=None, @@ -3170,15 +2840,14 @@ def compile( is_GQA=is_GQA, use_initial_state=use_initial_state, store_final_state=store_final_state, - enable_h=enable_h, - enable_o=enable_o, + enable_checkpoints=enable_checkpoints, split_k=split_k, log_gate=log_gate, dyn_sched=dyn_sched, ) return cute.compile( - _host, + host, cfg, q_cute, k_cute, @@ -3187,8 +2856,8 @@ def compile( beta_cute, o_cute, cu_seqlens_cute, - s_in_cute, - s_out_cute, + state_in_cute, + state_out_cute, work_items_cute, work_count_cute, sched_ctr_cute, @@ -3212,7 +2881,7 @@ def chunk_gdn_sm100( output_state, scale: float, checkpoint_every_n_tokens: int = 0, - output_h=None, + output_state_checkpoints=None, work_items=None, work_count=None, sched_ctr=None, @@ -3234,44 +2903,31 @@ def chunk_gdn_sm100( gate: ``(total_tokens, HO)`` float32, forget gate — raw linear alpha, or the natural-log decay when ``log_gate`` beta: ``(total_tokens, HO)`` float32, update gate - output: ``(total_tokens, HO, DK)`` float16/bfloat16, pre-allocated, - or None to skip the O output path entirely (state/H-only run; - requires output_h or output_state) + output: ``(total_tokens, HO, DK)`` float16/bfloat16, pre-allocated cu_seqlens: ``(num_seqs + 1,)`` int32 initial_state: ``(num_seqs, HO, DK, DK)`` float32/bfloat16, or None output_state: ``(num_seqs, HO, DK, DK)`` float32/bfloat16, or None scale: attention scale factor (must not be 0) - checkpoint_every_n_tokens: emit an H entry every N tokens (0 = off) - output_h: ``(total_h, HO, DK, DK)`` bfloat16, or None. H[j] is the + checkpoint_every_n_tokens: emit a checkpoint entry every N tokens (0 = off) + output_state_checkpoints: ``(total_checkpoints, HO, DK, DK)`` bfloat16, or None. Entry j is the state after ``(j + 1) * N`` tokens, STRICTLY BEFORE the sequence - end -- the end-of-sequence state is only ``output_state`` (S, - fp32-capable). With ``N == B_T`` this is the per-chunk state + end -- the end-of-sequence state is only ``output_state`` + (fp32-capable). With ``N == B_T`` this is the per-chunk checkpoint series the backward pass consumes. - work_items: ``(max_items, 6)`` int32 split-K work-item table from + work_items: ``(max_items, 8)`` int32 split-K work-item table from ``common/split_k.py``, or None for the one-tile-per-(b,h) layout. With a table, each item computes chunks - ``[cstart, wend)`` and writes O/H only for ``[wstart, wend)``. + ``[cstart, wend)`` and writes O/checkpoints only for ``[wstart, wend)``. work_count: ``(1,)`` int32 device-side item count (required with work_items) log_gate: ``gate`` holds natural-log decay values; the gate warp skips its log2 (rescales by 1/ln2) instead of exponentiating upstream - workspace: ``(>= get_workspace_size(B, HQ, HV) // 8,)`` int64, + workspace: ``(>= tensormap_workspace_bytes(module, B) // 8,)`` int64, 128-byte aligned; holds the per-(b,h) TMA descriptors (contents managed here — reuse the same buffer across calls) stream: CUDA stream handle (``cudaStream_t`` as an int) """ - if checkpoint_every_n_tokens < 0: - raise ValueError("checkpoint_every_n_tokens must be non-negative") - if checkpoint_every_n_tokens > 0: - if checkpoint_every_n_tokens % CFG.B_T != 0: - raise ValueError(f"checkpoint_every_n_tokens must be a multiple of the chunk " f"size ({CFG.B_T}), got {checkpoint_every_n_tokens}") - if output_h is None: - raise ValueError("output_h must be provided when checkpoint_every_n_tokens > 0") - if str(output_h.dtype).split(".")[-1] != str(q.dtype).split(".")[-1]: - raise ValueError(f"output_h dtype must match the io dtype (fp32 state belongs to " f"output_state): got {output_h.dtype} with io {q.dtype}") - elif output_h is not None: - raise ValueError("output_h must be None when checkpoint_every_n_tokens == 0") HQ = q.shape[1] HV = v.shape[1] DK = q.shape[2] @@ -3279,20 +2935,10 @@ def chunk_gdn_sm100( is_GQA = HQ >= HV use_initial_state = initial_state is not None store_final_state = output_state is not None - enable_h = checkpoint_every_n_tokens > 0 - enable_o = output is not None + enable_checkpoints = checkpoint_every_n_tokens > 0 split_k = work_items is not None dyn_sched = sched_ctr is not None - if not enable_o and not (enable_h or store_final_state): - raise ValueError("output=None requires output_h or output_state") - if split_k: - if work_count is None: - raise ValueError("work_count is required with work_items") - if enable_h and checkpoint_every_n_tokens != CFG.B_T: - raise ValueError(f"split-K H checkpoints require checkpoint_every_n_tokens == {CFG.B_T}, got {checkpoint_every_n_tokens}") - elif work_count is not None: - raise ValueError("work_count must be None without work_items") - io_dtype = _cutlass_io_dtype(q.dtype) + io_dtype = get_dtype(q.dtype) if initial_state is not None: state_dtype_src = initial_state.dtype @@ -3300,26 +2946,21 @@ def chunk_gdn_sm100( state_dtype_src = output_state.dtype else: state_dtype_src = None - state_dtype = _cutlass_state_dtype(state_dtype_src) if state_dtype_src is not None else cutlass.Float32 + state_dtype = get_dtype(state_dtype_src) if state_dtype_src is not None else cutlass.Float32 - ws_words = get_workspace_size(B, HQ, HV) // 8 - if workspace.shape[0] < ws_words: - raise ValueError(f"workspace too small: need {ws_words} int64 words, " f"got {workspace.shape[0]}") - if _data_ptr(workspace) % 128 != 0: - raise ValueError("workspace must be 128-byte aligned") cu_stream = cuda.CUstream(int(stream)) - cache = _get_compiled_cache( + cache = get_compiled_cache( str(q.dtype), str(state_dtype_src), + str(cu_seqlens.dtype), HQ, k.shape[1], HV, is_GQA, use_initial_state, store_final_state, - enable_h, - enable_o, + enable_checkpoints, split_k, log_gate, dyn_sched, @@ -3336,28 +2977,26 @@ def chunk_gdn_sm100( gate_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) beta_cute = from_dlpack(beta, assumed_align=16) beta_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) - o_cute = None - if enable_o: - o_cute = from_dlpack(output, assumed_align=16) - o_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - cu_seqlens_cute = from_dlpack(cu_seqlens, assumed_align=4).mark_layout_dynamic() + o_cute = from_dlpack(output, assumed_align=16) + o_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) + cu_seqlens_cute = from_dlpack(cu_seqlens, assumed_align=8 if str(cu_seqlens.dtype).endswith("int64") else 4).mark_layout_dynamic() - s_in_cute = None + state_in_cute = None if use_initial_state: - s_in_cute = from_dlpack(initial_state, assumed_align=16) - s_in_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=DK) + state_in_cute = from_dlpack(initial_state, assumed_align=16) + state_in_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=DK) - s_out_cute = None + state_out_cute = None if store_final_state: - s_out_cute = from_dlpack(output_state, assumed_align=16) - s_out_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=DK) + state_out_cute = from_dlpack(output_state, assumed_align=16) + state_out_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=DK) workspace_cute = from_dlpack(workspace, assumed_align=128).mark_layout_dynamic() work_items_cute = None work_count_cute = None if split_k: - work_items_cute = from_dlpack(work_items, assumed_align=4) + work_items_cute = from_dlpack(work_items, assumed_align=16) work_items_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) work_count_cute = from_dlpack(work_count, assumed_align=4).mark_layout_dynamic() @@ -3371,12 +3010,11 @@ def chunk_gdn_sm100( is_GQA, use_initial_state, store_final_state, - enable_h, - enable_o, + enable_checkpoints, split_k, log_gate, dyn_sched, - num_sm=_device_sm_count(), + num_sm=multiprocessor_count(current_device_id()), q_cute=q_cute, k_cute=k_cute, v_cute=v_cute, @@ -3384,8 +3022,8 @@ def chunk_gdn_sm100( beta_cute=beta_cute, o_cute=o_cute, cu_seqlens_cute=cu_seqlens_cute, - s_in_cute=s_in_cute, - s_out_cute=s_out_cute, + state_in_cute=state_in_cute, + state_out_cute=state_out_cute, work_items_cute=work_items_cute, work_count_cute=work_count_cute, sched_ctr_cute=sched_ctr_cute, @@ -3397,11 +3035,8 @@ def chunk_gdn_sm100( compiled = cache["compiled"] - # The descriptors encode cu_seqlens' CONTENTS, which no key built from the - # buffers can track. The skip this replaces asked torch's _version counter, - # so it was sound for torch callers and silently stale for every other - # producer. Rebuilding unconditionally measured free: 131 vs 135 us of host - # time, 157 either way once the launches are waited on. + # desc build runs every execute by contract (cu contents are data; + # buffer pointers may change) — capture-safe, single tiny launch if "build_descs" not in cache: q_bc = from_dlpack(q, assumed_align=16) q_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) @@ -3409,18 +3044,17 @@ def chunk_gdn_sm100( k_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) v_bc = from_dlpack(v, assumed_align=16) v_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - o_bc = None - if enable_o: - o_bc = from_dlpack(output, assumed_align=16) - o_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - cu_bc = from_dlpack(cu_seqlens, assumed_align=4).mark_layout_dynamic() - s_bc = None - if enable_h: - s_bc = from_dlpack(output_h, assumed_align=16) - s_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2, 3), divisibility=1) + o_bc = from_dlpack(output, assumed_align=16) + o_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) + cu_bc = from_dlpack(cu_seqlens, assumed_align=8 if str(cu_seqlens.dtype).endswith("int64") else 4).mark_layout_dynamic() + checkpoints_bc = None + cu_ckpt_bc = None + if enable_checkpoints: + checkpoints_bc = from_dlpack(output_state_checkpoints, assumed_align=16) + checkpoints_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2, 3), divisibility=1) ws_bc = from_dlpack(workspace, assumed_align=128).mark_layout_dynamic() cache["build_descs"] = cute.compile( - _build_descs, + build_descs, io_dtype, CFG.B_T, q_bc, @@ -3428,8 +3062,8 @@ def chunk_gdn_sm100( v_bc, o_bc, cu_bc, - s_bc, - cutlass.Int32(checkpoint_every_n_tokens if enable_h else 1), + checkpoints_bc, + cutlass.Int32(checkpoint_every_n_tokens if enable_checkpoints else 1), ws_bc, cu_stream, options="--enable-tvm-ffi", @@ -3440,12 +3074,11 @@ def chunk_gdn_sm100( v, output, cu_seqlens, - output_h, - checkpoint_every_n_tokens if enable_h else 1, + output_state_checkpoints, + checkpoint_every_n_tokens if enable_checkpoints else 1, workspace, cu_stream, ) - compiled( q, k, diff --git a/python/cudnn/linear_attention/frost/kernel/gdn_recompute_config.py b/python/cudnn/linear_attention/frost/kernel/gdn_recompute_config.py new file mode 100644 index 000000000..1086ef7a5 --- /dev/null +++ b/python/cudnn/linear_attention/frost/kernel/gdn_recompute_config.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# This kernel is derived from cuDNN, NVIDIA Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gated DeltaNet (GDN) Cutlass-primitives recompute (state/H-only) kernel config (fixed +compile-time constants; the per-compile attributes live on ``GdnCfg`` in the kernel file). + +Target arch: Blackwell SM100 (GB200) / SM103 (GB300). +""" + +from dataclasses import dataclass +from typing import Tuple + + +@dataclass(frozen=True) +class Cfg: + # --- tile shape --- + B_T: int = 64 # chunk size / token tile (the mma N or K of every GEMM) + D_K: int = 128 # key head dim (contraction of GEMMs 1/3, M of GEMM 7) + D_V: int = 128 # value head dim (M of GEMMs 3/5, N of GEMM 7) + + # --- TMA descriptor pool --- + + # --- warp assignments (12 warps total) --- + COMPUTE_GROUP_0_WARP_IDS: Tuple[int, ...] = (0, 1, 2, 3) # T-pairwise / kk_epi / inverse + COMPUTE_GROUP_1_WARP_IDS: Tuple[int, ...] = (4, 5, 6, 7) # kv_decay_v / v-k*state / epi ops + LOAD_GATE_BETA_WARP_ID: int = 8 # gate/beta chunk loads + TMEM lifecycle + TMA_KV_WARP_ID: int = 9 + MMA_WARP_ID: int = 10 # sole tcgen05 issuer: KK pairs + KS/NV/KV per chunk + EPILOGUE_WARP_ID: int = 11 + + # --- register split --- + NUM_REGS_COMPUTE_GROUP_0: int = 224 + NUM_REGS_COMPUTE_GROUP_1: int = 256 + NUM_REGS_OTHER: int = 24 + + THREADS_PER_WARP: int = 32 + + CLUSTER_SHAPE_MNK: Tuple[int, int, int] = (1, 1, 1) + + # --- SMEM stage counts --- + SMEM_SCHED_STAGES: int = 2 + SMEM_KQ_STAGES: int = 4 + SMEM_V_STAGES: int = 2 + SMEM_AINV_STAGES: int = 3 + SMEM_GATE_STAGES: int = 3 + SMEM_BETA_STAGES: int = 3 + + # --- TMEM stage counts --- + TMEM_KV_ACC_STAGES: int = 1 + TMEM_STATE_INP_STAGES: int = 1 + TMEM_CG0_ACC_STAGES: int = 2 + TMEM_CG1_ACC_STAGES: int = 1 + + BUFFER_ALIGN_BYTES: int = 1024 + + +CFG = Cfg() diff --git a/python/cudnn/linear_attention/frost/kernel/gdn_recompute_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn_recompute_f16.py new file mode 100644 index 000000000..52c178bf7 --- /dev/null +++ b/python/cudnn/linear_attention/frost/kernel/gdn_recompute_f16.py @@ -0,0 +1,2660 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# This kernel is derived from cuDNN, NVIDIA Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Chunked Gated Delta Net (GDN) recompute (state/checkpoint-only) kernel for Blackwell SM100 +(Cutlass primitives): the prefill state/checkpoint pipeline with the Q/O path removed. + +Algorithm overview (per chunk c, tokens [cC, (c+1)C)): + Inputs : K[BT,DK], V[BT,DV], Gate[BT] (scalar gate), Beta[BT] (scalar LR) + State : S_prev[DK,DV] (recurrent state, held in TMEM) + + Preprocessing (compute warp group 0): + cumsumlog[t] = sum_{l=0}^{t} log(Gate_l) cumulative log of gates + cumprod[t] = exp(cumsumlog[t]) cumulative product of gates + T_pairwise[i,j] = cumprod[i] / cumprod[j] (i>=j) inter-token transfer weights + (stored in registers; 128 regs/thread) + + GEMM 1 - KK : W_kk[BT,BT] = K @ K^T (lower-triangular intra scores) + GEMM 3 - K*state : KS[BT,DV] = K @ S_prev (key applied to state) + GEMM 5 - new V : NV[BT,DV] = A_inv @ V (corrected value vectors) + where A_inv = (I + M_kk)^{-1}, M_kk[i,j] = T[i,j]*Beta[i]*W_kk[i,j] (lower-tri, hierarchical blockwise inverse) + GEMM 7 - KV update : S_upd[DK,DV] = K^T @ delta (state update, BT contraction) + where delta[BT,DV] = V - KS (delta rule residuals, after decay) + + Epilogue: + S_next = cumprod[BT-1] * S_prev + S_upd (update state in TMEM) + +Chunks run in PAIRS (CG0 warp halves invert chunk 0 / chunk 1 in parallel); +odd counts pad with a neutral zero-filled chunk. + +SMEM layout (stage counts live in gdn_recompute_config.py; +enable_checkpoints compiles trim K stages to fit the checkpoint buffer): + Buffer Size (B) Stages + K (two-box stage) 32768 4 + V 16384 3 + A_inverse / new_V 8192 2 + checkpoint staging DK*DV*2 1 <-- enable_checkpoints only + cumsumlog / cumprod / Beta 256 3 + sched ticket ring 4 2 <-- dyn_sched publish ring + +TMEM layout (512 columns): + Buffer Cols + state 128 <-- DKxDV fp32 = 128x128x4B + state inp 64 <-- fp16 state staging (GEMM 3 A operand) + cg0 shared acc 128 <-- 2-stage ring: KK0/KK1 + cg1 shared acc 64 <-- 1-stage ring: KS then NV + v_k_state+NV / decay_V inp 64 <-- slot 0 = v_k_state then NV, slot 1 = decay_V + +Warp assignments (12 warps = 384 threads): + warps 0-3 : compute group 0 - T-pairwise x2, KK_epi x2, pair inverse + warps 4-7 : compute group 1 - state restage/rescale, V-K*state, + new_V_epi + warp 8 : Gate/Beta loads + warp 9 : TMA load warp - loads K, V + warp 10 : MMA warp - fused KK pairs + k_state/NV/KV per chunk; + TMEM lifecycle + warp 11 : epilogue warp - checkpoint TMA stores +""" + +import functools +from dataclasses import dataclass +from typing import NamedTuple, Optional, Type, Tuple + +import cuda.bindings.driver as cuda + +import cutlass +import cutlass.cute as cute +import cutlass.experimental.primitives as nvvm +import cutlass.experimental.cuda.tensor_map as tma +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import min + +from ..common.thd import emit_checkpoint_seq_descs, emit_seq_descs, TENSOR_MAP_QWORDS +from ..common.split_k import decode_work_item +from ..common.host import get_dtype +from cudnn.frost.buffers import current_device_id, data_ptr +from cudnn.frost.device import multiprocessor_count + +RCP_LN2 = 1.4426950408889634 # 1/ln(2): natural-log gates -> the kernel's log2 domain +from cudnn.frost.tile_dsl.barrier import ( + MBarrier, + Producer, + PipelineState, + advance, +) +from cudnn.frost.tile_dsl.handles import MmaDesc, SmemTile, tma_slice_runtime_desc +from cudnn.frost.tile_dsl.mma import mma_ss, mma_step_k8, mma_ts_step, mma_step +from cudnn.frost.tile_dsl.pointwise import fadd2, fp32_to_fp16, f16x2_to_f32, fmul2, opaque_f32_zero, sub_f16x2 +from cudnn.frost.tile_dsl.swizzle import swizzle_lin_128b, swizzle_xor_128b +from cudnn.frost.tile_dsl.tma import ( + tma_load_tile, + tma_store_tile, + tma_store_commit, + tma_store_wait, + tma_tensormap_acquire, +) +from .gdn_recompute_config import CFG + + +class GdnBars(NamedTuple): + """GDN pipeline mbarrier inventory. + + Every pipeline is a ``_ready``/``_done`` MBarrier pair over one ring: a + slot is acquired for filling by waiting ``_done`` and committed by + arriving ``_ready``; the reading side waits ``_ready`` and releases the + slot by arriving ``_done``. + """ + + mb_kq_ready: MBarrier + mb_kq_done: MBarrier + mb_v_ready: MBarrier + mb_v_done: MBarrier + + mb_gate_ready: MBarrier + mb_gate_done: MBarrier + mb_beta_ready: MBarrier + mb_beta_done: MBarrier + + mb_state_acc_ready: MBarrier + mb_state_acc_scale_done: MBarrier + mb_cg0_acc_ready: MBarrier + mb_cg0_acc_done: MBarrier + mb_k_state_ready: MBarrier + mb_nv_ready: MBarrier + + mb_ainv_ready: MBarrier + mb_ainv_done: MBarrier + mb_state_inp_ready: MBarrier + mb_v_k_state_inp_ready: MBarrier + mb_decay_v_inp_ready: MBarrier + + mb_checkpoint_tmastg_ready: MBarrier + mb_checkpoint_tmastg_done: MBarrier + + mb_tmem_done: MBarrier + mb_sched_ready: MBarrier + mb_sched_done: MBarrier + + +def make_gdn_bars(cfg) -> GdnBars: + """GdnBars factory. MUST be called from inside ``kernel`` (allocates SMEM).""" + ONE_LANE = 1 + MMA_ARRIVERS = len([cfg.mma_warp_id]) + KQ_RELEASE_SITES = 1 + GATE_WARP = cfg.threads_per_warp * len([cfg.load_gate_beta_warp_id]) + EPI_WARP = cfg.threads_per_warp * len([cfg.epilogue_warp_id]) + CG0_THREADS = cfg.threads_per_warp * len(cfg.compute_group_0_warp_ids) + CG1_THREADS = cfg.threads_per_warp * len(cfg.compute_group_1_warp_ids) + CG0_PLUS_CG1 = CG0_THREADS + CG1_THREADS + + def alloc(n): + return cutlass.Array(cutlass.Int64, n, space=cutlass.AddressSpace.smem, alignment=16) + + return GdnBars( + mb_kq_ready=MBarrier(alloc(cfg.smem_kq_stages), stages=cfg.smem_kq_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), + mb_kq_done=MBarrier(alloc(cfg.smem_kq_stages), stages=cfg.smem_kq_stages, init_count=KQ_RELEASE_SITES, producer=Producer.MMA_COMMIT), + mb_v_ready=MBarrier(alloc(cfg.smem_v_stages), stages=cfg.smem_v_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), + mb_v_done=MBarrier(alloc(cfg.smem_v_stages), stages=cfg.smem_v_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_gate_ready=MBarrier(alloc(cfg.smem_gate_stages), stages=cfg.smem_gate_stages, init_count=GATE_WARP, producer=Producer.THREAD), + mb_gate_done=MBarrier(alloc(cfg.smem_gate_stages), stages=cfg.smem_gate_stages, init_count=CG0_PLUS_CG1, producer=Producer.THREAD), + mb_beta_ready=MBarrier(alloc(cfg.smem_beta_stages), stages=cfg.smem_beta_stages, init_count=GATE_WARP, producer=Producer.THREAD), + mb_beta_done=MBarrier(alloc(cfg.smem_beta_stages), stages=cfg.smem_beta_stages, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_state_acc_ready=MBarrier(alloc(cfg.tmem_state_acc_stages), stages=cfg.tmem_state_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_state_acc_scale_done=MBarrier(alloc(cfg.tmem_state_acc_stages), stages=cfg.tmem_state_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_cg0_acc_ready=MBarrier(alloc(cfg.tmem_cg0_acc_stages), stages=cfg.tmem_cg0_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_cg0_acc_done=MBarrier(alloc(cfg.tmem_cg0_acc_stages), stages=cfg.tmem_cg0_acc_stages, init_count=CG0_THREADS // 2, producer=Producer.THREAD), + mb_k_state_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_nv_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_ainv_ready=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_ainv_done=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_state_inp_ready=MBarrier(alloc(cfg.tmem_state_inp_stages), stages=cfg.tmem_state_inp_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_v_k_state_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_decay_v_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_checkpoint_tmastg_ready=MBarrier( + alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=len(cfg.compute_group_1_warp_ids), producer=Producer.THREAD + ), + mb_checkpoint_tmastg_done=MBarrier(alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=EPI_WARP, producer=Producer.THREAD), + mb_tmem_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_sched_ready=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=1, producer=Producer.THREAD), + mb_sched_done=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=11, producer=Producer.THREAD), + ) + + +@cute.jit +def invert_diagonal_NxN(cfg, base_int, d, tidx, N: int = 8): + """Gauss-Jordan inversion of one diagonal NxN block in-place (f16 SMEM).""" + tidx_in_group = tidx % N + BT = cfg.b_t + + row_lin_base = (d * N + tidx_in_group) * BT + d * N + row_phys = swizzle_lin_128b(row_lin_base, row_stride_log2=6) + row_ptr = ( + cute.make_ptr( + cfg.io_dtype, + base_int, + mem_space=cute.AddressSpace.smem, + assumed_align=cfg.buffer_align_bytes, + ) + + row_phys + ) + + row = [(row_ptr + j).load().to(cutlass.Float32) for j in range(N)] + for i in cutlass.range_constexpr(N): + row[i] = cutlass.Float32(1.0) if tidx_in_group == i else row[i] + for src_row in cutlass.range_constexpr(N - 1): + row_scale = -row[src_row] + for i in cutlass.range_constexpr(src_row): + shfl_val = nvvm.shfl_sync(0xFFFFFFFF, row[i], src_row, 0b1100000011111, kind=nvvm.Shfl.IDX) + row[i] = row[i] + row_scale * shfl_val if tidx_in_group > src_row else row[i] + row[src_row] = row_scale if tidx_in_group > src_row else row[src_row] + + for j in cutlass.range_constexpr(N): + (row_ptr + j).store(row[j].to(cfg.io_dtype)) + + +@cute.jit +def blockwise_diagonal_8x8_to_16x16(cfg, base_int, d0, lane_id): + """Off-diagonal correction 8x8 -> 16x16 (C <- -D^{-1} C A^{-1}).""" + bpe = cfg.io_dtype.width // 8 + lds1 = (lane_id % 8) * 64 + d = nvvm.ldmatrix( + cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + 8 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + 1, + nvvm.MMALayout.ROW, + ) + c = nvvm.ldmatrix( + cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + 1, + nvvm.MMALayout.COL, + ) + + # ---- T = -(D^{-1} @ C) ------------------------------------------------------- + c_regs = cutlass.Array(cutlass.Float32, 4, alignment=16, space=cutlass.AddressSpace.rmem) + for i in cutlass.range_constexpr(4): + c_regs[i] = cutlass.Float32(0.0) + mma_step_k8(c_regs, [d, d], [c], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) + for i in cutlass.range_constexpr(4): + c_regs[i] = -c_regs[i] + a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(2)] + + # ---- C = T @ A^{-1} ---------------------------------------------------------- + ai = nvvm.ldmatrix( + cutlass.inttoptr(base_int + swizzle_lin_128b(d0 * 64 + d0 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + 1, + nvvm.MMALayout.COL, + ) + o_regs = cutlass.Array(cutlass.Float32, 4, alignment=16, space=cutlass.AddressSpace.rmem) + for i in cutlass.range_constexpr(4): + o_regs[i] = cutlass.Float32(0.0) + mma_step_k8(o_regs, a_f16, [ai], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) + o_f16 = fp32_to_fp16(o_regs[0], o_regs[1], dtype=cfg.io_dtype) + + # ---- store corrected C ------------------------------------------------------- + nvvm.stmatrix( + cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + o_f16, + nvvm.MMALayout.ROW, + ) + + +@cute.jit +def blockwise_diagonal_16x16_to_32x32(cfg, base_int, d0, lane_id): + """Off-diagonal correction 16x16 -> 32x32.""" + bpe = cfg.io_dtype.width // 8 + lds4 = (lane_id % 16) * 64 + (lane_id // 16) * 8 + d = list( + nvvm.ldmatrix( + cutlass.inttoptr( + base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + 16 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 + ), + 4, + nvvm.MMALayout.ROW, + ) + ) + c = list( + nvvm.ldmatrix( + cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + 4, + nvvm.MMALayout.COL, + ) + ) + + # ---- T = -(D^{-1} @ C) ------------------------------------------------------- + c_regs = cutlass.Array(cutlass.Float32, 8, alignment=16, space=cutlass.AddressSpace.rmem) + for i in cutlass.range_constexpr(8): + c_regs[i] = cutlass.Float32(0.0) + mma_step(c_regs, d, c, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) + for i in cutlass.range_constexpr(8): + c_regs[i] = -c_regs[i] + a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + + # ---- C = T @ A^{-1} ---------------------------------------------------------- + ai = list( + nvvm.ldmatrix( + cutlass.inttoptr(base_int + swizzle_lin_128b(d0 * 64 + d0 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + 4, + nvvm.MMALayout.COL, + ) + ) + o_regs = cutlass.Array(cutlass.Float32, 8, alignment=16, space=cutlass.AddressSpace.rmem) + for i in cutlass.range_constexpr(8): + o_regs[i] = cutlass.Float32(0.0) + mma_step(o_regs, a_f16, ai, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) + o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + + # ---- store corrected C ------------------------------------------------------- + nvvm.stmatrix( + cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + o_f16, + nvvm.MMALayout.ROW, + ) + + +@cute.jit +def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): + """Off-diagonal correction 32x32 -> 64x64 (2 warps, one 16-row M-band each).""" + band = warp_id % 2 + bpe = cfg.io_dtype.width // 8 + lds4 = (lane_id % 16) * 64 + (lane_id // 16) * 8 + a_regs = [] + for vs in cutlass.range_constexpr(2): + a_regs += list( + nvvm.ldmatrix( + cutlass.inttoptr( + base_int + swizzle_lin_128b((32 + band * 16) * 64 + 32 + vs * 16 + lds4, row_stride_log2=6) * bpe, + cutlass.AddressSpace.smem, + cutlass.BFloat16, + ), + 4, + nvvm.MMALayout.ROW, + ) + ) + b_regs = [] + for vs in cutlass.range_constexpr(4): + b_regs += list( + nvvm.ldmatrix( + cutlass.inttoptr( + base_int + swizzle_lin_128b((32 + (vs // 2) * 16) * 64 + (vs % 2) * 16 + lds4, row_stride_log2=6) * bpe, + cutlass.AddressSpace.smem, + cutlass.BFloat16, + ), + 4, + nvvm.MMALayout.COL, + ) + ) + + # ---- T = -(D^{-1} @ C) ------------------------------------------------------- + c_regs = cutlass.Array(cutlass.Float32, 16, alignment=16, space=cutlass.AddressSpace.rmem) + for i in cutlass.range_constexpr(16): + c_regs[i] = cutlass.Float32(0.0) + for ks in cutlass.range_constexpr(2): + mma_step(c_regs, a_regs, b_regs[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) + for i in cutlass.range_constexpr(16): + c_regs[i] = -c_regs[i] + a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + + # ---- C = T @ A^{-1} ---------------------------------------------------------- + ai_regs = [] + for vs in cutlass.range_constexpr(4): + ai_regs += list( + nvvm.ldmatrix( + cutlass.inttoptr( + base_int + swizzle_lin_128b(((vs // 2) * 16) * 64 + (vs % 2) * 16 + lds4, row_stride_log2=6) * bpe, + cutlass.AddressSpace.smem, + cutlass.BFloat16, + ), + 4, + nvvm.MMALayout.COL, + ) + ) + o_regs = cutlass.Array(cutlass.Float32, 16, alignment=16, space=cutlass.AddressSpace.rmem) + for i in cutlass.range_constexpr(16): + o_regs[i] = cutlass.Float32(0.0) + for ks in cutlass.range_constexpr(2): + mma_step(o_regs, a_f16, ai_regs[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) + o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + + # ---- store corrected C ------------------------------------------------------- + nvvm.barrier_cta_sync_aligned( + cfg.inverse_barrier_id, + thread_count=cfg.inverse_barrier_threads, + ) + nvvm.stmatrix( + cutlass.inttoptr(base_int + swizzle_lin_128b((32 + band * 16) * 64 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + o_f16[0:4], + nvvm.MMALayout.ROW, + ) + nvvm.stmatrix( + cutlass.inttoptr(base_int + swizzle_lin_128b((32 + band * 16) * 64 + 16 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), + o_f16[4:8], + nvvm.MMALayout.ROW, + ) + + +# ---- Dynamic tile scheduler ------------------------------------------------------ + + +@cute.jit +def sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): + """TMA-LDG-warp side: pull the next tile off the global ticket, publish it.""" + if cutlass.const_expr(cfg.dyn_sched): + bars.mb_sched_done[sched_state.idx].wait(sched_state.phase) + if nvvm.elect_sync(): + fetched = cutlass.Int32(nvvm.atomicrmw("add", mSched.iterator, cutlass.Int32(1), mem_order="relaxed", syncscope="gpu")) + sSched[sched_state.idx] = num_ctas + fetched + nvvm.bar_warp_sync(cute.arch.FULL_MASK) + next_tile = sSched[sched_state.idx] + if nvvm.elect_sync(): + bars.mb_sched_ready[sched_state.idx].arrive() + return next_tile, advance(sched_state, cfg.sched_stages) + return tile_idx + num_ctas, sched_state + + +@cute.jit +def sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): + """Consumer side: read the TMA-LDG warp's published next tile.""" + if cutlass.const_expr(cfg.dyn_sched): + bars.mb_sched_ready[sched_state.idx].wait(sched_state.phase) + next_tile = sSched[sched_state.idx] + if nvvm.elect_sync(): + bars.mb_sched_done[sched_state.idx].arrive() + return next_tile, advance(sched_state, cfg.sched_stages) + return tile_idx + num_ctas, sched_state + + +@cute.jit +def tmastg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + checkpoint_every_n_tokens, + tidx, + sCheckpoint_raw, + desc_checkpoint_base, + sSched, + bars, +): + """Epilogue warp role (warp 11): persistent scheduler loop issuing the + per-chunk checkpoint TMA stores.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + lidx = tidx % cfg.threads_per_warp + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + + if cutlass.const_expr(cfg.enable_checkpoints): + checkpoint_granu = 64 + sCheckpoint_tma = SmemTile( + base=sCheckpoint_raw, + elems_per_stage=(cfg.checkpoint_cosize // cfg.smem_checkpoint_stages), + stages=cfg.smem_checkpoint_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=cfg.d_v // checkpoint_granu, + tma_granu_elems=checkpoint_granu, + tma_subtile_stride_elems=cfg.d_k * checkpoint_granu, + ) + checkpoint_store_cnt = cutlass.Int32(0) + ckpt_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) + heads_out = cutlass.Int32(cfg.n_heads_out) + desc_qwords = cutlass.Int32(TENSOR_MAP_QWORDS) + + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + n_local = wend - cstart + n_padded = ((n_local + 1) // 2) * 2 + + head_o = head_idx + slot = batch_idx * desc_qwords + if cutlass.const_expr(cfg.enable_checkpoints): + desc_checkpoint_slot = (desc_checkpoint_base + slot).tospace(cutlass.AddressSpace.generic) + checkpoint_coord = wstart - 1 if wstart > 0 else cutlass.Int32(0) + checkpoint_mod = (cstart + cutlass.Int32(1)) % ckpt_chunks + if elect_one: + tma_tensormap_acquire(desc_checkpoint_slot) + + if n_local > 0: + for local_idx in cutlass.range(n_padded): + chunk_idx = cstart + local_idx + + did_checkpoint = cutlass.Int32(0) + if cutlass.const_expr(cfg.enable_checkpoints): + checkpoint_stage = checkpoint_store_cnt % cfg.smem_checkpoint_stages + checkpoint_phase = (checkpoint_store_cnt // cfg.smem_checkpoint_stages) & cutlass.Int32(1) + if chunk_idx >= wstart - 1 and chunk_idx < wend - 1: + if checkpoint_mod == 0: + bars.mb_checkpoint_tmastg_ready[checkpoint_stage].wait(checkpoint_phase) + checkpoint_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), checkpoint_coord, head_o) + tma_store_tile(sCheckpoint_tma[checkpoint_stage], checkpoint_slice, acquire=False) + tma_store_commit() + checkpoint_coord += 1 + did_checkpoint = cutlass.Int32(1) + checkpoint_mod = checkpoint_mod + cutlass.Int32(1) + checkpoint_mod = cutlass.Int32(0) if checkpoint_mod == ckpt_chunks else checkpoint_mod + + if did_checkpoint == 1: + tma_store_wait(0) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].arrive() + checkpoint_store_cnt = checkpoint_store_cnt + 1 + + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def gate_beta_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + tidx, + mGate, + mBeta, + sCumsumlog, + sCumprod, + sBeta, + sSched, + bars, +): + """Gate/Beta producer (warp 8): persistent scheduler loop + the + cumsum/cumprod/Beta chunk loads.""" + + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + gate_index = PipelineState.start(phase=1) + beta_index = PipelineState.start(phase=1) + lidx = tidx % cfg.threads_per_warp + + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + n_local = wend - cstart + n_padded = ((n_local + 1) // 2) * 2 + if n_local > 0: + for local_idx in cutlass.range(n_padded): + # ---- Gate load: GMEM -> SMEM (OOB neutral) ----------------------- + chunk_idx = cstart + local_idx + n_cols = cfg.b_t // cfg.threads_per_warp + chunk_offset = batch_start + chunk_idx * cfg.b_t + gGateSeq = mGate[None, head_idx] + gBeta = cute.domain_offset((chunk_offset,), mBeta[None, head_idx]) + + gate_idx = gate_index.idx + gate_phase = gate_index.phase + gate_index = advance(gate_index, cfg.smem_gate_stages) + + pos_valid = [None] * n_cols + gate_vals = [cutlass.Float32(0.0)] * n_cols + oob_neutral = cutlass.Float32(0.0) if cutlass.const_expr(cfg.log_gate) else cutlass.Float32(1.0) + for col in cutlass.range_constexpr(n_cols): + tok = chunk_offset + lidx + col * cfg.threads_per_warp + pos_valid[col] = cute.elem_less(tok, batch_end) + tok_clamped = min(tok, batch_end - 1) + gate_vals[col] = gGateSeq[tok_clamped] if pos_valid[col] else oob_neutral + + if cutlass.const_expr(cfg.log_gate): + for col in cutlass.range_constexpr(n_cols): + gate_vals[col] = gate_vals[col] * cutlass.Float32(RCP_LN2) + else: + for col in cutlass.range_constexpr(n_cols): + gate_vals[col] = cute.math.log2(gate_vals[col] + 1e-10, fastmath=True) + for offset in [1, 2, 4, 8, 16]: + for col in cutlass.range_constexpr(n_cols): + n = nvvm.shfl_sync(0xFFFFFFFF, gate_vals[col], offset, 0, kind=nvvm.Shfl.UP) + if lidx >= offset: + gate_vals[col] = gate_vals[col] + n + for col in cutlass.range_constexpr(1, n_cols): + last_v = nvvm.shfl_sync( + 0xFFFFFFFF, + gate_vals[col - 1], + cfg.threads_per_warp - 1, + cfg.threads_per_warp - 1, + kind=nvvm.Shfl.IDX, + ) + gate_vals[col] += last_v + + bars.mb_gate_done[gate_idx].wait(gate_phase) + for col in cutlass.range_constexpr(n_cols): + pos = lidx + col * cfg.threads_per_warp + sCumsumlog[pos, 0, gate_idx] = gate_vals[col] + sCumprod[pos, 0, gate_idx] = cute.math.exp2(gate_vals[col], fastmath=True) + + bars.mb_gate_ready[gate_idx].arrive() + + # ---- Beta load: GMEM -> SMEM (per-element cp.async) -------------------------- + beta_idx = beta_index.idx + bars.mb_beta_done[beta_idx].wait(beta_index.phase) + beta_index = advance(beta_index, cfg.smem_beta_stages) + for col in cutlass.range_constexpr(n_cols): + pos = lidx + col * cfg.threads_per_warp + src = gBeta.iterator + gBeta.layout((pos,)) + dst = sBeta.iterator + sBeta.layout((pos, 0, beta_idx)) + cp_size = cutlass.Int32(4) * cutlass.Int32(pos_valid[col]) + nvvm.cp_async_shared_global(dst, src, 4, nvvm.LoadCacheModifier.CA, cp_size=cp_size) + nvvm.cp_async_mbarrier_arrive(bars.mb_beta_ready[beta_idx].smem_ptr, noinc=True) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + for _ in range(cfg.smem_gate_stages): + bars.mb_gate_done[gate_index.idx].wait(gate_index.phase) + gate_index = advance(gate_index, cfg.smem_gate_stages) + for _ in range(cfg.smem_beta_stages): + bars.mb_beta_done[beta_index.idx].wait(beta_index.phase) + beta_index = advance(beta_index, cfg.smem_beta_stages) + + +@cute.jit +def mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + tmem_hold, + sKQ, + sKQ_trans, + sAinv, + sSched, + bars, +): + """MMA issuer role (warp 10): persistent scheduler loop issuing every + tcgen05 GEMM.""" + elect_one = nvvm.elect_sync() + + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + kv_acc_index = PipelineState.start(phase=1) + kq_index = PipelineState.start(phase=0) + cg0_acc_index = PipelineState.start(phase=1) + kq_fused_index = PipelineState.start(phase=0) + ainv_index = PipelineState.start(phase=0) + state_inp_index = PipelineState.start(phase=0) + v_k_state_inp_ready = PipelineState.start(phase=0) + decay_v_inp_ready = PipelineState.start(phase=0) + + nvvm.tcgen05_alloc(tmem_hold, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync_aligned( + cfg.tmem_alloc_barrier_id, + thread_count=cfg.tmem_alloc_barrier_threads, + ) + + # ---- chunk-invariant GEMM descriptors ---------------------------------------- + idesc_kk = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=2 * cfg.b_t, + ) + bmm_kk_desc = MmaDesc( + M=2 * cfg.b_t, + N=cfg.b_t, + K=cfg.d_k, + bpe_a=cfg.io_dtype.width // 8, + bpe_b=cfg.io_dtype.width // 8, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_kk, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bpe = cfg.io_dtype.width // 8 + idesc_k_state = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + ) + bmm_k_state_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.d_k, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + atranspose=False, + cta_group=1, + idesc=idesc_k_state, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_nv_ts = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + ) + bmm_nv_ts_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + atranspose=False, + cta_group=1, + idesc=idesc_nv_ts, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_kv = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.d_v, + m_dim=cfg.d_k, + b_major=1, + ) + bmm_kv_desc = MmaDesc( + M=cfg.d_k, + N=cfg.d_v, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + atranspose=False, + cta_group=1, + idesc=idesc_kv, + kind=nvvm.Tcgen05MMAKind.F16, + ) + KQ_SEG = (2 * cfg.b_t * 64 * bpe) >> 4 + KQ_BOX = (cfg.b_t * 64 * bpe) >> 4 + KQ_HALF_K = (cfg.d_k // 16) // 2 + KQ_A_HALF = KQ_HALF_K * bmm_k_state_desc.tmem_advance_A + + KV_ACC_STAGE_COLS = cfg.d_v + STATE_INP_STAGE_COLS = cfg.d_k // 2 + INP_SLOT_COLS = cfg.b_t // 2 + + tmem_base = tmem_hold.load() + tmem_cg0_acc_col_f = tmem_base + cfg.tmem_cg0_acc_offset + tmem_state_col = tmem_base + cfg.tmem_state_acc_offset + tmem_state_inp_col = tmem_base + cfg.tmem_state_inp_offset + tmem_inp_col = tmem_base + cfg.tmem_v_k_state_decay_inp_offset + v_k_state_inp_ptr = nvvm.make_tmem_ptr(tmem_inp_col, cutlass.Int8) + delta_inp_ptr = nvvm.make_tmem_ptr(tmem_inp_col + INP_SLOT_COLS, cutlass.Int8) + k_state_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_cg1_acc_offset, cutlass.Float32) + nv_acc_ptr = k_state_acc_ptr + acc_cg0_0 = nvvm.make_tmem_ptr(tmem_cg0_acc_col_f, cutlass.Float32) + acc_cg0_1 = nvvm.make_tmem_ptr(tmem_cg0_acc_col_f + cfg.b_t, cutlass.Float32) + + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + n_local = wend - cstart + n_padded = ((n_local + 1) // 2) * 2 + n_pairs = n_padded // 2 + + # ---- KK pair 0 = K(S) @ K^T (both members issued ahead of the loop) ------ + if n_pairs > 0: + member0_acc_idx = cg0_acc_index.idx + bars.mb_cg0_acc_done[member0_acc_idx].wait(cg0_acc_index.phase) + cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) + kqf_idx = kq_fused_index.idx + bars.mb_kq_ready[kqf_idx].wait(kq_fused_index.phase) + kq_fused_index = advance(kq_fused_index, cfg.smem_kq_stages) + desc_kqf = sKQ[kqf_idx].desc() + mma_ss(bmm_kk_desc, desc_kqf, desc_kqf, acc_cg0_0, accumulate=False, k_count=KQ_HALF_K) + mma_ss(bmm_kk_desc, desc_kqf + KQ_SEG, desc_kqf + KQ_SEG, acc_cg0_0, accumulate=True, k_count=KQ_HALF_K) + if elect_one: + bars.mb_cg0_acc_ready[member0_acc_idx].arrive(cta_group=1) + member1_acc_idx = cg0_acc_index.idx + bars.mb_cg0_acc_done[member1_acc_idx].wait(cg0_acc_index.phase) + cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) + kqf_idx = kq_fused_index.idx + bars.mb_kq_ready[kqf_idx].wait(kq_fused_index.phase) + kq_fused_index = advance(kq_fused_index, cfg.smem_kq_stages) + desc_kqf = sKQ[kqf_idx].desc() + desc_kqf_member1 = desc_kqf + KQ_BOX + mma_ss(bmm_kk_desc, desc_kqf, desc_kqf_member1, acc_cg0_1, accumulate=False, k_count=KQ_HALF_K) + mma_ss(bmm_kk_desc, desc_kqf + KQ_SEG, desc_kqf_member1 + KQ_SEG, acc_cg0_1, accumulate=True, k_count=KQ_HALF_K) + if elect_one: + bars.mb_cg0_acc_ready[member1_acc_idx].arrive(cta_group=1) + + for local_idx in cutlass.range(n_padded): # noqa: B007 + if cutlass.const_expr(cfg.use_initial_state): + if local_idx == 0: + if elect_one: + bars.mb_state_acc_ready[kv_acc_index.idx].arrive(cta_group=1) + kv_acc_index = advance(kv_acc_index, cfg.tmem_state_acc_stages) + have_state = cutlass.Boolean(True) if cutlass.const_expr(cfg.use_initial_state) else local_idx > 0 + + kq_idx = kq_index.idx + member = local_idx & 1 + state_inp_idx = state_inp_index.idx + ainv_idx = ainv_index.idx + kv_acc_idx = kv_acc_index.idx + kq_member_off = member * KQ_BOX + desc_k = sKQ[kq_idx].desc() + kq_member_off + desc_ainv = sAinv[ainv_idx].desc() + desc_kt = sKQ_trans[kq_idx].desc() + kq_member_off + state_a_ptr = nvvm.make_tmem_ptr(tmem_state_inp_col + state_inp_idx * STATE_INP_STAGE_COLS, cutlass.Int8) + state_acc_ptr = nvvm.make_tmem_ptr(tmem_state_col + kv_acc_idx * KV_ACC_STAGE_COLS, cutlass.Float32) + + kq_index = advance(kq_index, cfg.smem_kq_stages) + + # ---- KK pair lookahead (member 1) = K(S) @ K^T ----------------------- + if member == 1: + if (local_idx >> 1) + 1 < n_pairs: + member1_acc_idx = cg0_acc_index.idx + bars.mb_cg0_acc_done[member1_acc_idx].wait(cg0_acc_index.phase) + cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) + kqf_idx = kq_fused_index.idx + bars.mb_kq_ready[kqf_idx].wait(kq_fused_index.phase) + kq_fused_index = advance(kq_fused_index, cfg.smem_kq_stages) + desc_kqf = sKQ[kqf_idx].desc() + desc_kqf_member1 = desc_kqf + KQ_BOX + mma_ss(bmm_kk_desc, desc_kqf, desc_kqf_member1, acc_cg0_1, accumulate=False, k_count=KQ_HALF_K) + mma_ss(bmm_kk_desc, desc_kqf + KQ_SEG, desc_kqf_member1 + KQ_SEG, acc_cg0_1, accumulate=True, k_count=KQ_HALF_K) + if elect_one: + bars.mb_cg0_acc_ready[member1_acc_idx].arrive(cta_group=1) + + # ---- k_state = state(T) @ K^T (GEMM 3) ---------------------------------------- + if have_state: + bars.mb_state_inp_ready[state_inp_idx].wait(state_inp_index.phase) + state_inp_index = advance(state_inp_index, cfg.tmem_state_inp_stages) + + for k in cutlass.range_constexpr(KQ_HALF_K): + mma_ts_step(bmm_k_state_desc, state_a_ptr, desc_k, k_state_acc_ptr, k, cutlass.Boolean(k > 0)) + for k in cutlass.range_constexpr(KQ_HALF_K): + mma_ts_step(bmm_k_state_desc, state_a_ptr.subview(KQ_A_HALF), desc_k + KQ_SEG, k_state_acc_ptr, k, cutlass.Boolean(True)) + if elect_one: + bars.mb_k_state_ready[0].arrive(cta_group=1) + + # ---- NV = v_k_state(T) @ A_inv (GEMM 5) ------------------------------------ + bars.mb_ainv_ready[ainv_idx].wait(ainv_index.phase) + ainv_index = advance(ainv_index, cfg.smem_ainv_stages) + bars.mb_v_k_state_inp_ready[0].wait(v_k_state_inp_ready.phase) + v_k_state_inp_ready = advance(v_k_state_inp_ready, 1) + for k in cutlass.range_constexpr(cfg.b_t // 16): + mma_ts_step(bmm_nv_ts_desc, v_k_state_inp_ptr, desc_ainv, nv_acc_ptr, k, cutlass.Boolean(k > 0)) + if elect_one: + bars.mb_nv_ready[0].arrive(cta_group=1) + bars.mb_ainv_done[ainv_idx].arrive(cta_group=1) + + # ---- KK pair lookahead (member 0) = K(S) @ K^T ----------------------- + if member == 0: + if (local_idx >> 1) + 1 < n_pairs: + member0_acc_idx = cg0_acc_index.idx + bars.mb_cg0_acc_done[member0_acc_idx].wait(cg0_acc_index.phase) + cg0_acc_index = advance(cg0_acc_index, cfg.tmem_cg0_acc_stages) + kqf_idx = kq_fused_index.idx + bars.mb_kq_ready[kqf_idx].wait(kq_fused_index.phase) + kq_fused_index = advance(kq_fused_index, cfg.smem_kq_stages) + desc_kqf = sKQ[kqf_idx].desc() + mma_ss(bmm_kk_desc, desc_kqf, desc_kqf, acc_cg0_0, accumulate=False, k_count=KQ_HALF_K) + mma_ss(bmm_kk_desc, desc_kqf + KQ_SEG, desc_kqf + KQ_SEG, acc_cg0_0, accumulate=True, k_count=KQ_HALF_K) + if elect_one: + bars.mb_cg0_acc_ready[member0_acc_idx].arrive(cta_group=1) + + # ---- state += decay_V(T) @ K (GEMM 7) ------------------------------------ + bars.mb_decay_v_inp_ready[0].wait(decay_v_inp_ready.phase) + decay_v_inp_ready = advance(decay_v_inp_ready, 1) + kv_acc_index = advance(kv_acc_index, cfg.tmem_state_acc_stages) + for k in cutlass.range_constexpr(cfg.b_t // 16): + mma_ts_step(bmm_kv_desc, delta_inp_ptr, desc_kt, state_acc_ptr, k, cutlass.Boolean(True) if cutlass.const_expr(k > 0) else have_state) + if elect_one: + bars.mb_state_acc_ready[kv_acc_idx].arrive(cta_group=1) + bars.mb_kq_done[kq_idx].arrive(cta_group=1) + + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + bars.mb_tmem_done[0].wait(0) + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) + nvvm.tcgen05_dealloc( + nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), + cutlass.Int32(512), + group=nvvm.CTAGroup.CTA_1, + ) + + +@cute.jit +def tmaldg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sKQ_raw, + sV_raw, + desc_k_base, + desc_v_base, + mSched, + sSched, + bars, +): + """TMA-LDG warp role (warp 9): persistent scheduler loop + per-chunk + K/V G->S TMA loads.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + kq_index = PipelineState.start(phase=1) + v_index = PipelineState.start(phase=1) + sched_state = PipelineState.start(phase=1) + tile_idx = cutlass.Int32(bidx) + + bpe = cfg.io_dtype.width // 8 + granu = 128 // bpe + bt = cfg.b_t + kq_stage_elems = cfg.kq_cosize // cfg.smem_kq_stages + kq_box_elems = kq_stage_elems // 4 + sKQ_lo_tma = SmemTile( + base=sKQ_raw, + elems_per_stage=kq_stage_elems, + stages=cfg.smem_kq_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=2, + tma_granu_elems=granu, + tma_subtile_stride_elems=2 * bt * granu, + ) + sV_tma = SmemTile( + base=sV_raw, + elems_per_stage=cfg.v_cosize // cfg.smem_v_stages, + stages=cfg.smem_v_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=2, + tma_granu_elems=granu, + tma_subtile_stride_elems=4096, + ) + heads_out = cutlass.Int32(cfg.n_heads_out) + desc_qwords = cutlass.Int32(TENSOR_MAP_QWORDS) + + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + + head_k = head_idx if cfg.k_ratio == 1 else head_idx // cutlass.Int32(cfg.k_ratio) + head_v = head_idx if cfg.v_ratio == 1 else head_idx // cutlass.Int32(cfg.v_ratio) + slot = batch_idx * desc_qwords + desc_k_slot = (desc_k_base + slot).tospace(cutlass.AddressSpace.generic) + desc_v_slot = (desc_v_base + slot).tospace(cutlass.AddressSpace.generic) + if elect_one: + tma_tensormap_acquire(desc_k_slot) + tma_tensormap_acquire(desc_v_slot) + + wend_padded = cstart + ((wend - cstart + 1) // 2) * 2 + if wend_padded > cstart: + kq_idx = kq_index.idx + bars.mb_kq_done[kq_idx].wait(kq_index.phase) + kq_index = advance(kq_index, cfg.smem_kq_stages) + if elect_one: + bars.mb_kq_ready[kq_idx].arrive(n_bytes=cfg.tma_kq_bytes) + tok_coord = cstart * cutlass.Int32(cfg.b_t) + k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, tok_coord) + kq_tile = sKQ_lo_tma[kq_idx] + tma_load_tile(kq_tile, k_slice, bars.mb_kq_ready[kq_idx].smem_ptr, acquire=False) + for chunk_idx in cutlass.range(cstart + 1, wend_padded): + tok_coord = chunk_idx * cutlass.Int32(cfg.b_t) + + # ---- K load ------------------------------------------------------ + kq_idx = kq_index.idx + bars.mb_kq_done[kq_idx].wait(kq_index.phase) + kq_index = advance(kq_index, cfg.smem_kq_stages) + if elect_one: + bars.mb_kq_ready[kq_idx].arrive(n_bytes=cfg.tma_kq_bytes) + member = (chunk_idx - cstart) & 1 + k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, tok_coord) + kq_tile = sKQ_lo_tma[kq_idx] + if member == 0: + tma_load_tile(kq_tile, k_slice, bars.mb_kq_ready[kq_idx].smem_ptr, acquire=False) + else: + tma_load_tile(kq_tile.shifted(kq_box_elems), k_slice, bars.mb_kq_ready[kq_idx].smem_ptr, acquire=False) + + # ---- V load ------------------------------------------------------ + v_idx = v_index.idx + bars.mb_v_done[v_idx].wait(v_index.phase) + v_index = advance(v_index, cfg.smem_v_stages) + if elect_one: + bars.mb_v_ready[v_idx].arrive(n_bytes=cfg.tma_v_bytes) + v_tok = (chunk_idx - 1) * cutlass.Int32(cfg.b_t) + v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, v_tok) + tma_load_tile(sV_tma[v_idx], v_slice, bars.mb_v_ready[v_idx].smem_ptr, acquire=False) + + v_idx = v_index.idx + bars.mb_v_done[v_idx].wait(v_index.phase) + v_index = advance(v_index, cfg.smem_v_stages) + if elect_one: + bars.mb_v_ready[v_idx].arrive(n_bytes=cfg.tma_v_bytes) + v_tok = (wend_padded - 1) * cutlass.Int32(cfg.b_t) + v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, v_tok) + tma_load_tile(sV_tma[v_idx], v_slice, bars.mb_v_ready[v_idx].smem_ptr, acquire=False) + + tile_idx, sched_state = sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) + + for _ in range(cfg.smem_kq_stages): + bars.mb_kq_done[kq_index.idx].wait(kq_index.phase) + kq_index = advance(kq_index, cfg.smem_kq_stages) + for _ in range(cfg.smem_v_stages): + bars.mb_v_done[v_index.idx].wait(v_index.phase) + v_index = advance(v_index, cfg.smem_v_stages) + + +@cute.jit +def compute0_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + tidx, + tmem_hold, + sCumsumlog, + sBeta, + sAinv, + sCheckpoint_raw, + checkpoint_every_n_tokens, + sSched, + bars, +): + """Compute warp-group 0 role (warps 0-3): persistent scheduler loop + building the per-chunk beta-scaled A_inv operands.""" + + nvvm.setmaxregister(cfg.num_regs_compute_group_0, nvvm.SetMaxRegisterAction.INCREASE) + gate_index = PipelineState.start(phase=0) + beta_index = PipelineState.start(phase=0) + cg0_acc_ready = PipelineState.start(phase=0) + ainv_index = PipelineState.start(phase=1) + + nvvm.barrier_cta_sync_aligned( + cfg.tmem_alloc_barrier_id, + thread_count=cfg.tmem_alloc_barrier_threads, + ) + tmem_base = tmem_hold.load() + + num_threads_cg0 = cfg.threads_per_warp * len(cfg.compute_group_0_warp_ids) + cg0_tidx = tidx % num_threads_cg0 + warp_id = cg0_tidx // cfg.threads_per_warp + lane_id = cg0_tidx % cfg.threads_per_warp + inverse_local_warp = warp_id % 2 + pair_half = warp_id // 2 + half_row_base = inverse_local_warp * 32 + bpe = cfg.io_dtype.width // 8 + num_vals = 32 + FRAG_COLS = 16 + ACC_N_FRAGS = cfg.b_t // FRAG_COLS + store_row = warp_id * 16 + lane_id % 16 + store_row_frag = lane_id % 16 + store_col = (lane_id // 16) * 8 + tmem_warp_row = warp_id * cfg.threads_per_warp + tmem_cg0_acc_col = tmem_base + cfg.tmem_cg0_acc_offset + ACC_STAGE_COLS = cfg.b_t + mask_zero = opaque_f32_zero() + crow_lo = warp_id * 16 + lane_id // 4 + crow_hi = crow_lo + 8 + + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + n_local = wend - cstart + n_pairs = (n_local + 1) // 2 + n_padded = n_pairs * 2 + + for pair_i in cutlass.range(n_pairs): + # ---- Gate rows for this warp's KK member role ------------------------ + gate0_idx = gate_index.idx + bars.mb_gate_ready[gate0_idx].wait(gate_index.phase) + gate_index = advance(gate_index, cfg.smem_gate_stages) + gate1_idx = gate_index.idx + bars.mb_gate_ready[gate1_idx].wait(gate_index.phase) + gate_index = advance(gate_index, cfg.smem_gate_stages) + kk_gate_idx = gate1_idx if pair_half == 1 else gate0_idx + + row_u0_lo = half_row_base + lane_id // 4 + row_u0_hi = row_u0_lo + 8 + row_u1_lo = row_u0_lo + 16 + row_u1_hi = row_u0_lo + 24 + + kk_row_cumsumlog = [] + for r in (row_u0_lo, row_u0_hi, row_u1_lo, row_u1_hi): + kk_row_cumsumlog.append(sCumsumlog[r, 0, kk_gate_idx]) + kk_col_cumsumlog = [] + for g in cutlass.range_constexpr(8): + for b in cutlass.range_constexpr(2): + ccol = (lane_id % 4) * 2 + g * 8 + b + kk_col_cumsumlog.append(sCumsumlog[ccol, 0, kk_gate_idx]) + + decay_t_kk = [] + for u in cutlass.range_constexpr(2): + for k in cutlass.range_constexpr(num_vals): + hi_row = ((k // 2) % 2) == 1 + crow_u0 = row_u0_hi if cutlass.const_expr(hi_row) else row_u0_lo + crow_u1 = row_u1_hi if cutlass.const_expr(hi_row) else row_u1_lo + crow = crow_u1 if cutlass.const_expr(u == 1) else crow_u0 + ccol = (lane_id % 4) * 2 + ((k // 4) * 8 + k % 2) + is_lower = crow >= ccol + row_cumsumlog = kk_row_cumsumlog[u * 2 + (1 if hi_row else 0)] + col = (k // 4) * 2 + (k % 2) + decay_t_kk.append(cute.math.exp2(row_cumsumlog - kk_col_cumsumlog[col], fastmath=True) if is_lower else mask_zero) + bars.mb_gate_done[gate0_idx].arrive() + bars.mb_gate_done[gate1_idx].arrive() + + beta0_idx = beta_index.idx + bars.mb_beta_ready[beta0_idx].wait(beta_index.phase) + beta_index = advance(beta_index, cfg.smem_beta_stages) + beta1_idx = beta_index.idx + bars.mb_beta_ready[beta1_idx].wait(beta_index.phase) + beta_index = advance(beta_index, cfg.smem_beta_stages) + kk_beta_idx = beta1_idx if pair_half == 1 else beta0_idx + kk_beta = [] + for r in (row_u0_lo, row_u0_hi, row_u1_lo, row_u1_hi): + kk_beta.append(sBeta[r, 0, kk_beta_idx]) + + # ---- KK_epi (each warp pair stages its own member) ------------------- + acc0_idx = cg0_acc_ready.idx + acc0_phase = cg0_acc_ready.phase + cg0_acc_ready = advance(cg0_acc_ready, cfg.tmem_cg0_acc_stages) + acc1_idx = cg0_acc_ready.idx + acc1_phase = cg0_acc_ready.phase + cg0_acc_ready = advance(cg0_acc_ready, cfg.tmem_cg0_acc_stages) + kk_acc_idx = acc1_idx if pair_half == 1 else acc0_idx + kk_acc_phase = acc1_phase if pair_half == 1 else acc0_phase + + ainv0_idx = ainv_index.idx + ainv0_phase = ainv_index.phase + ainv_index = advance(ainv_index, cfg.smem_ainv_stages) + ainv1_idx = ainv_index.idx + ainv1_phase = ainv_index.phase + ainv_index = advance(ainv_index, cfg.smem_ainv_stages) + kk_ainv_idx = ainv1_idx if pair_half == 1 else ainv0_idx + kk_ainv_phase = ainv1_phase if pair_half == 1 else ainv0_phase + + bars.mb_cg0_acc_ready[kk_acc_idx].wait(kk_acc_phase) + + ainv0_base = sAinv[ainv0_idx].base + ainv1_base = sAinv[ainv1_idx].base + kk_base = ainv1_base if pair_half == 1 else ainv0_base + + kk_vec0 = nvvm.tcgen05_ld( + "16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_cg0_acc_col + kk_acc_idx * ACC_STAGE_COLS, cutlass.Float32), num=8 + ) + kk_vec1 = nvvm.tcgen05_ld( + "16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + 16) << 16) + tmem_cg0_acc_col + kk_acc_idx * ACC_STAGE_COLS, cutlass.Float32), num=8 + ) + nvvm.tcgen05_wait("load") + bars.mb_cg0_acc_done[kk_acc_idx].arrive() + bars.mb_ainv_done[kk_ainv_idx].wait(kk_ainv_phase) + for u in cutlass.range_constexpr(2): + kk_vec = kk_vec1 if cutlass.const_expr(u == 1) else kk_vec0 + kk_f16 = [] + for k in cutlass.range_constexpr(num_vals // 2): + row_beta = kk_beta[u * 2 + 1] if cutlass.const_expr((k % 2) == 1) else kk_beta[u * 2] + p0, p1 = fmul2(kk_vec[2 * k], kk_vec[2 * k + 1], decay_t_kk[u * num_vals + 2 * k], decay_t_kk[u * num_vals + 2 * k + 1]) + v0, v1 = fmul2(p0, p1, row_beta, row_beta) + kk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) + st_row = half_row_base + u * 16 + store_row_frag + for c in cutlass.range_constexpr(ACC_N_FRAGS): + nvvm.stmatrix( + cutlass.inttoptr( + kk_base + (st_row * cfg.b_t + swizzle_xor_128b(st_row, store_col + c * FRAG_COLS)) * bpe, + cutlass.AddressSpace.smem, + cutlass.BFloat16, + ), + [kk_f16[c * 4 + 0], kk_f16[c * 4 + 1], kk_f16[c * 4 + 2], kk_f16[c * 4 + 3]], + nvvm.MMALayout.ROW, + ) + + # ---- pair inverse: warps 0-1 own matrix 0, warps 2-3 matrix 1 -------- + inv_base = ainv1_base if warp_id >= 2 else ainv0_base + + # diagonal 8x8 Gauss-Jordan, all four warps + nvvm.barrier_cta_sync_aligned( + cfg.inverse_barrier_id, + thread_count=cfg.inverse_barrier_threads, + ) + invert_diagonal_NxN(cfg, inv_base, (inverse_local_warp * cfg.threads_per_warp + lane_id) // 8, cg0_tidx, 8) + nvvm.barrier_cta_sync_aligned( + cfg.inverse_barrier_id, + thread_count=cfg.inverse_barrier_threads, + ) + + # 8x8 -> 16x16 (both matrices per warp) + blockwise_diagonal_8x8_to_16x16(cfg, ainv0_base, warp_id * 16, lane_id) + blockwise_diagonal_8x8_to_16x16(cfg, ainv1_base, warp_id * 16, lane_id) + nvvm.barrier_cta_sync_aligned( + cfg.inverse_barrier_id, + thread_count=cfg.inverse_barrier_threads, + ) + + # 16x16 -> 32x32, one tile per warp within the group + blockwise_diagonal_16x16_to_32x32(cfg, inv_base, inverse_local_warp * 32, lane_id) + nvvm.barrier_cta_sync_aligned( + cfg.inverse_barrier_id, + thread_count=cfg.inverse_barrier_threads, + ) + + # 32x32 -> 64x64, two warps per matrix + blockwise_diagonal_32x32_to_64x64(cfg, inv_base, inverse_local_warp, lane_id) + nvvm.barrier_cta_sync_aligned( + cfg.inverse_barrier_id, + thread_count=cfg.inverse_barrier_threads, + ) + + # ---- Beta column-scaling + publish, stage 0 -------------------------- + beta_col = [] + for k in cutlass.range_constexpr(num_vals): + beta_col.append(sBeta[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, beta0_idx]) + ainv_f16 = [] + for c in cutlass.range_constexpr(ACC_N_FRAGS): + ainv_f16 += list( + nvvm.ldmatrix( + cutlass.inttoptr( + ainv0_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, + cutlass.AddressSpace.smem, + cutlass.BFloat16, + ), + 4, + nvvm.MMALayout.ROW, + ) + ) + ainv_scaled = [] + for j in cutlass.range_constexpr(num_vals // 2): + lo, hi = f16x2_to_f32(ainv_f16[j], dtype=cfg.io_dtype) + s0, s1 = fmul2(lo, hi, beta_col[2 * j], beta_col[2 * j + 1]) + ainv_scaled.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) + for c in cutlass.range_constexpr(ACC_N_FRAGS): + nvvm.stmatrix( + cutlass.inttoptr( + ainv0_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, + cutlass.AddressSpace.smem, + cutlass.BFloat16, + ), + [ainv_scaled[c * 4 + 0], ainv_scaled[c * 4 + 1], ainv_scaled[c * 4 + 2], ainv_scaled[c * 4 + 3]], + nvvm.MMALayout.ROW, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_ainv_ready[ainv0_idx].arrive() + bars.mb_beta_done[beta0_idx].arrive() + + # ---- Beta column-scaling + publish, stage 1 -------------------------- + beta_col = [] + for k in cutlass.range_constexpr(num_vals): + beta_col.append(sBeta[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, beta1_idx]) + ainv_f16 = [] + for c in cutlass.range_constexpr(ACC_N_FRAGS): + ainv_f16 += list( + nvvm.ldmatrix( + cutlass.inttoptr( + ainv1_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, + cutlass.AddressSpace.smem, + cutlass.BFloat16, + ), + 4, + nvvm.MMALayout.ROW, + ) + ) + ainv_scaled = [] + for j in cutlass.range_constexpr(num_vals // 2): + lo, hi = f16x2_to_f32(ainv_f16[j], dtype=cfg.io_dtype) + s0, s1 = fmul2(lo, hi, beta_col[2 * j], beta_col[2 * j + 1]) + ainv_scaled.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) + for c in cutlass.range_constexpr(ACC_N_FRAGS): + nvvm.stmatrix( + cutlass.inttoptr( + ainv1_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, + cutlass.AddressSpace.smem, + cutlass.BFloat16, + ), + [ainv_scaled[c * 4 + 0], ainv_scaled[c * 4 + 1], ainv_scaled[c * 4 + 2], ainv_scaled[c * 4 + 3]], + nvvm.MMALayout.ROW, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_ainv_ready[ainv1_idx].arrive() + bars.mb_beta_done[beta1_idx].arrive() + + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + for _ in range(cfg.smem_ainv_stages): + bars.mb_ainv_done[ainv_index.idx].wait(ainv_index.phase) + ainv_index = advance(ainv_index, cfg.smem_ainv_stages) + + +@cute.jit +def compute1_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + tidx, + warp_idx, + tmem_hold, + sV, + sCumsumlog, + sCumprod, + sBeta, + sCheckpoint_raw, + mState_init, + mState_out, + checkpoint_every_n_tokens, + sSched, + bars, +): + """Compute warp-group 1 role (warps 4-7): persistent scheduler loop + owning the recurrent state from seed to final store.""" + elect_one = nvvm.elect_sync() + + v_index = PipelineState.start(phase=0) + gate_index = PipelineState.start(phase=0) + kv_acc_index = PipelineState.start(phase=0) + k_state_ready_index = PipelineState.start(phase=0) + nv_ready_index = PipelineState.start(phase=0) + state_acc_seed_index = PipelineState.start(phase=1) + state_inp_cnt = cutlass.Int32(0) + kv_done_idx = cutlass.Int32(0) + + nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) + nvvm.barrier_cta_sync_aligned( + cfg.tmem_alloc_barrier_id, + thread_count=cfg.tmem_alloc_barrier_threads, + ) + tmem_base = tmem_hold.load() + + num_threads_cg1 = cfg.threads_per_warp * len(cfg.compute_group_1_warp_ids) + cg1_tidx = tidx % num_threads_cg1 + lane_id = cg1_tidx % cfg.threads_per_warp + tmem_warp_row = (cg1_tidx // cfg.threads_per_warp) * cfg.threads_per_warp + ldtm_width = 32 + sttm_width = ldtm_width // 2 + num_state_subs = cutlass.const_expr(cfg.d_v // ldtm_width) + tmem_state_col = tmem_base + cfg.tmem_state_acc_offset + tmem_state_inp_col = tmem_base + cfg.tmem_state_inp_offset + tmem_inp_col = tmem_base + cfg.tmem_v_k_state_decay_inp_offset + INP_SLOT_COLS = cfg.b_t // 2 + tmem_k_state_col = tmem_base + cfg.tmem_cg1_acc_offset + tmem_nv_col = tmem_k_state_col + tmem_v_k_state_col = tmem_inp_col + tmem_decay_v_col = tmem_inp_col + INP_SLOT_COLS + v_frag_tok = cg1_tidx % 8 + (cg1_tidx // 16 % 2) * 8 + v_frag_col = (cg1_tidx // 8 % 2) * 8 + (cg1_tidx // 32 % 2) * 32 + v_frag_slab = (cg1_tidx // 64) * 4096 + v_stage_elems = cfg.v_cosize // cfg.smem_v_stages + sV_base = cute.make_ptr(cfg.io_dtype, sV[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) + num_vals = 32 + if cutlass.const_expr(cfg.enable_checkpoints): + sCheckpoint_base_int = sCheckpoint_raw.data_ptr().toint() + checkpoint_cnt = cutlass.Int32(0) + checkpoint_frag_row = cg1_tidx % 8 + (cg1_tidx // 16 % 2) * 8 + checkpoint_frag_col = (cg1_tidx // 8 % 2) * 8 + (cg1_tidx // 32 % 2) * 32 + + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + n_local = wend - cstart + n_padded = ((n_local + 1) // 2) * 2 + if cutlass.const_expr(cfg.enable_checkpoints): + ckpt_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) + checkpoint_mod = cstart % ckpt_chunks + if n_local > 0: + if cutlass.const_expr(cfg.use_initial_state): + # ---- initial-state seed: initial_state GMEM -> state TMEM --------------- + gState_init = mState_init[None, None, head_idx, batch_idx] + kv_init_idx = state_acc_seed_index.idx + bars.mb_state_acc_scale_done[kv_init_idx].wait(state_acc_seed_index.phase) + state_acc_seed_index = advance(state_acc_seed_index, cfg.tmem_state_acc_stages) + seed_from_initial_state = cstart == 0 + if cutlass.const_expr(cfg.split_k): + if seed_from_initial_state: + for sub in cutlass.range_constexpr(num_state_subs): + words = [] + for k in cutlass.range_constexpr(32): + v = gState_init[sub * ldtm_width + k, cg1_tidx] + if cutlass.const_expr(cfg.state_dtype != cfg.acc_dtype): + v = v.to(cfg.acc_dtype) + words.append(v) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), + cutlass.Vector.from_elements(tuple(words), cutlass.Float32), + ) + else: + for sub in cutlass.range_constexpr(num_state_subs): + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), + cutlass.Vector.from_elements(tuple(cutlass.Float32(0.0) for _ in range(32)), cutlass.Float32), + ) + else: + for sub in cutlass.range_constexpr(num_state_subs): + words = [] + for k in cutlass.range_constexpr(32): + v = gState_init[sub * ldtm_width + k, cg1_tidx] + if cutlass.const_expr(cfg.state_dtype != cfg.acc_dtype): + v = v.to(cfg.acc_dtype) + words.append(v) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), + cutlass.Vector.from_elements(tuple(words), cutlass.Float32), + ) + nvvm.tcgen05_wait("store") + + nvvm.barrier_cta_sync_aligned( + cfg.init_state_store_barrier_id, + thread_count=cfg.init_state_store_barrier_threads, + ) + + for local_idx in cutlass.range(n_padded): # noqa: B007 + chunk_idx = cstart + local_idx + if cutlass.const_expr(cfg.enable_checkpoints): + do_checkpoint_now = checkpoint_mod == 0 + checkpoint_mod = checkpoint_mod + cutlass.Int32(1) + checkpoint_mod = cutlass.Int32(0) if checkpoint_mod == ckpt_chunks else checkpoint_mod + valid_state = local_idx > 0 + if cutlass.const_expr(cfg.use_initial_state): + valid_state = cutlass.Boolean(True) + state_acc_seed_index = advance(state_acc_seed_index, cfg.tmem_state_acc_stages) + + gate_idx = gate_index.idx + bars.mb_gate_ready[gate_idx].wait(gate_index.phase) + gate_index = advance(gate_index, cfg.smem_gate_stages) + cumprod_total = sCumprod[sCumprod.shape[0] - 1, 0, gate_idx] + + # ---- state restage + rescale ------------------------------------- + if valid_state: + kv_idx = kv_acc_index.idx + bars.mb_state_acc_ready[kv_idx].wait(kv_acc_index.phase) + kv_acc_index = advance(kv_acc_index, cfg.tmem_state_acc_stages) + kv_done_idx = kv_idx + + state_regs = [[cutlass.Float32(0.0) for _ in range(num_state_subs)] for _ in range(32)] + state_inp_stage_idx = state_inp_cnt % cfg.tmem_state_inp_stages + state_vecs = [] + for sub in cutlass.range_constexpr(num_state_subs): + state_vecs.append( + nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), num=32) + ) + for sub in cutlass.range_constexpr(num_state_subs): + for k in cutlass.range_constexpr(32): + state_regs[k][sub] = state_vecs[sub][k] + state_f16 = [fp32_to_fp16(state_regs[2 * j][sub], state_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_inp_col + sub * sttm_width, cutlass.Int32), + cutlass.Vector.from_elements(tuple(state_f16), cutlass.Int32), + ) + nvvm.tcgen05_wait("store") + bars.mb_state_inp_ready[state_inp_stage_idx].arrive() + state_inp_cnt = state_inp_cnt + 1 + + if cutlass.const_expr(cfg.enable_checkpoints): + # ---- state checkpoint ---------------------------------------- + do_checkpoint = do_checkpoint_now and chunk_idx > 0 and chunk_idx < wend + if cutlass.const_expr(cfg.split_k): + do_checkpoint = do_checkpoint and chunk_idx >= wstart + if do_checkpoint: + checkpoint_regs = [[cutlass.Int32(0) for _ in range(16)] for _ in range(4)] + for b in cutlass.range_constexpr(2): + for col_half in cutlass.range_constexpr(2): + checkpoint_vec = nvvm.tcgen05_ld( + "16x256b", + nvvm.make_tmem_ptr(((tmem_warp_row + b * 16) << 16) + tmem_state_col + col_half * 64, cutlass.Float32), + num=8, + ) + for j in cutlass.range_constexpr(16): + checkpoint_regs[b * 2 + col_half][j] = fp32_to_fp16( + checkpoint_vec[2 * j], checkpoint_vec[2 * j + 1], dtype=cfg.io_dtype + ) + checkpoint_stage = checkpoint_cnt % cfg.smem_checkpoint_stages + checkpoint_phase_done = cutlass.Int32(1) ^ ((checkpoint_cnt // cfg.smem_checkpoint_stages) & cutlass.Int32(1)) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].wait(checkpoint_phase_done) + for b in cutlass.range_constexpr(2): + for col_half in cutlass.range_constexpr(2): + checkpoint_base = checkpoint_stage * cfg.d_k * cfg.d_v + (cg1_tidx // 64) * cfg.d_k * 64 + for c in cutlass.range_constexpr(4): + checkpoint_row = col_half * 64 + checkpoint_frag_row + c * 16 + nvvm.stmatrix( + cutlass.inttoptr( + sCheckpoint_base_int + + (checkpoint_base + checkpoint_row * 64 + swizzle_xor_128b(checkpoint_row, checkpoint_frag_col + b * 16)) * 2, + cutlass.AddressSpace.smem, + cfg.io_dtype, + ), + [ + checkpoint_regs[b * 2 + col_half][c * 4 + 0], + checkpoint_regs[b * 2 + col_half][c * 4 + 1], + checkpoint_regs[b * 2 + col_half][c * 4 + 2], + checkpoint_regs[b * 2 + col_half][c * 4 + 3], + ], + nvvm.MMALayout.COL, + ) + nvvm.fence_proxy("async.shared", space="cta") + if elect_one: + bars.mb_checkpoint_tmastg_ready[checkpoint_stage].arrive() + checkpoint_cnt = checkpoint_cnt + 1 + + for sub in cutlass.range_constexpr(num_state_subs): + state_scaled = [] + for j in cutlass.range_constexpr(16): + s0, s1 = fmul2(state_regs[2 * j][sub], state_regs[2 * j + 1][sub], cumprod_total, cumprod_total) + state_scaled += [s0, s1] + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), + cutlass.Vector.from_elements(tuple(state_scaled), cutlass.Float32), + ) + nvvm.tcgen05_wait("store") + bars.mb_state_acc_scale_done[kv_idx].arrive() + + # ---- per-row Gate register builds -------------------------------- + cumprod_vals = [] + for k in cutlass.range_constexpr(num_vals): + cumprod_vals.append(sCumprod[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, gate_idx]) + last_cumsumlog = sCumsumlog[cfg.b_t - 1, 0, gate_idx] + cumsumlog_vals = [] + for k in cutlass.range_constexpr(num_vals): + cumsumlog_vals.append(sCumsumlog[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, gate_idx]) + decay_scale_vals = [] + for k in cutlass.range_constexpr(0, num_vals, 2): + d0, d1 = fadd2(last_cumsumlog, last_cumsumlog, -cumsumlog_vals[k], -cumsumlog_vals[k + 1]) + decay_scale_vals.append(cute.math.exp2(d0, fastmath=True)) + decay_scale_vals.append(cute.math.exp2(d1, fastmath=True)) + bars.mb_gate_done[gate_idx].arrive() + + # ---- V - K*state (packed 16-bit) --------------------------------- + v_idx = v_index.idx + bars.mb_v_ready[v_idx].wait(v_index.phase) + v_index = advance(v_index, cfg.smem_v_stages) + + v_words = [[cutlass.Int32(0), cutlass.Int32(0)] for _ in range(16)] + for c in cutlass.range_constexpr(8): + tok_block = cutlass.const_expr(c % 4) + sub = cutlass.const_expr(c // 4) + v_f16 = nvvm.ldmatrix( + ( + sV_base + + v_idx * v_stage_elems + + v_frag_slab + + (v_frag_tok + tok_block * 16) * 64 + + swizzle_xor_128b(v_frag_tok + tok_block * 16, v_frag_col + sub * 16) + ).raw_ptr(), + 4, + nvvm.MMALayout.COL, + ) + for i in cutlass.range_constexpr(4): + v_words[4 * tok_block + i][sub] = v_f16[i] + if valid_state: + bars.mb_k_state_ready[0].wait(k_state_ready_index.phase) + k_state_ready_index = advance(k_state_ready_index, 1) + + for sub in cutlass.range_constexpr(2): + k_state_vec = nvvm.tcgen05_ld( + "16x256b", + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_k_state_col, cutlass.Float32), + num=8, + ) + for j in cutlass.range_constexpr(16): + s0, s1 = fmul2(k_state_vec[2 * j], k_state_vec[2 * j + 1], cumprod_vals[2 * j], cumprod_vals[2 * j + 1]) + k_state_word = fp32_to_fp16(s0, s1, dtype=cfg.io_dtype) + v_words[j][sub] = sub_f16x2(v_words[j][sub], k_state_word, cfg.io_dtype) + for sub in cutlass.range_constexpr(2): + nvvm.tcgen05_st( + "16x128b", + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_v_k_state_col, cutlass.Int32), + cutlass.Vector.from_elements(tuple(v_words[j][sub] for j in range(16)), cutlass.Int32), + ) + nvvm.tcgen05_wait("store") + bars.mb_v_k_state_inp_ready[0].arrive() + + # ---- new_V_epi + decay_V publish --------------------------------- + bars.mb_nv_ready[0].wait(nv_ready_index.phase) + nv_ready_index = advance(nv_ready_index, 1) + bars.mb_v_done[v_idx].arrive() + + nv_regs = [[cutlass.Float32(0.0), cutlass.Float32(0.0)] for _ in range(32)] + nv_vecs = [] + for sub in cutlass.range_constexpr(2): + nv_vecs.append( + nvvm.tcgen05_ld( + "16x256b", + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_nv_col, cutlass.Float32), + num=8, + ) + ) + for sub in cutlass.range_constexpr(2): + for k in cutlass.range_constexpr(32): + nv_regs[k][sub] = nv_vecs[sub][k] + + for sub in cutlass.range_constexpr(2): + for j in cutlass.range_constexpr(16): + nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub] = fmul2( + nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], decay_scale_vals[2 * j], decay_scale_vals[2 * j + 1] + ) + decay_f16 = [fp32_to_fp16(nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] + nvvm.tcgen05_st( + "16x128b", + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_decay_v_col, cutlass.Int32), + cutlass.Vector.from_elements(tuple(decay_f16), cutlass.Int32), + ) + nvvm.tcgen05_wait("store") + bars.mb_decay_v_inp_ready[0].arrive() + + # ---- final state: state TMEM -> GMEM ----------------------------------- + if n_local > 0: + kv_last_idx = kv_acc_index.idx + bars.mb_state_acc_ready[kv_last_idx].wait(kv_acc_index.phase) + kv_acc_index = advance(kv_acc_index, cfg.tmem_state_acc_stages) + if cutlass.const_expr(cfg.store_final_state): + if cutlass.const_expr(cfg.split_k): + if wend == num_chunks_b: + gState_out = mState_out[None, None, head_idx, batch_idx] + for sub in cutlass.range_constexpr(num_state_subs): + state_vec = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), num=32 + ) + for k in cutlass.range_constexpr(32): + val = state_vec[k] + if cutlass.const_expr(cfg.state_dtype != cfg.acc_dtype): + val = val.to(cfg.state_dtype) + gState_out[sub * ldtm_width + k, cg1_tidx] = val + else: + gState_out = mState_out[None, None, head_idx, batch_idx] + for sub in cutlass.range_constexpr(num_state_subs): + state_vec = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_col + sub * ldtm_width, cutlass.Float32), num=32 + ) + for k in cutlass.range_constexpr(32): + val = state_vec[k] + if cutlass.const_expr(cfg.state_dtype != cfg.acc_dtype): + val = val.to(cfg.state_dtype) + gState_out[sub * ldtm_width + k, cg1_tidx] = val + bars.mb_state_acc_scale_done[kv_last_idx].arrive() + else: + bars.mb_state_acc_scale_done[kv_last_idx].arrive() + else: + if cutlass.const_expr(cfg.store_final_state): + write_passthrough = True + if cutlass.const_expr(cfg.split_k): + write_passthrough = wend == num_chunks_b + if write_passthrough: + gState_out = mState_out[None, None, head_idx, batch_idx] + if cutlass.const_expr(cfg.use_initial_state): + gState_in = mState_init[None, None, head_idx, batch_idx] + for r in cutlass.range(num_state_subs * ldtm_width): + gState_out[r, cg1_tidx] = gState_in[r, cg1_tidx] + else: + for sub in cutlass.range_constexpr(num_state_subs): + for k in cutlass.range_constexpr(32): + gState_out[sub * ldtm_width + k, cg1_tidx] = cutlass.Float32(0.0).to(cfg.state_dtype) + + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + bars.mb_tmem_done[0].arrive() + + if cutlass.const_expr(cfg.enable_checkpoints): + for _ in range(cfg.smem_checkpoint_stages): + checkpoint_stage = checkpoint_cnt % cfg.smem_checkpoint_stages + checkpoint_phase_done = cutlass.Int32(1) ^ ((checkpoint_cnt // cfg.smem_checkpoint_stages) & cutlass.Int32(1)) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].wait(checkpoint_phase_done) + checkpoint_cnt = checkpoint_cnt + 1 + + +@cute.kernel +def build_all_descs_kernel( + base_k: cutlass.GridConstant[tma.TensorMap], + base_v: cutlass.GridConstant[tma.TensorMap], + base_checkpoint: cutlass.GridConstant[tma.TensorMap], + desc_ws: cute.Tensor, + cu_seqlens: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + state_checkpoints_out: Optional[cute.Tensor], + n_batch: cutlass.Int32, + k_row_stride: cutlass.Int32, + v_row_stride: cutlass.Int32, + checkpoint_row_stride: cutlass.Int32, + checkpoint_every_n: cutlass.Int32, +) -> None: + """Single-launch builder for the per-BATCH descriptor arrays (one warp + per array).""" + tidx, _, _ = cute.arch.thread_idx() + widx = cutlass.Int32(tidx) // cutlass.Int32(32) + arr_words = n_batch * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_k_arr = cute.make_tensor(desc_ws.iterator, cute.make_layout((arr_words,), stride=(1,))) + desc_v_arr = cute.make_tensor(desc_ws.iterator + arr_words, cute.make_layout((arr_words,), stride=(1,))) + desc_checkpoint_arr = cute.make_tensor(desc_ws.iterator + 2 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + + if widx == 0: + if nvvm.elect_sync(): + emit_seq_descs(base_k, desc_k_arr, cu_seqlens, k, n_batch, k_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 1: + if nvvm.elect_sync(): + emit_seq_descs(base_v, desc_v_arr, cu_seqlens, v, n_batch, v_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if cutlass.const_expr(state_checkpoints_out is not None): + if widx == 2: + if nvvm.elect_sync(): + emit_checkpoint_seq_descs( + base_checkpoint, desc_checkpoint_arr, cu_seqlens, state_checkpoints_out, n_batch, checkpoint_row_stride, checkpoint_every_n, 2 + ) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + + +@cute.jit +def build_descs( + io_dtype: cutlass.Constexpr, + b_t: cutlass.Constexpr[int], + k: cute.Tensor, + v: cute.Tensor, + cu_seqlens: cute.Tensor, + state_checkpoints_out: Optional[cute.Tensor], + checkpoint_every_n: cutlass.Int32, + tensormap_workspace: cute.Tensor, + stream: cuda.CUstream, +): + """Build the 3 per-batch TMA-descriptor arrays (K, V, checkpoints) into + ``tensormap_workspace``.""" + h_k = k.shape[1] + h_v = v.shape[1] + batch_size = cu_seqlens.shape[0] - 1 + d_v = v.shape[2] + bpe = io_dtype.width // 8 + granu = 128 // bpe + bt = b_t + + k_row_stride, k_head_stride = k.stride[0], k.stride[1] + v_row_stride, v_head_stride = v.stride[0], v.stride[1] + + seqlen = k.shape[0] + d_k = k.shape[2] + k_headed = cute.make_tensor(k.iterator, cute.make_layout((seqlen, h_k, d_k), stride=(k_row_stride, k_head_stride, 1))) + v_headed = cute.make_tensor(v.iterator, cute.make_layout((d_v, h_v, seqlen), stride=(1, v_head_stride, v_row_stride))) + swz128 = tma.TensorMapSwizzle.s128b + base_desc_k = tma.create_tensor_map_tiled_from_view(k_headed, box_dims=(bt, 1, granu), stride_order=(2, 1, 0), swizzle=swz128) + base_desc_v = tma.create_tensor_map_tiled_from_view(v_headed, box_dims=(granu, 1, bt), stride_order=(0, 1, 2), swizzle=swz128) + + base_desc_checkpoint = base_desc_v + if cutlass.const_expr(state_checkpoints_out is not None): + d_k_state = state_checkpoints_out.shape[2] + d_v_state = state_checkpoints_out.shape[3] + checkpoint_granu = 128 // (state_checkpoints_out.element_type.width // 8) + checkpoint_view = cute.make_tensor( + state_checkpoints_out.iterator, + cute.make_layout( + (d_v_state, d_k_state, state_checkpoints_out.shape[0], state_checkpoints_out.shape[1]), + stride=(state_checkpoints_out.stride[3], state_checkpoints_out.stride[2], state_checkpoints_out.stride[0], state_checkpoints_out.stride[1]), + ), + ) + base_desc_checkpoint = tma.create_tensor_map_tiled_from_view( + checkpoint_view, box_dims=(checkpoint_granu, d_k_state, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz128 + ) + + n_warps = 3 if state_checkpoints_out is not None else 2 + build_all_descs_kernel( + base_desc_k, + base_desc_v, + base_desc_checkpoint, + tensormap_workspace, + cu_seqlens, + k, + v, + state_checkpoints_out, + cutlass.Int32(batch_size), + cutlass.Int32(k_row_stride), + cutlass.Int32(v_row_stride), + cutlass.Int32(state_checkpoints_out.stride[0] if state_checkpoints_out is not None else 0), + checkpoint_every_n, + ).launch(grid=(1, 1, 1), block=(32 * n_warps, 1, 1), stream=stream) + + +@cute.jit +def host( + cfg: cutlass.Constexpr, + k: cute.Tensor, + v: cute.Tensor, + gate: cute.Tensor, + beta: cute.Tensor, + cu_seqlens: cute.Tensor, + state_in: Optional[cute.Tensor], + state_out: Optional[cute.Tensor], + work_items: Optional[cute.Tensor], + work_count: Optional[cute.Tensor], + sched_ctr: Optional[cute.Tensor], + checkpoint_every_n_tokens: cutlass.Int32, + tensormap_workspace: cute.Tensor, + stream: cuda.CUstream, +): + h_k = k.shape[1] + h_v = v.shape[1] + batch_size = cu_seqlens.shape[0] - 1 + heads_out = gate.shape[1] + + # ---- GQA reshapes: fold the head group into a -------------------------------- + if cutlass.const_expr(cfg.is_GQA): + h_ratio = heads_out // h_v + h_native = h_v + k = cute.make_tensor( + k.iterator, + cute.make_layout( + (k.shape[0], k.shape[2], (h_ratio, h_v)), + stride=(k.stride[0], k.stride[2], (0, k.stride[1])), + ), + ) + v = cute.make_tensor( + v.iterator, + cute.make_layout( + (v.shape[2], v.shape[0], (h_ratio, h_v)), + stride=(v.stride[2], v.stride[0], (0, v.stride[1])), + ), + ) + else: + h_ratio = h_v // h_k + h_native = h_k + k = cute.make_tensor( + k.iterator, + cute.make_layout( + (k.shape[0], k.shape[2], (h_ratio, h_k)), + stride=(k.stride[0], k.stride[2], (0, k.stride[1])), + ), + ) + v = cute.make_tensor( + v.iterator, + cute.make_layout( + (v.shape[2], v.shape[0], (h_ratio, h_k)), + stride=(v.stride[2], v.stride[0], (v.stride[1], h_ratio * v.stride[1])), + ), + ) + + gate = cute.make_tensor( + gate.iterator, + cute.make_layout( + (gate.shape[0], (h_ratio, h_native)), + stride=(gate.stride[0], (gate.stride[1], h_ratio * gate.stride[1])), + ), + ) + beta = cute.make_tensor( + beta.iterator, + cute.make_layout( + (beta.shape[0], (h_ratio, h_native)), + stride=(beta.stride[0], (beta.stride[1], h_ratio * beta.stride[1])), + ), + ) + if cutlass.const_expr(state_in is not None): + state_in = cute.make_tensor( + state_in.iterator, + cute.make_layout( + (state_in.shape[2], state_in.shape[3], (h_ratio, h_native), state_in.shape[0]), + stride=( + state_in.stride[2], + state_in.stride[3], + (state_in.stride[1], h_ratio * state_in.stride[1]), + state_in.stride[0], + ), + ), + ) + if cutlass.const_expr(state_out is not None): + state_out = cute.make_tensor( + state_out.iterator, + cute.make_layout( + (state_out.shape[2], state_out.shape[3], (h_ratio, h_native), state_out.shape[0]), + stride=( + state_out.stride[2], + state_out.stride[3], + (state_out.stride[1], h_ratio * state_out.stride[1]), + state_out.stride[0], + ), + ), + ) + + # ---- SMEM sizing: per-buffer element cosizes --------------------------------- + bpe = cfg.io_dtype.width // 8 + kq_tile_elems = 2 * cfg.b_t * cfg.d_k + v_tile_elems = cfg.d_v * cfg.b_t + ainv_tile_elems = cfg.b_t * cfg.b_t + cfg.kq_cosize = kq_tile_elems * cfg.smem_kq_stages + cfg.v_cosize = v_tile_elems * cfg.smem_v_stages + cfg.ainv_cosize = ainv_tile_elems * cfg.smem_ainv_stages + cfg.checkpoint_cosize = cfg.d_k * cfg.d_v * cfg.smem_checkpoint_stages + + cumsumlog_smem_layout_staged = cute.make_layout((cfg.b_t, 1, cfg.smem_gate_stages)) + beta_smem_layout_staged = cute.make_layout((cfg.b_t, 1, cfg.smem_beta_stages)) + + cfg.tma_kq_bytes = (kq_tile_elems // 2) * bpe + cfg.tma_v_bytes = v_tile_elems * bpe + + cfg.n_heads_out = heads_out + cfg.k_ratio = heads_out // h_k + cfg.v_ratio = heads_out // h_v + num_descs = batch_size + + # ---- launch ------------------------------------------------------------------ + total_tiles = batch_size * heads_out + grid_shape = (cfg.max_active_clusters, 1, 1) + + kernel( + cfg, + gate, + beta, + cu_seqlens, + state_in, + state_out, + work_items, + work_count, + sched_ctr, + checkpoint_every_n_tokens, + cumsumlog_smem_layout_staged, + beta_smem_layout_staged, + total_tiles, + k, + v, + tensormap_workspace, + cutlass.Int32(num_descs), + ).launch( + grid=grid_shape, + block=(cfg.threads_per_cta, 1, 1), + cluster=cfg.cluster_shape_mnk, + stream=stream, + min_blocks_per_mp=1, + ) + + +@cute.kernel +def kernel( + cfg: cutlass.Constexpr, + mGate: cute.Tensor, + mBeta: cute.Tensor, + cu_seqlens: cute.Tensor, + mState_init: Optional[cute.Tensor], + mState_out: Optional[cute.Tensor], + mWorkItems: Optional[cute.Tensor], + mCount: Optional[cute.Tensor], + mSched: Optional[cute.Tensor], + checkpoint_every_n_tokens: cutlass.Int32, + cumsumlog_smem_layout_staged: cute.Layout, + beta_smem_layout_staged: cute.Layout, + total_tiles: cutlass.Int32, + mK, + mV, + tensormap_workspace: cute.Tensor, + n_desc: cutlass.Int32, +): + """Main GDN chunked kernel: warp-specialized dispatch over (batch, head) + tiles.""" + tidx, _, _ = cute.arch.thread_idx() + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + bidx = cute.arch.block_idx()[0] + num_ctas = cute.arch.grid_dim()[0] + + if cutlass.const_expr(cfg.split_k): + total_tiles = mCount[0] + if cutlass.const_expr(cfg.dyn_sched): + assert mSched is not None, "mSched must be provided if dyn_sched is True" + + if cutlass.const_expr(cfg.use_initial_state): + assert mState_init is not None, "mState_init must be provided if use_initial_state is True" + else: + assert mState_init is None, "mState_init must be None if use_initial_state is False" + if cutlass.const_expr(cfg.store_final_state): + assert mState_out is not None, "mState_out must be provided if store_final_state is True" + else: + assert mState_out is None, "mState_out must be None if store_final_state is False" + + desc_base_words = tensormap_workspace.iterator.raw_ptr() + desc_qwords = cutlass.Int32(TENSOR_MAP_QWORDS) + arr_words = n_desc * desc_qwords + desc_k_base = desc_base_words + desc_v_base = desc_base_words + arr_words + desc_checkpoint_base = desc_base_words + cutlass.Int32(2) * arr_words + + SMEM = cutlass.AddressSpace.smem + + bpe = cfg.io_dtype.width // 8 + SWZ = 2 + LEAD = 16 + STRIDE = 8 * 128 + KT_LEAD = (cfg.d_v // 2) * 128 + V_LEAD = (cfg.d_v // 2) * 128 + if cutlass.const_expr(cfg.enable_checkpoints): + sCheckpoint_raw = cutlass.Array( + cfg.io_dtype, + cfg.checkpoint_cosize, + space=cutlass.AddressSpace.smem, + alignment=cfg.buffer_align_bytes, + ) + else: + sCheckpoint_raw = None + sKQ_raw = cutlass.Array( + cfg.io_dtype, + cfg.kq_cosize, + space=cutlass.AddressSpace.smem, + alignment=cfg.buffer_align_bytes, + ) + sKQ = SmemTile( + base=sKQ_raw.data_ptr().toint(), + elems_per_stage=(cfg.kq_cosize // cfg.smem_kq_stages) * bpe, + stages=cfg.smem_kq_stages, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sKQ_trans = SmemTile( + base=sKQ_raw.data_ptr().toint(), + elems_per_stage=(cfg.kq_cosize // cfg.smem_kq_stages) * bpe, + stages=cfg.smem_kq_stages, + leading_byte_offset=2 * KT_LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + bars = make_gdn_bars(cfg) + tmem_hold = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=16) + sSched = cutlass.Array(cutlass.Int32, cfg.sched_stages, space=SMEM, alignment=16) + cumsumlog_raw = cutlass.Array(cutlass.Float32, cute.cosize(cumsumlog_smem_layout_staged), space=SMEM, alignment=128) + cumprod_raw = cutlass.Array(cutlass.Float32, cute.cosize(cumsumlog_smem_layout_staged), space=SMEM, alignment=128) + beta_raw = cutlass.Array(cutlass.Float32, cute.cosize(beta_smem_layout_staged), space=SMEM, alignment=128) + sAinv_raw = cutlass.Array( + cfg.io_dtype, + cfg.ainv_cosize, + space=cutlass.AddressSpace.smem, + alignment=cfg.buffer_align_bytes, + ) + sAinv = SmemTile( + base=sAinv_raw.data_ptr().toint(), + elems_per_stage=(cfg.ainv_cosize // cfg.smem_ainv_stages) * bpe, + stages=cfg.smem_ainv_stages, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sV_raw = cutlass.Array( + cfg.io_dtype, + cfg.v_cosize, + space=cutlass.AddressSpace.smem, + alignment=cfg.buffer_align_bytes, + ) + sV = SmemTile( + base=sV_raw.data_ptr().toint(), + elems_per_stage=(cfg.v_cosize // cfg.smem_v_stages) * bpe, + stages=cfg.smem_v_stages, + leading_byte_offset=V_LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sCumsumlog = cute.make_tensor( + cute.make_ptr(cutlass.Float32, cumsumlog_raw.data_ptr().toint(), mem_space=cute.AddressSpace.smem, assumed_align=128), + cumsumlog_smem_layout_staged, + ) + sCumprod = cute.make_tensor( + cute.make_ptr(cutlass.Float32, cumprod_raw.data_ptr().toint(), mem_space=cute.AddressSpace.smem, assumed_align=128), + cumsumlog_smem_layout_staged, + ) + sBeta = cute.make_tensor( + cute.make_ptr(cutlass.Float32, beta_raw.data_ptr().toint(), mem_space=cute.AddressSpace.smem, assumed_align=128), + beta_smem_layout_staged, + ) + + # ---- mbarrier init (all threads) --------------------------------------------- + for s in range(cfg.smem_kq_stages): + bars.mb_kq_ready[s].init() + bars.mb_kq_done[s].init() + for s in range(cfg.smem_v_stages): + bars.mb_v_ready[s].init() + bars.mb_v_done[s].init() + for s in range(cfg.smem_gate_stages): + bars.mb_gate_ready[s].init() + bars.mb_gate_done[s].init() + for s in range(cfg.smem_beta_stages): + bars.mb_beta_ready[s].init() + bars.mb_beta_done[s].init() + for s in range(cfg.tmem_state_acc_stages): + bars.mb_state_acc_ready[s].init() + bars.mb_state_acc_scale_done[s].init() + for s in range(cfg.tmem_cg0_acc_stages): + bars.mb_cg0_acc_ready[s].init() + bars.mb_cg0_acc_done[s].init() + bars.mb_k_state_ready[0].init() + bars.mb_nv_ready[0].init() + for s in range(cfg.smem_ainv_stages): + bars.mb_ainv_ready[s].init() + bars.mb_ainv_done[s].init() + for s in range(cfg.tmem_state_inp_stages): + bars.mb_state_inp_ready[s].init() + for b in (bars.mb_v_k_state_inp_ready, bars.mb_decay_v_inp_ready): + b[0].init() + for s in range(cfg.smem_checkpoint_stages): + bars.mb_checkpoint_tmastg_ready[s].init() + bars.mb_checkpoint_tmastg_done[s].init() + for s_ in range(cfg.sched_stages): + bars.mb_sched_ready[s_].init() + bars.mb_sched_done[s_].init() + bars.mb_tmem_done[0].init() + + nvvm.fence_mbarrier_init() + nvvm.barrier_cta_sync() + + # ---- warp specialization ----------------------------------------------------- + + if warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]: + compute0_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + tidx, + tmem_hold=tmem_hold, + sCumsumlog=sCumsumlog, + sBeta=sBeta, + sAinv=sAinv, + sCheckpoint_raw=sCheckpoint_raw, + checkpoint_every_n_tokens=checkpoint_every_n_tokens, + sSched=sSched, + bars=bars, + ) + + if warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]: + compute1_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + tidx, + warp_idx=warp_idx, + tmem_hold=tmem_hold, + sV=sV, + sCumsumlog=sCumsumlog, + sCumprod=sCumprod, + sBeta=sBeta, + sCheckpoint_raw=sCheckpoint_raw, + mState_init=mState_init, + mState_out=mState_out, + checkpoint_every_n_tokens=checkpoint_every_n_tokens, + sSched=sSched, + bars=bars, + ) + + elif warp_idx == cfg.load_gate_beta_warp_id: + gate_beta_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + tidx=tidx, + mGate=mGate, + mBeta=mBeta, + sCumsumlog=sCumsumlog, + sCumprod=sCumprod, + sBeta=sBeta, + sSched=sSched, + bars=bars, + ) + + elif warp_idx == cfg.mma_warp_id: + mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + tmem_hold=tmem_hold, + sKQ=sKQ, + sKQ_trans=sKQ_trans, + sAinv=sAinv, + sSched=sSched, + bars=bars, + ) + + elif warp_idx == cfg.tma_kv_warp_id: + tmaldg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sKQ_raw=sKQ_raw, + sV_raw=sV_raw, + desc_k_base=desc_k_base, + desc_v_base=desc_v_base, + mSched=mSched, + sSched=sSched, + bars=bars, + ) + + if warp_idx == cfg.epilogue_warp_id: + tmastg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + checkpoint_every_n_tokens=checkpoint_every_n_tokens, + tidx=tidx, + sCheckpoint_raw=sCheckpoint_raw, + desc_checkpoint_base=desc_checkpoint_base, + sSched=sSched, + bars=bars, + ) + + +@dataclass +class GdnRecomputeCfg: + """Per-compile GDN kernel knob, built by ``build_cfg``. + + The per-compile parameters (dtypes, GQA, state flags) are the + ``cute.compile`` cache keys; the rest is derived from the module-global + ``CFG`` constants. ``host`` stamps the shape-derived fields at trace + time. Passed ``cfg``-first (a ``cutlass.Constexpr``) into ``host`` / + ``kernel`` and every warp body. + """ + + io_dtype: Type[cutlass.Numeric] + acc_dtype: Type[cutlass.Numeric] + state_dtype: Type[cutlass.Numeric] + max_active_clusters: int + is_GQA: bool + use_initial_state: bool + store_final_state: bool + enable_checkpoints: bool + split_k: bool = False + log_gate: bool = False + dyn_sched: bool = False + sched_stages: int = CFG.SMEM_SCHED_STAGES + + # ---- fixed constants stamped from CFG by build_cfg --------------------------- + b_t: int = CFG.B_T + d_k: int = CFG.D_K + d_v: int = CFG.D_V + compute_group_0_warp_ids: Tuple[int, ...] = CFG.COMPUTE_GROUP_0_WARP_IDS + compute_group_1_warp_ids: Tuple[int, ...] = CFG.COMPUTE_GROUP_1_WARP_IDS + load_gate_beta_warp_id: int = CFG.LOAD_GATE_BETA_WARP_ID + tma_kv_warp_id: int = CFG.TMA_KV_WARP_ID + mma_warp_id: int = CFG.MMA_WARP_ID + epilogue_warp_id: int = CFG.EPILOGUE_WARP_ID + num_regs_compute_group_0: int = CFG.NUM_REGS_COMPUTE_GROUP_0 + num_regs_compute_group_1: int = CFG.NUM_REGS_COMPUTE_GROUP_1 + num_regs_other: int = CFG.NUM_REGS_OTHER + threads_per_warp: int = CFG.THREADS_PER_WARP + threads_per_cta: int = 0 + cluster_shape_mnk: Tuple[int, int, int] = CFG.CLUSTER_SHAPE_MNK + + # ---- named barrier slots (ids 1-4; 0 is the CTA-wide sync) ------------------- + tmem_alloc_barrier_id: int = 1 + tmem_alloc_barrier_threads: int = 0 + inverse_barrier_id: int = 2 + inverse_barrier_threads: int = 0 + init_state_store_barrier_id: int = 4 + init_state_store_barrier_threads: int = 0 + + # ---- SMEM / TMEM stage counts + TMEM column offsets -------------------------- + smem_kq_stages: int = CFG.SMEM_KQ_STAGES + smem_v_stages: int = CFG.SMEM_V_STAGES + smem_ainv_stages: int = CFG.SMEM_AINV_STAGES + smem_checkpoint_stages: int = 1 + smem_gate_stages: int = CFG.SMEM_GATE_STAGES + smem_beta_stages: int = CFG.SMEM_BETA_STAGES + tmem_state_acc_stages: int = CFG.TMEM_KV_ACC_STAGES + tmem_state_inp_stages: int = CFG.TMEM_STATE_INP_STAGES + tmem_cg0_acc_stages: int = CFG.TMEM_CG0_ACC_STAGES + tmem_cg1_acc_stages: int = CFG.TMEM_CG1_ACC_STAGES + tmem_state_acc_offset: int = 0 + tmem_state_inp_offset: int = 0 + tmem_cg0_acc_offset: int = 0 + tmem_cg1_acc_offset: int = 0 + tmem_v_k_state_decay_inp_offset: int = 0 + buffer_align_bytes: int = CFG.BUFFER_ALIGN_BYTES + + # ---- stamped by host at trace time (shape-derived) -------------------------- + kq_cosize: int = 0 + v_cosize: int = 0 + ainv_cosize: int = 0 + checkpoint_cosize: int = 0 + tma_kq_bytes: int = 0 + tma_v_bytes: int = 0 + n_heads_out: int = 0 + k_ratio: int = 1 + v_ratio: int = 1 + + +def build_cfg( + io_dtype: Type[cutlass.Numeric], + state_dtype: Type[cutlass.Numeric], + *, + max_active_clusters: int, + is_GQA: bool, + use_initial_state: bool, + store_final_state: bool = True, + enable_checkpoints: bool = False, + split_k: bool = False, + log_gate: bool = False, + dyn_sched: bool = False, +) -> GdnRecomputeCfg: + """Build the per-compile ``GdnRecomputeCfg`` (io_dtype ∈ {Float16, BFloat16}; + acc is always Float32).""" + if io_dtype not in (cutlass.Float16, cutlass.BFloat16): + raise ValueError(f"io_dtype={io_dtype} not supported; only Float16 and BFloat16 are supported") + cfg = GdnRecomputeCfg( + io_dtype=io_dtype, + acc_dtype=cutlass.Float32, + state_dtype=state_dtype, + max_active_clusters=max_active_clusters, + is_GQA=is_GQA, + use_initial_state=use_initial_state, + store_final_state=store_final_state, + enable_checkpoints=enable_checkpoints, + split_k=split_k, + log_gate=log_gate, + dyn_sched=dyn_sched, + ) + cfg.smem_checkpoint_stages = 1 + if enable_checkpoints: + cfg.smem_kq_stages = 3 + if not use_initial_state: + cfg.num_regs_compute_group_1 = 232 + cfg.num_regs_other = 48 + n_cg0 = len(cfg.compute_group_0_warp_ids) + n_cg1 = len(cfg.compute_group_1_warp_ids) + cfg.threads_per_cta = cfg.threads_per_warp * (4 + n_cg0 + n_cg1) + cfg.tmem_alloc_barrier_threads = cfg.threads_per_warp * (1 + n_cg0 + n_cg1) + cfg.inverse_barrier_threads = cfg.threads_per_warp * n_cg0 + cfg.init_state_store_barrier_threads = cfg.threads_per_warp * n_cg1 + cfg.tmem_state_acc_offset = 0 + cfg.tmem_state_inp_offset = cfg.tmem_state_acc_offset + cfg.tmem_state_acc_stages * 128 + cfg.tmem_cg0_acc_offset = cfg.tmem_state_inp_offset + cfg.tmem_state_inp_stages * 64 + cfg.tmem_cg1_acc_offset = cfg.tmem_cg0_acc_offset + cfg.tmem_cg0_acc_stages * 64 + cfg.tmem_v_k_state_decay_inp_offset = cfg.tmem_cg1_acc_offset + cfg.tmem_cg1_acc_stages * 64 + return cfg + + +TENSORMAP_DESC_ARRAYS = 3 # per-batch runtime TMA descriptors: K, V, checkpoints +TENSORMAP_STATIC_SLOTS = 0 + + +# --------------------------------------------------------------------------- + + +@functools.cache +def get_compiled_cache( + io_dtype_str: str, + state_dtype_str: str, + cu_dtype_str: str, + HK: int, + HV: int, + HO: int, + is_GQA: bool, + use_initial_state: bool, + store_final_state: bool, + enable_checkpoints: bool, + split_k: bool, + log_gate: bool, + dyn_sched: bool, +): + """Return a mutable dict that lazily stores the compiled kernel.""" + return {} + + +def compile( + io_dtype, + state_dtype, + is_GQA: bool, + use_initial_state: bool, + store_final_state: bool, + enable_checkpoints: bool, + split_k: bool = False, + log_gate: bool = False, + dyn_sched: bool = False, + *, + num_sm: int, + k_cute, + v_cute, + gate_cute, + beta_cute, + cu_seqlens_cute, + state_in_cute, + state_out_cute, + work_items_cute=None, + work_count_cute=None, + sched_ctr_cute=None, + checkpoint_every_n_tokens, + workspace_cute, + stream, +): + """JIT-compile the chunked GDN recompute kernel for one static config.""" + cfg = build_cfg( + io_dtype, + state_dtype, + max_active_clusters=num_sm, + is_GQA=is_GQA, + use_initial_state=use_initial_state, + store_final_state=store_final_state, + enable_checkpoints=enable_checkpoints, + split_k=split_k, + log_gate=log_gate, + dyn_sched=dyn_sched, + ) + + return cute.compile( + host, + cfg, + k_cute, + v_cute, + gate_cute, + beta_cute, + cu_seqlens_cute, + state_in_cute, + state_out_cute, + work_items_cute, + work_count_cute, + sched_ctr_cute, + checkpoint_every_n_tokens, + workspace_cute, + stream, + options="--enable-tvm-ffi --opt-level 3", + ) + + +def chunk_gdn_recompute_sm100( + k, + v, + gate, + beta, + cu_seqlens, + initial_state, + output_state, + checkpoint_every_n_tokens: int = 0, + output_state_checkpoints=None, + work_items=None, + work_count=None, + sched_ctr=None, + log_gate: bool = False, + *, + workspace, + stream, +) -> None: + """Execute the Blackwell chunked GDN recompute kernel (state/checkpoint-only, + THD / varlen entry). + + All tensors are contiguous, DLPack-compatible CUDA tensors on the same + device. Compile-cache-and-replay: the kernel is compiled once per static + config (dtypes, head counts, state flags) and replayed afterwards. + + Args: + k: ``(total_tokens, HK, DK)`` float16/bfloat16 + v: ``(total_tokens, HV, DK)`` float16/bfloat16 + gate: ``(total_tokens, HO)`` float32, forget gate — raw linear + alpha, or the natural-log decay when ``log_gate`` + beta: ``(total_tokens, HO)`` float32, update gate + cu_seqlens: ``(num_seqs + 1,)`` int32 + initial_state: ``(num_seqs, HO, DK, DK)`` float32/bfloat16, or None + output_state: ``(num_seqs, HO, DK, DK)`` float32/bfloat16, or None + checkpoint_every_n_tokens: emit a checkpoint entry every N tokens (0 = off) + output_state_checkpoints: ``(total_checkpoints, HO, DK, DK)`` bfloat16, or None. Entry j is the + state after ``(j + 1) * N`` tokens, STRICTLY BEFORE the sequence + end -- the end-of-sequence state is only ``output_state`` + (fp32-capable). With ``N == B_T`` this is the per-chunk checkpoint + series the backward pass consumes. + work_items: ``(max_items, 8)`` int32 split-K work-item table from + ``common/split_k.py``, or None for the one-tile-per-(b,h) + layout. With a table, each item computes chunks + ``[cstart, wend)`` and writes checkpoints only for ``[wstart, wend)``. + work_count: ``(1,)`` int32 device-side item count (required with + work_items) + log_gate: ``gate`` holds natural-log decay values; the gate warp + skips its log2 (rescales by 1/ln2) instead of exponentiating + upstream + workspace: ``(>= tensormap_workspace_bytes(module, B) // 8,)`` int64, + 128-byte aligned; holds the per-(b,h) TMA descriptors (contents + managed here — reuse the same buffer across calls) + stream: CUDA stream handle (``cudaStream_t`` as an int) + """ + HK = k.shape[1] + HV = v.shape[1] + HO = gate.shape[1] + DK = k.shape[2] + B = cu_seqlens.shape[0] - 1 + is_GQA = HK >= HV + use_initial_state = initial_state is not None + store_final_state = output_state is not None + enable_checkpoints = checkpoint_every_n_tokens > 0 + split_k = work_items is not None + dyn_sched = sched_ctr is not None + if not (enable_checkpoints or store_final_state): + raise ValueError("output_state_checkpoints or output_state is required") + io_dtype = get_dtype(k.dtype) + + if initial_state is not None: + state_dtype_src = initial_state.dtype + elif output_state is not None: + state_dtype_src = output_state.dtype + else: + state_dtype_src = None + state_dtype = get_dtype(state_dtype_src) if state_dtype_src is not None else cutlass.Float32 + + cu_stream = cuda.CUstream(int(stream)) + + cache = get_compiled_cache( + str(k.dtype), + str(state_dtype_src), + str(cu_seqlens.dtype), + HK, + HV, + HO, + is_GQA, + use_initial_state, + store_final_state, + enable_checkpoints, + split_k, + log_gate, + dyn_sched, + ) + + if "compiled" not in cache: + k_cute = from_dlpack(k, assumed_align=16) + k_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) + v_cute = from_dlpack(v, assumed_align=16) + v_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) + gate_cute = from_dlpack(gate, assumed_align=16) + gate_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) + beta_cute = from_dlpack(beta, assumed_align=16) + beta_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) + cu_seqlens_cute = from_dlpack(cu_seqlens, assumed_align=8 if str(cu_seqlens.dtype).endswith("int64") else 4).mark_layout_dynamic() + + state_in_cute = None + if use_initial_state: + state_in_cute = from_dlpack(initial_state, assumed_align=16) + state_in_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=DK) + + state_out_cute = None + if store_final_state: + state_out_cute = from_dlpack(output_state, assumed_align=16) + state_out_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=DK) + + workspace_cute = from_dlpack(workspace, assumed_align=128).mark_layout_dynamic() + + work_items_cute = None + work_count_cute = None + if split_k: + work_items_cute = from_dlpack(work_items, assumed_align=16) + work_items_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) + work_count_cute = from_dlpack(work_count, assumed_align=4).mark_layout_dynamic() + + sched_ctr_cute = None + if dyn_sched: + sched_ctr_cute = from_dlpack(sched_ctr, assumed_align=4).mark_layout_dynamic() + + cache["compiled"] = compile( + io_dtype, + state_dtype, + is_GQA, + use_initial_state, + store_final_state, + enable_checkpoints, + split_k, + log_gate, + dyn_sched, + num_sm=multiprocessor_count(current_device_id()), + k_cute=k_cute, + v_cute=v_cute, + gate_cute=gate_cute, + beta_cute=beta_cute, + cu_seqlens_cute=cu_seqlens_cute, + state_in_cute=state_in_cute, + state_out_cute=state_out_cute, + work_items_cute=work_items_cute, + work_count_cute=work_count_cute, + sched_ctr_cute=sched_ctr_cute, + checkpoint_every_n_tokens=checkpoint_every_n_tokens, + workspace_cute=workspace_cute, + stream=cu_stream, + ) + + compiled = cache["compiled"] + + # desc build runs every execute by contract (cu contents are data; + # buffer pointers may change) — capture-safe, single tiny launch + if "build_descs" not in cache: + k_bc = from_dlpack(k, assumed_align=16) + k_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) + v_bc = from_dlpack(v, assumed_align=16) + v_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) + cu_bc = from_dlpack(cu_seqlens, assumed_align=8 if str(cu_seqlens.dtype).endswith("int64") else 4).mark_layout_dynamic() + checkpoints_bc = None + cu_ckpt_bc = None + if enable_checkpoints: + checkpoints_bc = from_dlpack(output_state_checkpoints, assumed_align=16) + checkpoints_bc.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2, 3), divisibility=1) + ws_bc = from_dlpack(workspace, assumed_align=128).mark_layout_dynamic() + cache["build_descs"] = cute.compile( + build_descs, + io_dtype, + CFG.B_T, + k_bc, + v_bc, + cu_bc, + checkpoints_bc, + cutlass.Int32(checkpoint_every_n_tokens if enable_checkpoints else 1), + ws_bc, + cu_stream, + options="--enable-tvm-ffi", + ) + cache["build_descs"]( + k, + v, + cu_seqlens, + output_state_checkpoints, + checkpoint_every_n_tokens if enable_checkpoints else 1, + workspace, + cu_stream, + ) + compiled( + k, + v, + gate, + beta, + cu_seqlens, + initial_state, + output_state, + work_items, + work_count, + sched_ctr, + checkpoint_every_n_tokens, + workspace, + cu_stream, + ) diff --git a/python/cudnn/linear_attention/frost/kernel/kda_bprop_config.py b/python/cudnn/linear_attention/frost/kernel/kda_bprop_config.py index 7814626c5..d832fc85e 100644 --- a/python/cudnn/linear_attention/frost/kernel/kda_bprop_config.py +++ b/python/cudnn/linear_attention/frost/kernel/kda_bprop_config.py @@ -15,23 +15,56 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Kimi Delta Attention (KDA) Cutlass DSL backward kernel config — STUB. - -The FROST KDA backward is not implemented yet (see ``kda_bprop_f16.py``); -these constants mirror the prefill config's tile shape for when the backward -kernel lands. +"""Kimi Delta Attention (KDA) Cutlass DSL backward kernel config (fixed +compile-time constants). The BT=16 backward mirrors the prefill's 16-warp +(512-thread) specialization; the derived SMEM/TMEM sizes and offsets are +stamped by ``build_cfg`` in ``kda_bprop_f16.py``. Target arch: Blackwell SM100 (GB200) / SM103 (GB300). """ from dataclasses import dataclass +from typing import Tuple @dataclass(frozen=True) class Cfg: - B_T: int = 16 - D_K: int = 128 - D_V: int = 128 + # --- tile shape --- + B_T: int = 16 # chunk-inner token tile (BT=16 KDA schedule) + D_K: int = 128 # query/key head dim + D_V: int = 128 # value head dim + + # --- warp assignments (16 warps = 512 threads) --- + COMPUTE_GROUP_0_WARP_IDS: Tuple[int, ...] = (0, 1, 2, 3) # forward gate cumsum + decay-operand materialize + COMPUTE_GROUP_1_WARP_IDS: Tuple[int, ...] = (4, 5, 6, 7) # value-side TMEM staging / restages / dH capture + COMPUTE_GROUP_2_WARP_IDS: Tuple[int, ...] = (8, 9, 10, 11) # dq/dk-bank drain, dG assembly + reverse cumsum + SUPER_MMA_WARP_ID: int = 12 # register-MMA kk/qk/dA/dM + Neumann inverse + TCGEN05_MMA_WARP_ID: int = 13 # tcgen05 GEMM schedule + TMA_WARP_ID: int = 14 # q/k/v/gate/do/S(H) TMA loads + EPILOGUE_WARP_ID: int = 15 # dq/dk/dv TMA stores only + + # --- register split --- + NUM_REGS_COMPUTE_GROUP_0: int = 144 + NUM_REGS_COMPUTE_GROUP_1: int = 168 + NUM_REGS_COMPUTE_GROUP_2: int = 144 + NUM_REGS_OTHER: int = 56 + + THREADS_PER_WARP: int = 32 + + BUFFER_ALIGN_BYTES: int = 1024 + + # --- SMEM / TMEM ring stage counts --- + SMEM_RAW_STAGES: int = 2 + SMEM_S_STAGES: int = 1 + SMEM_DECAY_STAGES: int = 2 + SMEM_QK_STAGES: int = 2 + SMEM_STATE_SCALE_DIAG_STAGES: int = 2 + SMEM_DQ_STAGES: int = 1 + SMEM_DK_STAGES: int = 1 + SMEM_DG_STAGES: int = 1 + SMEM_DV_STAGES: int = 2 + + CLUSTER_SHAPE_MNK: Tuple[int, int, int] = (1, 1, 1) CFG = Cfg() diff --git a/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py b/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py index fd4c8a513..0100f93db 100644 --- a/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py @@ -1,41 +1,3822 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# This kernel is derived from cuDNN, NVIDIA Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Kimi Delta Attention (KDA) Cutlass DSL backward kernel — STUB. - -The FROST KDA backward is not implemented yet. Because the KDA chunk size is -small, the design recomputes the forward per-chunk states inside the backward -(no H store in the prefill kernel). Until the backward kernel lands, this -module raises ``NotImplementedError``; the cuTile KDA engine -(``cudnn.engines.KdaCuTileEngine``) provides KDA gradients. - -Target arch: Blackwell SM100 (GB200) / SM103 (GB300). +"""Chunked Kimi Delta Attention (KDA) BPROP kernel for Blackwell SM100/SM103 +(Cutlass DSL), BT=16 tiling with a per-key-channel decay. Framework-neutral +entry ``chunk_kda_bwd_sm100``. + +Algorithm overview (per chunk c, iterated c = NT-1 .. 0; within-chunk +log2-domain gate cumsum G[t,d], eG = 2^G, eGl = 2^G[BT-1]): + Inputs : Q/K[BT,DK], V/dO[BT,DV], S = state_checkpoints[c-1] (state ENTERING chunk c, KV), + Gate[BT,DK], Beta[BT] + State : dH[DV,DK] (state gradient, fp32 TMEM, accumulated backward) + + Operands (WG0, prefill recompute): K_decay = eG.K, K_inv = K/eG, + K_restore = (eGl/eG).K, Q_decay = eG.Q, diag(eGl). + + Forward recompute: A_kk = K_decay@K_inv^T; A_inv = (I + Beta.tril(A_kk,-1))^-1 + (register Neumann); A_qk = tril_incl(Q_decay@K_inv^T); W = Beta.(V - S^T K_decay); + U = A_inv@W. + + Backward math: + dU = K_restore@dH + A_qk^T@dO (Q_decay carries scale, so A_qk does too) + dY = A_inv^T@dU dV = Beta.dY + dA = tril_incl(dO@U^T) (unscaled) dM = dY@U^T dM_strict = Beta_row.strict(dM) + dQ = eG.scale.(dO@S^T + dA@K_inv) + dK = eG.dK_decay + dK_inv/eG + (eGl/eG).dK_restore where (sign-flipped parts) + dK_decay part = (Beta.dY)@S^T + dM_strict@K_inv (= -dK_decay) + dK_inv part = dA^T@(scale.Q_decay) - dM_strict^T@K_decay (= dK_inv; one TMEM + acc, the minus rides the staged -dM_strict tile) + dK_restore part = U@dH^T (= +dK_restore) + dBeta = sum_v dY.W / Beta - sum_{j= 1; chunk 0 seeds from `initial_state`); dq/dk/dv io at HO heads; dg `[T, HO, DK]` fp32 +(natural-log gate domain); dbeta `[T, HO]` fp32; d_initial_state / +d_final_state fp32 `[N, HO, DK, DV]` (K-major). + +Warp assignments (16 warps = 512 threads): + warps 0-3 : WG0 - Gate prefix scan + decay/restore operands (all chunks) + warps 4-7 : WG1 - state SMEM->TMEM copy, value-side TMEM staging, dstate capture, dBeta + warps 8-11 : WG2 - dQ/dK part drain, dG assembly, reverse cumsum, dGate + warp 12 : super-MMA - register KK/QK/dA/dM + Neumann inverse + warp 13 : tcgen05-MMA - the 15-GEMM backward schedule + warp 14 : TMA load - Q/K/V/Gate/dO/entering-state loads + Beta scalars + warp 15 : epilogue - dQ/dK/dV/dGate TMA stores """ -from __future__ import annotations +from dataclasses import dataclass +from functools import lru_cache +from typing import NamedTuple, Optional, Type + +import cuda.bindings.driver as cuda_driver +import cutlass +import cutlass.experimental.cuda as cuda +import cutlass.experimental.primitives as nvvm +import cutlass.cute as cute +from cutlass.cute.runtime import from_dlpack + +from ..common.split_k import decode_work_item +from ..common.host import get_dtype +from cudnn.frost.buffers import current_device_id, data_ptr +from cudnn.frost.device import multiprocessor_count +from ..common.thd import TENSOR_MAP_QWORDS, emit_copy_desc, emit_checkpoint_seq_descs, emit_seq_descs +from .kda_bprop_config import CFG +from cudnn.frost.tile_dsl.barrier import ( + advance, + MBarrier, + PipelineState, + Producer, +) +from cudnn.frost.tile_dsl.handles import MmaDesc, SmemTile, tma_slice_runtime_desc +from cudnn.frost.tile_dsl.mma import mma_ss, mma_step, mma_ts_step +from cudnn.frost.tile_dsl.swizzle import swizzle_lin_S, swizzle_xor_128b +from cudnn.frost.tile_dsl.tma import tma_load_tile, tma_store_commit, tma_store_tile, tma_store_wait, tma_tensormap_acquire +from cudnn.frost.tile_dsl.pointwise import ( + opaque_f32_zero, + f16x2_to_f32, + fmul2, + ffma2, + movmatrix_16b, + mul_f16x2, + fp32_to_fp16, + sub_f16x2, +) + +LOG2_E: float = 1.4426950408889634 + +L2_NORM_EPS: float = 1.0e-12 + + +class KdaBwdBars(NamedTuple): + """Every inter-warp handoff as an ``MBarrier`` over its ring. Consumers + track ``(idx, phase)`` inline; the producer tag selects the arrive + lowering (``TMA_LOAD``/``MMA_COMMIT``/``THREAD``). + + Buffers read by both the MMA warp and a compute/warp group carry mixed + arrive counts (one MMA commit + N thread arrivers) so the producer only + reuses the slot once every reader is done.""" + + mb_q_ready: MBarrier + mb_q_done: MBarrier + mb_k_ready: MBarrier + mb_k_done: MBarrier + mb_gate_ready: MBarrier + mb_gate_done: MBarrier + mb_do_ready: MBarrier + mb_do_done: MBarrier + mb_do_epi_done: MBarrier + mb_v_ready: MBarrier + mb_v_done: MBarrier + mb_state_ready: MBarrier + mb_state_done: MBarrier + mb_state_cg0_done: MBarrier + + mb_beta_ready: MBarrier + mb_beta_done: MBarrier + + mb_state_k_acc_ready: MBarrier + mb_du_acc_ready: MBarrier + mb_update_acc_ready: MBarrier + mb_dy_acc_ready: MBarrier + mb_dq_acc_ready: MBarrier + mb_dk_decay_part_ready: MBarrier + mb_dk_inv_part_ready: MBarrier + mb_dk_restore_part_ready: MBarrier + mb_parts_done: MBarrier + + mb_state_inp_ready: MBarrier + mb_state_inp_done: MBarrier + mb_state_inp_cg2_done: MBarrier + mb_rhs_ready: MBarrier + mb_du_inp_ready: MBarrier + mb_du_inp_done: MBarrier + mb_neg_beta_dy_inp_ready: MBarrier + mb_neg_beta_dy_inp_done: MBarrier + + mb_k_decay_inv_ready: MBarrier + mb_q_decay_k_restore_ready: MBarrier + mb_decay_done: MBarrier + mb_decay_super_done: MBarrier + mb_a_ready: MBarrier + mb_a_done: MBarrier + mb_aqk_ready: MBarrier + mb_aqk_done: MBarrier + mb_da_ready: MBarrier + mb_da_done: MBarrier + mb_dm_ready: MBarrier + mb_dm_done: MBarrier + mb_u_smem_ready: MBarrier + mb_u_done: MBarrier + mb_u_warps_done: MBarrier + mb_sdy_ready: MBarrier + mb_sdy_done: MBarrier + mb_dbeta_m_ready: MBarrier + mb_dbeta_m_done: MBarrier + + mb_dstate_acc_ready: MBarrier + mb_dstate_acc_done: MBarrier + mb_dstate_inp_ready: MBarrier + mb_dstate_inp_done: MBarrier + mb_dstate_smem_ready: MBarrier + mb_dstate_smem_done: MBarrier + mb_dstate_smem_cg2_done: MBarrier + + mb_dq_tmastg_ready: MBarrier + mb_dq_tmastg_done: MBarrier + mb_dk_tmastg_ready: MBarrier + mb_dk_tmastg_done: MBarrier + mb_dv_tmastg_ready: MBarrier + mb_dv_tmastg_done: MBarrier + mb_dv_epi_done: MBarrier + mb_dg_tmastg_ready: MBarrier + mb_dg_tmastg_done: MBarrier + + mb_dstate0_stored: MBarrier + + mb_sched_ready: MBarrier + mb_sched_done: MBarrier + + +def make_kda_bwd_bars(cfg) -> KdaBwdBars: + """Bars factory. MUST be called from inside ``kernel`` (allocates the + mbarrier rings in SMEM ahead of the data buffers).""" + + def alloc(n): + return cutlass.Array(cutlass.Int64, n, space=cutlass.AddressSpace.smem, alignment=8) + + WARP = cfg.threads_per_warp + CG0 = len(cfg.compute_group_0_warp_ids) * WARP + CG2 = len(cfg.compute_group_2_warp_ids) * WARP + CG1 = len(cfg.compute_group_1_warp_ids) * WARP + MMA = 1 + + return KdaBwdBars( + mb_q_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_q_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0 + CG2, producer=Producer.THREAD), + mb_k_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_k_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0 + CG2, producer=Producer.THREAD), + mb_gate_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_gate_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0 + CG2, producer=Producer.THREAD), + mb_do_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_do_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_do_epi_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=WARP, producer=Producer.THREAD), + mb_v_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_v_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG1, producer=Producer.THREAD), + mb_state_ready=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_state_done=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_state_cg0_done=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=CG0, producer=Producer.THREAD), + mb_beta_ready=MBarrier(alloc(cfg.smem_beta_stages), stages=cfg.smem_beta_stages, init_count=WARP, producer=Producer.THREAD), + mb_beta_done=MBarrier(alloc(cfg.smem_beta_stages), stages=cfg.smem_beta_stages, init_count=WARP + CG1, producer=Producer.THREAD), + mb_state_k_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_du_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_update_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dy_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dq_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dk_decay_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dk_inv_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dk_restore_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_parts_done=MBarrier(alloc(1), stages=1, init_count=CG2, producer=Producer.THREAD), + mb_state_inp_ready=MBarrier(alloc(2), stages=2, init_count=CG0, producer=Producer.THREAD), + mb_state_inp_done=MBarrier(alloc(2), stages=2, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_state_inp_cg2_done=MBarrier(alloc(2), stages=2, init_count=CG2, producer=Producer.THREAD), + mb_rhs_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_du_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_du_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_neg_beta_dy_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_neg_beta_dy_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_k_decay_inv_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0, producer=Producer.THREAD), + mb_q_decay_k_restore_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0, producer=Producer.THREAD), + mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=2 * WARP, producer=Producer.THREAD), + mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_a_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_aqk_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_aqk_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_da_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_da_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dm_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_dm_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_u_smem_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_u_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_u_warps_done=MBarrier(alloc(1), stages=1, init_count=2 * WARP, producer=Producer.THREAD), + mb_sdy_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_sdy_done=MBarrier(alloc(1), stages=1, init_count=WARP, producer=Producer.THREAD), + mb_dbeta_m_ready=MBarrier(alloc(1), stages=1, init_count=WARP, producer=Producer.THREAD), + mb_dbeta_m_done=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_dstate_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dstate_acc_done=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_dstate_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_dstate_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dstate_smem_ready=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_dstate_smem_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dstate_smem_cg2_done=MBarrier(alloc(1), stages=1, init_count=CG2, producer=Producer.THREAD), + mb_dq_tmastg_ready=MBarrier(alloc(cfg.smem_dq_stages), stages=cfg.smem_dq_stages, init_count=CG2, producer=Producer.THREAD), + mb_dq_tmastg_done=MBarrier(alloc(cfg.smem_dq_stages), stages=cfg.smem_dq_stages, init_count=WARP, producer=Producer.THREAD), + mb_dk_tmastg_ready=MBarrier(alloc(cfg.smem_dk_stages), stages=cfg.smem_dk_stages, init_count=CG2, producer=Producer.THREAD), + mb_dk_tmastg_done=MBarrier(alloc(cfg.smem_dk_stages), stages=cfg.smem_dk_stages, init_count=WARP, producer=Producer.THREAD), + mb_dv_tmastg_ready=MBarrier(alloc(cfg.smem_dv_stages), stages=cfg.smem_dv_stages, init_count=CG1, producer=Producer.THREAD), + mb_dv_tmastg_done=MBarrier(alloc(cfg.smem_dv_stages), stages=cfg.smem_dv_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dv_epi_done=MBarrier(alloc(cfg.smem_dv_stages), stages=cfg.smem_dv_stages, init_count=WARP, producer=Producer.THREAD), + mb_dg_tmastg_ready=MBarrier(alloc(cfg.smem_dg_stages), stages=cfg.smem_dg_stages, init_count=CG2, producer=Producer.THREAD), + mb_dg_tmastg_done=MBarrier(alloc(cfg.smem_dg_stages), stages=cfg.smem_dg_stages, init_count=WARP, producer=Producer.THREAD), + mb_dstate0_stored=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_sched_ready=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=1, producer=Producer.THREAD), + mb_sched_done=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=15, producer=Producer.THREAD), + ) + + +# ---- Dynamic tile scheduler ------------------------------------------------------ + + +@cute.jit +def sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): + """TMA-warp side: pull the next tile off the global ticket, publish it.""" + if cutlass.const_expr(cfg.dyn_sched): + bars.mb_sched_done[sched_state.idx].wait(sched_state.phase) + if nvvm.elect_sync(): + fetched = cutlass.Int32(nvvm.atomicrmw("add", mSched.iterator, cutlass.Int32(1), mem_order="relaxed", syncscope="gpu")) + sSched[sched_state.idx] = num_ctas + fetched + nvvm.bar_warp_sync(cute.arch.FULL_MASK) + next_tile = sSched[sched_state.idx] + if nvvm.elect_sync(): + bars.mb_sched_ready[sched_state.idx].arrive() + return next_tile, advance(sched_state, cfg.sched_stages) + return tile_idx + num_ctas, sched_state + +@cute.jit +def sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): + """Consumer side: read the TMA warp's published next tile.""" + if cutlass.const_expr(cfg.dyn_sched): + bars.mb_sched_ready[sched_state.idx].wait(sched_state.phase) + next_tile = sSched[sched_state.idx] + if nvvm.elect_sync(): + bars.mb_sched_done[sched_state.idx].arrive() + return next_tile, advance(sched_state, cfg.sched_stages) + return tile_idx + num_ctas, sched_state -def get_workspace_size(B: int, HQ: int, HV: int) -> int: - return 0 +# ---- Warp bodies ----------------------------------------------------------------- -def chunk_kda_bwd_sm100(*args, **kwargs) -> None: - """Not implemented — the FROST KDA backward kernel is a stub.""" - raise NotImplementedError( - "FROST KDA backward is not implemented yet (recompute-in-bprop kernel is a stub); " - "use the cuTile KDA engine (cudnn.engines.KdaCuTileEngine) for KDA gradients." + +@cute.jit +def epilogue_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_inv_raw, + sQ_decay_raw, + sDo_raw, + sU_raw, + sQk_raw, + sDq_raw, + sDk_raw, + sDv_raw, + sDg_raw, + desc_dq_base, + desc_dk_base, + desc_dv_base, + desc_dg_base, + bars, +) -> None: + """Epilogue warp role (warp 15): the register-MMA A_qk/dA tiles and the + dQ/dK/dV/dGate TMA stores.""" + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + + # ---- ldmatrix/stmatrix lane decode ------------------------------------------- + rhs_row_coord = lane % 8 + (cutlass.Int32(8) if (lane // 16) else cutlass.Int32(0)) + rhs_col_offset = cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0) + lhs_row_coord = lane % 8 + (cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0)) + lhs_col_offset = cutlass.Int32(8) if ((lane // 8) // 2) else cutlass.Int32(0) + stsm_row_coord = lane & 7 + stsm_col_coord = cutlass.Int32(0) + if (lane // 8) & 1: + stsm_row_coord = stsm_row_coord + cutlass.Int32(8) + if lane // 8 >= 2: + stsm_col_coord = cutlass.Int32(8) + stsm_idx = swizzle_lin_S(stsm_row_coord * cfg.b_t + stsm_col_coord, bbits=1, mbase=3, sshift=3) + row_lo = lane // 4 + row_hi = row_lo + cutlass.Int32(8) + raw_index = PipelineState.start(phase=0) + u_index = PipelineState.start(phase=0) + chunk_serial_base = cutlass.Int32(0) + + sDq_tma = SmemTile( + base=sDq_raw, + elems_per_stage=(cfg.b_t * cfg.d_k), + stages=cfg.smem_dq_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.b_t * 64, + ) + sDk_tma = SmemTile( + base=sDk_raw, + elems_per_stage=(cfg.b_t * cfg.d_k), + stages=cfg.smem_dk_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.b_t * 64, + ) + sDv_tma = SmemTile( + base=sDv_raw, + elems_per_stage=(cfg.b_t * cfg.d_v), + stages=cfg.smem_dv_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.b_t * 64, + ) + sDg_tma = SmemTile( + base=sDg_raw, + elems_per_stage=(cfg.b_t * cfg.d_k), + stages=cfg.smem_dg_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 32), + tma_granu_elems=32, + tma_subtile_stride_elems=cfg.b_t * 32, + ) + dq_index = PipelineState.start(phase=0) + dk_index = PipelineState.start(phase=0) + dv_index = PipelineState.start(phase=0) + dg_index = PipelineState.start(phase=0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + head_o = head_idx + slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_dq_slot = (desc_dq_base + slot).tospace(cutlass.AddressSpace.generic) + desc_dk_slot = (desc_dk_base + slot).tospace(cutlass.AddressSpace.generic) + desc_dv_slot = (desc_dv_base + slot).tospace(cutlass.AddressSpace.generic) + desc_dg_slot = (desc_dg_base + slot).tospace(cutlass.AddressSpace.generic) + if nvvm.elect_sync(): + tma_tensormap_acquire(desc_dq_slot) + tma_tensormap_acquire(desc_dk_slot) + tma_tensormap_acquire(desc_dv_slot) + tma_tensormap_acquire(desc_dg_slot) + num_compute_chunks = cend - wstart + pend_start = cutlass.Int32(0) + pend_writes = cutlass.Boolean(False) + for rev_idx in cutlass.range(num_compute_chunks, unroll=1): + chunk_idx = cend - cutlass.Int32(1) - rev_idx + chunk_start = chunk_idx * cfg.b_t + writes = chunk_idx < wend + chunk_serial = chunk_serial_base + rev_idx + decay_stage = chunk_serial % cfg.smem_decay_stages + qk_stage = chunk_serial % cfg.smem_qk_stages + raw_stage = raw_index.idx + sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sQ_decay_ptr = sQ_decay_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sDo_ptr = sDo_raw.data_ptr() + raw_stage * (cfg.d_v * cfg.b_t) + sQk_ptr = sQk_raw.data_ptr() + qk_stage * (cfg.qk_tiles * cfg.b_t * cfg.b_t) + + # ---- A_qk = tril_incl(Q_decay @ K_inv^T) ----------------------------- + bars.mb_aqk_done[qk_stage].wait(((chunk_serial // cfg.smem_qk_stages) + 1) % 2) + bars.mb_q_decay_k_restore_ready[decay_stage].wait((chunk_serial // cfg.smem_decay_stages) % 2) + qk_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + qk_acc[accum_idx] = cutlass.Float32(0.0) + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + a_col = k_block * 16 + lhs_col_offset + a_seg = a_col // 64 + a_vec = nvvm.ldmatrix( + sQ_decay_ptr + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + b_col = k_block * 16 + rhs_col_offset + b_seg = b_col // 64 + b_vec = nvvm.ldmatrix( + sK_inv_ptr + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + mma_step( + qk_acc, + (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), + (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + qk_acc[accum_idx] = qk_acc[accum_idx] if row_coord >= col_coord else cutlass.Float32(0.0) + nvvm.stmatrix( + sQk_ptr + stsm_idx, + [ + fp32_to_fp16(qk_acc[0], qk_acc[1], dtype=cfg.io_dtype), + fp32_to_fp16(qk_acc[2], qk_acc[3], dtype=cfg.io_dtype), + fp32_to_fp16(qk_acc[4], qk_acc[5], dtype=cfg.io_dtype), + fp32_to_fp16(qk_acc[6], qk_acc[7], dtype=cfg.io_dtype), + ], + nvvm.MMALayout.ROW, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_aqk_ready[qk_stage].arrive() + bars.mb_decay_super_done[decay_stage].arrive() + + # ---- dA = tril_incl(dO @ U^T) ---------------------------------------- + bars.mb_da_done[qk_stage].wait(((chunk_serial // cfg.smem_qk_stages) + 1) % 2) + bars.mb_u_smem_ready.wait(u_index.phase) + u_index = advance(u_index, 1) + da_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + da_acc[accum_idx] = cutlass.Float32(0.0) + for k_block in cutlass.range_constexpr(cfg.d_v // 16): + a_col = k_block * 16 + lhs_col_offset + a_seg = a_col // 64 + a_vec = nvvm.ldmatrix( + sDo_ptr + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + b_col = k_block * 16 + rhs_col_offset + b_seg = b_col // 64 + b_vec = nvvm.ldmatrix( + sU_raw.data_ptr() + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + mma_step( + da_acc, + (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), + (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + bars.mb_u_warps_done.arrive() + bars.mb_do_epi_done[raw_stage].arrive() + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + da_acc[accum_idx] = da_acc[accum_idx] if row_coord >= col_coord else cutlass.Float32(0.0) + nvvm.stmatrix( + sQk_ptr + 2 * (cfg.b_t * cfg.b_t) + stsm_idx, + [ + fp32_to_fp16(da_acc[0], da_acc[1], dtype=cfg.io_dtype), + fp32_to_fp16(da_acc[2], da_acc[3], dtype=cfg.io_dtype), + fp32_to_fp16(da_acc[4], da_acc[5], dtype=cfg.io_dtype), + fp32_to_fp16(da_acc[6], da_acc[7], dtype=cfg.io_dtype), + ], + nvvm.MMALayout.ROW, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_da_ready[qk_stage].arrive() + raw_index = advance(raw_index, cfg.smem_raw_stages) + + # ---- dQ/dK/dGate/dV: previous chunk, one-behind store ladder --------- + if rev_idx > 0: + bars.mb_dq_tmastg_ready[dq_index.idx].wait(dq_index.phase) + bars.mb_dk_tmastg_ready[dk_index.idx].wait(dk_index.phase) + bars.mb_dg_tmastg_ready[dg_index.idx].wait(dg_index.phase) + bars.mb_dv_tmastg_ready[dv_index.idx].wait(dv_index.phase) + if pend_writes: + dq_slice = tma_slice_runtime_desc(desc_dq_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDq_tma[dq_index.idx], dq_slice, acquire=False) + tma_store_commit() + dk_slice = tma_slice_runtime_desc(desc_dk_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDk_tma[dk_index.idx], dk_slice, acquire=False) + tma_store_commit() + dg_slice = tma_slice_runtime_desc(desc_dg_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDg_tma[dg_index.idx], dg_slice, acquire=False) + tma_store_commit() + dv_slice = tma_slice_runtime_desc(desc_dv_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDv_tma[dv_index.idx], dv_slice, acquire=False) + tma_store_commit() + tma_store_wait(3) + bars.mb_dq_tmastg_done[dq_index.idx].arrive() + tma_store_wait(2) + bars.mb_dk_tmastg_done[dk_index.idx].arrive() + tma_store_wait(1) + bars.mb_dg_tmastg_done[dg_index.idx].arrive() + tma_store_wait(0) + bars.mb_dv_epi_done[dv_index.idx].arrive() + dq_index = advance(dq_index, cfg.smem_dq_stages) + dk_index = advance(dk_index, cfg.smem_dk_stages) + dg_index = advance(dg_index, cfg.smem_dg_stages) + dv_index = advance(dv_index, cfg.smem_dv_stages) + pend_start = chunk_start + pend_writes = writes + + # ---- tile tail: drain the last chunk's dQ/dK/dGate/dV -------------------- + if num_compute_chunks > 0: + bars.mb_dq_tmastg_ready[dq_index.idx].wait(dq_index.phase) + bars.mb_dk_tmastg_ready[dk_index.idx].wait(dk_index.phase) + bars.mb_dg_tmastg_ready[dg_index.idx].wait(dg_index.phase) + bars.mb_dv_tmastg_ready[dv_index.idx].wait(dv_index.phase) + if pend_writes: + dq_slice = tma_slice_runtime_desc(desc_dq_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDq_tma[dq_index.idx], dq_slice, acquire=False) + tma_store_commit() + dk_slice = tma_slice_runtime_desc(desc_dk_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDk_tma[dk_index.idx], dk_slice, acquire=False) + tma_store_commit() + dg_slice = tma_slice_runtime_desc(desc_dg_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDg_tma[dg_index.idx], dg_slice, acquire=False) + tma_store_commit() + dv_slice = tma_slice_runtime_desc(desc_dv_slot, cutlass.Int32(0), head_o, pend_start) + tma_store_tile(sDv_tma[dv_index.idx], dv_slice, acquire=False) + tma_store_commit() + tma_store_wait(3) + bars.mb_dq_tmastg_done[dq_index.idx].arrive() + tma_store_wait(2) + bars.mb_dk_tmastg_done[dk_index.idx].arrive() + tma_store_wait(1) + bars.mb_dg_tmastg_done[dg_index.idx].arrive() + tma_store_wait(0) + bars.mb_dv_epi_done[dv_index.idx].arrive() + dq_index = advance(dq_index, cfg.smem_dq_stages) + dk_index = advance(dk_index, cfg.smem_dk_stages) + dg_index = advance(dg_index, cfg.smem_dg_stages) + dv_index = advance(dv_index, cfg.smem_dv_stages) + + chunk_serial_base += num_compute_chunks + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def super_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_decay_raw, + sK_inv_raw, + sU_raw, + sDy_raw, + sQk_raw, + sBeta_raw, + sBetaM_raw, + bars, +) -> None: + """Super-MMA warp role (warp 12): the Neumann A_inv and dM register MMAs + plus the dBeta M-term row sums.""" + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + raw_index = PipelineState.start(phase=0) + dbeta_m_done = PipelineState.start(phase=1) + sdy_index = PipelineState.start(phase=0) + + # ---- ldmatrix lane decode ---------------------------------------------------- + rhs_row_coord = lane % 8 + (cutlass.Int32(8) if (lane // 16) else cutlass.Int32(0)) + rhs_col_offset = cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0) + lhs_row_coord = lane % 8 + (cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0)) + lhs_col_offset = cutlass.Int32(8) if ((lane // 8) // 2) else cutlass.Int32(0) + stsm_row_coord = lane & 7 + stsm_col_coord = cutlass.Int32(0) + if (lane // 8) & 1: + stsm_row_coord = stsm_row_coord + cutlass.Int32(8) + if lane // 8 >= 2: + stsm_col_coord = cutlass.Int32(8) + stsm_idx = swizzle_lin_S(stsm_row_coord * cfg.b_t + stsm_col_coord, bbits=1, mbase=3, sshift=3) + row_lo = lane // 4 + row_hi = row_lo + cutlass.Int32(8) + + chunk_serial_base = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + num_compute_chunks = cend - wstart + for rev_idx in cutlass.range(num_compute_chunks, unroll=1): + chunk_serial = chunk_serial_base + rev_idx + decay_stage = chunk_serial % cfg.smem_decay_stages + qk_stage = chunk_serial % cfg.smem_qk_stages + raw_stage = raw_index.idx + sBeta_ptr = sBeta_raw.data_ptr() + (chunk_serial % cfg.smem_beta_stages) * cfg.b_t + sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sK_decay_ptr = sK_decay_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sQk_ptr = sQk_raw.data_ptr() + qk_stage * (cfg.qk_tiles * cfg.b_t * cfg.b_t) + + bars.mb_a_done[qk_stage].wait(((chunk_serial // cfg.smem_qk_stages) + 1) % 2) + + # ---- KK = K_decay @ K_inv^T ------------------------------------------ + bars.mb_k_decay_inv_ready[decay_stage].wait((chunk_serial // cfg.smem_decay_stages) % 2) + kk_lhs_row = lhs_row_coord + kk_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + kk_acc[accum_idx] = cutlass.Float32(0.0) + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + a_col = k_block * 16 + lhs_col_offset + a_seg = a_col // 64 + a_vec = nvvm.ldmatrix( + sK_decay_ptr + a_seg * (cfg.b_t * 64) + kk_lhs_row * 64 + swizzle_xor_128b(kk_lhs_row, a_col - a_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + b_col = k_block * 16 + rhs_col_offset + b_seg = b_col // 64 + b_vec = nvvm.ldmatrix( + sK_inv_ptr + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + mma_step( + kk_acc, + (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), + (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + + # ---- L = Beta * tril(KK, -1) ----------------------------------------- + bars.mb_beta_ready[chunk_serial % cfg.smem_beta_stages].wait((chunk_serial // cfg.smem_beta_stages) % 2) + beta_lo = (sBeta_ptr + row_lo).load().to(cutlass.Float32) + beta_hi = (sBeta_ptr + row_hi).load().to(cutlass.Float32) + bars.mb_beta_done[chunk_serial % cfg.smem_beta_stages].arrive() + l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + beta_scale = beta_lo if accum_idx % 4 < 2 else beta_hi + lower = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + l_frag[accum_idx] = lower * beta_scale + l_a0 = fp32_to_fp16(l_frag[0], l_frag[1], dtype=cfg.io_dtype) + l_a1 = fp32_to_fp16(l_frag[2], l_frag[3], dtype=cfg.io_dtype) + l_a2 = fp32_to_fp16(l_frag[4], l_frag[5], dtype=cfg.io_dtype) + l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) + l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) + + ainv_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + eye = cutlass.Float32(1.0) if row_coord == col_coord else cutlass.Float32(0.0) + ainv_acc[accum_idx] = eye - l_values[accum_idx] + + lpow_a0, lpow_a1, lpow_a2, lpow_a3 = l_a0, l_a1, l_a2, l_a3 + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(l_a0), movmatrix_16b(l_a1), movmatrix_16b(l_a2), movmatrix_16b(l_a3) + for _round in cutlass.range_constexpr(3): + sq_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + sq_acc[accum_idx] = cutlass.Float32(0.0) + mma_step( + sq_acc, + (lpow_a0, lpow_a1, lpow_a2, lpow_a3), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + lpow_a0 = fp32_to_fp16(sq_acc[0], sq_acc[1], dtype=cfg.io_dtype) + lpow_a1 = fp32_to_fp16(sq_acc[2], sq_acc[3], dtype=cfg.io_dtype) + lpow_a2 = fp32_to_fp16(sq_acc[4], sq_acc[5], dtype=cfg.io_dtype) + lpow_a3 = fp32_to_fp16(sq_acc[6], sq_acc[7], dtype=cfg.io_dtype) + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(lpow_a0), movmatrix_16b(lpow_a1), movmatrix_16b(lpow_a2), movmatrix_16b(lpow_a3) + upd_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + upd_acc[accum_idx] = cutlass.Float32(0.0) + ainv_p0 = fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype) + ainv_p1 = fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype) + ainv_p2 = fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype) + ainv_p3 = fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype) + mma_step( + upd_acc, + (ainv_p0, ainv_p1, ainv_p2, ainv_p3), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + ainv_lo0, ainv_hi0 = f16x2_to_f32(ainv_p0, dtype=cfg.io_dtype) + ainv_lo1, ainv_hi1 = f16x2_to_f32(ainv_p1, dtype=cfg.io_dtype) + ainv_lo2, ainv_hi2 = f16x2_to_f32(ainv_p2, dtype=cfg.io_dtype) + ainv_lo3, ainv_hi3 = f16x2_to_f32(ainv_p3, dtype=cfg.io_dtype) + ainv_acc[0] = ainv_lo0 + upd_acc[0] + ainv_acc[1] = ainv_hi0 + upd_acc[1] + ainv_acc[2] = ainv_lo1 + upd_acc[2] + ainv_acc[3] = ainv_hi1 + upd_acc[3] + ainv_acc[4] = ainv_lo2 + upd_acc[4] + ainv_acc[5] = ainv_hi2 + upd_acc[5] + ainv_acc[6] = ainv_lo3 + upd_acc[6] + ainv_acc[7] = ainv_hi3 + upd_acc[7] + + nvvm.stmatrix( + sQk_ptr + 1 * (cfg.b_t * cfg.b_t) + stsm_idx, + [ + fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype), + ], + nvvm.MMALayout.ROW, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_a_ready[qk_stage].arrive() + bars.mb_decay_super_done[decay_stage].arrive() + + # ---- dM = dY @ U^T --------------------------------------------------- + bars.mb_dm_done[qk_stage].wait(((chunk_serial // cfg.smem_qk_stages) + 1) % 2) + bars.mb_sdy_ready.wait(sdy_index.phase) + sdy_index = advance(sdy_index, 1) + dm_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + dm_acc[accum_idx] = cutlass.Float32(0.0) + for k_block in cutlass.range_constexpr(cfg.d_v // 16): + a_col = k_block * 16 + lhs_col_offset + a_seg = a_col // 64 + a_vec = nvvm.ldmatrix( + sDy_raw.data_ptr() + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + b_col = k_block * 16 + rhs_col_offset + b_seg = b_col // 64 + b_vec = nvvm.ldmatrix( + sU_raw.data_ptr() + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + mma_step( + dm_acc, + (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), + (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + bars.mb_u_warps_done.arrive() + bars.mb_sdy_done.arrive() + + dm_strict = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + beta_scale = beta_lo if accum_idx % 4 < 2 else beta_hi + val = dm_acc[accum_idx] * beta_scale if row_coord > col_coord else cutlass.Float32(0.0) + dm_strict[accum_idx] = val + w0 = fp32_to_fp16(dm_strict[0], dm_strict[1], dtype=cfg.io_dtype) + w1 = fp32_to_fp16(dm_strict[2], dm_strict[3], dtype=cfg.io_dtype) + w2 = fp32_to_fp16(dm_strict[4], dm_strict[5], dtype=cfg.io_dtype) + w3 = fp32_to_fp16(dm_strict[6], dm_strict[7], dtype=cfg.io_dtype) + nvvm.stmatrix(sQk_ptr + 3 * (cfg.b_t * cfg.b_t) + stsm_idx, [w0, w1, w2, w3], nvvm.MMALayout.ROW, shape=nvvm.StoreShape.M8N8) + nw0 = fp32_to_fp16(-dm_strict[0], -dm_strict[1], dtype=cfg.io_dtype) + nw1 = fp32_to_fp16(-dm_strict[2], -dm_strict[3], dtype=cfg.io_dtype) + nw2 = fp32_to_fp16(-dm_strict[4], -dm_strict[5], dtype=cfg.io_dtype) + nw3 = fp32_to_fp16(-dm_strict[6], -dm_strict[7], dtype=cfg.io_dtype) + nvvm.stmatrix(sQk_ptr + 4 * (cfg.b_t * cfg.b_t) + stsm_idx, [nw0, nw1, nw2, nw3], nvvm.MMALayout.ROW, shape=nvvm.StoreShape.M8N8) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dm_ready[qk_stage].arrive() + + bars.mb_dbeta_m_done.wait(dbeta_m_done.phase) + dbeta_m_done = advance(dbeta_m_done, 1) + bsum_lo = cutlass.Float32(0.0) + bsum_hi = cutlass.Float32(0.0) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + e = dm_acc[accum_idx] * kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + if cutlass.const_expr(accum_idx % 4 < 2): + bsum_lo = bsum_lo + e + else: + bsum_hi = bsum_hi + e + bsum_lo = bsum_lo + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, bsum_lo, 1, 31, kind=nvvm.Shfl.BFLY)) + bsum_lo = bsum_lo + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, bsum_lo, 2, 31, kind=nvvm.Shfl.BFLY)) + bsum_hi = bsum_hi + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, bsum_hi, 1, 31, kind=nvvm.Shfl.BFLY)) + bsum_hi = bsum_hi + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, bsum_hi, 2, 31, kind=nvvm.Shfl.BFLY)) + if lane % 4 == 0: + sBetaM_raw[row_lo] = -bsum_lo + sBetaM_raw[row_hi] = -bsum_hi + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dbeta_m_ready.arrive() + raw_index = advance(raw_index, cfg.smem_raw_stages) + chunk_serial_base += num_compute_chunks + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def tcgen05_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + tmem_base_holder, + sState_alt, + sK_decay_lead16, + sK_inv_lead16, + sK_inv_amaj, + sK_restore_lead16, + sDo_lead16, + sDo_amaj, + sQ_decay_trans, + sK_decay_trans, + sU_lead16, + sDv_lead16, + sDstate_alt, + sQk, + sState_scale_diag, + bars, +) -> None: + """tcgen05-MMA warp role (warp 13): issues every tcgen05 GEMM and owns + the TMEM lifecycle.""" + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + tmem_base = tmem_base_holder.load() + bpe = cfg.io_dtype.width // 8 + + # ---- chunk-invariant GEMM descriptors ---------------------------------------- + idesc_mv_nt = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + ) + idesc_state_k_at = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + a_major=1, + ) + bmm_state_k_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.d_k, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + atranspose=True, + cta_group=1, + idesc=idesc_state_k_at, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bmm_dvinter_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.d_k, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_mv_nt, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_du_at = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + a_major=1, + b_major=1, ) + bmm_du_at_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + atranspose=True, + cta_group=1, + idesc=idesc_du_at, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dstate_q_at = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.d_k, + m_dim=cfg.d_v, + a_major=1, + b_major=1, + ) + bmm_dstate_q_at_desc = MmaDesc( + M=cfg.d_v, + N=cfg.d_k, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + atranspose=True, + cta_group=1, + idesc=idesc_dstate_q_at, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bmm_qk_ts_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_mv_nt, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_mv_nt_t = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + b_major=1, + ) + bmm_qk_ts_t_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + cta_group=1, + idesc=idesc_mv_nt_t, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_diag = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=16, + m_dim=cfg.d_v, + ) + bmm_diag_desc = MmaDesc( + M=cfg.d_v, + N=16, + K=16, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_diag, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dstate_k = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.d_k, + m_dim=cfg.d_v, + b_major=1, + ) + bmm_dstate_k_desc = MmaDesc( + M=cfg.d_v, + N=cfg.d_k, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + cta_group=1, + idesc=idesc_dstate_k, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dstate = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_k, + ) + bmm_dstate_ts_desc = MmaDesc( + M=cfg.d_k, + N=cfg.b_t, + K=cfg.d_v, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_dstate, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dstate_at = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_k, + a_major=1, + ) + bmm_dstate_at_desc = MmaDesc( + M=cfg.d_k, + N=cfg.b_t, + K=cfg.d_v, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + atranspose=True, + cta_group=1, + idesc=idesc_dstate_at, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dgp = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_k, + ) + bmm_dgrad_ts_desc = MmaDesc( + M=cfg.d_k, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_dgp, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dgp_at = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_k, + a_major=1, + ) + bmm_dgrad_at_desc = MmaDesc( + M=cfg.d_k, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + atranspose=True, + cta_group=1, + idesc=idesc_dgp_at, + kind=nvvm.Tcgen05MMAKind.F16, + ) + idesc_dgp_at_t = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_k, + a_major=1, + b_major=1, + ) + bmm_dgrad_at_t_desc = MmaDesc( + M=cfg.d_k, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + atranspose=True, + cta_group=1, + idesc=idesc_dgp_at_t, + kind=nvvm.Tcgen05MMAKind.F16, + ) + + state_index = PipelineState.start(phase=0) + rhs_index = PipelineState.start(phase=0) + dstate_inp_index = PipelineState.start(phase=0) + dstate_acc_done_index = PipelineState.start(phase=1) + du_inp_index = PipelineState.start(phase=0) + neg_beta_dy_index = PipelineState.start(phase=0) + u_smem_index = PipelineState.start(phase=0) + dstate_smem_index = PipelineState.start(phase=0) + parts_done_index = PipelineState.start(phase=1) + + dstate0_index = PipelineState.start(phase=0) + + chunk_serial_base = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + num_compute_chunks = cend - wstart + for rev_idx in cutlass.range(num_compute_chunks, unroll=1): + chunk_serial = chunk_serial_base + rev_idx + decay_stage = chunk_serial % cfg.smem_decay_stages + qk_stage = chunk_serial % cfg.smem_qk_stages + decay_phase = (chunk_serial // cfg.smem_decay_stages) % 2 + qk_phase = (chunk_serial // cfg.smem_qk_stages) % 2 + has_dstate = cutlass.Boolean(rev_idx > 0) + if cutlass.const_expr(cfg.use_dstate_in): + has_dstate = cutlass.Boolean(True) + raw_stage_idx = chunk_serial % cfg.smem_raw_stages + + sQk_ptr = sQk[qk_stage] + chunk_idx = cend - cutlass.Int32(1) - rev_idx + + # ---- state_k = state(S) @ K_decay^T -------------------------------------- + bars.mb_k_decay_inv_ready[decay_stage].wait(decay_phase) + if chunk_idx >= FIRST_STATE_CHUNK: + bars.mb_state_ready[state_index.idx].wait(state_index.phase) + mma_ss( + bmm_state_k_desc, + sState_alt[state_index.idx].desc(), + sK_decay_lead16[decay_stage].desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_state_k_acc_offset), cutlass.Float32), + accumulate=False, + ) + if nvvm.elect_sync(): + bars.mb_state_k_acc_ready.arrive(cta_group=1) + bars.mb_state_done[state_index.idx].arrive(cta_group=1) + state_index = advance(state_index, cfg.smem_state_stages) + + # ---- dQ inter = state(T) @ dO^T ------------------------------------------ + bars.mb_parts_done.wait(parts_done_index.phase) + parts_done_index = advance(parts_done_index, 1) + bars.mb_state_inp_ready[chunk_serial % 2].wait((chunk_serial // 2) % 2) + bars.mb_do_ready[raw_stage_idx].wait((chunk_serial // cfg.smem_raw_stages) % 2) + if chunk_idx >= FIRST_STATE_CHUNK: + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_state_inp_offset + (chunk_serial % 2) * (cfg.d_v // 2)), cutlass.Int8) + b_desc = sDo_lead16[raw_stage_idx].desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dq_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_dstate_ts_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_dstate_ts_desc.sps_B): + mma_ts_step( + bmm_dstate_ts_desc, + a_ptr.subview(sub * bmm_dstate_ts_desc.sps_B * bmm_dstate_ts_desc.tmem_advance_A), + b_desc + sub * (bmm_dstate_ts_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + + # ---- dU inter = dstate(T) @ K_restore ------------------------------------ + bars.mb_q_decay_k_restore_ready[decay_stage].wait(decay_phase) + if has_dstate: + bars.mb_dstate_inp_ready.wait(dstate_inp_index.phase) + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dstate_inp_offset), cutlass.Int8) + b_desc = sK_restore_lead16[decay_stage].desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_du_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_dvinter_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_dvinter_desc.sps_B): + mma_ts_step( + bmm_dvinter_desc, + a_ptr.subview(sub * bmm_dvinter_desc.sps_B * bmm_dvinter_desc.tmem_advance_A), + b_desc + sub * (bmm_dvinter_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + + # ---- dU intra += dO^T(S) @ A_qk -------------------------------------- + bars.mb_aqk_ready[qk_stage].wait(qk_phase) + mma_ss( + bmm_du_at_desc, + sDo_amaj[raw_stage_idx].desc(), + sQk_ptr.desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_du_acc_offset), cutlass.Float32), + accumulate=has_dstate, + ) + if nvvm.elect_sync(): + bars.mb_du_acc_ready.arrive(cta_group=1) + bars.mb_aqk_done[qk_stage].arrive(cta_group=1) + + # ---- dstate decay = dstate(T) @ diag(eGl) ------------------------------------ + bars.mb_dstate_acc_done.wait(dstate_acc_done_index.phase) + dstate_acc_done_index = advance(dstate_acc_done_index, 1) + if has_dstate: + desc_diag = sState_scale_diag[decay_stage].desc() + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dstate_inp_offset) + k_block * 8, cutlass.Int8) + b_desc = desc_diag.advance_start_address(k_block * 256 * 2) + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dstate_acc_offset) + k_block * 16, cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_diag_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_diag_desc.sps_B): + mma_ts_step( + bmm_diag_desc, + a_ptr.subview(sub * bmm_diag_desc.sps_B * bmm_diag_desc.tmem_advance_A), + b_desc + sub * (bmm_diag_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + if nvvm.elect_sync(): + bars.mb_dstate_inp_done.arrive(cta_group=1) + dstate_inp_index = advance(dstate_inp_index, 1) + + # ---- dstate q-term += dO^T(S) @ Q_decay ---------------------------------- + mma_ss( + bmm_dstate_q_at_desc, + sDo_amaj[raw_stage_idx].desc(), + sQ_decay_trans[decay_stage].desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dstate_acc_offset), cutlass.Float32), + accumulate=has_dstate, + ) + + # ---- U = W(T) @ A_inv ------------------------------------------------ + bars.mb_a_ready[qk_stage].wait(qk_phase) + bars.mb_rhs_ready.wait(rhs_index.phase) + rhs_index = advance(rhs_index, 1) + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_rhs_inp_offset), cutlass.Int8) + b_desc = sQk_ptr.shifted(1 * (cfg.b_t * cfg.b_t)).desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_update_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_qk_ts_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_qk_ts_desc.sps_B): + mma_ts_step( + bmm_qk_ts_desc, + a_ptr.subview(sub * bmm_qk_ts_desc.sps_B * bmm_qk_ts_desc.tmem_advance_A), + b_desc + sub * (bmm_qk_ts_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + if nvvm.elect_sync(): + bars.mb_update_acc_ready.arrive(cta_group=1) + bars.mb_do_done[raw_stage_idx].arrive(cta_group=1) + + # ---- dY = dU(T) @ A_inv ---------------------------------------------- + bars.mb_du_inp_ready.wait(du_inp_index.phase) + du_inp_index = advance(du_inp_index, 1) + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_du_inp_offset), cutlass.Int8) + dy_b_desc = sQk_ptr.shifted(1 * (cfg.b_t * cfg.b_t)).desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dy_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_qk_ts_t_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_qk_ts_t_desc.sps_B): + mma_ts_step( + bmm_qk_ts_t_desc, + a_ptr.subview(sub * bmm_qk_ts_t_desc.sps_B * bmm_qk_ts_t_desc.tmem_advance_A), + dy_b_desc + sub * (bmm_qk_ts_t_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + if nvvm.elect_sync(): + bars.mb_dy_acc_ready.arrive(cta_group=1) + bars.mb_du_inp_done.arrive(cta_group=1) + bars.mb_a_done[qk_stage].arrive(cta_group=1) + + # ---- dK_restore part = dstate(S) @ U^T ----------------------------------- + bars.mb_u_smem_ready.wait(u_smem_index.phase) + u_smem_index = advance(u_smem_index, 1) + if has_dstate: + bars.mb_dstate_smem_ready.wait(dstate_smem_index.phase) + dstate_smem_index = advance(dstate_smem_index, 1) + mma_ss( + bmm_dstate_at_desc, + sDstate_alt[0].desc(), + sU_lead16[0].desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_restore_acc_offset), cutlass.Float32), + accumulate=False, + ) + if nvvm.elect_sync(): + bars.mb_dk_restore_part_ready.arrive(cta_group=1) + bars.mb_dstate_smem_done.arrive(cta_group=1) + if nvvm.elect_sync(): + bars.mb_u_done.arrive(cta_group=1) + + # ---- dstate dY-term += -Beta.dY(T) @ K_decay ----------------------------- + bars.mb_neg_beta_dy_inp_ready.wait(neg_beta_dy_index.phase) + neg_beta_dy_index = advance(neg_beta_dy_index, 1) + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_neg_beta_dy_inp_offset), cutlass.Int8) + b_desc = sK_decay_trans[decay_stage].desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dstate_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_dstate_k_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_dstate_k_desc.sps_B): + mma_ts_step( + bmm_dstate_k_desc, + a_ptr.subview(sub * bmm_dstate_k_desc.sps_B * bmm_dstate_k_desc.tmem_advance_A), + b_desc + sub * (bmm_dstate_k_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(True), + ) + if nvvm.elect_sync(): + bars.mb_dstate_acc_ready.arrive(cta_group=1) + bars.mb_neg_beta_dy_inp_done.arrive(cta_group=1) + + # ---- dK_inv part = scale.Q_decay^T(S) @ dA --------------------------- + bars.mb_da_ready[qk_stage].wait(qk_phase) + mma_ss( + bmm_dgrad_at_t_desc, + sQ_decay_trans[decay_stage].desc(), + sQk_ptr.shifted(2 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), + accumulate=False, + ) + + # ---- dQ attn += K_inv^T(S) @ dA^T ------------------------------------ + if chunk_idx >= FIRST_STATE_CHUNK: + mma_ss( + bmm_dgrad_at_desc, + sK_inv_amaj[decay_stage].desc(), + sQk_ptr.shifted(2 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dq_acc_offset), cutlass.Float32), + accumulate=True, + ) + if chunk_idx < FIRST_STATE_CHUNK: + mma_ss( + bmm_dgrad_at_desc, + sK_inv_amaj[decay_stage].desc(), + sQk_ptr.shifted(2 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dq_acc_offset), cutlass.Float32), + accumulate=False, + ) + if nvvm.elect_sync(): + bars.mb_dq_acc_ready.arrive(cta_group=1) + bars.mb_da_done[qk_stage].arrive(cta_group=1) + + # ---- dK_decay part = state(T) @ (Beta.dY)^T ------------------------------ + bars.mb_dv_tmastg_ready[chunk_serial % cfg.smem_dv_stages].wait((chunk_serial // cfg.smem_dv_stages) % 2) + if chunk_idx >= FIRST_STATE_CHUNK: + a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_state_inp_offset + (chunk_serial % 2) * (cfg.d_v // 2)), cutlass.Int8) + b_desc = sDv_lead16[chunk_serial % cfg.smem_dv_stages].desc() + c_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_decay_acc_offset), cutlass.Float32) + for sub in cutlass.range_constexpr(bmm_dstate_ts_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_dstate_ts_desc.sps_B): + mma_ts_step( + bmm_dstate_ts_desc, + a_ptr.subview(sub * bmm_dstate_ts_desc.sps_B * bmm_dstate_ts_desc.tmem_advance_A), + b_desc + sub * (bmm_dstate_ts_desc.smem_subtile_B >> 4), + c_ptr, + k, + cutlass.Boolean(sub + k > 0), + ) + if nvvm.elect_sync(): + bars.mb_dv_tmastg_done[chunk_serial % cfg.smem_dv_stages].arrive(cta_group=1) + bars.mb_state_inp_done[chunk_serial % 2].arrive(cta_group=1) + + # ---- dK_inv part += K_decay^T(S) @ -dM_strict ------------------------ + bars.mb_dm_ready[qk_stage].wait(qk_phase) + mma_ss( + bmm_dgrad_at_t_desc, + sK_decay_trans[decay_stage].desc(), + sQk_ptr.shifted(4 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), + accumulate=True, + ) + if nvvm.elect_sync(): + bars.mb_dk_inv_part_ready.arrive(cta_group=1) + + # ---- dK_decay part += K_inv^T(S) @ dM_strict^T ----------------------- + if chunk_idx >= FIRST_STATE_CHUNK: + mma_ss( + bmm_dgrad_at_desc, + sK_inv_amaj[decay_stage].desc(), + sQk_ptr.shifted(3 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), + accumulate=True, + ) + if chunk_idx < FIRST_STATE_CHUNK: + mma_ss( + bmm_dgrad_at_desc, + sK_inv_amaj[decay_stage].desc(), + sQk_ptr.shifted(3 * (cfg.b_t * cfg.b_t)).desc(), + nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), + accumulate=False, + ) + if nvvm.elect_sync(): + bars.mb_dk_decay_part_ready.arrive(cta_group=1) + bars.mb_dm_done[qk_stage].arrive(cta_group=1) + bars.mb_decay_done[decay_stage].arrive(cta_group=1) + + # ---- tile end: WG1's dstate0 drain gates the next tile's dstate reuse ------------ + bars.mb_dstate0_stored.wait(dstate0_index.phase) + dstate0_index = advance(dstate0_index, 1) + chunk_serial_base += num_compute_chunks + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + nvvm.tcgen05_dealloc( + nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), + cutlass.Int32(512), + group=nvvm.CTAGroup.CTA_1, + ) + + +@cute.jit +def tmaldg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + mSched, + sSched, + lane, + sQ_raw, + sK_raw, + sV_raw, + sGate_raw, + sDo_raw, + sState_raw, + desc_q_base, + desc_k_base, + desc_v_base, + desc_gate_base, + desc_do_base, + desc_checkpoint_base, + desc_initial_state_base, + bars, +) -> None: + """TMA-LDG warp role (warp 14): persistent tile-scheduler loop issuing + every G->S TMA load.""" + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + + sQ_tma = SmemTile( + base=sQ_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sK_tma = SmemTile( + base=sK_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sV_tma = SmemTile( + base=sV_raw, + elems_per_stage=(cfg.d_v * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sGate_tma = SmemTile( + base=sGate_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 32), + tma_granu_elems=32, + tma_subtile_stride_elems=(cfg.b_t * 32), + ) + sDo_tma = SmemTile( + base=sDo_raw, + elems_per_stage=(cfg.d_v * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sState_tma = SmemTile( + base=sState_raw, + elems_per_stage=(cfg.d_k * cfg.d_v), + stages=cfg.smem_state_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.d_k * 64, + ) + raw_index = PipelineState.start(phase=1) + state_index = PipelineState.start(phase=1) + sched_state = PipelineState.start(phase=1) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + next_tile, sched_state = sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) + head_o = head_idx + head_q = head_idx if cfg.q_ratio == 1 else head_idx // cutlass.Int32(cfg.q_ratio) + head_k = head_idx if cfg.k_ratio == 1 else head_idx // cutlass.Int32(cfg.k_ratio) + head_v = head_idx if cfg.v_ratio == 1 else head_idx // cutlass.Int32(cfg.v_ratio) + slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_q_slot = (desc_q_base + slot).tospace(cutlass.AddressSpace.generic) + desc_k_slot = (desc_k_base + slot).tospace(cutlass.AddressSpace.generic) + desc_v_slot = (desc_v_base + slot).tospace(cutlass.AddressSpace.generic) + desc_gate_slot = (desc_gate_base + slot).tospace(cutlass.AddressSpace.generic) + desc_do_slot = (desc_do_base + slot).tospace(cutlass.AddressSpace.generic) + desc_checkpoint_slot = (desc_checkpoint_base + slot).tospace(cutlass.AddressSpace.generic) + desc_initial_state_slot = (desc_initial_state_base + cutlass.Int32(0)).tospace(cutlass.AddressSpace.generic) + if nvvm.elect_sync(): + tma_tensormap_acquire(desc_q_slot) + tma_tensormap_acquire(desc_k_slot) + tma_tensormap_acquire(desc_v_slot) + tma_tensormap_acquire(desc_gate_slot) + tma_tensormap_acquire(desc_do_slot) + tma_tensormap_acquire(desc_checkpoint_slot) + if cutlass.const_expr(cfg.use_initial_state): + tma_tensormap_acquire(desc_initial_state_slot) + num_compute_chunks = cend - wstart + for rev_idx in cutlass.range(num_compute_chunks, unroll=1): + chunk_idx = cend - cutlass.Int32(1) - rev_idx + chunk_start = chunk_idx * cfg.b_t + + # ---- Q load ---------------------------------------------------------- + bars.mb_q_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_q_ready[raw_index.idx].arrive(n_bytes=cfg.tma_q_bytes) + q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), head_q, chunk_start) + tma_load_tile(sQ_tma[raw_index.idx], q_slice, bars.mb_q_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- K load ---------------------------------------------------------- + bars.mb_k_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_k_ready[raw_index.idx].arrive(n_bytes=cfg.tma_k_bytes) + k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, chunk_start) + tma_load_tile(sK_tma[raw_index.idx], k_slice, bars.mb_k_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- Gate load ------------------------------------------------------- + bars.mb_gate_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_gate_ready[raw_index.idx].arrive(n_bytes=cfg.tma_gate_bytes) + gate_slice = tma_slice_runtime_desc(desc_gate_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sGate_tma[raw_index.idx], gate_slice, bars.mb_gate_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- dO load --------------------------------------------------------- + bars.mb_do_done[raw_index.idx].wait(raw_index.phase) + bars.mb_do_epi_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_do_ready[raw_index.idx].arrive(n_bytes=cfg.tma_do_bytes) + do_slice = tma_slice_runtime_desc(desc_do_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sDo_tma[raw_index.idx], do_slice, bars.mb_do_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- V load ---------------------------------------------------------- + bars.mb_v_done[raw_index.idx].wait(raw_index.phase) + if nvvm.elect_sync(): + bars.mb_v_ready[raw_index.idx].arrive(n_bytes=cfg.tma_v_bytes) + v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, chunk_start) + tma_load_tile(sV_tma[raw_index.idx], v_slice, bars.mb_v_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- entering state: state_checkpoints[c - 1] (sequence-local), or initial_state for chunk 0 ---- + if chunk_idx >= FIRST_STATE_CHUNK: + state_idx = state_index.idx + bars.mb_state_done[state_idx].wait(state_index.phase) + bars.mb_state_cg0_done[state_idx].wait(state_index.phase) + state_index = advance(state_index, cfg.smem_state_stages) + if nvvm.elect_sync(): + bars.mb_state_ready[state_idx].arrive(n_bytes=cfg.tma_state_bytes) + if cutlass.const_expr(cfg.use_initial_state): + if chunk_idx == 0: + initial_state_slice = tma_slice_runtime_desc(desc_initial_state_slot, cutlass.Int32(0), cutlass.Int32(0), head_o, batch_idx) + tma_load_tile(sState_tma[state_idx], initial_state_slice, bars.mb_state_ready[state_idx].smem_ptr, acquire=False) + else: + state_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), chunk_idx - cutlass.Int32(1), head_o) + tma_load_tile(sState_tma[state_idx], state_slice, bars.mb_state_ready[state_idx].smem_ptr, acquire=False) + else: + state_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), chunk_idx - FIRST_STATE_CHUNK, head_o) + tma_load_tile(sState_tma[state_idx], state_slice, bars.mb_state_ready[state_idx].smem_ptr, acquire=False) + raw_index = advance(raw_index, cfg.smem_raw_stages) + tile_idx = next_tile + + +@cute.jit +def compute0_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_base_holder, + warp_idx, + scale, + mBeta, + sBeta_raw, + sK_inv_raw, + sGate_raw, + sK_raw, + sQ_raw, + sState_raw, + sNorm_raw, + sK_decay_raw, + sK_restore_raw, + sQ_decay_raw, + sState_scale_diag_raw, + bars, +) -> None: + """WG0 warp role (warps 0-3): persistent tile-scheduler loop + gate prefix + scan and the decay/restore operand materialization into tcgen05 SMEM for + EVERY chunk (no ping-pong: the backward pipeline is drain-bound). Also + stashes the per-row q/k inverse norms for WG2's dG assembly and copies + H -> TMEM f16 at the chunk tail.""" + nvvm.setmaxregister(cfg.num_regs_compute_group_0, nvvm.SetMaxRegisterAction.INCREASE) + cg0_warp = warp_idx - cfg.compute_group_0_warp_ids[0] + tmem_base = tmem_base_holder.load() + tmem_col = tmem_base & 0xFFFF + tmem_row = tmem_base >> 16 + tmem_subpartition = warp_idx % (cfg.d_v // cfg.threads_per_warp) + value_dim = tmem_subpartition * cfg.threads_per_warp + lane + row_addr = (tmem_row + tmem_subpartition * cfg.threads_per_warp) << 16 + state_index = PipelineState.start(phase=0) + chunk_serial_base = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + num_compute_chunks = cend - wstart + for rev_idx in cutlass.range(num_compute_chunks, unroll=1): + chunk_idx = cend - cutlass.Int32(1) - rev_idx + chunk_serial = chunk_serial_base + rev_idx + chunk_start = chunk_idx * cfg.b_t + decay_stage = chunk_serial % cfg.smem_decay_stages + raw_stage = chunk_serial % cfg.smem_raw_stages + sQ_ptr = sQ_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sK_ptr = sK_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sGate_ptr = sGate_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sK_decay_ptr = sK_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sQ_decay_ptr = sQ_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sK_restore_ptr = sK_restore_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sState_scale_diag_ptr = sState_scale_diag_raw.data_ptr() + decay_stage * ((cfg.d_k // 16) * 256) + + # ---- beta scalars: gathered in the inputs-wait shadow ---------------- + if cg0_warp == 0: + beta_stage = chunk_serial % cfg.smem_beta_stages + bars.mb_beta_done[beta_stage].wait(((chunk_serial // cfg.smem_beta_stages) + 1) % 2) + if lane < cfg.b_t: + token_idx = chunk_start + lane + beta_value = cutlass.Float32(0.0) + if token_idx < seqlen_b: + beta_value = mBeta[batch_start + token_idx, head_idx].to(cutlass.Float32) + sBeta_raw[beta_stage * cfg.b_t + lane] = beta_value + bars.mb_beta_ready[beta_stage].arrive() + + bars.mb_gate_ready[raw_stage].wait((chunk_serial // cfg.smem_raw_stages) % 2) + bars.mb_q_ready[raw_stage].wait((chunk_serial // cfg.smem_raw_stages) % 2) + bars.mb_k_ready[raw_stage].wait((chunk_serial // cfg.smem_raw_stages) % 2) + + row_group_start = cg0_warp * (cfg.b_t // len(cfg.compute_group_0_warp_ids)) + lane_row_group = lane // 8 + lane_in_row_group = lane - lane_row_group * 8 + decay_row = row_group_start + lane_row_group + + g_prefix_ptr = sGate_ptr + prefix_dim = cg0_warp * cfg.threads_per_warp + lane + # ---- gate prefix scan: cumulative log-gate per key channel ----------- + gate_raw = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + for row in cutlass.range_constexpr(cfg.b_t): + f32_segment = prefix_dim // 32 + f32_segment_dim = prefix_dim - f32_segment * 32 + prefix_idx = f32_segment * (cfg.b_t * 32) + row * 32 + swizzle_xor_128b(row, f32_segment_dim, elem_bytes=4) + gate_raw[row] = (sGate_ptr + prefix_idx).load() + g_prefix_regs = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + for row in cutlass.range_constexpr(cfg.b_t): + gate = gate_raw[row] + token_idx = chunk_idx * cutlass.Int32(cfg.b_t) + cutlass.Int32(row) + if token_idx < seqlen_b: + gate = gate * cutlass.Float32(LOG2_E) + else: + gate = cutlass.Float32(0.0) + g_prefix_regs[row] = gate + + prefix_acc = cutlass.Float32(0.0) + for row_pair in cutlass.range_constexpr(cfg.b_t // 2): + row0 = row_pair * 2 + row1 = row0 + 1 + gate0 = g_prefix_regs[row0] + gate1 = g_prefix_regs[row1] + pair_vec = nvvm.add_packed_f32x2( + cutlass.Vector.from_elements((prefix_acc, gate0), cutlass.Float32), + cutlass.Vector.from_elements((gate0, gate1), cutlass.Float32), + ftz=False, + rnd="rn", + ) + prefix0, row_pair_sum = cutlass.Float32(pair_vec[0]), cutlass.Float32(pair_vec[1]) + prefix1 = prefix_acc + row_pair_sum + g_prefix_regs[row0] = prefix0 + g_prefix_regs[row1] = prefix1 + prefix_acc = prefix1 + + for row in cutlass.range_constexpr(cfg.b_t): + g_prefix_regs[row] = cute.math.exp2(g_prefix_regs[row], fastmath=True) + + exp_g_last = g_prefix_regs[cfg.b_t - 1] + # ---- decay-slot guard: previous use fully consumed ------------------- + operand_done_phase = ((chunk_serial // cfg.smem_decay_stages) + 1) % 2 + bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) + + for row in cutlass.range_constexpr(cfg.b_t): + f32_segment = prefix_dim // 32 + f32_segment_dim = prefix_dim - f32_segment * 32 + prefix_idx = f32_segment * (cfg.b_t * 32) + row * 32 + swizzle_xor_128b(row, f32_segment_dim, elem_bytes=4) + (sGate_ptr + prefix_idx).store(g_prefix_regs[row]) + + # ---- state-scale diag: stage exp2(g_last) decay blocks --------------- + block = prefix_dim // cutlass.Int32(16) + coord = prefix_dim - block * cutlass.Int32(16) + linear_idx = block * cutlass.Int32(256) + coord * cutlass.Int32(16) + coord + diag_idx = swizzle_lin_S(linear_idx, bbits=1, mbase=3, sshift=3) + sState_scale_diag_ptr[diag_idx] = exp_g_last.to(cfg.io_dtype) + + nvvm.barrier_cta_sync(cfg.cg0_sync_barrier_id, thread_count=cfg.cg0_threads) + + k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) + raw_q_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + # ---- optional q/k L2-norm + K_inv staging ---------------------------- + if cutlass.const_expr(cfg.l2norm): + qk0_lo = opaque_f32_zero() + qk0_hi = opaque_f32_zero() + qk1_lo = opaque_f32_zero() + qk1_hi = opaque_f32_zero() + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + raw_f16_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) + raw_q_vec = (sQ_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_k_vec = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_q_vec_f32 = raw_q_vec.to(cutlass.Float32) + raw_k_vec_f32 = raw_k_vec.to(cutlass.Float32) + for dim_offset in cutlass.range_constexpr(8): + q_val = raw_q_vec_f32[dim_offset] + k_val = raw_k_vec_f32[dim_offset] + raw_q_regs[reg_base + dim_offset] = q_val + raw_k_regs[reg_base + dim_offset] = k_val + if cutlass.const_expr(cfg.l2norm): + if cutlass.const_expr(dim_offset % 2 == 0): + qk0_lo, qk0_hi = ffma2(q_val, k_val, q_val, k_val, qk0_lo, qk0_hi) + else: + qk1_lo, qk1_hi = ffma2(q_val, k_val, q_val, k_val, qk1_lo, qk1_hi) + + q_inv_norm = opaque_f32_zero() + cutlass.Float32(1.0) + k_inv_norm = opaque_f32_zero() + cutlass.Float32(1.0) + if cutlass.const_expr(cfg.l2norm): + q_sum_sq = qk0_lo + qk1_lo + k_sum_sq = qk0_hi + qk1_hi + q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 4, 31, kind=nvvm.Shfl.BFLY)) + q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 2, 31, kind=nvvm.Shfl.BFLY)) + q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 1, 31, kind=nvvm.Shfl.BFLY)) + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 4, 31, kind=nvvm.Shfl.BFLY)) + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 2, 31, kind=nvvm.Shfl.BFLY)) + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 1, 31, kind=nvvm.Shfl.BFLY)) + norm_floor_sq = cutlass.Float32(L2_NORM_EPS * L2_NORM_EPS) + q_inv_norm = cute.math.rsqrt(cute.math.max(q_sum_sq, norm_floor_sq), fastmath=True) + k_inv_norm = cute.math.rsqrt(cute.math.max(k_sum_sq, norm_floor_sq), fastmath=True) + if lane_in_row_group == 0: + sNorm_raw[raw_stage * (2 * cfg.b_t) + decay_row] = q_inv_norm + sNorm_raw[raw_stage * (2 * cfg.b_t) + cfg.b_t + decay_row] = k_inv_norm + q_stage_norm = q_inv_norm * scale + + # ---- decay/restore operands: exp2(+-g) applied per key channel ------- + exp_g_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + exp_g_last_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + for f32_group in cutlass.range_constexpr(2): + f32_dim_base = dim_base + f32_group * 4 + f32_segment = f32_dim_base // 32 + f32_segment_dim = f32_dim_base - f32_segment * 32 + g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) + exp_g_vec = (g_prefix_ptr + g_prefix_idx).load(count=4, alignment=16) + exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) + exp_g_last_vec = (g_prefix_ptr + exp_g_last_idx).load(count=4, alignment=16) + f32_reg_base = reg_base + f32_group * 4 + exp_g_regs[f32_reg_base] = exp_g_vec[0] + exp_g_regs[f32_reg_base + 1] = exp_g_vec[1] + exp_g_regs[f32_reg_base + 2] = exp_g_vec[2] + exp_g_regs[f32_reg_base + 3] = exp_g_vec[3] + exp_g_last_regs[f32_reg_base] = exp_g_last_vec[0] + exp_g_last_regs[f32_reg_base + 1] = exp_g_last_vec[1] + exp_g_last_regs[f32_reg_base + 2] = exp_g_last_vec[2] + exp_g_last_regs[f32_reg_base + 3] = exp_g_last_vec[3] + + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + # ---- k decay + k inv operands: exp2(+g) * k / exp2(-g) * k ------- + k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + for pair_idx in cutlass.range_constexpr(4): + dim0 = pair_idx * 2 + dim1 = dim0 + 1 + raw_reg_idx0 = reg_base + dim0 + raw_reg_idx1 = reg_base + dim1 + k_value0, k_value1 = fmul2(raw_k_regs[raw_reg_idx0], raw_k_regs[raw_reg_idx1], k_inv_norm, k_inv_norm) + k_pair = fp32_to_fp16(k_value0, k_value1, dtype=cfg.io_dtype) + exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) + k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) + exp_neg_g0 = cute.math.rcp(exp_g_regs[raw_reg_idx0], approx=True, ftz=True) + exp_neg_g1 = cute.math.rcp(exp_g_regs[raw_reg_idx1], approx=True, ftz=True) + exp_neg_pair = fp32_to_fp16(exp_neg_g0, exp_neg_g1, dtype=cfg.io_dtype) + k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_pair, exp_neg_pair, cfg.io_dtype) + + k_inv_vec = cutlass.Vector.from_elements( + ( + k_inv_words[dim_half * 4], + k_inv_words[dim_half * 4 + 1], + k_inv_words[dim_half * 4 + 2], + k_inv_words[dim_half * 4 + 3], + ), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + k_decay_vec = cutlass.Vector.from_elements( + (k_decay_words[0], k_decay_words[1], k_decay_words[2], k_decay_words[3]), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + op_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) + (sK_inv_ptr + op_idx).store(k_inv_vec, alignment=16) + (sK_decay_ptr + op_idx).store(k_decay_vec, alignment=16) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_k_decay_inv_ready[decay_stage].arrive() + + # ---- q decay + k_restore operands ------------------------------------ + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + q_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_restore_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + for pair_idx in cutlass.range_constexpr(4): + dim0 = pair_idx * 2 + dim1 = dim0 + 1 + raw_reg_idx0 = reg_base + dim0 + raw_reg_idx1 = reg_base + dim1 + q_value0, q_value1 = fmul2(raw_q_regs[raw_reg_idx0], raw_q_regs[raw_reg_idx1], q_stage_norm, q_stage_norm) + q_pair = fp32_to_fp16(q_value0, q_value1, dtype=cfg.io_dtype) + exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) + q_decay_words[pair_idx] = mul_f16x2(q_pair, exp_g_pair, cfg.io_dtype) + exp_g_last_pair = fp32_to_fp16(exp_g_last_regs[raw_reg_idx0], exp_g_last_regs[raw_reg_idx1], dtype=cfg.io_dtype) + k_restore_words[pair_idx] = mul_f16x2(k_inv_words[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) + + q_decay_vec = cutlass.Vector.from_elements( + (q_decay_words[0], q_decay_words[1], q_decay_words[2], q_decay_words[3]), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + k_restore_vec = cutlass.Vector.from_elements( + (k_restore_words[0], k_restore_words[1], k_restore_words[2], k_restore_words[3]), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + op_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) + (sQ_decay_ptr + op_idx).store(q_decay_vec, alignment=16) + (sK_restore_ptr + op_idx).store(k_restore_vec, alignment=16) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_q_decay_k_restore_ready[decay_stage].arrive() + bars.mb_q_done[raw_stage].arrive() + bars.mb_k_done[raw_stage].arrive() + bars.mb_gate_done[raw_stage].arrive() + + # ---- state copy: SMEM -> TMEM f16 ---------------------------------------- + bars.mb_state_inp_done[chunk_serial % 2].wait(((chunk_serial // 2) + 1) % 2) + bars.mb_state_inp_cg2_done[chunk_serial % 2].wait(((chunk_serial // 2) + 1) % 2) + if chunk_idx >= FIRST_STATE_CHUNK: + bars.mb_state_ready[state_index.idx].wait(state_index.phase) + state_src = sState_raw.data_ptr() + state_index.idx * (cfg.d_k * cfg.d_v) + for v_seg in cutlass.range_constexpr(2): + for v_col8 in cutlass.range_constexpr(8): + state_vec = (state_src + v_seg * (cfg.d_k * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, v_col8 * 8, elem_bytes=2)).load( + count=8, alignment=16 + ) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr( + row_addr + (tmem_col + cfg.tmem_state_inp_offset + (chunk_serial % 2) * (cfg.d_v // 2) + v_seg * 32 + v_col8 * 4), cutlass.Int8 + ), + state_vec.bitcast(cutlass.Int32), + ) + nvvm.tcgen05_wait("store") + bars.mb_state_cg0_done[state_index.idx].arrive() + state_index = advance(state_index, cfg.smem_state_stages) + bars.mb_state_inp_ready[chunk_serial % 2].arrive() + chunk_serial_base += num_compute_chunks + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def compute1_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_base_holder, + warp_idx, + mDbeta, + mDstate0, + mDstate_in, + sV_raw, + sDo_raw, + sBeta_raw, + sU_raw, + sDy_raw, + sDv_raw, + sDstate_raw, + sRed_raw, + sBetaM_raw, + bars, +) -> None: + """WG1 warp role (warps 4-7): the value-side TMEM staging. Per chunk: + rhs W = beta*(v - state_k) -> TMEM f16 (prefill's staging); + U readback -> sU; dU restage -> TMEM f16; dY + readback -> sdV(+beta.dY) / sdY(plain) / TMEM(-beta.dY) + the dbeta + v-term dot and store; end-of-chunk dH capture (dh acc -> dh_inp f16 + + sdH); tile edges: dht seeding and the dS0 drain.""" + nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) + tmem_base = tmem_base_holder.load() + tmem_col = tmem_base & 0xFFFF + tmem_row = tmem_base >> 16 + tmem_subpartition = warp_idx % (cfg.d_v // cfg.threads_per_warp) + token_row_coord = (lane // 16) * 8 + (lane & 7) + value_col_offset = ((lane // 8) & 1) * 8 + value_dim = tmem_subpartition * cfg.threads_per_warp + lane + value_dim_base = tmem_subpartition * cfg.threads_per_warp + cg1_tidx = warp_idx % 4 * cfg.threads_per_warp + lane + + raw_index = PipelineState.start(phase=0) + state_k_index = PipelineState.start(phase=0) + update_index = PipelineState.start(phase=0) + du_acc_index = PipelineState.start(phase=0) + dy_acc_index = PipelineState.start(phase=0) + du_inp_done_index = PipelineState.start(phase=1) + neg_beta_dy_done_index = PipelineState.start(phase=1) + u_done_index = PipelineState.start(phase=1) + sdy_done_index = PipelineState.start(phase=1) + dstate_ready_index = PipelineState.start(phase=0) + dstate_inp_done_index = PipelineState.start(phase=1) + dstate_smem_done_index = PipelineState.start(phase=1) + dbeta_m_index = PipelineState.start(phase=0) + dv_done_index = PipelineState.start(phase=1) + + chunk_serial_base = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + num_compute_chunks = cend - wstart + + # ---- dht seeding: dh acc + dh_inp f16 + sdH ------------------------------ + if cutlass.const_expr(cfg.use_dstate_in): + if num_compute_chunks > 0: + seed_true = cend == num_chunks_b + bars.mb_dstate_inp_done.wait(dstate_inp_done_index.phase) + dstate_inp_done_index = advance(dstate_inp_done_index, 1) + bars.mb_dstate_smem_done.wait(dstate_smem_done_index.phase) + bars.mb_dstate_smem_cg2_done.wait(dstate_smem_done_index.phase) + dstate_smem_done_index = advance(dstate_smem_done_index, 1) + row_addr = (tmem_row + tmem_subpartition * cfg.threads_per_warp) << 16 + for sub in cutlass.range_constexpr(cfg.d_k // 16): + seed_block = cutlass.Array(cutlass.Float32, 16, alignment=16) + for kk_i in cutlass.range_constexpr(16): + dval = mDstate_in[batch_idx, head_idx, sub * 16 + kk_i, value_dim].to(cutlass.Float32) + if cutlass.const_expr(cfg.split_k): + dval = dval if seed_true else cutlass.Float32(0.0) + seed_block[kk_i] = dval + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_acc_offset + sub * 16), cutlass.Float32), + seed_block[0:16], + ) + packed_seed = cutlass.Array(cutlass.Int32, 8, alignment=16) + for pc in cutlass.range_constexpr(8): + packed_seed[pc] = fp32_to_fp16(seed_block[2 * pc], seed_block[2 * pc + 1], dtype=cfg.io_dtype) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_inp_offset + sub * 8), cutlass.Int8), + packed_seed[0:8], + ) + for half in cutlass.range_constexpr(2): + d_base = sub * 16 + half * 8 + h_vec = cutlass.Vector.from_elements( + (packed_seed[half * 4], packed_seed[half * 4 + 1], packed_seed[half * 4 + 2], packed_seed[half * 4 + 3]), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + h_addr = (d_base // 64) * (cfg.d_v * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, d_base % 64, elem_bytes=2) + (sDstate_raw.data_ptr() + h_addr).store(h_vec, alignment=16) + nvvm.tcgen05_wait("store") + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dstate_inp_ready.arrive() + bars.mb_dstate_smem_ready.arrive() + + for rev_idx in cutlass.range(num_compute_chunks, unroll=1): + chunk_idx = cend - cutlass.Int32(1) - rev_idx + chunk_serial = chunk_serial_base + rev_idx + sV_ptr = sV_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) + sDo_ptr = sDo_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) + sBeta_ptr = sBeta_raw.data_ptr() + (chunk_serial % cfg.smem_beta_stages) * cfg.b_t + row_addr_lo = tmem_row << 16 + row_addr_hi = (tmem_row + 16) << 16 + row_id0 = tmem_row + value_dim_base + row_id1 = row_id0 + 16 + has_dstate = cutlass.Boolean(rev_idx > 0) + if cutlass.const_expr(cfg.use_dstate_in): + has_dstate = cutlass.Boolean(True) + + bars.mb_v_ready[raw_index.idx].wait((chunk_serial // cfg.smem_raw_stages) % 2) + + # ---- rhs staging: W = Beta * (V - state_k) --------------------------- + projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset + input_col_id = tmem_col + cfg.tmem_rhs_inp_offset + raw_v_regs0 = nvvm.ldmatrix( + sV_ptr + + (value_dim_base + value_col_offset) // 64 * (cfg.b_t * 64) + + token_row_coord * 64 + + swizzle_xor_128b(token_row_coord, (value_dim_base + value_col_offset) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + raw_v_regs1 = nvvm.ldmatrix( + sV_ptr + + (value_dim_base + 16 + value_col_offset) // 64 * (cfg.b_t * 64) + + token_row_coord * 64 + + swizzle_xor_128b(token_row_coord, (value_dim_base + 16 + value_col_offset) % 64, elem_bytes=2), + 4, + nvvm.MMALayout.COL, + ) + + bars.mb_beta_ready[chunk_serial % cfg.smem_beta_stages].wait((chunk_serial // cfg.smem_beta_stages) % 2) + beta_pairs = cutlass.Array(cutlass.Int32, 2, space=cutlass.AddressSpace.rmem) + for half in cutlass.range_constexpr(2): + token0 = ((half * 4 + (lane & 3)) ^ 4) * 2 + beta0 = (sBeta_ptr + token0).load().to(cutlass.Float32) + beta1 = (sBeta_ptr + token0 + 1).load().to(cutlass.Float32) + beta_pairs[half] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) + diff_w0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + diff_w1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + if chunk_idx >= FIRST_STATE_CHUNK: + bars.mb_state_k_acc_ready.wait(state_k_index.phase) + state_k_index = advance(state_k_index, 1) + state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_pair = fp32_to_fp16(state_k0[frag_pair], state_k0[frag_pair + 1], dtype=cfg.io_dtype) + diff_pair = sub_f16x2(raw_v_regs0[raw_matrix], state_k_pair, cfg.io_dtype) + diff_w0[reg_idx ^ 2] = diff_pair + packed_rhs0[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], diff_pair, cfg.io_dtype) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_pair = fp32_to_fp16(state_k1[frag_pair], state_k1[frag_pair + 1], dtype=cfg.io_dtype) + diff_pair = sub_f16x2(raw_v_regs1[raw_matrix], state_k_pair, cfg.io_dtype) + diff_w1[reg_idx ^ 2] = diff_pair + packed_rhs1[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], diff_pair, cfg.io_dtype) + if chunk_idx < FIRST_STATE_CHUNK: + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + diff_w0[reg_idx ^ 2] = raw_v_regs0[raw_matrix] + packed_rhs0[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], raw_v_regs0[raw_matrix], cfg.io_dtype) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + diff_w1[reg_idx ^ 2] = raw_v_regs1[raw_matrix] + packed_rhs1[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], raw_v_regs1[raw_matrix], cfg.io_dtype) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + input_col_id, cutlass.Int8), packed_rhs0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + input_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_wait("store") + bars.mb_rhs_ready.arrive() + + # ---- dU restage: dU acc -> TMEM f16 A operand ------------------------ + bars.mb_du_acc_ready.wait(du_acc_index.phase) + du_acc_index = advance(du_acc_index, 1) + bars.mb_du_inp_done.wait(du_inp_done_index.phase) + du_inp_done_index = advance(du_inp_done_index, 1) + du_col_id = tmem_col + cfg.tmem_du_acc_offset + du0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + du_col_id, cutlass.Float32), num=2) + du1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + du_col_id, cutlass.Float32), num=2) + + du_packed0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + du_packed1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + frag_pair = reg_idx * 2 + du_packed0[reg_idx] = fp32_to_fp16(du0[frag_pair], du0[frag_pair + 1], dtype=cfg.io_dtype) + du_packed1[reg_idx] = fp32_to_fp16(du1[frag_pair], du1[frag_pair + 1], dtype=cfg.io_dtype) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_packed0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_packed1[0:4]) + nvvm.tcgen05_wait("store") + bars.mb_du_inp_ready.arrive() + + # ---- U readback -> sU ------------------------------------------------ + bars.mb_update_acc_ready.wait(update_index.phase) + update_index = advance(update_index, 1) + bars.mb_u_done.wait(u_done_index.phase) + bars.mb_u_warps_done.wait(u_done_index.phase) + u_done_index = advance(u_done_index, 1) + u_col_id = tmem_col + cfg.tmem_update_acc_offset + u0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + u_col_id, cutlass.Float32), num=2) + u1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + u_col_id, cutlass.Float32), num=2) + + u_words0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + u_words1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + u_words0[reg_idx] = fp32_to_fp16(u0[2 * reg_idx], u0[2 * reg_idx + 1], dtype=cfg.io_dtype) + u_words1[reg_idx] = fp32_to_fp16(u1[2 * reg_idx], u1[2 * reg_idx + 1], dtype=cfg.io_dtype) + nvvm.stmatrix( + sU_raw.data_ptr() + + (value_dim_base + value_col_offset) // 64 * (cfg.b_t * 64) + + token_row_coord * 64 + + swizzle_xor_128b(token_row_coord, (value_dim_base + value_col_offset) % 64, elem_bytes=2), + u_words0.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.stmatrix( + sU_raw.data_ptr() + + (value_dim_base + 16 + value_col_offset) // 64 * (cfg.b_t * 64) + + token_row_coord * 64 + + swizzle_xor_128b(token_row_coord, (value_dim_base + 16 + value_col_offset) % 64, elem_bytes=2), + u_words1.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_u_smem_ready.arrive() + + # ---- dY readback ------------------------------------------------------- + bars.mb_dy_acc_ready.wait(dy_acc_index.phase) + dy_acc_index = advance(dy_acc_index, 1) + dy_col_id = tmem_col + cfg.tmem_dy_acc_offset + dy0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + dy_col_id, cutlass.Float32), num=2) + dy1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + dy_col_id, cutlass.Float32), num=2) + + # ---- dY -> sdY: pack + store + publish (super warp's dM operand) ------- + addr_lo0 = ( + (value_dim_base + value_col_offset) // 64 * (cfg.b_t * 64) + + token_row_coord * 64 + + swizzle_xor_128b(token_row_coord, (value_dim_base + value_col_offset) % 64, elem_bytes=2) + ) + addr_lo1 = ( + (value_dim_base + 16 + value_col_offset) // 64 * (cfg.b_t * 64) + + token_row_coord * 64 + + swizzle_xor_128b(token_row_coord, (value_dim_base + 16 + value_col_offset) % 64, elem_bytes=2) + ) + dy_p0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + dy_p1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + dy_p0[reg_idx] = fp32_to_fp16(dy0[2 * reg_idx], dy0[2 * reg_idx + 1], dtype=cfg.io_dtype) + dy_p1[reg_idx] = fp32_to_fp16(dy1[2 * reg_idx], dy1[2 * reg_idx + 1], dtype=cfg.io_dtype) + bars.mb_sdy_done.wait(sdy_done_index.phase) + sdy_done_index = advance(sdy_done_index, 1) + nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo0, dy_p0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) + nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo1, dy_p1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) + bars.mb_sdy_ready.arrive() + + # ---- beta scalars -> beta.dY -> sdV (epilogue TMA + GEMM 13 operand) --- + beta_c0 = (sBeta_ptr + (lane % 4) * 2).load().to(cutlass.Float32) + beta_c1 = (sBeta_ptr + (lane % 4) * 2 + 1).load().to(cutlass.Float32) + beta_c8 = (sBeta_ptr + (lane % 4) * 2 + 8).load().to(cutlass.Float32) + beta_c9 = (sBeta_ptr + (lane % 4) * 2 + 9).load().to(cutlass.Float32) + bars.mb_beta_done[chunk_serial % cfg.smem_beta_stages].arrive() + beta_dy0 = cutlass.Array(cutlass.Float32, 8, alignment=16) + beta_dy1 = cutlass.Array(cutlass.Float32, 8, alignment=16) + for e2 in cutlass.range_constexpr(4): + e = 2 * e2 + b_lo = beta_c8 if cutlass.const_expr(e >= 4) else beta_c0 + b_hi = beta_c9 if cutlass.const_expr(e >= 4) else beta_c1 + beta_dy0[e], beta_dy0[e + 1] = fmul2(dy0[e], dy0[e + 1], b_lo, b_hi) + beta_dy1[e], beta_dy1[e + 1] = fmul2(dy1[e], dy1[e + 1], b_lo, b_hi) + beta_dy_p0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_dy_p1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + beta_dy_p0[reg_idx] = fp32_to_fp16(beta_dy0[2 * reg_idx], beta_dy0[2 * reg_idx + 1], dtype=cfg.io_dtype) + beta_dy_p1[reg_idx] = fp32_to_fp16(beta_dy1[2 * reg_idx], beta_dy1[2 * reg_idx + 1], dtype=cfg.io_dtype) + dv_stage = chunk_serial % cfg.smem_dv_stages + sdv_stage_base = dv_stage * (cfg.b_t * cfg.d_v) + bars.mb_dv_tmastg_done[dv_stage].wait(dv_done_index.phase) + bars.mb_dv_epi_done[dv_stage].wait(dv_done_index.phase) + dv_done_index = advance(dv_done_index, cfg.smem_dv_stages) + nvvm.stmatrix( + sDv_raw.data_ptr() + sdv_stage_base + addr_lo0, beta_dy_p0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8 + ) + nvvm.stmatrix( + sDv_raw.data_ptr() + sdv_stage_base + addr_lo1, beta_dy_p1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8 + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dv_tmastg_ready[dv_stage].arrive() + + # ---- -beta.dY -> TMEM: A operand of the dH ds-term --------------------- + neg_beta_dy0 = cutlass.Array(cutlass.Float32, 8, alignment=16) + neg_beta_dy1 = cutlass.Array(cutlass.Float32, 8, alignment=16) + for e in cutlass.range_constexpr(8): + neg_beta_dy0[e] = -beta_dy0[e] + neg_beta_dy1[e] = -beta_dy1[e] + neg_beta_dy_p0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + neg_beta_dy_p1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + frag_pair = reg_idx * 2 + neg_beta_dy_p0[reg_idx] = fp32_to_fp16(neg_beta_dy0[frag_pair], neg_beta_dy0[frag_pair + 1], dtype=cfg.io_dtype) + neg_beta_dy_p1[reg_idx] = fp32_to_fp16(neg_beta_dy1[frag_pair], neg_beta_dy1[frag_pair + 1], dtype=cfg.io_dtype) + bars.mb_neg_beta_dy_inp_done.wait(neg_beta_dy_done_index.phase) + neg_beta_dy_done_index = advance(neg_beta_dy_done_index, 1) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_neg_beta_dy_inp_offset), cutlass.Int8), neg_beta_dy_p0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_neg_beta_dy_inp_offset), cutlass.Int8), neg_beta_dy_p1[0:4]) + nvvm.tcgen05_wait("store") + bars.mb_neg_beta_dy_inp_ready.arrive() + + # ---- dbeta v-term partials: dY.(v - state_k), diff_w from rhs staging -- + tok4 = cutlass.Array(cutlass.Float32, 4, alignment=16) + for s in cutlass.range_constexpr(4): + tok4[s] = cutlass.Float32(0.0) + for j in cutlass.range_constexpr(4): + d0_lo, d0_hi = f16x2_to_f32(diff_w0[j], dtype=cfg.io_dtype) + d1_lo, d1_hi = f16x2_to_f32(diff_w1[j], dtype=cfg.io_dtype) + s_lo = cutlass.const_expr(2 * (j // 2)) + s_hi = cutlass.const_expr(2 * (j // 2) + 1) + t_lo, t_hi = ffma2(dy0[2 * j], dy0[2 * j + 1], d0_lo, d0_hi, tok4[s_lo], tok4[s_hi]) + t_lo, t_hi = ffma2(dy1[2 * j], dy1[2 * j + 1], d1_lo, d1_hi, t_lo, t_hi) + tok4[s_lo] = t_lo + tok4[s_hi] = t_hi + + # ---- dbeta = sum_v dY.(v - state_k) + M-term ------------------------- + nvvm.barrier_cta_sync(cfg.cg1_sync_barrier_id, thread_count=cfg.cg1_threads) + for off in cutlass.range_constexpr(3): + step = cutlass.const_expr(4 << off) + for s in cutlass.range_constexpr(4): + tok4[s] = tok4[s] + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, tok4[s], step, 31, kind=nvvm.Shfl.BFLY)) + if lane < 4: + for s in cutlass.range_constexpr(4): + sRed_raw[(warp_idx % 4) * cfg.b_t + (lane % 4) * 2 + (s % 2) + 8 * (s // 2)] = tok4[s] + bars.mb_dbeta_m_ready.wait(dbeta_m_index.phase) + dbeta_m_index = advance(dbeta_m_index, 1) + nvvm.barrier_cta_sync(cfg.cg1_sync_barrier_id, thread_count=cfg.cg1_threads) + if cg1_tidx < cfg.b_t: + acc = cutlass.Float32(0.0) + for w in cutlass.range_constexpr(4): + acc = acc + sRed_raw[w * cfg.b_t + cg1_tidx] + db_val = acc + sBetaM_raw[cg1_tidx] + token_idx = chunk_idx * cutlass.Int32(cfg.b_t) + cg1_tidx + if token_idx < seqlen_b and chunk_idx < wend: + mDbeta[batch_start + token_idx, head_idx] = db_val + nvvm.barrier_cta_sync(cfg.cg1_sync_barrier_id, thread_count=cfg.cg1_threads) + bars.mb_dbeta_m_done.arrive() + bars.mb_v_done[raw_index.idx].arrive() + + # ---- dH capture for the next ----------------------------------------- + bars.mb_dstate_acc_ready.wait(dstate_ready_index.phase) + dstate_ready_index = advance(dstate_ready_index, 1) + if rev_idx + cutlass.Int32(1) < num_compute_chunks: + bars.mb_dstate_inp_done.wait(dstate_inp_done_index.phase) + dstate_inp_done_index = advance(dstate_inp_done_index, 1) + bars.mb_dstate_smem_done.wait(dstate_smem_done_index.phase) + bars.mb_dstate_smem_cg2_done.wait(dstate_smem_done_index.phase) + dstate_smem_done_index = advance(dstate_smem_done_index, 1) + row_addr = (tmem_row + tmem_subpartition * cfg.threads_per_warp) << 16 + for sub in cutlass.range_constexpr(cfg.d_k // 32): + dstate_block = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_acc_offset + sub * 32), cutlass.Float32), num=32 + ) + packed_dstate = cutlass.Array(cutlass.Int32, 16, alignment=16) + for pc in cutlass.range_constexpr(16): + packed_dstate[pc] = fp32_to_fp16(dstate_block[2 * pc], dstate_block[2 * pc + 1], dtype=cfg.io_dtype) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_inp_offset + sub * 16), cutlass.Int8), + packed_dstate[0:16], + ) + for half in cutlass.range_constexpr(4): + d_base = sub * 32 + half * 8 + h_vec = cutlass.Vector.from_elements( + (packed_dstate[half * 4], packed_dstate[half * 4 + 1], packed_dstate[half * 4 + 2], packed_dstate[half * 4 + 3]), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + h_addr = (d_base // 64) * (cfg.d_v * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, d_base % 64, elem_bytes=2) + (sDstate_raw.data_ptr() + h_addr).store(h_vec, alignment=16) + nvvm.tcgen05_wait("store") + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dstate_inp_ready.arrive() + bars.mb_dstate_smem_ready.arrive() + bars.mb_dstate_acc_done.arrive() + raw_index = advance(raw_index, cfg.smem_raw_stages) + + # ---- tile end: dS0 drain / zero-length pass-through ---------------------- + if cutlass.const_expr(mDstate0 is not None): + if num_compute_chunks > 0: + if wstart == 0: + row_addr = (tmem_row + tmem_subpartition * cfg.threads_per_warp) << 16 + for sub in cutlass.range_constexpr(cfg.d_k // 32): + dstate0_vec = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_acc_offset + sub * 32), cutlass.Float32), num=32 + ) + for kk_i in cutlass.range_constexpr(32): + mDstate0[batch_idx, head_idx, sub * 32 + kk_i, value_dim] = dstate0_vec[kk_i] + else: + for key_dim_base in cutlass.range_constexpr(0, cfg.d_k, 32): + for kk_i in cutlass.range_constexpr(32): + kd = key_dim_base + kk_i + if cutlass.const_expr(cfg.use_dstate_in): + mDstate0[batch_idx, head_idx, kd, value_dim] = mDstate_in[batch_idx, head_idx, kd, value_dim] + else: + mDstate0[batch_idx, head_idx, kd, value_dim] = cutlass.Float32(0.0) + bars.mb_dstate0_stored.arrive() + chunk_serial_base += num_compute_chunks + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def compute2_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_base_holder, + warp_idx, + sQ_raw, + sK_raw, + sGate_raw, + sNorm_raw, + sDq_raw, + sDk_raw, + sRed1_raw, + sDstate_raw, + sDg_raw, + scale, + bars, +) -> None: + """WG2 warp role (warps 8-11): the gradient drain. Each thread owns one + key channel d for all 16 tokens: reads the dq accumulator and the four dk + parts from TMEM, assembles dq/dk with the per-channel gate factors, + applies the in-kernel L2-norm backward row projection, assembles the + per-channel dG including the g_last terms, reverse-cumsums it in + registers, stages dgate for the epilogue's TMA store, and + stages dq/dk for the epilogue's TMA stores.""" + nvvm.setmaxregister(cfg.num_regs_compute_group_2, nvvm.SetMaxRegisterAction.INCREASE) + tmem_base = tmem_base_holder.load() + tmem_col = tmem_base & 0xFFFF + tmem_row = tmem_base >> 16 + tmem_subpartition = warp_idx % 4 + channel = tmem_subpartition * cfg.threads_per_warp + lane + row_addr = (tmem_row + tmem_subpartition * cfg.threads_per_warp) << 16 + cg2_tidx = channel + + raw_index = PipelineState.start(phase=0) + dq_acc_index = PipelineState.start(phase=0) + dk_decay_part_index = PipelineState.start(phase=0) + dk_inv_part_index = PipelineState.start(phase=0) + dk_restore_part_index = PipelineState.start(phase=0) + dgl_dstate_smem_index = PipelineState.start(phase=0) + chunk_serial_base = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + FIRST_STATE_CHUNK = 0 if cfg.use_initial_state else 1 + SFIRST_MIN = 1 if cfg.use_initial_state else 2 + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + num_compute_chunks = cend - wstart + for rev_idx in cutlass.range(num_compute_chunks, unroll=1): + chunk_idx = cend - cutlass.Int32(1) - rev_idx + chunk_serial = chunk_serial_base + rev_idx + chunk_start = chunk_idx * cfg.b_t + raw_stage = chunk_serial % cfg.smem_raw_stages + decay_stage = chunk_serial % cfg.smem_decay_stages + has_dstate = cutlass.Boolean(rev_idx > 0) + if cutlass.const_expr(cfg.use_dstate_in): + has_dstate = cutlass.Boolean(True) + sQ_ptr = sQ_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sK_ptr = sK_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sGate_ptr = sGate_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + writes = chunk_idx < wend + + bars.mb_k_decay_inv_ready[decay_stage].wait((chunk_serial // cfg.smem_decay_stages) % 2) + + # ---- per-channel gate factors ---------------------------------------- + f32_seg = channel // 32 + f32_dim = channel - f32_seg * 32 + f16_seg = channel // 64 + f16_dim = channel - f16_seg * 64 + eg = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + for t in cutlass.range_constexpr(cfg.b_t): + eg[t] = (sGate_ptr + f32_seg * (cfg.b_t * 32) + t * 32 + swizzle_xor_128b(t, f32_dim, elem_bytes=4)).load() + egl = eg[cfg.b_t - 1] + + # ---- dGlast hdot: sum_v sdH[v, c] * S0[c, v] ------------------------- + dgl_val = cutlass.Float32(0.0) + if has_dstate: + bars.mb_dstate_smem_ready.wait(dgl_dstate_smem_index.phase) + dgl_dstate_smem_index = advance(dgl_dstate_smem_index, 1) + bars.mb_state_inp_ready[chunk_serial % 2].wait((chunk_serial // 2) % 2) + for pl in cutlass.range_constexpr(2): + for row_half in cutlass.range_constexpr(2): + state_words = nvvm.tcgen05_ld( + "32x32b", + nvvm.make_tmem_ptr( + row_addr + (tmem_col + cfg.tmem_state_inp_offset + (chunk_serial % 2) * (cfg.d_v // 2) + pl * 32 + row_half * 16), cutlass.Float32 + ), + num=16, + ) + hacc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for i in cutlass.range_constexpr(8): + hacc[i] = opaque_f32_zero() + for j in cutlass.range_constexpr(16): + v0 = pl * 64 + row_half * 32 + 2 * j + state_pair = cutlass.Vector.from_elements((state_words[j],), cutlass.Float32).bitcast(cfg.io_dtype) + dstate_addr0 = (channel // 64) * (cfg.d_v * 64) + v0 * 64 + swizzle_xor_128b(v0, channel % 64, elem_bytes=2) + dstate_addr1 = (channel // 64) * (cfg.d_v * 64) + (v0 + 1) * 64 + swizzle_xor_128b(v0 + 1, channel % 64, elem_bytes=2) + hval0 = (sDstate_raw.data_ptr() + dstate_addr0).load().to(cutlass.Float32) + hval1 = (sDstate_raw.data_ptr() + dstate_addr1).load().to(cutlass.Float32) + hacc[(2 * j) % 8] = hacc[(2 * j) % 8] + hval0 * state_pair[0].to(cutlass.Float32) + hacc[(2 * j + 1) % 8] = hacc[(2 * j + 1) % 8] + hval1 * state_pair[1].to(cutlass.Float32) + part_a = (hacc[0] + hacc[4]) + (hacc[1] + hacc[5]) + part_b = (hacc[2] + hacc[6]) + (hacc[3] + hacc[7]) + dgl_val = dgl_val + (part_a + part_b) + bars.mb_dstate_smem_cg2_done.arrive() + bars.mb_state_inp_cg2_done[chunk_serial % 2].arrive() + + # ---- part-drain accumulators ------------------------------------------- + dq_n = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + dk_n = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + dg_reg = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + dgl_acc = cutlass.Array(cutlass.Float32, 4, alignment=16) + for i in cutlass.range_constexpr(4): + dgl_acc[i] = opaque_f32_zero() + for t in cutlass.range_constexpr(cfg.b_t): + dk_n[t] = cutlass.Float32(0.0) + + # ---- dk_restore part drain: (eGl/eG) scale + dGlast k-dot -------------- + if has_dstate: + bars.mb_dk_restore_part_ready.wait(dk_restore_part_index.phase) + dk_restore_part_index = advance(dk_restore_part_index, 1) + dk_restore_part = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_restore_acc_offset), cutlass.Float32), num=cfg.b_t + ) + for t in cutlass.range_constexpr(cfg.b_t): + dk_hat = egl * cute.math.rcp(eg[t], approx=True, ftz=True) * dk_restore_part[t] + dk_n[t] = dk_hat + k_v = (sK_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) + if cutlass.const_expr(cfg.l2norm): + k_v = k_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + cfg.b_t + t] + dgl_acc[t % 4] = dgl_acc[t % 4] + k_v * dk_hat + + # ---- dq acc drain: eG.scale --------------------------------------------- + bars.mb_dq_acc_ready.wait(dq_acc_index.phase) + dq_acc_index = advance(dq_acc_index, 1) + dq_acc = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dq_acc_offset), cutlass.Float32), num=cfg.b_t) + for t2 in cutlass.range_constexpr(cfg.b_t // 2): + t = 2 * t2 + es_lo, es_hi = fmul2(eg[t], eg[t + 1], scale, scale) + dq_n[t], dq_n[t + 1] = fmul2(es_lo, es_hi, dq_acc[t], dq_acc[t + 1]) + + # ---- dk_inv part drain: (dA - dM) term, 1/eG scale ---------------------- + bars.mb_dk_inv_part_ready.wait(dk_inv_part_index.phase) + dk_inv_part_index = advance(dk_inv_part_index, 1) + dk_inv_part = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), num=cfg.b_t) + for t in cutlass.range_constexpr(cfg.b_t): + dk_n[t] = dk_n[t] + dk_inv_part[t] * cute.math.rcp(eg[t], approx=True, ftz=True) + + # ---- dk_decay part drain: -eG scale, seeds dG --------------------------- + bars.mb_dk_decay_part_ready.wait(dk_decay_part_index.phase) + dk_decay_part_index = advance(dk_decay_part_index, 1) + dk_decay_part = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), num=cfg.b_t) + for t in cutlass.range_constexpr(cfg.b_t): + dg_reg[t] = -eg[t] * dk_decay_part[t] + dk_n[t] = dk_n[t] + dg_reg[t] + + # ---- all parts drained: release the accs to the next chunk's MMAs ------ + bars.mb_parts_done.arrive() + + # ---- dG finalize ----------------------------------------------------- + for t in cutlass.range_constexpr(cfg.b_t): + q_v = (sQ_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) + k_v = (sK_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) + if cutlass.const_expr(cfg.l2norm): + q_v = q_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + t] + k_v = k_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + cfg.b_t + t] + dg_reg[t] = q_v * dq_n[t] + k_v * (cutlass.Float32(2.0) * dg_reg[t] - dk_n[t]) + dg_reg[cfg.b_t - 1] = dg_reg[cfg.b_t - 1] + ((dgl_acc[0] + dgl_acc[1]) + (dgl_acc[2] + dgl_acc[3])) + + # ---- L2-norm backward row projection --------------------------------- + if cutlass.const_expr(cfg.l2norm): + for grad, src_ptr, inv_off in ((dq_n, sQ_ptr, 0), (dk_n, sK_ptr, cfg.b_t)): + dots = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + for t in cutlass.range_constexpr(cfg.b_t): + qn_v = (src_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) + dots[t] = grad[t] * qn_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + inv_off + t] + for off in cutlass.range_constexpr(5): + step = cutlass.const_expr(1 << off) + for t in cutlass.range_constexpr(cfg.b_t): + dots[t] = dots[t] + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, dots[t], step, 31, kind=nvvm.Shfl.BFLY)) + if lane == 0: + for t in cutlass.range_constexpr(cfg.b_t): + sRed1_raw[tmem_subpartition * cfg.b_t + t] = dots[t] + nvvm.barrier_cta_sync(cfg.cg2_sync_barrier_id, thread_count=cfg.cg2_threads) + for t in cutlass.range_constexpr(cfg.b_t): + qn_v = (src_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) + total_dot = sRed1_raw[t] + sRed1_raw[cfg.b_t + t] + sRed1_raw[2 * cfg.b_t + t] + sRed1_raw[3 * cfg.b_t + t] + norm_t = sNorm_raw[raw_stage * (2 * cfg.b_t) + inv_off + t] + grad[t] = (grad[t] - qn_v * norm_t * total_dot) * norm_t + nvvm.barrier_cta_sync(cfg.cg2_sync_barrier_id, thread_count=cfg.cg2_threads) + + # ---- stage dq/dk for the epilogue TMA stores ------------------------- + dq_stage = chunk_serial % cfg.smem_dq_stages + dk_stage = chunk_serial % cfg.smem_dk_stages + bars.mb_dq_tmastg_done[dq_stage].wait(((chunk_serial // cfg.smem_dq_stages) + 1) % 2) + bars.mb_dk_tmastg_done[dk_stage].wait(((chunk_serial // cfg.smem_dk_stages) + 1) % 2) + dq_base = dq_stage * (cfg.b_t * cfg.d_k) + dk_base = dk_stage * (cfg.b_t * cfg.d_k) + for t in cutlass.range_constexpr(cfg.b_t): + out_idx = f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2) + (sDq_raw.data_ptr() + dq_base + out_idx).store(dq_n[t].to(cfg.io_dtype)) + (sDk_raw.data_ptr() + dk_base + out_idx).store(dk_n[t].to(cfg.io_dtype)) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dq_tmastg_ready[dq_stage].arrive() + bars.mb_dk_tmastg_ready[dk_stage].arrive() + + # ---- dGlast add ------------------------------------------------------ + if has_dstate: + if chunk_idx >= FIRST_STATE_CHUNK: + dg_reg[cfg.b_t - 1] = dg_reg[cfg.b_t - 1] + egl * dgl_val + + # ---- dG reverse cumsum ----------------------------------------------- + suffix = cutlass.Float32(0.0) + for rt in cutlass.range_constexpr(cfg.b_t): + t = cfg.b_t - 1 - rt + suffix = suffix + dg_reg[t] + dg_reg[t] = suffix + + # ---- stage dGate for the epilogue TMA store -------------------------- + dg_stage = chunk_serial % cfg.smem_dg_stages + bars.mb_dg_tmastg_done[dg_stage].wait(((chunk_serial // cfg.smem_dg_stages) + 1) % 2) + for t in cutlass.range_constexpr(cfg.b_t): + dg_idx = f32_seg * (cfg.b_t * 32) + t * 32 + swizzle_xor_128b(t, f32_dim, elem_bytes=4) + (sDg_raw.data_ptr() + dg_idx).store(dg_reg[t]) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_dg_tmastg_ready[dg_stage].arrive() + + bars.mb_q_done[raw_stage].arrive() + bars.mb_k_done[raw_stage].arrive() + bars.mb_gate_done[raw_stage].arrive() + raw_index = advance(raw_index, cfg.smem_raw_stages) + chunk_serial_base += num_compute_chunks + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +# --------------------------------------------------------------------------- +# Host-side assembly +# --------------------------------------------------------------------------- + + +@cute.kernel +def build_all_descs_kernel( + base_q: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_k: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_v: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_gate: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_do: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_dq: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_dk: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_dv: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_dg: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_checkpoint: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_initial_state: cutlass.GridConstant[cuda.tensor_map.TensorMap], + desc_ws: cute.Tensor, + cu_seqlens: cute.Tensor, + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + gate: cute.Tensor, + do: cute.Tensor, + dq: cute.Tensor, + dk: cute.Tensor, + dv: cute.Tensor, + dg: cute.Tensor, + state_checkpoints: cute.Tensor, + state0: cute.Tensor | None, + n_batch: cutlass.Int32, + q_row_stride: cutlass.Int32, + k_row_stride: cutlass.Int32, + v_row_stride: cutlass.Int32, + gate_row_stride: cutlass.Int32, + do_row_stride: cutlass.Int32, + dq_row_stride: cutlass.Int32, + dk_row_stride: cutlass.Int32, + dv_row_stride: cutlass.Int32, + dg_row_stride: cutlass.Int32, + checkpoint_row_stride: cutlass.Int32, + checkpoint_every_n: cutlass.Int32, +) -> None: + """Single-launch builder for the per-batch descriptor arrays (one warp + per array).""" + tidx, _, _ = cute.arch.thread_idx() + widx = cutlass.Int32(tidx) // cutlass.Int32(32) + arr_words = n_batch * cutlass.Int32(TENSOR_MAP_QWORDS) + sub0 = cute.make_tensor(desc_ws.iterator, cute.make_layout((arr_words,), stride=(1,))) + sub1 = cute.make_tensor(desc_ws.iterator + arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub2 = cute.make_tensor(desc_ws.iterator + 2 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub3 = cute.make_tensor(desc_ws.iterator + 3 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub4 = cute.make_tensor(desc_ws.iterator + 4 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub5 = cute.make_tensor(desc_ws.iterator + 5 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub6 = cute.make_tensor(desc_ws.iterator + 6 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub7 = cute.make_tensor(desc_ws.iterator + 7 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub8 = cute.make_tensor(desc_ws.iterator + 8 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub9 = cute.make_tensor(desc_ws.iterator + 9 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + sub10 = cute.make_tensor(desc_ws.iterator + 10 * arr_words, cute.make_layout((cutlass.Int32(TENSOR_MAP_QWORDS),), stride=(1,))) + + if widx == 0: + if nvvm.elect_sync(): + emit_seq_descs(base_q, sub0, cu_seqlens, q, n_batch, q_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 1: + if nvvm.elect_sync(): + emit_seq_descs(base_k, sub1, cu_seqlens, k, n_batch, k_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 2: + if nvvm.elect_sync(): + emit_seq_descs(base_v, sub2, cu_seqlens, v, n_batch, v_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 3: + if nvvm.elect_sync(): + emit_seq_descs(base_gate, sub3, cu_seqlens, gate, n_batch, gate_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 4: + if nvvm.elect_sync(): + emit_seq_descs(base_do, sub4, cu_seqlens, do, n_batch, do_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 5: + if nvvm.elect_sync(): + emit_seq_descs(base_dq, sub5, cu_seqlens, dq, n_batch, dq_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 6: + if nvvm.elect_sync(): + emit_seq_descs(base_dk, sub6, cu_seqlens, dk, n_batch, dk_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 7: + if nvvm.elect_sync(): + emit_seq_descs(base_dv, sub7, cu_seqlens, dv, n_batch, dv_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 8: + if nvvm.elect_sync(): + emit_seq_descs(base_dg, sub8, cu_seqlens, dg, n_batch, dg_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 9: + if nvvm.elect_sync(): + emit_checkpoint_seq_descs(base_checkpoint, sub9, cu_seqlens, state_checkpoints, n_batch, checkpoint_row_stride, checkpoint_every_n, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if cutlass.const_expr(state0 is not None): + if widx == 10: + if nvvm.elect_sync(): + emit_copy_desc(base_initial_state, sub10) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + + +@cute.jit +def build_descs( + io_dtype: cutlass.Constexpr, + b_t: cutlass.Constexpr[int], + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + gate: cute.Tensor, + do: cute.Tensor, + dq: cute.Tensor, + dk: cute.Tensor, + dv: cute.Tensor, + dg: cute.Tensor, + state_checkpoints: cute.Tensor, + state0: cute.Tensor | None, + cu_seqlens: cute.Tensor, + tensormap_workspace: cute.Tensor, + stream: cuda_driver.CUstream, +): + """Build the 11 per-(batch, head) capped TMA-descriptor arrays into + ``tensormap_workspace`` (sequence-relative coordinates; tail loads + zero-fill and tail stores clip in hardware).""" + h_q = q.shape[1] + h_k = k.shape[1] + h_v = v.shape[1] + ho = gate.shape[1] + batch_size = cu_seqlens.shape[0] - 1 + d_k = q.shape[2] + d_v = v.shape[2] + bpe = io_dtype.width // 8 + tma_granu_elems = 128 // bpe + seqlen = q.shape[0] + + def headed(t, dim, hn): + return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) + + swz = cuda.TensorMapSwizzle.s128b + base_q = cuda.create_tensor_map_tiled_from_view(headed(q, d_k, h_q), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_do = cuda.create_tensor_map_tiled_from_view(headed(do, d_v, ho), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dq = cuda.create_tensor_map_tiled_from_view(headed(dq, d_k, ho), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dk = cuda.create_tensor_map_tiled_from_view(headed(dk, d_k, ho), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dv = cuda.create_tensor_map_tiled_from_view(headed(dv, d_v, ho), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dg = cuda.create_tensor_map_tiled_from_view(headed(dg, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + + checkpoint_view = cute.make_tensor( + state_checkpoints.iterator, + cute.make_layout( + (d_v, d_k, state_checkpoints.shape[0], ho), + stride=(state_checkpoints.stride[3], state_checkpoints.stride[2], state_checkpoints.stride[0], state_checkpoints.stride[1]), + ), + ) + base_checkpoint = cuda.create_tensor_map_tiled_from_view(checkpoint_view, box_dims=(64, d_k, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz) + base_initial_state = base_checkpoint + if cutlass.const_expr(state0 is not None): + initial_state_view = cute.make_tensor( + state0.iterator, + cute.make_layout( + (d_v, d_k, ho, batch_size), + stride=(state0.stride[3], state0.stride[2], state0.stride[1], state0.stride[0]), + ), + ) + base_initial_state = cuda.create_tensor_map_tiled_from_view(initial_state_view, box_dims=(64, d_k, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz) + + n_warps = 11 if state0 is not None else 10 + build_all_descs_kernel( + base_q, + base_k, + base_v, + base_gate, + base_do, + base_dq, + base_dk, + base_dv, + base_dg, + base_checkpoint, + base_initial_state, + tensormap_workspace, + cu_seqlens, + q, + k, + v, + gate, + do, + dq, + dk, + dv, + dg, + state_checkpoints, + state0, + cutlass.Int32(batch_size), + cutlass.Int32(q.stride[0]), + cutlass.Int32(k.stride[0]), + cutlass.Int32(v.stride[0]), + cutlass.Int32(gate.stride[0]), + cutlass.Int32(do.stride[0]), + cutlass.Int32(dq.stride[0]), + cutlass.Int32(dk.stride[0]), + cutlass.Int32(dv.stride[0]), + cutlass.Int32(dg.stride[0]), + cutlass.Int32(state_checkpoints.stride[0]), + cutlass.Int32(b_t), + ).launch(grid=(1, 1, 1), block=(32 * n_warps, 1, 1), stream=stream) + + +@cute.jit +def host( + cfg: cutlass.Constexpr, + beta: cute.Tensor, + state_checkpoints: cute.Tensor, + mState_init: cute.Tensor | None, + dg: cute.Tensor, + dbeta: cute.Tensor, + cu_seqlens: cute.Tensor, + d_initial_state: cute.Tensor | None, + d_final_state: cute.Tensor | None, + work_items: cute.Tensor | None, + work_count: cute.Tensor | None, + sched_ctr: cute.Tensor | None, + tensormap_workspace: cute.Tensor, + scale: cutlass.Float32, + stream, +) -> None: + num_sequences = cu_seqlens.shape[0] - 1 + ho = dg.shape[1] + + # ---- launch ------------------------------------------------------------------ + total_tiles = num_sequences * ho + n_desc = num_sequences + grid_shape = (cfg.max_active_clusters, 1, 1) + kernel( + cfg, + tensormap_workspace, + n_desc, + beta, + cu_seqlens, + dg, + dbeta, + d_initial_state, + d_final_state, + work_items, + work_count, + sched_ctr, + total_tiles, + scale, + ).launch( + grid=grid_shape, + block=(cfg.threads_per_cta, 1, 1), + stream=stream, + min_blocks_per_mp=1, + ) + + +@cute.kernel +def kernel( + cfg: cutlass.Constexpr, + tensormap_workspace: cute.Tensor, + n_desc: cutlass.Int32, + mBeta: cute.Tensor, + cu_seqlens: cute.Tensor, + mDg: cute.Tensor, + mDbeta: cute.Tensor, + mDstate0: cute.Tensor | None, + mDstate_in: cute.Tensor | None, + mWorkItems: cute.Tensor | None, + mCount: cute.Tensor | None, + mSched: cute.Tensor | None, + total_tiles: cutlass.Int32, + scale: cutlass.Float32, +) -> None: + """BT=16 KDA backward kernel (persistent, 16 warps).""" + tidx, _, _ = cute.arch.thread_idx() + bidx = cute.arch.block_idx()[0] + num_ctas = cute.arch.grid_dim()[0] + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + lane = tidx % cfg.threads_per_warp + + if cutlass.const_expr(cfg.split_k): + total_tiles = mCount[0] + assert mBeta.element_type == cutlass.Float32 + assert cu_seqlens.element_type in (cutlass.Int32, cutlass.Int64) + assert mDg.element_type == cutlass.Float32 and mDbeta.element_type == cutlass.Float32 + + desc_base_words = tensormap_workspace.iterator.raw_ptr() + arr_words = n_desc * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_q_base = desc_base_words + desc_k_base = desc_base_words + arr_words + desc_v_base = desc_base_words + cutlass.Int32(2) * arr_words + desc_gate_base = desc_base_words + cutlass.Int32(3) * arr_words + desc_do_base = desc_base_words + cutlass.Int32(4) * arr_words + desc_dq_base = desc_base_words + cutlass.Int32(5) * arr_words + desc_dk_base = desc_base_words + cutlass.Int32(6) * arr_words + desc_dv_base = desc_base_words + cutlass.Int32(7) * arr_words + desc_dg_base = desc_base_words + cutlass.Int32(8) * arr_words + desc_checkpoint_base = desc_base_words + cutlass.Int32(9) * arr_words + desc_initial_state_base = desc_base_words + cutlass.Int32(10) * arr_words + + SMEM = cutlass.AddressSpace.smem + bars = make_kda_bwd_bars(cfg) + tmem_base_holder = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=4) + sSched = cutlass.Array(cutlass.Int32, cfg.sched_stages, space=SMEM, alignment=16) + bpe = cfg.io_dtype.width // 8 + SWZ = 2 + LEAD = 16 + STRIDE = 8 * 128 + STATE_ALT_LEAD = cfg.d_v * 128 + + sQ_raw = cutlass.Array(cfg.io_dtype, cfg.raw_qk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_raw = cutlass.Array(cfg.io_dtype, cfg.raw_qk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sV_raw = cutlass.Array(cfg.io_dtype, cfg.raw_v_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sDo_raw = cutlass.Array(cfg.io_dtype, cfg.raw_v_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sGate_raw = cutlass.Array(cutlass.Float32, cfg.raw_gate_cosize, space=SMEM, alignment=1024) + sBeta_raw = cutlass.Array(cutlass.Float32, cfg.smem_beta_stages * cfg.b_t, space=SMEM, alignment=64) + sNorm_raw = cutlass.Array(cutlass.Float32, cfg.smem_raw_stages * 2 * cfg.b_t, space=SMEM, alignment=64) + sK_decay_raw = cutlass.Array(cfg.io_dtype, cfg.operand_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_inv_raw = cutlass.Array(cfg.io_dtype, cfg.operand_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_restore_raw = cutlass.Array(cfg.io_dtype, cfg.operand_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sQ_decay_raw = cutlass.Array(cfg.io_dtype, cfg.operand_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sState_scale_diag_raw = cutlass.Array(cfg.io_dtype, cfg.diag_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sQk_raw = cutlass.Array(cfg.io_dtype, cfg.qk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sState_raw = cutlass.Array(cfg.io_dtype, cfg.state_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sDstate_raw = cutlass.Array(cfg.io_dtype, cfg.d_k * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) + sU_raw = cutlass.Array(cfg.io_dtype, cfg.b_t * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) + sDy_raw = cutlass.Array(cfg.io_dtype, cfg.b_t * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) + sDq_raw = cutlass.Array(cfg.io_dtype, cfg.dq_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sDk_raw = cutlass.Array(cfg.io_dtype, cfg.dk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sDv_raw = cutlass.Array(cfg.io_dtype, cfg.dv_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sDg_raw = cutlass.Array(cutlass.Float32, cfg.dg_cosize, space=SMEM, alignment=1024) + sRed_raw = cutlass.Array(cutlass.Float32, 4 * cfg.b_t, space=SMEM, alignment=64) + sRed1_raw = cutlass.Array(cutlass.Float32, 4 * cfg.b_t, space=SMEM, alignment=64) + sBetaM_raw = cutlass.Array(cutlass.Float32, cfg.b_t, space=SMEM, alignment=64) + + sState_alt = SmemTile( + base=sState_raw.data_ptr().toint(), + elems_per_stage=((cfg.state_cosize) // (cfg.smem_state_stages)) * bpe, + stages=cfg.smem_state_stages, + leading_byte_offset=STATE_ALT_LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sK_decay_lead16 = SmemTile( + base=sK_decay_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sK_inv_lead16 = SmemTile( + base=sK_inv_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sK_restore_lead16 = SmemTile( + base=sK_restore_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sDo_lead16 = SmemTile( + base=sDo_raw.data_ptr().toint(), + elems_per_stage=((cfg.raw_v_cosize) // (cfg.smem_raw_stages)) * bpe, + stages=cfg.smem_raw_stages, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sDo_amaj = SmemTile( + base=sDo_raw.data_ptr().toint(), + elems_per_stage=((cfg.raw_v_cosize) // (cfg.smem_raw_stages)) * bpe, + stages=cfg.smem_raw_stages, + leading_byte_offset=cfg.b_t * 128, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sU_lead16 = SmemTile( + base=sU_raw.data_ptr().toint(), + elems_per_stage=((cfg.b_t * cfg.d_v) // (1)) * bpe, + stages=1, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sDv_lead16 = SmemTile( + base=sDv_raw.data_ptr().toint(), + elems_per_stage=((cfg.dv_cosize) // (cfg.smem_dv_stages)) * bpe, + stages=cfg.smem_dv_stages, + leading_byte_offset=LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sDstate_alt = SmemTile( + base=sDstate_raw.data_ptr().toint(), + elems_per_stage=((cfg.d_k * cfg.d_v) // (1)) * bpe, + stages=1, + leading_byte_offset=STATE_ALT_LEAD, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + + sQ_decay_trans = SmemTile( + base=sQ_decay_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=cfg.b_t * 128, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sK_inv_amaj = SmemTile( + base=sK_inv_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=cfg.b_t * 128, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sK_decay_trans = SmemTile( + base=sK_decay_raw.data_ptr().toint(), + elems_per_stage=((cfg.operand_cosize) // (cfg.smem_decay_stages)) * bpe, + stages=cfg.smem_decay_stages, + leading_byte_offset=cfg.b_t * 128, + stride_byte_offset=STRIDE, + layout=SWZ, + ) + sState_scale_diag = SmemTile( + base=sState_scale_diag_raw, + elems_per_stage=((cfg.d_k // 16) * 256), + stages=cfg.smem_decay_stages, + leading_byte_offset=16, + stride_byte_offset=(8 * 16 * 2), + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, + ) + sQk = SmemTile( + base=sQk_raw, + elems_per_stage=(cfg.qk_tiles * cfg.b_t * cfg.b_t), + stages=cfg.smem_qk_stages, + leading_byte_offset=16, + stride_byte_offset=(8 * cfg.b_t * 2), + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, + ) + + if warp_idx == cfg.tma_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.smem_raw_stages): + bars.mb_q_ready[stage].init() + bars.mb_q_done[stage].init() + bars.mb_k_ready[stage].init() + bars.mb_k_done[stage].init() + bars.mb_gate_ready[stage].init() + bars.mb_gate_done[stage].init() + bars.mb_do_ready[stage].init() + bars.mb_do_done[stage].init() + bars.mb_do_epi_done[stage].init() + bars.mb_v_ready[stage].init() + bars.mb_v_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_beta_stages): + bars.mb_beta_ready[stage].init() + bars.mb_beta_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_state_stages): + bars.mb_state_ready[stage].init() + bars.mb_state_done[stage].init() + bars.mb_state_cg0_done[stage].init() + for stage in cutlass.range_constexpr(2): + bars.mb_state_inp_ready[stage].init() + bars.mb_state_inp_done[stage].init() + bars.mb_state_inp_cg2_done[stage].init() + elif warp_idx == cfg.tcgen05_mma_warp_id: + if nvvm.elect_sync(): + bars.mb_state_k_acc_ready.init() + bars.mb_rhs_ready.init() + bars.mb_update_acc_ready.init() + bars.mb_u_smem_ready.init() + bars.mb_u_done.init() + bars.mb_u_warps_done.init() + bars.mb_du_acc_ready.init() + bars.mb_du_inp_ready.init() + bars.mb_du_inp_done.init() + bars.mb_dy_acc_ready.init() + bars.mb_neg_beta_dy_inp_ready.init() + bars.mb_neg_beta_dy_inp_done.init() + bars.mb_sdy_ready.init() + bars.mb_sdy_done.init() + bars.mb_dstate_acc_ready.init() + bars.mb_dstate_acc_done.init() + bars.mb_dstate_inp_ready.init() + bars.mb_dstate_inp_done.init() + bars.mb_dstate_smem_ready.init() + bars.mb_dstate_smem_done.init() + bars.mb_dstate_smem_cg2_done.init() + bars.mb_dq_acc_ready.init() + bars.mb_dk_decay_part_ready.init() + bars.mb_dk_inv_part_ready.init() + bars.mb_dk_restore_part_ready.init() + bars.mb_parts_done.init() + bars.mb_dbeta_m_ready.init() + bars.mb_dbeta_m_done.init() + bars.mb_dstate0_stored.init() + elif warp_idx == cfg.super_mma_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.smem_decay_stages): + bars.mb_k_decay_inv_ready[stage].init() + bars.mb_q_decay_k_restore_ready[stage].init() + bars.mb_decay_done[stage].init() + bars.mb_decay_super_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_qk_stages): + bars.mb_a_ready[stage].init() + bars.mb_aqk_ready[stage].init() + bars.mb_da_ready[stage].init() + bars.mb_dm_ready[stage].init() + bars.mb_aqk_done[stage].init() + bars.mb_a_done[stage].init() + bars.mb_da_done[stage].init() + bars.mb_dm_done[stage].init() + elif warp_idx == cfg.epilogue_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.smem_dq_stages): + bars.mb_dq_tmastg_ready[stage].init() + bars.mb_dq_tmastg_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_dk_stages): + bars.mb_dk_tmastg_ready[stage].init() + bars.mb_dk_tmastg_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_dg_stages): + bars.mb_dg_tmastg_ready[stage].init() + bars.mb_dg_tmastg_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_dv_stages): + bars.mb_dv_tmastg_ready[stage].init() + bars.mb_dv_tmastg_done[stage].init() + bars.mb_dv_epi_done[stage].init() + for stage in cutlass.range_constexpr(cfg.sched_stages): + bars.mb_sched_ready[stage].init() + bars.mb_sched_done[stage].init() + diag_zero = cfg.io_dtype(0.0) + for diag_idx in cutlass.range(tidx, cfg.diag_cosize, cfg.threads_per_cta, unroll=1): + sState_scale_diag_raw[diag_idx] = diag_zero + nvvm.fence_mbarrier_init() + nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) + is_tmem_user = ( + (warp_idx >= cfg.compute_group_2_warp_ids[0] and warp_idx <= cfg.compute_group_2_warp_ids[-1]) + or (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) + or (warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]) + or warp_idx == cfg.tcgen05_mma_warp_id + ) + if is_tmem_user: + if warp_idx == cfg.tcgen05_mma_warp_id: + nvvm.tcgen05_alloc(tmem_base_holder, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) + if warp_idx == cfg.tcgen05_mma_warp_id: + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) + + if warp_idx == cfg.tma_warp_id: + tmaldg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + mSched, + sSched, + lane, + sQ_raw, + sK_raw, + sV_raw, + sGate_raw, + sDo_raw, + sState_raw, + desc_q_base, + desc_k_base, + desc_v_base, + desc_gate_base, + desc_do_base, + desc_checkpoint_base, + desc_initial_state_base, + bars, + ) + elif warp_idx == cfg.super_mma_warp_id: + super_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_decay_raw, + sK_inv_raw, + sU_raw, + sDy_raw, + sQk_raw, + sBeta_raw, + sBetaM_raw, + bars, + ) + elif warp_idx == cfg.tcgen05_mma_warp_id: + tcgen05_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + tmem_base_holder, + sState_alt, + sK_decay_lead16, + sK_inv_lead16, + sK_inv_amaj, + sK_restore_lead16, + sDo_lead16, + sDo_amaj, + sQ_decay_trans, + sK_decay_trans, + sU_lead16, + sDv_lead16, + sDstate_alt, + sQk, + sState_scale_diag, + bars, + ) + elif warp_idx == cfg.epilogue_warp_id: + epilogue_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_inv_raw, + sQ_decay_raw, + sDo_raw, + sU_raw, + sQk_raw, + sDq_raw, + sDk_raw, + sDv_raw, + sDg_raw, + desc_dq_base, + desc_dk_base, + desc_dv_base, + desc_dg_base, + bars, + ) + elif warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]: + compute0_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_base_holder, + warp_idx, + scale, + mBeta, + sBeta_raw, + sK_inv_raw, + sGate_raw, + sK_raw, + sQ_raw, + sState_raw, + sNorm_raw, + sK_decay_raw, + sK_restore_raw, + sQ_decay_raw, + sState_scale_diag_raw, + bars, + ) + elif warp_idx >= cfg.compute_group_2_warp_ids[0] and warp_idx <= cfg.compute_group_2_warp_ids[-1]: + compute2_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_base_holder, + warp_idx, + sQ_raw, + sK_raw, + sGate_raw, + sNorm_raw, + sDq_raw, + sDk_raw, + sRed1_raw, + sDstate_raw, + sDg_raw, + scale, + bars, + ) + elif warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]: + compute1_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + tmem_base_holder, + warp_idx, + mDbeta, + mDstate0, + mDstate_in, + sV_raw, + sDo_raw, + sBeta_raw, + sU_raw, + sDy_raw, + sDv_raw, + sDstate_raw, + sRed_raw, + sBetaM_raw, + bars, + ) + + +@dataclass +class KdaBwdCfg: + """Kernel cfg (fixed BT=16 schedule constants; derived TMEM column offsets + and SMEM buffer cosizes are stamped by ``build_cfg``).""" + + io_dtype: Type[cutlass.Numeric] + use_dstate_in: bool + use_dstate0: bool + l2norm: bool + use_initial_state: bool + q_ratio: int + k_ratio: int + v_ratio: int + n_heads_out: int + max_active_clusters: int + split_k: bool = False + dyn_sched: bool = False + sched_stages: int = 8 + + # ---- fixed constants stamped from CFG by build_cfg --------------------------- + compute_group_0_warp_ids: tuple = CFG.COMPUTE_GROUP_0_WARP_IDS + compute_group_2_warp_ids: tuple = CFG.COMPUTE_GROUP_2_WARP_IDS + compute_group_1_warp_ids: tuple = CFG.COMPUTE_GROUP_1_WARP_IDS + super_mma_warp_id: int = CFG.SUPER_MMA_WARP_ID + tcgen05_mma_warp_id: int = CFG.TCGEN05_MMA_WARP_ID + tma_warp_id: int = CFG.TMA_WARP_ID + epilogue_warp_id: int = CFG.EPILOGUE_WARP_ID + b_t: int = CFG.B_T + d_k: int = CFG.D_K + d_v: int = CFG.D_V + threads_per_warp: int = CFG.THREADS_PER_WARP + threads_per_cta: int = 0 + num_regs_compute_group_0: int = CFG.NUM_REGS_COMPUTE_GROUP_0 + num_regs_compute_group_1: int = CFG.NUM_REGS_COMPUTE_GROUP_1 + num_regs_compute_group_2: int = CFG.NUM_REGS_COMPUTE_GROUP_2 + num_regs_other: int = CFG.NUM_REGS_OTHER + + # ---- named barrier slots (ids 1-4; 0 is the CTA-wide sync) ------------------- + cg0_sync_barrier_id: int = 1 + cg0_threads: int = 0 + cg2_sync_barrier_id: int = 2 + cg2_threads: int = 0 + tmem_lifecycle_barrier_id: int = 3 + tmem_user_threads: int = 0 + cg1_sync_barrier_id: int = 4 + cg1_threads: int = 0 + + # ---- SMEM / TMEM stage counts + TMEM column offsets -------------------------- + smem_raw_stages: int = CFG.SMEM_RAW_STAGES + smem_state_stages: int = CFG.SMEM_S_STAGES + smem_decay_stages: int = CFG.SMEM_DECAY_STAGES + smem_qk_stages: int = CFG.SMEM_QK_STAGES + smem_dq_stages: int = CFG.SMEM_DQ_STAGES + smem_dk_stages: int = CFG.SMEM_DK_STAGES + smem_dg_stages: int = CFG.SMEM_DG_STAGES + smem_dv_stages: int = CFG.SMEM_DV_STAGES + smem_beta_stages: int = 4 + qk_tiles: int = 5 + tmem_dstate_acc_offset: int = 0 + tmem_dstate_inp_offset: int = 0 + tmem_state_k_acc_offset: int = 0 + tmem_update_acc_offset: int = 0 + tmem_du_acc_offset: int = 0 + tmem_dy_acc_offset: int = 0 + tmem_dq_acc_offset: int = 0 + tmem_dk_decay_acc_offset: int = 0 + tmem_dk_inv_acc_offset: int = 0 + tmem_dk_restore_acc_offset: int = 0 + tmem_qk_raw_stages: int = 4 + tmem_qraw_inp_offset: int = 0 + tmem_kraw_inp_offset: int = 0 + tmem_rhs_inp_offset: int = 0 + tmem_du_inp_offset: int = 0 + tmem_neg_beta_dy_inp_offset: int = 0 + tmem_state_inp_offset: int = 0 + buffer_align_bytes: int = CFG.BUFFER_ALIGN_BYTES + + # ---- buffer cosizes / TMA bytes stamped by build_cfg ------------------------- + raw_qk_cosize: int = 0 + raw_v_cosize: int = 0 + raw_gate_cosize: int = 0 + operand_cosize: int = 0 + diag_cosize: int = 0 + qk_cosize: int = 0 + state_cosize: int = 0 + dq_cosize: int = 0 + dk_cosize: int = 0 + dg_cosize: int = 0 + dv_cosize: int = 0 + + # TMA transaction bytes per stage + tma_q_bytes: int = 0 + tma_k_bytes: int = 0 + tma_gate_bytes: int = 0 + tma_do_bytes: int = 0 + tma_v_bytes: int = 0 + tma_state_bytes: int = 0 + + +def build_cfg( + io_dtype: Type[cutlass.Numeric], + *, + use_dstate_in: bool, + use_dstate0: bool, + l2norm: bool, + use_initial_state: bool, + q_ratio: int, + k_ratio: int, + v_ratio: int, + n_heads_out: int, + max_active_clusters: int, + split_k: bool = False, + dyn_sched: bool = False, +) -> KdaBwdCfg: + if io_dtype not in (cutlass.Float16, cutlass.BFloat16): + raise ValueError(f"io_dtype={io_dtype} not supported; only Float16 and BFloat16 are supported") + cfg = KdaBwdCfg( + io_dtype=io_dtype, + use_dstate_in=use_dstate_in, + use_dstate0=use_dstate0, + l2norm=l2norm, + use_initial_state=use_initial_state, + q_ratio=q_ratio, + k_ratio=k_ratio, + v_ratio=v_ratio, + n_heads_out=n_heads_out, + max_active_clusters=max_active_clusters, + split_k=split_k, + dyn_sched=dyn_sched, + ) + cfg.threads_per_cta = 16 * cfg.threads_per_warp + cfg.cg0_threads = len(cfg.compute_group_0_warp_ids) * cfg.threads_per_warp + cfg.cg2_threads = len(cfg.compute_group_2_warp_ids) * cfg.threads_per_warp + cfg.cg1_threads = len(cfg.compute_group_1_warp_ids) * cfg.threads_per_warp + cfg.tmem_user_threads = ( + 1 + len(cfg.compute_group_2_warp_ids) + len(cfg.compute_group_1_warp_ids) + len(cfg.compute_group_0_warp_ids) + ) * cfg.threads_per_warp + + cfg.tmem_dstate_acc_offset = 0 + cfg.tmem_dstate_inp_offset = cfg.d_k + cfg.tmem_state_inp_offset = cfg.tmem_dstate_inp_offset + cfg.d_k // 2 + cfg.tmem_state_k_acc_offset = cfg.tmem_state_inp_offset + cfg.d_v + cfg.tmem_update_acc_offset = cfg.tmem_state_k_acc_offset + cfg.b_t + cfg.tmem_du_acc_offset = cfg.tmem_update_acc_offset + cfg.b_t + cfg.tmem_dy_acc_offset = cfg.tmem_du_acc_offset + cfg.b_t + cfg.tmem_dq_acc_offset = cfg.tmem_dy_acc_offset + cfg.b_t + cfg.tmem_dk_decay_acc_offset = cfg.tmem_dq_acc_offset + cfg.b_t + cfg.tmem_dk_inv_acc_offset = cfg.tmem_dk_decay_acc_offset + cfg.b_t + cfg.tmem_dk_restore_acc_offset = cfg.tmem_dk_inv_acc_offset + cfg.b_t + cfg.tmem_rhs_inp_offset = cfg.tmem_dk_restore_acc_offset + cfg.b_t + cfg.tmem_du_inp_offset = cfg.tmem_rhs_inp_offset + cfg.b_t // 2 + cfg.tmem_neg_beta_dy_inp_offset = cfg.tmem_du_inp_offset + cfg.b_t // 2 + assert cfg.tmem_neg_beta_dy_inp_offset + cfg.b_t // 2 <= 512 + + cfg.raw_qk_cosize = cfg.smem_raw_stages * cfg.d_k * cfg.b_t + cfg.raw_v_cosize = cfg.smem_raw_stages * cfg.d_v * cfg.b_t + cfg.raw_gate_cosize = cfg.smem_raw_stages * cfg.d_k * cfg.b_t + cfg.operand_cosize = cfg.smem_decay_stages * cfg.b_t * cfg.d_k + cfg.diag_cosize = cfg.smem_decay_stages * (cfg.d_k // 16) * 256 + cfg.qk_cosize = cfg.smem_qk_stages * cfg.qk_tiles * cfg.b_t * cfg.b_t + cfg.state_cosize = cfg.smem_state_stages * cfg.d_k * cfg.d_v + cfg.dq_cosize = cfg.smem_dq_stages * cfg.b_t * cfg.d_k + cfg.dk_cosize = cfg.smem_dk_stages * cfg.b_t * cfg.d_k + cfg.dg_cosize = cfg.smem_dg_stages * cfg.b_t * cfg.d_k + cfg.dv_cosize = cfg.smem_dv_stages * cfg.b_t * cfg.d_v + cfg.tma_state_bytes = cfg.d_k * cfg.d_v * (io_dtype.width // 8) + cfg.tma_q_bytes = cfg.d_k * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_k_bytes = cfg.d_k * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_gate_bytes = cfg.d_k * cfg.b_t * 4 + cfg.tma_do_bytes = cfg.d_v * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_v_bytes = cfg.d_v * cfg.b_t * (cfg.io_dtype.width // 8) + return cfg + + +TENSORMAP_DESC_ARRAYS = 10 # per-batch runtime TMA descriptors: Q, K, V, Gate, dO, state_checkpoints, dQ, dK, dV, dG +TENSORMAP_STATIC_SLOTS = 1 # initial_state + + +# ---- Torch adapter / host-side compilation --------------------------------------- + + +@lru_cache(maxsize=None) +def get_compiled_cache( + io_dtype_str: str, + cu_dtype_str: str, + HQ: int, + HK: int, + HV: int, + use_dstate_in: bool, + use_dstate0: bool, + l2norm: bool, + use_initial_state: bool, + split_k: bool, + dyn_sched: bool, +): + return {} + + +def chunk_kda_bwd_sm100( + q, + k, + v, + gate, + beta, + do, + state_checkpoints, + dq, + dk, + dv, + dg, + dbeta, + cu_seqlens, + scale: float, + *, + initial_state=None, + d_initial_state=None, + d_final_state=None, + use_qk_l2norm_in_kernel: bool = False, + work_items=None, + work_count=None, + sched_ctr=None, + tensormap_workspace, + stream, +) -> None: + """Execute the Blackwell BT=16 chunked KDA backward kernel. + + All tensors must be contiguous and on the same CUDA device. + + Args: + q: ``(total_tokens, HQ, DK)`` float16/bfloat16 + k: ``(total_tokens, HK, DK)`` float16/bfloat16 + v: ``(total_tokens, HV, DV)`` float16/bfloat16 + gate: ``(total_tokens, HO, DK)`` fp32 natural-log per-channel decay + beta: ``(total_tokens, HO)`` fp32 post-sigmoid + do: ``(total_tokens, HO, DV)`` io dtype + state_checkpoints: ``(total_checkpoints, HO, DK, DV)`` io dtype (KV, v contiguous), the PLAIN per-chunk checkpoint series with no initial-state + slot: sequence-local entry ``c - 1`` is the state ENTERING chunk c >= 1 + of sequence b; chunk 0 seeds from ``initial_state`` + dq/dk/dv: io dtype at ``HO = max(HQ, HV)`` heads, pre-allocated + dg: ``(total_tokens, HO, DK)`` fp32 (dL/d ln alpha), pre-allocated + dbeta: ``(total_tokens, HO)`` fp32, pre-allocated + cu_seqlens: ``(num_seqs + 1,)`` int32 + scale: attention scale factor + initial_state: ``(num_seqs, HO, DK, DV)`` io dtype (KV) — the state + entering chunk 0 (engine-provided zeros when the graph has none) + d_initial_state: fp32 ``(num_seqs, HO, DK, DV)`` OUT (dL/d initial state), or None + d_final_state: fp32 ``(num_seqs, HO, DK, DV)`` IN (dL/d final state) + use_qk_l2norm_in_kernel: q/k arrive raw; the kernel normalizes for the + recompute math and chains the L2-norm backward into dq/dk + work_items/work_count: split-K table (``common/split_k.py``); + each item computes chunks ``[wstart, cend)`` backward and writes + gradients only for ``[wstart, wend)`` + sched_ctr: ``(2,)`` int32 zeroed scratch enabling the dynamic + (work-stealing) tile scheduler + tensormap_workspace: ``tensormap_workspace_bytes(module, B)`` bytes, + 128-byte aligned, for the per-(batch, head) TMA-descriptor + arrays (tail chunks clip/zero-fill in hardware) + """ + HQ = q.shape[1] + HK = k.shape[1] + HV = v.shape[1] + HO = max(HQ, HV) + use_dstate_in = d_final_state is not None + use_dstate0 = d_initial_state is not None + use_initial_state = initial_state is not None + split_k = work_items is not None + dyn_sched = sched_ctr is not None + for name, t in (("state_checkpoints", state_checkpoints),) + ((("initial_state", initial_state),) if use_initial_state else ()): + if str(t.dtype).split(".")[-1] != str(q.dtype).split(".")[-1]: + raise ValueError(f"{name} dtype must match the io dtype: got {t.dtype} with io {q.dtype}") + for name, hh in (("HQ", HQ), ("HK", HK), ("HV", HV)): + if HO % hh != 0: + raise ValueError(f"{name}={hh} must divide {HO}") + B = cu_seqlens.shape[0] - 1 + + cu_stream = cuda_driver.CUstream(int(stream)) + cache = get_compiled_cache( + str(q.dtype), + str(cu_seqlens.dtype), + HQ, + HK, + HV, + use_dstate_in, + use_dstate0, + use_qk_l2norm_in_kernel, + use_initial_state, + split_k, + dyn_sched, + ) + + if "compiled" not in cache: + io_dtype = get_dtype(q.dtype) + cfg = build_cfg( + io_dtype, + use_dstate_in=use_dstate_in, + use_dstate0=use_dstate0, + l2norm=use_qk_l2norm_in_kernel, + use_initial_state=use_initial_state, + q_ratio=HO // HQ, + k_ratio=HO // HK, + v_ratio=HO // HV, + n_heads_out=HO, + max_active_clusters=multiprocessor_count(current_device_id()), + split_k=split_k, + dyn_sched=dyn_sched, + ) + + dstate0_cute = None + if use_dstate0: + dstate0_cute = from_dlpack(d_initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) + dstate_in_cute = None + if use_dstate_in: + dstate_in_cute = from_dlpack(d_final_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) + wi_cute = None + wc_cute = None + if split_k: + wi_cute = from_dlpack(work_items, assumed_align=16) + wi_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) + wc_cute = from_dlpack(work_count, assumed_align=4).mark_layout_dynamic() + sc_cute = None + if dyn_sched: + sc_cute = from_dlpack(sched_ctr, assumed_align=4).mark_layout_dynamic() + + tensormap_ws_cute = from_dlpack(tensormap_workspace, assumed_align=128).mark_layout_dynamic() + + beta_cute = from_dlpack(beta, assumed_align=4).mark_layout_dynamic(leading_dim=len(beta.shape) - 1) + state_checkpoints_cute = from_dlpack(state_checkpoints, assumed_align=16).mark_layout_dynamic(leading_dim=len(state_checkpoints.shape) - 1) + initial_state_cute = ( + from_dlpack(initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=len(initial_state.shape) - 1) if use_initial_state else None + ) + dg_cute = from_dlpack(dg, assumed_align=16).mark_layout_dynamic(leading_dim=len(dg.shape) - 1) + dbeta_cute = from_dlpack(dbeta, assumed_align=4).mark_layout_dynamic(leading_dim=len(dbeta.shape) - 1) + cache["compiled"] = cute.compile( + host, + cfg, + beta_cute, + state_checkpoints_cute, + initial_state_cute, + dg_cute, + dbeta_cute, + from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic(), + dstate0_cute, + dstate_in_cute, + wi_cute, + wc_cute, + sc_cute, + tensormap_ws_cute, + scale, + cu_stream, + options="--enable-tvm-ffi --opt-level 2", + ) + + # ---- per-(batch, head) descriptor arrays: rebuild on input change ------------ + # desc build runs every execute by contract (cu contents are data; + # buffer pointers may change) — capture-safe, single tiny launch + if "build_descs" not in cache: + io_dtype = get_dtype(q.dtype) + + q_bd = from_dlpack(q, assumed_align=16).mark_layout_dynamic(leading_dim=2) + k_bd = from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2) + v_bd = from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2) + gate_bd = from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=2) + do_bd = from_dlpack(do, assumed_align=16).mark_layout_dynamic(leading_dim=2) + dq_bd = from_dlpack(dq, assumed_align=16).mark_layout_dynamic(leading_dim=2) + dk_bd = from_dlpack(dk, assumed_align=16).mark_layout_dynamic(leading_dim=2) + dv_bd = from_dlpack(dv, assumed_align=16).mark_layout_dynamic(leading_dim=2) + dg_bd = from_dlpack(dg, assumed_align=16).mark_layout_dynamic(leading_dim=2) + state_checkpoints_bd = from_dlpack(state_checkpoints, assumed_align=16).mark_layout_dynamic(leading_dim=3) + initial_state_bd = from_dlpack(initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) if use_initial_state else None + + cu_bd = from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic() + ws_bd = from_dlpack(tensormap_workspace, assumed_align=128).mark_layout_dynamic() + cache["build_descs"] = cute.compile( + build_descs, + io_dtype, + CFG.B_T, + q_bd, + k_bd, + v_bd, + gate_bd, + do_bd, + dq_bd, + dk_bd, + dv_bd, + dg_bd, + state_checkpoints_bd, + initial_state_bd, + cu_bd, + ws_bd, + cu_stream, + options="--enable-tvm-ffi", + ) + cache["build_descs"](q, k, v, gate, do, dq, dk, dv, dg, state_checkpoints, initial_state, cu_seqlens, tensormap_workspace, cu_stream) + cache["compiled"]( + beta, + state_checkpoints, + initial_state, + dg, + dbeta, + cu_seqlens, + d_initial_state, + d_final_state, + work_items, + work_count, + sched_ctr, + tensormap_workspace, + scale, + cu_stream, + ) + + +# ---- Engine-side helpers: checkpoint-series bounds + entry-0 state seeding ---------------- diff --git a/python/cudnn/linear_attention/frost/kernel/kda_prefill_config.py b/python/cudnn/linear_attention/frost/kernel/kda_prefill_config.py index c7900f207..1579cd68a 100644 --- a/python/cudnn/linear_attention/frost/kernel/kda_prefill_config.py +++ b/python/cudnn/linear_attention/frost/kernel/kda_prefill_config.py @@ -56,7 +56,7 @@ class Cfg: SMEM_SCHED_STAGES: int = 8 SMEM_O_STAGES: int = 2 SMEM_DECAY_STAGES: int = 2 - SMEM_PAIRWISE_STAGES: int = 2 + SMEM_QK_STAGES: int = 2 SMEM_STATE_SCALE_DIAG_STAGES: int = 3 QK_SCALE_READY_STAGES: int = 3 TMEM_Q_STATE_ACC_STAGES: int = 2 diff --git a/python/cudnn/linear_attention/frost/kernel/kda_prefill_f16.py b/python/cudnn/linear_attention/frost/kernel/kda_prefill_f16.py index 71263b08b..c4098549e 100644 --- a/python/cudnn/linear_attention/frost/kernel/kda_prefill_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/kda_prefill_f16.py @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Chunked Kimi Delta Attention (KDA) prefill kernel for Blackwell SM100 +"""Chunked Kimi Delta Attention (KDA) prefill kernel for Blackwell SM100/SM103 (Cutlass DSL), BT=16 tiling with a per-key-channel decay. Framework-neutral entry ``chunk_kda_sm100``. @@ -29,12 +29,12 @@ Pipeline (direct CUTLASS primitives, chunk_idx-size 16 KDA schedule): - load q/k/v/gate/beta - optional in-kernel L2-norm of q/k (L2NORM specialization) - exp2(g), exp2(-g), stage final-token exp2(g) as exp2(g_last) - super-MMA: kk/qk/Neumann inverse + apply beta + load Q/K/V/Gate/Beta + optional in-kernel L2-norm of Q/K (L2NORM specialization) + exp2(G), exp2(-G), stage final-token exp2(G) as exp2(G_last) + super-MMA: KK/QK/Neumann inverse + apply Beta tcgen05-MMA: state*k/state*q/new-v/kv-update/qkv - store o, periodic state checkpoints, final state + store O, periodic state checkpoints, final state ABI: q `[T, HQ, DK]`, k `[T, HK, DK]`, v `[T, HV, DV]`, gate `[T, HO, DK]` fp32 (natural-log decay unless SAFE_GATE, which applies the @@ -45,29 +45,28 @@ head ratios are compile-time specializations. Warp assignments (16 warps = 512 threads): - warps 0-7 : compute group 0 - gate prefix scan + decay/restore operands - warps 8-11 : compute group 1 - TMEM value side, o drain, state stores - warp 12 : super-MMA - register-MMA kk^T + Neumann inverse + warps 0-7 : compute group 0 - Gate prefix scan + decay/restore operands + warps 8-11 : compute group 1 - TMEM value side, O drain, state stores + warp 12 : super-MMA - register-MMA KK^T + Neumann inverse warp 13 : tcgen05-MMA - the six state GEMMs + the TMEM lifecycle warp 14 : TMA load - per-chunk input G->S loads - warp 15 : epilogue - register-MMA qk + the O TMA store + warp 15 : epilogue - register-MMA QK + the O TMA store SMEM layout (~221 KB total): Buffer Bytes Stages q / k / v raw 32768 8 <-- SW128 TMA ring (io dtype) gate raw 65536 8 <-- fp32 prefix-scan source beta 512 8 <-- fp32 per-token scalars - dt_bias (+a_log slot) 516 1 <-- SAFE_GATE only K_inv 8192 2 <-- token-major ldmatrix/tcgen05 B operand K decay / Q decay 2x 8192 2 <-- tcgen05 SW128 K-box-major A/B operands K restore 8192 2 <-- tcgen05 B operand for the state update state-scale diag 12288 3 <-- per-k-atom decay diagonal blocks - pairwise (A_inv / qk) 2048 2 <-- SW32 16x16 register-MMA tiles + qk (A_inv / qk) 2048 2 <-- SW32 16x16 register-MMA tiles o staging 8192 2 <-- W128 output drain TMEM layout (272 of 512 columns): Buffer Cols Purpose - state 0-127 S[DK,DV] fp32 recurrent state + state 0-127 state[DK,DV] fp32 recurrent state state inp 128-191 packed b16 A operand view of the state q_state_acc 192-223 2-stage state*q -> o accumulator state_k_acc 224-239 state*k fp32 accumulator @@ -77,11 +76,11 @@ GEMM schedule (tcgen05-MMA warp, in issue order per chunk): state*k -> state_k_acc - state*q -> q_state_acc (the o acc) + state*q -> q_state_acc (the O acc) state decay (diag blocks) update = A_inv @ rhs -> update_acc - final_state += update @ k_restore - o += qk @ update -> q_state_acc + final_state += update @ K_restore + O += QK @ update -> q_state_acc Requires a cutlass DSL build providing `cutlass.experimental.*`; not available in the pip nvidia-cutlass-dsl releases. @@ -99,24 +98,27 @@ from cutlass.cute.runtime import from_dlpack from ..common.split_k import decode_work_item -from ..common.thd import TENSOR_MAP_QWORDS, build_h_descs_kernel, build_qkv_load_descs_kernel +from ..common.host import get_dtype +from cudnn.frost.buffers import current_device_id, data_ptr +from cudnn.frost.device import multiprocessor_count +from ..common.thd import TENSOR_MAP_QWORDS, emit_checkpoint_seq_descs, emit_seq_descs from .kda_prefill_config import CFG from cudnn.frost.tile_dsl.barrier import ( advance, - arrive, MBarrier, PipelineState, Producer, - wait, ) from cudnn.frost.tile_dsl.handles import GmemTileTma, MmaDesc, SmemTile, tma_slice_runtime_desc -from cudnn.frost.tile_dsl.mma import mma_step, mma_ts +from cudnn.frost.tile_dsl.mma import mma_step, mma_ts_step from cudnn.frost.tile_dsl.swizzle import swizzle_lin_128b, swizzle_lin_S, swizzle_xor_128b from cudnn.frost.tile_dsl.tma import tma_load_tile, tma_store_commit, tma_store_tile, tma_store_wait, tma_tensormap_acquire from cudnn.frost.tile_dsl.pointwise import ( opaque_f32_zero, + f16x2_to_f32, fadd2, fmul2, + ffma2, movmatrix_16b, mul_f16x2, fp32_to_fp16, @@ -136,40 +138,58 @@ class KdaBars(NamedTuple): - """Every inter-warp handoff as an ``MBarrier`` over its ring (mirrors GDN's - ``GdnBars``). Consumers track ``(idx, phase)`` inline; the producer tag selects + """Every inter-warp handoff as an ``MBarrier`` over its ring. Consumers + track ``(idx, phase)`` inline; the producer tag selects the arrive lowering (``TMA_LOAD``/``MMA_COMMIT``/``THREAD``).""" - mb_tma_done: MBarrier - mb_inputs_ready: MBarrier - mb_inputs_done: MBarrier + mb_q_ready: MBarrier + mb_q_done: MBarrier + mb_k_ready: MBarrier + mb_k_done: MBarrier + mb_v_ready: MBarrier + mb_v_done: MBarrier + mb_gate_ready: MBarrier + mb_gate_done: MBarrier + + mb_beta_ready: MBarrier + mb_beta_done: MBarrier + mb_o_acc_ready: MBarrier mb_o_acc_done: MBarrier mb_state_k_acc_ready: MBarrier mb_update_acc_ready: MBarrier + mb_state_inp_ready: MBarrier - mb_state_scale_diag_done: MBarrier - mb_kk_qk_super_mma_done: MBarrier - mb_kk_qk_mma_done: MBarrier - mb_k_restore_done: MBarrier mb_rhs_ready: MBarrier mb_update_ready: MBarrier - mb_final_state_stored: MBarrier + mb_a_ready: MBarrier + mb_a_inv_done: MBarrier mb_qk_acc_ready: MBarrier - mb_a_done: MBarrier + mb_qk_done: MBarrier mb_qk_scale_ready: MBarrier - mb_k_decay_cg0_ready: MBarrier + mb_state_scale_diag_done: MBarrier + mb_k_decay_inv_cg0_ready: MBarrier + mb_decay_done: MBarrier + mb_decay_super_done: MBarrier + mb_k_restore_done: MBarrier + + mb_state_read_done: MBarrier + mb_state_updated: MBarrier + mb_final_state_stored: MBarrier + mb_o_tmastg_ready: MBarrier mb_o_tmastg_done: MBarrier + + mb_checkpoint_tmastg_ready: MBarrier + mb_checkpoint_tmastg_done: MBarrier + mb_sched_ready: MBarrier mb_sched_done: MBarrier - mb_h_tmastg_ready: MBarrier - mb_h_tmastg_done: MBarrier def make_kda_bars(cfg) -> KdaBars: - """Bars factory. MUST be called from inside ``_kernel`` (allocates the + """Bars factory. MUST be called from inside ``kernel`` (allocates the mbarrier rings in SMEM ahead of the data buffers).""" def alloc(n): @@ -180,71 +200,62 @@ def alloc(n): CG1_THREADS = len(cfg.compute_group_1_warp_ids) * WARP return KdaBars( - mb_tma_done=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.TMA_LOAD), - mb_inputs_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=WARP, producer=Producer.THREAD), - mb_inputs_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_q_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_q_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_k_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_k_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_v_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_v_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_gate_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_gate_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_beta_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=WARP, producer=Producer.THREAD), + mb_beta_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=WARP + CG1_THREADS, producer=Producer.THREAD), mb_o_acc_ready=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), mb_o_acc_done=MBarrier(alloc(cfg.tmem_q_state_acc_stages), stages=cfg.tmem_q_state_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_state_k_acc_ready=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), mb_update_acc_ready=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), mb_state_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_state_scale_diag_done=MBarrier( - alloc(cfg.smem_state_scale_diag_stages), - stages=cfg.smem_state_scale_diag_stages, - init_count=1, - producer=Producer.MMA_COMMIT, - ), - mb_kk_qk_super_mma_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=2 * WARP, producer=Producer.THREAD), - mb_kk_qk_mma_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), - mb_k_restore_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_rhs_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_update_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_final_state_stored=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_a_ready=MBarrier(alloc(cfg.smem_pairwise_stages), stages=cfg.smem_pairwise_stages, init_count=WARP, producer=Producer.THREAD), - mb_qk_acc_ready=MBarrier(alloc(cfg.smem_pairwise_stages), stages=cfg.smem_pairwise_stages, init_count=WARP, producer=Producer.THREAD), - mb_a_done=MBarrier(alloc(cfg.smem_pairwise_stages), stages=cfg.smem_pairwise_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_a_inv_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_qk_acc_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_qk_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_qk_scale_ready=MBarrier( alloc(cfg.qk_scale_ready_stages), stages=cfg.qk_scale_ready_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD, ), - mb_k_decay_cg0_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_state_scale_diag_done=MBarrier( + alloc(cfg.smem_state_scale_diag_stages), + stages=cfg.smem_state_scale_diag_stages, + init_count=1, + producer=Producer.MMA_COMMIT, + ), + mb_k_decay_inv_cg0_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=2 * WARP, producer=Producer.THREAD), + mb_k_restore_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_state_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_state_updated=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), + mb_final_state_stored=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_tmastg_ready=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_tmastg_done=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=WARP, producer=Producer.THREAD), + mb_checkpoint_tmastg_ready=MBarrier( + alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=CG1_THREADS, producer=Producer.THREAD + ), + mb_checkpoint_tmastg_done=MBarrier(alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=WARP, producer=Producer.THREAD), mb_sched_ready=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=1, producer=Producer.THREAD), mb_sched_done=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=15, producer=Producer.THREAD), - # H staging handshake: CG1 fills sH (ready), the epilogue TMA-stores - # and frees it (done); single stage - mb_h_tmastg_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_h_tmastg_done=MBarrier(alloc(1), stages=1, init_count=WARP, producer=Producer.THREAD), ) -# --------------------------------------------------------------------------- -# Device-side helpers / warp bodies -# --------------------------------------------------------------------------- - - -@cute.jit -def _gate_log2(cfg, raw_gate: cutlass.Float32) -> cutlass.Float32: - """Map raw gate to the log2-domain decay increment used by KDA.""" - - if cutlass.const_expr(cfg.safe_gate): - half = cutlass.Float32(0.5) - sigmoid = cute.math.tanh(raw_gate * half, approx=True) * half + half - return cfg.gate_scale_log2 * sigmoid - # Default ABI: gate arrives in natural-log space - return raw_gate * cutlass.Float32(LOG2_E) - - -# --------------------------------------------------------------------------- -# Dynamic tile scheduler: global-ticket ring -# --------------------------------------------------------------------------- +# ---- Dynamic tile scheduler ------------------------------------------------------ @cute.jit -def _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): +def sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): """TMA-warp side: pull the next tile off the global ticket, publish it.""" if cutlass.const_expr(cfg.dyn_sched): bars.mb_sched_done[sched_state.idx].wait(sched_state.phase) @@ -260,7 +271,7 @@ def _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ct @cute.jit -def _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): +def sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): """Consumer side: read the TMA warp's published next tile.""" if cutlass.const_expr(cfg.dyn_sched): bars.mb_sched_ready[sched_state.idx].wait(sched_state.phase) @@ -272,7 +283,7 @@ def _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): @cute.jit -def _tmaldg_warp( +def tmaldg_warp( cfg, total_tiles, bidx, @@ -281,26 +292,20 @@ def _tmaldg_warp( mWorkItems, mSched, sSched, - tma_tx_bytes, lane, - mBeta, sQ_raw, sK_raw, sV_raw, sGate_raw, - sBeta_raw, desc_q_base, desc_k_base, desc_v_base, desc_gate_base, bars, ) -> None: - """TMA-LDG warp role (warp 14): persistent tile-scheduler loop + per-chunk - q/k/v/gate G->S loads on one shared tx-count mbarrier plus the per-token - beta scalar stage. Loads go through the per-(batch, head) descriptor - array: head grouping and the sequence base live in each descriptor and - the token extent is capped per sequence, so coordinates are - sequence-relative and tail chunks zero-fill in hardware.""" + """TMA-LDG warp role (warp 14): persistent scheduler loop issuing the + per-chunk Q/K/V/Gate G->S loads.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) sQ_tma = SmemTile( base=sQ_raw, @@ -346,7 +351,6 @@ def _tmaldg_warp( tma_granu_elems=32, tma_subtile_stride_elems=(cfg.b_t * 32), ) - tma_index = PipelineState.start(phase=0) raw_index = PipelineState.start(phase=1) sched_state = PipelineState.start(phase=1) tile_idx = cutlass.Int32(bidx) @@ -355,51 +359,56 @@ def _tmaldg_warp( cfg, tile_idx, cu_seqlens, mWorkItems ) head_o = head_idx - slot = (batch_idx * cutlass.Int32(cfg.n_heads_out) + head_idx) * cutlass.Int32(TENSOR_MAP_QWORDS) + head_q = head_idx if cfg.q_ratio == 1 else head_idx // cutlass.Int32(cfg.q_ratio) + head_k = head_idx if cfg.k_ratio == 1 else head_idx // cutlass.Int32(cfg.k_ratio) + head_v = head_idx if cfg.v_ratio == 1 else head_idx // cutlass.Int32(cfg.v_ratio) + slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) desc_q_slot = (desc_q_base + slot).tospace(cutlass.AddressSpace.generic) desc_k_slot = (desc_k_base + slot).tospace(cutlass.AddressSpace.generic) desc_v_slot = (desc_v_base + slot).tospace(cutlass.AddressSpace.generic) desc_gate_slot = (desc_gate_base + slot).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): + if elect_one: tma_tensormap_acquire(desc_q_slot) tma_tensormap_acquire(desc_k_slot) tma_tensormap_acquire(desc_v_slot) tma_tensormap_acquire(desc_gate_slot) for chunk_idx in cutlass.range(cstart, wend, 1, unroll=1): chunk_start = chunk_idx * cfg.b_t - bars.mb_inputs_done[raw_index.idx].wait(raw_index.phase) - # ---- q/k/v/gate TMA loads + per-token beta scalar stage ------------ - if nvvm.elect_sync(): - bars.mb_tma_done.arrive(n_bytes=tma_tx_bytes) - q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), chunk_start) - tma_load_tile(sQ_tma[raw_index.idx], q_slice, bars.mb_tma_done.smem_ptr, acquire=False) - k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), chunk_start) - tma_load_tile(sK_tma[raw_index.idx], k_slice, bars.mb_tma_done.smem_ptr, acquire=False) - v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), chunk_start) - tma_load_tile(sV_tma[raw_index.idx], v_slice, bars.mb_tma_done.smem_ptr, acquire=False) - gate_slice = tma_slice_runtime_desc(desc_gate_slot, cutlass.Int32(0), chunk_start) - tma_load_tile(sGate_tma[raw_index.idx], gate_slice, bars.mb_tma_done.smem_ptr, acquire=False) - - if lane < cfg.b_t: - token_idx = chunk_start + lane - beta_value = cutlass.Float32(0.0) - if token_idx < seqlen_b: - beta_value = mBeta[batch_start + token_idx, head_o].to(cutlass.Float32) - if cutlass.const_expr(cfg.beta_sigmoid): - # Roundtrip through the io dtype to bit-match host-side mBeta.sigmoid() - half = cutlass.Float32(0.5) - beta_value = (cute.math.tanh(beta_value * half, approx=True) * half + half).to(mBeta.element_type).to(cutlass.Float32) - sBeta_raw[raw_index.idx * cfg.b_t + lane] = beta_value - - bars.mb_tma_done.wait(tma_index.phase) - tma_index = advance(tma_index, 1) - bars.mb_inputs_ready[raw_index.idx].arrive() + + # ---- Q load ---------------------------------------------------------- + bars.mb_q_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_q_ready[raw_index.idx].arrive(n_bytes=cfg.tma_q_bytes) + q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), head_q, chunk_start) + tma_load_tile(sQ_tma[raw_index.idx], q_slice, bars.mb_q_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- K load ---------------------------------------------------------- + bars.mb_k_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_k_ready[raw_index.idx].arrive(n_bytes=cfg.tma_k_bytes) + k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, chunk_start) + tma_load_tile(sK_tma[raw_index.idx], k_slice, bars.mb_k_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- Gate load ------------------------------------------------------- + bars.mb_gate_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_gate_ready[raw_index.idx].arrive(n_bytes=cfg.tma_gate_bytes) + gate_slice = tma_slice_runtime_desc(desc_gate_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sGate_tma[raw_index.idx], gate_slice, bars.mb_gate_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- V load ---------------------------------------------------------- + bars.mb_v_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_v_ready[raw_index.idx].arrive(n_bytes=cfg.tma_v_bytes) + v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, chunk_start) + tma_load_tile(sV_tma[raw_index.idx], v_slice, bars.mb_v_ready[raw_index.idx].smem_ptr, acquire=False) + raw_index = advance(raw_index, cfg.smem_raw_stages) - tile_idx, sched_state = _sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) + tile_idx, sched_state = sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) @cute.jit -def _super_mma_warp( +def super_mma_warp( cfg, total_tiles, bidx, @@ -409,23 +418,24 @@ def _super_mma_warp( sSched, lane, sK_inv_raw, - sPairwise_raw, + sQk_raw, sBeta_raw, sK_decay_raw, bars, ) -> None: - """Super-MMA warp role (warp 12): persistent tile-scheduler loop + - register-MMA kk^T, L = beta*tril(kk), and the Neumann-series A_inv, - staged to pairwise SMEM.""" + """Super-MMA warp role (warp 12): persistent scheduler loop computing the + Neumann-series A_inv via register MMA.""" nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) raw_index = PipelineState.start(phase=0) - # ---- ldmatrix/stmatrix lane decode --------------------------------- + a_inv_free = PipelineState.start(phase=1) + k_decay_ready = PipelineState.start(phase=0) + + # ---- ldmatrix/stmatrix lane decode ------------------------------------------- rhs_row_coord = lane % 8 + (cutlass.Int32(8) if (lane // 16) else cutlass.Int32(0)) rhs_col_offset = cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0) lhs_row_coord = lane % 8 + (cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0)) lhs_col_offset = cutlass.Int32(8) if ((lane // 8) // 2) else cutlass.Int32(0) - decay_key_mask = cutlass.Int32(8) ^ ((lhs_row_coord & cutlass.Int32(2)) * cutlass.Int32(16)) - elems_per_128b = cutlass.Int32(64) + decay_key_mask = cutlass.Int32(8) stsm_row_coord = lane & 7 stsm_col_coord = cutlass.Int32(0) if (lane // 8) & 1: @@ -433,26 +443,29 @@ def _super_mma_warp( if lane // 8 >= 2: stsm_col_coord = cutlass.Int32(8) stsm_idx = swizzle_lin_S(stsm_row_coord * cfg.b_t + (stsm_col_coord ^ (cfg.b_t // 2)), bbits=1, mbase=3, sshift=3) - gbase = cutlass.Int32(0) + cum_chunk_base = cutlass.Int32(0) sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) while tile_idx < total_tiles: batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) - sk_nt = wend - cstart # processed chunks; ring bookkeeping runs on gbase + li - for li in cutlass.range(sk_nt, unroll=1): - gc = gbase + li - decay_stage = gc % cfg.smem_decay_stages - pairwise_stage = gc % cfg.smem_pairwise_stages + num_chunks_tile = wend - cstart # processed chunks; ring bookkeeping runs on cum_chunk_base + local_chunk_idx + for local_chunk_idx in cutlass.range(num_chunks_tile, unroll=1): + cum_chunk = cum_chunk_base + local_chunk_idx + decay_stage = k_decay_ready.idx + qk_stage = a_inv_free.idx sBeta_ptr = sBeta_raw.data_ptr() + raw_index.idx * cfg.b_t sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) sK_decay_ptr = sK_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) - sPairwise_ptr = sPairwise_raw.data_ptr() + pairwise_stage * (2 * cfg.b_t * cfg.b_t) + sQk_ptr = sQk_raw.data_ptr() + qk_stage * (2 * cfg.b_t * cfg.b_t) + + bars.mb_a_inv_done[qk_stage].wait(a_inv_free.phase) + a_inv_free = advance(a_inv_free, cfg.smem_qk_stages) + bars.mb_k_decay_inv_cg0_ready[decay_stage].wait(k_decay_ready.phase) + k_decay_ready = advance(k_decay_ready, cfg.smem_decay_stages) - bars.mb_a_done[pairwise_stage].wait(((gc // cfg.smem_pairwise_stages) + 1) % 2) - bars.mb_k_decay_cg0_ready[decay_stage].wait((gc // cfg.smem_decay_stages) % 2) - # ---- kk^T register MMA over the K blocks --------------------------- + # ---- KK = K_decay @ K_inv^T ------------------------------------------ kk_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): kk_acc[accum_idx] = cutlass.Float32(0.0) @@ -471,18 +484,11 @@ def _super_mma_warp( ) # Load A operand storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask - storage_slice = storage_key // elems_per_128b - key_in_slice = storage_key - storage_slice * elems_per_128b - storage_phase = key_in_slice // cutlass.Int32(16) - byte_in_slice = ( - lhs_row_coord * cutlass.Int32(128) - + storage_phase * cutlass.Int32(32) - + (key_in_slice - storage_phase * cutlass.Int32(16)) * cutlass.Int32(2) - ) + storage_slice = storage_key // 64 kk_lhs_vec = nvvm.ldmatrix( sK_decay_ptr - + storage_slice * cutlass.Int32(cfg.b_t) * elems_per_128b - + ((byte_in_slice ^ ((lhs_row_coord & cutlass.Int32(7)) << 4)) // cutlass.Int32(2)), + + storage_slice * (cfg.b_t * 64) + + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) @@ -496,30 +502,31 @@ def _super_mma_warp( N=16, ab_dtype=cfg.io_dtype, ) - # ---- L = beta * tril(kk, -1) fragment ------------------------------ + + # ---- L = Beta * tril(KK, -1) fragment -------------------------------- + bars.mb_beta_ready[raw_index.idx].wait(raw_index.phase) row_lo = lane // 4 row_hi = row_lo + cutlass.Int32(8) beta_lo = (sBeta_ptr + row_lo).load().to(cutlass.Float32) beta_hi = (sBeta_ptr + row_hi).load().to(cutlass.Float32) l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): - row_coord = row_lo - beta_scale = beta_lo - if cutlass.const_expr(accum_idx % 4 >= 2): - row_coord = row_hi - beta_scale = beta_hi + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) if cutlass.const_expr(accum_idx % 2 == 1): col_coord = col_coord + cutlass.Int32(1) - lower = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) - l_frag[accum_idx] = lower * beta_scale + l_frag[accum_idx] = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + for pair in cutlass.range_constexpr(4): + beta_scale = beta_hi if cutlass.const_expr(pair % 2 == 1) else beta_lo + l_frag[2 * pair], l_frag[2 * pair + 1] = fmul2(l_frag[2 * pair], l_frag[2 * pair + 1], beta_scale, beta_scale) + bars.mb_beta_done[raw_index.idx].arrive() l_a0 = fp32_to_fp16(l_frag[0], l_frag[1], dtype=cfg.io_dtype) l_a1 = fp32_to_fp16(l_frag[2], l_frag[3], dtype=cfg.io_dtype) l_a2 = fp32_to_fp16(l_frag[4], l_frag[5], dtype=cfg.io_dtype) l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) - # ---- A_inv = I - L, then three Neumann doubling rounds ------------- + # ---- A_inv = I - L, then three Neumann doubling rounds --------------- ainv_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): row_coord = row_lo @@ -532,15 +539,16 @@ def _super_mma_warp( ainv_acc[accum_idx] = eye - l_values[accum_idx] lpow_a0, lpow_a1, lpow_a2, lpow_a3 = l_a0, l_a1, l_a2, l_a3 + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(l_a0), movmatrix_16b(l_a1), movmatrix_16b(l_a2), movmatrix_16b(l_a3) for _round in cutlass.range_constexpr(3): - # Lpow <- Lpow @ Lpow (packed A-layout fragments, B via movmatrix) + # ---- Lpow = Lpow @ Lpow ------------------------------------------ sq_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): sq_acc[accum_idx] = cutlass.Float32(0.0) mma_step( sq_acc, (lpow_a0, lpow_a1, lpow_a2, lpow_a3), - (movmatrix_16b(lpow_a0), movmatrix_16b(lpow_a1), movmatrix_16b(lpow_a2), movmatrix_16b(lpow_a3)), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), k_step=0, M=16, N=16, @@ -550,29 +558,39 @@ def _super_mma_warp( lpow_a1 = fp32_to_fp16(sq_acc[2], sq_acc[3], dtype=cfg.io_dtype) lpow_a2 = fp32_to_fp16(sq_acc[4], sq_acc[5], dtype=cfg.io_dtype) lpow_a3 = fp32_to_fp16(sq_acc[6], sq_acc[7], dtype=cfg.io_dtype) - # A_inv <- A_inv + A_inv @ Lpow, keeping A_inv in registers + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(lpow_a0), movmatrix_16b(lpow_a1), movmatrix_16b(lpow_a2), movmatrix_16b(lpow_a3) + # ---- A_inv += A_inv @ Lpow --------------------------------------- upd_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): upd_acc[accum_idx] = cutlass.Float32(0.0) + ainv_p0 = fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype) + ainv_p1 = fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype) + ainv_p2 = fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype) + ainv_p3 = fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype) mma_step( upd_acc, - ( - fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype), - fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype), - fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype), - fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype), - ), - (movmatrix_16b(lpow_a0), movmatrix_16b(lpow_a1), movmatrix_16b(lpow_a2), movmatrix_16b(lpow_a3)), + (ainv_p0, ainv_p1, ainv_p2, ainv_p3), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype, ) - for accum_idx in cutlass.range_constexpr(8): - ainv_acc[accum_idx] = ainv_acc[accum_idx].to(cfg.io_dtype).to(cutlass.Float32) + upd_acc[accum_idx] + ainv_lo0, ainv_hi0 = f16x2_to_f32(ainv_p0, dtype=cfg.io_dtype) + ainv_lo1, ainv_hi1 = f16x2_to_f32(ainv_p1, dtype=cfg.io_dtype) + ainv_lo2, ainv_hi2 = f16x2_to_f32(ainv_p2, dtype=cfg.io_dtype) + ainv_lo3, ainv_hi3 = f16x2_to_f32(ainv_p3, dtype=cfg.io_dtype) + ainv_acc[0] = ainv_lo0 + upd_acc[0] + ainv_acc[1] = ainv_hi0 + upd_acc[1] + ainv_acc[2] = ainv_lo1 + upd_acc[2] + ainv_acc[3] = ainv_hi1 + upd_acc[3] + ainv_acc[4] = ainv_lo2 + upd_acc[4] + ainv_acc[5] = ainv_hi2 + upd_acc[5] + ainv_acc[6] = ainv_lo3 + upd_acc[6] + ainv_acc[7] = ainv_hi3 + upd_acc[7] nvvm.stmatrix( - sPairwise_ptr + (cfg.b_t * cfg.b_t) + stsm_idx, + sQk_ptr + (cfg.b_t * cfg.b_t) + stsm_idx, [ fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype), fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype), @@ -583,15 +601,15 @@ def _super_mma_warp( shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_a_ready[pairwise_stage].arrive() - bars.mb_kk_qk_super_mma_done[decay_stage].arrive() + bars.mb_a_ready[qk_stage].arrive() + bars.mb_decay_super_done[decay_stage].arrive() raw_index = advance(raw_index, cfg.smem_raw_stages) - gbase += sk_nt - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + cum_chunk_base += num_chunks_tile + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) @cute.jit -def _tcgen05_mma_warp( +def tcgen05_mma_warp( cfg, total_tiles, bidx, @@ -599,24 +617,37 @@ def _tcgen05_mma_warp( cu_seqlens, mWorkItems, sSched, - tmem_hold, - sPairwise, + tmem_base_slot, + sQk, sK_decay, sK_restore, sQ_decay, sState_scale_diag, bars, ) -> None: - """tcgen05-MMA warp role (warp 13): persistent tile-scheduler loop, issues - all six state GEMMs in dependency order and owns the TMEM lifecycle.""" + """tcgen05-MMA warp role (warp 13): persistent scheduler loop issuing + every tcgen05 GEMM and owning the TMEM lifecycle.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) - tmem_base = tmem_hold.load() + tmem_base = tmem_base_slot.load() + state_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) + state_dsts = tuple(nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset + k * 16, cutlass.Float32) for k in range(cfg.d_k // 16)) + state_k_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_k_acc_offset, cutlass.Float32) + update_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_acc_offset, cutlass.Float32) + rhs_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_rhs_inp_offset, cutlass.Int8) + update_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_inp_offset, cutlass.Int8) + state_dst_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset, cutlass.Float32) state_inp_index = PipelineState.start(phase=0) + state_read_index = PipelineState.start(phase=0) rhs_index = PipelineState.start(phase=0) update_index = PipelineState.start(phase=0) final_state_index = PipelineState.start(phase=0) qk_scale_index = PipelineState.start(phase=0) - # ---- chunk-invariant GEMM descriptors ------------------------------ + k_decay_ready = PipelineState.start(phase=0) + qk_ready = PipelineState.start(phase=0) + o_acc_free = PipelineState.start(phase=1) + + # ---- chunk-invariant GEMM descriptors ---------------------------------------- bpe = cfg.io_dtype.width // 8 idesc_acc = nvvm.Tcgen05InstrDesc.build( c_dtype=cutlass.Float32, @@ -666,7 +697,7 @@ def _tcgen05_mma_warp( idesc=idesc_diag, kind=nvvm.Tcgen05MMAKind.F16, ) - bmm_pairwise_desc = MmaDesc( + bmm_qk_desc = MmaDesc( M=cfg.d_v, N=cfg.b_t, K=cfg.b_t, @@ -690,128 +721,130 @@ def _tcgen05_mma_warp( idesc=idesc_final_state, kind=nvvm.Tcgen05MMAKind.F16, ) - gbase = cutlass.Int32(0) + STATE_A_SEG = bmm_state_desc.sps_B * bmm_state_desc.tmem_advance_A + STATE_B_SEG = bmm_state_desc.smem_subtile_B >> 4 + cum_chunk_base = cutlass.Int32(0) sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) while tile_idx < total_tiles: batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) - sk_nt = wend - cstart - for li in cutlass.range(sk_nt, unroll=1): - gc = gbase + li - q_state_acc_stage = gc % cfg.tmem_q_state_acc_stages - decay_stage = gc % cfg.smem_decay_stages + num_chunks_tile = wend - cstart + for local_chunk_idx in cutlass.range(num_chunks_tile, unroll=1): + cum_chunk = cum_chunk_base + local_chunk_idx + q_state_acc_stage = o_acc_free.idx + decay_stage = k_decay_ready.idx state_scale_diag_stage = qk_scale_index.idx - pairwise_stage = gc % cfg.smem_pairwise_stages + qk_stage = qk_ready.idx sK_decay_stage = sK_decay[decay_stage] sQ_decay_stage = sQ_decay[decay_stage] sK_restore_stage = sK_restore[decay_stage] sState_scale_diag_stage = sState_scale_diag[state_scale_diag_stage] - sPairwise_stage = sPairwise[pairwise_stage] + sQk_stage = sQk[qk_stage] - # ---- state*k -> state_k_acc ---------------------------------------- - bars.mb_k_decay_cg0_ready[decay_stage].wait((gc // cfg.smem_decay_stages) % 2) + # ---- state_k = S(T) @ K_decay^T -------------------------------------- + bars.mb_k_decay_inv_cg0_ready[decay_stage].wait(k_decay_ready.phase) + k_decay_ready = advance(k_decay_ready, cfg.smem_decay_stages) bars.mb_state_inp_ready.wait(state_inp_index.phase) state_inp_index = advance(state_inp_index, 1) desc_k_decay = sK_decay_stage.desc() - state_a_tmem_base = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) - mma_ts( - bmm_state_desc, - state_a_tmem_base, - desc_k_decay, - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_k_acc_offset, cutlass.Float32), - accumulate=False, - ) + for s in cutlass.range_constexpr(bmm_state_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_state_desc.sps_B): + mma_ts_step( + bmm_state_desc, + state_inp_ptr.subview(s * STATE_A_SEG), + desc_k_decay + s * STATE_B_SEG, + state_k_acc_ptr, + k, + cutlass.Boolean(s + k > 0), + ) - if nvvm.elect_sync(): + if elect_one: bars.mb_state_k_acc_ready.arrive(cta_group=1) - # ---- state*q -> q_state_acc (stays live until qk@update fuses into o) + # ---- q_state = state(T) @ Q_decay^T --------------------------------- bars.mb_qk_scale_ready[qk_scale_index.idx].wait(qk_scale_index.phase) - bars.mb_o_acc_done[q_state_acc_stage].wait(((gc // cfg.tmem_q_state_acc_stages + cutlass.Int32(1)) % cutlass.Int32(2))) + bars.mb_o_acc_done[q_state_acc_stage].wait(o_acc_free.phase) + o_acc_free = advance(o_acc_free, cfg.tmem_q_state_acc_stages) desc_q_decay = sQ_decay_stage.desc() - state_a_tmem_base = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) - mma_ts( - bmm_state_desc, - state_a_tmem_base, - desc_q_decay, - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_q_state_acc_offset + q_state_acc_stage * cfg.b_t, cutlass.Float32), - accumulate=False, - ) + q_state_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_q_state_acc_offset + q_state_acc_stage * cfg.b_t, cutlass.Float32) + for s in cutlass.range_constexpr(bmm_state_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_state_desc.sps_B): + mma_ts_step( + bmm_state_desc, + state_inp_ptr.subview(s * STATE_A_SEG), + desc_q_decay + s * STATE_B_SEG, + q_state_acc_ptr, + k, + cutlass.Boolean(s + k > 0), + ) - if nvvm.elect_sync(): - bars.mb_kk_qk_mma_done[decay_stage].arrive(cta_group=1) + if elect_one: + bars.mb_decay_done[decay_stage].arrive(cta_group=1) - # ---- state decay (per-k-atom diag blocks) ---------------------------- + # ---- S decay = S(T) @ diag(exp2(G_last)) --------- + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_read_done.wait(state_read_index.phase) + state_read_index = advance(state_read_index, 1) desc_diag = sState_scale_diag_stage.desc() for k_block in cutlass.range_constexpr(cfg.d_k // 16): - mma_ts( + mma_ts_step( bmm_diag_desc, - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset + k_block * 8, cutlass.Int8), + state_inp_ptr.subview(k_block * bmm_diag_desc.tmem_advance_A), desc_diag.advance_start_address(k_block * 256 * 2), - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_offset + k_block * 16, cutlass.Float32), - accumulate=False, + state_dsts[k_block], + 0, + cutlass.Boolean(False), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_state_scale_diag_done[state_scale_diag_stage].arrive(cta_group=1) - # ---- update = A_inv @ rhs -> update_acc ------------------------------ - bars.mb_a_ready[pairwise_stage].wait((gc // cfg.smem_pairwise_stages) % 2) + # ---- U = rhs(T) @ A_inv ---------------------------------------------- + bars.mb_a_ready[qk_stage].wait(qk_ready.phase) bars.mb_rhs_ready.wait(rhs_index.phase) rhs_index = advance(rhs_index, 1) - lhs_tmem = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_rhs_inp_offset, cutlass.Int8) - desc_pairwise = sPairwise_stage.shifted((cfg.b_t * cfg.b_t)).desc() - mma_ts( - bmm_pairwise_desc, - lhs_tmem, - desc_pairwise, - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_acc_offset, cutlass.Float32), - accumulate=False, - ) - if nvvm.elect_sync(): + desc_qk = sQk_stage.shifted((cfg.b_t * cfg.b_t)).desc() + mma_ts_step(bmm_qk_desc, rhs_inp_ptr, desc_qk, update_acc_ptr, 0, cutlass.Boolean(False)) + if elect_one: + bars.mb_a_inv_done[qk_stage].arrive(cta_group=1) bars.mb_update_acc_ready.arrive(cta_group=1) - # ---- final_state += update @ k_restore ------------------------------- + # ---- final_state += U(T) @ K_restore --------------------------------- bars.mb_update_ready.wait(update_index.phase) update_index = advance(update_index, 1) - update_tmem = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_inp_offset, cutlass.Int8) desc_k_restore = sK_restore_stage.desc() - mma_ts( - bmm_final_state_desc, - update_tmem, - desc_k_restore, - nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_offset, cutlass.Float32), - accumulate=True, - ) - if nvvm.elect_sync(): + mma_ts_step(bmm_final_state_desc, update_inp_ptr, desc_k_restore, state_dst_ptr, 0, cutlass.Boolean(True)) + if elect_one: bars.mb_k_restore_done[decay_stage].arrive(cta_group=1) - - # ---- o += qk @ update -> q_state_acc ---------------------------------- - bars.mb_qk_acc_ready[pairwise_stage].wait((gc // cfg.smem_pairwise_stages) % 2) - lhs_tmem = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_inp_offset, cutlass.Int8) - desc_pairwise = sPairwise_stage.desc() - mma_ts( - bmm_pairwise_desc, - lhs_tmem, - desc_pairwise, + bars.mb_state_updated.arrive(cta_group=1) + + # ---- O += U(T) @ QK -------------------------------------------------- + bars.mb_qk_acc_ready[qk_stage].wait(qk_ready.phase) + qk_ready = advance(qk_ready, cfg.smem_qk_stages) + desc_qk = sQk_stage.desc() + mma_ts_step( + bmm_qk_desc, + update_inp_ptr, + desc_qk, nvvm.make_tmem_ptr(tmem_base + cfg.tmem_q_state_acc_offset + q_state_acc_stage * cfg.b_t, cutlass.Float32), - accumulate=True, + 0, + cutlass.Boolean(True), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_o_acc_ready.arrive(cta_group=1) - bars.mb_a_done[pairwise_stage].arrive(cta_group=1) + bars.mb_qk_done[qk_stage].arrive(cta_group=1) qk_scale_index = advance(qk_scale_index, cfg.smem_state_scale_diag_stages) bars.mb_final_state_stored.wait(final_state_index.phase) final_state_index = advance(final_state_index, 1) - gbase += sk_nt - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + cum_chunk_base += num_chunks_tile + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) nvvm.tcgen05_dealloc( nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), cutlass.Int32(512), @@ -820,7 +853,7 @@ def _tcgen05_mma_warp( @cute.jit -def _epilogue_warp( +def epilogue_warp( cfg, total_tiles, bidx, @@ -832,27 +865,23 @@ def _epilogue_warp( mO, sK_inv_raw, sO_raw, - sPairwise_raw, + sQk_raw, sQ_decay_raw, - sH_raw, + sCheckpoint_raw, desc_o_base, - desc_h_base, + desc_checkpoint_base, checkpoint_every_n_tokens, bars, ) -> None: - """Epilogue warp role (warp 15): persistent tile-scheduler loop + - register-MMA qk (causal), A_inv qk staging, the O TMA store drain - (split-K warmup chunks drain the SMEM stage but never store), and the - per-chunk H TMA store when checkpoints are enabled. O and H stores go - through the per-(batch, head) descriptor arrays: the token / entry - extents are capped per sequence, so partial tails are clipped by the - hardware and the H coordinate is sequence-local.""" + """Epilogue warp role (warp 15): persistent scheduler loop computing the + causal QK via register MMA and draining the O/checkpoint TMA stores.""" + elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) if cutlass.const_expr(cfg.enable_checkpoints): - sH_tma = SmemTile( - base=sH_raw, + sCheckpoint_tma = SmemTile( + base=sCheckpoint_raw, elems_per_stage=(cfg.d_k * cfg.d_v), - stages=1, + stages=cfg.smem_checkpoint_stages, leading_byte_offset=0, stride_byte_offset=0, layout=0, @@ -860,7 +889,7 @@ def _epilogue_warp( tma_granu_elems=64, tma_subtile_stride_elems=cfg.d_k * 64, ) - h_ready_index = PipelineState.start(phase=0) + checkpoint_ready_index = PipelineState.start(phase=0) sO_tma = SmemTile( base=sO_raw, elems_per_stage=(cfg.b_t * cfg.d_v), @@ -873,13 +902,14 @@ def _epilogue_warp( tma_subtile_stride_elems=cfg.b_t * 64, ) qk_scale_index = PipelineState.start(phase=0) - # ---- ldmatrix/stmatrix lane decode --------------------------------- + qk_free = PipelineState.start(phase=1) + + # ---- ldmatrix/stmatrix lane decode ------------------------------------------- rhs_row_coord = lane % 8 + (cutlass.Int32(8) if (lane // 16) else cutlass.Int32(0)) rhs_col_offset = cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0) lhs_row_coord = lane % 8 + (cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0)) lhs_col_offset = cutlass.Int32(8) if ((lane // 8) // 2) else cutlass.Int32(0) - decay_key_mask = cutlass.Int32(8) ^ ((lhs_row_coord & cutlass.Int32(2)) * cutlass.Int32(16)) - elems_per_128b = cutlass.Int32(64) + decay_key_mask = cutlass.Int32(8) stsm_row_coord = lane & 7 stsm_col_coord = cutlass.Int32(0) if (lane // 8) & 1: @@ -887,7 +917,7 @@ def _epilogue_warp( if lane // 8 >= 2: stsm_col_coord = cutlass.Int32(8) stsm_idx = swizzle_lin_S(stsm_row_coord * cfg.b_t + (stsm_col_coord ^ (cfg.b_t // 2)), bbits=1, mbase=3, sshift=3) - gbase = cutlass.Int32(0) + cum_chunk_base = cutlass.Int32(0) sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) while tile_idx < total_tiles: @@ -895,27 +925,33 @@ def _epilogue_warp( cfg, tile_idx, cu_seqlens, mWorkItems ) head_o = head_idx - o_slot = (batch_idx * cutlass.Int32(cfg.n_heads_out) + head_idx) * cutlass.Int32(TENSOR_MAP_QWORDS) + o_slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) desc_o_slot = (desc_o_base + o_slot).tospace(cutlass.AddressSpace.generic) if cutlass.const_expr(cfg.enable_checkpoints): - desc_h_slot = (desc_h_base + o_slot).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): - tma_tensormap_acquire(desc_h_slot) - if nvvm.elect_sync(): + desc_checkpoint_slot = (desc_checkpoint_base + o_slot).tospace(cutlass.AddressSpace.generic) + checkpoint_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) + checkpoint_quot = (cstart + cutlass.Int32(1)) // checkpoint_chunks + checkpoint_mod = (cstart + cutlass.Int32(1)) % checkpoint_chunks + if elect_one: + tma_tensormap_acquire(desc_checkpoint_slot) + if elect_one: tma_tensormap_acquire(desc_o_slot) - sk_nt = wend - cstart - for li in cutlass.range(sk_nt, unroll=1): - chunk_idx = cstart + li - gc = gbase + li - decay_stage = gc % cfg.smem_decay_stages - pairwise_stage = gc % cfg.smem_pairwise_stages + num_chunks_tile = wend - cstart + for local_chunk_idx in cutlass.range(num_chunks_tile, unroll=1): + chunk_idx = cstart + local_chunk_idx + cum_chunk = cum_chunk_base + local_chunk_idx + decay_stage = cum_chunk % cfg.smem_decay_stages + qk_stage = qk_free.idx + sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) sQ_decay_ptr = sQ_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) - sPairwise_ptr = sPairwise_raw.data_ptr() + pairwise_stage * (2 * cfg.b_t * cfg.b_t) + sQk_ptr = sQk_raw.data_ptr() + qk_stage * (2 * cfg.b_t * cfg.b_t) - bars.mb_a_done[pairwise_stage].wait(((gc // cfg.smem_pairwise_stages) + 1) % 2) + bars.mb_qk_done[qk_stage].wait(qk_free.phase) + qk_free = advance(qk_free, cfg.smem_qk_stages) bars.mb_qk_scale_ready[qk_scale_index.idx].wait(qk_scale_index.phase) - # ---- qk register MMA (inclusive-causal) ---------------------------- + + # ---- QK = Q_decay @ K_inv^T ----------------------------------------- qk_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): qk_acc[accum_idx] = cutlass.Float32(0.0) @@ -934,18 +970,11 @@ def _epilogue_warp( ) # Load A operand storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask - storage_slice = storage_key // elems_per_128b - key_in_slice = storage_key - storage_slice * elems_per_128b - storage_phase = key_in_slice // cutlass.Int32(16) - byte_in_slice = ( - lhs_row_coord * cutlass.Int32(128) - + storage_phase * cutlass.Int32(32) - + (key_in_slice - storage_phase * cutlass.Int32(16)) * cutlass.Int32(2) - ) + storage_slice = storage_key // 64 qk_lhs_vec = nvvm.ldmatrix( sQ_decay_ptr - + storage_slice * cutlass.Int32(cfg.b_t) * elems_per_128b - + ((byte_in_slice ^ ((lhs_row_coord & cutlass.Int32(7)) << 4)) // cutlass.Int32(2)), + + storage_slice * (cfg.b_t * 64) + + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) @@ -970,7 +999,7 @@ def _epilogue_warp( qk_acc[accum_idx] = qk_acc[accum_idx] if row_coord >= col_coord else cutlass.Float32(0.0) nvvm.stmatrix( - sPairwise_ptr + stsm_idx, + sQk_ptr + stsm_idx, [ fp32_to_fp16(qk_acc[0], qk_acc[1], dtype=cfg.io_dtype), fp32_to_fp16(qk_acc[2], qk_acc[3], dtype=cfg.io_dtype), @@ -981,64 +1010,94 @@ def _epilogue_warp( shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_qk_acc_ready[pairwise_stage].arrive() - bars.mb_kk_qk_super_mma_done[decay_stage].arrive() + bars.mb_qk_acc_ready[qk_stage].arrive() + bars.mb_decay_super_done[decay_stage].arrive() qk_scale_index = advance(qk_scale_index, cfg.qk_scale_ready_stages) - # ---- O drain: staged output tile -> GMEM TMA store ----------------- - if li > 0: + # ---- checkpoint + O drain: checkpoint stores first (CG1 stages the checkpoint before O) ------------- + if local_chunk_idx > 0: output_chunk = chunk_idx - cutlass.Int32(1) output_chunk_start = output_chunk * cfg.b_t - o_stage = (gc - cutlass.Int32(1)) % cfg.smem_o_stages - bars.mb_o_tmastg_ready[o_stage].wait(((gc - cutlass.Int32(1)) // cfg.smem_o_stages) % 2) - o_slice = tma_slice_runtime_desc(desc_o_slot, cutlass.Int32(0), output_chunk_start) + o_stage = (cum_chunk - cutlass.Int32(1)) % cfg.smem_o_stages + did_checkpoint = cutlass.Int32(0) + checkpoint_stage = cutlass.Int32(0) + if cutlass.const_expr(cfg.enable_checkpoints): + # ---- checkpoint store ---------------------------------------- + do_checkpoint = checkpoint_mod == 0 + if cutlass.const_expr(cfg.split_k): + do_checkpoint = do_checkpoint and chunk_idx >= wstart + checkpoint_stage = checkpoint_ready_index.idx + if do_checkpoint: + bars.mb_checkpoint_tmastg_ready[checkpoint_ready_index.idx].wait(checkpoint_ready_index.phase) + checkpoint_ready_index = advance(checkpoint_ready_index, cfg.smem_checkpoint_stages) + checkpoint_entry = checkpoint_quot - cutlass.Int32(1) + checkpoint_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), checkpoint_entry, head_o) + tma_store_tile(sCheckpoint_tma[checkpoint_stage], checkpoint_slice, acquire=False) + tma_store_commit() + did_checkpoint = cutlass.Int32(1) + checkpoint_mod = checkpoint_mod + cutlass.Int32(1) + if checkpoint_mod == checkpoint_chunks: + checkpoint_mod = cutlass.Int32(0) + checkpoint_quot = checkpoint_quot + cutlass.Int32(1) + bars.mb_o_tmastg_ready[o_stage].wait(((cum_chunk - cutlass.Int32(1)) // cfg.smem_o_stages) % 2) + o_slice = tma_slice_runtime_desc(desc_o_slot, cutlass.Int32(0), head_o, output_chunk_start) + did_o = cutlass.Int32(0) if cutlass.const_expr(cfg.split_k): - # warmup chunks stage O to SMEM but never store it if output_chunk >= wstart: tma_store_tile(sO_tma[o_stage], o_slice, acquire=False) tma_store_commit() + did_o = cutlass.Int32(1) else: tma_store_tile(sO_tma[o_stage], o_slice, acquire=False) tma_store_commit() - tma_store_wait(0) - bars.mb_o_tmastg_done[o_stage].arrive() - # ---- H store: CG1 staged the state entering chunk_idx ---------- - if cutlass.const_expr(cfg.enable_checkpoints): - if li > 0: - tokens_done = chunk_idx * cutlass.Int32(cfg.b_t) - do_h = tokens_done % checkpoint_every_n_tokens == 0 - if cutlass.const_expr(cfg.split_k): - do_h = do_h and chunk_idx >= wstart - if do_h: - bars.mb_h_tmastg_ready.wait(h_ready_index.phase) - h_ready_index = advance(h_ready_index, 1) - # sequence-local entry: the per-(b,h) descriptor folds the - # sequence base into GLOBAL_ADDRESS and caps the extent - h_entry = tokens_done // checkpoint_every_n_tokens - cutlass.Int32(1) - h_slice = tma_slice_runtime_desc(desc_h_slot, cutlass.Int32(0), cutlass.Int32(0), h_entry) - tma_store_tile(sH_tma[0], h_slice, acquire=False) - tma_store_commit() + did_o = cutlass.Int32(1) + if cutlass.const_expr(cfg.enable_checkpoints): + if did_checkpoint == 1 and did_o == 1: + tma_store_wait(1) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].arrive() tma_store_wait(0) - bars.mb_h_tmastg_done.arrive() - # ---- last computed chunk drain (always owned: it is wend - 1) ------ - if sk_nt > 0: + bars.mb_o_tmastg_done[o_stage].arrive() + if did_checkpoint == 1 and did_o == 0: + tma_store_wait(0) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].arrive() + bars.mb_o_tmastg_done[o_stage].arrive() + if did_checkpoint == 0: + if did_o == 1: + tma_store_wait(0) + bars.mb_o_tmastg_done[o_stage].arrive() + else: + tma_store_wait(0) + bars.mb_o_tmastg_done[o_stage].arrive() + + # ---- last computed chunk drain (always owned: it is wend - 1) ------------ + if num_chunks_tile > 0: output_chunk = wend - cutlass.Int32(1) - og = gbase + sk_nt - cutlass.Int32(1) + last_cum_chunk = cum_chunk_base + num_chunks_tile - cutlass.Int32(1) output_chunk_start = output_chunk * cfg.b_t - o_stage = og % cfg.smem_o_stages - bars.mb_o_tmastg_ready[o_stage].wait((og // cfg.smem_o_stages) % 2) - # a partial last chunk is clipped by the descriptor's token extent - o_slice = tma_slice_runtime_desc(desc_o_slot, cutlass.Int32(0), output_chunk_start) + o_stage = last_cum_chunk % cfg.smem_o_stages + bars.mb_o_tmastg_ready[o_stage].wait((last_cum_chunk // cfg.smem_o_stages) % 2) + o_slice = tma_slice_runtime_desc(desc_o_slot, cutlass.Int32(0), head_o, output_chunk_start) tma_store_tile(sO_tma[o_stage], o_slice, acquire=False) tma_store_commit() tma_store_wait(0) bars.mb_o_tmastg_done[o_stage].arrive() - gbase += sk_nt - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + cum_chunk_base += num_chunks_tile + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def gate_scale(cfg, raw_gate: cutlass.Float32) -> cutlass.Float32: + """Map raw gate to the log2-domain decay increment used by KDA.""" + + if cutlass.const_expr(cfg.safe_gate): + half = cutlass.Float32(0.5) + sigmoid = cute.math.tanh(raw_gate * half, approx=True) * half + half + return cfg.gate_scale_log2 * sigmoid + return raw_gate * cutlass.Float32(LOG2_E) @cute.jit -def _compute0_warp_group( +def compute0_warp_group( cfg, total_tiles, bidx, @@ -1053,18 +1112,18 @@ def _compute0_warp_group( mDt_bias, sK_inv_raw, sGate_raw, + mBeta, + sBeta_raw, sK_raw, sQ_raw, - sV_raw, sK_decay_raw, sK_restore_raw, sQ_decay_raw, sState_scale_diag_raw, bars, ) -> None: - """CG0 warp role (warps 0-7, two ping-pong groups): persistent - tile-scheduler loop + gate prefix scan and the decay/restore operand - materialization into tcgen05 SMEM.""" + """CG0 warp role (warps 0-7, two ping-pong groups): persistent scheduler + loop for the Gate prefix scan and decay/restore operand materialization.""" nvvm.setmaxregister(cfg.num_regs_compute_group_0, nvvm.SetMaxRegisterAction.INCREASE) cg0_warp = warp_idx - cfg.compute_group_0_warp_ids[0] cg0_group_id = cg0_warp // cfg.cg0_warps_per_group @@ -1072,32 +1131,33 @@ def _compute0_warp_group( prefix_dim = cg0_local_warp * cfg.threads_per_warp + lane cg0_a_log_exp = cutlass.Float32(1.0) cg0_dt_bias_value = cutlass.Float32(0.0) - gbase = cutlass.Int32(0) + cum_chunk_base = cutlass.Int32(0) sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) + opaque_one = opaque_f32_zero() + cutlass.Float32(1.0) while tile_idx < total_tiles: batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( cfg, tile_idx, cu_seqlens, mWorkItems ) head_o = head_idx - sk_nt = wend - cstart + num_chunks_tile = wend - cstart if cutlass.const_expr(cfg.safe_gate): - # per-head safe-gate constants straight from GMEM (the head - # changes per tile, so there is no SMEM staging to handshake) - if sk_nt > 0: + if num_chunks_tile > 0: cg0_a_log_exp = cute.math.exp2(mA_log[head_o].to(cutlass.Float32) * LOG2_E, fastmath=True) cg0_dt_bias_value = mDt_bias[head_o, prefix_dim].to(cutlass.Float32) - for li in cutlass.range(cg0_group_id, sk_nt, cfg.cg0_group_count, unroll=1): - chunk_idx = cstart + li - gc = gbase + li + group_cum_chunk_start = cum_chunk_base + cutlass.Int32(cg0_group_id) + diag_ring_idx = group_cum_chunk_start % cutlass.Int32(cfg.smem_state_scale_diag_stages) + diag_ring_phase = (group_cum_chunk_start // cutlass.Int32(cfg.smem_state_scale_diag_stages)) % cutlass.Int32(2) + for local_chunk_idx in cutlass.range(cg0_group_id, num_chunks_tile, cfg.cg0_group_count, unroll=1): + chunk_idx = cstart + local_chunk_idx + cum_chunk = cum_chunk_base + local_chunk_idx chunk_start = chunk_idx * cfg.b_t - decay_stage = gc % cfg.smem_decay_stages - raw_stage = gc % cfg.smem_raw_stages - state_scale_diag_stage = gc % cfg.smem_state_scale_diag_stages + decay_stage = cum_chunk % cfg.smem_decay_stages + raw_stage = cum_chunk % cfg.smem_raw_stages + state_scale_diag_stage = diag_ring_idx qk_scale_ready_stage = state_scale_diag_stage sQ_ptr = sQ_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) sK_ptr = sK_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) - sV_ptr = sV_raw.data_ptr() + raw_stage * (cfg.d_v * cfg.b_t) sGate_ptr = sGate_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) sK_decay_ptr = sK_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) @@ -1105,65 +1165,36 @@ def _compute0_warp_group( sK_restore_ptr = sK_restore_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) sState_scale_diag_ptr = sState_scale_diag_raw.data_ptr() + state_scale_diag_stage * ((cfg.d_k // 16) * 256) - bars.mb_inputs_ready[raw_stage].wait((gc // cfg.smem_raw_stages) % 2) - - # ---- tail chunk: zero-fill raw staging past seqlen ----------------- - if chunk_start + cutlass.Int32(cfg.b_t) > seqlen_b: - if cg0_local_warp == 0: - f16_zero = mQ.element_type(0.0) - f16_zero_vec = cutlass.Vector.from_elements( - ( - f16_zero, - f16_zero, - f16_zero, - f16_zero, - f16_zero, - f16_zero, - f16_zero, - f16_zero, - ), - mQ.element_type, - ) - f32_zero = cutlass.Float32(0.0) - f32_zero_vec = cutlass.Vector.from_elements( - (f32_zero, f32_zero, f32_zero, f32_zero), - cutlass.Float32, - ) - for row in cutlass.range_constexpr(cfg.b_t): - token_idx = chunk_start + cutlass.Int32(row) - if token_idx >= seqlen_b: - if lane < (cfg.d_k // 8): - f16_dim_base = lane * 8 - f16_segment = f16_dim_base // 64 - f16_segment_dim = f16_dim_base - f16_segment * 64 - f16_idx = f16_segment * (cfg.b_t * 64) + row * 64 + swizzle_xor_128b(row, f16_segment_dim, elem_bytes=2) - (sQ_ptr + f16_idx).store(f16_zero_vec, alignment=16) - (sK_ptr + f16_idx).store(f16_zero_vec, alignment=16) - (sV_ptr + f16_idx).store(f16_zero_vec, alignment=16) - if lane < (cfg.d_k // 4): - f32_dim_base = lane * 4 - f32_segment = f32_dim_base // 32 - f32_segment_dim = f32_dim_base - f32_segment * 32 - f32_idx = f32_segment * (cfg.b_t * 32) + row * 32 + swizzle_xor_128b(row, f32_segment_dim, elem_bytes=4) - (sGate_ptr + f32_idx).store(f32_zero_vec, alignment=16) - nvvm.barrier_cta_sync(cfg.nbar_cg0_group0_id + cg0_group_id, thread_count=cfg.cg0_threads_per_group) + # ---- Beta scalars --------------------------------------------------- + if cg0_local_warp == 0: + bars.mb_beta_done[raw_stage].wait(((cum_chunk // cfg.smem_raw_stages) + 1) % 2) + if lane < cfg.b_t: + token_idx = chunk_idx * cfg.b_t + lane + beta_value = cutlass.Float32(0.0) + if token_idx < seqlen_b: + beta_value = mBeta[batch_start + token_idx, head_o].to(cutlass.Float32) + if cutlass.const_expr(cfg.beta_sigmoid): + half = cutlass.Float32(0.5) + beta_value = (cute.math.tanh(beta_value * half, approx=True) * half + half).to(mBeta.element_type).to(cutlass.Float32) + sBeta_raw[raw_stage * cfg.b_t + lane] = beta_value + bars.mb_beta_ready[raw_stage].arrive() + bars.mb_gate_ready[raw_stage].wait((cum_chunk // cfg.smem_raw_stages) % 2) row_group_start = cg0_local_warp * (cfg.b_t // cfg.cg0_warps_per_group) lane_row_group = lane // 8 lane_in_row_group = lane - lane_row_group * 8 decay_row = row_group_start + lane_row_group - - g_prefix_ptr = sGate_ptr + decay_key_mask = cutlass.Int32(8) prefix_dim = cg0_local_warp * cfg.threads_per_warp + lane - # ---- gate prefix scan: cumulative log-gate per key channel -------- - # gathers first, math second: a load consumed in the same iteration - # serializes on LDS latency + + # ---- Gate prefix scan ----------------------------------------------- + f32_segment = prefix_dim // 32 + prefix_seg_base = f32_segment * (cfg.b_t * 32) + prefix_col = prefix_dim - f32_segment * 32 gate_raw = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) for row in cutlass.range_constexpr(cfg.b_t): - f32_segment = prefix_dim // 32 - f32_segment_dim = prefix_dim - f32_segment * 32 - prefix_idx = f32_segment * (cfg.b_t * 32) + row * 32 + swizzle_xor_128b(row, f32_segment_dim, elem_bytes=4) + prefix_idx = prefix_seg_base + swizzle_xor_128b(row, row * 32 + prefix_col, elem_bytes=4) gate_raw[row] = (sGate_ptr + prefix_idx).load() g_prefix_regs = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) if cutlass.const_expr(cfg.safe_gate): @@ -1174,11 +1205,11 @@ def _compute0_warp_group( row1 = row0 + 1 gate0 = cg0_a_log_exp * (gate_raw[row0] + cg0_dt_bias_value) gate1 = cg0_a_log_exp * (gate_raw[row1] + cg0_dt_bias_value) - gate0 = _gate_log2( + gate0 = gate_scale( cfg, gate0, ) - gate1 = _gate_log2( + gate1 = gate_scale( cfg, gate1, ) @@ -1191,7 +1222,7 @@ def _compute0_warp_group( gate = gate_raw[row] token_idx = chunk_idx * cutlass.Int32(cfg.b_t) + cutlass.Int32(row) if token_idx < seqlen_b: - gate = _gate_log2( + gate = gate_scale( cfg, gate, ) @@ -1211,19 +1242,17 @@ def _compute0_warp_group( g_prefix_regs[row1] = prefix1 prefix_acc = prefix1 + # ---- exp2(G): stage prefixes + final-token decay --------------------- for row in cutlass.range_constexpr(cfg.b_t): g_prefix_regs[row] = cute.math.exp2(g_prefix_regs[row], fastmath=True) - # ---- exp2(g): stage prefixes + final-token decay ------------------ exp_g_last = g_prefix_regs[cfg.b_t - 1] for row in cutlass.range_constexpr(cfg.b_t): - f32_segment = prefix_dim // 32 - f32_segment_dim = prefix_dim - f32_segment * 32 - prefix_idx = f32_segment * (cfg.b_t * 32) + row * 32 + swizzle_xor_128b(row, f32_segment_dim, elem_bytes=4) + prefix_idx = prefix_seg_base + swizzle_xor_128b(row, row * 32 + prefix_col, elem_bytes=4) (sGate_ptr + prefix_idx).store(g_prefix_regs[row]) - # ---- state-scale diag: stage exp2(g_last) decay blocks ------------- - bars.mb_state_scale_diag_done[state_scale_diag_stage].wait((gc // cfg.smem_state_scale_diag_stages + 1) % 2) + # ---- state-scale diag: stage exp2(G_last) decay blocks --------------- + bars.mb_state_scale_diag_done[state_scale_diag_stage].wait(diag_ring_phase ^ cutlass.Int32(1)) block = prefix_dim // cutlass.Int32(16) coord = prefix_dim - block * cutlass.Int32(16) storage_col = coord ^ cutlass.Int32((cfg.b_t // 2)) @@ -1231,14 +1260,20 @@ def _compute0_warp_group( diag_idx = swizzle_lin_S(linear_idx, bbits=1, mbase=3, sshift=3) sState_scale_diag_ptr[diag_idx] = exp_g_last.to(cfg.io_dtype) - nvvm.barrier_cta_sync(cfg.nbar_cg0_group0_id + cg0_group_id, thread_count=cfg.cg0_threads_per_group) + nvvm.barrier_cta_sync(cfg.cg0_group_sync_barrier_base_id + cg0_group_id, thread_count=cfg.cg0_threads_per_group) + bars.mb_q_ready[raw_stage].wait((cum_chunk // cfg.smem_raw_stages) % 2) + bars.mb_k_ready[raw_stage].wait((cum_chunk // cfg.smem_raw_stages) % 2) k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) raw_q_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) - # ---- optional q/k L2-norm + K_inv staging -------------------------- - q_sum_sq = cutlass.Float32(0.0) - k_sum_sq = cutlass.Float32(0.0) + + # ---- optional Q/K L2-norm ------------------------------------------- + if cutlass.const_expr(cfg.l2norm): + q_sq_even = opaque_f32_zero() + k_sq_even = opaque_f32_zero() + q_sq_odd = opaque_f32_zero() + k_sq_odd = opaque_f32_zero() for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 @@ -1254,14 +1289,17 @@ def _compute0_warp_group( k_val = raw_k_vec_f32[dim_offset] raw_q_regs[reg_base + dim_offset] = q_val raw_k_regs[reg_base + dim_offset] = k_val - q_sum_sq = q_sum_sq + q_val * q_val - k_sum_sq = k_sum_sq + k_val * k_val + if cutlass.const_expr(cfg.l2norm): + if cutlass.const_expr(dim_offset % 2 == 0): + q_sq_even, k_sq_even = ffma2(q_val, k_val, q_val, k_val, q_sq_even, k_sq_even) + else: + q_sq_odd, k_sq_odd = ffma2(q_val, k_val, q_val, k_val, q_sq_odd, k_sq_odd) - # opaque 1.0: keeps the no-l2norm packed-mul operands out of libNVVM's - # constant folder (the documented inline_ptx "n"-constraint ICE) - q_inv_norm = opaque_f32_zero() + cutlass.Float32(1.0) - k_inv_norm = opaque_f32_zero() + cutlass.Float32(1.0) + q_inv_norm = opaque_one + k_inv_norm = opaque_one if cutlass.const_expr(cfg.l2norm): + q_sum_sq = q_sq_even + q_sq_odd + k_sum_sq = k_sq_even + k_sq_odd q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 4, 31, kind=nvvm.Shfl.BFLY)) q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 2, 31, kind=nvvm.Shfl.BFLY)) q_sum_sq = q_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, q_sum_sq, 1, 31, kind=nvvm.Shfl.BFLY)) @@ -1272,11 +1310,9 @@ def _compute0_warp_group( q_inv_norm = cute.math.rsqrt(cute.math.max(q_sum_sq, norm_floor_sq), fastmath=True) k_inv_norm = cute.math.rsqrt(cute.math.max(k_sum_sq, norm_floor_sq), fastmath=True) - # ---- decay/restore operands: exp2(+-g) applied per key channel ----- + # ---- decay/restore operands: exp2(+-G) ------------------------------ exp_g_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) exp_g_last_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) - # both halves' prefix gathers up front: the sK_inv/sK_decay stores - # below would otherwise pin the second half's loads behind them for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 @@ -1285,23 +1321,19 @@ def _compute0_warp_group( f32_segment = f32_dim_base // 32 f32_segment_dim = f32_dim_base - f32_segment * 32 g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) - exp_g_vec = (g_prefix_ptr + g_prefix_idx).load(count=4, alignment=16) + exp_g_vec = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) - exp_g_last_vec = (g_prefix_ptr + exp_g_last_idx).load(count=4, alignment=16) + exp_g_last_vec = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) f32_reg_base = reg_base + f32_group * 4 - exp_g_regs[f32_reg_base] = exp_g_vec[0] - exp_g_regs[f32_reg_base + 1] = exp_g_vec[1] - exp_g_regs[f32_reg_base + 2] = exp_g_vec[2] - exp_g_regs[f32_reg_base + 3] = exp_g_vec[3] - exp_g_last_regs[f32_reg_base] = exp_g_last_vec[0] - exp_g_last_regs[f32_reg_base + 1] = exp_g_last_vec[1] - exp_g_last_regs[f32_reg_base + 2] = exp_g_last_vec[2] - exp_g_last_regs[f32_reg_base + 3] = exp_g_last_vec[3] + for j in cutlass.range_constexpr(4): + exp_g_regs[f32_reg_base + j] = exp_g_vec[j] + exp_g_last_regs[f32_reg_base + j] = exp_g_last_vec[j] for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 - # ---- k decay operand: exp2(g) * k ------------------------------ + + # ---- K decay + K_inv operands: K * exp2(+G) and K * exp2(-G) ----- k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 @@ -1314,7 +1346,8 @@ def _compute0_warp_group( k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) exp_neg_g0 = cute.math.rcp(exp_g_regs[raw_reg_idx0], approx=True, ftz=True) exp_neg_g1 = cute.math.rcp(exp_g_regs[raw_reg_idx1], approx=True, ftz=True) - k_inv_words[dim_half * 4 + pair_idx] = fp32_to_fp16(k_value0 * exp_neg_g0, k_value1 * exp_neg_g1, dtype=cfg.io_dtype) + exp_neg_pair = fp32_to_fp16(exp_neg_g0, exp_neg_g1, dtype=cfg.io_dtype) + k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_pair, exp_neg_pair, cfg.io_dtype) k_inv_vec = cutlass.Vector.from_elements( ( @@ -1335,30 +1368,23 @@ def _compute0_warp_group( cutlass.Int32, ).bitcast(cfg.io_dtype) if cutlass.const_expr(dim_half == 0): - operand_done_phase = ((gc // cfg.smem_decay_stages) + 1) % 2 - bars.mb_kk_qk_super_mma_done[decay_stage].wait(operand_done_phase) - bars.mb_kk_qk_mma_done[decay_stage].wait(operand_done_phase) + operand_done_phase = ((cum_chunk // cfg.smem_decay_stages) + 1) % 2 + bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 k_inv_swizzled_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) (sK_inv_ptr + k_inv_swizzled_idx).store(k_inv_vec, alignment=16) - key_mask = cutlass.Int32(8) ^ (decay_row & cutlass.Int32(2)) * cutlass.Int32(16) - decay_storage_dim_base = dim_base ^ key_mask - decay_linear_idx_base = decay_row * cfg.d_k + decay_storage_dim_base - sw128_elems_per_128b = 128 // 2 - sw128_row = decay_linear_idx_base // cfg.d_k - sw128_col = decay_linear_idx_base - sw128_row * cfg.d_k - sw128_slice = sw128_col // sw128_elems_per_128b - sw128_col_in_slice = sw128_col - sw128_slice * sw128_elems_per_128b - sw128_slice_linear = sw128_row * sw128_elems_per_128b + sw128_col_in_slice - sw128_byte = sw128_slice_linear * 2 - sw128_mask = (sw128_byte >> 7 & 7) << 4 - decay_swizzled_idx_base = sw128_slice * cfg.b_t * sw128_elems_per_128b + (sw128_byte ^ sw128_mask) // 2 - (sK_decay_ptr + decay_swizzled_idx_base).store(k_decay_vec, alignment=16) + storage_key = dim_base ^ decay_key_mask + storage_slice = storage_key // 64 + decay_swizzled_idx = storage_slice * (cfg.b_t * 64) + swizzle_xor_128b( + decay_row, decay_row * 64 + storage_key - storage_slice * 64, elem_bytes=2 + ) + (sK_decay_ptr + decay_swizzled_idx).store(k_decay_vec, alignment=16) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_k_decay_cg0_ready[decay_stage].arrive() + bars.mb_k_decay_inv_cg0_ready[decay_stage].arrive() - # ---- q decay operand ----------------------------------------------- + # ---- Q_decay operand: Q * q_inv_norm -------------------------------- for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 @@ -1382,23 +1408,15 @@ def _compute0_warp_group( ), cutlass.Int32, ).bitcast(cfg.io_dtype) - key_mask = cutlass.Int32(8) ^ (decay_row & cutlass.Int32(2)) * cutlass.Int32(16) - decay_storage_dim_base = dim_base ^ key_mask - decay_linear_idx_base = decay_row * cfg.d_k + decay_storage_dim_base - sw128_elems_per_128b = 128 // 2 - sw128_row = decay_linear_idx_base // cfg.d_k - sw128_col = decay_linear_idx_base - sw128_row * cfg.d_k - sw128_slice = sw128_col // sw128_elems_per_128b - sw128_col_in_slice = sw128_col - sw128_slice * sw128_elems_per_128b - sw128_slice_linear = sw128_row * sw128_elems_per_128b + sw128_col_in_slice - sw128_byte = sw128_slice_linear * 2 - sw128_mask = (sw128_byte >> 7 & 7) << 4 - decay_swizzled_idx_base = sw128_slice * cfg.b_t * sw128_elems_per_128b + (sw128_byte ^ sw128_mask) // 2 - (sQ_decay_ptr + decay_swizzled_idx_base).store(q_decay_vec, alignment=16) - - bars.mb_k_restore_done[decay_stage].wait(((gc // cfg.smem_decay_stages + 1) % 2)) - - # ---- k_restore operand ---------------------------------------------- + storage_key = dim_base ^ decay_key_mask + storage_slice = storage_key // 64 + decay_swizzled_idx = storage_slice * (cfg.b_t * 64) + swizzle_xor_128b( + decay_row, decay_row * 64 + storage_key - storage_slice * 64, elem_bytes=2 + ) + (sQ_decay_ptr + decay_swizzled_idx).store(q_decay_vec, alignment=16) + + # ---- K_restore operand: K_inv * exp_g_last -------------------------- + bars.mb_k_restore_done[decay_stage].wait(((cum_chunk // cfg.smem_decay_stages + 1) % 2)) for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 @@ -1424,12 +1442,19 @@ def _compute0_warp_group( (sK_restore_ptr + k_restore_idx).store(k_restore_vec, alignment=16) nvvm.fence_proxy("async.shared", space="cta") bars.mb_qk_scale_ready[qk_scale_ready_stage].arrive() - gbase += sk_nt - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_q_done[raw_stage].arrive() + bars.mb_k_done[raw_stage].arrive() + bars.mb_gate_done[raw_stage].arrive() + diag_ring_idx = diag_ring_idx + cutlass.Int32(cfg.cg0_group_count) + wrapped = diag_ring_idx >= cutlass.Int32(cfg.smem_state_scale_diag_stages) + diag_ring_idx = diag_ring_idx - cutlass.Int32(cfg.smem_state_scale_diag_stages) if wrapped else diag_ring_idx + diag_ring_phase = diag_ring_phase ^ (cutlass.Int32(1) if wrapped else cutlass.Int32(0)) + cum_chunk_base += num_chunks_tile + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) @cute.jit -def _compute1_warp_group( +def compute1_warp_group( cfg, total_tiles, bidx, @@ -1438,41 +1463,65 @@ def _compute1_warp_group( mWorkItems, sSched, lane, - tmem_hold, + tmem_base_slot, warp_idx, - mS_out, - mS_init, + mState_out, + mState_init, mO, sO_raw, sBeta_raw, sV_raw, - sH_raw, + sCheckpoint_raw, checkpoint_every_n_tokens, scale, bars, ) -> None: - """CG1 warp role (warps 8-11): persistent tile-scheduler loop + value-side - TMEM staging (state input, rhs, update), output drain to SMEM, and - checkpoint/final-state stores (split-K: only owned entries).""" + """CG1 warp role (warps 8-11): persistent scheduler loop for the + value-side TMEM staging, output drain, and state stores.""" nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) sO_ptr = sO_raw.data_ptr() - sH_ptr = sH_raw.data_ptr() if cutlass.const_expr(cfg.enable_checkpoints) else sO_raw.data_ptr() - h_done_index = PipelineState.start(phase=1) # sH starts free - tmem_base = tmem_hold.load() + sCheckpoint_ptr = sCheckpoint_raw.data_ptr() if cutlass.const_expr(cfg.enable_checkpoints) else sO_raw.data_ptr() + checkpoint_done_index = PipelineState.start(phase=1) + tmem_base = tmem_base_slot.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 - tmem_sp = warp_idx % (cfg.d_v // cfg.threads_per_warp) - # ldmatrix.x4/stmatrix.x4 COL lane decode shared by the v loads and o stores - ov_tok = (lane // 16) * 8 + (lane & 7) - ov_col = ((lane // 8) & 1) * 8 - row_id = tmem_row + tmem_sp * cfg.threads_per_warp - value_dim = tmem_sp * cfg.threads_per_warp + lane + tmem_subpartition = warp_idx % (cfg.d_v // cfg.threads_per_warp) + ov_token_coord = (lane // 16) * 8 + (lane & 7) + ov_col_coord = ((lane // 8) & 1) * 8 + row_id = tmem_row + tmem_subpartition * cfg.threads_per_warp + value_dim = tmem_subpartition * cfg.threads_per_warp + lane + value_dim_base = tmem_subpartition * cfg.threads_per_warp + row_addr = row_id << 16 + row16_addr = (row_id + 16) << 16 + st_row_addr = tmem_row << 16 + st_row16_addr = (tmem_row + 16) << 16 + state_col_id = tmem_col + cfg.tmem_state_acc_offset + packed_col_id = tmem_col + cfg.tmem_state_inp_offset + state_k_col_id = tmem_col + cfg.tmem_state_k_acc_offset + rhs_inp_col_id = tmem_col + cfg.tmem_rhs_inp_offset + update_acc_addr = row_addr + tmem_col + cfg.tmem_update_acc_offset + update_inp_addr = st_row_addr + tmem_col + cfg.tmem_update_inp_offset + q_state_col_base = tmem_col + cfg.tmem_q_state_acc_offset + ov_swz_off0 = ( + (value_dim_base + ov_col_coord) // 64 * (cfg.b_t * 64) + + ov_token_coord * 64 + + swizzle_xor_128b(ov_token_coord, (value_dim_base + ov_col_coord) % 64, elem_bytes=2) + ) + ov_swz_off = ( + (value_dim_base + 16 + ov_col_coord) // 64 * (cfg.b_t * 64) + + ov_token_coord * 64 + + swizzle_xor_128b(ov_token_coord, (value_dim_base + 16 + ov_col_coord) % 64, elem_bytes=2) + ) + checkpoint_swz_off0 = (value_dim_base + ov_col_coord) // 64 * (cfg.d_k * 64) + checkpoint_swz_col0 = (value_dim_base + ov_col_coord) % 64 + checkpoint_swz_off = (value_dim_base + 16 + ov_col_coord) // 64 * (cfg.d_k * 64) + checkpoint_swz_col = (value_dim_base + 16 + ov_col_coord) % 64 state_k_acc_index = PipelineState.start(phase=0) update_acc_index = PipelineState.start(phase=0) o_acc_index = PipelineState.start(phase=0) - kr_index = PipelineState.start(phase=0) # CG1's per-chunk mb_k_restore_done wait slot - raw_index = PipelineState.start(phase=0) # raw-ring slot for the sV/sBeta reads + inputs_done arrives - gbase = cutlass.Int32(0) + state_upd_index = PipelineState.start(phase=0) + raw_index = PipelineState.start(phase=0) + cum_chunk_base = cutlass.Int32(0) sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) while tile_idx < total_tiles: @@ -1480,42 +1529,68 @@ def _compute1_warp_group( cfg, tile_idx, cu_seqlens, mWorkItems ) head_o = head_idx - sk_nt = wend - cstart - - if sk_nt > 0: - # ---- first chunk: seed state TMEM from mS_init (else zeros); - # split-K warmup items (cstart > 0) rebuild their state from zero - seed_from_s0 = cstart == 0 - for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): - state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) - for col in cutlass.range_constexpr(32): - key_dim = key_block_start + col - state_value = cutlass.Float32(0.0) - if cutlass.const_expr(mS_init is not None): - state_value = mS_init[batch_idx, head_o, key_dim, value_dim].to(cutlass.Float32) - if cutlass.const_expr(cfg.split_k): - state_value = state_value if seed_from_s0 else cutlass.Float32(0.0) - state_block[col] = state_value + num_chunks_tile = wend - cstart - nvvm.tcgen05_st( - "32x32b", - nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_offset + key_block_start), cutlass.Float32), - state_block[0:32], - ) + if num_chunks_tile > 0: + # ---- first chunk: seed state TMEM from mState_init (else zeros) ---------- + seed_from_initial_state = cstart == 0 + if cutlass.const_expr(mState_init is not None and cfg.split_k): + if seed_from_initial_state: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + state_block[col] = mState_init[batch_idx, head_o, key_dim, value_dim].to(cutlass.Float32) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + else: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + state_block[col] = cutlass.Float32(0.0) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + elif cutlass.const_expr(mState_init is not None): + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + state_block[col] = mState_init[batch_idx, head_o, key_dim, value_dim].to(cutlass.Float32) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + else: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + state_block[col] = cutlass.Float32(0.0) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) nvvm.tcgen05_wait("store") sV_ptr = sV_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) sBeta_ptr = sBeta_raw.data_ptr() + raw_index.idx * cfg.b_t - row_addr = (tmem_row + tmem_sp * cfg.threads_per_warp) << 16 - state_col_id = tmem_col + cfg.tmem_state_offset - # ---- state -> packed b16 A operand (TMEM roundtrip) ---------------- + # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- state_blocks = [] for sub in cutlass.range_constexpr(cfg.d_k // 16): state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) - nvvm.tcgen05_wait("load") - packed_col_id = tmem_col + cfg.tmem_state_inp_offset for sub in cutlass.range_constexpr(cfg.d_k // 16): packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) for packed_col in cutlass.range_constexpr(8): @@ -1529,50 +1604,38 @@ def _compute1_warp_group( nvvm.tcgen05_wait("store") bars.mb_state_inp_ready.arrive() + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_read_done.arrive() - # ---- rhs staging: rhs input = beta * (v - state*k) ----------------- + # ---- rhs staging: rhs input = Beta * (V - state_k) ------------------- + bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) + bars.mb_beta_ready[raw_index.idx].wait(raw_index.phase) bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) - projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset - input_col_id = tmem_col + cfg.tmem_rhs_inp_offset - value_dim_base = tmem_sp * cfg.threads_per_warp - - # ---- read back state*k acc + raw v fragments ----------------------- - row_id0 = tmem_row + value_dim_base - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) - - row_id1 = row_id0 + 16 - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_k_col_id, cutlass.Float32), num=2) + state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_k_col_id, cutlass.Float32), num=2) raw_v_regs0 = nvvm.ldmatrix( - sV_ptr - + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + sV_ptr + ov_swz_off0, 4, nvvm.MMALayout.COL, ) raw_v_regs1 = nvvm.ldmatrix( - sV_ptr - + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + sV_ptr + ov_swz_off, 4, nvvm.MMALayout.COL, ) - nvvm.tcgen05_wait("load") - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_pairs = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - packed_col = (reg_idx // 2) * 4 + (lane & 3) - source_pair = packed_col ^ 4 - token0 = source_pair * 2 - token1 = token0 + 1 + token0 = (((reg_idx // 2) * 4 + (lane & 3)) ^ 4) * 2 beta0 = (sBeta_ptr + token0).load().to(cutlass.Float32) - beta1 = (sBeta_ptr + token1).load().to(cutlass.Float32) + beta1 = (sBeta_ptr + token0 + 1).load().to(cutlass.Float32) + beta_pairs[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) + packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] - beta_pair = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) diff_pair = sub_f16x2( raw_v_regs0[raw_matrix], @@ -1580,23 +1643,16 @@ def _compute1_warp_group( cfg.io_dtype, ) packed_rhs0[reg_idx] = mul_f16x2( - beta_pair, + beta_pairs[reg_idx], diff_pair, cfg.io_dtype, ) packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - packed_col = (reg_idx // 2) * 4 + (lane & 3) - source_pair = packed_col ^ 4 - token0 = source_pair * 2 - token1 = token0 + 1 - beta0 = (sBeta_ptr + token0).load().to(cutlass.Float32) - beta1 = (sBeta_ptr + token1).load().to(cutlass.Float32) raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] - beta_pair = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) diff_pair = sub_f16x2( raw_v_regs1[raw_matrix], @@ -1604,28 +1660,26 @@ def _compute1_warp_group( cfg.io_dtype, ) packed_rhs1[reg_idx] = mul_f16x2( - beta_pair, + beta_pairs[reg_idx], diff_pair, cfg.io_dtype, ) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), packed_rhs0[0:4]) - - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), packed_rhs1[0:4]) - + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs1[0:4]) nvvm.tcgen05_wait("store") state_k_acc_index = advance(state_k_acc_index, 1) - bars.mb_inputs_done[raw_index.idx].arrive() + bars.mb_v_done[raw_index.idx].arrive() + bars.mb_beta_done[raw_index.idx].arrive() bars.mb_rhs_ready.arrive() - # ---- update readback -> packed b16 A operand ----------------------- + # ---- update repack: acc TMEM -> packed b16 TMEM --------------------- bars.mb_update_acc_ready.wait(update_acc_index.phase) update = nvvm.tcgen05_ld( "32x32b", - nvvm.make_tmem_ptr((tmem_row + tmem_sp * cfg.threads_per_warp << 16) + (tmem_col + cfg.tmem_update_acc_offset), cutlass.Float32), + nvvm.make_tmem_ptr(update_acc_addr, cutlass.Float32), num=cfg.b_t, ) - nvvm.tcgen05_wait("load") packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): @@ -1636,53 +1690,48 @@ def _compute1_warp_group( nvvm.tcgen05_st( "32x32b", - nvvm.make_tmem_ptr((tmem_row << 16) + (tmem_col + cfg.tmem_update_inp_offset), cutlass.Int8), + nvvm.make_tmem_ptr(update_inp_addr, cutlass.Int8), packed_update[0 : (cfg.b_t // 2)], ) nvvm.tcgen05_wait("store") update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_updated.wait(state_upd_index.phase) + state_upd_index = advance(state_upd_index, 1) - bars.mb_k_restore_done[kr_index.idx].wait(kr_index.phase) - kr_index = advance(kr_index, cfg.smem_decay_stages) raw_index = advance(raw_index, cfg.smem_raw_stages) - # the first chunk is peeled above so this steady-state loop always - # drains the prior chunk's output - for li in cutlass.range(1, sk_nt, 1, unroll=1): - chunk_idx = cstart + li - gc = gbase + li + if cutlass.const_expr(cfg.enable_checkpoints): + cg1_checkpoint_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) + cg1_checkpoint_mod = (cstart + cutlass.Int32(1)) % cg1_checkpoint_chunks + for local_chunk_idx in cutlass.range(1, num_chunks_tile, 1, unroll=1): + chunk_idx = cstart + local_chunk_idx + cum_chunk = cum_chunk_base + local_chunk_idx sV_ptr = sV_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) sBeta_ptr = sBeta_raw.data_ptr() + raw_index.idx * cfg.b_t prev_output_chunk = chunk_idx - cutlass.Int32(1) - prev_og = gc - cutlass.Int32(1) - prev_o_stage = prev_og % cfg.smem_o_stages - prev_q_state_acc_stage = prev_og % cfg.tmem_q_state_acc_stages + prev_cum_chunk = cum_chunk - cutlass.Int32(1) + prev_o_stage = prev_cum_chunk % cfg.smem_o_stages + prev_q_state_acc_stage = prev_cum_chunk % cfg.tmem_q_state_acc_stages prev_o_stage_base = prev_o_stage * (cfg.b_t * cfg.d_v) - # H entry gate: the state read by this restage entered chunk_idx, - # i.e. the state after chunk_idx * b_t tokens (strictly before the - # sequence end -- the end state is only final_state); split-K - # warmup reconstructions below wstart belong to the previous item - do_h = False + do_checkpoint = False if cutlass.const_expr(cfg.enable_checkpoints): - do_h = (chunk_idx * cutlass.Int32(cfg.b_t)) % checkpoint_every_n_tokens == 0 + do_checkpoint = cg1_checkpoint_mod == 0 + cg1_checkpoint_mod = cg1_checkpoint_mod + cutlass.Int32(1) + cg1_checkpoint_mod = cutlass.Int32(0) if cg1_checkpoint_mod == cg1_checkpoint_chunks else cg1_checkpoint_mod if cutlass.const_expr(cfg.split_k): - do_h = do_h and chunk_idx >= wstart - if do_h: - # sH is free once the epilogue's previous TMA store retired - bars.mb_h_tmastg_done.wait(h_done_index.phase) - h_done_index = advance(h_done_index, 1) - row_addr = (tmem_row + tmem_sp * cfg.threads_per_warp) << 16 - state_col_id = tmem_col + cfg.tmem_state_offset - # ---- state -> packed b16 A operand (TMEM roundtrip) ---------------- + do_checkpoint = do_checkpoint and chunk_idx >= wstart + + # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- + if cutlass.const_expr(not cfg.enable_checkpoints): + bars.mb_state_updated.wait(state_upd_index.phase) + state_upd_index = advance(state_upd_index, 1) state_blocks = [] for sub in cutlass.range_constexpr(cfg.d_k // 16): state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) - bars.mb_o_tmastg_done[prev_o_stage].wait(((prev_og // cfg.smem_o_stages) + 1) % 2) - nvvm.tcgen05_wait("load") - packed_col_id = tmem_col + cfg.tmem_state_inp_offset for sub in cutlass.range_constexpr(cfg.d_k // 16): packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) for packed_col in cutlass.range_constexpr(8): @@ -1693,35 +1742,62 @@ def _compute1_warp_group( nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + sub * 8, cutlass.Int8), packed_state[0:8], ) - if cutlass.const_expr(cfg.enable_checkpoints): - # stage this sub's state to sH TRANSPOSED (KV: k rows, v - # contiguous in 64-v slabs, swizzled — the GDN H layout); - # each thread scatters its 16 k values down one v column - if do_h: - h_col = value_dim % cutlass.Int32(64) - h_seg = (value_dim // cutlass.Int32(64)) * (cfg.d_k * cutlass.Int32(64)) - k_base = sub * 16 - for j in cutlass.range_constexpr(16): - hv = state_blocks[sub][j].to(cfg.io_dtype) - (sH_ptr + h_seg + cutlass.Int32((k_base + j) * 64) + swizzle_xor_128b(cutlass.Int32(k_base + j), h_col, elem_bytes=2)).store(hv) - nvvm.tcgen05_wait("store") bars.mb_state_inp_ready.arrive() + + # ---- checkpoint store ----------------------------------------------- if cutlass.const_expr(cfg.enable_checkpoints): - if do_h: + if do_checkpoint: + checkpoint_stage = checkpoint_done_index.idx + bars.mb_checkpoint_tmastg_done[checkpoint_stage].wait(checkpoint_done_index.phase) + checkpoint_done_index = advance(checkpoint_done_index, cfg.smem_checkpoint_stages) + checkpoint_stage_base = checkpoint_stage * (cfg.d_k * cfg.d_v) + for slab in cutlass.range_constexpr(cfg.d_k // 16): + checkpoint_ld0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + slab * 16, cutlass.Float32), num=2) + checkpoint_ld1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + slab * 16, cutlass.Float32), num=2) + checkpoint_st0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_st1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + checkpoint_st0[reg_idx] = fp32_to_fp16(checkpoint_ld0[2 * reg_idx], checkpoint_ld0[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_st1[reg_idx] = fp32_to_fp16(checkpoint_ld1[2 * reg_idx], checkpoint_ld1[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_row = slab * 16 + ov_token_coord + nvvm.stmatrix( + sCheckpoint_ptr + + checkpoint_stage_base + + checkpoint_swz_off0 + + checkpoint_row * 64 + + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col0, elem_bytes=2), + checkpoint_st0.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.stmatrix( + sCheckpoint_ptr + + checkpoint_stage_base + + checkpoint_swz_off + + checkpoint_row * 64 + + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col, elem_bytes=2), + checkpoint_st1.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.tcgen05_wait("load") + bars.mb_state_read_done.arrive() nvvm.fence_proxy("async.shared", space="cta") - bars.mb_h_tmastg_ready.arrive() + bars.mb_checkpoint_tmastg_ready[checkpoint_stage].arrive() + else: + bars.mb_state_read_done.arrive() + bars.mb_o_acc_ready.wait(o_acc_index.phase) o_acc_index = advance(o_acc_index, 1) - projection_col_id = tmem_col + cfg.tmem_q_state_acc_offset + prev_q_state_acc_stage * cfg.b_t - value_dim_base = tmem_sp * cfg.threads_per_warp + projection_col_id = q_state_col_base + prev_q_state_acc_stage * cfg.b_t + + loaded0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + projection_col_id, cutlass.Float32), num=2) + loaded1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + projection_col_id, cutlass.Float32), num=2) - row_id0 = tmem_row + value_dim_base - row_id1 = row_id0 + 16 - loaded0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) - loaded1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + bars.mb_o_tmastg_done[prev_o_stage].wait(((prev_cum_chunk // cfg.smem_o_stages) + 1) % 2) - # ---- output drain: q_state_acc -> scaled b16 -> SMEM stmatrix ------- + # ---- output drain: O acc TMEM -> scaled b16 SMEM -------------------- stsm_regs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) stsm_regs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): @@ -1731,74 +1807,51 @@ def _compute1_warp_group( stsm_regs1[reg_idx] = fp32_to_fp16(scaled1_0, scaled1_1, dtype=mO.element_type) nvvm.stmatrix( - sO_ptr - + prev_o_stage_base - + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + sO_ptr + prev_o_stage_base + ov_swz_off0, stsm_regs0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) nvvm.stmatrix( - sO_ptr - + prev_o_stage_base - + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + sO_ptr + prev_o_stage_base + ov_swz_off, stsm_regs1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) - # release only after the stmatrix pair: the STSM->F2FP->FMUL2->LDTM - # register chain pins the TMEM reads complete without a wait("load") bars.mb_o_acc_done[prev_q_state_acc_stage].arrive() nvvm.fence_proxy("async.shared", space="cta") bars.mb_o_tmastg_ready[prev_o_stage].arrive() bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) - # ---- rhs staging: rhs input = beta * (v - state*k) ----------------- - projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset - input_col_id = tmem_col + cfg.tmem_rhs_inp_offset - value_dim_base = tmem_sp * cfg.threads_per_warp - # ---- read back state*k acc + raw v fragments ----------------------- - row_id0 = tmem_row + value_dim_base - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) - - row_id1 = row_id0 + 16 - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + # ---- rhs staging: rhs input = Beta * (V - state_k) ------------------- + bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) + bars.mb_beta_ready[raw_index.idx].wait(raw_index.phase) + state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_k_col_id, cutlass.Float32), num=2) + state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_k_col_id, cutlass.Float32), num=2) raw_v_regs0 = nvvm.ldmatrix( - sV_ptr - + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + sV_ptr + ov_swz_off0, 4, nvvm.MMALayout.COL, ) raw_v_regs1 = nvvm.ldmatrix( - sV_ptr - + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + sV_ptr + ov_swz_off, 4, nvvm.MMALayout.COL, ) - nvvm.tcgen05_wait("load") - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_pairs = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - packed_col = (reg_idx // 2) * 4 + (lane & 3) - source_pair = packed_col ^ 4 - token0 = source_pair * 2 - token1 = token0 + 1 + token0 = (((reg_idx // 2) * 4 + (lane & 3)) ^ 4) * 2 beta0 = (sBeta_ptr + token0).load().to(cutlass.Float32) - beta1 = (sBeta_ptr + token1).load().to(cutlass.Float32) + beta1 = (sBeta_ptr + token0 + 1).load().to(cutlass.Float32) + beta_pairs[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) + packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] - beta_pair = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) diff_pair = sub_f16x2( raw_v_regs0[raw_matrix], @@ -1806,23 +1859,16 @@ def _compute1_warp_group( cfg.io_dtype, ) packed_rhs0[reg_idx] = mul_f16x2( - beta_pair, + beta_pairs[reg_idx], diff_pair, cfg.io_dtype, ) packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - packed_col = (reg_idx // 2) * 4 + (lane & 3) - source_pair = packed_col ^ 4 - token0 = source_pair * 2 - token1 = token0 + 1 - beta0 = (sBeta_ptr + token0).load().to(cutlass.Float32) - beta1 = (sBeta_ptr + token1).load().to(cutlass.Float32) raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] - beta_pair = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) diff_pair = sub_f16x2( raw_v_regs1[raw_matrix], @@ -1830,28 +1876,26 @@ def _compute1_warp_group( cfg.io_dtype, ) packed_rhs1[reg_idx] = mul_f16x2( - beta_pair, + beta_pairs[reg_idx], diff_pair, cfg.io_dtype, ) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), packed_rhs0[0:4]) - - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), packed_rhs1[0:4]) - + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs1[0:4]) nvvm.tcgen05_wait("store") state_k_acc_index = advance(state_k_acc_index, 1) - bars.mb_inputs_done[raw_index.idx].arrive() + bars.mb_v_done[raw_index.idx].arrive() + bars.mb_beta_done[raw_index.idx].arrive() bars.mb_rhs_ready.arrive() - # ---- update readback -> packed b16 A operand ----------------------- + # ---- update repack: acc TMEM -> packed b16 TMEM --------------------- bars.mb_update_acc_ready.wait(update_acc_index.phase) update = nvvm.tcgen05_ld( "32x32b", - nvvm.make_tmem_ptr((tmem_row + tmem_sp * cfg.threads_per_warp << 16) + (tmem_col + cfg.tmem_update_acc_offset), cutlass.Float32), + nvvm.make_tmem_ptr(update_acc_addr, cutlass.Float32), num=cfg.b_t, ) - nvvm.tcgen05_wait("load") packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): @@ -1862,35 +1906,36 @@ def _compute1_warp_group( nvvm.tcgen05_st( "32x32b", - nvvm.make_tmem_ptr((tmem_row << 16) + (tmem_col + cfg.tmem_update_inp_offset), cutlass.Int8), + nvvm.make_tmem_ptr(update_inp_addr, cutlass.Int8), packed_update[0 : (cfg.b_t // 2)], ) nvvm.tcgen05_wait("store") update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() - bars.mb_k_restore_done[kr_index.idx].wait(kr_index.phase) - kr_index = advance(kr_index, cfg.smem_decay_stages) + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_updated.wait(state_upd_index.phase) + state_upd_index = advance(state_upd_index, 1) raw_index = advance(raw_index, cfg.smem_raw_stages) - if sk_nt > 0: - og = gbase + sk_nt - cutlass.Int32(1) - final_o_stage = og % cfg.smem_o_stages - final_q_state_acc_stage = og % cfg.tmem_q_state_acc_stages + if num_chunks_tile > 0: + if cutlass.const_expr(not cfg.enable_checkpoints): + bars.mb_state_updated.wait(state_upd_index.phase) + state_upd_index = advance(state_upd_index, 1) + last_cum_chunk = cum_chunk_base + num_chunks_tile - cutlass.Int32(1) + final_o_stage = last_cum_chunk % cfg.smem_o_stages + final_q_state_acc_stage = last_cum_chunk % cfg.tmem_q_state_acc_stages final_o_stage_base = final_o_stage * (cfg.b_t * cfg.d_v) - bars.mb_o_tmastg_done[final_o_stage].wait(((og // cfg.smem_o_stages) + 1) % 2) + bars.mb_o_tmastg_done[final_o_stage].wait(((last_cum_chunk // cfg.smem_o_stages) + 1) % 2) bars.mb_o_acc_ready.wait(o_acc_index.phase) o_acc_index = advance(o_acc_index, 1) - projection_col_id = tmem_col + cfg.tmem_q_state_acc_offset + final_q_state_acc_stage * cfg.b_t - value_dim_base = tmem_sp * cfg.threads_per_warp + projection_col_id = q_state_col_base + final_q_state_acc_stage * cfg.b_t - row_id0 = tmem_row + value_dim_base - row_id1 = row_id0 + 16 - loaded0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) - loaded1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + loaded0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + projection_col_id, cutlass.Float32), num=2) + loaded1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + projection_col_id, cutlass.Float32), num=2) - # ---- output drain: q_state_acc -> scaled b16 -> SMEM stmatrix ------- + # ---- output drain: O acc TMEM -> scaled b16 SMEM -------------------- stsm_regs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) stsm_regs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): @@ -1900,66 +1945,52 @@ def _compute1_warp_group( stsm_regs1[reg_idx] = fp32_to_fp16(scaled1_0, scaled1_1, dtype=mO.element_type) nvvm.stmatrix( - sO_ptr - + final_o_stage_base - + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), + sO_ptr + final_o_stage_base + ov_swz_off0, stsm_regs0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) nvvm.stmatrix( - sO_ptr - + final_o_stage_base - + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) - + ov_tok * 64 - + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), + sO_ptr + final_o_stage_base + ov_swz_off, stsm_regs1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) - # release only after the stmatrix pair: the STSM->F2FP->FMUL2->LDTM - # register chain pins the TMEM reads complete without a wait("load") bars.mb_o_acc_done[final_q_state_acc_stage].arrive() nvvm.fence_proxy("async.shared", space="cta") bars.mb_o_tmastg_ready[final_o_stage].arrive() - # split-K: only the item owning the sequence's last chunk holds the - # true end-of-sequence state (legacy tiles always do: wend == nc) owns_final = wend == num_chunks_b - # ---- final-state drain: final_state acc -> GMEM -------------------- - if cutlass.const_expr(mS_out is not None): + + # ---- final-state drain: state acc TMEM -> GMEM --------------------------- + if cutlass.const_expr(mState_out is not None): if seqlen_b > 0: if owns_final: for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): loaded = nvvm.tcgen05_ld( "32x32b", - nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_offset + key_block_start), cutlass.Float32), + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), num=32, ) - nvvm.tcgen05_wait("load") for col in cutlass.range_constexpr(32): key_dim = key_block_start + col - mS_out[batch_idx, head_o, key_dim, value_dim] = loaded[col].to(mS_out.element_type) + mState_out[batch_idx, head_o, key_dim, value_dim] = loaded[col].to(mState_out.element_type) else: - # zero-length sequence: the state passes through untouched - # (S0 when seeded, zeros otherwise); pure GMEM, no TMEM for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): for col in cutlass.range_constexpr(32): key_dim = key_block_start + col - if cutlass.const_expr(mS_init is not None): - mS_out[batch_idx, head_o, key_dim, value_dim] = mS_init[batch_idx, head_o, key_dim, value_dim] + if cutlass.const_expr(mState_init is not None): + mState_out[batch_idx, head_o, key_dim, value_dim] = mState_init[batch_idx, head_o, key_dim, value_dim] else: - mS_out[batch_idx, head_o, key_dim, value_dim] = cutlass.Float32(0.0).to(mS_out.element_type) + mState_out[batch_idx, head_o, key_dim, value_dim] = cutlass.Float32(0.0).to(mState_out.element_type) bars.mb_final_state_stored.arrive() - gbase += sk_nt - tile_idx, sched_state = _sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + cum_chunk_base += num_chunks_tile + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) @cute.jit -def _host( +def host( cfg: cutlass.Constexpr, q: cute.Tensor, k: cute.Tensor, @@ -1981,15 +2012,13 @@ def _host( stream, ) -> None: num_sequences = cu_seqlens.shape[0] - 1 - ho = raw_gate.shape[1] - # ---- persistent launch: the grid only needs to cover the tiles ------ - total_tiles = num_sequences * ho - # CUDA-graph-stable launch: fixed SM-count grid + n_heads_out = raw_gate.shape[1] + total_tiles = num_sequences * n_heads_out grid_shape = (cfg.max_active_clusters, 1, 1) - _kernel( + kernel( cfg, tensormap_workspace, - cutlass.Int32(num_sequences * ho), + cutlass.Int32(num_sequences), q, k, v, @@ -2016,7 +2045,7 @@ def _host( @cute.kernel -def _kernel( +def kernel( cfg: cutlass.Constexpr, tensormap_workspace: cute.Tensor, n_desc: cutlass.Int32, @@ -2028,9 +2057,9 @@ def _kernel( mDt_bias: cute.Tensor | None, mBeta: cute.Tensor, cu_seqlens: cute.Tensor, - mS_init: cute.Tensor | None, + mState_init: cute.Tensor | None, mO: cute.Tensor, - mS_out: cute.Tensor | None, + mState_out: cute.Tensor | None, mWorkItems: cute.Tensor | None, mCount: cute.Tensor | None, mSched: cute.Tensor | None, @@ -2038,13 +2067,7 @@ def _kernel( scale: cutlass.Float32, checkpoint_every_n_tokens: cutlass.Int32, ) -> None: - """BT=16 KDA forward kernel (persistent). - - Grid: `(min(tiles, SM count), 1, 1)`. Every warp role runs a - tile-scheduler loop over the tiles — one packed sequence/head each, or - one split-K work item — iterating its chunks in order. Source heads - follow repeat_interleave: head_x = head_idx // X_RATIO. - """ + """BT=16 KDA forward kernel (persistent, tile-scheduled).""" tidx, _, _ = cute.arch.thread_idx() bidx = cute.arch.block_idx()[0] @@ -2062,16 +2085,15 @@ def _kernel( assert mBeta.element_type == beta_expected assert cu_seqlens.element_type in (cutlass.Int32, cutlass.Int64) if cutlass.const_expr(cfg.use_initial_state): - assert mS_init is not None and mS_init.element_type in (cutlass.BFloat16, cutlass.Float32) + assert mState_init is not None and mState_init.element_type in (cutlass.BFloat16, cutlass.Float32) else: - assert mS_init is None, "mS_init must be None if use_initial_state is False" + assert mState_init is None, "mState_init must be None if use_initial_state is False" if cutlass.const_expr(cfg.store_final_state): - assert mS_out is not None and mS_out.element_type in (cutlass.BFloat16, cutlass.Float32) + assert mState_out is not None and mState_out.element_type in (cutlass.BFloat16, cutlass.Float32) else: - assert mS_out is None, "mS_out must be None if store_final_state is False" - if cutlass.const_expr(mS_init is not None and mS_out is not None): - assert mS_init.element_type == mS_out.element_type - # per-(batch, head) TMA-descriptor arrays: [q, k, v, gate, o] + assert mState_out is None, "mState_out must be None if store_final_state is False" + if cutlass.const_expr(mState_init is not None and mState_out is not None): + assert mState_init.element_type == mState_out.element_type desc_base_words = tensormap_workspace.iterator.raw_ptr() arr_words = n_desc * cutlass.Int32(TENSOR_MAP_QWORDS) desc_q_base = desc_base_words @@ -2079,19 +2101,17 @@ def _kernel( desc_v_base = desc_base_words + cutlass.Int32(2) * arr_words desc_gate_base = desc_base_words + cutlass.Int32(3) * arr_words desc_o_base = desc_base_words + cutlass.Int32(4) * arr_words - desc_h_base = desc_base_words + cutlass.Int32(5) * arr_words + desc_checkpoint_base = desc_base_words + cutlass.Int32(5) * arr_words # Buffers are declaration-ordered and intentionally non-aliased. SMEM = cutlass.AddressSpace.smem bars = make_kda_bars(cfg) - # The hand-written K-box-major SW128 mapping is normalized to phase 0, - # so both tcgen05 and ldmatrix can share 1KB-aligned operand buffers. - tmem_hold = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=4) + tmem_base_slot = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=4) sSched = cutlass.Array(cutlass.Int32, cfg.sched_stages, space=SMEM, alignment=16) sK_decay_raw = cutlass.Array(cfg.io_dtype, cfg.k_decay_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) sQ_decay_raw = cutlass.Array(cfg.io_dtype, cfg.q_decay_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) sK_restore_raw = cutlass.Array(cfg.io_dtype, cfg.k_restore_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) - sPairwise_raw = cutlass.Array(cfg.io_dtype, cfg.pairwise_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sQk_raw = cutlass.Array(cfg.io_dtype, cfg.qk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) sQ_raw = cutlass.Array(mQ.element_type, cfg.q_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) sK_raw = cutlass.Array(mK.element_type, cfg.k_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) sV_raw = cutlass.Array(mV.element_type, cfg.v_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) @@ -2102,17 +2122,14 @@ def _kernel( mO.element_type, cfg.o_cosize, space=SMEM, - # The scalar CG1 store computes W128 offsets relative to this buffer. - # Align to the full s128b period so absolute SMEM address bits do not - # add a hidden phase to the TMA store-side swizzle. alignment=cfg.buffer_align_bytes, ) sBeta_raw = cutlass.Array(cutlass.Float32, cfg.beta_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) - sH_raw = ( - cutlass.Array(cfg.io_dtype, cfg.d_k * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) if cutlass.const_expr(cfg.enable_checkpoints) else sO_raw + sCheckpoint_raw = ( + cutlass.Array(cfg.io_dtype, cfg.smem_checkpoint_stages * cfg.d_k * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) + if cutlass.const_expr(cfg.enable_checkpoints) + else sO_raw ) - # K-box-major SW128 staging: 16B leading offset, 1KB stride; the decay - # stores apply a row-group key xor so tcgen05 B reads logical [DK, BT]. sK_decay = SmemTile( base=sK_decay_raw, elems_per_stage=(cfg.d_k * cfg.b_t), @@ -2145,27 +2162,28 @@ def _kernel( stride_byte_offset=(8 * 16 * 2), layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, ) - sPairwise = SmemTile( - base=sPairwise_raw, + sQk = SmemTile( + base=sQk_raw, elems_per_stage=(2 * cfg.b_t * cfg.b_t), - stages=cfg.smem_pairwise_stages, + stages=cfg.smem_qk_stages, leading_byte_offset=16, stride_byte_offset=(8 * cfg.b_t * 2), layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, ) - tma_tx_bytes = cutlass.const_expr( - cfg.d_k * cfg.b_t * mQ.element_type.width // 8 - + cfg.d_k * cfg.b_t * mK.element_type.width // 8 - + cfg.d_v * cfg.b_t * mV.element_type.width // 8 - + cfg.d_k * cfg.b_t * mGate.element_type.width // 8 - ) if warp_idx == cfg.tma_warp_id: if nvvm.elect_sync(): - bars.mb_tma_done.init() for stage in cutlass.range_constexpr(cfg.smem_raw_stages): - bars.mb_inputs_ready[stage].init() - bars.mb_inputs_done[stage].init() + bars.mb_q_ready[stage].init() + bars.mb_k_ready[stage].init() + bars.mb_v_ready[stage].init() + bars.mb_gate_ready[stage].init() + bars.mb_beta_ready[stage].init() + bars.mb_beta_done[stage].init() + bars.mb_q_done[stage].init() + bars.mb_k_done[stage].init() + bars.mb_v_done[stage].init() + bars.mb_gate_done[stage].init() elif warp_idx == cfg.tcgen05_mma_warp_id: if nvvm.elect_sync(): bars.mb_o_acc_ready.init() @@ -2173,26 +2191,28 @@ def _kernel( bars.mb_o_acc_done[stage].init() bars.mb_state_k_acc_ready.init() bars.mb_update_acc_ready.init() + bars.mb_state_updated.init() bars.mb_state_inp_ready.init() for stage in cutlass.range_constexpr(cfg.smem_state_scale_diag_stages): bars.mb_state_scale_diag_done[stage].init() for stage in cutlass.range_constexpr(cfg.smem_decay_stages): - bars.mb_kk_qk_super_mma_done[stage].init() - bars.mb_kk_qk_mma_done[stage].init() + bars.mb_decay_done[stage].init() + bars.mb_decay_super_done[stage].init() bars.mb_k_restore_done[stage].init() bars.mb_rhs_ready.init() bars.mb_update_ready.init() bars.mb_final_state_stored.init() elif warp_idx == cfg.super_mma_warp_id: if nvvm.elect_sync(): - for stage in cutlass.range_constexpr(cfg.smem_pairwise_stages): + for stage in cutlass.range_constexpr(cfg.smem_qk_stages): bars.mb_a_ready[stage].init() bars.mb_qk_acc_ready[stage].init() - bars.mb_a_done[stage].init() + bars.mb_a_inv_done[stage].init() + bars.mb_qk_done[stage].init() for stage in cutlass.range_constexpr(cfg.qk_scale_ready_stages): bars.mb_qk_scale_ready[stage].init() for stage in cutlass.range_constexpr(cfg.smem_decay_stages): - bars.mb_k_decay_cg0_ready[stage].init() + bars.mb_k_decay_inv_cg0_ready[stage].init() elif warp_idx == cfg.epilogue_warp_id: if nvvm.elect_sync(): for stage in cutlass.range_constexpr(cfg.smem_o_stages): @@ -2202,8 +2222,10 @@ def _kernel( bars.mb_sched_ready[stage].init() bars.mb_sched_done[stage].init() if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_h_tmastg_ready.init() - bars.mb_h_tmastg_done.init() + for stage in cutlass.range_constexpr(cfg.smem_checkpoint_stages): + bars.mb_checkpoint_tmastg_ready[stage].init() + bars.mb_checkpoint_tmastg_done[stage].init() + bars.mb_state_read_done.init() diag_zero = cfg.io_dtype(0.0) for diag_idx in cutlass.range(tidx, cfg.state_scale_diag_cosize, cfg.threads_per_cta, unroll=1): sState_scale_diag_raw[diag_idx] = diag_zero @@ -2211,11 +2233,11 @@ def _kernel( nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) if (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) or warp_idx == cfg.tcgen05_mma_warp_id: if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_alloc(tmem_hold, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.nbar_tmem_lifecycle_id, thread_count=cfg.tmem_user_threads) + nvvm.tcgen05_alloc(tmem_base_slot, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) if warp_idx == cfg.tcgen05_mma_warp_id: nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.nbar_tmem_lifecycle_id, thread_count=cfg.tmem_user_threads) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) # Actual SMEM/TMEM buffers for the BT=16 schedule: # q/k/v : 16 x 128 each @@ -2234,7 +2256,7 @@ def _kernel( # is `[DV, DK] @ [DK, BT] -> [DV, BT]` with M=128. if warp_idx == cfg.tma_warp_id: - _tmaldg_warp( + tmaldg_warp( cfg, total_tiles, bidx, @@ -2243,14 +2265,11 @@ def _kernel( mWorkItems, mSched, sSched, - tma_tx_bytes, lane, - mBeta, sQ_raw, sK_raw, sV_raw, sGate_raw, - sBeta_raw, desc_q_base, desc_k_base, desc_v_base, @@ -2258,7 +2277,7 @@ def _kernel( bars, ) elif warp_idx == cfg.super_mma_warp_id: - _super_mma_warp( + super_mma_warp( cfg, total_tiles, bidx, @@ -2268,13 +2287,13 @@ def _kernel( sSched, lane, sK_inv_raw, - sPairwise_raw, + sQk_raw, sBeta_raw, sK_decay_raw, bars, ) elif warp_idx == cfg.tcgen05_mma_warp_id: - _tcgen05_mma_warp( + tcgen05_mma_warp( cfg, total_tiles, bidx, @@ -2282,8 +2301,8 @@ def _kernel( cu_seqlens, mWorkItems, sSched, - tmem_hold, - sPairwise, + tmem_base_slot, + sQk, sK_decay, sK_restore, sQ_decay, @@ -2291,7 +2310,7 @@ def _kernel( bars, ) elif warp_idx == cfg.epilogue_warp_id: - _epilogue_warp( + epilogue_warp( cfg, total_tiles, bidx, @@ -2303,16 +2322,16 @@ def _kernel( mO, sK_inv_raw, sO_raw, - sPairwise_raw, + sQk_raw, sQ_decay_raw, - sH_raw, + sCheckpoint_raw, desc_o_base, - desc_h_base, + desc_checkpoint_base, checkpoint_every_n_tokens, bars, ) elif warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]: - _compute0_warp_group( + compute0_warp_group( cfg, total_tiles, bidx, @@ -2327,9 +2346,10 @@ def _kernel( mDt_bias, sK_inv_raw, sGate_raw, + mBeta, + sBeta_raw, sK_raw, sQ_raw, - sV_raw, sK_decay_raw, sK_restore_raw, sQ_decay_raw, @@ -2337,7 +2357,7 @@ def _kernel( bars, ) elif warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]: - _compute1_warp_group( + compute1_warp_group( cfg, total_tiles, bidx, @@ -2346,15 +2366,15 @@ def _kernel( mWorkItems, sSched, lane, - tmem_hold, + tmem_base_slot, warp_idx, - mS_out, - mS_init, + mState_out, + mState_init, mO, sO_raw, sBeta_raw, sV_raw, - sH_raw, + sCheckpoint_raw, checkpoint_every_n_tokens, scale, bars, @@ -2366,8 +2386,7 @@ class KdaCfg: """Kernel cfg (fixed BT=16 schedule constants; derived TMEM column offsets and SMEM buffer cosizes are stamped by ``build_cfg``; per-stage sizes are inlined at the use sites). Passed ``cfg``-first (a ``cutlass.Constexpr``) - into ``_host`` / ``_kernel`` and every warp body, mirroring GDN's - ``GdnCfg``.""" + into ``host`` / ``kernel`` and every warp body.""" io_dtype: Type[cutlass.Numeric] state_dtype: Type[cutlass.Numeric] @@ -2383,8 +2402,6 @@ class KdaCfg: v_ratio: int n_heads_out: int max_active_clusters: int - # split-K: tiles come from a work-item table (see common/split_k.py); - # each item computes chunks [cstart, wend) and writes only [wstart, wend) split_k: bool = False dyn_sched: bool = False sched_stages: int = CFG.SMEM_SCHED_STAGES @@ -2404,24 +2421,25 @@ class KdaCfg: cg0_group_count: int = 2 cg0_warps_per_group: int = 4 cg0_threads_per_group: int = 0 - nbar_cg0_group0_id: int = 1 # CG0 group g syncs on nbar id 1 + g + cg0_group_sync_barrier_base_id: int = 1 # CG0 group g syncs on nbar id 1 + g tmem_user_threads: int = 0 - nbar_tmem_lifecycle_id: int = 3 + tmem_lifecycle_barrier_id: int = 3 num_regs_compute_group_0: int = CFG.NUM_REGS_COMPUTE_GROUP_0 num_regs_compute_group_1: int = CFG.NUM_REGS_COMPUTE_GROUP_1 num_regs_other: int = CFG.NUM_REGS_OTHER - # --- SMEM / TMEM ring stage counts --- + # ---- SMEM / TMEM ring stage counts ------------------------------------------- smem_raw_stages: int = CFG.SMEM_RAW_STAGES smem_o_stages: int = CFG.SMEM_O_STAGES + smem_checkpoint_stages: int = 1 smem_decay_stages: int = CFG.SMEM_DECAY_STAGES - smem_pairwise_stages: int = CFG.SMEM_PAIRWISE_STAGES + smem_qk_stages: int = CFG.SMEM_QK_STAGES smem_state_scale_diag_stages: int = CFG.SMEM_STATE_SCALE_DIAG_STAGES qk_scale_ready_stages: int = CFG.QK_SCALE_READY_STAGES tmem_q_state_acc_stages: int = CFG.TMEM_Q_STATE_ACC_STAGES - # --- TMEM column offsets (state doubles as the final_state acc) --- - tmem_state_offset: int = 0 + # ---- TMEM column offsets (state doubles as the final_state acc) -------------- + tmem_state_acc_offset: int = 0 tmem_state_inp_offset: int = 0 tmem_q_state_acc_offset: int = 0 tmem_state_k_acc_offset: int = 0 @@ -2429,7 +2447,7 @@ class KdaCfg: tmem_rhs_inp_offset: int = 0 tmem_update_inp_offset: int = 0 - # --- SMEM buffer cosizes --- + # ---- SMEM buffer cosizes ----------------------------------------------------- q_cosize: int = 0 k_cosize: int = 0 v_cosize: int = 0 @@ -2441,7 +2459,13 @@ class KdaCfg: k_restore_cosize: int = 0 state_scale_diag_cosize: int = 0 o_cosize: int = 0 - pairwise_cosize: int = 0 + + # TMA transaction bytes per stage + tma_q_bytes: int = 0 + tma_k_bytes: int = 0 + tma_v_bytes: int = 0 + tma_gate_bytes: int = 0 + qk_cosize: int = 0 def build_cfg( @@ -2486,16 +2510,15 @@ def build_cfg( dyn_sched=dyn_sched, ) if enable_checkpoints: - # the 32 KB H staging buffer must fit next to the raw ring: trim the - # q/k/v/gate TMA lookahead for H compiles - cfg.smem_raw_stages = 6 + cfg.smem_raw_stages = 5 + cfg.smem_checkpoint_stages = 2 cfg.threads_per_cta = 16 * cfg.threads_per_warp cfg.cg0_threads_per_group = cfg.cg0_warps_per_group * cfg.threads_per_warp cfg.tmem_user_threads = (1 + len(cfg.compute_group_1_warp_ids)) * cfg.threads_per_warp if cfg.smem_state_scale_diag_stages != cfg.qk_scale_ready_stages: raise ValueError("diag and qk-scale ready rings must share their rolling stage") - cfg.tmem_state_inp_offset = cfg.tmem_state_offset + cfg.d_k + cfg.tmem_state_inp_offset = cfg.tmem_state_acc_offset + cfg.d_k cfg.tmem_q_state_acc_offset = cfg.tmem_state_inp_offset + (cfg.d_k // 2) cfg.tmem_state_k_acc_offset = cfg.tmem_q_state_acc_offset + cfg.tmem_q_state_acc_stages * cfg.b_t cfg.tmem_update_acc_offset = cfg.tmem_state_k_acc_offset + cfg.b_t @@ -2514,19 +2537,86 @@ def build_cfg( cfg.k_restore_cosize = cfg.smem_decay_stages * cfg.d_k * cfg.b_t cfg.state_scale_diag_cosize = cfg.smem_state_scale_diag_stages * (cfg.d_k // 16) * 256 cfg.o_cosize = cfg.smem_o_stages * cfg.b_t * cfg.d_v - cfg.pairwise_cosize = cfg.smem_pairwise_stages * 2 * cfg.b_t * cfg.b_t + cfg.qk_cosize = cfg.smem_qk_stages * 2 * cfg.b_t * cfg.b_t + cfg.tma_q_bytes = cfg.d_k * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_k_bytes = cfg.d_k * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_v_bytes = cfg.d_v * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_gate_bytes = cfg.d_k * cfg.b_t * 4 return cfg -def get_workspace_size(B: int, HQ: int, HV: int) -> int: - """Bytes for the per-(batch, head) TMA-descriptor arrays (q, k, v, gate, - o, h) + 128 alignment slack.""" - HO = HQ if HQ >= HV else HV - return TENSOR_MAP_QWORDS * 8 * (6 * B * HO) + 128 +TENSORMAP_DESC_ARRAYS = 6 # per-batch runtime TMA descriptors: Q, K, V, Gate, O, state_checkpoints +TENSORMAP_STATIC_SLOTS = 0 + + +@cute.kernel +def build_all_descs_kernel( + base_q: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_k: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_v: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_gate: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_o: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_checkpoint: cutlass.GridConstant[cuda.tensor_map.TensorMap], + desc_ws: cute.Tensor, + cu_seqlens: cute.Tensor, + q: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + gate: cute.Tensor, + o: cute.Tensor, + state_checkpoints: cute.Tensor | None, + n_batch: cutlass.Int32, + q_token_stride: cutlass.Int32, + k_token_stride: cutlass.Int32, + v_token_stride: cutlass.Int32, + gate_token_stride: cutlass.Int32, + o_token_stride: cutlass.Int32, + checkpoint_entry_stride: cutlass.Int32, + checkpoint_every_n: cutlass.Int32, +) -> None: + """Single-launch builder for the per-BATCH descriptor arrays (one warp + per array).""" + tidx, _, _ = cute.arch.thread_idx() + widx = cutlass.Int32(tidx) // cutlass.Int32(32) + arr_words = n_batch * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_q_arr = cute.make_tensor(desc_ws.iterator, cute.make_layout((arr_words,), stride=(1,))) + desc_k_arr = cute.make_tensor(desc_ws.iterator + arr_words, cute.make_layout((arr_words,), stride=(1,))) + desc_v_arr = cute.make_tensor(desc_ws.iterator + 2 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + desc_gate_arr = cute.make_tensor(desc_ws.iterator + 3 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + desc_o_arr = cute.make_tensor(desc_ws.iterator + 4 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + desc_checkpoint_arr = cute.make_tensor(desc_ws.iterator + 5 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + + if widx == 0: + if nvvm.elect_sync(): + emit_seq_descs(base_q, desc_q_arr, cu_seqlens, q, n_batch, q_token_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 1: + if nvvm.elect_sync(): + emit_seq_descs(base_k, desc_k_arr, cu_seqlens, k, n_batch, k_token_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 2: + if nvvm.elect_sync(): + emit_seq_descs(base_v, desc_v_arr, cu_seqlens, v, n_batch, v_token_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 3: + if nvvm.elect_sync(): + emit_seq_descs(base_gate, desc_gate_arr, cu_seqlens, gate, n_batch, gate_token_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 4: + if nvvm.elect_sync(): + emit_seq_descs(base_o, desc_o_arr, cu_seqlens, o, n_batch, o_token_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if cutlass.const_expr(state_checkpoints is not None): + if widx == 5: + if nvvm.elect_sync(): + emit_checkpoint_seq_descs( + base_checkpoint, desc_checkpoint_arr, cu_seqlens, state_checkpoints, n_batch, checkpoint_entry_stride, checkpoint_every_n, 2 + ) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) @cute.jit -def _build_descs( +def build_descs( io_dtype: cutlass.Constexpr, b_t: cutlass.Constexpr[int], q: cute.Tensor, @@ -2534,139 +2624,77 @@ def _build_descs( v: cute.Tensor, gate: cute.Tensor, o: cute.Tensor, - h: cute.Tensor | None, + state_checkpoints: cute.Tensor | None, cu_seqlens: cute.Tensor, tensormap_workspace: cute.Tensor, - h_every_n: cutlass.Int32, + checkpoint_every_n: cutlass.Int32, stream: cuda_driver.CUstream, ): """Build the 6 per-(batch, head) TMA-descriptor arrays (q, k, v, gate, - o, h) into ``tensormap_workspace``. - - Launched on every execute: the descriptors fold cu_seqlens contents into - GLOBAL_ADDRESS and GLOBAL_DIM, which the host cannot read without a D2H sync. Each descriptor - folds the sequence base + head offset into GLOBAL_ADDRESS (Int64) and - caps the token GLOBAL_DIM to the sequence length, so the main kernel's - coordinates are sequence-relative and tail chunks clip in hardware. The - H descriptor is 3-D ``(dv, dk, entry)`` over the packed ``[total_h, HO, - DK, DV]`` series; its per-sequence entry offsets ((seqlen-1)//N, - prefix-summed) are derived on device and its entry extent is capped per - sequence, so H store coordinates are sequence-local.""" + o, state_checkpoints) into ``tensormap_workspace``.""" h_q = q.shape[1] h_k = k.shape[1] h_v = v.shape[1] - ho = gate.shape[1] + n_heads_out = gate.shape[1] batch_size = cu_seqlens.shape[0] - 1 d_k = q.shape[2] d_v = v.shape[2] bpe = io_dtype.width // 8 - granu = 128 // bpe + tma_granu_elems = 128 // bpe seqlen = q.shape[0] - def _head0(t, dim, heads): - # 2-D (dim, token) head-0 view: box (granu, b_t) matches the main - # kernel's SMEM staging byte-for-byte - return cute.make_tensor(t.iterator, cute.make_layout((dim, seqlen), stride=(1, heads * dim))) + def headed(t, dim, hn): + return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) swz = cuda.TensorMapSwizzle.s128b - base_q = cuda.create_tensor_map_tiled_from_view(_head0(q, d_k, h_q), box_dims=(granu, b_t), stride_order=(0, 1), swizzle=swz) - base_k = cuda.create_tensor_map_tiled_from_view(_head0(k, d_k, h_k), box_dims=(granu, b_t), stride_order=(0, 1), swizzle=swz) - base_v = cuda.create_tensor_map_tiled_from_view(_head0(v, d_v, h_v), box_dims=(granu, b_t), stride_order=(0, 1), swizzle=swz) - base_gate = cuda.create_tensor_map_tiled_from_view(_head0(gate, d_k, ho), box_dims=(32, b_t), stride_order=(0, 1), swizzle=swz) - base_o = cuda.create_tensor_map_tiled_from_view(_head0(o, d_v, ho), box_dims=(granu, b_t), stride_order=(0, 1), swizzle=swz) - - arr_words = (batch_size * ho) * TENSOR_MAP_QWORDS - ws_iter = tensormap_workspace.iterator - - def _sub(i): - return cute.make_tensor(ws_iter + i * arr_words, cute.make_layout((arr_words,), stride=(1,))) - - build_qkv_load_descs_kernel( - base_q, _sub(0), cu_seqlens, q, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(ho // h_q), cutlass.Int32(d_k), cutlass.Int32(h_q * d_k), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_k, _sub(1), cu_seqlens, k, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(ho // h_k), cutlass.Int32(d_k), cutlass.Int32(h_k * d_k), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_v, _sub(2), cu_seqlens, v, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(ho // h_v), cutlass.Int32(d_v), cutlass.Int32(h_v * d_v), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_gate, _sub(3), cu_seqlens, gate, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(1), cutlass.Int32(d_k), cutlass.Int32(ho * d_k), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - build_qkv_load_descs_kernel( - base_o, _sub(4), cu_seqlens, o, cutlass.Int32(batch_size), cutlass.Int32(ho), cutlass.Int32(1), cutlass.Int32(d_v), cutlass.Int32(ho * d_v), 1 - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - if cutlass.const_expr(h is not None): - h_view = cute.make_tensor( - h.iterator, + base_q = cuda.create_tensor_map_tiled_from_view(headed(q, d_k, h_q), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, n_heads_out), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_o = cuda.create_tensor_map_tiled_from_view(headed(o, d_v, n_heads_out), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + + base_checkpoint = base_o + if cutlass.const_expr(state_checkpoints is not None): + checkpoint_view = cute.make_tensor( + state_checkpoints.iterator, cute.make_layout( - (h.shape[3], h.shape[2], h.shape[0]), - stride=(h.stride[3], h.stride[2], h.stride[0]), + (state_checkpoints.shape[3], state_checkpoints.shape[2], state_checkpoints.shape[0], n_heads_out), + stride=(state_checkpoints.stride[3], state_checkpoints.stride[2], state_checkpoints.stride[0], state_checkpoints.stride[1]), ), ) - base_h = cuda.create_tensor_map_tiled_from_view(h_view, box_dims=(granu, d_k, 1), stride_order=(0, 1, 2), swizzle=swz) - build_h_descs_kernel( - base_h, - _sub(5), - cu_seqlens, - h, - cutlass.Int32(batch_size), - cutlass.Int32(ho), - cutlass.Int32(h.stride[1]), - cutlass.Int32(h.stride[0]), - h_every_n, - 2, - ).launch(grid=(1, 1, 1), block=(32, 1, 1), stream=stream) - - -# --------------------------------------------------------------------------- -# Torch adapter / host-side compilation -# --------------------------------------------------------------------------- - - -def _device_sm_count() -> int: - """Multiprocessor count of the current device (runtime API: auto-inits - the primary context, so this works before any other CUDA call).""" - from cuda.bindings import runtime as _rt - - err, dev = _rt.cudaGetDevice() - if int(err) != 0: - raise RuntimeError(f"cudaGetDevice failed: {err}") - err, count = _rt.cudaDeviceGetAttribute(_rt.cudaDeviceAttr.cudaDevAttrMultiProcessorCount, dev) - if int(err) != 0: - raise RuntimeError(f"cudaDeviceGetAttribute failed: {err}") - return count - - -def _data_ptr(t) -> int: - """Device address of a tensor-like (``data_ptr()`` or the CUDA array - interface).""" - fn = getattr(t, "data_ptr", None) - if fn is not None: - return fn() - return t.__cuda_array_interface__["data"][0] - - -def _cutlass_io_dtype(dtype): - name = str(dtype) - if "bfloat16" in name: - return cutlass.BFloat16 - if "float16" in name or "half" in name: - return cutlass.Float16 - raise ValueError(f"Unsupported dtype {dtype}, expected bfloat16 or float16") - - -def _cutlass_state_dtype(dtype): - name = str(dtype) - if "bfloat16" in name: - return cutlass.BFloat16 - if "float32" in name: - return cutlass.Float32 - raise ValueError(f"Unsupported state dtype {dtype}, expected float32 or bfloat16") + base_checkpoint = cuda.create_tensor_map_tiled_from_view(checkpoint_view, box_dims=(tma_granu_elems, d_k, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz) + n_warps = 6 if state_checkpoints is not None else 5 + build_all_descs_kernel( + base_q, + base_k, + base_v, + base_gate, + base_o, + base_checkpoint, + tensormap_workspace, + cu_seqlens, + q, + k, + v, + gate, + o, + state_checkpoints, + cutlass.Int32(batch_size), + cutlass.Int32(q.stride[0]), + cutlass.Int32(k.stride[0]), + cutlass.Int32(v.stride[0]), + cutlass.Int32(gate.stride[0]), + cutlass.Int32(o.stride[0]), + cutlass.Int32(state_checkpoints.stride[0] if state_checkpoints is not None else 0), + checkpoint_every_n, + ).launch(grid=(1, 1, 1), block=(32 * n_warps, 1, 1), stream=stream) + + +# ---- Torch adapter / host-side compilation --------------------------------------- @lru_cache(maxsize=None) -def _get_compiled_cache( +def get_compiled_cache( io_dtype_str: str, state_dtype_str: str, cu_dtype_str: str, @@ -2713,9 +2741,9 @@ def compile( dt_bias_cute, beta_cute, cu_seqlens_cute, - s_in_cute, + state_in_cute, o_cute, - s_out_cute, + state_out_cute, work_items_cute=None, work_count_cute=None, sched_ctr_cute=None, @@ -2745,7 +2773,7 @@ def compile( ) return cute.compile( - _host, + host, cfg, q_cute, k_cute, @@ -2755,9 +2783,9 @@ def compile( dt_bias_cute, beta_cute, cu_seqlens_cute, - s_in_cute, + state_in_cute, o_cute, - s_out_cute, + state_out_cute, work_items_cute, work_count_cute, sched_ctr_cute, @@ -2781,7 +2809,7 @@ def chunk_kda_sm100( output_state, scale: float, checkpoint_every_n_tokens: int = 0, - output_checkpoints=None, + output_state_checkpoints=None, use_qk_l2norm_in_kernel: bool = False, safe_gate: bool = False, gate_lower_bound: float = DEFAULT_GATE_LOWER_BOUND, @@ -2797,7 +2825,9 @@ def chunk_kda_sm100( ) -> None: """Execute the Blackwell BT=16 chunked KDA prefill kernel. - All tensors must be contiguous and on the same CUDA device. + All tensors must be on the same CUDA device with a stride-1 innermost + dim; outer strides are free (padded / permuted views are read through + the TMA descriptors and dynamic layouts). Args: q: ``(total_tokens, HQ, DK)`` float16/bfloat16 @@ -2813,13 +2843,13 @@ def chunk_kda_sm100( initial_state: ``(num_seqs, HO, DK, DV)`` float32/bfloat16, or None output_state: ``(num_seqs, HO, DK, DV)`` float32/bfloat16, or None scale: attention scale factor (must not be 0) - checkpoint_every_n_tokens: emit an H entry every N tokens (0 = off). - H[j] is the state after ``(j + 1) * N`` tokens, STRICTLY BEFORE + checkpoint_every_n_tokens: emit a state checkpoint every N tokens (0 = off). + state_checkpoints[j] is the state after ``(j + 1) * N`` tokens, STRICTLY BEFORE the sequence end — the end-of-sequence state is only - ``output_state``. With ``N == B_T`` this is the per-chunk state + ``output_state``. With ``N == B_T`` this is the per-chunk checkpoint series the backward pass consumes. - output_checkpoints: ``(total_h, HO, DK, DV)`` io-dtype (KV, v - contiguous — the GDN H layout); the per-sequence entry offsets + output_state_checkpoints: ``(total_checkpoints, HO, DK, DV)`` io-dtype (KV, v + contiguous); the per-sequence entry offsets are derived on device from ``cu_seqlens`` ((seqlen-1)//N, prefix-summed), so there is no cu_checkpoints array use_qk_l2norm_in_kernel: L2-normalize q/k rows inside the kernel @@ -2827,7 +2857,7 @@ def chunk_kda_sm100( a_log: ``(HO,)`` float32, safe-gate per-head log-amplitude (None = 0) dt_bias: ``(HO, DK)`` float32, safe-gate channel bias (None = 0) use_beta_sigmoid_in_kernel: ``beta`` holds logits; sigmoid in-kernel - work_items: ``(max_items, 6)`` int32 split-K work-item table from + work_items: ``(max_items, 8)`` int32 split-K work-item table from ``common/split_k.py``, or None for the one-tile-per-(b,h) layout. With a table, each item computes chunks ``[cstart, wend)`` and writes O/checkpoints only for @@ -2847,11 +2877,11 @@ def chunk_kda_sm100( store_final_state = output_state is not None enable_checkpoints = checkpoint_every_n_tokens > 0 if enable_checkpoints: - if output_checkpoints is None: - raise ValueError("checkpoint_every_n_tokens > 0 requires output_checkpoints") - if str(output_checkpoints.dtype).split(".")[-1] != str(q.dtype).split(".")[-1]: + if output_state_checkpoints is None: + raise ValueError("checkpoint_every_n_tokens > 0 requires output_state_checkpoints") + if str(output_state_checkpoints.dtype).split(".")[-1] != str(q.dtype).split(".")[-1]: raise ValueError( - f"output_checkpoints dtype must match the io dtype (fp32 state belongs to output_state): got {output_checkpoints.dtype} with io {q.dtype}" + f"output_state_checkpoints dtype must match the io dtype (fp32 state belongs to output_state): got {output_state_checkpoints.dtype} with io {q.dtype}" ) split_k = work_items is not None dyn_sched = sched_ctr is not None @@ -2883,11 +2913,9 @@ def chunk_kda_sm100( if not safe_gate: a_log = None dt_bias = None - if _data_ptr(tensormap_workspace) % 128 != 0: - raise ValueError("tensormap_workspace must be 128-byte aligned") cu_stream = cuda_driver.CUstream(int(stream)) - cache = _get_compiled_cache( + cache = get_compiled_cache( str(q.dtype), str(state_dtype_src), str(cu_seqlens.dtype), @@ -2906,38 +2934,30 @@ def chunk_kda_sm100( ) if "compiled" not in cache: - io_dtype = _cutlass_io_dtype(q.dtype) - state_dtype = _cutlass_state_dtype(state_dtype_src) - q_cute = from_dlpack(q, assumed_align=16) - q_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - k_cute = from_dlpack(k, assumed_align=16) - k_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - v_cute = from_dlpack(v, assumed_align=16) - v_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - gate_cute = from_dlpack(gate, assumed_align=16) - gate_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) + io_dtype = get_dtype(q.dtype) + state_dtype = get_dtype(state_dtype_src) + q_cute = from_dlpack(q, assumed_align=16).mark_layout_dynamic(leading_dim=2) + k_cute = from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2) + v_cute = from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2) + gate_cute = from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=2) a_log_cute = from_dlpack(a_log, assumed_align=4) if a_log is not None else None dt_bias_cute = from_dlpack(dt_bias, assumed_align=16) if dt_bias is not None else None - beta_cute = from_dlpack(beta, assumed_align=4) - beta_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) - o_cute = from_dlpack(output, assumed_align=16) - o_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) + beta_cute = from_dlpack(beta, assumed_align=4).mark_layout_dynamic(leading_dim=1) + o_cute = from_dlpack(output, assumed_align=16).mark_layout_dynamic(leading_dim=2) cu_seqlens_cute = from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic() - s_in_cute = None + state_in_cute = None if use_initial_state: - s_in_cute = from_dlpack(initial_state, assumed_align=16) - s_in_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=CFG.D_K) + state_in_cute = from_dlpack(initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) - s_out_cute = None + state_out_cute = None if store_final_state: - s_out_cute = from_dlpack(output_state, assumed_align=16) - s_out_cute.mark_layout_dynamic().mark_compact_shape_dynamic(mode=3, stride_order=(0, 1, 2, 3), divisibility=CFG.D_K) + state_out_cute = from_dlpack(output_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) work_items_cute = None work_count_cute = None if split_k: - work_items_cute = from_dlpack(work_items, assumed_align=4) + work_items_cute = from_dlpack(work_items, assumed_align=16) work_items_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) work_count_cute = from_dlpack(work_count, assumed_align=4).mark_layout_dynamic() @@ -2963,7 +2983,7 @@ def chunk_kda_sm100( HO, split_k, dyn_sched, - num_sm=_device_sm_count(), + num_sm=multiprocessor_count(current_device_id()), q_cute=q_cute, k_cute=k_cute, v_cute=v_cute, @@ -2972,9 +2992,9 @@ def chunk_kda_sm100( dt_bias_cute=dt_bias_cute, beta_cute=beta_cute, cu_seqlens_cute=cu_seqlens_cute, - s_in_cute=s_in_cute, + state_in_cute=state_in_cute, o_cute=o_cute, - s_out_cute=s_out_cute, + state_out_cute=state_out_cute, work_items_cute=work_items_cute, work_count_cute=work_count_cute, sched_ctr_cute=sched_ctr_cute, @@ -2985,36 +3005,26 @@ def chunk_kda_sm100( ) compiled = cache["compiled"] - - # The descriptors encode cu_seqlens' CONTENTS, which no key built from the - # buffers can track. The skip this replaces asked torch's _version counter, - # so it was sound for a torch caller and silently stale for every other - # producer. Rebuilding unconditionally measures free: 131 vs 135 us of host - # time, and 157 either way once the launches are waited on. - h_for_descs = output_checkpoints if enable_checkpoints else None - if cache.get("build_descs_has_h") != (h_for_descs is not None): + state_checkpoints_for_descs = output_state_checkpoints if enable_checkpoints else None + # desc build runs every execute by contract (cu contents are data; + # buffer pointers may change) — capture-safe, single tiny launch + if cache.get("build_descs_has_state_checkpoints") != (state_checkpoints_for_descs is not None): cache.pop("build_descs", None) - cache["build_descs_has_h"] = h_for_descs is not None + cache["build_descs_has_state_checkpoints"] = state_checkpoints_for_descs is not None if "build_descs" not in cache: - io_dtype = _cutlass_io_dtype(q.dtype) - q_bd = from_dlpack(q, assumed_align=16) - q_bd.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - k_bd = from_dlpack(k, assumed_align=16) - k_bd.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - v_bd = from_dlpack(v, assumed_align=16) - v_bd.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - gate_bd = from_dlpack(gate, assumed_align=16) - gate_bd.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) - o_bd = from_dlpack(output, assumed_align=16) - o_bd.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2), divisibility=1) + io_dtype = get_dtype(q.dtype) + q_bd = from_dlpack(q, assumed_align=16).mark_layout_dynamic(leading_dim=2) + k_bd = from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2) + v_bd = from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2) + gate_bd = from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=2) + o_bd = from_dlpack(output, assumed_align=16).mark_layout_dynamic(leading_dim=2) cu_bd = from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic() ws_bd = from_dlpack(tensormap_workspace, assumed_align=128).mark_layout_dynamic() - h_bd = None - if h_for_descs is not None: - h_bd = from_dlpack(h_for_descs, assumed_align=16) - h_bd.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1, 2, 3), divisibility=1) + state_checkpoints_bd = None + if state_checkpoints_for_descs is not None: + state_checkpoints_bd = from_dlpack(state_checkpoints_for_descs, assumed_align=16).mark_layout_dynamic(leading_dim=3) cache["build_descs"] = cute.compile( - _build_descs, + build_descs, io_dtype, CFG.B_T, q_bd, @@ -3022,7 +3032,7 @@ def chunk_kda_sm100( v_bd, gate_bd, o_bd, - h_bd, + state_checkpoints_bd, cu_bd, ws_bd, cutlass.Int32(checkpoint_every_n_tokens), @@ -3035,13 +3045,12 @@ def chunk_kda_sm100( v, gate, output, - h_for_descs, + state_checkpoints_for_descs, cu_seqlens, tensormap_workspace, checkpoint_every_n_tokens, cu_stream, ) - compiled( q, k, diff --git a/python/cudnn/linear_attention/frost/kernel/kda_recompute_config.py b/python/cudnn/linear_attention/frost/kernel/kda_recompute_config.py new file mode 100644 index 000000000..4b0e7c4f7 --- /dev/null +++ b/python/cudnn/linear_attention/frost/kernel/kda_recompute_config.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# This kernel is derived from cuDNN, NVIDIA Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kimi Delta Attention (KDA) Cutlass DSL recompute (state/H-only) kernel +config (fixed compile-time constants). The BT=16 KDA schedule uses a 16-warp +(512-thread) specialization with a per-key-channel decay; the derived +SMEM/TMEM sizes and offsets are stamped by ``build_cfg`` in +``kda_recompute_f16.py``. + +Target arch: Blackwell SM100 (GB200) / SM103 (GB300). +""" + +from dataclasses import dataclass +from typing import Tuple + + +@dataclass(frozen=True) +class Cfg: + # --- tile shape --- + B_T: int = 16 # chunk-inner token tile (BT=16 KDA schedule) + D_K: int = 128 # query/key head dim + D_V: int = 128 # value head dim + + # --- warp assignments (16 warps = 512 threads) --- + COMPUTE_GROUP_0_WARP_IDS: Tuple[int, ...] = (0, 1, 2, 3, 4, 5, 6, 7) # decay-operand materialize (2-group ping-pong) + COMPUTE_GROUP_1_WARP_IDS: Tuple[int, ...] = (8, 9, 10, 11) # value-side TMEM / state staging + SUPER_MMA_WARP_ID: int = 12 # register-MMA kk + Neumann inverse + TCGEN05_MMA_WARP_ID: int = 13 # tcgen05 state GEMMs + TMA_WARP_ID: int = 14 # k/v/gate TMA loads + EPILOGUE_WARP_ID: int = 15 # H TMA store + + # --- register split --- + NUM_REGS_COMPUTE_GROUP_0: int = 160 + NUM_REGS_COMPUTE_GROUP_1: int = 136 + NUM_REGS_OTHER: int = 56 + + THREADS_PER_WARP: int = 32 + + BUFFER_ALIGN_BYTES: int = 1024 + + # --- SMEM / TMEM ring stage counts --- + SMEM_RAW_STAGES: int = 8 + SMEM_SCHED_STAGES: int = 8 + SMEM_DECAY_STAGES: int = 2 + SMEM_QK_STAGES: int = 2 + SMEM_STATE_SCALE_DIAG_STAGES: int = 3 + QK_SCALE_READY_STAGES: int = 3 + + CLUSTER_SHAPE_MNK: Tuple[int, int, int] = (1, 1, 1) + + +CFG = Cfg() diff --git a/python/cudnn/linear_attention/frost/kernel/kda_recompute_f16.py b/python/cudnn/linear_attention/frost/kernel/kda_recompute_f16.py new file mode 100644 index 000000000..bf627ca34 --- /dev/null +++ b/python/cudnn/linear_attention/frost/kernel/kda_recompute_f16.py @@ -0,0 +1,2579 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# This kernel is derived from cuDNN, NVIDIA Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Chunked Kimi Delta Attention (KDA) recompute (state/checkpoints-only) kernel for +Blackwell SM100/SM103 (Cutlass DSL), BT=16 tiling with a per-key-channel decay. +Framework-neutral entry ``chunk_kda_recompute_sm100``. + +Persistent kernel: the grid is the SM count and every warp role +runs a tile-scheduler loop (``decode_work_item``); a tile is one (batch, +head) sequence, or one split-K work item computing chunks ``[cstart, wend)`` +and writing checkpoints only for the owned ``[wstart, wend)`` (see +``common/split_k.py``; warmup chunks rebuild the incoming state from +zero). All ring stage/phase bookkeeping runs on cumulative per-CTA chunk +counters so pipelines flow seamlessly across tiles. + +Pipeline (direct CUTLASS primitives, chunk_idx-size 16 KDA schedule): + + load K/V/Gate/Beta + optional in-kernel L2-norm of K (L2NORM specialization) + exp2(g), exp2(-g), stage final-token exp2(g) as exp2(g_last) + super-MMA: KK/Neumann inverse + apply Beta + tcgen05-MMA: State*K/new-v/kv-update + store periodic state checkpoints, final state + +ABI: k `[T, HK, DK]`, v `[T, HV, DV]`, gate +`[T, HO, DK]` fp32 (natural-log decay unless SAFE_GATE, which applies the +safe-gate transform from raw gate + a_log/dt_bias), beta `[T, HO]` fp32 +post-sigmoid, cu_seqlens int32, states/checkpoints `[N, HO, DV, DK]` (VK). +GQA/GVA head broadcast follows repeat_interleave: source head = +head_idx // (HO // H_x). State presence, L2NORM, SAFE_GATE, checkpoints, and the +head ratios are compile-time specializations. + +Warp assignments (16 warps = 512 threads): + warps 0-7 : compute group 0 - Gate prefix scan + decay/restore operands + warps 8-11 : compute group 1 - TMEM value side, state stores + warp 12 : super-MMA - register-MMA KK^T + Neumann inverse + warp 13 : tcgen05-MMA - the four state GEMMs + the TMEM lifecycle + warp 14 : TMA load - per-chunk input G->S loads + warp 15 : epilogue - the checkpoint TMA store + +SMEM layout: + Buffer Bytes Stages + K / V raw 32768 8 <-- SW128 TMA ring (io dtype) + Gate raw 65536 8 <-- fp32 prefix-scan source + Beta 512 8 <-- fp32 per-token scalars + K_inv 8192 2 <-- token-major ldmatrix/tcgen05 B operand + K decay 8192 2 <-- tcgen05 SW128 K-box-major A/B operands + K restore 8192 2 <-- tcgen05 B operand for the state update + state-scale diag 12288 3 <-- per-k-atom decay diagonal blocks + A_inv 2048 2 <-- SW32 16x16 register-MMA tiles + +TMEM layout (240 of 512 columns): + Buffer Cols Purpose + state 0-127 state[DK,DV] fp32 recurrent state + state inp 128-191 packed b16 A operand view of the state + state_k_acc 192-207 State*K fp32 accumulator + update_acc 208-223 update fp32 accumulator + rhs input 224-231 packed b16 A operand: Beta * (V - State*K) + update input 232-239 packed b16 A operand: the update readback + +GEMM schedule (tcgen05-MMA warp, in issue order per chunk): + State*K -> state_k_acc + State decay (diag blocks) + update = A_inv @ rhs -> update_acc + final_state += update @ K_restore + +Requires a cutlass DSL build providing `cutlass.experimental.*`; not +available in the pip nvidia-cutlass-dsl releases. +""" + +from dataclasses import dataclass +from functools import lru_cache +from typing import Callable, NamedTuple, Optional, Type + +import cuda.bindings.driver as cuda_driver +import cutlass +import cutlass.experimental.cuda as cuda +import cutlass.experimental.primitives as nvvm +import cutlass.cute as cute +from cutlass.cute.runtime import from_dlpack + +from ..common.split_k import decode_work_item +from ..common.host import get_dtype +from cudnn.frost.buffers import current_device_id, data_ptr +from cudnn.frost.device import multiprocessor_count +from ..common.thd import TENSOR_MAP_QWORDS, emit_checkpoint_seq_descs, emit_seq_descs +from .kda_recompute_config import CFG +from cudnn.frost.tile_dsl.barrier import ( + advance, + MBarrier, + PipelineState, + Producer, +) +from cudnn.frost.tile_dsl.handles import GmemTileTma, MmaDesc, SmemTile, tma_slice_runtime_desc +from cudnn.frost.tile_dsl.mma import mma_step, mma_ts_step +from cudnn.frost.tile_dsl.swizzle import swizzle_lin_128b, swizzle_lin_S, swizzle_xor_128b +from cudnn.frost.tile_dsl.tma import tma_load_tile, tma_store_commit, tma_store_tile, tma_store_wait, tma_tensormap_acquire +from cudnn.frost.tile_dsl.pointwise import ( + opaque_f32_zero, + f16x2_to_f32, + fadd2, + fmul2, + ffma2, + movmatrix_16b, + mul_f16x2, + fp32_to_fp16, + sub_f16x2, +) + +LOG2_E: float = 1.4426950408889634 + + +DEFAULT_GATE_LOWER_BOUND: float = -5.0 + + +# Host-side API defaults. + + +L2_NORM_EPS: float = 1.0e-12 + + +class KdaBars(NamedTuple): + """Every inter-warp handoff as an ``MBarrier`` over its ring. Consumers track ``(idx, phase)`` inline; the producer tag selects + the arrive lowering (``TMA_LOAD``/``MMA_COMMIT``/``THREAD``).""" + + mb_k_ready: MBarrier + mb_k_done: MBarrier + mb_v_ready: MBarrier + mb_v_done: MBarrier + mb_gate_ready: MBarrier + mb_gate_done: MBarrier + + mb_beta_ready: MBarrier + mb_beta_done: MBarrier + + mb_state_k_acc_ready: MBarrier + mb_update_acc_ready: MBarrier + + mb_state_inp_ready: MBarrier + mb_rhs_ready: MBarrier + mb_update_ready: MBarrier + + mb_a_ready: MBarrier + mb_a_inv_done: MBarrier + mb_qk_scale_ready: MBarrier + mb_state_scale_diag_done: MBarrier + mb_k_decay_inv_cg0_ready: MBarrier + mb_decay_done: MBarrier + mb_decay_super_done: MBarrier + mb_k_restore_done: MBarrier + + mb_state_updated: MBarrier + mb_state_read_done: MBarrier + mb_final_state_stored: MBarrier + + mb_checkpoint_tmastg_ready: MBarrier + mb_checkpoint_tmastg_done: MBarrier + + mb_sched_ready: MBarrier + mb_sched_done: MBarrier + + +def make_kda_bars(cfg) -> KdaBars: + """Bars factory. MUST be called from inside ``kernel`` (allocates the + mbarrier rings in SMEM ahead of the data buffers).""" + + def alloc(n): + return cutlass.Array(cutlass.Int64, n, space=cutlass.AddressSpace.smem, alignment=8) + + WARP = cfg.threads_per_warp + CG0_GROUP_THREADS = cfg.cg0_warps_per_group * WARP + CG1_THREADS = len(cfg.compute_group_1_warp_ids) * WARP + + return KdaBars( + mb_k_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_k_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_v_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_v_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_gate_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=1, producer=Producer.TMA_LOAD), + mb_gate_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_beta_ready=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=WARP, producer=Producer.THREAD), + mb_beta_done=MBarrier(alloc(cfg.smem_raw_stages), stages=cfg.smem_raw_stages, init_count=WARP + CG1_THREADS, producer=Producer.THREAD), + mb_state_k_acc_ready=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), + mb_update_acc_ready=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), + mb_state_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_rhs_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_update_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_a_inv_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_qk_scale_ready=MBarrier( + alloc(cfg.qk_scale_ready_stages), + stages=cfg.qk_scale_ready_stages, + init_count=CG0_GROUP_THREADS, + producer=Producer.THREAD, + ), + mb_state_scale_diag_done=MBarrier( + alloc(cfg.smem_state_scale_diag_stages), + stages=cfg.smem_state_scale_diag_stages, + init_count=1, + producer=Producer.MMA_COMMIT, + ), + mb_k_decay_inv_cg0_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), + mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=WARP, producer=Producer.THREAD), + mb_k_restore_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_state_updated=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), + mb_state_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_final_state_stored=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_checkpoint_tmastg_ready=MBarrier( + alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=CG1_THREADS, producer=Producer.THREAD + ), + mb_checkpoint_tmastg_done=MBarrier(alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=WARP, producer=Producer.THREAD), + mb_sched_ready=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=1, producer=Producer.THREAD), + mb_sched_done=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=15, producer=Producer.THREAD), + ) + + +# ---- Dynamic tile scheduler ------------------------------------------------------ + + +@cute.jit +def sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas): + """TMA-warp side: pull the next tile off the global ticket, publish it.""" + if cutlass.const_expr(cfg.dyn_sched): + bars.mb_sched_done[sched_state.idx].wait(sched_state.phase) + if nvvm.elect_sync(): + fetched = cutlass.Int32(nvvm.atomicrmw("add", mSched.iterator, cutlass.Int32(1), mem_order="relaxed", syncscope="gpu")) + sSched[sched_state.idx] = num_ctas + fetched + nvvm.bar_warp_sync(cute.arch.FULL_MASK) + next_tile = sSched[sched_state.idx] + if nvvm.elect_sync(): + bars.mb_sched_ready[sched_state.idx].arrive() + return next_tile, advance(sched_state, cfg.sched_stages) + return tile_idx + num_ctas, sched_state + + +@cute.jit +def sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas): + """Consumer side: read the TMA warp's published next tile.""" + if cutlass.const_expr(cfg.dyn_sched): + bars.mb_sched_ready[sched_state.idx].wait(sched_state.phase) + next_tile = sSched[sched_state.idx] + if nvvm.elect_sync(): + bars.mb_sched_done[sched_state.idx].arrive() + return next_tile, advance(sched_state, cfg.sched_stages) + return tile_idx + num_ctas, sched_state + + +@cute.jit +def tmaldg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + mSched, + sSched, + lane, + sK_raw, + sV_raw, + sGate_raw, + desc_k_base, + desc_v_base, + desc_gate_base, + bars, +) -> None: + """TMA-LDG warp role (warp 14): persistent scheduler loop issuing the + per-chunk K/V/Gate G->S loads.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + sK_tma = SmemTile( + base=sK_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sV_tma = SmemTile( + base=sV_raw, + elems_per_stage=(cfg.d_v * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=(cfg.b_t * 64), + ) + sGate_tma = SmemTile( + base=sGate_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_raw_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_k // 32), + tma_granu_elems=32, + tma_subtile_stride_elems=(cfg.b_t * 32), + ) + raw_index = PipelineState.start(phase=1) + sched_state = PipelineState.start(phase=1) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + head_o = head_idx + head_k = head_idx if cfg.k_ratio == 1 else head_idx // cutlass.Int32(cfg.k_ratio) + head_v = head_idx if cfg.v_ratio == 1 else head_idx // cutlass.Int32(cfg.v_ratio) + slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_k_slot = (desc_k_base + slot).tospace(cutlass.AddressSpace.generic) + desc_v_slot = (desc_v_base + slot).tospace(cutlass.AddressSpace.generic) + desc_gate_slot = (desc_gate_base + slot).tospace(cutlass.AddressSpace.generic) + if elect_one: + tma_tensormap_acquire(desc_k_slot) + tma_tensormap_acquire(desc_v_slot) + tma_tensormap_acquire(desc_gate_slot) + for chunk_idx in cutlass.range(cstart, wend, 1, unroll=1): + chunk_start = chunk_idx * cfg.b_t + + # ---- K load ---------------------------------------------------------- + bars.mb_k_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_k_ready[raw_index.idx].arrive(n_bytes=cfg.tma_k_bytes) + k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, chunk_start) + tma_load_tile(sK_tma[raw_index.idx], k_slice, bars.mb_k_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- Gate load ------------------------------------------------------- + bars.mb_gate_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_gate_ready[raw_index.idx].arrive(n_bytes=cfg.tma_gate_bytes) + gate_slice = tma_slice_runtime_desc(desc_gate_slot, cutlass.Int32(0), head_o, chunk_start) + tma_load_tile(sGate_tma[raw_index.idx], gate_slice, bars.mb_gate_ready[raw_index.idx].smem_ptr, acquire=False) + + # ---- V load ---------------------------------------------------------- + bars.mb_v_done[raw_index.idx].wait(raw_index.phase) + if elect_one: + bars.mb_v_ready[raw_index.idx].arrive(n_bytes=cfg.tma_v_bytes) + v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, chunk_start) + tma_load_tile(sV_tma[raw_index.idx], v_slice, bars.mb_v_ready[raw_index.idx].smem_ptr, acquire=False) + + raw_index = advance(raw_index, cfg.smem_raw_stages) + tile_idx, sched_state = sched_publish_next(cfg, bars, sSched, mSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def super_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_inv_raw, + sQk_raw, + sBeta_raw, + sK_decay_raw, + bars, +) -> None: + """Super-MMA warp role (warp 12): persistent scheduler loop computing the + register-MMA Neumann-series A_inv.""" + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + raw_index = PipelineState.start(phase=0) + a_inv_free = PipelineState.start(phase=1) + k_decay_ready = PipelineState.start(phase=0) + + # ---- ldmatrix/stmatrix lane decode ------------------------------------------- + rhs_row_coord = lane % 8 + (cutlass.Int32(8) if (lane // 16) else cutlass.Int32(0)) + rhs_col_offset = cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0) + lhs_row_coord = lane % 8 + (cutlass.Int32(8) if ((lane // 8) % 2) else cutlass.Int32(0)) + lhs_col_offset = cutlass.Int32(8) if ((lane // 8) // 2) else cutlass.Int32(0) + decay_key_mask = cutlass.Int32(8) + stsm_row_coord = lane & 7 + stsm_col_coord = cutlass.Int32(0) + if (lane // 8) & 1: + stsm_row_coord = stsm_row_coord + cutlass.Int32(8) + if lane // 8 >= 2: + stsm_col_coord = cutlass.Int32(8) + stsm_idx = swizzle_lin_S(stsm_row_coord * cfg.b_t + (stsm_col_coord ^ (cfg.b_t // 2)), bbits=1, mbase=3, sshift=3) + cum_chunk_base = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + num_chunks_tile = wend - cstart # processed chunks; ring bookkeeping runs on cum_chunk_base + local_chunk_idx + for local_chunk_idx in cutlass.range(num_chunks_tile, unroll=1): + cum_chunk = cum_chunk_base + local_chunk_idx + decay_stage = k_decay_ready.idx + qk_stage = a_inv_free.idx + sBeta_ptr = sBeta_raw.data_ptr() + raw_index.idx * cfg.b_t + sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sK_decay_ptr = sK_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sQk_ptr = sQk_raw.data_ptr() + qk_stage * (2 * cfg.b_t * cfg.b_t) + + bars.mb_a_inv_done[qk_stage].wait(a_inv_free.phase) + a_inv_free = advance(a_inv_free, cfg.smem_qk_stages) + bars.mb_k_decay_inv_cg0_ready[decay_stage].wait(k_decay_ready.phase) + k_decay_ready = advance(k_decay_ready, cfg.smem_decay_stages) + + # ---- KK = K_decay @ K_inv^T ------------------------------------------ + kk_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + kk_acc[accum_idx] = cutlass.Float32(0.0) + + for k_block in cutlass.range_constexpr((cfg.d_k // 16)): + # Load B operand + k_inv_col = k_block * 16 + rhs_col_offset + k_inv_segment = k_inv_col // 64 + rhs_vec = nvvm.ldmatrix( + sK_inv_ptr + + k_inv_segment * (cfg.b_t * 64) + + rhs_row_coord * 64 + + swizzle_xor_128b(rhs_row_coord, k_inv_col - k_inv_segment * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + # Load A operand + storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask + storage_slice = storage_key // 64 + kk_lhs_vec = nvvm.ldmatrix( + sK_decay_ptr + + storage_slice * (cfg.b_t * 64) + + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), + 4, + nvvm.MMALayout.ROW, + ) + + mma_step( + kk_acc, + (kk_lhs_vec[0], kk_lhs_vec[1], kk_lhs_vec[2], kk_lhs_vec[3]), + (rhs_vec[0], rhs_vec[1], rhs_vec[2], rhs_vec[3]), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + + # ---- L = Beta * tril(KK, -1) fragment -------------------------------- + bars.mb_beta_ready[raw_index.idx].wait(raw_index.phase) + row_lo = lane // 4 + row_hi = row_lo + cutlass.Int32(8) + beta_lo = (sBeta_ptr + row_lo).load().to(cutlass.Float32) + beta_hi = (sBeta_ptr + row_hi).load().to(cutlass.Float32) + l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + l_frag[accum_idx] = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + for pair in cutlass.range_constexpr(4): + beta_scale = beta_hi if cutlass.const_expr(pair % 2 == 1) else beta_lo + l_frag[2 * pair], l_frag[2 * pair + 1] = fmul2(l_frag[2 * pair], l_frag[2 * pair + 1], beta_scale, beta_scale) + bars.mb_beta_done[raw_index.idx].arrive() + l_a0 = fp32_to_fp16(l_frag[0], l_frag[1], dtype=cfg.io_dtype) + l_a1 = fp32_to_fp16(l_frag[2], l_frag[3], dtype=cfg.io_dtype) + l_a2 = fp32_to_fp16(l_frag[4], l_frag[5], dtype=cfg.io_dtype) + l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) + l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) + + # ---- A_inv = I - L, then three Neumann doubling rounds --------------- + ainv_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + row_coord = row_lo + if cutlass.const_expr(accum_idx % 4 >= 2): + row_coord = row_hi + col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) + if cutlass.const_expr(accum_idx % 2 == 1): + col_coord = col_coord + cutlass.Int32(1) + eye = cutlass.Float32(1.0) if row_coord == col_coord else cutlass.Float32(0.0) + ainv_acc[accum_idx] = eye - l_values[accum_idx] + + lpow_a0, lpow_a1, lpow_a2, lpow_a3 = l_a0, l_a1, l_a2, l_a3 + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(l_a0), movmatrix_16b(l_a1), movmatrix_16b(l_a2), movmatrix_16b(l_a3) + for _round in cutlass.range_constexpr(3): + # ---- Lpow = Lpow @ Lpow ------------------------------------------ + sq_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + sq_acc[accum_idx] = cutlass.Float32(0.0) + mma_step( + sq_acc, + (lpow_a0, lpow_a1, lpow_a2, lpow_a3), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + lpow_a0 = fp32_to_fp16(sq_acc[0], sq_acc[1], dtype=cfg.io_dtype) + lpow_a1 = fp32_to_fp16(sq_acc[2], sq_acc[3], dtype=cfg.io_dtype) + lpow_a2 = fp32_to_fp16(sq_acc[4], sq_acc[5], dtype=cfg.io_dtype) + lpow_a3 = fp32_to_fp16(sq_acc[6], sq_acc[7], dtype=cfg.io_dtype) + mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3 = movmatrix_16b(lpow_a0), movmatrix_16b(lpow_a1), movmatrix_16b(lpow_a2), movmatrix_16b(lpow_a3) + # ---- A_inv += A_inv @ Lpow --------------------------------------- + upd_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) + for accum_idx in cutlass.range_constexpr(8): + upd_acc[accum_idx] = cutlass.Float32(0.0) + ainv_p0 = fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype) + ainv_p1 = fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype) + ainv_p2 = fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype) + ainv_p3 = fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype) + mma_step( + upd_acc, + (ainv_p0, ainv_p1, ainv_p2, ainv_p3), + (mov_lpow0, mov_lpow1, mov_lpow2, mov_lpow3), + k_step=0, + M=16, + N=16, + ab_dtype=cfg.io_dtype, + ) + ainv_lo0, ainv_hi0 = f16x2_to_f32(ainv_p0, dtype=cfg.io_dtype) + ainv_lo1, ainv_hi1 = f16x2_to_f32(ainv_p1, dtype=cfg.io_dtype) + ainv_lo2, ainv_hi2 = f16x2_to_f32(ainv_p2, dtype=cfg.io_dtype) + ainv_lo3, ainv_hi3 = f16x2_to_f32(ainv_p3, dtype=cfg.io_dtype) + ainv_acc[0] = ainv_lo0 + upd_acc[0] + ainv_acc[1] = ainv_hi0 + upd_acc[1] + ainv_acc[2] = ainv_lo1 + upd_acc[2] + ainv_acc[3] = ainv_hi1 + upd_acc[3] + ainv_acc[4] = ainv_lo2 + upd_acc[4] + ainv_acc[5] = ainv_hi2 + upd_acc[5] + ainv_acc[6] = ainv_lo3 + upd_acc[6] + ainv_acc[7] = ainv_hi3 + upd_acc[7] + + nvvm.stmatrix( + sQk_ptr + (cfg.b_t * cfg.b_t) + stsm_idx, + [ + fp32_to_fp16(ainv_acc[0], ainv_acc[1], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[2], ainv_acc[3], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[4], ainv_acc[5], dtype=cfg.io_dtype), + fp32_to_fp16(ainv_acc[6], ainv_acc[7], dtype=cfg.io_dtype), + ], + nvvm.MMALayout.ROW, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_a_ready[qk_stage].arrive() + bars.mb_decay_super_done[decay_stage].arrive() + raw_index = advance(raw_index, cfg.smem_raw_stages) + cum_chunk_base += num_chunks_tile + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def tcgen05_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + sTmem_base, + sQk, + sK_decay, + sK_restore, + sState_scale_diag, + bars, +) -> None: + """tcgen05-MMA warp role (warp 13): persistent scheduler loop issuing + every state GEMM and owning the TMEM lifecycle.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + tmem_base = sTmem_base.load() + state_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) + state_dsts = tuple(nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset + k * 16, cutlass.Float32) for k in range(cfg.d_k // 16)) + state_k_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_k_acc_offset, cutlass.Float32) + update_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_acc_offset, cutlass.Float32) + rhs_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_rhs_inp_offset, cutlass.Int8) + update_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_update_inp_offset, cutlass.Int8) + state_dst_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset, cutlass.Float32) + state_inp_index = PipelineState.start(phase=0) + state_read_index = PipelineState.start(phase=0) + rhs_index = PipelineState.start(phase=0) + update_index = PipelineState.start(phase=0) + final_state_index = PipelineState.start(phase=0) + qk_scale_index = PipelineState.start(phase=0) + k_decay_ready = PipelineState.start(phase=0) + qk_ready = PipelineState.start(phase=0) + + # ---- chunk-invariant GEMM descriptors ---------------------------------------- + bpe = cfg.io_dtype.width // 8 + idesc_acc = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.b_t, + m_dim=cfg.d_v, + b_major=0, + ) + idesc_diag = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=16, + m_dim=cfg.d_v, + b_major=0, + ) + idesc_final_state = nvvm.Tcgen05InstrDesc.build( + c_dtype=cutlass.Float32, + a_dtype=cfg.io_dtype, + b_dtype=cfg.io_dtype, + n_dim=cfg.d_k, + m_dim=cfg.d_v, + b_major=1, + ) + bmm_state_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.d_k, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_acc, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bmm_diag_desc = MmaDesc( + M=cfg.d_v, + N=16, + K=16, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_diag, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bmm_qk_desc = MmaDesc( + M=cfg.d_v, + N=cfg.b_t, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=False, + cta_group=1, + idesc=idesc_acc, + kind=nvvm.Tcgen05MMAKind.F16, + ) + bmm_final_state_desc = MmaDesc( + M=cfg.d_v, + N=cfg.d_k, + K=cfg.b_t, + bpe_a=bpe, + bpe_b=bpe, + tile_k_hw=16, + btranspose=True, + cta_group=1, + idesc=idesc_final_state, + kind=nvvm.Tcgen05MMAKind.F16, + ) + STATE_A_SEG = bmm_state_desc.sps_B * bmm_state_desc.tmem_advance_A + STATE_B_SEG = bmm_state_desc.smem_subtile_B >> 4 + cum_chunk_base = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + num_chunks_tile = wend - cstart + for local_chunk_idx in cutlass.range(num_chunks_tile, unroll=1): + cum_chunk = cum_chunk_base + local_chunk_idx + decay_stage = k_decay_ready.idx + state_scale_diag_stage = qk_scale_index.idx + qk_stage = qk_ready.idx + sK_decay_stage = sK_decay[decay_stage] + sK_restore_stage = sK_restore[decay_stage] + sState_scale_diag_stage = sState_scale_diag[state_scale_diag_stage] + sQk_stage = sQk[qk_stage] + + # ---- State*K = State(T) @ K_decay^T ---------------------------------- + bars.mb_k_decay_inv_cg0_ready[decay_stage].wait(k_decay_ready.phase) + k_decay_ready = advance(k_decay_ready, cfg.smem_decay_stages) + bars.mb_state_inp_ready.wait(state_inp_index.phase) + state_inp_index = advance(state_inp_index, 1) + desc_k_decay = sK_decay_stage.desc() + + for s in cutlass.range_constexpr(bmm_state_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_state_desc.sps_B): + mma_ts_step( + bmm_state_desc, + state_inp_ptr.subview(s * STATE_A_SEG), + desc_k_decay + s * STATE_B_SEG, + state_k_acc_ptr, + k, + cutlass.Boolean(s + k > 0), + ) + + if elect_one: + bars.mb_state_k_acc_ready.arrive(cta_group=1) + bars.mb_decay_done[decay_stage].arrive(cta_group=1) + + bars.mb_qk_scale_ready[qk_scale_index.idx].wait(qk_scale_index.phase) + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_read_done.wait(state_read_index.phase) + state_read_index = advance(state_read_index, 1) + + # ---- State decay = State(T) @ exp2(g_last) diag (per-k-atom blocks) ---- + desc_diag = sState_scale_diag_stage.desc() + + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + mma_ts_step( + bmm_diag_desc, + state_inp_ptr.subview(k_block * bmm_diag_desc.tmem_advance_A), + desc_diag.advance_start_address(k_block * 256 * 2), + state_dsts[k_block], + 0, + cutlass.Boolean(False), + ) + + if elect_one: + bars.mb_state_scale_diag_done[state_scale_diag_stage].arrive(cta_group=1) + + # ---- update = rhs(T) @ A_inv ----------------------------------------- + bars.mb_a_ready[qk_stage].wait(qk_ready.phase) + bars.mb_rhs_ready.wait(rhs_index.phase) + rhs_index = advance(rhs_index, 1) + desc_qk = sQk_stage.shifted((cfg.b_t * cfg.b_t)).desc() + mma_ts_step(bmm_qk_desc, rhs_inp_ptr, desc_qk, update_acc_ptr, 0, cutlass.Boolean(False)) + if elect_one: + bars.mb_a_inv_done[qk_stage].arrive(cta_group=1) + bars.mb_update_acc_ready.arrive(cta_group=1) + + # ---- final_state += update(T) @ K_restore ---------------------------- + bars.mb_update_ready.wait(update_index.phase) + update_index = advance(update_index, 1) + desc_k_restore = sK_restore_stage.desc() + + mma_ts_step(bmm_final_state_desc, update_inp_ptr, desc_k_restore, state_dst_ptr, 0, cutlass.Boolean(True)) + if elect_one: + bars.mb_k_restore_done[decay_stage].arrive(cta_group=1) + bars.mb_state_updated.arrive(cta_group=1) + + qk_ready = advance(qk_ready, cfg.smem_qk_stages) + qk_scale_index = advance(qk_scale_index, cfg.smem_state_scale_diag_stages) + + bars.mb_final_state_stored.wait(final_state_index.phase) + final_state_index = advance(final_state_index, 1) + cum_chunk_base += num_chunks_tile + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + nvvm.tcgen05_dealloc( + nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), + cutlass.Int32(512), + group=nvvm.CTAGroup.CTA_1, + ) + + +@cute.jit +def epilogue_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + sCheckpoint_raw, + desc_checkpoint_base, + checkpoint_every_n_tokens, + bars, +) -> None: + """Epilogue warp role (warp 15): persistent scheduler loop issuing the + per-chunk checkpoint TMA stores.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + if cutlass.const_expr(cfg.enable_checkpoints): + sCheckpoint_tma = SmemTile( + base=sCheckpoint_raw, + elems_per_stage=(cfg.d_k * cfg.d_v), + stages=cfg.smem_checkpoint_stages, + leading_byte_offset=0, + stride_byte_offset=0, + layout=0, + tma_loads_per_tile=(cfg.d_v // 64), + tma_granu_elems=64, + tma_subtile_stride_elems=cfg.d_k * 64, + ) + checkpoint_ready_index = PipelineState.start(phase=0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + if cutlass.const_expr(cfg.enable_checkpoints): + head_o = head_idx + checkpoint_slot = batch_idx * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_checkpoint_slot = (desc_checkpoint_base + checkpoint_slot).tospace(cutlass.AddressSpace.generic) + checkpoint_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) + checkpoint_quot = (cstart + cutlass.Int32(1)) // checkpoint_chunks + checkpoint_mod = (cstart + cutlass.Int32(1)) % checkpoint_chunks + if elect_one: + tma_tensormap_acquire(desc_checkpoint_slot) + num_chunks_tile = wend - cstart + for local_chunk_idx in cutlass.range(num_chunks_tile, unroll=1): + chunk_idx = cstart + local_chunk_idx + if local_chunk_idx > 0: + # ---- checkpoint store ---------------------------------------- + do_checkpoint = checkpoint_mod == 0 + if cutlass.const_expr(cfg.split_k): + do_checkpoint = do_checkpoint and chunk_idx >= wstart + if do_checkpoint: + checkpoint_stage = checkpoint_ready_index.idx + bars.mb_checkpoint_tmastg_ready[checkpoint_stage].wait(checkpoint_ready_index.phase) + checkpoint_ready_index = advance(checkpoint_ready_index, cfg.smem_checkpoint_stages) + checkpoint_entry = checkpoint_quot - cutlass.Int32(1) + checkpoint_slice = tma_slice_runtime_desc(desc_checkpoint_slot, cutlass.Int32(0), cutlass.Int32(0), checkpoint_entry, head_o) + tma_store_tile(sCheckpoint_tma[checkpoint_stage], checkpoint_slice, acquire=False) + tma_store_commit() + tma_store_wait(0) + bars.mb_checkpoint_tmastg_done[checkpoint_stage].arrive() + checkpoint_mod = checkpoint_mod + cutlass.Int32(1) + if checkpoint_mod == checkpoint_chunks: + checkpoint_mod = cutlass.Int32(0) + checkpoint_quot = checkpoint_quot + cutlass.Int32(1) + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def gate_scale(cfg, raw_gate: cutlass.Float32) -> cutlass.Float32: + """Map raw gate to the log2-domain decay increment used by KDA.""" + + if cutlass.const_expr(cfg.safe_gate): + half = cutlass.Float32(0.5) + sigmoid = cute.math.tanh(raw_gate * half, approx=True) * half + half + return cfg.gate_scale_log2 * sigmoid + # Default ABI: Gate arrives in natural-log space + return raw_gate * cutlass.Float32(LOG2_E) + + +@cute.jit +def compute0_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + warp_idx, + mA_log, + mDt_bias, + sK_inv_raw, + sGate_raw, + mBeta, + sBeta_raw, + sK_raw, + sK_decay_raw, + sK_restore_raw, + sState_scale_diag_raw, + bars, +) -> None: + """CG0 warp-group role (warps 0-7): persistent scheduler loop running the + Gate prefix scan and staging the decay/restore operands.""" + nvvm.setmaxregister(cfg.num_regs_compute_group_0, nvvm.SetMaxRegisterAction.INCREASE) + cg0_warp = warp_idx - cfg.compute_group_0_warp_ids[0] + cg0_group_id = cg0_warp // cfg.cg0_warps_per_group + cg0_local_warp = cg0_warp % cfg.cg0_warps_per_group + prefix_dim = cg0_local_warp * cfg.threads_per_warp + lane + cg0_a_log_exp = cutlass.Float32(1.0) + cg0_dt_bias_value = cutlass.Float32(0.0) + cum_chunk_base = cutlass.Int32(0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + opaque_one = opaque_f32_zero() + cutlass.Float32(1.0) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + head_o = head_idx + num_chunks_tile = wend - cstart + if cutlass.const_expr(cfg.safe_gate): + if num_chunks_tile > 0: + cg0_a_log_exp = cute.math.exp2(mA_log[head_o].to(cutlass.Float32) * LOG2_E, fastmath=True) + cg0_dt_bias_value = mDt_bias[head_o, prefix_dim].to(cutlass.Float32) + group_cum_chunk_start = cum_chunk_base + cutlass.Int32(cg0_group_id) + diag_ring_stage = group_cum_chunk_start % cutlass.Int32(cfg.smem_state_scale_diag_stages) + diag_ring_phase = (group_cum_chunk_start // cutlass.Int32(cfg.smem_state_scale_diag_stages)) % cutlass.Int32(2) + for local_chunk_idx in cutlass.range(cg0_group_id, num_chunks_tile, cfg.cg0_group_count, unroll=1): + chunk_idx = cstart + local_chunk_idx + cum_chunk = cum_chunk_base + local_chunk_idx + chunk_start = chunk_idx * cfg.b_t + decay_stage = cum_chunk % cfg.smem_decay_stages + raw_stage = cum_chunk % cfg.smem_raw_stages + state_scale_diag_stage = diag_ring_stage + qk_scale_ready_stage = state_scale_diag_stage + sK_ptr = sK_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sGate_ptr = sGate_raw.data_ptr() + raw_stage * (cfg.d_k * cfg.b_t) + sK_inv_ptr = sK_inv_raw.data_ptr() + decay_stage * (cfg.b_t * cfg.d_k) + sK_decay_ptr = sK_decay_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sK_restore_ptr = sK_restore_raw.data_ptr() + decay_stage * (cfg.d_k * cfg.b_t) + sState_scale_diag_ptr = sState_scale_diag_raw.data_ptr() + state_scale_diag_stage * ((cfg.d_k // 16) * 256) + + # ---- Beta scalars --------------------------------------------------- + if cg0_local_warp == 0: + bars.mb_beta_done[raw_stage].wait(((cum_chunk // cfg.smem_raw_stages) + 1) % 2) + if lane < cfg.b_t: + token_idx = chunk_idx * cfg.b_t + lane + beta_value = cutlass.Float32(0.0) + if token_idx < seqlen_b: + beta_value = mBeta[batch_start + token_idx, head_o].to(cutlass.Float32) + if cutlass.const_expr(cfg.beta_sigmoid): + half = cutlass.Float32(0.5) + beta_value = (cute.math.tanh(beta_value * half, approx=True) * half + half).to(mBeta.element_type).to(cutlass.Float32) + sBeta_raw[raw_stage * cfg.b_t + lane] = beta_value + bars.mb_beta_ready[raw_stage].arrive() + bars.mb_gate_ready[raw_stage].wait((cum_chunk // cfg.smem_raw_stages) % 2) + + row_group_start = cg0_local_warp * (cfg.b_t // cfg.cg0_warps_per_group) + lane_row_group = lane // 8 + lane_in_row_group = lane - lane_row_group * 8 + decay_row = row_group_start + lane_row_group + decay_key_mask = cutlass.Int32(8) + + prefix_dim = cg0_local_warp * cfg.threads_per_warp + lane + + # ---- Gate prefix scan ----------------------------------------------- + f32_segment = prefix_dim // 32 + prefix_seg_base = f32_segment * (cfg.b_t * 32) + prefix_col = prefix_dim - f32_segment * 32 + gate_raw = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + for row in cutlass.range_constexpr(cfg.b_t): + prefix_idx = prefix_seg_base + swizzle_xor_128b(row, row * 32 + prefix_col, elem_bytes=4) + gate_raw[row] = (sGate_ptr + prefix_idx).load() + g_prefix_regs = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + if cutlass.const_expr(cfg.safe_gate): + valid_rows = seqlen_b - chunk_idx * cutlass.Int32(cfg.b_t) + valid_mask = cutlass.vector.create_mask([cfg.b_t], [valid_rows]) + for row_pair in cutlass.range_constexpr(cfg.b_t // 2): + row0 = row_pair * 2 + row1 = row0 + 1 + gate0 = cg0_a_log_exp * (gate_raw[row0] + cg0_dt_bias_value) + gate1 = cg0_a_log_exp * (gate_raw[row1] + cg0_dt_bias_value) + gate0 = gate_scale( + cfg, + gate0, + ) + gate1 = gate_scale( + cfg, + gate1, + ) + gate_pair = cutlass.Vector.from_elements((gate0, gate1), cutlass.Float32) + gate_pair = cutlass.vector.where(valid_mask[row0 : row1 + 1], gate_pair, 0.0) + g_prefix_regs[row0] = gate_pair[0] + g_prefix_regs[row1] = gate_pair[1] + else: + for row in cutlass.range_constexpr(cfg.b_t): + gate = gate_raw[row] + token_idx = chunk_idx * cutlass.Int32(cfg.b_t) + cutlass.Int32(row) + if token_idx < seqlen_b: + gate = gate_scale( + cfg, + gate, + ) + else: + gate = cutlass.Float32(0.0) + g_prefix_regs[row] = gate + + prefix_acc = cutlass.Float32(0.0) + for row_pair in cutlass.range_constexpr(cfg.b_t // 2): + row0 = row_pair * 2 + row1 = row0 + 1 + gate0 = g_prefix_regs[row0] + gate1 = g_prefix_regs[row1] + prefix0, row_pair_sum = fadd2(prefix_acc, gate0, gate0, gate1) + prefix1 = prefix_acc + row_pair_sum + g_prefix_regs[row0] = prefix0 + g_prefix_regs[row1] = prefix1 + prefix_acc = prefix1 + + # ---- exp2(g): stage prefixes + final-token decay --------------------- + for row in cutlass.range_constexpr(cfg.b_t): + g_prefix_regs[row] = cute.math.exp2(g_prefix_regs[row], fastmath=True) + + exp_g_last = g_prefix_regs[cfg.b_t - 1] + for row in cutlass.range_constexpr(cfg.b_t): + prefix_idx = prefix_seg_base + swizzle_xor_128b(row, row * 32 + prefix_col, elem_bytes=4) + (sGate_ptr + prefix_idx).store(g_prefix_regs[row]) + + # ---- state-scale diag: stage exp2(g_last) decay blocks --------------- + bars.mb_state_scale_diag_done[state_scale_diag_stage].wait(diag_ring_phase ^ cutlass.Int32(1)) + block = prefix_dim // cutlass.Int32(16) + coord = prefix_dim - block * cutlass.Int32(16) + storage_col = coord ^ cutlass.Int32((cfg.b_t // 2)) + linear_idx = block * cutlass.Int32(256) + coord * cutlass.Int32(16) + storage_col + diag_idx = swizzle_lin_S(linear_idx, bbits=1, mbase=3, sshift=3) + sState_scale_diag_ptr[diag_idx] = exp_g_last.to(cfg.io_dtype) + + nvvm.barrier_cta_sync(cfg.cg0_group_sync_barrier_base_id + cg0_group_id, thread_count=cfg.cg0_threads_per_group) + + bars.mb_k_ready[raw_stage].wait((cum_chunk // cfg.smem_raw_stages) % 2) + k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) + raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + + # ---- optional K L2-norm + K_inv staging ------------------------------ + if cutlass.const_expr(cfg.l2norm): + kk0_lo = opaque_f32_zero() + kk0_hi = opaque_f32_zero() + kk1_lo = opaque_f32_zero() + kk1_hi = opaque_f32_zero() + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + raw_f16_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) + raw_k_vec = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_k_vec_f32 = raw_k_vec.to(cutlass.Float32) + for dim_offset in cutlass.range_constexpr(8): + k_val = raw_k_vec_f32[dim_offset] + raw_k_regs[reg_base + dim_offset] = k_val + if cutlass.const_expr(cfg.l2norm): + if cutlass.const_expr(dim_offset % 2 == 0): + kk0_lo, kk0_hi = ffma2(k_val, k_val, k_val, k_val, kk0_lo, kk0_hi) + else: + kk1_lo, kk1_hi = ffma2(k_val, k_val, k_val, k_val, kk1_lo, kk1_hi) + + k_inv_norm = opaque_one + if cutlass.const_expr(cfg.l2norm): + k_sum_sq = kk0_hi + kk1_hi + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 4, 31, kind=nvvm.Shfl.BFLY)) + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 2, 31, kind=nvvm.Shfl.BFLY)) + k_sum_sq = k_sum_sq + cutlass.Float32(nvvm.shfl_sync(0xFFFFFFFF, k_sum_sq, 1, 31, kind=nvvm.Shfl.BFLY)) + norm_floor_sq = cutlass.Float32(L2_NORM_EPS * L2_NORM_EPS) + k_inv_norm = cute.math.rsqrt(cute.math.max(k_sum_sq, norm_floor_sq), fastmath=True) + + # ---- decay/restore operands: exp2(+-g) applied per key channel ------- + exp_g_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + exp_g_last_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + for f32_group in cutlass.range_constexpr(2): + f32_dim_base = dim_base + f32_group * 4 + f32_segment = f32_dim_base // 32 + f32_segment_dim = f32_dim_base - f32_segment * 32 + g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) + exp_g_vec = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) + exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) + exp_g_last_vec = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) + f32_reg_base = reg_base + f32_group * 4 + for j in cutlass.range_constexpr(4): + exp_g_regs[f32_reg_base + j] = exp_g_vec[j] + exp_g_last_regs[f32_reg_base + j] = exp_g_last_vec[j] + + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + + # ---- K decay + K_inv operands: K * exp2(+g) and K * exp2(-g) ----- + k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + for pair_idx in cutlass.range_constexpr(4): + dim0 = pair_idx * 2 + dim1 = dim0 + 1 + raw_reg_idx0 = reg_base + dim0 + raw_reg_idx1 = reg_base + dim1 + k_value0, k_value1 = fmul2(raw_k_regs[raw_reg_idx0], raw_k_regs[raw_reg_idx1], k_inv_norm, k_inv_norm) + k_pair = fp32_to_fp16(k_value0, k_value1, dtype=cfg.io_dtype) + exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) + k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) + exp_neg_g0 = cute.math.rcp(exp_g_regs[raw_reg_idx0], approx=True, ftz=True) + exp_neg_g1 = cute.math.rcp(exp_g_regs[raw_reg_idx1], approx=True, ftz=True) + exp_neg_pair = fp32_to_fp16(exp_neg_g0, exp_neg_g1, dtype=cfg.io_dtype) + k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_pair, exp_neg_pair, cfg.io_dtype) + + k_inv_vec = cutlass.Vector.from_elements( + ( + k_inv_words[dim_half * 4], + k_inv_words[dim_half * 4 + 1], + k_inv_words[dim_half * 4 + 2], + k_inv_words[dim_half * 4 + 3], + ), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + k_decay_vec = cutlass.Vector.from_elements( + ( + k_decay_words[0], + k_decay_words[1], + k_decay_words[2], + k_decay_words[3], + ), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + if cutlass.const_expr(dim_half == 0): + operand_done_phase = ((cum_chunk // cfg.smem_decay_stages) + 1) % 2 + bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + k_inv_swizzled_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) + (sK_inv_ptr + k_inv_swizzled_idx).store(k_inv_vec, alignment=16) + storage_key = dim_base ^ decay_key_mask + storage_slice = storage_key // 64 + decay_swizzled_idx = storage_slice * (cfg.b_t * 64) + swizzle_xor_128b( + decay_row, decay_row * 64 + storage_key - storage_slice * 64, elem_bytes=2 + ) + (sK_decay_ptr + decay_swizzled_idx).store(k_decay_vec, alignment=16) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_k_decay_inv_cg0_ready[decay_stage].arrive() + + # ---- K_restore operand: K_inv * exp_g_last -------------------------- + bars.mb_k_restore_done[decay_stage].wait(((cum_chunk // cfg.smem_decay_stages + 1) % 2)) + for dim_half in cutlass.range_constexpr(2): + dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 + reg_base = dim_half * 8 + k_restore_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + for pair_idx in cutlass.range_constexpr(4): + dim0 = pair_idx * 2 + dim1 = dim0 + 1 + exp_g_last_pair = fp32_to_fp16(exp_g_last_regs[reg_base + dim0], exp_g_last_regs[reg_base + dim1], dtype=cfg.io_dtype) + k_restore_words[pair_idx] = mul_f16x2(k_inv_words[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) + storage_row = decay_row ^ (cfg.b_t // 2) + f16_segment = dim_base // 64 + f16_segment_dim = dim_base - f16_segment * 64 + k_restore_idx = f16_segment * (cfg.b_t * 64) + storage_row * 64 + swizzle_xor_128b(storage_row, f16_segment_dim, elem_bytes=2) + k_restore_vec = cutlass.Vector.from_elements( + ( + k_restore_words[0], + k_restore_words[1], + k_restore_words[2], + k_restore_words[3], + ), + cutlass.Int32, + ).bitcast(cfg.io_dtype) + (sK_restore_ptr + k_restore_idx).store(k_restore_vec, alignment=16) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_qk_scale_ready[qk_scale_ready_stage].arrive() + bars.mb_k_done[raw_stage].arrive() + bars.mb_gate_done[raw_stage].arrive() + diag_ring_stage = diag_ring_stage + cutlass.Int32(cfg.cg0_group_count) + wrapped = diag_ring_stage >= cutlass.Int32(cfg.smem_state_scale_diag_stages) + diag_ring_stage = diag_ring_stage - cutlass.Int32(cfg.smem_state_scale_diag_stages) if wrapped else diag_ring_stage + diag_ring_phase = diag_ring_phase ^ (cutlass.Int32(1) if wrapped else cutlass.Int32(0)) + cum_chunk_base += num_chunks_tile + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def compute1_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sTmem_base, + warp_idx, + mState_out, + mState_init, + sBeta_raw, + sV_raw, + sCheckpoint_raw, + checkpoint_every_n_tokens, + bars, +) -> None: + """CG1 warp-group role (warps 8-11): persistent scheduler loop staging the + value-side TMEM operands and storing the checkpoint/final states.""" + nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) + sCheckpoint_ptr = sCheckpoint_raw.data_ptr() if cutlass.const_expr(cfg.enable_checkpoints) else sV_raw.data_ptr() + checkpoint_done_index = PipelineState.start(phase=1) + tmem_base = sTmem_base.load() + tmem_col = tmem_base & 0xFFFF + tmem_row = tmem_base >> 16 + tmem_subpartition = warp_idx % (cfg.d_v // cfg.threads_per_warp) + # ldmatrix.x4 COL lane decode for the V loads + frag_row_coord = (lane // 16) * 8 + (lane & 7) + frag_col_offset = ((lane // 8) & 1) * 8 + row_id = tmem_row + tmem_subpartition * cfg.threads_per_warp + value_dim = tmem_subpartition * cfg.threads_per_warp + lane + value_dim_base = tmem_subpartition * cfg.threads_per_warp + row_addr = row_id << 16 + row16_addr = (row_id + 16) << 16 + st_row_addr = tmem_row << 16 + st_row16_addr = (tmem_row + 16) << 16 + state_col_id = tmem_col + cfg.tmem_state_acc_offset + packed_col_id = tmem_col + cfg.tmem_state_inp_offset + statek_col_id = tmem_col + cfg.tmem_state_k_acc_offset + rhs_inp_col_id = tmem_col + cfg.tmem_rhs_inp_offset + upd_acc_addr = row_addr + tmem_col + cfg.tmem_update_acc_offset + upd_inp_addr = st_row_addr + tmem_col + cfg.tmem_update_inp_offset + v_swz_off0 = ( + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + frag_col_offset) % 64, elem_bytes=2) + ) + v_swz_off = ( + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + + frag_row_coord * 64 + + swizzle_xor_128b(frag_row_coord, (value_dim_base + 16 + frag_col_offset) % 64, elem_bytes=2) + ) + checkpoint_swz_off0 = (value_dim_base + frag_col_offset) // 64 * (cfg.d_k * 64) + checkpoint_swz_col0 = (value_dim_base + frag_col_offset) % 64 + checkpoint_swz_off = (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.d_k * 64) + checkpoint_swz_col = (value_dim_base + 16 + frag_col_offset) % 64 + state_k_acc_index = PipelineState.start(phase=0) + update_acc_index = PipelineState.start(phase=0) + state_upd_index = PipelineState.start(phase=0) + raw_index = PipelineState.start(phase=0) + sched_state = PipelineState.start(phase=0) + tile_idx = cutlass.Int32(bidx) + while tile_idx < total_tiles: + batch_idx, head_idx, batch_start, batch_end, seqlen_b, num_chunks_b, wstart, wend, cstart, cend = decode_work_item( + cfg, tile_idx, cu_seqlens, mWorkItems + ) + head_o = head_idx + num_chunks_tile = wend - cstart + + if num_chunks_tile > 0: + # ---- first chunk: seed state TMEM from mState_init (else zeros) ---------- + seed_from_initial_state = cstart == 0 + if cutlass.const_expr(mState_init is not None and cfg.split_k): + if seed_from_initial_state: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + state_block[col] = mState_init[batch_idx, head_o, key_dim, value_dim].to(cutlass.Float32) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + else: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + state_block[col] = cutlass.Float32(0.0) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + elif cutlass.const_expr(mState_init is not None): + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + state_block[col] = mState_init[batch_idx, head_o, key_dim, value_dim].to(cutlass.Float32) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + else: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) + for col in cutlass.range_constexpr(32): + state_block[col] = cutlass.Float32(0.0) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + state_block[0:32], + ) + + nvvm.tcgen05_wait("store") + sV_ptr = sV_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) + sBeta_ptr = sBeta_raw.data_ptr() + raw_index.idx * cfg.b_t + + # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- + state_blocks = [] + for sub in cutlass.range_constexpr(cfg.d_k // 16): + state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) + + for sub in cutlass.range_constexpr(cfg.d_k // 16): + packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) + for packed_col in cutlass.range_constexpr(8): + source_pair = packed_col ^ 4 + packed_state[packed_col] = fp32_to_fp16(state_blocks[sub][2 * source_pair], state_blocks[sub][2 * source_pair + 1], dtype=cfg.io_dtype) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + sub * 8, cutlass.Int8), + packed_state[0:8], + ) + + nvvm.tcgen05_wait("store") + bars.mb_state_inp_ready.arrive() + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_read_done.arrive() + + # ---- rhs staging: rhs input = Beta * (V - State*K) ------------------- + bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) + bars.mb_beta_ready[raw_index.idx].wait(raw_index.phase) + bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) + + # ---- read back State*K acc + raw V fragments ------------------------- + state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + statek_col_id, cutlass.Float32), num=2) + + state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + statek_col_id, cutlass.Float32), num=2) + + raw_v_regs0 = nvvm.ldmatrix( + sV_ptr + v_swz_off0, + 4, + nvvm.MMALayout.COL, + ) + raw_v_regs1 = nvvm.ldmatrix( + sV_ptr + v_swz_off, + 4, + nvvm.MMALayout.COL, + ) + + beta_pairs = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + token0 = (((reg_idx // 2) * 4 + (lane & 3)) ^ 4) * 2 + beta0 = (sBeta_ptr + token0).load().to(cutlass.Float32) + beta1 = (sBeta_ptr + token0 + 1).load().to(cutlass.Float32) + beta_pairs[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) + packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] + state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) + diff_pair = sub_f16x2( + raw_v_regs0[raw_matrix], + state_k_pair, + cfg.io_dtype, + ) + packed_rhs0[reg_idx] = mul_f16x2( + beta_pairs[reg_idx], + diff_pair, + cfg.io_dtype, + ) + + packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] + state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) + diff_pair = sub_f16x2( + raw_v_regs1[raw_matrix], + state_k_pair, + cfg.io_dtype, + ) + packed_rhs1[reg_idx] = mul_f16x2( + beta_pairs[reg_idx], + diff_pair, + cfg.io_dtype, + ) + + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_wait("store") + state_k_acc_index = advance(state_k_acc_index, 1) + bars.mb_v_done[raw_index.idx].arrive() + bars.mb_beta_done[raw_index.idx].arrive() + bars.mb_rhs_ready.arrive() + + # ---- update repack: acc TMEM -> packed b16 TMEM --------------------- + bars.mb_update_acc_ready.wait(update_acc_index.phase) + update = nvvm.tcgen05_ld( + "32x32b", + nvvm.make_tmem_ptr(upd_acc_addr, cutlass.Float32), + num=cfg.b_t, + ) + + packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): + source_pair = packed_col ^ 4 + token0 = source_pair * 2 + token1 = token0 + 1 + packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr(upd_inp_addr, cutlass.Int8), + packed_update[0 : (cfg.b_t // 2)], + ) + nvvm.tcgen05_wait("store") + update_acc_index = advance(update_acc_index, 1) + bars.mb_update_ready.arrive() + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_updated.wait(state_upd_index.phase) + state_upd_index = advance(state_upd_index, 1) + + raw_index = advance(raw_index, cfg.smem_raw_stages) + + if cutlass.const_expr(cfg.enable_checkpoints): + cg1_checkpoint_chunks = checkpoint_every_n_tokens // cutlass.Int32(cfg.b_t) + cg1_checkpoint_mod = (cstart + cutlass.Int32(1)) % cg1_checkpoint_chunks + for local_chunk_idx in cutlass.range(1, num_chunks_tile, 1, unroll=1): + chunk_idx = cstart + local_chunk_idx + sV_ptr = sV_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) + sBeta_ptr = sBeta_raw.data_ptr() + raw_index.idx * cfg.b_t + + do_checkpoint = False + if cutlass.const_expr(cfg.enable_checkpoints): + do_checkpoint = cg1_checkpoint_mod == 0 + cg1_checkpoint_mod = cg1_checkpoint_mod + cutlass.Int32(1) + cg1_checkpoint_mod = cutlass.Int32(0) if cg1_checkpoint_mod == cg1_checkpoint_chunks else cg1_checkpoint_mod + if cutlass.const_expr(cfg.split_k): + do_checkpoint = do_checkpoint and chunk_idx >= wstart + + # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- + if cutlass.const_expr(not cfg.enable_checkpoints): + bars.mb_state_updated.wait(state_upd_index.phase) + state_upd_index = advance(state_upd_index, 1) + state_blocks = [] + for sub in cutlass.range_constexpr(cfg.d_k // 16): + state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) + + for sub in cutlass.range_constexpr(cfg.d_k // 16): + packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) + for packed_col in cutlass.range_constexpr(8): + source_pair = packed_col ^ 4 + packed_state[packed_col] = fp32_to_fp16(state_blocks[sub][2 * source_pair], state_blocks[sub][2 * source_pair + 1], dtype=cfg.io_dtype) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + sub * 8, cutlass.Int8), + packed_state[0:8], + ) + nvvm.tcgen05_wait("store") + bars.mb_state_inp_ready.arrive() + + # ---- checkpoint store ----------------------------------------------- + if cutlass.const_expr(cfg.enable_checkpoints): + if do_checkpoint: + checkpoint_stage = checkpoint_done_index.idx + bars.mb_checkpoint_tmastg_done[checkpoint_stage].wait(checkpoint_done_index.phase) + checkpoint_done_index = advance(checkpoint_done_index, cfg.smem_checkpoint_stages) + checkpoint_stage_base = checkpoint_stage * (cfg.d_k * cfg.d_v) + for slab in cutlass.range_constexpr(cfg.d_k // 16): + checkpoint_ld0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + slab * 16, cutlass.Float32), num=2) + checkpoint_ld1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + slab * 16, cutlass.Float32), num=2) + checkpoint_st0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_st1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + checkpoint_st0[reg_idx] = fp32_to_fp16(checkpoint_ld0[2 * reg_idx], checkpoint_ld0[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_st1[reg_idx] = fp32_to_fp16(checkpoint_ld1[2 * reg_idx], checkpoint_ld1[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_row = slab * 16 + frag_row_coord + nvvm.stmatrix( + sCheckpoint_ptr + + checkpoint_stage_base + + checkpoint_swz_off0 + + checkpoint_row * 64 + + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col0, elem_bytes=2), + checkpoint_st0.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.stmatrix( + sCheckpoint_ptr + + checkpoint_stage_base + + checkpoint_swz_off + + checkpoint_row * 64 + + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col, elem_bytes=2), + checkpoint_st1.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, + ) + nvvm.fence_proxy("async.shared", space="cta") + bars.mb_checkpoint_tmastg_ready[checkpoint_stage].arrive() + nvvm.tcgen05_wait("load") + bars.mb_state_read_done.arrive() + else: + bars.mb_state_read_done.arrive() + + bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) + + # ---- rhs staging: rhs input = Beta * (V - State*K) ------------------- + bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) + bars.mb_beta_ready[raw_index.idx].wait(raw_index.phase) + + # ---- read back State*K acc + raw V fragments ------------------------- + state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + statek_col_id, cutlass.Float32), num=2) + state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + statek_col_id, cutlass.Float32), num=2) + + raw_v_regs0 = nvvm.ldmatrix( + sV_ptr + v_swz_off0, + 4, + nvvm.MMALayout.COL, + ) + raw_v_regs1 = nvvm.ldmatrix( + sV_ptr + v_swz_off, + 4, + nvvm.MMALayout.COL, + ) + + beta_pairs = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + token0 = (((reg_idx // 2) * 4 + (lane & 3)) ^ 4) * 2 + beta0 = (sBeta_ptr + token0).load().to(cutlass.Float32) + beta1 = (sBeta_ptr + token0 + 1).load().to(cutlass.Float32) + beta_pairs[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) + packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] + state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) + diff_pair = sub_f16x2( + raw_v_regs0[raw_matrix], + state_k_pair, + cfg.io_dtype, + ) + packed_rhs0[reg_idx] = mul_f16x2( + beta_pairs[reg_idx], + diff_pair, + cfg.io_dtype, + ) + + packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + for reg_idx in cutlass.range_constexpr(4): + raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) + frag_pair = (reg_idx ^ 2) * 2 + state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] + state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) + diff_pair = sub_f16x2( + raw_v_regs1[raw_matrix], + state_k_pair, + cfg.io_dtype, + ) + packed_rhs1[reg_idx] = mul_f16x2( + beta_pairs[reg_idx], + diff_pair, + cfg.io_dtype, + ) + + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_wait("store") + state_k_acc_index = advance(state_k_acc_index, 1) + bars.mb_v_done[raw_index.idx].arrive() + bars.mb_beta_done[raw_index.idx].arrive() + bars.mb_rhs_ready.arrive() + + # ---- update repack: acc TMEM -> packed b16 TMEM --------------------- + bars.mb_update_acc_ready.wait(update_acc_index.phase) + update = nvvm.tcgen05_ld( + "32x32b", + nvvm.make_tmem_ptr(upd_acc_addr, cutlass.Float32), + num=cfg.b_t, + ) + + packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): + source_pair = packed_col ^ 4 + token0 = source_pair * 2 + token1 = token0 + 1 + packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr(upd_inp_addr, cutlass.Int8), + packed_update[0 : (cfg.b_t // 2)], + ) + nvvm.tcgen05_wait("store") + update_acc_index = advance(update_acc_index, 1) + bars.mb_update_ready.arrive() + + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_updated.wait(state_upd_index.phase) + state_upd_index = advance(state_upd_index, 1) + raw_index = advance(raw_index, cfg.smem_raw_stages) + + if num_chunks_tile > 0: + if cutlass.const_expr(not cfg.enable_checkpoints): + bars.mb_state_updated.wait(state_upd_index.phase) + state_upd_index = advance(state_upd_index, 1) + + owns_final = wend == num_chunks_b + + # ---- final-state drain: state acc TMEM -> GMEM --------------------------- + if cutlass.const_expr(mState_out is not None): + if seqlen_b > 0: + if owns_final: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + loaded = nvvm.tcgen05_ld( + "32x32b", + nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), + num=32, + ) + + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + mState_out[batch_idx, head_o, key_dim, value_dim] = loaded[col].to(mState_out.element_type) + else: + for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): + for col in cutlass.range_constexpr(32): + key_dim = key_block_start + col + if cutlass.const_expr(mState_init is not None): + mState_out[batch_idx, head_o, key_dim, value_dim] = mState_init[batch_idx, head_o, key_dim, value_dim] + else: + mState_out[batch_idx, head_o, key_dim, value_dim] = cutlass.Float32(0.0).to(mState_out.element_type) + bars.mb_final_state_stored.arrive() + tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + + +@cute.jit +def host( + cfg: cutlass.Constexpr, + k: cute.Tensor, + v: cute.Tensor, + raw_gate: cute.Tensor, + a_log: cute.Tensor | None, + dt_bias: cute.Tensor | None, + beta: cute.Tensor, + cu_seqlens: cute.Tensor, + initial_state: cute.Tensor | None, + final_state: cute.Tensor | None, + work_items: cute.Tensor | None, + work_count: cute.Tensor | None, + sched_ctr: cute.Tensor | None, + tensormap_workspace: cute.Tensor, + checkpoint_every_n_tokens: cutlass.Int32, + stream, +) -> None: + num_sequences = cu_seqlens.shape[0] - 1 + ho = raw_gate.shape[1] + total_tiles = num_sequences * ho + grid_shape = (cfg.max_active_clusters, 1, 1) + kernel( + cfg, + tensormap_workspace, + cutlass.Int32(num_sequences), + k, + v, + raw_gate, + a_log, + dt_bias, + beta, + cu_seqlens, + initial_state, + final_state, + work_items, + work_count, + sched_ctr, + total_tiles, + checkpoint_every_n_tokens, + ).launch( + grid=grid_shape, + block=(cfg.threads_per_cta, 1, 1), + stream=stream, + min_blocks_per_mp=1, + ) + + +@cute.kernel +def kernel( + cfg: cutlass.Constexpr, + tensormap_workspace: cute.Tensor, + n_desc: cutlass.Int32, + mK: cute.Tensor, + mV: cute.Tensor, + mGate: cute.Tensor, + mA_log: cute.Tensor | None, + mDt_bias: cute.Tensor | None, + mBeta: cute.Tensor, + cu_seqlens: cute.Tensor, + mState_init: cute.Tensor | None, + mState_out: cute.Tensor | None, + mWorkItems: cute.Tensor | None, + mCount: cute.Tensor | None, + mSched: cute.Tensor | None, + total_tiles: cutlass.Int32, + checkpoint_every_n_tokens: cutlass.Int32, +) -> None: + """BT=16 KDA recompute (state/checkpoints-only) persistent kernel body: every warp + role runs a tile-scheduler loop over the tiles.""" + + tidx, _, _ = cute.arch.thread_idx() + bidx = cute.arch.block_idx()[0] + num_ctas = cute.arch.grid_dim()[0] + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + lane = tidx % cfg.threads_per_warp + + if cutlass.const_expr(cfg.split_k): + total_tiles = mCount[0] + if cutlass.const_expr(cfg.dyn_sched): + assert mSched is not None and mSched.element_type == cutlass.Int32 + assert mK.element_type == cfg.io_dtype and mV.element_type == cfg.io_dtype + assert mGate.element_type == cutlass.Float32 + beta_expected = cfg.io_dtype if cutlass.const_expr(cfg.beta_sigmoid) else cutlass.Float32 + assert mBeta.element_type == beta_expected + assert cu_seqlens.element_type in (cutlass.Int32, cutlass.Int64) + if cutlass.const_expr(cfg.use_initial_state): + assert mState_init is not None and mState_init.element_type in (cutlass.BFloat16, cutlass.Float32) + else: + assert mState_init is None, "mState_init must be None if use_initial_state is False" + if cutlass.const_expr(cfg.store_final_state): + assert mState_out is not None and mState_out.element_type in (cutlass.BFloat16, cutlass.Float32) + else: + assert mState_out is None, "mState_out must be None if store_final_state is False" + if cutlass.const_expr(mState_init is not None and mState_out is not None): + assert mState_init.element_type == mState_out.element_type + desc_base_words = tensormap_workspace.iterator.raw_ptr() + arr_words = n_desc * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_k_base = desc_base_words + desc_v_base = desc_base_words + arr_words + desc_gate_base = desc_base_words + cutlass.Int32(2) * arr_words + desc_checkpoint_base = desc_base_words + cutlass.Int32(3) * arr_words + + # Buffers are declaration-ordered and intentionally non-aliased. + SMEM = cutlass.AddressSpace.smem + bars = make_kda_bars(cfg) + sTmem_base = cutlass.Array(cutlass.Int32, 1, space=SMEM, alignment=4) + sSched = cutlass.Array(cutlass.Int32, cfg.sched_stages, space=SMEM, alignment=16) + sK_decay_raw = cutlass.Array(cfg.io_dtype, cfg.k_decay_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_restore_raw = cutlass.Array(cfg.io_dtype, cfg.k_restore_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sQk_raw = cutlass.Array(cfg.io_dtype, cfg.qk_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_raw = cutlass.Array(mK.element_type, cfg.k_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sV_raw = cutlass.Array(mV.element_type, cfg.v_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sGate_raw = cutlass.Array(cutlass.Float32, cfg.gate_cosize, space=SMEM, alignment=1024) + sState_scale_diag_raw = cutlass.Array(cfg.io_dtype, cfg.state_scale_diag_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sK_inv_raw = cutlass.Array(cfg.io_dtype, cfg.k_inv_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sBeta_raw = cutlass.Array(cutlass.Float32, cfg.beta_cosize, space=SMEM, alignment=cfg.buffer_align_bytes) + sCheckpoint_raw = ( + cutlass.Array(cfg.io_dtype, cfg.smem_checkpoint_stages * cfg.d_k * cfg.d_v, space=SMEM, alignment=cfg.buffer_align_bytes) + if cutlass.const_expr(cfg.enable_checkpoints) + else sV_raw + ) + sK_decay = SmemTile( + base=sK_decay_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_decay_stages, + leading_byte_offset=16, + stride_byte_offset=1024, + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_128B, + ) + sK_restore = SmemTile( + base=sK_restore_raw, + elems_per_stage=(cfg.d_k * cfg.b_t), + stages=cfg.smem_decay_stages, + leading_byte_offset=(cfg.b_t * (cfg.d_v // 2) * 2), + stride_byte_offset=(8 * (cfg.d_v // 2) * 2), + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_128B, + ) + sState_scale_diag = SmemTile( + base=sState_scale_diag_raw, + elems_per_stage=((cfg.d_k // 16) * 256), + stages=cfg.smem_state_scale_diag_stages, + leading_byte_offset=16, + stride_byte_offset=(8 * 16 * 2), + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, + ) + sQk = SmemTile( + base=sQk_raw, + elems_per_stage=(2 * cfg.b_t * cfg.b_t), + stages=cfg.smem_qk_stages, + leading_byte_offset=16, + stride_byte_offset=(8 * cfg.b_t * 2), + layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, + ) + + if warp_idx == cfg.tma_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.smem_raw_stages): + bars.mb_k_ready[stage].init() + bars.mb_v_ready[stage].init() + bars.mb_gate_ready[stage].init() + bars.mb_beta_ready[stage].init() + bars.mb_beta_done[stage].init() + bars.mb_k_done[stage].init() + bars.mb_v_done[stage].init() + bars.mb_gate_done[stage].init() + elif warp_idx == cfg.tcgen05_mma_warp_id: + if nvvm.elect_sync(): + bars.mb_state_k_acc_ready.init() + bars.mb_update_acc_ready.init() + bars.mb_state_updated.init() + bars.mb_state_inp_ready.init() + for stage in cutlass.range_constexpr(cfg.smem_state_scale_diag_stages): + bars.mb_state_scale_diag_done[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_decay_stages): + bars.mb_decay_done[stage].init() + bars.mb_decay_super_done[stage].init() + bars.mb_k_restore_done[stage].init() + bars.mb_rhs_ready.init() + bars.mb_update_ready.init() + bars.mb_final_state_stored.init() + elif warp_idx == cfg.super_mma_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.smem_qk_stages): + bars.mb_a_ready[stage].init() + bars.mb_a_inv_done[stage].init() + for stage in cutlass.range_constexpr(cfg.qk_scale_ready_stages): + bars.mb_qk_scale_ready[stage].init() + for stage in cutlass.range_constexpr(cfg.smem_decay_stages): + bars.mb_k_decay_inv_cg0_ready[stage].init() + elif warp_idx == cfg.epilogue_warp_id: + if nvvm.elect_sync(): + for stage in cutlass.range_constexpr(cfg.sched_stages): + bars.mb_sched_ready[stage].init() + bars.mb_sched_done[stage].init() + if cutlass.const_expr(cfg.enable_checkpoints): + for stage in cutlass.range_constexpr(cfg.smem_checkpoint_stages): + bars.mb_checkpoint_tmastg_ready[stage].init() + bars.mb_checkpoint_tmastg_done[stage].init() + bars.mb_state_read_done.init() + diag_zero = cfg.io_dtype(0.0) + for diag_idx in cutlass.range(tidx, cfg.state_scale_diag_cosize, cfg.threads_per_cta, unroll=1): + sState_scale_diag_raw[diag_idx] = diag_zero + nvvm.fence_mbarrier_init() + nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) + if (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) or warp_idx == cfg.tcgen05_mma_warp_id: + if warp_idx == cfg.tcgen05_mma_warp_id: + nvvm.tcgen05_alloc(sTmem_base, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) + if warp_idx == cfg.tcgen05_mma_warp_id: + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) + + # Actual SMEM/TMEM buffers for the BT=16 schedule: + # K/V : 16 x 128 each + # Gate_log2 : 16 x 128 + # Beta : 16 + # K inverse norm : 16, staged once per decay stage + # exp_g_last : 128, CG0-local and staged once per decay stage + # state-scale diag : 8 x 16 x 16 input dtype, zeroed once in the prologue + # K_decay : tcgen05 SW128 operand shared with super-MMA + # K_restore : tcgen05 SW128 N-major final-state operand + # K_inv : 16 x 128 token-major for super-MMA RHS + # A inverse : 16 x 16, plus transposed tcgen05 operands + # State : external/kernel ABI is VK `[DV, DK]`; reference + # math can view it as KV `[DK, DV]` by transposing. + # The TS A-staging path keeps VK in TMEM so State*K + # is `[DV, DK] @ [DK, BT] -> [DV, BT]` with M=128. + + if warp_idx == cfg.tma_warp_id: + tmaldg_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + mSched, + sSched, + lane, + sK_raw, + sV_raw, + sGate_raw, + desc_k_base, + desc_v_base, + desc_gate_base, + bars, + ) + elif warp_idx == cfg.super_mma_warp_id: + super_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sK_inv_raw, + sQk_raw, + sBeta_raw, + sK_decay_raw, + bars, + ) + elif warp_idx == cfg.tcgen05_mma_warp_id: + tcgen05_mma_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + sTmem_base, + sQk, + sK_decay, + sK_restore, + sState_scale_diag, + bars, + ) + elif warp_idx == cfg.epilogue_warp_id: + epilogue_warp( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + sCheckpoint_raw, + desc_checkpoint_base, + checkpoint_every_n_tokens, + bars, + ) + elif warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]: + compute0_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + warp_idx, + mA_log, + mDt_bias, + sK_inv_raw, + sGate_raw, + mBeta, + sBeta_raw, + sK_raw, + sK_decay_raw, + sK_restore_raw, + sState_scale_diag_raw, + bars, + ) + elif warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]: + compute1_warp_group( + cfg, + total_tiles, + bidx, + num_ctas, + cu_seqlens, + mWorkItems, + sSched, + lane, + sTmem_base, + warp_idx, + mState_out, + mState_init, + sBeta_raw, + sV_raw, + sCheckpoint_raw, + checkpoint_every_n_tokens, + bars, + ) + + +@dataclass +class KdaRecomputeCfg: + """Kernel cfg (fixed BT=16 schedule constants; derived TMEM column offsets + and SMEM buffer cosizes are stamped by ``build_cfg``; per-stage sizes are + inlined at the use sites). Passed ``cfg``-first (a ``cutlass.Constexpr``) + into ``host`` / ``kernel`` and every warp body.""" + + io_dtype: Type[cutlass.Numeric] + state_dtype: Type[cutlass.Numeric] + use_initial_state: bool + store_final_state: bool + enable_checkpoints: bool + l2norm: bool + safe_gate: bool + gate_scale_log2: float + beta_sigmoid: bool + k_ratio: int + v_ratio: int + n_heads_out: int + max_active_clusters: int + split_k: bool = False + dyn_sched: bool = False + sched_stages: int = CFG.SMEM_SCHED_STAGES + + compute_group_0_warp_ids: tuple[int, ...] = CFG.COMPUTE_GROUP_0_WARP_IDS + compute_group_1_warp_ids: tuple[int, ...] = CFG.COMPUTE_GROUP_1_WARP_IDS + super_mma_warp_id: int = CFG.SUPER_MMA_WARP_ID + tcgen05_mma_warp_id: int = CFG.TCGEN05_MMA_WARP_ID + tma_warp_id: int = CFG.TMA_WARP_ID + epilogue_warp_id: int = CFG.EPILOGUE_WARP_ID + b_t: int = CFG.B_T + d_k: int = CFG.D_K + d_v: int = CFG.D_V + threads_per_warp: int = CFG.THREADS_PER_WARP + buffer_align_bytes: int = CFG.BUFFER_ALIGN_BYTES + threads_per_cta: int = 0 + cg0_group_count: int = 2 + cg0_warps_per_group: int = 4 + cg0_threads_per_group: int = 0 + cg0_group_sync_barrier_base_id: int = 1 # CG0 group g syncs on nbar id 1 + g + tmem_user_threads: int = 0 + tmem_lifecycle_barrier_id: int = 3 + num_regs_compute_group_0: int = CFG.NUM_REGS_COMPUTE_GROUP_0 + num_regs_compute_group_1: int = CFG.NUM_REGS_COMPUTE_GROUP_1 + num_regs_other: int = CFG.NUM_REGS_OTHER + + # ---- SMEM / TMEM ring stage counts ------------------------------------------- + smem_raw_stages: int = CFG.SMEM_RAW_STAGES + smem_checkpoint_stages: int = 1 + smem_decay_stages: int = CFG.SMEM_DECAY_STAGES + smem_qk_stages: int = CFG.SMEM_QK_STAGES + smem_state_scale_diag_stages: int = CFG.SMEM_STATE_SCALE_DIAG_STAGES + qk_scale_ready_stages: int = CFG.QK_SCALE_READY_STAGES + + # ---- TMEM column offsets (state doubles as the final_state acc) -------------- + tmem_state_acc_offset: int = 0 + tmem_state_inp_offset: int = 0 + tmem_state_k_acc_offset: int = 0 + tmem_update_acc_offset: int = 0 + tmem_rhs_inp_offset: int = 0 + tmem_update_inp_offset: int = 0 + + # ---- SMEM buffer cosizes ----------------------------------------------------- + k_cosize: int = 0 + v_cosize: int = 0 + gate_cosize: int = 0 + beta_cosize: int = 0 + k_inv_cosize: int = 0 + k_decay_cosize: int = 0 + k_restore_cosize: int = 0 + state_scale_diag_cosize: int = 0 + + # TMA transaction bytes per stage + tma_k_bytes: int = 0 + tma_v_bytes: int = 0 + tma_gate_bytes: int = 0 + qk_cosize: int = 0 + + +def build_cfg( + io_dtype: Type[cutlass.Numeric], + state_dtype: Type[cutlass.Numeric], + *, + use_initial_state: bool, + store_final_state: bool, + enable_checkpoints: bool, + l2norm: bool, + safe_gate: bool, + gate_scale_log2: float, + beta_sigmoid: bool, + k_ratio: int, + v_ratio: int, + n_heads_out: int, + max_active_clusters: int, + split_k: bool = False, + dyn_sched: bool = False, +) -> KdaRecomputeCfg: + """Build the per-compile ``KdaRecomputeCfg`` (io_dtype in {Float16, BFloat16}); + fills the derived TMEM column offsets and SMEM buffer cosizes.""" + if io_dtype not in (cutlass.Float16, cutlass.BFloat16): + raise ValueError(f"io_dtype={io_dtype} not supported; only Float16 and BFloat16 are supported") + cfg = KdaRecomputeCfg( + io_dtype=io_dtype, + state_dtype=state_dtype, + use_initial_state=use_initial_state, + store_final_state=store_final_state, + enable_checkpoints=enable_checkpoints, + l2norm=l2norm, + safe_gate=safe_gate, + gate_scale_log2=gate_scale_log2, + beta_sigmoid=beta_sigmoid, + k_ratio=k_ratio, + v_ratio=v_ratio, + n_heads_out=n_heads_out, + max_active_clusters=max_active_clusters, + split_k=split_k, + dyn_sched=dyn_sched, + ) + if enable_checkpoints: + cfg.smem_raw_stages = 6 + cfg.smem_checkpoint_stages = 2 + cfg.threads_per_cta = 16 * cfg.threads_per_warp + cfg.cg0_threads_per_group = cfg.cg0_warps_per_group * cfg.threads_per_warp + cfg.tmem_user_threads = (1 + len(cfg.compute_group_1_warp_ids)) * cfg.threads_per_warp + if cfg.smem_state_scale_diag_stages != cfg.qk_scale_ready_stages: + raise ValueError("diag and qk-scale ready rings must share their rolling stage") + + cfg.tmem_state_inp_offset = cfg.tmem_state_acc_offset + cfg.d_k + cfg.tmem_state_k_acc_offset = cfg.tmem_state_inp_offset + (cfg.d_k // 2) + cfg.tmem_update_acc_offset = cfg.tmem_state_k_acc_offset + cfg.b_t + cfg.tmem_rhs_inp_offset = cfg.tmem_update_acc_offset + cfg.b_t + cfg.tmem_update_inp_offset = cfg.tmem_rhs_inp_offset + (cfg.b_t // 2) + assert (cfg.tmem_update_inp_offset + (cfg.b_t // 2)) <= 512 + + cfg.k_cosize = cfg.smem_raw_stages * cfg.d_k * cfg.b_t + cfg.v_cosize = cfg.smem_raw_stages * cfg.d_v * cfg.b_t + cfg.gate_cosize = cfg.smem_raw_stages * cfg.d_k * cfg.b_t + cfg.beta_cosize = cfg.smem_raw_stages * cfg.b_t + cfg.k_inv_cosize = cfg.smem_decay_stages * cfg.b_t * cfg.d_k + cfg.k_decay_cosize = cfg.smem_decay_stages * cfg.d_k * cfg.b_t + cfg.k_restore_cosize = cfg.smem_decay_stages * cfg.d_k * cfg.b_t + cfg.state_scale_diag_cosize = cfg.smem_state_scale_diag_stages * (cfg.d_k // 16) * 256 + cfg.qk_cosize = cfg.smem_qk_stages * 2 * cfg.b_t * cfg.b_t + cfg.tma_k_bytes = cfg.d_k * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_v_bytes = cfg.d_v * cfg.b_t * (cfg.io_dtype.width // 8) + cfg.tma_gate_bytes = cfg.d_k * cfg.b_t * 4 + return cfg + + +TENSORMAP_DESC_ARRAYS = 4 # per-batch runtime TMA descriptors: K, V, Gate, state_checkpoints +TENSORMAP_STATIC_SLOTS = 0 + + +@cute.kernel +def build_all_descs_kernel( + base_k: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_v: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_gate: cutlass.GridConstant[cuda.tensor_map.TensorMap], + base_checkpoint: cutlass.GridConstant[cuda.tensor_map.TensorMap], + desc_ws: cute.Tensor, + cu_seqlens: cute.Tensor, + k: cute.Tensor, + v: cute.Tensor, + gate: cute.Tensor, + state_checkpoints: cute.Tensor | None, + n_batch: cutlass.Int32, + k_row_stride: cutlass.Int32, + v_row_stride: cutlass.Int32, + gate_row_stride: cutlass.Int32, + checkpoint_row_stride: cutlass.Int32, + checkpoint_every_n: cutlass.Int32, +) -> None: + """Single-launch builder kernel: one warp emits each per-batch TMA + descriptor array.""" + tidx, _, _ = cute.arch.thread_idx() + widx = cutlass.Int32(tidx) // cutlass.Int32(32) + arr_words = n_batch * cutlass.Int32(TENSOR_MAP_QWORDS) + desc_words_k = cute.make_tensor(desc_ws.iterator, cute.make_layout((arr_words,), stride=(1,))) + desc_words_v = cute.make_tensor(desc_ws.iterator + arr_words, cute.make_layout((arr_words,), stride=(1,))) + desc_words_gate = cute.make_tensor(desc_ws.iterator + 2 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + desc_words_checkpoint = cute.make_tensor(desc_ws.iterator + 3 * arr_words, cute.make_layout((arr_words,), stride=(1,))) + + if widx == 0: + if nvvm.elect_sync(): + emit_seq_descs(base_k, desc_words_k, cu_seqlens, k, n_batch, k_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 1: + if nvvm.elect_sync(): + emit_seq_descs(base_v, desc_words_v, cu_seqlens, v, n_batch, v_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if widx == 2: + if nvvm.elect_sync(): + emit_seq_descs(base_gate, desc_words_gate, cu_seqlens, gate, n_batch, gate_row_stride, 2) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + if cutlass.const_expr(state_checkpoints is not None): + if widx == 3: + if nvvm.elect_sync(): + emit_checkpoint_seq_descs( + base_checkpoint, desc_words_checkpoint, cu_seqlens, state_checkpoints, n_batch, checkpoint_row_stride, checkpoint_every_n, 2 + ) + nvvm.fence_proxy_release(nvvm.MemScope.GPU, from_proxy=nvvm.Proxy.GENERIC, to_proxy=nvvm.Proxy.TENSORMAP) + + +@cute.jit +def build_descs( + io_dtype: cutlass.Constexpr, + b_t: cutlass.Constexpr[int], + k: cute.Tensor, + v: cute.Tensor, + gate: cute.Tensor, + state_checkpoints: cute.Tensor | None, + cu_seqlens: cute.Tensor, + tensormap_workspace: cute.Tensor, + checkpoint_every_n: cutlass.Int32, + stream: cuda_driver.CUstream, +): + """Build the per-batch K/V/Gate/checkpoint TMA-descriptor arrays into + ``tensormap_workspace``.""" + h_k = k.shape[1] + h_v = v.shape[1] + ho = gate.shape[1] + batch_size = cu_seqlens.shape[0] - 1 + d_k = k.shape[2] + d_v = v.shape[2] + bpe = io_dtype.width // 8 + tma_granu_elems = 128 // bpe + seqlen = k.shape[0] + + def headed(t, dim, hn): + return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) + + swz = cuda.TensorMapSwizzle.s128b + base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + + base_checkpoint = base_gate + if cutlass.const_expr(state_checkpoints is not None): + checkpoint_view = cute.make_tensor( + state_checkpoints.iterator, + cute.make_layout( + (state_checkpoints.shape[3], state_checkpoints.shape[2], state_checkpoints.shape[0], ho), + stride=(state_checkpoints.stride[3], state_checkpoints.stride[2], state_checkpoints.stride[0], state_checkpoints.stride[1]), + ), + ) + base_checkpoint = cuda.create_tensor_map_tiled_from_view(checkpoint_view, box_dims=(tma_granu_elems, d_k, 1, 1), stride_order=(0, 1, 2, 3), swizzle=swz) + n_warps = 4 if state_checkpoints is not None else 3 + build_all_descs_kernel( + base_k, + base_v, + base_gate, + base_checkpoint, + tensormap_workspace, + cu_seqlens, + k, + v, + gate, + state_checkpoints, + cutlass.Int32(batch_size), + cutlass.Int32(k.stride[0]), + cutlass.Int32(v.stride[0]), + cutlass.Int32(gate.stride[0]), + cutlass.Int32(state_checkpoints.stride[0] if state_checkpoints is not None else 0), + checkpoint_every_n, + ).launch(grid=(1, 1, 1), block=(32 * n_warps, 1, 1), stream=stream) + + +# ---- Torch adapter / host-side compilation --------------------------------------- + + +@lru_cache(maxsize=None) +def get_compiled_cache( + io_dtype_str: str, + state_dtype_str: str, + cu_dtype_str: str, + HO: int, + HK: int, + HV: int, + use_initial_state: bool, + store_final_state: bool, + enable_checkpoints: bool, + l2norm: bool, + safe_gate: bool, + gate_lower_bound: float, + beta_sigmoid: bool, + split_k: bool, + dyn_sched: bool, +): + """Return a mutable dict that lazily stores the compiled kernel.""" + return {} + + +def compile( + io_dtype, + state_dtype, + use_initial_state: bool, + store_final_state: bool, + enable_checkpoints: bool, + l2norm: bool, + safe_gate: bool, + gate_scale_log2: float, + beta_sigmoid: bool, + k_ratio: int, + v_ratio: int, + n_heads_out: int, + split_k: bool = False, + dyn_sched: bool = False, + *, + num_sm: int, + k_cute, + v_cute, + gate_cute, + a_log_cute, + dt_bias_cute, + beta_cute, + cu_seqlens_cute, + state_in_cute, + state_out_cute, + work_items_cute=None, + work_count_cute=None, + sched_ctr_cute=None, + tensormap_ws_cute, + checkpoint_every_n_tokens, + stream, +): + """JIT-compile the chunked KDA recompute kernel for one static config.""" + cfg = build_cfg( + io_dtype, + state_dtype, + use_initial_state=use_initial_state, + store_final_state=store_final_state, + enable_checkpoints=enable_checkpoints, + l2norm=l2norm, + safe_gate=safe_gate, + gate_scale_log2=gate_scale_log2, + beta_sigmoid=beta_sigmoid, + k_ratio=k_ratio, + v_ratio=v_ratio, + n_heads_out=n_heads_out, + max_active_clusters=num_sm, + split_k=split_k, + dyn_sched=dyn_sched, + ) + + return cute.compile( + host, + cfg, + k_cute, + v_cute, + gate_cute, + a_log_cute, + dt_bias_cute, + beta_cute, + cu_seqlens_cute, + state_in_cute, + state_out_cute, + work_items_cute, + work_count_cute, + sched_ctr_cute, + tensormap_ws_cute, + checkpoint_every_n_tokens, + stream, + options="--enable-tvm-ffi --opt-level 2", + ) + + +def chunk_kda_recompute_sm100( + k, + v, + gate, + beta, + cu_seqlens, + initial_state, + output_state, + checkpoint_every_n_tokens: int = 0, + output_state_checkpoints=None, + use_qk_l2norm_in_kernel: bool = False, + safe_gate: bool = False, + gate_lower_bound: float = DEFAULT_GATE_LOWER_BOUND, + a_log=None, + dt_bias=None, + use_beta_sigmoid_in_kernel: bool = False, + work_items=None, + work_count=None, + sched_ctr=None, + *, + tensormap_workspace, + stream, +) -> None: + """Execute the Blackwell BT=16 chunked KDA recompute (state/checkpoints-only) + kernel. + + All tensors must be on the same CUDA device with a stride-1 innermost + dim; outer strides are free (padded / permuted views are read through + the TMA descriptors and dynamic layouts). + + Args: + k: ``(total_tokens, HK, DK)`` float16/bfloat16 + v: ``(total_tokens, HV, DV)`` float16/bfloat16 + gate: ``(total_tokens, HO, DK)`` float32. Natural-log decay unless + ``safe_gate``, which applies the safe-gate transform + ``lower_bound * sigmoid(exp(a_log) * (gate + dt_bias))``. + beta: ``(total_tokens, HO)``. Post-sigmoid float32, or io-dtype + logits when ``use_beta_sigmoid_in_kernel`` + cu_seqlens: ``(num_seqs + 1,)`` int32 + initial_state: ``(num_seqs, HO, DK, DV)`` float32/bfloat16, or None + output_state: ``(num_seqs, HO, DK, DV)`` float32/bfloat16, or None + checkpoint_every_n_tokens: emit a state checkpoint every N tokens (0 = off). + state_checkpoints[j] is the state after ``(j + 1) * N`` tokens, STRICTLY BEFORE + the sequence end — the end-of-sequence state is only + ``output_state``. With ``N == B_T`` this is the per-chunk checkpoint + series the backward pass consumes. + output_state_checkpoints: ``(total_checkpoints, HO, DK, DV)`` io-dtype (KV, V + contiguous); the per-sequence entry offsets + are derived on device from ``cu_seqlens`` ((seqlen-1)//N, + prefix-summed), so there is no cu_checkpoints array + use_qk_l2norm_in_kernel: L2-normalize k rows inside the kernel + safe_gate: interpret ``gate`` through the safe-gate transform + a_log: ``(HO,)`` float32, safe-gate per-head log-amplitude (None = 0) + dt_bias: ``(HO, DK)`` float32, safe-gate channel bias (None = 0) + use_beta_sigmoid_in_kernel: ``beta`` holds logits; sigmoid in-kernel + work_items: ``(max_items, 8)`` int32 split-K work-item table from + ``common/split_k.py``, or None for the one-tile-per-(b,h) + layout. With a table, each item computes chunks + ``[cstart, wend)`` and writes checkpoints only for + ``[wstart, wend)``. + work_count: ``(1,)`` int32 device-side item count (required with + work_items) + sched_ctr: ``(2,)`` int32 device scratch ``[ticket, done]`` enabling + the dynamic (work-stealing) tile scheduler; must be zeroed before + every launch (``build_split_table`` does this when it is passed as + ``sched_ctr``). None keeps the static CTA stride. + """ + HK = k.shape[1] + HV = v.shape[1] + HO = gate.shape[1] + use_initial_state = initial_state is not None + store_final_state = output_state is not None + enable_checkpoints = checkpoint_every_n_tokens > 0 + if enable_checkpoints: + if output_state_checkpoints is None: + raise ValueError("checkpoint_every_n_tokens > 0 requires output_state_checkpoints") + if str(output_state_checkpoints.dtype).split(".")[-1] != str(k.dtype).split(".")[-1]: + raise ValueError( + f"output_state_checkpoints dtype must match the io dtype (fp32 state belongs to output_state): got {output_state_checkpoints.dtype} with io {k.dtype}" + ) + split_k = work_items is not None + dyn_sched = sched_ctr is not None + if split_k: + if work_count is None: + raise ValueError("work_count is required with work_items") + if enable_checkpoints and checkpoint_every_n_tokens != CFG.B_T: + raise ValueError(f"split-K checkpoints require checkpoint_every_n_tokens == {CFG.B_T}, got {checkpoint_every_n_tokens}") + elif work_count is not None: + raise ValueError("work_count must be None without work_items") + + if initial_state is not None: + state_dtype_src = initial_state.dtype + elif output_state is not None: + state_dtype_src = output_state.dtype + else: + state_dtype_src = "float32" + + for name, h in (("HK", HK), ("HV", HV)): + if HO % h != 0: + raise ValueError(f"{name}={h} must divide sab heads {HO}") + k_ratio = HO // HK + v_ratio = HO // HV + gate_scale_log2 = gate_lower_bound * LOG2_E + + if safe_gate and (a_log is None or dt_bias is None): + raise ValueError("safe_gate requires a_log and dt_bias") + if not safe_gate: + a_log = None + dt_bias = None + cu_stream = cuda_driver.CUstream(int(stream)) + + cache = get_compiled_cache( + str(k.dtype), + str(state_dtype_src), + str(cu_seqlens.dtype), + HO, + HK, + HV, + use_initial_state, + store_final_state, + enable_checkpoints, + use_qk_l2norm_in_kernel, + safe_gate, + gate_lower_bound, + use_beta_sigmoid_in_kernel, + split_k, + dyn_sched, + ) + + if "compiled" not in cache: + io_dtype = get_dtype(k.dtype) + state_dtype = get_dtype(state_dtype_src) + k_cute = from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2) + v_cute = from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2) + gate_cute = from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=2) + a_log_cute = from_dlpack(a_log, assumed_align=4) if a_log is not None else None + dt_bias_cute = from_dlpack(dt_bias, assumed_align=16) if dt_bias is not None else None + beta_cute = from_dlpack(beta, assumed_align=4).mark_layout_dynamic(leading_dim=1) + cu_seqlens_cute = from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic() + + state_in_cute = None + if use_initial_state: + state_in_cute = from_dlpack(initial_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) + + state_out_cute = None + if store_final_state: + state_out_cute = from_dlpack(output_state, assumed_align=16).mark_layout_dynamic(leading_dim=3) + + work_items_cute = None + work_count_cute = None + if split_k: + work_items_cute = from_dlpack(work_items, assumed_align=16) + work_items_cute.mark_compact_shape_dynamic(mode=0, stride_order=(0, 1), divisibility=1) + work_count_cute = from_dlpack(work_count, assumed_align=4).mark_layout_dynamic() + + sched_ctr_cute = None + if dyn_sched: + sched_ctr_cute = from_dlpack(sched_ctr, assumed_align=4).mark_layout_dynamic() + + tensormap_ws_cute = from_dlpack(tensormap_workspace, assumed_align=128).mark_layout_dynamic() + + cache["compiled"] = compile( + io_dtype, + state_dtype, + use_initial_state, + store_final_state, + enable_checkpoints, + use_qk_l2norm_in_kernel, + safe_gate, + gate_scale_log2, + use_beta_sigmoid_in_kernel, + k_ratio, + v_ratio, + HO, + split_k, + dyn_sched, + num_sm=multiprocessor_count(current_device_id()), + k_cute=k_cute, + v_cute=v_cute, + gate_cute=gate_cute, + a_log_cute=a_log_cute, + dt_bias_cute=dt_bias_cute, + beta_cute=beta_cute, + cu_seqlens_cute=cu_seqlens_cute, + state_in_cute=state_in_cute, + state_out_cute=state_out_cute, + work_items_cute=work_items_cute, + work_count_cute=work_count_cute, + sched_ctr_cute=sched_ctr_cute, + tensormap_ws_cute=tensormap_ws_cute, + checkpoint_every_n_tokens=checkpoint_every_n_tokens, + stream=cu_stream, + ) + + compiled = cache["compiled"] + state_checkpoints_for_descs = output_state_checkpoints if enable_checkpoints else None + # desc build runs every execute by contract (cu contents are data; + # buffer pointers may change) — capture-safe, single tiny launch + if cache.get("build_descs_has_state_checkpoints") != (state_checkpoints_for_descs is not None): + cache.pop("build_descs", None) + cache["build_descs_has_state_checkpoints"] = state_checkpoints_for_descs is not None + if "build_descs" not in cache: + io_dtype = get_dtype(k.dtype) + k_bd = from_dlpack(k, assumed_align=16).mark_layout_dynamic(leading_dim=2) + v_bd = from_dlpack(v, assumed_align=16).mark_layout_dynamic(leading_dim=2) + gate_bd = from_dlpack(gate, assumed_align=16).mark_layout_dynamic(leading_dim=2) + cu_bd = from_dlpack(cu_seqlens, assumed_align=8).mark_layout_dynamic() + ws_bd = from_dlpack(tensormap_workspace, assumed_align=128).mark_layout_dynamic() + state_checkpoints_bd = None + if state_checkpoints_for_descs is not None: + state_checkpoints_bd = from_dlpack(state_checkpoints_for_descs, assumed_align=16).mark_layout_dynamic(leading_dim=3) + cache["build_descs"] = cute.compile( + build_descs, + io_dtype, + CFG.B_T, + k_bd, + v_bd, + gate_bd, + state_checkpoints_bd, + cu_bd, + ws_bd, + cutlass.Int32(checkpoint_every_n_tokens), + cu_stream, + options="--enable-tvm-ffi", + ) + cache["build_descs"]( + k, + v, + gate, + state_checkpoints_for_descs, + cu_seqlens, + tensormap_workspace, + checkpoint_every_n_tokens, + cu_stream, + ) + compiled( + k, + v, + gate, + a_log, + dt_bias, + beta, + cu_seqlens, + initial_state if use_initial_state else None, + output_state if store_final_state else None, + work_items, + work_count, + sched_ctr, + tensormap_workspace, + checkpoint_every_n_tokens, + cu_stream, + ) diff --git a/python/cudnn/linear_attention/graph_analyzer.py b/python/cudnn/linear_attention/graph_analyzer.py new file mode 100644 index 000000000..b8a359d29 --- /dev/null +++ b/python/cudnn/linear_attention/graph_analyzer.py @@ -0,0 +1,363 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Engine-agnostic linear-attention graph analysis: ``graph.nodes`` -> :class:`LaGraphFacts`. + +One analyzer serves the three LA families (gdn / kda / gdn2 — single dedicated +nodes sharing the THD port vocabulary). :func:`analyze` is the callable each +family names in ``engines/manifest.py``; PLANNING runs it once per frozen +graph and attaches the record, so the family's engines read that same record +back instead of each parsing the node. + +Also hosts the engine-side helpers shared by the LA engines: the +check_support gates over the facts record, the execute-time buffer-layout +gate, and the compiled-plan wrapper. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional + +import cudnn +from cudnn.engines.base import CompiledPlan, resolve_node_buffers +from cudnn.frost import buffers + +BUFFER_NAME_FROM_CUDNN = { + cudnn.data_type.HALF: "float16", + cudnn.data_type.BFLOAT16: "bfloat16", + cudnn.data_type.FLOAT: "float32", + cudnn.data_type.INT32: "int32", + cudnn.data_type.INT64: "int64", +} + + +def to_buffer_dtype(dt) -> str: + """cudnn.data_type -> the buffer-level dtype name (``buffers.DTYPES`` vocabulary).""" + return BUFFER_NAME_FROM_CUDNN[dt] + + +# node type -> (op family, is_bwd) +LA_NODE_OPS = { + cudnn.NodeType.GDN: ("GDN", False), + cudnn.NodeType.GDN_BWD: ("GDN", True), + cudnn.NodeType.KDA: ("KDA", False), + cudnn.NodeType.KDA_BWD: ("KDA", True), + cudnn.NodeType.GDN2: ("GDN2", False), + cudnn.NodeType.GDN2_BWD: ("GDN2", True), +} + + +@dataclass(frozen=True) +class LaGraphFacts: + """What a single-LA graph asks for. Pure description — no support + judgment: head-dim limits, dtype sets, and feature coverage are per-engine + knowledge, matched in each engine's ``check_support``. ``invalid`` is the + one exception: a graph-consistency error (malformed regardless of which + kernel would run — a missing required port, ``d_initial_state`` without + ``initial_state``, safe-gate inputs without the attribute); when set, + every engine is ineligible.""" + + invalid: Optional[str] = None + + op: str = "" # "GDN" | "KDA" | "GDN2" + is_bwd: bool = False + + # geometry (THD; zeros when the port ranks are not declared) + thd_layout: bool = True # Q/K/V are rank-3 [total_T, heads, dim] + h_q: int = 0 + h_k: int = 0 + h_v: int = 0 + h_o: int = 0 + d_qk: int = 0 + d_v: int = 0 + gates_at_ho: bool = True # Gate/Beta(/W) carry HO = max(h_q, h_v) heads + + # dtypes (cudnn.data_type vocabulary; None = unset/inferred) + io_dtype: Any = None # Q's dtype + uniform_io: bool = True # Q/K/V dtypes agree + g_dtype: Any = None + beta_dtype: Any = None + w_dtype: Any = None + cu_dtype: Any = None + a_log_dtype: Any = None + dt_bias_dtype: Any = None + do_dtype: Any = None + state_checkpoints_dtype: Any = None + state_checkpoints_out_dtype: Any = None # fwd checkpoint OUTPUT port dtype + d_final_state_dtype: Any = None + state_dtype: Any = None + final_state_dtype: Any = None + state_pair_match: bool = True + o_dtype: Any = None + dq_dtype: Any = None + dk_dtype: Any = None + dv_dtype: Any = None + dg_dtype: Any = None + dbeta_dtype: Any = None + dw_dtype: Any = None + d_initial_state_dtype: Any = None + + # ports present / requested + has_initial_state: bool = False + wants_d_initial_state: bool = False + wants_state_checkpoints: bool = False # fwd checkpoint-series output + + # attributes + scale: Optional[float] = None + use_qk_l2norm: bool = False + safe_gate: bool = False + use_beta_sigmoid: bool = False + checkpoint_every_n_tokens: int = 0 + + +def analyze(graph: "cudnn.pygraph") -> Optional[LaGraphFacts]: + """Facts for a single-LA graph, or None if the graph is anything else. + + Pure: attaching and caching is the graph's job (create_execution_plans + -> _attach_facts). This is the callable the LA families name in their + manifest ``analyzer`` entries.""" + nodes = list(graph.nodes) + if len(nodes) != 1: + return None + node = nodes[0] + kind = LA_NODE_OPS.get(node.node_type) + if kind is None: + return None + op, is_bwd = kind + ins, outs, params = node.inputs, node.outputs, node.params + + required_in = ["q", "k", "v", "g", "beta", "cu_seqlens"] + if op == "GDN2": + required_in.append("w") + if is_bwd: + required_in.append("dO") + required_out = (["dQ", "dK", "dV", "dG", "dBeta"] + (["dW"] if op == "GDN2" else [])) if is_bwd else ["O"] + + safe_gate = bool(params.get("safe_gate", False)) + ckpt = int(params.get("checkpoint_every_n_tokens", 0) or 0) + invalid = None + missing_in = [p for p in required_in if p not in ins] + missing_out = [p for p in required_out if p not in outs] + if missing_in: + invalid = f"{node.node_type.name} node '{node.name}' is missing input(s) {missing_in}" + elif missing_out: + invalid = f"{node.node_type.name} node '{node.name}' is missing output(s) {missing_out}" + elif "d_initial_state" in outs and "initial_state" not in ins: + invalid = "d_initial_state requires initial_state" + elif safe_gate and ("a_log" not in ins or "dt_bias" not in ins): + invalid = "safe_gate requires a_log and dt_bias inputs" + elif not safe_gate and ("a_log" in ins or "dt_bias" in ins): + invalid = "a_log/dt_bias require safe_gate=True" + elif params.get("gate_lower_bound") is not None and not safe_gate: + invalid = "gate_lower_bound requires safe_gate=True" + elif ckpt < 0: + invalid = "checkpoint_every_n_tokens must be non-negative" + elif not is_bwd and ckpt > 0 and "state_checkpoints" not in outs: + invalid = "checkpoint_every_n_tokens > 0 requires the state_checkpoints output" + elif not is_bwd and ckpt == 0 and "state_checkpoints" in outs: + invalid = "state_checkpoints output requires checkpoint_every_n_tokens > 0" + if invalid is not None: + return LaGraphFacts(invalid=invalid, op=op, is_bwd=is_bwd) + + in_dt = {name: t.get_data_type() for name, t in ins.items()} + out_dt = {name: t.get_data_type() for name, t in outs.items()} + q, k, v = ins["q"], ins["k"], ins["v"] + + thd_layout = all(t.dim and len(t.dim) == 3 for t in (q, k, v)) + if thd_layout: + _, h_q, d_qk = (int(d) for d in q.dim) + h_k, h_v, d_v = int(k.dim[1]), int(v.dim[1]), int(v.dim[2]) + else: + h_q = h_k = h_v = d_qk = d_v = 0 + h_o = max(h_q, h_v) + gates_at_ho = all(t is None or not t.dim or int(t.dim[1]) == h_o for t in (ins["g"], ins["beta"], ins.get("w"))) + io_dtypes = {in_dt["q"], in_dt["k"], in_dt["v"]} - {None} + state_dtypes = {in_dt.get("initial_state"), out_dt.get("final_state")} - {None} + scale = params.get("scale") + + return LaGraphFacts( + op=op, + is_bwd=is_bwd, + thd_layout=thd_layout, + h_q=h_q, + h_k=h_k, + h_v=h_v, + h_o=h_o, + d_qk=d_qk, + d_v=d_v, + gates_at_ho=gates_at_ho, + io_dtype=in_dt["q"], + uniform_io=len(io_dtypes) <= 1, + g_dtype=in_dt["g"], + beta_dtype=in_dt["beta"], + w_dtype=in_dt.get("w"), + cu_dtype=in_dt["cu_seqlens"], + a_log_dtype=in_dt.get("a_log"), + dt_bias_dtype=in_dt.get("dt_bias"), + do_dtype=in_dt.get("dO"), + state_checkpoints_dtype=in_dt.get("state_checkpoints"), + state_checkpoints_out_dtype=out_dt.get("state_checkpoints"), + d_final_state_dtype=in_dt.get("d_final_state"), + state_dtype=in_dt.get("initial_state"), + final_state_dtype=out_dt.get("final_state"), + state_pair_match=len(state_dtypes) <= 1, + o_dtype=out_dt.get("O"), + dq_dtype=out_dt.get("dQ"), + dk_dtype=out_dt.get("dK"), + dv_dtype=out_dt.get("dV"), + dg_dtype=out_dt.get("dG"), + dbeta_dtype=out_dt.get("dBeta"), + dw_dtype=out_dt.get("dW"), + d_initial_state_dtype=out_dt.get("d_initial_state"), + has_initial_state="initial_state" in ins, + wants_d_initial_state="d_initial_state" in outs, + wants_state_checkpoints="state_checkpoints" in outs, + scale=float(scale) if scale is not None else None, + use_qk_l2norm=bool(params.get("use_qk_l2norm", False)), + safe_gate=safe_gate, + use_beta_sigmoid=bool(params.get("use_beta_sigmoid", False)), + checkpoint_every_n_tokens=ckpt, + ) + + +# --------------------------------------------------------------------------- +# Engine-side helpers shared by the LA engines +# --------------------------------------------------------------------------- + + +def require(engine: str, port: str, got, want) -> None: + """check_support dtype gate over a facts field: unset passes (the kernel + validates the buffer), anything else must be the kernel-native dtype.""" + if got is None: + return + wanted = want if isinstance(want, tuple) else (want,) + if got not in wanted: + names = "/".join(w.name for w in wanted) + raise NotImplementedError(f"{engine}: '{port}' must be {names} (the kernel-native dtype; no staging), got {got}") + + +def frost_la_gate(engine: str, facts, op: str) -> None: + """The FROST LA engines' shared check_support core: the analyzer record, + the device/DSL environment, and the gates common to all three kernels.""" + if facts is None or facts.op != op: + raise NotImplementedError(f"{engine} supports exactly one {op}/{op}_BWD node") + if facts.invalid: + raise NotImplementedError(f"{engine}: {facts.invalid}") + sm = buffers.current_sm() + if sm is None or not (100 <= sm <= 103): + raise NotImplementedError(f"{engine} requires SM100-SM103 (found {sm})") + installed, version = buffers.cutedsl_state() + if not installed: + raise NotImplementedError(f"{engine} requires the cutedsl extra (nvidia-cutlass-dsl), which is not installed") + if buffers.cutedsl_too_old(version): + want = ".".join(str(v) for v in buffers.CUTEDSL_MIN_VERSION) + raise NotImplementedError(f"{engine} requires nvidia-cutlass-dsl >= {want}; found {version[1]}") + if not facts.uniform_io: + raise NotImplementedError(f"{engine}: q/k/v dtypes must match") + require(engine, "q/k/v", facts.io_dtype, (cudnn.data_type.BFLOAT16, cudnn.data_type.HALF)) + if not facts.thd_layout: + raise NotImplementedError(f"{engine}: q/k/v must be THD [total_T, heads, dim]") + if facts.d_qk != 128 or facts.d_v != 128: + raise NotImplementedError(f"{engine}: head dims must be 128 (the recurrent state is 128x128), got K={facts.d_qk} V={facts.d_v}") + if facts.h_k not in (facts.h_q, facts.h_v): + raise NotImplementedError(f"{engine}: k heads ({facts.h_k}) must match q's ({facts.h_q}) or v's ({facts.h_v}; canonical GQA shares grouped k/v heads)") + if facts.h_v != facts.h_q and max(facts.h_q, facts.h_v) % min(facts.h_q, facts.h_v) != 0: + raise NotImplementedError(f"{engine}: q heads ({facts.h_q}) and v heads ({facts.h_v}) must be equal or one a multiple of the other") + require(engine, "g", facts.g_dtype, cudnn.data_type.FLOAT) + require(engine, "cu_seqlens", facts.cu_dtype, (cudnn.data_type.INT32, cudnn.data_type.INT64)) + + +class FrostLaPlan(CompiledPlan): + """A compiled LA executor, called with the graph's resolved node buffers.""" + + def __init__(self, compiled): + self.compiled = compiled + + def get_workspace_size(self) -> int: + return self.compiled.workspace_bytes() + + def execute(self, graph, uid_to_data, ctx) -> None: + node_buffers = resolve_node_buffers(graph, uid_to_data) + self.compiled(node_buffers, workspace=ctx.workspace, stream=ctx.stream) + + +def expect_table(node, align) -> dict: + """Build-time ``{port: (dims, dtype_name, align_bytes)}`` for + :func:`check_layouts`: bound buffers must match the node's frozen + geometry exactly (one graph per shape), and base pointers must satisfy + the kernel entry's ``assumed_align`` claim. ``align`` maps port name -> + bytes (family table read from the entry's from_dlpack calls; absent + ports default to 16).""" + table = {} + for ports in (node.inputs, node.outputs): + for name, t in ports.items(): + if t is None: + continue + dims = tuple(int(d) for d in t.dim) if t.dim else None + table[name] = (dims, BUFFER_NAME_FROM_CUDNN.get(t.get_data_type()), align.get(name, 16)) + return table + + +def check_layouts_compact(plan_name: str, expect, nb) -> None: + """Execute-time gate for the cuTile backend: every bound buffer must be + CONTIGUOUS (the kernels stage rank-merged views and whole-buffer zero + fills), and must match the node's build-time dims/dtype and base + alignment per ``expect`` (see :func:`expect_table`).""" + for ports in (nb.inputs, nb.outputs): + for name, b in ports.items(): + if b is None: + continue + ptr, shape, strides, dtype, _dev = buffers.probe(b) + exp = expect.get(name) if expect else None + if exp is not None: + dims, dtype_name, align = exp + if dims is not None and tuple(shape) != dims: + raise ValueError(f"{plan_name}: buffer for {name!r} must match the graph's build-time dims {dims}; got {tuple(shape)}") + if dtype_name is not None and dtype != dtype_name: + raise ValueError(f"{plan_name}: buffer for {name!r} must be {dtype_name} (the node's declared dtype); got {dtype}") + if align and ptr % align != 0: + raise ValueError(f"{plan_name}: buffer for {name!r} base pointer must be {align}-byte aligned; got 0x{ptr:x}") + if not buffers.is_contiguous(shape, strides): + raise ValueError( + f"{plan_name}: buffer for {name!r} must be contiguous (the cuTile backend stages rank-merged views); got shape {shape} strides {strides}" + ) + + +def check_layouts(plan_name: str, expect=None, compact=(), **bufs) -> None: + """Layout gate for stride-plumbed kernels (pass-through, no staging): + innermost dim stride-1, non-broadcast, non-overlapping; outer strides on + size>1 dims must be 16-byte multiples (TMA global-stride rule, and the + vectorized gate-scan / word-pair store alignment). ``expect`` (from + :func:`expect_table`) additionally pins each buffer to the node's + build-time dims/dtype and the entry's assumed base alignment; ``compact`` + names ports whose kernels compiled static-compact layouts (full + contiguity required, not just dense).""" + for name, b in bufs.items(): + if b is None: + continue + ptr, shape, strides, dtype, _dev = buffers.probe(b) + if not buffers.dense_layout_ok(shape, strides): + raise ValueError( + f"{plan_name}: buffer for {name!r} must have a stride-1 innermost dim and " + f"non-broadcast, non-overlapping strides (padded outer strides allowed); got shape {shape} strides {strides}" + ) + exp = expect.get(name) if expect else None + if exp is not None: + dims, dtype_name, align = exp + if dims is not None and tuple(shape) != dims: + raise ValueError(f"{plan_name}: buffer for {name!r} must match the graph's build-time dims {dims}; got {tuple(shape)}") + if dtype_name is not None and dtype != dtype_name: + raise ValueError(f"{plan_name}: buffer for {name!r} must be {dtype_name} (the node's declared dtype); got {dtype}") + if align and ptr % align != 0: + raise ValueError(f"{plan_name}: buffer for {name!r} base pointer must be {align}-byte aligned (kernel assumed_align); got 0x{ptr:x}") + if name in compact and not buffers.is_contiguous(shape, strides): + raise ValueError(f"{plan_name}: buffer for {name!r} must be contiguous (compiled static-compact); got shape {shape} strides {strides}") + if strides is None or buffers.is_contiguous(shape, strides): + continue + bpe = buffers.DTYPES[dtype][1] // 8 + for sz, st in zip(shape[:-1], strides[:-1]): + if sz != 1 and (st * bpe) % 16 != 0: + raise ValueError( + f"{plan_name}: buffer for {name!r} padded outer strides must be 16-byte multiples (TMA global-stride rule); got strides {strides} at {bpe} B/elem" + ) diff --git a/python/cudnn/linear_attention/ops/gdn.py b/python/cudnn/linear_attention/ops/gdn.py index a27186e4e..7e0d5d5cb 100644 --- a/python/cudnn/linear_attention/ops/gdn.py +++ b/python/cudnn/linear_attention/ops/gdn.py @@ -10,130 +10,209 @@ S_t = alpha_t (I - beta_t k_t^T k_t) S_{t-1} + beta_t k_t^T v_t, o_t = q_t S_t, -where ``alpha_t`` is a scalar per-token decay in ``(0, 1]`` and ``beta_t`` -is a scalar per-token write strength. +where ``S_t`` is the recurrent state, ``alpha_t`` a scalar per-token decay +in ``(0, 1]``, and ``beta_t`` a scalar per-token write strength. Layout follows the graph-API GDN node: THD — token-packed ``[total_tokens, heads, dim]`` tensors plus ``cu_seqlens`` sequence boundaries. -The op is a thin adapter over the graph API (the SDPA-op pattern): forward +The op is a thin adapter over the graph API: forward and backward execute cached single-node ``GDN`` / ``GDN_BWD`` pygraphs. -Engine selection happens at graph planning time over the registered python +Engine selection happens at graph planning time over the manifest's python engines: ``GdnFrostEngine`` (default on SM100/SM103) with ``GdnCuTileEngine`` as the fallback everywhere else. Registered through ``torch.library.custom_op`` so it composes with autograd, ``torch.compile``, and DDP. -The backward graph recomputes the forward's cheap intermediates (cumulative -gate, intra-chunk WY factor) — the ``GDN_BWD`` node contract keeps them off -the autograd wire. +Graph caching ensures cuDNN graphs are built once per unique configuration +and reused across calls. """ -from __future__ import annotations - import math from typing import Dict, Optional, Tuple import torch - import cudnn -from cudnn.linear_attention import engine_utils -_OP_NAMESPACE = "cudnn" -_OP_NAME = "gated_delta_net" +# --------------------------------------------------------------------------- +# Module-level state +# --------------------------------------------------------------------------- + -_TORCH_TO_CUDNN_DTYPE = { +_TORCH_DTYPE_TO_CUDNN = { torch.float16: cudnn.data_type.HALF, torch.bfloat16: cudnn.data_type.BFLOAT16, torch.float32: cudnn.data_type.FLOAT, + torch.int32: cudnn.data_type.INT32, + torch.int64: cudnn.data_type.INT64, } # one graph per static configuration (shapes, dtypes, scale, flags, device) -_fwd_graph_cache: Dict[tuple, tuple] = {} -_bwd_graph_cache: Dict[tuple, tuple] = {} -_ws_cache: Dict[int, "torch.Tensor"] = {} +_fprop_cache: Dict[tuple, tuple] = {} +_bprop_cache: Dict[tuple, tuple] = {} +_cudnn_handles: Dict[int, int] = {} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- def _graph_workspace(graph, device): - """Caller-side workspace for a compiled graph (the explicit-workspace - convention: query the plan's size, allocate, pass to execute).""" + """Caller-side workspace for a compiled graph (grow-only, held on the + graph object itself — same lifetime by construction, no id()-keyed + side table).""" if not graph._is_built: - # mirror execute()'s auto-build: plan via the router first (a bare - # build() would lower GDN to the backend, which has no lowering) + # plan first: a bare build() would lower GDN to the backend, + # which has no lowering if not graph._planning_done: graph.create_execution_plans() - engine_utils.apply_pin(graph) if graph.selected_engine is None: graph.build() else: graph.build_plans() size = graph.get_workspace_size() - ws = _ws_cache.get(id(graph)) + ws = getattr(graph, "_la_ops_workspace", None) if ws is None or ws.numel() < size or ws.device != device: ws = torch.empty(max(size, 1), dtype=torch.uint8, device=device) - _ws_cache[id(graph)] = ws + graph._la_ops_workspace = ws return ws -_handle_cache: Dict[int, int] = {} - - -def _graph_handle(device): - """Per-device cuDNN handle carrying the caller's current stream - (classic ``set_stream`` semantics).""" +def _get_handle(device): + """Per-device cuDNN handle carrying the caller's current stream.""" idx = device.index if device.index is not None else torch.cuda.current_device() - handle = _handle_cache.get(idx) + handle = _cudnn_handles.get(idx) if handle is None: with torch.cuda.device(idx): handle = cudnn.create_handle() - _handle_cache[idx] = handle + _cudnn_handles[idx] = handle cudnn.set_stream(handle=handle, stream=torch.cuda.current_stream(device).cuda_stream) return handle -def _cudnn_dtype(dtype: Optional[torch.dtype]): - return _TORCH_TO_CUDNN_DTYPE[dtype] if dtype is not None else None - - -# --------------------------------------------------------------------------- -# Forward -# --------------------------------------------------------------------------- +def _torch_dtype_to_cudnn(dtype: torch.dtype): + """Map a PyTorch dtype to a cuDNN data_type enum.""" + return _TORCH_DTYPE_TO_CUDNN[dtype] def _check_dtype(name, t, want) -> None: if t.dtype != want: - raise TypeError(f"{_OP_NAME}: {name} must be {want} (kernel-native; callers convert), got {t.dtype}") + raise TypeError(f"gated_delta_net: {name} must be {want} (kernel-native; callers convert), got {t.dtype}") + + +def _make_fprop_cache_key( + total, N, H, HK, HV, K, V, io_dtype, k_dtype, v_dtype, k_shape, v_shape, cu_dtype, scale, output_final_state, use_qk_l2norm, has_initial_state, ckpt, device +): + return ( + "fprop", + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + k_shape, + v_shape, + cu_dtype, + float(scale), + bool(output_final_state), + bool(use_qk_l2norm), + bool(has_initial_state), + ckpt, + device, + ) + + +def _make_bprop_cache_key( + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + do_dtype, + k_shape, + v_shape, + cu_dtype, + has_initial_state, + has_d_final_state, + scale, + use_qk_l2norm, + device, +): + return ( + "bprop", + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + do_dtype, + k_shape, + v_shape, + cu_dtype, + bool(has_initial_state), + bool(has_d_final_state), + float(scale), + bool(use_qk_l2norm), + device, + ) + +# --------------------------------------------------------------------------- +# Forward graph builder +# --------------------------------------------------------------------------- -def _build_fwd_graph(total, N, H, HV, K, V, io_dtype, g_dtype, beta_dtype, state_dtype, scale, output_final_state, use_qk_l2norm): + +def _build_fprop_graph(total, N, H, HK, HV, K, V, io_dtype, g_dtype, beta_dtype, state_dtype, cu_dtype, scale, output_final_state, use_qk_l2norm, ckpt): graph = cudnn.pygraph() HO = max(H, HV) q_t = graph.tensor([total, H, K], data_type=io_dtype, name="q") - k_t = graph.tensor([total, H, K], data_type=io_dtype, name="k") + k_t = graph.tensor([total, HK, K], data_type=io_dtype, name="k") v_t = graph.tensor([total, HV, V], data_type=io_dtype, name="v") g_t = graph.tensor([total, HO], data_type=g_dtype, name="g") beta_t = graph.tensor([total, HO], data_type=beta_dtype, name="beta") - cu_t = graph.tensor([N + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") - s0_t = None + cu_t = graph.tensor([N + 1], data_type=cu_dtype, name="cu_seqlens") + state0_t = None if state_dtype is not None: - s0_t = graph.tensor([N, HO, K, V], data_type=state_dtype, name="initial_state") - O_t, fs_t, _h_t = graph.gdn( + state0_t = graph.tensor([N, HO, K, V], data_type=state_dtype, name="initial_state") + O_t, fs_t, state_checkpoints_t = graph.gdn( q=q_t, k=k_t, v=v_t, g=g_t, beta=beta_t, cu_seqlens=cu_t, - initial_state=s0_t, + initial_state=state0_t, scale=scale, output_final_state=output_final_state, use_qk_l2norm=use_qk_l2norm, + checkpoint_every_n_tokens=ckpt, name="gdn", ) - return graph, dict(q=q_t, k=k_t, v=v_t, g=g_t, beta=beta_t, cu=cu_t, s0=s0_t, O=O_t, fs=fs_t) + return graph, dict(q=q_t, k=k_t, v=v_t, g=g_t, beta=beta_t, cu=cu_t, state0=state0_t, O=O_t, fs=fs_t, state_checkpoints=state_checkpoints_t) -@torch.library.custom_op(f"{_OP_NAMESPACE}::{_OP_NAME}_fwd", mutates_args=()) +# --------------------------------------------------------------------------- +# Forward custom op +# --------------------------------------------------------------------------- + + +@torch.library.custom_op("cudnn::gated_delta_net_fwd", mutates_args=()) def _gdn_fwd( q: torch.Tensor, k: torch.Tensor, @@ -145,117 +224,151 @@ def _gdn_fwd( initial_state: Optional[torch.Tensor] = None, output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, -) -> Tuple[torch.Tensor, torch.Tensor]: - """GDN forward via a cached single-node GDN pygraph (THD layout). + checkpoint_every_n_tokens: int = 0, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """GDN forward (internal): a cached single-node GDN pygraph, THD layout. - Returns ``(o, final_state)``; ``final_state`` is a zero-size tensor when - ``output_final_state`` is ``False``. + Returns ``(o, final_state, state_checkpoints)``; ``final_state`` / ``state_checkpoints`` are zero-size + tensors when ``output_final_state`` is ``False`` / + ``checkpoint_every_n_tokens`` is ``0``. """ total, H, K = q.shape - if k.shape[1] != H: - raise ValueError(f"k must carry the same head count as q ({H}), got {k.shape[1]}") + HK = k.shape[1] HV, V = v.shape[1], v.shape[2] + if HK not in (H, HV): + raise ValueError(f"k head count ({HK}) must match q's ({H}) or v's ({HV}); canonical GQA shares grouped k/v heads") N = cu_seqlens.shape[0] - 1 device = q.device - cu = cu_seqlens.to(torch.int32).contiguous() + if cu_seqlens.dtype not in (torch.int32, torch.int64): + raise ValueError(f"gated_delta_net: cu_seqlens must be int32 or int64; got {cu_seqlens.dtype}") + cu = cu_seqlens _check_dtype("g", g, torch.float32) _check_dtype("beta", beta, torch.float32) if initial_state is not None: _check_dtype("initial_state", initial_state, torch.float32) if initial_state.shape[0] != N: raise ValueError(f"initial_state must carry one state per sequence: got {initial_state.shape[0]} for {N} sequences") - g32 = g.contiguous() - beta32 = beta.contiguous() - s0 = initial_state.contiguous() if initial_state is not None else None - - key = ( + for _name, _t in (("k", k), ("v", v), ("g", g), ("beta", beta), ("cu_seqlens", cu_seqlens), ("initial_state", initial_state)): + if _t is not None and _t.device != device: + raise ValueError(f"gated_delta_net: {_name} must be on q's device ({device}); got {_t.device}") + g32 = g + beta32 = beta + state0 = initial_state if initial_state is not None else None + ckpt = int(checkpoint_every_n_tokens) + + cache_key = _make_fprop_cache_key( total, N, H, + HK, HV, K, V, q.dtype, - float(scale), - bool(output_final_state), - bool(use_qk_l2norm_in_kernel), - bool(s0 is not None), + k.dtype, + v.dtype, + tuple(k.shape), + tuple(v.shape), + cu_seqlens.dtype, + scale, + output_final_state, + use_qk_l2norm_in_kernel, + state0 is not None, + ckpt, device, ) - if key not in _fwd_graph_cache: - _fwd_graph_cache[key] = _build_fwd_graph( + if cache_key not in _fprop_cache: + _fprop_cache[cache_key] = _build_fprop_graph( total, N, H, + HK, HV, K, V, - _cudnn_dtype(q.dtype), + _torch_dtype_to_cudnn(q.dtype), cudnn.data_type.FLOAT, cudnn.data_type.FLOAT, - cudnn.data_type.FLOAT if s0 is not None else None, + cudnn.data_type.FLOAT if state0 is not None else None, + _torch_dtype_to_cudnn(cu_seqlens.dtype), float(scale), bool(output_final_state), bool(use_qk_l2norm_in_kernel), + ckpt, ) - graph, t = _fwd_graph_cache[key] + graph, t = _fprop_cache[cache_key] HO = max(H, HV) o = torch.empty(total, HO, V, dtype=q.dtype, device=device) variant_pack = { - t["q"]: q.contiguous(), - t["k"]: k.contiguous(), - t["v"]: v.contiguous(), + t["q"]: q, + t["k"]: k, + t["v"]: v, t["g"]: g32, t["beta"]: beta32, t["cu"]: cu, t["O"]: o, } - if s0 is not None: - variant_pack[t["s0"]] = s0 + if state0 is not None: + variant_pack[t["state0"]] = state0 final_state = torch.empty(0, dtype=torch.float32, device=device) if output_final_state: final_state = torch.empty(N, HO, K, V, dtype=torch.float32, device=device) variant_pack[t["fs"]] = final_state - graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_graph_handle(device)) - return o, final_state + state_checkpoints = torch.empty(0, dtype=q.dtype, device=device) + if ckpt > 0: + # shape-derived upper bound (sum floor((len_i-1)/ckpt) <= (total-N)//ckpt); + # the exact count is data-dependent (cu contents) — no sync on the hot path + total_checkpoints = max((total - N) // ckpt, 1) + state_checkpoints = torch.empty(total_checkpoints, HO, K, V, dtype=q.dtype, device=device) + variant_pack[t["state_checkpoints"]] = state_checkpoints + graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_get_handle(device)) + return o, final_state, state_checkpoints @_gdn_fwd.register_fake -def _gdn_fwd_fake(q, k, v, g, beta, cu_seqlens, scale, initial_state=None, output_final_state=False, use_qk_l2norm_in_kernel=False): +def _gdn_fwd_fake( + q, k, v, g, beta, cu_seqlens, scale, initial_state=None, output_final_state=False, use_qk_l2norm_in_kernel=False, checkpoint_every_n_tokens=0 +): total, H, K = q.shape - if k.shape[1] != H: - raise ValueError(f"k must carry the same head count as q ({H}), got {k.shape[1]}") + HK = k.shape[1] HV, V = v.shape[1], v.shape[2] + if HK not in (H, HV): + raise ValueError(f"k head count ({HK}) must match q's ({H}) or v's ({HV}); canonical GQA shares grouped k/v heads") HO = max(H, HV) N = cu_seqlens.shape[0] - 1 o = q.new_empty(total, HO, V) final = q.new_empty((N, HO, K, V) if output_final_state else (0,), dtype=torch.float32) - return o, final + if checkpoint_every_n_tokens > 0: + total_checkpoints = max((total - N) // int(checkpoint_every_n_tokens), 1) + state_checkpoints = q.new_empty(total_checkpoints, HO, K, V) + else: + state_checkpoints = q.new_empty(0) + return o, final, state_checkpoints # --------------------------------------------------------------------------- -# Backward +# Backward graph builder # --------------------------------------------------------------------------- -def _build_bwd_graph(total, N, H, HV, K, V, io_dtype, g_dtype, beta_dtype, state_dtype, dht_dtype, scale, use_qk_l2norm): +def _build_bprop_graph(total, N, H, HK, HV, K, V, io_dtype, g_dtype, beta_dtype, state_dtype, dstate_in_dtype, cu_dtype, scale, use_qk_l2norm): graph = cudnn.pygraph() HO = max(H, HV) q_t = graph.tensor([total, H, K], data_type=io_dtype, name="q") - k_t = graph.tensor([total, H, K], data_type=io_dtype, name="k") + k_t = graph.tensor([total, HK, K], data_type=io_dtype, name="k") v_t = graph.tensor([total, HV, V], data_type=io_dtype, name="v") g_t = graph.tensor([total, HO], data_type=g_dtype, name="g") beta_t = graph.tensor([total, HO], data_type=beta_dtype, name="beta") - cu_t = graph.tensor([N + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") + cu_t = graph.tensor([N + 1], data_type=cu_dtype, name="cu_seqlens") dO_t = graph.tensor([total, HO, V], data_type=io_dtype, name="dO") - s0_t = None + state0_t = None if state_dtype is not None: - s0_t = graph.tensor([N, HO, K, V], data_type=state_dtype, name="initial_state") + state0_t = graph.tensor([N, HO, K, V], data_type=state_dtype, name="initial_state") dfs_t = None - if dht_dtype is not None: - dfs_t = graph.tensor([N, HO, K, V], data_type=dht_dtype, name="d_final_state") - dQ_t, dK_t, dV_t, dG_t, dBeta_t, dS0_t = graph.gdn_bwd( + if dstate_in_dtype is not None: + dfs_t = graph.tensor([N, HO, K, V], data_type=dstate_in_dtype, name="d_final_state") + dQ_t, dK_t, dV_t, dG_t, dBeta_t, dstate0_t = graph.gdn_bwd( q=q_t, k=k_t, v=v_t, @@ -263,7 +376,7 @@ def _build_bwd_graph(total, N, H, HV, K, V, io_dtype, g_dtype, beta_dtype, state beta=beta_t, cu_seqlens=cu_t, dO=dO_t, - initial_state=s0_t, + initial_state=state0_t, d_final_state=dfs_t, scale=scale, use_qk_l2norm=use_qk_l2norm, @@ -277,18 +390,23 @@ def _build_bwd_graph(total, N, H, HV, K, V, io_dtype, g_dtype, beta_dtype, state beta=beta_t, cu=cu_t, dO=dO_t, - s0=s0_t, + state0=state0_t, dfs=dfs_t, dQ=dQ_t, dK=dK_t, dV=dV_t, dG=dG_t, dBeta=dBeta_t, - dS0=dS0_t, + dstate0=dstate0_t, ) -@torch.library.custom_op(f"{_OP_NAMESPACE}::{_OP_NAME}_bwd", mutates_args=()) +# --------------------------------------------------------------------------- +# Backward custom op +# --------------------------------------------------------------------------- + + +@torch.library.custom_op("cudnn::gated_delta_net_bwd", mutates_args=()) def _gdn_bwd( dO: torch.Tensor, q: torch.Tensor, @@ -302,105 +420,127 @@ def _gdn_bwd( d_final_state: Optional[torch.Tensor] = None, use_qk_l2norm_in_kernel: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """GDN backward via a cached single-node GDN_BWD pygraph (THD layout). + """GDN backward (internal): a cached single-node GDN_BWD pygraph, THD layout. Returns ``(dq, dk, dv, dg, dbeta, d_initial_state)``; ``d_initial_state`` is a zero-size tensor when ``initial_state`` is ``None``. """ total, H, K = q.shape - if k.shape[1] != H: - raise ValueError(f"k must carry the same head count as q ({H}), got {k.shape[1]}") + # autograd materializes reduction grads as broadcast (stride-0) + # views; densify ONLY those (dense callers pass through untouched) + if 0 in dO.stride(): + dO = dO.contiguous() + if d_final_state is not None and 0 in d_final_state.stride(): + d_final_state = d_final_state.contiguous() + HK = k.shape[1] HV, V = v.shape[1], v.shape[2] + if HK not in (H, HV): + raise ValueError(f"k head count ({HK}) must match q's ({H}) or v's ({HV}); canonical GQA shares grouped k/v heads") N = cu_seqlens.shape[0] - 1 device = q.device - cu = cu_seqlens.to(torch.int32).contiguous() + if cu_seqlens.dtype not in (torch.int32, torch.int64): + raise ValueError(f"gated_delta_net: cu_seqlens must be int32 or int64; got {cu_seqlens.dtype}") + cu = cu_seqlens _check_dtype("g", g, torch.float32) _check_dtype("beta", beta, torch.float32) if initial_state is not None: _check_dtype("initial_state", initial_state, torch.float32) if initial_state.shape[0] != N: raise ValueError(f"initial_state must carry one state per sequence: got {initial_state.shape[0]} for {N} sequences") - g32 = g.contiguous() - beta32 = beta.contiguous() - s0 = initial_state.contiguous() if initial_state is not None else None + for _name, _t in (("k", k), ("v", v), ("g", g), ("beta", beta), ("cu_seqlens", cu_seqlens), ("dO", dO), ("d_final_state", d_final_state)): + if _t is not None and _t.device != device: + raise ValueError(f"gated_delta_net: {_name} must be on q's device ({device}); got {_t.device}") + g32 = g + beta32 = beta + state0 = initial_state if initial_state is not None else None if d_final_state is not None: _check_dtype("d_final_state", d_final_state, torch.float32) - dht = d_final_state.contiguous() if d_final_state is not None else None + dstate_in = d_final_state if d_final_state is not None else None - key = ( + cache_key = _make_bprop_cache_key( total, N, H, + HK, HV, K, V, q.dtype, - bool(s0 is not None), - bool(dht is not None), - float(scale), - bool(use_qk_l2norm_in_kernel), + k.dtype, + v.dtype, + dO.dtype, + tuple(k.shape), + tuple(v.shape), + cu_seqlens.dtype, + state0 is not None, + dstate_in is not None, + scale, + use_qk_l2norm_in_kernel, device, ) - if key not in _bwd_graph_cache: - _bwd_graph_cache[key] = _build_bwd_graph( + if cache_key not in _bprop_cache: + _bprop_cache[cache_key] = _build_bprop_graph( total, N, H, + HK, HV, K, V, - _cudnn_dtype(q.dtype), + _torch_dtype_to_cudnn(q.dtype), cudnn.data_type.FLOAT, cudnn.data_type.FLOAT, - cudnn.data_type.FLOAT if s0 is not None else None, - cudnn.data_type.FLOAT if dht is not None else None, + cudnn.data_type.FLOAT if state0 is not None else None, + cudnn.data_type.FLOAT if dstate_in is not None else None, + _torch_dtype_to_cudnn(cu_seqlens.dtype), float(scale), bool(use_qk_l2norm_in_kernel), ) - graph, t = _bwd_graph_cache[key] + graph, t = _bprop_cache[cache_key] HO = max(H, HV) dq = torch.empty(total, H, K, dtype=q.dtype, device=device) - dk = torch.empty(total, H, K, dtype=k.dtype, device=device) - dv = torch.empty(total, HV, V, dtype=v.dtype, device=device) + dk = torch.empty(total, HK, K, dtype=q.dtype, device=device) + dv = torch.empty(total, HV, V, dtype=q.dtype, device=device) dg32 = torch.empty(total, HO, dtype=torch.float32, device=device) dbeta32 = torch.empty(total, HO, dtype=torch.float32, device=device) variant_pack = { - t["q"]: q.contiguous(), - t["k"]: k.contiguous(), - t["v"]: v.contiguous(), + t["q"]: q, + t["k"]: k, + t["v"]: v, t["g"]: g32, t["beta"]: beta32, t["cu"]: cu, - t["dO"]: dO.contiguous(), + t["dO"]: dO, t["dQ"]: dq, t["dK"]: dk, t["dV"]: dv, t["dG"]: dg32, t["dBeta"]: dbeta32, } - dh032 = None - if s0 is not None: - variant_pack[t["s0"]] = s0 - dh032 = torch.empty_like(s0) - variant_pack[t["dS0"]] = dh032 - if dht is not None: - variant_pack[t["dfs"]] = dht - graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_graph_handle(device)) - dh0 = dh032 if dh032 is not None else torch.empty(0, dtype=torch.float32, device=device) - return dq, dk, dv, dg32, dbeta32, dh0 + dstate0 = None + if state0 is not None: + variant_pack[t["state0"]] = state0 + dstate0 = torch.empty_like(state0) + variant_pack[t["dstate0"]] = dstate0 + if dstate_in is not None: + variant_pack[t["dfs"]] = dstate_in + graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_get_handle(device)) + if dstate0 is None: + dstate0 = torch.empty(0, dtype=torch.float32, device=device) + return dq, dk, dv, dg32, dbeta32, dstate0 @_gdn_bwd.register_fake def _gdn_bwd_fake(dO, q, k, v, g, beta, cu_seqlens, scale, initial_state=None, d_final_state=None, use_qk_l2norm_in_kernel=False): - dh0 = torch.empty_like(initial_state) if initial_state is not None else q.new_empty(0, dtype=torch.float32) + dstate0 = torch.empty_like(initial_state) if initial_state is not None else q.new_empty(0, dtype=torch.float32) return ( torch.empty_like(q), torch.empty_like(k), torch.empty_like(v), torch.empty_like(g), torch.empty_like(beta), - dh0, + dstate0, ) @@ -410,21 +550,22 @@ def _gdn_bwd_fake(dO, q, k, v, g, beta, cu_seqlens, scale, initial_state=None, d def _gdn_setup_context(ctx, inputs, output): - q, k, v, g, beta, cu_seqlens, scale, initial_state, output_final_state, use_qk_l2norm_in_kernel = inputs + q, k, v, g, beta, cu_seqlens, scale, initial_state, output_final_state, use_qk_l2norm_in_kernel, checkpoint_every_n_tokens = inputs # save_for_backward cannot hold None; keep initial_state as an attribute. ctx.save_for_backward(q, k, v, g, beta, cu_seqlens) ctx.initial_state = initial_state ctx.scale = scale ctx.use_qk_l2norm_in_kernel = use_qk_l2norm_in_kernel + ctx.mark_non_differentiable(output[2]) -def _gdn_backward(ctx, dO, dFinal): +def _gdn_backward(ctx, dO, dFinal, _dstate_checkpoints): q, k, v, g, beta, cu_seqlens = ctx.saved_tensors initial_state = ctx.initial_state - dht = dFinal if (dFinal is not None and dFinal.numel() > 0) else None - dq, dk, dv, dg, dbeta, dh0 = torch.ops.cudnn.gated_delta_net_bwd( - dO.contiguous(), + dstate_in = dFinal if (dFinal is not None and dFinal.numel() > 0) else None + dq, dk, dv, dg, dbeta, dstate0 = torch.ops.cudnn.gated_delta_net_bwd( + dO, q, k, v, @@ -433,11 +574,11 @@ def _gdn_backward(ctx, dO, dFinal): cu_seqlens, ctx.scale, initial_state=initial_state, - d_final_state=dht, + d_final_state=dstate_in, use_qk_l2norm_in_kernel=ctx.use_qk_l2norm_in_kernel, ) # q, k, v, g, beta, cu_seqlens, scale, initial_state, output_final_state, - # use_qk_l2norm_in_kernel + # use_qk_l2norm_in_kernel, checkpoint_every_n_tokens return ( dq, dk, @@ -446,14 +587,15 @@ def _gdn_backward(ctx, dO, dFinal): dbeta, None, None, - dh0 if initial_state is not None else None, + dstate0 if initial_state is not None else None, + None, None, None, ) torch.library.register_autograd( - f"{_OP_NAMESPACE}::{_OP_NAME}_fwd", + "cudnn::gated_delta_net_fwd", _gdn_backward, setup_context=_gdn_setup_context, ) @@ -475,14 +617,16 @@ def gated_delta_net( initial_state: Optional[torch.Tensor] = None, output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, + checkpoint_every_n_tokens: int = 0, ): """Gated DeltaNet (GDN) linear attention. THD layout (matches the graph-API GDN node): - q, k: ``[total_tokens, H, K]``; v: ``[total_tokens, HV, V]`` + q: ``[total_tokens, H, K]``; k: ``[total_tokens, HK, K]`` (HK = H, or + HK = HV for canonical GQA: grouped K/V heads shared across query groups); v: ``[total_tokens, HV, V]`` g, beta: ``[total_tokens, HO]`` with ``HO = max(H, HV)``; - cu_seqlens: ``[N+1]`` int32; o and the states live at HO heads + cu_seqlens: ``[N+1]`` int32; O and the states live at HO heads (initial_state / final_state: ``[N, HO, K, V]``) A dense batch of N equal-length sequences is expressed as @@ -500,18 +644,24 @@ def gated_delta_net( initial_state: optional recurrent state (otherwise zero). output_final_state: if ``True``, also return the per-sequence state after the last token. - use_qk_l2norm_in_kernel: if ``True``, L2-normalize the q/k rows inside + use_qk_l2norm_in_kernel: if ``True``, L2-normalize the Q/K rows inside the kernel. Engines that cannot honor it decline the graph. + checkpoint_every_n_tokens: if ``> 0``, also return the per-chunk + recurrent state series ``state_checkpoints`` (``[total_checkpoints, HO, K, V]`` io dtype, + one entry per N tokens strictly before each sequence end; the + FROST engine requires the kernel chunk size, 64). The series is + a non-differentiable dump. Returns: - ``(o, final_state)`` with ``o`` shaped like ``v``. ``final_state`` is - empty unless ``output_final_state=True``. + ``(o, final_state)`` with ``o`` shaped like ``v``, or + ``(o, final_state, state_checkpoints)`` when ``checkpoint_every_n_tokens > 0``. + ``final_state`` is empty unless ``output_final_state=True``. """ if q.dim() != 3: raise ValueError("expected THD [total_tokens, heads, dim] tensors") if scale is None: scale = 1.0 / math.sqrt(q.shape[-1]) - return torch.ops.cudnn.gated_delta_net_fwd( + o, final_state, state_checkpoints = torch.ops.cudnn.gated_delta_net_fwd( q, k, v, @@ -522,4 +672,8 @@ def gated_delta_net( initial_state=initial_state, output_final_state=bool(output_final_state), use_qk_l2norm_in_kernel=bool(use_qk_l2norm_in_kernel), + checkpoint_every_n_tokens=int(checkpoint_every_n_tokens), ) + if checkpoint_every_n_tokens > 0: + return o, final_state, state_checkpoints + return o, final_state diff --git a/python/cudnn/linear_attention/ops/gdn2.py b/python/cudnn/linear_attention/ops/gdn2.py index 57ca61dd3..e258bb00c 100644 --- a/python/cudnn/linear_attention/ops/gdn2.py +++ b/python/cudnn/linear_attention/ops/gdn2.py @@ -9,106 +9,199 @@ S_t = (I - k_t (beta_t . k_t)^T) Diag(exp(g_t)) S_{t-1} + k_t (w_t . v_t)^T, o_t = scale * q_t S_t, -with per-key-channel log decay ``g_t in R^K``, per-key erase gate +where ``S_t`` is the recurrent state, with per-key-channel log decay +``g_t in R^K``, per-key erase gate ``beta_t in R^K`` (applied inside the erase read of the decayed state), and a per-value write gate ``w_t in R^V`` (applied to the written value). Layout follows the graph-API GDN-2 node: THD — token-packed -``[total_tokens, heads, dim]`` q/k/v, ``g``/``beta`` ``[total_tokens, HO, K]``, +``[total_tokens, heads, dim]`` Q/K/V, ``g``/``beta`` ``[total_tokens, HO, K]``, ``w`` ``[total_tokens, HO, V]``, plus ``cu_seqlens`` boundaries. -The op is a thin adapter over the graph API (the SDPA-op pattern): forward -executes a cached single-node ``GDN2`` pygraph. Engine selection happens at -graph planning time over the registered python engines — ``Gdn2FrostEngine`` -is the only GDN-2 engine (SM100/SM103). The op is **forward only** (the -FROST GDN-2 backward kernel is a stub), so no autograd is registered; -differentiating through it raises. Registered through -``torch.library.custom_op`` so it composes with ``torch.compile``. -""" +The op is a thin adapter over the graph API: forward +and backward execute cached single-node ``GDN2`` / ``GDN2_BWD`` pygraphs. +Engine selection happens at graph planning time over the registered python +engines — ``Gdn2FrostEngine`` is the only GDN-2 engine (SM100/SM103). +Registered through ``torch.library.custom_op`` so it composes with autograd, +``torch.compile``, and DDP. -from __future__ import annotations +Graph caching ensures cuDNN graphs are built once per unique configuration +and reused across calls. +""" import math from typing import Dict, Optional, Tuple import torch - import cudnn -from cudnn.linear_attention import engine_utils -_OP_NAMESPACE = "cudnn" -_OP_NAME = "gated_delta_net_v2" +# --------------------------------------------------------------------------- +# Module-level state +# --------------------------------------------------------------------------- + -_TORCH_TO_CUDNN_DTYPE = { +_TORCH_DTYPE_TO_CUDNN = { torch.float16: cudnn.data_type.HALF, torch.bfloat16: cudnn.data_type.BFLOAT16, torch.float32: cudnn.data_type.FLOAT, + torch.int32: cudnn.data_type.INT32, + torch.int64: cudnn.data_type.INT64, } # one graph per static configuration (shapes, dtypes, scale, flags, device) -_fwd_graph_cache: Dict[tuple, tuple] = {} -_ws_cache: Dict[int, "torch.Tensor"] = {} +_fprop_cache: Dict[tuple, tuple] = {} +_bprop_cache: Dict[tuple, tuple] = {} +_cudnn_handles: Dict[int, int] = {} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- def _graph_workspace(graph, device): - """Caller-side workspace for a compiled graph (the explicit-workspace - convention: query the plan's size, allocate, pass to execute).""" + """Caller-side workspace for a compiled graph.""" if not graph._is_built: - # mirror execute()'s auto-build: plan via the router first (a bare - # build() would lower GDN2 to the backend, which has no lowering) + # mirror execute()'s auto-build: plan first (a bare build() would + # lower GDN2 to the backend, which has no lowering) if not graph._planning_done: graph.create_execution_plans() - engine_utils.apply_pin(graph) if graph.selected_engine is None: graph.build() else: graph.build_plans() size = graph.get_workspace_size() - ws = _ws_cache.get(id(graph)) + ws = getattr(graph, "_la_ops_workspace", None) if ws is None or ws.numel() < size or ws.device != device: ws = torch.empty(max(size, 1), dtype=torch.uint8, device=device) - _ws_cache[id(graph)] = ws + graph._la_ops_workspace = ws return ws -_handle_cache: Dict[int, int] = {} - - -def _graph_handle(device): - """Per-device cuDNN handle carrying the caller's current stream - (classic ``set_stream`` semantics).""" +def _get_handle(device): + """Per-device cuDNN handle carrying the caller's current stream.""" idx = device.index if device.index is not None else torch.cuda.current_device() - handle = _handle_cache.get(idx) + handle = _cudnn_handles.get(idx) if handle is None: with torch.cuda.device(idx): handle = cudnn.create_handle() - _handle_cache[idx] = handle + _cudnn_handles[idx] = handle cudnn.set_stream(handle=handle, stream=torch.cuda.current_stream(device).cuda_stream) return handle -def _cudnn_dtype(dtype: Optional[torch.dtype]): - return _TORCH_TO_CUDNN_DTYPE[dtype] if dtype is not None else None +def _torch_dtype_to_cudnn(dtype: torch.dtype): + """Map a PyTorch dtype to a cuDNN data_type enum.""" + return _TORCH_DTYPE_TO_CUDNN[dtype] def _check_dtype(name, t, want) -> None: if t.dtype != want: - raise TypeError(f"{_OP_NAME}: {name} must be {want} (kernel-native; callers convert), got {t.dtype}") + raise TypeError(f"gated_delta_net_v2: {name} must be {want} (kernel-native; callers convert), got {t.dtype}") + + +def _make_fprop_cache_key( + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + k_shape, + v_shape, + cu_dtype, + scale, + output_final_state, + use_qk_l2norm, + safe_gate, + gate_lower_bound, + has_initial_state, + ckpt, + device, +): + return ( + "fprop", + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + k_shape, + v_shape, + cu_dtype, + float(scale), + bool(output_final_state), + bool(use_qk_l2norm), + bool(safe_gate), + float(gate_lower_bound) if gate_lower_bound is not None else None, + bool(has_initial_state), + ckpt, + device, + ) + + +def _make_bprop_cache_key( + total, N, H, HK, HV, K, V, io_dtype, k_dtype, v_dtype, do_dtype, k_shape, v_shape, cu_dtype, state_dtype, dstate_in_dtype, scale, use_qk_l2norm, device +): + return ( + "bprop", + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + do_dtype, + k_shape, + v_shape, + cu_dtype, + state_dtype, + dstate_in_dtype, + float(scale), + bool(use_qk_l2norm), + device, + ) + + +# --------------------------------------------------------------------------- +# Forward graph builder +# --------------------------------------------------------------------------- -def _build_fwd_graph(total, N, H, HV, K, V, io_dtype, g_dtype, gate_dtype, state_dtype, scale, output_final_state, use_qk_l2norm): +def _build_fprop_graph( + total, N, H, HK, HV, K, V, io_dtype, g_dtype, gate_dtype, state_dtype, cu_dtype, scale, output_final_state, use_qk_l2norm, safe_gate, gate_lower_bound, ckpt +): graph = cudnn.pygraph() + HO = max(H, HV) q_t = graph.tensor([total, H, K], data_type=io_dtype, name="q") - k_t = graph.tensor([total, H, K], data_type=io_dtype, name="k") + k_t = graph.tensor([total, HK, K], data_type=io_dtype, name="k") v_t = graph.tensor([total, HV, V], data_type=io_dtype, name="v") - g_t = graph.tensor([total, HV, K], data_type=g_dtype, name="g") - beta_t = graph.tensor([total, HV, K], data_type=gate_dtype, name="beta") - w_t = graph.tensor([total, HV, V], data_type=gate_dtype, name="w") - cu_t = graph.tensor([N + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") - s0_t = None + g_t = graph.tensor([total, HO, K], data_type=g_dtype, name="g") + beta_t = graph.tensor([total, HO, K], data_type=gate_dtype, name="beta") + w_t = graph.tensor([total, HO, V], data_type=gate_dtype, name="w") + cu_t = graph.tensor([N + 1], data_type=cu_dtype, name="cu_seqlens") + state0_t = None if state_dtype is not None: - s0_t = graph.tensor([N, HV, K, V], data_type=state_dtype, name="initial_state") - O_t, fs_t, _h_t = graph.gdn2( + state0_t = graph.tensor([N, HO, K, V], data_type=state_dtype, name="initial_state") + a_log_t = None + dt_bias_t = None + if safe_gate: + a_log_t = graph.tensor([HO], data_type=cudnn.data_type.FLOAT, name="a_log") + dt_bias_t = graph.tensor([HO, K], data_type=cudnn.data_type.FLOAT, name="dt_bias") + O_t, fs_t, state_checkpoints_t = graph.gdn2( q=q_t, k=k_t, v=v_t, @@ -116,16 +209,40 @@ def _build_fwd_graph(total, N, H, HV, K, V, io_dtype, g_dtype, gate_dtype, state beta=beta_t, w=w_t, cu_seqlens=cu_t, - initial_state=s0_t, + initial_state=state0_t, + a_log=a_log_t, + dt_bias=dt_bias_t, scale=scale, output_final_state=output_final_state, use_qk_l2norm=use_qk_l2norm, + safe_gate=safe_gate, + gate_lower_bound=gate_lower_bound, + checkpoint_every_n_tokens=ckpt, name="gdn2", ) - return graph, dict(q=q_t, k=k_t, v=v_t, g=g_t, beta=beta_t, w=w_t, cu=cu_t, s0=s0_t, O=O_t, fs=fs_t) + return graph, dict( + q=q_t, + k=k_t, + v=v_t, + g=g_t, + beta=beta_t, + w=w_t, + cu=cu_t, + state0=state0_t, + a_log=a_log_t, + dt_bias=dt_bias_t, + O=O_t, + fs=fs_t, + state_checkpoints=state_checkpoints_t, + ) + + +# --------------------------------------------------------------------------- +# Forward custom op +# --------------------------------------------------------------------------- -@torch.library.custom_op(f"{_OP_NAMESPACE}::{_OP_NAME}_fwd", mutates_args=()) +@torch.library.custom_op("cudnn::gated_delta_net_v2_fwd", mutates_args=()) def _gdn2_fwd( q: torch.Tensor, k: torch.Tensor, @@ -138,92 +255,459 @@ def _gdn2_fwd( initial_state: Optional[torch.Tensor] = None, output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, -) -> Tuple[torch.Tensor, torch.Tensor]: - """GDN-2 forward via a cached single-node GDN2 pygraph (THD layout). - - Returns ``(o, final_state)``; ``final_state`` is a zero-size tensor when - ``output_final_state`` is ``False``. + safe_gate: bool = False, + gate_lower_bound: Optional[float] = None, + a_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + checkpoint_every_n_tokens: int = 0, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """GDN-2 forward (internal): a cached single-node GDN2 pygraph, THD layout. + + Returns ``(o, final_state, state_checkpoints)``; ``final_state`` / ``state_checkpoints`` are zero-size + tensors when ``output_final_state`` is ``False`` / + ``checkpoint_every_n_tokens`` is ``0``. """ total, H, K = q.shape - if k.shape[1] != H: - raise ValueError(f"k must carry the same head count as q ({H}), got {k.shape[1]}") + HK = k.shape[1] HV, V = v.shape[1], v.shape[2] + if HK not in (H, HV): + raise ValueError(f"k head count ({HK}) must match q's ({H}) or v's ({HV}); canonical GQA shares grouped k/v heads") + HO = max(H, HV) N = cu_seqlens.shape[0] - 1 device = q.device - cu = cu_seqlens.to(torch.int32).contiguous() + if cu_seqlens.dtype not in (torch.int32, torch.int64): + raise ValueError(f"gated_delta_net_2: cu_seqlens must be int32 or int64; got {cu_seqlens.dtype}") + cu = cu_seqlens _check_dtype("g", g, torch.float32) _check_dtype("beta", beta, q.dtype) _check_dtype("w", w, q.dtype) + if safe_gate: + if a_log is None or dt_bias is None: + raise ValueError("gated_delta_net_v2: safe_gate requires a_log and dt_bias") + _check_dtype("a_log", a_log, torch.float32) + _check_dtype("dt_bias", dt_bias, torch.float32) + elif a_log is not None or dt_bias is not None: + raise ValueError("gated_delta_net_v2: a_log/dt_bias require safe_gate=True") if initial_state is not None: _check_dtype("initial_state", initial_state, torch.float32) if initial_state.shape[0] != N: raise ValueError(f"initial_state must carry one state per sequence: got {initial_state.shape[0]} for {N} sequences") - g32 = g.contiguous() - beta_io = beta.contiguous() - w_io = w.contiguous() - s0 = initial_state.contiguous() if initial_state is not None else None - - key = ( + g32 = g + beta_io = beta + w_io = w + for _name, _t in ( + ("k", k), + ("v", v), + ("g", g), + ("beta", beta), + ("w", w), + ("cu_seqlens", cu_seqlens), + ("initial_state", initial_state), + ("a_log", a_log), + ("dt_bias", dt_bias), + ): + if _t is not None and _t.device != device: + raise ValueError(f"gated_delta_net_2: {_name} must be on q's device ({device}); got {_t.device}") + state0 = initial_state if initial_state is not None else None + ckpt = int(checkpoint_every_n_tokens) + + cache_key = _make_fprop_cache_key( total, N, H, + HK, HV, K, V, q.dtype, - bool(s0 is not None), - float(scale), - bool(output_final_state), - bool(use_qk_l2norm_in_kernel), + k.dtype, + v.dtype, + tuple(k.shape), + tuple(v.shape), + cu_seqlens.dtype, + scale, + output_final_state, + use_qk_l2norm_in_kernel, + safe_gate, + gate_lower_bound, + state0 is not None, + ckpt, device, ) - if key not in _fwd_graph_cache: - _fwd_graph_cache[key] = _build_fwd_graph( + if cache_key not in _fprop_cache: + _fprop_cache[cache_key] = _build_fprop_graph( total, N, H, + HK, HV, K, V, - _cudnn_dtype(q.dtype), + _torch_dtype_to_cudnn(q.dtype), cudnn.data_type.FLOAT, - _cudnn_dtype(q.dtype), - cudnn.data_type.FLOAT if s0 is not None else None, + _torch_dtype_to_cudnn(q.dtype), + cudnn.data_type.FLOAT if state0 is not None else None, + _torch_dtype_to_cudnn(cu_seqlens.dtype), float(scale), bool(output_final_state), bool(use_qk_l2norm_in_kernel), + bool(safe_gate), + float(gate_lower_bound) if gate_lower_bound is not None else None, + ckpt, ) - graph, t = _fwd_graph_cache[key] + graph, t = _fprop_cache[cache_key] - o = torch.empty(total, HV, V, dtype=q.dtype, device=device) + o = torch.empty(total, HO, V, dtype=q.dtype, device=device) variant_pack = { - t["q"]: q.contiguous(), - t["k"]: k.contiguous(), - t["v"]: v.contiguous(), + t["q"]: q, + t["k"]: k, + t["v"]: v, t["g"]: g32, t["beta"]: beta_io, t["w"]: w_io, t["cu"]: cu, t["O"]: o, } - if s0 is not None: - variant_pack[t["s0"]] = s0 + if state0 is not None: + variant_pack[t["state0"]] = state0 + if safe_gate: + variant_pack[t["a_log"]] = a_log + variant_pack[t["dt_bias"]] = dt_bias final_state = torch.empty(0, dtype=torch.float32, device=device) if output_final_state: - final_state = torch.empty(N, HV, K, V, dtype=torch.float32, device=device) + final_state = torch.empty(N, HO, K, V, dtype=torch.float32, device=device) variant_pack[t["fs"]] = final_state - graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_graph_handle(device)) - return o, final_state + state_checkpoints = torch.empty(0, dtype=q.dtype, device=device) + if ckpt > 0: + # shape-derived upper bound; exact count is data-dependent (cu contents) + total_checkpoints = max((total - N) // ckpt, 1) + state_checkpoints = torch.empty(total_checkpoints, HO, K, V, dtype=q.dtype, device=device) + variant_pack[t["state_checkpoints"]] = state_checkpoints + graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_get_handle(device)) + return o, final_state, state_checkpoints @_gdn2_fwd.register_fake -def _gdn2_fwd_fake(q, k, v, g, beta, w, cu_seqlens, scale, initial_state=None, output_final_state=False, use_qk_l2norm_in_kernel=False): - total, _H, K = q.shape +def _gdn2_fwd_fake( + q, + k, + v, + g, + beta, + w, + cu_seqlens, + scale, + initial_state=None, + output_final_state=False, + use_qk_l2norm_in_kernel=False, + safe_gate=False, + gate_lower_bound=None, + a_log=None, + dt_bias=None, + checkpoint_every_n_tokens=0, +): + total, H, K = q.shape HV, V = v.shape[1], v.shape[2] + HO = max(H, HV) N = cu_seqlens.shape[0] - 1 - o = q.new_empty(total, HV, V) - final = q.new_empty((N, HV, K, V) if output_final_state else (0,), dtype=torch.float32) - return o, final + o = q.new_empty(total, HO, V) + final = q.new_empty((N, HO, K, V) if output_final_state else (0,), dtype=torch.float32) + if checkpoint_every_n_tokens > 0: + total_checkpoints = max((total - N) // int(checkpoint_every_n_tokens), 1) + state_checkpoints = q.new_empty(total_checkpoints, HO, K, V) + else: + state_checkpoints = q.new_empty(0) + return o, final, state_checkpoints + + +# --------------------------------------------------------------------------- +# Backward graph builder +# --------------------------------------------------------------------------- + + +def _build_bprop_graph(total, N, H, HK, HV, K, V, io_dtype, g_dtype, gate_dtype, state_dtype, dstate_in_dtype, cu_dtype, scale, use_qk_l2norm): + graph = cudnn.pygraph() + HO = max(H, HV) + q_t = graph.tensor([total, H, K], data_type=io_dtype, name="q") + k_t = graph.tensor([total, HK, K], data_type=io_dtype, name="k") + v_t = graph.tensor([total, HV, V], data_type=io_dtype, name="v") + g_t = graph.tensor([total, HO, K], data_type=g_dtype, name="g") + beta_t = graph.tensor([total, HO, K], data_type=gate_dtype, name="beta") + w_t = graph.tensor([total, HO, V], data_type=gate_dtype, name="w") + cu_t = graph.tensor([N + 1], data_type=cu_dtype, name="cu_seqlens") + dO_t = graph.tensor([total, HO, V], data_type=io_dtype, name="dO") + state0_t = None + if state_dtype is not None: + state0_t = graph.tensor([N, HO, K, V], data_type=state_dtype, name="initial_state") + dfs_t = None + if dstate_in_dtype is not None: + dfs_t = graph.tensor([N, HO, K, V], data_type=dstate_in_dtype, name="d_final_state") + dQ_t, dK_t, dV_t, dG_t, dBeta_t, dW_t, dstate0_t = graph.gdn2_bwd( + q=q_t, + k=k_t, + v=v_t, + g=g_t, + beta=beta_t, + w=w_t, + cu_seqlens=cu_t, + dO=dO_t, + initial_state=state0_t, + d_final_state=dfs_t, + scale=scale, + use_qk_l2norm=use_qk_l2norm, + name="gdn2_bwd", + ) + return graph, dict( + q=q_t, + k=k_t, + v=v_t, + g=g_t, + beta=beta_t, + w=w_t, + cu=cu_t, + dO=dO_t, + state0=state0_t, + dfs=dfs_t, + dQ=dQ_t, + dK=dK_t, + dV=dV_t, + dG=dG_t, + dBeta=dBeta_t, + dW=dW_t, + dstate0=dstate0_t, + ) + + +# --------------------------------------------------------------------------- +# Backward custom op +# --------------------------------------------------------------------------- + + +@torch.library.custom_op("cudnn::gated_delta_net_v2_bwd", mutates_args=()) +def _gdn2_bwd( + dO: torch.Tensor, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + w: torch.Tensor, + cu_seqlens: torch.Tensor, + scale: float, + initial_state: Optional[torch.Tensor] = None, + d_final_state: Optional[torch.Tensor] = None, + use_qk_l2norm_in_kernel: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """GDN-2 backward (internal): a cached single-node GDN2_BWD pygraph, THD layout. + + Returns ``(dq, dk, dv, dg, dbeta, dw, d_initial_state)``; + ``d_initial_state`` is a zero-size tensor when ``initial_state`` is + ``None``. + """ + total, H, K = q.shape + # autograd materializes reduction grads as broadcast (stride-0) + # views; densify ONLY those (dense callers pass through untouched) + if 0 in dO.stride(): + dO = dO.contiguous() + if d_final_state is not None and 0 in d_final_state.stride(): + d_final_state = d_final_state.contiguous() + HK = k.shape[1] + HV, V = v.shape[1], v.shape[2] + if HK not in (H, HV): + raise ValueError(f"k head count ({HK}) must match q's ({H}) or v's ({HV}); canonical GQA shares grouped k/v heads") + HO = max(H, HV) + N = cu_seqlens.shape[0] - 1 + device = q.device + if cu_seqlens.dtype not in (torch.int32, torch.int64): + raise ValueError(f"gated_delta_net_2: cu_seqlens must be int32 or int64; got {cu_seqlens.dtype}") + cu = cu_seqlens + _check_dtype("g", g, torch.float32) + _check_dtype("beta", beta, q.dtype) + _check_dtype("w", w, q.dtype) + if initial_state is not None: + _check_dtype("initial_state", initial_state, torch.float32) + if initial_state.shape[0] != N: + raise ValueError(f"initial_state must carry one state per sequence: got {initial_state.shape[0]} for {N} sequences") + if d_final_state is not None: + _check_dtype("d_final_state", d_final_state, torch.float32) + for _name, _t in (("k", k), ("v", v), ("g", g), ("beta", beta), ("w", w), ("cu_seqlens", cu_seqlens), ("dO", dO), ("d_final_state", d_final_state)): + if _t is not None and _t.device != device: + raise ValueError(f"gated_delta_net_2: {_name} must be on q's device ({device}); got {_t.device}") + state0 = initial_state if initial_state is not None else None + dstate_in = d_final_state if d_final_state is not None else None + + cache_key = _make_bprop_cache_key( + total, + N, + H, + HK, + HV, + K, + V, + q.dtype, + k.dtype, + v.dtype, + dO.dtype, + tuple(k.shape), + tuple(v.shape), + cu_seqlens.dtype, + state0.dtype if state0 is not None else None, + dstate_in.dtype if dstate_in is not None else None, + scale, + use_qk_l2norm_in_kernel, + device, + ) + if cache_key not in _bprop_cache: + _bprop_cache[cache_key] = _build_bprop_graph( + total, + N, + H, + HK, + HV, + K, + V, + _torch_dtype_to_cudnn(q.dtype), + cudnn.data_type.FLOAT, + _torch_dtype_to_cudnn(q.dtype), + _torch_dtype_to_cudnn(state0.dtype) if state0 is not None else None, + _torch_dtype_to_cudnn(dstate_in.dtype) if dstate_in is not None else None, + _torch_dtype_to_cudnn(cu_seqlens.dtype), + float(scale), + bool(use_qk_l2norm_in_kernel), + ) + graph, t = _bprop_cache[cache_key] + + dq = torch.empty(total, H, K, dtype=q.dtype, device=device) + dk = torch.empty(total, HK, K, dtype=q.dtype, device=device) + dv = torch.empty(total, HV, V, dtype=q.dtype, device=device) + dg = torch.empty(total, HO, K, dtype=torch.float32, device=device) + dbeta = torch.empty(total, HO, K, dtype=beta.dtype, device=device) + dw = torch.empty(total, HO, V, dtype=w.dtype, device=device) + variant_pack = { + t["q"]: q, + t["k"]: k, + t["v"]: v, + t["g"]: g, + t["beta"]: beta, + t["w"]: w, + t["cu"]: cu, + t["dO"]: dO, + t["dQ"]: dq, + t["dK"]: dk, + t["dV"]: dv, + t["dG"]: dg, + t["dBeta"]: dbeta, + t["dW"]: dw, + } + dstate0 = None + if state0 is not None: + variant_pack[t["state0"]] = state0 + dstate0 = torch.empty_like(state0) + variant_pack[t["dstate0"]] = dstate0 + if dstate_in is not None: + variant_pack[t["dfs"]] = dstate_in + graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_get_handle(device)) + if dstate0 is None: + dstate0 = torch.empty(0, dtype=torch.float32, device=device) + return dq, dk, dv, dg, dbeta, dw, dstate0 + + +@_gdn2_bwd.register_fake +def _gdn2_bwd_fake(dO, q, k, v, g, beta, w, cu_seqlens, scale, initial_state=None, d_final_state=None, use_qk_l2norm_in_kernel=False): + dstate0 = torch.empty_like(initial_state) if initial_state is not None else q.new_empty(0, dtype=torch.float32) + return ( + torch.empty_like(q), + torch.empty_like(k), + torch.empty_like(v), + g.new_empty(g.shape, dtype=torch.float32), + torch.empty_like(beta), + torch.empty_like(w), + dstate0, + ) + + +# --------------------------------------------------------------------------- +# Autograd registration +# --------------------------------------------------------------------------- + + +def _gdn2_setup_context(ctx, inputs, output): + ( + q, + k, + v, + g, + beta, + w, + cu_seqlens, + scale, + initial_state, + output_final_state, + use_qk_l2norm_in_kernel, + safe_gate, + gate_lower_bound, + a_log, + dt_bias, + checkpoint_every_n_tokens, + ) = inputs + # save_for_backward cannot hold None; keep initial_state as an attribute. + ctx.save_for_backward(q, k, v, g, beta, w, cu_seqlens) + ctx.initial_state = initial_state + ctx.scale = scale + ctx.use_qk_l2norm_in_kernel = use_qk_l2norm_in_kernel + ctx.safe_gate = bool(safe_gate) + ctx.mark_non_differentiable(output[2]) + + +def _gdn2_backward(ctx, dO, dFinal, _dstate_checkpoints): + if ctx.safe_gate: + raise NotImplementedError("gated_delta_net_v2: safe_gate is forward-only (GDN2_BWD takes post-activation gates)") + q, k, v, g, beta, w, cu_seqlens = ctx.saved_tensors + initial_state = ctx.initial_state + + dstate_in = dFinal if (dFinal is not None and dFinal.numel() > 0) else None + dq, dk, dv, dg, dbeta, dw, dstate0 = torch.ops.cudnn.gated_delta_net_v2_bwd( + dO, + q, + k, + v, + g, + beta, + w, + cu_seqlens, + ctx.scale, + initial_state=initial_state, + d_final_state=dstate_in, + use_qk_l2norm_in_kernel=ctx.use_qk_l2norm_in_kernel, + ) + # q, k, v, g, beta, w, cu_seqlens, scale, initial_state, + # output_final_state, use_qk_l2norm_in_kernel, safe_gate, + # gate_lower_bound, a_log, dt_bias, checkpoint_every_n_tokens + return ( + dq, + dk, + dv, + dg, + dbeta, + dw, + None, + None, + dstate0 if initial_state is not None else None, + None, + None, + None, + None, + None, + None, + None, + ) + + +torch.library.register_autograd( + "cudnn::gated_delta_net_v2_fwd", + _gdn2_backward, + setup_context=_gdn2_setup_context, +) # --------------------------------------------------------------------------- @@ -243,41 +727,61 @@ def gated_delta_net_v2( initial_state: Optional[torch.Tensor] = None, output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, -) -> Tuple[torch.Tensor, torch.Tensor]: - """Gated DeltaNet v2 (GDN-2) linear attention — forward only. + safe_gate: bool = False, + gate_lower_bound: Optional[float] = None, + a_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + checkpoint_every_n_tokens: int = 0, +): + """Gated DeltaNet v2 (GDN-2) linear attention. THD layout (matches the graph-API GDN-2 node): - q, k: ``[total_tokens, H, K]``; v: ``[total_tokens, HV, V]``; - g, beta: ``[total_tokens, HV, K]``; w: ``[total_tokens, HV, V]``; + q: ``[total_tokens, H, K]``; k: ``[total_tokens, HK, K]`` (HK = H, or + HK = HV for canonical GQA: grouped k/v heads shared across query groups); v: ``[total_tokens, HV, V]``; + g, beta: ``[total_tokens, HO, K]``; w: ``[total_tokens, HO, V]``; cu_seqlens: ``[N+1]`` int32; - initial_state / final_state: ``[N, HV, K, V]``. + initial_state / final_state: ``[N, HO, K, V]`` + (``HO = max(H, HV)``: the gates, output, and state heads). A dense batch of N equal-length sequences is expressed as ``cu_seqlens = [0, T, 2T, ...]`` over the flattened tokens. Args: - g: per-key-channel log-space decay (``alpha = exp(g)``). - beta: per-key erase gate. - w: per-value write gate. + g: per-key-channel log-space decay (``alpha = exp(g)``), or raw + pre-activation logits when ``safe_gate=True``. + beta: per-key erase gate (io dtype, post-activation). + w: per-value write gate (io dtype, post-activation). cu_seqlens: ``[N+1]`` int32 sequence boundaries over the packed tokens. scale: attention scale applied to ``q``. Defaults to ``1 / sqrt(K)``. initial_state: optional recurrent state (otherwise zero). output_final_state: if ``True``, also return the per-sequence state after the last token. - use_qk_l2norm_in_kernel: if ``True``, L2-normalize the q/k rows inside - the kernel; if ``False``, pass q/k as given (the caller owns their + use_qk_l2norm_in_kernel: if ``True``, L2-normalize the Q/K rows inside + the kernel; if ``False``, pass Q/K as given (the caller owns their conditioning). + safe_gate: interpret ``g`` through the safe-gate transform + ``gate_lower_bound * sigmoid(exp(a_log) * (g + dt_bias))``. + Requires ``a_log`` and ``dt_bias``. Forward-only. + gate_lower_bound: safe-gate lower bound in log space (default -5.0). + a_log: ``[HO]`` float32 safe-gate per-head log-amplitude. + dt_bias: ``[HO, K]`` float32 safe-gate channel bias. + checkpoint_every_n_tokens: if ``> 0``, also return the per-chunk + recurrent state series ``state_checkpoints`` (``[total_checkpoints, HO, K, V]`` io dtype, + one entry per N tokens strictly before each sequence end; the + FROST engine requires the kernel chunk size, 16). The series is + a non-differentiable dump. Returns: - ``(o, final_state)`` with ``o`` shaped like ``v``. ``final_state`` is - empty unless ``output_final_state=True``. + ``(o, final_state)`` with ``o`` shaped like ``v``, or + ``(o, final_state, state_checkpoints)`` when ``checkpoint_every_n_tokens > 0``. + ``final_state`` is empty unless ``output_final_state=True``. """ if q.dim() != 3: raise ValueError("expected THD [total_tokens, heads, dim] tensors") if scale is None: scale = 1.0 / math.sqrt(q.shape[-1]) - return torch.ops.cudnn.gated_delta_net_v2_fwd( + o, final_state, state_checkpoints = torch.ops.cudnn.gated_delta_net_v2_fwd( q, k, v, @@ -289,4 +793,12 @@ def gated_delta_net_v2( initial_state=initial_state, output_final_state=bool(output_final_state), use_qk_l2norm_in_kernel=bool(use_qk_l2norm_in_kernel), + safe_gate=bool(safe_gate), + gate_lower_bound=float(gate_lower_bound) if gate_lower_bound is not None else None, + a_log=a_log, + dt_bias=dt_bias, + checkpoint_every_n_tokens=int(checkpoint_every_n_tokens), ) + if checkpoint_every_n_tokens > 0: + return o, final_state, state_checkpoints + return o, final_state diff --git a/python/cudnn/linear_attention/ops/kda.py b/python/cudnn/linear_attention/ops/kda.py index 7801da3f3..98c2c596d 100644 --- a/python/cudnn/linear_attention/ops/kda.py +++ b/python/cudnn/linear_attention/ops/kda.py @@ -20,124 +20,272 @@ per-key-channel log decay ``[total_tokens, heads, dim]``; ``beta`` is scalar ``[total_tokens, heads]``. -The op is a thin adapter over the graph API (the SDPA-op pattern): forward +The op is a thin adapter over the graph API: forward and backward execute cached single-node ``KDA`` / ``KDA_BWD`` pygraphs. -Engine selection happens at graph planning time over the registered python -engines: ``KdaFrostEngine`` (forward, default on SM100/SM103) with -``KdaCuTileEngine`` as the fallback — and the backward engine everywhere -(the FROST KDA backward kernel is a stub). Registered through +Engine selection happens at graph planning time over the manifest's python +engines: ``KdaFrostEngine`` (default on SM100/SM103, forward and backward) +with ``KdaCuTileEngine`` as the fallback. Registered through ``torch.library.custom_op`` so it composes with autograd, ``torch.compile``, and DDP. -The backward graph recomputes the forward's cheap intermediates (cumulative -gate, intra-chunk WY factors) — the ``KDA_BWD`` node contract keeps them off -the autograd wire. +Graph caching ensures cuDNN graphs are built once per unique configuration +and reused across calls. """ -from __future__ import annotations - import math from typing import Dict, Optional, Tuple import torch - import cudnn -from cudnn.linear_attention import engine_utils -_OP_NAMESPACE = "cudnn" -_OP_NAME = "kimi_delta_attention" +# --------------------------------------------------------------------------- +# Module-level state +# --------------------------------------------------------------------------- + -_TORCH_TO_CUDNN_DTYPE = { +_TORCH_DTYPE_TO_CUDNN = { torch.float16: cudnn.data_type.HALF, torch.bfloat16: cudnn.data_type.BFLOAT16, torch.float32: cudnn.data_type.FLOAT, + torch.int32: cudnn.data_type.INT32, + torch.int64: cudnn.data_type.INT64, } # one graph per static configuration (shapes, dtypes, scale, flags, device) -_fwd_graph_cache: Dict[tuple, tuple] = {} -_bwd_graph_cache: Dict[tuple, tuple] = {} -_ws_cache: Dict[int, "torch.Tensor"] = {} +_fprop_cache: Dict[tuple, tuple] = {} +_bprop_cache: Dict[tuple, tuple] = {} +_cudnn_handles: Dict[int, int] = {} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- def _graph_workspace(graph, device): - """Caller-side workspace for a compiled graph (the explicit-workspace - convention: query the plan's size, allocate, pass to execute).""" + """Caller-side workspace for a compiled graph.""" if not graph._is_built: - # mirror execute()'s auto-build: plan via the router first (a bare - # build() would lower KDA to the backend, which has no lowering) + # plan first (a bare build() would lower KDA to the backend, + # which has no lowering) if not graph._planning_done: graph.create_execution_plans() - engine_utils.apply_pin(graph) if graph.selected_engine is None: graph.build() else: graph.build_plans() size = graph.get_workspace_size() - ws = _ws_cache.get(id(graph)) + ws = getattr(graph, "_la_ops_workspace", None) if ws is None or ws.numel() < size or ws.device != device: ws = torch.empty(max(size, 1), dtype=torch.uint8, device=device) - _ws_cache[id(graph)] = ws + graph._la_ops_workspace = ws return ws -_handle_cache: Dict[int, int] = {} - - -def _graph_handle(device): - """Per-device cuDNN handle carrying the caller's current stream - (classic ``set_stream`` semantics).""" +def _get_handle(device): + """Per-device cuDNN handle carrying the caller's current stream.""" idx = device.index if device.index is not None else torch.cuda.current_device() - handle = _handle_cache.get(idx) + handle = _cudnn_handles.get(idx) if handle is None: with torch.cuda.device(idx): handle = cudnn.create_handle() - _handle_cache[idx] = handle + _cudnn_handles[idx] = handle cudnn.set_stream(handle=handle, stream=torch.cuda.current_stream(device).cuda_stream) return handle -def _cudnn_dtype(dtype: Optional[torch.dtype]): - return _TORCH_TO_CUDNN_DTYPE[dtype] if dtype is not None else None - - -# --------------------------------------------------------------------------- -# Forward -# --------------------------------------------------------------------------- +def _torch_dtype_to_cudnn(dtype: torch.dtype): + """Map a PyTorch dtype to a cuDNN data_type enum.""" + return _TORCH_DTYPE_TO_CUDNN[dtype] def _check_dtype(name, t, want) -> None: if t.dtype != want: - raise TypeError(f"{_OP_NAME}: {name} must be {want} (kernel-native; callers convert), got {t.dtype}") + raise TypeError(f"kimi_delta_attention: {name} must be {want} (kernel-native; callers convert), got {t.dtype}") + + +def _make_fprop_cache_key( + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + k_shape, + v_shape, + cu_dtype, + scale, + output_final_state, + use_qk_l2norm, + use_beta_sigmoid, + safe_gate, + gate_lower_bound, + has_initial_state, + ckpt, + device, +): + return ( + "fprop", + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + k_shape, + v_shape, + cu_dtype, + float(scale), + bool(output_final_state), + bool(use_qk_l2norm), + bool(use_beta_sigmoid), + bool(safe_gate), + float(gate_lower_bound) if gate_lower_bound is not None else None, + bool(has_initial_state), + ckpt, + device, + ) + + +def _make_bprop_cache_key( + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + do_dtype, + k_shape, + v_shape, + cu_dtype, + g_dtype, + beta_dtype, + state_dtype, + dstate_in_dtype, + scale, + use_qk_l2norm, + device, +): + return ( + "bprop", + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + do_dtype, + k_shape, + v_shape, + cu_dtype, + g_dtype, + beta_dtype, + state_dtype, + dstate_in_dtype, + float(scale), + bool(use_qk_l2norm), + device, + ) + + +# --------------------------------------------------------------------------- +# Forward graph builder +# --------------------------------------------------------------------------- -def _build_fwd_graph(total, N, H, HV, K, V, io_dtype, g_dtype, beta_dtype, state_dtype, scale, output_final_state, use_qk_l2norm): +def _build_fprop_graph( + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + g_dtype, + beta_dtype, + state_dtype, + cu_dtype, + scale, + output_final_state, + use_qk_l2norm, + use_beta_sigmoid, + safe_gate, + gate_lower_bound, + ckpt, +): graph = cudnn.pygraph() + HO = max(H, HV) q_t = graph.tensor([total, H, K], data_type=io_dtype, name="q") - k_t = graph.tensor([total, H, K], data_type=io_dtype, name="k") + k_t = graph.tensor([total, HK, K], data_type=io_dtype, name="k") v_t = graph.tensor([total, HV, V], data_type=io_dtype, name="v") - g_t = graph.tensor([total, HV, K], data_type=g_dtype, name="g") - beta_t = graph.tensor([total, HV], data_type=beta_dtype, name="beta") - cu_t = graph.tensor([N + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") - s0_t = None + g_t = graph.tensor([total, HO, K], data_type=g_dtype, name="g") + beta_t = graph.tensor([total, HO], data_type=beta_dtype, name="beta") + cu_t = graph.tensor([N + 1], data_type=cu_dtype, name="cu_seqlens") + state0_t = None if state_dtype is not None: - s0_t = graph.tensor([N, HV, K, V], data_type=state_dtype, name="initial_state") - O_t, fs_t, _h_t = graph.kda( + state0_t = graph.tensor([N, HO, K, V], data_type=state_dtype, name="initial_state") + a_log_t = None + dt_bias_t = None + if safe_gate: + a_log_t = graph.tensor([HO], data_type=cudnn.data_type.FLOAT, name="a_log") + dt_bias_t = graph.tensor([HO, K], data_type=cudnn.data_type.FLOAT, name="dt_bias") + O_t, fs_t, state_checkpoints_t = graph.kda( q=q_t, k=k_t, v=v_t, g=g_t, beta=beta_t, cu_seqlens=cu_t, - initial_state=s0_t, + initial_state=state0_t, + a_log=a_log_t, + dt_bias=dt_bias_t, scale=scale, output_final_state=output_final_state, use_qk_l2norm=use_qk_l2norm, + use_beta_sigmoid=use_beta_sigmoid, + safe_gate=safe_gate, + gate_lower_bound=gate_lower_bound, + checkpoint_every_n_tokens=ckpt, name="kda", ) - return graph, dict(q=q_t, k=k_t, v=v_t, g=g_t, beta=beta_t, cu=cu_t, s0=s0_t, O=O_t, fs=fs_t) + return graph, dict( + q=q_t, + k=k_t, + v=v_t, + g=g_t, + beta=beta_t, + cu=cu_t, + state0=state0_t, + a_log=a_log_t, + dt_bias=dt_bias_t, + O=O_t, + fs=fs_t, + state_checkpoints=state_checkpoints_t, + ) -@torch.library.custom_op(f"{_OP_NAMESPACE}::{_OP_NAME}_fwd", mutates_args=()) +# --------------------------------------------------------------------------- +# Forward custom op +# --------------------------------------------------------------------------- + + +@torch.library.custom_op("cudnn::kimi_delta_attention_fwd", mutates_args=()) def _kda_fwd( q: torch.Tensor, k: torch.Tensor, @@ -149,112 +297,193 @@ def _kda_fwd( initial_state: Optional[torch.Tensor] = None, output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, -) -> Tuple[torch.Tensor, torch.Tensor]: - """KDA forward via a cached single-node KDA pygraph (THD layout). - - Returns ``(o, final_state)``; ``final_state`` is a zero-size tensor when - ``output_final_state`` is ``False``. + use_beta_sigmoid_in_kernel: bool = False, + safe_gate: bool = False, + gate_lower_bound: Optional[float] = None, + a_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + checkpoint_every_n_tokens: int = 0, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """KDA forward (internal): a cached single-node KDA pygraph, THD layout. + + Returns ``(o, final_state, state_checkpoints)``; ``final_state`` / ``state_checkpoints`` are zero-size + tensors when ``output_final_state`` is ``False`` / + ``checkpoint_every_n_tokens`` is ``0``. """ total, H, K = q.shape - if k.shape[1] != H: - raise ValueError(f"k must carry the same head count as q ({H}), got {k.shape[1]}") + HK = k.shape[1] HV, V = v.shape[1], v.shape[2] + if HK not in (H, HV): + raise ValueError(f"k head count ({HK}) must match q's ({H}) or v's ({HV}); canonical GQA shares grouped k/v heads") + HO = max(H, HV) N = cu_seqlens.shape[0] - 1 device = q.device - cu = cu_seqlens.to(torch.int32).contiguous() + if cu_seqlens.dtype not in (torch.int32, torch.int64): + raise ValueError(f"kimi_delta_attention: cu_seqlens must be int32 or int64; got {cu_seqlens.dtype}") + cu = cu_seqlens _check_dtype("g", g, torch.float32) - _check_dtype("beta", beta, torch.float32) + if use_beta_sigmoid_in_kernel: + _check_dtype("beta", beta, q.dtype) + else: + _check_dtype("beta", beta, torch.float32) + if safe_gate: + if a_log is None or dt_bias is None: + raise ValueError("kimi_delta_attention: safe_gate requires a_log and dt_bias") + _check_dtype("a_log", a_log, torch.float32) + _check_dtype("dt_bias", dt_bias, torch.float32) + elif a_log is not None or dt_bias is not None: + raise ValueError("kimi_delta_attention: a_log/dt_bias require safe_gate=True") if initial_state is not None: _check_dtype("initial_state", initial_state, torch.float32) if initial_state.shape[0] != N: raise ValueError(f"initial_state must carry one state per sequence: got {initial_state.shape[0]} for {N} sequences") - g32 = g.contiguous() - beta32 = beta.contiguous() - s0 = initial_state.contiguous() if initial_state is not None else None - - key = ( + for _name, _t in ( + ("k", k), + ("v", v), + ("g", g), + ("beta", beta), + ("cu_seqlens", cu_seqlens), + ("initial_state", initial_state), + ("a_log", a_log), + ("dt_bias", dt_bias), + ): + if _t is not None and _t.device != device: + raise ValueError(f"kimi_delta_attention: {_name} must be on q's device ({device}); got {_t.device}") + state0 = initial_state if initial_state is not None else None + ckpt = int(checkpoint_every_n_tokens) + + cache_key = _make_fprop_cache_key( total, N, H, + HK, HV, K, V, q.dtype, - bool(s0 is not None), - float(scale), - bool(output_final_state), - bool(use_qk_l2norm_in_kernel), + k.dtype, + v.dtype, + tuple(k.shape), + tuple(v.shape), + cu_seqlens.dtype, + scale, + output_final_state, + use_qk_l2norm_in_kernel, + use_beta_sigmoid_in_kernel, + safe_gate, + gate_lower_bound, + state0 is not None, + ckpt, device, ) - if key not in _fwd_graph_cache: - _fwd_graph_cache[key] = _build_fwd_graph( + if cache_key not in _fprop_cache: + _fprop_cache[cache_key] = _build_fprop_graph( total, N, H, + HK, HV, K, V, - _cudnn_dtype(q.dtype), - cudnn.data_type.FLOAT, + _torch_dtype_to_cudnn(q.dtype), cudnn.data_type.FLOAT, - cudnn.data_type.FLOAT if s0 is not None else None, + _torch_dtype_to_cudnn(beta.dtype), + cudnn.data_type.FLOAT if state0 is not None else None, + _torch_dtype_to_cudnn(cu_seqlens.dtype), float(scale), bool(output_final_state), bool(use_qk_l2norm_in_kernel), + bool(use_beta_sigmoid_in_kernel), + bool(safe_gate), + float(gate_lower_bound) if gate_lower_bound is not None else None, + ckpt, ) - graph, t = _fwd_graph_cache[key] + graph, t = _fprop_cache[cache_key] - o = torch.empty(total, HV, V, dtype=q.dtype, device=device) + o = torch.empty(total, HO, V, dtype=q.dtype, device=device) variant_pack = { - t["q"]: q.contiguous(), - t["k"]: k.contiguous(), - t["v"]: v.contiguous(), - t["g"]: g32, - t["beta"]: beta32, + t["q"]: q, + t["k"]: k, + t["v"]: v, + t["g"]: g, + t["beta"]: beta, t["cu"]: cu, t["O"]: o, } - if s0 is not None: - variant_pack[t["s0"]] = s0 + if state0 is not None: + variant_pack[t["state0"]] = state0 + if safe_gate: + variant_pack[t["a_log"]] = a_log + variant_pack[t["dt_bias"]] = dt_bias final_state = torch.empty(0, dtype=torch.float32, device=device) if output_final_state: - final_state = torch.empty(N, HV, K, V, dtype=torch.float32, device=device) + final_state = torch.empty(N, HO, K, V, dtype=torch.float32, device=device) variant_pack[t["fs"]] = final_state - graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_graph_handle(device)) - return o, final_state + state_checkpoints = torch.empty(0, dtype=q.dtype, device=device) + if ckpt > 0: + # shape-derived upper bound; exact count is data-dependent (cu contents) + total_checkpoints = max((total - N) // ckpt, 1) + state_checkpoints = torch.empty(total_checkpoints, HO, K, V, dtype=q.dtype, device=device) + variant_pack[t["state_checkpoints"]] = state_checkpoints + graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_get_handle(device)) + return o, final_state, state_checkpoints @_kda_fwd.register_fake -def _kda_fwd_fake(q, k, v, g, beta, cu_seqlens, scale, initial_state=None, output_final_state=False, use_qk_l2norm_in_kernel=False): - total, _H, K = q.shape +def _kda_fwd_fake( + q, + k, + v, + g, + beta, + cu_seqlens, + scale, + initial_state=None, + output_final_state=False, + use_qk_l2norm_in_kernel=False, + use_beta_sigmoid_in_kernel=False, + safe_gate=False, + gate_lower_bound=None, + a_log=None, + dt_bias=None, + checkpoint_every_n_tokens=0, +): + total, H, K = q.shape HV, V = v.shape[1], v.shape[2] + HO = max(H, HV) N = cu_seqlens.shape[0] - 1 - o = q.new_empty(total, HV, V) - final = q.new_empty((N, HV, K, V) if output_final_state else (0,), dtype=torch.float32) - return o, final + o = q.new_empty(total, HO, V) + final = q.new_empty((N, HO, K, V) if output_final_state else (0,), dtype=torch.float32) + if checkpoint_every_n_tokens > 0: + total_checkpoints = max((total - N) // int(checkpoint_every_n_tokens), 1) + state_checkpoints = q.new_empty(total_checkpoints, HO, K, V) + else: + state_checkpoints = q.new_empty(0) + return o, final, state_checkpoints # --------------------------------------------------------------------------- -# Backward +# Backward graph builder # --------------------------------------------------------------------------- -def _build_bwd_graph(total, N, H, HV, K, V, io_dtype, g_dtype, beta_dtype, state_dtype, dht_dtype, scale, use_qk_l2norm): +def _build_bprop_graph(total, N, H, HK, HV, K, V, io_dtype, g_dtype, beta_dtype, state_dtype, dstate_in_dtype, cu_dtype, scale, use_qk_l2norm): graph = cudnn.pygraph() + HO = max(H, HV) q_t = graph.tensor([total, H, K], data_type=io_dtype, name="q") - k_t = graph.tensor([total, H, K], data_type=io_dtype, name="k") + k_t = graph.tensor([total, HK, K], data_type=io_dtype, name="k") v_t = graph.tensor([total, HV, V], data_type=io_dtype, name="v") - g_t = graph.tensor([total, HV, K], data_type=g_dtype, name="g") - beta_t = graph.tensor([total, HV], data_type=beta_dtype, name="beta") - cu_t = graph.tensor([N + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") - dO_t = graph.tensor([total, HV, V], data_type=io_dtype, name="dO") - s0_t = None + g_t = graph.tensor([total, HO, K], data_type=g_dtype, name="g") + beta_t = graph.tensor([total, HO], data_type=beta_dtype, name="beta") + cu_t = graph.tensor([N + 1], data_type=cu_dtype, name="cu_seqlens") + dO_t = graph.tensor([total, HO, V], data_type=io_dtype, name="dO") + state0_t = None if state_dtype is not None: - s0_t = graph.tensor([N, HV, K, V], data_type=state_dtype, name="initial_state") + state0_t = graph.tensor([N, HO, K, V], data_type=state_dtype, name="initial_state") dfs_t = None - if dht_dtype is not None: - dfs_t = graph.tensor([N, HV, K, V], data_type=dht_dtype, name="d_final_state") - dQ_t, dK_t, dV_t, dG_t, dBeta_t, dS0_t = graph.kda_bwd( + if dstate_in_dtype is not None: + dfs_t = graph.tensor([N, HO, K, V], data_type=dstate_in_dtype, name="d_final_state") + dQ_t, dK_t, dV_t, dG_t, dBeta_t, dstate0_t = graph.kda_bwd( q=q_t, k=k_t, v=v_t, @@ -262,7 +491,7 @@ def _build_bwd_graph(total, N, H, HV, K, V, io_dtype, g_dtype, beta_dtype, state beta=beta_t, cu_seqlens=cu_t, dO=dO_t, - initial_state=s0_t, + initial_state=state0_t, d_final_state=dfs_t, scale=scale, use_qk_l2norm=use_qk_l2norm, @@ -276,18 +505,23 @@ def _build_bwd_graph(total, N, H, HV, K, V, io_dtype, g_dtype, beta_dtype, state beta=beta_t, cu=cu_t, dO=dO_t, - s0=s0_t, + state0=state0_t, dfs=dfs_t, dQ=dQ_t, dK=dK_t, dV=dV_t, dG=dG_t, dBeta=dBeta_t, - dS0=dS0_t, + dstate0=dstate0_t, ) -@torch.library.custom_op(f"{_OP_NAMESPACE}::{_OP_NAME}_bwd", mutates_args=()) +# --------------------------------------------------------------------------- +# Backward custom op +# --------------------------------------------------------------------------- + + +@torch.library.custom_op("cudnn::kimi_delta_attention_bwd", mutates_args=()) def _kda_bwd( dO: torch.Tensor, q: torch.Tensor, @@ -301,18 +535,28 @@ def _kda_bwd( d_final_state: Optional[torch.Tensor] = None, use_qk_l2norm_in_kernel: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """KDA backward via a cached single-node KDA_BWD pygraph (THD layout). + """KDA backward (internal): a cached single-node KDA_BWD pygraph, THD layout. Returns ``(dq, dk, dv, dg, dbeta, d_initial_state)``; ``d_initial_state`` is a zero-size tensor when ``initial_state`` is ``None``. """ total, H, K = q.shape - if k.shape[1] != H: - raise ValueError(f"k must carry the same head count as q ({H}), got {k.shape[1]}") + # autograd materializes reduction grads as broadcast (stride-0) + # views; densify ONLY those (dense callers pass through untouched) + if 0 in dO.stride(): + dO = dO.contiguous() + if d_final_state is not None and 0 in d_final_state.stride(): + d_final_state = d_final_state.contiguous() + HK = k.shape[1] HV, V = v.shape[1], v.shape[2] + if HK not in (H, HV): + raise ValueError(f"k head count ({HK}) must match q's ({H}) or v's ({HV}); canonical GQA shares grouped k/v heads") + HO = max(H, HV) N = cu_seqlens.shape[0] - 1 device = q.device - cu = cu_seqlens.to(torch.int32).contiguous() + if cu_seqlens.dtype not in (torch.int32, torch.int64): + raise ValueError(f"kimi_delta_attention: cu_seqlens must be int32 or int64; got {cu_seqlens.dtype}") + cu = cu_seqlens _check_dtype("g", g, torch.float32) _check_dtype("beta", beta, torch.float32) if initial_state is not None: @@ -321,84 +565,97 @@ def _kda_bwd( raise ValueError(f"initial_state must carry one state per sequence: got {initial_state.shape[0]} for {N} sequences") if d_final_state is not None: _check_dtype("d_final_state", d_final_state, torch.float32) - s0 = initial_state.contiguous() if initial_state is not None else None - dht = d_final_state.contiguous() if d_final_state is not None else None + for _name, _t in (("k", k), ("v", v), ("g", g), ("beta", beta), ("cu_seqlens", cu_seqlens), ("dO", dO), ("d_final_state", d_final_state)): + if _t is not None and _t.device != device: + raise ValueError(f"kimi_delta_attention: {_name} must be on q's device ({device}); got {_t.device}") + state0 = initial_state if initial_state is not None else None + dstate_in = d_final_state if d_final_state is not None else None - key = ( + cache_key = _make_bprop_cache_key( total, N, H, + HK, HV, K, V, q.dtype, + k.dtype, + v.dtype, + dO.dtype, + tuple(k.shape), + tuple(v.shape), + cu_seqlens.dtype, g.dtype, beta.dtype, - s0.dtype if s0 is not None else None, - dht.dtype if dht is not None else None, - float(scale), - bool(use_qk_l2norm_in_kernel), + state0.dtype if state0 is not None else None, + dstate_in.dtype if dstate_in is not None else None, + scale, + use_qk_l2norm_in_kernel, device, ) - if key not in _bwd_graph_cache: - _bwd_graph_cache[key] = _build_bwd_graph( + if cache_key not in _bprop_cache: + _bprop_cache[cache_key] = _build_bprop_graph( total, N, H, + HK, HV, K, V, - _cudnn_dtype(q.dtype), - _cudnn_dtype(g.dtype), - _cudnn_dtype(beta.dtype), - _cudnn_dtype(s0.dtype) if s0 is not None else None, - _cudnn_dtype(dht.dtype) if dht is not None else None, + _torch_dtype_to_cudnn(q.dtype), + _torch_dtype_to_cudnn(g.dtype), + _torch_dtype_to_cudnn(beta.dtype), + _torch_dtype_to_cudnn(state0.dtype) if state0 is not None else None, + _torch_dtype_to_cudnn(dstate_in.dtype) if dstate_in is not None else None, + _torch_dtype_to_cudnn(cu_seqlens.dtype), float(scale), bool(use_qk_l2norm_in_kernel), ) - graph, t = _bwd_graph_cache[key] + graph, t = _bprop_cache[cache_key] dq = torch.empty(total, H, K, dtype=q.dtype, device=device) - dk = torch.empty(total, H, K, dtype=k.dtype, device=device) - dv = torch.empty(total, HV, V, dtype=v.dtype, device=device) - dg = torch.empty(total, HV, K, dtype=g.dtype, device=device) - dbeta = torch.empty(total, HV, dtype=beta.dtype, device=device) + dk = torch.empty(total, HK, K, dtype=q.dtype, device=device) + dv = torch.empty(total, HV, V, dtype=q.dtype, device=device) + dg = torch.empty(total, HO, K, dtype=g.dtype, device=device) + dbeta = torch.empty(total, HO, dtype=beta.dtype, device=device) variant_pack = { - t["q"]: q.contiguous(), - t["k"]: k.contiguous(), - t["v"]: v.contiguous(), - t["g"]: g.contiguous(), - t["beta"]: beta.contiguous(), + t["q"]: q, + t["k"]: k, + t["v"]: v, + t["g"]: g, + t["beta"]: beta, t["cu"]: cu, - t["dO"]: dO.contiguous(), + t["dO"]: dO, t["dQ"]: dq, t["dK"]: dk, t["dV"]: dv, t["dG"]: dg, t["dBeta"]: dbeta, } - dh032 = None - if s0 is not None: - variant_pack[t["s0"]] = s0 - dh032 = torch.empty_like(s0) - variant_pack[t["dS0"]] = dh032 - if dht is not None: - variant_pack[t["dfs"]] = dht - graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_graph_handle(device)) - dh0 = dh032 if dh032 is not None else torch.empty(0, dtype=torch.float32, device=device) - return dq, dk, dv, dg, dbeta, dh0 + dstate0 = None + if state0 is not None: + variant_pack[t["state0"]] = state0 + dstate0 = torch.empty_like(state0) + variant_pack[t["dstate0"]] = dstate0 + if dstate_in is not None: + variant_pack[t["dfs"]] = dstate_in + graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_get_handle(device)) + if dstate0 is None: + dstate0 = torch.empty(0, dtype=torch.float32, device=device) + return dq, dk, dv, dg, dbeta, dstate0 @_kda_bwd.register_fake def _kda_bwd_fake(dO, q, k, v, g, beta, cu_seqlens, scale, initial_state=None, d_final_state=None, use_qk_l2norm_in_kernel=False): - dh0 = torch.empty_like(initial_state) if initial_state is not None else q.new_empty(0, dtype=torch.float32) + dstate0 = torch.empty_like(initial_state) if initial_state is not None else q.new_empty(0, dtype=torch.float32) return ( torch.empty_like(q), torch.empty_like(k), torch.empty_like(v), torch.empty_like(g), torch.empty_like(beta), - dh0, + dstate0, ) @@ -408,21 +665,43 @@ def _kda_bwd_fake(dO, q, k, v, g, beta, cu_seqlens, scale, initial_state=None, d def _kda_setup_context(ctx, inputs, output): - q, k, v, g, beta, cu_seqlens, scale, initial_state, output_final_state, use_qk_l2norm_in_kernel = inputs + ( + q, + k, + v, + g, + beta, + cu_seqlens, + scale, + initial_state, + output_final_state, + use_qk_l2norm_in_kernel, + use_beta_sigmoid_in_kernel, + safe_gate, + gate_lower_bound, + a_log, + dt_bias, + checkpoint_every_n_tokens, + ) = inputs # save_for_backward cannot hold None; keep initial_state as an attribute. ctx.save_for_backward(q, k, v, g, beta, cu_seqlens) ctx.initial_state = initial_state ctx.scale = scale ctx.use_qk_l2norm_in_kernel = use_qk_l2norm_in_kernel + ctx.use_beta_sigmoid_in_kernel = bool(use_beta_sigmoid_in_kernel) + ctx.safe_gate = bool(safe_gate) + ctx.mark_non_differentiable(output[2]) -def _kda_backward(ctx, dO, dFinal): +def _kda_backward(ctx, dO, dFinal, _dstate_checkpoints): + if ctx.use_beta_sigmoid_in_kernel or ctx.safe_gate: + raise NotImplementedError("kimi_delta_attention: safe_gate/use_beta_sigmoid_in_kernel are forward-only (KDA_BWD takes post-activation gates)") q, k, v, g, beta, cu_seqlens = ctx.saved_tensors initial_state = ctx.initial_state - dht = dFinal if (dFinal is not None and dFinal.numel() > 0) else None - dq, dk, dv, dg, dbeta, dh0 = torch.ops.cudnn.kimi_delta_attention_bwd( - dO.contiguous(), + dstate_in = dFinal if (dFinal is not None and dFinal.numel() > 0) else None + dq, dk, dv, dg, dbeta, dstate0 = torch.ops.cudnn.kimi_delta_attention_bwd( + dO, q, k, v, @@ -431,11 +710,12 @@ def _kda_backward(ctx, dO, dFinal): cu_seqlens, ctx.scale, initial_state=initial_state, - d_final_state=dht, + d_final_state=dstate_in, use_qk_l2norm_in_kernel=ctx.use_qk_l2norm_in_kernel, ) # q, k, v, g, beta, cu_seqlens, scale, initial_state, output_final_state, - # use_qk_l2norm_in_kernel + # use_qk_l2norm_in_kernel, use_beta_sigmoid_in_kernel, safe_gate, + # gate_lower_bound, a_log, dt_bias, checkpoint_every_n_tokens return ( dq, dk, @@ -444,14 +724,20 @@ def _kda_backward(ctx, dO, dFinal): dbeta, None, None, - dh0 if initial_state is not None else None, + dstate0 if initial_state is not None else None, + None, + None, + None, + None, + None, + None, None, None, ) torch.library.register_autograd( - f"{_OP_NAMESPACE}::{_OP_NAME}_fwd", + "cudnn::kimi_delta_attention_fwd", _kda_backward, setup_context=_kda_setup_context, ) @@ -473,15 +759,23 @@ def kimi_delta_attention( initial_state: Optional[torch.Tensor] = None, output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, + use_beta_sigmoid_in_kernel: bool = False, + safe_gate: bool = False, + gate_lower_bound: Optional[float] = None, + a_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + checkpoint_every_n_tokens: int = 0, ): """Kimi Delta Attention (KDA) linear attention. THD layout (matches the graph-API KDA node): - q, k: ``[total_tokens, H, K]``; v: ``[total_tokens, HV, V]`` - g: ``[total_tokens, HV, K]`` (per-key-channel log decay); - beta: ``[total_tokens, HV]`` (scalar); cu_seqlens: ``[N+1]`` int32 - initial_state / final_state: ``[N, HV, K, V]`` + q: ``[total_tokens, H, K]``; k: ``[total_tokens, HK, K]`` (HK = H, or + HK = HV for canonical GQA: grouped K/V heads shared across query groups); v: ``[total_tokens, HV, V]`` + g: ``[total_tokens, HO, K]`` (per-key-channel log decay); + beta: ``[total_tokens, HO]`` (scalar); cu_seqlens: ``[N+1]`` int32 + initial_state / final_state: ``[N, HO, K, V]`` + (``HO = max(H, HV)``: the gates, output, and state heads) A dense batch of N equal-length sequences is expressed as ``cu_seqlens = [0, T, 2T, ...]`` over the flattened tokens. @@ -491,26 +785,42 @@ def kimi_delta_attention( ``d_initial_state`` are returned in float32. Args: - g: per-key-channel log-space decay (``alpha = exp(g) in (0, 1]^K``). - beta: per-token scalar write strength. + g: per-key-channel log-space decay (``alpha = exp(g) in (0, 1]^K``), + or raw pre-activation logits when ``safe_gate=True``. + beta: per-token scalar write strength (float32 post-sigmoid), or + io-dtype logits when ``use_beta_sigmoid_in_kernel=True``. cu_seqlens: ``[N+1]`` int32 sequence boundaries over the packed tokens. scale: attention scale applied to ``q``. Defaults to ``1 / sqrt(K)``. initial_state: optional recurrent state (otherwise zero). output_final_state: if ``True``, also return the per-sequence state after the last token. - use_qk_l2norm_in_kernel: if ``True``, L2-normalize the q/k rows inside - the kernel (the KDA model's feature map); if ``False``, pass q/k + use_qk_l2norm_in_kernel: if ``True``, L2-normalize the Q/K rows inside + the kernel (the KDA model's feature map); if ``False``, pass Q/K as given (the caller owns their conditioning). + use_beta_sigmoid_in_kernel: apply ``sigmoid(beta)`` inside the kernel. + Forward-only. + safe_gate: interpret ``g`` through the safe-gate transform + ``gate_lower_bound * sigmoid(exp(a_log) * (g + dt_bias))``. + Requires ``a_log`` and ``dt_bias``. Forward-only. + gate_lower_bound: safe-gate lower bound in log space (default -5.0). + a_log: ``[HO]`` float32 safe-gate per-head log-amplitude. + dt_bias: ``[HO, K]`` float32 safe-gate channel bias. + checkpoint_every_n_tokens: if ``> 0``, also return the per-chunk + recurrent state series ``state_checkpoints`` (``[total_checkpoints, HO, K, V]`` io dtype, + one entry per N tokens strictly before each sequence end; the + FROST engine requires the kernel chunk size, 16). The series is + a non-differentiable dump. Returns: - ``(o, final_state)`` with ``o`` shaped like ``v``. ``final_state`` is - empty unless ``output_final_state=True``. + ``(o, final_state)`` with ``o`` shaped like ``v``, or + ``(o, final_state, state_checkpoints)`` when ``checkpoint_every_n_tokens > 0``. + ``final_state`` is empty unless ``output_final_state=True``. """ if q.dim() != 3: raise ValueError("expected THD [total_tokens, heads, dim] tensors") if scale is None: scale = 1.0 / math.sqrt(q.shape[-1]) - return torch.ops.cudnn.kimi_delta_attention_fwd( + o, final_state, state_checkpoints = torch.ops.cudnn.kimi_delta_attention_fwd( q, k, v, @@ -521,4 +831,13 @@ def kimi_delta_attention( initial_state=initial_state, output_final_state=bool(output_final_state), use_qk_l2norm_in_kernel=bool(use_qk_l2norm_in_kernel), + use_beta_sigmoid_in_kernel=bool(use_beta_sigmoid_in_kernel), + safe_gate=bool(safe_gate), + gate_lower_bound=float(gate_lower_bound) if gate_lower_bound is not None else None, + a_log=a_log, + dt_bias=dt_bias, + checkpoint_every_n_tokens=int(checkpoint_every_n_tokens), ) + if checkpoint_every_n_tokens > 0: + return o, final_state, state_checkpoints + return o, final_state diff --git a/test/python/linear_attention/common.py b/test/python/linear_attention/common.py deleted file mode 100644 index a00cd27e8..000000000 --- a/test/python/linear_attention/common.py +++ /dev/null @@ -1,208 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -""" -Shared helpers for the GDN (Gated DeltaNet) test suite. - -Availability guards, run dispatcher, and comparison tolerances shared by the -fprop and bprop test files. -""" - -from __future__ import annotations - -import os -from typing import Optional - -import pytest -import torch - -_HAS_CUDA = torch.cuda.is_available() - -try: - import cuda.tile # noqa: F401 — the engines' own availability gate - - from cudnn.linear_attention.ops import gated_delta_net, kimi_delta_attention - - _HAS_CUTILE = True -except ImportError: - _HAS_CUTILE = False - -GDN_MARKS = [ - pytest.mark.L0, - pytest.mark.skipif(not _HAS_CUDA, reason="needs CUDA"), - pytest.mark.skipif(not _HAS_CUTILE, reason="needs the cuda.tile runtime"), -] - -KDA_MARKS = GDN_MARKS # same gate: both ops are lazy exports of the same package - -FWD_TOL = {torch.bfloat16: 2e-2, torch.float16: 1e-2} -STATE_TOL = {torch.bfloat16: 2e-2, torch.float16: 1e-2} -BWD_TOL = {torch.bfloat16: 4e-2, torch.float16: 2e-2} - -# (H, HV) pairs: H = q/k heads, HV = v/g/beta heads (GVA when HV > H). -HEAD_CONFIGS = [(1, 1), (3, 3), (1, 2), (2, 4), (16, 32), (16, 64)] -HEAD_CONFIGS_SMALL = [(1, 1), (2, 4)] - - -def assert_engine_declines(graph, engine_name: str) -> None: - """``engine_name`` must not serve ``graph``. - - Asserted against the ranked plan list rather than through a failing - ``build()``: a decline only advances the walk, so a sibling engine - (cuTile vs FROST) may still serve the graph — and does wherever both are - installed, which is why the build-fails form passes on a box missing one - of them and fails in CI.""" - try: - graph.create_execution_plans() - except Exception: - return # nothing claimed it at all, which is a stronger decline - names = [graph.get_plan_name_at_index(i) for i in range(len(graph.plans))] - assert not any(n.startswith(engine_name) for n in names), f"{engine_name} claimed a graph it must decline; plans={names}" - - -def run_gdn( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - g: torch.Tensor, - beta: torch.Tensor, - *, - scale: Optional[float] = None, - initial_state: Optional[torch.Tensor] = None, - output_final_state: bool = False, - cu_seqlens: Optional[torch.Tensor] = None, -): - """Run GDN through the public custom op (the op is THD-only: dense - ``[B, T, ...]`` inputs are flattened with ``cu_seqlens = [0, T, 2T, ...]``; - packed ``[1, total, ...]`` inputs are squeezed to the op's rank-3 THD - layout). Returns ``(o, final_state)`` with ``final_state`` normalized to - ``None`` when not requested.""" - g = g.float() # the op requires kernel-native fp32 gates; callers convert - beta = beta.float() - if cu_seqlens is None: - B, T = q.shape[0], q.shape[1] - cu = torch.arange(0, B + 1, dtype=torch.int32, device=q.device) * T - o, fs = gated_delta_net( - q.reshape(B * T, *q.shape[2:]), - k.reshape(B * T, *k.shape[2:]), - v.reshape(B * T, *v.shape[2:]), - g.reshape(B * T, *g.shape[2:]), - beta.reshape(B * T, *beta.shape[2:]), - cu, - scale=scale, - initial_state=initial_state, - output_final_state=output_final_state, - ) - o = o.reshape(B, T, *o.shape[1:]) - else: - o, fs = gated_delta_net( - q.squeeze(0), - k.squeeze(0), - v.squeeze(0), - g.squeeze(0), - beta.squeeze(0), - scale=scale, - initial_state=initial_state, - output_final_state=output_final_state, - cu_seqlens=cu_seqlens, - ) - o = o.unsqueeze(0) - if fs is None or fs.numel() == 0: - fs = None - return o, fs - - -def run_kda( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - g: torch.Tensor, - beta: torch.Tensor, - *, - scale: Optional[float] = None, - initial_state: Optional[torch.Tensor] = None, - output_final_state: bool = False, - use_qk_l2norm_in_kernel: bool = False, - cu_seqlens: Optional[torch.Tensor] = None, -): - """Run KDA through the public custom op (the op is THD-only: dense - ``[B, T, ...]`` inputs are flattened with ``cu_seqlens = [0, T, 2T, ...]``). - ``g`` is the per-key-channel log decay ([..., HV, K]); ``beta`` is scalar - ([..., HV]). Returns ``(o, final_state)`` with ``final_state`` normalized - to ``None`` when not requested.""" - g = g.float() # the op requires kernel-native fp32 gates; callers convert - beta = beta.float() - if cu_seqlens is None: - B, T = q.shape[0], q.shape[1] - cu = torch.arange(0, B + 1, dtype=torch.int32, device=q.device) * T - o, fs = kimi_delta_attention( - q.reshape(B * T, *q.shape[2:]), - k.reshape(B * T, *k.shape[2:]), - v.reshape(B * T, *v.shape[2:]), - g.reshape(B * T, *g.shape[2:]), - beta.reshape(B * T, *beta.shape[2:]), - cu, - scale=scale, - initial_state=initial_state, - output_final_state=output_final_state, - use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, - ) - o = o.reshape(B, T, *o.shape[1:]) - else: - o, fs = kimi_delta_attention( - q.squeeze(0), - k.squeeze(0), - v.squeeze(0), - g.squeeze(0), - beta.squeeze(0), - scale=scale, - initial_state=initial_state, - output_final_state=output_final_state, - use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, - cu_seqlens=cu_seqlens, - ) - o = o.unsqueeze(0) - if fs is None or fs.numel() == 0: - fs = None - return o, fs - - -DETERMINISM_REPEATS = int(os.environ.get("DETERMINISM_REPEATS", "8")) - - -def bitwise_bits(t: torch.Tensor) -> torch.Tensor: - return t.contiguous().view(torch.uint8) - - -def assert_bitwise_runs(launch, repeats=DETERMINISM_REPEATS, label=""): - """``launch()`` returns a tuple of freshly-written output tensors. Launch - ``repeats`` times back to back (single sync at the end) and require every - run to match run 0 bit for bit (barrier/fence races are timing-dependent; - any mismatching bit is a failure, there is no tolerance).""" - runs = [launch() for _ in range(repeats)] - torch.cuda.synchronize() - for out in runs[0]: - assert torch.isfinite(out.float()).all(), f"{label}: non-finite output in run 0" - for r, outs in enumerate(runs[1:], start=1): - for i, (a, b) in enumerate(zip(runs[0], outs)): - assert torch.equal(bitwise_bits(a), bitwise_bits(b)), f"{label}: output {i} differs between run 0 and run {r}" - - -def assert_concurrent_stream_runs(launch_a, launch_b, s1, s2, repeats=DETERMINISM_REPEATS): - """Two concurrent kernel instances on separate streams must not perturb - each other: every repeat must match its own single-stream baseline.""" - with torch.cuda.stream(s1): - base_a = launch_a() - torch.cuda.synchronize() - with torch.cuda.stream(s2): - base_b = launch_b() - torch.cuda.synchronize() - for r in range(repeats): - with torch.cuda.stream(s1): - out_a = launch_a() - with torch.cuda.stream(s2): - out_b = launch_b() - torch.cuda.synchronize() - for label, base, outs in (("A", base_a, out_a), ("B", base_b, out_b)): - for i, (x, y) in enumerate(zip(base, outs)): - assert torch.equal(bitwise_bits(x), bitwise_bits(y)), f"stream {label} output {i} differs on concurrent run {r}" diff --git a/test/python/linear_attention/conftest.py b/test/python/linear_attention/conftest.py index 00cccd912..147770ea1 100644 --- a/test/python/linear_attention/conftest.py +++ b/test/python/linear_attention/conftest.py @@ -2,16 +2,32 @@ # SPDX-License-Identifier: Apache-2.0 """ -Fixtures for the GDN test suite: seeded input factories for q/k/v and the -gate tensors (g = log decay, beta = write strength). +Fixtures for the linear-attention test suite: seeded input factories for +Q/K/V and the gate tensors (G = log decay, Beta = write strength). + +Also overlays the source ``python/cudnn`` dir onto the installed ``cudnn`` +package's ``__path__``: the built wheel may lack the engine subtrees the +suite pins (``cudnn.linear_attention.frost``). Unnecessary once the engines +ship in the built frontend package. """ from __future__ import annotations +from pathlib import Path + import pytest import torch import torch.nn.functional as F +try: + import cudnn + + _SRC_CUDNN = Path(__file__).resolve().parents[3] / "python" / "cudnn" + if _SRC_CUDNN.is_dir() and str(_SRC_CUDNN) not in cudnn.__path__: + cudnn.__path__.append(str(_SRC_CUDNN)) +except ImportError: + pass # test_la.py skips via importorskip + def multidist_randu(num_dists, dim, *, device, mean_std=0.05, lower=-0.25, upper=0.25): """Rows drawn from per-row uniform distributions with normally-distributed @@ -21,9 +37,9 @@ def multidist_randu(num_dists, dim, *, device, mean_std=0.05, lower=-0.25, upper def gen_qkv(B, T, H, HV, K, V, dtype, device="cuda"): - """Dense [B, T, heads, dim] q/k/v. k is l2-normalized along the feature - dim so the delta-rule update (I - beta k k^T) stays contractive for - beta in (0, 1].""" + """Dense [B, T, heads, dim] Q/K/V. K is l2-normalized along the feature + dim so the delta-rule update (I - Beta K K^T) stays contractive for + Beta in (0, 1].""" q = multidist_randu(B * T * H, K, device=device).reshape(B, T, H, K) k = multidist_randu(B * T * H, K, device=device).reshape(B, T, H, K) k = F.normalize(k, p=2.0, dim=-1) @@ -36,8 +52,8 @@ def gen_qkv(B, T, H, HV, K, V, dtype, device="cuda"): def gen_gates(B, T, HV, dtype, device="cuda", alpha=True, beta=True): - """Gate tensors [B, T, HV]. alpha off -> g = 0 (decay factor exactly 1); - beta off -> beta = 1 (plain delta rule).""" + """Gate tensors [B, T, HV]. alpha off -> G = 0 (decay factor exactly 1); + beta off -> Beta = 1 (plain delta rule).""" if alpha: a = torch.empty(B, T, HV, device=device, dtype=torch.float32).uniform_(0.1, 1.0) g = a.log() @@ -51,11 +67,11 @@ def gen_gates(B, T, HV, dtype, device="cuda", alpha=True, beta=True): def gen_kda_gates(B, T, HV, K, dtype, device="cuda", alpha=True, beta=True, lo=0.9): - """KDA gate tensors: g [B, T, HV, K] fp32 per-key-channel log decay, beta + """KDA gate tensors: G [B, T, HV, K] fp32 per-key-channel log decay, Beta [B, T, HV] scalar write strength. ``g`` stays fp32 (the per-channel decay kernel keeps it fp32); the per-channel cumulative product over a 64-token - chunk stays well within bf16 range for ``alpha >= lo``. alpha off -> g = 0 - (no decay); beta off -> beta = 1 (plain delta rule).""" + chunk stays well within bf16 range for ``alpha >= lo``. alpha off -> G = 0 + (no decay); beta off -> Beta = 1 (plain delta rule).""" if alpha: a = torch.empty(B, T, HV, K, device=device, dtype=torch.float32).uniform_(lo, 1.0) g = a.log() @@ -69,10 +85,10 @@ def gen_kda_gates(B, T, HV, K, dtype, device="cuda", alpha=True, beta=True, lo=0 def gen_gdn2_gates(B, T, HO, K, V, dtype, device="cuda", alpha=True, beta=True, w=True, lo=0.5): - """GDN-2 gate tensors: g [B, T, HO, K] fp32 per-key-channel log decay, - beta [B, T, HO, K] per-key erase gate, w [B, T, HO, V] per-value write - gate. beta/w are io-dtype (rounded before both kernel and reference see - them). alpha off -> g = 0; beta/w off -> ones.""" + """GDN-2 gate tensors: G [B, T, HO, K] fp32 per-key-channel log decay, + Beta [B, T, HO, K] per-key erase gate, W [B, T, HO, V] per-value write + gate. Beta/W are io-dtype (rounded before both kernel and reference see + them). alpha off -> G = 0; beta/w off -> ones.""" if alpha: a = torch.empty(B, T, HO, K, device=device, dtype=torch.float32).uniform_(lo, 1.0) g = a.log() diff --git a/test/python/linear_attention/cutile/__init__.py b/test/python/linear_attention/cutile/__init__.py deleted file mode 100644 index 52a7a9daf..000000000 --- a/test/python/linear_attention/cutile/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 diff --git a/test/python/linear_attention/cutile/conftest.py b/test/python/linear_attention/cutile/conftest.py deleted file mode 100644 index 1caa0845b..000000000 --- a/test/python/linear_attention/cutile/conftest.py +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""The cutile suite validates the cuTile engines and their kernels, so op -calls made here pin them — under default ranking the FROST engines would serve -the FROST-eligible shapes instead. - -The pin is enforced where it is applied: engine_utils.apply_pin() raises if the -pinned engine produced no plan, so a pin that stops working fails the first op -call rather than silently testing another engine. This suite once ran for -months against whichever engine the router picked, because the seam it pinned -through was dead and nothing checked.""" - -from __future__ import annotations - -import pytest - - -@pytest.fixture(autouse=True, scope="package") -def _pin_cutile_engines(): - from cudnn.linear_attention import engine_utils - from cudnn.linear_attention.ops import gdn, kda - - saved = engine_utils.pin_engines(("gdn_cutile", "kda_cutile")) - for m in (gdn, kda): - m._fwd_graph_cache.clear() - m._bwd_graph_cache.clear() - yield - engine_utils.pin_engines(saved) - for m in (gdn, kda): - m._fwd_graph_cache.clear() - m._bwd_graph_cache.clear() diff --git a/test/python/linear_attention/cutile/test_gdn_bprop.py b/test/python/linear_attention/cutile/test_gdn_bprop.py deleted file mode 100644 index 76e455af3..000000000 --- a/test/python/linear_attention/cutile/test_gdn_bprop.py +++ /dev/null @@ -1,272 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -""" -Backward (bprop) tests for the GDN cuTile kernel: gradients of q, k, v, g, -beta (and the initial state) are compared against autograd through the fp64 -recurrent reference in ``reference_gdn`` using a shared random upstream -gradient. - -Covers: fp16/bf16, MHA and GVA head configs, full-chunk and partial-chunk -sequence lengths, ragged varlen batches (cu_seqlens), zero-length sequences, -explicit/auto scale, alpha (decay gate) and beta (write strength) on/off, -chunked prefill with state carry-over, initial-state (dh0) and final-state -(dht) gradient paths, and head-dim variants including K != V and K = 256. -""" - -from __future__ import annotations - -import math -import random - -import pytest -import torch - -from ..common import ( - BWD_TOL, - FWD_TOL, - GDN_MARKS, - HEAD_CONFIGS, - HEAD_CONFIGS_SMALL, - run_gdn, -) -from ..conftest import gen_gates, gen_qkv -from ..reference_gdn import gdn_reference, rms_ratio - -pytestmark = GDN_MARKS - -_SEED = 42 - - -def _seed_all(seed=_SEED): - random.seed(seed) - torch.random.manual_seed(seed) - torch.cuda.manual_seed(seed) - - -def _run_bprop_case( - dtype, - H, - HV, - B, - T, - K=128, - V=128, - scale=None, - alpha=True, - beta=True, - initial_state=False, - state_grad=False, - cu_seqlens=None, - total_T=None, -): - """Run kernel fwd+bwd and reference fwd+bwd with a shared upstream - gradient; assert forward parity and per-input gradient parity.""" - _seed_all() - Teff = total_T or T - q0, k0, v0 = gen_qkv(B, Teff, H, HV, K, V, dtype) - g0, b0 = gen_gates(B, Teff, HV, dtype, alpha=alpha, beta=beta) - - S0_data = None - if initial_state: - N = B if cu_seqlens is None else cu_seqlens.numel() - 1 - S0_data = torch.randn(N, HV, K, V, device="cuda", dtype=torch.float32) * 0.05 - - w = torch.randn(B, Teff, HV, V, device="cuda", dtype=torch.float32) - if state_grad: - N = B if cu_seqlens is None else cu_seqlens.numel() - 1 - wf = torch.randn(N, HV, K, V, device="cuda", dtype=torch.float32) - - # --- kernel --- - leaves = { - "q": q0.clone().requires_grad_(), - "k": k0.clone().requires_grad_(), - "v": v0.clone().requires_grad_(), - "g": g0.clone().requires_grad_(), - "beta": b0.clone().requires_grad_(), - } - S0 = S0_data.clone().requires_grad_() if initial_state else None - o, fs = run_gdn( - leaves["q"], - leaves["k"], - leaves["v"], - leaves["g"], - leaves["beta"], - scale=scale, - initial_state=S0, - output_final_state=state_grad, - cu_seqlens=cu_seqlens, - ) - loss = (o.float() * w).sum() - if state_grad: - loss = loss + (fs.float() * wf).sum() - loss.backward() - torch.cuda.synchronize() - - # --- fp64 reference --- - ref_leaves = {n: t.detach().double().requires_grad_() for n, t in leaves.items()} - S0_ref = S0_data.detach().double().requires_grad_() if initial_state else None - o_ref, fs_ref = gdn_reference( - ref_leaves["q"], - ref_leaves["k"], - ref_leaves["v"], - ref_leaves["g"], - ref_leaves["beta"], - scale=scale, - initial_state=S0_ref, - cu_seqlens=cu_seqlens, - ) - loss_ref = (o_ref * w.double()).sum() - if state_grad: - loss_ref = loss_ref + (fs_ref * wf.double()).sum() - loss_ref.backward() - - r_o = rms_ratio(o, o_ref) - assert r_o < FWD_TOL[dtype], f"forward o rms ratio {r_o:.4g} >= {FWD_TOL[dtype]}" - - pairs = [(f"d{n}", leaves[n].grad, ref_leaves[n].grad) for n in leaves] - if initial_state: - pairs.append(("dh0", S0.grad, S0_ref.grad)) - for name, got, ref in pairs: - assert got is not None, f"no gradient for {name}" - assert torch.isfinite(got).all(), f"non-finite gradient for {name}" - r = rms_ratio(got, ref) - assert r < BWD_TOL[dtype], f"{name} rms ratio {r:.4g} >= {BWD_TOL[dtype]}" - - -@pytest.mark.parametrize("alpha,beta", [(True, True), (True, False), (False, True)]) -@pytest.mark.parametrize("scale", [1.0, "auto"]) -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS) -@pytest.mark.parametrize("B,T", [(1, 64), (1, 128), (1, 256), (2, 256)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_basic(dtype, B, T, H, HV, scale, alpha, beta): - scale = 1.0 / math.sqrt(128) if scale == "auto" else scale - _run_bprop_case(dtype, H, HV, B, T, scale=scale, alpha=alpha, beta=beta) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (2, 4), (16, 64)]) -@pytest.mark.parametrize("T", [31, 61, 91, 121, 251]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_nonfull(dtype, T, H, HV): - """Backward through sequence lengths that are not chunk multiples.""" - _run_bprop_case(dtype, H, HV, 1, T) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (2, 4), (16, 64)]) -@pytest.mark.parametrize( - "seq_lens", - [[256, 256], [511, 501], [64, 128, 512], [31, 63, 93, 123, 150, 500], [255, 257]], -) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_varlen_ragged(dtype, seq_lens, H, HV): - """Backward through ragged varlen batches (cu_seqlens path).""" - bounds = [0] - for sl in seq_lens: - bounds.append(bounds[-1] + sl) - cu = torch.tensor(bounds, dtype=torch.int32, device="cuda") - _run_bprop_case(dtype, H, HV, 1, None, cu_seqlens=cu, total_T=bounds[-1]) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (16, 64)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_zero_length_sequence(dtype, H, HV, T=64): - """A zero-length sequence in the varlen batch must not perturb the - gradients of the others.""" - _seed_all() - q, k, v = gen_qkv(1, T, H, HV, 128, 128, dtype) - g, b = gen_gates(1, T, HV, dtype) - w = torch.randn(1, T, HV, 128, device="cuda", dtype=torch.float32) - - def run(cu): - leaves = { - "q": q.clone().requires_grad_(), - "k": k.clone().requires_grad_(), - "v": v.clone().requires_grad_(), - "g": g.clone().requires_grad_(), - "beta": b.clone().requires_grad_(), - } - o, _ = run_gdn(leaves["q"], leaves["k"], leaves["v"], leaves["g"], leaves["beta"], cu_seqlens=cu) - (o.float() * w).sum().backward() - torch.cuda.synchronize() - return {n: t.grad for n, t in leaves.items()} - - ref = run(torch.tensor([0, T], dtype=torch.int32, device="cuda")) - got = run(torch.tensor([0, T, T], dtype=torch.int32, device="cuda")) - for name in ref: - torch.testing.assert_close(got[name], ref[name], atol=1e-3, rtol=1e-3, msg=f"d{name} perturbed by the zero-length sequence") - - -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL) -@pytest.mark.parametrize("T1,T2", [(128, 128), (64, 192), (192, 121)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_chunked_prefill(dtype, T1, T2, H, HV, B=2, K=128, V=128): - """Backward through a two-phase prefill: the state carried from part 1 - into part 2 chains the gradient (dht -> dh0) across the boundary; the - leaf gradients must match a single-shot fp64 reference.""" - _seed_all() - T = T1 + T2 - q0, k0, v0 = gen_qkv(B, T, H, HV, K, V, dtype) - g0, b0 = gen_gates(B, T, HV, dtype) - w = torch.randn(B, T, HV, V, device="cuda", dtype=torch.float32) - - leaves = { - "q": q0.clone().requires_grad_(), - "k": k0.clone().requires_grad_(), - "v": v0.clone().requires_grad_(), - "g": g0.clone().requires_grad_(), - "beta": b0.clone().requires_grad_(), - } - - def part(t0, t1, S0, output_final_state): - return run_gdn( - leaves["q"][:, t0:t1], - leaves["k"][:, t0:t1], - leaves["v"][:, t0:t1], - leaves["g"][:, t0:t1], - leaves["beta"][:, t0:t1], - initial_state=S0, - output_final_state=output_final_state, - ) - - o1, fs1 = part(0, T1, None, True) - o2, _ = part(T1, T, fs1, False) - o = torch.cat([o1, o2], dim=1) - (o.float() * w).sum().backward() - torch.cuda.synchronize() - - ref_leaves = {n: t.detach().double().requires_grad_() for n, t in leaves.items()} - o_ref, _ = gdn_reference(ref_leaves["q"], ref_leaves["k"], ref_leaves["v"], ref_leaves["g"], ref_leaves["beta"]) - (o_ref * w.double()).sum().backward() - - # The carried state round-trips through the op's output dtype, so allow - # slightly more than the single-shot backward tolerance. - tol = 1.5 * BWD_TOL[dtype] - for name in leaves: - r = rms_ratio(leaves[name].grad, ref_leaves[name].grad) - assert r < tol, f"chunked-prefill d{name} rms ratio {r:.4g} >= {tol}" - - -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL) -@pytest.mark.parametrize("T", [128, 251]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_initial_state(dtype, T, H, HV): - """Gradient flow through a random (non-zero) initial recurrent state - (dh0 path, no final-state gradient).""" - _run_bprop_case(dtype, H, HV, 1, T, initial_state=True) - - -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL) -@pytest.mark.parametrize("T", [128, 192, 251]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_state_grads(dtype, T, H, HV): - """Initial-state gradient (dh0) and final-state gradient (dht) paths: - the loss includes the final state and the initial state requires grad.""" - _run_bprop_case(dtype, H, HV, 1, T, initial_state=True, state_grad=True) - - -@pytest.mark.parametrize("K,V", [(64, 64), (64, 128), (128, 128), (256, 128)]) -@pytest.mark.parametrize("T", [128, 251]) -@pytest.mark.parametrize("dtype", [torch.bfloat16]) -def test_bprop_head_dims(dtype, T, K, V, H=2, HV=2): - """Backward through head-dim variants: K != V and the K = 256 bound.""" - _run_bprop_case(dtype, H, HV, 1, T, K=K, V=V) diff --git a/test/python/linear_attention/cutile/test_gdn_fprop.py b/test/python/linear_attention/cutile/test_gdn_fprop.py deleted file mode 100644 index fad0ef7c5..000000000 --- a/test/python/linear_attention/cutile/test_gdn_fprop.py +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -""" -Forward (fprop) tests for the GDN cuTile kernel, validated against the fp64 -recurrent reference in ``reference_gdn``. - -Covers: fp16/bf16, MHA and GVA head configs, full-chunk and partial-chunk -sequence lengths, ragged varlen batches (cu_seqlens), zero-length sequences, -explicit/auto scale, alpha (decay gate) and beta (write strength) on/off, -chunked prefill with state carry-over, initial state, and head-dim variants -including K != V and K = 256. -""" - -from __future__ import annotations - -import math -import random - -import pytest -import torch - -from ..common import ( - FWD_TOL, - GDN_MARKS, - HEAD_CONFIGS, - HEAD_CONFIGS_SMALL, - STATE_TOL, - run_gdn, -) -from ..conftest import gen_gates, gen_qkv -from ..reference_gdn import gdn_reference, rms_ratio - -pytestmark = GDN_MARKS - -_SEED = 42 - - -def _seed_all(seed=_SEED): - random.seed(seed) - torch.random.manual_seed(seed) - torch.cuda.manual_seed(seed) - - -def _check_fwd(o, o_ref, dtype, what="o"): - assert torch.isfinite(o).all(), f"non-finite values in {what}" - r = rms_ratio(o, o_ref) - assert r < FWD_TOL[dtype], f"{what} rms ratio {r:.4g} >= {FWD_TOL[dtype]}" - - -def _check_state(fs, fs_ref, dtype): - assert torch.isfinite(fs).all(), "non-finite values in final_state" - r = rms_ratio(fs, fs_ref) - assert r < STATE_TOL[dtype], f"final_state rms ratio {r:.4g} >= {STATE_TOL[dtype]}" - - -def _run_fprop_case( - dtype, - H, - HV, - B, - T, - K=128, - V=128, - scale=None, - alpha=True, - beta=True, - initial_state=False, - cu_seqlens=None, - total_T=None, -): - """Build inputs, run the kernel, and compare o + final_state against the - fp64 recurrent reference. ``cu_seqlens`` implies a packed batch with - B == 1 and T == total_T.""" - _seed_all() - q, k, v = gen_qkv(B, total_T or T, H, HV, K, V, dtype) - g, b = gen_gates(B, total_T or T, HV, dtype, alpha=alpha, beta=beta) - - S0 = None - if initial_state: - N = B if cu_seqlens is None else cu_seqlens.numel() - 1 - S0 = torch.randn(N, HV, K, V, device="cuda", dtype=torch.float32) * 0.05 - - o, fs = run_gdn(q, k, v, g, b, scale=scale, initial_state=S0, output_final_state=True, cu_seqlens=cu_seqlens) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = gdn_reference(q, k, v, g, b, scale=scale, initial_state=S0, cu_seqlens=cu_seqlens) - - _check_fwd(o, o_ref, dtype) - assert fs is not None - _check_state(fs, fs_ref, dtype) - - -@pytest.mark.parametrize("beta", [False, True]) -@pytest.mark.parametrize("alpha", [False, True]) -@pytest.mark.parametrize("scale", [1.0, "auto"]) -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS) -@pytest.mark.parametrize("B,T", [(1, 64), (1, 128), (1, 256), (2, 256)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_basic(dtype, B, T, H, HV, scale, alpha, beta): - if not alpha and not beta: - pytest.skip("output amplitude grows unbounded along the token dimension") - scale = 1.0 / math.sqrt(128) if scale == "auto" else scale - _run_fprop_case(dtype, H, HV, B, T, scale=scale, alpha=alpha, beta=beta) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (2, 4), (16, 64)]) -@pytest.mark.parametrize("T", [31, 61, 91, 121, 251]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_nonfull(dtype, T, H, HV): - """Sequence lengths that are not a multiple of the 64-token chunk.""" - _run_fprop_case(dtype, H, HV, 1, T) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (2, 4), (16, 64)]) -@pytest.mark.parametrize( - "seq_lens", - [[256, 256], [511, 501], [64, 128, 512], [31, 63, 93, 123, 150, 500]], -) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_varlen_ragged(dtype, seq_lens, H, HV): - """Ragged multi-sequence batches through the packed cu_seqlens path.""" - bounds = [0] - for sl in seq_lens: - bounds.append(bounds[-1] + sl) - cu = torch.tensor(bounds, dtype=torch.int32, device="cuda") - _run_fprop_case(dtype, H, HV, 1, None, cu_seqlens=cu, total_T=bounds[-1]) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (16, 64)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_zero_length_sequence(dtype, H, HV, T=64): - """A zero-length sequence in the varlen batch must not perturb the others.""" - _seed_all() - q, k, v = gen_qkv(1, T, H, HV, 128, 128, dtype) - g, b = gen_gates(1, T, HV, dtype) - cu = torch.tensor([0, T], dtype=torch.int32, device="cuda") - cu_with_empty = torch.tensor([0, T, T], dtype=torch.int32, device="cuda") - - o_ref, fs_ref = run_gdn(q, k, v, g, b, output_final_state=True, cu_seqlens=cu) - o, fs = run_gdn(q, k, v, g, b, output_final_state=True, cu_seqlens=cu_with_empty) - torch.cuda.synchronize() - - torch.testing.assert_close(o, o_ref, atol=1e-3, rtol=1e-3) - torch.testing.assert_close(fs[0], fs_ref[0], atol=1e-3, rtol=1e-3) - assert (fs[1] == 0).all(), "state of a zero-length sequence must stay zero" - - -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL) -@pytest.mark.parametrize("T1,T2", [(128, 128), (64, 192), (192, 121)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_chunked_prefill(dtype, T1, T2, H, HV, B=2, K=128, V=128): - """Two-phase prefill: the final state of part 1 feeds part 2 as its - initial state; the concatenated output must match a single-shot run.""" - _seed_all() - T = T1 + T2 - q, k, v = gen_qkv(B, T, H, HV, K, V, dtype) - g, b = gen_gates(B, T, HV, dtype) - - def part(t0, t1, S0): - return run_gdn( - q[:, t0:t1].contiguous(), - k[:, t0:t1].contiguous(), - v[:, t0:t1].contiguous(), - g[:, t0:t1].contiguous(), - b[:, t0:t1].contiguous(), - initial_state=S0, - output_final_state=True, - ) - - o1, fs1 = part(0, T1, None) - o2, fs2 = part(T1, T, fs1) - o = torch.cat([o1, o2], dim=1) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = gdn_reference(q, k, v, g, b) - - # The state round-trips through the op's output dtype between the two - # calls, so allow slightly more than the single-shot forward tolerance. - tol = 1.5 * FWD_TOL[dtype] - r_o = rms_ratio(o, o_ref) - r_s = rms_ratio(fs2, fs_ref) - assert r_o < tol, f"chunked-prefill o rms ratio {r_o:.4g} >= {tol}" - assert r_s < tol, f"chunked-prefill final_state rms ratio {r_s:.4g} >= {tol}" - - -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL) -@pytest.mark.parametrize("T", [128, 251]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_initial_state(dtype, T, H, HV): - """Random (non-zero) initial recurrent state.""" - _run_fprop_case(dtype, H, HV, 1, T, initial_state=True) - - -@pytest.mark.parametrize("K,V", [(64, 64), (64, 128), (128, 128), (256, 128)]) -@pytest.mark.parametrize("T", [128, 251]) -@pytest.mark.parametrize("dtype", [torch.bfloat16]) -def test_fprop_head_dims(dtype, T, K, V, H=2, HV=2): - """Head-dim variants: K != V and the K = 256 upper bound.""" - _run_fprop_case(dtype, H, HV, 1, T, K=K, V=V) - - -# --------------------------------------------------------------------------- -# Argument validation -# --------------------------------------------------------------------------- - - -def _dummy_inputs(H=2, HV=2, T=64, K=128, V=128): - q, k, v = gen_qkv(1, T, H, HV, K, V, torch.bfloat16) - g, b = gen_gates(1, T, HV, torch.bfloat16) - return q[0], k[0], v[0], g[0], b[0] - - -def _cu(*bounds): - return torch.tensor(bounds, dtype=torch.int32, device="cuda") - - -def test_invalid_qk_head_mismatch(): - q, k, v, g, b = _dummy_inputs() - with pytest.raises(Exception, match="head|No valid engine"): - run_gdn(q.unsqueeze(0), k[:, :1].unsqueeze(0), v.unsqueeze(0), g.unsqueeze(0), b.unsqueeze(0), cu_seqlens=_cu(0, 64)) - - -def test_invalid_gva_head_ratio(): - q, k, v, g, b = _dummy_inputs(H=2, HV=2) - with pytest.raises(Exception, match="divisible|multiple|head|No valid engine"): - run_gdn(q.unsqueeze(0), k.unsqueeze(0), v.repeat(1, 3, 1)[:, :3].unsqueeze(0), g.unsqueeze(0), b.unsqueeze(0), cu_seqlens=_cu(0, 64)) - - -def test_invalid_initial_state_count(): - q, k, v, g, b = _dummy_inputs(T=128) - S0 = torch.zeros(1, 2, 128, 128, device="cuda", dtype=torch.float32) - with pytest.raises(Exception, match="initial"): - run_gdn(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), g.unsqueeze(0), b.unsqueeze(0), cu_seqlens=_cu(0, 64, 128), initial_state=S0) diff --git a/test/python/linear_attention/cutile/test_kda_bprop.py b/test/python/linear_attention/cutile/test_kda_bprop.py deleted file mode 100644 index 658c032c7..000000000 --- a/test/python/linear_attention/cutile/test_kda_bprop.py +++ /dev/null @@ -1,273 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -""" -Backward (bprop) tests for the KDA cuTile kernel: gradients of q, k, v, g, -beta (and the initial state) are compared against autograd through the fp64 -recurrent reference in ``reference_kda`` using a shared random upstream -gradient. - -Covers: fp16/bf16, MHA and GVA head configs, full-chunk and partial-chunk -sequence lengths, ragged varlen batches (cu_seqlens), zero-length sequences, -explicit/auto scale, alpha (per-channel decay gate) and beta (write strength) -on/off, chunked prefill with state carry-over, initial-state (dh0) and -final-state (dht) gradient paths, and head-dim variants including K != V and -K = 256. -""" - -from __future__ import annotations - -import math -import random - -import pytest -import torch - -from ..common import ( - BWD_TOL, - FWD_TOL, - HEAD_CONFIGS, - HEAD_CONFIGS_SMALL, - KDA_MARKS, - run_kda, -) -from ..conftest import gen_kda_gates, gen_qkv -from ..reference_kda import kda_reference, rms_ratio - -pytestmark = KDA_MARKS - -_SEED = 42 - - -def _seed_all(seed=_SEED): - random.seed(seed) - torch.random.manual_seed(seed) - torch.cuda.manual_seed(seed) - - -def _run_bprop_case( - dtype, - H, - HV, - B, - T, - K=128, - V=128, - scale=None, - alpha=True, - beta=True, - initial_state=False, - state_grad=False, - cu_seqlens=None, - total_T=None, -): - """Run kernel fwd+bwd and reference fwd+bwd with a shared upstream - gradient; assert forward parity and per-input gradient parity.""" - _seed_all() - Teff = total_T or T - q0, k0, v0 = gen_qkv(B, Teff, H, HV, K, V, dtype) - g0, b0 = gen_kda_gates(B, Teff, HV, K, dtype, alpha=alpha, beta=beta) - - S0_data = None - if initial_state: - N = B if cu_seqlens is None else cu_seqlens.numel() - 1 - S0_data = torch.randn(N, HV, K, V, device="cuda", dtype=torch.float32) * 0.05 - - w = torch.randn(B, Teff, HV, V, device="cuda", dtype=torch.float32) - if state_grad: - N = B if cu_seqlens is None else cu_seqlens.numel() - 1 - wf = torch.randn(N, HV, K, V, device="cuda", dtype=torch.float32) - - # --- kernel --- - leaves = { - "q": q0.clone().requires_grad_(), - "k": k0.clone().requires_grad_(), - "v": v0.clone().requires_grad_(), - "g": g0.clone().requires_grad_(), - "beta": b0.clone().requires_grad_(), - } - S0 = S0_data.clone().requires_grad_() if initial_state else None - o, fs = run_kda( - leaves["q"], - leaves["k"], - leaves["v"], - leaves["g"], - leaves["beta"], - scale=scale, - initial_state=S0, - output_final_state=state_grad, - cu_seqlens=cu_seqlens, - ) - loss = (o.float() * w).sum() - if state_grad: - loss = loss + (fs.float() * wf).sum() - loss.backward() - torch.cuda.synchronize() - - # --- fp64 reference --- - ref_leaves = {n: t.detach().double().requires_grad_() for n, t in leaves.items()} - S0_ref = S0_data.detach().double().requires_grad_() if initial_state else None - o_ref, fs_ref = kda_reference( - ref_leaves["q"], - ref_leaves["k"], - ref_leaves["v"], - ref_leaves["g"], - ref_leaves["beta"], - scale=scale, - initial_state=S0_ref, - cu_seqlens=cu_seqlens, - ) - loss_ref = (o_ref * w.double()).sum() - if state_grad: - loss_ref = loss_ref + (fs_ref * wf.double()).sum() - loss_ref.backward() - - r_o = rms_ratio(o, o_ref) - assert r_o < FWD_TOL[dtype], f"forward o rms ratio {r_o:.4g} >= {FWD_TOL[dtype]}" - - pairs = [(f"d{n}", leaves[n].grad, ref_leaves[n].grad) for n in leaves] - if initial_state: - pairs.append(("dh0", S0.grad, S0_ref.grad)) - for name, got, ref in pairs: - assert got is not None, f"no gradient for {name}" - assert torch.isfinite(got).all(), f"non-finite gradient for {name}" - r = rms_ratio(got, ref) - assert r < BWD_TOL[dtype], f"{name} rms ratio {r:.4g} >= {BWD_TOL[dtype]}" - - -@pytest.mark.parametrize("alpha,beta", [(True, True), (True, False), (False, True)]) -@pytest.mark.parametrize("scale", [1.0, "auto"]) -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS) -@pytest.mark.parametrize("B,T", [(1, 64), (1, 128), (1, 256), (2, 256)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_basic(dtype, B, T, H, HV, scale, alpha, beta): - scale = 1.0 / math.sqrt(128) if scale == "auto" else scale - _run_bprop_case(dtype, H, HV, B, T, scale=scale, alpha=alpha, beta=beta) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (2, 4), (16, 64)]) -@pytest.mark.parametrize("T", [31, 61, 91, 121, 251]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_nonfull(dtype, T, H, HV): - """Backward through sequence lengths that are not chunk multiples.""" - _run_bprop_case(dtype, H, HV, 1, T) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (2, 4), (16, 64)]) -@pytest.mark.parametrize( - "seq_lens", - [[256, 256], [511, 501], [64, 128, 512], [31, 63, 93, 123, 150, 500], [255, 257]], -) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_varlen_ragged(dtype, seq_lens, H, HV): - """Backward through ragged varlen batches (cu_seqlens path).""" - bounds = [0] - for sl in seq_lens: - bounds.append(bounds[-1] + sl) - cu = torch.tensor(bounds, dtype=torch.int32, device="cuda") - _run_bprop_case(dtype, H, HV, 1, None, cu_seqlens=cu, total_T=bounds[-1]) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (16, 64)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_zero_length_sequence(dtype, H, HV, T=64): - """A zero-length sequence in the varlen batch must not perturb the - gradients of the others.""" - _seed_all() - q, k, v = gen_qkv(1, T, H, HV, 128, 128, dtype) - g, b = gen_kda_gates(1, T, HV, 128, dtype) - w = torch.randn(1, T, HV, 128, device="cuda", dtype=torch.float32) - - def run(cu): - leaves = { - "q": q.clone().requires_grad_(), - "k": k.clone().requires_grad_(), - "v": v.clone().requires_grad_(), - "g": g.clone().requires_grad_(), - "beta": b.clone().requires_grad_(), - } - o, _ = run_kda(leaves["q"], leaves["k"], leaves["v"], leaves["g"], leaves["beta"], cu_seqlens=cu) - (o.float() * w).sum().backward() - torch.cuda.synchronize() - return {n: t.grad for n, t in leaves.items()} - - ref = run(torch.tensor([0, T], dtype=torch.int32, device="cuda")) - got = run(torch.tensor([0, T, T], dtype=torch.int32, device="cuda")) - for name in ref: - torch.testing.assert_close(got[name], ref[name], atol=1e-3, rtol=1e-3, msg=f"d{name} perturbed by the zero-length sequence") - - -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL) -@pytest.mark.parametrize("T1,T2", [(128, 128), (64, 192), (192, 121)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_chunked_prefill(dtype, T1, T2, H, HV, B=2, K=128, V=128): - """Backward through a two-phase prefill: the state carried from part 1 - into part 2 chains the gradient (dht -> dh0) across the boundary; the - leaf gradients must match a single-shot fp64 reference.""" - _seed_all() - T = T1 + T2 - q0, k0, v0 = gen_qkv(B, T, H, HV, K, V, dtype) - g0, b0 = gen_kda_gates(B, T, HV, K, dtype) - w = torch.randn(B, T, HV, V, device="cuda", dtype=torch.float32) - - leaves = { - "q": q0.clone().requires_grad_(), - "k": k0.clone().requires_grad_(), - "v": v0.clone().requires_grad_(), - "g": g0.clone().requires_grad_(), - "beta": b0.clone().requires_grad_(), - } - - def part(t0, t1, S0, output_final_state): - return run_kda( - leaves["q"][:, t0:t1], - leaves["k"][:, t0:t1], - leaves["v"][:, t0:t1], - leaves["g"][:, t0:t1], - leaves["beta"][:, t0:t1], - initial_state=S0, - output_final_state=output_final_state, - ) - - o1, fs1 = part(0, T1, None, True) - o2, _ = part(T1, T, fs1, False) - o = torch.cat([o1, o2], dim=1) - (o.float() * w).sum().backward() - torch.cuda.synchronize() - - ref_leaves = {n: t.detach().double().requires_grad_() for n, t in leaves.items()} - o_ref, _ = kda_reference(ref_leaves["q"], ref_leaves["k"], ref_leaves["v"], ref_leaves["g"], ref_leaves["beta"]) - (o_ref * w.double()).sum().backward() - - # The carried state round-trips through the op's output dtype, so allow - # slightly more than the single-shot backward tolerance. - tol = 1.5 * BWD_TOL[dtype] - for name in leaves: - r = rms_ratio(leaves[name].grad, ref_leaves[name].grad) - assert r < tol, f"chunked-prefill d{name} rms ratio {r:.4g} >= {tol}" - - -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL) -@pytest.mark.parametrize("T", [128, 251]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_initial_state(dtype, T, H, HV): - """Gradient flow through a random (non-zero) initial recurrent state - (dh0 path, no final-state gradient).""" - _run_bprop_case(dtype, H, HV, 1, T, initial_state=True) - - -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL) -@pytest.mark.parametrize("T", [128, 192, 251]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_bprop_state_grads(dtype, T, H, HV): - """Initial-state gradient (dh0) and final-state gradient (dht) paths: - the loss includes the final state and the initial state requires grad.""" - _run_bprop_case(dtype, H, HV, 1, T, initial_state=True, state_grad=True) - - -@pytest.mark.parametrize("K,V", [(64, 64), (64, 128), (128, 128), (256, 128)]) -@pytest.mark.parametrize("T", [128, 251]) -@pytest.mark.parametrize("dtype", [torch.bfloat16]) -def test_bprop_head_dims(dtype, T, K, V, H=2, HV=2): - """Backward through head-dim variants: K != V and the K = 256 bound.""" - _run_bprop_case(dtype, H, HV, 1, T, K=K, V=V) diff --git a/test/python/linear_attention/cutile/test_kda_fprop.py b/test/python/linear_attention/cutile/test_kda_fprop.py deleted file mode 100644 index e7ce2ed3d..000000000 --- a/test/python/linear_attention/cutile/test_kda_fprop.py +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -""" -Forward (fprop) tests for the KDA cuTile kernel, validated against the fp64 -recurrent reference in ``reference_kda``. - -Covers: fp16/bf16, MHA and GVA head configs, full-chunk and partial-chunk -sequence lengths, ragged varlen batches (cu_seqlens), zero-length sequences, -explicit/auto scale, alpha (per-channel decay gate) and beta (write strength) -on/off, chunked prefill with state carry-over, initial state, and head-dim -variants including K != V and K = 256. -""" - -from __future__ import annotations - -import math -import random - -import pytest -import torch - -from ..common import ( - FWD_TOL, - HEAD_CONFIGS, - HEAD_CONFIGS_SMALL, - KDA_MARKS, - STATE_TOL, - run_kda, -) -from ..conftest import gen_kda_gates, gen_qkv -from ..reference_kda import kda_reference, rms_ratio - -pytestmark = KDA_MARKS - -_SEED = 42 - - -def _seed_all(seed=_SEED): - random.seed(seed) - torch.random.manual_seed(seed) - torch.cuda.manual_seed(seed) - - -def _check_fwd(o, o_ref, dtype, what="o"): - assert torch.isfinite(o).all(), f"non-finite values in {what}" - r = rms_ratio(o, o_ref) - assert r < FWD_TOL[dtype], f"{what} rms ratio {r:.4g} >= {FWD_TOL[dtype]}" - - -def _check_state(fs, fs_ref, dtype): - assert torch.isfinite(fs).all(), "non-finite values in final_state" - r = rms_ratio(fs, fs_ref) - assert r < STATE_TOL[dtype], f"final_state rms ratio {r:.4g} >= {STATE_TOL[dtype]}" - - -def _run_fprop_case( - dtype, - H, - HV, - B, - T, - K=128, - V=128, - scale=None, - alpha=True, - beta=True, - initial_state=False, - cu_seqlens=None, - total_T=None, -): - """Build inputs, run the kernel, and compare o + final_state against the - fp64 recurrent reference. ``cu_seqlens`` implies a packed batch with - B == 1 and T == total_T.""" - _seed_all() - q, k, v = gen_qkv(B, total_T or T, H, HV, K, V, dtype) - g, b = gen_kda_gates(B, total_T or T, HV, K, dtype, alpha=alpha, beta=beta) - - S0 = None - if initial_state: - N = B if cu_seqlens is None else cu_seqlens.numel() - 1 - S0 = torch.randn(N, HV, K, V, device="cuda", dtype=torch.float32) * 0.05 - - o, fs = run_kda(q, k, v, g, b, scale=scale, initial_state=S0, output_final_state=True, cu_seqlens=cu_seqlens) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = kda_reference(q, k, v, g, b, scale=scale, initial_state=S0, cu_seqlens=cu_seqlens) - - _check_fwd(o, o_ref, dtype) - assert fs is not None - _check_state(fs, fs_ref, dtype) - - -@pytest.mark.parametrize("beta", [False, True]) -@pytest.mark.parametrize("alpha", [False, True]) -@pytest.mark.parametrize("scale", [1.0, "auto"]) -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS) -@pytest.mark.parametrize("B,T", [(1, 64), (1, 128), (1, 256), (2, 256)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_basic(dtype, B, T, H, HV, scale, alpha, beta): - if not alpha and not beta: - pytest.skip("output amplitude grows unbounded along the token dimension") - scale = 1.0 / math.sqrt(128) if scale == "auto" else scale - _run_fprop_case(dtype, H, HV, B, T, scale=scale, alpha=alpha, beta=beta) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (2, 4), (16, 64)]) -@pytest.mark.parametrize("T", [31, 61, 91, 121, 251]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_nonfull(dtype, T, H, HV): - """Sequence lengths that are not a multiple of the 64-token chunk.""" - _run_fprop_case(dtype, H, HV, 1, T) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (2, 4), (16, 64)]) -@pytest.mark.parametrize( - "seq_lens", - [[256, 256], [511, 501], [64, 128, 512], [31, 63, 93, 123, 150, 500]], -) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_varlen_ragged(dtype, seq_lens, H, HV): - """Ragged multi-sequence batches through the packed cu_seqlens path.""" - bounds = [0] - for sl in seq_lens: - bounds.append(bounds[-1] + sl) - cu = torch.tensor(bounds, dtype=torch.int32, device="cuda") - _run_fprop_case(dtype, H, HV, 1, None, cu_seqlens=cu, total_T=bounds[-1]) - - -@pytest.mark.parametrize("H,HV", [(1, 1), (16, 64)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_zero_length_sequence(dtype, H, HV, T=64): - """A zero-length sequence in the varlen batch must not perturb the others.""" - _seed_all() - q, k, v = gen_qkv(1, T, H, HV, 128, 128, dtype) - g, b = gen_kda_gates(1, T, HV, 128, dtype) - cu = torch.tensor([0, T], dtype=torch.int32, device="cuda") - cu_with_empty = torch.tensor([0, T, T], dtype=torch.int32, device="cuda") - - o_ref, fs_ref = run_kda(q, k, v, g, b, output_final_state=True, cu_seqlens=cu) - o, fs = run_kda(q, k, v, g, b, output_final_state=True, cu_seqlens=cu_with_empty) - torch.cuda.synchronize() - - torch.testing.assert_close(o, o_ref, atol=1e-3, rtol=1e-3) - torch.testing.assert_close(fs[0], fs_ref[0], atol=1e-3, rtol=1e-3) - assert (fs[1] == 0).all(), "state of a zero-length sequence must stay zero" - - -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL) -@pytest.mark.parametrize("T1,T2", [(128, 128), (64, 192), (192, 121)]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_chunked_prefill(dtype, T1, T2, H, HV, B=2, K=128, V=128): - """Two-phase prefill: the final state of part 1 feeds part 2 as its - initial state; the concatenated output must match a single-shot run.""" - _seed_all() - T = T1 + T2 - q, k, v = gen_qkv(B, T, H, HV, K, V, dtype) - g, b = gen_kda_gates(B, T, HV, K, dtype) - - def part(t0, t1, S0): - return run_kda( - q[:, t0:t1].contiguous(), - k[:, t0:t1].contiguous(), - v[:, t0:t1].contiguous(), - g[:, t0:t1].contiguous(), - b[:, t0:t1].contiguous(), - initial_state=S0, - output_final_state=True, - ) - - o1, fs1 = part(0, T1, None) - o2, fs2 = part(T1, T, fs1) - o = torch.cat([o1, o2], dim=1) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = kda_reference(q, k, v, g, b) - - # The state round-trips through the op's output dtype between the two - # calls, so allow slightly more than the single-shot forward tolerance. - tol = 1.5 * FWD_TOL[dtype] - r_o = rms_ratio(o, o_ref) - r_s = rms_ratio(fs2, fs_ref) - assert r_o < tol, f"chunked-prefill o rms ratio {r_o:.4g} >= {tol}" - assert r_s < tol, f"chunked-prefill final_state rms ratio {r_s:.4g} >= {tol}" - - -@pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL) -@pytest.mark.parametrize("T", [128, 251]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_fprop_initial_state(dtype, T, H, HV): - """Random (non-zero) initial recurrent state.""" - _run_fprop_case(dtype, H, HV, 1, T, initial_state=True) - - -@pytest.mark.parametrize("K,V", [(64, 64), (64, 128), (128, 128), (256, 128)]) -@pytest.mark.parametrize("T", [128, 251]) -@pytest.mark.parametrize("dtype", [torch.bfloat16]) -def test_fprop_head_dims(dtype, T, K, V, H=2, HV=2): - """Head-dim variants: K != V and the K = 256 upper bound.""" - _run_fprop_case(dtype, H, HV, 1, T, K=K, V=V) - - -# --------------------------------------------------------------------------- -# Argument validation -# --------------------------------------------------------------------------- - - -def _dummy_inputs(H=2, HV=2, T=64, K=128, V=128): - q, k, v = gen_qkv(1, T, H, HV, K, V, torch.bfloat16) - g, b = gen_kda_gates(1, T, HV, K, torch.bfloat16) - return q[0], k[0], v[0], g[0], b[0] - - -def _cu(*bounds): - return torch.tensor(bounds, dtype=torch.int32, device="cuda") - - -def test_invalid_qk_head_mismatch(): - q, k, v, g, b = _dummy_inputs() - with pytest.raises(Exception, match="head|No valid engine"): - run_kda(q.unsqueeze(0), k[:, :1].unsqueeze(0), v.unsqueeze(0), g.unsqueeze(0), b.unsqueeze(0), cu_seqlens=_cu(0, 64)) - - -def test_invalid_gva_head_ratio(): - q, k, v, g, b = _dummy_inputs(H=2, HV=2) - with pytest.raises(Exception, match="divisible|multiple|head|No valid engine"): - run_kda(q.unsqueeze(0), k.unsqueeze(0), v.repeat(1, 3, 1)[:, :3].unsqueeze(0), g.unsqueeze(0), b.unsqueeze(0), cu_seqlens=_cu(0, 64)) - - -def test_invalid_initial_state_count(): - q, k, v, g, b = _dummy_inputs(T=128) - S0 = torch.zeros(1, 2, 128, 128, device="cuda", dtype=torch.float32) - with pytest.raises(Exception, match="initial"): - run_kda(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), g.unsqueeze(0), b.unsqueeze(0), cu_seqlens=_cu(0, 64, 128), initial_state=S0) diff --git a/test/python/linear_attention/frost/__init__.py b/test/python/linear_attention/frost/__init__.py deleted file mode 100644 index 52a7a9daf..000000000 --- a/test/python/linear_attention/frost/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 diff --git a/test/python/linear_attention/frost/conftest.py b/test/python/linear_attention/frost/conftest.py deleted file mode 100644 index 3a76f4b73..000000000 --- a/test/python/linear_attention/frost/conftest.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Make ``cudnn.linear_attention.frost`` importable: append the source ``python/cudnn`` dir -to the installed ``cudnn`` package's ``__path__`` (the wheel lacks the ``FROST`` -subtree). Unnecessary once the engine ships in the built frontend package.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -import cudnn - -_SRC_CUDNN = Path(__file__).resolve().parents[4] / "python" / "cudnn" -if _SRC_CUDNN.is_dir() and str(_SRC_CUDNN) not in cudnn.__path__: - cudnn.__path__.append(str(_SRC_CUDNN)) - -# The shared helpers live in the linear_attention test package root -# (test/python/linear_attention); pytest only prepends this file's own directory. -_TEST_PY = Path(__file__).resolve().parents[2] -if str(_TEST_PY) not in sys.path: - sys.path.insert(0, str(_TEST_PY)) diff --git a/test/python/linear_attention/frost/examples/01_gdn_prefill.py b/test/python/linear_attention/frost/examples/01_gdn_prefill.py new file mode 100644 index 000000000..547300e49 --- /dev/null +++ b/test/python/linear_attention/frost/examples/01_gdn_prefill.py @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Example 01: GDN (Gated DeltaNet) prefill (pure cuDNN frontend API). + +Per-token recurrence with scalar decay ``alpha_t = exp(g_t)`` and write +strength ``beta_t``:: + + S_t = alpha_t (I - beta_t k_t^T k_t) S_{t-1} + beta_t k_t^T v_t + o_t = q_t S_t + +THD layout: token-packed ``[total, H, D]`` tensors plus ``cu_seqlens`` +sequence boundaries; the final state comes back K-major ``[N, H, K, V]``. +""" + +from __future__ import annotations + +import math + +import cudnn +import torch + + +def _build_plans(g) -> None: + g.validate() + g.build_operation_graph() + g.create_execution_plans([cudnn.heur_mode.A]) + names = [g.get_plan_name_at_index(i) for i in range(len(g.plans))] + g.select_plan(names.index("gdn_frost")) # pin the FROST entry + g.check_support() + g.build_plans() + + +def _rms_ratio(out, ref): + out, ref = out.detach().double(), ref.detach().double() + return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() + + +def _reference(q, k, v, g, beta, cu, scale): + """fp64 per-token recurrence over the packed batch. Returns (o, final_state).""" + total, H, D = q.shape + V = v.shape[2] + q, k, v, g, beta = (x.double() for x in (q, k, v, g, beta)) + o = torch.zeros(total, H, V, dtype=torch.float64, device=q.device) + fs = torch.zeros(cu.numel() - 1, H, D, V, dtype=torch.float64, device=q.device) + for n in range(cu.numel() - 1): + S = torch.zeros(H, D, V, dtype=torch.float64, device=q.device) + for t in range(int(cu[n]), int(cu[n + 1])): + a, b = g[t].exp(), beta[t] # [H] scalar decay / write strength + residual = v[t] - a[:, None] * torch.einsum("hd,hdv->hv", k[t], S) + S = a[:, None, None] * S + b[:, None, None] * torch.einsum("hd,hv->hdv", k[t], residual) + o[t] = torch.einsum("hd,hdv->hv", q[t] * scale, S) + fs[n] = S + return o, fs + + +def main(seq_lens=(192, 320), H: int = 2, D: int = 128) -> None: + torch.manual_seed(0) + device = "cuda" + total, num_seqs = sum(seq_lens), len(seq_lens) + scale = 1.0 / math.sqrt(D) + + q = torch.randn(total, H, D, device=device).bfloat16() + k = torch.nn.functional.normalize(torch.randn(total, H, D, device=device), dim=-1).bfloat16() + v = torch.randn(total, H, D, device=device).bfloat16() + gate = torch.empty(total, H, device=device).uniform_(0.1, 1.0).log().contiguous() + beta = torch.rand(total, H, device=device).contiguous() + cu = torch.tensor([0, *torch.tensor(seq_lens).cumsum(0).tolist()], dtype=torch.int32, device=device) + + g = cudnn.pygraph() + q_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="q") + k_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="k") + v_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="v") + g_t = g.tensor([total, H], data_type=cudnn.data_type.FLOAT, name="g") + beta_t = g.tensor([total, H], data_type=cudnn.data_type.FLOAT, name="beta") + cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") + O_t, fs_t, _h_t = g.gdn( + q=q_t, + k=k_t, + v=v_t, + g=g_t, + beta=beta_t, + cu_seqlens=cu_t, + scale=scale, + output_final_state=True, + name="gdn", + ) + O_t.set_output(True).set_data_type(cudnn.data_type.BFLOAT16) + fs_t.set_output(True).set_data_type(cudnn.data_type.FLOAT) + _build_plans(g) + + o = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + fs = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device=device) + pack = {q_t: q, k_t: k, v_t: v, g_t: gate, beta_t: beta, cu_t: cu, O_t: o, fs_t: fs} + g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) + torch.cuda.synchronize() + + o_ref, fs_ref = _reference(q, k, v, gate, beta, cu, scale) + r_o, r_s = _rms_ratio(o, o_ref), _rms_ratio(fs, fs_ref) + assert r_o < 2e-2, f"o rms ratio {r_o:.4g}" + assert r_s < 2e-2, f"final_state rms ratio {r_s:.4g}" + print(f"[01] PASS gdn prefill seq_lens={list(seq_lens)} H={H} D={D} (o rms {r_o:.2e}, fs rms {r_s:.2e})") + + +if __name__ == "__main__": + main() diff --git a/test/python/linear_attention/frost/examples/02_gdn_backward.py b/test/python/linear_attention/frost/examples/02_gdn_backward.py new file mode 100644 index 000000000..6cc05443b --- /dev/null +++ b/test/python/linear_attention/frost/examples/02_gdn_backward.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Example 02: GDN (Gated DeltaNet) backward (pure cuDNN frontend API). + +The GDN_BWD node takes the forward inputs plus ``dO`` and returns +``(dQ, dK, dV, dG, dBeta, dS0)``. Without the optional per-chunk ``h`` input +the engine recomputes the forward state pass internally. Gradients are +checked against fp64 autograd through the per-token recurrence. +""" + +from __future__ import annotations + +import math + +import cudnn +import torch + + +def _build_plans(g) -> None: + g.validate() + g.build_operation_graph() + g.create_execution_plans([cudnn.heur_mode.A]) + names = [g.get_plan_name_at_index(i) for i in range(len(g.plans))] + g.select_plan(names.index("gdn_frost")) # pin the FROST entry + g.check_support() + g.build_plans() + + +def _rms_ratio(out, ref): + out, ref = out.detach().double(), ref.detach().double() + return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() + + +def _reference_o(q, k, v, g, beta, cu, scale): + """Differentiable fp64 per-token recurrence; returns o.""" + total, H, _D = q.shape + V = v.shape[2] + outs = [] + for n in range(cu.numel() - 1): + S = torch.zeros(H, q.shape[2], V, dtype=torch.float64, device=q.device) + for t in range(int(cu[n]), int(cu[n + 1])): + a, b = g[t].exp(), beta[t] + residual = v[t] - a[:, None] * torch.einsum("hd,hdv->hv", k[t], S) + S = a[:, None, None] * S + b[:, None, None] * torch.einsum("hd,hv->hdv", k[t], residual) + outs.append(torch.einsum("hd,hdv->hv", q[t] * scale, S)) + return torch.stack(outs, dim=0) + + +def main(seq_lens=(192, 320), H: int = 2, D: int = 128) -> None: + torch.manual_seed(0) + device = "cuda" + total, num_seqs = sum(seq_lens), len(seq_lens) + scale = 1.0 / math.sqrt(D) + + q = torch.randn(total, H, D, device=device).bfloat16() + k = torch.nn.functional.normalize(torch.randn(total, H, D, device=device), dim=-1).bfloat16() + v = torch.randn(total, H, D, device=device).bfloat16() + gate = torch.empty(total, H, device=device).uniform_(0.1, 1.0).log().contiguous() + beta = torch.rand(total, H, device=device).contiguous() + do = torch.randn(total, H, D, device=device).bfloat16() + cu = torch.tensor([0, *torch.tensor(seq_lens).cumsum(0).tolist()], dtype=torch.int32, device=device) + + g = cudnn.pygraph() + q_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="q") + k_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="k") + v_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="v") + g_t = g.tensor([total, H], data_type=cudnn.data_type.FLOAT, name="g") + beta_t = g.tensor([total, H], data_type=cudnn.data_type.FLOAT, name="beta") + cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") + do_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="dO") + dQ_t, dK_t, dV_t, dG_t, dBeta_t, _dS0_t = g.gdn_bwd( + q=q_t, + k=k_t, + v=v_t, + g=g_t, + beta=beta_t, + cu_seqlens=cu_t, + dO=do_t, + scale=scale, + name="gdn_bwd", + ) + for t_, dt in ( + (dQ_t, cudnn.data_type.BFLOAT16), + (dK_t, cudnn.data_type.BFLOAT16), + (dV_t, cudnn.data_type.BFLOAT16), + (dG_t, cudnn.data_type.FLOAT), + (dBeta_t, cudnn.data_type.FLOAT), + ): + t_.set_output(True).set_data_type(dt) + _build_plans(g) + + dq = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + dk = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + dv = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + dg = torch.empty(total, H, dtype=torch.float32, device=device) + db = torch.empty(total, H, dtype=torch.float32, device=device) + pack = {q_t: q, k_t: k, v_t: v, g_t: gate, beta_t: beta, cu_t: cu, do_t: do, dQ_t: dq, dK_t: dk, dV_t: dv, dG_t: dg, dBeta_t: db} + g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) + torch.cuda.synchronize() + + leaves = [x.double().requires_grad_(True) for x in (q, k, v, gate, beta)] + o_ref = _reference_o(*leaves, cu, scale) + grads = torch.autograd.grad((o_ref * do.double()).sum(), leaves) + for name, out, ref, tol in ( + ("dQ", dq, grads[0], 5e-2), + ("dK", dk, grads[1], 5e-2), + ("dV", dv, grads[2], 5e-2), + ("dG", dg, grads[3], 5e-2), + ("dBeta", db, grads[4], 5e-2), + ): + r = _rms_ratio(out, ref) + assert r < tol, f"{name} rms ratio {r:.4g}" + print(f"[02] PASS gdn backward (recompute) seq_lens={list(seq_lens)} H={H} D={D}") + + +if __name__ == "__main__": + main() diff --git a/test/python/linear_attention/frost/examples/03_kda_prefill.py b/test/python/linear_attention/frost/examples/03_kda_prefill.py new file mode 100644 index 000000000..5bc6a698c --- /dev/null +++ b/test/python/linear_attention/frost/examples/03_kda_prefill.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Example 03: KDA (Kimi Delta Attention) prefill (pure cuDNN frontend API). + +KDA replaces GDN's scalar decay with a per-key-channel decay +``a_t = exp(g_t) in R^K`` (the decayed state feeds the delta-rule residual):: + + S' = Diag(a_t) S_{t-1} + S_t = S' + beta_t k_t^T (v_t - k_t S') + o_t = q_t S_t + +``use_qk_l2norm=False`` passes q/k through as given, so this example feeds +pre-normalized rows. +""" + +from __future__ import annotations + +import math + +import cudnn +import torch + + +def _build_plans(g) -> None: + g.validate() + g.build_operation_graph() + g.create_execution_plans([cudnn.heur_mode.A]) + names = [g.get_plan_name_at_index(i) for i in range(len(g.plans))] + g.select_plan(names.index("kda_frost")) # pin the FROST entry + g.check_support() + g.build_plans() + + +def _rms_ratio(out, ref): + out, ref = out.detach().double(), ref.detach().double() + return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() + + +def _randu(rows, dim, device): + """Per-row uniform [-0.25, 0.25) with normally-distributed means: mildly + heterogeneous data that keeps the recurrence stable.""" + means = torch.randn(rows, 1, device=device) * 0.05 + return means + torch.rand(rows, dim, device=device) * 0.5 - 0.25 + + +def _reference(q, k, v, g, beta, cu, scale): + """fp64 per-token recurrence over the packed batch. Returns (o, final_state).""" + total, H, D = q.shape + V = v.shape[2] + q, k, v, g, beta = (x.double() for x in (q, k, v, g, beta)) + o = torch.zeros(total, H, V, dtype=torch.float64, device=q.device) + fs = torch.zeros(cu.numel() - 1, H, D, V, dtype=torch.float64, device=q.device) + for n in range(cu.numel() - 1): + S = torch.zeros(H, D, V, dtype=torch.float64, device=q.device) + for t in range(int(cu[n]), int(cu[n + 1])): + S = g[t].exp()[..., None] * S # per-key-channel decay first + residual = v[t] - torch.einsum("hd,hdv->hv", k[t], S) + S = S + beta[t][:, None, None] * torch.einsum("hd,hv->hdv", k[t], residual) + o[t] = torch.einsum("hd,hdv->hv", q[t] * scale, S) + fs[n] = S + return o, fs + + +def main(seq_lens=(192, 320), H: int = 2, D: int = 128) -> None: + torch.manual_seed(0) + device = "cuda" + total, num_seqs = sum(seq_lens), len(seq_lens) + scale = 1.0 / math.sqrt(D) + + q = torch.nn.functional.normalize(_randu(total * H, D, device), dim=-1).reshape(total, H, D).bfloat16() + k = torch.nn.functional.normalize(_randu(total * H, D, device), dim=-1).reshape(total, H, D).bfloat16() + v = _randu(total * H, D, device).reshape(total, H, D).bfloat16() + gate = torch.empty(total, H, D, device=device).uniform_(0.5, 1.0).log().contiguous() + beta = torch.rand(total, H, device=device).sigmoid().contiguous() + cu = torch.tensor([0, *torch.tensor(seq_lens).cumsum(0).tolist()], dtype=torch.int32, device=device) + + g = cudnn.pygraph() + q_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="q") + k_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="k") + v_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="v") + g_t = g.tensor([total, H, D], data_type=cudnn.data_type.FLOAT, name="g") + beta_t = g.tensor([total, H], data_type=cudnn.data_type.FLOAT, name="beta") + cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") + O_t, fs_t, _h_t = g.kda( + q=q_t, + k=k_t, + v=v_t, + g=g_t, + beta=beta_t, + cu_seqlens=cu_t, + scale=scale, + output_final_state=True, + use_qk_l2norm=False, + name="kda", + ) + O_t.set_output(True).set_data_type(cudnn.data_type.BFLOAT16) + fs_t.set_output(True).set_data_type(cudnn.data_type.FLOAT) + _build_plans(g) + + o = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + fs = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device=device) + pack = {q_t: q, k_t: k, v_t: v, g_t: gate, beta_t: beta, cu_t: cu, O_t: o, fs_t: fs} + g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) + torch.cuda.synchronize() + + o_ref, fs_ref = _reference(q, k, v, gate, beta, cu, scale) + torch.testing.assert_close(o.float(), o_ref.float(), atol=1e-1, rtol=1e-1) + r_s = _rms_ratio(fs, fs_ref) + assert r_s < 5e-2, f"final_state rms ratio {r_s:.4g}" + print(f"[03] PASS kda prefill seq_lens={list(seq_lens)} H={H} D={D} (fs rms {r_s:.2e})") + + +if __name__ == "__main__": + main() diff --git a/test/python/linear_attention/frost/examples/04_kda_backward.py b/test/python/linear_attention/frost/examples/04_kda_backward.py new file mode 100644 index 000000000..4a8b24360 --- /dev/null +++ b/test/python/linear_attention/frost/examples/04_kda_backward.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Example 04: KDA (Kimi Delta Attention) backward (pure cuDNN frontend API). + +The KDA_BWD node takes the forward inputs plus ``dO`` and returns +``(dQ, dK, dV, dG, dBeta)`` (per-key-channel ``dG``). Without the optional +per-chunk ``h`` input the engine recomputes the forward state pass +internally. Gradients are checked against fp64 autograd through the +per-token recurrence. +""" + +from __future__ import annotations + +import math + +import cudnn +import torch + + +def _build_plans(g) -> None: + g.validate() + g.build_operation_graph() + g.create_execution_plans([cudnn.heur_mode.A]) + names = [g.get_plan_name_at_index(i) for i in range(len(g.plans))] + g.select_plan(names.index("kda_frost")) # pin the FROST entry + g.check_support() + g.build_plans() + + +def _rms_ratio(out, ref): + out, ref = out.detach().double(), ref.detach().double() + return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() + + +def _reference_o(q, k, v, g, beta, cu, scale): + """Differentiable fp64 per-token recurrence; returns o.""" + total, H, D = q.shape + V = v.shape[2] + outs = [] + for n in range(cu.numel() - 1): + S = torch.zeros(H, D, V, dtype=torch.float64, device=q.device) + for t in range(int(cu[n]), int(cu[n + 1])): + S = g[t].exp()[..., None] * S + residual = v[t] - torch.einsum("hd,hdv->hv", k[t], S) + S = S + beta[t][:, None, None] * torch.einsum("hd,hv->hdv", k[t], residual) + outs.append(torch.einsum("hd,hdv->hv", q[t] * scale, S)) + return torch.stack(outs, dim=0) + + +def main(seq_lens=(192, 320), H: int = 2, D: int = 128) -> None: + torch.manual_seed(0) + device = "cuda" + total, num_seqs = sum(seq_lens), len(seq_lens) + scale = 1.0 / math.sqrt(D) + + q = torch.nn.functional.normalize(torch.randn(total, H, D, device=device), dim=-1).bfloat16() + k = torch.nn.functional.normalize(torch.randn(total, H, D, device=device), dim=-1).bfloat16() + v = torch.randn(total, H, D, device=device).bfloat16() + gate = torch.empty(total, H, D, device=device).uniform_(0.5, 1.0).log().contiguous() + beta = torch.rand(total, H, device=device).sigmoid().contiguous() + do = torch.randn(total, H, D, device=device).bfloat16() + cu = torch.tensor([0, *torch.tensor(seq_lens).cumsum(0).tolist()], dtype=torch.int32, device=device) + + g = cudnn.pygraph() + q_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="q") + k_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="k") + v_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="v") + g_t = g.tensor([total, H, D], data_type=cudnn.data_type.FLOAT, name="g") + beta_t = g.tensor([total, H], data_type=cudnn.data_type.FLOAT, name="beta") + cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") + do_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="dO") + outs = g.kda_bwd( + q=q_t, + k=k_t, + v=v_t, + g=g_t, + beta=beta_t, + cu_seqlens=cu_t, + dO=do_t, + scale=scale, + use_qk_l2norm=False, + name="kda_bwd", + ) + dtypes = (cudnn.data_type.BFLOAT16,) * 3 + (cudnn.data_type.FLOAT,) * 2 + grads_t = [out.set_output(True).set_data_type(dt) for out, dt in zip(outs, dtypes)] + _build_plans(g) + + dq = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + dk = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + dv = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + dg = torch.empty(total, H, D, dtype=torch.float32, device=device) + db = torch.empty(total, H, dtype=torch.float32, device=device) + pack = {q_t: q, k_t: k, v_t: v, g_t: gate, beta_t: beta, cu_t: cu, do_t: do} + pack.update(dict(zip(grads_t, (dq, dk, dv, dg, db)))) + g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) + torch.cuda.synchronize() + + leaves = [x.double().requires_grad_(True) for x in (q, k, v, gate, beta)] + o_ref = _reference_o(*leaves, cu, scale) + grads = torch.autograd.grad((o_ref * do.double()).sum(), leaves) + for name, out, ref in (("dQ", dq, grads[0]), ("dK", dk, grads[1]), ("dV", dv, grads[2]), ("dG", dg, grads[3]), ("dBeta", db, grads[4])): + r = _rms_ratio(out, ref) + assert r < 5e-2, f"{name} rms ratio {r:.4g}" + print(f"[04] PASS kda backward (recompute) seq_lens={list(seq_lens)} H={H} D={D}") + + +if __name__ == "__main__": + main() diff --git a/test/python/linear_attention/frost/examples/05_gdn2_prefill.py b/test/python/linear_attention/frost/examples/05_gdn2_prefill.py new file mode 100644 index 000000000..7a370c792 --- /dev/null +++ b/test/python/linear_attention/frost/examples/05_gdn2_prefill.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Example 05: GDN-2 (Gated DeltaNet v2) prefill (pure cuDNN frontend API). + +GDN-2 gates every channel: per-key decay ``a_t = exp(g_t) in R^K``, per-key +erase gate ``beta_t in R^K``, and per-value write gate ``w_t in R^V``:: + + S' = Diag(a_t) S_{t-1} + S_t = S' + k_t^T (w_t . v_t - (beta_t . k_t) S') + o_t = q_t S_t + +``beta``/``w`` are io-dtype post-sigmoid tensors; ``use_qk_l2norm=False`` +passes q/k through as given, so this example feeds pre-normalized rows. +""" + +from __future__ import annotations + +import math + +import cudnn +import torch + + +def _build_plans(g) -> None: + g.validate() + g.build_operation_graph() + g.create_execution_plans([cudnn.heur_mode.A]) + names = [g.get_plan_name_at_index(i) for i in range(len(g.plans))] + g.select_plan(names.index("gdn2_frost")) # pin the FROST entry + g.check_support() + g.build_plans() + + +def _rms_ratio(out, ref): + out, ref = out.detach().double(), ref.detach().double() + return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() + + +def _randu(rows, dim, device): + """Per-row uniform [-0.25, 0.25) with normally-distributed means: mildly + heterogeneous data that keeps the recurrence stable.""" + means = torch.randn(rows, 1, device=device) * 0.05 + return means + torch.rand(rows, dim, device=device) * 0.5 - 0.25 + + +def _reference(q, k, v, g, beta, w, cu, scale): + """fp64 per-token recurrence over the packed batch. Returns (o, final_state).""" + total, H, D = q.shape + V = v.shape[2] + q, k, v, g, beta, w = (x.double() for x in (q, k, v, g, beta, w)) + o = torch.zeros(total, H, V, dtype=torch.float64, device=q.device) + fs = torch.zeros(cu.numel() - 1, H, D, V, dtype=torch.float64, device=q.device) + for n in range(cu.numel() - 1): + S = torch.zeros(H, D, V, dtype=torch.float64, device=q.device) + for t in range(int(cu[n]), int(cu[n + 1])): + S = g[t].exp()[..., None] * S # per-key-channel decay first + erase = torch.einsum("hd,hdv->hv", beta[t] * k[t], S) + v_new = w[t] * v[t] - erase + S = S + torch.einsum("hd,hv->hdv", k[t], v_new) + o[t] = torch.einsum("hd,hdv->hv", q[t] * scale, S) + fs[n] = S + return o, fs + + +def main(seq_lens=(192, 320), H: int = 2, D: int = 128) -> None: + torch.manual_seed(0) + device = "cuda" + total, num_seqs = sum(seq_lens), len(seq_lens) + scale = 1.0 / math.sqrt(D) + + q = torch.nn.functional.normalize(_randu(total * H, D, device), dim=-1).reshape(total, H, D).bfloat16() + k = torch.nn.functional.normalize(_randu(total * H, D, device), dim=-1).reshape(total, H, D).bfloat16() + v = _randu(total * H, D, device).reshape(total, H, D).bfloat16() + gate = torch.empty(total, H, D, device=device).uniform_(0.5, 1.0).log().contiguous() + beta = (torch.rand(total, H, D, device=device).sigmoid() * 2.0).bfloat16().contiguous() + w = torch.rand(total, H, D, device=device).sigmoid().bfloat16().contiguous() + cu = torch.tensor([0, *torch.tensor(seq_lens).cumsum(0).tolist()], dtype=torch.int32, device=device) + + g = cudnn.pygraph() + q_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="q") + k_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="k") + v_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="v") + g_t = g.tensor([total, H, D], data_type=cudnn.data_type.FLOAT, name="g") + beta_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="beta") + w_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="w") + cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") + O_t, fs_t, _h_t = g.gdn2( + q=q_t, + k=k_t, + v=v_t, + g=g_t, + beta=beta_t, + w=w_t, + cu_seqlens=cu_t, + scale=scale, + output_final_state=True, + use_qk_l2norm=False, + name="gdn2", + ) + O_t.set_output(True).set_data_type(cudnn.data_type.BFLOAT16) + fs_t.set_output(True).set_data_type(cudnn.data_type.FLOAT) + _build_plans(g) + + o = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + fs = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device=device) + pack = {q_t: q, k_t: k, v_t: v, g_t: gate, beta_t: beta, w_t: w, cu_t: cu, O_t: o, fs_t: fs} + g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) + torch.cuda.synchronize() + + o_ref, fs_ref = _reference(q, k, v, gate, beta, w, cu, scale) + torch.testing.assert_close(o.float(), o_ref.float(), atol=1e-1, rtol=1e-1) + r_s = _rms_ratio(fs, fs_ref) + assert r_s < 5e-2, f"final_state rms ratio {r_s:.4g}" + print(f"[05] PASS gdn2 prefill seq_lens={list(seq_lens)} H={H} D={D} (fs rms {r_s:.2e})") + + +if __name__ == "__main__": + main() diff --git a/test/python/linear_attention/frost/examples/06_gdn2_backward.py b/test/python/linear_attention/frost/examples/06_gdn2_backward.py new file mode 100644 index 000000000..fffb7b11d --- /dev/null +++ b/test/python/linear_attention/frost/examples/06_gdn2_backward.py @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Example 06: GDN-2 (Gated DeltaNet v2) backward (pure cuDNN frontend API). + +The GDN2_BWD node takes the forward inputs plus ``dO`` and returns +``(dQ, dK, dV, dG, dBeta, dW)`` (per-key-channel ``dG``/``dBeta``, per-value +``dW``; ``dBeta``/``dW`` in io dtype). Without the optional per-chunk ``h`` +input the engine recomputes the forward state pass internally. Gradients are +checked against fp64 autograd through the per-token recurrence. +""" + +from __future__ import annotations + +import math + +import cudnn +import torch + + +def _build_plans(g) -> None: + g.validate() + g.build_operation_graph() + g.create_execution_plans([cudnn.heur_mode.A]) + names = [g.get_plan_name_at_index(i) for i in range(len(g.plans))] + g.select_plan(names.index("gdn2_frost")) # pin the FROST entry + g.check_support() + g.build_plans() + + +def _rms_ratio(out, ref): + out, ref = out.detach().double(), ref.detach().double() + return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() + + +def _reference_o(q, k, v, g, beta, w, cu, scale): + """Differentiable fp64 per-token recurrence; returns o.""" + total, H, D = q.shape + V = v.shape[2] + outs = [] + for n in range(cu.numel() - 1): + S = torch.zeros(H, D, V, dtype=torch.float64, device=q.device) + for t in range(int(cu[n]), int(cu[n + 1])): + S = g[t].exp()[..., None] * S + erase = torch.einsum("hd,hdv->hv", beta[t] * k[t], S) + v_new = w[t] * v[t] - erase + S = S + torch.einsum("hd,hv->hdv", k[t], v_new) + outs.append(torch.einsum("hd,hdv->hv", q[t] * scale, S)) + return torch.stack(outs, dim=0) + + +def main(seq_lens=(192, 320), H: int = 2, D: int = 128) -> None: + torch.manual_seed(0) + device = "cuda" + total, num_seqs = sum(seq_lens), len(seq_lens) + scale = 1.0 / math.sqrt(D) + + q = torch.nn.functional.normalize(torch.randn(total, H, D, device=device), dim=-1).bfloat16() + k = torch.nn.functional.normalize(torch.randn(total, H, D, device=device), dim=-1).bfloat16() + v = torch.randn(total, H, D, device=device).bfloat16() + gate = torch.empty(total, H, D, device=device).uniform_(0.5, 1.0).log().contiguous() + beta = torch.rand(total, H, D, device=device).sigmoid().bfloat16().contiguous() + w = torch.rand(total, H, D, device=device).sigmoid().bfloat16().contiguous() + do = torch.randn(total, H, D, device=device).bfloat16() + cu = torch.tensor([0, *torch.tensor(seq_lens).cumsum(0).tolist()], dtype=torch.int32, device=device) + + g = cudnn.pygraph() + q_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="q") + k_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="k") + v_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="v") + g_t = g.tensor([total, H, D], data_type=cudnn.data_type.FLOAT, name="g") + beta_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="beta") + w_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="w") + cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") + do_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="dO") + outs = g.gdn2_bwd( + q=q_t, + k=k_t, + v=v_t, + g=g_t, + beta=beta_t, + w=w_t, + cu_seqlens=cu_t, + dO=do_t, + scale=scale, + name="gdn2_bwd", + ) + dtypes = (cudnn.data_type.BFLOAT16,) * 3 + (cudnn.data_type.FLOAT, cudnn.data_type.BFLOAT16, cudnn.data_type.BFLOAT16) + grads_t = [out.set_output(True).set_data_type(dt) for out, dt in zip(outs, dtypes)] + _build_plans(g) + + dq = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + dk = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + dv = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + dg = torch.empty(total, H, D, dtype=torch.float32, device=device) + db = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + dw = torch.empty(total, H, D, dtype=torch.bfloat16, device=device) + pack = {q_t: q, k_t: k, v_t: v, g_t: gate, beta_t: beta, w_t: w, cu_t: cu, do_t: do} + pack.update(dict(zip(grads_t, (dq, dk, dv, dg, db, dw)))) + g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) + torch.cuda.synchronize() + + leaves = [x.double().requires_grad_(True) for x in (q, k, v, gate, beta, w)] + o_ref = _reference_o(*leaves, cu, scale) + grads = torch.autograd.grad((o_ref * do.double()).sum(), leaves) + for name, out, ref in ( + ("dQ", dq, grads[0]), + ("dK", dk, grads[1]), + ("dV", dv, grads[2]), + ("dG", dg, grads[3]), + ("dBeta", db, grads[4]), + ("dW", dw, grads[5]), + ): + r = _rms_ratio(out, ref) + assert r < 5e-2, f"{name} rms ratio {r:.4g}" + print(f"[06] PASS gdn2 backward (recompute) seq_lens={list(seq_lens)} H={H} D={D}") + + +if __name__ == "__main__": + main() diff --git a/test/python/linear_attention/frost/test_gdn2_bprop_kernel.py b/test/python/linear_attention/frost/test_gdn2_bprop_kernel.py deleted file mode 100644 index 1c23dbfdb..000000000 --- a/test/python/linear_attention/frost/test_gdn2_bprop_kernel.py +++ /dev/null @@ -1,44 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""FROST GDN-2 backward: currently a STUB on this branch (the small-chunk -design recomputes the forward states in the backward once the kernel lands). -The contract: ``Gdn2FrostEngine`` declines ``GDN2_BWD`` -graphs so the router can fall back.""" - -from __future__ import annotations - -import pytest - -import cudnn - -from linear_attention.common import assert_engine_declines - -pytestmark = pytest.mark.L0 - - -def test_gdn2_bwd_frost_engine_declines(): - - total, H, D = 256, 2, 128 - g = cudnn.pygraph() - q_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="q") - k_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="k") - v_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="v") - g_t = g.tensor([total, H, D], data_type=cudnn.data_type.FLOAT, name="g") - beta_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="beta") - w_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="w") - cu_t = g.tensor([2], data_type=cudnn.data_type.INT32, name="cu_seqlens") - dO_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="dO") - g.gdn2_bwd( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=beta_t, - w=w_t, - cu_seqlens=cu_t, - dO=dO_t, - scale=0.125, - name="gdn2_bwd", - ) - assert_engine_declines(g, "gdn2_frost") diff --git a/test/python/linear_attention/frost/test_gdn2_prefill_kernel.py b/test/python/linear_attention/frost/test_gdn2_prefill_kernel.py deleted file mode 100644 index 0fdefc555..000000000 --- a/test/python/linear_attention/frost/test_gdn2_prefill_kernel.py +++ /dev/null @@ -1,676 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""FROST GDN-2 prefill tests (``pygraph`` + ``Gdn2FrostEngine``) against the -fp64 recurrent reference.""" - -from __future__ import annotations - -import math -import random -from itertools import accumulate - -import pytest -import torch -import torch.nn.functional as F - -import cudnn # noqa: F401 (conftest extends cudnn.__path__ with the source tree) -from cudnn.linear_attention.frost import Gdn2FrostEngine - -from linear_attention.common import assert_bitwise_runs, assert_concurrent_stream_runs, assert_engine_declines -from linear_attention.conftest import multidist_randu -from linear_attention.reference_gdn2 import gdn2_reference, rms_ratio - -pytestmark = pytest.mark.L0 - -SEED = 42 - - -def _sm100_dsl_available() -> bool: - if not torch.cuda.is_available(): - return False - major, _minor = torch.cuda.get_device_capability() - if major != 10: - return False - try: - import cutlass.experimental.primitives # noqa: F401 -- often a sys.modules alias, invisible to find_spec - except ImportError: - return False - return True - - -requires_runtime = pytest.mark.skipif(not _sm100_dsl_available(), reason="needs an SM100-class GPU and the Cutlass DSL") - - -def _seed(seed=SEED): - random.seed(seed) - torch.random.manual_seed(seed) - torch.cuda.manual_seed(seed) - - -def _cu(seq_lens, device="cuda"): - return torch.tensor([0] + list(accumulate(seq_lens)), dtype=torch.int32, device=device) - - -def _gen_thd(seq_lens, H, HV, head_size, dtype, HK=None): - HK = H if HK is None else HK - total = sum(seq_lens) - q = multidist_randu(total * H, head_size, device="cuda").reshape(total, H, head_size) - k = F.normalize(multidist_randu(total * HK, head_size, device="cuda").reshape(total, HK, head_size), p=2.0, dim=-1) - v = multidist_randu(total * HV, head_size, device="cuda").reshape(total, HV, head_size) - return q.to(dtype).contiguous(), k.to(dtype).contiguous(), v.to(dtype).contiguous() - - -_DT = {torch.bfloat16: cudnn.data_type.BFLOAT16, torch.float16: cudnn.data_type.HALF, torch.float32: cudnn.data_type.FLOAT} - - -def _build_gdn2_engine_graph( - total, - H, - D, - num_seqs, - scale, - *, - io_dt=None, - HV=None, - use_qk_l2norm=True, - with_s0=False, - s0_dt=None, - fs_dt=None, - h_dt=None, - checkpoint_every_n_tokens=0, - use_beta_w_sigmoid=False, - bwd=False, -): - - io_dt = io_dt or cudnn.data_type.BFLOAT16 - HV = HV or H - HO = max(H, HV) - g = cudnn.pygraph() - q_t = g.tensor([total, H, D], data_type=io_dt, name="q") - k_t = g.tensor([total, H, D], data_type=io_dt, name="k") - v_t = g.tensor([total, HV, D], data_type=io_dt, name="v") - g_t = g.tensor([total, HO, D], data_type=cudnn.data_type.FLOAT, name="g") - beta_t = g.tensor([total, HO, D], data_type=io_dt, name="beta") - w_t = g.tensor([total, HO, D], data_type=io_dt, name="w") - cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") - t = dict(q=q_t, k=k_t, v=v_t, g=g_t, beta=beta_t, w=w_t, cu=cu_t) - if with_s0: - t["s0"] = g.tensor([num_seqs, HO, D, D], data_type=s0_dt or cudnn.data_type.FLOAT, name="initial_state") - if bwd: - t["dO"] = g.tensor([total, HO, D], data_type=io_dt, name="dO") - g.gdn2_bwd( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=beta_t, - w=w_t, - cu_seqlens=cu_t, - dO=t["dO"], - scale=scale, - name="gdn2_bwd", - ) - return g, t - O_t, fs_t, h_t = g.gdn2( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=beta_t, - w=w_t, - cu_seqlens=cu_t, - initial_state=t.get("s0"), - scale=scale, - output_final_state=True, - use_qk_l2norm=use_qk_l2norm, - checkpoint_every_n_tokens=checkpoint_every_n_tokens, - use_beta_w_sigmoid=use_beta_w_sigmoid, - name="gdn2", - ) - O_t.set_output(True).set_data_type(io_dt) - fs_t.set_output(True).set_data_type(fs_dt or cudnn.data_type.FLOAT) - t["O"], t["fs"] = O_t, fs_t - if h_t is not None: - h_t.set_output(True).set_data_type(h_dt or io_dt) - t["H"] = h_t - return g, t - - -def _run_gdn2(q, k, v, gate, beta, w, scale, cu, initial_state=None, output_state=None, out_h=None, every_n=0, use_qk_l2norm=True): - """Torch adapter over the graph. ``gate`` is the per-key- - channel natural-log decay (fp32); ``beta``/``w`` are the io-dtype - channel-wise gates; state ports are K-major ``[N, HO, K, V]``. GQA ``k`` - (and ``HV < HQ`` ``v``) are pre-broadcast: the node serves HK == HQ, HV a - multiple of HQ.""" - device = q.device - H, HV = q.shape[1], v.shape[1] - if k.shape[1] != H: - k = k.repeat_interleave(H // k.shape[1], dim=1) - if HV < H: - v = v.repeat_interleave(H // HV, dim=1) - HV = H - total, D = q.shape[0], q.shape[2] - HO = max(H, HV) - num_seqs = cu.shape[0] - 1 - if output_state is None: - output_state = torch.empty(num_seqs, HO, D, D, dtype=torch.float32, device=device) - g, t = _build_gdn2_engine_graph( - total, - H, - D, - num_seqs, - scale, - io_dt=_DT[q.dtype], - HV=HV, - use_qk_l2norm=use_qk_l2norm, - with_s0=initial_state is not None, - s0_dt=None if initial_state is None else _DT[initial_state.dtype], - fs_dt=_DT[output_state.dtype], - h_dt=None if out_h is None else _DT[out_h.dtype], - checkpoint_every_n_tokens=every_n, - ) - g.build() - output = torch.empty(total, HO, D, dtype=q.dtype, device=device) - pack = { - t["q"]: q.contiguous(), - t["k"]: k.contiguous(), - t["v"]: v.contiguous(), - t["g"]: gate.float().contiguous(), - t["beta"]: beta.contiguous(), - t["w"]: w.contiguous(), - t["cu"]: cu.to(torch.int32).contiguous(), - t["O"]: output, - t["fs"]: output_state, - } - if initial_state is not None: - pack[t["s0"]] = initial_state.contiguous() - if out_h is not None: - pack[t["H"]] = out_h - g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) - return output, output_state - - -def _run_and_check(dtype, H, HV, seq_lens, with_s0=False, HK=None, gate_lo=None): - """Run the engine on random inputs and compare against the fp64 recurrent - reference (the kernel L2-normalizes q/k internally). fp16 io needs a - higher gate floor: k * exp2(-cumsum(g)) grazes the fp16 range at 0.5.""" - _seed() - head_size = 128 - total = sum(seq_lens) - HO = max(H, HV) - num_seqs = len(seq_lens) - q, k, v = _gen_thd(seq_lens, H, HV, head_size, dtype, HK=HK) - lo = gate_lo if gate_lo is not None else (0.6 if dtype == torch.float16 else 0.5) - gate = torch.empty(total, HO, head_size, device="cuda").uniform_(lo, 1.0).log() - beta = (torch.rand(total, HO, head_size, device="cuda").sigmoid() * 2.0).to(dtype) - w = torch.rand(total, HO, head_size, device="cuda").sigmoid().to(dtype) - s0 = (torch.randn(num_seqs, HO, head_size, head_size, dtype=torch.float32, device="cuda") * 0.05).contiguous() if with_s0 else None - output_state = torch.full((num_seqs, HO, head_size, head_size), float("nan"), dtype=torch.float32, device="cuda") - - scale = 1.0 / math.sqrt(head_size) - output, _ = _run_gdn2(q, k, v, gate, beta, w, scale, _cu(seq_lens), initial_state=s0, output_state=output_state) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = gdn2_reference( - F.normalize(q.float(), dim=-1).unsqueeze(0), - F.normalize(k.float(), dim=-1).unsqueeze(0), - v.unsqueeze(0), - gate.unsqueeze(0), - beta.unsqueeze(0), - w.unsqueeze(0), - scale=scale, - initial_state=s0, - cu_seqlens=_cu(seq_lens), - ) - tol = 1.2e-1 if dtype == torch.float16 else 1e-1 - torch.testing.assert_close(output.float(), o_ref.squeeze(0).float(), atol=tol, rtol=tol) - assert rms_ratio(output_state, fs_ref) < 5e-2 # engine state ports are K-major, like the reference - - -@requires_runtime -@pytest.mark.parametrize("H,HV", [(1, 1), (2, 2), (2, 4)]) -@pytest.mark.parametrize("seq_lens", [[256], [256, 256], [64, 128, 512]]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_gdn2_kernel_basic(dtype, H, HV, seq_lens): - _run_and_check(dtype, H, HV, seq_lens) - - -@requires_runtime -@pytest.mark.parametrize( - "seq_lens", - [[1], [15], [16], [17], [63, 65], [240, 255, 257], [7] * 24 + [1] * 8, [2048], [33] * 200], - ids=lambda s: f"{len(s)}seqs_{sum(s)}tok", -) -def test_gdn2_kernel_seqlen_edges(seq_lens): - """Chunk-boundary lengths (BT=16 +/- 1), single-token, many-short-seq - packs, a long sequence (many mbarrier ring wraps), and more tiles than - SMs (persistent CTAs walking several tiles).""" - _run_and_check(torch.bfloat16, 2, 4, seq_lens) - - -@requires_runtime -@pytest.mark.parametrize("seq_lens", [[240, 255, 257], [7] * 24 + [1] * 8]) -def test_gdn2_kernel_seqlen_edges_fp16(seq_lens): - _run_and_check(torch.float16, 2, 4, seq_lens) - - -@requires_runtime -@pytest.mark.parametrize("seq_lens", [[256], [64, 129, 512]]) -def test_gdn2_kernel_initial_state(seq_lens): - _run_and_check(torch.bfloat16, 2, 2, seq_lens, with_s0=True) - - -@requires_runtime -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_gdn2_beta_w_sigmoid_in_kernel(dtype): - """use_beta_w_sigmoid: io-dtype beta/w logits + in-kernel sigmoid must - match the host-side sigmoid path (approx-tanh sigmoid and the io-dtype - roundtrip differ by at most ~1 ulp per gate).""" - _seed() - seq_lens, H, D = [256, 129], 2, 128 - total, num_seqs = sum(seq_lens), len(seq_lens) - q, k, v = _gen_thd(seq_lens, H, H, D, dtype) - gate = torch.empty(total, H, D, device="cuda").uniform_(0.5, 1.0).log().float().contiguous() - beta_logits = torch.randn(total, H, D, device="cuda") - w_logits = torch.randn(total, H, D, device="cuda") - scale = 1.0 / math.sqrt(D) - cu = _cu(seq_lens) - - outs = [] - for in_kernel in (True, False): - if in_kernel: - beta, w = beta_logits.to(dtype).contiguous(), w_logits.to(dtype).contiguous() - else: - beta = beta_logits.to(dtype).float().sigmoid().to(dtype).contiguous() - w = w_logits.to(dtype).float().sigmoid().to(dtype).contiguous() - g, t = _build_gdn2_engine_graph(total, H, D, num_seqs, scale, io_dt=_DT[dtype], use_beta_w_sigmoid=in_kernel) - g.build() - o = torch.empty(total, H, D, dtype=dtype, device="cuda") - fs = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device="cuda") - pack = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate, t["beta"]: beta, t["w"]: w, t["cu"]: cu, t["O"]: o, t["fs"]: fs} - g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device="cuda")) - torch.cuda.synchronize() - outs.append((o, fs)) - torch.testing.assert_close(outs[0][0].float(), outs[1][0].float(), atol=2.5e-2, rtol=2.5e-2) - assert rms_ratio(outs[0][1], outs[1][1]) < 2e-2 - - -@requires_runtime -@pytest.mark.parametrize("with_initial_state", [False, True]) -def test_gdn2_zero_length_sequence_state_passthrough(with_initial_state): - """A zero-length sequence's final-state slot gets the passthrough value: - its initial state when seeded, zeros otherwise.""" - _seed() - seq_len, H, D, sentinel = 256, 2, 128, 123.0 - q, k, v = _gen_thd([seq_len], H, H, D, torch.bfloat16) - gate = torch.empty(seq_len, H, D, device="cuda").uniform_(0.5, 1.0).log().float().contiguous() - beta = (torch.rand(seq_len, H, D, device="cuda").sigmoid() * 2.0).to(torch.bfloat16) - w = torch.rand(seq_len, H, D, device="cuda").sigmoid().to(torch.bfloat16) - s0 = torch.randn(2, H, D, D, dtype=torch.float32, device="cuda") if with_initial_state else None - fs = torch.full((2, H, D, D), sentinel, dtype=torch.float32, device="cuda") - _run_gdn2(q, k, v, gate, beta, w, 1.0 / math.sqrt(D), _cu([seq_len, 0]), initial_state=s0, output_state=fs) - torch.cuda.synchronize() - expected = s0[1] if with_initial_state else torch.zeros_like(fs[1]) - torch.testing.assert_close(fs[1], expected, atol=0, rtol=0) - - -@requires_runtime -@pytest.mark.parametrize("HQ,HK,HV", [(1, 1, 1), (4, 1, 1), (3, 3, 3), (6, 2, 2), (1, 1, 2), (2, 2, 4), (16, 16, 32), (16, 16, 64)]) -def test_gdn2_kernel_head_configs(HQ, HK, HV): - """The FlashInfer head-config matrix: GQA (HK < HQ), GVA (HV > HQ), odd - head counts, and production-sized 16/32/64-head grids (the adapter - pre-broadcasts k and any HV < HQ v to the node's HK == HQ contract).""" - _run_and_check(torch.bfloat16, HQ, HV, [64, 128, 512], HK=HK) - - -@requires_runtime -@pytest.mark.parametrize("HQ,HK,HV", [(2, 2, 4), (16, 16, 64)]) -def test_gdn2_kernel_head_configs_fp16(HQ, HK, HV): - _run_and_check(torch.float16, HQ, HV, [64, 128, 512], HK=HK) - - -@requires_runtime -def test_gdn2_chunked_prefill(): - """Splitting a sequence at a chunk boundary and reseeding from the fp32 - final state must reproduce the single-shot run (state roundtrip; the two - runs get different split tables, so the match is approximate).""" - _seed() - H, D, T1, T2 = 2, 128, 256, 192 - dtype = torch.bfloat16 - q, k, v = _gen_thd([T1 + T2], H, H, D, dtype) - gate = torch.empty(T1 + T2, H, D, device="cuda").uniform_(0.5, 1.0).log() - beta = (torch.rand(T1 + T2, H, D, device="cuda").sigmoid() * 2.0).to(dtype) - w = torch.rand(T1 + T2, H, D, device="cuda").sigmoid().to(dtype) - scale = 1.0 / math.sqrt(D) - - fs_full = torch.full((1, H, D, D), float("nan"), dtype=torch.float32, device="cuda") - o_full, _ = _run_gdn2(q, k, v, gate, beta, w, scale, _cu([T1 + T2]), output_state=fs_full) - - fs1 = torch.full((1, H, D, D), float("nan"), dtype=torch.float32, device="cuda") - o1, _ = _run_gdn2(q[:T1], k[:T1], v[:T1], gate[:T1], beta[:T1], w[:T1], scale, _cu([T1]), output_state=fs1) - fs2 = torch.full((1, H, D, D), float("nan"), dtype=torch.float32, device="cuda") - o2, _ = _run_gdn2( - q[T1:].contiguous(), - k[T1:].contiguous(), - v[T1:].contiguous(), - gate[T1:].contiguous(), - beta[T1:].contiguous(), - w[T1:].contiguous(), - scale, - _cu([T2]), - initial_state=fs1, - output_state=fs2, - ) - torch.cuda.synchronize() - - torch.testing.assert_close(o1.float(), o_full[:T1].float(), atol=2e-3, rtol=2e-3) - torch.testing.assert_close(o2.float(), o_full[T1:].float(), atol=2e-3, rtol=2e-3) - torch.testing.assert_close(fs2, fs_full, atol=2e-3, rtol=2e-3) - - -@requires_runtime -def test_gdn2_kernel_state_dtype_bf16(): - """bf16 initial/final state buffers (the io-downcast S0 path).""" - _seed() - seq_lens, H, D = [64, 128, 512], 2, 128 - total, num_seqs = sum(seq_lens), len(seq_lens) - dtype = torch.bfloat16 - q, k, v = _gen_thd(seq_lens, H, H, D, dtype) - gate = torch.empty(total, H, D, device="cuda").uniform_(0.5, 1.0).log() - beta = (torch.rand(total, H, D, device="cuda").sigmoid() * 2.0).to(dtype) - w = torch.rand(total, H, D, device="cuda").sigmoid().to(dtype) - scale = 1.0 / math.sqrt(D) - s0 = (torch.randn(num_seqs, H, D, D, dtype=torch.float32, device="cuda") * 0.05).to(dtype).contiguous() - fs = torch.full((num_seqs, H, D, D), float("nan"), dtype=dtype, device="cuda") - o, _ = _run_gdn2(q, k, v, gate, beta, w, scale, _cu(seq_lens), initial_state=s0, output_state=fs) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = gdn2_reference( - F.normalize(q.float(), dim=-1).unsqueeze(0), - F.normalize(k.float(), dim=-1).unsqueeze(0), - v.unsqueeze(0), - gate.unsqueeze(0), - beta.unsqueeze(0), - w.unsqueeze(0), - scale=scale, - initial_state=s0.float(), - cu_seqlens=_cu(seq_lens), - ) - torch.testing.assert_close(o.float(), o_ref.squeeze(0).float(), atol=1e-1, rtol=1e-1) - assert rms_ratio(fs.float(), fs_ref) < 5e-2 - - -# --------------------------------------------------------------------------- -# Gdn2FrostEngine: graph-level coverage through the router -# --------------------------------------------------------------------------- - - -def _gdn2_engine_inputs(seq_lens, H, D): - from linear_attention.conftest import gen_gdn2_gates, gen_qkv - - total = sum(seq_lens) - q, k, v = gen_qkv(1, total, H, H, D, D, torch.bfloat16) - gate, beta, w = gen_gdn2_gates(1, total, H, D, D, torch.bfloat16) - cu = torch.tensor([0] + list(accumulate(seq_lens)), dtype=torch.int32, device="cuda") - return (x.squeeze(0).contiguous() for x in (q, k, v, gate, beta, w)), cu - - -@requires_runtime -@pytest.mark.parametrize("seq_lens", [[256], [512, 512], [64, 128, 512]]) -def test_gdn2_frost_engine_matches_reference(seq_lens, H=2, D=128): - - _seed() - (q, k, v, gate, beta, w), cu = _gdn2_engine_inputs(seq_lens, H, D) - total, num_seqs = sum(seq_lens), len(seq_lens) - scale = 1.0 / math.sqrt(D) - - g, t = _build_gdn2_engine_graph(total, H, D, num_seqs, scale) - g.build() - assert isinstance(g.selected_engine, Gdn2FrostEngine) - assert g.get_workspace_size() > 0 # split-K work-item table + scheduler counters - - o_buf = torch.empty(total, H, D, dtype=torch.bfloat16, device="cuda") - fs_buf = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device="cuda") - pack = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate, t["beta"]: beta, t["w"]: w, t["cu"]: cu, t["O"]: o_buf, t["fs"]: fs_buf} - g.execute(pack, torch.empty(g.get_workspace_size(), dtype=torch.uint8, device="cuda")) - torch.cuda.synchronize() - - # the kernel L2-normalizes q/k internally - with torch.no_grad(): - o_ref, fs_ref = gdn2_reference( - F.normalize(q.float(), dim=-1).unsqueeze(0), - F.normalize(k.float(), dim=-1).unsqueeze(0), - v.unsqueeze(0), - gate.unsqueeze(0), - beta.unsqueeze(0), - w.unsqueeze(0), - scale=scale, - cu_seqlens=cu, - ) - torch.testing.assert_close(o_buf.float(), o_ref.squeeze(0).float(), atol=1e-1, rtol=1e-1) - r_s = rms_ratio(fs_buf, fs_ref) # engine state ports are K-major - assert r_s < 5e-2, f"final_state rms ratio {r_s:.4g}" - - -@requires_runtime -def test_gdn2_frost_engine_no_l2norm_matches_reference(seq_lens=(256,), H=2, D=128): - """use_qk_l2norm=False passes q/k through as given, so the test feeds - pre-normalized rows (the kernel's io-dtype arithmetic needs them).""" - - _seed() - (q, k, v, gate, beta, w), cu = _gdn2_engine_inputs(list(seq_lens), H, D) - q = F.normalize(q.float(), dim=-1).to(q.dtype) - k = F.normalize(k.float(), dim=-1).to(k.dtype) - total, num_seqs = sum(seq_lens), len(seq_lens) - scale = 1.0 / math.sqrt(D) - - g, t = _build_gdn2_engine_graph(total, H, D, num_seqs, scale, use_qk_l2norm=False) - g.build() - assert isinstance(g.selected_engine, Gdn2FrostEngine) - - o_buf = torch.empty(total, H, D, dtype=torch.bfloat16, device="cuda") - fs_buf = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device="cuda") - pack = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate, t["beta"]: beta, t["w"]: w, t["cu"]: cu, t["O"]: o_buf, t["fs"]: fs_buf} - g.execute(pack, torch.empty(g.get_workspace_size(), dtype=torch.uint8, device="cuda")) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = gdn2_reference( - q.float().unsqueeze(0), - k.float().unsqueeze(0), - v.unsqueeze(0), - gate.unsqueeze(0), - beta.unsqueeze(0), - w.unsqueeze(0), - scale=scale, - cu_seqlens=cu, - ) - torch.testing.assert_close(o_buf.float(), o_ref.squeeze(0).float(), atol=1e-1, rtol=1e-1) - r_s = rms_ratio(fs_buf, fs_ref) # engine state ports are K-major - assert r_s < 5e-2, f"final_state rms ratio {r_s:.4g}" - - -@requires_runtime -def test_gdn2_frost_engine_initial_state(seq_lens=(128, 256), H=2, D=128): - - _seed() - (q, k, v, gate, beta, w), cu = _gdn2_engine_inputs(seq_lens, H, D) - total, num_seqs = sum(seq_lens), len(seq_lens) - scale = 1.0 / math.sqrt(D) - s0 = torch.randn(num_seqs, H, D, D, dtype=torch.float32, device="cuda") * 0.05 - - g, t = _build_gdn2_engine_graph(total, H, D, num_seqs, scale, with_s0=True) - g.build() - assert isinstance(g.selected_engine, Gdn2FrostEngine) - - o_buf = torch.empty(total, H, D, dtype=torch.bfloat16, device="cuda") - fs_buf = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device="cuda") - pack = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate, t["beta"]: beta, t["w"]: w, t["cu"]: cu, t["s0"]: s0, t["O"]: o_buf, t["fs"]: fs_buf} - g.execute(pack, torch.empty(g.get_workspace_size(), dtype=torch.uint8, device="cuda")) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = gdn2_reference( - F.normalize(q.float(), dim=-1).unsqueeze(0), - F.normalize(k.float(), dim=-1).unsqueeze(0), - v.unsqueeze(0), - gate.unsqueeze(0), - beta.unsqueeze(0), - w.unsqueeze(0), - scale=scale, - initial_state=s0, - cu_seqlens=cu, - ) - torch.testing.assert_close(o_buf.float(), o_ref.squeeze(0).float(), atol=1e-1, rtol=1e-1) - r_s = rms_ratio(fs_buf, fs_ref) # engine state ports are K-major - assert r_s < 5e-2, f"final_state rms ratio {r_s:.4g}" - - -def test_gdn2_frost_engine_declines_bwd(): - """GDN2_BWD declines (stub backward kernel on this branch).""" - g, _t = _build_gdn2_engine_graph(256, 2, 128, 1, 0.125, bwd=True) - assert_engine_declines(g, "gdn2_frost") - - -@requires_runtime -def test_gdn2_frost_engine_declines_wrong_head_dim(): - g, _t = _build_gdn2_engine_graph(256, 2, 64, 1, 0.125) - assert_engine_declines(g, "gdn2_frost") - - -# --------------------------------------------------------------------------- -# Determinism stress: bitwise repeat runs + two-stream co-residency -# --------------------------------------------------------------------------- - -DET_VARLEN_MIX = [497, 16, 1, 480, 0, 253] # zero-length + single-token + odd tails - - -def _det_launch(seq_lens, H, HV, with_s0=False, stream=None): - q, k, v = _gen_thd(seq_lens, H, HV, 128, torch.bfloat16) - total, HO = sum(seq_lens), max(H, HV) - gate = torch.empty(total, HO, 128, device="cuda").uniform_(0.5, 1.0).log().float().contiguous() - beta = (torch.rand(total, HO, 128, device="cuda").sigmoid() * 2.0).to(torch.bfloat16).contiguous() - w = torch.rand(total, HO, 128, device="cuda").sigmoid().to(torch.bfloat16).contiguous() - s0 = (torch.randn(len(seq_lens), HO, 128, 128, dtype=torch.float32, device="cuda") * 0.05).contiguous() if with_s0 else None - cu = _cu(seq_lens) - scale = 1.0 / math.sqrt(128) - - def launch(): - if stream is not None: - with torch.cuda.stream(stream): - return _run_gdn2(q, k, v, gate, beta, w, scale, cu, initial_state=s0) - return _run_gdn2(q, k, v, gate, beta, w, scale, cu, initial_state=s0) - - return launch - - -@requires_runtime -@pytest.mark.parametrize("seq_lens", [DET_VARLEN_MIX, [4096]], ids=["varlen_mix", "long"]) -def test_gdn2_prefill_determinism(seq_lens): - _seed() - assert_bitwise_runs(_det_launch(seq_lens, 2, 4), label="gdn2") - - -@requires_runtime -def test_gdn2_prefill_determinism_initial_state(): - _seed() - assert_bitwise_runs(_det_launch([256, 640, 0, 33], 2, 2, with_s0=True), label="gdn2+s0") - - -@requires_runtime -def test_gdn2_concurrent_streams_determinism(): - _seed() - s1, s2 = torch.cuda.Stream(), torch.cuda.Stream() - assert_concurrent_stream_runs(_det_launch([1024, 31, 512], 2, 4, stream=s1), _det_launch([256, 999, 1, 128], 2, 4, stream=s2), s1, s2) - - -# --------------------------------------------------------------------------- -# split-K: strong decay on a long-sequence pack, so the table actually cuts -# (the per-channel gate scan itself is covered in test_kda_prefill_kernel.py) -# --------------------------------------------------------------------------- - - -@requires_runtime -@pytest.mark.parametrize("with_s0", [False, True]) -@pytest.mark.parametrize("heads", [(2, 2), (1, 4)]) # MHA, GVA -def test_gdn2_prefill_split_strong_decay(heads, with_s0): - """Strong decay (gate floor 0.3) saturates the scan's warmup threshold on - the 2048-token sequence, so the split table cuts it into several work - items; checked against the fp64 recurrent reference.""" - HQ, HV = heads - _run_and_check(torch.bfloat16, HQ, HV, [100, 2048, 0, 517], with_s0=with_s0, gate_lo=0.3) - - -# --------------------------------------------------------------------------- -# CUDA graph capture/replay across dynamic shapes (fixed SM-count grid) -# --------------------------------------------------------------------------- - - -@requires_runtime -def test_gdn2_prefill_cuda_graph_replay(): - """Capture the ENGINE execute once, replay across CHANGED effective - shapes: capacity buffers, cu_seqlens with zero-length tails; everything - the engine launches (sched memset, split table, desc rebuilds, kernel) - must be capture-safe, and every replay must match an eager engine launch - on the same data bit for bit.""" - T_cap, B_cap, H, D = 768, 4, 2, 128 - dev = "cuda" - _seed() - scale = 1.0 / math.sqrt(D) - q = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - k = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - v = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - gate = torch.zeros(T_cap, H, D, dtype=torch.float32, device=dev) - beta = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - w = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - cu = torch.zeros(B_cap + 1, dtype=torch.int32, device=dev) - o_graph = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - fs_graph = torch.zeros(B_cap, H, D, D, dtype=torch.float32, device=dev) - o_eager = torch.zeros_like(o_graph) - fs_eager = torch.zeros_like(fs_graph) - - g, t = _build_gdn2_engine_graph(T_cap, H, D, B_cap, scale) - g.build() - ws = torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=dev) - base = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate, t["beta"]: beta, t["w"]: w, t["cu"]: cu} - pack_graph = {**base, t["O"]: o_graph, t["fs"]: fs_graph} - pack_eager = {**base, t["O"]: o_eager, t["fs"]: fs_eager} - - def fill(seq_lens): - total = sum(seq_lens) - q[:total] = torch.randn(total, H, D, device=dev).bfloat16() * 0.5 - k[:total] = F.normalize(torch.randn(total, H, D, device=dev), dim=-1).bfloat16() - v[:total] = torch.randn(total, H, D, device=dev).bfloat16() * 0.5 - gate[:total] = torch.empty(total, H, D, device=dev).uniform_(0.5, 1.0).log() - beta[:total] = (torch.rand(total, H, D, device=dev).sigmoid() * 2.0).bfloat16() - w[:total] = torch.rand(total, H, D, device=dev).sigmoid().bfloat16() - bounds = [0] + list(accumulate(seq_lens)) - bounds += [bounds[-1]] * (B_cap + 1 - len(bounds)) - cu.copy_(torch.tensor(bounds, dtype=torch.int32)) - return total - - fill([256, 512, 0, 0]) - stream = torch.cuda.Stream() - handle = cudnn.create_handle() - cudnn.set_stream(handle, stream.cuda_stream) - with torch.cuda.stream(stream): - g.execute(pack_graph, ws, handle=handle) # warmup: compile + caches - torch.cuda.synchronize() - - cg = torch.cuda.CUDAGraph() - with torch.cuda.graph(cg, stream=stream): - g.execute(pack_graph, ws, handle=handle) - - eager_handle = cudnn.create_handle() - cudnn.set_stream(eager_handle, torch.cuda.current_stream().cuda_stream) - for seq_lens in ([256, 512, 0, 0], [100, 200, 56, 0], [768], [16, 0, 16, 736 - 32]): - total = fill(seq_lens) - cg.replay() - torch.cuda.synchronize() - g.execute(pack_eager, ws, handle=eager_handle) - torch.cuda.synchronize() - assert torch.equal(o_graph[:total], o_eager[:total]), f"graph o diverges from eager at {seq_lens}" - assert torch.equal(fs_graph, fs_eager), f"graph final_state diverges from eager at {seq_lens}" diff --git a/test/python/linear_attention/frost/test_gdn_bprop_kernel.py b/test/python/linear_attention/frost/test_gdn_bprop_kernel.py deleted file mode 100644 index 0594a4e10..000000000 --- a/test/python/linear_attention/frost/test_gdn_bprop_kernel.py +++ /dev/null @@ -1,964 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""FROST GDN backward tests (pygraph -GDN_BWD node on GdnFrostEngine) against fp64 autograd oracles.""" - -from __future__ import annotations - -import math -import random -from itertools import accumulate - -import pytest -import torch -import torch.nn.functional as F - -import cudnn # noqa: F401 (conftest extends cudnn.__path__ with the source tree) -from cudnn.linear_attention.frost import GdnFrostEngine - -from linear_attention.common import assert_bitwise_runs -from linear_attention.conftest import multidist_randu -from linear_attention.reference_gdn import gdn_reference, rms_ratio - -pytestmark = pytest.mark.L0 - -SEED = 42 - -BWD_TOL = 3e-2 - -HEAD_CONFIGS = [ - (1, 1, 1), - (4, 1, 1), - (3, 3, 3), - (6, 2, 2), - (1, 1, 2), - (2, 2, 4), - (16, 16, 32), - (16, 16, 64), -] - - -def _sm100_dsl_available() -> bool: - if not torch.cuda.is_available(): - return False - major, _minor = torch.cuda.get_device_capability() - if major != 10: - return False - try: - import cutlass.experimental.primitives # noqa: F401 — often a sys.modules alias, invisible to find_spec - except ImportError: - return False - return True - - -requires_runtime = pytest.mark.skipif( - not _sm100_dsl_available(), - reason="needs an SM100-class GPU and the Cutlass DSL", -) - - -def _seed(seed=SEED): - random.seed(seed) - torch.random.manual_seed(seed) - torch.cuda.manual_seed(seed) - - -def _cu(seq_lens, device="cuda"): - return torch.tensor([0] + list(accumulate(seq_lens)), dtype=torch.int32, device=device) - - -def _cu_h(seq_lens, device="cuda"): - counts = [max(sl - 1, 0) // 64 for sl in seq_lens] - return torch.tensor([0] + list(accumulate(counts)), dtype=torch.int32, device=device), sum(counts) - - -def _gen_case(seq_lens, HQ=2, HK=None, HV=None, head_size=128, dtype=torch.bfloat16, alpha_on=True, beta_on=True): - """THD inputs at native head counts; do/alpha/beta at HO = max(HQ, HV).""" - HK = HQ if HK is None else HK - HV = HQ if HV is None else HV - HO = max(HQ, HV) - _seed() - total = sum(seq_lens) - q = multidist_randu(total * HQ, head_size, device="cuda").reshape(total, HQ, head_size) - k = multidist_randu(total * HK, head_size, device="cuda").reshape(total, HK, head_size) - k = F.normalize(k, p=2.0, dim=-1) - v = multidist_randu(total * HV, head_size, device="cuda").reshape(total, HV, head_size) - do = multidist_randu(total * HO, head_size, device="cuda").reshape(total, HO, head_size) - alpha = torch.empty(total, HO, device="cuda").uniform_(0.1, 1.0) if alpha_on else torch.ones(total, HO, device="cuda") - beta = torch.rand(total, HO, device="cuda") if beta_on else torch.ones(total, HO, device="cuda") - return ( - q.to(dtype).contiguous(), - k.to(dtype).contiguous(), - v.to(dtype).contiguous(), - do.to(dtype).contiguous(), - alpha.contiguous(), - beta.contiguous(), - ) - - -def _build_bwd_graph(total, HQ, HV, D, num_seqs, scale, io_dt, *, h_shape=None, s0=False, dht=False): - - HO = max(HQ, HV) - g = cudnn.pygraph() - t = dict( - q=g.tensor([total, HQ, D], data_type=io_dt, name="q"), - k=g.tensor([total, HQ, D], data_type=io_dt, name="k"), - v=g.tensor([total, HV, D], data_type=io_dt, name="v"), - g=g.tensor([total, HO], data_type=cudnn.data_type.FLOAT, name="g"), - beta=g.tensor([total, HO], data_type=cudnn.data_type.FLOAT, name="beta"), - cu=g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens"), - dO=g.tensor([total, HO, D], data_type=io_dt, name="dO"), - ) - kwargs = {} - if h_shape is not None: - t["h"] = kwargs["h"] = g.tensor(list(h_shape), data_type=io_dt, name="h") - if s0: - t["s0"] = kwargs["initial_state"] = g.tensor([num_seqs, HO, D, D], data_type=cudnn.data_type.FLOAT, name="initial_state") - if dht: - t["dht"] = kwargs["d_final_state"] = g.tensor([num_seqs, HO, D, D], data_type=cudnn.data_type.FLOAT, name="d_final_state") - dQ_t, dK_t, dV_t, dG_t, dBeta_t, dS0_t = g.gdn_bwd( - q=t["q"], - k=t["k"], - v=t["v"], - g=t["g"], - beta=t["beta"], - cu_seqlens=t["cu"], - dO=t["dO"], - scale=float(scale), - name="gdn_bwd", - **kwargs, - ) - outs = [(dQ_t, io_dt), (dK_t, io_dt), (dV_t, io_dt), (dG_t, cudnn.data_type.FLOAT), (dBeta_t, cudnn.data_type.FLOAT)] - if s0: - outs.append((dS0_t, cudnn.data_type.FLOAT)) - for t_, dt in outs: - t_.set_output(True).set_data_type(dt) - t["grads"] = [o for o, _ in outs] - g.build() - return g, t - - -def _fwd_h(q, k, v, alpha, beta, scale, cu, seq_lens, s0=None, every_n=64): - """Forward through the engine with the per-chunk H output; returns h.""" - - device = q.device - total, HQ, HV, D = q.shape[0], q.shape[1], v.shape[1], q.shape[2] - HO = max(HQ, HV) - num_seqs = cu.shape[0] - 1 - io_dt = cudnn.data_type.BFLOAT16 if q.dtype == torch.bfloat16 else cudnn.data_type.HALF - g = cudnn.pygraph() - q_t = g.tensor([total, HQ, D], data_type=io_dt, name="q") - k_t = g.tensor([total, HQ, D], data_type=io_dt, name="k") - v_t = g.tensor([total, HV, D], data_type=io_dt, name="v") - g_t = g.tensor([total, HO], data_type=cudnn.data_type.FLOAT, name="g") - b_t = g.tensor([total, HO], data_type=cudnn.data_type.FLOAT, name="beta") - cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") - s0_t = g.tensor([num_seqs, HO, D, D], data_type=cudnn.data_type.FLOAT, name="initial_state") if s0 is not None else None - O_t, _fs_t, h_t = g.gdn( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=b_t, - cu_seqlens=cu_t, - initial_state=s0_t, - scale=float(scale), - checkpoint_every_n_tokens=every_n, - name="gdn", - ) - O_t.set_output(True).set_data_type(io_dt) - h_t.set_output(True).set_data_type(io_dt) - g.build() - total_h = sum(max(sl - 1, 0) // every_n for sl in seq_lens) - o = torch.empty(total, HO, D, dtype=q.dtype, device=device) - h = torch.full((max(total_h, 1), HO, D, D), float("nan"), dtype=q.dtype, device=device) - pack = {q_t: q, k_t: k, v_t: v, g_t: alpha.float().log().contiguous(), b_t: beta.float().contiguous(), cu_t: cu, O_t: o, h_t: h} - if s0 is not None: - pack[s0_t] = s0.contiguous() - g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) - torch.cuda.synchronize() - return h - - -def _run_bwd(q, k, v, alpha, beta, do, h, scale, cu, s0=None, dht=None): - """Engine-driven backward; ``h=None`` exercises the recompute path. - Returns (dq, dk, dv, dg, db) and appends ds0 when ``s0`` is given. - ``k`` with fewer heads than ``q`` is pre-broadcast to the node contract - and ``dk`` is group-reduced back to the caller's granularity.""" - device = q.device - total, HQ, HV, D = q.shape[0], q.shape[1], v.shape[1], q.shape[2] - HK = k.shape[1] - if HK != HQ: - k = k.repeat_interleave(HQ // HK, dim=1).contiguous() - HO = max(HQ, HV) - num_seqs = cu.shape[0] - 1 - io_dt = cudnn.data_type.BFLOAT16 if q.dtype == torch.bfloat16 else cudnn.data_type.HALF - g, t = _build_bwd_graph( - total, - HQ, - HV, - D, - num_seqs, - scale, - io_dt, - h_shape=None if h is None else h.shape, - s0=s0 is not None, - dht=dht is not None, - ) - dq = torch.full((total, HQ, D), float("nan"), dtype=q.dtype, device=device) - dk = torch.full((total, HQ, D), float("nan"), dtype=q.dtype, device=device) - dv = torch.full((total, HV, D), float("nan"), dtype=q.dtype, device=device) - dg = torch.full((total, HO), float("nan"), dtype=torch.float32, device=device) - db = torch.full((total, HO), float("nan"), dtype=torch.float32, device=device) - bufs = [dq, dk, dv, dg, db] - if s0 is not None: - bufs.append(torch.full((num_seqs, HO, D, D), float("nan"), dtype=torch.float32, device=device)) - pack = { - t["q"]: q, - t["k"]: k, - t["v"]: v, - t["g"]: alpha.float().log().contiguous(), - t["beta"]: beta.float().contiguous(), - t["cu"]: cu, - t["dO"]: do, - } - for ot, buf in zip(t["grads"], bufs): - pack[ot] = buf - if h is not None: - pack[t["h"]] = h - if s0 is not None: - pack[t["s0"]] = s0.contiguous() - if dht is not None: - pack[t["dht"]] = dht.contiguous() - g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) - torch.cuda.synchronize() - if HK != HQ: - bufs[1] = _reduce_ref(bufs[1], HK) - return tuple(bufs) - - -def _reference_grads(q, k, v, alpha, beta, do, scale, seq_lens): - """fp64 autograd oracle: grads of (o * dO).sum() at HO-head granularity. - - q/k/v leaves are pre-expanded to HO with the kernel's repeat_interleave - head mapping; the gate leaf is NATURAL-LOG (the kernel's dgate - convention) and alpha/beta are already HO-shaped.""" - HO = max(q.shape[1], v.shape[1]) - - def leaf(x): - r = HO // x.shape[1] - x = x.double().repeat_interleave(r, dim=1) if r > 1 else x.double() - return x.requires_grad_(True) - - qq, kk, vv = leaf(q), leaf(k), leaf(v) - gate = alpha.double().log().requires_grad_(True) - bb = beta.double().requires_grad_(True) - o, _fs = gdn_reference( - qq.unsqueeze(0), - kk.unsqueeze(0), - vv.unsqueeze(0), - gate.unsqueeze(0), - bb.unsqueeze(0), - scale=scale, - initial_state=None, - cu_seqlens=_cu(seq_lens, q.device), - ) - (o.squeeze(0) * do.double()).sum().backward() - return qq.grad, kk.grad, vv.grad, gate.grad, bb.grad - - -def _reduce_ref(g_ho, native_h): - ho = g_ho.shape[1] - if ho == native_h: - return g_ho - return g_ho.view(g_ho.shape[0], native_h, ho // native_h, *g_ho.shape[2:]).sum(2) - - -def _run_bprop_case(seq_lens, HQ=2, HK=None, HV=None, head_size=128, dtype=torch.bfloat16, scale=None, alpha_on=True, beta_on=True): - """Prefill H -> backward kernel -> compare all five gradients against the - fp64 autograd oracle at HO granularity.""" - scale = 1.0 / math.sqrt(head_size) if scale is None else scale - q, k, v, do, alpha, beta = _gen_case(seq_lens, HQ, HK, HV, head_size, dtype, alpha_on, beta_on) - cu = _cu(seq_lens) - dq, dk, dv, dg, db = _run_bwd(q, k, v, alpha, beta, do, None, scale, cu) - torch.cuda.synchronize() - dq_ref, dk_ref, dv_ref, dg_ref, db_ref = _reference_grads(q, k, v, alpha, beta, do, scale, seq_lens) - for name, got, ref, tol in ( - ("dq", dq, _reduce_ref(dq_ref, q.shape[1]), BWD_TOL), - ("dk", dk, _reduce_ref(dk_ref, k.shape[1]), BWD_TOL), - ("dv", dv, _reduce_ref(dv_ref, v.shape[1]), BWD_TOL), - ("dg", dg, dg_ref, BWD_TOL), - ("db", db, db_ref, BWD_TOL), - ): - assert torch.isfinite(got.float()).all(), f"non-finite values in {name}" - r = rms_ratio(got.float(), ref.float()) - assert r < tol, f"{name} rms ratio {r:.4g} >= {tol}" - - -# --------------------------------------------------------------------------- -# Comprehensive correctness (all five gradients per case) -# --------------------------------------------------------------------------- - - -@requires_runtime -@pytest.mark.parametrize("num_q_heads, num_k_heads, num_v_heads", HEAD_CONFIGS) -@pytest.mark.parametrize("seq_lens", [[256], [256, 256], [64, 128, 512]]) -@pytest.mark.parametrize( - "dtype", - [ - "float16", - "bfloat16", - ], -) -def test_bprop_kernel_basic(dtype, num_q_heads, num_k_heads, num_v_heads, seq_lens): - _run_bprop_case(seq_lens, num_q_heads, num_k_heads, num_v_heads, dtype=getattr(torch, dtype)) - - -@requires_runtime -@pytest.mark.parametrize("beta", [False, True]) -@pytest.mark.parametrize("alpha", [False, True]) -@pytest.mark.parametrize("scale", [1.0, "auto"]) -def test_bprop_kernel_gates_and_scale(scale, alpha, beta): - if not alpha and not beta: - pytest.skip("large diff due to output value amplitude explosion along token dimension") - scale = 1.0 / math.sqrt(128) if scale == "auto" else scale - _run_bprop_case([64, 128, 512], 3, 3, 3, scale=scale, alpha_on=alpha, beta_on=beta) - - -@requires_runtime -@pytest.mark.parametrize("num_q_heads, num_k_heads, num_v_heads", [(3, 3, 3), (4, 1, 1), (2, 2, 4)]) -@pytest.mark.parametrize("seq_lens", [[31], [251], [511, 501], [31, 63, 93, 123, 150, 500]]) -@pytest.mark.parametrize("dtype", ["bfloat16", "float16"]) -def test_bprop_kernel_nonfull(dtype, num_q_heads, num_k_heads, num_v_heads, seq_lens): - _run_bprop_case(seq_lens, num_q_heads, num_k_heads, num_v_heads, dtype=getattr(torch, dtype)) - - -@requires_runtime -@pytest.mark.parametrize("num_q_heads, num_k_heads, num_v_heads", [(1, 1, 1), (16, 16, 64)]) -@pytest.mark.parametrize("seq_len", [256, 255]) -def test_bprop_kernel_zero_length_sequence(num_q_heads, num_k_heads, num_v_heads, seq_len): - """A trailing zero-length sequence neither perturbs the gradients nor hangs.""" - head_size = 128 - scale = 1.0 / math.sqrt(head_size) - q, k, v, do, alpha, beta = _gen_case([seq_len], num_q_heads, num_k_heads, num_v_heads, head_size) - - def run(seq_lens): - cu = _cu(seq_lens) - out = _run_bwd(q, k, v, alpha, beta, do, None, scale, cu) - torch.cuda.synchronize() - return out - - ref = run([seq_len]) - got = run([seq_len, 0]) - for name, g, r in zip(("dq", "dk", "dv", "dg", "db"), got, ref): - torch.testing.assert_close(g, r, atol=2e-2, rtol=2e-2, msg=f"{name} perturbed by the zero-length sequence") - - -# --------------------------------------------------------------------------- -# FROST engine: GDN_BWD through the graph -# --------------------------------------------------------------------------- - - -def test_frost_gdn_bwd_engine_no_h(seq_lens=(128, 256), H=2, D=128): - """GDN_BWD without the h input: the engine reruns the forward with H - dumping and must match the explicit-h path bit for bit.""" - - _seed() - seq_lens = list(seq_lens) - q, k, v, do, alpha, beta = _gen_case(seq_lens, HQ=H, head_size=D) - device = q.device - scale = 1.0 / math.sqrt(D) - num_seqs = len(seq_lens) - total = q.shape[0] - cu = _cu(seq_lens, device) - cu_h_plain, total_h = _cu_h(seq_lens) - - h = _fwd_h(q, k, v, alpha, beta, scale, cu, seq_lens) - - results = {} - for with_h in (True, False): - g = cudnn.pygraph() - q_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="q") - k_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="k") - v_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="v") - g_t = g.tensor([total, H], data_type=cudnn.data_type.FLOAT, name="g") - beta_t = g.tensor([total, H], data_type=cudnn.data_type.FLOAT, name="beta") - cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") - do_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="dO") - kwargs = {} - if with_h: - kwargs["h"] = g.tensor([max(total_h, 1), H, D, D], data_type=cudnn.data_type.BFLOAT16, name="h") - dQ_t, dK_t, dV_t, dG_t, dBeta_t, _ = g.gdn_bwd( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=beta_t, - cu_seqlens=cu_t, - dO=do_t, - scale=scale, - name="gdn_bwd", - **kwargs, - ) - for t_, dt in ( - (dQ_t, cudnn.data_type.BFLOAT16), - (dK_t, cudnn.data_type.BFLOAT16), - (dV_t, cudnn.data_type.BFLOAT16), - (dG_t, cudnn.data_type.FLOAT), - (dBeta_t, cudnn.data_type.FLOAT), - ): - t_.set_output(True).set_data_type(dt) - g.build() - assert isinstance(g.selected_engine, GdnFrostEngine), f"frost engine must accept with_h={with_h}" - - dq = torch.empty(total, H, D, dtype=q.dtype, device=device) - dk = torch.empty(total, H, D, dtype=q.dtype, device=device) - dv = torch.empty(total, H, D, dtype=q.dtype, device=device) - dg = torch.empty(total, H, dtype=torch.float32, device=device) - dbeta = torch.empty(total, H, dtype=torch.float32, device=device) - pack = { - q_t: q, - k_t: k, - v_t: v, - g_t: alpha.log().contiguous(), - beta_t: beta.contiguous(), - cu_t: cu, - do_t: do, - dQ_t: dq, - dK_t: dk, - dV_t: dv, - dG_t: dg, - dBeta_t: dbeta, - } - if with_h: - pack[kwargs["h"]] = h - g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) - torch.cuda.synchronize() - results[with_h] = (dq, dk, dv, dg, dbeta) - - # both paths run the identical kernels on the identical log-decay inputs - for a, b, name in zip(results[True], results[False], ("dq", "dk", "dv", "dg", "dbeta")): - assert torch.equal(a, b), f"no-h recompute path diverges from explicit h on {name}" - - -def _reference_grads_s0(q, k, v, alpha, beta, do, scale, seq_lens, s0): - """fp64 autograd oracle including the initial-state leaf.""" - HO = max(q.shape[1], v.shape[1]) - - def leaf(x): - r = HO // x.shape[1] - x = x.double().repeat_interleave(r, dim=1) if r > 1 else x.double() - return x.requires_grad_(True) - - qq, kk, vv = leaf(q), leaf(k), leaf(v) - gate = alpha.double().log().requires_grad_(True) - bb = beta.double().requires_grad_(True) - ss0 = s0.double().requires_grad_(True) - o, _fs = gdn_reference( - qq.unsqueeze(0), - kk.unsqueeze(0), - vv.unsqueeze(0), - gate.unsqueeze(0), - bb.unsqueeze(0), - scale=scale, - initial_state=ss0, - cu_seqlens=_cu(seq_lens, q.device), - ) - (o.squeeze(0) * do.double()).sum().backward() - return qq.grad, kk.grad, vv.grad, gate.grad, bb.grad, ss0.grad - - -@requires_runtime -@pytest.mark.parametrize("heads", [(2, 2, 2), (2, 2, 4)]) -@pytest.mark.parametrize("seq_lens", [[256], [128, 512]]) -def test_bprop_d_initial_state(seq_lens, heads): - """dL/dS0 vs the fp64 autograd oracle with a nonzero initial state. - - The backward takes the plain per-chunk h plus the io-downcast S0 - (``initial_state``, read through its own descriptor set for chunk 0).""" - _seed() - q, k, v, do, alpha, beta = _gen_case(seq_lens, *heads) - HO = max(q.shape[1], v.shape[1]) - D = q.shape[2] - device = q.device - scale = 1.0 / (D**0.5) - num_seqs = len(seq_lens) - s0 = (torch.randn(num_seqs, HO, D, D, device=device, dtype=torch.float32) * 0.05).contiguous() - - cu = _cu(seq_lens, device) - dq, dk, dv, dg, db, ds0 = _run_bwd(q, k, v, alpha, beta, do, None, scale, cu, s0=s0) - - rq, rk, rv, rg, rb, rs0 = _reference_grads_s0(q, k, v, alpha, beta, do, scale, seq_lens, s0) - - def rms(a, b): - return (a.double() - b).pow(2).mean().sqrt().item() - - assert not ds0.isnan().any(), "d_initial_state has unwritten slots" - rq, rk, rv = _reduce_ref(rq, q.shape[1]), _reduce_ref(rk, k.shape[1]), _reduce_ref(rv, v.shape[1]) - assert rms(dq, rq) < 6e-2 and rms(dk, rk) < 6e-2 and rms(dv, rv) < 6e-2 - assert rms(dg, rg) < 6e-2 and rms(db, rb) < 6e-2 - assert rms(ds0, rs0) < 6e-2, f"ds0 rms {rms(ds0, rs0)}" - - -def _reference_grads_dht(q, k, v, alpha, beta, do, scale, seq_lens, dht): - """fp64 autograd oracle with a final-state-gradient loss term (dht).""" - HO = max(q.shape[1], v.shape[1]) - - def leaf(x): - r = HO // x.shape[1] - x = x.double().repeat_interleave(r, dim=1) if r > 1 else x.double() - return x.requires_grad_(True) - - qq, kk, vv = leaf(q), leaf(k), leaf(v) - gate = alpha.double().log().requires_grad_(True) - bb = beta.double().requires_grad_(True) - o, fs = gdn_reference( - qq.unsqueeze(0), - kk.unsqueeze(0), - vv.unsqueeze(0), - gate.unsqueeze(0), - bb.unsqueeze(0), - scale=scale, - initial_state=None, - cu_seqlens=_cu(seq_lens, q.device), - ) - ((o.squeeze(0) * do.double()).sum() + (fs * dht.double()).sum()).backward() - return qq.grad, kk.grad, vv.grad, gate.grad, bb.grad - - -@requires_runtime -@pytest.mark.parametrize("heads", [(2, 2, 2), (2, 2, 4)]) -@pytest.mark.parametrize("seq_lens", [[64], [256], [128, 512]]) -def test_bprop_d_final_state(seq_lens, heads): - """d_final_state seeds the backward dH at the first processed chunk; - every gradient picks up the propagated term.""" - _seed() - q, k, v, do, alpha, beta = _gen_case(seq_lens, *heads) - HO = max(q.shape[1], v.shape[1]) - D = q.shape[2] - device = q.device - scale = 1.0 / (D**0.5) - num_seqs = len(seq_lens) - dht = (torch.randn(num_seqs, HO, D, D, device=device, dtype=torch.float32) * 0.05).contiguous() - - cu = _cu(seq_lens, device) - dq, dk, dv, dg, db = _run_bwd(q, k, v, alpha, beta, do, None, scale, cu, dht=dht) - - rq, rk, rv, rg, rb = _reference_grads_dht(q, k, v, alpha, beta, do, scale, seq_lens, dht) - - def rms(a, b): - return (a.double() - b).pow(2).mean().sqrt().item() - - rq, rk, rv = _reduce_ref(rq, q.shape[1]), _reduce_ref(rk, k.shape[1]), _reduce_ref(rv, v.shape[1]) - for name, got, ref in (("dq", dq, rq), ("dk", dk, rk), ("dv", dv, rv), ("dg", dg, rg), ("db", db, rb)): - assert torch.isfinite(got.float()).all(), f"non-finite values in {name}" - assert rms(got, ref) < 6e-2, f"{name} rms {rms(got, ref):.4g}" - - -@requires_runtime -def test_frost_gdn_bwd_engine_initial_state(seq_lens=(128, 256), H=2, D=128): - """GDN_BWD through the pygraph engine path with initial_state: the engine - extends h with the per-sequence S0 entries; the node's state ports are - K-major [N, H, K, V] (the engine converts to the kernel orientation).""" - - _seed() - seq_lens = list(seq_lens) - q, k, v, do, alpha, beta = _gen_case(seq_lens, HQ=H, head_size=D) - device = q.device - scale = 1.0 / math.sqrt(D) - num_seqs = len(seq_lens) - total = q.shape[0] - s0 = (torch.randn(num_seqs, H, D, D, device=device, dtype=torch.float32) * 0.05).contiguous() - - cu = _cu(seq_lens, device) - _cu_h_plain, total_h = _cu_h(seq_lens) - h = _fwd_h(q, k, v, alpha, beta, scale, cu, seq_lens, s0=s0) - - g = cudnn.pygraph() - q_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="q") - k_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="k") - v_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="v") - g_t = g.tensor([total, H], data_type=cudnn.data_type.FLOAT, name="g") - beta_t = g.tensor([total, H], data_type=cudnn.data_type.FLOAT, name="beta") - cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") - do_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="dO") - h_t = g.tensor([max(total_h, 1), H, D, D], data_type=cudnn.data_type.BFLOAT16, name="h") - s0_t = g.tensor([num_seqs, H, D, D], data_type=cudnn.data_type.FLOAT, name="initial_state") - dQ_t, dK_t, dV_t, dG_t, dBeta_t, dS0_t = g.gdn_bwd( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=beta_t, - cu_seqlens=cu_t, - dO=do_t, - h=h_t, - initial_state=s0_t, - scale=scale, - name="gdn_bwd", - ) - for t_, dt in ( - (dQ_t, cudnn.data_type.BFLOAT16), - (dK_t, cudnn.data_type.BFLOAT16), - (dV_t, cudnn.data_type.BFLOAT16), - (dG_t, cudnn.data_type.FLOAT), - (dBeta_t, cudnn.data_type.FLOAT), - (dS0_t, cudnn.data_type.FLOAT), - ): - t_.set_output(True).set_data_type(dt) - - g.build() - assert isinstance(g.selected_engine, GdnFrostEngine), "frost engine must accept initial_state" - - dq = torch.empty(total, H, D, dtype=q.dtype, device=device) - dk = torch.empty(total, H, D, dtype=q.dtype, device=device) - dv = torch.empty(total, H, D, dtype=q.dtype, device=device) - dg = torch.empty(total, H, dtype=torch.float32, device=device) - dbeta = torch.empty(total, H, dtype=torch.float32, device=device) - ds0 = torch.full((num_seqs, H, D, D), float("nan"), dtype=torch.float32, device=device) - wsb = torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device) - g.execute( - { - q_t: q, - k_t: k, - v_t: v, - g_t: alpha.log().contiguous(), - beta_t: beta.contiguous(), - cu_t: cu, - do_t: do, - h_t: h, - s0_t: s0, - dQ_t: dq, - dK_t: dk, - dV_t: dv, - dG_t: dg, - dBeta_t: dbeta, - dS0_t: ds0, - }, - wsb, - ) - torch.cuda.synchronize() - - rq, rk, rv, rg, rb, rs0 = _reference_grads_s0(q, k, v, alpha, beta, do, scale, seq_lens, s0) - - def rms(a, b): - return (a.double() - b).pow(2).mean().sqrt().item() - - assert not ds0.isnan().any(), "d_initial_state has unwritten slots" - assert rms(dq, rq) < BWD_TOL and rms(dk, rk) < BWD_TOL and rms(dv, rv) < BWD_TOL - assert rms(dg, rg) < BWD_TOL and rms(dbeta, rb) < BWD_TOL - assert rms(ds0, rs0) < BWD_TOL, f"ds0 rms {rms(ds0, rs0)}" - - -# --------------------------------------------------------------------------- -# GVA: head-group reduction back to native q/k heads -# --------------------------------------------------------------------------- - - -def _reference_grads_native(q, k, v, alpha, beta, do, scale, seq_lens): - """fp64 autograd oracle at NATIVE head counts: gdn_reference expands the - head groups internally, so autograd sums each group's gradient.""" - qq = q.double().requires_grad_(True) - kk = k.double().requires_grad_(True) - vv = v.double().requires_grad_(True) - gate = alpha.double().log().requires_grad_(True) - bb = beta.double().requires_grad_(True) - o, _fs = gdn_reference( - qq.unsqueeze(0), - kk.unsqueeze(0), - vv.unsqueeze(0), - gate.unsqueeze(0), - bb.unsqueeze(0), - scale=scale, - initial_state=None, - cu_seqlens=_cu(seq_lens, q.device), - ) - (o.squeeze(0) * do.double()).sum().backward() - return qq.grad, kk.grad, vv.grad, gate.grad, bb.grad - - -@requires_runtime -@pytest.mark.parametrize("num_q_heads, num_v_heads", [(2, 4), (16, 64), (3, 6)]) -@pytest.mark.parametrize("seq_lens", [[256], [128, 512], [77, 178], [1]]) -def test_bprop_kernel_gva_native_heads(num_q_heads, num_v_heads, seq_lens): - """GVA through the node surface: the engine returns native-head grads.""" - scale = 1.0 / math.sqrt(128) - q, k, v, do, alpha, beta = _gen_case(seq_lens, num_q_heads, num_q_heads, num_v_heads) - cu = _cu(seq_lens) - dq, dk, dv, dg, db = _run_bwd(q, k, v, alpha, beta, do, None, scale, cu) - refs = _reference_grads_native(q, k, v, alpha, beta, do, scale, seq_lens) - for name, got, ref in (("dq", dq, refs[0]), ("dk", dk, refs[1]), ("dv", dv, refs[2]), ("dg", dg, refs[3]), ("db", db, refs[4])): - assert torch.isfinite(got.float()).all(), f"non-finite values in {name}" - r = rms_ratio(got.float(), ref.float()) - assert r < BWD_TOL, f"{name} rms ratio {r:.4g} >= {BWD_TOL}" - - -@requires_runtime -@pytest.mark.parametrize("num_q_heads, num_v_heads", [(2, 4), (16, 64)]) -def test_frost_gdn_bwd_engine_gva(num_q_heads, num_v_heads, seq_lens=(128, 256), D=128): - """GDN_BWD through the FROST engine with grouped value heads: dQ/dK come - back at the node's native q/k head counts via the head-group reduction.""" - - _seed() - seq_lens = list(seq_lens) - HQ, HV = num_q_heads, num_v_heads - q, k, v, do, alpha, beta = _gen_case(seq_lens, HQ=HQ, HV=HV, head_size=D) - device = q.device - scale = 1.0 / math.sqrt(D) - num_seqs = len(seq_lens) - total = q.shape[0] - cu = _cu(seq_lens, device) - cu_h, total_h = _cu_h(seq_lens) - h = _fwd_h(q, k, v, alpha, beta, scale, cu, seq_lens) - - g = cudnn.pygraph() - q_t = g.tensor([total, HQ, D], data_type=cudnn.data_type.BFLOAT16, name="q") - k_t = g.tensor([total, HQ, D], data_type=cudnn.data_type.BFLOAT16, name="k") - v_t = g.tensor([total, HV, D], data_type=cudnn.data_type.BFLOAT16, name="v") - g_t = g.tensor([total, HV], data_type=cudnn.data_type.FLOAT, name="g") - beta_t = g.tensor([total, HV], data_type=cudnn.data_type.FLOAT, name="beta") - cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") - do_t = g.tensor([total, HV, D], data_type=cudnn.data_type.BFLOAT16, name="dO") - h_t = g.tensor([max(total_h, 1), HV, D, D], data_type=cudnn.data_type.BFLOAT16, name="h") - dQ_t, dK_t, dV_t, dG_t, dBeta_t, _ = g.gdn_bwd( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=beta_t, - cu_seqlens=cu_t, - dO=do_t, - h=h_t, - scale=scale, - name="gdn_bwd", - ) - for t_, dt in ( - (dQ_t, cudnn.data_type.BFLOAT16), - (dK_t, cudnn.data_type.BFLOAT16), - (dV_t, cudnn.data_type.BFLOAT16), - (dG_t, cudnn.data_type.FLOAT), - (dBeta_t, cudnn.data_type.FLOAT), - ): - t_.set_output(True).set_data_type(dt) - g.build() - assert isinstance(g.selected_engine, GdnFrostEngine), "frost engine must accept GVA" - - dq = torch.full((total, HQ, D), float("nan"), dtype=q.dtype, device=device) - dk = torch.full((total, HQ, D), float("nan"), dtype=q.dtype, device=device) - dv = torch.full((total, HV, D), float("nan"), dtype=q.dtype, device=device) - dg = torch.full((total, HV), float("nan"), dtype=torch.float32, device=device) - dbeta = torch.full((total, HV), float("nan"), dtype=torch.float32, device=device) - g.execute( - { - q_t: q, - k_t: k, - v_t: v, - g_t: alpha.log().contiguous(), - beta_t: beta.contiguous(), - cu_t: cu, - do_t: do, - h_t: h, - dQ_t: dq, - dK_t: dk, - dV_t: dv, - dG_t: dg, - dBeta_t: dbeta, - }, - torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device), - ) - torch.cuda.synchronize() - - refs = _reference_grads_native(q, k, v, alpha, beta, do, scale, seq_lens) - for name, got, ref in (("dq", dq, refs[0]), ("dk", dk, refs[1]), ("dv", dv, refs[2]), ("dg", dg, refs[3]), ("db", dbeta, refs[4])): - assert torch.isfinite(got.float()).all(), f"non-finite values in {name}" - r = rms_ratio(got.float(), ref.float()) - assert r < BWD_TOL, f"{name} rms ratio {r:.4g} >= {BWD_TOL}" - - -@requires_runtime -@pytest.mark.parametrize("num_q_heads, num_k_heads, num_v_heads", [(4, 1, 1), (6, 2, 2)]) -@pytest.mark.parametrize("seq_lens", [[256], [128, 512]]) -def test_bprop_kernel_gqa_native_heads(num_q_heads, num_k_heads, num_v_heads, seq_lens): - """GQA through the node surface (q/k at HQ heads, v at native HV): the - engine returns native-head grads; dk/dg/db reduce to the HK/HV oracle.""" - HQ, HK, HV = num_q_heads, num_k_heads, num_v_heads - HO = HQ - D = 128 - scale = 1.0 / math.sqrt(D) - _seed() - total = sum(seq_lens) - q = multidist_randu(total * HQ, D, device="cuda").reshape(total, HQ, D) - k = F.normalize(multidist_randu(total * HK, D, device="cuda").reshape(total, HK, D), p=2.0, dim=-1) - v = multidist_randu(total * HV, D, device="cuda").reshape(total, HV, D) - do = multidist_randu(total * HO, D, device="cuda").reshape(total, HO, D) - q, k, v, do = (t.bfloat16().contiguous() for t in (q, k, v, do)) - alpha_hv = torch.empty(total, HV, device="cuda").uniform_(0.1, 1.0) - beta_hv = torch.rand(total, HV, device="cuda") - alpha = alpha_hv.repeat_interleave(HO // HV, dim=1).contiguous() - beta = beta_hv.repeat_interleave(HO // HV, dim=1).contiguous() - cu = _cu(seq_lens) - k_hq = k.repeat_interleave(HQ // HK, dim=1).contiguous() - dq, dk_hq, dv, dg_ho, db_ho = _run_bwd(q, k_hq, v, alpha, beta, do, None, scale, cu) - refs = _reference_grads_native(q, k, v, alpha_hv, beta_hv, do, scale, seq_lens) - for name, got, ref in ( - ("dq", dq, refs[0]), - ("dk", _reduce_ref(dk_hq, HK), refs[1]), - ("dv", dv, refs[2]), - ("dg", _reduce_ref(dg_ho, HV), refs[3]), - ("db", _reduce_ref(db_ho, HV), refs[4]), - ): - assert torch.isfinite(got.float()).all(), f"non-finite values in {name}" - r = rms_ratio(got.float(), ref.float()) - assert r < BWD_TOL, f"{name} rms ratio {r:.4g} >= {BWD_TOL}" - - -# --------------------------------------------------------------------------- -# split-K: strong decay on a long-sequence pack, so the backward's table -# actually cuts (see the partition-table unit tests in -# test_gdn_prefill_kernel.py) -# --------------------------------------------------------------------------- - - -@requires_runtime -@pytest.mark.parametrize("with_s0", [False, True]) -@pytest.mark.parametrize("with_dht", [False, True]) -def test_bprop_split_strong_decay(with_s0, with_dht): - """Strong decay saturates the scan's warmup threshold on the 2048-token - sequence, so the split table cuts it into several work items; all - gradients against the fp64 autograd oracle.""" - _seed() - seq_lens = [100, 2048, 0, 517] - H = 2 - total = sum(seq_lens) - q = torch.randn(total, H, 128, dtype=torch.bfloat16, device="cuda") * 0.5 - k = F.normalize(torch.randn(total, H, 128, device="cuda"), dim=-1).bfloat16() - v = torch.randn(total, H, 128, dtype=torch.bfloat16, device="cuda") * 0.5 - do = torch.randn(total, H, 128, dtype=torch.bfloat16, device="cuda") * 0.5 - alpha = (torch.rand(total, H, device="cuda") * 0.9 + 0.05).float() - beta = torch.rand(total, H, dtype=torch.float32, device="cuda") - cu = _cu(seq_lens) - B = len(seq_lens) - scale = 1.0 / math.sqrt(128) - s0 = (torch.randn(B, H, 128, 128, dtype=torch.float32, device="cuda") * 0.05) if with_s0 else None - dht = (torch.randn(B, H, 128, 128, dtype=torch.float32, device="cuda") * 0.05) if with_dht else None - - got = _run_bwd(q, k, v, alpha, beta, do, None, scale, cu, s0=s0, dht=dht) - - qq = q.double().requires_grad_(True) - kk = k.double().requires_grad_(True) - vv = v.double().requires_grad_(True) - gl = alpha.double().log().requires_grad_(True) - bb = beta.double().requires_grad_(True) - ss0 = s0.double().requires_grad_(True) if with_s0 else None - o, fs = gdn_reference( - qq.unsqueeze(0), - kk.unsqueeze(0), - vv.unsqueeze(0), - gl.unsqueeze(0), - bb.unsqueeze(0), - scale=scale, - initial_state=ss0, - cu_seqlens=cu, - ) - loss = (o.squeeze(0) * do.double()).sum() - if with_dht: - loss = loss + (fs * dht.double()).sum() - loss.backward() - ref = (qq.grad, kk.grad, vv.grad, gl.grad, bb.grad) + ((ss0.grad,) if with_s0 else ()) - - names = ("dq", "dk", "dv", "dg", "dbeta") + (("ds0",) if with_s0 else ()) - for name, a_, b_ in zip(names, got, ref): - assert not torch.isnan(a_.float()).any(), f"unwritten {name}" - assert rms_ratio(a_.float(), b_.float()) < 6e-2, name - if with_s0: - # a zero-length sequence passes the gradient through: ds0 = dht or 0 - zi = seq_lens.index(0) - expected = dht[zi] if with_dht else torch.zeros_like(got[5][zi]) - torch.testing.assert_close(got[5][zi], expected, atol=0, rtol=0) - - -# --------------------------------------------------------------------------- -# Determinism stress: bitwise repeat runs (fixed forward, repeated backward) -# --------------------------------------------------------------------------- - - -@requires_runtime -def test_gdn_bprop_determinism(): - seq_lens = [497, 16, 480, 256] - q, k, v, do, alpha, beta = _gen_case(seq_lens, HQ=2, HK=2, HV=2) - cu = _cu(seq_lens) - scale = 1.0 / math.sqrt(128) - assert_bitwise_runs(lambda: _run_bwd(q, k, v, alpha, beta, do, None, scale, cu), label="gdn_bwd") - - -# --------------------------------------------------------------------------- -# CUDA graph capture/replay across dynamic shapes (fixed SM-count grid) -# --------------------------------------------------------------------------- - - -@requires_runtime -def test_gdn_bprop_cuda_graph_replay(): - """Capture the ENGINE backward once (h=None: the forward-state recompute - happens INSIDE the capture), replay across CHANGED effective shapes; - every replay must match an eager engine backward bit for bit.""" - T_cap, B_cap, H, D = 768, 4, 2, 128 - dev = "cuda" - _seed() - scale = 1.0 / math.sqrt(D) - q = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - k = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - v = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - do = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - log_g = torch.zeros(T_cap, H, dtype=torch.float32, device=dev) - beta = torch.zeros(T_cap, H, dtype=torch.float32, device=dev) - cu = torch.zeros(B_cap + 1, dtype=torch.int32, device=dev) - - g, t = _build_bwd_graph(T_cap, H, H, D, B_cap, scale, cudnn.data_type.BFLOAT16) - ws = torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=dev) - base = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: log_g, t["beta"]: beta, t["cu"]: cu, t["dO"]: do} - - def bufs(): - return [torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) for _ in range(3)] + [ - torch.zeros(T_cap, H, dtype=torch.float32, device=dev) for _ in range(2) - ] - - grads_graph, grads_eager = bufs(), bufs() - pack_graph = {**base, **dict(zip(t["grads"], grads_graph))} - pack_eager = {**base, **dict(zip(t["grads"], grads_eager))} - - def fill(seq_lens): - total = sum(seq_lens) - q[:total] = torch.randn(total, H, D, device=dev).bfloat16() * 0.5 - k[:total] = F.normalize(torch.randn(total, H, D, device=dev), dim=-1).bfloat16() - v[:total] = torch.randn(total, H, D, device=dev).bfloat16() * 0.5 - do[:total] = torch.randn(total, H, D, device=dev).bfloat16() * 0.5 - log_g[:total] = torch.empty(total, H, device=dev).uniform_(0.1, 1.0).log() - beta[:total] = torch.rand(total, H, device=dev) - bounds = [0] + list(accumulate(seq_lens)) - bounds += [bounds[-1]] * (B_cap + 1 - len(bounds)) - cu.copy_(torch.tensor(bounds, dtype=torch.int32)) - return total - - fill([256, 448, 0, 0]) - stream = torch.cuda.Stream() - handle = cudnn.create_handle() - cudnn.set_stream(handle, stream.cuda_stream) - with torch.cuda.stream(stream): - g.execute(pack_graph, ws, handle=handle) # warmup: compile + caches - torch.cuda.synchronize() - - cg = torch.cuda.CUDAGraph() - with torch.cuda.graph(cg, stream=stream): - g.execute(pack_graph, ws, handle=handle) - - eager_handle = cudnn.create_handle() - cudnn.set_stream(eager_handle, torch.cuda.current_stream().cuda_stream) - for seq_lens in ([256, 448, 0, 0], [100, 200, 56, 0], [768], [64, 0, 64, 512]): - total = fill(seq_lens) - cg.replay() - torch.cuda.synchronize() - g.execute(pack_eager, ws, handle=eager_handle) - torch.cuda.synchronize() - for name, a_, b_ in zip(("dq", "dk", "dv", "dg", "db"), grads_graph, grads_eager): - assert torch.equal(a_[:total], b_[:total]), f"graph {name} diverges from eager at {seq_lens}" diff --git a/test/python/linear_attention/frost/test_gdn_prefill_kernel.py b/test/python/linear_attention/frost/test_gdn_prefill_kernel.py deleted file mode 100644 index 9d9f24752..000000000 --- a/test/python/linear_attention/frost/test_gdn_prefill_kernel.py +++ /dev/null @@ -1,644 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""FROST GDN prefill tests (pygraph GDN node on GdnFrostEngine) against the -fp64 recurrent reference.""" - -from __future__ import annotations - -import math -import random -from itertools import accumulate - -import pytest -import torch -import torch.nn.functional as F - -import cudnn # noqa: F401 (conftest extends cudnn.__path__ with the source tree) - -from linear_attention.common import FWD_TOL, STATE_TOL, assert_bitwise_runs, assert_concurrent_stream_runs, assert_engine_declines -from linear_attention.conftest import multidist_randu -from linear_attention.reference_gdn import gdn_reference, rms_ratio - -pytestmark = pytest.mark.L0 - -SEED = 42 - - -def _sm100_dsl_available() -> bool: - if not torch.cuda.is_available(): - return False - major, _minor = torch.cuda.get_device_capability() - if major != 10: - return False - try: - import cutlass.experimental.primitives # noqa: F401 — often a sys.modules alias, invisible to find_spec - except ImportError: - return False - return True - - -requires_runtime = pytest.mark.skipif( - not _sm100_dsl_available(), - reason="needs an SM100-class GPU and the Cutlass DSL", -) - - -def _seed(seed=SEED): - random.seed(seed) - torch.random.manual_seed(seed) - torch.cuda.manual_seed(seed) - - -def _cu(seq_lens, device="cuda"): - return torch.tensor([0] + list(accumulate(seq_lens)), dtype=torch.int32, device=device) - - -def _gen_thd(seq_lens, num_q_heads, num_k_heads, num_v_heads, head_size, dtype): - total = sum(seq_lens) - q = multidist_randu(total * num_q_heads, head_size, device="cuda").reshape(total, num_q_heads, head_size) - k = multidist_randu(total * num_k_heads, head_size, device="cuda").reshape(total, num_k_heads, head_size) - k = F.normalize(k, p=2.0, dim=-1) - v = multidist_randu(total * num_v_heads, head_size, device="cuda").reshape(total, num_v_heads, head_size) - return q.to(dtype).contiguous(), k.to(dtype).contiguous(), v.to(dtype).contiguous() - - -def _build_gdn_graph( - total, HQ, HV, head_size, num_seqs, scale, io_dt, *, output_final_state, s0_shape=None, s0_dt=None, fs_dt=None, checkpoint_every_n_tokens=0 -): - - HO = max(HQ, HV) - g = cudnn.pygraph() - t = dict( - q=g.tensor([total, HQ, head_size], data_type=io_dt, name="q"), - k=g.tensor([total, HQ, head_size], data_type=io_dt, name="k"), - v=g.tensor([total, HV, head_size], data_type=io_dt, name="v"), - g=g.tensor([total, HO], data_type=cudnn.data_type.FLOAT, name="g"), - beta=g.tensor([total, HO], data_type=cudnn.data_type.FLOAT, name="beta"), - cu=g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens"), - ) - if s0_shape is not None: - t["s0"] = g.tensor(list(s0_shape), data_type=s0_dt, name="initial_state") - O_t, fs_t, h_t = g.gdn( - q=t["q"], - k=t["k"], - v=t["v"], - g=t["g"], - beta=t["beta"], - cu_seqlens=t["cu"], - initial_state=t.get("s0"), - scale=float(scale), - output_final_state=output_final_state, - checkpoint_every_n_tokens=checkpoint_every_n_tokens, - name="gdn", - ) - if h_t is not None: - h_t.set_output(True).set_data_type(io_dt) - t["H"] = h_t - O_t.set_output(True).set_data_type(io_dt) - t["O"] = O_t - if output_final_state: - fs_t.set_output(True).set_data_type(fs_dt) - t["fs"] = fs_t - g.build() - return g, t - - -def chunk_gated_delta_rule( - q, - k, - v, - alpha, - beta, - scale, - initial_state, - output_final_state, - cu_seqlens, - output=None, - output_state=None, -): - """Torch adapter over the graph. ``alpha`` / ``beta`` are the - raw linear gates (``None`` -> ones); the node takes natural-log decay.""" - device = q.device - total, HQ = q.shape[0], q.shape[1] - HV = v.shape[1] - HO = max(HQ, HV) - head_size = q.shape[2] - num_seqs = cu_seqlens.shape[0] - 1 - io_dt = cudnn.data_type.BFLOAT16 if q.dtype == torch.bfloat16 else cudnn.data_type.HALF - - gate = (alpha if alpha is not None else torch.ones(total, HO, device=device)).float() - log_g = gate.log().contiguous() - beta_f = (beta if beta is not None else torch.ones(total, HO, device=device)).float().contiguous() - cu = cu_seqlens.to(torch.int32).contiguous() - if output is None: - output = torch.empty(total, HO, head_size, dtype=q.dtype, device=device) - if output_final_state and output_state is None: - output_state = torch.empty(num_seqs, HO, head_size, head_size, dtype=torch.float32, device=device) - if not output_final_state: - output_state = None - - s0_dt = None - if initial_state is not None: - s0_dt = cudnn.data_type.BFLOAT16 if initial_state.dtype == torch.bfloat16 else cudnn.data_type.FLOAT - fs_dt = None - if output_final_state: - fs_dt = cudnn.data_type.BFLOAT16 if output_state.dtype == torch.bfloat16 else cudnn.data_type.FLOAT - g, t = _build_gdn_graph( - total, - HQ, - HV, - head_size, - num_seqs, - scale, - io_dt, - output_final_state=output_final_state, - s0_shape=None if initial_state is None else initial_state.shape, - s0_dt=s0_dt, - fs_dt=fs_dt, - ) - pack = {t["q"]: q.contiguous(), t["k"]: k.contiguous(), t["v"]: v.contiguous(), t["g"]: log_g, t["beta"]: beta_f, t["cu"]: cu, t["O"]: output} - if initial_state is not None: - pack[t["s0"]] = initial_state.contiguous() - if output_final_state: - pack[t["fs"]] = output_state - ws = torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device) - g.execute(pack, ws) - return output, output_state - - -def _reference(q, k, v, alpha, beta, scale, seq_lens, initial_state=None): - """fp64 oracle on THD inputs; gates are raw linear (``None`` -> ones).""" - total = q.shape[0] - HO = max(q.shape[1], v.shape[1]) - device = q.device - gate = alpha if alpha is not None else torch.ones(total, HO, device=device) - beta_f = beta if beta is not None else torch.ones(total, HO, device=device) - with torch.no_grad(): - o, fs = gdn_reference( - q.unsqueeze(0), - k.unsqueeze(0), - v.unsqueeze(0), - gate.log().unsqueeze(0), - beta_f.unsqueeze(0), - scale=scale, - initial_state=initial_state, - cu_seqlens=_cu(seq_lens, device), - ) - return o.squeeze(0), fs - - -HEAD_CONFIGS = [ - (1, 1, 1), - (4, 1, 1), - (3, 3, 3), - (6, 2, 2), - (1, 1, 2), - (2, 2, 4), - (16, 16, 32), - (16, 16, 64), -] - - -def _run_prefill_case( - dtype, - num_q_heads, - num_k_heads, - num_v_heads, - seq_lens, - scale, - alpha, - beta, - head_size=128, -): - _seed() - total = sum(seq_lens) - num_seqs = len(seq_lens) - HO = max(num_q_heads, num_v_heads) - - dtype = getattr(torch, dtype) - q, k, v = _gen_thd(seq_lens, num_q_heads, num_k_heads, num_v_heads, head_size, dtype) - alpha_t = torch.empty(total, HO, device="cuda").uniform_(0.1, 1.0) if alpha else None - beta_t = torch.rand(total, HO, device="cuda") if beta else None - - our_o = torch.full((total, HO, head_size), float("nan"), dtype=dtype, device="cuda") - our_state = torch.full((num_seqs, HO, head_size, head_size), float("nan"), dtype=torch.float32, device="cuda") - chunk_gated_delta_rule(q, k, v, alpha_t, beta_t, scale, None, True, _cu(seq_lens), output=our_o, output_state=our_state) - torch.cuda.synchronize() - - ref_o, ref_state = _reference(q, k, v, alpha_t, beta_t, scale, seq_lens) - assert rms_ratio(our_o, ref_o) < FWD_TOL[dtype] - # The kernel state is K-major [N,HO,K,V], matching the reference. - assert rms_ratio(our_state, ref_state) < STATE_TOL[dtype] - - -@requires_runtime -@pytest.mark.parametrize("num_q_heads, num_k_heads, num_v_heads", HEAD_CONFIGS) -@pytest.mark.parametrize("seq_lens", [[256], [256, 256], [64, 128, 512]]) -@pytest.mark.parametrize("dtype", ["float16", "bfloat16"]) -def test_prefill_kernel_basic(dtype, num_q_heads, num_k_heads, num_v_heads, seq_lens): - _run_prefill_case( - dtype, - num_q_heads, - num_k_heads, - num_v_heads, - seq_lens, - scale=1.0 / math.sqrt(128), - alpha=True, - beta=True, - ) - - -@requires_runtime -@pytest.mark.parametrize("beta", [False, True]) -@pytest.mark.parametrize("alpha", [False, True]) -@pytest.mark.parametrize("scale", [1.0, "auto"]) -def test_prefill_kernel_gates_and_scale(scale, alpha, beta): - if not alpha and not beta: - pytest.skip("large diff due to output value amplitude explosion along token dimension") - scale = 1.0 / math.sqrt(128) if scale == "auto" else scale - _run_prefill_case("bfloat16", 3, 3, 3, [64, 128, 512], scale=scale, alpha=alpha, beta=beta) - - -@requires_runtime -@pytest.mark.parametrize("num_q_heads, num_k_heads, num_v_heads", [(3, 3, 3), (4, 1, 1), (2, 2, 4)]) -@pytest.mark.parametrize("seq_lens", [[31], [251], [511, 501], [31, 63, 93, 123, 150, 500]]) -@pytest.mark.parametrize("dtype", ["bfloat16"]) -def test_prefill_kernel_nonfull(dtype, num_q_heads, num_k_heads, num_v_heads, seq_lens): - _run_prefill_case( - dtype, - num_q_heads, - num_k_heads, - num_v_heads, - seq_lens, - scale=1.0 / math.sqrt(128), - alpha=True, - beta=True, - ) - - -@requires_runtime -@pytest.mark.parametrize("num_q_heads, num_k_heads, num_v_heads", [(1, 1, 1), (16, 16, 64)]) -@pytest.mark.parametrize("seq_len", [256, 255]) -def test_prefill_kernel_zero_length_sequence(num_q_heads, num_k_heads, num_v_heads, seq_len): - """A trailing zero-length sequence neither changes the output nor hangs.""" - _seed() - head_size = 128 - HO = max(num_q_heads, num_v_heads) - q, k, v = _gen_thd([seq_len], num_q_heads, num_k_heads, num_v_heads, head_size, torch.bfloat16) - alpha = torch.rand(seq_len, HO, device="cuda") - beta = torch.rand(seq_len, HO, device="cuda") - - ref_o, _ = chunk_gated_delta_rule(q, k, v, alpha, beta, 0.1, None, False, _cu([seq_len])) - our_o, _ = chunk_gated_delta_rule(q, k, v, alpha, beta, 0.1, None, False, _cu([seq_len, 0])) - torch.cuda.synchronize() - torch.testing.assert_close(our_o, ref_o, atol=2e-2, rtol=2e-2) - - -@requires_runtime -@pytest.mark.parametrize("with_initial_state", [False, True]) -def test_prefill_zero_length_sequence_state_passthrough(with_initial_state): - """A zero-length sequence's final-state slot gets the passthrough value: - its initial state when seeded, zeros otherwise.""" - _seed() - seq_len, head_size, num_heads, sentinel = 256, 128, 1, 123.0 - q, k, v = _gen_thd([seq_len], num_heads, num_heads, num_heads, head_size, torch.bfloat16) - alpha = torch.rand(seq_len, num_heads, device="cuda") - beta = torch.rand(seq_len, num_heads, device="cuda") - - s0 = torch.randn(2, num_heads, head_size, head_size, dtype=torch.float32, device="cuda") if with_initial_state else None - our_state = torch.full((2, num_heads, head_size, head_size), sentinel, dtype=torch.float32, device="cuda") - chunk_gated_delta_rule(q, k, v, alpha, beta, 0.1, s0, True, _cu([seq_len, 0]), output_state=our_state) - torch.cuda.synchronize() - expected = s0[1] if with_initial_state else torch.zeros_like(our_state[1]) - torch.testing.assert_close(our_state[1], expected, atol=0, rtol=0) - - -@requires_runtime -@pytest.mark.parametrize("num_q_heads, num_k_heads, num_v_heads", [(6, 2, 2), (2, 2, 4)]) -@pytest.mark.parametrize( - "seq_lens1, seq_lens2", - [([61], [128]), ([256, 256], [511, 501]), ([64, 128, 512], [123, 150, 500])], -) -def test_chunked_prefill(num_q_heads, num_k_heads, num_v_heads, seq_lens1, seq_lens2): - """Two-phase prefill carrying the state matches a single-shot reference.""" - _seed() - head_size = 128 - dtype = torch.bfloat16 - num_seqs = len(seq_lens1) - assert num_seqs == len(seq_lens2) - HO = max(num_q_heads, num_v_heads) - q1, k1, v1 = _gen_thd(seq_lens1, num_q_heads, num_k_heads, num_v_heads, head_size, dtype) - q2, k2, v2 = _gen_thd(seq_lens2, num_q_heads, num_k_heads, num_v_heads, head_size, dtype) - alpha1 = torch.empty(sum(seq_lens1), HO, device="cuda").uniform_(0.1, 1.0) - alpha2 = torch.empty(sum(seq_lens2), HO, device="cuda").uniform_(0.1, 1.0) - beta1 = torch.rand(sum(seq_lens1), HO, device="cuda") - beta2 = torch.rand(sum(seq_lens2), HO, device="cuda") - - scale = 1.0 / math.sqrt(head_size) - o1, state1 = chunk_gated_delta_rule(q1, k1, v1, alpha1, beta1, scale, None, True, _cu(seq_lens1)) - o2, state2 = chunk_gated_delta_rule(q2, k2, v2, alpha2, beta2, scale, state1, True, _cu(seq_lens2)) - torch.cuda.synchronize() - - def concat_varlen(t1, cua, t2, cub): - out = [] - for i in range(cua.size(0) - 1): - out.append(t1[cua[i] : cua[i + 1]]) - out.append(t2[cub[i] : cub[i + 1]]) - return torch.concat(out) - - cu1c, cu2c = _cu(seq_lens1).cpu(), _cu(seq_lens2).cpu() - our_o = concat_varlen(o1, cu1c, o2, cu2c) - q = concat_varlen(q1, cu1c, q2, cu2c) - k = concat_varlen(k1, cu1c, k2, cu2c) - v = concat_varlen(v1, cu1c, v2, cu2c) - alpha = concat_varlen(alpha1, cu1c, alpha2, cu2c) - beta = concat_varlen(beta1, cu1c, beta2, cu2c) - seq_lens = [a + b for a, b in zip(seq_lens1, seq_lens2)] - - ref_o, ref_state = _reference(q, k, v, alpha, beta, scale, seq_lens) - assert rms_ratio(our_o, ref_o) < FWD_TOL[dtype] - assert rms_ratio(state2, ref_state) < STATE_TOL[dtype] - - -@requires_runtime -@pytest.mark.parametrize("seq_lens", [[64], [256, 256], [64, 128, 512]]) -def test_prefill_kernel_state_dtype_bf16(seq_lens): - """bf16 recurrent state (initial + final) against the fp64 reference.""" - _seed() - head_size = 128 - num_heads = 3 - num_seqs = len(seq_lens) - total = sum(seq_lens) - q, k, v = _gen_thd(seq_lens, num_heads, num_heads, num_heads, head_size, torch.bfloat16) - alpha = torch.empty(total, num_heads, device="cuda").uniform_(0.1, 1.0) - beta = torch.rand(total, num_heads, device="cuda") - initial_state_ref = (torch.randn(num_seqs, num_heads, head_size, head_size, dtype=torch.float32, device="cuda") * 0.01).to(torch.bfloat16) - initial_state = initial_state_ref.contiguous() - - scale = 1.0 / math.sqrt(head_size) - our_state = torch.zeros(num_seqs, num_heads, head_size, head_size, dtype=torch.bfloat16, device="cuda") - our_o, _ = chunk_gated_delta_rule(q, k, v, alpha, beta, scale, initial_state, True, _cu(seq_lens), output_state=our_state) - torch.cuda.synchronize() - - ref_o, ref_state = _reference(q, k, v, alpha, beta, scale, seq_lens, initial_state=initial_state_ref.float()) - assert rms_ratio(our_o, ref_o) < 5e-2 - assert rms_ratio(our_state.float(), ref_state) < 5e-2 - - -# --------------------------------------------------------------------------- -# Per-chunk H output (fwd node surface; the GDN_BWD node's ``h`` source) -# --------------------------------------------------------------------------- - - -def _reference_h(q, k, v, alpha, beta, scale, seq_lens, every_n): - """fp64-chained per-chunk states: entry j of a sequence is the state after - (j + 1) * every_n tokens, strictly before the sequence end (bf16, K-major).""" - HO = max(q.shape[1], v.shape[1]) - - def expand(x): - r = HO // x.shape[1] - return (x.double().repeat_interleave(r, dim=1) if r > 1 else x.double()).unsqueeze(0) - - qq, kk, vv = expand(q), expand(k), expand(v) - gate = alpha.double().log().unsqueeze(0) - bb = beta.double().unsqueeze(0) - cu_piece = torch.tensor([0, every_n], dtype=torch.int32, device=q.device) - hs, off = [], 0 - for sl in seq_lens: - state = torch.zeros(1, HO, q.shape[2], v.shape[2], dtype=torch.float64, device=q.device) - for j in range(max(sl - 1, 0) // every_n): - a, b = off + j * every_n, off + (j + 1) * every_n - _o, state = gdn_reference( - qq[:, a:b], - kk[:, a:b], - vv[:, a:b], - gate[:, a:b], - bb[:, a:b], - scale=scale, - initial_state=state, - cu_seqlens=cu_piece, - ) - hs.append(state.squeeze(0)) - off += sl - return torch.stack(hs).to(q.dtype) if hs else torch.zeros(0, HO, q.shape[2], v.shape[2], dtype=q.dtype, device=q.device) - - -def _run_fwd_h(q, k, v, alpha, beta, scale, cu_seqlens, seq_lens, every_n): - """Engine-driven forward with the per-chunk H output; returns (o, fs, h).""" - device = q.device - total, HQ, HV, D = q.shape[0], q.shape[1], v.shape[1], q.shape[2] - HO = max(HQ, HV) - num_seqs = cu_seqlens.shape[0] - 1 - io_dt = cudnn.data_type.BFLOAT16 if q.dtype == torch.bfloat16 else cudnn.data_type.HALF - g, t = _build_gdn_graph( - total, - HQ, - HV, - D, - num_seqs, - scale, - io_dt, - output_final_state=True, - fs_dt=cudnn.data_type.FLOAT, - checkpoint_every_n_tokens=every_n, - ) - total_h = sum(max(sl - 1, 0) // every_n for sl in seq_lens) - o = torch.empty(total, HO, D, dtype=q.dtype, device=device) - fs = torch.empty(num_seqs, HO, D, D, dtype=torch.float32, device=device) - h = torch.full((max(total_h, 1), HO, D, D), float("nan"), dtype=q.dtype, device=device) - gate = (alpha if alpha is not None else torch.ones(total, HO, device=device)).float().log().contiguous() - beta_f = (beta if beta is not None else torch.ones(total, HO, device=device)).float().contiguous() - pack = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate, t["beta"]: beta_f, t["cu"]: cu_seqlens, t["O"]: o, t["fs"]: fs, t["H"]: h} - g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) - torch.cuda.synchronize() - return o, fs, h[:total_h] - - -@requires_runtime -@pytest.mark.parametrize("every_n", [64, 128]) -@pytest.mark.parametrize("seq_lens", [[256], [64, 128, 512], [255, 0, 511]]) -def test_fwd_h_matches_reference(seq_lens, every_n): - _seed() - q, k, v = _gen_thd(seq_lens, 2, 2, 2, 128, torch.bfloat16) - total = sum(seq_lens) - alpha = torch.empty(total, 2, device="cuda").uniform_(0.1, 1.0) - beta = torch.rand(total, 2, device="cuda") - scale = 1.0 / math.sqrt(128) - _o, _fs, h = _run_fwd_h(q, k, v, alpha, beta, scale, _cu(seq_lens), seq_lens, every_n) - assert not h.float().isnan().any(), "H has unwritten entries" - h_ref = _reference_h(q, k, v, alpha, beta, scale, seq_lens, every_n) - assert h.shape == h_ref.shape - if h.numel(): - r = rms_ratio(h.float(), h_ref.float()) - assert r < 5e-2, f"H rms ratio {r:.4g}" - - -@requires_runtime -def test_fwd_h_cutile_declines(): - - g = cudnn.pygraph() - q_t = g.tensor([256, 2, 128], data_type=cudnn.data_type.BFLOAT16, name="q") - k_t = g.tensor([256, 2, 128], data_type=cudnn.data_type.BFLOAT16, name="k") - v_t = g.tensor([256, 2, 128], data_type=cudnn.data_type.BFLOAT16, name="v") - g_t = g.tensor([256, 2], data_type=cudnn.data_type.FLOAT, name="g") - b_t = g.tensor([256, 2], data_type=cudnn.data_type.FLOAT, name="beta") - cu_t = g.tensor([2], data_type=cudnn.data_type.INT32, name="cu_seqlens") - O_t, _fs, h_t = g.gdn(q=q_t, k=k_t, v=v_t, g=g_t, beta=b_t, cu_seqlens=cu_t, scale=1.0, checkpoint_every_n_tokens=64, name="gdn") - O_t.set_output(True).set_data_type(cudnn.data_type.BFLOAT16) - h_t.set_output(True).set_data_type(cudnn.data_type.BFLOAT16) - assert_engine_declines(g, "gdn_cutile") # cuTile has no per-chunk H output - - -# --------------------------------------------------------------------------- -# split-K: strong decay on a long-sequence pack, so the table actually cuts -# --------------------------------------------------------------------------- - - -def _gen_split_case(seq_lens, HQ, HV): - total = sum(seq_lens) - HO = max(HQ, HV) - q = torch.randn(total, HQ, 128, dtype=torch.bfloat16, device="cuda") * 0.5 - k = F.normalize(torch.randn(total, HQ, 128, device="cuda"), dim=-1).bfloat16() - v = torch.randn(total, HV, 128, dtype=torch.bfloat16, device="cuda") * 0.5 - gate = (torch.rand(total, HO, device="cuda") * 0.9 + 0.05).float() - beta = torch.rand(total, HO, dtype=torch.float32, device="cuda") - return q, k, v, gate, beta, _cu(seq_lens) - - -@requires_runtime -@pytest.mark.parametrize("with_s0", [False, True]) -@pytest.mark.parametrize("heads", [(2, 2), (1, 4), (4, 1)]) # MHA, GVA, GQA -def test_prefill_split_strong_decay(heads, with_s0): - """Strong decay saturates the scan's warmup threshold on the 2048-token - sequence, so the split table cuts it into several work items; checked - against the fp64 recurrent reference.""" - _seed() - HQ, HV = heads - seq_lens = [100, 2048, 0, 517] - q, k, v, gate, beta, cu = _gen_split_case(seq_lens, HQ, HV) - HO = max(HQ, HV) - B = len(seq_lens) - s0 = (torch.randn(B, HO, 128, 128, dtype=torch.float32, device="cuda") * 0.05) if with_s0 else None - scale = 1.0 / math.sqrt(128) - - o = torch.full((sum(seq_lens), HO, 128), float("nan"), dtype=q.dtype, device="cuda") - fs = torch.full((B, HO, 128, 128), float("nan"), dtype=torch.float32, device="cuda") - chunk_gated_delta_rule(q, k, v, gate, beta, scale, s0, True, cu, output=o, output_state=fs) - torch.cuda.synchronize() - - o_ref, fs_ref = _reference(q, k, v, gate, beta, scale, seq_lens, initial_state=s0) - # zero-length sequences leave their final-state slot untouched - nz = torch.tensor([sl > 0 for sl in seq_lens], device="cuda") - assert not torch.isnan(o).any() and not torch.isnan(fs[nz]).any() - assert rms_ratio(o.float(), o_ref.float()) < FWD_TOL[torch.bfloat16] - assert rms_ratio(fs[nz], fs_ref[nz]) < STATE_TOL[torch.bfloat16] - - -# --------------------------------------------------------------------------- -# Determinism stress: bitwise repeat runs + two-stream co-residency -# --------------------------------------------------------------------------- - -DET_VARLEN_MIX = [497, 16, 1, 480, 0, 253] # zero-length + single-token + odd tails - - -def _det_launch(seq_lens, heads=(2, 2, 4), stream=None): - HQ, HK, HV = heads - HO = max(HQ, HV) - total = sum(seq_lens) - q, k, v = _gen_thd(seq_lens, HQ, HK, HV, 128, torch.bfloat16) - alpha = torch.empty(total, HO, device="cuda").uniform_(0.9, 1.0) - beta = torch.rand(total, HO, device="cuda").sigmoid() - cu = _cu(seq_lens) - scale = 1.0 / math.sqrt(128) - - def launch(): - if stream is not None: - with torch.cuda.stream(stream): - return chunk_gated_delta_rule(q, k, v, alpha, beta, scale, None, True, cu) - return chunk_gated_delta_rule(q, k, v, alpha, beta, scale, None, True, cu) - - return launch - - -@requires_runtime -@pytest.mark.parametrize("seq_lens", [DET_VARLEN_MIX, [4096]], ids=["varlen_mix", "long"]) -def test_gdn_prefill_determinism(seq_lens): - _seed() - assert_bitwise_runs(_det_launch(seq_lens), label="gdn") - - -@requires_runtime -def test_gdn_concurrent_streams_determinism(): - _seed() - s1, s2 = torch.cuda.Stream(), torch.cuda.Stream() - assert_concurrent_stream_runs(_det_launch([1024, 31, 512], stream=s1), _det_launch([256, 999, 1, 128], stream=s2), s1, s2) - - -# --------------------------------------------------------------------------- -# CUDA graph capture/replay across dynamic shapes (fixed SM-count grid) -# --------------------------------------------------------------------------- - - -@requires_runtime -def test_gdn_prefill_cuda_graph_replay(): - """Capture the ENGINE execute once, replay across CHANGED effective - shapes: capacity buffers, cu_seqlens with zero-length tails; everything - the engine launches (sched memset, split table, desc rebuilds, kernel) - must be capture-safe, and every replay must match an eager engine launch - on the same data bit for bit.""" - T_cap, B_cap, H, D = 768, 4, 2, 128 - dev = "cuda" - _seed() - scale = 1.0 / math.sqrt(D) - q = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - k = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - v = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - log_g = torch.zeros(T_cap, H, dtype=torch.float32, device=dev) - beta = torch.zeros(T_cap, H, dtype=torch.float32, device=dev) - cu = torch.zeros(B_cap + 1, dtype=torch.int32, device=dev) - o_graph = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - fs_graph = torch.zeros(B_cap, H, D, D, dtype=torch.float32, device=dev) - o_eager = torch.zeros_like(o_graph) - fs_eager = torch.zeros_like(fs_graph) - - g, t = _build_gdn_graph(T_cap, H, H, D, B_cap, scale, cudnn.data_type.BFLOAT16, output_final_state=True, fs_dt=cudnn.data_type.FLOAT) - ws = torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=dev) - base = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: log_g, t["beta"]: beta, t["cu"]: cu} - pack_graph = {**base, t["O"]: o_graph, t["fs"]: fs_graph} - pack_eager = {**base, t["O"]: o_eager, t["fs"]: fs_eager} - - def fill(seq_lens): - total = sum(seq_lens) - q[:total] = torch.randn(total, H, D, device=dev).bfloat16() * 0.5 - k[:total] = F.normalize(torch.randn(total, H, D, device=dev), dim=-1).bfloat16() - v[:total] = torch.randn(total, H, D, device=dev).bfloat16() * 0.5 - log_g[:total] = torch.empty(total, H, device=dev).uniform_(0.1, 1.0).log() - beta[:total] = torch.rand(total, H, device=dev) - bounds = [0] + list(accumulate(seq_lens)) - bounds += [bounds[-1]] * (B_cap + 1 - len(bounds)) - cu.copy_(torch.tensor(bounds, dtype=torch.int32)) - return total - - fill([256, 448, 0, 0]) - stream = torch.cuda.Stream() - handle = cudnn.create_handle() - cudnn.set_stream(handle, stream.cuda_stream) - with torch.cuda.stream(stream): - g.execute(pack_graph, ws, handle=handle) # warmup: compile + caches - torch.cuda.synchronize() - - cg = torch.cuda.CUDAGraph() - with torch.cuda.graph(cg, stream=stream): - g.execute(pack_graph, ws, handle=handle) - - eager_handle = cudnn.create_handle() - cudnn.set_stream(eager_handle, torch.cuda.current_stream().cuda_stream) - for seq_lens in ([256, 448, 0, 0], [100, 200, 56, 0], [768], [64, 0, 64, 512]): - total = fill(seq_lens) - cg.replay() - torch.cuda.synchronize() - g.execute(pack_eager, ws, handle=eager_handle) - torch.cuda.synchronize() - assert torch.equal(o_graph[:total], o_eager[:total]), f"graph o diverges from eager at {seq_lens}" - assert torch.equal(fs_graph, fs_eager), f"graph final_state diverges from eager at {seq_lens}" diff --git a/test/python/linear_attention/frost/test_kda_bprop_kernel.py b/test/python/linear_attention/frost/test_kda_bprop_kernel.py deleted file mode 100644 index 3f14fd0f8..000000000 --- a/test/python/linear_attention/frost/test_kda_bprop_kernel.py +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""FROST KDA backward: currently a STUB on this branch (the small-chunk -design recomputes the forward states in the backward once the kernel lands). -The contract: ``KdaFrostEngine`` declines ``KDA_BWD`` -graphs so the router can fall back.""" - -from __future__ import annotations - -import pytest - -import cudnn # noqa: F401 (conftest extends cudnn.__path__ with the source tree) - -from linear_attention.common import assert_engine_declines - -pytestmark = pytest.mark.L0 - - -def test_kda_bwd_frost_engine_declines(): - - total, H, D = 256, 2, 128 - g = cudnn.pygraph() - q_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="q") - k_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="k") - v_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="v") - g_t = g.tensor([total, H, D], data_type=cudnn.data_type.FLOAT, name="g") - beta_t = g.tensor([total, H], data_type=cudnn.data_type.FLOAT, name="beta") - cu_t = g.tensor([2], data_type=cudnn.data_type.INT32, name="cu_seqlens") - dO_t = g.tensor([total, H, D], data_type=cudnn.data_type.BFLOAT16, name="dO") - g.kda_bwd( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=beta_t, - cu_seqlens=cu_t, - dO=dO_t, - scale=0.125, - name="kda_bwd", - ) - assert_engine_declines(g, "kda_frost") # stub backward kernel on this branch diff --git a/test/python/linear_attention/frost/test_kda_prefill_kernel.py b/test/python/linear_attention/frost/test_kda_prefill_kernel.py deleted file mode 100644 index 0cf8b28aa..000000000 --- a/test/python/linear_attention/frost/test_kda_prefill_kernel.py +++ /dev/null @@ -1,757 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""FROST KDA prefill tests (``pygraph`` + ``KdaFrostEngine``) against the -fp64 recurrent reference.""" - -from __future__ import annotations - -import math -import random -from itertools import accumulate - -import pytest -import torch -import torch.nn.functional as F - -import cudnn # noqa: F401 (conftest extends cudnn.__path__ with the source tree) -from cudnn.linear_attention.frost import KdaFrostEngine - -from linear_attention.common import assert_bitwise_runs, assert_concurrent_stream_runs, assert_engine_declines -from linear_attention.conftest import multidist_randu -from linear_attention.reference_kda import kda_reference, rms_ratio - -pytestmark = pytest.mark.L0 - -SEED = 42 - - -def _sm100_dsl_available() -> bool: - if not torch.cuda.is_available(): - return False - major, _minor = torch.cuda.get_device_capability() - if major != 10: - return False - try: - import cutlass.experimental.primitives # noqa: F401 -- often a sys.modules alias, invisible to find_spec - except ImportError: - return False - return True - - -requires_runtime = pytest.mark.skipif(not _sm100_dsl_available(), reason="needs an SM100-class GPU and the Cutlass DSL") - - -def _seed(seed=SEED): - random.seed(seed) - torch.random.manual_seed(seed) - torch.cuda.manual_seed(seed) - - -def _cu(seq_lens, device="cuda"): - return torch.tensor([0] + list(accumulate(seq_lens)), dtype=torch.int32, device=device) - - -def _gen_thd(seq_lens, H, HV, head_size, dtype, HK=None): - HK = H if HK is None else HK - total = sum(seq_lens) - q = multidist_randu(total * H, head_size, device="cuda").reshape(total, H, head_size) - k = multidist_randu(total * HK, head_size, device="cuda").reshape(total, HK, head_size) - k = F.normalize(k, p=2.0, dim=-1) - v = multidist_randu(total * HV, head_size, device="cuda").reshape(total, HV, head_size) - return q.to(dtype).contiguous(), k.to(dtype).contiguous(), v.to(dtype).contiguous() - - -_DT = {torch.bfloat16: cudnn.data_type.BFLOAT16, torch.float16: cudnn.data_type.HALF, torch.float32: cudnn.data_type.FLOAT} - - -def _build_kda_engine_graph( - total, - H, - D, - num_seqs, - scale, - *, - io_dt=None, - HV=None, - use_qk_l2norm=True, - with_s0=False, - s0_dt=None, - fs_dt=None, - h_dt=None, - checkpoint_every_n_tokens=0, - use_beta_sigmoid=False, - safe_gate=False, - gate_lower_bound=None, - bwd=False, -): - - io_dt = io_dt or cudnn.data_type.BFLOAT16 - HV = HV or H - HO = max(H, HV) - g = cudnn.pygraph() - q_t = g.tensor([total, H, D], data_type=io_dt, name="q") - k_t = g.tensor([total, H, D], data_type=io_dt, name="k") - v_t = g.tensor([total, HV, D], data_type=io_dt, name="v") - g_t = g.tensor([total, HO, D], data_type=cudnn.data_type.FLOAT, name="g") - beta_t = g.tensor([total, HO], data_type=io_dt if use_beta_sigmoid else cudnn.data_type.FLOAT, name="beta") - cu_t = g.tensor([num_seqs + 1], data_type=cudnn.data_type.INT32, name="cu_seqlens") - t = dict(q=q_t, k=k_t, v=v_t, g=g_t, beta=beta_t, cu=cu_t) - if safe_gate: - t["a_log"] = g.tensor([HO], data_type=cudnn.data_type.FLOAT, name="a_log") - t["dt_bias"] = g.tensor([HO, D], data_type=cudnn.data_type.FLOAT, name="dt_bias") - if with_s0: - t["s0"] = g.tensor([num_seqs, HO, D, D], data_type=s0_dt or cudnn.data_type.FLOAT, name="initial_state") - if bwd: - t["dO"] = g.tensor([total, HO, D], data_type=io_dt, name="dO") - g.kda_bwd( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=beta_t, - cu_seqlens=cu_t, - dO=t["dO"], - scale=scale, - use_qk_l2norm=use_qk_l2norm, - name="kda_bwd", - ) - return g, t - O_t, fs_t, h_t = g.kda( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=beta_t, - cu_seqlens=cu_t, - initial_state=t.get("s0"), - scale=scale, - output_final_state=True, - use_qk_l2norm=use_qk_l2norm, - checkpoint_every_n_tokens=checkpoint_every_n_tokens, - use_beta_sigmoid=use_beta_sigmoid, - safe_gate=safe_gate, - gate_lower_bound=gate_lower_bound, - a_log=t.get("a_log"), - dt_bias=t.get("dt_bias"), - name="kda", - ) - O_t.set_output(True).set_data_type(io_dt) - fs_t.set_output(True).set_data_type(fs_dt or cudnn.data_type.FLOAT) - t["O"], t["fs"] = O_t, fs_t - if h_t is not None: - h_t.set_output(True).set_data_type(h_dt or io_dt) - t["H"] = h_t - return g, t - - -def _run_kda(q, k, v, gate, beta, scale, cu, initial_state=None, output_state=None, out_h=None, every_n=0, use_qk_l2norm=True): - """Torch adapter over the graph. ``gate`` is the per-key- - channel natural-log decay (fp32), ``beta`` the post-sigmoid scalar (fp32); - state ports are K-major ``[N, HO, K, V]``. GQA ``k`` (and ``HV < HQ`` - ``v``) are pre-broadcast: the node serves HK == HQ, HV a multiple of HQ.""" - device = q.device - H, HV = q.shape[1], v.shape[1] - if k.shape[1] != H: - k = k.repeat_interleave(H // k.shape[1], dim=1) - if HV < H: - v = v.repeat_interleave(H // HV, dim=1) - HV = H - total, D = q.shape[0], q.shape[2] - HO = max(H, HV) - num_seqs = cu.shape[0] - 1 - if output_state is None: - output_state = torch.empty(num_seqs, HO, D, D, dtype=torch.float32, device=device) - g, t = _build_kda_engine_graph( - total, - H, - D, - num_seqs, - scale, - io_dt=_DT[q.dtype], - HV=HV, - use_qk_l2norm=use_qk_l2norm, - with_s0=initial_state is not None, - s0_dt=None if initial_state is None else _DT[initial_state.dtype], - fs_dt=_DT[output_state.dtype], - h_dt=None if out_h is None else _DT[out_h.dtype], - checkpoint_every_n_tokens=every_n, - ) - g.build() - output = torch.empty(total, HO, D, dtype=q.dtype, device=device) - pack = { - t["q"]: q.contiguous(), - t["k"]: k.contiguous(), - t["v"]: v.contiguous(), - t["g"]: gate.float().contiguous(), - t["beta"]: beta.float().contiguous(), - t["cu"]: cu.to(torch.int32).contiguous(), - t["O"]: output, - t["fs"]: output_state, - } - if initial_state is not None: - pack[t["s0"]] = initial_state.contiguous() - if out_h is not None: - pack[t["H"]] = out_h - g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=device)) - return output, output_state - - -def _run_and_check(dtype, H, HV, seq_lens, with_s0=False, HK=None, gate_lo=None): - """Run the engine on random inputs and compare against the fp64 recurrent - reference (the kernel L2-normalizes q/k internally). fp16 io needs a - higher gate floor: k * exp2(-cumsum(g)) grazes the fp16 range at 0.5.""" - _seed() - head_size = 128 - total = sum(seq_lens) - HO = max(H, HV) - num_seqs = len(seq_lens) - q, k, v = _gen_thd(seq_lens, H, HV, head_size, dtype, HK=HK) - # BT=16 io-dtype Neumann inverse: stronger decay + post-sigmoid beta. - lo = gate_lo if gate_lo is not None else (0.6 if dtype == torch.float16 else 0.5) - gate = torch.empty(total, HO, head_size, device="cuda").uniform_(lo, 1.0).log() - beta = torch.rand(total, HO, device="cuda").sigmoid() - scale = 1.0 / math.sqrt(head_size) - s0 = (torch.randn(num_seqs, HO, head_size, head_size, dtype=torch.float32, device="cuda") * 0.05).contiguous() if with_s0 else None - output_state = torch.full((num_seqs, HO, head_size, head_size), float("nan"), dtype=torch.float32, device="cuda") - - o, fs = _run_kda(q, k, v, gate, beta, scale, _cu(seq_lens), initial_state=s0, output_state=output_state) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = kda_reference( - F.normalize(q.float(), dim=-1).unsqueeze(0), - F.normalize(k.float(), dim=-1).unsqueeze(0), - v.unsqueeze(0), - gate.unsqueeze(0), - beta.unsqueeze(0), - scale=scale, - initial_state=s0, - cu_seqlens=_cu(seq_lens), - ) - tol = 1.2e-1 if dtype == torch.float16 else 1e-1 - torch.testing.assert_close(o.float(), o_ref.squeeze(0).float(), atol=tol, rtol=tol) - assert rms_ratio(fs, fs_ref) < 5e-2 # kernel state is K-major, like the reference - - -@requires_runtime -@pytest.mark.parametrize("H,HV", [(1, 1), (2, 2), (2, 4)]) -@pytest.mark.parametrize("seq_lens", [[256], [256, 256], [64, 128, 512]]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_kda_kernel_basic(dtype, H, HV, seq_lens): - _run_and_check(dtype, H, HV, seq_lens) - - -@requires_runtime -@pytest.mark.parametrize( - "seq_lens", - [[1], [15], [16], [17], [63, 65], [240, 255, 257], [7] * 24 + [1] * 8, [2048], [33] * 200], - ids=lambda s: f"{len(s)}seqs_{sum(s)}tok", -) -def test_kda_kernel_seqlen_edges(seq_lens): - """Chunk-boundary lengths (BT=16 +/- 1), single-token, many-short-seq - packs, a long sequence (many mbarrier ring wraps), and more tiles than - SMs (persistent CTAs walking several tiles).""" - _run_and_check(torch.bfloat16, 2, 4, seq_lens) - - -@requires_runtime -@pytest.mark.parametrize("seq_lens", [[240, 255, 257], [7] * 24 + [1] * 8]) -def test_kda_kernel_seqlen_edges_fp16(seq_lens): - _run_and_check(torch.float16, 2, 4, seq_lens) - - -@requires_runtime -@pytest.mark.parametrize("seq_lens", [[256], [64, 129, 512]]) -def test_kda_kernel_initial_state(seq_lens): - _run_and_check(torch.bfloat16, 2, 2, seq_lens, with_s0=True) - - -@requires_runtime -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_kda_beta_sigmoid_in_kernel(dtype): - """use_beta_sigmoid: io-dtype beta logits + in-kernel sigmoid must match - the host-side sigmoid path (the kernel roundtrips through the io dtype; - the approx-tanh sigmoid differs by at most ~1 io-dtype ulp).""" - _seed() - seq_lens, H, D = [256, 129], 2, 128 - total, num_seqs = sum(seq_lens), len(seq_lens) - q, k, v = _gen_thd(seq_lens, H, H, D, dtype) - gate = torch.empty(total, H, D, device="cuda").uniform_(0.5, 1.0).log() - logits = torch.randn(total, H, device="cuda") - scale = 1.0 / math.sqrt(D) - cu = _cu(seq_lens) - - outs = [] - for in_kernel in (True, False): - beta = logits.to(dtype).contiguous() if in_kernel else logits.to(dtype).float().sigmoid().to(dtype).float().contiguous() - g, t = _build_kda_engine_graph(total, H, D, num_seqs, scale, io_dt=_DT[dtype], use_beta_sigmoid=in_kernel) - g.build() - o = torch.empty(total, H, D, dtype=dtype, device="cuda") - fs = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device="cuda") - pack = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate.float().contiguous(), t["beta"]: beta, t["cu"]: cu, t["O"]: o, t["fs"]: fs} - g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device="cuda")) - torch.cuda.synchronize() - outs.append((o, fs)) - torch.testing.assert_close(outs[0][0].float(), outs[1][0].float(), atol=2.5e-2, rtol=2.5e-2) - assert rms_ratio(outs[0][1], outs[1][1]) < 2e-2 - - -@requires_runtime -def test_kda_safe_gate(dtype=torch.bfloat16): - """safe_gate: the in-kernel transform lower_bound * sigmoid(exp(a_log) * - (g + dt_bias)) must match feeding the host-computed transform as a plain - natural-log gate.""" - _seed() - seq_lens, H, D = [256, 129], 2, 128 - total, num_seqs = sum(seq_lens), len(seq_lens) - q, k, v = _gen_thd(seq_lens, H, H, D, dtype) - raw_gate = torch.randn(total, H, D, device="cuda").contiguous() - a_log = (torch.randn(H, device="cuda") * 0.3).contiguous() - dt_bias = (torch.randn(H, D, device="cuda") * 0.3).contiguous() - beta = torch.rand(total, H, device="cuda").sigmoid().contiguous() - scale = 1.0 / math.sqrt(D) - cu = _cu(seq_lens) - lower_bound = -5.0 # the kernel default, pinned through the attr - - outs = [] - for safe_gate in (True, False): - if safe_gate: - gate = raw_gate - else: - gate = (lower_bound * torch.sigmoid(a_log.exp().view(1, H, 1) * (raw_gate + dt_bias.view(1, H, D)))).contiguous() - g, t = _build_kda_engine_graph(total, H, D, num_seqs, scale, io_dt=_DT[dtype], safe_gate=safe_gate, gate_lower_bound=lower_bound if safe_gate else None) - g.build() - o = torch.empty(total, H, D, dtype=dtype, device="cuda") - fs = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device="cuda") - pack = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate, t["beta"]: beta, t["cu"]: cu, t["O"]: o, t["fs"]: fs} - if safe_gate: - pack[t["a_log"]] = a_log - pack[t["dt_bias"]] = dt_bias - g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device="cuda")) - torch.cuda.synchronize() - outs.append((o, fs)) - torch.testing.assert_close(outs[0][0].float(), outs[1][0].float(), atol=2.5e-2, rtol=2.5e-2) - assert rms_ratio(outs[0][1], outs[1][1]) < 2e-2 - - -@requires_runtime -@pytest.mark.parametrize("with_initial_state", [False, True]) -def test_kda_zero_length_sequence_state_passthrough(with_initial_state): - """A zero-length sequence's final-state slot gets the passthrough value: - its initial state when seeded, zeros otherwise.""" - _seed() - seq_len, H, D, sentinel = 256, 2, 128, 123.0 - q, k, v = _gen_thd([seq_len], H, H, D, torch.bfloat16) - gate = torch.empty(seq_len, H, D, device="cuda").uniform_(0.5, 1.0).log() - beta = torch.rand(seq_len, H, device="cuda").sigmoid() - s0 = torch.randn(2, H, D, D, dtype=torch.float32, device="cuda") if with_initial_state else None - fs = torch.full((2, H, D, D), sentinel, dtype=torch.float32, device="cuda") - _run_kda(q, k, v, gate, beta, 1.0 / math.sqrt(D), _cu([seq_len, 0]), initial_state=s0, output_state=fs) - torch.cuda.synchronize() - expected = s0[1] if with_initial_state else torch.zeros_like(fs[1]) - torch.testing.assert_close(fs[1], expected, atol=0, rtol=0) - - -@requires_runtime -@pytest.mark.parametrize("HQ,HK,HV", [(1, 1, 1), (4, 1, 1), (3, 3, 3), (6, 2, 2), (1, 1, 2), (2, 2, 4), (16, 16, 32), (16, 16, 64)]) -def test_kda_kernel_head_configs(HQ, HK, HV): - """The FlashInfer head-config matrix: GQA (HK < HQ), GVA (HV > HQ), odd - head counts, and production-sized 16/32/64-head grids (the adapter - pre-broadcasts k and any HV < HQ v to the node's HK == HQ contract).""" - _run_and_check(torch.bfloat16, HQ, HV, [64, 128, 512], HK=HK) - - -@requires_runtime -@pytest.mark.parametrize("HQ,HK,HV", [(2, 2, 4), (16, 16, 64)]) -def test_kda_kernel_head_configs_fp16(HQ, HK, HV): - _run_and_check(torch.float16, HQ, HV, [64, 128, 512], HK=HK) - - -@requires_runtime -def test_kda_chunked_prefill(): - """Splitting a sequence at a chunk boundary and reseeding from the fp32 - final state must reproduce the single-shot run (state roundtrip; the two - runs get different split tables, so the match is approximate).""" - _seed() - H, D, T1, T2 = 2, 128, 256, 192 - dtype = torch.bfloat16 - q, k, v = _gen_thd([T1 + T2], H, H, D, dtype) - gate = torch.empty(T1 + T2, H, D, device="cuda").uniform_(0.5, 1.0).log() - beta = torch.rand(T1 + T2, H, device="cuda").sigmoid() - scale = 1.0 / math.sqrt(D) - - fs_full = torch.full((1, H, D, D), float("nan"), dtype=torch.float32, device="cuda") - o_full, _ = _run_kda(q, k, v, gate, beta, scale, _cu([T1 + T2]), output_state=fs_full) - - fs1 = torch.full((1, H, D, D), float("nan"), dtype=torch.float32, device="cuda") - o1, _ = _run_kda(q[:T1], k[:T1], v[:T1], gate[:T1], beta[:T1], scale, _cu([T1]), output_state=fs1) - fs2 = torch.full((1, H, D, D), float("nan"), dtype=torch.float32, device="cuda") - o2, _ = _run_kda( - q[T1:].contiguous(), - k[T1:].contiguous(), - v[T1:].contiguous(), - gate[T1:].contiguous(), - beta[T1:].contiguous(), - scale, - _cu([T2]), - initial_state=fs1, - output_state=fs2, - ) - torch.cuda.synchronize() - - torch.testing.assert_close(o1.float(), o_full[:T1].float(), atol=2e-3, rtol=2e-3) - torch.testing.assert_close(o2.float(), o_full[T1:].float(), atol=2e-3, rtol=2e-3) - torch.testing.assert_close(fs2, fs_full, atol=2e-3, rtol=2e-3) - - -@requires_runtime -def test_kda_kernel_state_dtype_bf16(): - """bf16 initial/final state buffers (the io-downcast S0 path).""" - _seed() - seq_lens, H, D = [64, 128, 512], 2, 128 - total, num_seqs = sum(seq_lens), len(seq_lens) - q, k, v = _gen_thd(seq_lens, H, H, D, torch.bfloat16) - gate = torch.empty(total, H, D, device="cuda").uniform_(0.5, 1.0).log() - beta = torch.rand(total, H, device="cuda").sigmoid() - scale = 1.0 / math.sqrt(D) - s0 = (torch.randn(num_seqs, H, D, D, dtype=torch.float32, device="cuda") * 0.05).to(torch.bfloat16).contiguous() - fs = torch.full((num_seqs, H, D, D), float("nan"), dtype=torch.bfloat16, device="cuda") - o, _ = _run_kda(q, k, v, gate, beta, scale, _cu(seq_lens), initial_state=s0, output_state=fs) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = kda_reference( - F.normalize(q.float(), dim=-1).unsqueeze(0), - F.normalize(k.float(), dim=-1).unsqueeze(0), - v.unsqueeze(0), - gate.unsqueeze(0), - beta.unsqueeze(0), - scale=scale, - initial_state=s0.float(), - cu_seqlens=_cu(seq_lens), - ) - torch.testing.assert_close(o.float(), o_ref.squeeze(0).float(), atol=1e-1, rtol=1e-1) - assert rms_ratio(fs.float(), fs_ref) < 5e-2 - - -# --------------------------------------------------------------------------- -# KdaFrostEngine: graph-level coverage through the router -# --------------------------------------------------------------------------- - - -def _kda_engine_inputs(seq_lens, H, D): - from linear_attention.conftest import gen_kda_gates, gen_qkv - - total = sum(seq_lens) - q, k, v = gen_qkv(1, total, H, H, D, D, torch.bfloat16) - # BT=16 io-dtype Neumann inverse: stronger decay + post-sigmoid beta. - gate, beta = gen_kda_gates(1, total, H, D, torch.bfloat16, lo=0.5) - beta = beta.float().sigmoid() - cu = torch.tensor([0] + list(accumulate(seq_lens)), dtype=torch.int32, device="cuda") - return (x.squeeze(0).contiguous() for x in (q, k, v, gate, beta)), cu - - -@requires_runtime -@pytest.mark.parametrize("seq_lens", [[256], [512, 512], [64, 128, 512]]) -def test_kda_frost_engine_matches_reference(seq_lens, H=2, D=128): - - _seed() - (q, k, v, gate, beta), cu = _kda_engine_inputs(seq_lens, H, D) - total, num_seqs = sum(seq_lens), len(seq_lens) - scale = 1.0 / math.sqrt(D) - - g, t = _build_kda_engine_graph(total, H, D, num_seqs, scale) - g.build() - assert isinstance(g.selected_engine, KdaFrostEngine) - assert g.get_workspace_size() > 0 # split-K work-item table + scheduler counters - - o_buf = torch.empty(total, H, D, dtype=torch.bfloat16, device="cuda") - fs_buf = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device="cuda") - pack = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate, t["beta"]: beta, t["cu"]: cu, t["O"]: o_buf, t["fs"]: fs_buf} - g.execute(pack, torch.empty(g.get_workspace_size(), dtype=torch.uint8, device="cuda")) - torch.cuda.synchronize() - - # the kernel L2-normalizes q/k internally (use_qk_l2norm) - with torch.no_grad(): - o_ref, fs_ref = kda_reference( - F.normalize(q.float(), dim=-1).unsqueeze(0), - F.normalize(k.float(), dim=-1).unsqueeze(0), - v.unsqueeze(0), - gate.unsqueeze(0), - beta.unsqueeze(0), - scale=scale, - cu_seqlens=cu, - ) - torch.testing.assert_close(o_buf.float(), o_ref.squeeze(0).float(), atol=1e-1, rtol=1e-1) - r_s = rms_ratio(fs_buf, fs_ref) # engine state ports are K-major - assert r_s < 5e-2, f"final_state rms ratio {r_s:.4g}" - - -@requires_runtime -def test_kda_frost_engine_initial_state(seq_lens=(128, 256), H=2, D=128): - - _seed() - (q, k, v, gate, beta), cu = _kda_engine_inputs(seq_lens, H, D) - total, num_seqs = sum(seq_lens), len(seq_lens) - scale = 1.0 / math.sqrt(D) - s0 = torch.randn(num_seqs, H, D, D, dtype=torch.float32, device="cuda") * 0.05 - - g, t = _build_kda_engine_graph(total, H, D, num_seqs, scale, with_s0=True) - g.build() - assert isinstance(g.selected_engine, KdaFrostEngine) - - o_buf = torch.empty(total, H, D, dtype=torch.bfloat16, device="cuda") - fs_buf = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device="cuda") - pack = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate, t["beta"]: beta, t["cu"]: cu, t["s0"]: s0, t["O"]: o_buf, t["fs"]: fs_buf} - g.execute(pack, torch.empty(g.get_workspace_size(), dtype=torch.uint8, device="cuda")) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = kda_reference( - F.normalize(q.float(), dim=-1).unsqueeze(0), - F.normalize(k.float(), dim=-1).unsqueeze(0), - v.unsqueeze(0), - gate.unsqueeze(0), - beta.unsqueeze(0), - scale=scale, - initial_state=s0, - cu_seqlens=cu, - ) - torch.testing.assert_close(o_buf.float(), o_ref.squeeze(0).float(), atol=1e-1, rtol=1e-1) - r_s = rms_ratio(fs_buf, fs_ref) # engine state ports are K-major - assert r_s < 5e-2, f"final_state rms ratio {r_s:.4g}" - - -@requires_runtime -def test_kda_frost_engine_no_l2norm_matches_reference(seq_lens=(256,), H=2, D=128): - """use_qk_l2norm=False passes q/k through as given, so the test feeds - pre-normalized rows (the kernel's io-dtype arithmetic needs them).""" - - _seed() - (q, k, v, gate, beta), cu = _kda_engine_inputs(list(seq_lens), H, D) - q = F.normalize(q.float(), dim=-1).to(q.dtype) - k = F.normalize(k.float(), dim=-1).to(k.dtype) - total, num_seqs = sum(seq_lens), len(seq_lens) - scale = 1.0 / math.sqrt(D) - - g, t = _build_kda_engine_graph(total, H, D, num_seqs, scale, use_qk_l2norm=False) - g.build() - assert isinstance(g.selected_engine, KdaFrostEngine) - - o_buf = torch.empty(total, H, D, dtype=torch.bfloat16, device="cuda") - fs_buf = torch.empty(num_seqs, H, D, D, dtype=torch.float32, device="cuda") - pack = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate, t["beta"]: beta, t["cu"]: cu, t["O"]: o_buf, t["fs"]: fs_buf} - g.execute(pack, torch.empty(g.get_workspace_size(), dtype=torch.uint8, device="cuda")) - torch.cuda.synchronize() - - with torch.no_grad(): - o_ref, fs_ref = kda_reference( - q.float().unsqueeze(0), - k.float().unsqueeze(0), - v.unsqueeze(0), - gate.unsqueeze(0), - beta.unsqueeze(0), - scale=scale, - cu_seqlens=cu, - ) - torch.testing.assert_close(o_buf.float(), o_ref.squeeze(0).float(), atol=1e-1, rtol=1e-1) - r_s = rms_ratio(fs_buf, fs_ref) # engine state ports are K-major - assert r_s < 5e-2, f"final_state rms ratio {r_s:.4g}" - - -def test_kda_frost_engine_declines_bwd(): - """KDA_BWD declines (stub backward kernel on this branch).""" - g, _t = _build_kda_engine_graph(256, 2, 128, 1, 0.125, bwd=True) - assert_engine_declines(g, "kda_frost") - - -@requires_runtime -def test_kda_frost_engine_declines_wrong_head_dim(): - g, _t = _build_kda_engine_graph(256, 2, 64, 1, 0.125) - assert_engine_declines(g, "kda_frost") - - -# --------------------------------------------------------------------------- -# Determinism stress: bitwise repeat runs + two-stream co-residency -# --------------------------------------------------------------------------- - -DET_VARLEN_MIX = [497, 16, 1, 480, 0, 253] # zero-length + single-token + odd tails - - -def _det_launch(seq_lens, H, HV, with_s0=False, stream=None): - q, k, v = _gen_thd(seq_lens, H, HV, 128, torch.bfloat16) - total, HO = sum(seq_lens), max(H, HV) - gate = torch.empty(total, HO, 128, device="cuda").uniform_(0.5, 1.0).log().float().contiguous() - beta = torch.rand(total, HO, device="cuda").sigmoid().float().contiguous() - s0 = (torch.randn(len(seq_lens), HO, 128, 128, dtype=torch.float32, device="cuda") * 0.05).contiguous() if with_s0 else None - cu = _cu(seq_lens) - scale = 1.0 / math.sqrt(128) - - def launch(): - if stream is not None: - with torch.cuda.stream(stream): - return _run_kda(q, k, v, gate, beta, scale, cu, initial_state=s0) - return _run_kda(q, k, v, gate, beta, scale, cu, initial_state=s0) - - return launch - - -@requires_runtime -@pytest.mark.parametrize("seq_lens", [DET_VARLEN_MIX, [4096]], ids=["varlen_mix", "long"]) -def test_kda_prefill_determinism(seq_lens): - _seed() - assert_bitwise_runs(_det_launch(seq_lens, 2, 4), label="kda") - - -@requires_runtime -def test_kda_prefill_determinism_initial_state(): - _seed() - assert_bitwise_runs(_det_launch([256, 640, 0, 33], 2, 2, with_s0=True), label="kda+s0") - - -@requires_runtime -def test_kda_concurrent_streams_determinism(): - _seed() - s1, s2 = torch.cuda.Stream(), torch.cuda.Stream() - assert_concurrent_stream_runs(_det_launch([1024, 31, 512], 2, 4, stream=s1), _det_launch([256, 999, 1, 128], 2, 4, stream=s2), s1, s2) - - -# --------------------------------------------------------------------------- -# split-K: strong decay on a long-sequence pack, so the table actually cuts -# --------------------------------------------------------------------------- - - -@requires_runtime -@pytest.mark.parametrize("with_s0", [False, True]) -@pytest.mark.parametrize("heads", [(2, 2), (1, 4)]) # MHA, GVA -def test_kda_prefill_split_strong_decay(heads, with_s0): - """Strong decay (gate floor 0.3) saturates the scan's warmup threshold on - the 2048-token sequence, so the split table cuts it into several work - items; checked against the fp64 recurrent reference.""" - HQ, HV = heads - _run_and_check(torch.bfloat16, HQ, HV, [100, 2048, 0, 517], with_s0=with_s0, gate_lo=0.3) - - -@requires_runtime -def test_kda_prefill_split_safe_gate(dtype=torch.bfloat16): - """safe_gate + split-K: the partition scan applies the gate transform - itself, so cuts land on true decay values; checked against the fp64 - reference fed the host-computed transform.""" - _seed() - HO = 2 - seq_lens = [100, 2048, 0, 517] - total = sum(seq_lens) - B = len(seq_lens) - q, k, v = _gen_thd(seq_lens, HO, HO, 128, dtype) - raw_gate = (torch.randn(total, HO, 128, device="cuda") + 1.0).contiguous() - a_log = (torch.randn(HO, device="cuda") * 0.3).contiguous() - dt_bias = (torch.randn(HO, 128, device="cuda") * 0.3).contiguous() - beta = torch.rand(total, HO, device="cuda").sigmoid().float() - cu = _cu(seq_lens) - scale = 1.0 / math.sqrt(128) - lower_bound = -5.0 - - g, t = _build_kda_engine_graph(total, HO, 128, B, scale, safe_gate=True, gate_lower_bound=lower_bound) - g.build() - o = torch.full((total, HO, 128), float("nan"), dtype=dtype, device="cuda") - fs = torch.full((B, HO, 128, 128), float("nan"), dtype=torch.float32, device="cuda") - pack = { - t["q"]: q, - t["k"]: k, - t["v"]: v, - t["g"]: raw_gate, - t["beta"]: beta, - t["cu"]: cu, - t["O"]: o, - t["fs"]: fs, - t["a_log"]: a_log, - t["dt_bias"]: dt_bias, - } - g.execute(pack, torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device="cuda")) - torch.cuda.synchronize() - assert not torch.isnan(o).any() and not torch.isnan(fs).any() - - gate = lower_bound * torch.sigmoid(a_log.exp().view(1, HO, 1) * (raw_gate + dt_bias.view(1, HO, 128))) - with torch.no_grad(): - o_ref, fs_ref = kda_reference( - F.normalize(q.float(), dim=-1).unsqueeze(0), - F.normalize(k.float(), dim=-1).unsqueeze(0), - v.unsqueeze(0), - gate.unsqueeze(0), - beta.unsqueeze(0), - scale=scale, - cu_seqlens=cu, - ) - torch.testing.assert_close(o.float(), o_ref.squeeze(0).float(), atol=1e-1, rtol=1e-1) - assert rms_ratio(fs, fs_ref) < 5e-2 - - -# --------------------------------------------------------------------------- -# CUDA graph capture/replay across dynamic shapes (fixed SM-count grid) -# --------------------------------------------------------------------------- - - -@requires_runtime -def test_kda_prefill_cuda_graph_replay(): - """Capture the ENGINE execute once, replay across CHANGED effective - shapes: capacity buffers, cu_seqlens with zero-length tails; everything - the engine launches (sched memset, split table, desc rebuilds, kernel) - must be capture-safe, and every replay must match an eager engine launch - on the same data bit for bit.""" - T_cap, B_cap, H, D = 768, 4, 2, 128 - dev = "cuda" - _seed() - scale = 1.0 / math.sqrt(D) - q = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - k = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - v = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - gate = torch.zeros(T_cap, H, D, dtype=torch.float32, device=dev) - beta = torch.zeros(T_cap, H, dtype=torch.float32, device=dev) - cu = torch.zeros(B_cap + 1, dtype=torch.int32, device=dev) - o_graph = torch.zeros(T_cap, H, D, dtype=torch.bfloat16, device=dev) - fs_graph = torch.zeros(B_cap, H, D, D, dtype=torch.float32, device=dev) - o_eager = torch.zeros_like(o_graph) - fs_eager = torch.zeros_like(fs_graph) - - g, t = _build_kda_engine_graph(T_cap, H, D, B_cap, scale) - g.build() - ws = torch.empty(max(g.get_workspace_size(), 1), dtype=torch.uint8, device=dev) - base = {t["q"]: q, t["k"]: k, t["v"]: v, t["g"]: gate, t["beta"]: beta, t["cu"]: cu} - pack_graph = {**base, t["O"]: o_graph, t["fs"]: fs_graph} - pack_eager = {**base, t["O"]: o_eager, t["fs"]: fs_eager} - - def fill(seq_lens): - total = sum(seq_lens) - q[:total] = torch.randn(total, H, D, device=dev).bfloat16() * 0.5 - k[:total] = F.normalize(torch.randn(total, H, D, device=dev), dim=-1).bfloat16() - v[:total] = torch.randn(total, H, D, device=dev).bfloat16() * 0.5 - gate[:total] = torch.empty(total, H, D, device=dev).uniform_(0.5, 1.0).log() - beta[:total] = torch.rand(total, H, device=dev) - bounds = [0] + list(accumulate(seq_lens)) - bounds += [bounds[-1]] * (B_cap + 1 - len(bounds)) - cu.copy_(torch.tensor(bounds, dtype=torch.int32)) - return total - - fill([256, 512, 0, 0]) - stream = torch.cuda.Stream() - handle = cudnn.create_handle() - cudnn.set_stream(handle, stream.cuda_stream) - with torch.cuda.stream(stream): - g.execute(pack_graph, ws, handle=handle) # warmup: compile + caches - torch.cuda.synchronize() - - cg = torch.cuda.CUDAGraph() - with torch.cuda.graph(cg, stream=stream): - g.execute(pack_graph, ws, handle=handle) - - eager_handle = cudnn.create_handle() - cudnn.set_stream(eager_handle, torch.cuda.current_stream().cuda_stream) - for seq_lens in ([256, 512, 0, 0], [100, 200, 56, 0], [768], [16, 0, 16, 736 - 32]): - total = fill(seq_lens) - cg.replay() - torch.cuda.synchronize() - g.execute(pack_eager, ws, handle=eager_handle) - torch.cuda.synchronize() - assert torch.equal(o_graph[:total], o_eager[:total]), f"graph o diverges from eager at {seq_lens}" - assert torch.equal(fs_graph, fs_eager), f"graph final_state diverges from eager at {seq_lens}" diff --git a/test/python/linear_attention/ops/__init__.py b/test/python/linear_attention/ops/__init__.py deleted file mode 100644 index 52a7a9daf..000000000 --- a/test/python/linear_attention/ops/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 diff --git a/test/python/linear_attention/ops/test_gdn2_op.py b/test/python/linear_attention/ops/test_gdn2_op.py deleted file mode 100644 index 225bca7fa..000000000 --- a/test/python/linear_attention/ops/test_gdn2_op.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""GDN-2 custom-op tests (``cudnn.linear_attention.ops.gated_delta_net_v2``): -THD layout and torch.compile against the fp64 recurrent reference. GDN-2 runs -on the FROST SM100 engine only (forward), so these tests require an -SM100-class GPU with the Cutlass DSL runtime.""" - -from __future__ import annotations - -import importlib.util -import math - -import pytest -import torch -import torch.nn.functional as F - -from cudnn.linear_attention.ops import gated_delta_net_v2 - -from ..conftest import gen_gdn2_gates, gen_qkv -from ..reference_gdn2 import gdn2_reference, rms_ratio - -pytestmark = pytest.mark.L0 - - -def _frost_gdn2_available() -> bool: - if not torch.cuda.is_available(): - return False - major, _minor = torch.cuda.get_device_capability() - if major != 10 or importlib.util.find_spec("cutlass") is None: - return False - try: - import cutlass.experimental.primitives # noqa: F401 — the engine's own availability gate - except Exception: # noqa: BLE001 - return False - return True - - -requires_runtime = pytest.mark.skipif(not _frost_gdn2_available(), reason="needs an SM100-class GPU and the Cutlass DSL GDN-2 prefill kernel runtime") - - -def _inputs(B, T, H, K=128, V=128, dtype=torch.bfloat16, seed=0): - torch.random.manual_seed(seed) - torch.cuda.manual_seed(seed) - q, k, v = gen_qkv(B, T, H, H, K, V, dtype) - g, beta, w = gen_gdn2_gates(B, T, H, K, V, dtype) - return q, k, v, g, beta, w - - -def _thd(x): - return x.reshape(-1, *x.shape[2:]) - - -def _cu(B, T): - return torch.arange(0, B + 1, dtype=torch.int32, device="cuda") * T - - -@requires_runtime -class TestGdn2Op: - def test_forward_parity(self, B=1, T=256, H=2): - q, k, v, g, beta, w = _inputs(B, T, H) - scale = 1.0 / math.sqrt(128) - o, _ = gated_delta_net_v2(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _thd(w), _cu(B, T), scale=scale, use_qk_l2norm_in_kernel=True) - with torch.no_grad(): - o_ref, _ = gdn2_reference(F.normalize(q.float(), dim=-1), F.normalize(k.float(), dim=-1), v, g, beta, w, scale=scale) - torch.testing.assert_close(o.view_as(o_ref).float(), o_ref.float(), atol=1e-1, rtol=1e-1) - - def test_forward_parity_no_l2norm(self, B=1, T=256, H=2): - q, k, v, g, beta, w = _inputs(B, T, H) - q = F.normalize(q.float(), dim=-1).to(q.dtype) - k = F.normalize(k.float(), dim=-1).to(k.dtype) - scale = 1.0 / math.sqrt(128) - o, _ = gated_delta_net_v2(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _thd(w), _cu(B, T), scale=scale, use_qk_l2norm_in_kernel=False) - with torch.no_grad(): - o_ref, _ = gdn2_reference(q.float(), k.float(), v, g, beta, w, scale=scale) - torch.testing.assert_close(o.view_as(o_ref).float(), o_ref.float(), atol=1e-1, rtol=1e-1) - - def test_forward_parity_thd(self): - seq_lens = [64, 192] - total = sum(seq_lens) - q, k, v, g, beta, w = (x.squeeze(0) for x in _inputs(1, total, 2)) - cu = torch.tensor([0, 64, 256], dtype=torch.int32, device="cuda") - o, _ = gated_delta_net_v2(q, k, v, g, beta, w, cu, use_qk_l2norm_in_kernel=True) - with torch.no_grad(): - o_ref, _ = gdn2_reference( - F.normalize(q.float(), dim=-1).unsqueeze(0), - F.normalize(k.float(), dim=-1).unsqueeze(0), - v.unsqueeze(0), - g.unsqueeze(0), - beta.unsqueeze(0), - w.unsqueeze(0), - cu_seqlens=cu, - ) - torch.testing.assert_close(o.float(), o_ref.squeeze(0).float(), atol=1e-1, rtol=1e-1) - - def test_initial_state(self, B=1, T=256, H=2): - q, k, v, g, beta, w = _inputs(B, T, H) - S0 = torch.randn(B, H, 128, 128, dtype=torch.float32, device="cuda") * 0.05 - scale = 1.0 / math.sqrt(128) - o, fs = gated_delta_net_v2( - _thd(q), - _thd(k), - _thd(v), - _thd(g), - _thd(beta), - _thd(w), - _cu(B, T), - scale=scale, - initial_state=S0, - output_final_state=True, - use_qk_l2norm_in_kernel=True, - ) - with torch.no_grad(): - o_ref, fs_ref = gdn2_reference(F.normalize(q.float(), dim=-1), F.normalize(k.float(), dim=-1), v, g, beta, w, scale=scale, initial_state=S0) - torch.testing.assert_close(o.view_as(o_ref).float(), o_ref.float(), atol=1e-1, rtol=1e-1) - assert rms_ratio(fs, fs_ref) < 5e-2 - - def test_default_scale(self): - q, k, v, g, beta, w = (_thd(x) for x in _inputs(1, 128, 1)) - cu = _cu(1, 128) - o_default, _ = gated_delta_net_v2(q, k, v, g, beta, w, cu, use_qk_l2norm_in_kernel=True) - o_explicit, _ = gated_delta_net_v2(q, k, v, g, beta, w, cu, scale=1.0 / math.sqrt(128), use_qk_l2norm_in_kernel=True) - torch.testing.assert_close(o_default, o_explicit) - - def test_no_final_state_returns_empty(self): - q, k, v, g, beta, w = (_thd(x) for x in _inputs(1, 128, 1)) - _o, final = gated_delta_net_v2(q, k, v, g, beta, w, _cu(1, 128), output_final_state=False, use_qk_l2norm_in_kernel=True) - assert final.numel() == 0 - - def test_torch_compile_forward(self): - q, k, v, g, beta, w = (_thd(x) for x in _inputs(1, 128, 1)) - cu = _cu(1, 128) - compiled = torch.compile(gated_delta_net_v2, fullgraph=True) - o_eager, _ = gated_delta_net_v2(q, k, v, g, beta, w, cu, use_qk_l2norm_in_kernel=True) - o_comp, _ = compiled(q, k, v, g, beta, w, cu, use_qk_l2norm_in_kernel=True) - torch.testing.assert_close(o_eager, o_comp) diff --git a/test/python/linear_attention/ops/test_gdn_op.py b/test/python/linear_attention/ops/test_gdn_op.py deleted file mode 100644 index 7063385b4..000000000 --- a/test/python/linear_attention/ops/test_gdn_op.py +++ /dev/null @@ -1,205 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""GDN custom-op tests (``cudnn.linear_attention.gated_delta_net``): THD -layout, autograd, and torch.compile against the fp64 recurrent reference.""" - -from __future__ import annotations - -import importlib.util -import math - -import pytest -import torch - -from cudnn.linear_attention.ops import gated_delta_net - -from ..common import FWD_TOL, GDN_MARKS, STATE_TOL -from ..conftest import gen_gates, gen_qkv -from ..reference_gdn import gdn_reference, rms_ratio - -pytestmark = GDN_MARKS - - -def _frost_gdn_available() -> bool: - if not torch.cuda.is_available(): - return False - major, _minor = torch.cuda.get_device_capability() - if major != 10 or importlib.util.find_spec("cutlass") is None: - return False - try: - import cutlass.experimental.primitives # noqa: F401 — the engine's own availability gate - except Exception: # noqa: BLE001 - return False - return True - - -requires_frost = pytest.mark.skipif(not _frost_gdn_available(), reason="GDN with HV < HQ (GQA-style v broadcast) is served by the FROST engine only") - - -def _inputs(B, T, H, HV, K, V, dtype=torch.bfloat16, seed=0): - torch.random.manual_seed(seed) - torch.cuda.manual_seed(seed) - q, k, v = gen_qkv(B, T, H, HV, K, V, dtype) - g, beta = gen_gates(B, T, HV, torch.float32) # the op requires kernel-native fp32 gates - return q, k, v, g, beta - - -def _thd(x): - return x.reshape(-1, *x.shape[2:]) - - -def _cu(B, T): - return torch.arange(0, B + 1, dtype=torch.int32, device="cuda") * T - - -class TestGdnOp: - @pytest.mark.parametrize( - "B,T,H,K,V", - [ - (1, 128, 1, 64, 64), - (2, 256, 4, 64, 64), - (1, 128, 2, 64, 128), # K != V - ], - ) - def test_forward_parity(self, B, T, H, K, V): - q, k, v, g, beta = _inputs(B, T, H, H, K, V) - scale = 1.0 / math.sqrt(K) - o, _ = gated_delta_net(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), scale=scale) - with torch.no_grad(): - o_ref, _ = gdn_reference(q, k, v, g, beta, scale=scale) - assert rms_ratio(o.view_as(o_ref), o_ref) < FWD_TOL[torch.bfloat16] - - def test_forward_parity_gva(self, B=1, T=192, H=2, HV=4, K=64, V=64): - q, k, v, g, beta = _inputs(B, T, H, HV, K, V) - o, fs = gated_delta_net(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), output_final_state=True) - with torch.no_grad(): - o_ref, fs_ref = gdn_reference(q, k, v, g, beta) - assert rms_ratio(o.view_as(o_ref), o_ref) < FWD_TOL[torch.bfloat16] - assert rms_ratio(fs, fs_ref) < STATE_TOL[torch.bfloat16] - - @pytest.mark.parametrize("B,T,H,K,V", [(1, 128, 1, 64, 64), (2, 128, 2, 64, 64)]) - def test_backward_runs(self, B, T, H, K, V): - q, k, v, g, beta = (_thd(x).detach().requires_grad_(True) for x in _inputs(B, T, H, H, K, V)) - o, _ = gated_delta_net(q, k, v, g, beta, _cu(B, T)) - o.sum().backward() - for name, t in [("q", q), ("k", k), ("v", v), ("g", g), ("beta", beta)]: - assert t.grad is not None, f"no grad for {name}" - assert torch.isfinite(t.grad).all(), f"non-finite grad for {name}" - - def test_default_scale(self): - q, k, v, g, beta = (_thd(x) for x in _inputs(1, 128, 1, 1, 64, 64)) - cu = _cu(1, 128) - o_default, _ = gated_delta_net(q, k, v, g, beta, cu) - o_explicit, _ = gated_delta_net(q, k, v, g, beta, cu, scale=1.0 / math.sqrt(64)) - torch.testing.assert_close(o_default, o_explicit) - - def test_no_final_state_returns_empty(self): - q, k, v, g, beta = (_thd(x) for x in _inputs(1, 128, 1, 1, 64, 64)) - _o, final = gated_delta_net(q, k, v, g, beta, _cu(1, 128), output_final_state=False) - assert final.numel() == 0 - - def test_initial_state(self): - B, T, H, K, V = 1, 128, 2, 64, 64 - q, k, v, g, beta = _inputs(B, T, H, H, K, V) - S0 = torch.randn(B, H, K, V, dtype=torch.float32, device="cuda") * 0.05 - o, fs = gated_delta_net(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), initial_state=S0, output_final_state=True) - with torch.no_grad(): - o_ref, fs_ref = gdn_reference(q, k, v, g, beta, initial_state=S0) - assert rms_ratio(o.view_as(o_ref), o_ref) < FWD_TOL[torch.bfloat16] - assert rms_ratio(fs, fs_ref) < STATE_TOL[torch.bfloat16] - - def test_packed_matches_per_sequence(self): - B, T, H, K, V = 2, 128, 2, 64, 64 - q, k, v, g, beta = _inputs(B, T, H, H, K, V) - o, fs = gated_delta_net(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), output_final_state=True) - for b in range(B): - o_b, fs_b = gated_delta_net(q[b], k[b], v[b], g[b], beta[b], _cu(1, T), output_final_state=True) - torch.testing.assert_close(o[b * T : (b + 1) * T], o_b) - torch.testing.assert_close(fs[b], fs_b[0]) - - def test_thd_ragged_parity_and_backward(self): - seq_lens = [64, 192] - H, K, V = 2, 64, 64 - total = sum(seq_lens) - q, k, v, g, beta = (x.squeeze(0).detach().requires_grad_(True) for x in _inputs(1, total, H, H, K, V)) - cu = torch.tensor([0, 64, 256], dtype=torch.int32, device="cuda") - - o, _ = gated_delta_net(q, k, v, g, beta, cu_seqlens=cu) - with torch.no_grad(): - o_ref, _ = gdn_reference(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), g.unsqueeze(0), beta.unsqueeze(0), cu_seqlens=cu) - assert rms_ratio(o, o_ref.squeeze(0)) < FWD_TOL[torch.bfloat16] - - o.sum().backward() - for name, t in [("q", q), ("k", k), ("v", v), ("g", g), ("beta", beta)]: - assert t.grad is not None and torch.isfinite(t.grad).all(), f"bad grad for {name}" - - @requires_frost - def test_forward_parity_gqa(self, B=1, T=192, H=4, HV=1, K=128, V=128): - """GQA (q heads group over v heads): gates/o/state at HO = H; the - reference expands v over the head group (each output head runs its - own q/k against the shared v).""" - torch.random.manual_seed(0) - torch.cuda.manual_seed(0) - q, k, v = gen_qkv(B, T, H, HV, K, V, torch.bfloat16) - g, beta = gen_gates(B, T, H, torch.float32) # the op requires kernel-native fp32 gates - o, fs = gated_delta_net(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), output_final_state=True) - v_exp = v.repeat_interleave(H // HV, dim=2) - with torch.no_grad(): - o_ref, fs_ref = gdn_reference(q, k, v_exp, g, beta) - assert rms_ratio(o.view_as(o_ref), o_ref) < FWD_TOL[torch.bfloat16] - assert rms_ratio(fs, fs_ref) < STATE_TOL[torch.bfloat16] - - @requires_frost - def test_backward_parity_gqa(self, B=1, T=128, H=4, HV=2, K=128, V=128): - """GQA gradients vs the op's own (validated) MHA path on the expanded - v: dQ/dK/dG/dBeta match directly, dV matches the head-group sum.""" - torch.random.manual_seed(0) - torch.cuda.manual_seed(0) - q, k, v = gen_qkv(B, T, H, HV, K, V, torch.bfloat16) - g, beta = gen_gates(B, T, H, torch.float32) # the op requires kernel-native fp32 gates - r = H // HV - cu = _cu(B, T) - - gqa = {n: _thd(x).detach().requires_grad_(True) for n, x in (("q", q), ("k", k), ("v", v), ("g", g), ("beta", beta))} - o, _ = gated_delta_net(gqa["q"], gqa["k"], gqa["v"], gqa["g"], gqa["beta"], cu) - w = torch.randn_like(o.float()) - (o.float() * w).sum().backward() - - mha = {n: _thd(x).detach().requires_grad_(True) for n, x in (("q", q), ("k", k), ("v", v.repeat_interleave(r, dim=2)), ("g", g), ("beta", beta))} - o_ref, _ = gated_delta_net(mha["q"], mha["k"], mha["v"], mha["g"], mha["beta"], cu) - (o_ref.float() * w).sum().backward() - - torch.testing.assert_close(o, o_ref, atol=1e-3, rtol=1e-3) - for name in ("q", "k", "g", "beta"): - torch.testing.assert_close(gqa[name].grad, mha[name].grad, atol=1e-3, rtol=1e-3, msg=f"d{name} mismatch") - dv_ref = mha["v"].grad.view(B * T, HV, r, V).sum(2) - torch.testing.assert_close(gqa["v"].grad.float(), dv_ref.float(), atol=1e-2, rtol=1e-2, msg="dV mismatch") - - def test_forward_parity_qk_l2norm(self, B=1, T=192, H=2, K=64, V=64): - import torch.nn.functional as F - - q, k, v, g, beta = _inputs(B, T, H, H, K, V) - scale = 1.0 / math.sqrt(K) - o, fs = gated_delta_net(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), scale=scale, output_final_state=True, use_qk_l2norm_in_kernel=True) - with torch.no_grad(): - o_ref, fs_ref = gdn_reference(F.normalize(q.float(), dim=-1), F.normalize(k.float(), dim=-1), v, g, beta, scale=scale) - assert rms_ratio(o.view_as(o_ref), o_ref) < FWD_TOL[torch.bfloat16] - assert rms_ratio(fs, fs_ref) < STATE_TOL[torch.bfloat16] - - @pytest.mark.parametrize("H,HV", [(2, 2), (2, 4)]) - def test_backward_runs_qk_l2norm(self, H, HV, B=1, T=128, K=64, V=64): - q, k, v, g, beta = (_thd(x).detach().requires_grad_(True) for x in _inputs(B, T, H, HV, K, V)) - o, _ = gated_delta_net(q, k, v, g, beta, _cu(B, T), use_qk_l2norm_in_kernel=True) - o.sum().backward() - for name, t in [("q", q), ("k", k), ("v", v), ("g", g), ("beta", beta)]: - assert t.grad is not None, f"no grad for {name}" - assert torch.isfinite(t.grad).all(), f"non-finite grad for {name}" - - def test_torch_compile_forward(self): - q, k, v, g, beta = (_thd(x) for x in _inputs(1, 128, 1, 1, 64, 64)) - cu = _cu(1, 128) - compiled = torch.compile(gated_delta_net, fullgraph=True) - o_eager, _ = gated_delta_net(q, k, v, g, beta, cu) - o_comp, _ = compiled(q, k, v, g, beta, cu) - torch.testing.assert_close(o_eager, o_comp) diff --git a/test/python/linear_attention/ops/test_kda_op.py b/test/python/linear_attention/ops/test_kda_op.py deleted file mode 100644 index 20369fc4a..000000000 --- a/test/python/linear_attention/ops/test_kda_op.py +++ /dev/null @@ -1,234 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""KDA custom-op tests (``cudnn.linear_attention.ops.kimi_delta_attention``): -THD layout, autograd, and torch.compile against the fp64 recurrent -reference.""" - -from __future__ import annotations - -import importlib.util -import math - -import pytest -import torch - -from cudnn.linear_attention.ops import kimi_delta_attention - -from ..common import FWD_TOL, KDA_MARKS, STATE_TOL -from ..conftest import gen_kda_gates, gen_qkv -from ..reference_kda import kda_reference, rms_ratio - -pytestmark = KDA_MARKS - - -def _frost_kda_available() -> bool: - if not torch.cuda.is_available(): - return False - major, _minor = torch.cuda.get_device_capability() - if major != 10 or importlib.util.find_spec("cutlass") is None: - return False - try: - import cutlass.experimental.primitives # noqa: F401 — the engine's own availability gate - except Exception: # noqa: BLE001 - return False - return True - - -def _inputs(B, T, H, HV, K, V, dtype=torch.bfloat16, seed=0): - torch.random.manual_seed(seed) - torch.cuda.manual_seed(seed) - q, k, v = gen_qkv(B, T, H, HV, K, V, dtype) - g, beta = gen_kda_gates(B, T, HV, K, torch.float32) # the op requires kernel-native fp32 gates - return q, k, v, g, beta - - -def _thd(x): - return x.reshape(-1, *x.shape[2:]) - - -def _cu(B, T): - return torch.arange(0, B + 1, dtype=torch.int32, device="cuda") * T - - -class TestKdaOp: - @pytest.mark.parametrize( - "B,T,H,K,V", - [ - (1, 128, 1, 64, 64), - (2, 256, 4, 64, 64), - (1, 128, 2, 64, 128), # K != V - ], - ) - def test_forward_parity(self, B, T, H, K, V): - q, k, v, g, beta = _inputs(B, T, H, H, K, V) - scale = 1.0 / math.sqrt(K) - o, _ = kimi_delta_attention(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), scale=scale) - with torch.no_grad(): - o_ref, _ = kda_reference(q, k, v, g, beta, scale=scale) - assert rms_ratio(o.view_as(o_ref), o_ref) < FWD_TOL[torch.bfloat16] - - def test_forward_parity_gva(self, B=1, T=192, H=2, HV=4, K=64, V=64): - q, k, v, g, beta = _inputs(B, T, H, HV, K, V) - o, fs = kimi_delta_attention(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), output_final_state=True) - with torch.no_grad(): - o_ref, fs_ref = kda_reference(q, k, v, g, beta) - assert rms_ratio(o.view_as(o_ref), o_ref) < FWD_TOL[torch.bfloat16] - assert rms_ratio(fs, fs_ref) < STATE_TOL[torch.bfloat16] - - @pytest.mark.parametrize("B,T,H,K,V", [(1, 128, 1, 64, 64), (2, 128, 2, 64, 64)]) - def test_backward_runs(self, B, T, H, K, V): - q, k, v, g, beta = (_thd(x).detach().requires_grad_(True) for x in _inputs(B, T, H, H, K, V)) - o, _ = kimi_delta_attention(q, k, v, g, beta, _cu(B, T)) - o.sum().backward() - for name, t in [("q", q), ("k", k), ("v", v), ("g", g), ("beta", beta)]: - assert t.grad is not None, f"no grad for {name}" - assert torch.isfinite(t.grad).all(), f"non-finite grad for {name}" - - def test_default_scale(self): - q, k, v, g, beta = (_thd(x) for x in _inputs(1, 128, 1, 1, 64, 64)) - cu = _cu(1, 128) - o_default, _ = kimi_delta_attention(q, k, v, g, beta, cu) - o_explicit, _ = kimi_delta_attention(q, k, v, g, beta, cu, scale=1.0 / math.sqrt(64)) - torch.testing.assert_close(o_default, o_explicit) - - def test_no_final_state_returns_empty(self): - q, k, v, g, beta = (_thd(x) for x in _inputs(1, 128, 1, 1, 64, 64)) - _o, final = kimi_delta_attention(q, k, v, g, beta, _cu(1, 128), output_final_state=False) - assert final.numel() == 0 - - def test_initial_state(self): - B, T, H, K, V = 1, 128, 2, 64, 64 - q, k, v, g, beta = _inputs(B, T, H, H, K, V) - S0 = torch.randn(B, H, K, V, dtype=torch.float32, device="cuda") * 0.05 - o, fs = kimi_delta_attention(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), initial_state=S0, output_final_state=True) - with torch.no_grad(): - o_ref, fs_ref = kda_reference(q, k, v, g, beta, initial_state=S0) - assert rms_ratio(o.view_as(o_ref), o_ref) < FWD_TOL[torch.bfloat16] - assert rms_ratio(fs, fs_ref) < STATE_TOL[torch.bfloat16] - - def test_packed_matches_per_sequence(self): - B, T, H, K, V = 2, 128, 2, 64, 64 - q, k, v, g, beta = _inputs(B, T, H, H, K, V) - o, fs = kimi_delta_attention(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), output_final_state=True) - for b in range(B): - o_b, fs_b = kimi_delta_attention(q[b], k[b], v[b], g[b], beta[b], _cu(1, T), output_final_state=True) - torch.testing.assert_close(o[b * T : (b + 1) * T], o_b) - torch.testing.assert_close(fs[b], fs_b[0]) - - def test_thd_ragged_parity_and_backward(self): - seq_lens = [64, 192] - H, K, V = 2, 64, 64 - total = sum(seq_lens) - q, k, v, g, beta = (x.squeeze(0).detach().requires_grad_(True) for x in _inputs(1, total, H, H, K, V)) - cu = torch.tensor([0, 64, 256], dtype=torch.int32, device="cuda") - - o, _ = kimi_delta_attention(q, k, v, g, beta, cu_seqlens=cu) - with torch.no_grad(): - o_ref, _ = kda_reference(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), g.unsqueeze(0), beta.unsqueeze(0), cu_seqlens=cu) - assert rms_ratio(o, o_ref.squeeze(0)) < FWD_TOL[torch.bfloat16] - - o.sum().backward() - for name, t in [("q", q), ("k", k), ("v", v), ("g", g), ("beta", beta)]: - assert t.grad is not None and torch.isfinite(t.grad).all(), f"bad grad for {name}" - - def test_torch_compile_forward(self): - q, k, v, g, beta = (_thd(x) for x in _inputs(1, 128, 1, 1, 64, 64)) - cu = _cu(1, 128) - compiled = torch.compile(kimi_delta_attention, fullgraph=True) - o_eager, _ = kimi_delta_attention(q, k, v, g, beta, cu) - o_comp, _ = compiled(q, k, v, g, beta, cu) - torch.testing.assert_close(o_eager, o_comp) - - def test_forward_parity_qk_l2norm(self, B=1, T=256, H=2, K=128, V=128): - """D=128 + use_qk_l2norm: routes to KdaFrostEngine on SM100/SM103 (the - cuTile engine serves it elsewhere — both honor the in-kernel norm).""" - import torch.nn.functional as F - - q, k, v, g, beta = _inputs(B, T, H, H, K, V) - # the FROST BT=16 kernel's Neumann inverse wants stronger decay and a - # post-sigmoid beta (see its kernel suite) - g = torch.empty(B, T, H, K, device="cuda").uniform_(0.5, 1.0).log() - beta = beta.float().sigmoid().to(beta.dtype) - scale = 1.0 / math.sqrt(K) - o, fs = kimi_delta_attention( - _thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), scale=scale, output_final_state=True, use_qk_l2norm_in_kernel=True - ) - with torch.no_grad(): - o_ref, fs_ref = kda_reference(F.normalize(q.float(), dim=-1), F.normalize(k.float(), dim=-1), v, g, beta, scale=scale) - torch.testing.assert_close(o.view_as(o_ref).float(), o_ref.float(), atol=1e-1, rtol=1e-1) - assert rms_ratio(fs, fs_ref) < 5e-2 - - def test_forward_parity_no_l2norm_frost(self, B=1, T=256, H=2, K=128, V=128): - """D=128 without the in-kernel norm routes to KdaFrostEngine on - SM100/SM103, so q/k are pre-normalized here.""" - import torch.nn.functional as F - - q, k, v, g, beta = _inputs(B, T, H, H, K, V) - g = torch.empty(B, T, H, K, device="cuda").uniform_(0.5, 1.0).log() - beta = beta.float().sigmoid().to(beta.dtype) - q = F.normalize(q.float(), dim=-1).to(q.dtype) - k = F.normalize(k.float(), dim=-1).to(k.dtype) - scale = 1.0 / math.sqrt(K) - o, fs = kimi_delta_attention(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), scale=scale, output_final_state=True) - with torch.no_grad(): - o_ref, fs_ref = kda_reference(q.float(), k.float(), v, g, beta, scale=scale) - torch.testing.assert_close(o.view_as(o_ref).float(), o_ref.float(), atol=1e-1, rtol=1e-1) - assert rms_ratio(fs, fs_ref) < 5e-2 - - def test_backward_runs_qk_l2norm(self, B=1, T=128, H=2, K=128, V=128): - """D=128 + use_qk_l2norm: forward may run on KdaFrostEngine, backward - always falls back to the cuTile engine (FROST KDA_BWD is a stub).""" - q, k, v, g, beta = (_thd(x).detach().requires_grad_(True) for x in _inputs(B, T, H, H, K, V)) - o, _ = kimi_delta_attention(q, k, v, g, beta, _cu(B, T), use_qk_l2norm_in_kernel=True) - o.sum().backward() - for name, t in [("q", q), ("k", k), ("v", v), ("g", g), ("beta", beta)]: - assert t.grad is not None, f"no grad for {name}" - assert torch.isfinite(t.grad).all(), f"non-finite grad for {name}" - - def test_thd_ragged_parity_frost(self): - """D=128 ragged varlen: the FROST path on SM100/SM103 (cuTile elsewhere).""" - import torch.nn.functional as F - - seq_lens = [64, 192] - total, H, K, V = sum(seq_lens), 2, 128, 128 - q, k, v, g, beta = _inputs(1, total, H, H, K, V) - g = torch.empty(1, total, H, K, device="cuda").uniform_(0.5, 1.0).log() - beta = beta.float().sigmoid().to(beta.dtype) - cu = torch.tensor([0, 64, 256], dtype=torch.int32, device="cuda") - o, fs = kimi_delta_attention(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), cu, output_final_state=True, use_qk_l2norm_in_kernel=True) - with torch.no_grad(): - o_ref, fs_ref = kda_reference(F.normalize(q.float(), dim=-1), F.normalize(k.float(), dim=-1), v, g, beta, cu_seqlens=cu) - torch.testing.assert_close(o.view_as(o_ref.squeeze(0)).float(), o_ref.squeeze(0).float(), atol=1e-1, rtol=1e-1) - assert rms_ratio(fs, fs_ref) < 5e-2 - - def test_initial_state_frost(self, B=1, T=256, H=2, K=128, V=128): - """D=128 + initial state on the FROST path.""" - import torch.nn.functional as F - - q, k, v, g, beta = _inputs(B, T, H, H, K, V) - g = torch.empty(B, T, H, K, device="cuda").uniform_(0.5, 1.0).log() - beta = beta.float().sigmoid().to(beta.dtype) - S0 = torch.randn(B, H, K, V, dtype=torch.float32, device="cuda") * 0.05 - o, fs = kimi_delta_attention( - _thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(B, T), initial_state=S0, output_final_state=True, use_qk_l2norm_in_kernel=True - ) - with torch.no_grad(): - o_ref, fs_ref = kda_reference(F.normalize(q.float(), dim=-1), F.normalize(k.float(), dim=-1), v, g, beta, initial_state=S0) - torch.testing.assert_close(o.view_as(o_ref).float(), o_ref.float(), atol=1e-1, rtol=1e-1) - assert rms_ratio(fs, fs_ref) < 5e-2 - - def test_frost_engine_selected(self): - """On SM100/SM103 with the DSL runtime, D=128 + l2norm graphs must - actually lower to KdaFrostEngine (guards the router ranking).""" - if not _frost_kda_available(): - pytest.skip("needs an SM100-class GPU and the Cutlass DSL KDA prefill kernel runtime") - from cudnn.linear_attention.frost.kda_engine import KdaFrostEngine - from cudnn.linear_attention.ops import kda as kda_ops - - kda_ops._fwd_graph_cache.clear() - q, k, v, g, beta = _inputs(1, 128, 2, 2, 128, 128) - g = torch.empty(1, 128, 2, 128, device="cuda").uniform_(0.5, 1.0).log() - beta = beta.float().sigmoid().to(beta.dtype) - kimi_delta_attention(_thd(q), _thd(k), _thd(v), _thd(g), _thd(beta), _cu(1, 128), use_qk_l2norm_in_kernel=True) - assert any(isinstance(graph.selected_engine, KdaFrostEngine) for graph, _t in kda_ops._fwd_graph_cache.values()) diff --git a/test/python/linear_attention/reference_gdn.py b/test/python/linear_attention/reference_gdn.py index a828f58d6..d88dd93e5 100644 --- a/test/python/linear_attention/reference_gdn.py +++ b/test/python/linear_attention/reference_gdn.py @@ -9,8 +9,8 @@ S_t = alpha_t (I - beta_t k_t^T k_t) S_{t-1} + beta_t k_t^T v_t o_t = q_t S_t -with scalar per-token decay ``alpha_t = exp(g_t)`` and write strength -``beta_t``. Supports grouped heads (every input's head count must divide +where ``S_t`` is the recurrent state, with scalar per-token decay +``alpha_t = exp(g_t)`` and write strength ``beta_t``. Supports grouped heads (every input's head count must divide ``HO = max(Hq, Hv)``; heads are replicated onto the HO output heads), an optional initial state, and varlen packed batches via ``cu_seqlens``. @@ -33,21 +33,21 @@ def rms_ratio(out: torch.Tensor, ref: torch.Tensor) -> float: return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() -def _recurrent_dense(q, k, v, alpha, beta, S0): - """Dense recurrence in [B, HV, T, *] layout, fp64. Returns (o, S_T).""" +def _recurrent_dense(q, k, v, alpha, beta, state0): + """Dense recurrence in [B, HV, T, *] layout, fp64. Returns (o, final state).""" T = q.shape[2] - S = S0 + state = state0 outs = [] for t in range(T): kt = k[:, :, t, :] vt = v[:, :, t, :] at = alpha[:, :, t] bt = beta[:, :, t] - kt_S = (kt.unsqueeze(-2) @ S).squeeze(-2) - residual = vt - at[..., None] * kt_S - S = at[..., None, None] * S + bt[..., None, None] * (kt.unsqueeze(-1) @ residual.unsqueeze(-2)) - outs.append((q[:, :, t, :].unsqueeze(-2) @ S).squeeze(-2)) - return torch.stack(outs, dim=2), S + kt_state = (kt.unsqueeze(-2) @ state).squeeze(-2) + residual = vt - at[..., None] * kt_state + state = at[..., None, None] * state + bt[..., None, None] * (kt.unsqueeze(-1) @ residual.unsqueeze(-2)) + outs.append((q[:, :, t, :].unsqueeze(-2) @ state).squeeze(-2)) + return torch.stack(outs, dim=2), state def gdn_reference( @@ -103,11 +103,11 @@ def expand(x): if cu_seqlens is None: B = q.shape[0] if initial_state is None: - S0 = torch.zeros(B, HV, K, V, dtype=torch.float64, device=q.device) + state0 = torch.zeros(B, HV, K, V, dtype=torch.float64, device=q.device) else: - S0 = initial_state.double() - o, S = _recurrent_dense(qf, kf, vf, alphaf, betaf, S0) - return o.permute(0, 2, 1, 3), S + state0 = initial_state.double() + o, state = _recurrent_dense(qf, kf, vf, alphaf, betaf, state0) + return o.permute(0, 2, 1, 3), state assert q.shape[0] == 1, "cu_seqlens requires packed batch B == 1" bounds = cu_seqlens.tolist() @@ -115,15 +115,15 @@ def expand(x): for n in range(len(bounds) - 1): s, e = bounds[n], bounds[n + 1] if initial_state is None: - S0 = torch.zeros(1, HV, K, V, dtype=torch.float64, device=q.device) + state0 = torch.zeros(1, HV, K, V, dtype=torch.float64, device=q.device) else: - S0 = initial_state[n : n + 1].double() + state0 = initial_state[n : n + 1].double() if e == s: - states.append(S0) + states.append(state0) continue - o_n, S_n = _recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], S0) + o_n, state_n = _recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], state0) outs.append(o_n) - states.append(S_n) + states.append(state_n) if outs: o = torch.cat(outs, dim=2).permute(0, 2, 1, 3) else: diff --git a/test/python/linear_attention/reference_gdn2.py b/test/python/linear_attention/reference_gdn2.py index b8d9ecf26..9b35ac573 100644 --- a/test/python/linear_attention/reference_gdn2.py +++ b/test/python/linear_attention/reference_gdn2.py @@ -6,8 +6,8 @@ GDN-2 generalizes GDN's scalar gates to three channel-wise gates: a per-key decay ``alpha_t = exp(g_t) in (0, 1]^K``, a per-key erase gate ``beta_t in -R^K``, and a NEW per-value write gate ``w_t in R^V`` (``k``/``q`` already -feature-mapped, ``q`` pre-scaled): +R^K``, and a NEW per-value write gate ``w_t in R^V`` (``S_t`` is the +recurrent state; ``k``/``q`` already feature-mapped, ``q`` pre-scaled): S_t = (I - k_t (beta_t . k_t)^T) Diag(alpha_t) S_{t-1} + k_t (w_t . v_t)^T o_t = q_t S_t @@ -46,12 +46,12 @@ def rms_ratio(out: torch.Tensor, ref: torch.Tensor) -> float: return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() -def _recurrent_dense(q, k, v, alpha, beta, w, S0): - """Dense recurrence in [B, HV, T, *] layout, fp64. Returns (o, S_T). +def _recurrent_dense(q, k, v, alpha, beta, w, state0): + """Dense recurrence in [B, HV, T, *] layout, fp64. Returns (o, final state). alpha, beta: per-key-channel [B, HV, T, K]; w: per-value-channel [B, HV, T, V].""" T = q.shape[2] - S = S0 + state = state0 outs = [] for t in range(T): kt = k[:, :, t, :] # [B, HV, K] @@ -59,12 +59,12 @@ def _recurrent_dense(q, k, v, alpha, beta, w, S0): at = alpha[:, :, t, :] # [B, HV, K] (= exp(g_t)) bt = beta[:, :, t, :] # [B, HV, K] wt = w[:, :, t, :] # [B, HV, V] - S = at[..., None] * S # per-K-channel decay first - erase = ((bt * kt).unsqueeze(-2) @ S).squeeze(-2) # (beta . k)^T S_dec: [B, HV, V] + state = at[..., None] * state # per-K-channel decay first + erase = ((bt * kt).unsqueeze(-2) @ state).squeeze(-2) # (beta . k)^T on the decayed state: [B, HV, V] v_new = wt * vt - erase - S = S + kt.unsqueeze(-1) @ v_new.unsqueeze(-2) # k (x) v_new - outs.append((q[:, :, t, :].unsqueeze(-2) @ S).squeeze(-2)) - return torch.stack(outs, dim=2), S + state = state + kt.unsqueeze(-1) @ v_new.unsqueeze(-2) # k (x) v_new + outs.append((q[:, :, t, :].unsqueeze(-2) @ state).squeeze(-2)) + return torch.stack(outs, dim=2), state def gdn2_reference( @@ -126,11 +126,11 @@ def expand(x): if cu_seqlens is None: B = q.shape[0] if initial_state is None: - S0 = torch.zeros(B, HV, K, V, dtype=torch.float64, device=q.device) + state0 = torch.zeros(B, HV, K, V, dtype=torch.float64, device=q.device) else: - S0 = initial_state.double() - o, S = _recurrent_dense(qf, kf, vf, alphaf, betaf, wf, S0) - return o.permute(0, 2, 1, 3), S + state0 = initial_state.double() + o, state = _recurrent_dense(qf, kf, vf, alphaf, betaf, wf, state0) + return o.permute(0, 2, 1, 3), state assert q.shape[0] == 1, "cu_seqlens requires packed batch B == 1" bounds = cu_seqlens.tolist() @@ -138,15 +138,15 @@ def expand(x): for n in range(len(bounds) - 1): s, e = bounds[n], bounds[n + 1] if initial_state is None: - S0 = torch.zeros(1, HV, K, V, dtype=torch.float64, device=q.device) + state0 = torch.zeros(1, HV, K, V, dtype=torch.float64, device=q.device) else: - S0 = initial_state[n : n + 1].double() + state0 = initial_state[n : n + 1].double() if e == s: - states.append(S0) + states.append(state0) continue - o_n, S_n = _recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], wf[:, :, s:e], S0) + o_n, state_n = _recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], wf[:, :, s:e], state0) outs.append(o_n) - states.append(S_n) + states.append(state_n) if outs: o = torch.cat(outs, dim=2).permute(0, 2, 1, 3) else: diff --git a/test/python/linear_attention/reference_kda.py b/test/python/linear_attention/reference_kda.py index 5e8f5ffd3..86508c637 100644 --- a/test/python/linear_attention/reference_kda.py +++ b/test/python/linear_attention/reference_kda.py @@ -12,9 +12,9 @@ S_t = (I - beta_t k_t^T k_t) Diag(alpha_t) S_{t-1} + beta_t k_t^T v_t o_t = q_t S_t -with scalar per-token write strength ``beta_t``. The decay is applied FIRST -(``S_dec = Diag(alpha_t) S_{t-1}``) and the delta-rule correction reads the -already-decayed state, so unlike GDN the order matters. Supports grouped +where ``S_t`` is the recurrent state and ``beta_t`` the scalar per-token +write strength. The decay is applied first and the delta-rule correction +reads the already-decayed state, so unlike GDN the order matters. Supports grouped heads (every input's head count must divide ``HO = max(Hq, Hv)``; heads are replicated onto the HO output heads), an optional initial state, and varlen packed batches via ``cu_seqlens``. @@ -38,24 +38,24 @@ def rms_ratio(out: torch.Tensor, ref: torch.Tensor) -> float: return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() -def _recurrent_dense(q, k, v, alpha, beta, S0): - """Dense recurrence in [B, HV, T, *] layout, fp64. Returns (o, S_T). +def _recurrent_dense(q, k, v, alpha, beta, state0): + """Dense recurrence in [B, HV, T, *] layout, fp64. Returns (o, final state). ``alpha`` is per-key-channel: [B, HV, T, K] (GDN's is scalar [B, HV, T]).""" T = q.shape[2] - S = S0 + state = state0 outs = [] for t in range(T): kt = k[:, :, t, :] vt = v[:, :, t, :] at = alpha[:, :, t, :] # [B, HV, K] per-channel decay bt = beta[:, :, t] - S = at[..., None] * S # decay first: Diag(alpha) S, per-K-row scaling - kt_S = (kt.unsqueeze(-2) @ S).squeeze(-2) - residual = vt - kt_S # reads the already-decayed state (no scalar alpha here) - S = S + bt[..., None, None] * (kt.unsqueeze(-1) @ residual.unsqueeze(-2)) - outs.append((q[:, :, t, :].unsqueeze(-2) @ S).squeeze(-2)) - return torch.stack(outs, dim=2), S + state = at[..., None] * state # decay first: Diag(alpha) state, per-K-row scaling + kt_state = (kt.unsqueeze(-2) @ state).squeeze(-2) + residual = vt - kt_state # reads the already-decayed state (no scalar alpha here) + state = state + bt[..., None, None] * (kt.unsqueeze(-1) @ residual.unsqueeze(-2)) + outs.append((q[:, :, t, :].unsqueeze(-2) @ state).squeeze(-2)) + return torch.stack(outs, dim=2), state def kda_reference( @@ -112,11 +112,11 @@ def expand(x): if cu_seqlens is None: B = q.shape[0] if initial_state is None: - S0 = torch.zeros(B, HV, K, V, dtype=torch.float64, device=q.device) + state0 = torch.zeros(B, HV, K, V, dtype=torch.float64, device=q.device) else: - S0 = initial_state.double() - o, S = _recurrent_dense(qf, kf, vf, alphaf, betaf, S0) - return o.permute(0, 2, 1, 3), S + state0 = initial_state.double() + o, state = _recurrent_dense(qf, kf, vf, alphaf, betaf, state0) + return o.permute(0, 2, 1, 3), state assert q.shape[0] == 1, "cu_seqlens requires packed batch B == 1" bounds = cu_seqlens.tolist() @@ -124,15 +124,15 @@ def expand(x): for n in range(len(bounds) - 1): s, e = bounds[n], bounds[n + 1] if initial_state is None: - S0 = torch.zeros(1, HV, K, V, dtype=torch.float64, device=q.device) + state0 = torch.zeros(1, HV, K, V, dtype=torch.float64, device=q.device) else: - S0 = initial_state[n : n + 1].double() + state0 = initial_state[n : n + 1].double() if e == s: - states.append(S0) + states.append(state0) continue - o_n, S_n = _recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], S0) + o_n, state_n = _recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], state0) outs.append(o_n) - states.append(S_n) + states.append(state_n) if outs: o = torch.cat(outs, dim=2).permute(0, 2, 1, 3) else: diff --git a/test/python/linear_attention/test_la.py b/test/python/linear_attention/test_la.py new file mode 100644 index 000000000..b9e5c5c3a --- /dev/null +++ b/test/python/linear_attention/test_la.py @@ -0,0 +1,911 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Backend-parametrized test suite for the linear-attention ops. + +One suite for the public API (``gated_delta_net`` / ``kimi_delta_attention`` / +``gated_delta_net_v2``), parametrized over the python engine backends that can +serve it. Each test pins one backend's engines onto the ops and validates +against the fp64 recurrent references. A pinned engine that declines a +configuration waives the test (``cudnnGraphNotSupportedError`` -> skip), so +the support surface is owned by the engines' ``check_support``, not by a +suite-side matrix; a backend that is not installed skips the same way. + +Determinism soaks and CUDA-graph replay run on the backends that promise +those contracts (currently FROST only). +""" + +from __future__ import annotations + +import contextlib +import math +import os +from types import SimpleNamespace + +import pytest + +torch = pytest.importorskip("torch") +cudnn = pytest.importorskip("cudnn") +la_ops = pytest.importorskip("cudnn.linear_attention.ops") + +import torch.nn.functional as F # noqa: E402 + +from .conftest import gen_qkv # noqa: E402 +from .reference_gdn import gdn_reference, rms_ratio # noqa: E402 +from .reference_gdn2 import gdn2_reference # noqa: E402 +from .reference_kda import kda_reference # noqa: E402 + +pytestmark = [ + pytest.mark.L0, + pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA"), +] + +VARIANTS = ("gdn", "kda", "gdn2") +CHUNK = {"gdn": 64, "kda": 16, "gdn2": 16} + +FWD_TOL = {torch.bfloat16: 2e-2, torch.float16: 1e-2} +STATE_TOL = {torch.bfloat16: 2e-2, torch.float16: 1e-2} +BWD_TOL = {torch.bfloat16: 4e-2, torch.float16: 3e-2} +STATE_GRAD_TOL = 6e-2 + +# (H, HV) pairs: H = Q/K heads, HV = V heads; gates/O/states live at HO = max. +HEAD_CONFIGS = [(1, 1), (3, 3), (1, 2), (2, 4), (16, 32), (16, 64)] +HEAD_CONFIGS_SMALL = [(1, 1), (2, 4)] +# (H, HK, HV) triples; HK == HV < H is canonical GQA (FlashInfer's grouped +# native K), HK == H > HV is the expanded-k form. +GQA_CONFIGS = [(4, 4, 1), (6, 6, 2), (4, 1, 1), (6, 2, 2)] + +RAGGED_SEQ_LENS = [ + [256, 256], + [511, 501], + [64, 128, 512], + [31, 63, 93, 123, 150, 500], + [7] * 24 + [1] * 8, + [2048], +] +EDGE_LENS = [1, 15, 16, 17, 31, 63, 64, 65, 121, 251, 257] + +DETERMINISM_REPEATS = int(os.environ.get("DETERMINISM_REPEATS", "8")) + +_DTYPE_IDS = {torch.bfloat16: "bf16", torch.float16: "fp16"} + + +# --------------------------------------------------------------------------- +# Backend pinning +# --------------------------------------------------------------------------- + + +def _op_modules(): + from cudnn.linear_attention.ops import gdn, gdn2, kda + + return {"gdn": gdn, "kda": kda, "gdn2": gdn2} + + +def _family_engines(backend_name): + """The backend's engine instances per family, ids assigned by the manifest.""" + from cudnn.engines import manifest + + suffix = "_" + backend_name + families = {} + for family in manifest.MANIFEST: + if family.name in VARIANTS: + engines = manifest.instantiate(family, family.offered_ids()) + families[family.name] = [e for e in engines if e.name.endswith(suffix)] + return families + + +def _clear_op_caches(): + for mod in _op_modules().values(): + mod._fprop_cache.clear() + mod._bprop_cache.clear() + + +@pytest.fixture(params=("frost", "cutile")) +def backend(request, monkeypatch): + """Pin one backend by scoping the manifest's candidate list for the LA + families to it (ranking consumes the candidate list, so filtering it is + the pin seam). The op graph caches are keyed without the pin, so they are + cleared around each test.""" + name = request.param + families = _family_engines(name) + missing = [v for v in VARIANTS if v not in families] + if missing: + pytest.fail(f"the engine manifest offers no {missing} families — a stale installed cudnn package is likely shadowing the source tree") + from cudnn.engines import manifest + + real_engines_for = manifest.engines_for + suffix = "_" + name + + def pinned_engines_for(graph): + engines = real_engines_for(graph) + family = manifest.family_for(graph) + if family is not None and family.name in VARIANTS: + return [e for e in engines if e.name.endswith(suffix)] + return engines + + monkeypatch.setattr(manifest, "engines_for", pinned_engines_for) + _clear_op_caches() + try: + yield SimpleNamespace(name=name, engines=families) + finally: + _clear_op_caches() + + +@contextlib.contextmanager +def _waive_unsupported(backend, variant): + """An empty pin (op call would fall back to manifest routing) or an engine + decline waives the test; the engines own the support surface.""" + if not backend.engines[variant]: + pytest.skip(f"the {backend.name} backend has no {variant} engine") + try: + yield + except cudnn.cudnnGraphNotSupportedError as exc: + pytest.skip(f"{backend.name} {variant} declined: {exc}") + + +# --------------------------------------------------------------------------- +# Case generation and dispatch +# --------------------------------------------------------------------------- + + +def _seed(seed=0): + torch.random.manual_seed(seed) + torch.cuda.manual_seed(seed) + + +def _gate_lo(dtype): + return 0.6 if dtype == torch.float16 else 0.5 + + +def _gen_gates(variant, B, T, HO, K, V, dtype, *, alpha=True, beta=True, w=True, lo=None, device="cuda"): + if lo is None: + lo = _gate_lo(dtype) + gshape = (B, T, HO) if variant == "gdn" else (B, T, HO, K) + if alpha: + g = torch.empty(gshape, device=device, dtype=torch.float32).uniform_(lo, 1.0).log() + else: + g = torch.zeros(gshape, device=device, dtype=torch.float32) + if variant == "gdn2": + b = (torch.rand(B, T, HO, K, device=device).sigmoid() * 2.0).to(dtype) if beta else torch.ones(B, T, HO, K, device=device, dtype=dtype) + wt = torch.rand(B, T, HO, V, device=device).sigmoid().to(dtype) if w else torch.ones(B, T, HO, V, device=device, dtype=dtype) + return {"g": g, "beta": b, "w": wt} + b = torch.rand(B, T, HO, device=device) if beta else torch.ones(B, T, HO, device=device) + return {"g": g, "beta": b} + + +def _case(variant, dtype, *, B=1, T=None, seq_lens=None, H=2, HK=None, HV=None, K=128, V=128, alpha=True, beta=True, w=True, lo=None, seed=0): + """Dense ``(B, T)`` or packed varlen (``seq_lens``, B == 1) inputs plus the + matching ``cu_seqlens``. ``HK`` defaults to ``H``; ``HK == HV < H`` is + canonical (native grouped K) GQA.""" + _seed(seed) + HV = H if HV is None else HV + HK = H if HK is None else HK + HO = max(H, HV) + if seq_lens is not None: + total = sum(seq_lens) + bounds = [0] + for sl in seq_lens: + bounds.append(bounds[-1] + sl) + cu = torch.tensor(bounds, dtype=torch.int32, device="cuda") + B, T, N, varlen = 1, total, len(seq_lens), True + else: + cu = torch.arange(0, B + 1, dtype=torch.int32, device="cuda") * T + N, varlen = B, False + q, k, v = gen_qkv(B, T, H, HV, K, V, dtype) + if HK != H: + from .conftest import multidist_randu + + k = F.normalize(multidist_randu(B * T * HK, K, device="cuda").reshape(B, T, HK, K), p=2.0, dim=-1).to(dtype).contiguous() + gates = _gen_gates(variant, B, T, HO, K, V, dtype, alpha=alpha, beta=beta, w=w, lo=lo) + return SimpleNamespace(variant=variant, dtype=dtype, q=q, k=k, v=v, gates=gates, cu=cu, B=B, T=T, N=N, H=H, HK=HK, HV=HV, HO=HO, K=K, V=V, varlen=varlen) + + +def _thd(x): + return x.reshape(-1, *x.shape[2:]) + + +def _op(variant): + return {"gdn": la_ops.gated_delta_net, "kda": la_ops.kimi_delta_attention, "gdn2": la_ops.gated_delta_net_v2}[variant] + + +def _op_args(case, cu=None): + args = [_thd(case.q), _thd(case.k), _thd(case.v), _thd(case.gates["g"]), _thd(case.gates["beta"])] + if case.variant == "gdn2": + args.append(_thd(case.gates["w"])) + args.append(case.cu if cu is None else cu) + return args + + +def _run_fwd(backend, case, *, cu=None, **kw): + with _waive_unsupported(backend, case.variant): + return _op(case.variant)(*_op_args(case, cu=cu), **kw) + + +def _reference(case, *, scale=None, initial_state=None, l2norm=False, cu=None): + fn = {"gdn": gdn_reference, "kda": kda_reference, "gdn2": gdn2_reference}[case.variant] + q, k = case.q, case.k + if l2norm: + q = F.normalize(q.float(), dim=-1) + k = F.normalize(k.float(), dim=-1) + args = [q, k, case.v, case.gates["g"], case.gates["beta"]] + if case.variant == "gdn2": + args.append(case.gates["w"]) + kwargs = dict(scale=scale, initial_state=initial_state) + if case.varlen or cu is not None: + kwargs["cu_seqlens"] = case.cu if cu is None else cu + with torch.no_grad(): + return fn(*args, **kwargs) + + +def _check(name, out, ref, tol): + out = out.float() + assert torch.isfinite(out).all(), f"non-finite values in {name}" + r = rms_ratio(out.reshape(ref.shape), ref) + assert r < tol, f"{name} rms ratio {r:.4g} >= {tol}" + + +def _check_fwd(case, o, fs, *, scale=None, initial_state=None, l2norm=False, tol_mult=1.0): + o_ref, fs_ref = _reference(case, scale=scale, initial_state=initial_state, l2norm=l2norm) + _check("o", o, o_ref, tol_mult * FWD_TOL[case.dtype]) + if fs is not None and fs.numel(): + _check("final_state", fs, fs_ref, tol_mult * STATE_TOL[case.dtype]) + + +# --------------------------------------------------------------------------- +# Backend pin seam +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_backend_pin_selects_engine(backend, variant): + """The pinned engine actually serves the graph — a dead pin would silently + validate whatever the default routing picks.""" + case = _case(variant, torch.bfloat16, T=4 * CHUNK[variant]) + with _waive_unsupported(backend, variant): + _op(variant)(*_op_args(case)) + mod = _op_modules()[variant] + names = {g.selected_engine.name for g, _t in mod._fprop_cache.values() if g.selected_engine is not None} + assert names == {f"{variant}_{backend.name}"}, f"expected only {variant}_{backend.name} to serve, got {names}" + + +# --------------------------------------------------------------------------- +# Forward parity +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("H,HV", HEAD_CONFIGS) +@pytest.mark.parametrize("B,T", [(1, 64), (1, 128), (2, 256)]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16], ids=_DTYPE_IDS.get) +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_basic(backend, variant, dtype, B, T, H, HV): + if dtype == torch.float16 and (H, HV) not in HEAD_CONFIGS_SMALL: + pytest.skip("fp16 runs the small head matrix") + case = _case(variant, dtype, B=B, T=T, H=H, HV=HV) + o, fs = _run_fwd(backend, case, output_final_state=True) + _check_fwd(case, o, fs) + + +@pytest.mark.parametrize("alpha,beta,w", [(True, False, True), (False, True, True), (True, True, False)], ids=["no_beta", "no_alpha", "no_w"]) +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_gate_combinations(backend, variant, alpha, beta, w): + if variant != "gdn" and not alpha: + pytest.skip("the delta-rule inverse needs decay (matches FI's use_g=False skip)") + if variant != "gdn2" and not w: + pytest.skip("w is a GDN-2 gate") + case = _case(variant, torch.bfloat16, T=192, alpha=alpha, beta=beta, w=w) + o, fs = _run_fwd(backend, case, output_final_state=True) + _check_fwd(case, o, fs) + + +@pytest.mark.parametrize("scale", [0.5, 1.0, None], ids=["half", "one", "auto"]) +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_scale(backend, variant, scale): + case = _case(variant, torch.bfloat16, T=192) + o, fs = _run_fwd(backend, case, scale=scale, output_final_state=True) + _check_fwd(case, o, fs, scale=scale) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_default_scale_matches_explicit(backend, variant): + case = _case(variant, torch.bfloat16, T=128) + o_default, _ = _run_fwd(backend, case) + o_explicit, _ = _run_fwd(backend, case, scale=1.0 / math.sqrt(case.K)) + torch.testing.assert_close(o_default, o_explicit) + + +@pytest.mark.parametrize("T", EDGE_LENS) +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_seqlen_edges(backend, variant, T): + """Lengths straddling the kernels' chunk boundaries (16 and 64).""" + case = _case(variant, torch.bfloat16, T=T) + o, fs = _run_fwd(backend, case, output_final_state=True) + _check_fwd(case, o, fs) + + +@pytest.mark.parametrize("H,HV", [(1, 1), (2, 4)]) +@pytest.mark.parametrize("seq_lens", RAGGED_SEQ_LENS, ids=lambda sl: f"{len(sl)}seqs_{sum(sl)}tok") +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_varlen_ragged(backend, variant, seq_lens, H, HV): + case = _case(variant, torch.bfloat16, seq_lens=seq_lens, H=H, HV=HV) + o, fs = _run_fwd(backend, case, output_final_state=True) + _check_fwd(case, o, fs) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_many_short_sequences(backend, variant): + """A 200-sequence packed batch matches the same sequences run one by one.""" + T = 33 + case = _case(variant, torch.bfloat16, seq_lens=[T] * 200) + o, fs = _run_fwd(backend, case, output_final_state=True) + cu1 = torch.tensor([0, T], dtype=torch.int32, device="cuda") + for n in (0, 1, 99, 199): + # clone: sliced views can start at non-16B-aligned offsets, which the + # kernels' buffer contract rejects + sl = slice(T * n, T * (n + 1)) + args = [ + _thd(case.q)[sl].clone(), + _thd(case.k)[sl].clone(), + _thd(case.v)[sl].clone(), + _thd(case.gates["g"])[sl].clone(), + _thd(case.gates["beta"])[sl].clone(), + ] + if case.variant == "gdn2": + args.append(_thd(case.gates["w"])[sl].clone()) + with _waive_unsupported(backend, variant): + o_n, fs_n = _op(variant)(*args, cu1, output_final_state=True) + _check(f"o[seq {n}]", o[sl], o_n.float(), FWD_TOL[torch.bfloat16]) + _check(f"final_state[seq {n}]", fs[n], fs_n[0].float(), STATE_TOL[torch.bfloat16]) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_zero_length_sequences(backend, variant): + """Empty sequences must not perturb their neighbors; their state rows stay + zero (or pass the initial state through when one is given).""" + case = _case(variant, torch.bfloat16, seq_lens=[64, 128]) + o_base, fs_base = _run_fwd(backend, case, output_final_state=True) + cu = torch.tensor([0, 64, 64, 192, 192], dtype=torch.int32, device="cuda") + o, fs = _run_fwd(backend, case, cu=cu, output_final_state=True) + torch.testing.assert_close(o, o_base, atol=1e-3, rtol=1e-3) + torch.testing.assert_close(fs[0], fs_base[0], atol=1e-3, rtol=1e-3) + torch.testing.assert_close(fs[2], fs_base[1], atol=1e-3, rtol=1e-3) + assert (fs[1] == 0).all() and (fs[3] == 0).all(), "zero-length sequence states must stay zero" + state0 = torch.randn(4, case.HO, case.K, case.V, device="cuda", dtype=torch.float32) * 0.05 + _o, fs_state0 = _run_fwd(backend, case, cu=cu, initial_state=state0, output_final_state=True) + torch.testing.assert_close(fs_state0[1], state0[1], atol=0.0, rtol=0.0) + torch.testing.assert_close(fs_state0[3], state0[3], atol=0.0, rtol=0.0) + + +@pytest.mark.parametrize("T", [128, 251]) +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_initial_state(backend, variant, T): + case = _case(variant, torch.bfloat16, T=T) + state0 = torch.randn(case.N, case.HO, case.K, case.V, device="cuda", dtype=torch.float32) * 0.05 + o, fs = _run_fwd(backend, case, initial_state=state0, output_final_state=True) + _check_fwd(case, o, fs, initial_state=state0) + + +@pytest.mark.parametrize("T1,T2", [(128, 128), (64, 192), (192, 121)]) +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_chunked_prefill(backend, variant, T1, T2): + """Two-phase prefill: part 1's final state feeds part 2; the concatenated + output matches a single-shot reference (state round-trips through fp32).""" + case = _case(variant, torch.bfloat16, B=2, T=T1 + T2) + + def part(t0, t1, state0): + sub = SimpleNamespace(**vars(case)) + sub.q, sub.k, sub.v = (x[:, t0:t1].contiguous() for x in (case.q, case.k, case.v)) + sub.gates = {n: g[:, t0:t1].contiguous() for n, g in case.gates.items()} + sub.T = t1 - t0 + sub.cu = torch.arange(0, case.B + 1, dtype=torch.int32, device="cuda") * sub.T + return _run_fwd(backend, sub, initial_state=state0, output_final_state=True) + + o1, fs1 = part(0, T1, None) + o2, fs2 = part(T1, T1 + T2, fs1) + o = torch.cat([o1.reshape(case.B, T1, case.HO, case.V), o2.reshape(case.B, T2, case.HO, case.V)], dim=1) + o_ref, fs_ref = _reference(case) + _check("o", o, o_ref, 1.5 * FWD_TOL[case.dtype]) + _check("final_state", fs2, fs_ref, 1.5 * STATE_TOL[case.dtype]) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_packed_matches_per_sequence(backend, variant): + B, T = 3, 128 + case = _case(variant, torch.bfloat16, B=B, T=T) + o, fs = _run_fwd(backend, case, output_final_state=True) + cu1 = torch.tensor([0, T], dtype=torch.int32, device="cuda") + for b in range(B): + args = [case.q[b], case.k[b], case.v[b], case.gates["g"][b], case.gates["beta"][b]] + if variant == "gdn2": + args.append(case.gates["w"][b]) + with _waive_unsupported(backend, variant): + o_b, fs_b = _op(variant)(*args, cu1, output_final_state=True) + torch.testing.assert_close(o[b * T : (b + 1) * T], o_b) + torch.testing.assert_close(fs[b], fs_b[0]) + + +@pytest.mark.parametrize( + "variant,K,V", + [("gdn", 64, 64), ("gdn", 64, 128), ("gdn", 128, 128), ("gdn", 256, 128), ("kda", 64, 64), ("kda", 64, 128), ("kda", 128, 128), ("gdn2", 128, 128)], +) +def test_fwd_head_dims(backend, variant, K, V): + """K/V head-dim variants; engines that only serve K = V = 128 decline.""" + case = _case(variant, torch.bfloat16, T=192, K=K, V=V) + o, fs = _run_fwd(backend, case, output_final_state=True) + _check_fwd(case, o, fs) + + +@pytest.mark.parametrize("H,HK,HV", GQA_CONFIGS) +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_gqa(backend, variant, H, HK, HV): + """Grouped-query heads (H > HV): gates/O/states live at HO = H. Covers + both canonical GQA (native K at HK == HV) and the expanded-k form.""" + case = _case(variant, torch.bfloat16, T=192, H=H, HK=HK, HV=HV) + o, fs = _run_fwd(backend, case, output_final_state=True) + _check_fwd(case, o, fs) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_qk_l2norm(backend, variant): + """In-kernel Q/K L2 norm matches the reference on pre-normalized inputs.""" + case = _case(variant, torch.bfloat16, T=256) + o, fs = _run_fwd(backend, case, output_final_state=True, use_qk_l2norm_in_kernel=True) + _check_fwd(case, o, fs, l2norm=True) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_strong_decay_varlen(backend, variant): + case = _case(variant, torch.bfloat16, seq_lens=[100, 2048, 0, 517], lo=0.1 if variant == "gdn" else 0.3) + o, fs = _run_fwd(backend, case, output_final_state=True) + _check_fwd(case, o, fs) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_fwd_output_contract(backend, variant): + """O is io-dtype at HO heads; final_state is fp32 and empty unless requested.""" + case = _case(variant, torch.bfloat16, T=128, H=2, HV=4) + o, fs = _run_fwd(backend, case) + assert o.shape == (case.T, case.HO, case.V) and o.dtype == case.dtype + assert fs.numel() == 0 + _o, fs = _run_fwd(backend, case, output_final_state=True) + assert fs.shape == (case.N, case.HO, case.K, case.V) and fs.dtype == torch.float32 + + +# --------------------------------------------------------------------------- +# Backward parity (oracle: fp64 autograd through the references) +# --------------------------------------------------------------------------- + + +def _assert_bwd_parity(backend, case, *, scale=None, use_initial_state=False, use_dfs=False, l2norm=False, gate_grad_tol=None, seed=1): + variant, tol = case.variant, BWD_TOL[case.dtype] + tensors = {"q": case.q, "k": case.k, "v": case.v, "g": case.gates["g"], "beta": case.gates["beta"]} + if variant == "gdn2": + tensors["w"] = case.gates["w"] + op_leaves = {n: _thd(t).detach().clone().requires_grad_(True) for n, t in tensors.items()} + ref_leaves = {n: t.detach().double().requires_grad_(True) for n, t in tensors.items()} + _seed(seed) + state0_op = state0_ref = None + if use_initial_state: + state0 = torch.randn(case.N, case.HO, case.K, case.V, device="cuda", dtype=torch.float32) * 0.05 + state0_op = state0.detach().clone().requires_grad_(True) + state0_ref = state0.detach().double().requires_grad_(True) + + with _waive_unsupported(backend, variant): + args = [op_leaves["q"], op_leaves["k"], op_leaves["v"], op_leaves["g"], op_leaves["beta"]] + if variant == "gdn2": + args.append(op_leaves["w"]) + args.append(case.cu) + o, fs = _op(variant)(*args, scale=scale, initial_state=state0_op, output_final_state=True, use_qk_l2norm_in_kernel=l2norm) + dO = torch.randn_like(o) + outputs, grad_outputs = [o], [dO] + dFS = None + if use_dfs: + dFS = torch.randn_like(fs) * 0.1 + outputs.append(fs) + grad_outputs.append(dFS) + grad_inputs = list(op_leaves.values()) + ([state0_op] if use_initial_state else []) + grads = torch.autograd.grad(outputs, grad_inputs, grad_outputs) + + qd, kd = ref_leaves["q"], ref_leaves["k"] + if l2norm: + qd, kd = F.normalize(qd, dim=-1), F.normalize(kd, dim=-1) + ref_fn = {"gdn": gdn_reference, "kda": kda_reference, "gdn2": gdn2_reference}[variant] + ref_args = [qd, kd, ref_leaves["v"], ref_leaves["g"], ref_leaves["beta"]] + if variant == "gdn2": + ref_args.append(ref_leaves["w"]) + ref_kwargs = dict(scale=scale, initial_state=state0_ref) + if case.varlen: + ref_kwargs["cu_seqlens"] = case.cu + o_ref, fs_ref = ref_fn(*ref_args, **ref_kwargs) + ref_outputs, ref_gos = [o_ref], [dO.double().reshape(o_ref.shape)] + if use_dfs: + ref_outputs.append(fs_ref) + ref_gos.append(dFS.double().reshape(fs_ref.shape)) + ref_grads = torch.autograd.grad(ref_outputs, list(ref_leaves.values()) + ([state0_ref] if use_initial_state else []), ref_gos) + + names = list(op_leaves) + (["initial_state"] if use_initial_state else []) + for name, got, want in zip(names, grads, ref_grads): + if name == "initial_state": + tol_n = STATE_GRAD_TOL + elif name in ("g", "beta", "w") and gate_grad_tol is not None: + tol_n = gate_grad_tol + else: + tol_n = tol + _check(f"d{name}", got, want, tol_n) + + +@pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL + [(16, 64)]) +@pytest.mark.parametrize("T", [64, 128, 251]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16], ids=_DTYPE_IDS.get) +@pytest.mark.parametrize("variant", VARIANTS) +def test_bwd_parity(backend, variant, dtype, T, H, HV): + if dtype == torch.float16 and (T != 128 or (H, HV) != (1, 1)): + pytest.skip("fp16 runs one representative backward config") + if (H, HV) == (16, 64) and T != 128: + pytest.skip("the large GVA config runs one length") + _assert_bwd_parity(backend, _case(variant, dtype, T=T, H=H, HV=HV)) + + +@pytest.mark.parametrize("H,HK,HV", GQA_CONFIGS) +@pytest.mark.parametrize("variant", VARIANTS) +def test_bwd_gqa(backend, variant, H, HK, HV): + _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=128, H=H, HK=HK, HV=HV)) + + +@pytest.mark.parametrize("seq_lens", [[64, 192], [31, 63, 93, 123]], ids=["two", "ragged"]) +@pytest.mark.parametrize("variant", VARIANTS) +def test_bwd_varlen(backend, variant, seq_lens): + _assert_bwd_parity(backend, _case(variant, torch.bfloat16, seq_lens=seq_lens)) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_bwd_zero_length_sequence(backend, variant): + _assert_bwd_parity(backend, _case(variant, torch.bfloat16, seq_lens=[64, 0, 128])) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_bwd_initial_state(backend, variant): + _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=128), use_initial_state=True) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_bwd_d_final_state(backend, variant): + _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=128), use_initial_state=True, use_dfs=True) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_bwd_d_final_state_partial_chunk(backend, variant): + _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=251), use_dfs=True) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_bwd_scale(backend, variant): + """A non-default scale must reach the backward path (the engines carry an + independent 1/sqrt(K) default that would mask a dropped plumb).""" + _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=128), scale=1.0) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_bwd_qk_l2norm(backend, variant): + """dQ/dK must include the in-kernel normalization's own backward.""" + _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=128), l2norm=True) + + +def test_bwd_no_decay_gate_grad_floor(backend): + """GDN with alpha off: dGate/dBeta are cancelling-reduction noise floors; + the data grads stay at full tolerance.""" + _assert_bwd_parity(backend, _case("gdn", torch.bfloat16, T=192, alpha=False), gate_grad_tol=0.3) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_bwd_with_checkpoints(backend, variant): + """The checkpoint dump is non-differentiable and must not block backward.""" + ckpt = CHUNK[variant] + case = _case(variant, torch.bfloat16, T=4 * ckpt) + q_t = _thd(case.q).detach().clone().requires_grad_(True) + g_t = _thd(case.gates["g"]).detach().clone().requires_grad_(True) + args = [q_t, _thd(case.k), _thd(case.v), g_t, _thd(case.gates["beta"])] + if variant == "gdn2": + args.append(_thd(case.gates["w"])) + with _waive_unsupported(backend, variant): + o, fs, state_checkpoints = _op(variant)(*args, case.cu, output_final_state=True, checkpoint_every_n_tokens=ckpt) + assert not state_checkpoints.requires_grad + (o.sum() + fs.sum()).backward() + for name, t in (("q", q_t), ("g", g_t)): + assert t.grad is not None and torch.isfinite(t.grad).all(), f"bad grad for {name}" + + +# --------------------------------------------------------------------------- +# Checkpoints (per-chunk state series) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_checkpoints_match_prefix_final_states(backend, variant): + """state_checkpoints[j] is the state after (j+1)*ckpt tokens, strictly before the end.""" + ckpt = CHUNK[variant] + T = 5 * ckpt + case = _case(variant, torch.bfloat16, T=T) + o, fs, state_checkpoints = _run_fwd(backend, case, output_final_state=True, checkpoint_every_n_tokens=ckpt) + assert state_checkpoints.shape == ((T - 1) // ckpt, case.HO, case.K, case.V) + assert state_checkpoints.dtype == case.dtype + for j in (0, state_checkpoints.shape[0] - 1): + n = (j + 1) * ckpt + args = [_thd(case.q)[:n], _thd(case.k)[:n], _thd(case.v)[:n], _thd(case.gates["g"])[:n], _thd(case.gates["beta"])[:n]] + if variant == "gdn2": + args.append(_thd(case.gates["w"])[:n]) + cu_n = torch.tensor([0, n], dtype=torch.int32, device="cuda") + with _waive_unsupported(backend, variant): + _o, fs_p = _op(variant)(*args, cu_n, output_final_state=True) + _check(f"state_checkpoints[{j}]", state_checkpoints[j], fs_p[0], STATE_TOL[case.dtype]) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_checkpoints_varlen(backend, variant): + """Entries pack per sequence in order (one per ckpt tokens strictly before + each sequence end); each entry matches its sequence's truncated prefix.""" + ckpt = CHUNK[variant] + seq_lens = [3 * ckpt + 5, ckpt - 1, 0, 2 * ckpt] + case = _case(variant, torch.bfloat16, seq_lens=seq_lens) + _o, _fs, state_checkpoints = _run_fwd(backend, case, output_final_state=True, checkpoint_every_n_tokens=ckpt) + counts = [max(sl - 1, 0) // ckpt for sl in seq_lens] + # shape[0] is the shape-derived capacity bound; the packed prefix holds + # sum(counts) real rows (per-sequence, in order), the tail is uninitialized + assert state_checkpoints.shape[0] == max((sum(seq_lens) - len(seq_lens)) // ckpt, 1) + bounds = case.cu.tolist() + base = 0 + for n, cnt in enumerate(counts): + for j in sorted({0, cnt - 1} if cnt else set()): + n0 = bounds[n] + ntok = (j + 1) * ckpt + args = [_thd(t)[n0 : n0 + ntok].clone() for t in (case.q, case.k, case.v, case.gates["g"], case.gates["beta"])] + if variant == "gdn2": + args.append(_thd(case.gates["w"])[n0 : n0 + ntok].clone()) + cu_n = torch.tensor([0, ntok], dtype=torch.int32, device="cuda") + with _waive_unsupported(backend, variant): + _op_, fs_p = _op(variant)(*args, cu_n, output_final_state=True) + _check(f"state_checkpoints[seq {n}][{j}]", state_checkpoints[base + j], fs_p[0], STATE_TOL[case.dtype]) + base += cnt + + +# --------------------------------------------------------------------------- +# Raw-logit gate modes (safe gate, in-kernel Beta sigmoid) +# --------------------------------------------------------------------------- + + +def _safe_gate_case(variant, T=256, H=2, K=128, V=128, seed=7): + case = _case(variant, torch.bfloat16, T=T, H=H, K=K, V=V, seed=seed) + _seed(seed + 1) + graw = torch.randn(1, T, case.HO, K, device="cuda", dtype=torch.float32) + a_log = torch.zeros(case.HO, dtype=torch.float32, device="cuda") + dt_bias = torch.zeros(case.HO, K, dtype=torch.float32, device="cuda") + return case, graw, a_log, dt_bias + + +@pytest.mark.parametrize("variant", ["kda", "gdn2"]) +def test_safe_gate_forward_parity(backend, variant): + """Raw logits with a_log = 0 / dt_bias = 0 match the post-activation path + fed the host-side transform ``lb * sigmoid(g)``.""" + lb = -5.0 + case, graw, a_log, dt_bias = _safe_gate_case(variant) + kw = dict(output_final_state=True, use_qk_l2norm_in_kernel=True) + raw_kw = dict(kw, safe_gate=True, gate_lower_bound=lb, a_log=a_log, dt_bias=dt_bias) + raw_gates = dict(case.gates, g=graw) + if variant == "kda": + braw = torch.randn(1, case.T, case.HO, device="cuda").to(case.dtype) + raw_gates["beta"] = braw + raw_kw["use_beta_sigmoid_in_kernel"] = True + eff_beta = braw.float().sigmoid() + else: + eff_beta = case.gates["beta"] + raw_case = SimpleNamespace(**{**vars(case), "gates": raw_gates}) + eff_case = SimpleNamespace(**{**vars(case), "gates": dict(case.gates, g=lb * torch.sigmoid(graw), beta=eff_beta)}) + o_raw, fs_raw = _run_fwd(backend, raw_case, **raw_kw) + o_eff, fs_eff = _run_fwd(backend, eff_case, **kw) + torch.testing.assert_close(o_raw.float(), o_eff.float(), atol=2.5e-2, rtol=2.5e-2) + assert rms_ratio(fs_raw, fs_eff) < 2e-2 + + +@pytest.mark.parametrize("variant", ["kda", "gdn2"]) +def test_safe_gate_backward_raises(backend, variant): + """Raw-logit gate modes are forward-only by contract.""" + lb = -5.0 + case, graw, a_log, dt_bias = _safe_gate_case(variant, T=128) + raw_gates = dict(case.gates, g=graw) + kw = dict(safe_gate=True, gate_lower_bound=lb, a_log=a_log, dt_bias=dt_bias, use_qk_l2norm_in_kernel=True) + if variant == "kda": + raw_gates["beta"] = torch.randn(1, case.T, case.HO, device="cuda").to(case.dtype) + kw["use_beta_sigmoid_in_kernel"] = True + raw_case = SimpleNamespace(**{**vars(case), "gates": raw_gates}) + g_leaf = _thd(raw_gates["g"]).detach().clone().requires_grad_(True) + args = [_thd(raw_case.q).detach().clone().requires_grad_(True), _thd(raw_case.k), _thd(raw_case.v), g_leaf, _thd(raw_gates["beta"])] + if variant == "gdn2": + args.append(_thd(raw_gates["w"])) + with _waive_unsupported(backend, variant): + o, _ = _op(variant)(*args, case.cu, **kw) + with pytest.raises(NotImplementedError, match="forward-only"): + o.sum().backward() + + +def test_beta_sigmoid_in_kernel(backend): + """KDA: io-dtype Beta logits with the in-kernel sigmoid match the + post-activation fp32 path.""" + case = _case("kda", torch.bfloat16, T=256) + _seed(11) + braw = torch.randn(1, case.T, case.HO, device="cuda").to(case.dtype) + raw_case = SimpleNamespace(**{**vars(case), "gates": dict(case.gates, beta=braw)}) + eff_case = SimpleNamespace(**{**vars(case), "gates": dict(case.gates, beta=braw.float().sigmoid())}) + o_raw, fs_raw = _run_fwd(backend, raw_case, output_final_state=True, use_beta_sigmoid_in_kernel=True) + o_eff, fs_eff = _run_fwd(backend, eff_case, output_final_state=True) + torch.testing.assert_close(o_raw.float(), o_eff.float(), atol=2.5e-2, rtol=2.5e-2) + assert rms_ratio(fs_raw, fs_eff) < 2e-2 + + +# --------------------------------------------------------------------------- +# torch.compile +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_torch_compile_forward(backend, variant): + case = _case(variant, torch.bfloat16, T=128) + with _waive_unsupported(backend, variant): + o_eager, _ = _op(variant)(*_op_args(case)) + compiled = torch.compile(_op(variant), fullgraph=True) + o_comp, _ = compiled(*_op_args(case)) + torch.testing.assert_close(o_eager, o_comp) + + +# --------------------------------------------------------------------------- +# Argument validation (op-level contract; raises before engine selection) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_invalid_rank_raises(variant): + case = _case(variant, torch.bfloat16, T=64) + with pytest.raises(ValueError, match="THD"): + _op(variant)(case.q, _thd(case.k), _thd(case.v), *[_thd(case.gates[n]) for n in case.gates], case.cu) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_invalid_qk_head_mismatch_raises(variant): + case = _case(variant, torch.bfloat16, T=64, H=2) + args = [_thd(case.q), _thd(case.k)[:, :1].contiguous(), _thd(case.v), _thd(case.gates["g"]), _thd(case.gates["beta"])] + if variant == "gdn2": + args.append(_thd(case.gates["w"])) + with pytest.raises(ValueError, match="head count"): + _op(variant)(*args, case.cu) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_invalid_gate_dtype_raises(variant): + case = _case(variant, torch.bfloat16, T=64) + args = [_thd(case.q), _thd(case.k), _thd(case.v), _thd(case.gates["g"]).to(torch.bfloat16), _thd(case.gates["beta"])] + if variant == "gdn2": + args.append(_thd(case.gates["w"])) + with pytest.raises(TypeError, match="must be"): + _op(variant)(*args, case.cu) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_invalid_initial_state_count_raises(variant): + case = _case(variant, torch.bfloat16, T=64) + state0 = torch.zeros(3, case.HO, case.K, case.V, device="cuda", dtype=torch.float32) + with pytest.raises(ValueError, match="initial"): + _op(variant)(*_op_args(case), initial_state=state0) + + +@pytest.mark.parametrize("variant", ["kda", "gdn2"]) +def test_invalid_safe_gate_args_raise(variant): + case = _case(variant, torch.bfloat16, T=64) + with pytest.raises(ValueError, match="safe_gate"): + _op(variant)(*_op_args(case), safe_gate=True) + with pytest.raises(ValueError, match="safe_gate"): + _op(variant)(*_op_args(case), a_log=torch.zeros(case.HO, device="cuda")) + + +# --------------------------------------------------------------------------- +# Determinism (contract held by the FROST backend) +# --------------------------------------------------------------------------- + + +def _bits(t): + return t.contiguous().view(torch.uint8) + + +def _assert_bitwise_runs(launch, repeats=DETERMINISM_REPEATS, label=""): + """Back-to-back launches (single sync) must match run 0 bit for bit — + barrier/fence races are timing-dependent, so there is no tolerance.""" + runs = [launch() for _ in range(repeats)] + torch.cuda.synchronize() + for out in runs[0]: + assert torch.isfinite(out.float()).all(), f"{label}: non-finite output in run 0" + for r, outs in enumerate(runs[1:], start=1): + for i, (a, b) in enumerate(zip(runs[0], outs)): + assert torch.equal(_bits(a), _bits(b)), f"{label}: output {i} differs between run 0 and run {r}" + + +@pytest.mark.parametrize("backend", ["frost"], indirect=True) +@pytest.mark.parametrize("variant", VARIANTS) +def test_determinism_fwd(backend, variant): + case = _case(variant, torch.bfloat16, seq_lens=[497, 16, 1, 480, 0, 253]) + state0 = torch.randn(case.N, case.HO, case.K, case.V, device="cuda", dtype=torch.float32) * 0.05 + + def launch(): + o, fs = _run_fwd(backend, case, initial_state=state0, output_final_state=True) + return o, fs + + _assert_bitwise_runs(launch, label=f"{variant} fwd") + + +@pytest.mark.parametrize("backend", ["frost"], indirect=True) +@pytest.mark.parametrize("variant", VARIANTS) +def test_determinism_bwd(backend, variant): + case = _case(variant, torch.bfloat16, seq_lens=[497, 16, 1, 480, 0, 253]) + leaves = [_thd(case.q).detach().clone().requires_grad_(True), _thd(case.k).detach().clone().requires_grad_(True)] + args = [leaves[0], leaves[1], _thd(case.v), _thd(case.gates["g"]), _thd(case.gates["beta"])] + if variant == "gdn2": + args.append(_thd(case.gates["w"])) + with _waive_unsupported(backend, variant): + o, _fs = _op(variant)(*args, case.cu) + dO = torch.randn_like(o) + + def launch(): + return torch.autograd.grad([o], leaves, [dO], retain_graph=True) + + _assert_bitwise_runs(launch, label=f"{variant} bwd") + + +@pytest.mark.parametrize("backend", ["frost"], indirect=True) +@pytest.mark.parametrize("variant", VARIANTS) +def test_determinism_two_streams(backend, variant): + """Two concurrent instances on separate streams must not perturb each + other: every repeat matches its own single-stream baseline.""" + case_a = _case(variant, torch.bfloat16, seq_lens=[497, 16, 1, 480, 0, 253], seed=0) + case_b = _case(variant, torch.bfloat16, B=2, T=512, seed=1) + launch_a = lambda: _run_fwd(backend, case_a, output_final_state=True) # noqa: E731 + launch_b = lambda: _run_fwd(backend, case_b, output_final_state=True) # noqa: E731 + s1, s2 = torch.cuda.Stream(), torch.cuda.Stream() + # order the side streams behind the input generation (default stream) + torch.cuda.synchronize() + + with torch.cuda.stream(s1): + base_a = launch_a() + torch.cuda.synchronize() + with torch.cuda.stream(s2): + base_b = launch_b() + torch.cuda.synchronize() + for r in range(DETERMINISM_REPEATS): + with torch.cuda.stream(s1): + out_a = launch_a() + with torch.cuda.stream(s2): + out_b = launch_b() + torch.cuda.synchronize() + for label, base, outs in (("A", base_a, out_a), ("B", base_b, out_b)): + for i, (x, y) in enumerate(zip(base, outs)): + assert torch.equal(_bits(x), _bits(y)), f"stream {label} output {i} differs on concurrent run {r}" + + +# --------------------------------------------------------------------------- +# CUDA-graph replay (contract held by the FROST backend) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("backend", ["frost"], indirect=True) +@pytest.mark.parametrize("variant", VARIANTS) +def test_cuda_graph_replay_fwd(backend, variant): + case = _case(variant, torch.bfloat16, B=2, T=256) + + def launch(): + return _op(variant)(*_op_args(case), output_final_state=True) + + with _waive_unsupported(backend, variant): + eager = launch() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = launch() + graph.replay() + torch.cuda.synchronize() + for i, (a, b) in enumerate(zip(eager, captured)): + assert torch.equal(_bits(a), _bits(b)), f"replayed output {i} differs from eager" From f3ca28c7f52409052443746289f470e09a062c0f Mon Sep 17 00:00:00 2001 From: Joshua Park Date: Tue, 11 Aug 2026 16:02:02 -0700 Subject: [PATCH 2/9] 2 --- .../benchmark_single_linear_attention.py | 4 +-- benchmark/linear_attention/plot_results.py | 22 +++++++++---- python/cudnn/_pygraph.py | 3 +- python/cudnn/frost/buffers.py | 7 ++--- python/cudnn/frost/tile_dsl/barrier.py | 31 +++++-------------- .../linear_attention/cutile/kda_engine.py | 7 +++-- .../cudnn/linear_attention/frost/__init__.py | 3 +- .../linear_attention/frost/common/downcast.py | 2 ++ .../frost/common/head_reduce.py | 2 ++ .../linear_attention/frost/gdn2_engine.py | 2 +- .../linear_attention/frost/kda_engine.py | 10 +++--- .../frost/kernel/gdn2_bprop_f16.py | 2 +- .../frost/kernel/gdn_bprop_f16.py | 1 - .../frost/kernel/kda_bprop_f16.py | 5 +-- .../cudnn/linear_attention/graph_analyzer.py | 2 +- python/cudnn/linear_attention/ops/gdn.py | 6 ++-- python/cudnn/linear_attention/ops/gdn2.py | 16 +++++----- python/cudnn/linear_attention/ops/kda.py | 8 +++-- .../frost/examples/03_kda_prefill.py | 3 +- .../frost/examples/05_gdn2_prefill.py | 3 +- .../frost/examples/06_gdn2_backward.py | 2 +- test/python/linear_attention/test_la.py | 10 ++++-- 22 files changed, 79 insertions(+), 72 deletions(-) diff --git a/benchmark/linear_attention/benchmark_single_linear_attention.py b/benchmark/linear_attention/benchmark_single_linear_attention.py index f4926d9d6..15a761535 100644 --- a/benchmark/linear_attention/benchmark_single_linear_attention.py +++ b/benchmark/linear_attention/benchmark_single_linear_attention.py @@ -624,12 +624,12 @@ def flash_kda_linear_attention(query, key, value, gate, beta, write_gate, state0 try: from fla.ops.kda import chunk_kda except ImportError as e: - raise RuntimeError(f"The installed fla does not provide KDA (fla.ops.kda): {e}") + raise RuntimeError(f"The installed fla does not provide KDA (fla.ops.kda): {e}") from e else: # gdn2 try: from fla.ops.gdn2 import chunk_gdn2 except ImportError as e: - raise RuntimeError(f"The installed fla does not provide GDN2 (fla.ops.gdn2): {e}") + raise RuntimeError(f"The installed fla does not provide GDN2 (fla.ops.gdn2): {e}") from e ## FLA takes dense (B, T, H, D) tensors; g is the log-space decay ## (raw safe-gate logits in forward-only kda runs). --store_on adds diff --git a/benchmark/linear_attention/plot_results.py b/benchmark/linear_attention/plot_results.py index d8277e704..bed2c0df9 100644 --- a/benchmark/linear_attention/plot_results.py +++ b/benchmark/linear_attention/plot_results.py @@ -15,6 +15,7 @@ import argparse from pathlib import Path +from typing import List, Optional import pandas as pd import matplotlib @@ -27,11 +28,12 @@ BACKEND_CONFIG = { "fla": {"name": "FLA (Triton)", "color": "#FF8C00", "order": 0}, "flash_qla": {"name": "FlashQLA (TileLang)", "color": "#6495ED", "order": 1}, - "cudnn": {"name": "cuDNN", "color": "#76b900", "order": 2}, + "flash_kda": {"name": "FlashKDA", "color": "#9370DB", "order": 2}, + "cudnn": {"name": "cuDNN", "color": "#76b900", "order": 3}, } # Backends dropped from every chart (rows may still exist in older CSVs). -UNAVAILABLE_BACKENDS = ("flash_kda",) +UNAVAILABLE_BACKENDS = () LABEL_FONT_SIZE = 10 LEGEND_FONT_SIZE = 8 @@ -56,7 +58,7 @@ ] -def get_backend_display_name(backend: str, cudnn_version: str = None) -> str: +def get_backend_display_name(backend: str, cudnn_version: Optional[str] = None) -> str: base_name = BACKEND_CONFIG.get(backend, {}).get("name", backend) if backend == "cudnn" and cudnn_version: base_name = f"{base_name} {cudnn_version}" @@ -64,7 +66,13 @@ def get_backend_display_name(backend: str, cudnn_version: str = None) -> str: def generate_charts( - df: pd.DataFrame, output_dir: Path, gpu_name: str = "", cudnn_version: str = None, variant: str = "gdn", batch_sizes: list = None, x_axis: str = "seqlen" + df: pd.DataFrame, + output_dir: Path, + gpu_name: str = "", + cudnn_version: Optional[str] = None, + variant: str = "gdn", + batch_sizes: Optional[List[int]] = None, + x_axis: str = "seqlen", ) -> list: output_dir.mkdir(parents=True, exist_ok=True) @@ -89,7 +97,6 @@ def generate_charts( for group_val in sorted(df[group_col].unique()): sub = df[df[group_col] == group_val].copy() sub.sort_values([x_col, "backend_order"], inplace=True) - hue_order = list(sub.sort_values("backend_order")["backend_display"].drop_duplicates()) fwd_df = sub[sub["fwd_tflops"] > 0] bwd_df = sub[sub["bwd_tflops"] > 0] @@ -120,6 +127,7 @@ def generate_charts( ): if ax is None or pass_df.empty: continue + hue_order = list(pass_df.sort_values("backend_order")["backend_display"].drop_duplicates()) sns.barplot( data=pass_df, x=x_col, @@ -160,7 +168,9 @@ def main(): parser.add_argument("--cudnn-version", default=None, help="cuDNN backend version for the legend (e.g. 9.24.0)") parser.add_argument("--variant", default="gdn", help="Linear attention variant to plot") parser.add_argument("--batch-sizes", default=None, help="Comma-separated batch sizes to plot (default: all in the CSV)") - parser.add_argument("--x-axis", default="seqlen", choices=("seqlen", "batch"), help="Bar-group axis: seqlen (one chart per batch) or batch (one chart per seqlen)") + parser.add_argument( + "--x-axis", default="seqlen", choices=("seqlen", "batch"), help="Bar-group axis: seqlen (one chart per batch) or batch (one chart per seqlen)" + ) args = parser.parse_args() batch_sizes = [int(b) for b in args.batch_sizes.split(",")] if args.batch_sizes else None diff --git a/python/cudnn/_pygraph.py b/python/cudnn/_pygraph.py index 96af2a43b..af31d5271 100644 --- a/python/cudnn/_pygraph.py +++ b/python/cudnn/_pygraph.py @@ -2453,8 +2453,7 @@ def _linear_attention_state_checkpoints_dims(node): cu = node.inputs["cu_seqlens"].dim if node.inputs.get("cu_seqlens") is not None else None if not n or not q or not v or not cu: return None - b = int(cu[0]) - 1 - return [max((v[0] - b) // n, 1), max(q[1], v[1]), q[2], v[2]] + return [max(v[0] // n, 1), max(q[1], v[1]), q[2], v[2]] def _linear_attention_o_dims(node): diff --git a/python/cudnn/frost/buffers.py b/python/cudnn/frost/buffers.py index 416aea873..eef7c9f05 100644 --- a/python/cudnn/frost/buffers.py +++ b/python/cudnn/frost/buffers.py @@ -480,10 +480,7 @@ def cutedsl_state(): ``version`` is ``(distribution, version)`` or None: the public wheel is nvidia-cutlass-dsl, internal RCs ship as nvidia-cutlass-dsl-internal, and the two number themselves differently. Presence is what gates; the pair only - refines the message and feeds :func:`cutedsl_too_old`. The internal dist is - checked first: when both are installed the RC's packages shadow the public - wheel on sys.path, and a stale public version string must not veto a - machine that works. + refines the message and feeds :func:`cutedsl_too_old`. """ global _DSL_STATE if _DSL_STATE is None: @@ -495,7 +492,7 @@ def cutedsl_state(): except (ImportError, ValueError): installed = False version = None - for dist in ("nvidia-cutlass-dsl-internal", "nvidia-cutlass-dsl"): + for dist in ("nvidia-cutlass-dsl", "nvidia-cutlass-dsl-internal"): try: version = (dist, importlib.metadata.version(dist)) break diff --git a/python/cudnn/frost/tile_dsl/barrier.py b/python/cudnn/frost/tile_dsl/barrier.py index 64a7d1967..553313ffd 100644 --- a/python/cudnn/frost/tile_dsl/barrier.py +++ b/python/cudnn/frost/tile_dsl/barrier.py @@ -23,23 +23,14 @@ def start(cls, phase: int = 0): return cls(idx=cutlass.Int32(0), phase=cutlass.Int32(phase)) -@cute.jit -def advance(state, stages, step: int = 1): - if cutlass.const_expr(stages < 1): +def advance(state, stages): + if stages < 1: raise ValueError(f"PipelineState.advance requires stages >= 1, got {stages}") - incr = state.idx + cutlass.Int32(step) - if cutlass.const_expr(stages & (stages - 1) == 0 or step > stages): - # pow2 folds to mask+shift; >1-wrap steps need the divmod - stages_i = cutlass.Int32(stages) - new_idx = incr % stages_i - flip = (incr // stages_i) & cutlass.Int32(1) - new_phase = state.phase ^ flip - else: - # non-pow2, at most one wrap: compare-select beats the magic-multiply - wrapped = incr >= cutlass.Int32(stages) - new_idx = incr - cutlass.Int32(stages) if wrapped else incr - flip = cutlass.Int32(1) if wrapped else cutlass.Int32(0) - new_phase = state.phase ^ flip + incr = state.idx + cutlass.Int32(1) + stages_i = cutlass.Int32(stages) + new_idx = incr % stages_i + flip = incr // stages_i + new_phase = state.phase ^ flip return PipelineState(idx=new_idx, phase=new_phase) @@ -219,14 +210,8 @@ def arrive( arrive_expect_tx(self.smem_ptr, n_bytes, pred=pred) elif cutlass.const_expr(self.producer == int(Producer.MMA_COMMIT)): - # commit barriers take tcgen05 commits ONLY; mixing thread arrives - # onto a commit barrier is banned (split into a THREAD sibling) if cta_group is None: - raise TypeError( - "MBarrier(producer=MMA_COMMIT) only accepts the commit form " - "(.arrive(cta_group=...)). Thread arrivals must go to a " - "separate MBarrier(producer=THREAD) sibling; waiters wait both." - ) + raise TypeError("MBarrier(producer=MMA_COMMIT).arrive() requires " "cta_group= (Python compile-time int).") commit_mma(self.smem_ptr, mcast_mask, cta_group, pred=pred) else: diff --git a/python/cudnn/linear_attention/cutile/kda_engine.py b/python/cudnn/linear_attention/cutile/kda_engine.py index 1373be1e3..8ca2f5c00 100644 --- a/python/cudnn/linear_attention/cutile/kda_engine.py +++ b/python/cudnn/linear_attention/cutile/kda_engine.py @@ -265,9 +265,10 @@ def check_support(self, graph: "pygraph") -> None: ): if got not in (f32, None): raise NotImplementedError(f"KdaCuTileEngine: '{port}' must be fp32 (callers convert), got {got}") - glb = graph.nodes and list(graph.nodes)[0].params.get("gate_lower_bound") - if glb is not None and glb is not False and float(glb) >= 0: - raise NotImplementedError(f"KdaCuTileEngine: gate_lower_bound must be negative (log-gate floor), got {glb}") + node = next(iter(graph.nodes), None) + glb = node.params.get("gate_lower_bound") if node is not None else None + if glb is not None and glb is not False and not (-5.0 <= float(glb) < 0): + raise NotImplementedError(f"KdaCuTileEngine: gate_lower_bound must be in [-5, 0) (chunk_kda log-gate floor), got {glb}") if not facts.uniform_io: raise NotImplementedError("KdaCuTileEngine: q/k/v dtypes must match") if facts.io_dtype not in (cudnn.data_type.HALF, cudnn.data_type.BFLOAT16, None): diff --git a/python/cudnn/linear_attention/frost/__init__.py b/python/cudnn/linear_attention/frost/__init__.py index 85c2dd693..2ff16ba10 100644 --- a/python/cudnn/linear_attention/frost/__init__.py +++ b/python/cudnn/linear_attention/frost/__init__.py @@ -4,7 +4,8 @@ """cudnn.linear_attention.frost: the FROST linear-attention engines — Gated DeltaNet, Kimi Delta Attention, and Gated DeltaNet v2 on the SM100 chunked kernels built on Cutlass primitives. All three serve forward and -backward on SM100/SM103 and rank ahead of the cuTile fallbacks.""" +backward on SM100/SM103 and rank ahead of the cuTile fallbacks, except +GDN-2, which does not have a cuTile fallback.""" # Lazy: importing one family's engine must not drag its neighbours in. import importlib diff --git a/python/cudnn/linear_attention/frost/common/downcast.py b/python/cudnn/linear_attention/frost/common/downcast.py index 0451ff59c..f6b19a9a4 100644 --- a/python/cudnn/linear_attention/frost/common/downcast.py +++ b/python/cudnn/linear_attention/frost/common/downcast.py @@ -73,6 +73,8 @@ def downcast_state(initial_state, out, *, stream): n_seq, ho, k, v = (int(s_) for s_ in out.shape) if v % 8 != 0: raise ValueError(f"state V dim must be a multiple of 8 (8-element staging chunks); got {v}") + if v > 1024: + raise ValueError(f"state V dim must be <= 1024 (one 128-thread block stages a full row); got {v}") threads_per_row = v // 8 rows_per_cta = max(128 // threads_per_row, 1) n_blocks = (k + rows_per_cta - 1) // rows_per_cta diff --git a/python/cudnn/linear_attention/frost/common/head_reduce.py b/python/cudnn/linear_attention/frost/common/head_reduce.py index bae195548..d9078cc0b 100644 --- a/python/cudnn/linear_attention/frost/common/head_reduce.py +++ b/python/cudnn/linear_attention/frost/common/head_reduce.py @@ -138,6 +138,8 @@ def head_group_reduce(src, dst, *, stream) -> None: dst_strides = tuple(reversed(dst_strides)) if not is_fp32 and any(st % 2 != 0 for st, sz in zip(dst_strides[:-1], dst.shape[:-1]) if sz != 1): raise ValueError(f"head_group_reduce: f16/bf16 dst outer strides must be even (word-pair stores), got {dst_strides}") + if dst.shape[-1] != 1 and dst_strides[-1] != 1: + raise ValueError(f"head_group_reduce: dst innermost dim must be stride-1, got strides {dst_strides}") out_row_words = dst_strides[0] if is_fp32 else dst_strides[0] // 2 out_head_words = (dst_strides[1] if is_fp32 else dst_strides[1] // 2) if len(dst.shape) == 3 else 1 diff --git a/python/cudnn/linear_attention/frost/gdn2_engine.py b/python/cudnn/linear_attention/frost/gdn2_engine.py index b10a05333..3f05efeee 100644 --- a/python/cudnn/linear_attention/frost/gdn2_engine.py +++ b/python/cudnn/linear_attention/frost/gdn2_engine.py @@ -279,7 +279,7 @@ def __init__(self, node, bwd_mod, regen_mod): # chunk-0 entering state, io dtype (downcast initial_state or zeros) self.off_state0_io = layout.add(B * HO * K * V * 2) if self.has_state0 else None if not self.has_state_checkpoints: - self.state_checkpoints_rows = max(total // self.b_t, 1) + self.state_checkpoints_rows = max(total // self.b_t + B, 1) self.off_state_checkpoints = layout.add(self.state_checkpoints_rows * HO * K * V * 2) self.regen_tm_bytes = tensormap_workspace_bytes(regen_mod, B) self.off_regen_tensormaps = layout.add(self.regen_tm_bytes, align=128) diff --git a/python/cudnn/linear_attention/frost/kda_engine.py b/python/cudnn/linear_attention/frost/kda_engine.py index e6405e1f7..52298aa90 100644 --- a/python/cudnn/linear_attention/frost/kda_engine.py +++ b/python/cudnn/linear_attention/frost/kda_engine.py @@ -235,10 +235,10 @@ def __call__(self, node_buffers, *, workspace=None, stream=None) -> Any: class CompiledKdaBwd: """Compiled FROST KDA backward plan. When the graph carries no ``state_checkpoints`` - input, the workspace holds a regenerated per-chunk checkpoint series (entry-0 - initial-state slots seeded on device, then the recompute kernel re-run - with shifted checkpoint bounds). GVA/GQA gradients land in HO-head - scratch and are reduced back to the native head counts.""" + input, one recompute pass over the forward inputs regenerates the per-chunk + checkpoint series into workspace (checkpoint stride ``b_t``). GVA/GQA + gradients land in HO-head scratch and are reduced back to the native head + counts.""" def __init__(self, node, bwd_mod, regen_mod): from .common.split_k import WORK_ITEM_FIELDS, chunk_scratch_rows, compute_ideal_chunks, max_work_items @@ -282,7 +282,7 @@ def __init__(self, node, bwd_mod, regen_mod): # chunk-0 entering state, io dtype (downcast initial_state; absent = in-kernel zeros) self.off_state0_io = layout.add(B * HO * K * V * 2) if self.has_state0 else None if not self.has_state_checkpoints: - self.state_checkpoints_rows = max(total // self.b_t, 1) + self.state_checkpoints_rows = max(total // self.b_t + B, 1) self.off_state_checkpoints = layout.add(self.state_checkpoints_rows * HO * K * V * 2) self.regen_tm_bytes = tensormap_workspace_bytes(regen_mod, B) self.off_regen_tensormaps = layout.add(self.regen_tm_bytes, align=128) diff --git a/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py index 53d79d5f5..2e32c26b3 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py @@ -2564,10 +2564,10 @@ def compute2_warp_group( # ---- dGlast hdot: sum_v sdH[v, c] * S0[c, v] ------------------------- dgl_val = cutlass.Float32(0.0) + bars.mb_state_inp_ready[gc % 2].wait((gc // 2) % 2) if has_dstate: bars.mb_dstate_smem_ready.wait(dgl_dstate_smem_index.phase) dgl_dstate_smem_index = advance(dgl_dstate_smem_index, 1) - bars.mb_state_inp_ready[gc % 2].wait((gc // 2) % 2) for pl in cutlass.range_constexpr(2): for row_half in cutlass.range_constexpr(2): state_words = nvvm.tcgen05_ld( diff --git a/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py index 6df986764..4ab6cbead 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py @@ -4581,7 +4581,6 @@ def chunk_gdn_bwd_sm100( ) if "compiled" not in cache: - cu_seqlens_cute = from_dlpack(cu_seqlens, assumed_align=4).mark_layout_dynamic() workspace_cute = from_dlpack(workspace, assumed_align=128).mark_layout_dynamic() diff --git a/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py b/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py index 0100f93db..916ee6385 100644 --- a/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py @@ -2509,16 +2509,17 @@ def compute2_warp_group( # ---- dGlast hdot: sum_v sdH[v, c] * S0[c, v] ------------------------- dgl_val = cutlass.Float32(0.0) + bars.mb_state_inp_ready[chunk_serial % 2].wait((chunk_serial // 2) % 2) if has_dstate: bars.mb_dstate_smem_ready.wait(dgl_dstate_smem_index.phase) dgl_dstate_smem_index = advance(dgl_dstate_smem_index, 1) - bars.mb_state_inp_ready[chunk_serial % 2].wait((chunk_serial // 2) % 2) for pl in cutlass.range_constexpr(2): for row_half in cutlass.range_constexpr(2): state_words = nvvm.tcgen05_ld( "32x32b", nvvm.make_tmem_ptr( - row_addr + (tmem_col + cfg.tmem_state_inp_offset + (chunk_serial % 2) * (cfg.d_v // 2) + pl * 32 + row_half * 16), cutlass.Float32 + row_addr + (tmem_col + cfg.tmem_state_inp_offset + (chunk_serial % 2) * (cfg.d_v // 2) + pl * 32 + row_half * 16), + cutlass.Float32, ), num=16, ) diff --git a/python/cudnn/linear_attention/graph_analyzer.py b/python/cudnn/linear_attention/graph_analyzer.py index b8a359d29..63765275e 100644 --- a/python/cudnn/linear_attention/graph_analyzer.py +++ b/python/cudnn/linear_attention/graph_analyzer.py @@ -171,7 +171,7 @@ def analyze(graph: "cudnn.pygraph") -> Optional[LaGraphFacts]: else: h_q = h_k = h_v = d_qk = d_v = 0 h_o = max(h_q, h_v) - gates_at_ho = all(t is None or not t.dim or int(t.dim[1]) == h_o for t in (ins["g"], ins["beta"], ins.get("w"))) + gates_at_ho = all(t is None or not t.dim or (len(t.dim) > 1 and int(t.dim[1]) == h_o) for t in (ins["g"], ins["beta"], ins.get("w"))) io_dtypes = {in_dt["q"], in_dt["k"], in_dt["v"]} - {None} state_dtypes = {in_dt.get("initial_state"), out_dt.get("final_state")} - {None} scale = params.get("scale") diff --git a/python/cudnn/linear_attention/ops/gdn.py b/python/cudnn/linear_attention/ops/gdn.py index 7e0d5d5cb..37f96b995 100644 --- a/python/cudnn/linear_attention/ops/gdn.py +++ b/python/cudnn/linear_attention/ops/gdn.py @@ -317,9 +317,7 @@ def _gdn_fwd( variant_pack[t["fs"]] = final_state state_checkpoints = torch.empty(0, dtype=q.dtype, device=device) if ckpt > 0: - # shape-derived upper bound (sum floor((len_i-1)/ckpt) <= (total-N)//ckpt); - # the exact count is data-dependent (cu contents) — no sync on the hot path - total_checkpoints = max((total - N) // ckpt, 1) + total_checkpoints = max(total // ckpt, 1) state_checkpoints = torch.empty(total_checkpoints, HO, K, V, dtype=q.dtype, device=device) variant_pack[t["state_checkpoints"]] = state_checkpoints graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_get_handle(device)) @@ -340,7 +338,7 @@ def _gdn_fwd_fake( o = q.new_empty(total, HO, V) final = q.new_empty((N, HO, K, V) if output_final_state else (0,), dtype=torch.float32) if checkpoint_every_n_tokens > 0: - total_checkpoints = max((total - N) // int(checkpoint_every_n_tokens), 1) + total_checkpoints = max(total // int(checkpoint_every_n_tokens), 1) state_checkpoints = q.new_empty(total_checkpoints, HO, K, V) else: state_checkpoints = q.new_empty(0) diff --git a/python/cudnn/linear_attention/ops/gdn2.py b/python/cudnn/linear_attention/ops/gdn2.py index e258bb00c..540f8573b 100644 --- a/python/cudnn/linear_attention/ops/gdn2.py +++ b/python/cudnn/linear_attention/ops/gdn2.py @@ -276,7 +276,7 @@ def _gdn2_fwd( N = cu_seqlens.shape[0] - 1 device = q.device if cu_seqlens.dtype not in (torch.int32, torch.int64): - raise ValueError(f"gated_delta_net_2: cu_seqlens must be int32 or int64; got {cu_seqlens.dtype}") + raise ValueError(f"gated_delta_net_v2: cu_seqlens must be int32 or int64; got {cu_seqlens.dtype}") cu = cu_seqlens _check_dtype("g", g, torch.float32) _check_dtype("beta", beta, q.dtype) @@ -307,7 +307,7 @@ def _gdn2_fwd( ("dt_bias", dt_bias), ): if _t is not None and _t.device != device: - raise ValueError(f"gated_delta_net_2: {_name} must be on q's device ({device}); got {_t.device}") + raise ValueError(f"gated_delta_net_v2: {_name} must be on q's device ({device}); got {_t.device}") state0 = initial_state if initial_state is not None else None ckpt = int(checkpoint_every_n_tokens) @@ -379,8 +379,7 @@ def _gdn2_fwd( variant_pack[t["fs"]] = final_state state_checkpoints = torch.empty(0, dtype=q.dtype, device=device) if ckpt > 0: - # shape-derived upper bound; exact count is data-dependent (cu contents) - total_checkpoints = max((total - N) // ckpt, 1) + total_checkpoints = max(total // ckpt, 1) state_checkpoints = torch.empty(total_checkpoints, HO, K, V, dtype=q.dtype, device=device) variant_pack[t["state_checkpoints"]] = state_checkpoints graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_get_handle(device)) @@ -407,13 +406,16 @@ def _gdn2_fwd_fake( checkpoint_every_n_tokens=0, ): total, H, K = q.shape + HK = k.shape[1] HV, V = v.shape[1], v.shape[2] + if HK not in (H, HV): + raise ValueError(f"k head count ({HK}) must match q's ({H}) or v's ({HV}); canonical GQA shares grouped k/v heads") HO = max(H, HV) N = cu_seqlens.shape[0] - 1 o = q.new_empty(total, HO, V) final = q.new_empty((N, HO, K, V) if output_final_state else (0,), dtype=torch.float32) if checkpoint_every_n_tokens > 0: - total_checkpoints = max((total - N) // int(checkpoint_every_n_tokens), 1) + total_checkpoints = max(total // int(checkpoint_every_n_tokens), 1) state_checkpoints = q.new_empty(total_checkpoints, HO, K, V) else: state_checkpoints = q.new_empty(0) @@ -519,7 +521,7 @@ def _gdn2_bwd( N = cu_seqlens.shape[0] - 1 device = q.device if cu_seqlens.dtype not in (torch.int32, torch.int64): - raise ValueError(f"gated_delta_net_2: cu_seqlens must be int32 or int64; got {cu_seqlens.dtype}") + raise ValueError(f"gated_delta_net_v2: cu_seqlens must be int32 or int64; got {cu_seqlens.dtype}") cu = cu_seqlens _check_dtype("g", g, torch.float32) _check_dtype("beta", beta, q.dtype) @@ -532,7 +534,7 @@ def _gdn2_bwd( _check_dtype("d_final_state", d_final_state, torch.float32) for _name, _t in (("k", k), ("v", v), ("g", g), ("beta", beta), ("w", w), ("cu_seqlens", cu_seqlens), ("dO", dO), ("d_final_state", d_final_state)): if _t is not None and _t.device != device: - raise ValueError(f"gated_delta_net_2: {_name} must be on q's device ({device}); got {_t.device}") + raise ValueError(f"gated_delta_net_v2: {_name} must be on q's device ({device}); got {_t.device}") state0 = initial_state if initial_state is not None else None dstate_in = d_final_state if d_final_state is not None else None diff --git a/python/cudnn/linear_attention/ops/kda.py b/python/cudnn/linear_attention/ops/kda.py index 98c2c596d..7c39add0c 100644 --- a/python/cudnn/linear_attention/ops/kda.py +++ b/python/cudnn/linear_attention/ops/kda.py @@ -421,8 +421,7 @@ def _kda_fwd( variant_pack[t["fs"]] = final_state state_checkpoints = torch.empty(0, dtype=q.dtype, device=device) if ckpt > 0: - # shape-derived upper bound; exact count is data-dependent (cu contents) - total_checkpoints = max((total - N) // ckpt, 1) + total_checkpoints = max(total // ckpt, 1) state_checkpoints = torch.empty(total_checkpoints, HO, K, V, dtype=q.dtype, device=device) variant_pack[t["state_checkpoints"]] = state_checkpoints graph.execute(variant_pack, workspace=_graph_workspace(graph, device), handle=_get_handle(device)) @@ -449,13 +448,16 @@ def _kda_fwd_fake( checkpoint_every_n_tokens=0, ): total, H, K = q.shape + HK = k.shape[1] HV, V = v.shape[1], v.shape[2] + if HK not in (H, HV): + raise ValueError(f"k head count ({HK}) must match q's ({H}) or v's ({HV}); canonical GQA shares grouped k/v heads") HO = max(H, HV) N = cu_seqlens.shape[0] - 1 o = q.new_empty(total, HO, V) final = q.new_empty((N, HO, K, V) if output_final_state else (0,), dtype=torch.float32) if checkpoint_every_n_tokens > 0: - total_checkpoints = max((total - N) // int(checkpoint_every_n_tokens), 1) + total_checkpoints = max(total // int(checkpoint_every_n_tokens), 1) state_checkpoints = q.new_empty(total_checkpoints, HO, K, V) else: state_checkpoints = q.new_empty(0) diff --git a/test/python/linear_attention/frost/examples/03_kda_prefill.py b/test/python/linear_attention/frost/examples/03_kda_prefill.py index 5bc6a698c..393870680 100644 --- a/test/python/linear_attention/frost/examples/03_kda_prefill.py +++ b/test/python/linear_attention/frost/examples/03_kda_prefill.py @@ -105,7 +105,8 @@ def main(seq_lens=(192, 320), H: int = 2, D: int = 128) -> None: torch.cuda.synchronize() o_ref, fs_ref = _reference(q, k, v, gate, beta, cu, scale) - torch.testing.assert_close(o.float(), o_ref.float(), atol=1e-1, rtol=1e-1) + r_o = _rms_ratio(o, o_ref) + assert r_o < 5e-2, f"o rms ratio {r_o:.4g}" r_s = _rms_ratio(fs, fs_ref) assert r_s < 5e-2, f"final_state rms ratio {r_s:.4g}" print(f"[03] PASS kda prefill seq_lens={list(seq_lens)} H={H} D={D} (fs rms {r_s:.2e})") diff --git a/test/python/linear_attention/frost/examples/05_gdn2_prefill.py b/test/python/linear_attention/frost/examples/05_gdn2_prefill.py index 7a370c792..19da0445b 100644 --- a/test/python/linear_attention/frost/examples/05_gdn2_prefill.py +++ b/test/python/linear_attention/frost/examples/05_gdn2_prefill.py @@ -109,7 +109,8 @@ def main(seq_lens=(192, 320), H: int = 2, D: int = 128) -> None: torch.cuda.synchronize() o_ref, fs_ref = _reference(q, k, v, gate, beta, w, cu, scale) - torch.testing.assert_close(o.float(), o_ref.float(), atol=1e-1, rtol=1e-1) + r_o = _rms_ratio(o, o_ref) + assert r_o < 5e-2, f"o rms ratio {r_o:.4g}" r_s = _rms_ratio(fs, fs_ref) assert r_s < 5e-2, f"final_state rms ratio {r_s:.4g}" print(f"[05] PASS gdn2 prefill seq_lens={list(seq_lens)} H={H} D={D} (fs rms {r_s:.2e})") diff --git a/test/python/linear_attention/frost/examples/06_gdn2_backward.py b/test/python/linear_attention/frost/examples/06_gdn2_backward.py index fffb7b11d..8bfb6a7d4 100644 --- a/test/python/linear_attention/frost/examples/06_gdn2_backward.py +++ b/test/python/linear_attention/frost/examples/06_gdn2_backward.py @@ -59,7 +59,7 @@ def main(seq_lens=(192, 320), H: int = 2, D: int = 128) -> None: k = torch.nn.functional.normalize(torch.randn(total, H, D, device=device), dim=-1).bfloat16() v = torch.randn(total, H, D, device=device).bfloat16() gate = torch.empty(total, H, D, device=device).uniform_(0.5, 1.0).log().contiguous() - beta = torch.rand(total, H, D, device=device).sigmoid().bfloat16().contiguous() + beta = (torch.rand(total, H, D, device=device).sigmoid() * 2.0).bfloat16().contiguous() w = torch.rand(total, H, D, device=device).sigmoid().bfloat16().contiguous() do = torch.randn(total, H, D, device=device).bfloat16() cu = torch.tensor([0, *torch.tensor(seq_lens).cumsum(0).tolist()], dtype=torch.int32, device=device) diff --git a/test/python/linear_attention/test_la.py b/test/python/linear_attention/test_la.py index b9e5c5c3a..e1bd782d4 100644 --- a/test/python/linear_attention/test_la.py +++ b/test/python/linear_attention/test_la.py @@ -701,7 +701,7 @@ def test_safe_gate_forward_parity(backend, variant): eff_case = SimpleNamespace(**{**vars(case), "gates": dict(case.gates, g=lb * torch.sigmoid(graw), beta=eff_beta)}) o_raw, fs_raw = _run_fwd(backend, raw_case, **raw_kw) o_eff, fs_eff = _run_fwd(backend, eff_case, **kw) - torch.testing.assert_close(o_raw.float(), o_eff.float(), atol=2.5e-2, rtol=2.5e-2) + _check("o", o_raw, o_eff.double(), 2e-2) assert rms_ratio(fs_raw, fs_eff) < 2e-2 @@ -736,7 +736,7 @@ def test_beta_sigmoid_in_kernel(backend): eff_case = SimpleNamespace(**{**vars(case), "gates": dict(case.gates, beta=braw.float().sigmoid())}) o_raw, fs_raw = _run_fwd(backend, raw_case, output_final_state=True, use_beta_sigmoid_in_kernel=True) o_eff, fs_eff = _run_fwd(backend, eff_case, output_final_state=True) - torch.testing.assert_close(o_raw.float(), o_eff.float(), atol=2.5e-2, rtol=2.5e-2) + _check("o", o_raw, o_eff.double(), 2e-2) assert rms_ratio(fs_raw, fs_eff) < 2e-2 @@ -901,6 +901,12 @@ def launch(): with _waive_unsupported(backend, variant): eager = launch() + warmup = torch.cuda.Stream() + warmup.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(warmup): + for _ in range(3): + launch() + torch.cuda.current_stream().wait_stream(warmup) torch.cuda.synchronize() graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): From 075ddef682a607bbc00cdc01c09b9c2ec61e6604 Mon Sep 17 00:00:00 2001 From: Joshua Park Date: Tue, 11 Aug 2026 23:41:55 -0700 Subject: [PATCH 3/9] 3 --- .../frost/kernel/gdn2_bprop_f16.py | 514 +++++++++-------- .../frost/kernel/gdn2_prefill_f16.py | 408 +++++++------ .../frost/kernel/gdn2_recompute_f16.py | 344 ++++++----- .../frost/kernel/gdn_bprop_f16.py | 407 ++++++------- .../frost/kernel/gdn_prefill_f16.py | 173 +++--- .../frost/kernel/gdn_recompute_f16.py | 122 ++-- .../frost/kernel/kda_bprop_f16.py | 502 ++++++++-------- .../frost/kernel/kda_prefill_f16.py | 538 +++++++++--------- .../frost/kernel/kda_recompute_f16.py | 304 +++++----- python/cudnn/linear_attention/ops/gdn.py | 73 ++- python/cudnn/linear_attention/ops/gdn2.py | 60 +- python/cudnn/linear_attention/ops/kda.py | 40 +- test/python/linear_attention/reference_gdn.py | 31 +- .../python/linear_attention/reference_gdn2.py | 35 +- test/python/linear_attention/reference_kda.py | 31 +- test/python/linear_attention/test_la.py | 498 ++++++++-------- 16 files changed, 2092 insertions(+), 1988 deletions(-) diff --git a/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py index 2e32c26b3..4ec599f8b 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn2_bprop_f16.py @@ -125,10 +125,10 @@ class Gdn2BwdBars(NamedTuple): mb_k_decay_inv_ready: MBarrier mb_q_decay_k_restore_ready: MBarrier - mb_decay_done: MBarrier + mb_decay_tcgen05_done: MBarrier mb_decay_super_done: MBarrier - mb_a_ready: MBarrier + mb_a_inv_ready: MBarrier mb_aqk_ready: MBarrier mb_da_ready: MBarrier mb_dm_ready: MBarrier @@ -161,10 +161,10 @@ class Gdn2BwdBars(NamedTuple): mb_dstate_smem_cg2_done: MBarrier mb_dq_acc_ready: MBarrier - mb_dk_decay_part_ready: MBarrier - mb_dk_inv_part_ready: MBarrier - mb_dk_restore_part_ready: MBarrier - mb_parts_done: MBarrier + mb_dk_decay_part_acc_ready: MBarrier + mb_dk_inv_part_acc_ready: MBarrier + mb_dk_restore_part_acc_ready: MBarrier + mb_parts_acc_done: MBarrier mb_dq_tmastg_ready: MBarrier mb_dq_tmastg_done: MBarrier @@ -178,7 +178,8 @@ class Gdn2BwdBars(NamedTuple): mb_db_tmastg_done: MBarrier mb_dwo_tmastg_ready: MBarrier mb_dwo_tmastg_done: MBarrier - mb_dstate0_stored: MBarrier + mb_dstate0_acc_stored: MBarrier + mb_tmem_done: MBarrier mb_sched_ready: MBarrier mb_sched_done: MBarrier @@ -221,9 +222,9 @@ def alloc(n): mb_state_inp_cg2_done=MBarrier(alloc(2), stages=2, init_count=CG2, producer=Producer.THREAD), mb_k_decay_inv_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0, producer=Producer.THREAD), mb_q_decay_k_restore_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0, producer=Producer.THREAD), - mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_decay_tcgen05_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=MMA, producer=Producer.MMA_COMMIT), mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=2 * WARP, producer=Producer.THREAD), - mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_a_inv_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), mb_aqk_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), mb_da_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), mb_dm_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), @@ -254,10 +255,10 @@ def alloc(n): mb_dstate_smem_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), mb_dstate_smem_cg2_done=MBarrier(alloc(1), stages=1, init_count=CG2, producer=Producer.THREAD), mb_dq_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), - mb_dk_decay_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), - mb_dk_inv_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), - mb_dk_restore_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), - mb_parts_done=MBarrier(alloc(1), stages=1, init_count=CG2, producer=Producer.THREAD), + mb_dk_decay_part_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dk_inv_part_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dk_restore_part_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_parts_acc_done=MBarrier(alloc(1), stages=1, init_count=CG2, producer=Producer.THREAD), mb_dq_tmastg_ready=MBarrier(alloc(cfg.smem_dq_stages), stages=cfg.smem_dq_stages, init_count=CG2, producer=Producer.THREAD), mb_dq_tmastg_done=MBarrier(alloc(cfg.smem_dq_stages), stages=cfg.smem_dq_stages, init_count=WARP, producer=Producer.THREAD), mb_dk_tmastg_ready=MBarrier(alloc(cfg.smem_dk_stages), stages=cfg.smem_dk_stages, init_count=CG2, producer=Producer.THREAD), @@ -270,7 +271,8 @@ def alloc(n): mb_db_tmastg_done=MBarrier(alloc(cfg.smem_db_stages), stages=cfg.smem_db_stages, init_count=WARP, producer=Producer.THREAD), mb_dwo_tmastg_ready=MBarrier(alloc(cfg.smem_dwo_stages), stages=cfg.smem_dwo_stages, init_count=CG1, producer=Producer.THREAD), mb_dwo_tmastg_done=MBarrier(alloc(cfg.smem_dwo_stages), stages=cfg.smem_dwo_stages, init_count=WARP, producer=Producer.THREAD), - mb_dstate0_stored=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_dstate0_acc_stored=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_tmem_done=MBarrier(alloc(1), stages=1, init_count=CG1 + CG2, producer=Producer.THREAD), mb_sched_ready=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=1, producer=Producer.THREAD), mb_sched_done=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=15, producer=Producer.THREAD), ) @@ -341,6 +343,8 @@ def epilogue_warp( ) -> None: """Epilogue warp role (warp 15): register-MMA A_qk/dA tiles and the gradient TMA stores.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) # ---- ldmatrix/stmatrix lane decode ------------------------------------------- @@ -448,7 +452,7 @@ def epilogue_warp( desc_dg_slot = (desc_dg_base + slot).tospace(cutlass.AddressSpace.generic) desc_db_slot = (desc_db_base + slot).tospace(cutlass.AddressSpace.generic) desc_dwo_slot = (desc_dwo_base + slot).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): + if elect_one: tma_tensormap_acquire(desc_dq_slot) tma_tensormap_acquire(desc_dk_slot) tma_tensormap_acquire(desc_dv_slot) @@ -480,22 +484,22 @@ def epilogue_warp( for k_block in cutlass.range_constexpr(cfg.d_k // 16): a_col = k_block * 16 + lhs_col_offset a_seg = a_col // 64 - a_vec = nvvm.ldmatrix( + a_frag = nvvm.ldmatrix( sQ_decay_ptr + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) b_col = k_block * 16 + rhs_col_offset b_seg = b_col // 64 - b_vec = nvvm.ldmatrix( + b_frag = nvvm.ldmatrix( sK_inv_ptr + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) mma_step( qk_acc, - (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), - (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + (a_frag[0], a_frag[1], a_frag[2], a_frag[3]), + (b_frag[0], b_frag[1], b_frag[2], b_frag[3]), k_step=0, M=16, N=16, @@ -532,22 +536,22 @@ def epilogue_warp( for k_block in cutlass.range_constexpr(cfg.d_v // 16): a_col = k_block * 16 + lhs_col_offset a_seg = a_col // 64 - a_vec = nvvm.ldmatrix( + a_frag = nvvm.ldmatrix( sDo_ptr + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) b_col = k_block * 16 + rhs_col_offset b_seg = b_col // 64 - b_vec = nvvm.ldmatrix( + b_frag = nvvm.ldmatrix( sU_raw.data_ptr() + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) mma_step( da_acc, - (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), - (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + (a_frag[0], a_frag[1], a_frag[2], a_frag[3]), + (b_frag[0], b_frag[1], b_frag[2], b_frag[3]), k_step=0, M=16, N=16, @@ -742,22 +746,22 @@ def super_mma_warp( for k_block in cutlass.range_constexpr(cfg.d_k // 16): a_col = k_block * 16 + lhs_col_offset a_seg = a_col // 64 - a_vec = nvvm.ldmatrix( + a_frag = nvvm.ldmatrix( sK_decay_ptr + a_seg * (cfg.b_t * 64) + kk_lhs_row * 64 + swizzle_xor_128b(kk_lhs_row, a_col - a_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) b_col = k_block * 16 + rhs_col_offset b_seg = b_col // 64 - b_vec = nvvm.ldmatrix( + b_frag = nvvm.ldmatrix( sK_inv_ptr + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) mma_step( kk_acc, - (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), - (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + (a_frag[0], a_frag[1], a_frag[2], a_frag[3]), + (b_frag[0], b_frag[1], b_frag[2], b_frag[3]), k_step=0, M=16, N=16, @@ -765,18 +769,18 @@ def super_mma_warp( ) # ---- L = tril(KK, -1) ------------------------------------------------ - l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) + l_regs = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) if cutlass.const_expr(accum_idx % 2 == 1): col_coord = col_coord + cutlass.Int32(1) lower = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) - l_frag[accum_idx] = lower - l_a0 = fp32_to_fp16(l_frag[0], l_frag[1], dtype=cfg.io_dtype) - l_a1 = fp32_to_fp16(l_frag[2], l_frag[3], dtype=cfg.io_dtype) - l_a2 = fp32_to_fp16(l_frag[4], l_frag[5], dtype=cfg.io_dtype) - l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) + l_regs[accum_idx] = lower + l_a0 = fp32_to_fp16(l_regs[0], l_regs[1], dtype=cfg.io_dtype) + l_a1 = fp32_to_fp16(l_regs[2], l_regs[3], dtype=cfg.io_dtype) + l_a2 = fp32_to_fp16(l_regs[4], l_regs[5], dtype=cfg.io_dtype) + l_a3 = fp32_to_fp16(l_regs[6], l_regs[7], dtype=cfg.io_dtype) l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) ainv_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) @@ -849,7 +853,7 @@ def super_mma_warp( shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_a_ready[qk_stage].arrive() + bars.mb_a_inv_ready[qk_stage].arrive() bars.mb_decay_super_done[decay_stage].arrive() # ---- dM = dY @ U^T --------------------------------------------------- @@ -862,22 +866,22 @@ def super_mma_warp( for k_block in cutlass.range_constexpr(cfg.d_v // 16): a_col = k_block * 16 + lhs_col_offset a_seg = a_col // 64 - a_vec = nvvm.ldmatrix( + a_frag = nvvm.ldmatrix( sDy_raw.data_ptr() + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) b_col = k_block * 16 + rhs_col_offset b_seg = b_col // 64 - b_vec = nvvm.ldmatrix( + b_frag = nvvm.ldmatrix( sU_raw.data_ptr() + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) mma_step( dm_acc, - (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), - (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + (a_frag[0], a_frag[1], a_frag[2], a_frag[3]), + (b_frag[0], b_frag[1], b_frag[2], b_frag[3]), k_step=0, M=16, N=16, @@ -886,22 +890,22 @@ def super_mma_warp( bars.mb_u_warps_done.arrive() bars.mb_sdy_super_done.arrive() - dm_strict = cutlass.Array(cutlass.Float32, 8, alignment=16) + dm_strict_regs = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) if cutlass.const_expr(accum_idx % 2 == 1): col_coord = col_coord + cutlass.Int32(1) - dm_strict[accum_idx] = dm_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) - w0 = fp32_to_fp16(dm_strict[0], dm_strict[1], dtype=cfg.io_dtype) - w1 = fp32_to_fp16(dm_strict[2], dm_strict[3], dtype=cfg.io_dtype) - w2 = fp32_to_fp16(dm_strict[4], dm_strict[5], dtype=cfg.io_dtype) - w3 = fp32_to_fp16(dm_strict[6], dm_strict[7], dtype=cfg.io_dtype) + dm_strict_regs[accum_idx] = dm_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + w0 = fp32_to_fp16(dm_strict_regs[0], dm_strict_regs[1], dtype=cfg.io_dtype) + w1 = fp32_to_fp16(dm_strict_regs[2], dm_strict_regs[3], dtype=cfg.io_dtype) + w2 = fp32_to_fp16(dm_strict_regs[4], dm_strict_regs[5], dtype=cfg.io_dtype) + w3 = fp32_to_fp16(dm_strict_regs[6], dm_strict_regs[7], dtype=cfg.io_dtype) nvvm.stmatrix(sQk_ptr + 3 * (cfg.b_t * cfg.b_t) + stsm_idx, [w0, w1, w2, w3], nvvm.MMALayout.ROW, shape=nvvm.StoreShape.M8N8) - nw0 = fp32_to_fp16(-dm_strict[0], -dm_strict[1], dtype=cfg.io_dtype) - nw1 = fp32_to_fp16(-dm_strict[2], -dm_strict[3], dtype=cfg.io_dtype) - nw2 = fp32_to_fp16(-dm_strict[4], -dm_strict[5], dtype=cfg.io_dtype) - nw3 = fp32_to_fp16(-dm_strict[6], -dm_strict[7], dtype=cfg.io_dtype) + nw0 = fp32_to_fp16(-dm_strict_regs[0], -dm_strict_regs[1], dtype=cfg.io_dtype) + nw1 = fp32_to_fp16(-dm_strict_regs[2], -dm_strict_regs[3], dtype=cfg.io_dtype) + nw2 = fp32_to_fp16(-dm_strict_regs[4], -dm_strict_regs[5], dtype=cfg.io_dtype) + nw3 = fp32_to_fp16(-dm_strict_regs[6], -dm_strict_regs[7], dtype=cfg.io_dtype) nvvm.stmatrix(sQk_ptr + 4 * (cfg.b_t * cfg.b_t) + stsm_idx, [nw0, nw1, nw2, nw3], nvvm.MMALayout.ROW, shape=nvvm.StoreShape.M8N8) nvvm.fence_proxy("async.shared", space="cta") bars.mb_dm_ready[qk_stage].arrive() @@ -938,7 +942,11 @@ def tcgen05_mma_warp( ) -> None: """tcgen05-MMA warp role (warp 13): issues every tcgen05 GEMM and owns the TMEM lifecycle.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + nvvm.tcgen05_alloc(tmem_hold, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_hold.load() bpe = cfg.io_dtype.width // 8 @@ -1260,13 +1268,13 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_state_k_acc_offset), cutlass.Float32), accumulate=False, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_state_k_acc_ready.arrive(cta_group=1) bars.mb_state_done[state_index.idx].arrive(cta_group=1) state_index = advance(state_index, cfg.smem_state_stages) # ---- dQ inter = state(T) @ dO^T -------------------------------------- - bars.mb_parts_done.wait(parts_done_index.phase) + bars.mb_parts_acc_done.wait(parts_done_index.phase) parts_done_index = advance(parts_done_index, 1) bars.mb_state_inp_ready[gc % 2].wait((gc // 2) % 2) bars.mb_do_ready[raw_stage_idx].wait((gc // cfg.smem_raw_stages) % 2) @@ -1312,7 +1320,7 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_du_acc_offset), cutlass.Float32), accumulate=has_dstate, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_du_acc_ready.arrive(cta_group=1) bars.mb_aqk_done[qk_stage].arrive(cta_group=1) @@ -1335,7 +1343,7 @@ def tcgen05_mma_warp( k, cutlass.Boolean(sub + k > 0), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dstate_inp_done.arrive(cta_group=1) dstate_inp_index = advance(dstate_inp_index, 1) @@ -1349,7 +1357,7 @@ def tcgen05_mma_warp( ) # ---- U = rhs(T) @ A_inv ---------------------------------------------- - bars.mb_a_ready[qk_stage].wait(qk_phase) + bars.mb_a_inv_ready[qk_stage].wait(qk_phase) bars.mb_rhs_ready.wait(rhs_index.phase) rhs_index = advance(rhs_index, 1) a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_rhs_inp_offset), cutlass.Int8) @@ -1365,7 +1373,7 @@ def tcgen05_mma_warp( k, cutlass.Boolean(sub + k > 0), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_update_acc_ready.arrive(cta_group=1) bars.mb_do_done[raw_stage_idx].arrive(cta_group=1) @@ -1385,7 +1393,7 @@ def tcgen05_mma_warp( k, cutlass.Boolean(sub + k > 0), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dy_acc_ready.arrive(cta_group=1) bars.mb_du_inp_done.arrive(cta_group=1) bars.mb_a_done[qk_stage].arrive(cta_group=1) @@ -1403,10 +1411,10 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_restore_acc_offset), cutlass.Float32), accumulate=False, ) - if nvvm.elect_sync(): - bars.mb_dk_restore_part_ready.arrive(cta_group=1) + if elect_one: + bars.mb_dk_restore_part_acc_ready.arrive(cta_group=1) bars.mb_dstate_smem_done.arrive(cta_group=1) - if nvvm.elect_sync(): + if elect_one: bars.mb_u_done.arrive(cta_group=1) # ---- dstate k-term += -dY(T) @ K_decay ------------------------------- @@ -1425,7 +1433,7 @@ def tcgen05_mma_warp( k, cutlass.Boolean(True), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dstate_acc_ready.arrive(cta_group=1) bars.mb_neg_dy_inp_done.arrive(cta_group=1) @@ -1456,7 +1464,7 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dq_acc_offset), cutlass.Float32), accumulate=False, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dq_acc_ready.arrive(cta_group=1) bars.mb_da_done[qk_stage].arrive(cta_group=1) @@ -1476,7 +1484,7 @@ def tcgen05_mma_warp( k, cutlass.Boolean(sub + k > 0), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_sdy_done.arrive(cta_group=1) bars.mb_state_inp_done[gc % 2].arrive(cta_group=1) @@ -1489,8 +1497,8 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), accumulate=True, ) - if nvvm.elect_sync(): - bars.mb_dk_inv_part_ready.arrive(cta_group=1) + if elect_one: + bars.mb_dk_inv_part_acc_ready.arrive(cta_group=1) # ---- dK_decay part += K_inv^T(S) @ dM_strict^T ----------------------- if chunk_idx >= FIRST_STATE_CHUNK: @@ -1509,16 +1517,18 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), accumulate=False, ) - if nvvm.elect_sync(): - bars.mb_dk_decay_part_ready.arrive(cta_group=1) + if elect_one: + bars.mb_dk_decay_part_acc_ready.arrive(cta_group=1) bars.mb_dm_done[qk_stage].arrive(cta_group=1) - bars.mb_decay_done[decay_stage].arrive(cta_group=1) + bars.mb_decay_tcgen05_done[decay_stage].arrive(cta_group=1) # ---- tile end: WG1's dstate0 drain gates the next tile's dstate reuse ---- - bars.mb_dstate0_stored.wait(dstate0_index.phase) + bars.mb_dstate0_acc_stored.wait(dstate0_index.phase) dstate0_index = advance(dstate0_index, 1) gbase += sk_nt tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].wait(0) + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) nvvm.tcgen05_dealloc( nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), cutlass.Int32(512), @@ -1565,6 +1575,8 @@ def tmaldg_warp( ) -> None: """TMA-LDG warp role (warp 14): persistent scheduler loop issuing every G->S operand load.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) sQ_tma = SmemTile( @@ -1680,7 +1692,7 @@ def tmaldg_warp( desc_w_slot = (desc_w_base + slot).tospace(cutlass.AddressSpace.generic) desc_checkpoint_slot = (desc_checkpoint_base + slot).tospace(cutlass.AddressSpace.generic) desc_initial_state_slot = (desc_initial_state_base + cutlass.Int32(0)).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): + if elect_one: tma_tensormap_acquire(desc_q_slot) tma_tensormap_acquire(desc_k_slot) tma_tensormap_acquire(desc_v_slot) @@ -1698,28 +1710,28 @@ def tmaldg_warp( # ---- Q load ---------------------------------------------------------- bars.mb_q_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_q_ready[raw_index.idx].arrive(n_bytes=q_tx_bytes) q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), head_q, chunk_start) tma_load_tile(sQ_tma[raw_index.idx], q_slice, bars.mb_q_ready[raw_index.idx].smem_ptr, acquire=False) # ---- K load ---------------------------------------------------------- bars.mb_k_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_k_ready[raw_index.idx].arrive(n_bytes=k_tx_bytes) k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, chunk_start) tma_load_tile(sK_tma[raw_index.idx], k_slice, bars.mb_k_ready[raw_index.idx].smem_ptr, acquire=False) # ---- Gate load ------------------------------------------------------- bars.mb_gate_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_gate_ready[raw_index.idx].arrive(n_bytes=gate_tx_bytes) gate_slice = tma_slice_runtime_desc(desc_gate_slot, cutlass.Int32(0), head_o, chunk_start) tma_load_tile(sGate_tma[raw_index.idx], gate_slice, bars.mb_gate_ready[raw_index.idx].smem_ptr, acquire=False) # ---- Beta load ------------------------------------------------------- bars.mb_beta_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_beta_ready[raw_index.idx].arrive(n_bytes=beta_tx_bytes) beta_slice = tma_slice_runtime_desc(desc_beta_slot, cutlass.Int32(0), head_o, chunk_start) tma_load_tile(sBeta_tma[raw_index.idx], beta_slice, bars.mb_beta_ready[raw_index.idx].smem_ptr, acquire=False) @@ -1730,7 +1742,7 @@ def tmaldg_warp( bars.mb_state_done[state_idx].wait(state_index.phase) bars.mb_state_cg0_done[state_idx].wait(state_index.phase) state_index = advance(state_index, cfg.smem_state_stages) - if nvvm.elect_sync(): + if elect_one: bars.mb_state_ready[state_idx].arrive(n_bytes=cfg.tma_state_bytes) if cutlass.const_expr(cfg.use_initial_state): if chunk_idx == 0: @@ -1746,21 +1758,21 @@ def tmaldg_warp( # ---- dO load --------------------------------------------------------- bars.mb_do_done[raw_index.idx].wait(raw_index.phase) bars.mb_do_epi_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_do_ready[raw_index.idx].arrive(n_bytes=do_tx_bytes) do_slice = tma_slice_runtime_desc(desc_do_slot, cutlass.Int32(0), head_o, chunk_start) tma_load_tile(sDo_tma[raw_index.idx], do_slice, bars.mb_do_ready[raw_index.idx].smem_ptr, acquire=False) # ---- V load ---------------------------------------------------------- bars.mb_v_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_v_ready[raw_index.idx].arrive(n_bytes=v_tx_bytes) v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, chunk_start) tma_load_tile(sV_tma[raw_index.idx], v_slice, bars.mb_v_ready[raw_index.idx].smem_ptr, acquire=False) # ---- W load ---------------------------------------------------------- bars.mb_w_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_w_ready[raw_index.idx].arrive(n_bytes=w_tx_bytes) w_slice = tma_slice_runtime_desc(desc_w_slot, cutlass.Int32(0), head_o, chunk_start) tma_load_tile(sW_tma[raw_index.idx], w_slice, bars.mb_w_ready[raw_index.idx].smem_ptr, acquire=False) @@ -1804,6 +1816,7 @@ def compute0_warp_group( H -> TMEM f16 at the chunk tail.""" nvvm.setmaxregister(cfg.num_regs_compute_group_0, nvvm.SetMaxRegisterAction.INCREASE) cg0_warp = warp_idx - cfg.compute_group_0_warp_ids[0] + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_hold.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 @@ -1893,7 +1906,7 @@ def compute0_warp_group( exp_g_last = g_prefix_regs[cfg.b_t - 1] # ---- decay-slot guard: previous use fully consumed ------------------- operand_done_phase = ((gc // cfg.smem_decay_stages) + 1) % 2 - bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_tcgen05_done[decay_stage].wait(operand_done_phase) bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) for row in cutlass.range_constexpr(cfg.b_t): @@ -1911,7 +1924,7 @@ def compute0_warp_group( nvvm.barrier_cta_sync(cfg.cg0_sync_barrier_id, thread_count=cfg.cg0_threads) - k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) + k_inv_pack = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) raw_q_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) raw_beta_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) @@ -1927,18 +1940,18 @@ def compute0_warp_group( f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 raw_f16_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) - raw_q_vec = (sQ_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_k_vec = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_beta_vec = (sBetaP_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_q_vec_f32 = raw_q_vec.to(cutlass.Float32) - raw_k_vec_f32 = raw_k_vec.to(cutlass.Float32) - raw_beta_vec_f32 = raw_beta_vec.to(cutlass.Float32) + raw_q_frag = (sQ_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_k_frag = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_beta_frag = (sBetaP_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_q_frag_f32 = raw_q_frag.to(cutlass.Float32) + raw_k_frag_f32 = raw_k_frag.to(cutlass.Float32) + raw_beta_frag_f32 = raw_beta_frag.to(cutlass.Float32) for dim_offset in cutlass.range_constexpr(8): - q_val = raw_q_vec_f32[dim_offset] - k_val = raw_k_vec_f32[dim_offset] + q_val = raw_q_frag_f32[dim_offset] + k_val = raw_k_frag_f32[dim_offset] raw_q_regs[reg_base + dim_offset] = q_val raw_k_regs[reg_base + dim_offset] = k_val - raw_beta_regs[reg_base + dim_offset] = raw_beta_vec_f32[dim_offset] + raw_beta_regs[reg_base + dim_offset] = raw_beta_frag_f32[dim_offset] if cutlass.const_expr(cfg.l2norm): if cutlass.const_expr(dim_offset % 2 == 0): qk0_lo, qk0_hi = ffma2(q_val, k_val, q_val, k_val, qk0_lo, qk0_hi) @@ -1975,24 +1988,24 @@ def compute0_warp_group( f32_segment = f32_dim_base // 32 f32_segment_dim = f32_dim_base - f32_segment * 32 g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) - exp_g_vec = (g_prefix_ptr + g_prefix_idx).load(count=4, alignment=16) + exp_g_frag = (g_prefix_ptr + g_prefix_idx).load(count=4, alignment=16) exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) - exp_g_last_vec = (g_prefix_ptr + exp_g_last_idx).load(count=4, alignment=16) + exp_g_last_frag = (g_prefix_ptr + exp_g_last_idx).load(count=4, alignment=16) f32_reg_base = reg_base + f32_group * 4 - exp_g_regs[f32_reg_base] = exp_g_vec[0] - exp_g_regs[f32_reg_base + 1] = exp_g_vec[1] - exp_g_regs[f32_reg_base + 2] = exp_g_vec[2] - exp_g_regs[f32_reg_base + 3] = exp_g_vec[3] - exp_g_last_regs[f32_reg_base] = exp_g_last_vec[0] - exp_g_last_regs[f32_reg_base + 1] = exp_g_last_vec[1] - exp_g_last_regs[f32_reg_base + 2] = exp_g_last_vec[2] - exp_g_last_regs[f32_reg_base + 3] = exp_g_last_vec[3] + exp_g_regs[f32_reg_base] = exp_g_frag[0] + exp_g_regs[f32_reg_base + 1] = exp_g_frag[1] + exp_g_regs[f32_reg_base + 2] = exp_g_frag[2] + exp_g_regs[f32_reg_base + 3] = exp_g_frag[3] + exp_g_last_regs[f32_reg_base] = exp_g_last_frag[0] + exp_g_last_regs[f32_reg_base + 1] = exp_g_last_frag[1] + exp_g_last_regs[f32_reg_base + 2] = exp_g_last_frag[2] + exp_g_last_regs[f32_reg_base + 3] = exp_g_last_frag[3] for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 # ---- k decay + k inv operands: exp2(g) * beta * k / exp2(-g) * k - - k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_decay_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 @@ -2002,24 +2015,24 @@ def compute0_warp_group( k_beta0, k_beta1 = fmul2(k_value0, k_value1, raw_beta_regs[raw_reg_idx0], raw_beta_regs[raw_reg_idx1]) k_pair = fp32_to_fp16(k_beta0, k_beta1, dtype=cfg.io_dtype) exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) - k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) + k_decay_pack[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) exp_neg_g0 = cute.math.rcp(exp_g_regs[raw_reg_idx0], approx=True, ftz=True) exp_neg_g1 = cute.math.rcp(exp_g_regs[raw_reg_idx1], approx=True, ftz=True) exp_neg_pair = fp32_to_fp16(exp_neg_g0, exp_neg_g1, dtype=cfg.io_dtype) k_norm_pair = fp32_to_fp16(k_value0, k_value1, dtype=cfg.io_dtype) - k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_norm_pair, exp_neg_pair, cfg.io_dtype) + k_inv_pack[dim_half * 4 + pair_idx] = mul_f16x2(k_norm_pair, exp_neg_pair, cfg.io_dtype) k_inv_vec = cutlass.Vector.from_elements( ( - k_inv_words[dim_half * 4], - k_inv_words[dim_half * 4 + 1], - k_inv_words[dim_half * 4 + 2], - k_inv_words[dim_half * 4 + 3], + k_inv_pack[dim_half * 4], + k_inv_pack[dim_half * 4 + 1], + k_inv_pack[dim_half * 4 + 2], + k_inv_pack[dim_half * 4 + 3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) k_decay_vec = cutlass.Vector.from_elements( - (k_decay_words[0], k_decay_words[1], k_decay_words[2], k_decay_words[3]), + (k_decay_pack[0], k_decay_pack[1], k_decay_pack[2], k_decay_pack[3]), cutlass.Int32, ).bitcast(cfg.io_dtype) f16_segment = dim_base // 64 @@ -2034,8 +2047,8 @@ def compute0_warp_group( for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 - q_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) - k_restore_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + q_decay_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_restore_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 @@ -2044,16 +2057,16 @@ def compute0_warp_group( q_value0, q_value1 = fmul2(raw_q_regs[raw_reg_idx0], raw_q_regs[raw_reg_idx1], q_stage_norm, q_stage_norm) q_pair = fp32_to_fp16(q_value0, q_value1, dtype=cfg.io_dtype) exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) - q_decay_words[pair_idx] = mul_f16x2(q_pair, exp_g_pair, cfg.io_dtype) + q_decay_pack[pair_idx] = mul_f16x2(q_pair, exp_g_pair, cfg.io_dtype) exp_g_last_pair = fp32_to_fp16(exp_g_last_regs[raw_reg_idx0], exp_g_last_regs[raw_reg_idx1], dtype=cfg.io_dtype) - k_restore_words[pair_idx] = mul_f16x2(k_inv_words[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) + k_restore_pack[pair_idx] = mul_f16x2(k_inv_pack[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) q_decay_vec = cutlass.Vector.from_elements( - (q_decay_words[0], q_decay_words[1], q_decay_words[2], q_decay_words[3]), + (q_decay_pack[0], q_decay_pack[1], q_decay_pack[2], q_decay_pack[3]), cutlass.Int32, ).bitcast(cfg.io_dtype) k_restore_vec = cutlass.Vector.from_elements( - (k_restore_words[0], k_restore_words[1], k_restore_words[2], k_restore_words[3]), + (k_restore_pack[0], k_restore_pack[1], k_restore_pack[2], k_restore_pack[3]), cutlass.Int32, ).bitcast(cfg.io_dtype) f16_segment = dim_base // 64 @@ -2076,7 +2089,7 @@ def compute0_warp_group( state_src = sState_raw.data_ptr() + state_index.idx * (cfg.d_k * cfg.d_v) for pl in cutlass.range_constexpr(2): for g8 in cutlass.range_constexpr(8): - state_vec = (state_src + pl * (cfg.d_k * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, g8 * 8, elem_bytes=2)).load( + state_frag = (state_src + pl * (cfg.d_k * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, g8 * 8, elem_bytes=2)).load( count=8, alignment=16 ) nvvm.tcgen05_st( @@ -2084,7 +2097,7 @@ def compute0_warp_group( nvvm.make_tmem_ptr( state_copy_addr + (tmem_col + cfg.tmem_state_inp_offset + (gc % 2) * (cfg.d_v // 2) + pl * 32 + g8 * 4), cutlass.Int8 ), - state_vec.bitcast(cutlass.Int32), + state_frag.bitcast(cutlass.Int32), ) nvvm.tcgen05_wait("store") bars.mb_state_cg0_done[state_index.idx].arrive() @@ -2125,6 +2138,7 @@ def compute1_warp_group( v-term dot and store; end-of-chunk dH capture (dh acc -> dh_inp f16 + sdH); tile edges: dht seeding and the dS0 drain.""" nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_hold.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 @@ -2183,22 +2197,22 @@ def compute1_warp_group( nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_acc_offset + sub * 16), cutlass.Float32), seed_block[0:16], ) - packed_seed = cutlass.Array(cutlass.Int32, 8, alignment=16) + seed_pack = cutlass.Array(cutlass.Int32, 8, alignment=16) for pc in cutlass.range_constexpr(8): - packed_seed[pc] = fp32_to_fp16(seed_block[2 * pc], seed_block[2 * pc + 1], dtype=cfg.io_dtype) + seed_pack[pc] = fp32_to_fp16(seed_block[2 * pc], seed_block[2 * pc + 1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_inp_offset + sub * 8), cutlass.Int8), - packed_seed[0:8], + seed_pack[0:8], ) for half in cutlass.range_constexpr(2): d_base = sub * 16 + half * 8 - h_vec = cutlass.Vector.from_elements( - (packed_seed[half * 4], packed_seed[half * 4 + 1], packed_seed[half * 4 + 2], packed_seed[half * 4 + 3]), + h_pack = cutlass.Vector.from_elements( + (seed_pack[half * 4], seed_pack[half * 4 + 1], seed_pack[half * 4 + 2], seed_pack[half * 4 + 3]), cutlass.Int32, ).bitcast(cfg.io_dtype) h_addr = (d_base // 64) * (cfg.d_v * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, d_base % 64, elem_bytes=2) - (sDstate_raw.data_ptr() + h_addr).store(h_vec, alignment=16) + (sDstate_raw.data_ptr() + h_addr).store(h_pack, alignment=16) nvvm.tcgen05_wait("store") nvvm.fence_proxy("async.shared", space="cta") bars.mb_dstate_inp_ready.arrive() @@ -2225,7 +2239,7 @@ def compute1_warp_group( # ---- rhs staging: W = w*V - state_k -> TMEM f16 ---------------------- projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset input_col_id = tmem_col + cfg.tmem_rhs_inp_offset - raw_v_regs0 = nvvm.ldmatrix( + raw_v_frag0 = nvvm.ldmatrix( sV_ptr + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) + ov_tok * 64 @@ -2233,7 +2247,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_v_regs1 = nvvm.ldmatrix( + raw_v_frag1 = nvvm.ldmatrix( sV_ptr + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) + ov_tok * 64 @@ -2241,7 +2255,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_w_regs0 = nvvm.ldmatrix( + raw_w_frag0 = nvvm.ldmatrix( sW_ptr + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) + ov_tok * 64 @@ -2249,7 +2263,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_w_regs1 = nvvm.ldmatrix( + raw_w_frag1 = nvvm.ldmatrix( sW_ptr + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) + ov_tok * 64 @@ -2258,34 +2272,34 @@ def compute1_warp_group( nvvm.MMALayout.COL, ) - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) if chunk_idx >= FIRST_STATE_CHUNK: bars.mb_state_k_acc_ready.wait(state_k_index.phase) state_k_index = advance(state_k_index, 1) - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_pair = fp32_to_fp16(state_k0[frag_pair], state_k0[frag_pair + 1], dtype=cfg.io_dtype) - wv_pair = mul_f16x2(raw_w_regs0[raw_matrix], raw_v_regs0[raw_matrix], cfg.io_dtype) - packed_rhs0[reg_idx ^ 2] = sub_f16x2(wv_pair, state_k_pair, cfg.io_dtype) + state_k_pair = fp32_to_fp16(state_k_vec0[frag_pair], state_k_vec0[frag_pair + 1], dtype=cfg.io_dtype) + wv_pair = mul_f16x2(raw_w_frag0[raw_matrix], raw_v_frag0[raw_matrix], cfg.io_dtype) + rhs_pack0[reg_idx ^ 2] = sub_f16x2(wv_pair, state_k_pair, cfg.io_dtype) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_pair = fp32_to_fp16(state_k1[frag_pair], state_k1[frag_pair + 1], dtype=cfg.io_dtype) - wv_pair = mul_f16x2(raw_w_regs1[raw_matrix], raw_v_regs1[raw_matrix], cfg.io_dtype) - packed_rhs1[reg_idx ^ 2] = sub_f16x2(wv_pair, state_k_pair, cfg.io_dtype) + state_k_pair = fp32_to_fp16(state_k_vec1[frag_pair], state_k_vec1[frag_pair + 1], dtype=cfg.io_dtype) + wv_pair = mul_f16x2(raw_w_frag1[raw_matrix], raw_v_frag1[raw_matrix], cfg.io_dtype) + rhs_pack1[reg_idx ^ 2] = sub_f16x2(wv_pair, state_k_pair, cfg.io_dtype) if chunk_idx < FIRST_STATE_CHUNK: for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) - packed_rhs0[reg_idx ^ 2] = mul_f16x2(raw_w_regs0[raw_matrix], raw_v_regs0[raw_matrix], cfg.io_dtype) + rhs_pack0[reg_idx ^ 2] = mul_f16x2(raw_w_frag0[raw_matrix], raw_v_frag0[raw_matrix], cfg.io_dtype) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) - packed_rhs1[reg_idx ^ 2] = mul_f16x2(raw_w_regs1[raw_matrix], raw_v_regs1[raw_matrix], cfg.io_dtype) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + input_col_id, cutlass.Int8), packed_rhs0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + input_col_id, cutlass.Int8), packed_rhs1[0:4]) + rhs_pack1[reg_idx ^ 2] = mul_f16x2(raw_w_frag1[raw_matrix], raw_v_frag1[raw_matrix], cfg.io_dtype) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + input_col_id, cutlass.Int8), rhs_pack0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + input_col_id, cutlass.Int8), rhs_pack1[0:4]) nvvm.tcgen05_wait("store") bars.mb_rhs_ready.arrive() @@ -2295,17 +2309,17 @@ def compute1_warp_group( bars.mb_du_inp_done.wait(du_inp_done_index.phase) du_inp_done_index = advance(du_inp_done_index, 1) du_col_id = tmem_col + cfg.tmem_du_acc_offset - du0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + du_col_id, cutlass.Float32), num=2) - du1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + du_col_id, cutlass.Float32), num=2) + du_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + du_col_id, cutlass.Float32), num=2) + du_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + du_col_id, cutlass.Float32), num=2) - du_packed0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - du_packed1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + du_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + du_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): frag_pair = reg_idx * 2 - du_packed0[reg_idx] = fp32_to_fp16(du0[frag_pair], du0[frag_pair + 1], dtype=cfg.io_dtype) - du_packed1[reg_idx] = fp32_to_fp16(du1[frag_pair], du1[frag_pair + 1], dtype=cfg.io_dtype) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_packed0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_packed1[0:4]) + du_pack0[reg_idx] = fp32_to_fp16(du_vec0[frag_pair], du_vec0[frag_pair + 1], dtype=cfg.io_dtype) + du_pack1[reg_idx] = fp32_to_fp16(du_vec1[frag_pair], du_vec1[frag_pair + 1], dtype=cfg.io_dtype) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_pack0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_pack1[0:4]) nvvm.tcgen05_wait("store") bars.mb_du_inp_ready.arrive() @@ -2316,20 +2330,20 @@ def compute1_warp_group( bars.mb_u_warps_done.wait(u_done_index.phase) u_done_index = advance(u_done_index, 1) u_col_id = tmem_col + cfg.tmem_update_acc_offset - u0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + u_col_id, cutlass.Float32), num=2) - u1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + u_col_id, cutlass.Float32), num=2) + u_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + u_col_id, cutlass.Float32), num=2) + u_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + u_col_id, cutlass.Float32), num=2) - u_words0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - u_words1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + u_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + u_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - u_words0[reg_idx] = fp32_to_fp16(u0[2 * reg_idx], u0[2 * reg_idx + 1], dtype=cfg.io_dtype) - u_words1[reg_idx] = fp32_to_fp16(u1[2 * reg_idx], u1[2 * reg_idx + 1], dtype=cfg.io_dtype) + u_pack0[reg_idx] = fp32_to_fp16(u_vec0[2 * reg_idx], u_vec0[2 * reg_idx + 1], dtype=cfg.io_dtype) + u_pack1[reg_idx] = fp32_to_fp16(u_vec1[2 * reg_idx], u_vec1[2 * reg_idx + 1], dtype=cfg.io_dtype) nvvm.stmatrix( sU_raw.data_ptr() + (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) + ov_tok * 64 + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2), - u_words0.data_ptr().load(count=4, alignment=4), + u_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -2338,7 +2352,7 @@ def compute1_warp_group( + (value_dim_base + 16 + ov_col) // 64 * (cfg.b_t * 64) + ov_tok * 64 + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2), - u_words1.data_ptr().load(count=4, alignment=4), + u_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -2349,8 +2363,8 @@ def compute1_warp_group( bars.mb_dy_acc_ready.wait(dy_acc_index.phase) dy_acc_index = advance(dy_acc_index, 1) dy_col_id = tmem_col + cfg.tmem_dy_acc_offset - dy0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + dy_col_id, cutlass.Float32), num=2) - dy1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + dy_col_id, cutlass.Float32), num=2) + dy_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + dy_col_id, cutlass.Float32), num=2) + dy_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + dy_col_id, cutlass.Float32), num=2) # ---- dY -> sdY: pack + store + publish (super warp's dM operand) ------- addr_lo0 = (value_dim_base + ov_col) // 64 * (cfg.b_t * 64) + ov_tok * 64 + swizzle_xor_128b(ov_tok, (value_dim_base + ov_col) % 64, elem_bytes=2) @@ -2359,34 +2373,34 @@ def compute1_warp_group( + ov_tok * 64 + swizzle_xor_128b(ov_tok, (value_dim_base + 16 + ov_col) % 64, elem_bytes=2) ) - dy_p0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - dy_p1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + dy_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + dy_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - dy_p0[reg_idx] = fp32_to_fp16(dy0[2 * reg_idx], dy0[2 * reg_idx + 1], dtype=cfg.io_dtype) - dy_p1[reg_idx] = fp32_to_fp16(dy1[2 * reg_idx], dy1[2 * reg_idx + 1], dtype=cfg.io_dtype) + dy_pack0[reg_idx] = fp32_to_fp16(dy_vec0[2 * reg_idx], dy_vec0[2 * reg_idx + 1], dtype=cfg.io_dtype) + dy_pack1[reg_idx] = fp32_to_fp16(dy_vec1[2 * reg_idx], dy_vec1[2 * reg_idx + 1], dtype=cfg.io_dtype) bars.mb_sdy_done.wait(sdy_done_index.phase) bars.mb_sdy_super_done.wait(sdy_done_index.phase) sdy_done_index = advance(sdy_done_index, 1) - nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo0, dy_p0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) - nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo1, dy_p1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) + nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo0, dy_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) + nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo1, dy_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) bars.mb_sdy_ready.arrive() # ---- -dY -> TMEM: A operand of the dH ds-term -------------------------- - neg_dy0 = cutlass.Array(cutlass.Float32, 8, alignment=16) - neg_dy1 = cutlass.Array(cutlass.Float32, 8, alignment=16) + neg_dy_regs0 = cutlass.Array(cutlass.Float32, 8, alignment=16) + neg_dy_regs1 = cutlass.Array(cutlass.Float32, 8, alignment=16) for e in cutlass.range_constexpr(8): - neg_dy0[e] = -dy0[e] - neg_dy1[e] = -dy1[e] - neg_dy_p0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - neg_dy_p1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + neg_dy_regs0[e] = -dy_vec0[e] + neg_dy_regs1[e] = -dy_vec1[e] + neg_dy_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + neg_dy_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): frag_pair = reg_idx * 2 - neg_dy_p0[reg_idx] = fp32_to_fp16(neg_dy0[frag_pair], neg_dy0[frag_pair + 1], dtype=cfg.io_dtype) - neg_dy_p1[reg_idx] = fp32_to_fp16(neg_dy1[frag_pair], neg_dy1[frag_pair + 1], dtype=cfg.io_dtype) + neg_dy_pack0[reg_idx] = fp32_to_fp16(neg_dy_regs0[frag_pair], neg_dy_regs0[frag_pair + 1], dtype=cfg.io_dtype) + neg_dy_pack1[reg_idx] = fp32_to_fp16(neg_dy_regs1[frag_pair], neg_dy_regs1[frag_pair + 1], dtype=cfg.io_dtype) bars.mb_neg_dy_inp_done.wait(neg_dy_done_index.phase) neg_dy_done_index = advance(neg_dy_done_index, 1) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_neg_dy_inp_offset), cutlass.Int8), neg_dy_p0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_neg_dy_inp_offset), cutlass.Int8), neg_dy_p1[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_neg_dy_inp_offset), cutlass.Int8), neg_dy_pack0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_neg_dy_inp_offset), cutlass.Int8), neg_dy_pack1[0:4]) nvvm.tcgen05_wait("store") bars.mb_neg_dy_inp_ready.arrive() @@ -2426,25 +2440,25 @@ def compute1_warp_group( dstate_smem_done_index = advance(dstate_smem_done_index, 1) row_addr = (tmem_row + tmem_sp * cfg.threads_per_warp) << 16 for sub in cutlass.range_constexpr(cfg.d_k // 32): - dstate_block = nvvm.tcgen05_ld( + dstate_vec = nvvm.tcgen05_ld( "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_acc_offset + sub * 32), cutlass.Float32), num=32 ) - packed_dstate = cutlass.Array(cutlass.Int32, 16, alignment=16) + dstate_pack = cutlass.Array(cutlass.Int32, 16, alignment=16) for pc in cutlass.range_constexpr(16): - packed_dstate[pc] = fp32_to_fp16(dstate_block[2 * pc], dstate_block[2 * pc + 1], dtype=cfg.io_dtype) + dstate_pack[pc] = fp32_to_fp16(dstate_vec[2 * pc], dstate_vec[2 * pc + 1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_inp_offset + sub * 16), cutlass.Int8), - packed_dstate[0:16], + dstate_pack[0:16], ) for half in cutlass.range_constexpr(4): d_base = sub * 32 + half * 8 - h_vec = cutlass.Vector.from_elements( - (packed_dstate[half * 4], packed_dstate[half * 4 + 1], packed_dstate[half * 4 + 2], packed_dstate[half * 4 + 3]), + h_pack = cutlass.Vector.from_elements( + (dstate_pack[half * 4], dstate_pack[half * 4 + 1], dstate_pack[half * 4 + 2], dstate_pack[half * 4 + 3]), cutlass.Int32, ).bitcast(cfg.io_dtype) h_addr = (d_base // 64) * (cfg.d_v * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, d_base % 64, elem_bytes=2) - (sDstate_raw.data_ptr() + h_addr).store(h_vec, alignment=16) + (sDstate_raw.data_ptr() + h_addr).store(h_pack, alignment=16) nvvm.tcgen05_wait("store") nvvm.fence_proxy("async.shared", space="cta") bars.mb_dstate_inp_ready.arrive() @@ -2471,10 +2485,12 @@ def compute1_warp_group( mDstate0[batch_idx, head_idx, kd, value_dim] = mDstate_in[batch_idx, head_idx, kd, value_dim] else: mDstate0[batch_idx, head_idx, kd, value_dim] = cutlass.Float32(0.0) - bars.mb_dstate0_stored.arrive() + bars.mb_dstate0_acc_stored.arrive() gbase += sk_nt tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].arrive() + @cute.jit def compute2_warp_group( @@ -2511,6 +2527,7 @@ def compute2_warp_group( stages dq/dk for the epilogue's TMA stores.""" nvvm.setmaxregister(cfg.num_regs_compute_group_2, nvvm.SetMaxRegisterAction.INCREASE) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_hold.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 @@ -2570,7 +2587,7 @@ def compute2_warp_group( dgl_dstate_smem_index = advance(dgl_dstate_smem_index, 1) for pl in cutlass.range_constexpr(2): for row_half in cutlass.range_constexpr(2): - state_words = nvvm.tcgen05_ld( + state_vec = nvvm.tcgen05_ld( "32x32b", nvvm.make_tmem_ptr( row_addr + (tmem_col + cfg.tmem_state_inp_offset + (gc % 2) * (cfg.d_v // 2) + pl * 32 + row_half * 16), cutlass.Float32 @@ -2582,7 +2599,7 @@ def compute2_warp_group( hacc[i] = opaque_f32_zero() for j in cutlass.range_constexpr(16): v0 = pl * 64 + row_half * 32 + 2 * j - state_pair = cutlass.Vector.from_elements((state_words[j],), cutlass.Float32).bitcast(cfg.io_dtype) + state_pair = cutlass.Vector.from_elements((state_vec[j],), cutlass.Float32).bitcast(cfg.io_dtype) dstate_addr0 = (channel // 64) * (cfg.d_v * 64) + v0 * 64 + swizzle_xor_128b(v0, channel % 64, elem_bytes=2) dstate_addr1 = (channel // 64) * (cfg.d_v * 64) + (v0 + 1) * 64 + swizzle_xor_128b(v0 + 1, channel % 64, elem_bytes=2) hval0 = (sDstate_raw.data_ptr() + dstate_addr0).load().to(cutlass.Float32) @@ -2598,8 +2615,8 @@ def compute2_warp_group( # ---- part-drain accumulators ------------------------------------------- dq_n = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) dk_n = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) - db_reg = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) - dg_reg = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + db_regs = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + dg_regs = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) dgl_acc = cutlass.Array(cutlass.Float32, 4, alignment=16) for i in cutlass.range_constexpr(4): dgl_acc[i] = opaque_f32_zero() @@ -2608,13 +2625,13 @@ def compute2_warp_group( # ---- dk_restore part drain: (eGl/eG) scale + dGlast k-dot -------------- if has_dstate: - bars.mb_dk_restore_part_ready.wait(dk_restore_part_index.phase) + bars.mb_dk_restore_part_acc_ready.wait(dk_restore_part_index.phase) dk_restore_part_index = advance(dk_restore_part_index, 1) - dk_restore_part = nvvm.tcgen05_ld( + dk_restore_part_vec = nvvm.tcgen05_ld( "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_restore_acc_offset), cutlass.Float32), num=cfg.b_t ) for t in cutlass.range_constexpr(cfg.b_t): - dk_hat = egl * cute.math.rcp(eg[t], approx=True, ftz=True) * dk_restore_part[t] + dk_hat = egl * cute.math.rcp(eg[t], approx=True, ftz=True) * dk_restore_part_vec[t] dk_n[t] = dk_hat k_v = (sK_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) if cutlass.const_expr(cfg.l2norm): @@ -2624,34 +2641,36 @@ def compute2_warp_group( # ---- dq acc drain: eG.scale --------------------------------------------- bars.mb_dq_acc_ready.wait(dq_acc_index.phase) dq_acc_index = advance(dq_acc_index, 1) - dq_acc = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dq_acc_offset), cutlass.Float32), num=cfg.b_t) + dq_vec = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dq_acc_offset), cutlass.Float32), num=cfg.b_t) for t2 in cutlass.range_constexpr(cfg.b_t // 2): t = 2 * t2 es_lo, es_hi = fmul2(eg[t], eg[t + 1], scale, scale) - dq_n[t], dq_n[t + 1] = fmul2(es_lo, es_hi, dq_acc[t], dq_acc[t + 1]) + dq_n[t], dq_n[t + 1] = fmul2(es_lo, es_hi, dq_vec[t], dq_vec[t + 1]) # ---- dk_inv part drain: (dA - dM) term, 1/eG scale ---------------------- - bars.mb_dk_inv_part_ready.wait(dk_inv_part_index.phase) + bars.mb_dk_inv_part_acc_ready.wait(dk_inv_part_index.phase) dk_inv_part_index = advance(dk_inv_part_index, 1) - dk_inv_part = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), num=cfg.b_t) + dk_inv_part_vec = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), num=cfg.b_t) for t in cutlass.range_constexpr(cfg.b_t): - dk_n[t] = dk_n[t] + dk_inv_part[t] * cute.math.rcp(eg[t], approx=True, ftz=True) + dk_n[t] = dk_n[t] + dk_inv_part_vec[t] * cute.math.rcp(eg[t], approx=True, ftz=True) # ---- dk_decay part drain: -eG scale, seeds db and dG -------------------- - bars.mb_dk_decay_part_ready.wait(dk_decay_part_index.phase) + bars.mb_dk_decay_part_acc_ready.wait(dk_decay_part_index.phase) dk_decay_part_index = advance(dk_decay_part_index, 1) - dk_decay_part = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), num=cfg.b_t) + dk_decay_part_vec = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), num=cfg.b_t + ) for t in cutlass.range_constexpr(cfg.b_t): - dk_decay = -eg[t] * dk_decay_part[t] + dk_decay = -eg[t] * dk_decay_part_vec[t] k_v = (sK_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) if cutlass.const_expr(cfg.l2norm): k_v = k_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + cfg.b_t + t] - db_reg[t] = k_v * dk_decay - dg_reg[t] = (sBetaP_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) * dk_decay - dk_n[t] = dk_n[t] + dg_reg[t] + db_regs[t] = k_v * dk_decay + dg_regs[t] = (sBetaP_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) * dk_decay + dk_n[t] = dk_n[t] + dg_regs[t] # ---- all parts drained: release the accs to the next chunk's MMAs ------ - bars.mb_parts_done.arrive() + bars.mb_parts_acc_done.arrive() # ---- dG finalize ----------------------------------------------------- for t in cutlass.range_constexpr(cfg.b_t): @@ -2660,12 +2679,12 @@ def compute2_warp_group( if cutlass.const_expr(cfg.l2norm): q_v = q_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + t] k_v = k_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + cfg.b_t + t] - dg_reg[t] = ( + dg_regs[t] = ( q_v * dq_n[t] - + (sBetaP_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) * db_reg[t] - - k_v * (dk_n[t] - dg_reg[t]) + + (sBetaP_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) * db_regs[t] + - k_v * (dk_n[t] - dg_regs[t]) ) - dg_reg[cfg.b_t - 1] = dg_reg[cfg.b_t - 1] + ((dgl_acc[0] + dgl_acc[1]) + (dgl_acc[2] + dgl_acc[3])) + dg_regs[cfg.b_t - 1] = dg_regs[cfg.b_t - 1] + ((dgl_acc[0] + dgl_acc[1]) + (dgl_acc[2] + dgl_acc[3])) # ---- L2-norm backward row projection --------------------------------- if cutlass.const_expr(cfg.l2norm): @@ -2707,14 +2726,14 @@ def compute2_warp_group( # ---- dGlast add ------------------------------------------------------ if has_dstate: if chunk_idx >= FIRST_STATE_CHUNK: - dg_reg[cfg.b_t - 1] = dg_reg[cfg.b_t - 1] + egl * dgl_val + dg_regs[cfg.b_t - 1] = dg_regs[cfg.b_t - 1] + egl * dgl_val # ---- dG reverse cumsum ----------------------------------------------- suffix = cutlass.Float32(0.0) for rt in cutlass.range_constexpr(cfg.b_t): t = cfg.b_t - 1 - rt - suffix = suffix + dg_reg[t] - dg_reg[t] = suffix + suffix = suffix + dg_regs[t] + dg_regs[t] = suffix # ---- stage dGate + db for the epilogue TMA stores -------------------- dg_stage = gc % cfg.smem_dg_stages @@ -2723,9 +2742,9 @@ def compute2_warp_group( bars.mb_db_tmastg_done[db_stage].wait(((gc // cfg.smem_db_stages) + 1) % 2) for t in cutlass.range_constexpr(cfg.b_t): dg_idx = f32_seg * (cfg.b_t * 32) + t * 32 + swizzle_xor_128b(t, f32_dim, elem_bytes=4) - (sDg_raw.data_ptr() + dg_idx).store(dg_reg[t]) + (sDg_raw.data_ptr() + dg_idx).store(dg_regs[t]) db_idx = f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2) - (sDb_raw.data_ptr() + db_idx).store(db_reg[t].to(cfg.io_dtype)) + (sDb_raw.data_ptr() + db_idx).store(db_regs[t].to(cfg.io_dtype)) nvvm.fence_proxy("async.shared", space="cta") bars.mb_dg_tmastg_ready[dg_stage].arrive() bars.mb_db_tmastg_ready[db_stage].arrive() @@ -2738,6 +2757,8 @@ def compute2_warp_group( gbase += sk_nt tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].arrive() + # --------------------------------------------------------------------------- # Host-side assembly @@ -2915,23 +2936,34 @@ def build_descs( granu = 128 // bpe seqlen = q.shape[0] - def headed(t, dim, hn): - return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) + q_headed = cute.make_tensor(q.iterator, cute.make_layout((d_k, h_q, seqlen), stride=(1, q.stride[1], q.stride[0]))) + k_headed = cute.make_tensor(k.iterator, cute.make_layout((d_k, h_k, seqlen), stride=(1, k.stride[1], k.stride[0]))) + v_headed = cute.make_tensor(v.iterator, cute.make_layout((d_v, h_v, seqlen), stride=(1, v.stride[1], v.stride[0]))) + gate_headed = cute.make_tensor(gate.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, gate.stride[1], gate.stride[0]))) + do_headed = cute.make_tensor(do.iterator, cute.make_layout((d_v, ho, seqlen), stride=(1, do.stride[1], do.stride[0]))) + beta_headed = cute.make_tensor(beta.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, beta.stride[1], beta.stride[0]))) + w_headed = cute.make_tensor(w.iterator, cute.make_layout((d_v, ho, seqlen), stride=(1, w.stride[1], w.stride[0]))) + dq_headed = cute.make_tensor(dq.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, dq.stride[1], dq.stride[0]))) + dk_headed = cute.make_tensor(dk.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, dk.stride[1], dk.stride[0]))) + dv_headed = cute.make_tensor(dv.iterator, cute.make_layout((d_v, ho, seqlen), stride=(1, dv.stride[1], dv.stride[0]))) + dg_headed = cute.make_tensor(dg.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, dg.stride[1], dg.stride[0]))) + dwo_headed = cute.make_tensor(dwo.iterator, cute.make_layout((d_v, ho, seqlen), stride=(1, dwo.stride[1], dwo.stride[0]))) + dbo_headed = cute.make_tensor(dbo.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, dbo.stride[1], dbo.stride[0]))) swz = cuda.TensorMapSwizzle.s128b - base_q = cuda.create_tensor_map_tiled_from_view(headed(q, d_k, h_q), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_do = cuda.create_tensor_map_tiled_from_view(headed(do, d_v, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_beta = cuda.create_tensor_map_tiled_from_view(headed(beta, d_k, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_w = cuda.create_tensor_map_tiled_from_view(headed(w, d_v, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_dq = cuda.create_tensor_map_tiled_from_view(headed(dq, d_k, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_dk = cuda.create_tensor_map_tiled_from_view(headed(dk, d_k, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_dv = cuda.create_tensor_map_tiled_from_view(headed(dv, d_v, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_dg = cuda.create_tensor_map_tiled_from_view(headed(dg, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_dwo = cuda.create_tensor_map_tiled_from_view(headed(dwo, d_v, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_dbo = cuda.create_tensor_map_tiled_from_view(headed(dbo, d_k, ho), box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_q = cuda.create_tensor_map_tiled_from_view(q_headed, box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_k = cuda.create_tensor_map_tiled_from_view(k_headed, box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(v_headed, box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(gate_headed, box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_do = cuda.create_tensor_map_tiled_from_view(do_headed, box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_beta = cuda.create_tensor_map_tiled_from_view(beta_headed, box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_w = cuda.create_tensor_map_tiled_from_view(w_headed, box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dq = cuda.create_tensor_map_tiled_from_view(dq_headed, box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dk = cuda.create_tensor_map_tiled_from_view(dk_headed, box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dv = cuda.create_tensor_map_tiled_from_view(dv_headed, box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dg = cuda.create_tensor_map_tiled_from_view(dg_headed, box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dwo = cuda.create_tensor_map_tiled_from_view(dwo_headed, box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dbo = cuda.create_tensor_map_tiled_from_view(dbo_headed, box_dims=(granu, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) checkpoint_view = cute.make_tensor( state_checkpoints.iterator, @@ -3258,8 +3290,9 @@ def kernel( v_tx_bytes = cutlass.const_expr(cfg.d_v * cfg.b_t * bpe) w_tx_bytes = cutlass.const_expr(cfg.d_v * cfg.b_t * bpe) + elect_one = nvvm.elect_sync() if warp_idx == cfg.tma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_raw_stages): bars.mb_q_ready[stage].init() bars.mb_q_done[stage].init() @@ -3285,7 +3318,7 @@ def kernel( bars.mb_state_inp_done[stage].init() bars.mb_state_inp_cg2_done[stage].init() elif warp_idx == cfg.tcgen05_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: bars.mb_state_k_acc_ready.init() bars.mb_rhs_ready.init() bars.mb_update_acc_ready.init() @@ -3309,20 +3342,21 @@ def kernel( bars.mb_dstate_smem_done.init() bars.mb_dstate_smem_cg2_done.init() bars.mb_dq_acc_ready.init() - bars.mb_dk_decay_part_ready.init() - bars.mb_dk_inv_part_ready.init() - bars.mb_dk_restore_part_ready.init() - bars.mb_parts_done.init() - bars.mb_dstate0_stored.init() + bars.mb_dk_decay_part_acc_ready.init() + bars.mb_dk_inv_part_acc_ready.init() + bars.mb_dk_restore_part_acc_ready.init() + bars.mb_parts_acc_done.init() + bars.mb_dstate0_acc_stored.init() + bars.mb_tmem_done[0].init() elif warp_idx == cfg.super_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_decay_stages): bars.mb_k_decay_inv_ready[stage].init() bars.mb_q_decay_k_restore_ready[stage].init() - bars.mb_decay_done[stage].init() + bars.mb_decay_tcgen05_done[stage].init() bars.mb_decay_super_done[stage].init() for stage in cutlass.range_constexpr(cfg.smem_qk_stages): - bars.mb_a_ready[stage].init() + bars.mb_a_inv_ready[stage].init() bars.mb_aqk_ready[stage].init() bars.mb_da_ready[stage].init() bars.mb_dm_ready[stage].init() @@ -3331,7 +3365,7 @@ def kernel( bars.mb_da_done[stage].init() bars.mb_dm_done[stage].init() elif warp_idx == cfg.epilogue_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_dq_stages): bars.mb_dq_tmastg_ready[stage].init() bars.mb_dq_tmastg_done[stage].init() @@ -3358,20 +3392,6 @@ def kernel( sState_scale_diag_raw[diag_idx] = diag_zero nvvm.fence_mbarrier_init() nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) - is_tmem_user = ( - (warp_idx >= cfg.compute_group_2_warp_ids[0] and warp_idx <= cfg.compute_group_2_warp_ids[-1]) - or (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) - or (warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]) - or warp_idx == cfg.tcgen05_mma_warp_id - ) - if is_tmem_user: - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_alloc(tmem_hold, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - if warp_idx == cfg.tma_warp_id: tmaldg_warp( cfg, diff --git a/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_f16.py index 276b6cec0..e8fd2ab3c 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn2_prefill_f16.py @@ -159,18 +159,18 @@ class Gdn2Bars(NamedTuple): mb_rhs_ready: MBarrier mb_update_ready: MBarrier - mb_a_ready: MBarrier + mb_a_inv_ready: MBarrier mb_a_done: MBarrier mb_qk_acc_ready: MBarrier mb_k_decay_inv_cg0_ready: MBarrier - mb_decay_done: MBarrier + mb_decay_tcgen05_done: MBarrier mb_decay_super_done: MBarrier mb_qk_scale_ready: MBarrier - mb_k_restore_done: MBarrier + mb_k_restore_acc_done: MBarrier mb_state_scale_diag_done: MBarrier - mb_final_state_stored: MBarrier - mb_state_read_done: MBarrier + mb_tmem_done: MBarrier + mb_state_acc_read_done: MBarrier mb_o_tmastg_ready: MBarrier mb_o_tmastg_done: MBarrier @@ -213,11 +213,11 @@ def alloc(n): mb_state_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_rhs_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_update_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_a_inv_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), mb_a_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_qk_acc_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), mb_k_decay_inv_cg0_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), - mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_decay_tcgen05_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=2 * WARP, producer=Producer.THREAD), mb_qk_scale_ready=MBarrier( alloc(cfg.qk_scale_ready_stages), @@ -225,15 +225,15 @@ def alloc(n): init_count=CG0_GROUP_THREADS, producer=Producer.THREAD, ), - mb_k_restore_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_k_restore_acc_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_state_scale_diag_done=MBarrier( alloc(cfg.smem_state_scale_diag_stages), stages=cfg.smem_state_scale_diag_stages, init_count=1, producer=Producer.MMA_COMMIT, ), - mb_final_state_stored=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_state_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_tmem_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_state_acc_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_tmastg_ready=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_tmastg_done=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=WARP, producer=Producer.THREAD), mb_checkpoint_tmastg_ready=MBarrier( @@ -515,7 +515,7 @@ def super_mma_warp( # Load B operand k_inv_col = k_block * 16 + rhs_col_offset k_inv_segment = k_inv_col // 64 - rhs_vec = nvvm.ldmatrix( + rhs_frag = nvvm.ldmatrix( sK_inv_ptr + k_inv_segment * (cfg.b_t * 64) + rhs_row_coord * 64 @@ -526,7 +526,7 @@ def super_mma_warp( # Load A operand storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask storage_slice = storage_key // 64 - kk_lhs_vec = nvvm.ldmatrix( + kk_lhs_frag = nvvm.ldmatrix( sK_decay_ptr + storage_slice * (cfg.b_t * 64) + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), @@ -536,8 +536,8 @@ def super_mma_warp( mma_step( kk_acc, - (kk_lhs_vec[0], kk_lhs_vec[1], kk_lhs_vec[2], kk_lhs_vec[3]), - (rhs_vec[0], rhs_vec[1], rhs_vec[2], rhs_vec[3]), + (kk_lhs_frag[0], kk_lhs_frag[1], kk_lhs_frag[2], kk_lhs_frag[3]), + (rhs_frag[0], rhs_frag[1], rhs_frag[2], rhs_frag[3]), k_step=0, M=16, N=16, @@ -547,7 +547,7 @@ def super_mma_warp( # ---- L = tril(KK, -1) fragment --------------------------------------- row_lo = lane // 4 row_hi = row_lo + cutlass.Int32(8) - l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) + l_regs = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): row_coord = row_lo if cutlass.const_expr(accum_idx % 4 >= 2): @@ -555,11 +555,11 @@ def super_mma_warp( col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) if cutlass.const_expr(accum_idx % 2 == 1): col_coord = col_coord + cutlass.Int32(1) - l_frag[accum_idx] = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) - l_a0 = fp32_to_fp16(l_frag[0], l_frag[1], dtype=cfg.io_dtype) - l_a1 = fp32_to_fp16(l_frag[2], l_frag[3], dtype=cfg.io_dtype) - l_a2 = fp32_to_fp16(l_frag[4], l_frag[5], dtype=cfg.io_dtype) - l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) + l_regs[accum_idx] = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + l_a0 = fp32_to_fp16(l_regs[0], l_regs[1], dtype=cfg.io_dtype) + l_a1 = fp32_to_fp16(l_regs[2], l_regs[3], dtype=cfg.io_dtype) + l_a2 = fp32_to_fp16(l_regs[4], l_regs[5], dtype=cfg.io_dtype) + l_a3 = fp32_to_fp16(l_regs[6], l_regs[7], dtype=cfg.io_dtype) l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) # ---- A_inv = I - L, then three Neumann doubling rounds --------------- @@ -637,7 +637,7 @@ def super_mma_warp( shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_a_ready[qk_stage].arrive() + bars.mb_a_inv_ready[qk_stage].arrive() bars.mb_decay_super_done[decay_stage].arrive() global_chunk_base += num_tile_chunks tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) @@ -664,6 +664,8 @@ def tcgen05_mma_warp( every tcgen05 GEMM.""" elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + nvvm.tcgen05_alloc(sTmem_base, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = sTmem_base.load() state_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) state_dsts = tuple(nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset + k * 16, cutlass.Float32) for k in range(cfg.d_k // 16)) @@ -676,7 +678,6 @@ def tcgen05_mma_warp( state_read_index = PipelineState.start(phase=0) rhs_index = PipelineState.start(phase=0) update_index = PipelineState.start(phase=0) - final_state_index = PipelineState.start(phase=0) qk_scale_index = PipelineState.start(phase=0) # ---- chunk-invariant GEMM descriptors ---------------------------------------- @@ -813,11 +814,11 @@ def tcgen05_mma_warp( ) if elect_one: - bars.mb_decay_done[decay_stage].arrive(cta_group=1) + bars.mb_decay_tcgen05_done[decay_stage].arrive(cta_group=1) # WAR: CG1's post-publish f32 checkpoint read must retire before the overwrite if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_read_done.wait(state_read_index.phase) + bars.mb_state_acc_read_done.wait(state_read_index.phase) state_read_index = advance(state_read_index, 1) # ---- state decay = state(T) @ diag(exp2(g_last)) (per-k-atom blocks) ---- @@ -837,7 +838,7 @@ def tcgen05_mma_warp( bars.mb_state_scale_diag_done[state_scale_diag_stage].arrive(cta_group=1) # ---- update = rhs(T) @ A_inv ----------------------------------------- - bars.mb_a_ready[qk_stage].wait((global_chunk // cfg.smem_qk_stages) % 2) + bars.mb_a_inv_ready[qk_stage].wait((global_chunk // cfg.smem_qk_stages) % 2) bars.mb_rhs_ready.wait(rhs_index.phase) rhs_index = advance(rhs_index, 1) desc_qk = sQk_stage.shifted((cfg.b_t * cfg.b_t)).desc() @@ -852,7 +853,7 @@ def tcgen05_mma_warp( mma_ts_step(bmm_final_state_desc, update_inp_ptr, desc_k_restore, state_dst_ptr, 0, cutlass.Boolean(True)) if elect_one: - bars.mb_k_restore_done[decay_stage].arrive(cta_group=1) + bars.mb_k_restore_acc_done[decay_stage].arrive(cta_group=1) # ---- O += update(T) @ QK --------------------------------------------- bars.mb_qk_acc_ready[qk_stage].wait((global_chunk // cfg.smem_qk_stages) % 2) @@ -870,10 +871,10 @@ def tcgen05_mma_warp( bars.mb_a_done[qk_stage].arrive(cta_group=1) qk_scale_index = advance(qk_scale_index, cfg.smem_state_scale_diag_stages) - bars.mb_final_state_stored.wait(final_state_index.phase) - final_state_index = advance(final_state_index, 1) global_chunk_base += num_tile_chunks tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].wait(0) + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) nvvm.tcgen05_dealloc( nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), cutlass.Int32(512), @@ -986,7 +987,7 @@ def epilogue_warp( # Load B operand k_inv_col = k_block * 16 + rhs_col_offset k_inv_segment = k_inv_col // 64 - rhs_vec = nvvm.ldmatrix( + rhs_frag = nvvm.ldmatrix( sK_inv_ptr + k_inv_segment * (cfg.b_t * 64) + rhs_row_coord * 64 @@ -997,7 +998,7 @@ def epilogue_warp( # Load A operand storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask storage_slice = storage_key // 64 - qk_lhs_vec = nvvm.ldmatrix( + qk_lhs_frag = nvvm.ldmatrix( sQ_decay_ptr + storage_slice * (cfg.b_t * 64) + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), @@ -1007,8 +1008,8 @@ def epilogue_warp( mma_step( qk_acc, - (qk_lhs_vec[0], qk_lhs_vec[1], qk_lhs_vec[2], qk_lhs_vec[3]), - (rhs_vec[0], rhs_vec[1], rhs_vec[2], rhs_vec[3]), + (qk_lhs_frag[0], qk_lhs_frag[1], qk_lhs_frag[2], qk_lhs_frag[3]), + (rhs_frag[0], rhs_frag[1], rhs_frag[2], rhs_frag[3]), k_step=0, M=16, N=16, @@ -1281,7 +1282,7 @@ def compute0_warp_group( nvvm.barrier_cta_sync(cfg.cg0_group_sync_barrier_base_id + cg0_group_id, thread_count=cfg.cg0_threads_per_group) - k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) + k_inv_pack = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) raw_q_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) raw_beta_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) @@ -1298,12 +1299,12 @@ def compute0_warp_group( f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 raw_f16_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) - raw_q_vec = (sQ_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_k_vec = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_beta_vec = (sBeta_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_q_vec_f32 = raw_q_vec.to(cutlass.Float32) - raw_k_vec_f32 = raw_k_vec.to(cutlass.Float32) - raw_beta_vec_f32 = raw_beta_vec.to(cutlass.Float32) + raw_q_frag = (sQ_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_k_frag = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_beta_frag = (sBeta_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_q_vec_f32 = raw_q_frag.to(cutlass.Float32) + raw_k_vec_f32 = raw_k_frag.to(cutlass.Float32) + raw_beta_vec_f32 = raw_beta_frag.to(cutlass.Float32) for dim_offset in cutlass.range_constexpr(8): q_val = raw_q_vec_f32[dim_offset] k_val = raw_k_vec_f32[dim_offset] @@ -1344,28 +1345,28 @@ def compute0_warp_group( f32_segment = f32_dim_base // 32 f32_segment_dim = f32_dim_base - f32_segment * 32 g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) - exp_g_vec = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) + exp_g_frag = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) f32_segment = f32_dim_base // 32 f32_segment_dim = f32_dim_base - f32_segment * 32 exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) - exp_g_last_vec = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) + exp_g_last_frag = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) half_reg_base = f32_group * 4 f32_reg_base = reg_base + half_reg_base - exp_g_regs[f32_reg_base] = exp_g_vec[0] - exp_g_regs[f32_reg_base + 1] = exp_g_vec[1] - exp_g_regs[f32_reg_base + 2] = exp_g_vec[2] - exp_g_regs[f32_reg_base + 3] = exp_g_vec[3] - exp_neg_g_regs[half_reg_base] = cute.math.rcp(exp_g_vec[0], approx=True, ftz=True) - exp_neg_g_regs[half_reg_base + 1] = cute.math.rcp(exp_g_vec[1], approx=True, ftz=True) - exp_neg_g_regs[half_reg_base + 2] = cute.math.rcp(exp_g_vec[2], approx=True, ftz=True) - exp_neg_g_regs[half_reg_base + 3] = cute.math.rcp(exp_g_vec[3], approx=True, ftz=True) - exp_g_last_regs[f32_reg_base] = exp_g_last_vec[0] - exp_g_last_regs[f32_reg_base + 1] = exp_g_last_vec[1] - exp_g_last_regs[f32_reg_base + 2] = exp_g_last_vec[2] - exp_g_last_regs[f32_reg_base + 3] = exp_g_last_vec[3] + exp_g_regs[f32_reg_base] = exp_g_frag[0] + exp_g_regs[f32_reg_base + 1] = exp_g_frag[1] + exp_g_regs[f32_reg_base + 2] = exp_g_frag[2] + exp_g_regs[f32_reg_base + 3] = exp_g_frag[3] + exp_neg_g_regs[half_reg_base] = cute.math.rcp(exp_g_frag[0], approx=True, ftz=True) + exp_neg_g_regs[half_reg_base + 1] = cute.math.rcp(exp_g_frag[1], approx=True, ftz=True) + exp_neg_g_regs[half_reg_base + 2] = cute.math.rcp(exp_g_frag[2], approx=True, ftz=True) + exp_neg_g_regs[half_reg_base + 3] = cute.math.rcp(exp_g_frag[3], approx=True, ftz=True) + exp_g_last_regs[f32_reg_base] = exp_g_last_frag[0] + exp_g_last_regs[f32_reg_base + 1] = exp_g_last_frag[1] + exp_g_last_regs[f32_reg_base + 2] = exp_g_last_frag[2] + exp_g_last_regs[f32_reg_base + 3] = exp_g_last_frag[3] # ---- K decay + K_inv operands: K * exp2(+g) and K * exp2(-g) ----- - k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_decay_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 @@ -1375,32 +1376,32 @@ def compute0_warp_group( k_beta0, k_beta1 = fmul2(k_value0, k_value1, raw_beta_regs[raw_reg_idx0], raw_beta_regs[raw_reg_idx1]) k_pair = fp32_to_fp16(k_beta0, k_beta1, dtype=cfg.io_dtype) exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) - k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) + k_decay_pack[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) exp_neg_pair = fp32_to_fp16(exp_neg_g_regs[dim0], exp_neg_g_regs[dim1], dtype=cfg.io_dtype) k_norm_pair = fp32_to_fp16(k_value0, k_value1, dtype=cfg.io_dtype) - k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_norm_pair, exp_neg_pair, cfg.io_dtype) + k_inv_pack[dim_half * 4 + pair_idx] = mul_f16x2(k_norm_pair, exp_neg_pair, cfg.io_dtype) k_inv_vec = cutlass.Vector.from_elements( ( - k_inv_words[dim_half * 4], - k_inv_words[dim_half * 4 + 1], - k_inv_words[dim_half * 4 + 2], - k_inv_words[dim_half * 4 + 3], + k_inv_pack[dim_half * 4], + k_inv_pack[dim_half * 4 + 1], + k_inv_pack[dim_half * 4 + 2], + k_inv_pack[dim_half * 4 + 3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) k_decay_vec = cutlass.Vector.from_elements( ( - k_decay_words[0], - k_decay_words[1], - k_decay_words[2], - k_decay_words[3], + k_decay_pack[0], + k_decay_pack[1], + k_decay_pack[2], + k_decay_pack[3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) if cutlass.const_expr(dim_half == 0): operand_done_phase = ((global_chunk // cfg.smem_decay_stages) + 1) % 2 - bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_tcgen05_done[decay_stage].wait(operand_done_phase) bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 @@ -1423,7 +1424,7 @@ def compute0_warp_group( for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 - q_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + q_decay_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 @@ -1432,14 +1433,14 @@ def compute0_warp_group( q_value0, q_value1 = fmul2(raw_q_regs[raw_reg_idx0], raw_q_regs[raw_reg_idx1], q_inv_norm, q_inv_norm) q_pair = fp32_to_fp16(q_value0, q_value1, dtype=cfg.io_dtype) exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) - q_decay_words[pair_idx] = mul_f16x2(q_pair, exp_g_pair, cfg.io_dtype) + q_decay_pack[pair_idx] = mul_f16x2(q_pair, exp_g_pair, cfg.io_dtype) q_decay_vec = cutlass.Vector.from_elements( ( - q_decay_words[0], - q_decay_words[1], - q_decay_words[2], - q_decay_words[3], + q_decay_pack[0], + q_decay_pack[1], + q_decay_pack[2], + q_decay_pack[3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) @@ -1451,26 +1452,26 @@ def compute0_warp_group( (sQ_decay_ptr + decay_swizzled_idx).store(q_decay_vec, alignment=16) # ---- K_restore operand: K_inv * exp_g_last -------------------------- - bars.mb_k_restore_done[decay_stage].wait(((global_chunk // cfg.smem_decay_stages + 1) % 2)) + bars.mb_k_restore_acc_done[decay_stage].wait(((global_chunk // cfg.smem_decay_stages + 1) % 2)) for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 - k_restore_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_restore_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 exp_g_last_pair = fp32_to_fp16(exp_g_last_regs[reg_base + dim0], exp_g_last_regs[reg_base + dim1], dtype=cfg.io_dtype) - k_restore_words[pair_idx] = mul_f16x2(k_inv_words[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) + k_restore_pack[pair_idx] = mul_f16x2(k_inv_pack[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) storage_row = decay_row ^ (cfg.b_t // 2) f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 k_restore_idx = f16_segment * (cfg.b_t * 64) + storage_row * 64 + swizzle_xor_128b(storage_row, f16_segment_dim, elem_bytes=2) k_restore_vec = cutlass.Vector.from_elements( ( - k_restore_words[0], - k_restore_words[1], - k_restore_words[2], - k_restore_words[3], + k_restore_pack[0], + k_restore_pack[1], + k_restore_pack[2], + k_restore_pack[3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) @@ -1518,6 +1519,7 @@ def compute1_warp_group( sO_ptr = sO_raw.data_ptr() sCheckpoint_ptr = sCheckpoint_raw.data_ptr() if cutlass.const_expr(cfg.enable_checkpoints) else sO_raw.data_ptr() checkpoint_done_index = PipelineState.start(phase=1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = sTmem_base.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 @@ -1529,7 +1531,7 @@ def compute1_warp_group( state_k_acc_index = PipelineState.start(phase=0) update_acc_index = PipelineState.start(phase=0) o_acc_index = PipelineState.start(phase=0) - k_restore_index = PipelineState.start(phase=0) # CG1's per-chunk mb_k_restore_done wait slot + k_restore_index = PipelineState.start(phase=0) # CG1's per-chunk mb_k_restore_acc_done wait slot raw_index = PipelineState.start(phase=0) # raw-ring slot for the sV/sW reads + inputs_done arrives global_chunk_base = cutlass.Int32(0) sched_state = PipelineState.start(phase=0) @@ -1600,28 +1602,26 @@ def compute1_warp_group( state_col_id = tmem_col + cfg.tmem_state_acc_offset # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- - state_blocks = [] + state_vecs = [] for k_block in cutlass.range_constexpr(cfg.d_k // 16): - state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) + state_vecs.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) packed_col_id = tmem_col + cfg.tmem_state_inp_offset for k_block in cutlass.range_constexpr(cfg.d_k // 16): - packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) + state_pack = cutlass.Array(cutlass.Int32, 8, alignment=16) for packed_col in cutlass.range_constexpr(8): source_pair = packed_col ^ 4 - packed_state[packed_col] = fp32_to_fp16( - state_blocks[k_block][2 * source_pair], state_blocks[k_block][2 * source_pair + 1], dtype=cfg.io_dtype - ) + state_pack[packed_col] = fp32_to_fp16(state_vecs[k_block][2 * source_pair], state_vecs[k_block][2 * source_pair + 1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + k_block * 8, cutlass.Int8), - packed_state[0:8], + state_pack[0:8], ) nvvm.tcgen05_wait("store") bars.mb_state_inp_ready.arrive() if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_read_done.arrive() + bars.mb_state_acc_read_done.arrive() # ---- rhs staging: rhs input = W*V - state*(Beta*K) ------------------- bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) @@ -1633,12 +1633,12 @@ def compute1_warp_group( # ---- read back state*K acc + raw V fragments ------------------------- row_id0 = tmem_row + value_dim_base - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) row_id1 = row_id0 + 16 - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) - raw_v_regs0 = nvvm.ldmatrix( + raw_v_frag0 = nvvm.ldmatrix( sV_ptr + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 @@ -1646,7 +1646,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_v_regs1 = nvvm.ldmatrix( + raw_v_frag1 = nvvm.ldmatrix( sV_ptr + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 @@ -1654,7 +1654,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_w_regs0 = nvvm.ldmatrix( + raw_w_frag0 = nvvm.ldmatrix( sW_ptr + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 @@ -1662,7 +1662,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_w_regs1 = nvvm.ldmatrix( + raw_w_frag1 = nvvm.ldmatrix( sW_ptr + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 @@ -1671,45 +1671,45 @@ def compute1_warp_group( nvvm.MMALayout.COL, ) - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec0[frag_pair], state_k_vec0[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) - w_pair = raw_w_regs0[raw_matrix] + w_pair = raw_w_frag0[raw_matrix] wv_pair = mul_f16x2( w_pair, - raw_v_regs0[raw_matrix], + raw_v_frag0[raw_matrix], cfg.io_dtype, ) - packed_rhs0[reg_idx] = sub_f16x2( + rhs_pack0[reg_idx] = sub_f16x2( wv_pair, state_k_pair, cfg.io_dtype, ) - packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec1[frag_pair], state_k_vec1[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) - w_pair = raw_w_regs1[raw_matrix] + w_pair = raw_w_frag1[raw_matrix] wv_pair = mul_f16x2( w_pair, - raw_v_regs1[raw_matrix], + raw_v_frag1[raw_matrix], cfg.io_dtype, ) - packed_rhs1[reg_idx] = sub_f16x2( + rhs_pack1[reg_idx] = sub_f16x2( wv_pair, state_k_pair, cfg.io_dtype, ) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), packed_rhs0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), rhs_pack0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), rhs_pack1[0:4]) nvvm.tcgen05_wait("store") state_k_acc_index = advance(state_k_acc_index, 1) @@ -1725,23 +1725,23 @@ def compute1_warp_group( num=cfg.b_t, ) - packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + update_pack = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): source_pair = packed_col ^ 4 token0 = source_pair * 2 token1 = token0 + 1 - packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + update_pack[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_row << 16) + (tmem_col + cfg.tmem_update_inp_offset), cutlass.Int8), - packed_update[0 : (cfg.b_t // 2)], + update_pack[0 : (cfg.b_t // 2)], ) nvvm.tcgen05_wait("store") update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() - bars.mb_k_restore_done[k_restore_index.idx].wait(k_restore_index.phase) + bars.mb_k_restore_acc_done[k_restore_index.idx].wait(k_restore_index.phase) k_restore_index = advance(k_restore_index, cfg.smem_decay_stages) raw_index = advance(raw_index, cfg.smem_raw_stages) @@ -1770,29 +1770,26 @@ def compute1_warp_group( state_col_id = tmem_col + cfg.tmem_state_acc_offset # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- - state_blocks = [] + state_vecs = [] for k_block in cutlass.range_constexpr(cfg.d_k // 16): - state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) - bars.mb_o_tmastg_done[prev_o_stage].wait(((prev_global_chunk // cfg.smem_o_stages) + 1) % 2) + state_vecs.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) packed_col_id = tmem_col + cfg.tmem_state_inp_offset for k_block in cutlass.range_constexpr(cfg.d_k // 16): - packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) + state_pack = cutlass.Array(cutlass.Int32, 8, alignment=16) for packed_col in cutlass.range_constexpr(8): source_pair = packed_col ^ 4 - packed_state[packed_col] = fp32_to_fp16( - state_blocks[k_block][2 * source_pair], state_blocks[k_block][2 * source_pair + 1], dtype=cfg.io_dtype - ) + state_pack[packed_col] = fp32_to_fp16(state_vecs[k_block][2 * source_pair], state_vecs[k_block][2 * source_pair + 1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + k_block * 8, cutlass.Int8), - packed_state[0:8], + state_pack[0:8], ) nvvm.tcgen05_wait("store") bars.mb_state_inp_ready.arrive() # ---- checkpoint: post-publish f32 fragment read; the decay GEMM's ------ - # overwrite is held by mb_state_read_done ------------------------------ + # overwrite is held by mb_state_acc_read_done ------------------------------ if cutlass.const_expr(cfg.enable_checkpoints): if do_checkpoint: checkpoint_stage = checkpoint_done_index.idx @@ -1806,13 +1803,13 @@ def compute1_warp_group( checkpoint_swz_off = (checkpoint_vbase + 16 + frag_col_offset) // 64 * (cfg.d_k * 64) checkpoint_swz_col = (checkpoint_vbase + 16 + frag_col_offset) % 64 for k_block in cutlass.range_constexpr(cfg.d_k // 16): - checkpoint_ld0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) - checkpoint_ld1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) - checkpoint_st0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - checkpoint_st1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) + checkpoint_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) + checkpoint_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - checkpoint_st0[reg_idx] = fp32_to_fp16(checkpoint_ld0[2 * reg_idx], checkpoint_ld0[2 * reg_idx + 1], dtype=cfg.io_dtype) - checkpoint_st1[reg_idx] = fp32_to_fp16(checkpoint_ld1[2 * reg_idx], checkpoint_ld1[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_pack0[reg_idx] = fp32_to_fp16(checkpoint_vec0[2 * reg_idx], checkpoint_vec0[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_pack1[reg_idx] = fp32_to_fp16(checkpoint_vec1[2 * reg_idx], checkpoint_vec1[2 * reg_idx + 1], dtype=cfg.io_dtype) checkpoint_row = k_block * 16 + frag_row_coord nvvm.stmatrix( sCheckpoint_ptr @@ -1820,7 +1817,7 @@ def compute1_warp_group( + checkpoint_swz_off0 + checkpoint_row * 64 + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col0, elem_bytes=2), - checkpoint_st0.data_ptr().load(count=4, alignment=4), + checkpoint_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -1830,16 +1827,16 @@ def compute1_warp_group( + checkpoint_swz_off + checkpoint_row * 64 + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col, elem_bytes=2), - checkpoint_st1.data_ptr().load(count=4, alignment=4), + checkpoint_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) nvvm.tcgen05_wait("load") - bars.mb_state_read_done.arrive() + bars.mb_state_acc_read_done.arrive() nvvm.fence_proxy("async.shared", space="cta") bars.mb_checkpoint_tmastg_ready[checkpoint_stage].arrive() else: - bars.mb_state_read_done.arrive() + bars.mb_state_acc_read_done.arrive() bars.mb_o_acc_ready.wait(o_acc_index.phase) o_acc_index = advance(o_acc_index, 1) projection_col_id = tmem_col + cfg.tmem_q_state_acc_offset + prev_q_state_acc_stage * cfg.b_t @@ -1847,25 +1844,26 @@ def compute1_warp_group( row_id0 = tmem_row + value_dim_base row_id1 = row_id0 + 16 - loaded0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) - loaded1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + loaded_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + loaded_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) # ---- output drain: O acc TMEM -> scaled b16 SMEM -------------------- - stsm_regs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - stsm_regs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + stsm_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + stsm_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - scaled0_0, scaled0_1 = fmul2(loaded0[2 * reg_idx], loaded0[2 * reg_idx + 1], scale, scale) - scaled1_0, scaled1_1 = fmul2(loaded1[2 * reg_idx], loaded1[2 * reg_idx + 1], scale, scale) - stsm_regs0[reg_idx] = fp32_to_fp16(scaled0_0, scaled0_1, dtype=mO.element_type) - stsm_regs1[reg_idx] = fp32_to_fp16(scaled1_0, scaled1_1, dtype=mO.element_type) + scaled0_0, scaled0_1 = fmul2(loaded_vec0[2 * reg_idx], loaded_vec0[2 * reg_idx + 1], scale, scale) + scaled1_0, scaled1_1 = fmul2(loaded_vec1[2 * reg_idx], loaded_vec1[2 * reg_idx + 1], scale, scale) + stsm_pack0[reg_idx] = fp32_to_fp16(scaled0_0, scaled0_1, dtype=mO.element_type) + stsm_pack1[reg_idx] = fp32_to_fp16(scaled1_0, scaled1_1, dtype=mO.element_type) + bars.mb_o_tmastg_done[prev_o_stage].wait(((prev_global_chunk // cfg.smem_o_stages) + 1) % 2) nvvm.stmatrix( sO_ptr + prev_o_stage_base + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 + swizzle_xor_128b(frag_row_coord, (value_dim_base + frag_col_offset) % 64, elem_bytes=2), - stsm_regs0.data_ptr().load(count=4, alignment=4), + stsm_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -1875,7 +1873,7 @@ def compute1_warp_group( + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 + swizzle_xor_128b(frag_row_coord, (value_dim_base + 16 + frag_col_offset) % 64, elem_bytes=2), - stsm_regs1.data_ptr().load(count=4, alignment=4), + stsm_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -1883,9 +1881,8 @@ def compute1_warp_group( nvvm.fence_proxy("async.shared", space="cta") bars.mb_o_tmastg_ready[prev_o_stage].arrive() - bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) - # ---- rhs staging: rhs input = W*V - state*(Beta*K) ------------------- + bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) bars.mb_w_ready[raw_index.idx].wait(raw_index.phase) projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset @@ -1894,12 +1891,12 @@ def compute1_warp_group( # ---- read back state*K acc + raw V fragments ------------------------- row_id0 = tmem_row + value_dim_base - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) row_id1 = row_id0 + 16 - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) - raw_v_regs0 = nvvm.ldmatrix( + raw_v_frag0 = nvvm.ldmatrix( sV_ptr + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 @@ -1907,7 +1904,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_v_regs1 = nvvm.ldmatrix( + raw_v_frag1 = nvvm.ldmatrix( sV_ptr + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 @@ -1915,7 +1912,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_w_regs0 = nvvm.ldmatrix( + raw_w_frag0 = nvvm.ldmatrix( sW_ptr + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 @@ -1923,7 +1920,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_w_regs1 = nvvm.ldmatrix( + raw_w_frag1 = nvvm.ldmatrix( sW_ptr + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 @@ -1932,45 +1929,45 @@ def compute1_warp_group( nvvm.MMALayout.COL, ) - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec0[frag_pair], state_k_vec0[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) - w_pair = raw_w_regs0[raw_matrix] + w_pair = raw_w_frag0[raw_matrix] wv_pair = mul_f16x2( w_pair, - raw_v_regs0[raw_matrix], + raw_v_frag0[raw_matrix], cfg.io_dtype, ) - packed_rhs0[reg_idx] = sub_f16x2( + rhs_pack0[reg_idx] = sub_f16x2( wv_pair, state_k_pair, cfg.io_dtype, ) - packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec1[frag_pair], state_k_vec1[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) - w_pair = raw_w_regs1[raw_matrix] + w_pair = raw_w_frag1[raw_matrix] wv_pair = mul_f16x2( w_pair, - raw_v_regs1[raw_matrix], + raw_v_frag1[raw_matrix], cfg.io_dtype, ) - packed_rhs1[reg_idx] = sub_f16x2( + rhs_pack1[reg_idx] = sub_f16x2( wv_pair, state_k_pair, cfg.io_dtype, ) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), packed_rhs0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), rhs_pack0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), rhs_pack1[0:4]) nvvm.tcgen05_wait("store") state_k_acc_index = advance(state_k_acc_index, 1) @@ -1986,23 +1983,23 @@ def compute1_warp_group( num=cfg.b_t, ) - packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + update_pack = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): source_pair = packed_col ^ 4 token0 = source_pair * 2 token1 = token0 + 1 - packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + update_pack[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_row << 16) + (tmem_col + cfg.tmem_update_inp_offset), cutlass.Int8), - packed_update[0 : (cfg.b_t // 2)], + update_pack[0 : (cfg.b_t // 2)], ) nvvm.tcgen05_wait("store") update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() - bars.mb_k_restore_done[k_restore_index.idx].wait(k_restore_index.phase) + bars.mb_k_restore_acc_done[k_restore_index.idx].wait(k_restore_index.phase) k_restore_index = advance(k_restore_index, cfg.smem_decay_stages) raw_index = advance(raw_index, cfg.smem_raw_stages) @@ -2012,7 +2009,6 @@ def compute1_warp_group( final_o_stage = last_global_chunk % cfg.smem_o_stages final_q_state_acc_stage = last_global_chunk % cfg.tmem_q_state_acc_stages final_o_stage_base = final_o_stage * (cfg.b_t * cfg.d_v) - bars.mb_o_tmastg_done[final_o_stage].wait(((last_global_chunk // cfg.smem_o_stages) + 1) % 2) bars.mb_o_acc_ready.wait(o_acc_index.phase) o_acc_index = advance(o_acc_index, 1) @@ -2021,25 +2017,26 @@ def compute1_warp_group( row_id0 = tmem_row + value_dim_base row_id1 = row_id0 + 16 - loaded0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) - loaded1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + loaded_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + loaded_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) # ---- output drain: O acc TMEM -> scaled b16 SMEM -------------------- - stsm_regs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - stsm_regs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + stsm_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + stsm_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - scaled0_0, scaled0_1 = fmul2(loaded0[2 * reg_idx], loaded0[2 * reg_idx + 1], scale, scale) - scaled1_0, scaled1_1 = fmul2(loaded1[2 * reg_idx], loaded1[2 * reg_idx + 1], scale, scale) - stsm_regs0[reg_idx] = fp32_to_fp16(scaled0_0, scaled0_1, dtype=mO.element_type) - stsm_regs1[reg_idx] = fp32_to_fp16(scaled1_0, scaled1_1, dtype=mO.element_type) + scaled0_0, scaled0_1 = fmul2(loaded_vec0[2 * reg_idx], loaded_vec0[2 * reg_idx + 1], scale, scale) + scaled1_0, scaled1_1 = fmul2(loaded_vec1[2 * reg_idx], loaded_vec1[2 * reg_idx + 1], scale, scale) + stsm_pack0[reg_idx] = fp32_to_fp16(scaled0_0, scaled0_1, dtype=mO.element_type) + stsm_pack1[reg_idx] = fp32_to_fp16(scaled1_0, scaled1_1, dtype=mO.element_type) + bars.mb_o_tmastg_done[final_o_stage].wait(((last_global_chunk // cfg.smem_o_stages) + 1) % 2) nvvm.stmatrix( sO_ptr + final_o_stage_base + (value_dim_base + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 + swizzle_xor_128b(frag_row_coord, (value_dim_base + frag_col_offset) % 64, elem_bytes=2), - stsm_regs0.data_ptr().load(count=4, alignment=4), + stsm_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -2049,7 +2046,7 @@ def compute1_warp_group( + (value_dim_base + 16 + frag_col_offset) // 64 * (cfg.b_t * 64) + frag_row_coord * 64 + swizzle_xor_128b(frag_row_coord, (value_dim_base + 16 + frag_col_offset) % 64, elem_bytes=2), - stsm_regs1.data_ptr().load(count=4, alignment=4), + stsm_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -2081,10 +2078,11 @@ def compute1_warp_group( mState_out[batch_idx, head_o, key_dim, value_dim] = mState_init[batch_idx, head_o, key_dim, value_dim] else: mState_out[batch_idx, head_o, key_dim, value_dim] = cutlass.Float32(0.0).to(mState_out.element_type) - bars.mb_final_state_stored.arrive() global_chunk_base += num_tile_chunks tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].arrive() + @cute.jit def host( @@ -2280,8 +2278,9 @@ def kernel( layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, ) + elect_one = nvvm.elect_sync() if warp_idx == cfg.tma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_raw_stages): bars.mb_q_ready[stage].init() bars.mb_q_done[stage].init() @@ -2296,7 +2295,7 @@ def kernel( bars.mb_w_ready[stage].init() bars.mb_w_done[stage].init() elif warp_idx == cfg.tcgen05_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: bars.mb_o_acc_ready.init() for stage in cutlass.range_constexpr(cfg.tmem_q_state_acc_stages): bars.mb_o_acc_done[stage].init() @@ -2306,16 +2305,16 @@ def kernel( for stage in cutlass.range_constexpr(cfg.smem_state_scale_diag_stages): bars.mb_state_scale_diag_done[stage].init() for stage in cutlass.range_constexpr(cfg.smem_decay_stages): - bars.mb_decay_done[stage].init() + bars.mb_decay_tcgen05_done[stage].init() bars.mb_decay_super_done[stage].init() - bars.mb_k_restore_done[stage].init() + bars.mb_k_restore_acc_done[stage].init() bars.mb_rhs_ready.init() bars.mb_update_ready.init() - bars.mb_final_state_stored.init() + bars.mb_tmem_done[0].init() elif warp_idx == cfg.super_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_qk_stages): - bars.mb_a_ready[stage].init() + bars.mb_a_inv_ready[stage].init() bars.mb_qk_acc_ready[stage].init() bars.mb_a_done[stage].init() for stage in cutlass.range_constexpr(cfg.qk_scale_ready_stages): @@ -2323,7 +2322,7 @@ def kernel( for stage in cutlass.range_constexpr(cfg.smem_decay_stages): bars.mb_k_decay_inv_cg0_ready[stage].init() elif warp_idx == cfg.epilogue_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_o_stages): bars.mb_o_tmastg_ready[stage].init() bars.mb_o_tmastg_done[stage].init() @@ -2334,32 +2333,12 @@ def kernel( for stage in cutlass.range_constexpr(cfg.smem_checkpoint_stages): bars.mb_checkpoint_tmastg_ready[stage].init() bars.mb_checkpoint_tmastg_done[stage].init() - bars.mb_state_read_done.init() + bars.mb_state_acc_read_done.init() diag_zero = cfg.io_dtype(0.0) for diag_idx in cutlass.range(tidx, cfg.state_scale_diag_cosize, cfg.threads_per_cta, unroll=1): sState_scale_diag_raw[diag_idx] = diag_zero nvvm.fence_mbarrier_init() nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) - if (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) or warp_idx == cfg.tcgen05_mma_warp_id: - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_alloc(sTmem_base, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - - # Actual SMEM/TMEM buffers for the BT=16 schedule: - # Q/K/V : 16 x 128 each - # Gate_log2 : 16 x 128 - # Beta : 16 - # Q/K inverse norm : 16 each, staged once per decay stage - # exp_g_last : 128, CG0-local and staged once per decay stage - # state-scale diag : 8 x 16 x 16 input dtype, zeroed once in the prologue - # Q_decay/K_decay : tcgen05 SW128 operands shared with super-MMA - # K_restore : tcgen05 SW128 N-major final-state operand - # K_inv : 16 x 128 token-major for super-MMA RHS - # A inverse/QK : 16 x 16 each, plus transposed tcgen05 operands - if warp_idx == cfg.tma_warp_id: tmaldg_warp( cfg, @@ -2784,17 +2763,22 @@ def build_descs( tma_granu_elems = 128 // bpe seqlen = q.shape[0] - def headed(t, dim, hn): - return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) + q_headed = cute.make_tensor(q.iterator, cute.make_layout((d_k, h_q, seqlen), stride=(1, q.stride[1], q.stride[0]))) + k_headed = cute.make_tensor(k.iterator, cute.make_layout((d_k, h_k, seqlen), stride=(1, k.stride[1], k.stride[0]))) + v_headed = cute.make_tensor(v.iterator, cute.make_layout((d_v, h_v, seqlen), stride=(1, v.stride[1], v.stride[0]))) + gate_headed = cute.make_tensor(gate.iterator, cute.make_layout((d_k, n_heads_out, seqlen), stride=(1, gate.stride[1], gate.stride[0]))) + beta_headed = cute.make_tensor(beta.iterator, cute.make_layout((d_k, n_heads_out, seqlen), stride=(1, beta.stride[1], beta.stride[0]))) + w_headed = cute.make_tensor(w.iterator, cute.make_layout((d_v, n_heads_out, seqlen), stride=(1, w.stride[1], w.stride[0]))) + o_headed = cute.make_tensor(o.iterator, cute.make_layout((d_v, n_heads_out, seqlen), stride=(1, o.stride[1], o.stride[0]))) swz = cuda.TensorMapSwizzle.s128b - base_q = cuda.create_tensor_map_tiled_from_view(headed(q, d_k, h_q), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, n_heads_out), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_beta = cuda.create_tensor_map_tiled_from_view(headed(beta, d_k, n_heads_out), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_w = cuda.create_tensor_map_tiled_from_view(headed(w, d_v, n_heads_out), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_o = cuda.create_tensor_map_tiled_from_view(headed(o, d_v, n_heads_out), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_q = cuda.create_tensor_map_tiled_from_view(q_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_k = cuda.create_tensor_map_tiled_from_view(k_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(v_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(gate_headed, box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_beta = cuda.create_tensor_map_tiled_from_view(beta_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_w = cuda.create_tensor_map_tiled_from_view(w_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_o = cuda.create_tensor_map_tiled_from_view(o_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) base_checkpoint = base_o if cutlass.const_expr(state_checkpoints is not None): diff --git a/python/cudnn/linear_attention/frost/kernel/gdn2_recompute_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn2_recompute_f16.py index db8b56d14..5c5709b9b 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn2_recompute_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn2_recompute_f16.py @@ -153,16 +153,16 @@ class Gdn2Bars(NamedTuple): mb_update_ready: MBarrier mb_k_decay_inv_cg0_ready: MBarrier - mb_decay_done: MBarrier + mb_decay_tcgen05_done: MBarrier mb_decay_super_done: MBarrier - mb_k_restore_done: MBarrier + mb_k_restore_acc_done: MBarrier mb_qk_scale_ready: MBarrier mb_state_scale_diag_done: MBarrier - mb_a_ready: MBarrier + mb_a_inv_ready: MBarrier mb_a_done: MBarrier - mb_state_read_done: MBarrier - mb_final_state_stored: MBarrier + mb_state_acc_read_done: MBarrier + mb_tmem_done: MBarrier mb_checkpoint_tmastg_ready: MBarrier mb_checkpoint_tmastg_done: MBarrier @@ -199,9 +199,9 @@ def alloc(n): mb_rhs_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_update_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_k_decay_inv_cg0_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), - mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_decay_tcgen05_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=WARP, producer=Producer.THREAD), - mb_k_restore_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_k_restore_acc_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_qk_scale_ready=MBarrier( alloc(cfg.qk_scale_ready_stages), stages=cfg.qk_scale_ready_stages, @@ -214,10 +214,10 @@ def alloc(n): init_count=1, producer=Producer.MMA_COMMIT, ), - mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_a_inv_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), mb_a_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=1, producer=Producer.MMA_COMMIT), - mb_state_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_final_state_stored=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_state_acc_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_tmem_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_checkpoint_tmastg_ready=MBarrier( alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=CG1_THREADS, producer=Producer.THREAD ), @@ -474,7 +474,7 @@ def super_mma_warp( # Load B operand k_inv_col = k_block * 16 + rhs_col_offset k_inv_segment = k_inv_col // 64 - rhs_vec = nvvm.ldmatrix( + rhs_frag = nvvm.ldmatrix( sK_inv_ptr + k_inv_segment * (cfg.b_t * 64) + rhs_row_coord * 64 @@ -485,7 +485,7 @@ def super_mma_warp( # Load A operand storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask storage_slice = storage_key // 64 - kk_lhs_vec = nvvm.ldmatrix( + kk_lhs_frag = nvvm.ldmatrix( sK_decay_ptr + storage_slice * (cfg.b_t * 64) + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), @@ -495,8 +495,8 @@ def super_mma_warp( mma_step( kk_acc, - (kk_lhs_vec[0], kk_lhs_vec[1], kk_lhs_vec[2], kk_lhs_vec[3]), - (rhs_vec[0], rhs_vec[1], rhs_vec[2], rhs_vec[3]), + (kk_lhs_frag[0], kk_lhs_frag[1], kk_lhs_frag[2], kk_lhs_frag[3]), + (rhs_frag[0], rhs_frag[1], rhs_frag[2], rhs_frag[3]), k_step=0, M=16, N=16, @@ -506,7 +506,7 @@ def super_mma_warp( # ---- L = tril(KK, -1) fragment --------------------------------------- row_lo = lane // 4 row_hi = row_lo + cutlass.Int32(8) - l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) + l_regs = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): row_coord = row_lo if cutlass.const_expr(accum_idx % 4 >= 2): @@ -514,11 +514,11 @@ def super_mma_warp( col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) if cutlass.const_expr(accum_idx % 2 == 1): col_coord = col_coord + cutlass.Int32(1) - l_frag[accum_idx] = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) - l_a0 = fp32_to_fp16(l_frag[0], l_frag[1], dtype=cfg.io_dtype) - l_a1 = fp32_to_fp16(l_frag[2], l_frag[3], dtype=cfg.io_dtype) - l_a2 = fp32_to_fp16(l_frag[4], l_frag[5], dtype=cfg.io_dtype) - l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) + l_regs[accum_idx] = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + l_a0 = fp32_to_fp16(l_regs[0], l_regs[1], dtype=cfg.io_dtype) + l_a1 = fp32_to_fp16(l_regs[2], l_regs[3], dtype=cfg.io_dtype) + l_a2 = fp32_to_fp16(l_regs[4], l_regs[5], dtype=cfg.io_dtype) + l_a3 = fp32_to_fp16(l_regs[6], l_regs[7], dtype=cfg.io_dtype) l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) # ---- A_inv = I - L, then three Neumann doubling rounds --------------- @@ -596,7 +596,7 @@ def super_mma_warp( shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_a_ready[qk_stage].arrive() + bars.mb_a_inv_ready[qk_stage].arrive() bars.mb_decay_super_done[decay_stage].arrive() global_chunk_base += num_chunks_tile tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) @@ -622,6 +622,8 @@ def tcgen05_mma_warp( tcgen05 GEMM.""" elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + nvvm.tcgen05_alloc(tmem_base_holder, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_base_holder.load() state_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) state_dsts = tuple(nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset + k * 16, cutlass.Float32) for k in range(cfg.d_k // 16)) @@ -634,7 +636,6 @@ def tcgen05_mma_warp( state_read_index = PipelineState.start(phase=0) rhs_index = PipelineState.start(phase=0) update_index = PipelineState.start(phase=0) - final_state_index = PipelineState.start(phase=0) qk_scale_index = PipelineState.start(phase=0) # ---- chunk-invariant GEMM descriptors ---------------------------------------- @@ -750,10 +751,10 @@ def tcgen05_mma_warp( if elect_one: bars.mb_state_k_acc_ready.arrive(cta_group=1) - bars.mb_decay_done[decay_stage].arrive(cta_group=1) + bars.mb_decay_tcgen05_done[decay_stage].arrive(cta_group=1) if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_read_done.wait(state_read_index.phase) + bars.mb_state_acc_read_done.wait(state_read_index.phase) state_read_index = advance(state_read_index, 1) # ---- state decay = state(T) @ diag(exp2(g_last)) (per-k-atom blocks) -- @@ -774,7 +775,7 @@ def tcgen05_mma_warp( bars.mb_state_scale_diag_done[state_scale_diag_stage].arrive(cta_group=1) # ---- update_acc = rhs(T) @ A_inv^T ----------------------------------- - bars.mb_a_ready[qk_stage].wait((global_chunk // cfg.smem_qk_stages) % 2) + bars.mb_a_inv_ready[qk_stage].wait((global_chunk // cfg.smem_qk_stages) % 2) bars.mb_rhs_ready.wait(rhs_index.phase) rhs_index = advance(rhs_index, 1) desc_qk = sQk_stage.desc() @@ -790,14 +791,14 @@ def tcgen05_mma_warp( mma_ts_step(bmm_final_state_desc, update_inp_ptr, desc_k_restore, state_dst_ptr, 0, cutlass.Boolean(True)) if elect_one: - bars.mb_k_restore_done[decay_stage].arrive(cta_group=1) + bars.mb_k_restore_acc_done[decay_stage].arrive(cta_group=1) qk_scale_index = advance(qk_scale_index, cfg.smem_state_scale_diag_stages) - bars.mb_final_state_stored.wait(final_state_index.phase) - final_state_index = advance(final_state_index, 1) global_chunk_base += num_chunks_tile tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].wait(0) + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) nvvm.tcgen05_dealloc( nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), cutlass.Int32(512), @@ -1040,7 +1041,7 @@ def compute0_warp_group( nvvm.barrier_cta_sync(cfg.cg0_group_sync_barrier_base_id + cg0_group_id, thread_count=cfg.cg0_threads_per_group) - k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) + k_inv_pack = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) raw_beta_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) @@ -1056,14 +1057,14 @@ def compute0_warp_group( f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 raw_f16_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) - raw_k_vec = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_beta_vec = (sBeta_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_k_vec_f32 = raw_k_vec.to(cutlass.Float32) - raw_beta_vec_f32 = raw_beta_vec.to(cutlass.Float32) + raw_k_frag = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_beta_frag = (sBeta_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_k_frag_f32 = raw_k_frag.to(cutlass.Float32) + raw_beta_frag_f32 = raw_beta_frag.to(cutlass.Float32) for dim_offset in cutlass.range_constexpr(8): - k_val = raw_k_vec_f32[dim_offset] + k_val = raw_k_frag_f32[dim_offset] raw_k_regs[reg_base + dim_offset] = k_val - beta_val = raw_beta_vec_f32[dim_offset] + beta_val = raw_beta_frag_f32[dim_offset] raw_beta_regs[reg_base + dim_offset] = beta_val if cutlass.const_expr(cfg.l2norm): if cutlass.const_expr(dim_offset % 2 == 0): @@ -1092,28 +1093,28 @@ def compute0_warp_group( f32_segment = f32_dim_base // 32 f32_segment_dim = f32_dim_base - f32_segment * 32 g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) - exp_g_vec = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) + exp_g_frag = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) f32_segment = f32_dim_base // 32 f32_segment_dim = f32_dim_base - f32_segment * 32 exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) - exp_g_last_vec = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) + exp_g_last_frag = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) half_reg_base = f32_group * 4 f32_reg_base = reg_base + half_reg_base - exp_g_regs[f32_reg_base] = exp_g_vec[0] - exp_g_regs[f32_reg_base + 1] = exp_g_vec[1] - exp_g_regs[f32_reg_base + 2] = exp_g_vec[2] - exp_g_regs[f32_reg_base + 3] = exp_g_vec[3] - exp_neg_g_regs[half_reg_base] = cute.math.rcp(exp_g_vec[0], approx=True, ftz=True) - exp_neg_g_regs[half_reg_base + 1] = cute.math.rcp(exp_g_vec[1], approx=True, ftz=True) - exp_neg_g_regs[half_reg_base + 2] = cute.math.rcp(exp_g_vec[2], approx=True, ftz=True) - exp_neg_g_regs[half_reg_base + 3] = cute.math.rcp(exp_g_vec[3], approx=True, ftz=True) - exp_g_last_regs[f32_reg_base] = exp_g_last_vec[0] - exp_g_last_regs[f32_reg_base + 1] = exp_g_last_vec[1] - exp_g_last_regs[f32_reg_base + 2] = exp_g_last_vec[2] - exp_g_last_regs[f32_reg_base + 3] = exp_g_last_vec[3] + exp_g_regs[f32_reg_base] = exp_g_frag[0] + exp_g_regs[f32_reg_base + 1] = exp_g_frag[1] + exp_g_regs[f32_reg_base + 2] = exp_g_frag[2] + exp_g_regs[f32_reg_base + 3] = exp_g_frag[3] + exp_neg_g_regs[half_reg_base] = cute.math.rcp(exp_g_frag[0], approx=True, ftz=True) + exp_neg_g_regs[half_reg_base + 1] = cute.math.rcp(exp_g_frag[1], approx=True, ftz=True) + exp_neg_g_regs[half_reg_base + 2] = cute.math.rcp(exp_g_frag[2], approx=True, ftz=True) + exp_neg_g_regs[half_reg_base + 3] = cute.math.rcp(exp_g_frag[3], approx=True, ftz=True) + exp_g_last_regs[f32_reg_base] = exp_g_last_frag[0] + exp_g_last_regs[f32_reg_base + 1] = exp_g_last_frag[1] + exp_g_last_regs[f32_reg_base + 2] = exp_g_last_frag[2] + exp_g_last_regs[f32_reg_base + 3] = exp_g_last_frag[3] # ---- K_decay + K_inv operands: K * exp2(+g) and K * exp2(-g) ----- - k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_decay_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 @@ -1123,32 +1124,32 @@ def compute0_warp_group( k_beta0, k_beta1 = fmul2(k_value0, k_value1, raw_beta_regs[raw_reg_idx0], raw_beta_regs[raw_reg_idx1]) k_pair = fp32_to_fp16(k_beta0, k_beta1, dtype=cfg.io_dtype) exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) - k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) + k_decay_pack[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) exp_neg_pair = fp32_to_fp16(exp_neg_g_regs[dim0], exp_neg_g_regs[dim1], dtype=cfg.io_dtype) k_norm_pair = fp32_to_fp16(k_value0, k_value1, dtype=cfg.io_dtype) - k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_norm_pair, exp_neg_pair, cfg.io_dtype) + k_inv_pack[dim_half * 4 + pair_idx] = mul_f16x2(k_norm_pair, exp_neg_pair, cfg.io_dtype) k_inv_vec = cutlass.Vector.from_elements( ( - k_inv_words[dim_half * 4], - k_inv_words[dim_half * 4 + 1], - k_inv_words[dim_half * 4 + 2], - k_inv_words[dim_half * 4 + 3], + k_inv_pack[dim_half * 4], + k_inv_pack[dim_half * 4 + 1], + k_inv_pack[dim_half * 4 + 2], + k_inv_pack[dim_half * 4 + 3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) k_decay_vec = cutlass.Vector.from_elements( ( - k_decay_words[0], - k_decay_words[1], - k_decay_words[2], - k_decay_words[3], + k_decay_pack[0], + k_decay_pack[1], + k_decay_pack[2], + k_decay_pack[3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) if cutlass.const_expr(dim_half == 0): operand_done_phase = ((global_chunk // cfg.smem_decay_stages) + 1) % 2 - bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_tcgen05_done[decay_stage].wait(operand_done_phase) bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 @@ -1167,26 +1168,26 @@ def compute0_warp_group( bars.mb_beta_done[raw_stage].arrive() # ---- K_restore operand: K_inv * exp_g_last -------------------------- - bars.mb_k_restore_done[decay_stage].wait(((global_chunk // cfg.smem_decay_stages + 1) % 2)) + bars.mb_k_restore_acc_done[decay_stage].wait(((global_chunk // cfg.smem_decay_stages + 1) % 2)) for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 - k_restore_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_restore_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 exp_g_last_pair = fp32_to_fp16(exp_g_last_regs[reg_base + dim0], exp_g_last_regs[reg_base + dim1], dtype=cfg.io_dtype) - k_restore_words[pair_idx] = mul_f16x2(k_inv_words[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) + k_restore_pack[pair_idx] = mul_f16x2(k_inv_pack[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) storage_row = decay_row ^ (cfg.b_t // 2) f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 k_restore_idx = f16_segment * (cfg.b_t * 64) + storage_row * 64 + swizzle_xor_128b(storage_row, f16_segment_dim, elem_bytes=2) k_restore_vec = cutlass.Vector.from_elements( ( - k_restore_words[0], - k_restore_words[1], - k_restore_words[2], - k_restore_words[3], + k_restore_pack[0], + k_restore_pack[1], + k_restore_pack[2], + k_restore_pack[3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) @@ -1230,6 +1231,7 @@ def compute1_warp_group( nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) sCheckpoint_ptr = sCheckpoint_raw.data_ptr() checkpoint_done_index = PipelineState.start(phase=1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_base_holder.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 @@ -1241,7 +1243,7 @@ def compute1_warp_group( value_dim = tmem_subpartition * cfg.threads_per_warp + lane state_k_acc_index = PipelineState.start(phase=0) update_acc_index = PipelineState.start(phase=0) - k_restore_index = PipelineState.start(phase=0) # CG1's per-chunk mb_k_restore_done wait slot + k_restore_index = PipelineState.start(phase=0) # CG1's per-chunk mb_k_restore_acc_done wait slot raw_index = PipelineState.start(phase=0) # raw-ring slot for the sV/sW reads + inputs_done arrives sched_state = PipelineState.start(phase=0) tile_idx = cutlass.Int32(bidx) @@ -1311,28 +1313,26 @@ def compute1_warp_group( state_col_id = tmem_col + cfg.tmem_state_acc_offset # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- - state_blocks = [] + state_vecs = [] for k_block in cutlass.range_constexpr(cfg.d_k // 16): - state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) + state_vecs.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) packed_col_id = tmem_col + cfg.tmem_state_inp_offset for k_block in cutlass.range_constexpr(cfg.d_k // 16): - packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) + state_pack = cutlass.Array(cutlass.Int32, 8, alignment=16) for packed_col in cutlass.range_constexpr(8): source_pair = packed_col ^ 4 - packed_state[packed_col] = fp32_to_fp16( - state_blocks[k_block][2 * source_pair], state_blocks[k_block][2 * source_pair + 1], dtype=cfg.io_dtype - ) + state_pack[packed_col] = fp32_to_fp16(state_vecs[k_block][2 * source_pair], state_vecs[k_block][2 * source_pair + 1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + k_block * 8, cutlass.Int8), - packed_state[0:8], + state_pack[0:8], ) nvvm.tcgen05_wait("store") bars.mb_state_inp_ready.arrive() if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_read_done.arrive() + bars.mb_state_acc_read_done.arrive() # ---- rhs staging: rhs input = W*V - state*(Beta*K) ------------------- bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) @@ -1344,12 +1344,12 @@ def compute1_warp_group( # ---- read back state*K acc + raw V fragments ------------------------- row_id0 = tmem_row + value_dim_base - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) row_id1 = row_id0 + 16 - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) - raw_v_regs0 = nvvm.ldmatrix( + raw_v_frag0 = nvvm.ldmatrix( sV_ptr + (value_dim_base + ldsm_col_offset) // 64 * (cfg.b_t * 64) + ldsm_row_coord * 64 @@ -1357,7 +1357,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_v_regs1 = nvvm.ldmatrix( + raw_v_frag1 = nvvm.ldmatrix( sV_ptr + (value_dim_base + 16 + ldsm_col_offset) // 64 * (cfg.b_t * 64) + ldsm_row_coord * 64 @@ -1365,7 +1365,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_w_regs0 = nvvm.ldmatrix( + raw_w_frag0 = nvvm.ldmatrix( sW_ptr + (value_dim_base + ldsm_col_offset) // 64 * (cfg.b_t * 64) + ldsm_row_coord * 64 @@ -1373,7 +1373,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_w_regs1 = nvvm.ldmatrix( + raw_w_frag1 = nvvm.ldmatrix( sW_ptr + (value_dim_base + 16 + ldsm_col_offset) // 64 * (cfg.b_t * 64) + ldsm_row_coord * 64 @@ -1382,45 +1382,45 @@ def compute1_warp_group( nvvm.MMALayout.COL, ) - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec0[frag_pair], state_k_vec0[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) - w_pair = raw_w_regs0[raw_matrix] + w_pair = raw_w_frag0[raw_matrix] wv_pair = mul_f16x2( w_pair, - raw_v_regs0[raw_matrix], + raw_v_frag0[raw_matrix], cfg.io_dtype, ) - packed_rhs0[reg_idx] = sub_f16x2( + rhs_pack0[reg_idx] = sub_f16x2( wv_pair, state_k_pair, cfg.io_dtype, ) - packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec1[frag_pair], state_k_vec1[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) - w_pair = raw_w_regs1[raw_matrix] + w_pair = raw_w_frag1[raw_matrix] wv_pair = mul_f16x2( w_pair, - raw_v_regs1[raw_matrix], + raw_v_frag1[raw_matrix], cfg.io_dtype, ) - packed_rhs1[reg_idx] = sub_f16x2( + rhs_pack1[reg_idx] = sub_f16x2( wv_pair, state_k_pair, cfg.io_dtype, ) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), packed_rhs0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), rhs_pack0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), rhs_pack1[0:4]) nvvm.tcgen05_wait("store") state_k_acc_index = advance(state_k_acc_index, 1) @@ -1436,23 +1436,23 @@ def compute1_warp_group( num=cfg.b_t, ) - packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + update_pack = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): source_pair = packed_col ^ 4 token0 = source_pair * 2 token1 = token0 + 1 - packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + update_pack[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_row << 16) + (tmem_col + cfg.tmem_update_inp_offset), cutlass.Int8), - packed_update[0 : (cfg.b_t // 2)], + update_pack[0 : (cfg.b_t // 2)], ) nvvm.tcgen05_wait("store") update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() - bars.mb_k_restore_done[k_restore_index.idx].wait(k_restore_index.phase) + bars.mb_k_restore_acc_done[k_restore_index.idx].wait(k_restore_index.phase) k_restore_index = advance(k_restore_index, cfg.smem_decay_stages) raw_index = advance(raw_index, cfg.smem_raw_stages) @@ -1475,22 +1475,20 @@ def compute1_warp_group( state_col_id = tmem_col + cfg.tmem_state_acc_offset # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- - state_blocks = [] + state_vecs = [] for k_block in cutlass.range_constexpr(cfg.d_k // 16): - state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) + state_vecs.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=16)) packed_col_id = tmem_col + cfg.tmem_state_inp_offset for k_block in cutlass.range_constexpr(cfg.d_k // 16): - packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) + state_pack = cutlass.Array(cutlass.Int32, 8, alignment=16) for packed_col in cutlass.range_constexpr(8): source_pair = packed_col ^ 4 - packed_state[packed_col] = fp32_to_fp16( - state_blocks[k_block][2 * source_pair], state_blocks[k_block][2 * source_pair + 1], dtype=cfg.io_dtype - ) + state_pack[packed_col] = fp32_to_fp16(state_vecs[k_block][2 * source_pair], state_vecs[k_block][2 * source_pair + 1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + k_block * 8, cutlass.Int8), - packed_state[0:8], + state_pack[0:8], ) nvvm.tcgen05_wait("store") bars.mb_state_inp_ready.arrive() @@ -1509,13 +1507,13 @@ def compute1_warp_group( checkpoint_swz_off = (checkpoint_vbase + 16 + ldsm_col_offset) // 64 * (cfg.d_k * 64) checkpoint_swz_col = (checkpoint_vbase + 16 + ldsm_col_offset) % 64 for k_block in cutlass.range_constexpr(cfg.d_k // 16): - checkpoint_ld0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) - checkpoint_ld1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) - checkpoint_st0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - checkpoint_st1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) + checkpoint_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + k_block * 16, cutlass.Float32), num=2) + checkpoint_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - checkpoint_st0[reg_idx] = fp32_to_fp16(checkpoint_ld0[2 * reg_idx], checkpoint_ld0[2 * reg_idx + 1], dtype=cfg.io_dtype) - checkpoint_st1[reg_idx] = fp32_to_fp16(checkpoint_ld1[2 * reg_idx], checkpoint_ld1[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_pack0[reg_idx] = fp32_to_fp16(checkpoint_vec0[2 * reg_idx], checkpoint_vec0[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_pack1[reg_idx] = fp32_to_fp16(checkpoint_vec1[2 * reg_idx], checkpoint_vec1[2 * reg_idx + 1], dtype=cfg.io_dtype) checkpoint_row = k_block * 16 + ldsm_row_coord nvvm.stmatrix( sCheckpoint_ptr @@ -1523,7 +1521,7 @@ def compute1_warp_group( + checkpoint_swz_off0 + checkpoint_row * 64 + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col0, elem_bytes=2), - checkpoint_st0.data_ptr().load(count=4, alignment=4), + checkpoint_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -1533,19 +1531,18 @@ def compute1_warp_group( + checkpoint_swz_off + checkpoint_row * 64 + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col, elem_bytes=2), - checkpoint_st1.data_ptr().load(count=4, alignment=4), + checkpoint_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) nvvm.tcgen05_wait("load") - bars.mb_state_read_done.arrive() + bars.mb_state_acc_read_done.arrive() nvvm.fence_proxy("async.shared", space="cta") bars.mb_checkpoint_tmastg_ready[checkpoint_stage].arrive() else: - bars.mb_state_read_done.arrive() - bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) - + bars.mb_state_acc_read_done.arrive() # ---- rhs staging: rhs input = W*V - state*(Beta*K) ------------------- + bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) bars.mb_w_ready[raw_index.idx].wait(raw_index.phase) projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset @@ -1554,12 +1551,12 @@ def compute1_warp_group( # ---- read back state*K acc + raw V fragments ------------------------- row_id0 = tmem_row + value_dim_base - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) row_id1 = row_id0 + 16 - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) - raw_v_regs0 = nvvm.ldmatrix( + raw_v_frag0 = nvvm.ldmatrix( sV_ptr + (value_dim_base + ldsm_col_offset) // 64 * (cfg.b_t * 64) + ldsm_row_coord * 64 @@ -1567,7 +1564,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_v_regs1 = nvvm.ldmatrix( + raw_v_frag1 = nvvm.ldmatrix( sV_ptr + (value_dim_base + 16 + ldsm_col_offset) // 64 * (cfg.b_t * 64) + ldsm_row_coord * 64 @@ -1575,7 +1572,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_w_regs0 = nvvm.ldmatrix( + raw_w_frag0 = nvvm.ldmatrix( sW_ptr + (value_dim_base + ldsm_col_offset) // 64 * (cfg.b_t * 64) + ldsm_row_coord * 64 @@ -1583,7 +1580,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_w_regs1 = nvvm.ldmatrix( + raw_w_frag1 = nvvm.ldmatrix( sW_ptr + (value_dim_base + 16 + ldsm_col_offset) // 64 * (cfg.b_t * 64) + ldsm_row_coord * 64 @@ -1592,45 +1589,45 @@ def compute1_warp_group( nvvm.MMALayout.COL, ) - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec0[frag_pair], state_k_vec0[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) - w_pair = raw_w_regs0[raw_matrix] + w_pair = raw_w_frag0[raw_matrix] wv_pair = mul_f16x2( w_pair, - raw_v_regs0[raw_matrix], + raw_v_frag0[raw_matrix], cfg.io_dtype, ) - packed_rhs0[reg_idx] = sub_f16x2( + rhs_pack0[reg_idx] = sub_f16x2( wv_pair, state_k_pair, cfg.io_dtype, ) - packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec1[frag_pair], state_k_vec1[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) - w_pair = raw_w_regs1[raw_matrix] + w_pair = raw_w_frag1[raw_matrix] wv_pair = mul_f16x2( w_pair, - raw_v_regs1[raw_matrix], + raw_v_frag1[raw_matrix], cfg.io_dtype, ) - packed_rhs1[reg_idx] = sub_f16x2( + rhs_pack1[reg_idx] = sub_f16x2( wv_pair, state_k_pair, cfg.io_dtype, ) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), packed_rhs0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row << 16) + input_col_id, cutlass.Int8), rhs_pack0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr((tmem_row + 16 << 16) + input_col_id, cutlass.Int8), rhs_pack1[0:4]) nvvm.tcgen05_wait("store") state_k_acc_index = advance(state_k_acc_index, 1) @@ -1646,23 +1643,23 @@ def compute1_warp_group( num=cfg.b_t, ) - packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + update_pack = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): source_pair = packed_col ^ 4 token0 = source_pair * 2 token1 = token0 + 1 - packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + update_pack[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_row << 16) + (tmem_col + cfg.tmem_update_inp_offset), cutlass.Int8), - packed_update[0 : (cfg.b_t // 2)], + update_pack[0 : (cfg.b_t // 2)], ) nvvm.tcgen05_wait("store") update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() - bars.mb_k_restore_done[k_restore_index.idx].wait(k_restore_index.phase) + bars.mb_k_restore_acc_done[k_restore_index.idx].wait(k_restore_index.phase) k_restore_index = advance(k_restore_index, cfg.smem_decay_stages) raw_index = advance(raw_index, cfg.smem_raw_stages) @@ -1673,7 +1670,7 @@ def compute1_warp_group( if seqlen_b > 0: if owns_final: for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): - loaded = nvvm.tcgen05_ld( + state_vec = nvvm.tcgen05_ld( "32x32b", nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), num=32, @@ -1681,7 +1678,7 @@ def compute1_warp_group( for col in cutlass.range_constexpr(32): key_dim = key_block_start + col - mState_out[batch_idx, head_o, key_dim, value_dim] = loaded[col].to(mState_out.element_type) + mState_out[batch_idx, head_o, key_dim, value_dim] = state_vec[col].to(mState_out.element_type) else: for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): for col in cutlass.range_constexpr(32): @@ -1690,9 +1687,10 @@ def compute1_warp_group( mState_out[batch_idx, head_o, key_dim, value_dim] = mState_init[batch_idx, head_o, key_dim, value_dim] else: mState_out[batch_idx, head_o, key_dim, value_dim] = cutlass.Float32(0.0).to(mState_out.element_type) - bars.mb_final_state_stored.arrive() tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].arrive() + @cute.jit def host( @@ -1856,8 +1854,9 @@ def kernel( layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, ) + elect_one = nvvm.elect_sync() if warp_idx == cfg.tma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_raw_stages): bars.mb_k_ready[stage].init() bars.mb_k_done[stage].init() @@ -1870,30 +1869,30 @@ def kernel( bars.mb_w_ready[stage].init() bars.mb_w_done[stage].init() elif warp_idx == cfg.tcgen05_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: bars.mb_state_k_acc_ready.init() bars.mb_update_acc_ready.init() bars.mb_state_inp_ready.init() for stage in cutlass.range_constexpr(cfg.smem_state_scale_diag_stages): bars.mb_state_scale_diag_done[stage].init() for stage in cutlass.range_constexpr(cfg.smem_decay_stages): - bars.mb_decay_done[stage].init() + bars.mb_decay_tcgen05_done[stage].init() bars.mb_decay_super_done[stage].init() - bars.mb_k_restore_done[stage].init() + bars.mb_k_restore_acc_done[stage].init() bars.mb_rhs_ready.init() bars.mb_update_ready.init() - bars.mb_final_state_stored.init() + bars.mb_tmem_done[0].init() elif warp_idx == cfg.super_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_qk_stages): - bars.mb_a_ready[stage].init() + bars.mb_a_inv_ready[stage].init() bars.mb_a_done[stage].init() for stage in cutlass.range_constexpr(cfg.qk_scale_ready_stages): bars.mb_qk_scale_ready[stage].init() for stage in cutlass.range_constexpr(cfg.smem_decay_stages): bars.mb_k_decay_inv_cg0_ready[stage].init() elif warp_idx == cfg.epilogue_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.sched_stages): bars.mb_sched_ready[stage].init() bars.mb_sched_done[stage].init() @@ -1901,36 +1900,12 @@ def kernel( for stage in cutlass.range_constexpr(cfg.smem_checkpoint_stages): bars.mb_checkpoint_tmastg_ready[stage].init() bars.mb_checkpoint_tmastg_done[stage].init() - bars.mb_state_read_done.init() + bars.mb_state_acc_read_done.init() diag_zero = cfg.io_dtype(0.0) for diag_idx in cutlass.range(tidx, cfg.state_scale_diag_cosize, cfg.threads_per_cta, unroll=1): sState_scale_diag_raw[diag_idx] = diag_zero nvvm.fence_mbarrier_init() nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) - if (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) or warp_idx == cfg.tcgen05_mma_warp_id: - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_alloc(tmem_base_holder, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - - # Actual SMEM/TMEM buffers for the BT=16 schedule: - # K/V : 16 x 128 each - # gate_log2 : 16 x 128 - # Beta : 16 - # K inverse norm : 16, staged once per decay stage - # exp_g_last : 128, CG0-local and staged once per decay stage - # state-scale diag : 8 x 16 x 16 input dtype, zeroed once in the prologue - # K_decay : tcgen05 SW128 operands shared with super-MMA - # K_restore : tcgen05 SW128 N-major final-state operand - # K_inv : 16 x 128 token-major for super-MMA RHS - # A inverse : 16 x 16, plus transposed tcgen05 operands - # state : external/kernel ABI is VK `[DV, DK]`; reference - # math can view it as KV `[DK, DV]` by transposing. - # The TS A-staging path keeps VK in TMEM so state*K - # is `[DV, DK] @ [DK, BT] -> [DV, BT]` with M=128. - if warp_idx == cfg.tma_warp_id: tmaldg_warp( cfg, @@ -2291,15 +2266,18 @@ def build_descs( tma_box_elems = 128 // bpe seqlen = k.shape[0] - def headed(t, dim, hn): - return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) + k_headed = cute.make_tensor(k.iterator, cute.make_layout((d_k, h_k, seqlen), stride=(1, k.stride[1], k.stride[0]))) + v_headed = cute.make_tensor(v.iterator, cute.make_layout((d_v, h_v, seqlen), stride=(1, v.stride[1], v.stride[0]))) + gate_headed = cute.make_tensor(gate.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, gate.stride[1], gate.stride[0]))) + beta_headed = cute.make_tensor(beta.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, beta.stride[1], beta.stride[0]))) + w_headed = cute.make_tensor(w.iterator, cute.make_layout((d_v, ho, seqlen), stride=(1, w.stride[1], w.stride[0]))) swz = cuda.TensorMapSwizzle.s128b - base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_beta = cuda.create_tensor_map_tiled_from_view(headed(beta, d_k, ho), box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_w = cuda.create_tensor_map_tiled_from_view(headed(w, d_v, ho), box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_k = cuda.create_tensor_map_tiled_from_view(k_headed, box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(v_headed, box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(gate_headed, box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_beta = cuda.create_tensor_map_tiled_from_view(beta_headed, box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_w = cuda.create_tensor_map_tiled_from_view(w_headed, box_dims=(tma_box_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) base_checkpoint = base_k if cutlass.const_expr(state_checkpoints is not None): diff --git a/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py index 4ab6cbead..a93473470 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn_bprop_f16.py @@ -165,13 +165,10 @@ class GdnBwdBars(NamedTuple): mb_k_cg0_done: MBarrier mb_v_ready: MBarrier mb_v_mma_done: MBarrier - mb_v_cg1_done: MBarrier mb_do_ready: MBarrier mb_do_mma_done: MBarrier - mb_do_cg1_done: MBarrier mb_state_ready: MBarrier - mb_state_done: MBarrier - mb_state_cg0_done: MBarrier + mb_state_mma_done: MBarrier mb_gate_ready: MBarrier mb_gate_done: MBarrier @@ -179,16 +176,16 @@ class GdnBwdBars(NamedTuple): mb_beta_done: MBarrier mb_dstate_acc_ready: MBarrier - mb_dstate_acc_done: MBarrier - mb_du_scale_ready: MBarrier - mb_du_scale_done: MBarrier - mb_du_total_ready: MBarrier - mb_dk_scale_ready: MBarrier - mb_dk_scale_done: MBarrier - mb_dk_attn_ready: MBarrier - mb_dk_attn_done: MBarrier - mb_dk_total_ready: MBarrier - mb_dk_total_done: MBarrier + mb_dstate_scale_acc_done: MBarrier + mb_du_scale_acc_ready: MBarrier + mb_du_scale_acc_done: MBarrier + mb_du_total_acc_ready: MBarrier + mb_dk_scale_acc_ready: MBarrier + mb_dk_scale_acc_done: MBarrier + mb_dk_attn_acc_ready: MBarrier + mb_dk_attn_acc_done: MBarrier + mb_dk_total_acc_ready: MBarrier + mb_dk_total_acc_done: MBarrier mb_dq_acc_scale_ready: MBarrier mb_dq_acc_scale_done: MBarrier mb_dq_acc_total_ready: MBarrier @@ -200,26 +197,22 @@ class GdnBwdBars(NamedTuple): mb_u_acc_ready: MBarrier mb_dy_acc_ready: MBarrier mb_da_acc_ready: MBarrier - mb_dm_ready: MBarrier - mb_dm_done: MBarrier - mb_dk_state_path_ready: MBarrier + mb_dm_acc_ready: MBarrier + mb_dm_acc_done: MBarrier + mb_dk_state_path_acc_ready: MBarrier mb_dstate_inp_ready: MBarrier mb_dstate_inp_done: MBarrier mb_do_prime_inp_ready: MBarrier - mb_do_prime_inp_done: MBarrier mb_du_inp_ready: MBarrier mb_dyp_inp_ready: MBarrier - mb_dyp_inp_done: MBarrier mb_y_ready: MBarrier mb_ainv_ready: MBarrier - mb_ainv_done: MBarrier mb_qk_ready: MBarrier mb_qk_done: MBarrier mb_u_ready: MBarrier mb_dstate_smem_ready: MBarrier - mb_dstate_smem_done: MBarrier mb_state_dot_dstate_done: MBarrier mb_da_ready: MBarrier @@ -262,13 +255,10 @@ def alloc(n): mb_k_cg0_done=MBarrier(alloc(cfg.smem_k_stages), stages=cfg.smem_k_stages, init_count=CG0_THREADS, producer=Producer.THREAD), mb_v_ready=MBarrier(alloc(cfg.smem_v_stages), stages=cfg.smem_v_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), mb_v_mma_done=MBarrier(alloc(cfg.smem_v_stages), stages=cfg.smem_v_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_v_cg1_done=MBarrier(alloc(cfg.smem_v_stages), stages=cfg.smem_v_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_do_ready=MBarrier(alloc(cfg.smem_do_stages), stages=cfg.smem_do_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), mb_do_mma_done=MBarrier(alloc(cfg.smem_do_stages), stages=cfg.smem_do_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_do_cg1_done=MBarrier(alloc(cfg.smem_do_stages), stages=cfg.smem_do_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_state_ready=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=ONE_LANE, producer=Producer.TMA_LOAD), - mb_state_done=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=1, producer=Producer.MMA_COMMIT), - mb_state_cg0_done=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_state_mma_done=MBarrier(alloc(cfg.smem_state_stages), stages=cfg.smem_state_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_gate_ready=MBarrier(alloc(cfg.smem_gate_stages), stages=cfg.smem_gate_stages, init_count=GATE_WARP, producer=Producer.THREAD), mb_gate_done=MBarrier(alloc(cfg.smem_gate_stages), stages=cfg.smem_gate_stages, init_count=CG0_PLUS_CG1, producer=Producer.THREAD), mb_beta_ready=MBarrier(alloc(cfg.smem_beta_stages), stages=cfg.smem_beta_stages, init_count=GATE_WARP, producer=Producer.THREAD), @@ -276,17 +266,19 @@ def alloc(n): mb_dstate_acc_ready=MBarrier( alloc(cfg.tmem_dstate_acc_stages), stages=cfg.tmem_dstate_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT ), - mb_dstate_acc_done=MBarrier(alloc(cfg.tmem_dstate_acc_stages), stages=cfg.tmem_dstate_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_dstate_scale_acc_done=MBarrier( + alloc(cfg.tmem_dstate_acc_stages), stages=cfg.tmem_dstate_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD + ), # five sequential per-chunk productions share the dV/dK TMEM slot - mb_du_scale_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_du_scale_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_du_total_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dk_scale_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dk_scale_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_dk_attn_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dk_attn_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_dk_total_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dk_total_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_du_scale_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_du_scale_acc_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_du_total_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_dk_scale_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_dk_scale_acc_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_dk_attn_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_dk_attn_acc_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_dk_total_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_dk_total_acc_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_dq_acc_scale_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_dq_acc_scale_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), mb_dq_acc_total_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), @@ -300,27 +292,23 @@ def alloc(n): mb_u_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_dy_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_da_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dm_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_dm_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_dk_state_path_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_dm_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_dm_acc_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), + mb_dk_state_path_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_dstate_inp_ready=MBarrier(alloc(cfg.tmem_dstate_inp_stages), stages=cfg.tmem_dstate_inp_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_dstate_inp_done=MBarrier( alloc(cfg.tmem_dstate_inp_stages), stages=cfg.tmem_dstate_inp_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT ), # f16 input restages at STATIC columns: dO' alone; dU and dY' overlap mb_do_prime_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_do_prime_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_du_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_dyp_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_dyp_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_y_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_ainv_ready=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=CG0_THREADS, producer=Producer.THREAD), - mb_ainv_done=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_qk_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=CG0_THREADS, producer=Producer.THREAD), mb_qk_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_u_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_dstate_smem_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_dstate_smem_done=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_state_dot_dstate_done=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), mb_da_ready=MBarrier(alloc(1), stages=1, init_count=CG0_THREADS, producer=Producer.THREAD), mb_dbeta_cg1_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), @@ -456,7 +444,7 @@ def blockwise_diagonal_8x8_to_16x16(cfg, base_int, raw_base, d0, lane_id): mma_step_k8(c_regs, [d, d], [c], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) for i in cutlass.range_constexpr(4): c_regs[i] = -c_regs[i] - a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(2)] + a_pack = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(2)] # ---- C = T @ A^{-1} ---------------------------------------------------------- ai = nvvm.ldmatrix( @@ -467,13 +455,13 @@ def blockwise_diagonal_8x8_to_16x16(cfg, base_int, raw_base, d0, lane_id): o_regs = cutlass.Array(cutlass.Float32, 4, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(4): o_regs[i] = cutlass.Float32(0.0) - mma_step_k8(o_regs, a_f16, [ai], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) - o_f16 = fp32_to_fp16(o_regs[0], o_regs[1], dtype=cfg.io_dtype) + mma_step_k8(o_regs, a_pack, [ai], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) + o_pack = fp32_to_fp16(o_regs[0], o_regs[1], dtype=cfg.io_dtype) # ---- store corrected C ------------------------------------------------------- nvvm.stmatrix( cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + ldsm_x1_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - o_f16, + o_pack, nvvm.MMALayout.ROW, ) @@ -509,7 +497,7 @@ def blockwise_diagonal_16x16_to_32x32(cfg, base_int, raw_base, d0, lane_id): mma_step(c_regs, d, c, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) for i in cutlass.range_constexpr(8): c_regs[i] = -c_regs[i] - a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + a_pack = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] # ---- C = T @ A^{-1} ---------------------------------------------------------- ai = list( @@ -522,13 +510,13 @@ def blockwise_diagonal_16x16_to_32x32(cfg, base_int, raw_base, d0, lane_id): o_regs = cutlass.Array(cutlass.Float32, 8, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(8): o_regs[i] = cutlass.Float32(0.0) - mma_step(o_regs, a_f16, ai, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) - o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + mma_step(o_regs, a_pack, ai, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) + o_pack = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] # ---- store corrected C ------------------------------------------------------- nvvm.stmatrix( cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - o_f16, + o_pack, nvvm.MMALayout.ROW, ) @@ -539,9 +527,9 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, raw_base, warp_id, lane_id) band = warp_id % 2 bpe = cfg.io_dtype.width // 8 ldsm_x4_off = (lane_id % 16) * 64 + (lane_id // 16) * 8 - a_regs = [] + a_frags = [] for vs in cutlass.range_constexpr(2): - a_regs += list( + a_frags += list( nvvm.ldmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b((32 + band * 16) * 64 + 32 + vs * 16 + ldsm_x4_off, row_stride_log2=6) * bpe, @@ -552,9 +540,9 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, raw_base, warp_id, lane_id) nvvm.MMALayout.ROW, ) ) - b_regs = [] + b_frags = [] for vs in cutlass.range_constexpr(4): - b_regs += list( + b_frags += list( nvvm.ldmatrix( cutlass.inttoptr( raw_base + swizzle_lin_128b((32 + (vs // 2) * 16) * 64 + (vs % 2) * 16 + ldsm_x4_off, row_stride_log2=6) * bpe, @@ -571,15 +559,15 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, raw_base, warp_id, lane_id) for i in cutlass.range_constexpr(16): c_regs[i] = cutlass.Float32(0.0) for ks in cutlass.range_constexpr(2): - mma_step(c_regs, a_regs, b_regs[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) + mma_step(c_regs, a_frags, b_frags[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) for i in cutlass.range_constexpr(16): c_regs[i] = -c_regs[i] - a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + a_pack = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] # ---- C = T @ A^{-1} ---------------------------------------------------------- - ai_regs = [] + ai_frags = [] for vs in cutlass.range_constexpr(4): - ai_regs += list( + ai_frags += list( nvvm.ldmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b(((vs // 2) * 16) * 64 + (vs % 2) * 16 + ldsm_x4_off, row_stride_log2=6) * bpe, @@ -594,8 +582,8 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, raw_base, warp_id, lane_id) for i in cutlass.range_constexpr(16): o_regs[i] = cutlass.Float32(0.0) for ks in cutlass.range_constexpr(2): - mma_step(o_regs, a_f16, ai_regs[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) - o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + mma_step(o_regs, a_pack, ai_frags[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) + o_pack = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] # ---- store corrected C ------------------------------------------------------- nvvm.barrier_cta_sync_aligned( @@ -606,14 +594,14 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, raw_base, warp_id, lane_id) cutlass.inttoptr( base_int + swizzle_lin_128b((32 + band * 16) * 64 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 ), - o_f16[0:4], + o_pack[0:4], nvvm.MMALayout.ROW, ) nvvm.stmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b((32 + band * 16) * 64 + 16 + ldsm_x4_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 ), - o_f16[4:8], + o_pack[4:8], nvvm.MMALayout.ROW, ) @@ -1381,7 +1369,7 @@ def mma_warp( if have_dstate: bars.mb_dstate_inp_ready[dstate_inp_idx].wait(dstate_inp_index.phase) dstate_inp_index = advance(dstate_inp_index, cfg.tmem_dstate_inp_stages) - bars.mb_dk_total_done[0].wait(dk_total_index.phase) + bars.mb_dk_total_acc_done[0].wait(dk_total_index.phase) dk_total_index = advance(dk_total_index, 1) if have_dstate: @@ -1397,9 +1385,10 @@ def mma_warp( cutlass.Boolean(sub + k > 0), ) if elect_one: - bars.mb_du_scale_ready[0].arrive(cta_group=1) + bars.mb_du_scale_acc_ready[0].arrive(cta_group=1) bars.mb_dstate_inp_done[dstate_inp_idx].arrive(cta_group=1) + # ---- dQ inter = state(S) @ dO^T ------------------------------------------ do_idx = do_index.idx bars.mb_do_ready[do_idx].wait(do_index.phase) do_index = advance(do_index, cfg.smem_do_stages) @@ -1423,7 +1412,7 @@ def mma_warp( bars.mb_qk_ready[du_qk_idx].wait(qk_index.phase) qk_index = advance(qk_index, cfg.smem_qk_stages) if have_dstate: - bars.mb_du_scale_done[0].wait(du_scale_index.phase) + bars.mb_du_scale_acc_done[0].wait(du_scale_index.phase) du_scale_index = advance(du_scale_index, 1) desc_do_mnmaj = d_do0 @@ -1436,13 +1425,13 @@ def mma_warp( accumulate=have_dstate, ) if elect_one: - bars.mb_du_total_ready[0].arrive(cta_group=1) + bars.mb_du_total_acc_ready[0].arrive(cta_group=1) # ---- dstate update += dO'^T(T) @ Q --------------------------------------- bars.mb_do_prime_inp_ready[0].wait(do_prime_inp_ready.phase) do_prime_inp_ready = advance(do_prime_inp_ready, 1) dstate_idx = dstate_acc_index.idx - bars.mb_dstate_acc_done[dstate_idx].wait(dstate_acc_index.phase) + bars.mb_dstate_scale_acc_done[dstate_idx].wait(dstate_acc_index.phase) dstate_acc_index = advance(dstate_acc_index, cfg.tmem_dstate_acc_stages) desc_q_t = d_q_trans0 @@ -1456,8 +1445,6 @@ def mma_warp( k, cutlass.Boolean(True) if cutlass.const_expr(sub + k > 0) else have_dstate, ) - if elect_one: - bars.mb_do_prime_inp_done[0].arrive(cta_group=1) # ---- U^T recompute = Y^T(T) @ T^T ------------------------------------ ainv_idx = ainv_index.idx @@ -1499,7 +1486,6 @@ def mma_warp( ) if elect_one: bars.mb_dy_acc_ready[0].arrive(cta_group=1) - bars.mb_ainv_done[ainv_idx].arrive(cta_group=1) # ---- dK_inter = dstate_entry(S) @ U^T ------------------------------------ bars.mb_u_ready[0].wait(u_index.phase) @@ -1518,8 +1504,7 @@ def mma_warp( accumulate=False, ) if elect_one: - bars.mb_dk_scale_ready[0].arrive(cta_group=1) - bars.mb_dstate_smem_done[0].arrive(cta_group=1) + bars.mb_dk_scale_acc_ready[0].arrive(cta_group=1) # ---- dA_eff = dO(S) @ U^T -------------------------------------------- desc_do_kmaj_da = d_do_kmaj0 @@ -1550,7 +1535,7 @@ def mma_warp( accumulate=False, ) if elect_one: - bars.mb_dm_ready[0].arrive(cta_group=1) + bars.mb_dm_acc_ready[0].arrive(cta_group=1) bars.mb_v_mma_done[v_idx].arrive(cta_group=1) # ---- dstate update-term += dY'^T(T) @ K -------------------------------------- @@ -1570,13 +1555,12 @@ def mma_warp( ) if elect_one: bars.mb_dstate_acc_ready[dstate_idx].arrive(cta_group=1) - bars.mb_dyp_inp_done[0].arrive(cta_group=1) # ---- dK attn += Q^T(S) @ dA ------------------------------------------ bars.mb_da_ready[0].wait(da_ready_index.phase) da_ready_index = advance(da_ready_index, 1) if have_dstate: - bars.mb_dk_scale_done[0].wait(dk_scale_index.phase) + bars.mb_dk_scale_acc_done[0].wait(dk_scale_index.phase) dk_scale_index = advance(dk_scale_index, 1) desc_q_mnmaj_dk_attn = d_q_trans0 @@ -1589,7 +1573,7 @@ def mma_warp( accumulate=have_dstate, ) if elect_one: - bars.mb_dk_attn_ready[0].arrive(cta_group=1) + bars.mb_dk_attn_acc_ready[0].arrive(cta_group=1) bars.mb_q_mma_done[q_idx].arrive(cta_group=1) # ---- dQ attn += K^T(S) @ dA ------------------------------------------ @@ -1633,15 +1617,15 @@ def mma_warp( accumulate=False, ) if elect_one: - bars.mb_dk_state_path_ready[0].arrive(cta_group=1) - bars.mb_state_done[state_idx].arrive(cta_group=1) + bars.mb_dk_state_path_acc_ready[0].arrive(cta_group=1) + bars.mb_state_mma_done[state_idx].arrive(cta_group=1) if elect_one: bars.mb_sdv_done[0].arrive(cta_group=1) # ---- dK dM-terms += K^T(S) @ dM^T + K^T(S) @ dM ---------------------- - bars.mb_dm_done[0].wait(dm_index.phase) + bars.mb_dm_acc_done[0].wait(dm_index.phase) dm_index = advance(dm_index, 1) - bars.mb_dk_attn_done[0].wait(dk_attn_index.phase) + bars.mb_dk_attn_acc_done[0].wait(dk_attn_index.phase) dk_attn_index = advance(dk_attn_index, 1) desc_k_mnmaj_dm_terms = d_k_trans0.advance_start_address(k_idx * K_STAGE_BYTES) desc_dm_t = d_dm_trans0 @@ -1661,12 +1645,12 @@ def mma_warp( accumulate=True, ) if elect_one: - bars.mb_dk_total_ready[0].arrive(cta_group=1) + bars.mb_dk_total_acc_ready[0].arrive(cta_group=1) bars.mb_k_mma_done[k_idx].arrive(cta_group=1) tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) - bars.mb_dk_total_done[0].wait(dk_total_index.phase) + bars.mb_dk_total_acc_done[0].wait(dk_total_index.phase) bars.mb_tmem_done[0].wait(0) nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) @@ -1830,7 +1814,6 @@ def tmaldg_warp( # ---- V load ---------------------------------------------------------- v_idx = v_index.idx bars.mb_v_mma_done[v_idx].wait(v_index.phase) - bars.mb_v_cg1_done[v_idx].wait(v_index.phase) v_index = advance(v_index, cfg.smem_v_stages) if elect_one: bars.mb_v_ready[v_idx].arrive(n_bytes=cfg.tma_v_bytes) @@ -1840,7 +1823,6 @@ def tmaldg_warp( # ---- dO load --------------------------------------------------------- do_idx = do_index.idx bars.mb_do_mma_done[do_idx].wait(do_index.phase) - bars.mb_do_cg1_done[do_idx].wait(do_index.phase) do_index = advance(do_index, cfg.smem_do_stages) if elect_one: bars.mb_do_ready[do_idx].arrive(n_bytes=cfg.tma_do_bytes) @@ -1850,8 +1832,7 @@ def tmaldg_warp( # ---- entering state: checkpoint c - 1, or initial_state for chunk 0 when given ---------- if chunk_idx >= FIRST_STATE_CHUNK: state_idx = state_index.idx - bars.mb_state_done[state_idx].wait(state_index.phase) - bars.mb_state_cg0_done[state_idx].wait(state_index.phase) + bars.mb_state_mma_done[state_idx].wait(state_index.phase) state_index = advance(state_index, cfg.smem_state_stages) if elect_one: bars.mb_state_ready[state_idx].arrive(n_bytes=cfg.tma_state_bytes) @@ -1878,11 +1859,9 @@ def tmaldg_warp( k_index = advance(k_index, cfg.smem_k_stages) for _ in range(cfg.smem_v_stages): bars.mb_v_mma_done[v_index.idx].wait(v_index.phase) - bars.mb_v_cg1_done[v_index.idx].wait(v_index.phase) v_index = advance(v_index, cfg.smem_v_stages) for _ in range(cfg.smem_do_stages): bars.mb_do_mma_done[do_index.idx].wait(do_index.phase) - bars.mb_do_cg1_done[do_index.idx].wait(do_index.phase) do_index = advance(do_index, cfg.smem_do_stages) @@ -2034,7 +2013,6 @@ def compute0_warp_group( # ---- KK epi: M_kk[i,j] = W_kk[i,j] * T[i,j] * Beta[i] --------------- ainv_idx = ainv_index.idx - ainv_phase = ainv_index.phase ainv_index = advance(ainv_index, cfg.smem_ainv_stages) bars.mb_kk_acc_ready[0].wait(cg0_kk_ready.phase) cg0_kk_ready = advance(cg0_kk_ready, 1) @@ -2042,11 +2020,11 @@ def compute0_warp_group( ainv_base = sAinv[ainv_idx].base kk_base = sKK[ainv_idx].base kk_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_kk_col, cutlass.Float32), num=8) - kk_f16 = [] + kk_pack = [] for k in cutlass.range_constexpr(num_vals // 2): p0, p1 = fmul2(kk_vec[2 * k], kk_vec[2 * k + 1], decay_t[2 * k], decay_t[2 * k + 1]) v0, v1 = fmul2(p0, p1, gBeta[2 * k], gBeta[2 * k + 1]) - kk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) + kk_pack.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( cutlass.inttoptr( @@ -2054,7 +2032,7 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [kk_f16[c * 4 + 0], kk_f16[c * 4 + 1], kk_f16[c * 4 + 2], kk_f16[c * 4 + 3]], + [kk_pack[c * 4 + 0], kk_pack[c * 4 + 1], kk_pack[c * 4 + 2], kk_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) if chunk_idx < cend - FIRST_STATE_CHUNK: @@ -2069,11 +2047,11 @@ def compute0_warp_group( qk_base = sQk[qk_idx].base qk_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_a_col, cutlass.Float32), num=8) - qk_f16 = [] + qk_pack = [] for k in cutlass.range_constexpr(num_vals // 2): p0, p1 = fmul2(qk_vec[2 * k], qk_vec[2 * k + 1], decay_t[2 * k], decay_t[2 * k + 1]) v0, v1 = fmul2(p0, p1, scale, scale) - qk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) + qk_pack.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) bars.mb_qk_done[qk_idx].wait(qk_phase) for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( @@ -2082,14 +2060,13 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [qk_f16[c * 4 + 0], qk_f16[c * 4 + 1], qk_f16[c * 4 + 2], qk_f16[c * 4 + 3]], + [qk_pack[c * 4 + 0], qk_pack[c * 4 + 1], qk_pack[c * 4 + 2], qk_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_qk_ready[qk_idx].arrive() # ---- blockwise inverse: A_inv = ------------------------------------- - bars.mb_ainv_done[ainv_idx].wait(ainv_phase) nvvm.barrier_cta_sync_aligned( cfg.inverse_barrier_id, thread_count=cfg.inverse_barrier_threads, @@ -2125,9 +2102,9 @@ def compute0_warp_group( beta_col = [] for k in cutlass.range_constexpr(num_vals): beta_col.append(sBeta[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, beta_idx]) - ainv_f16 = [] + ainv_frags = [] for c in cutlass.range_constexpr(ACC_N_FRAGS): - ainv_f16 += list( + ainv_frags += list( nvvm.ldmatrix( cutlass.inttoptr( ainv_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, @@ -2138,11 +2115,11 @@ def compute0_warp_group( nvvm.MMALayout.ROW, ) ) - ainv_scaled = [] + ainv_pack = [] for j in cutlass.range_constexpr(num_vals // 2): - lo, hi = f16x2_to_f32(ainv_f16[j], dtype=cfg.io_dtype) + lo, hi = f16x2_to_f32(ainv_frags[j], dtype=cfg.io_dtype) s0, s1 = fmul2(lo, hi, beta_col[2 * j], beta_col[2 * j + 1]) - ainv_scaled.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) + ainv_pack.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( cutlass.inttoptr( @@ -2150,7 +2127,7 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [ainv_scaled[c * 4 + 0], ainv_scaled[c * 4 + 1], ainv_scaled[c * 4 + 2], ainv_scaled[c * 4 + 3]], + [ainv_pack[c * 4 + 0], ainv_pack[c * 4 + 1], ainv_pack[c * 4 + 2], ainv_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) @@ -2191,17 +2168,17 @@ def compute0_warp_group( state_dot_dstate_hi = [opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero()] for oct_ in cutlass.range_constexpr(cfg.d_v // 8): state_dot_off = (oct_ // 8) * (cfg.d_k * 64) + cg0_tidx * 64 + swizzle_xor_128b(cg0_tidx, (oct_ % 8) * 8) - dstate_w = cute.make_tensor( + dstate_frag = cute.make_tensor( cute.make_ptr(cutlass.Int32, sdstate_base + state_dot_off * bpe, mem_space=cute.AddressSpace.smem, assumed_align=16), cute.make_layout(4), ).load() - state_w = cute.make_tensor( + state_frag = cute.make_tensor( cute.make_ptr(cutlass.Int32, sstate_base + state_dot_off * bpe, mem_space=cute.AddressSpace.smem, assumed_align=16), cute.make_layout(4), ).load() for w in cutlass.range_constexpr(4): - dstate_lo, dstate_hi = f16x2_to_f32(dstate_w[w], dtype=cfg.io_dtype) - state_lo, state_hi = f16x2_to_f32(state_w[w], dtype=cfg.io_dtype) + dstate_lo, dstate_hi = f16x2_to_f32(dstate_frag[w], dtype=cfg.io_dtype) + state_lo, state_hi = f16x2_to_f32(state_frag[w], dtype=cfg.io_dtype) state_dot_dstate_lo[w], state_dot_dstate_hi[w] = ffma2( dstate_lo, dstate_hi, state_lo, state_hi, state_dot_dstate_lo[w], state_dot_dstate_hi[w] ) @@ -2211,16 +2188,13 @@ def compute0_warp_group( for off in [1, 2, 4, 8, 16]: state_dot_dstate += nvvm.shfl_sync(0xFFFFFFFF, state_dot_dstate, off, 31, kind=nvvm.Shfl.BFLY) bars.mb_state_dot_dstate_done[0].arrive() - if chunk_idx < cend - FIRST_STATE_CHUNK: - nvvm.fence_proxy("async.shared", space="cta") - bars.mb_state_cg0_done[0].arrive() # ---- dK inter rescale ------------------------------------------------ cg0_k_idx = cg0_k_index.idx cg0_k_index = advance(cg0_k_index, cfg.smem_k_stages) k_dot_dk_inter = cutlass.Float32(0.0) if chunk_idx + DSTATE_IN0 >= 1: - bars.mb_dk_scale_ready[0].wait(cg0_dk_scale_ready.phase) + bars.mb_dk_scale_acc_ready[0].wait(cg0_dk_scale_ready.phase) cg0_dk_scale_ready = advance(cg0_dk_scale_ready, 1) k_dot_dk_inter_lo = [opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero()] k_dot_dk_inter_hi = [opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero(), opaque_f32_zero()] @@ -2237,14 +2211,14 @@ def compute0_warp_group( nvvm.tcgen05_st("16x256b", dki_ptrs[sub], cutlass.Vector.from_elements(tuple(dki_scaled), cutlass.Float32)) for m0 in cutlass.range_constexpr(4): frag_addr = frag_slab_off + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) - k_f16 = nvvm.ldmatrix((sK_base_p + cg0_k_idx * k_stage_elems_cg0 + frag_addr).raw_ptr(), 4, nvvm.MMALayout.COL) + k_frag = nvvm.ldmatrix((sK_base_p + cg0_k_idx * k_stage_elems_cg0 + frag_addr).raw_ptr(), 4, nvvm.MMALayout.COL) for i in cutlass.range_constexpr(4): - k_lo, k_hi = f16x2_to_f32(k_f16[i], dtype=cfg.io_dtype) + k_lo, k_hi = f16x2_to_f32(k_frag[i], dtype=cfg.io_dtype) k_dot_dk_inter_lo[i], k_dot_dk_inter_hi[i] = ffma2( dki_scaled[8 * m0 + 2 * i], dki_scaled[8 * m0 + 2 * i + 1], k_lo, k_hi, k_dot_dk_inter_lo[i], k_dot_dk_inter_hi[i] ) nvvm.tcgen05_wait("store") - bars.mb_dk_scale_done[0].arrive() + bars.mb_dk_scale_acc_done[0].arrive() k_dot_dk_inter = ((k_dot_dk_inter_lo[0] + k_dot_dk_inter_lo[1]) + (k_dot_dk_inter_lo[2] + k_dot_dk_inter_lo[3])) + ( (k_dot_dk_inter_hi[0] + k_dot_dk_inter_hi[1]) + (k_dot_dk_inter_hi[2] + k_dot_dk_inter_hi[3]) ) @@ -2257,11 +2231,11 @@ def compute0_warp_group( da_base = sDa[0].base da_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_da_col, cutlass.Float32), num=8) - da_f16 = [] + da_pack = [] for k in cutlass.range_constexpr(num_vals // 2): p0, p1 = fmul2(da_vec[2 * k], da_vec[2 * k + 1], decay_t[2 * k], decay_t[2 * k + 1]) v0, v1 = fmul2(p0, p1, scale, scale) - da_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) + da_pack.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( cutlass.inttoptr( @@ -2269,24 +2243,24 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [da_f16[c * 4 + 0], da_f16[c * 4 + 1], da_f16[c * 4 + 2], da_f16[c * 4 + 3]], + [da_pack[c * 4 + 0], da_pack[c * 4 + 1], da_pack[c * 4 + 2], da_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_da_ready[0].arrive() # ---- dM epilogue ----------------------------------------------------- - bars.mb_dm_ready[0].wait(dm_ready_index.phase) + bars.mb_dm_acc_ready[0].wait(dm_ready_index.phase) dm_ready_index = advance(dm_ready_index, 1) dm_base = sDm[0].base dm_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dm_core_col, cutlass.Float32), num=8) - dm_f16 = [] + dm_pack = [] for k in cutlass.range_constexpr(num_vals // 2): p0, p1 = fmul2(dm_vec[2 * k], dm_vec[2 * k + 1], decay_t_strict[2 * k], decay_t_strict[2 * k + 1]) v0 = cutlass.Float32(0.0) - p0 v1 = cutlass.Float32(0.0) - p1 - dm_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) + dm_pack.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( cutlass.inttoptr( @@ -2294,25 +2268,25 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [dm_f16[c * 4 + 0], dm_f16[c * 4 + 1], dm_f16[c * 4 + 2], dm_f16[c * 4 + 3]], + [dm_pack[c * 4 + 0], dm_pack[c * 4 + 1], dm_pack[c * 4 + 2], dm_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_dm_done[0].arrive() + bars.mb_dm_acc_done[0].arrive() # ---- dK attn read ---------------------------------------------------- - bars.mb_dk_attn_ready[0].wait(cg0_dk_attn_ready.phase) + bars.mb_dk_attn_acc_ready[0].wait(cg0_dk_attn_ready.phase) cg0_dk_attn_ready = advance(cg0_dk_attn_ready, 1) - dks_frag = [] + dks_regs = [] for sub in cutlass.range_constexpr(2): dks_vec = nvvm.tcgen05_ld( "16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_acc_col, cutlass.Float32), num=8, ) - dks_frag.append([dks_vec[k] for k in range(32)]) + dks_regs.append([dks_vec[k] for k in range(32)]) nvvm.tcgen05_wait("load") - bars.mb_dk_attn_done[0].arrive() + bars.mb_dk_attn_acc_done[0].arrive() # ---- dBeta/dGate M-terms: E = strict ⊙ dM_core ⊙ M_kk(sKK). ---------- kk_frag = [] @@ -2366,16 +2340,16 @@ def compute0_warp_group( for sub in cutlass.range_constexpr(2): for m0 in cutlass.range_constexpr(4): frag_addr = frag_slab_off + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) - k_f16 = nvvm.ldmatrix((sK_base_p + cg0_k_idx * k_stage_elems_cg0 + frag_addr).raw_ptr(), 4, nvvm.MMALayout.COL) + k_frag = nvvm.ldmatrix((sK_base_p + cg0_k_idx * k_stage_elems_cg0 + frag_addr).raw_ptr(), 4, nvvm.MMALayout.COL) for i in cutlass.range_constexpr(4): - k_lo, k_hi = f16x2_to_f32(k_f16[i], dtype=cfg.io_dtype) + k_lo, k_hi = f16x2_to_f32(k_frag[i], dtype=cfg.io_dtype) frag_e0 = cutlass.const_expr(8 * m0 + 2 * i) part_e0 = cutlass.const_expr((frag_e0 // 4) * 2 + (frag_e0 % 2)) if cutlass.const_expr(sub == 0 and i % 2 == 0): - part_k[part_e0], part_k[part_e0 + 1] = fmul2(dks_frag[sub][frag_e0], dks_frag[sub][frag_e0 + 1], k_lo, k_hi) + part_k[part_e0], part_k[part_e0 + 1] = fmul2(dks_regs[sub][frag_e0], dks_regs[sub][frag_e0 + 1], k_lo, k_hi) else: part_k[part_e0], part_k[part_e0 + 1] = ffma2( - dks_frag[sub][frag_e0], dks_frag[sub][frag_e0 + 1], k_lo, k_hi, part_k[part_e0], part_k[part_e0 + 1] + dks_regs[sub][frag_e0], dks_regs[sub][frag_e0 + 1], k_lo, k_hi, part_k[part_e0], part_k[part_e0 + 1] ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_k_cg0_done[cg0_k_idx].arrive() @@ -2407,16 +2381,16 @@ def compute0_warp_group( for sub in cutlass.range_constexpr(2): for m0 in cutlass.range_constexpr(4): frag_addr = frag_slab_off + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) - k_f16 = nvvm.ldmatrix((sK_base_p + cg0_k_idx * k_stage_elems_cg0 + frag_addr).raw_ptr(), 4, nvvm.MMALayout.COL) + k_frag = nvvm.ldmatrix((sK_base_p + cg0_k_idx * k_stage_elems_cg0 + frag_addr).raw_ptr(), 4, nvvm.MMALayout.COL) for i in cutlass.range_constexpr(4): - k_lo, k_hi = f16x2_to_f32(k_f16[i], dtype=cfg.io_dtype) + k_lo, k_hi = f16x2_to_f32(k_frag[i], dtype=cfg.io_dtype) frag_e0 = cutlass.const_expr(8 * m0 + 2 * i) part_e0 = cutlass.const_expr((frag_e0 // 4) * 2 + (frag_e0 % 2)) if cutlass.const_expr(sub == 0 and i % 2 == 0): - part_k[part_e0], part_k[part_e0 + 1] = fmul2(dks_frag[sub][frag_e0], dks_frag[sub][frag_e0 + 1], k_lo, k_hi) + part_k[part_e0], part_k[part_e0 + 1] = fmul2(dks_regs[sub][frag_e0], dks_regs[sub][frag_e0 + 1], k_lo, k_hi) else: part_k[part_e0], part_k[part_e0 + 1] = ffma2( - dks_frag[sub][frag_e0], dks_frag[sub][frag_e0 + 1], k_lo, k_hi, part_k[part_e0], part_k[part_e0 + 1] + dks_regs[sub][frag_e0], dks_regs[sub][frag_e0 + 1], k_lo, k_hi, part_k[part_e0], part_k[part_e0 + 1] ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_k_cg0_done[cg0_k_idx].arrive() @@ -2446,9 +2420,6 @@ def compute0_warp_group( for _ in range(cfg.smem_qk_stages): bars.mb_qk_done[qk_index.idx].wait(qk_index.phase) qk_index = advance(qk_index, cfg.smem_qk_stages) - for _ in range(cfg.smem_ainv_stages): - bars.mb_ainv_done[ainv_index.idx].wait(ainv_index.phase) - ainv_index = advance(ainv_index, cfg.smem_ainv_stages) bars.mb_tmem_done[0].arrive() @@ -2496,9 +2467,6 @@ def compute1_warp_group( cg1_du_total_ready = PipelineState.start(phase=0) cg1_dk_total_ready = PipelineState.start(phase=0) dstate_acc_index = PipelineState.start(phase=0) - do_prime_inp_free = PipelineState.start(phase=1) - dyp_inp_free = PipelineState.start(phase=1) - dstate_smem_index = PipelineState.start(phase=1) cg1_state_dot_dstate_index = PipelineState.start(phase=0) dq_index = PipelineState.start(phase=1) cg1_beta_index = PipelineState.start(phase=0) @@ -2560,9 +2528,7 @@ def compute1_warp_group( sdQ_base = cute.make_ptr(cfg.io_dtype, sdQ[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) sdK_base = cute.make_ptr(cfg.io_dtype, sdK[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) sDstate_base_int = sDstate[0].base - # dBeta/dGate reduction scratch: f32 view of the owned dQ stage sred_base = cute.make_ptr(cutlass.Float32, sdQ[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) - # part_q scratch in sDstate sdstate_red = cute.make_ptr(cutlass.Float32, sDstate[0].base, mem_space=cute.AddressSpace.smem, assumed_align=cfg.buffer_align_bytes) dstate_done_idx = cutlass.Int32(0) cg1_warp_id = cg1_tidx // cfg.threads_per_warp @@ -2597,17 +2563,15 @@ def compute1_warp_group( nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dstate_acc_col + sub * ldtm_width, cutlass.Float32), cutlass.Vector.from_elements(tuple(dstate_in_vals), cutlass.Float32), ) - dstate_in_f16 = [fp32_to_fp16(dstate_in_vals[2 * j], dstate_in_vals[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] + dstate_in_pack = [fp32_to_fp16(dstate_in_vals[2 * j], dstate_in_vals[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dstate_inp_col + sub * sttm_width, cutlass.Int32), - cutlass.Vector.from_elements(tuple(dstate_in_f16), cutlass.Int32), + cutlass.Vector.from_elements(tuple(dstate_in_pack), cutlass.Int32), ) nvvm.tcgen05_wait("store") bars.mb_dstate_inp_ready[dstate_inp_idx].arrive() - bars.mb_dstate_smem_done[0].wait(dstate_smem_index.phase) - dstate_smem_index = advance(dstate_smem_index, 1) for b in cutlass.range_constexpr(2): for col_half in cutlass.range_constexpr(2): dstate_smem_vec = nvvm.tcgen05_ld( @@ -2615,7 +2579,7 @@ def compute1_warp_group( nvvm.make_tmem_ptr(((tmem_warp_row + b * 16) << 16) + tmem_dstate_acc_col + col_half * 64, cutlass.Float32), num=8, ) - dstate_smem_f16 = [fp32_to_fp16(dstate_smem_vec[2 * j], dstate_smem_vec[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] + dstate_smem_pack = [fp32_to_fp16(dstate_smem_vec[2 * j], dstate_smem_vec[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] for c in cutlass.range_constexpr(4): dstate_smem_row = col_half * 64 + frag_row + c * 16 nvvm.stmatrix( @@ -2625,7 +2589,7 @@ def compute1_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [dstate_smem_f16[c * 4 + 0], dstate_smem_f16[c * 4 + 1], dstate_smem_f16[c * 4 + 2], dstate_smem_f16[c * 4 + 3]], + [dstate_smem_pack[c * 4 + 0], dstate_smem_pack[c * 4 + 1], dstate_smem_pack[c * 4 + 2], dstate_smem_pack[c * 4 + 3]], nvvm.MMALayout.COL, ) nvvm.fence_proxy("async.shared", space="cta") @@ -2660,7 +2624,7 @@ def compute1_warp_group( cutlass.Vector.from_elements(tuple(dstate_rescaled), cutlass.Float32), ) nvvm.tcgen05_wait("store") - bars.mb_dstate_acc_done[dstate_done_idx].arrive() + bars.mb_dstate_scale_acc_done[dstate_done_idx].arrive() num_vals = 32 cumprod_fp32 = [] @@ -2670,8 +2634,6 @@ def compute1_warp_group( cumprod_vals = [cumprod_fp32[(k // 4) * 2 + (k % 2)] for k in range(num_vals)] # ---- dO' restage: dO * cumprod_vals * scale -> shared_inp TMEM ----------- - bars.mb_do_prime_inp_done[0].wait(do_prime_inp_free.phase) - do_prime_inp_free = advance(do_prime_inp_free, 1) do_idx = do_index.idx bars.mb_do_ready[do_idx].wait(do_index.phase) do_index = advance(do_index, cfg.smem_do_stages) @@ -2679,7 +2641,7 @@ def compute1_warp_group( for c in cutlass.range_constexpr(8): m0 = cutlass.const_expr(c % 4) sub = cutlass.const_expr(c // 4) - do_f16 = nvvm.ldmatrix( + do_frag = nvvm.ldmatrix( ( sdO_base + do_idx * do_stage_elems @@ -2691,7 +2653,7 @@ def compute1_warp_group( nvvm.MMALayout.COL, ) for i in cutlass.range_constexpr(4): - lo, hi = f16x2_to_f32(do_f16[i], dtype=cfg.io_dtype) + lo, hi = f16x2_to_f32(do_frag[i], dtype=cfg.io_dtype) p0, p1 = fmul2(lo, hi, cumprod_vals[8 * m0 + 2 * i], cumprod_vals[8 * m0 + 2 * i + 1]) do_regs[8 * m0 + 2 * i][sub], do_regs[8 * m0 + 2 * i + 1][sub] = fmul2(p0, p1, scale, scale) for sub in cutlass.range_constexpr(2): @@ -2715,7 +2677,7 @@ def compute1_warp_group( for i in cutlass.range_constexpr(16): decay_scale_fp32.append(cute.math.exp2(last_cumsumlog - col_cs_fp32[i], fastmath=True)) decay_scale_vals = [decay_scale_fp32[(k // 4) * 2 + (k % 2)] for k in range(num_vals)] - bars.mb_du_scale_ready[0].wait(cg1_du_scale_ready.phase) + bars.mb_du_scale_acc_ready[0].wait(cg1_du_scale_ready.phase) cg1_du_scale_ready = advance(cg1_du_scale_ready, 1) dv_ptrs = [] dv_vecs = [] @@ -2729,18 +2691,18 @@ def compute1_warp_group( dv_scaled += [s0, s1] nvvm.tcgen05_st("16x256b", dv_ptrs[sub], cutlass.Vector.from_elements(tuple(dv_scaled), cutlass.Float32)) nvvm.tcgen05_wait("store") - bars.mb_du_scale_done[0].arrive() + bars.mb_du_scale_acc_done[0].arrive() # ---- V - K*state: delta = V - cumprod*(K @ state), in place ------------- v_idx = v_index.idx bars.mb_v_ready[v_idx].wait(v_index.phase) v_index = advance(v_index, cfg.smem_v_stages) if chunk_idx >= FIRST_STATE_CHUNK: - v_words = [[cutlass.Int32(0), cutlass.Int32(0)] for _ in range(16)] + v_frags = [[cutlass.Int32(0), cutlass.Int32(0)] for _ in range(16)] for c in cutlass.range_constexpr(8): m0 = cutlass.const_expr(c % 4) sub = cutlass.const_expr(c // 4) - v_f16 = nvvm.ldmatrix( + v_frag = nvvm.ldmatrix( ( sV_base + v_idx * v_stage_elems @@ -2752,7 +2714,7 @@ def compute1_warp_group( nvvm.MMALayout.COL, ) for i in cutlass.range_constexpr(4): - v_words[4 * m0 + i][sub] = v_f16[i] + v_frags[4 * m0 + i][sub] = v_frag[i] bars.mb_k_state_acc_ready[0].wait(cg1_k_state_ready.phase) cg1_k_state_ready = advance(cg1_k_state_ready, 1) for sub in cutlass.range_constexpr(2): @@ -2767,7 +2729,7 @@ def compute1_warp_group( g0, g1 = fmul2(k_state_vec[2 * j], k_state_vec[2 * j + 1], cumprod_vals[2 * j], cumprod_vals[2 * j + 1]) g_k_state_word = fp32_to_fp16(g0, g1, dtype=cfg.io_dtype) g_k_state_pack.append(g_k_state_word) - y_pack.append(sub_f16x2(v_words[j][sub], g_k_state_word, cfg.io_dtype)) + y_pack.append(sub_f16x2(v_frags[j][sub], g_k_state_word, cfg.io_dtype)) nvvm.tcgen05_st( "16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), @@ -2782,7 +2744,7 @@ def compute1_warp_group( for sub in cutlass.range_constexpr(2): v_pack = [] for m0 in cutlass.range_constexpr(4): - v_f16 = nvvm.ldmatrix( + v_frag = nvvm.ldmatrix( ( sV_base + v_idx * v_stage_elems @@ -2794,7 +2756,7 @@ def compute1_warp_group( nvvm.MMALayout.COL, ) for i in cutlass.range_constexpr(4): - v_pack.append(v_f16[i]) + v_pack.append(v_frag[i]) nvvm.tcgen05_st( "16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), @@ -2804,9 +2766,7 @@ def compute1_warp_group( bars.mb_y_ready[0].arrive() # ---- dU restage: dV acc ---------------------------------------------- - bars.mb_dyp_inp_done[0].wait(dyp_inp_free.phase) - dyp_inp_free = advance(dyp_inp_free, 1) - bars.mb_du_total_ready[0].wait(cg1_du_total_ready.phase) + bars.mb_du_total_acc_ready[0].wait(cg1_du_total_ready.phase) cg1_du_total_ready = advance(cg1_du_total_ready, 1) for sub in cutlass.range_constexpr(2): du_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_acc_col, cutlass.Float32), num=8) @@ -2818,7 +2778,6 @@ def compute1_warp_group( ) nvvm.tcgen05_wait("store") bars.mb_du_inp_ready[0].arrive() - bars.mb_do_cg1_done[do_idx].arrive() # ---- U readout ------------------------------------------------------- bars.mb_u_acc_ready[0].wait(cg1_u_ready.phase) @@ -2829,7 +2788,7 @@ def compute1_warp_group( u_regs.append([u_vec[k] for k in range(32)]) for sub in cutlass.range_constexpr(2): for m0 in cutlass.range_constexpr(4): - u_f16 = [fp32_to_fp16(u_regs[sub][8 * m0 + 2 * j], u_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + u_pack = [fp32_to_fp16(u_regs[sub][8 * m0 + 2 * j], u_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] nvvm.stmatrix( ( sV_base @@ -2838,12 +2797,11 @@ def compute1_warp_group( + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) ).raw_ptr(), - u_f16, + u_pack, nvvm.MMALayout.COL, ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_u_ready[0].arrive() - bars.mb_v_cg1_done[v_idx].arrive() # ---- dY -------------------------------------------------------------- bars.mb_dy_acc_ready[0].wait(cg1_dy_ready.phase) @@ -2882,7 +2840,7 @@ def compute1_warp_group( sdv_done_index = advance(sdv_done_index, 1) for sub in cutlass.range_constexpr(2): for m0 in cutlass.range_constexpr(4): - dv_f16 = [fp32_to_fp16(dy_regs[sub][8 * m0 + 2 * j], dy_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + dv_pack = [fp32_to_fp16(dy_regs[sub][8 * m0 + 2 * j], dy_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] nvvm.stmatrix( ( sdV_base @@ -2891,14 +2849,11 @@ def compute1_warp_group( + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) ).raw_ptr(), - dv_f16, + dv_pack, nvvm.MMALayout.COL, ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_dv_tmastg_ready[dv_stg_idx].arrive() - dk_stg_idx = dk_index.idx - bars.mb_dk_tmastg_done[dk_stg_idx].wait(dk_index.phase) - dk_index = advance(dk_index, cfg.smem_dk_stages) dq_stg_idx = dq_index.idx bars.mb_dq_tmastg_done[dq_stg_idx].wait(dq_index.phase) dq_index = advance(dq_index, cfg.smem_dq_stages) @@ -2907,9 +2862,9 @@ def compute1_warp_group( # ---- dBeta/dGate v-terms: dBeta_t += rowsum(dV ⊙ Y)_t / Beta_t ------- part_y = [cutlass.Float32(0.0)] * 16 for sub in cutlass.range_constexpr(2): - y_pk = nvvm.tcgen05_ld("16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), num=8) + y_vec = nvvm.tcgen05_ld("16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_y_col, cutlass.Int32), num=8) for j in cutlass.range_constexpr(16): - lo, hi = f16x2_to_f32(y_pk[j], dtype=cfg.io_dtype) + lo, hi = f16x2_to_f32(y_vec[j], dtype=cfg.io_dtype) frag_e0 = cutlass.const_expr(2 * j) part_e0 = cutlass.const_expr((frag_e0 // 4) * 2 + (frag_e0 % 2)) if cutlass.const_expr(sub == 0 and j % 2 == 0): @@ -2923,9 +2878,11 @@ def compute1_warp_group( if chunk_idx >= FIRST_STATE_CHUNK: part_g = [cutlass.Float32(0.0)] * 16 for sub in cutlass.range_constexpr(2): - g_k_state_pk = nvvm.tcgen05_ld("16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_g_k_state_col, cutlass.Int32), num=8) + g_k_state_vec = nvvm.tcgen05_ld( + "16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_g_k_state_col, cutlass.Int32), num=8 + ) for j in cutlass.range_constexpr(16): - lo, hi = f16x2_to_f32(g_k_state_pk[j], dtype=cfg.io_dtype) + lo, hi = f16x2_to_f32(g_k_state_vec[j], dtype=cfg.io_dtype) frag_e0 = cutlass.const_expr(2 * j) part_e0 = cutlass.const_expr((frag_e0 // 4) * 2 + (frag_e0 % 2)) if cutlass.const_expr(sub == 0 and j % 2 == 0): @@ -2983,14 +2940,14 @@ def compute1_warp_group( # ---- dQ final read -> sdQ -------------------------------------------- bars.mb_dq_acc_total_ready[0].wait(dq_total_ready_index.phase) dq_total_ready_index = advance(dq_total_ready_index, 1) - dq_frag = [] + dq_regs = [] for sub in cutlass.range_constexpr(2): - dq_vals = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dstate_inp_col, cutlass.Float32), num=8) - dq_frag.append([dq_vals[k] for k in range(32)]) + dq_vec = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dstate_inp_col, cutlass.Float32), num=8) + dq_regs.append([dq_vec[k] for k in range(32)]) for m0 in cutlass.range_constexpr(4): frag_addr = frag_slab_off + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) - dq_f16 = [fp32_to_fp16(dq_vals[8 * m0 + 2 * j], dq_vals[8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] - nvvm.stmatrix((sdQ_base + dq_stg_idx * dq_stage_elems + frag_addr).raw_ptr(), dq_f16, nvvm.MMALayout.COL) + dq_pack = [fp32_to_fp16(dq_vec[8 * m0 + 2 * j], dq_vec[8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + nvvm.stmatrix((sdQ_base + dq_stg_idx * dq_stage_elems + frag_addr).raw_ptr(), dq_pack, nvvm.MMALayout.COL) nvvm.fence_proxy("async.shared", space="cta") bars.mb_dq_acc_total_done[0].arrive() bars.mb_dq_tmastg_ready[dq_stg_idx].arrive() @@ -3004,10 +2961,10 @@ def compute1_warp_group( frag_e0 = cutlass.const_expr(8 * m0 + 2 * i) part_e0 = cutlass.const_expr((frag_e0 // 4) * 2 + (frag_e0 % 2)) if cutlass.const_expr(sub == 0 and i % 2 == 0): - part_q[part_e0], part_q[part_e0 + 1] = fmul2(dq_frag[sub][frag_e0], dq_frag[sub][frag_e0 + 1], q_lo, q_hi) + part_q[part_e0], part_q[part_e0 + 1] = fmul2(dq_regs[sub][frag_e0], dq_regs[sub][frag_e0 + 1], q_lo, q_hi) else: part_q[part_e0], part_q[part_e0 + 1] = ffma2( - dq_frag[sub][frag_e0], dq_frag[sub][frag_e0 + 1], q_lo, q_hi, part_q[part_e0], part_q[part_e0 + 1] + dq_regs[sub][frag_e0], dq_regs[sub][frag_e0 + 1], q_lo, q_hi, part_q[part_e0], part_q[part_e0 + 1] ) part_q_lo, part_q_hi = warp_reduce_scatter_frag_16_elems(part_q, lane_id) tok0 = (lane_id // 4) * 8 + (lane_id % 4) * 2 @@ -3042,18 +2999,21 @@ def compute1_warp_group( for k in cutlass.range_constexpr(32): dstate_regs[k][sub] = dstate_vec[k] - dstate_f16 = [fp32_to_fp16(dstate_regs[2 * j][sub], dstate_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] + dstate_pack = [fp32_to_fp16(dstate_regs[2 * j][sub], dstate_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_dstate_inp_col + sub * sttm_width, cutlass.Int32), - cutlass.Vector.from_elements(tuple(dstate_f16), cutlass.Int32), + cutlass.Vector.from_elements(tuple(dstate_pack), cutlass.Int32), ) nvvm.tcgen05_wait("store") bars.mb_dstate_inp_ready[dstate_inp_idx].arrive() # ---- dK fold --------------------------------------------------------- + dk_stg_idx = dk_index.idx + bars.mb_dk_tmastg_done[dk_stg_idx].wait(dk_index.phase) + dk_index = advance(dk_index, cfg.smem_dk_stages) if chunk_idx >= FIRST_STATE_CHUNK: - bars.mb_dk_state_path_ready[0].wait(cg1_dk_state_path_ready.phase) + bars.mb_dk_state_path_acc_ready[0].wait(cg1_dk_state_path_ready.phase) cg1_dk_state_path_ready = advance(cg1_dk_state_path_ready, 1) dk_state_path_vecs = [] for sub in cutlass.range_constexpr(2): @@ -3073,7 +3033,7 @@ def compute1_warp_group( dk_state_path_row += [n0, n1] dk_state_path_regs.append(dk_state_path_row) - bars.mb_dk_total_ready[0].wait(cg1_dk_total_ready.phase) + bars.mb_dk_total_acc_ready[0].wait(cg1_dk_total_ready.phase) cg1_dk_total_ready = advance(cg1_dk_total_ready, 1) dmr_vecs = [] for sub in cutlass.range_constexpr(2): @@ -3081,11 +3041,11 @@ def compute1_warp_group( nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_acc_col, cutlass.Float32), num=8) ) nvvm.tcgen05_wait("load") - bars.mb_dk_total_done[0].arrive() + bars.mb_dk_total_acc_done[0].arrive() for sub in cutlass.range_constexpr(2): dk_sum = [dmr_vecs[sub][k] + dk_state_path_regs[sub][k] for k in range(32)] for m0 in cutlass.range_constexpr(4): - dk_f16 = [fp32_to_fp16(dk_sum[8 * m0 + 2 * j], dk_sum[8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + dk_pack = [fp32_to_fp16(dk_sum[8 * m0 + 2 * j], dk_sum[8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] nvvm.stmatrix( ( sdK_base @@ -3094,13 +3054,13 @@ def compute1_warp_group( + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) ).raw_ptr(), - dk_f16, + dk_pack, nvvm.MMALayout.COL, ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_dk_tmastg_ready[dk_stg_idx].arrive() if chunk_idx < FIRST_STATE_CHUNK: - bars.mb_dk_total_ready[0].wait(cg1_dk_total_ready.phase) + bars.mb_dk_total_acc_ready[0].wait(cg1_dk_total_ready.phase) cg1_dk_total_ready = advance(cg1_dk_total_ready, 1) dmr_vecs = [] for sub in cutlass.range_constexpr(2): @@ -3108,10 +3068,10 @@ def compute1_warp_group( nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_dvdk_acc_col, cutlass.Float32), num=8) ) nvvm.tcgen05_wait("load") - bars.mb_dk_total_done[0].arrive() + bars.mb_dk_total_acc_done[0].arrive() for sub in cutlass.range_constexpr(2): for m0 in cutlass.range_constexpr(4): - dk_f16 = [fp32_to_fp16(dmr_vecs[sub][8 * m0 + 2 * j], dmr_vecs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + dk_pack = [fp32_to_fp16(dmr_vecs[sub][8 * m0 + 2 * j], dmr_vecs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] nvvm.stmatrix( ( sdK_base @@ -3120,7 +3080,7 @@ def compute1_warp_group( + (frag_row + m0 * 16) * 64 + swizzle_xor_128b(frag_row + m0 * 16, frag_col + sub * 16) ).raw_ptr(), - dk_f16, + dk_pack, nvvm.MMALayout.COL, ) nvvm.fence_proxy("async.shared", space="cta") @@ -3130,8 +3090,6 @@ def compute1_warp_group( if chunk_idx >= wstart + 1: bars.mb_state_dot_dstate_done[0].wait(cg1_state_dot_dstate_index.phase) cg1_state_dot_dstate_index = advance(cg1_state_dot_dstate_index, 1) - bars.mb_dstate_smem_done[0].wait(dstate_smem_index.phase) - dstate_smem_index = advance(dstate_smem_index, 1) for b in cutlass.range_constexpr(2): for col_half in cutlass.range_constexpr(2): dstate_smem_vec = nvvm.tcgen05_ld( @@ -3139,7 +3097,7 @@ def compute1_warp_group( nvvm.make_tmem_ptr(((tmem_warp_row + b * 16) << 16) + tmem_dstate_acc_col + col_half * 64, cutlass.Float32), num=8, ) - dstate_smem_f16 = [fp32_to_fp16(dstate_smem_vec[2 * j], dstate_smem_vec[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] + dstate_smem_pack = [fp32_to_fp16(dstate_smem_vec[2 * j], dstate_smem_vec[2 * j + 1], dtype=cfg.io_dtype) for j in range(16)] for c in cutlass.range_constexpr(4): dstate_smem_row = col_half * 64 + frag_row + c * 16 nvvm.stmatrix( @@ -3149,7 +3107,7 @@ def compute1_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [dstate_smem_f16[c * 4 + 0], dstate_smem_f16[c * 4 + 1], dstate_smem_f16[c * 4 + 2], dstate_smem_f16[c * 4 + 3]], + [dstate_smem_pack[c * 4 + 0], dstate_smem_pack[c * 4 + 1], dstate_smem_pack[c * 4 + 2], dstate_smem_pack[c * 4 + 3]], nvvm.MMALayout.COL, ) nvvm.fence_proxy("async.shared", space="cta") @@ -3182,7 +3140,7 @@ def compute1_warp_group( for kk in cutlass.range_constexpr(32): gDstate0[sub * ldtm_width + kk, cg1_tidx] = dstate0_vec[kk] if cutlass.const_expr(not cfg.use_dstate_in): - bars.mb_dstate_acc_done[dstate_idx].arrive() + bars.mb_dstate_scale_acc_done[dstate_idx].arrive() else: if cutlass.const_expr(cfg.use_dstate0): write_passthrough = True @@ -3346,17 +3304,11 @@ def build_descs( q_headed = cute.make_tensor(q.iterator, cute.make_layout((seqlen, h_q, d_k), stride=(q_row_stride, q_head_stride, 1))) k_headed = cute.make_tensor(k.iterator, cute.make_layout((seqlen, h_k, d_k), stride=(k_row_stride, k_head_stride, 1))) - def trans_headed(t, d, hn): - return cute.make_tensor( - t.iterator, - cute.make_layout((d, hn, seqlen), stride=(1, t.stride[1], t.stride[0])), - ) - - v_headed = trans_headed(v, d_v, h_v) - do_headed = trans_headed(do_, d_v, heads_out) - dq_headed = trans_headed(dq, dq.shape[2], heads_out) - dk_headed = trans_headed(dk, dk.shape[2], heads_out) - dv_headed = trans_headed(dv, d_v, heads_out) + v_headed = cute.make_tensor(v.iterator, cute.make_layout((d_v, h_v, seqlen), stride=(1, v.stride[1], v.stride[0]))) + do_headed = cute.make_tensor(do_.iterator, cute.make_layout((d_v, heads_out, seqlen), stride=(1, do_.stride[1], do_.stride[0]))) + dq_headed = cute.make_tensor(dq.iterator, cute.make_layout((dq.shape[2], heads_out, seqlen), stride=(1, dq.stride[1], dq.stride[0]))) + dk_headed = cute.make_tensor(dk.iterator, cute.make_layout((dk.shape[2], heads_out, seqlen), stride=(1, dk.stride[1], dk.stride[0]))) + dv_headed = cute.make_tensor(dv.iterator, cute.make_layout((d_v, heads_out, seqlen), stride=(1, dv.stride[1], dv.stride[0]))) swz128 = tma.TensorMapSwizzle.s128b base_desc_q = tma.create_tensor_map_tiled_from_view(q_headed, box_dims=(bt, 1, granule_elems), stride_order=(2, 1, 0), swizzle=swz128) base_desc_k = tma.create_tensor_map_tiled_from_view(k_headed, box_dims=(bt, 1, granule_elems), stride_order=(2, 1, 0), swizzle=swz128) @@ -3995,15 +3947,12 @@ def kernel( for s in range(cfg.smem_v_stages): bars.mb_v_ready[s].init() bars.mb_v_mma_done[s].init() - bars.mb_v_cg1_done[s].init() for s in range(cfg.smem_do_stages): bars.mb_do_ready[s].init() bars.mb_do_mma_done[s].init() - bars.mb_do_cg1_done[s].init() for s in range(cfg.smem_state_stages): bars.mb_state_ready[s].init() - bars.mb_state_done[s].init() - bars.mb_state_cg0_done[s].init() + bars.mb_state_mma_done[s].init() for s in range(cfg.smem_gate_stages): bars.mb_gate_ready[s].init() bars.mb_gate_done[s].init() @@ -4012,17 +3961,17 @@ def kernel( bars.mb_beta_done[s].init() for s in range(cfg.tmem_dstate_acc_stages): bars.mb_dstate_acc_ready[s].init() - bars.mb_dstate_acc_done[s].init() + bars.mb_dstate_scale_acc_done[s].init() for b in ( - bars.mb_du_scale_ready, - bars.mb_du_scale_done, - bars.mb_du_total_ready, - bars.mb_dk_scale_ready, - bars.mb_dk_scale_done, - bars.mb_dk_attn_ready, - bars.mb_dk_attn_done, - bars.mb_dk_total_ready, - bars.mb_dk_total_done, + bars.mb_du_scale_acc_ready, + bars.mb_du_scale_acc_done, + bars.mb_du_total_acc_ready, + bars.mb_dk_scale_acc_ready, + bars.mb_dk_scale_acc_done, + bars.mb_dk_attn_acc_ready, + bars.mb_dk_attn_acc_done, + bars.mb_dk_total_acc_ready, + bars.mb_dk_total_acc_done, ): b[0].init() for b in ( @@ -4036,7 +3985,6 @@ def kernel( b[0].init() for s in range(cfg.smem_ainv_stages): bars.mb_ainv_ready[s].init() - bars.mb_ainv_done[s].init() for s in range(cfg.smem_qk_stages): bars.mb_qk_ready[s].init() bars.mb_qk_done[s].init() @@ -4045,10 +3993,8 @@ def kernel( bars.mb_dstate_inp_done[s].init() for b in ( bars.mb_do_prime_inp_ready, - bars.mb_do_prime_inp_done, bars.mb_du_inp_ready, bars.mb_dyp_inp_ready, - bars.mb_dyp_inp_done, ): b[0].init() for s in range(cfg.smem_dq_stages): @@ -4064,19 +4010,18 @@ def kernel( bars.mb_sdv_done[0].init() bars.mb_u_ready[0].init() bars.mb_dstate_smem_ready[0].init() - bars.mb_dstate_smem_done[0].init() bars.mb_da_ready[0].init() bars.mb_dq_acc_scale_ready[0].init() bars.mb_dq_acc_scale_done[0].init() bars.mb_dq_acc_total_ready[0].init() bars.mb_dq_acc_total_done[0].init() bars.mb_da_acc_ready[0].init() - bars.mb_dm_ready[0].init() - bars.mb_dm_done[0].init() + bars.mb_dm_acc_ready[0].init() + bars.mb_dm_acc_done[0].init() bars.mb_dbeta_cg1_ready[0].init() bars.mb_dgate_cg1_ready[0].init() bars.mb_state_dot_dstate_done[0].init() - bars.mb_dk_state_path_ready[0].init() + bars.mb_dk_state_path_acc_ready[0].init() bars.mb_tmem_done[0].init() nvvm.fence_mbarrier_init() diff --git a/python/cudnn/linear_attention/frost/kernel/gdn_prefill_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn_prefill_f16.py index 16f670fc0..e280fcd8c 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn_prefill_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn_prefill_f16.py @@ -144,7 +144,7 @@ class GdnBars(NamedTuple): mb_state_acc_scale_done: MBarrier mb_o_acc_ready: MBarrier mb_o_acc_done: MBarrier - mb_o_state_scale_acc_ready: MBarrier + mb_o_final_acc_ready: MBarrier mb_o_state_scale_acc_done: MBarrier mb_cg0_acc_ready: MBarrier mb_cg0_acc_done: MBarrier @@ -159,8 +159,8 @@ class GdnBars(NamedTuple): mb_qk_ready: MBarrier mb_qk_done: MBarrier - mb_k_state_ready: MBarrier - mb_nv_ready: MBarrier + mb_k_state_acc_ready: MBarrier + mb_nv_acc_ready: MBarrier mb_o_tmastg_ready: MBarrier mb_o_tmastg_done: MBarrier @@ -201,7 +201,7 @@ def alloc(n): mb_state_acc_scale_done=MBarrier(alloc(cfg.tmem_state_acc_stages), stages=cfg.tmem_state_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_acc_ready=MBarrier(alloc(cfg.tmem_q_state_acc_stages), stages=cfg.tmem_q_state_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_o_acc_done=MBarrier(alloc(cfg.tmem_q_state_acc_stages), stages=cfg.tmem_q_state_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_o_state_scale_acc_ready=MBarrier( + mb_o_final_acc_ready=MBarrier( alloc(cfg.tmem_q_state_acc_stages), stages=cfg.tmem_q_state_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT ), mb_o_state_scale_acc_done=MBarrier( @@ -217,8 +217,8 @@ def alloc(n): mb_ainv_done=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_qk_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=CG0_THREADS // 2, producer=Producer.THREAD), mb_qk_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_k_state_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_nv_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_k_state_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_nv_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_o_tmastg_ready=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_tmastg_done=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=EPI_WARP, producer=Producer.THREAD), mb_checkpoint_tmastg_ready=MBarrier( @@ -290,10 +290,10 @@ def blockwise_diagonal_8x8_to_16x16(cfg, base_int, d0, lane_id): mma_step_k8(c_regs, [d, d], [c], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) for i in cutlass.range_constexpr(4): c_regs[i] = -c_regs[i] - a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(2)] + a_pack = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(2)] # ---- C = T @ A^{-1} ---------------------------------------------------------- - ai = nvvm.ldmatrix( + ai_frag = nvvm.ldmatrix( cutlass.inttoptr(base_int + swizzle_lin_128b(d0 * 64 + d0 + ldsm_x1_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), 1, nvvm.MMALayout.COL, @@ -301,15 +301,15 @@ def blockwise_diagonal_8x8_to_16x16(cfg, base_int, d0, lane_id): o_regs = cutlass.Array(cutlass.Float32, 4, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(4): o_regs[i] = cutlass.Float32(0.0) - mma_step_k8(o_regs, a_f16, [ai], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) - o_f16 = fp32_to_fp16(o_regs[0], o_regs[1], dtype=cfg.io_dtype) + mma_step_k8(o_regs, a_pack, [ai_frag], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) + o_pack = fp32_to_fp16(o_regs[0], o_regs[1], dtype=cfg.io_dtype) # ---- store corrected C ------------------------------------------------------- nvvm.stmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + ldsm_x1_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 ), - o_f16, + o_pack, nvvm.MMALayout.ROW, ) @@ -345,10 +345,10 @@ def blockwise_diagonal_16x16_to_32x32(cfg, base_int, d0, lane_id): mma_step(c_regs, d, c, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) for i in cutlass.range_constexpr(8): c_regs[i] = -c_regs[i] - a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + a_pack = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] # ---- C = T @ A^{-1} ---------------------------------------------------------- - ai = list( + ai_frag = list( nvvm.ldmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b(d0 * 64 + d0 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 @@ -360,15 +360,15 @@ def blockwise_diagonal_16x16_to_32x32(cfg, base_int, d0, lane_id): o_regs = cutlass.Array(cutlass.Float32, 8, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(8): o_regs[i] = cutlass.Float32(0.0) - mma_step(o_regs, a_f16, ai, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) - o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + mma_step(o_regs, a_pack, ai_frag, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) + o_pack = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] # ---- store corrected C ------------------------------------------------------- nvvm.stmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 ), - o_f16, + o_pack, nvvm.MMALayout.ROW, ) @@ -379,9 +379,9 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): band = warp_id % 2 bpe = cfg.io_dtype.width // 8 ldsm_x4_lane_off = (lane_id % 16) * 64 + (lane_id // 16) * 8 - a_regs = [] + a_frags = [] for vs in cutlass.range_constexpr(2): - a_regs += list( + a_frags += list( nvvm.ldmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b((32 + band * 16) * 64 + 32 + vs * 16 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, @@ -392,9 +392,9 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): nvvm.MMALayout.ROW, ) ) - b_regs = [] + b_frags = [] for vs in cutlass.range_constexpr(4): - b_regs += list( + b_frags += list( nvvm.ldmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b((32 + (vs // 2) * 16) * 64 + (vs % 2) * 16 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, @@ -411,15 +411,15 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): for i in cutlass.range_constexpr(16): c_regs[i] = cutlass.Float32(0.0) for ks in cutlass.range_constexpr(2): - mma_step(c_regs, a_regs, b_regs[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) + mma_step(c_regs, a_frags, b_frags[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) for i in cutlass.range_constexpr(16): c_regs[i] = -c_regs[i] - a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + a_pack = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] # ---- C = T @ A^{-1} ---------------------------------------------------------- - ai_regs = [] + ai_frags = [] for vs in cutlass.range_constexpr(4): - ai_regs += list( + ai_frags += list( nvvm.ldmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b(((vs // 2) * 16) * 64 + (vs % 2) * 16 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, @@ -434,8 +434,8 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): for i in cutlass.range_constexpr(16): o_regs[i] = cutlass.Float32(0.0) for ks in cutlass.range_constexpr(2): - mma_step(o_regs, a_f16, ai_regs[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) - o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + mma_step(o_regs, a_pack, ai_frags[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) + o_pack = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] # ---- store corrected C ------------------------------------------------------- nvvm.barrier_cta_sync_aligned( @@ -446,14 +446,14 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): cutlass.inttoptr( base_int + swizzle_lin_128b((32 + band * 16) * 64 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 ), - o_f16[0:4], + o_pack[0:4], nvvm.MMALayout.ROW, ) nvvm.stmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b((32 + band * 16) * 64 + 16 + ldsm_x4_lane_off, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16 ), - o_f16[4:8], + o_pack[4:8], nvvm.MMALayout.ROW, ) @@ -919,7 +919,7 @@ def mma_warp( q_state_acc_idx = o_acc_index.idx ainv_idx = ainv_index.idx qk_idx = qk_index.idx - q_state2_idx = o_state_scale_index.idx + o_scale_idx = o_state_scale_index.idx kv_acc_idx = kv_acc_index.idx kq_member_off = member * KQ_BOX desc_k = sKQ[kq_idx].desc() + kq_member_off @@ -929,7 +929,7 @@ def mma_warp( desc_kt = sKQ_trans[kq_idx].desc() + kq_member_off state_a_ptr = nvvm.make_tmem_ptr(tmem_state_inp_col + state_inp_idx * STATE_INP_STAGE_COLS, cutlass.Int8) q_state_acc_ptr = nvvm.make_tmem_ptr(tmem_q_state_col + q_state_acc_idx * ACC_STAGE_COLS, cutlass.Float32) - qkv_acc_ptr = nvvm.make_tmem_ptr(tmem_q_state_col + q_state2_idx * ACC_STAGE_COLS, cutlass.Float32) + qkv_acc_ptr = nvvm.make_tmem_ptr(tmem_q_state_col + o_scale_idx * ACC_STAGE_COLS, cutlass.Float32) state_acc_ptr = nvvm.make_tmem_ptr(tmem_state_col + kv_acc_idx * KV_ACC_STAGE_COLS, cutlass.Float32) kq_index = advance(kq_index, cfg.smem_kq_stages) @@ -960,7 +960,7 @@ def mma_warp( for k in cutlass.range_constexpr(KQ_HALF_K): mma_ts_step(bmm_q_state_desc, state_a_ptr.subview(KQ_A_HALF), desc_k + KQ_SEG, k_state_acc_ptr, k, cutlass.Boolean(True)) if elect_one: - bars.mb_k_state_ready[0].arrive(cta_group=1) + bars.mb_k_state_acc_ready[0].arrive(cta_group=1) # ---- q_state^T (GEMM 4) = state^T(T) @ Q^T ------------------------------------ bars.mb_o_acc_done[q_state_acc_idx].wait(o_acc_index.phase) @@ -981,7 +981,7 @@ def mma_warp( for k in cutlass.range_constexpr(cfg.b_t // 16): mma_ts_step(bmm_qkv_ts_desc, v_k_state_inp_ptr, desc_ainv, nv_acc_ptr, k, cutlass.Boolean(k > 0)) if elect_one: - bars.mb_nv_ready[0].arrive(cta_group=1) + bars.mb_nv_acc_ready[0].arrive(cta_group=1) bars.mb_ainv_done[ainv_idx].arrive(cta_group=1) # ---- KK/QK lookahead (member 0) = [K;Q](S) @ K^T --------------------- @@ -1003,7 +1003,7 @@ def mma_warp( bars.mb_qk_ready[qk_idx].wait(qk_index.phase) qk_index = advance(qk_index, cfg.smem_qk_stages) if have_state: - bars.mb_o_state_scale_acc_done[q_state2_idx].wait(o_state_scale_index.phase) + bars.mb_o_state_scale_acc_done[o_scale_idx].wait(o_state_scale_index.phase) o_state_scale_index = advance(o_state_scale_index, cfg.tmem_q_state_acc_stages) bars.mb_nv_inp_ready[0].wait(nv_inp_ready.phase) nv_inp_ready = advance(nv_inp_ready, 1) @@ -1011,7 +1011,7 @@ def mma_warp( mma_ts_step(bmm_qkv_ts_desc, v_k_state_inp_ptr, desc_nv, qkv_acc_ptr, k, cutlass.Boolean(True) if cutlass.const_expr(k > 0) else have_state) if elect_one: bars.mb_qk_done[qk_idx].arrive(cta_group=1) - bars.mb_o_state_scale_acc_ready[q_state2_idx].arrive(cta_group=1) + bars.mb_o_final_acc_ready[o_scale_idx].arrive(cta_group=1) # ---- state^T (GEMM 7) += decay_V^T(T) @ K -------------------------------- bars.mb_decay_v_inp_ready[0].wait(decay_v_inp_ready.phase) @@ -1304,7 +1304,6 @@ def compute0_warp_group( kk_acc_idx = acc1_idx if pair_half == 1 else acc0_idx kk_acc_phase = acc1_phase if pair_half == 1 else acc0_phase qk_acc_idx = acc0_idx if pair_half == 1 else acc1_idx - qk_acc_phase = acc0_phase if pair_half == 1 else acc1_phase ainv0_idx = ainv_index.idx ainv0_phase = ainv_index.phase @@ -1329,12 +1328,12 @@ def compute0_warp_group( bars.mb_ainv_done[kk_ainv_idx].wait(kk_ainv_phase) for u in cutlass.range_constexpr(2): kk_vec = kk_vec1 if cutlass.const_expr(u == 1) else kk_vec0 - kk_f16 = [] + kk_pack = [] for k in cutlass.range_constexpr(num_vals // 2): b0 = kk_beta[u * 2 + 1] if cutlass.const_expr((k % 2) == 1) else kk_beta[u * 2] p0, p1 = fmul2(kk_vec[2 * k], kk_vec[2 * k + 1], decay_t_kk[u * num_vals + 2 * k], decay_t_kk[u * num_vals + 2 * k + 1]) v0, v1 = fmul2(p0, p1, b0, b0) - kk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) + kk_pack.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) st_row = half_row_base + u * 16 + store_row_frag for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( @@ -1343,7 +1342,7 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [kk_f16[c * 4 + 0], kk_f16[c * 4 + 1], kk_f16[c * 4 + 2], kk_f16[c * 4 + 3]], + [kk_pack[c * 4 + 0], kk_pack[c * 4 + 1], kk_pack[c * 4 + 2], kk_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) @@ -1387,9 +1386,9 @@ def compute0_warp_group( beta_col = [] for k in cutlass.range_constexpr(num_vals): beta_col.append(sBeta[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, beta0_idx]) - ainv_f16 = [] + ainv_frags = [] for c in cutlass.range_constexpr(ACC_N_FRAGS): - ainv_f16 += list( + ainv_frags += list( nvvm.ldmatrix( cutlass.inttoptr( ainv0_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, @@ -1400,11 +1399,11 @@ def compute0_warp_group( nvvm.MMALayout.ROW, ) ) - ainv_scaled = [] + ainv_pack = [] for j in cutlass.range_constexpr(num_vals // 2): - lo, hi = f16x2_to_f32(ainv_f16[j], dtype=cfg.io_dtype) + lo, hi = f16x2_to_f32(ainv_frags[j], dtype=cfg.io_dtype) s0, s1 = fmul2(lo, hi, beta_col[2 * j], beta_col[2 * j + 1]) - ainv_scaled.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) + ainv_pack.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( cutlass.inttoptr( @@ -1412,7 +1411,7 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [ainv_scaled[c * 4 + 0], ainv_scaled[c * 4 + 1], ainv_scaled[c * 4 + 2], ainv_scaled[c * 4 + 3]], + [ainv_pack[c * 4 + 0], ainv_pack[c * 4 + 1], ainv_pack[c * 4 + 2], ainv_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) nvvm.fence_proxy("async.shared", space="cta") @@ -1423,9 +1422,9 @@ def compute0_warp_group( beta_col = [] for k in cutlass.range_constexpr(num_vals): beta_col.append(sBeta[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, beta1_idx]) - ainv_f16 = [] + ainv_frags = [] for c in cutlass.range_constexpr(ACC_N_FRAGS): - ainv_f16 += list( + ainv_frags += list( nvvm.ldmatrix( cutlass.inttoptr( ainv1_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, @@ -1436,11 +1435,11 @@ def compute0_warp_group( nvvm.MMALayout.ROW, ) ) - ainv_scaled = [] + ainv_pack = [] for j in cutlass.range_constexpr(num_vals // 2): - lo, hi = f16x2_to_f32(ainv_f16[j], dtype=cfg.io_dtype) + lo, hi = f16x2_to_f32(ainv_frags[j], dtype=cfg.io_dtype) s0, s1 = fmul2(lo, hi, beta_col[2 * j], beta_col[2 * j + 1]) - ainv_scaled.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) + ainv_pack.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( cutlass.inttoptr( @@ -1448,7 +1447,7 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [ainv_scaled[c * 4 + 0], ainv_scaled[c * 4 + 1], ainv_scaled[c * 4 + 2], ainv_scaled[c * 4 + 3]], + [ainv_pack[c * 4 + 0], ainv_pack[c * 4 + 1], ainv_pack[c * 4 + 2], ainv_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) nvvm.fence_proxy("async.shared", space="cta") @@ -1465,8 +1464,6 @@ def compute0_warp_group( my_qk_idx = qk0_idx if pair_half == 1 else qk1_idx my_qk_phase = qk0_phase if pair_half == 1 else qk1_phase - bars.mb_cg0_acc_ready[qk_acc_idx].wait(qk_acc_phase) - qk_base = sQk[my_qk_idx].base qk_vec0 = nvvm.tcgen05_ld( "16x256b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_cg0_acc_col + qk_acc_idx * ACC_STAGE_COLS, cutlass.Float32), num=8 @@ -1479,11 +1476,11 @@ def compute0_warp_group( bars.mb_qk_done[my_qk_idx].wait(my_qk_phase) for u in cutlass.range_constexpr(2): qk_vec = qk_vec1 if cutlass.const_expr(u == 1) else qk_vec0 - qk_f16 = [] + qk_pack = [] for k in cutlass.range_constexpr(num_vals // 2): p0, p1 = fmul2(qk_vec[2 * k], qk_vec[2 * k + 1], decay_t_qk[u * num_vals + 2 * k], decay_t_qk[u * num_vals + 2 * k + 1]) v0, v1 = fmul2(p0, p1, scale, scale) - qk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) + qk_pack.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) st_row = half_row_base + u * 16 + store_row_frag for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( @@ -1492,7 +1489,7 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [qk_f16[c * 4 + 0], qk_f16[c * 4 + 1], qk_f16[c * 4 + 2], qk_f16[c * 4 + 3]], + [qk_pack[c * 4 + 0], qk_pack[c * 4 + 1], qk_pack[c * 4 + 2], qk_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) nvvm.fence_proxy("async.shared", space="cta") @@ -1539,7 +1536,7 @@ def compute1_warp_group( gate_index = PipelineState.start(phase=0) kv_acc_index = PipelineState.start(phase=0) o_acc_ready_index = PipelineState.start(phase=0) - o_scale_ready_index = PipelineState.start(phase=0) + o_final_acc_ready_index = PipelineState.start(phase=0) k_state_ready_index = PipelineState.start(phase=0) nv_ready_index = PipelineState.start(phase=0) state_acc_seed_index = PipelineState.start(phase=1) @@ -1679,11 +1676,11 @@ def compute1_warp_group( for sub in cutlass.range_constexpr(num_state_subs): for k in cutlass.range_constexpr(32): state_regs[k][sub] = state_vecs[sub][k] - state_f16 = [fp32_to_fp16(state_regs[2 * j][sub], state_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] + state_pack = [fp32_to_fp16(state_regs[2 * j][sub], state_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_inp_col + sub * sttm_width, cutlass.Int32), - cutlass.Vector.from_elements(tuple(state_f16), cutlass.Int32), + cutlass.Vector.from_elements(tuple(state_pack), cutlass.Int32), ) nvvm.tcgen05_wait("store") bars.mb_state_inp_ready[state_inp_stage_idx].arrive() @@ -1695,7 +1692,7 @@ def compute1_warp_group( if cutlass.const_expr(cfg.split_k): do_checkpoint = do_checkpoint and chunk_idx >= wstart if do_checkpoint: - checkpoint_regs = [[cutlass.Int32(0) for _ in range(16)] for _ in range(4)] + checkpoint_pack = [[cutlass.Int32(0) for _ in range(16)] for _ in range(4)] for b in cutlass.range_constexpr(2): for col_half in cutlass.range_constexpr(2): checkpoint_vec = nvvm.tcgen05_ld( @@ -1704,7 +1701,7 @@ def compute1_warp_group( num=8, ) for j in cutlass.range_constexpr(16): - checkpoint_regs[b * 2 + col_half][j] = fp32_to_fp16( + checkpoint_pack[b * 2 + col_half][j] = fp32_to_fp16( checkpoint_vec[2 * j], checkpoint_vec[2 * j + 1], dtype=cfg.io_dtype ) checkpoint_stage = checkpoint_cnt % cfg.smem_checkpoint_stages @@ -1723,10 +1720,10 @@ def compute1_warp_group( cfg.io_dtype, ), [ - checkpoint_regs[b * 2 + col_half][c * 4 + 0], - checkpoint_regs[b * 2 + col_half][c * 4 + 1], - checkpoint_regs[b * 2 + col_half][c * 4 + 2], - checkpoint_regs[b * 2 + col_half][c * 4 + 3], + checkpoint_pack[b * 2 + col_half][c * 4 + 0], + checkpoint_pack[b * 2 + col_half][c * 4 + 1], + checkpoint_pack[b * 2 + col_half][c * 4 + 2], + checkpoint_pack[b * 2 + col_half][c * 4 + 3], ], nvvm.MMALayout.COL, ) @@ -1768,11 +1765,11 @@ def compute1_warp_group( bars.mb_v_ready[v_idx].wait(v_index.phase) v_index = advance(v_index, cfg.smem_v_stages) - v_words = [[cutlass.Int32(0), cutlass.Int32(0)] for _ in range(16)] + v_frags = [[cutlass.Int32(0), cutlass.Int32(0)] for _ in range(16)] for c in cutlass.range_constexpr(8): m0 = cutlass.const_expr(c % 4) sub = cutlass.const_expr(c // 4) - v_f16 = nvvm.ldmatrix( + v_frag = nvvm.ldmatrix( ( sV_base + v_idx * v_stage_elems @@ -1784,9 +1781,9 @@ def compute1_warp_group( nvvm.MMALayout.COL, ) for i in cutlass.range_constexpr(4): - v_words[4 * m0 + i][sub] = v_f16[i] + v_frags[4 * m0 + i][sub] = v_frag[i] if valid_state: - bars.mb_k_state_ready[0].wait(k_state_ready_index.phase) + bars.mb_k_state_acc_ready[0].wait(k_state_ready_index.phase) k_state_ready_index = advance(k_state_ready_index, 1) for sub in cutlass.range_constexpr(2): @@ -1798,12 +1795,12 @@ def compute1_warp_group( for j in cutlass.range_constexpr(16): s0, s1 = fmul2(k_state_vec[2 * j], k_state_vec[2 * j + 1], cumprod_vals[2 * j], cumprod_vals[2 * j + 1]) k_state_word = fp32_to_fp16(s0, s1, dtype=cfg.io_dtype) - v_words[j][sub] = sub_f16x2(v_words[j][sub], k_state_word, cfg.io_dtype) + v_frags[j][sub] = sub_f16x2(v_frags[j][sub], k_state_word, cfg.io_dtype) for sub in cutlass.range_constexpr(2): nvvm.tcgen05_st( "16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_v_k_state_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(v_words[j][sub] for j in range(16)), cutlass.Int32), + cutlass.Vector.from_elements(tuple(v_frags[j][sub] for j in range(16)), cutlass.Int32), ) nvvm.tcgen05_wait("store") bars.mb_v_k_state_inp_ready[0].arrive() @@ -1833,7 +1830,7 @@ def compute1_warp_group( bars.mb_o_state_scale_acc_done[q_state_idx].arrive() # ---- new_V_epi + decay_V publish --------------------------------- - bars.mb_nv_ready[0].wait(nv_ready_index.phase) + bars.mb_nv_acc_ready[0].wait(nv_ready_index.phase) nv_ready_index = advance(nv_ready_index, 1) bars.mb_v_done[v_idx].arrive() @@ -1851,11 +1848,11 @@ def compute1_warp_group( for k in cutlass.range_constexpr(32): nv_regs[k][sub] = nv_vecs[sub][k] - nv_f16 = [fp32_to_fp16(nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] + nv_pack = [fp32_to_fp16(nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_nv_inp_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(nv_f16), cutlass.Int32), + cutlass.Vector.from_elements(tuple(nv_pack), cutlass.Int32), ) nvvm.tcgen05_wait("store") bars.mb_nv_inp_ready[0].arrive() @@ -1865,36 +1862,36 @@ def compute1_warp_group( nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub] = fmul2( nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], decay_scale_vals[2 * j], decay_scale_vals[2 * j + 1] ) - decay_f16 = [fp32_to_fp16(nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] + decay_pack = [fp32_to_fp16(nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_decay_v_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(decay_f16), cutlass.Int32), + cutlass.Vector.from_elements(tuple(decay_pack), cutlass.Int32), ) nvvm.tcgen05_wait("store") bars.mb_decay_v_inp_ready[0].arrive() # ---- QKV_epilogue: O acc TMEM -> sO SMEM ------------------------- - o_idx = o_index.idx - bars.mb_o_tmastg_done[o_idx].wait(o_index.phase) - o_index = advance(o_index, cfg.smem_o_stages) - q_state2_idx = o_scale_ready_index.idx - bars.mb_o_state_scale_acc_ready[q_state2_idx].wait(o_scale_ready_index.phase) - o_scale_ready_index = advance(o_scale_ready_index, cfg.tmem_q_state_acc_stages) + o_scale_idx = o_final_acc_ready_index.idx + bars.mb_o_final_acc_ready[o_scale_idx].wait(o_final_acc_ready_index.phase) + o_final_acc_ready_index = advance(o_final_acc_ready_index, cfg.tmem_q_state_acc_stages) o_regs = [] for sub in cutlass.range_constexpr(2): o_vec = nvvm.tcgen05_ld( "16x256b", - nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_q_state_col + q_state2_idx * ACC_STAGE_COLS, cutlass.Float32), + nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_q_state_col + o_scale_idx * ACC_STAGE_COLS, cutlass.Float32), num=8, ) o_regs.append([o_vec[k] for k in range(32)]) nvvm.tcgen05_wait("load") - bars.mb_o_acc_done[q_state2_idx].arrive() + bars.mb_o_acc_done[o_scale_idx].arrive() + o_idx = o_index.idx + bars.mb_o_tmastg_done[o_idx].wait(o_index.phase) + o_index = advance(o_index, cfg.smem_o_stages) for sub in cutlass.range_constexpr(2): for m0 in cutlass.range_constexpr(4): - o_f16 = [fp32_to_fp16(o_regs[sub][8 * m0 + 2 * j], o_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + o_pack = [fp32_to_fp16(o_regs[sub][8 * m0 + 2 * j], o_regs[sub][8 * m0 + 2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] nvvm.stmatrix( ( sO_base @@ -1903,7 +1900,7 @@ def compute1_warp_group( + (v_o_smem_tok + m0 * 16) * 64 + swizzle_xor_128b(v_o_smem_tok + m0 * 16, v_o_smem_col + sub * 16) ).raw_ptr(), - o_f16, + o_pack, nvvm.MMALayout.COL, ) nvvm.fence_proxy("async.shared", space="cta") @@ -2487,13 +2484,13 @@ def kernel( for s in range(cfg.tmem_q_state_acc_stages): bars.mb_o_acc_ready[s].init() bars.mb_o_acc_done[s].init() - bars.mb_o_state_scale_acc_ready[s].init() + bars.mb_o_final_acc_ready[s].init() bars.mb_o_state_scale_acc_done[s].init() for s in range(cfg.tmem_cg0_acc_stages): bars.mb_cg0_acc_ready[s].init() bars.mb_cg0_acc_done[s].init() - bars.mb_k_state_ready[0].init() - bars.mb_nv_ready[0].init() + bars.mb_k_state_acc_ready[0].init() + bars.mb_nv_acc_ready[0].init() for s in range(cfg.smem_ainv_stages): bars.mb_ainv_ready[s].init() bars.mb_ainv_done[s].init() diff --git a/python/cudnn/linear_attention/frost/kernel/gdn_recompute_f16.py b/python/cudnn/linear_attention/frost/kernel/gdn_recompute_f16.py index 52c178bf7..09bfecdc4 100644 --- a/python/cudnn/linear_attention/frost/kernel/gdn_recompute_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/gdn_recompute_f16.py @@ -134,8 +134,8 @@ class GdnBars(NamedTuple): mb_state_acc_scale_done: MBarrier mb_cg0_acc_ready: MBarrier mb_cg0_acc_done: MBarrier - mb_k_state_ready: MBarrier - mb_nv_ready: MBarrier + mb_k_state_acc_ready: MBarrier + mb_nv_acc_ready: MBarrier mb_ainv_ready: MBarrier mb_ainv_done: MBarrier @@ -178,8 +178,8 @@ def alloc(n): mb_state_acc_scale_done=MBarrier(alloc(cfg.tmem_state_acc_stages), stages=cfg.tmem_state_acc_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_cg0_acc_ready=MBarrier(alloc(cfg.tmem_cg0_acc_stages), stages=cfg.tmem_cg0_acc_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_cg0_acc_done=MBarrier(alloc(cfg.tmem_cg0_acc_stages), stages=cfg.tmem_cg0_acc_stages, init_count=CG0_THREADS // 2, producer=Producer.THREAD), - mb_k_state_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), - mb_nv_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_k_state_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), + mb_nv_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_ainv_ready=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=CG0_THREADS, producer=Producer.THREAD), mb_ainv_done=MBarrier(alloc(cfg.smem_ainv_stages), stages=cfg.smem_ainv_stages, init_count=MMA_ARRIVERS, producer=Producer.MMA_COMMIT), mb_state_inp_ready=MBarrier(alloc(cfg.tmem_state_inp_stages), stages=cfg.tmem_state_inp_stages, init_count=CG1_THREADS, producer=Producer.THREAD), @@ -250,7 +250,7 @@ def blockwise_diagonal_8x8_to_16x16(cfg, base_int, d0, lane_id): mma_step_k8(c_regs, [d, d], [c], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) for i in cutlass.range_constexpr(4): c_regs[i] = -c_regs[i] - a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(2)] + a_pack = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(2)] # ---- C = T @ A^{-1} ---------------------------------------------------------- ai = nvvm.ldmatrix( @@ -261,13 +261,13 @@ def blockwise_diagonal_8x8_to_16x16(cfg, base_int, d0, lane_id): o_regs = cutlass.Array(cutlass.Float32, 4, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(4): o_regs[i] = cutlass.Float32(0.0) - mma_step_k8(o_regs, a_f16, [ai], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) - o_f16 = fp32_to_fp16(o_regs[0], o_regs[1], dtype=cfg.io_dtype) + mma_step_k8(o_regs, a_pack, [ai], k_step=0, M=16, N=8, ab_dtype=cfg.io_dtype) + o_pack = fp32_to_fp16(o_regs[0], o_regs[1], dtype=cfg.io_dtype) # ---- store corrected C ------------------------------------------------------- nvvm.stmatrix( cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 8) * 64 + d0 + lds1, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - o_f16, + o_pack, nvvm.MMALayout.ROW, ) @@ -301,7 +301,7 @@ def blockwise_diagonal_16x16_to_32x32(cfg, base_int, d0, lane_id): mma_step(c_regs, d, c, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) for i in cutlass.range_constexpr(8): c_regs[i] = -c_regs[i] - a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + a_pack = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] # ---- C = T @ A^{-1} ---------------------------------------------------------- ai = list( @@ -314,13 +314,13 @@ def blockwise_diagonal_16x16_to_32x32(cfg, base_int, d0, lane_id): o_regs = cutlass.Array(cutlass.Float32, 8, alignment=16, space=cutlass.AddressSpace.rmem) for i in cutlass.range_constexpr(8): o_regs[i] = cutlass.Float32(0.0) - mma_step(o_regs, a_f16, ai, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) - o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] + mma_step(o_regs, a_pack, ai, k_step=0, M=16, N=16, ab_dtype=cfg.io_dtype) + o_pack = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(4)] # ---- store corrected C ------------------------------------------------------- nvvm.stmatrix( cutlass.inttoptr(base_int + swizzle_lin_128b((d0 + 16) * 64 + d0 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - o_f16, + o_pack, nvvm.MMALayout.ROW, ) @@ -331,9 +331,9 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): band = warp_id % 2 bpe = cfg.io_dtype.width // 8 lds4 = (lane_id % 16) * 64 + (lane_id // 16) * 8 - a_regs = [] + a_frags = [] for vs in cutlass.range_constexpr(2): - a_regs += list( + a_frags += list( nvvm.ldmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b((32 + band * 16) * 64 + 32 + vs * 16 + lds4, row_stride_log2=6) * bpe, @@ -344,9 +344,9 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): nvvm.MMALayout.ROW, ) ) - b_regs = [] + b_frags = [] for vs in cutlass.range_constexpr(4): - b_regs += list( + b_frags += list( nvvm.ldmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b((32 + (vs // 2) * 16) * 64 + (vs % 2) * 16 + lds4, row_stride_log2=6) * bpe, @@ -363,15 +363,15 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): for i in cutlass.range_constexpr(16): c_regs[i] = cutlass.Float32(0.0) for ks in cutlass.range_constexpr(2): - mma_step(c_regs, a_regs, b_regs[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) + mma_step(c_regs, a_frags, b_frags[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) for i in cutlass.range_constexpr(16): c_regs[i] = -c_regs[i] - a_f16 = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + a_pack = [fp32_to_fp16(c_regs[2 * j], c_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] # ---- C = T @ A^{-1} ---------------------------------------------------------- - ai_regs = [] + ai_frags = [] for vs in cutlass.range_constexpr(4): - ai_regs += list( + ai_frags += list( nvvm.ldmatrix( cutlass.inttoptr( base_int + swizzle_lin_128b(((vs // 2) * 16) * 64 + (vs % 2) * 16 + lds4, row_stride_log2=6) * bpe, @@ -386,8 +386,8 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): for i in cutlass.range_constexpr(16): o_regs[i] = cutlass.Float32(0.0) for ks in cutlass.range_constexpr(2): - mma_step(o_regs, a_f16, ai_regs[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) - o_f16 = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] + mma_step(o_regs, a_pack, ai_frags[ks * 8 : ks * 8 + 8], k_step=ks, M=16, N=32, ab_dtype=cfg.io_dtype) + o_pack = [fp32_to_fp16(o_regs[2 * j], o_regs[2 * j + 1], dtype=cfg.io_dtype) for j in range(8)] # ---- store corrected C ------------------------------------------------------- nvvm.barrier_cta_sync_aligned( @@ -396,12 +396,12 @@ def blockwise_diagonal_32x32_to_64x64(cfg, base_int, warp_id, lane_id): ) nvvm.stmatrix( cutlass.inttoptr(base_int + swizzle_lin_128b((32 + band * 16) * 64 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - o_f16[0:4], + o_pack[0:4], nvvm.MMALayout.ROW, ) nvvm.stmatrix( cutlass.inttoptr(base_int + swizzle_lin_128b((32 + band * 16) * 64 + 16 + lds4, row_stride_log2=6) * bpe, cutlass.AddressSpace.smem, cutlass.BFloat16), - o_f16[4:8], + o_pack[4:8], nvvm.MMALayout.ROW, ) @@ -848,7 +848,7 @@ def mma_warp( for k in cutlass.range_constexpr(KQ_HALF_K): mma_ts_step(bmm_k_state_desc, state_a_ptr.subview(KQ_A_HALF), desc_k + KQ_SEG, k_state_acc_ptr, k, cutlass.Boolean(True)) if elect_one: - bars.mb_k_state_ready[0].arrive(cta_group=1) + bars.mb_k_state_acc_ready[0].arrive(cta_group=1) # ---- NV = v_k_state(T) @ A_inv (GEMM 5) ------------------------------------ bars.mb_ainv_ready[ainv_idx].wait(ainv_index.phase) @@ -858,7 +858,7 @@ def mma_warp( for k in cutlass.range_constexpr(cfg.b_t // 16): mma_ts_step(bmm_nv_ts_desc, v_k_state_inp_ptr, desc_ainv, nv_acc_ptr, k, cutlass.Boolean(k > 0)) if elect_one: - bars.mb_nv_ready[0].arrive(cta_group=1) + bars.mb_nv_acc_ready[0].arrive(cta_group=1) bars.mb_ainv_done[ainv_idx].arrive(cta_group=1) # ---- KK pair lookahead (member 0) = K(S) @ K^T ----------------------- @@ -1173,12 +1173,12 @@ def compute0_warp_group( bars.mb_ainv_done[kk_ainv_idx].wait(kk_ainv_phase) for u in cutlass.range_constexpr(2): kk_vec = kk_vec1 if cutlass.const_expr(u == 1) else kk_vec0 - kk_f16 = [] + kk_pack = [] for k in cutlass.range_constexpr(num_vals // 2): row_beta = kk_beta[u * 2 + 1] if cutlass.const_expr((k % 2) == 1) else kk_beta[u * 2] p0, p1 = fmul2(kk_vec[2 * k], kk_vec[2 * k + 1], decay_t_kk[u * num_vals + 2 * k], decay_t_kk[u * num_vals + 2 * k + 1]) v0, v1 = fmul2(p0, p1, row_beta, row_beta) - kk_f16.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) + kk_pack.append(fp32_to_fp16(v0, v1, dtype=cfg.io_dtype)) st_row = half_row_base + u * 16 + store_row_frag for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( @@ -1187,7 +1187,7 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [kk_f16[c * 4 + 0], kk_f16[c * 4 + 1], kk_f16[c * 4 + 2], kk_f16[c * 4 + 3]], + [kk_pack[c * 4 + 0], kk_pack[c * 4 + 1], kk_pack[c * 4 + 2], kk_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) @@ -1231,9 +1231,9 @@ def compute0_warp_group( beta_col = [] for k in cutlass.range_constexpr(num_vals): beta_col.append(sBeta[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, beta0_idx]) - ainv_f16 = [] + ainv_frags = [] for c in cutlass.range_constexpr(ACC_N_FRAGS): - ainv_f16 += list( + ainv_frags += list( nvvm.ldmatrix( cutlass.inttoptr( ainv0_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, @@ -1244,11 +1244,11 @@ def compute0_warp_group( nvvm.MMALayout.ROW, ) ) - ainv_scaled = [] + ainv_pack = [] for j in cutlass.range_constexpr(num_vals // 2): - lo, hi = f16x2_to_f32(ainv_f16[j], dtype=cfg.io_dtype) + lo, hi = f16x2_to_f32(ainv_frags[j], dtype=cfg.io_dtype) s0, s1 = fmul2(lo, hi, beta_col[2 * j], beta_col[2 * j + 1]) - ainv_scaled.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) + ainv_pack.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( cutlass.inttoptr( @@ -1256,7 +1256,7 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [ainv_scaled[c * 4 + 0], ainv_scaled[c * 4 + 1], ainv_scaled[c * 4 + 2], ainv_scaled[c * 4 + 3]], + [ainv_pack[c * 4 + 0], ainv_pack[c * 4 + 1], ainv_pack[c * 4 + 2], ainv_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) nvvm.fence_proxy("async.shared", space="cta") @@ -1267,9 +1267,9 @@ def compute0_warp_group( beta_col = [] for k in cutlass.range_constexpr(num_vals): beta_col.append(sBeta[(lane_id % 4) * 2 + ((k // 4) * 8 + k % 2), 0, beta1_idx]) - ainv_f16 = [] + ainv_frags = [] for c in cutlass.range_constexpr(ACC_N_FRAGS): - ainv_f16 += list( + ainv_frags += list( nvvm.ldmatrix( cutlass.inttoptr( ainv1_base + (store_row * cfg.b_t + swizzle_xor_128b(store_row, store_col + c * FRAG_COLS)) * bpe, @@ -1280,11 +1280,11 @@ def compute0_warp_group( nvvm.MMALayout.ROW, ) ) - ainv_scaled = [] + ainv_pack = [] for j in cutlass.range_constexpr(num_vals // 2): - lo, hi = f16x2_to_f32(ainv_f16[j], dtype=cfg.io_dtype) + lo, hi = f16x2_to_f32(ainv_frags[j], dtype=cfg.io_dtype) s0, s1 = fmul2(lo, hi, beta_col[2 * j], beta_col[2 * j + 1]) - ainv_scaled.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) + ainv_pack.append(fp32_to_fp16(s0, s1, dtype=cfg.io_dtype)) for c in cutlass.range_constexpr(ACC_N_FRAGS): nvvm.stmatrix( cutlass.inttoptr( @@ -1292,7 +1292,7 @@ def compute0_warp_group( cutlass.AddressSpace.smem, cutlass.BFloat16, ), - [ainv_scaled[c * 4 + 0], ainv_scaled[c * 4 + 1], ainv_scaled[c * 4 + 2], ainv_scaled[c * 4 + 3]], + [ainv_pack[c * 4 + 0], ainv_pack[c * 4 + 1], ainv_pack[c * 4 + 2], ainv_pack[c * 4 + 3]], nvvm.MMALayout.ROW, ) nvvm.fence_proxy("async.shared", space="cta") @@ -1467,11 +1467,11 @@ def compute1_warp_group( for sub in cutlass.range_constexpr(num_state_subs): for k in cutlass.range_constexpr(32): state_regs[k][sub] = state_vecs[sub][k] - state_f16 = [fp32_to_fp16(state_regs[2 * j][sub], state_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] + state_pack = [fp32_to_fp16(state_regs[2 * j][sub], state_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_warp_row << 16) + tmem_state_inp_col + sub * sttm_width, cutlass.Int32), - cutlass.Vector.from_elements(tuple(state_f16), cutlass.Int32), + cutlass.Vector.from_elements(tuple(state_pack), cutlass.Int32), ) nvvm.tcgen05_wait("store") bars.mb_state_inp_ready[state_inp_stage_idx].arrive() @@ -1483,7 +1483,7 @@ def compute1_warp_group( if cutlass.const_expr(cfg.split_k): do_checkpoint = do_checkpoint and chunk_idx >= wstart if do_checkpoint: - checkpoint_regs = [[cutlass.Int32(0) for _ in range(16)] for _ in range(4)] + checkpoint_pack = [[cutlass.Int32(0) for _ in range(16)] for _ in range(4)] for b in cutlass.range_constexpr(2): for col_half in cutlass.range_constexpr(2): checkpoint_vec = nvvm.tcgen05_ld( @@ -1492,7 +1492,7 @@ def compute1_warp_group( num=8, ) for j in cutlass.range_constexpr(16): - checkpoint_regs[b * 2 + col_half][j] = fp32_to_fp16( + checkpoint_pack[b * 2 + col_half][j] = fp32_to_fp16( checkpoint_vec[2 * j], checkpoint_vec[2 * j + 1], dtype=cfg.io_dtype ) checkpoint_stage = checkpoint_cnt % cfg.smem_checkpoint_stages @@ -1511,10 +1511,10 @@ def compute1_warp_group( cfg.io_dtype, ), [ - checkpoint_regs[b * 2 + col_half][c * 4 + 0], - checkpoint_regs[b * 2 + col_half][c * 4 + 1], - checkpoint_regs[b * 2 + col_half][c * 4 + 2], - checkpoint_regs[b * 2 + col_half][c * 4 + 3], + checkpoint_pack[b * 2 + col_half][c * 4 + 0], + checkpoint_pack[b * 2 + col_half][c * 4 + 1], + checkpoint_pack[b * 2 + col_half][c * 4 + 2], + checkpoint_pack[b * 2 + col_half][c * 4 + 3], ], nvvm.MMALayout.COL, ) @@ -1556,11 +1556,11 @@ def compute1_warp_group( bars.mb_v_ready[v_idx].wait(v_index.phase) v_index = advance(v_index, cfg.smem_v_stages) - v_words = [[cutlass.Int32(0), cutlass.Int32(0)] for _ in range(16)] + v_frags = [[cutlass.Int32(0), cutlass.Int32(0)] for _ in range(16)] for c in cutlass.range_constexpr(8): tok_block = cutlass.const_expr(c % 4) sub = cutlass.const_expr(c // 4) - v_f16 = nvvm.ldmatrix( + v_frag = nvvm.ldmatrix( ( sV_base + v_idx * v_stage_elems @@ -1572,9 +1572,9 @@ def compute1_warp_group( nvvm.MMALayout.COL, ) for i in cutlass.range_constexpr(4): - v_words[4 * tok_block + i][sub] = v_f16[i] + v_frags[4 * tok_block + i][sub] = v_frag[i] if valid_state: - bars.mb_k_state_ready[0].wait(k_state_ready_index.phase) + bars.mb_k_state_acc_ready[0].wait(k_state_ready_index.phase) k_state_ready_index = advance(k_state_ready_index, 1) for sub in cutlass.range_constexpr(2): @@ -1585,19 +1585,19 @@ def compute1_warp_group( ) for j in cutlass.range_constexpr(16): s0, s1 = fmul2(k_state_vec[2 * j], k_state_vec[2 * j + 1], cumprod_vals[2 * j], cumprod_vals[2 * j + 1]) - k_state_word = fp32_to_fp16(s0, s1, dtype=cfg.io_dtype) - v_words[j][sub] = sub_f16x2(v_words[j][sub], k_state_word, cfg.io_dtype) + k_state_pack = fp32_to_fp16(s0, s1, dtype=cfg.io_dtype) + v_frags[j][sub] = sub_f16x2(v_frags[j][sub], k_state_pack, cfg.io_dtype) for sub in cutlass.range_constexpr(2): nvvm.tcgen05_st( "16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_v_k_state_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(v_words[j][sub] for j in range(16)), cutlass.Int32), + cutlass.Vector.from_elements(tuple(v_frags[j][sub] for j in range(16)), cutlass.Int32), ) nvvm.tcgen05_wait("store") bars.mb_v_k_state_inp_ready[0].arrive() # ---- new_V_epi + decay_V publish --------------------------------- - bars.mb_nv_ready[0].wait(nv_ready_index.phase) + bars.mb_nv_acc_ready[0].wait(nv_ready_index.phase) nv_ready_index = advance(nv_ready_index, 1) bars.mb_v_done[v_idx].arrive() @@ -1620,11 +1620,11 @@ def compute1_warp_group( nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub] = fmul2( nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], decay_scale_vals[2 * j], decay_scale_vals[2 * j + 1] ) - decay_f16 = [fp32_to_fp16(nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] + decay_pack = [fp32_to_fp16(nv_regs[2 * j][sub], nv_regs[2 * j + 1][sub], dtype=cfg.io_dtype) for j in range(16)] nvvm.tcgen05_st( "16x128b", nvvm.make_tmem_ptr(((tmem_warp_row + sub * 16) << 16) + tmem_decay_v_col, cutlass.Int32), - cutlass.Vector.from_elements(tuple(decay_f16), cutlass.Int32), + cutlass.Vector.from_elements(tuple(decay_pack), cutlass.Int32), ) nvvm.tcgen05_wait("store") bars.mb_decay_v_inp_ready[0].arrive() @@ -2102,8 +2102,8 @@ def kernel( for s in range(cfg.tmem_cg0_acc_stages): bars.mb_cg0_acc_ready[s].init() bars.mb_cg0_acc_done[s].init() - bars.mb_k_state_ready[0].init() - bars.mb_nv_ready[0].init() + bars.mb_k_state_acc_ready[0].init() + bars.mb_nv_acc_ready[0].init() for s in range(cfg.smem_ainv_stages): bars.mb_ainv_ready[s].init() bars.mb_ainv_done[s].init() diff --git a/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py b/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py index 916ee6385..2cdb29da5 100644 --- a/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/kda_bprop_f16.py @@ -123,10 +123,10 @@ class KdaBwdBars(NamedTuple): mb_update_acc_ready: MBarrier mb_dy_acc_ready: MBarrier mb_dq_acc_ready: MBarrier - mb_dk_decay_part_ready: MBarrier - mb_dk_inv_part_ready: MBarrier - mb_dk_restore_part_ready: MBarrier - mb_parts_done: MBarrier + mb_dk_decay_part_acc_ready: MBarrier + mb_dk_inv_part_acc_ready: MBarrier + mb_dk_restore_part_acc_ready: MBarrier + mb_parts_acc_done: MBarrier mb_state_inp_ready: MBarrier mb_state_inp_done: MBarrier @@ -139,9 +139,9 @@ class KdaBwdBars(NamedTuple): mb_k_decay_inv_ready: MBarrier mb_q_decay_k_restore_ready: MBarrier - mb_decay_done: MBarrier + mb_decay_tcgen05_done: MBarrier mb_decay_super_done: MBarrier - mb_a_ready: MBarrier + mb_a_inv_ready: MBarrier mb_a_done: MBarrier mb_aqk_ready: MBarrier mb_aqk_done: MBarrier @@ -175,7 +175,8 @@ class KdaBwdBars(NamedTuple): mb_dg_tmastg_ready: MBarrier mb_dg_tmastg_done: MBarrier - mb_dstate0_stored: MBarrier + mb_dstate0_acc_stored: MBarrier + mb_tmem_done: MBarrier mb_sched_ready: MBarrier mb_sched_done: MBarrier @@ -216,10 +217,10 @@ def alloc(n): mb_update_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), mb_dy_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), mb_dq_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), - mb_dk_decay_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), - mb_dk_inv_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), - mb_dk_restore_part_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), - mb_parts_done=MBarrier(alloc(1), stages=1, init_count=CG2, producer=Producer.THREAD), + mb_dk_decay_part_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dk_inv_part_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_dk_restore_part_acc_ready=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_parts_acc_done=MBarrier(alloc(1), stages=1, init_count=CG2, producer=Producer.THREAD), mb_state_inp_ready=MBarrier(alloc(2), stages=2, init_count=CG0, producer=Producer.THREAD), mb_state_inp_done=MBarrier(alloc(2), stages=2, init_count=MMA, producer=Producer.MMA_COMMIT), mb_state_inp_cg2_done=MBarrier(alloc(2), stages=2, init_count=CG2, producer=Producer.THREAD), @@ -230,9 +231,9 @@ def alloc(n): mb_neg_beta_dy_inp_done=MBarrier(alloc(1), stages=1, init_count=MMA, producer=Producer.MMA_COMMIT), mb_k_decay_inv_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0, producer=Producer.THREAD), mb_q_decay_k_restore_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0, producer=Producer.THREAD), - mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=MMA, producer=Producer.MMA_COMMIT), + mb_decay_tcgen05_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=MMA, producer=Producer.MMA_COMMIT), mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=2 * WARP, producer=Producer.THREAD), - mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_a_inv_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), mb_a_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA, producer=Producer.MMA_COMMIT), mb_aqk_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), mb_aqk_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=MMA, producer=Producer.MMA_COMMIT), @@ -263,7 +264,8 @@ def alloc(n): mb_dv_epi_done=MBarrier(alloc(cfg.smem_dv_stages), stages=cfg.smem_dv_stages, init_count=WARP, producer=Producer.THREAD), mb_dg_tmastg_ready=MBarrier(alloc(cfg.smem_dg_stages), stages=cfg.smem_dg_stages, init_count=CG2, producer=Producer.THREAD), mb_dg_tmastg_done=MBarrier(alloc(cfg.smem_dg_stages), stages=cfg.smem_dg_stages, init_count=WARP, producer=Producer.THREAD), - mb_dstate0_stored=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_dstate0_acc_stored=MBarrier(alloc(1), stages=1, init_count=CG1, producer=Producer.THREAD), + mb_tmem_done=MBarrier(alloc(1), stages=1, init_count=CG1 + CG2, producer=Producer.THREAD), mb_sched_ready=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=1, producer=Producer.THREAD), mb_sched_done=MBarrier(alloc(cfg.sched_stages), stages=cfg.sched_stages, init_count=15, producer=Producer.THREAD), ) @@ -330,6 +332,8 @@ def epilogue_warp( ) -> None: """Epilogue warp role (warp 15): the register-MMA A_qk/dA tiles and the dQ/dK/dV/dGate TMA stores.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) # ---- ldmatrix/stmatrix lane decode ------------------------------------------- @@ -411,7 +415,7 @@ def epilogue_warp( desc_dk_slot = (desc_dk_base + slot).tospace(cutlass.AddressSpace.generic) desc_dv_slot = (desc_dv_base + slot).tospace(cutlass.AddressSpace.generic) desc_dg_slot = (desc_dg_base + slot).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): + if elect_one: tma_tensormap_acquire(desc_dq_slot) tma_tensormap_acquire(desc_dk_slot) tma_tensormap_acquire(desc_dv_slot) @@ -441,22 +445,22 @@ def epilogue_warp( for k_block in cutlass.range_constexpr(cfg.d_k // 16): a_col = k_block * 16 + lhs_col_offset a_seg = a_col // 64 - a_vec = nvvm.ldmatrix( + a_frag = nvvm.ldmatrix( sQ_decay_ptr + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) b_col = k_block * 16 + rhs_col_offset b_seg = b_col // 64 - b_vec = nvvm.ldmatrix( + b_frag = nvvm.ldmatrix( sK_inv_ptr + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) mma_step( qk_acc, - (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), - (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + (a_frag[0], a_frag[1], a_frag[2], a_frag[3]), + (b_frag[0], b_frag[1], b_frag[2], b_frag[3]), k_step=0, M=16, N=16, @@ -493,22 +497,22 @@ def epilogue_warp( for k_block in cutlass.range_constexpr(cfg.d_v // 16): a_col = k_block * 16 + lhs_col_offset a_seg = a_col // 64 - a_vec = nvvm.ldmatrix( + a_frag = nvvm.ldmatrix( sDo_ptr + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) b_col = k_block * 16 + rhs_col_offset b_seg = b_col // 64 - b_vec = nvvm.ldmatrix( + b_frag = nvvm.ldmatrix( sU_raw.data_ptr() + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) mma_step( da_acc, - (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), - (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + (a_frag[0], a_frag[1], a_frag[2], a_frag[3]), + (b_frag[0], b_frag[1], b_frag[2], b_frag[3]), k_step=0, M=16, N=16, @@ -679,22 +683,22 @@ def super_mma_warp( for k_block in cutlass.range_constexpr(cfg.d_k // 16): a_col = k_block * 16 + lhs_col_offset a_seg = a_col // 64 - a_vec = nvvm.ldmatrix( + a_frag = nvvm.ldmatrix( sK_decay_ptr + a_seg * (cfg.b_t * 64) + kk_lhs_row * 64 + swizzle_xor_128b(kk_lhs_row, a_col - a_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) b_col = k_block * 16 + rhs_col_offset b_seg = b_col // 64 - b_vec = nvvm.ldmatrix( + b_frag = nvvm.ldmatrix( sK_inv_ptr + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) mma_step( kk_acc, - (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), - (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + (a_frag[0], a_frag[1], a_frag[2], a_frag[3]), + (b_frag[0], b_frag[1], b_frag[2], b_frag[3]), k_step=0, M=16, N=16, @@ -706,7 +710,7 @@ def super_mma_warp( beta_lo = (sBeta_ptr + row_lo).load().to(cutlass.Float32) beta_hi = (sBeta_ptr + row_hi).load().to(cutlass.Float32) bars.mb_beta_done[chunk_serial % cfg.smem_beta_stages].arrive() - l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) + l_regs = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) @@ -714,11 +718,11 @@ def super_mma_warp( col_coord = col_coord + cutlass.Int32(1) beta_scale = beta_lo if accum_idx % 4 < 2 else beta_hi lower = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) - l_frag[accum_idx] = lower * beta_scale - l_a0 = fp32_to_fp16(l_frag[0], l_frag[1], dtype=cfg.io_dtype) - l_a1 = fp32_to_fp16(l_frag[2], l_frag[3], dtype=cfg.io_dtype) - l_a2 = fp32_to_fp16(l_frag[4], l_frag[5], dtype=cfg.io_dtype) - l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) + l_regs[accum_idx] = lower * beta_scale + l_a0 = fp32_to_fp16(l_regs[0], l_regs[1], dtype=cfg.io_dtype) + l_a1 = fp32_to_fp16(l_regs[2], l_regs[3], dtype=cfg.io_dtype) + l_a2 = fp32_to_fp16(l_regs[4], l_regs[5], dtype=cfg.io_dtype) + l_a3 = fp32_to_fp16(l_regs[6], l_regs[7], dtype=cfg.io_dtype) l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) ainv_acc = cutlass.Array(cutlass.Float32, 8, alignment=16) @@ -791,7 +795,7 @@ def super_mma_warp( shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_a_ready[qk_stage].arrive() + bars.mb_a_inv_ready[qk_stage].arrive() bars.mb_decay_super_done[decay_stage].arrive() # ---- dM = dY @ U^T --------------------------------------------------- @@ -804,22 +808,22 @@ def super_mma_warp( for k_block in cutlass.range_constexpr(cfg.d_v // 16): a_col = k_block * 16 + lhs_col_offset a_seg = a_col // 64 - a_vec = nvvm.ldmatrix( + a_frag = nvvm.ldmatrix( sDy_raw.data_ptr() + a_seg * (cfg.b_t * 64) + lhs_row_coord * 64 + swizzle_xor_128b(lhs_row_coord, a_col - a_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) b_col = k_block * 16 + rhs_col_offset b_seg = b_col // 64 - b_vec = nvvm.ldmatrix( + b_frag = nvvm.ldmatrix( sU_raw.data_ptr() + b_seg * (cfg.b_t * 64) + rhs_row_coord * 64 + swizzle_xor_128b(rhs_row_coord, b_col - b_seg * 64, elem_bytes=2), 4, nvvm.MMALayout.ROW, ) mma_step( dm_acc, - (a_vec[0], a_vec[1], a_vec[2], a_vec[3]), - (b_vec[0], b_vec[1], b_vec[2], b_vec[3]), + (a_frag[0], a_frag[1], a_frag[2], a_frag[3]), + (b_frag[0], b_frag[1], b_frag[2], b_frag[3]), k_step=0, M=16, N=16, @@ -828,7 +832,7 @@ def super_mma_warp( bars.mb_u_warps_done.arrive() bars.mb_sdy_done.arrive() - dm_strict = cutlass.Array(cutlass.Float32, 8, alignment=16) + dm_strict_regs = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) @@ -836,16 +840,16 @@ def super_mma_warp( col_coord = col_coord + cutlass.Int32(1) beta_scale = beta_lo if accum_idx % 4 < 2 else beta_hi val = dm_acc[accum_idx] * beta_scale if row_coord > col_coord else cutlass.Float32(0.0) - dm_strict[accum_idx] = val - w0 = fp32_to_fp16(dm_strict[0], dm_strict[1], dtype=cfg.io_dtype) - w1 = fp32_to_fp16(dm_strict[2], dm_strict[3], dtype=cfg.io_dtype) - w2 = fp32_to_fp16(dm_strict[4], dm_strict[5], dtype=cfg.io_dtype) - w3 = fp32_to_fp16(dm_strict[6], dm_strict[7], dtype=cfg.io_dtype) + dm_strict_regs[accum_idx] = val + w0 = fp32_to_fp16(dm_strict_regs[0], dm_strict_regs[1], dtype=cfg.io_dtype) + w1 = fp32_to_fp16(dm_strict_regs[2], dm_strict_regs[3], dtype=cfg.io_dtype) + w2 = fp32_to_fp16(dm_strict_regs[4], dm_strict_regs[5], dtype=cfg.io_dtype) + w3 = fp32_to_fp16(dm_strict_regs[6], dm_strict_regs[7], dtype=cfg.io_dtype) nvvm.stmatrix(sQk_ptr + 3 * (cfg.b_t * cfg.b_t) + stsm_idx, [w0, w1, w2, w3], nvvm.MMALayout.ROW, shape=nvvm.StoreShape.M8N8) - nw0 = fp32_to_fp16(-dm_strict[0], -dm_strict[1], dtype=cfg.io_dtype) - nw1 = fp32_to_fp16(-dm_strict[2], -dm_strict[3], dtype=cfg.io_dtype) - nw2 = fp32_to_fp16(-dm_strict[4], -dm_strict[5], dtype=cfg.io_dtype) - nw3 = fp32_to_fp16(-dm_strict[6], -dm_strict[7], dtype=cfg.io_dtype) + nw0 = fp32_to_fp16(-dm_strict_regs[0], -dm_strict_regs[1], dtype=cfg.io_dtype) + nw1 = fp32_to_fp16(-dm_strict_regs[2], -dm_strict_regs[3], dtype=cfg.io_dtype) + nw2 = fp32_to_fp16(-dm_strict_regs[4], -dm_strict_regs[5], dtype=cfg.io_dtype) + nw3 = fp32_to_fp16(-dm_strict_regs[6], -dm_strict_regs[7], dtype=cfg.io_dtype) nvvm.stmatrix(sQk_ptr + 4 * (cfg.b_t * cfg.b_t) + stsm_idx, [nw0, nw1, nw2, nw3], nvvm.MMALayout.ROW, shape=nvvm.StoreShape.M8N8) nvvm.fence_proxy("async.shared", space="cta") bars.mb_dm_ready[qk_stage].arrive() @@ -906,7 +910,11 @@ def tcgen05_mma_warp( ) -> None: """tcgen05-MMA warp role (warp 13): issues every tcgen05 GEMM and owns the TMEM lifecycle.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + nvvm.tcgen05_alloc(tmem_base_holder, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_base_holder.load() bpe = cfg.io_dtype.width // 8 @@ -1216,13 +1224,13 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_state_k_acc_offset), cutlass.Float32), accumulate=False, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_state_k_acc_ready.arrive(cta_group=1) bars.mb_state_done[state_index.idx].arrive(cta_group=1) state_index = advance(state_index, cfg.smem_state_stages) # ---- dQ inter = state(T) @ dO^T ------------------------------------------ - bars.mb_parts_done.wait(parts_done_index.phase) + bars.mb_parts_acc_done.wait(parts_done_index.phase) parts_done_index = advance(parts_done_index, 1) bars.mb_state_inp_ready[chunk_serial % 2].wait((chunk_serial // 2) % 2) bars.mb_do_ready[raw_stage_idx].wait((chunk_serial // cfg.smem_raw_stages) % 2) @@ -1268,7 +1276,7 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_du_acc_offset), cutlass.Float32), accumulate=has_dstate, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_du_acc_ready.arrive(cta_group=1) bars.mb_aqk_done[qk_stage].arrive(cta_group=1) @@ -1291,7 +1299,7 @@ def tcgen05_mma_warp( k, cutlass.Boolean(sub + k > 0), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dstate_inp_done.arrive(cta_group=1) dstate_inp_index = advance(dstate_inp_index, 1) @@ -1305,7 +1313,7 @@ def tcgen05_mma_warp( ) # ---- U = W(T) @ A_inv ------------------------------------------------ - bars.mb_a_ready[qk_stage].wait(qk_phase) + bars.mb_a_inv_ready[qk_stage].wait(qk_phase) bars.mb_rhs_ready.wait(rhs_index.phase) rhs_index = advance(rhs_index, 1) a_ptr = nvvm.make_tmem_ptr((tmem_base + cfg.tmem_rhs_inp_offset), cutlass.Int8) @@ -1321,7 +1329,7 @@ def tcgen05_mma_warp( k, cutlass.Boolean(sub + k > 0), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_update_acc_ready.arrive(cta_group=1) bars.mb_do_done[raw_stage_idx].arrive(cta_group=1) @@ -1341,7 +1349,7 @@ def tcgen05_mma_warp( k, cutlass.Boolean(sub + k > 0), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dy_acc_ready.arrive(cta_group=1) bars.mb_du_inp_done.arrive(cta_group=1) bars.mb_a_done[qk_stage].arrive(cta_group=1) @@ -1359,10 +1367,10 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_restore_acc_offset), cutlass.Float32), accumulate=False, ) - if nvvm.elect_sync(): - bars.mb_dk_restore_part_ready.arrive(cta_group=1) + if elect_one: + bars.mb_dk_restore_part_acc_ready.arrive(cta_group=1) bars.mb_dstate_smem_done.arrive(cta_group=1) - if nvvm.elect_sync(): + if elect_one: bars.mb_u_done.arrive(cta_group=1) # ---- dstate dY-term += -Beta.dY(T) @ K_decay ----------------------------- @@ -1381,7 +1389,7 @@ def tcgen05_mma_warp( k, cutlass.Boolean(True), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dstate_acc_ready.arrive(cta_group=1) bars.mb_neg_beta_dy_inp_done.arrive(cta_group=1) @@ -1412,7 +1420,7 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dq_acc_offset), cutlass.Float32), accumulate=False, ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dq_acc_ready.arrive(cta_group=1) bars.mb_da_done[qk_stage].arrive(cta_group=1) @@ -1432,7 +1440,7 @@ def tcgen05_mma_warp( k, cutlass.Boolean(sub + k > 0), ) - if nvvm.elect_sync(): + if elect_one: bars.mb_dv_tmastg_done[chunk_serial % cfg.smem_dv_stages].arrive(cta_group=1) bars.mb_state_inp_done[chunk_serial % 2].arrive(cta_group=1) @@ -1445,8 +1453,8 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), accumulate=True, ) - if nvvm.elect_sync(): - bars.mb_dk_inv_part_ready.arrive(cta_group=1) + if elect_one: + bars.mb_dk_inv_part_acc_ready.arrive(cta_group=1) # ---- dK_decay part += K_inv^T(S) @ dM_strict^T ----------------------- if chunk_idx >= FIRST_STATE_CHUNK: @@ -1465,16 +1473,18 @@ def tcgen05_mma_warp( nvvm.make_tmem_ptr((tmem_base + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), accumulate=False, ) - if nvvm.elect_sync(): - bars.mb_dk_decay_part_ready.arrive(cta_group=1) + if elect_one: + bars.mb_dk_decay_part_acc_ready.arrive(cta_group=1) bars.mb_dm_done[qk_stage].arrive(cta_group=1) - bars.mb_decay_done[decay_stage].arrive(cta_group=1) + bars.mb_decay_tcgen05_done[decay_stage].arrive(cta_group=1) # ---- tile end: WG1's dstate0 drain gates the next tile's dstate reuse ------------ - bars.mb_dstate0_stored.wait(dstate0_index.phase) + bars.mb_dstate0_acc_stored.wait(dstate0_index.phase) dstate0_index = advance(dstate0_index, 1) chunk_serial_base += num_compute_chunks tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].wait(0) + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) nvvm.tcgen05_dealloc( nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), cutlass.Int32(512), @@ -1510,6 +1520,8 @@ def tmaldg_warp( ) -> None: """TMA-LDG warp role (warp 14): persistent tile-scheduler loop issuing every G->S TMA load.""" + elect_one = nvvm.elect_sync() + nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) sQ_tma = SmemTile( @@ -1601,7 +1613,7 @@ def tmaldg_warp( desc_do_slot = (desc_do_base + slot).tospace(cutlass.AddressSpace.generic) desc_checkpoint_slot = (desc_checkpoint_base + slot).tospace(cutlass.AddressSpace.generic) desc_initial_state_slot = (desc_initial_state_base + cutlass.Int32(0)).tospace(cutlass.AddressSpace.generic) - if nvvm.elect_sync(): + if elect_one: tma_tensormap_acquire(desc_q_slot) tma_tensormap_acquire(desc_k_slot) tma_tensormap_acquire(desc_v_slot) @@ -1617,21 +1629,21 @@ def tmaldg_warp( # ---- Q load ---------------------------------------------------------- bars.mb_q_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_q_ready[raw_index.idx].arrive(n_bytes=cfg.tma_q_bytes) q_slice = tma_slice_runtime_desc(desc_q_slot, cutlass.Int32(0), head_q, chunk_start) tma_load_tile(sQ_tma[raw_index.idx], q_slice, bars.mb_q_ready[raw_index.idx].smem_ptr, acquire=False) # ---- K load ---------------------------------------------------------- bars.mb_k_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_k_ready[raw_index.idx].arrive(n_bytes=cfg.tma_k_bytes) k_slice = tma_slice_runtime_desc(desc_k_slot, cutlass.Int32(0), head_k, chunk_start) tma_load_tile(sK_tma[raw_index.idx], k_slice, bars.mb_k_ready[raw_index.idx].smem_ptr, acquire=False) # ---- Gate load ------------------------------------------------------- bars.mb_gate_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_gate_ready[raw_index.idx].arrive(n_bytes=cfg.tma_gate_bytes) gate_slice = tma_slice_runtime_desc(desc_gate_slot, cutlass.Int32(0), head_o, chunk_start) tma_load_tile(sGate_tma[raw_index.idx], gate_slice, bars.mb_gate_ready[raw_index.idx].smem_ptr, acquire=False) @@ -1639,14 +1651,14 @@ def tmaldg_warp( # ---- dO load --------------------------------------------------------- bars.mb_do_done[raw_index.idx].wait(raw_index.phase) bars.mb_do_epi_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_do_ready[raw_index.idx].arrive(n_bytes=cfg.tma_do_bytes) do_slice = tma_slice_runtime_desc(desc_do_slot, cutlass.Int32(0), head_o, chunk_start) tma_load_tile(sDo_tma[raw_index.idx], do_slice, bars.mb_do_ready[raw_index.idx].smem_ptr, acquire=False) # ---- V load ---------------------------------------------------------- bars.mb_v_done[raw_index.idx].wait(raw_index.phase) - if nvvm.elect_sync(): + if elect_one: bars.mb_v_ready[raw_index.idx].arrive(n_bytes=cfg.tma_v_bytes) v_slice = tma_slice_runtime_desc(desc_v_slot, cutlass.Int32(0), head_v, chunk_start) tma_load_tile(sV_tma[raw_index.idx], v_slice, bars.mb_v_ready[raw_index.idx].smem_ptr, acquire=False) @@ -1657,7 +1669,7 @@ def tmaldg_warp( bars.mb_state_done[state_idx].wait(state_index.phase) bars.mb_state_cg0_done[state_idx].wait(state_index.phase) state_index = advance(state_index, cfg.smem_state_stages) - if nvvm.elect_sync(): + if elect_one: bars.mb_state_ready[state_idx].arrive(n_bytes=cfg.tma_state_bytes) if cutlass.const_expr(cfg.use_initial_state): if chunk_idx == 0: @@ -1707,6 +1719,7 @@ def compute0_warp_group( H -> TMEM f16 at the chunk tail.""" nvvm.setmaxregister(cfg.num_regs_compute_group_0, nvvm.SetMaxRegisterAction.INCREASE) cg0_warp = warp_idx - cfg.compute_group_0_warp_ids[0] + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_base_holder.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 @@ -1803,7 +1816,7 @@ def compute0_warp_group( exp_g_last = g_prefix_regs[cfg.b_t - 1] # ---- decay-slot guard: previous use fully consumed ------------------- operand_done_phase = ((chunk_serial // cfg.smem_decay_stages) + 1) % 2 - bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_tcgen05_done[decay_stage].wait(operand_done_phase) bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) for row in cutlass.range_constexpr(cfg.b_t): @@ -1821,7 +1834,7 @@ def compute0_warp_group( nvvm.barrier_cta_sync(cfg.cg0_sync_barrier_id, thread_count=cfg.cg0_threads) - k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) + k_inv_pack = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) raw_q_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) # ---- optional q/k L2-norm + K_inv staging ---------------------------- @@ -1836,13 +1849,13 @@ def compute0_warp_group( f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 raw_f16_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) - raw_q_vec = (sQ_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_k_vec = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_q_vec_f32 = raw_q_vec.to(cutlass.Float32) - raw_k_vec_f32 = raw_k_vec.to(cutlass.Float32) + raw_q_frag = (sQ_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_k_frag = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_q_frag_f32 = raw_q_frag.to(cutlass.Float32) + raw_k_frag_f32 = raw_k_frag.to(cutlass.Float32) for dim_offset in cutlass.range_constexpr(8): - q_val = raw_q_vec_f32[dim_offset] - k_val = raw_k_vec_f32[dim_offset] + q_val = raw_q_frag_f32[dim_offset] + k_val = raw_k_frag_f32[dim_offset] raw_q_regs[reg_base + dim_offset] = q_val raw_k_regs[reg_base + dim_offset] = k_val if cutlass.const_expr(cfg.l2norm): @@ -1881,24 +1894,24 @@ def compute0_warp_group( f32_segment = f32_dim_base // 32 f32_segment_dim = f32_dim_base - f32_segment * 32 g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) - exp_g_vec = (g_prefix_ptr + g_prefix_idx).load(count=4, alignment=16) + exp_g_frag = (g_prefix_ptr + g_prefix_idx).load(count=4, alignment=16) exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) - exp_g_last_vec = (g_prefix_ptr + exp_g_last_idx).load(count=4, alignment=16) + exp_g_last_frag = (g_prefix_ptr + exp_g_last_idx).load(count=4, alignment=16) f32_reg_base = reg_base + f32_group * 4 - exp_g_regs[f32_reg_base] = exp_g_vec[0] - exp_g_regs[f32_reg_base + 1] = exp_g_vec[1] - exp_g_regs[f32_reg_base + 2] = exp_g_vec[2] - exp_g_regs[f32_reg_base + 3] = exp_g_vec[3] - exp_g_last_regs[f32_reg_base] = exp_g_last_vec[0] - exp_g_last_regs[f32_reg_base + 1] = exp_g_last_vec[1] - exp_g_last_regs[f32_reg_base + 2] = exp_g_last_vec[2] - exp_g_last_regs[f32_reg_base + 3] = exp_g_last_vec[3] + exp_g_regs[f32_reg_base] = exp_g_frag[0] + exp_g_regs[f32_reg_base + 1] = exp_g_frag[1] + exp_g_regs[f32_reg_base + 2] = exp_g_frag[2] + exp_g_regs[f32_reg_base + 3] = exp_g_frag[3] + exp_g_last_regs[f32_reg_base] = exp_g_last_frag[0] + exp_g_last_regs[f32_reg_base + 1] = exp_g_last_frag[1] + exp_g_last_regs[f32_reg_base + 2] = exp_g_last_frag[2] + exp_g_last_regs[f32_reg_base + 3] = exp_g_last_frag[3] for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 # ---- k decay + k inv operands: exp2(+g) * k / exp2(-g) * k ------- - k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_decay_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 @@ -1907,23 +1920,23 @@ def compute0_warp_group( k_value0, k_value1 = fmul2(raw_k_regs[raw_reg_idx0], raw_k_regs[raw_reg_idx1], k_inv_norm, k_inv_norm) k_pair = fp32_to_fp16(k_value0, k_value1, dtype=cfg.io_dtype) exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) - k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) + k_decay_pack[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) exp_neg_g0 = cute.math.rcp(exp_g_regs[raw_reg_idx0], approx=True, ftz=True) exp_neg_g1 = cute.math.rcp(exp_g_regs[raw_reg_idx1], approx=True, ftz=True) exp_neg_pair = fp32_to_fp16(exp_neg_g0, exp_neg_g1, dtype=cfg.io_dtype) - k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_pair, exp_neg_pair, cfg.io_dtype) + k_inv_pack[dim_half * 4 + pair_idx] = mul_f16x2(k_pair, exp_neg_pair, cfg.io_dtype) k_inv_vec = cutlass.Vector.from_elements( ( - k_inv_words[dim_half * 4], - k_inv_words[dim_half * 4 + 1], - k_inv_words[dim_half * 4 + 2], - k_inv_words[dim_half * 4 + 3], + k_inv_pack[dim_half * 4], + k_inv_pack[dim_half * 4 + 1], + k_inv_pack[dim_half * 4 + 2], + k_inv_pack[dim_half * 4 + 3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) k_decay_vec = cutlass.Vector.from_elements( - (k_decay_words[0], k_decay_words[1], k_decay_words[2], k_decay_words[3]), + (k_decay_pack[0], k_decay_pack[1], k_decay_pack[2], k_decay_pack[3]), cutlass.Int32, ).bitcast(cfg.io_dtype) f16_segment = dim_base // 64 @@ -1938,8 +1951,8 @@ def compute0_warp_group( for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 - q_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) - k_restore_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + q_decay_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_restore_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 @@ -1948,16 +1961,16 @@ def compute0_warp_group( q_value0, q_value1 = fmul2(raw_q_regs[raw_reg_idx0], raw_q_regs[raw_reg_idx1], q_stage_norm, q_stage_norm) q_pair = fp32_to_fp16(q_value0, q_value1, dtype=cfg.io_dtype) exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) - q_decay_words[pair_idx] = mul_f16x2(q_pair, exp_g_pair, cfg.io_dtype) + q_decay_pack[pair_idx] = mul_f16x2(q_pair, exp_g_pair, cfg.io_dtype) exp_g_last_pair = fp32_to_fp16(exp_g_last_regs[raw_reg_idx0], exp_g_last_regs[raw_reg_idx1], dtype=cfg.io_dtype) - k_restore_words[pair_idx] = mul_f16x2(k_inv_words[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) + k_restore_pack[pair_idx] = mul_f16x2(k_inv_pack[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) q_decay_vec = cutlass.Vector.from_elements( - (q_decay_words[0], q_decay_words[1], q_decay_words[2], q_decay_words[3]), + (q_decay_pack[0], q_decay_pack[1], q_decay_pack[2], q_decay_pack[3]), cutlass.Int32, ).bitcast(cfg.io_dtype) k_restore_vec = cutlass.Vector.from_elements( - (k_restore_words[0], k_restore_words[1], k_restore_words[2], k_restore_words[3]), + (k_restore_pack[0], k_restore_pack[1], k_restore_pack[2], k_restore_pack[3]), cutlass.Int32, ).bitcast(cfg.io_dtype) f16_segment = dim_base // 64 @@ -1979,7 +1992,7 @@ def compute0_warp_group( state_src = sState_raw.data_ptr() + state_index.idx * (cfg.d_k * cfg.d_v) for v_seg in cutlass.range_constexpr(2): for v_col8 in cutlass.range_constexpr(8): - state_vec = (state_src + v_seg * (cfg.d_k * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, v_col8 * 8, elem_bytes=2)).load( + state_frag = (state_src + v_seg * (cfg.d_k * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, v_col8 * 8, elem_bytes=2)).load( count=8, alignment=16 ) nvvm.tcgen05_st( @@ -1987,7 +2000,7 @@ def compute0_warp_group( nvvm.make_tmem_ptr( row_addr + (tmem_col + cfg.tmem_state_inp_offset + (chunk_serial % 2) * (cfg.d_v // 2) + v_seg * 32 + v_col8 * 4), cutlass.Int8 ), - state_vec.bitcast(cutlass.Int32), + state_frag.bitcast(cutlass.Int32), ) nvvm.tcgen05_wait("store") bars.mb_state_cg0_done[state_index.idx].arrive() @@ -2030,6 +2043,7 @@ def compute1_warp_group( v-term dot and store; end-of-chunk dH capture (dh acc -> dh_inp f16 + sdH); tile edges: dht seeding and the dS0 drain.""" nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_base_holder.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 @@ -2088,18 +2102,18 @@ def compute1_warp_group( nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_acc_offset + sub * 16), cutlass.Float32), seed_block[0:16], ) - packed_seed = cutlass.Array(cutlass.Int32, 8, alignment=16) + seed_pack = cutlass.Array(cutlass.Int32, 8, alignment=16) for pc in cutlass.range_constexpr(8): - packed_seed[pc] = fp32_to_fp16(seed_block[2 * pc], seed_block[2 * pc + 1], dtype=cfg.io_dtype) + seed_pack[pc] = fp32_to_fp16(seed_block[2 * pc], seed_block[2 * pc + 1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_inp_offset + sub * 8), cutlass.Int8), - packed_seed[0:8], + seed_pack[0:8], ) for half in cutlass.range_constexpr(2): d_base = sub * 16 + half * 8 h_vec = cutlass.Vector.from_elements( - (packed_seed[half * 4], packed_seed[half * 4 + 1], packed_seed[half * 4 + 2], packed_seed[half * 4 + 3]), + (seed_pack[half * 4], seed_pack[half * 4 + 1], seed_pack[half * 4 + 2], seed_pack[half * 4 + 3]), cutlass.Int32, ).bitcast(cfg.io_dtype) h_addr = (d_base // 64) * (cfg.d_v * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, d_base % 64, elem_bytes=2) @@ -2128,7 +2142,7 @@ def compute1_warp_group( # ---- rhs staging: W = Beta * (V - state_k) --------------------------- projection_col_id = tmem_col + cfg.tmem_state_k_acc_offset input_col_id = tmem_col + cfg.tmem_rhs_inp_offset - raw_v_regs0 = nvvm.ldmatrix( + raw_v_frag0 = nvvm.ldmatrix( sV_ptr + (value_dim_base + value_col_offset) // 64 * (cfg.b_t * 64) + token_row_coord * 64 @@ -2136,7 +2150,7 @@ def compute1_warp_group( 4, nvvm.MMALayout.COL, ) - raw_v_regs1 = nvvm.ldmatrix( + raw_v_frag1 = nvvm.ldmatrix( sV_ptr + (value_dim_base + 16 + value_col_offset) // 64 * (cfg.b_t * 64) + token_row_coord * 64 @@ -2154,38 +2168,38 @@ def compute1_warp_group( beta_pairs[half] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) diff_w0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) diff_w1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) if chunk_idx >= FIRST_STATE_CHUNK: bars.mb_state_k_acc_ready.wait(state_k_index.phase) state_k_index = advance(state_k_index, 1) - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + projection_col_id, cutlass.Float32), num=2) + state_k_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + projection_col_id, cutlass.Float32), num=2) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_pair = fp32_to_fp16(state_k0[frag_pair], state_k0[frag_pair + 1], dtype=cfg.io_dtype) - diff_pair = sub_f16x2(raw_v_regs0[raw_matrix], state_k_pair, cfg.io_dtype) + state_k_pair = fp32_to_fp16(state_k_vec0[frag_pair], state_k_vec0[frag_pair + 1], dtype=cfg.io_dtype) + diff_pair = sub_f16x2(raw_v_frag0[raw_matrix], state_k_pair, cfg.io_dtype) diff_w0[reg_idx ^ 2] = diff_pair - packed_rhs0[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], diff_pair, cfg.io_dtype) + rhs_pack0[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], diff_pair, cfg.io_dtype) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_pair = fp32_to_fp16(state_k1[frag_pair], state_k1[frag_pair + 1], dtype=cfg.io_dtype) - diff_pair = sub_f16x2(raw_v_regs1[raw_matrix], state_k_pair, cfg.io_dtype) + state_k_pair = fp32_to_fp16(state_k_vec1[frag_pair], state_k_vec1[frag_pair + 1], dtype=cfg.io_dtype) + diff_pair = sub_f16x2(raw_v_frag1[raw_matrix], state_k_pair, cfg.io_dtype) diff_w1[reg_idx ^ 2] = diff_pair - packed_rhs1[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], diff_pair, cfg.io_dtype) + rhs_pack1[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], diff_pair, cfg.io_dtype) if chunk_idx < FIRST_STATE_CHUNK: for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) - diff_w0[reg_idx ^ 2] = raw_v_regs0[raw_matrix] - packed_rhs0[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], raw_v_regs0[raw_matrix], cfg.io_dtype) + diff_w0[reg_idx ^ 2] = raw_v_frag0[raw_matrix] + rhs_pack0[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], raw_v_frag0[raw_matrix], cfg.io_dtype) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) - diff_w1[reg_idx ^ 2] = raw_v_regs1[raw_matrix] - packed_rhs1[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], raw_v_regs1[raw_matrix], cfg.io_dtype) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + input_col_id, cutlass.Int8), packed_rhs0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + input_col_id, cutlass.Int8), packed_rhs1[0:4]) + diff_w1[reg_idx ^ 2] = raw_v_frag1[raw_matrix] + rhs_pack1[reg_idx ^ 2] = mul_f16x2(beta_pairs[reg_idx // 2], raw_v_frag1[raw_matrix], cfg.io_dtype) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + input_col_id, cutlass.Int8), rhs_pack0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + input_col_id, cutlass.Int8), rhs_pack1[0:4]) nvvm.tcgen05_wait("store") bars.mb_rhs_ready.arrive() @@ -2195,17 +2209,17 @@ def compute1_warp_group( bars.mb_du_inp_done.wait(du_inp_done_index.phase) du_inp_done_index = advance(du_inp_done_index, 1) du_col_id = tmem_col + cfg.tmem_du_acc_offset - du0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + du_col_id, cutlass.Float32), num=2) - du1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + du_col_id, cutlass.Float32), num=2) + du_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + du_col_id, cutlass.Float32), num=2) + du_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + du_col_id, cutlass.Float32), num=2) - du_packed0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - du_packed1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + du_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + du_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): frag_pair = reg_idx * 2 - du_packed0[reg_idx] = fp32_to_fp16(du0[frag_pair], du0[frag_pair + 1], dtype=cfg.io_dtype) - du_packed1[reg_idx] = fp32_to_fp16(du1[frag_pair], du1[frag_pair + 1], dtype=cfg.io_dtype) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_packed0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_packed1[0:4]) + du_pack0[reg_idx] = fp32_to_fp16(du_vec0[frag_pair], du_vec0[frag_pair + 1], dtype=cfg.io_dtype) + du_pack1[reg_idx] = fp32_to_fp16(du_vec1[frag_pair], du_vec1[frag_pair + 1], dtype=cfg.io_dtype) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_pack0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_du_inp_offset), cutlass.Int8), du_pack1[0:4]) nvvm.tcgen05_wait("store") bars.mb_du_inp_ready.arrive() @@ -2216,20 +2230,20 @@ def compute1_warp_group( bars.mb_u_warps_done.wait(u_done_index.phase) u_done_index = advance(u_done_index, 1) u_col_id = tmem_col + cfg.tmem_update_acc_offset - u0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + u_col_id, cutlass.Float32), num=2) - u1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + u_col_id, cutlass.Float32), num=2) + u_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + u_col_id, cutlass.Float32), num=2) + u_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + u_col_id, cutlass.Float32), num=2) - u_words0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - u_words1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + u_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + u_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - u_words0[reg_idx] = fp32_to_fp16(u0[2 * reg_idx], u0[2 * reg_idx + 1], dtype=cfg.io_dtype) - u_words1[reg_idx] = fp32_to_fp16(u1[2 * reg_idx], u1[2 * reg_idx + 1], dtype=cfg.io_dtype) + u_pack0[reg_idx] = fp32_to_fp16(u_vec0[2 * reg_idx], u_vec0[2 * reg_idx + 1], dtype=cfg.io_dtype) + u_pack1[reg_idx] = fp32_to_fp16(u_vec1[2 * reg_idx], u_vec1[2 * reg_idx + 1], dtype=cfg.io_dtype) nvvm.stmatrix( sU_raw.data_ptr() + (value_dim_base + value_col_offset) // 64 * (cfg.b_t * 64) + token_row_coord * 64 + swizzle_xor_128b(token_row_coord, (value_dim_base + value_col_offset) % 64, elem_bytes=2), - u_words0.data_ptr().load(count=4, alignment=4), + u_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -2238,7 +2252,7 @@ def compute1_warp_group( + (value_dim_base + 16 + value_col_offset) // 64 * (cfg.b_t * 64) + token_row_coord * 64 + swizzle_xor_128b(token_row_coord, (value_dim_base + 16 + value_col_offset) % 64, elem_bytes=2), - u_words1.data_ptr().load(count=4, alignment=4), + u_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -2249,8 +2263,8 @@ def compute1_warp_group( bars.mb_dy_acc_ready.wait(dy_acc_index.phase) dy_acc_index = advance(dy_acc_index, 1) dy_col_id = tmem_col + cfg.tmem_dy_acc_offset - dy0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + dy_col_id, cutlass.Float32), num=2) - dy1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + dy_col_id, cutlass.Float32), num=2) + dy_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id0 << 16) + dy_col_id, cutlass.Float32), num=2) + dy_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr((row_id1 << 16) + dy_col_id, cutlass.Float32), num=2) # ---- dY -> sdY: pack + store + publish (super warp's dM operand) ------- addr_lo0 = ( @@ -2263,15 +2277,15 @@ def compute1_warp_group( + token_row_coord * 64 + swizzle_xor_128b(token_row_coord, (value_dim_base + 16 + value_col_offset) % 64, elem_bytes=2) ) - dy_p0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - dy_p1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + dy_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + dy_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - dy_p0[reg_idx] = fp32_to_fp16(dy0[2 * reg_idx], dy0[2 * reg_idx + 1], dtype=cfg.io_dtype) - dy_p1[reg_idx] = fp32_to_fp16(dy1[2 * reg_idx], dy1[2 * reg_idx + 1], dtype=cfg.io_dtype) + dy_pack0[reg_idx] = fp32_to_fp16(dy_vec0[2 * reg_idx], dy_vec0[2 * reg_idx + 1], dtype=cfg.io_dtype) + dy_pack1[reg_idx] = fp32_to_fp16(dy_vec1[2 * reg_idx], dy_vec1[2 * reg_idx + 1], dtype=cfg.io_dtype) bars.mb_sdy_done.wait(sdy_done_index.phase) sdy_done_index = advance(sdy_done_index, 1) - nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo0, dy_p0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) - nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo1, dy_p1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) + nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo0, dy_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) + nvvm.stmatrix(sDy_raw.data_ptr() + addr_lo1, dy_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8) bars.mb_sdy_ready.arrive() # ---- beta scalars -> beta.dY -> sdV (epilogue TMA + GEMM 13 operand) --- @@ -2280,49 +2294,55 @@ def compute1_warp_group( beta_c8 = (sBeta_ptr + (lane % 4) * 2 + 8).load().to(cutlass.Float32) beta_c9 = (sBeta_ptr + (lane % 4) * 2 + 9).load().to(cutlass.Float32) bars.mb_beta_done[chunk_serial % cfg.smem_beta_stages].arrive() - beta_dy0 = cutlass.Array(cutlass.Float32, 8, alignment=16) - beta_dy1 = cutlass.Array(cutlass.Float32, 8, alignment=16) + beta_dy_regs0 = cutlass.Array(cutlass.Float32, 8, alignment=16) + beta_dy_regs1 = cutlass.Array(cutlass.Float32, 8, alignment=16) for e2 in cutlass.range_constexpr(4): e = 2 * e2 b_lo = beta_c8 if cutlass.const_expr(e >= 4) else beta_c0 b_hi = beta_c9 if cutlass.const_expr(e >= 4) else beta_c1 - beta_dy0[e], beta_dy0[e + 1] = fmul2(dy0[e], dy0[e + 1], b_lo, b_hi) - beta_dy1[e], beta_dy1[e + 1] = fmul2(dy1[e], dy1[e + 1], b_lo, b_hi) - beta_dy_p0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - beta_dy_p1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_dy_regs0[e], beta_dy_regs0[e + 1] = fmul2(dy_vec0[e], dy_vec0[e + 1], b_lo, b_hi) + beta_dy_regs1[e], beta_dy_regs1[e + 1] = fmul2(dy_vec1[e], dy_vec1[e + 1], b_lo, b_hi) + beta_dy_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_dy_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - beta_dy_p0[reg_idx] = fp32_to_fp16(beta_dy0[2 * reg_idx], beta_dy0[2 * reg_idx + 1], dtype=cfg.io_dtype) - beta_dy_p1[reg_idx] = fp32_to_fp16(beta_dy1[2 * reg_idx], beta_dy1[2 * reg_idx + 1], dtype=cfg.io_dtype) + beta_dy_pack0[reg_idx] = fp32_to_fp16(beta_dy_regs0[2 * reg_idx], beta_dy_regs0[2 * reg_idx + 1], dtype=cfg.io_dtype) + beta_dy_pack1[reg_idx] = fp32_to_fp16(beta_dy_regs1[2 * reg_idx], beta_dy_regs1[2 * reg_idx + 1], dtype=cfg.io_dtype) dv_stage = chunk_serial % cfg.smem_dv_stages sdv_stage_base = dv_stage * (cfg.b_t * cfg.d_v) bars.mb_dv_tmastg_done[dv_stage].wait(dv_done_index.phase) bars.mb_dv_epi_done[dv_stage].wait(dv_done_index.phase) dv_done_index = advance(dv_done_index, cfg.smem_dv_stages) nvvm.stmatrix( - sDv_raw.data_ptr() + sdv_stage_base + addr_lo0, beta_dy_p0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8 + sDv_raw.data_ptr() + sdv_stage_base + addr_lo0, + beta_dy_pack0.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, ) nvvm.stmatrix( - sDv_raw.data_ptr() + sdv_stage_base + addr_lo1, beta_dy_p1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8 + sDv_raw.data_ptr() + sdv_stage_base + addr_lo1, + beta_dy_pack1.data_ptr().load(count=4, alignment=4), + nvvm.MMALayout.COL, + shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_dv_tmastg_ready[dv_stage].arrive() # ---- -beta.dY -> TMEM: A operand of the dH ds-term --------------------- - neg_beta_dy0 = cutlass.Array(cutlass.Float32, 8, alignment=16) - neg_beta_dy1 = cutlass.Array(cutlass.Float32, 8, alignment=16) + neg_beta_dy_regs0 = cutlass.Array(cutlass.Float32, 8, alignment=16) + neg_beta_dy_regs1 = cutlass.Array(cutlass.Float32, 8, alignment=16) for e in cutlass.range_constexpr(8): - neg_beta_dy0[e] = -beta_dy0[e] - neg_beta_dy1[e] = -beta_dy1[e] - neg_beta_dy_p0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - neg_beta_dy_p1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + neg_beta_dy_regs0[e] = -beta_dy_regs0[e] + neg_beta_dy_regs1[e] = -beta_dy_regs1[e] + neg_beta_dy_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + neg_beta_dy_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): frag_pair = reg_idx * 2 - neg_beta_dy_p0[reg_idx] = fp32_to_fp16(neg_beta_dy0[frag_pair], neg_beta_dy0[frag_pair + 1], dtype=cfg.io_dtype) - neg_beta_dy_p1[reg_idx] = fp32_to_fp16(neg_beta_dy1[frag_pair], neg_beta_dy1[frag_pair + 1], dtype=cfg.io_dtype) + neg_beta_dy_pack0[reg_idx] = fp32_to_fp16(neg_beta_dy_regs0[frag_pair], neg_beta_dy_regs0[frag_pair + 1], dtype=cfg.io_dtype) + neg_beta_dy_pack1[reg_idx] = fp32_to_fp16(neg_beta_dy_regs1[frag_pair], neg_beta_dy_regs1[frag_pair + 1], dtype=cfg.io_dtype) bars.mb_neg_beta_dy_inp_done.wait(neg_beta_dy_done_index.phase) neg_beta_dy_done_index = advance(neg_beta_dy_done_index, 1) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_neg_beta_dy_inp_offset), cutlass.Int8), neg_beta_dy_p0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_neg_beta_dy_inp_offset), cutlass.Int8), neg_beta_dy_p1[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_lo + (tmem_col + cfg.tmem_neg_beta_dy_inp_offset), cutlass.Int8), neg_beta_dy_pack0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(row_addr_hi + (tmem_col + cfg.tmem_neg_beta_dy_inp_offset), cutlass.Int8), neg_beta_dy_pack1[0:4]) nvvm.tcgen05_wait("store") bars.mb_neg_beta_dy_inp_ready.arrive() @@ -2335,8 +2355,8 @@ def compute1_warp_group( d1_lo, d1_hi = f16x2_to_f32(diff_w1[j], dtype=cfg.io_dtype) s_lo = cutlass.const_expr(2 * (j // 2)) s_hi = cutlass.const_expr(2 * (j // 2) + 1) - t_lo, t_hi = ffma2(dy0[2 * j], dy0[2 * j + 1], d0_lo, d0_hi, tok4[s_lo], tok4[s_hi]) - t_lo, t_hi = ffma2(dy1[2 * j], dy1[2 * j + 1], d1_lo, d1_hi, t_lo, t_hi) + t_lo, t_hi = ffma2(dy_vec0[2 * j], dy_vec0[2 * j + 1], d0_lo, d0_hi, tok4[s_lo], tok4[s_hi]) + t_lo, t_hi = ffma2(dy_vec1[2 * j], dy_vec1[2 * j + 1], d1_lo, d1_hi, t_lo, t_hi) tok4[s_lo] = t_lo tok4[s_hi] = t_hi @@ -2375,21 +2395,21 @@ def compute1_warp_group( dstate_smem_done_index = advance(dstate_smem_done_index, 1) row_addr = (tmem_row + tmem_subpartition * cfg.threads_per_warp) << 16 for sub in cutlass.range_constexpr(cfg.d_k // 32): - dstate_block = nvvm.tcgen05_ld( + dstate_vec = nvvm.tcgen05_ld( "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_acc_offset + sub * 32), cutlass.Float32), num=32 ) - packed_dstate = cutlass.Array(cutlass.Int32, 16, alignment=16) + dstate_pack = cutlass.Array(cutlass.Int32, 16, alignment=16) for pc in cutlass.range_constexpr(16): - packed_dstate[pc] = fp32_to_fp16(dstate_block[2 * pc], dstate_block[2 * pc + 1], dtype=cfg.io_dtype) + dstate_pack[pc] = fp32_to_fp16(dstate_vec[2 * pc], dstate_vec[2 * pc + 1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dstate_inp_offset + sub * 16), cutlass.Int8), - packed_dstate[0:16], + dstate_pack[0:16], ) for half in cutlass.range_constexpr(4): d_base = sub * 32 + half * 8 h_vec = cutlass.Vector.from_elements( - (packed_dstate[half * 4], packed_dstate[half * 4 + 1], packed_dstate[half * 4 + 2], packed_dstate[half * 4 + 3]), + (dstate_pack[half * 4], dstate_pack[half * 4 + 1], dstate_pack[half * 4 + 2], dstate_pack[half * 4 + 3]), cutlass.Int32, ).bitcast(cfg.io_dtype) h_addr = (d_base // 64) * (cfg.d_v * 64) + value_dim * 64 + swizzle_xor_128b(value_dim, d_base % 64, elem_bytes=2) @@ -2420,10 +2440,12 @@ def compute1_warp_group( mDstate0[batch_idx, head_idx, kd, value_dim] = mDstate_in[batch_idx, head_idx, kd, value_dim] else: mDstate0[batch_idx, head_idx, kd, value_dim] = cutlass.Float32(0.0) - bars.mb_dstate0_stored.arrive() + bars.mb_dstate0_acc_stored.arrive() chunk_serial_base += num_compute_chunks tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].arrive() + @cute.jit def compute2_warp_group( @@ -2457,6 +2479,7 @@ def compute2_warp_group( registers, stages dgate for the epilogue's TMA store, and stages dq/dk for the epilogue's TMA stores.""" nvvm.setmaxregister(cfg.num_regs_compute_group_2, nvvm.SetMaxRegisterAction.INCREASE) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_base_holder.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 @@ -2515,7 +2538,7 @@ def compute2_warp_group( dgl_dstate_smem_index = advance(dgl_dstate_smem_index, 1) for pl in cutlass.range_constexpr(2): for row_half in cutlass.range_constexpr(2): - state_words = nvvm.tcgen05_ld( + state_vec = nvvm.tcgen05_ld( "32x32b", nvvm.make_tmem_ptr( row_addr + (tmem_col + cfg.tmem_state_inp_offset + (chunk_serial % 2) * (cfg.d_v // 2) + pl * 32 + row_half * 16), @@ -2528,7 +2551,7 @@ def compute2_warp_group( hacc[i] = opaque_f32_zero() for j in cutlass.range_constexpr(16): v0 = pl * 64 + row_half * 32 + 2 * j - state_pair = cutlass.Vector.from_elements((state_words[j],), cutlass.Float32).bitcast(cfg.io_dtype) + state_pair = cutlass.Vector.from_elements((state_vec[j],), cutlass.Float32).bitcast(cfg.io_dtype) dstate_addr0 = (channel // 64) * (cfg.d_v * 64) + v0 * 64 + swizzle_xor_128b(v0, channel % 64, elem_bytes=2) dstate_addr1 = (channel // 64) * (cfg.d_v * 64) + (v0 + 1) * 64 + swizzle_xor_128b(v0 + 1, channel % 64, elem_bytes=2) hval0 = (sDstate_raw.data_ptr() + dstate_addr0).load().to(cutlass.Float32) @@ -2544,7 +2567,7 @@ def compute2_warp_group( # ---- part-drain accumulators ------------------------------------------- dq_n = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) dk_n = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) - dg_reg = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) + dg_regs = cutlass.Array(cutlass.Float32, cfg.b_t, alignment=16) dgl_acc = cutlass.Array(cutlass.Float32, 4, alignment=16) for i in cutlass.range_constexpr(4): dgl_acc[i] = opaque_f32_zero() @@ -2553,13 +2576,13 @@ def compute2_warp_group( # ---- dk_restore part drain: (eGl/eG) scale + dGlast k-dot -------------- if has_dstate: - bars.mb_dk_restore_part_ready.wait(dk_restore_part_index.phase) + bars.mb_dk_restore_part_acc_ready.wait(dk_restore_part_index.phase) dk_restore_part_index = advance(dk_restore_part_index, 1) - dk_restore_part = nvvm.tcgen05_ld( + dk_restore_part_vec = nvvm.tcgen05_ld( "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_restore_acc_offset), cutlass.Float32), num=cfg.b_t ) for t in cutlass.range_constexpr(cfg.b_t): - dk_hat = egl * cute.math.rcp(eg[t], approx=True, ftz=True) * dk_restore_part[t] + dk_hat = egl * cute.math.rcp(eg[t], approx=True, ftz=True) * dk_restore_part_vec[t] dk_n[t] = dk_hat k_v = (sK_ptr + f16_seg * (cfg.b_t * 64) + t * 64 + swizzle_xor_128b(t, f16_dim, elem_bytes=2)).load().to(cutlass.Float32) if cutlass.const_expr(cfg.l2norm): @@ -2569,29 +2592,31 @@ def compute2_warp_group( # ---- dq acc drain: eG.scale --------------------------------------------- bars.mb_dq_acc_ready.wait(dq_acc_index.phase) dq_acc_index = advance(dq_acc_index, 1) - dq_acc = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dq_acc_offset), cutlass.Float32), num=cfg.b_t) + dq_vec = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dq_acc_offset), cutlass.Float32), num=cfg.b_t) for t2 in cutlass.range_constexpr(cfg.b_t // 2): t = 2 * t2 es_lo, es_hi = fmul2(eg[t], eg[t + 1], scale, scale) - dq_n[t], dq_n[t + 1] = fmul2(es_lo, es_hi, dq_acc[t], dq_acc[t + 1]) + dq_n[t], dq_n[t + 1] = fmul2(es_lo, es_hi, dq_vec[t], dq_vec[t + 1]) # ---- dk_inv part drain: (dA - dM) term, 1/eG scale ---------------------- - bars.mb_dk_inv_part_ready.wait(dk_inv_part_index.phase) + bars.mb_dk_inv_part_acc_ready.wait(dk_inv_part_index.phase) dk_inv_part_index = advance(dk_inv_part_index, 1) - dk_inv_part = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), num=cfg.b_t) + dk_inv_part_vec = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_inv_acc_offset), cutlass.Float32), num=cfg.b_t) for t in cutlass.range_constexpr(cfg.b_t): - dk_n[t] = dk_n[t] + dk_inv_part[t] * cute.math.rcp(eg[t], approx=True, ftz=True) + dk_n[t] = dk_n[t] + dk_inv_part_vec[t] * cute.math.rcp(eg[t], approx=True, ftz=True) # ---- dk_decay part drain: -eG scale, seeds dG --------------------------- - bars.mb_dk_decay_part_ready.wait(dk_decay_part_index.phase) + bars.mb_dk_decay_part_acc_ready.wait(dk_decay_part_index.phase) dk_decay_part_index = advance(dk_decay_part_index, 1) - dk_decay_part = nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), num=cfg.b_t) + dk_decay_part_vec = nvvm.tcgen05_ld( + "32x32b", nvvm.make_tmem_ptr(row_addr + (tmem_col + cfg.tmem_dk_decay_acc_offset), cutlass.Float32), num=cfg.b_t + ) for t in cutlass.range_constexpr(cfg.b_t): - dg_reg[t] = -eg[t] * dk_decay_part[t] - dk_n[t] = dk_n[t] + dg_reg[t] + dg_regs[t] = -eg[t] * dk_decay_part_vec[t] + dk_n[t] = dk_n[t] + dg_regs[t] # ---- all parts drained: release the accs to the next chunk's MMAs ------ - bars.mb_parts_done.arrive() + bars.mb_parts_acc_done.arrive() # ---- dG finalize ----------------------------------------------------- for t in cutlass.range_constexpr(cfg.b_t): @@ -2600,8 +2625,8 @@ def compute2_warp_group( if cutlass.const_expr(cfg.l2norm): q_v = q_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + t] k_v = k_v * sNorm_raw[raw_stage * (2 * cfg.b_t) + cfg.b_t + t] - dg_reg[t] = q_v * dq_n[t] + k_v * (cutlass.Float32(2.0) * dg_reg[t] - dk_n[t]) - dg_reg[cfg.b_t - 1] = dg_reg[cfg.b_t - 1] + ((dgl_acc[0] + dgl_acc[1]) + (dgl_acc[2] + dgl_acc[3])) + dg_regs[t] = q_v * dq_n[t] + k_v * (cutlass.Float32(2.0) * dg_regs[t] - dk_n[t]) + dg_regs[cfg.b_t - 1] = dg_regs[cfg.b_t - 1] + ((dgl_acc[0] + dgl_acc[1]) + (dgl_acc[2] + dgl_acc[3])) # ---- L2-norm backward row projection --------------------------------- if cutlass.const_expr(cfg.l2norm): @@ -2643,21 +2668,21 @@ def compute2_warp_group( # ---- dGlast add ------------------------------------------------------ if has_dstate: if chunk_idx >= FIRST_STATE_CHUNK: - dg_reg[cfg.b_t - 1] = dg_reg[cfg.b_t - 1] + egl * dgl_val + dg_regs[cfg.b_t - 1] = dg_regs[cfg.b_t - 1] + egl * dgl_val # ---- dG reverse cumsum ----------------------------------------------- suffix = cutlass.Float32(0.0) for rt in cutlass.range_constexpr(cfg.b_t): t = cfg.b_t - 1 - rt - suffix = suffix + dg_reg[t] - dg_reg[t] = suffix + suffix = suffix + dg_regs[t] + dg_regs[t] = suffix # ---- stage dGate for the epilogue TMA store -------------------------- dg_stage = chunk_serial % cfg.smem_dg_stages bars.mb_dg_tmastg_done[dg_stage].wait(((chunk_serial // cfg.smem_dg_stages) + 1) % 2) for t in cutlass.range_constexpr(cfg.b_t): dg_idx = f32_seg * (cfg.b_t * 32) + t * 32 + swizzle_xor_128b(t, f32_dim, elem_bytes=4) - (sDg_raw.data_ptr() + dg_idx).store(dg_reg[t]) + (sDg_raw.data_ptr() + dg_idx).store(dg_regs[t]) nvvm.fence_proxy("async.shared", space="cta") bars.mb_dg_tmastg_ready[dg_stage].arrive() @@ -2668,6 +2693,8 @@ def compute2_warp_group( chunk_serial_base += num_compute_chunks tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].arrive() + # --------------------------------------------------------------------------- # Host-side assembly @@ -2810,19 +2837,26 @@ def build_descs( tma_granu_elems = 128 // bpe seqlen = q.shape[0] - def headed(t, dim, hn): - return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) + q_headed = cute.make_tensor(q.iterator, cute.make_layout((d_k, h_q, seqlen), stride=(1, q.stride[1], q.stride[0]))) + k_headed = cute.make_tensor(k.iterator, cute.make_layout((d_k, h_k, seqlen), stride=(1, k.stride[1], k.stride[0]))) + v_headed = cute.make_tensor(v.iterator, cute.make_layout((d_v, h_v, seqlen), stride=(1, v.stride[1], v.stride[0]))) + gate_headed = cute.make_tensor(gate.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, gate.stride[1], gate.stride[0]))) + do_headed = cute.make_tensor(do.iterator, cute.make_layout((d_v, ho, seqlen), stride=(1, do.stride[1], do.stride[0]))) + dq_headed = cute.make_tensor(dq.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, dq.stride[1], dq.stride[0]))) + dk_headed = cute.make_tensor(dk.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, dk.stride[1], dk.stride[0]))) + dv_headed = cute.make_tensor(dv.iterator, cute.make_layout((d_v, ho, seqlen), stride=(1, dv.stride[1], dv.stride[0]))) + dg_headed = cute.make_tensor(dg.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, dg.stride[1], dg.stride[0]))) swz = cuda.TensorMapSwizzle.s128b - base_q = cuda.create_tensor_map_tiled_from_view(headed(q, d_k, h_q), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_do = cuda.create_tensor_map_tiled_from_view(headed(do, d_v, ho), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_dq = cuda.create_tensor_map_tiled_from_view(headed(dq, d_k, ho), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_dk = cuda.create_tensor_map_tiled_from_view(headed(dk, d_k, ho), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_dv = cuda.create_tensor_map_tiled_from_view(headed(dv, d_v, ho), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_dg = cuda.create_tensor_map_tiled_from_view(headed(dg, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_q = cuda.create_tensor_map_tiled_from_view(q_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_k = cuda.create_tensor_map_tiled_from_view(k_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(v_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(gate_headed, box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_do = cuda.create_tensor_map_tiled_from_view(do_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dq = cuda.create_tensor_map_tiled_from_view(dq_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dk = cuda.create_tensor_map_tiled_from_view(dk_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dv = cuda.create_tensor_map_tiled_from_view(dv_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_dg = cuda.create_tensor_map_tiled_from_view(dg_headed, box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) checkpoint_view = cute.make_tensor( state_checkpoints.iterator, @@ -3125,8 +3159,9 @@ def kernel( layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, ) + elect_one = nvvm.elect_sync() if warp_idx == cfg.tma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_raw_stages): bars.mb_q_ready[stage].init() bars.mb_q_done[stage].init() @@ -3151,7 +3186,7 @@ def kernel( bars.mb_state_inp_done[stage].init() bars.mb_state_inp_cg2_done[stage].init() elif warp_idx == cfg.tcgen05_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: bars.mb_state_k_acc_ready.init() bars.mb_rhs_ready.init() bars.mb_update_acc_ready.init() @@ -3174,22 +3209,23 @@ def kernel( bars.mb_dstate_smem_done.init() bars.mb_dstate_smem_cg2_done.init() bars.mb_dq_acc_ready.init() - bars.mb_dk_decay_part_ready.init() - bars.mb_dk_inv_part_ready.init() - bars.mb_dk_restore_part_ready.init() - bars.mb_parts_done.init() + bars.mb_dk_decay_part_acc_ready.init() + bars.mb_dk_inv_part_acc_ready.init() + bars.mb_dk_restore_part_acc_ready.init() + bars.mb_parts_acc_done.init() bars.mb_dbeta_m_ready.init() bars.mb_dbeta_m_done.init() - bars.mb_dstate0_stored.init() + bars.mb_dstate0_acc_stored.init() + bars.mb_tmem_done[0].init() elif warp_idx == cfg.super_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_decay_stages): bars.mb_k_decay_inv_ready[stage].init() bars.mb_q_decay_k_restore_ready[stage].init() - bars.mb_decay_done[stage].init() + bars.mb_decay_tcgen05_done[stage].init() bars.mb_decay_super_done[stage].init() for stage in cutlass.range_constexpr(cfg.smem_qk_stages): - bars.mb_a_ready[stage].init() + bars.mb_a_inv_ready[stage].init() bars.mb_aqk_ready[stage].init() bars.mb_da_ready[stage].init() bars.mb_dm_ready[stage].init() @@ -3198,7 +3234,7 @@ def kernel( bars.mb_da_done[stage].init() bars.mb_dm_done[stage].init() elif warp_idx == cfg.epilogue_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_dq_stages): bars.mb_dq_tmastg_ready[stage].init() bars.mb_dq_tmastg_done[stage].init() @@ -3220,20 +3256,6 @@ def kernel( sState_scale_diag_raw[diag_idx] = diag_zero nvvm.fence_mbarrier_init() nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) - is_tmem_user = ( - (warp_idx >= cfg.compute_group_2_warp_ids[0] and warp_idx <= cfg.compute_group_2_warp_ids[-1]) - or (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) - or (warp_idx >= cfg.compute_group_0_warp_ids[0] and warp_idx <= cfg.compute_group_0_warp_ids[-1]) - or warp_idx == cfg.tcgen05_mma_warp_id - ) - if is_tmem_user: - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_alloc(tmem_base_holder, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - if warp_idx == cfg.tma_warp_id: tmaldg_warp( cfg, diff --git a/python/cudnn/linear_attention/frost/kernel/kda_prefill_f16.py b/python/cudnn/linear_attention/frost/kernel/kda_prefill_f16.py index c4098549e..5c6deebd6 100644 --- a/python/cudnn/linear_attention/frost/kernel/kda_prefill_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/kda_prefill_f16.py @@ -163,20 +163,20 @@ class KdaBars(NamedTuple): mb_rhs_ready: MBarrier mb_update_ready: MBarrier - mb_a_ready: MBarrier + mb_a_inv_ready: MBarrier mb_a_inv_done: MBarrier mb_qk_acc_ready: MBarrier mb_qk_done: MBarrier mb_qk_scale_ready: MBarrier mb_state_scale_diag_done: MBarrier mb_k_decay_inv_cg0_ready: MBarrier - mb_decay_done: MBarrier + mb_decay_tcgen05_done: MBarrier mb_decay_super_done: MBarrier mb_k_restore_done: MBarrier - mb_state_read_done: MBarrier - mb_state_updated: MBarrier - mb_final_state_stored: MBarrier + mb_state_acc_read_done: MBarrier + mb_state_acc_done: MBarrier + mb_tmem_done: MBarrier mb_o_tmastg_ready: MBarrier mb_o_tmastg_done: MBarrier @@ -217,7 +217,7 @@ def alloc(n): mb_state_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_rhs_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_update_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_a_inv_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), mb_a_inv_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_qk_acc_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), mb_qk_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=1, producer=Producer.MMA_COMMIT), @@ -234,12 +234,12 @@ def alloc(n): producer=Producer.MMA_COMMIT, ), mb_k_decay_inv_cg0_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), - mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_decay_tcgen05_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=2 * WARP, producer=Producer.THREAD), mb_k_restore_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), - mb_state_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_state_updated=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), - mb_final_state_stored=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_state_acc_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_state_acc_done=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), + mb_tmem_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_tmastg_ready=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=CG1_THREADS, producer=Producer.THREAD), mb_o_tmastg_done=MBarrier(alloc(cfg.smem_o_stages), stages=cfg.smem_o_stages, init_count=WARP, producer=Producer.THREAD), mb_checkpoint_tmastg_ready=MBarrier( @@ -474,7 +474,7 @@ def super_mma_warp( # Load B operand k_inv_col = k_block * 16 + rhs_col_offset k_inv_segment = k_inv_col // 64 - rhs_vec = nvvm.ldmatrix( + rhs_frag = nvvm.ldmatrix( sK_inv_ptr + k_inv_segment * (cfg.b_t * 64) + rhs_row_coord * 64 @@ -485,7 +485,7 @@ def super_mma_warp( # Load A operand storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask storage_slice = storage_key // 64 - kk_lhs_vec = nvvm.ldmatrix( + kk_lhs_frag = nvvm.ldmatrix( sK_decay_ptr + storage_slice * (cfg.b_t * 64) + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), @@ -495,8 +495,8 @@ def super_mma_warp( mma_step( kk_acc, - (kk_lhs_vec[0], kk_lhs_vec[1], kk_lhs_vec[2], kk_lhs_vec[3]), - (rhs_vec[0], rhs_vec[1], rhs_vec[2], rhs_vec[3]), + (kk_lhs_frag[0], kk_lhs_frag[1], kk_lhs_frag[2], kk_lhs_frag[3]), + (rhs_frag[0], rhs_frag[1], rhs_frag[2], rhs_frag[3]), k_step=0, M=16, N=16, @@ -509,21 +509,21 @@ def super_mma_warp( row_hi = row_lo + cutlass.Int32(8) beta_lo = (sBeta_ptr + row_lo).load().to(cutlass.Float32) beta_hi = (sBeta_ptr + row_hi).load().to(cutlass.Float32) - l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) + l_regs = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) if cutlass.const_expr(accum_idx % 2 == 1): col_coord = col_coord + cutlass.Int32(1) - l_frag[accum_idx] = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + l_regs[accum_idx] = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) for pair in cutlass.range_constexpr(4): beta_scale = beta_hi if cutlass.const_expr(pair % 2 == 1) else beta_lo - l_frag[2 * pair], l_frag[2 * pair + 1] = fmul2(l_frag[2 * pair], l_frag[2 * pair + 1], beta_scale, beta_scale) + l_regs[2 * pair], l_regs[2 * pair + 1] = fmul2(l_regs[2 * pair], l_regs[2 * pair + 1], beta_scale, beta_scale) bars.mb_beta_done[raw_index.idx].arrive() - l_a0 = fp32_to_fp16(l_frag[0], l_frag[1], dtype=cfg.io_dtype) - l_a1 = fp32_to_fp16(l_frag[2], l_frag[3], dtype=cfg.io_dtype) - l_a2 = fp32_to_fp16(l_frag[4], l_frag[5], dtype=cfg.io_dtype) - l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) + l_a0 = fp32_to_fp16(l_regs[0], l_regs[1], dtype=cfg.io_dtype) + l_a1 = fp32_to_fp16(l_regs[2], l_regs[3], dtype=cfg.io_dtype) + l_a2 = fp32_to_fp16(l_regs[4], l_regs[5], dtype=cfg.io_dtype) + l_a3 = fp32_to_fp16(l_regs[6], l_regs[7], dtype=cfg.io_dtype) l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) # ---- A_inv = I - L, then three Neumann doubling rounds --------------- @@ -601,7 +601,7 @@ def super_mma_warp( shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_a_ready[qk_stage].arrive() + bars.mb_a_inv_ready[qk_stage].arrive() bars.mb_decay_super_done[decay_stage].arrive() raw_index = advance(raw_index, cfg.smem_raw_stages) cum_chunk_base += num_chunks_tile @@ -629,6 +629,8 @@ def tcgen05_mma_warp( every tcgen05 GEMM and owning the TMEM lifecycle.""" elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + nvvm.tcgen05_alloc(tmem_base_slot, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_base_slot.load() state_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) state_dsts = tuple(nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset + k * 16, cutlass.Float32) for k in range(cfg.d_k // 16)) @@ -641,7 +643,6 @@ def tcgen05_mma_warp( state_read_index = PipelineState.start(phase=0) rhs_index = PipelineState.start(phase=0) update_index = PipelineState.start(phase=0) - final_state_index = PipelineState.start(phase=0) qk_scale_index = PipelineState.start(phase=0) k_decay_ready = PipelineState.start(phase=0) qk_ready = PipelineState.start(phase=0) @@ -733,6 +734,7 @@ def tcgen05_mma_warp( num_chunks_tile = wend - cstart for local_chunk_idx in cutlass.range(num_chunks_tile, unroll=1): cum_chunk = cum_chunk_base + local_chunk_idx + have_state = cutlass.Boolean(True) if cutlass.const_expr(cfg.use_initial_state) else local_chunk_idx > 0 q_state_acc_stage = o_acc_free.idx decay_stage = k_decay_ready.idx state_scale_diag_stage = qk_scale_index.idx @@ -746,66 +748,68 @@ def tcgen05_mma_warp( # ---- state_k = S(T) @ K_decay^T -------------------------------------- bars.mb_k_decay_inv_cg0_ready[decay_stage].wait(k_decay_ready.phase) k_decay_ready = advance(k_decay_ready, cfg.smem_decay_stages) - bars.mb_state_inp_ready.wait(state_inp_index.phase) - state_inp_index = advance(state_inp_index, 1) - desc_k_decay = sK_decay_stage.desc() - - for s in cutlass.range_constexpr(bmm_state_desc.num_subtiles_B): - for k in cutlass.range_constexpr(bmm_state_desc.sps_B): - mma_ts_step( - bmm_state_desc, - state_inp_ptr.subview(s * STATE_A_SEG), - desc_k_decay + s * STATE_B_SEG, - state_k_acc_ptr, - k, - cutlass.Boolean(s + k > 0), - ) + if have_state: + bars.mb_state_inp_ready.wait(state_inp_index.phase) + state_inp_index = advance(state_inp_index, 1) + desc_k_decay = sK_decay_stage.desc() + + for s in cutlass.range_constexpr(bmm_state_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_state_desc.sps_B): + mma_ts_step( + bmm_state_desc, + state_inp_ptr.subview(s * STATE_A_SEG), + desc_k_decay + s * STATE_B_SEG, + state_k_acc_ptr, + k, + cutlass.Boolean(s + k > 0), + ) - if elect_one: - bars.mb_state_k_acc_ready.arrive(cta_group=1) + if elect_one: + bars.mb_state_k_acc_ready.arrive(cta_group=1) # ---- q_state = state(T) @ Q_decay^T --------------------------------- bars.mb_qk_scale_ready[qk_scale_index.idx].wait(qk_scale_index.phase) bars.mb_o_acc_done[q_state_acc_stage].wait(o_acc_free.phase) o_acc_free = advance(o_acc_free, cfg.tmem_q_state_acc_stages) - desc_q_decay = sQ_decay_stage.desc() - q_state_acc_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_q_state_acc_offset + q_state_acc_stage * cfg.b_t, cutlass.Float32) - for s in cutlass.range_constexpr(bmm_state_desc.num_subtiles_B): - for k in cutlass.range_constexpr(bmm_state_desc.sps_B): - mma_ts_step( - bmm_state_desc, - state_inp_ptr.subview(s * STATE_A_SEG), - desc_q_decay + s * STATE_B_SEG, - q_state_acc_ptr, - k, - cutlass.Boolean(s + k > 0), - ) + if have_state: + desc_q_decay = sQ_decay_stage.desc() + for s in cutlass.range_constexpr(bmm_state_desc.num_subtiles_B): + for k in cutlass.range_constexpr(bmm_state_desc.sps_B): + mma_ts_step( + bmm_state_desc, + state_inp_ptr.subview(s * STATE_A_SEG), + desc_q_decay + s * STATE_B_SEG, + q_state_acc_ptr, + k, + cutlass.Boolean(s + k > 0), + ) if elect_one: - bars.mb_decay_done[decay_stage].arrive(cta_group=1) + bars.mb_decay_tcgen05_done[decay_stage].arrive(cta_group=1) # ---- S decay = S(T) @ diag(exp2(G_last)) --------- if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_read_done.wait(state_read_index.phase) - state_read_index = advance(state_read_index, 1) - desc_diag = sState_scale_diag_stage.desc() - - for k_block in cutlass.range_constexpr(cfg.d_k // 16): - mma_ts_step( - bmm_diag_desc, - state_inp_ptr.subview(k_block * bmm_diag_desc.tmem_advance_A), - desc_diag.advance_start_address(k_block * 256 * 2), - state_dsts[k_block], - 0, - cutlass.Boolean(False), - ) + if have_state: + bars.mb_state_acc_read_done.wait(state_read_index.phase) + state_read_index = advance(state_read_index, 1) + if have_state: + desc_diag = sState_scale_diag_stage.desc() + for k_block in cutlass.range_constexpr(cfg.d_k // 16): + mma_ts_step( + bmm_diag_desc, + state_inp_ptr.subview(k_block * bmm_diag_desc.tmem_advance_A), + desc_diag.advance_start_address(k_block * 256 * 2), + state_dsts[k_block], + 0, + cutlass.Boolean(False), + ) if elect_one: bars.mb_state_scale_diag_done[state_scale_diag_stage].arrive(cta_group=1) # ---- U = rhs(T) @ A_inv ---------------------------------------------- - bars.mb_a_ready[qk_stage].wait(qk_ready.phase) + bars.mb_a_inv_ready[qk_stage].wait(qk_ready.phase) bars.mb_rhs_ready.wait(rhs_index.phase) rhs_index = advance(rhs_index, 1) desc_qk = sQk_stage.shifted((cfg.b_t * cfg.b_t)).desc() @@ -819,10 +823,10 @@ def tcgen05_mma_warp( update_index = advance(update_index, 1) desc_k_restore = sK_restore_stage.desc() - mma_ts_step(bmm_final_state_desc, update_inp_ptr, desc_k_restore, state_dst_ptr, 0, cutlass.Boolean(True)) + mma_ts_step(bmm_final_state_desc, update_inp_ptr, desc_k_restore, state_dst_ptr, 0, have_state) if elect_one: bars.mb_k_restore_done[decay_stage].arrive(cta_group=1) - bars.mb_state_updated.arrive(cta_group=1) + bars.mb_state_acc_done.arrive(cta_group=1) # ---- O += U(T) @ QK -------------------------------------------------- bars.mb_qk_acc_ready[qk_stage].wait(qk_ready.phase) @@ -834,17 +838,17 @@ def tcgen05_mma_warp( desc_qk, nvvm.make_tmem_ptr(tmem_base + cfg.tmem_q_state_acc_offset + q_state_acc_stage * cfg.b_t, cutlass.Float32), 0, - cutlass.Boolean(True), + have_state, ) if elect_one: bars.mb_o_acc_ready.arrive(cta_group=1) bars.mb_qk_done[qk_stage].arrive(cta_group=1) qk_scale_index = advance(qk_scale_index, cfg.smem_state_scale_diag_stages) - bars.mb_final_state_stored.wait(final_state_index.phase) - final_state_index = advance(final_state_index, 1) cum_chunk_base += num_chunks_tile tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].wait(0) + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) nvvm.tcgen05_dealloc( nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), cutlass.Int32(512), @@ -960,7 +964,7 @@ def epilogue_warp( # Load B operand k_inv_col = k_block * 16 + rhs_col_offset k_inv_segment = k_inv_col // 64 - rhs_vec = nvvm.ldmatrix( + rhs_frag = nvvm.ldmatrix( sK_inv_ptr + k_inv_segment * (cfg.b_t * 64) + rhs_row_coord * 64 @@ -971,7 +975,7 @@ def epilogue_warp( # Load A operand storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask storage_slice = storage_key // 64 - qk_lhs_vec = nvvm.ldmatrix( + qk_lhs_frag = nvvm.ldmatrix( sQ_decay_ptr + storage_slice * (cfg.b_t * 64) + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), @@ -981,8 +985,8 @@ def epilogue_warp( mma_step( qk_acc, - (qk_lhs_vec[0], qk_lhs_vec[1], qk_lhs_vec[2], qk_lhs_vec[3]), - (rhs_vec[0], rhs_vec[1], rhs_vec[2], rhs_vec[3]), + (qk_lhs_frag[0], qk_lhs_frag[1], qk_lhs_frag[2], qk_lhs_frag[3]), + (rhs_frag[0], rhs_frag[1], rhs_frag[2], rhs_frag[3]), k_step=0, M=16, N=16, @@ -1264,7 +1268,7 @@ def compute0_warp_group( bars.mb_q_ready[raw_stage].wait((cum_chunk // cfg.smem_raw_stages) % 2) bars.mb_k_ready[raw_stage].wait((cum_chunk // cfg.smem_raw_stages) % 2) - k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) + k_inv_pack = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) raw_q_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) @@ -1280,10 +1284,10 @@ def compute0_warp_group( f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 raw_f16_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) - raw_q_vec = (sQ_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_k_vec = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_q_vec_f32 = raw_q_vec.to(cutlass.Float32) - raw_k_vec_f32 = raw_k_vec.to(cutlass.Float32) + raw_q_frag = (sQ_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_k_frag = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_q_vec_f32 = raw_q_frag.to(cutlass.Float32) + raw_k_vec_f32 = raw_k_frag.to(cutlass.Float32) for dim_offset in cutlass.range_constexpr(8): q_val = raw_q_vec_f32[dim_offset] k_val = raw_k_vec_f32[dim_offset] @@ -1321,20 +1325,20 @@ def compute0_warp_group( f32_segment = f32_dim_base // 32 f32_segment_dim = f32_dim_base - f32_segment * 32 g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) - exp_g_vec = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) + exp_g_frag = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) - exp_g_last_vec = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) + exp_g_last_frag = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) f32_reg_base = reg_base + f32_group * 4 for j in cutlass.range_constexpr(4): - exp_g_regs[f32_reg_base + j] = exp_g_vec[j] - exp_g_last_regs[f32_reg_base + j] = exp_g_last_vec[j] + exp_g_regs[f32_reg_base + j] = exp_g_frag[j] + exp_g_last_regs[f32_reg_base + j] = exp_g_last_frag[j] for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 # ---- K decay + K_inv operands: K * exp2(+G) and K * exp2(-G) ----- - k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_decay_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 @@ -1343,33 +1347,33 @@ def compute0_warp_group( k_value0, k_value1 = fmul2(raw_k_regs[raw_reg_idx0], raw_k_regs[raw_reg_idx1], k_inv_norm, k_inv_norm) k_pair = fp32_to_fp16(k_value0, k_value1, dtype=cfg.io_dtype) exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) - k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) + k_decay_pack[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) exp_neg_g0 = cute.math.rcp(exp_g_regs[raw_reg_idx0], approx=True, ftz=True) exp_neg_g1 = cute.math.rcp(exp_g_regs[raw_reg_idx1], approx=True, ftz=True) exp_neg_pair = fp32_to_fp16(exp_neg_g0, exp_neg_g1, dtype=cfg.io_dtype) - k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_pair, exp_neg_pair, cfg.io_dtype) + k_inv_pack[dim_half * 4 + pair_idx] = mul_f16x2(k_pair, exp_neg_pair, cfg.io_dtype) k_inv_vec = cutlass.Vector.from_elements( ( - k_inv_words[dim_half * 4], - k_inv_words[dim_half * 4 + 1], - k_inv_words[dim_half * 4 + 2], - k_inv_words[dim_half * 4 + 3], + k_inv_pack[dim_half * 4], + k_inv_pack[dim_half * 4 + 1], + k_inv_pack[dim_half * 4 + 2], + k_inv_pack[dim_half * 4 + 3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) k_decay_vec = cutlass.Vector.from_elements( ( - k_decay_words[0], - k_decay_words[1], - k_decay_words[2], - k_decay_words[3], + k_decay_pack[0], + k_decay_pack[1], + k_decay_pack[2], + k_decay_pack[3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) if cutlass.const_expr(dim_half == 0): operand_done_phase = ((cum_chunk // cfg.smem_decay_stages) + 1) % 2 - bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_tcgen05_done[decay_stage].wait(operand_done_phase) bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 @@ -1388,7 +1392,7 @@ def compute0_warp_group( for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 - q_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + q_decay_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 @@ -1397,14 +1401,14 @@ def compute0_warp_group( q_value0, q_value1 = fmul2(raw_q_regs[raw_reg_idx0], raw_q_regs[raw_reg_idx1], q_inv_norm, q_inv_norm) q_pair = fp32_to_fp16(q_value0, q_value1, dtype=cfg.io_dtype) exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) - q_decay_words[pair_idx] = mul_f16x2(q_pair, exp_g_pair, cfg.io_dtype) + q_decay_pack[pair_idx] = mul_f16x2(q_pair, exp_g_pair, cfg.io_dtype) q_decay_vec = cutlass.Vector.from_elements( ( - q_decay_words[0], - q_decay_words[1], - q_decay_words[2], - q_decay_words[3], + q_decay_pack[0], + q_decay_pack[1], + q_decay_pack[2], + q_decay_pack[3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) @@ -1420,22 +1424,22 @@ def compute0_warp_group( for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 - k_restore_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_restore_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 exp_g_last_pair = fp32_to_fp16(exp_g_last_regs[reg_base + dim0], exp_g_last_regs[reg_base + dim1], dtype=cfg.io_dtype) - k_restore_words[pair_idx] = mul_f16x2(k_inv_words[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) + k_restore_pack[pair_idx] = mul_f16x2(k_inv_pack[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) storage_row = decay_row ^ (cfg.b_t // 2) f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 k_restore_idx = f16_segment * (cfg.b_t * 64) + storage_row * 64 + swizzle_xor_128b(storage_row, f16_segment_dim, elem_bytes=2) k_restore_vec = cutlass.Vector.from_elements( ( - k_restore_words[0], - k_restore_words[1], - k_restore_words[2], - k_restore_words[3], + k_restore_pack[0], + k_restore_pack[1], + k_restore_pack[2], + k_restore_pack[3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) @@ -1482,6 +1486,7 @@ def compute1_warp_group( sO_ptr = sO_raw.data_ptr() sCheckpoint_ptr = sCheckpoint_raw.data_ptr() if cutlass.const_expr(cfg.enable_checkpoints) else sO_raw.data_ptr() checkpoint_done_index = PipelineState.start(phase=1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = tmem_base_slot.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 @@ -1532,7 +1537,7 @@ def compute1_warp_group( num_chunks_tile = wend - cstart if num_chunks_tile > 0: - # ---- first chunk: seed state TMEM from mState_init (else zeros) ---------- + # ---- first chunk: seed state TMEM from mState_init ---------- seed_from_initial_state = cstart == 0 if cutlass.const_expr(mState_init is not None and cfg.split_k): if seed_from_initial_state: @@ -1570,105 +1575,102 @@ def compute1_warp_group( nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), state_block[0:32], ) - else: - for key_block_start in cutlass.range_constexpr(0, cfg.d_k, 32): - state_block = cutlass.Array(cutlass.Float32, 32, alignment=16) - for col in cutlass.range_constexpr(32): - state_block[col] = cutlass.Float32(0.0) - - nvvm.tcgen05_st( - "32x32b", - nvvm.make_tmem_ptr((row_id << 16) + (tmem_col + cfg.tmem_state_acc_offset + key_block_start), cutlass.Float32), - state_block[0:32], - ) - - nvvm.tcgen05_wait("store") + if cutlass.const_expr(mState_init is not None): + nvvm.tcgen05_wait("store") sV_ptr = sV_raw.data_ptr() + raw_index.idx * (cfg.d_v * cfg.b_t) sBeta_ptr = sBeta_raw.data_ptr() + raw_index.idx * cfg.b_t # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- - state_blocks = [] - for sub in cutlass.range_constexpr(cfg.d_k // 16): - state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) - - for sub in cutlass.range_constexpr(cfg.d_k // 16): - packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) - for packed_col in cutlass.range_constexpr(8): - source_pair = packed_col ^ 4 - packed_state[packed_col] = fp32_to_fp16(state_blocks[sub][2 * source_pair], state_blocks[sub][2 * source_pair + 1], dtype=cfg.io_dtype) - nvvm.tcgen05_st( - "32x32b", - nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + sub * 8, cutlass.Int8), - packed_state[0:8], - ) - - nvvm.tcgen05_wait("store") - bars.mb_state_inp_ready.arrive() - if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_read_done.arrive() + if cutlass.const_expr(mState_init is not None): + state_vecs = [] + for sub in cutlass.range_constexpr(cfg.d_k // 16): + state_vecs.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) + + for sub in cutlass.range_constexpr(cfg.d_k // 16): + packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) + for packed_col in cutlass.range_constexpr(8): + source_pair = packed_col ^ 4 + packed_state[packed_col] = fp32_to_fp16(state_vecs[sub][2 * source_pair], state_vecs[sub][2 * source_pair + 1], dtype=cfg.io_dtype) + nvvm.tcgen05_st( + "32x32b", + nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + sub * 8, cutlass.Int8), + packed_state[0:8], + ) + nvvm.tcgen05_wait("store") + bars.mb_state_inp_ready.arrive() + if cutlass.const_expr(cfg.enable_checkpoints): + bars.mb_state_acc_read_done.arrive() # ---- rhs staging: rhs input = Beta * (V - state_k) ------------------- bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) bars.mb_beta_ready[raw_index.idx].wait(raw_index.phase) - bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) + if cutlass.const_expr(mState_init is not None): + bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_k_col_id, cutlass.Float32), num=2) - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_k_col_id, cutlass.Float32), num=2) - raw_v_regs0 = nvvm.ldmatrix( + state_k_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_k_col_id, cutlass.Float32), num=2) + state_k_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_k_col_id, cutlass.Float32), num=2) + raw_v_frag0 = nvvm.ldmatrix( sV_ptr + ov_swz_off0, 4, nvvm.MMALayout.COL, ) - raw_v_regs1 = nvvm.ldmatrix( + raw_v_frag1 = nvvm.ldmatrix( sV_ptr + ov_swz_off, 4, nvvm.MMALayout.COL, ) - beta_pairs = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_pack = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): token0 = (((reg_idx // 2) * 4 + (lane & 3)) ^ 4) * 2 beta0 = (sBeta_ptr + token0).load().to(cutlass.Float32) beta1 = (sBeta_ptr + token0 + 1).load().to(cutlass.Float32) - beta_pairs[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_pack[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) + rhs_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] - state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) - diff_pair = sub_f16x2( - raw_v_regs0[raw_matrix], - state_k_pair, - cfg.io_dtype, - ) - packed_rhs0[reg_idx] = mul_f16x2( - beta_pairs[reg_idx], + if cutlass.const_expr(mState_init is not None): + state_k_val0, state_k_val1 = state_k_vec0[frag_pair], state_k_vec0[frag_pair + 1] + state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) + diff_pair = sub_f16x2( + raw_v_frag0[raw_matrix], + state_k_pair, + cfg.io_dtype, + ) + else: + diff_pair = raw_v_frag0[raw_matrix] + rhs_pack0[reg_idx] = mul_f16x2( + beta_pack[reg_idx], diff_pair, cfg.io_dtype, ) - packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] - state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) - diff_pair = sub_f16x2( - raw_v_regs1[raw_matrix], - state_k_pair, - cfg.io_dtype, - ) - packed_rhs1[reg_idx] = mul_f16x2( - beta_pairs[reg_idx], + if cutlass.const_expr(mState_init is not None): + state_k_val0, state_k_val1 = state_k_vec1[frag_pair], state_k_vec1[frag_pair + 1] + state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) + diff_pair = sub_f16x2( + raw_v_frag1[raw_matrix], + state_k_pair, + cfg.io_dtype, + ) + else: + diff_pair = raw_v_frag1[raw_matrix] + rhs_pack1[reg_idx] = mul_f16x2( + beta_pack[reg_idx], diff_pair, cfg.io_dtype, ) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), rhs_pack0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), rhs_pack1[0:4]) nvvm.tcgen05_wait("store") - state_k_acc_index = advance(state_k_acc_index, 1) + if cutlass.const_expr(mState_init is not None): + state_k_acc_index = advance(state_k_acc_index, 1) bars.mb_v_done[raw_index.idx].arrive() bars.mb_beta_done[raw_index.idx].arrive() bars.mb_rhs_ready.arrive() @@ -1681,23 +1683,23 @@ def compute1_warp_group( num=cfg.b_t, ) - packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + update_pack = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): source_pair = packed_col ^ 4 token0 = source_pair * 2 token1 = token0 + 1 - packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + update_pack[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr(update_inp_addr, cutlass.Int8), - packed_update[0 : (cfg.b_t // 2)], + update_pack[0 : (cfg.b_t // 2)], ) nvvm.tcgen05_wait("store") update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_updated.wait(state_upd_index.phase) + bars.mb_state_acc_done.wait(state_upd_index.phase) state_upd_index = advance(state_upd_index, 1) raw_index = advance(raw_index, cfg.smem_raw_stages) @@ -1726,17 +1728,17 @@ def compute1_warp_group( # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- if cutlass.const_expr(not cfg.enable_checkpoints): - bars.mb_state_updated.wait(state_upd_index.phase) + bars.mb_state_acc_done.wait(state_upd_index.phase) state_upd_index = advance(state_upd_index, 1) - state_blocks = [] + state_vecs = [] for sub in cutlass.range_constexpr(cfg.d_k // 16): - state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) + state_vecs.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) for sub in cutlass.range_constexpr(cfg.d_k // 16): packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) for packed_col in cutlass.range_constexpr(8): source_pair = packed_col ^ 4 - packed_state[packed_col] = fp32_to_fp16(state_blocks[sub][2 * source_pair], state_blocks[sub][2 * source_pair + 1], dtype=cfg.io_dtype) + packed_state[packed_col] = fp32_to_fp16(state_vecs[sub][2 * source_pair], state_vecs[sub][2 * source_pair + 1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + sub * 8, cutlass.Int8), @@ -1753,13 +1755,13 @@ def compute1_warp_group( checkpoint_done_index = advance(checkpoint_done_index, cfg.smem_checkpoint_stages) checkpoint_stage_base = checkpoint_stage * (cfg.d_k * cfg.d_v) for slab in cutlass.range_constexpr(cfg.d_k // 16): - checkpoint_ld0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + slab * 16, cutlass.Float32), num=2) - checkpoint_ld1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + slab * 16, cutlass.Float32), num=2) - checkpoint_st0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - checkpoint_st1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + slab * 16, cutlass.Float32), num=2) + checkpoint_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + slab * 16, cutlass.Float32), num=2) + checkpoint_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - checkpoint_st0[reg_idx] = fp32_to_fp16(checkpoint_ld0[2 * reg_idx], checkpoint_ld0[2 * reg_idx + 1], dtype=cfg.io_dtype) - checkpoint_st1[reg_idx] = fp32_to_fp16(checkpoint_ld1[2 * reg_idx], checkpoint_ld1[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_pack0[reg_idx] = fp32_to_fp16(checkpoint_vec0[2 * reg_idx], checkpoint_vec0[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_pack1[reg_idx] = fp32_to_fp16(checkpoint_vec1[2 * reg_idx], checkpoint_vec1[2 * reg_idx + 1], dtype=cfg.io_dtype) checkpoint_row = slab * 16 + ov_token_coord nvvm.stmatrix( sCheckpoint_ptr @@ -1767,7 +1769,7 @@ def compute1_warp_group( + checkpoint_swz_off0 + checkpoint_row * 64 + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col0, elem_bytes=2), - checkpoint_st0.data_ptr().load(count=4, alignment=4), + checkpoint_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -1777,44 +1779,42 @@ def compute1_warp_group( + checkpoint_swz_off + checkpoint_row * 64 + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col, elem_bytes=2), - checkpoint_st1.data_ptr().load(count=4, alignment=4), + checkpoint_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) nvvm.tcgen05_wait("load") - bars.mb_state_read_done.arrive() + bars.mb_state_acc_read_done.arrive() nvvm.fence_proxy("async.shared", space="cta") bars.mb_checkpoint_tmastg_ready[checkpoint_stage].arrive() else: - bars.mb_state_read_done.arrive() + bars.mb_state_acc_read_done.arrive() bars.mb_o_acc_ready.wait(o_acc_index.phase) o_acc_index = advance(o_acc_index, 1) projection_col_id = q_state_col_base + prev_q_state_acc_stage * cfg.b_t - - loaded0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + projection_col_id, cutlass.Float32), num=2) - loaded1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + projection_col_id, cutlass.Float32), num=2) - - bars.mb_o_tmastg_done[prev_o_stage].wait(((prev_cum_chunk // cfg.smem_o_stages) + 1) % 2) + loaded_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + projection_col_id, cutlass.Float32), num=2) + loaded_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + projection_col_id, cutlass.Float32), num=2) # ---- output drain: O acc TMEM -> scaled b16 SMEM -------------------- - stsm_regs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - stsm_regs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + stsm_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + stsm_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - scaled0_0, scaled0_1 = fmul2(loaded0[2 * reg_idx], loaded0[2 * reg_idx + 1], scale, scale) - scaled1_0, scaled1_1 = fmul2(loaded1[2 * reg_idx], loaded1[2 * reg_idx + 1], scale, scale) - stsm_regs0[reg_idx] = fp32_to_fp16(scaled0_0, scaled0_1, dtype=mO.element_type) - stsm_regs1[reg_idx] = fp32_to_fp16(scaled1_0, scaled1_1, dtype=mO.element_type) + scaled0_0, scaled0_1 = fmul2(loaded_vec0[2 * reg_idx], loaded_vec0[2 * reg_idx + 1], scale, scale) + scaled1_0, scaled1_1 = fmul2(loaded_vec1[2 * reg_idx], loaded_vec1[2 * reg_idx + 1], scale, scale) + stsm_pack0[reg_idx] = fp32_to_fp16(scaled0_0, scaled0_1, dtype=mO.element_type) + stsm_pack1[reg_idx] = fp32_to_fp16(scaled1_0, scaled1_1, dtype=mO.element_type) + bars.mb_o_tmastg_done[prev_o_stage].wait(((prev_cum_chunk // cfg.smem_o_stages) + 1) % 2) nvvm.stmatrix( sO_ptr + prev_o_stage_base + ov_swz_off0, - stsm_regs0.data_ptr().load(count=4, alignment=4), + stsm_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) nvvm.stmatrix( sO_ptr + prev_o_stage_base + ov_swz_off, - stsm_regs1.data_ptr().load(count=4, alignment=4), + stsm_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -1822,67 +1822,66 @@ def compute1_warp_group( nvvm.fence_proxy("async.shared", space="cta") bars.mb_o_tmastg_ready[prev_o_stage].arrive() - bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) - # ---- rhs staging: rhs input = Beta * (V - state_k) ------------------- + bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) bars.mb_beta_ready[raw_index.idx].wait(raw_index.phase) - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_k_col_id, cutlass.Float32), num=2) - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_k_col_id, cutlass.Float32), num=2) - raw_v_regs0 = nvvm.ldmatrix( + state_k_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_k_col_id, cutlass.Float32), num=2) + state_k_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_k_col_id, cutlass.Float32), num=2) + raw_v_frag0 = nvvm.ldmatrix( sV_ptr + ov_swz_off0, 4, nvvm.MMALayout.COL, ) - raw_v_regs1 = nvvm.ldmatrix( + raw_v_frag1 = nvvm.ldmatrix( sV_ptr + ov_swz_off, 4, nvvm.MMALayout.COL, ) - beta_pairs = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_pack = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): token0 = (((reg_idx // 2) * 4 + (lane & 3)) ^ 4) * 2 beta0 = (sBeta_ptr + token0).load().to(cutlass.Float32) beta1 = (sBeta_ptr + token0 + 1).load().to(cutlass.Float32) - beta_pairs[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_pack[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) + rhs_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec0[frag_pair], state_k_vec0[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) diff_pair = sub_f16x2( - raw_v_regs0[raw_matrix], + raw_v_frag0[raw_matrix], state_k_pair, cfg.io_dtype, ) - packed_rhs0[reg_idx] = mul_f16x2( - beta_pairs[reg_idx], + rhs_pack0[reg_idx] = mul_f16x2( + beta_pack[reg_idx], diff_pair, cfg.io_dtype, ) - packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec1[frag_pair], state_k_vec1[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) diff_pair = sub_f16x2( - raw_v_regs1[raw_matrix], + raw_v_frag1[raw_matrix], state_k_pair, cfg.io_dtype, ) - packed_rhs1[reg_idx] = mul_f16x2( - beta_pairs[reg_idx], + rhs_pack1[reg_idx] = mul_f16x2( + beta_pack[reg_idx], diff_pair, cfg.io_dtype, ) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), rhs_pack0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), rhs_pack1[0:4]) nvvm.tcgen05_wait("store") state_k_acc_index = advance(state_k_acc_index, 1) bars.mb_v_done[raw_index.idx].arrive() @@ -1897,62 +1896,62 @@ def compute1_warp_group( num=cfg.b_t, ) - packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + update_pack = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): source_pair = packed_col ^ 4 token0 = source_pair * 2 token1 = token0 + 1 - packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + update_pack[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr(update_inp_addr, cutlass.Int8), - packed_update[0 : (cfg.b_t // 2)], + update_pack[0 : (cfg.b_t // 2)], ) nvvm.tcgen05_wait("store") update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_updated.wait(state_upd_index.phase) + bars.mb_state_acc_done.wait(state_upd_index.phase) state_upd_index = advance(state_upd_index, 1) raw_index = advance(raw_index, cfg.smem_raw_stages) if num_chunks_tile > 0: if cutlass.const_expr(not cfg.enable_checkpoints): - bars.mb_state_updated.wait(state_upd_index.phase) + bars.mb_state_acc_done.wait(state_upd_index.phase) state_upd_index = advance(state_upd_index, 1) last_cum_chunk = cum_chunk_base + num_chunks_tile - cutlass.Int32(1) final_o_stage = last_cum_chunk % cfg.smem_o_stages final_q_state_acc_stage = last_cum_chunk % cfg.tmem_q_state_acc_stages final_o_stage_base = final_o_stage * (cfg.b_t * cfg.d_v) - bars.mb_o_tmastg_done[final_o_stage].wait(((last_cum_chunk // cfg.smem_o_stages) + 1) % 2) bars.mb_o_acc_ready.wait(o_acc_index.phase) o_acc_index = advance(o_acc_index, 1) projection_col_id = q_state_col_base + final_q_state_acc_stage * cfg.b_t - loaded0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + projection_col_id, cutlass.Float32), num=2) - loaded1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + projection_col_id, cutlass.Float32), num=2) + loaded_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + projection_col_id, cutlass.Float32), num=2) + loaded_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + projection_col_id, cutlass.Float32), num=2) # ---- output drain: O acc TMEM -> scaled b16 SMEM -------------------- - stsm_regs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - stsm_regs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + stsm_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + stsm_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - scaled0_0, scaled0_1 = fmul2(loaded0[2 * reg_idx], loaded0[2 * reg_idx + 1], scale, scale) - scaled1_0, scaled1_1 = fmul2(loaded1[2 * reg_idx], loaded1[2 * reg_idx + 1], scale, scale) - stsm_regs0[reg_idx] = fp32_to_fp16(scaled0_0, scaled0_1, dtype=mO.element_type) - stsm_regs1[reg_idx] = fp32_to_fp16(scaled1_0, scaled1_1, dtype=mO.element_type) + scaled0_0, scaled0_1 = fmul2(loaded_vec0[2 * reg_idx], loaded_vec0[2 * reg_idx + 1], scale, scale) + scaled1_0, scaled1_1 = fmul2(loaded_vec1[2 * reg_idx], loaded_vec1[2 * reg_idx + 1], scale, scale) + stsm_pack0[reg_idx] = fp32_to_fp16(scaled0_0, scaled0_1, dtype=mO.element_type) + stsm_pack1[reg_idx] = fp32_to_fp16(scaled1_0, scaled1_1, dtype=mO.element_type) + bars.mb_o_tmastg_done[final_o_stage].wait(((last_cum_chunk // cfg.smem_o_stages) + 1) % 2) nvvm.stmatrix( sO_ptr + final_o_stage_base + ov_swz_off0, - stsm_regs0.data_ptr().load(count=4, alignment=4), + stsm_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) nvvm.stmatrix( sO_ptr + final_o_stage_base + ov_swz_off, - stsm_regs1.data_ptr().load(count=4, alignment=4), + stsm_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -1984,10 +1983,11 @@ def compute1_warp_group( mState_out[batch_idx, head_o, key_dim, value_dim] = mState_init[batch_idx, head_o, key_dim, value_dim] else: mState_out[batch_idx, head_o, key_dim, value_dim] = cutlass.Float32(0.0).to(mState_out.element_type) - bars.mb_final_state_stored.arrive() cum_chunk_base += num_chunks_tile tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].arrive() + @cute.jit def host( @@ -2171,8 +2171,9 @@ def kernel( layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, ) + elect_one = nvvm.elect_sync() if warp_idx == cfg.tma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_raw_stages): bars.mb_q_ready[stage].init() bars.mb_k_ready[stage].init() @@ -2185,27 +2186,27 @@ def kernel( bars.mb_v_done[stage].init() bars.mb_gate_done[stage].init() elif warp_idx == cfg.tcgen05_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: bars.mb_o_acc_ready.init() for stage in cutlass.range_constexpr(cfg.tmem_q_state_acc_stages): bars.mb_o_acc_done[stage].init() bars.mb_state_k_acc_ready.init() bars.mb_update_acc_ready.init() - bars.mb_state_updated.init() + bars.mb_state_acc_done.init() bars.mb_state_inp_ready.init() for stage in cutlass.range_constexpr(cfg.smem_state_scale_diag_stages): bars.mb_state_scale_diag_done[stage].init() for stage in cutlass.range_constexpr(cfg.smem_decay_stages): - bars.mb_decay_done[stage].init() + bars.mb_decay_tcgen05_done[stage].init() bars.mb_decay_super_done[stage].init() bars.mb_k_restore_done[stage].init() bars.mb_rhs_ready.init() bars.mb_update_ready.init() - bars.mb_final_state_stored.init() + bars.mb_tmem_done[0].init() elif warp_idx == cfg.super_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_qk_stages): - bars.mb_a_ready[stage].init() + bars.mb_a_inv_ready[stage].init() bars.mb_qk_acc_ready[stage].init() bars.mb_a_inv_done[stage].init() bars.mb_qk_done[stage].init() @@ -2214,7 +2215,7 @@ def kernel( for stage in cutlass.range_constexpr(cfg.smem_decay_stages): bars.mb_k_decay_inv_cg0_ready[stage].init() elif warp_idx == cfg.epilogue_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_o_stages): bars.mb_o_tmastg_ready[stage].init() bars.mb_o_tmastg_done[stage].init() @@ -2225,36 +2226,12 @@ def kernel( for stage in cutlass.range_constexpr(cfg.smem_checkpoint_stages): bars.mb_checkpoint_tmastg_ready[stage].init() bars.mb_checkpoint_tmastg_done[stage].init() - bars.mb_state_read_done.init() + bars.mb_state_acc_read_done.init() diag_zero = cfg.io_dtype(0.0) for diag_idx in cutlass.range(tidx, cfg.state_scale_diag_cosize, cfg.threads_per_cta, unroll=1): sState_scale_diag_raw[diag_idx] = diag_zero nvvm.fence_mbarrier_init() nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) - if (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) or warp_idx == cfg.tcgen05_mma_warp_id: - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_alloc(tmem_base_slot, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - - # Actual SMEM/TMEM buffers for the BT=16 schedule: - # q/k/v : 16 x 128 each - # gate_log2 : 16 x 128 - # beta : 16 - # q/k inverse norm : 16 each, staged once per decay stage - # exp_g_last : 128, CG0-local and staged once per decay stage - # state-scale diag : 8 x 16 x 16 input dtype, zeroed once in the prologue - # q_decay/k_decay : tcgen05 SW128 operands shared with super-MMA - # k_restore : tcgen05 SW128 N-major final-state operand - # k_inv : 16 x 128 token-major for super-MMA RHS - # A inverse/QK : 16 x 16 each, plus transposed tcgen05 operands - # state : external/kernel ABI is VK `[DV, DK]`; reference - # math can view it as KV `[DK, DV]` by transposing. - # The TS A-staging path keeps VK in TMEM so state*k - # is `[DV, DK] @ [DK, BT] -> [DV, BT]` with M=128. - if warp_idx == cfg.tma_warp_id: tmaldg_warp( cfg, @@ -2643,15 +2620,18 @@ def build_descs( tma_granu_elems = 128 // bpe seqlen = q.shape[0] - def headed(t, dim, hn): - return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) + q_headed = cute.make_tensor(q.iterator, cute.make_layout((d_k, h_q, seqlen), stride=(1, q.stride[1], q.stride[0]))) + k_headed = cute.make_tensor(k.iterator, cute.make_layout((d_k, h_k, seqlen), stride=(1, k.stride[1], k.stride[0]))) + v_headed = cute.make_tensor(v.iterator, cute.make_layout((d_v, h_v, seqlen), stride=(1, v.stride[1], v.stride[0]))) + gate_headed = cute.make_tensor(gate.iterator, cute.make_layout((d_k, n_heads_out, seqlen), stride=(1, gate.stride[1], gate.stride[0]))) + o_headed = cute.make_tensor(o.iterator, cute.make_layout((d_v, n_heads_out, seqlen), stride=(1, o.stride[1], o.stride[0]))) swz = cuda.TensorMapSwizzle.s128b - base_q = cuda.create_tensor_map_tiled_from_view(headed(q, d_k, h_q), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, n_heads_out), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_o = cuda.create_tensor_map_tiled_from_view(headed(o, d_v, n_heads_out), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_q = cuda.create_tensor_map_tiled_from_view(q_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_k = cuda.create_tensor_map_tiled_from_view(k_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(v_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(gate_headed, box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_o = cuda.create_tensor_map_tiled_from_view(o_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) base_checkpoint = base_o if cutlass.const_expr(state_checkpoints is not None): diff --git a/python/cudnn/linear_attention/frost/kernel/kda_recompute_f16.py b/python/cudnn/linear_attention/frost/kernel/kda_recompute_f16.py index bf627ca34..733459055 100644 --- a/python/cudnn/linear_attention/frost/kernel/kda_recompute_f16.py +++ b/python/cudnn/linear_attention/frost/kernel/kda_recompute_f16.py @@ -154,18 +154,18 @@ class KdaBars(NamedTuple): mb_rhs_ready: MBarrier mb_update_ready: MBarrier - mb_a_ready: MBarrier + mb_a_inv_ready: MBarrier mb_a_inv_done: MBarrier mb_qk_scale_ready: MBarrier mb_state_scale_diag_done: MBarrier mb_k_decay_inv_cg0_ready: MBarrier - mb_decay_done: MBarrier + mb_decay_tcgen05_done: MBarrier mb_decay_super_done: MBarrier mb_k_restore_done: MBarrier - mb_state_updated: MBarrier - mb_state_read_done: MBarrier - mb_final_state_stored: MBarrier + mb_state_acc_done: MBarrier + mb_state_acc_read_done: MBarrier + mb_tmem_done: MBarrier mb_checkpoint_tmastg_ready: MBarrier mb_checkpoint_tmastg_done: MBarrier @@ -199,7 +199,7 @@ def alloc(n): mb_state_inp_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_rhs_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_update_ready=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_a_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), + mb_a_inv_ready=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=WARP, producer=Producer.THREAD), mb_a_inv_done=MBarrier(alloc(cfg.smem_qk_stages), stages=cfg.smem_qk_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_qk_scale_ready=MBarrier( alloc(cfg.qk_scale_ready_stages), @@ -214,12 +214,12 @@ def alloc(n): producer=Producer.MMA_COMMIT, ), mb_k_decay_inv_cg0_ready=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=CG0_GROUP_THREADS, producer=Producer.THREAD), - mb_decay_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), + mb_decay_tcgen05_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), mb_decay_super_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=WARP, producer=Producer.THREAD), mb_k_restore_done=MBarrier(alloc(cfg.smem_decay_stages), stages=cfg.smem_decay_stages, init_count=1, producer=Producer.MMA_COMMIT), - mb_state_updated=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), - mb_state_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), - mb_final_state_stored=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_state_acc_done=MBarrier(alloc(1), stages=1, init_count=1, producer=Producer.MMA_COMMIT), + mb_state_acc_read_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), + mb_tmem_done=MBarrier(alloc(1), stages=1, init_count=CG1_THREADS, producer=Producer.THREAD), mb_checkpoint_tmastg_ready=MBarrier( alloc(cfg.smem_checkpoint_stages), stages=cfg.smem_checkpoint_stages, init_count=CG1_THREADS, producer=Producer.THREAD ), @@ -429,7 +429,7 @@ def super_mma_warp( # Load B operand k_inv_col = k_block * 16 + rhs_col_offset k_inv_segment = k_inv_col // 64 - rhs_vec = nvvm.ldmatrix( + rhs_frag = nvvm.ldmatrix( sK_inv_ptr + k_inv_segment * (cfg.b_t * 64) + rhs_row_coord * 64 @@ -440,7 +440,7 @@ def super_mma_warp( # Load A operand storage_key = (k_block * 16 + lhs_col_offset) ^ decay_key_mask storage_slice = storage_key // 64 - kk_lhs_vec = nvvm.ldmatrix( + kk_lhs_frag = nvvm.ldmatrix( sK_decay_ptr + storage_slice * (cfg.b_t * 64) + swizzle_xor_128b(lhs_row_coord, lhs_row_coord * 64 + storage_key - storage_slice * 64, elem_bytes=2), @@ -450,8 +450,8 @@ def super_mma_warp( mma_step( kk_acc, - (kk_lhs_vec[0], kk_lhs_vec[1], kk_lhs_vec[2], kk_lhs_vec[3]), - (rhs_vec[0], rhs_vec[1], rhs_vec[2], rhs_vec[3]), + (kk_lhs_frag[0], kk_lhs_frag[1], kk_lhs_frag[2], kk_lhs_frag[3]), + (rhs_frag[0], rhs_frag[1], rhs_frag[2], rhs_frag[3]), k_step=0, M=16, N=16, @@ -464,21 +464,21 @@ def super_mma_warp( row_hi = row_lo + cutlass.Int32(8) beta_lo = (sBeta_ptr + row_lo).load().to(cutlass.Float32) beta_hi = (sBeta_ptr + row_hi).load().to(cutlass.Float32) - l_frag = cutlass.Array(cutlass.Float32, 8, alignment=16) + l_regs = cutlass.Array(cutlass.Float32, 8, alignment=16) for accum_idx in cutlass.range_constexpr(8): row_coord = row_hi if cutlass.const_expr(accum_idx % 4 >= 2) else row_lo col_coord = (accum_idx // 4) * 8 + 2 * (lane % 4) if cutlass.const_expr(accum_idx % 2 == 1): col_coord = col_coord + cutlass.Int32(1) - l_frag[accum_idx] = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) + l_regs[accum_idx] = kk_acc[accum_idx] if row_coord > col_coord else cutlass.Float32(0.0) for pair in cutlass.range_constexpr(4): beta_scale = beta_hi if cutlass.const_expr(pair % 2 == 1) else beta_lo - l_frag[2 * pair], l_frag[2 * pair + 1] = fmul2(l_frag[2 * pair], l_frag[2 * pair + 1], beta_scale, beta_scale) + l_regs[2 * pair], l_regs[2 * pair + 1] = fmul2(l_regs[2 * pair], l_regs[2 * pair + 1], beta_scale, beta_scale) bars.mb_beta_done[raw_index.idx].arrive() - l_a0 = fp32_to_fp16(l_frag[0], l_frag[1], dtype=cfg.io_dtype) - l_a1 = fp32_to_fp16(l_frag[2], l_frag[3], dtype=cfg.io_dtype) - l_a2 = fp32_to_fp16(l_frag[4], l_frag[5], dtype=cfg.io_dtype) - l_a3 = fp32_to_fp16(l_frag[6], l_frag[7], dtype=cfg.io_dtype) + l_a0 = fp32_to_fp16(l_regs[0], l_regs[1], dtype=cfg.io_dtype) + l_a1 = fp32_to_fp16(l_regs[2], l_regs[3], dtype=cfg.io_dtype) + l_a2 = fp32_to_fp16(l_regs[4], l_regs[5], dtype=cfg.io_dtype) + l_a3 = fp32_to_fp16(l_regs[6], l_regs[7], dtype=cfg.io_dtype) l_values = cutlass.Vector.from_elements((l_a0, l_a1, l_a2, l_a3), cutlass.Int32).bitcast(cfg.io_dtype).to(cutlass.Float32) # ---- A_inv = I - L, then three Neumann doubling rounds --------------- @@ -556,7 +556,7 @@ def super_mma_warp( shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") - bars.mb_a_ready[qk_stage].arrive() + bars.mb_a_inv_ready[qk_stage].arrive() bars.mb_decay_super_done[decay_stage].arrive() raw_index = advance(raw_index, cfg.smem_raw_stages) cum_chunk_base += num_chunks_tile @@ -583,6 +583,8 @@ def tcgen05_mma_warp( every state GEMM and owning the TMEM lifecycle.""" elect_one = nvvm.elect_sync() nvvm.setmaxregister(cfg.num_regs_other, nvvm.SetMaxRegisterAction.DECREASE) + nvvm.tcgen05_alloc(sTmem_base, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = sTmem_base.load() state_inp_ptr = nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_inp_offset, cutlass.Int8) state_dsts = tuple(nvvm.make_tmem_ptr(tmem_base + cfg.tmem_state_acc_offset + k * 16, cutlass.Float32) for k in range(cfg.d_k // 16)) @@ -595,7 +597,6 @@ def tcgen05_mma_warp( state_read_index = PipelineState.start(phase=0) rhs_index = PipelineState.start(phase=0) update_index = PipelineState.start(phase=0) - final_state_index = PipelineState.start(phase=0) qk_scale_index = PipelineState.start(phase=0) k_decay_ready = PipelineState.start(phase=0) qk_ready = PipelineState.start(phase=0) @@ -714,11 +715,11 @@ def tcgen05_mma_warp( if elect_one: bars.mb_state_k_acc_ready.arrive(cta_group=1) - bars.mb_decay_done[decay_stage].arrive(cta_group=1) + bars.mb_decay_tcgen05_done[decay_stage].arrive(cta_group=1) bars.mb_qk_scale_ready[qk_scale_index.idx].wait(qk_scale_index.phase) if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_read_done.wait(state_read_index.phase) + bars.mb_state_acc_read_done.wait(state_read_index.phase) state_read_index = advance(state_read_index, 1) # ---- State decay = State(T) @ exp2(g_last) diag (per-k-atom blocks) ---- @@ -738,7 +739,7 @@ def tcgen05_mma_warp( bars.mb_state_scale_diag_done[state_scale_diag_stage].arrive(cta_group=1) # ---- update = rhs(T) @ A_inv ----------------------------------------- - bars.mb_a_ready[qk_stage].wait(qk_ready.phase) + bars.mb_a_inv_ready[qk_stage].wait(qk_ready.phase) bars.mb_rhs_ready.wait(rhs_index.phase) rhs_index = advance(rhs_index, 1) desc_qk = sQk_stage.shifted((cfg.b_t * cfg.b_t)).desc() @@ -755,15 +756,15 @@ def tcgen05_mma_warp( mma_ts_step(bmm_final_state_desc, update_inp_ptr, desc_k_restore, state_dst_ptr, 0, cutlass.Boolean(True)) if elect_one: bars.mb_k_restore_done[decay_stage].arrive(cta_group=1) - bars.mb_state_updated.arrive(cta_group=1) + bars.mb_state_acc_done.arrive(cta_group=1) qk_ready = advance(qk_ready, cfg.smem_qk_stages) qk_scale_index = advance(qk_scale_index, cfg.smem_state_scale_diag_stages) - bars.mb_final_state_stored.wait(final_state_index.phase) - final_state_index = advance(final_state_index, 1) cum_chunk_base += num_chunks_tile tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].wait(0) + nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) nvvm.tcgen05_dealloc( nvvm.make_tmem_ptr(tmem_base, cutlass.Int8), cutlass.Int32(512), @@ -1016,7 +1017,7 @@ def compute0_warp_group( nvvm.barrier_cta_sync(cfg.cg0_group_sync_barrier_base_id + cg0_group_id, thread_count=cfg.cg0_threads_per_group) bars.mb_k_ready[raw_stage].wait((cum_chunk // cfg.smem_raw_stages) % 2) - k_inv_words = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) + k_inv_pack = cutlass.Array(cutlass.Int32, 2 * 4, alignment=16) raw_k_regs = cutlass.Array(cutlass.Float32, 2 * 8, alignment=16) # ---- optional K L2-norm + K_inv staging ------------------------------ @@ -1031,8 +1032,8 @@ def compute0_warp_group( f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 raw_f16_idx = f16_segment * (cfg.b_t * 64) + decay_row * 64 + swizzle_xor_128b(decay_row, f16_segment_dim, elem_bytes=2) - raw_k_vec = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) - raw_k_vec_f32 = raw_k_vec.to(cutlass.Float32) + raw_k_frag = (sK_ptr + raw_f16_idx).load(count=8, alignment=16) + raw_k_vec_f32 = raw_k_frag.to(cutlass.Float32) for dim_offset in cutlass.range_constexpr(8): k_val = raw_k_vec_f32[dim_offset] raw_k_regs[reg_base + dim_offset] = k_val @@ -1062,20 +1063,20 @@ def compute0_warp_group( f32_segment = f32_dim_base // 32 f32_segment_dim = f32_dim_base - f32_segment * 32 g_prefix_idx = f32_segment * (cfg.b_t * 32) + decay_row * 32 + swizzle_xor_128b(decay_row, f32_segment_dim, elem_bytes=4) - exp_g_vec = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) + exp_g_frag = (sGate_ptr + g_prefix_idx).load(count=4, alignment=16) exp_g_last_idx = f32_segment * (cfg.b_t * 32) + (cfg.b_t - 1) * 32 + swizzle_xor_128b((cfg.b_t - 1), f32_segment_dim, elem_bytes=4) - exp_g_last_vec = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) + exp_g_last_frag = (sGate_ptr + exp_g_last_idx).load(count=4, alignment=16) f32_reg_base = reg_base + f32_group * 4 for j in cutlass.range_constexpr(4): - exp_g_regs[f32_reg_base + j] = exp_g_vec[j] - exp_g_last_regs[f32_reg_base + j] = exp_g_last_vec[j] + exp_g_regs[f32_reg_base + j] = exp_g_frag[j] + exp_g_last_regs[f32_reg_base + j] = exp_g_last_frag[j] for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 # ---- K decay + K_inv operands: K * exp2(+g) and K * exp2(-g) ----- - k_decay_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_decay_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 @@ -1084,33 +1085,33 @@ def compute0_warp_group( k_value0, k_value1 = fmul2(raw_k_regs[raw_reg_idx0], raw_k_regs[raw_reg_idx1], k_inv_norm, k_inv_norm) k_pair = fp32_to_fp16(k_value0, k_value1, dtype=cfg.io_dtype) exp_g_pair = fp32_to_fp16(exp_g_regs[raw_reg_idx0], exp_g_regs[raw_reg_idx1], dtype=cfg.io_dtype) - k_decay_words[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) + k_decay_pack[pair_idx] = mul_f16x2(k_pair, exp_g_pair, cfg.io_dtype) exp_neg_g0 = cute.math.rcp(exp_g_regs[raw_reg_idx0], approx=True, ftz=True) exp_neg_g1 = cute.math.rcp(exp_g_regs[raw_reg_idx1], approx=True, ftz=True) exp_neg_pair = fp32_to_fp16(exp_neg_g0, exp_neg_g1, dtype=cfg.io_dtype) - k_inv_words[dim_half * 4 + pair_idx] = mul_f16x2(k_pair, exp_neg_pair, cfg.io_dtype) + k_inv_pack[dim_half * 4 + pair_idx] = mul_f16x2(k_pair, exp_neg_pair, cfg.io_dtype) k_inv_vec = cutlass.Vector.from_elements( ( - k_inv_words[dim_half * 4], - k_inv_words[dim_half * 4 + 1], - k_inv_words[dim_half * 4 + 2], - k_inv_words[dim_half * 4 + 3], + k_inv_pack[dim_half * 4], + k_inv_pack[dim_half * 4 + 1], + k_inv_pack[dim_half * 4 + 2], + k_inv_pack[dim_half * 4 + 3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) k_decay_vec = cutlass.Vector.from_elements( ( - k_decay_words[0], - k_decay_words[1], - k_decay_words[2], - k_decay_words[3], + k_decay_pack[0], + k_decay_pack[1], + k_decay_pack[2], + k_decay_pack[3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) if cutlass.const_expr(dim_half == 0): operand_done_phase = ((cum_chunk // cfg.smem_decay_stages) + 1) % 2 - bars.mb_decay_done[decay_stage].wait(operand_done_phase) + bars.mb_decay_tcgen05_done[decay_stage].wait(operand_done_phase) bars.mb_decay_super_done[decay_stage].wait(operand_done_phase) f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 @@ -1130,22 +1131,22 @@ def compute0_warp_group( for dim_half in cutlass.range_constexpr(2): dim_base = dim_half * (cfg.d_k // 2) + lane_in_row_group * 8 reg_base = dim_half * 8 - k_restore_words = cutlass.Array(cutlass.Int32, 4, alignment=16) + k_restore_pack = cutlass.Array(cutlass.Int32, 4, alignment=16) for pair_idx in cutlass.range_constexpr(4): dim0 = pair_idx * 2 dim1 = dim0 + 1 exp_g_last_pair = fp32_to_fp16(exp_g_last_regs[reg_base + dim0], exp_g_last_regs[reg_base + dim1], dtype=cfg.io_dtype) - k_restore_words[pair_idx] = mul_f16x2(k_inv_words[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) + k_restore_pack[pair_idx] = mul_f16x2(k_inv_pack[dim_half * 4 + pair_idx], exp_g_last_pair, cfg.io_dtype) storage_row = decay_row ^ (cfg.b_t // 2) f16_segment = dim_base // 64 f16_segment_dim = dim_base - f16_segment * 64 k_restore_idx = f16_segment * (cfg.b_t * 64) + storage_row * 64 + swizzle_xor_128b(storage_row, f16_segment_dim, elem_bytes=2) k_restore_vec = cutlass.Vector.from_elements( ( - k_restore_words[0], - k_restore_words[1], - k_restore_words[2], - k_restore_words[3], + k_restore_pack[0], + k_restore_pack[1], + k_restore_pack[2], + k_restore_pack[3], ), cutlass.Int32, ).bitcast(cfg.io_dtype) @@ -1187,6 +1188,7 @@ def compute1_warp_group( nvvm.setmaxregister(cfg.num_regs_compute_group_1, nvvm.SetMaxRegisterAction.INCREASE) sCheckpoint_ptr = sCheckpoint_raw.data_ptr() if cutlass.const_expr(cfg.enable_checkpoints) else sV_raw.data_ptr() checkpoint_done_index = PipelineState.start(phase=1) + nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) tmem_base = sTmem_base.load() tmem_col = tmem_base & 0xFFFF tmem_row = tmem_base >> 16 @@ -1290,25 +1292,25 @@ def compute1_warp_group( sBeta_ptr = sBeta_raw.data_ptr() + raw_index.idx * cfg.b_t # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- - state_blocks = [] + state_vecs = [] for sub in cutlass.range_constexpr(cfg.d_k // 16): - state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) + state_vecs.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) for sub in cutlass.range_constexpr(cfg.d_k // 16): - packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) + state_pack = cutlass.Array(cutlass.Int32, 8, alignment=16) for packed_col in cutlass.range_constexpr(8): source_pair = packed_col ^ 4 - packed_state[packed_col] = fp32_to_fp16(state_blocks[sub][2 * source_pair], state_blocks[sub][2 * source_pair + 1], dtype=cfg.io_dtype) + state_pack[packed_col] = fp32_to_fp16(state_vecs[sub][2 * source_pair], state_vecs[sub][2 * source_pair + 1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + sub * 8, cutlass.Int8), - packed_state[0:8], + state_pack[0:8], ) nvvm.tcgen05_wait("store") bars.mb_state_inp_ready.arrive() if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_read_done.arrive() + bars.mb_state_acc_read_done.arrive() # ---- rhs staging: rhs input = Beta * (V - State*K) ------------------- bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) @@ -1316,63 +1318,63 @@ def compute1_warp_group( bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) # ---- read back State*K acc + raw V fragments ------------------------- - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + statek_col_id, cutlass.Float32), num=2) + state_k_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + statek_col_id, cutlass.Float32), num=2) - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + statek_col_id, cutlass.Float32), num=2) + state_k_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + statek_col_id, cutlass.Float32), num=2) - raw_v_regs0 = nvvm.ldmatrix( + raw_v_frag0 = nvvm.ldmatrix( sV_ptr + v_swz_off0, 4, nvvm.MMALayout.COL, ) - raw_v_regs1 = nvvm.ldmatrix( + raw_v_frag1 = nvvm.ldmatrix( sV_ptr + v_swz_off, 4, nvvm.MMALayout.COL, ) - beta_pairs = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_pack = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): token0 = (((reg_idx // 2) * 4 + (lane & 3)) ^ 4) * 2 beta0 = (sBeta_ptr + token0).load().to(cutlass.Float32) beta1 = (sBeta_ptr + token0 + 1).load().to(cutlass.Float32) - beta_pairs[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_pack[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) + rhs_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec0[frag_pair], state_k_vec0[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) diff_pair = sub_f16x2( - raw_v_regs0[raw_matrix], + raw_v_frag0[raw_matrix], state_k_pair, cfg.io_dtype, ) - packed_rhs0[reg_idx] = mul_f16x2( - beta_pairs[reg_idx], + rhs_pack0[reg_idx] = mul_f16x2( + beta_pack[reg_idx], diff_pair, cfg.io_dtype, ) - packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec1[frag_pair], state_k_vec1[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) diff_pair = sub_f16x2( - raw_v_regs1[raw_matrix], + raw_v_frag1[raw_matrix], state_k_pair, cfg.io_dtype, ) - packed_rhs1[reg_idx] = mul_f16x2( - beta_pairs[reg_idx], + rhs_pack1[reg_idx] = mul_f16x2( + beta_pack[reg_idx], diff_pair, cfg.io_dtype, ) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), rhs_pack0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), rhs_pack1[0:4]) nvvm.tcgen05_wait("store") state_k_acc_index = advance(state_k_acc_index, 1) bars.mb_v_done[raw_index.idx].arrive() @@ -1387,23 +1389,23 @@ def compute1_warp_group( num=cfg.b_t, ) - packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + update_pack = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): source_pair = packed_col ^ 4 token0 = source_pair * 2 token1 = token0 + 1 - packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + update_pack[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr(upd_inp_addr, cutlass.Int8), - packed_update[0 : (cfg.b_t // 2)], + update_pack[0 : (cfg.b_t // 2)], ) nvvm.tcgen05_wait("store") update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_updated.wait(state_upd_index.phase) + bars.mb_state_acc_done.wait(state_upd_index.phase) state_upd_index = advance(state_upd_index, 1) raw_index = advance(raw_index, cfg.smem_raw_stages) @@ -1426,21 +1428,21 @@ def compute1_warp_group( # ---- state repack: acc TMEM -> packed b16 TMEM ---------------------- if cutlass.const_expr(not cfg.enable_checkpoints): - bars.mb_state_updated.wait(state_upd_index.phase) + bars.mb_state_acc_done.wait(state_upd_index.phase) state_upd_index = advance(state_upd_index, 1) - state_blocks = [] + state_vecs = [] for sub in cutlass.range_constexpr(cfg.d_k // 16): - state_blocks.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) + state_vecs.append(nvvm.tcgen05_ld("32x32b", nvvm.make_tmem_ptr(row_addr + state_col_id + sub * 16, cutlass.Float32), num=16)) for sub in cutlass.range_constexpr(cfg.d_k // 16): - packed_state = cutlass.Array(cutlass.Int32, 8, alignment=16) + state_pack = cutlass.Array(cutlass.Int32, 8, alignment=16) for packed_col in cutlass.range_constexpr(8): source_pair = packed_col ^ 4 - packed_state[packed_col] = fp32_to_fp16(state_blocks[sub][2 * source_pair], state_blocks[sub][2 * source_pair + 1], dtype=cfg.io_dtype) + state_pack[packed_col] = fp32_to_fp16(state_vecs[sub][2 * source_pair], state_vecs[sub][2 * source_pair + 1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr((tmem_row << 16) + packed_col_id + sub * 8, cutlass.Int8), - packed_state[0:8], + state_pack[0:8], ) nvvm.tcgen05_wait("store") bars.mb_state_inp_ready.arrive() @@ -1453,13 +1455,13 @@ def compute1_warp_group( checkpoint_done_index = advance(checkpoint_done_index, cfg.smem_checkpoint_stages) checkpoint_stage_base = checkpoint_stage * (cfg.d_k * cfg.d_v) for slab in cutlass.range_constexpr(cfg.d_k // 16): - checkpoint_ld0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + slab * 16, cutlass.Float32), num=2) - checkpoint_ld1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + slab * 16, cutlass.Float32), num=2) - checkpoint_st0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) - checkpoint_st1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + state_col_id + slab * 16, cutlass.Float32), num=2) + checkpoint_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + state_col_id + slab * 16, cutlass.Float32), num=2) + checkpoint_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + checkpoint_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): - checkpoint_st0[reg_idx] = fp32_to_fp16(checkpoint_ld0[2 * reg_idx], checkpoint_ld0[2 * reg_idx + 1], dtype=cfg.io_dtype) - checkpoint_st1[reg_idx] = fp32_to_fp16(checkpoint_ld1[2 * reg_idx], checkpoint_ld1[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_pack0[reg_idx] = fp32_to_fp16(checkpoint_vec0[2 * reg_idx], checkpoint_vec0[2 * reg_idx + 1], dtype=cfg.io_dtype) + checkpoint_pack1[reg_idx] = fp32_to_fp16(checkpoint_vec1[2 * reg_idx], checkpoint_vec1[2 * reg_idx + 1], dtype=cfg.io_dtype) checkpoint_row = slab * 16 + frag_row_coord nvvm.stmatrix( sCheckpoint_ptr @@ -1467,7 +1469,7 @@ def compute1_warp_group( + checkpoint_swz_off0 + checkpoint_row * 64 + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col0, elem_bytes=2), - checkpoint_st0.data_ptr().load(count=4, alignment=4), + checkpoint_pack0.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) @@ -1477,80 +1479,79 @@ def compute1_warp_group( + checkpoint_swz_off + checkpoint_row * 64 + swizzle_xor_128b(checkpoint_row, checkpoint_swz_col, elem_bytes=2), - checkpoint_st1.data_ptr().load(count=4, alignment=4), + checkpoint_pack1.data_ptr().load(count=4, alignment=4), nvvm.MMALayout.COL, shape=nvvm.StoreShape.M8N8, ) nvvm.fence_proxy("async.shared", space="cta") bars.mb_checkpoint_tmastg_ready[checkpoint_stage].arrive() nvvm.tcgen05_wait("load") - bars.mb_state_read_done.arrive() + bars.mb_state_acc_read_done.arrive() else: - bars.mb_state_read_done.arrive() - - bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) + bars.mb_state_acc_read_done.arrive() # ---- rhs staging: rhs input = Beta * (V - State*K) ------------------- + bars.mb_state_k_acc_ready.wait(state_k_acc_index.phase) bars.mb_v_ready[raw_index.idx].wait(raw_index.phase) bars.mb_beta_ready[raw_index.idx].wait(raw_index.phase) # ---- read back State*K acc + raw V fragments ------------------------- - state_k0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + statek_col_id, cutlass.Float32), num=2) - state_k1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + statek_col_id, cutlass.Float32), num=2) + state_k_vec0 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row_addr + statek_col_id, cutlass.Float32), num=2) + state_k_vec1 = nvvm.tcgen05_ld("16x256b", nvvm.make_tmem_ptr(row16_addr + statek_col_id, cutlass.Float32), num=2) - raw_v_regs0 = nvvm.ldmatrix( + raw_v_frag0 = nvvm.ldmatrix( sV_ptr + v_swz_off0, 4, nvvm.MMALayout.COL, ) - raw_v_regs1 = nvvm.ldmatrix( + raw_v_frag1 = nvvm.ldmatrix( sV_ptr + v_swz_off, 4, nvvm.MMALayout.COL, ) - beta_pairs = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_pack = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): token0 = (((reg_idx // 2) * 4 + (lane & 3)) ^ 4) * 2 beta0 = (sBeta_ptr + token0).load().to(cutlass.Float32) beta1 = (sBeta_ptr + token0 + 1).load().to(cutlass.Float32) - beta_pairs[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) - packed_rhs0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + beta_pack[reg_idx] = fp32_to_fp16(beta0, beta1, dtype=cfg.io_dtype) + rhs_pack0 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k0[frag_pair], state_k0[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec0[frag_pair], state_k_vec0[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) diff_pair = sub_f16x2( - raw_v_regs0[raw_matrix], + raw_v_frag0[raw_matrix], state_k_pair, cfg.io_dtype, ) - packed_rhs0[reg_idx] = mul_f16x2( - beta_pairs[reg_idx], + rhs_pack0[reg_idx] = mul_f16x2( + beta_pack[reg_idx], diff_pair, cfg.io_dtype, ) - packed_rhs1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) + rhs_pack1 = cutlass.Array(cutlass.Int32, 4, space=cutlass.AddressSpace.rmem) for reg_idx in cutlass.range_constexpr(4): raw_matrix = (1 - (reg_idx // 2)) * 2 + (reg_idx & 1) frag_pair = (reg_idx ^ 2) * 2 - state_k_val0, state_k_val1 = state_k1[frag_pair], state_k1[frag_pair + 1] + state_k_val0, state_k_val1 = state_k_vec1[frag_pair], state_k_vec1[frag_pair + 1] state_k_pair = fp32_to_fp16(state_k_val0, state_k_val1, dtype=cfg.io_dtype) diff_pair = sub_f16x2( - raw_v_regs1[raw_matrix], + raw_v_frag1[raw_matrix], state_k_pair, cfg.io_dtype, ) - packed_rhs1[reg_idx] = mul_f16x2( - beta_pairs[reg_idx], + rhs_pack1[reg_idx] = mul_f16x2( + beta_pack[reg_idx], diff_pair, cfg.io_dtype, ) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs0[0:4]) - nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), packed_rhs1[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row_addr + rhs_inp_col_id, cutlass.Int8), rhs_pack0[0:4]) + nvvm.tcgen05_st("16x128b", nvvm.make_tmem_ptr(st_row16_addr + rhs_inp_col_id, cutlass.Int8), rhs_pack1[0:4]) nvvm.tcgen05_wait("store") state_k_acc_index = advance(state_k_acc_index, 1) bars.mb_v_done[raw_index.idx].arrive() @@ -1565,30 +1566,30 @@ def compute1_warp_group( num=cfg.b_t, ) - packed_update = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) + update_pack = cutlass.Array(cutlass.Int32, (cfg.b_t // 2), alignment=16) for packed_col in cutlass.range_constexpr((cfg.b_t // 2)): source_pair = packed_col ^ 4 token0 = source_pair * 2 token1 = token0 + 1 - packed_update[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) + update_pack[packed_col] = fp32_to_fp16(update[token0], update[token1], dtype=cfg.io_dtype) nvvm.tcgen05_st( "32x32b", nvvm.make_tmem_ptr(upd_inp_addr, cutlass.Int8), - packed_update[0 : (cfg.b_t // 2)], + update_pack[0 : (cfg.b_t // 2)], ) nvvm.tcgen05_wait("store") update_acc_index = advance(update_acc_index, 1) bars.mb_update_ready.arrive() if cutlass.const_expr(cfg.enable_checkpoints): - bars.mb_state_updated.wait(state_upd_index.phase) + bars.mb_state_acc_done.wait(state_upd_index.phase) state_upd_index = advance(state_upd_index, 1) raw_index = advance(raw_index, cfg.smem_raw_stages) if num_chunks_tile > 0: if cutlass.const_expr(not cfg.enable_checkpoints): - bars.mb_state_updated.wait(state_upd_index.phase) + bars.mb_state_acc_done.wait(state_upd_index.phase) state_upd_index = advance(state_upd_index, 1) owns_final = wend == num_chunks_b @@ -1615,9 +1616,10 @@ def compute1_warp_group( mState_out[batch_idx, head_o, key_dim, value_dim] = mState_init[batch_idx, head_o, key_dim, value_dim] else: mState_out[batch_idx, head_o, key_dim, value_dim] = cutlass.Float32(0.0).to(mState_out.element_type) - bars.mb_final_state_stored.arrive() tile_idx, sched_state = sched_next_tile(cfg, bars, sSched, sched_state, tile_idx, num_ctas) + bars.mb_tmem_done[0].arrive() + @cute.jit def host( @@ -1775,8 +1777,9 @@ def kernel( layout=nvvm.Tcgen05SmemSwizzle.SWIZZLE_32B, ) + elect_one = nvvm.elect_sync() if warp_idx == cfg.tma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_raw_stages): bars.mb_k_ready[stage].init() bars.mb_v_ready[stage].init() @@ -1787,31 +1790,31 @@ def kernel( bars.mb_v_done[stage].init() bars.mb_gate_done[stage].init() elif warp_idx == cfg.tcgen05_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: bars.mb_state_k_acc_ready.init() bars.mb_update_acc_ready.init() - bars.mb_state_updated.init() + bars.mb_state_acc_done.init() bars.mb_state_inp_ready.init() for stage in cutlass.range_constexpr(cfg.smem_state_scale_diag_stages): bars.mb_state_scale_diag_done[stage].init() for stage in cutlass.range_constexpr(cfg.smem_decay_stages): - bars.mb_decay_done[stage].init() + bars.mb_decay_tcgen05_done[stage].init() bars.mb_decay_super_done[stage].init() bars.mb_k_restore_done[stage].init() bars.mb_rhs_ready.init() bars.mb_update_ready.init() - bars.mb_final_state_stored.init() + bars.mb_tmem_done[0].init() elif warp_idx == cfg.super_mma_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.smem_qk_stages): - bars.mb_a_ready[stage].init() + bars.mb_a_inv_ready[stage].init() bars.mb_a_inv_done[stage].init() for stage in cutlass.range_constexpr(cfg.qk_scale_ready_stages): bars.mb_qk_scale_ready[stage].init() for stage in cutlass.range_constexpr(cfg.smem_decay_stages): bars.mb_k_decay_inv_cg0_ready[stage].init() elif warp_idx == cfg.epilogue_warp_id: - if nvvm.elect_sync(): + if elect_one: for stage in cutlass.range_constexpr(cfg.sched_stages): bars.mb_sched_ready[stage].init() bars.mb_sched_done[stage].init() @@ -1819,36 +1822,12 @@ def kernel( for stage in cutlass.range_constexpr(cfg.smem_checkpoint_stages): bars.mb_checkpoint_tmastg_ready[stage].init() bars.mb_checkpoint_tmastg_done[stage].init() - bars.mb_state_read_done.init() + bars.mb_state_acc_read_done.init() diag_zero = cfg.io_dtype(0.0) for diag_idx in cutlass.range(tidx, cfg.state_scale_diag_cosize, cfg.threads_per_cta, unroll=1): sState_scale_diag_raw[diag_idx] = diag_zero nvvm.fence_mbarrier_init() nvvm.barrier_cta_sync(0, thread_count=cfg.threads_per_cta) - if (warp_idx >= cfg.compute_group_1_warp_ids[0] and warp_idx <= cfg.compute_group_1_warp_ids[-1]) or warp_idx == cfg.tcgen05_mma_warp_id: - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_alloc(sTmem_base, cutlass.Int32(512), group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - if warp_idx == cfg.tcgen05_mma_warp_id: - nvvm.tcgen05_relinquish_alloc_permit(group=nvvm.CTAGroup.CTA_1) - nvvm.barrier_cta_sync(cfg.tmem_lifecycle_barrier_id, thread_count=cfg.tmem_user_threads) - - # Actual SMEM/TMEM buffers for the BT=16 schedule: - # K/V : 16 x 128 each - # Gate_log2 : 16 x 128 - # Beta : 16 - # K inverse norm : 16, staged once per decay stage - # exp_g_last : 128, CG0-local and staged once per decay stage - # state-scale diag : 8 x 16 x 16 input dtype, zeroed once in the prologue - # K_decay : tcgen05 SW128 operand shared with super-MMA - # K_restore : tcgen05 SW128 N-major final-state operand - # K_inv : 16 x 128 token-major for super-MMA RHS - # A inverse : 16 x 16, plus transposed tcgen05 operands - # State : external/kernel ABI is VK `[DV, DK]`; reference - # math can view it as KV `[DK, DV]` by transposing. - # The TS A-staging path keeps VK in TMEM so State*K - # is `[DV, DK] @ [DK, BT] -> [DV, BT]` with M=128. - if warp_idx == cfg.tma_warp_id: tmaldg_warp( cfg, @@ -2187,13 +2166,14 @@ def build_descs( tma_granu_elems = 128 // bpe seqlen = k.shape[0] - def headed(t, dim, hn): - return cute.make_tensor(t.iterator, cute.make_layout((dim, hn, seqlen), stride=(1, t.stride[1], t.stride[0]))) + k_headed = cute.make_tensor(k.iterator, cute.make_layout((d_k, h_k, seqlen), stride=(1, k.stride[1], k.stride[0]))) + v_headed = cute.make_tensor(v.iterator, cute.make_layout((d_v, h_v, seqlen), stride=(1, v.stride[1], v.stride[0]))) + gate_headed = cute.make_tensor(gate.iterator, cute.make_layout((d_k, ho, seqlen), stride=(1, gate.stride[1], gate.stride[0]))) swz = cuda.TensorMapSwizzle.s128b - base_k = cuda.create_tensor_map_tiled_from_view(headed(k, d_k, h_k), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_v = cuda.create_tensor_map_tiled_from_view(headed(v, d_v, h_v), box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) - base_gate = cuda.create_tensor_map_tiled_from_view(headed(gate, d_k, ho), box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_k = cuda.create_tensor_map_tiled_from_view(k_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_v = cuda.create_tensor_map_tiled_from_view(v_headed, box_dims=(tma_granu_elems, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) + base_gate = cuda.create_tensor_map_tiled_from_view(gate_headed, box_dims=(32, 1, b_t), stride_order=(0, 1, 2), swizzle=swz) base_checkpoint = base_gate if cutlass.const_expr(state_checkpoints is not None): diff --git a/python/cudnn/linear_attention/ops/gdn.py b/python/cudnn/linear_attention/ops/gdn.py index 37f96b995..11bfa2e2d 100644 --- a/python/cudnn/linear_attention/ops/gdn.py +++ b/python/cudnn/linear_attention/ops/gdn.py @@ -58,6 +58,20 @@ # --------------------------------------------------------------------------- +def select_plan(graph, plan_name): + """Pin one execution plan by name on a freshly built graph (create the + plans, select by name, check support); ``None`` keeps default routing.""" + if plan_name is None: + return + graph.create_execution_plans() + names = [graph.get_plan_name_at_index(i) for i in range(len(graph.plans))] + matches = [i for i, n in enumerate(names) if n == plan_name or n.startswith(plan_name + "[")] + if not matches: + raise cudnn.cudnnGraphNotSupportedError(f"no {plan_name} plan for this graph (offered: {names})") + graph.select_plan(matches[0]) + graph.check_support() + + def _graph_workspace(graph, device): """Caller-side workspace for a compiled graph (grow-only, held on the graph object itself — same lifetime by construction, no id()-keyed @@ -102,7 +116,26 @@ def _check_dtype(name, t, want) -> None: def _make_fprop_cache_key( - total, N, H, HK, HV, K, V, io_dtype, k_dtype, v_dtype, k_shape, v_shape, cu_dtype, scale, output_final_state, use_qk_l2norm, has_initial_state, ckpt, device + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + k_shape, + v_shape, + cu_dtype, + scale, + output_final_state, + use_qk_l2norm, + has_initial_state, + ckpt, + device, + plan_name, ): return ( "fprop", @@ -125,6 +158,7 @@ def _make_fprop_cache_key( bool(has_initial_state), ckpt, device, + plan_name, ) @@ -148,6 +182,7 @@ def _make_bprop_cache_key( scale, use_qk_l2norm, device, + plan_name, ): return ( "bprop", @@ -170,6 +205,7 @@ def _make_bprop_cache_key( float(scale), bool(use_qk_l2norm), device, + plan_name, ) @@ -225,6 +261,7 @@ def _gdn_fwd( output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, checkpoint_every_n_tokens: int = 0, + plan_name: Optional[str] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """GDN forward (internal): a cached single-node GDN pygraph, THD layout. @@ -276,6 +313,7 @@ def _gdn_fwd( state0 is not None, ckpt, device, + plan_name, ) if cache_key not in _fprop_cache: _fprop_cache[cache_key] = _build_fprop_graph( @@ -296,6 +334,8 @@ def _gdn_fwd( bool(use_qk_l2norm_in_kernel), ckpt, ) + select_plan(_fprop_cache[cache_key][0], plan_name) + graph, t = _fprop_cache[cache_key] HO = max(H, HV) @@ -326,7 +366,18 @@ def _gdn_fwd( @_gdn_fwd.register_fake def _gdn_fwd_fake( - q, k, v, g, beta, cu_seqlens, scale, initial_state=None, output_final_state=False, use_qk_l2norm_in_kernel=False, checkpoint_every_n_tokens=0 + q, + k, + v, + g, + beta, + cu_seqlens, + scale, + initial_state=None, + output_final_state=False, + use_qk_l2norm_in_kernel=False, + checkpoint_every_n_tokens=0, + plan_name: Optional[str] = None, ): total, H, K = q.shape HK = k.shape[1] @@ -417,6 +468,7 @@ def _gdn_bwd( initial_state: Optional[torch.Tensor] = None, d_final_state: Optional[torch.Tensor] = None, use_qk_l2norm_in_kernel: bool = False, + plan_name: Optional[str] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """GDN backward (internal): a cached single-node GDN_BWD pygraph, THD layout. @@ -475,6 +527,7 @@ def _gdn_bwd( scale, use_qk_l2norm_in_kernel, device, + plan_name, ) if cache_key not in _bprop_cache: _bprop_cache[cache_key] = _build_bprop_graph( @@ -494,6 +547,8 @@ def _gdn_bwd( float(scale), bool(use_qk_l2norm_in_kernel), ) + select_plan(_bprop_cache[cache_key][0], plan_name) + graph, t = _bprop_cache[cache_key] HO = max(H, HV) @@ -530,7 +585,7 @@ def _gdn_bwd( @_gdn_bwd.register_fake -def _gdn_bwd_fake(dO, q, k, v, g, beta, cu_seqlens, scale, initial_state=None, d_final_state=None, use_qk_l2norm_in_kernel=False): +def _gdn_bwd_fake(dO, q, k, v, g, beta, cu_seqlens, scale, initial_state=None, d_final_state=None, use_qk_l2norm_in_kernel=False, plan_name=None): dstate0 = torch.empty_like(initial_state) if initial_state is not None else q.new_empty(0, dtype=torch.float32) return ( torch.empty_like(q), @@ -548,12 +603,13 @@ def _gdn_bwd_fake(dO, q, k, v, g, beta, cu_seqlens, scale, initial_state=None, d def _gdn_setup_context(ctx, inputs, output): - q, k, v, g, beta, cu_seqlens, scale, initial_state, output_final_state, use_qk_l2norm_in_kernel, checkpoint_every_n_tokens = inputs + q, k, v, g, beta, cu_seqlens, scale, initial_state, output_final_state, use_qk_l2norm_in_kernel, checkpoint_every_n_tokens, plan_name = inputs # save_for_backward cannot hold None; keep initial_state as an attribute. ctx.save_for_backward(q, k, v, g, beta, cu_seqlens) ctx.initial_state = initial_state ctx.scale = scale ctx.use_qk_l2norm_in_kernel = use_qk_l2norm_in_kernel + ctx.plan_name = plan_name ctx.mark_non_differentiable(output[2]) @@ -574,9 +630,10 @@ def _gdn_backward(ctx, dO, dFinal, _dstate_checkpoints): initial_state=initial_state, d_final_state=dstate_in, use_qk_l2norm_in_kernel=ctx.use_qk_l2norm_in_kernel, + plan_name=ctx.plan_name, ) # q, k, v, g, beta, cu_seqlens, scale, initial_state, output_final_state, - # use_qk_l2norm_in_kernel, checkpoint_every_n_tokens + # use_qk_l2norm_in_kernel, checkpoint_every_n_tokens, plan_name return ( dq, dk, @@ -589,6 +646,7 @@ def _gdn_backward(ctx, dO, dFinal, _dstate_checkpoints): None, None, None, + None, ) @@ -616,6 +674,7 @@ def gated_delta_net( output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, checkpoint_every_n_tokens: int = 0, + plan_name: Optional[str] = None, ): """Gated DeltaNet (GDN) linear attention. @@ -650,6 +709,9 @@ def gated_delta_net( FROST engine requires the kernel chunk size, 64). The series is a non-differentiable dump. + plan_name: optionally pin one execution plan by name (the plan + API's ``get_plan_name_at_index`` names, e.g. ``gdn_frost``); a + graph offering no such plan raises ``cudnnGraphNotSupportedError``. Returns: ``(o, final_state)`` with ``o`` shaped like ``v``, or ``(o, final_state, state_checkpoints)`` when ``checkpoint_every_n_tokens > 0``. @@ -671,6 +733,7 @@ def gated_delta_net( output_final_state=bool(output_final_state), use_qk_l2norm_in_kernel=bool(use_qk_l2norm_in_kernel), checkpoint_every_n_tokens=int(checkpoint_every_n_tokens), + plan_name=plan_name, ) if checkpoint_every_n_tokens > 0: return o, final_state, state_checkpoints diff --git a/python/cudnn/linear_attention/ops/gdn2.py b/python/cudnn/linear_attention/ops/gdn2.py index 540f8573b..d64f2778c 100644 --- a/python/cudnn/linear_attention/ops/gdn2.py +++ b/python/cudnn/linear_attention/ops/gdn2.py @@ -59,6 +59,20 @@ # --------------------------------------------------------------------------- +def select_plan(graph, plan_name): + """Pin one execution plan by name on a freshly built graph (create the + plans, select by name, check support); ``None`` keeps default routing.""" + if plan_name is None: + return + graph.create_execution_plans() + names = [graph.get_plan_name_at_index(i) for i in range(len(graph.plans))] + matches = [i for i, n in enumerate(names) if n == plan_name or n.startswith(plan_name + "[")] + if not matches: + raise cudnn.cudnnGraphNotSupportedError(f"no {plan_name} plan for this graph (offered: {names})") + graph.select_plan(matches[0]) + graph.check_support() + + def _graph_workspace(graph, device): """Caller-side workspace for a compiled graph.""" if not graph._is_built: @@ -122,6 +136,7 @@ def _make_fprop_cache_key( has_initial_state, ckpt, device, + plan_name, ): return ( "fprop", @@ -146,11 +161,31 @@ def _make_fprop_cache_key( bool(has_initial_state), ckpt, device, + plan_name, ) def _make_bprop_cache_key( - total, N, H, HK, HV, K, V, io_dtype, k_dtype, v_dtype, do_dtype, k_shape, v_shape, cu_dtype, state_dtype, dstate_in_dtype, scale, use_qk_l2norm, device + total, + N, + H, + HK, + HV, + K, + V, + io_dtype, + k_dtype, + v_dtype, + do_dtype, + k_shape, + v_shape, + cu_dtype, + state_dtype, + dstate_in_dtype, + scale, + use_qk_l2norm, + device, + plan_name, ): return ( "bprop", @@ -173,6 +208,7 @@ def _make_bprop_cache_key( float(scale), bool(use_qk_l2norm), device, + plan_name, ) @@ -260,6 +296,7 @@ def _gdn2_fwd( a_log: Optional[torch.Tensor] = None, dt_bias: Optional[torch.Tensor] = None, checkpoint_every_n_tokens: int = 0, + plan_name: Optional[str] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """GDN-2 forward (internal): a cached single-node GDN2 pygraph, THD layout. @@ -333,6 +370,7 @@ def _gdn2_fwd( state0 is not None, ckpt, device, + plan_name, ) if cache_key not in _fprop_cache: _fprop_cache[cache_key] = _build_fprop_graph( @@ -355,6 +393,8 @@ def _gdn2_fwd( float(gate_lower_bound) if gate_lower_bound is not None else None, ckpt, ) + select_plan(_fprop_cache[cache_key][0], plan_name) + graph, t = _fprop_cache[cache_key] o = torch.empty(total, HO, V, dtype=q.dtype, device=device) @@ -404,6 +444,7 @@ def _gdn2_fwd_fake( a_log=None, dt_bias=None, checkpoint_every_n_tokens=0, + plan_name: Optional[str] = None, ): total, H, K = q.shape HK = k.shape[1] @@ -499,6 +540,7 @@ def _gdn2_bwd( initial_state: Optional[torch.Tensor] = None, d_final_state: Optional[torch.Tensor] = None, use_qk_l2norm_in_kernel: bool = False, + plan_name: Optional[str] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """GDN-2 backward (internal): a cached single-node GDN2_BWD pygraph, THD layout. @@ -558,6 +600,7 @@ def _gdn2_bwd( scale, use_qk_l2norm_in_kernel, device, + plan_name, ) if cache_key not in _bprop_cache: _bprop_cache[cache_key] = _build_bprop_graph( @@ -577,6 +620,8 @@ def _gdn2_bwd( float(scale), bool(use_qk_l2norm_in_kernel), ) + select_plan(_bprop_cache[cache_key][0], plan_name) + graph, t = _bprop_cache[cache_key] dq = torch.empty(total, H, K, dtype=q.dtype, device=device) @@ -615,7 +660,7 @@ def _gdn2_bwd( @_gdn2_bwd.register_fake -def _gdn2_bwd_fake(dO, q, k, v, g, beta, w, cu_seqlens, scale, initial_state=None, d_final_state=None, use_qk_l2norm_in_kernel=False): +def _gdn2_bwd_fake(dO, q, k, v, g, beta, w, cu_seqlens, scale, initial_state=None, d_final_state=None, use_qk_l2norm_in_kernel=False, plan_name=None): dstate0 = torch.empty_like(initial_state) if initial_state is not None else q.new_empty(0, dtype=torch.float32) return ( torch.empty_like(q), @@ -651,12 +696,14 @@ def _gdn2_setup_context(ctx, inputs, output): a_log, dt_bias, checkpoint_every_n_tokens, + plan_name, ) = inputs # save_for_backward cannot hold None; keep initial_state as an attribute. ctx.save_for_backward(q, k, v, g, beta, w, cu_seqlens) ctx.initial_state = initial_state ctx.scale = scale ctx.use_qk_l2norm_in_kernel = use_qk_l2norm_in_kernel + ctx.plan_name = plan_name ctx.safe_gate = bool(safe_gate) ctx.mark_non_differentiable(output[2]) @@ -681,10 +728,11 @@ def _gdn2_backward(ctx, dO, dFinal, _dstate_checkpoints): initial_state=initial_state, d_final_state=dstate_in, use_qk_l2norm_in_kernel=ctx.use_qk_l2norm_in_kernel, + plan_name=ctx.plan_name, ) # q, k, v, g, beta, w, cu_seqlens, scale, initial_state, # output_final_state, use_qk_l2norm_in_kernel, safe_gate, - # gate_lower_bound, a_log, dt_bias, checkpoint_every_n_tokens + # gate_lower_bound, a_log, dt_bias, checkpoint_every_n_tokens, plan_name return ( dq, dk, @@ -702,6 +750,7 @@ def _gdn2_backward(ctx, dO, dFinal, _dstate_checkpoints): None, None, None, + None, ) @@ -734,6 +783,7 @@ def gated_delta_net_v2( a_log: Optional[torch.Tensor] = None, dt_bias: Optional[torch.Tensor] = None, checkpoint_every_n_tokens: int = 0, + plan_name: Optional[str] = None, ): """Gated DeltaNet v2 (GDN-2) linear attention. @@ -774,6 +824,9 @@ def gated_delta_net_v2( FROST engine requires the kernel chunk size, 16). The series is a non-differentiable dump. + plan_name: optionally pin one execution plan by name (the plan + API's ``get_plan_name_at_index`` names, e.g. ``gdn_frost``); a + graph offering no such plan raises ``cudnnGraphNotSupportedError``. Returns: ``(o, final_state)`` with ``o`` shaped like ``v``, or ``(o, final_state, state_checkpoints)`` when ``checkpoint_every_n_tokens > 0``. @@ -800,6 +853,7 @@ def gated_delta_net_v2( a_log=a_log, dt_bias=dt_bias, checkpoint_every_n_tokens=int(checkpoint_every_n_tokens), + plan_name=plan_name, ) if checkpoint_every_n_tokens > 0: return o, final_state, state_checkpoints diff --git a/python/cudnn/linear_attention/ops/kda.py b/python/cudnn/linear_attention/ops/kda.py index 7c39add0c..e38e82ee4 100644 --- a/python/cudnn/linear_attention/ops/kda.py +++ b/python/cudnn/linear_attention/ops/kda.py @@ -62,6 +62,20 @@ # --------------------------------------------------------------------------- +def select_plan(graph, plan_name): + """Pin one execution plan by name on a freshly built graph (create the + plans, select by name, check support); ``None`` keeps default routing.""" + if plan_name is None: + return + graph.create_execution_plans() + names = [graph.get_plan_name_at_index(i) for i in range(len(graph.plans))] + matches = [i for i, n in enumerate(names) if n == plan_name or n.startswith(plan_name + "[")] + if not matches: + raise cudnn.cudnnGraphNotSupportedError(f"no {plan_name} plan for this graph (offered: {names})") + graph.select_plan(matches[0]) + graph.check_support() + + def _graph_workspace(graph, device): """Caller-side workspace for a compiled graph.""" if not graph._is_built: @@ -126,6 +140,7 @@ def _make_fprop_cache_key( has_initial_state, ckpt, device, + plan_name, ): return ( "fprop", @@ -151,6 +166,7 @@ def _make_fprop_cache_key( bool(has_initial_state), ckpt, device, + plan_name, ) @@ -176,6 +192,7 @@ def _make_bprop_cache_key( scale, use_qk_l2norm, device, + plan_name, ): return ( "bprop", @@ -200,6 +217,7 @@ def _make_bprop_cache_key( float(scale), bool(use_qk_l2norm), device, + plan_name, ) @@ -303,6 +321,7 @@ def _kda_fwd( a_log: Optional[torch.Tensor] = None, dt_bias: Optional[torch.Tensor] = None, checkpoint_every_n_tokens: int = 0, + plan_name: Optional[str] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """KDA forward (internal): a cached single-node KDA pygraph, THD layout. @@ -375,6 +394,7 @@ def _kda_fwd( state0 is not None, ckpt, device, + plan_name, ) if cache_key not in _fprop_cache: _fprop_cache[cache_key] = _build_fprop_graph( @@ -398,6 +418,8 @@ def _kda_fwd( float(gate_lower_bound) if gate_lower_bound is not None else None, ckpt, ) + select_plan(_fprop_cache[cache_key][0], plan_name) + graph, t = _fprop_cache[cache_key] o = torch.empty(total, HO, V, dtype=q.dtype, device=device) @@ -446,6 +468,7 @@ def _kda_fwd_fake( a_log=None, dt_bias=None, checkpoint_every_n_tokens=0, + plan_name: Optional[str] = None, ): total, H, K = q.shape HK = k.shape[1] @@ -536,6 +559,7 @@ def _kda_bwd( initial_state: Optional[torch.Tensor] = None, d_final_state: Optional[torch.Tensor] = None, use_qk_l2norm_in_kernel: bool = False, + plan_name: Optional[str] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """KDA backward (internal): a cached single-node KDA_BWD pygraph, THD layout. @@ -595,6 +619,7 @@ def _kda_bwd( scale, use_qk_l2norm_in_kernel, device, + plan_name, ) if cache_key not in _bprop_cache: _bprop_cache[cache_key] = _build_bprop_graph( @@ -614,6 +639,8 @@ def _kda_bwd( float(scale), bool(use_qk_l2norm_in_kernel), ) + select_plan(_bprop_cache[cache_key][0], plan_name) + graph, t = _bprop_cache[cache_key] dq = torch.empty(total, H, K, dtype=q.dtype, device=device) @@ -649,7 +676,7 @@ def _kda_bwd( @_kda_bwd.register_fake -def _kda_bwd_fake(dO, q, k, v, g, beta, cu_seqlens, scale, initial_state=None, d_final_state=None, use_qk_l2norm_in_kernel=False): +def _kda_bwd_fake(dO, q, k, v, g, beta, cu_seqlens, scale, initial_state=None, d_final_state=None, use_qk_l2norm_in_kernel=False, plan_name=None): dstate0 = torch.empty_like(initial_state) if initial_state is not None else q.new_empty(0, dtype=torch.float32) return ( torch.empty_like(q), @@ -684,12 +711,14 @@ def _kda_setup_context(ctx, inputs, output): a_log, dt_bias, checkpoint_every_n_tokens, + plan_name, ) = inputs # save_for_backward cannot hold None; keep initial_state as an attribute. ctx.save_for_backward(q, k, v, g, beta, cu_seqlens) ctx.initial_state = initial_state ctx.scale = scale ctx.use_qk_l2norm_in_kernel = use_qk_l2norm_in_kernel + ctx.plan_name = plan_name ctx.use_beta_sigmoid_in_kernel = bool(use_beta_sigmoid_in_kernel) ctx.safe_gate = bool(safe_gate) ctx.mark_non_differentiable(output[2]) @@ -714,10 +743,11 @@ def _kda_backward(ctx, dO, dFinal, _dstate_checkpoints): initial_state=initial_state, d_final_state=dstate_in, use_qk_l2norm_in_kernel=ctx.use_qk_l2norm_in_kernel, + plan_name=ctx.plan_name, ) # q, k, v, g, beta, cu_seqlens, scale, initial_state, output_final_state, # use_qk_l2norm_in_kernel, use_beta_sigmoid_in_kernel, safe_gate, - # gate_lower_bound, a_log, dt_bias, checkpoint_every_n_tokens + # gate_lower_bound, a_log, dt_bias, checkpoint_every_n_tokens, plan_name return ( dq, dk, @@ -735,6 +765,7 @@ def _kda_backward(ctx, dO, dFinal, _dstate_checkpoints): None, None, None, + None, ) @@ -767,6 +798,7 @@ def kimi_delta_attention( a_log: Optional[torch.Tensor] = None, dt_bias: Optional[torch.Tensor] = None, checkpoint_every_n_tokens: int = 0, + plan_name: Optional[str] = None, ): """Kimi Delta Attention (KDA) linear attention. @@ -813,6 +845,9 @@ def kimi_delta_attention( FROST engine requires the kernel chunk size, 16). The series is a non-differentiable dump. + plan_name: optionally pin one execution plan by name (the plan + API's ``get_plan_name_at_index`` names, e.g. ``gdn_frost``); a + graph offering no such plan raises ``cudnnGraphNotSupportedError``. Returns: ``(o, final_state)`` with ``o`` shaped like ``v``, or ``(o, final_state, state_checkpoints)`` when ``checkpoint_every_n_tokens > 0``. @@ -839,6 +874,7 @@ def kimi_delta_attention( a_log=a_log, dt_bias=dt_bias, checkpoint_every_n_tokens=int(checkpoint_every_n_tokens), + plan_name=plan_name, ) if checkpoint_every_n_tokens > 0: return o, final_state, state_checkpoints diff --git a/test/python/linear_attention/reference_gdn.py b/test/python/linear_attention/reference_gdn.py index d88dd93e5..4df5f1120 100644 --- a/test/python/linear_attention/reference_gdn.py +++ b/test/python/linear_attention/reference_gdn.py @@ -33,7 +33,7 @@ def rms_ratio(out: torch.Tensor, ref: torch.Tensor) -> float: return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() -def _recurrent_dense(q, k, v, alpha, beta, state0): +def recurrent_dense(q, k, v, alpha, beta, state0): """Dense recurrence in [B, HV, T, *] layout, fp64. Returns (o, final state).""" T = q.shape[2] state = state0 @@ -80,15 +80,22 @@ def gdn_reference( HO = max(q.shape[2], v.shape[2]) - def expand(x): - r = HO // x.shape[2] - return x.repeat_interleave(r, dim=2) if r > 1 else x - - qf = expand(q.double() * scale) - kf = expand(k.double()) - vf = expand(v.double()) - alphaf = expand(g.double().exp()) - betaf = expand(beta.double()) + qf = q.double() * scale + kf = k.double() + vf = v.double() + alphaf = g.double().exp() + betaf = beta.double() + # expand tensors for grouped heads (view, no copy), as in the sdpa references + if q.shape[2] != HO: + qf = qf.unsqueeze(3).expand(-1, -1, -1, HO // q.shape[2], -1).reshape(q.shape[0], q.shape[1], HO, -1) + if k.shape[2] != HO: + kf = kf.unsqueeze(3).expand(-1, -1, -1, HO // k.shape[2], -1).reshape(k.shape[0], k.shape[1], HO, -1) + if v.shape[2] != HO: + vf = vf.unsqueeze(3).expand(-1, -1, -1, HO // v.shape[2], -1).reshape(v.shape[0], v.shape[1], HO, -1) + if g.shape[2] != HO: + alphaf = alphaf.unsqueeze(3).expand(-1, -1, -1, HO // g.shape[2]).reshape(g.shape[0], g.shape[1], HO) + if beta.shape[2] != HO: + betaf = betaf.unsqueeze(3).expand(-1, -1, -1, HO // beta.shape[2]).reshape(beta.shape[0], beta.shape[1], HO) HV = HO # [B, T, HV, *] -> [B, HV, T, *] @@ -106,7 +113,7 @@ def expand(x): state0 = torch.zeros(B, HV, K, V, dtype=torch.float64, device=q.device) else: state0 = initial_state.double() - o, state = _recurrent_dense(qf, kf, vf, alphaf, betaf, state0) + o, state = recurrent_dense(qf, kf, vf, alphaf, betaf, state0) return o.permute(0, 2, 1, 3), state assert q.shape[0] == 1, "cu_seqlens requires packed batch B == 1" @@ -121,7 +128,7 @@ def expand(x): if e == s: states.append(state0) continue - o_n, state_n = _recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], state0) + o_n, state_n = recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], state0) outs.append(o_n) states.append(state_n) if outs: diff --git a/test/python/linear_attention/reference_gdn2.py b/test/python/linear_attention/reference_gdn2.py index 9b35ac573..777eb80e9 100644 --- a/test/python/linear_attention/reference_gdn2.py +++ b/test/python/linear_attention/reference_gdn2.py @@ -46,7 +46,7 @@ def rms_ratio(out: torch.Tensor, ref: torch.Tensor) -> float: return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() -def _recurrent_dense(q, k, v, alpha, beta, w, state0): +def recurrent_dense(q, k, v, alpha, beta, w, state0): """Dense recurrence in [B, HV, T, *] layout, fp64. Returns (o, final state). alpha, beta: per-key-channel [B, HV, T, K]; w: per-value-channel [B, HV, T, V].""" @@ -101,16 +101,25 @@ def gdn2_reference( HO = max(q.shape[2], v.shape[2]) - def expand(x): - r = HO // x.shape[2] - return x.repeat_interleave(r, dim=2) if r > 1 else x - - qf = expand(q.double() * scale) - kf = expand(k.double()) - vf = expand(v.double()) - alphaf = expand(g.double().exp()) # [B, T, HO, K] - betaf = expand(beta.double()) # [B, T, HO, K] - wf = expand(w.double()) # [B, T, HO, V] + qf = q.double() * scale + kf = k.double() + vf = v.double() + alphaf = g.double().exp() # [B, T, HO, K] + betaf = beta.double() # [B, T, HO, K] + wf = w.double() # [B, T, HO, V] + # expand tensors for grouped heads (view, no copy), as in the sdpa references + if q.shape[2] != HO: + qf = qf.unsqueeze(3).expand(-1, -1, -1, HO // q.shape[2], -1).reshape(q.shape[0], q.shape[1], HO, -1) + if k.shape[2] != HO: + kf = kf.unsqueeze(3).expand(-1, -1, -1, HO // k.shape[2], -1).reshape(k.shape[0], k.shape[1], HO, -1) + if v.shape[2] != HO: + vf = vf.unsqueeze(3).expand(-1, -1, -1, HO // v.shape[2], -1).reshape(v.shape[0], v.shape[1], HO, -1) + if g.shape[2] != HO: + alphaf = alphaf.unsqueeze(3).expand(-1, -1, -1, HO // g.shape[2], -1).reshape(g.shape[0], g.shape[1], HO, -1) + if beta.shape[2] != HO: + betaf = betaf.unsqueeze(3).expand(-1, -1, -1, HO // beta.shape[2], -1).reshape(beta.shape[0], beta.shape[1], HO, -1) + if w.shape[2] != HO: + wf = wf.unsqueeze(3).expand(-1, -1, -1, HO // w.shape[2], -1).reshape(w.shape[0], w.shape[1], HO, -1) # [B, T, HO, *] -> [B, HO, T, *] qf = qf.permute(0, 2, 1, 3) @@ -129,7 +138,7 @@ def expand(x): state0 = torch.zeros(B, HV, K, V, dtype=torch.float64, device=q.device) else: state0 = initial_state.double() - o, state = _recurrent_dense(qf, kf, vf, alphaf, betaf, wf, state0) + o, state = recurrent_dense(qf, kf, vf, alphaf, betaf, wf, state0) return o.permute(0, 2, 1, 3), state assert q.shape[0] == 1, "cu_seqlens requires packed batch B == 1" @@ -144,7 +153,7 @@ def expand(x): if e == s: states.append(state0) continue - o_n, state_n = _recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], wf[:, :, s:e], state0) + o_n, state_n = recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], wf[:, :, s:e], state0) outs.append(o_n) states.append(state_n) if outs: diff --git a/test/python/linear_attention/reference_kda.py b/test/python/linear_attention/reference_kda.py index 86508c637..b2468864d 100644 --- a/test/python/linear_attention/reference_kda.py +++ b/test/python/linear_attention/reference_kda.py @@ -38,7 +38,7 @@ def rms_ratio(out: torch.Tensor, ref: torch.Tensor) -> float: return ((out - ref).pow(2).mean().sqrt() / ref.pow(2).mean().sqrt().clamp_min(1e-12)).item() -def _recurrent_dense(q, k, v, alpha, beta, state0): +def recurrent_dense(q, k, v, alpha, beta, state0): """Dense recurrence in [B, HV, T, *] layout, fp64. Returns (o, final state). ``alpha`` is per-key-channel: [B, HV, T, K] (GDN's is scalar [B, HV, T]).""" @@ -89,15 +89,22 @@ def kda_reference( HO = max(q.shape[2], v.shape[2]) - def expand(x): - r = HO // x.shape[2] - return x.repeat_interleave(r, dim=2) if r > 1 else x - - qf = expand(q.double() * scale) - kf = expand(k.double()) - vf = expand(v.double()) - alphaf = expand(g.double().exp()) # [B, T, HO, K] - betaf = expand(beta.double()) + qf = q.double() * scale + kf = k.double() + vf = v.double() + alphaf = g.double().exp() # [B, T, HO, K] + betaf = beta.double() + # expand tensors for grouped heads (view, no copy), as in the sdpa references + if q.shape[2] != HO: + qf = qf.unsqueeze(3).expand(-1, -1, -1, HO // q.shape[2], -1).reshape(q.shape[0], q.shape[1], HO, -1) + if k.shape[2] != HO: + kf = kf.unsqueeze(3).expand(-1, -1, -1, HO // k.shape[2], -1).reshape(k.shape[0], k.shape[1], HO, -1) + if v.shape[2] != HO: + vf = vf.unsqueeze(3).expand(-1, -1, -1, HO // v.shape[2], -1).reshape(v.shape[0], v.shape[1], HO, -1) + if g.shape[2] != HO: + alphaf = alphaf.unsqueeze(3).expand(-1, -1, -1, HO // g.shape[2], -1).reshape(g.shape[0], g.shape[1], HO, -1) + if beta.shape[2] != HO: + betaf = betaf.unsqueeze(3).expand(-1, -1, -1, HO // beta.shape[2], -1).reshape(beta.shape[0], beta.shape[1], HO, -1) # [B, T, HO, *] -> [B, HO, T, *] qf = qf.permute(0, 2, 1, 3) @@ -115,7 +122,7 @@ def expand(x): state0 = torch.zeros(B, HV, K, V, dtype=torch.float64, device=q.device) else: state0 = initial_state.double() - o, state = _recurrent_dense(qf, kf, vf, alphaf, betaf, state0) + o, state = recurrent_dense(qf, kf, vf, alphaf, betaf, state0) return o.permute(0, 2, 1, 3), state assert q.shape[0] == 1, "cu_seqlens requires packed batch B == 1" @@ -130,7 +137,7 @@ def expand(x): if e == s: states.append(state0) continue - o_n, state_n = _recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], state0) + o_n, state_n = recurrent_dense(qf[:, :, s:e], kf[:, :, s:e], vf[:, :, s:e], alphaf[:, :, s:e], betaf[:, :, s:e], state0) outs.append(o_n) states.append(state_n) if outs: diff --git a/test/python/linear_attention/test_la.py b/test/python/linear_attention/test_la.py index e1bd782d4..1468cd321 100644 --- a/test/python/linear_attention/test_la.py +++ b/test/python/linear_attention/test_la.py @@ -5,8 +5,8 @@ One suite for the public API (``gated_delta_net`` / ``kimi_delta_attention`` / ``gated_delta_net_v2``), parametrized over the python engine backends that can -serve it. Each test pins one backend's engines onto the ops and validates -against the fp64 recurrent references. A pinned engine that declines a +serve it. Each test pins one backend's plan onto the ops' graphs (plan-API +``select_plan`` by name) and validates against the fp64 recurrent references. A pinned engine that declines a configuration waives the test (``cudnnGraphNotSupportedError`` -> skip), so the support surface is owned by the engines' ``check_support``, not by a suite-side matrix; a backend that is not installed skips the same way. @@ -18,10 +18,8 @@ from __future__ import annotations import contextlib +import functools import math -import os -from types import SimpleNamespace - import pytest torch = pytest.importorskip("torch") @@ -51,9 +49,7 @@ # (H, HV) pairs: H = Q/K heads, HV = V heads; gates/O/states live at HO = max. HEAD_CONFIGS = [(1, 1), (3, 3), (1, 2), (2, 4), (16, 32), (16, 64)] HEAD_CONFIGS_SMALL = [(1, 1), (2, 4)] -# (H, HK, HV) triples; HK == HV < H is canonical GQA (FlashInfer's grouped -# native K), HK == H > HV is the expanded-k form. -GQA_CONFIGS = [(4, 4, 1), (6, 6, 2), (4, 1, 1), (6, 2, 2)] +GQA_CONFIGS = [(4, 4, 1), (6, 6, 2), (4, 1, 1), (6, 2, 2), (1, 2, 2), (2, 4, 4)] RAGGED_SEQ_LENS = [ [256, 256], @@ -65,9 +61,10 @@ ] EDGE_LENS = [1, 15, 16, 17, 31, 63, 64, 65, 121, 251, 257] -DETERMINISM_REPEATS = int(os.environ.get("DETERMINISM_REPEATS", "8")) +DETERMINISM_REPEATS = 8 +SEED = 888 -_DTYPE_IDS = {torch.bfloat16: "bf16", torch.float16: "fp16"} +DTYPE_IDS = {torch.bfloat16: "bf16", torch.float16: "fp16"} # --------------------------------------------------------------------------- @@ -75,13 +72,13 @@ # --------------------------------------------------------------------------- -def _op_modules(): +def op_modules(): from cudnn.linear_attention.ops import gdn, gdn2, kda return {"gdn": gdn, "kda": kda, "gdn2": gdn2} -def _family_engines(backend_name): +def family_engines(backend_name): """The backend's engine instances per family, ids assigned by the manifest.""" from cudnn.engines import manifest @@ -94,47 +91,70 @@ def _family_engines(backend_name): return families -def _clear_op_caches(): - for mod in _op_modules().values(): +def clear_op_caches(): + for mod in op_modules().values(): mod._fprop_cache.clear() mod._bprop_cache.clear() +class Case: + """One test configuration: inputs, gates, cu_seqlens and the geometry.""" + + __slots__ = ("variant", "dtype", "q", "k", "v", "gates", "cu", "B", "T", "N", "H", "HK", "HV", "HO", "K", "V", "varlen") + + def __init__(self, **fields): + for name in self.__slots__: + setattr(self, name, fields.pop(name)) + assert not fields, f"unknown Case fields: {sorted(fields)}" + + def clone(self, **overrides): + fields = {name: getattr(self, name) for name in self.__slots__} + fields.update(overrides) + return Case(**fields) + + +class Backend: + """The pinned backend: its name, its engine instances per family, and the + plan name a graph must offer (``_``, e.g. ``gdn_frost``).""" + + __slots__ = ("name", "engines") + + def __init__(self, name, engines): + self.name = name + self.engines = engines + + def plan(self, variant): + return f"{variant}_{self.name}" + + +def pinned_op(backend, variant): + """The variant's op with the backend's plan pinned (ops-level + ``plan_name``, the examples' ``select_plan`` paradigm).""" + return functools.partial(op(variant), plan_name=backend.plan(variant)) + + @pytest.fixture(params=("frost", "cutile")) -def backend(request, monkeypatch): - """Pin one backend by scoping the manifest's candidate list for the LA - families to it (ranking consumes the candidate list, so filtering it is - the pin seam). The op graph caches are keyed without the pin, so they are - cleared around each test.""" +def backend(request): + """One backend per run of each test; the tests pass its plan name to the + ops. The op graph caches are cleared around each test so the pin + assertions only ever see this test's graphs.""" name = request.param - families = _family_engines(name) + families = family_engines(name) missing = [v for v in VARIANTS if v not in families] if missing: pytest.fail(f"the engine manifest offers no {missing} families — a stale installed cudnn package is likely shadowing the source tree") - from cudnn.engines import manifest - - real_engines_for = manifest.engines_for - suffix = "_" + name - - def pinned_engines_for(graph): - engines = real_engines_for(graph) - family = manifest.family_for(graph) - if family is not None and family.name in VARIANTS: - return [e for e in engines if e.name.endswith(suffix)] - return engines - - monkeypatch.setattr(manifest, "engines_for", pinned_engines_for) - _clear_op_caches() + clear_op_caches() try: - yield SimpleNamespace(name=name, engines=families) + yield Backend(name, families) finally: - _clear_op_caches() + clear_op_caches() @contextlib.contextmanager -def _waive_unsupported(backend, variant): - """An empty pin (op call would fall back to manifest routing) or an engine - decline waives the test; the engines own the support surface.""" +def waive_unsupported(backend, variant): + """A backend with no engine for the family, or an engine decline (no plan + offered / check_support raise), waives the test; the engines own the + support surface.""" if not backend.engines[variant]: pytest.skip(f"the {backend.name} backend has no {variant} engine") try: @@ -148,18 +168,18 @@ def _waive_unsupported(backend, variant): # --------------------------------------------------------------------------- -def _seed(seed=0): +def set_seed(seed=SEED): torch.random.manual_seed(seed) torch.cuda.manual_seed(seed) -def _gate_lo(dtype): +def gate_lo(dtype): return 0.6 if dtype == torch.float16 else 0.5 -def _gen_gates(variant, B, T, HO, K, V, dtype, *, alpha=True, beta=True, w=True, lo=None, device="cuda"): +def gen_gates(variant, B, T, HO, K, V, dtype, *, alpha=True, beta=True, w=True, lo=None, device="cuda"): if lo is None: - lo = _gate_lo(dtype) + lo = gate_lo(dtype) gshape = (B, T, HO) if variant == "gdn" else (B, T, HO, K) if alpha: g = torch.empty(gshape, device=device, dtype=torch.float32).uniform_(lo, 1.0).log() @@ -173,11 +193,11 @@ def _gen_gates(variant, B, T, HO, K, V, dtype, *, alpha=True, beta=True, w=True, return {"g": g, "beta": b} -def _case(variant, dtype, *, B=1, T=None, seq_lens=None, H=2, HK=None, HV=None, K=128, V=128, alpha=True, beta=True, w=True, lo=None, seed=0): +def make_case(variant, dtype, *, B=1, T=None, seq_lens=None, H=2, HK=None, HV=None, K=128, V=128, alpha=True, beta=True, w=True, lo=None, seed=SEED): """Dense ``(B, T)`` or packed varlen (``seq_lens``, B == 1) inputs plus the matching ``cu_seqlens``. ``HK`` defaults to ``H``; ``HK == HV < H`` is canonical (native grouped K) GQA.""" - _seed(seed) + set_seed(seed) HV = H if HV is None else HV HK = H if HK is None else HK HO = max(H, HV) @@ -196,32 +216,32 @@ def _case(variant, dtype, *, B=1, T=None, seq_lens=None, H=2, HK=None, HV=None, from .conftest import multidist_randu k = F.normalize(multidist_randu(B * T * HK, K, device="cuda").reshape(B, T, HK, K), p=2.0, dim=-1).to(dtype).contiguous() - gates = _gen_gates(variant, B, T, HO, K, V, dtype, alpha=alpha, beta=beta, w=w, lo=lo) - return SimpleNamespace(variant=variant, dtype=dtype, q=q, k=k, v=v, gates=gates, cu=cu, B=B, T=T, N=N, H=H, HK=HK, HV=HV, HO=HO, K=K, V=V, varlen=varlen) + gates = gen_gates(variant, B, T, HO, K, V, dtype, alpha=alpha, beta=beta, w=w, lo=lo) + return Case(variant=variant, dtype=dtype, q=q, k=k, v=v, gates=gates, cu=cu, B=B, T=T, N=N, H=H, HK=HK, HV=HV, HO=HO, K=K, V=V, varlen=varlen) -def _thd(x): +def to_thd(x): return x.reshape(-1, *x.shape[2:]) -def _op(variant): +def op(variant): return {"gdn": la_ops.gated_delta_net, "kda": la_ops.kimi_delta_attention, "gdn2": la_ops.gated_delta_net_v2}[variant] -def _op_args(case, cu=None): - args = [_thd(case.q), _thd(case.k), _thd(case.v), _thd(case.gates["g"]), _thd(case.gates["beta"])] +def op_args(case, cu=None): + args = [to_thd(case.q), to_thd(case.k), to_thd(case.v), to_thd(case.gates["g"]), to_thd(case.gates["beta"])] if case.variant == "gdn2": - args.append(_thd(case.gates["w"])) + args.append(to_thd(case.gates["w"])) args.append(case.cu if cu is None else cu) return args -def _run_fwd(backend, case, *, cu=None, **kw): - with _waive_unsupported(backend, case.variant): - return _op(case.variant)(*_op_args(case, cu=cu), **kw) +def run_fwd(backend, case, *, cu=None, **kw): + with waive_unsupported(backend, case.variant): + return pinned_op(backend, case.variant)(*op_args(case, cu=cu), **kw) -def _reference(case, *, scale=None, initial_state=None, l2norm=False, cu=None): +def reference(case, *, scale=None, initial_state=None, l2norm=False, cu=None): fn = {"gdn": gdn_reference, "kda": kda_reference, "gdn2": gdn2_reference}[case.variant] q, k = case.q, case.k if l2norm: @@ -237,18 +257,18 @@ def _reference(case, *, scale=None, initial_state=None, l2norm=False, cu=None): return fn(*args, **kwargs) -def _check(name, out, ref, tol): +def check(name, out, ref, tol): out = out.float() assert torch.isfinite(out).all(), f"non-finite values in {name}" r = rms_ratio(out.reshape(ref.shape), ref) assert r < tol, f"{name} rms ratio {r:.4g} >= {tol}" -def _check_fwd(case, o, fs, *, scale=None, initial_state=None, l2norm=False, tol_mult=1.0): - o_ref, fs_ref = _reference(case, scale=scale, initial_state=initial_state, l2norm=l2norm) - _check("o", o, o_ref, tol_mult * FWD_TOL[case.dtype]) +def check_fwd(case, o, fs, *, scale=None, initial_state=None, l2norm=False, tol_mult=1.0): + o_ref, fs_ref = reference(case, scale=scale, initial_state=initial_state, l2norm=l2norm) + check("o", o, o_ref, tol_mult * FWD_TOL[case.dtype]) if fs is not None and fs.numel(): - _check("final_state", fs, fs_ref, tol_mult * STATE_TOL[case.dtype]) + check("final_state", fs, fs_ref, tol_mult * STATE_TOL[case.dtype]) # --------------------------------------------------------------------------- @@ -260,11 +280,11 @@ def _check_fwd(case, o, fs, *, scale=None, initial_state=None, l2norm=False, tol def test_backend_pin_selects_engine(backend, variant): """The pinned engine actually serves the graph — a dead pin would silently validate whatever the default routing picks.""" - case = _case(variant, torch.bfloat16, T=4 * CHUNK[variant]) - with _waive_unsupported(backend, variant): - _op(variant)(*_op_args(case)) - mod = _op_modules()[variant] - names = {g.selected_engine.name for g, _t in mod._fprop_cache.values() if g.selected_engine is not None} + case = make_case(variant, torch.bfloat16, T=4 * CHUNK[variant]) + with waive_unsupported(backend, variant): + pinned_op(backend, variant)(*op_args(case)) + mod = op_modules()[variant] + names = {g.selected_engine.name for g, entry in mod._fprop_cache.values() if g.selected_engine is not None} assert names == {f"{variant}_{backend.name}"}, f"expected only {variant}_{backend.name} to serve, got {names}" @@ -275,14 +295,14 @@ def test_backend_pin_selects_engine(backend, variant): @pytest.mark.parametrize("H,HV", HEAD_CONFIGS) @pytest.mark.parametrize("B,T", [(1, 64), (1, 128), (2, 256)]) -@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16], ids=_DTYPE_IDS.get) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16], ids=DTYPE_IDS.get) @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_basic(backend, variant, dtype, B, T, H, HV): if dtype == torch.float16 and (H, HV) not in HEAD_CONFIGS_SMALL: pytest.skip("fp16 runs the small head matrix") - case = _case(variant, dtype, B=B, T=T, H=H, HV=HV) - o, fs = _run_fwd(backend, case, output_final_state=True) - _check_fwd(case, o, fs) + case = make_case(variant, dtype, B=B, T=T, H=H, HV=HV) + o, fs = run_fwd(backend, case, output_final_state=True) + check_fwd(case, o, fs) @pytest.mark.parametrize("alpha,beta,w", [(True, False, True), (False, True, True), (True, True, False)], ids=["no_beta", "no_alpha", "no_w"]) @@ -292,24 +312,24 @@ def test_fwd_gate_combinations(backend, variant, alpha, beta, w): pytest.skip("the delta-rule inverse needs decay (matches FI's use_g=False skip)") if variant != "gdn2" and not w: pytest.skip("w is a GDN-2 gate") - case = _case(variant, torch.bfloat16, T=192, alpha=alpha, beta=beta, w=w) - o, fs = _run_fwd(backend, case, output_final_state=True) - _check_fwd(case, o, fs) + case = make_case(variant, torch.bfloat16, T=192, alpha=alpha, beta=beta, w=w) + o, fs = run_fwd(backend, case, output_final_state=True) + check_fwd(case, o, fs) @pytest.mark.parametrize("scale", [0.5, 1.0, None], ids=["half", "one", "auto"]) @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_scale(backend, variant, scale): - case = _case(variant, torch.bfloat16, T=192) - o, fs = _run_fwd(backend, case, scale=scale, output_final_state=True) - _check_fwd(case, o, fs, scale=scale) + case = make_case(variant, torch.bfloat16, T=192) + o, fs = run_fwd(backend, case, scale=scale, output_final_state=True) + check_fwd(case, o, fs, scale=scale) @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_default_scale_matches_explicit(backend, variant): - case = _case(variant, torch.bfloat16, T=128) - o_default, _ = _run_fwd(backend, case) - o_explicit, _ = _run_fwd(backend, case, scale=1.0 / math.sqrt(case.K)) + case = make_case(variant, torch.bfloat16, T=128) + o_default, _ = run_fwd(backend, case) + o_explicit, _ = run_fwd(backend, case, scale=1.0 / math.sqrt(case.K)) torch.testing.assert_close(o_default, o_explicit) @@ -317,60 +337,60 @@ def test_fwd_default_scale_matches_explicit(backend, variant): @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_seqlen_edges(backend, variant, T): """Lengths straddling the kernels' chunk boundaries (16 and 64).""" - case = _case(variant, torch.bfloat16, T=T) - o, fs = _run_fwd(backend, case, output_final_state=True) - _check_fwd(case, o, fs) + case = make_case(variant, torch.bfloat16, T=T) + o, fs = run_fwd(backend, case, output_final_state=True) + check_fwd(case, o, fs) @pytest.mark.parametrize("H,HV", [(1, 1), (2, 4)]) @pytest.mark.parametrize("seq_lens", RAGGED_SEQ_LENS, ids=lambda sl: f"{len(sl)}seqs_{sum(sl)}tok") @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_varlen_ragged(backend, variant, seq_lens, H, HV): - case = _case(variant, torch.bfloat16, seq_lens=seq_lens, H=H, HV=HV) - o, fs = _run_fwd(backend, case, output_final_state=True) - _check_fwd(case, o, fs) + case = make_case(variant, torch.bfloat16, seq_lens=seq_lens, H=H, HV=HV) + o, fs = run_fwd(backend, case, output_final_state=True) + check_fwd(case, o, fs) @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_many_short_sequences(backend, variant): """A 200-sequence packed batch matches the same sequences run one by one.""" T = 33 - case = _case(variant, torch.bfloat16, seq_lens=[T] * 200) - o, fs = _run_fwd(backend, case, output_final_state=True) + case = make_case(variant, torch.bfloat16, seq_lens=[T] * 200) + o, fs = run_fwd(backend, case, output_final_state=True) cu1 = torch.tensor([0, T], dtype=torch.int32, device="cuda") for n in (0, 1, 99, 199): # clone: sliced views can start at non-16B-aligned offsets, which the # kernels' buffer contract rejects sl = slice(T * n, T * (n + 1)) args = [ - _thd(case.q)[sl].clone(), - _thd(case.k)[sl].clone(), - _thd(case.v)[sl].clone(), - _thd(case.gates["g"])[sl].clone(), - _thd(case.gates["beta"])[sl].clone(), + to_thd(case.q)[sl].clone(), + to_thd(case.k)[sl].clone(), + to_thd(case.v)[sl].clone(), + to_thd(case.gates["g"])[sl].clone(), + to_thd(case.gates["beta"])[sl].clone(), ] if case.variant == "gdn2": - args.append(_thd(case.gates["w"])[sl].clone()) - with _waive_unsupported(backend, variant): - o_n, fs_n = _op(variant)(*args, cu1, output_final_state=True) - _check(f"o[seq {n}]", o[sl], o_n.float(), FWD_TOL[torch.bfloat16]) - _check(f"final_state[seq {n}]", fs[n], fs_n[0].float(), STATE_TOL[torch.bfloat16]) + args.append(to_thd(case.gates["w"])[sl].clone()) + with waive_unsupported(backend, variant): + o_n, fs_n = pinned_op(backend, variant)(*args, cu1, output_final_state=True) + check(f"o[seq {n}]", o[sl], o_n.float(), FWD_TOL[torch.bfloat16]) + check(f"final_state[seq {n}]", fs[n], fs_n[0].float(), STATE_TOL[torch.bfloat16]) @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_zero_length_sequences(backend, variant): """Empty sequences must not perturb their neighbors; their state rows stay zero (or pass the initial state through when one is given).""" - case = _case(variant, torch.bfloat16, seq_lens=[64, 128]) - o_base, fs_base = _run_fwd(backend, case, output_final_state=True) + case = make_case(variant, torch.bfloat16, seq_lens=[64, 128]) + o_base, fs_base = run_fwd(backend, case, output_final_state=True) cu = torch.tensor([0, 64, 64, 192, 192], dtype=torch.int32, device="cuda") - o, fs = _run_fwd(backend, case, cu=cu, output_final_state=True) + o, fs = run_fwd(backend, case, cu=cu, output_final_state=True) torch.testing.assert_close(o, o_base, atol=1e-3, rtol=1e-3) torch.testing.assert_close(fs[0], fs_base[0], atol=1e-3, rtol=1e-3) torch.testing.assert_close(fs[2], fs_base[1], atol=1e-3, rtol=1e-3) assert (fs[1] == 0).all() and (fs[3] == 0).all(), "zero-length sequence states must stay zero" state0 = torch.randn(4, case.HO, case.K, case.V, device="cuda", dtype=torch.float32) * 0.05 - _o, fs_state0 = _run_fwd(backend, case, cu=cu, initial_state=state0, output_final_state=True) + o, fs_state0 = run_fwd(backend, case, cu=cu, initial_state=state0, output_final_state=True) torch.testing.assert_close(fs_state0[1], state0[1], atol=0.0, rtol=0.0) torch.testing.assert_close(fs_state0[3], state0[3], atol=0.0, rtol=0.0) @@ -378,10 +398,10 @@ def test_fwd_zero_length_sequences(backend, variant): @pytest.mark.parametrize("T", [128, 251]) @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_initial_state(backend, variant, T): - case = _case(variant, torch.bfloat16, T=T) + case = make_case(variant, torch.bfloat16, T=T) state0 = torch.randn(case.N, case.HO, case.K, case.V, device="cuda", dtype=torch.float32) * 0.05 - o, fs = _run_fwd(backend, case, initial_state=state0, output_final_state=True) - _check_fwd(case, o, fs, initial_state=state0) + o, fs = run_fwd(backend, case, initial_state=state0, output_final_state=True) + check_fwd(case, o, fs, initial_state=state0) @pytest.mark.parametrize("T1,T2", [(128, 128), (64, 192), (192, 121)]) @@ -389,36 +409,36 @@ def test_fwd_initial_state(backend, variant, T): def test_fwd_chunked_prefill(backend, variant, T1, T2): """Two-phase prefill: part 1's final state feeds part 2; the concatenated output matches a single-shot reference (state round-trips through fp32).""" - case = _case(variant, torch.bfloat16, B=2, T=T1 + T2) + case = make_case(variant, torch.bfloat16, B=2, T=T1 + T2) def part(t0, t1, state0): - sub = SimpleNamespace(**vars(case)) + sub = case.clone() sub.q, sub.k, sub.v = (x[:, t0:t1].contiguous() for x in (case.q, case.k, case.v)) sub.gates = {n: g[:, t0:t1].contiguous() for n, g in case.gates.items()} sub.T = t1 - t0 sub.cu = torch.arange(0, case.B + 1, dtype=torch.int32, device="cuda") * sub.T - return _run_fwd(backend, sub, initial_state=state0, output_final_state=True) + return run_fwd(backend, sub, initial_state=state0, output_final_state=True) o1, fs1 = part(0, T1, None) o2, fs2 = part(T1, T1 + T2, fs1) o = torch.cat([o1.reshape(case.B, T1, case.HO, case.V), o2.reshape(case.B, T2, case.HO, case.V)], dim=1) - o_ref, fs_ref = _reference(case) - _check("o", o, o_ref, 1.5 * FWD_TOL[case.dtype]) - _check("final_state", fs2, fs_ref, 1.5 * STATE_TOL[case.dtype]) + o_ref, fs_ref = reference(case) + check("o", o, o_ref, 1.5 * FWD_TOL[case.dtype]) + check("final_state", fs2, fs_ref, 1.5 * STATE_TOL[case.dtype]) @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_packed_matches_per_sequence(backend, variant): B, T = 3, 128 - case = _case(variant, torch.bfloat16, B=B, T=T) - o, fs = _run_fwd(backend, case, output_final_state=True) + case = make_case(variant, torch.bfloat16, B=B, T=T) + o, fs = run_fwd(backend, case, output_final_state=True) cu1 = torch.tensor([0, T], dtype=torch.int32, device="cuda") for b in range(B): args = [case.q[b], case.k[b], case.v[b], case.gates["g"][b], case.gates["beta"][b]] if variant == "gdn2": args.append(case.gates["w"][b]) - with _waive_unsupported(backend, variant): - o_b, fs_b = _op(variant)(*args, cu1, output_final_state=True) + with waive_unsupported(backend, variant): + o_b, fs_b = pinned_op(backend, variant)(*args, cu1, output_final_state=True) torch.testing.assert_close(o[b * T : (b + 1) * T], o_b) torch.testing.assert_close(fs[b], fs_b[0]) @@ -429,44 +449,44 @@ def test_fwd_packed_matches_per_sequence(backend, variant): ) def test_fwd_head_dims(backend, variant, K, V): """K/V head-dim variants; engines that only serve K = V = 128 decline.""" - case = _case(variant, torch.bfloat16, T=192, K=K, V=V) - o, fs = _run_fwd(backend, case, output_final_state=True) - _check_fwd(case, o, fs) + case = make_case(variant, torch.bfloat16, T=192, K=K, V=V) + o, fs = run_fwd(backend, case, output_final_state=True) + check_fwd(case, o, fs) @pytest.mark.parametrize("H,HK,HV", GQA_CONFIGS) @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_gqa(backend, variant, H, HK, HV): - """Grouped-query heads (H > HV): gates/O/states live at HO = H. Covers - both canonical GQA (native K at HK == HV) and the expanded-k form.""" - case = _case(variant, torch.bfloat16, T=192, H=H, HK=HK, HV=HV) - o, fs = _run_fwd(backend, case, output_final_state=True) - _check_fwd(case, o, fs) + """Grouped heads: canonical GQA (native K at HK == HV), the expanded-k + form, and shared-kv GVA (HK == HV > H); gates/O/states live at HO = max(H, HV).""" + case = make_case(variant, torch.bfloat16, T=192, H=H, HK=HK, HV=HV) + o, fs = run_fwd(backend, case, output_final_state=True) + check_fwd(case, o, fs) @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_qk_l2norm(backend, variant): """In-kernel Q/K L2 norm matches the reference on pre-normalized inputs.""" - case = _case(variant, torch.bfloat16, T=256) - o, fs = _run_fwd(backend, case, output_final_state=True, use_qk_l2norm_in_kernel=True) - _check_fwd(case, o, fs, l2norm=True) + case = make_case(variant, torch.bfloat16, T=256) + o, fs = run_fwd(backend, case, output_final_state=True, use_qk_l2norm_in_kernel=True) + check_fwd(case, o, fs, l2norm=True) @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_strong_decay_varlen(backend, variant): - case = _case(variant, torch.bfloat16, seq_lens=[100, 2048, 0, 517], lo=0.1 if variant == "gdn" else 0.3) - o, fs = _run_fwd(backend, case, output_final_state=True) - _check_fwd(case, o, fs) + case = make_case(variant, torch.bfloat16, seq_lens=[100, 2048, 0, 517], lo=0.1 if variant == "gdn" else 0.3) + o, fs = run_fwd(backend, case, output_final_state=True) + check_fwd(case, o, fs) @pytest.mark.parametrize("variant", VARIANTS) def test_fwd_output_contract(backend, variant): """O is io-dtype at HO heads; final_state is fp32 and empty unless requested.""" - case = _case(variant, torch.bfloat16, T=128, H=2, HV=4) - o, fs = _run_fwd(backend, case) + case = make_case(variant, torch.bfloat16, T=128, H=2, HV=4) + o, fs = run_fwd(backend, case) assert o.shape == (case.T, case.HO, case.V) and o.dtype == case.dtype assert fs.numel() == 0 - _o, fs = _run_fwd(backend, case, output_final_state=True) + o, fs = run_fwd(backend, case, output_final_state=True) assert fs.shape == (case.N, case.HO, case.K, case.V) and fs.dtype == torch.float32 @@ -475,26 +495,26 @@ def test_fwd_output_contract(backend, variant): # --------------------------------------------------------------------------- -def _assert_bwd_parity(backend, case, *, scale=None, use_initial_state=False, use_dfs=False, l2norm=False, gate_grad_tol=None, seed=1): +def assert_bwd_parity(backend, case, *, scale=None, use_initial_state=False, use_dfs=False, l2norm=False, gate_grad_tol=None, seed=SEED + 1): variant, tol = case.variant, BWD_TOL[case.dtype] tensors = {"q": case.q, "k": case.k, "v": case.v, "g": case.gates["g"], "beta": case.gates["beta"]} if variant == "gdn2": tensors["w"] = case.gates["w"] - op_leaves = {n: _thd(t).detach().clone().requires_grad_(True) for n, t in tensors.items()} + op_leaves = {n: to_thd(t).detach().clone().requires_grad_(True) for n, t in tensors.items()} ref_leaves = {n: t.detach().double().requires_grad_(True) for n, t in tensors.items()} - _seed(seed) + set_seed(seed) state0_op = state0_ref = None if use_initial_state: state0 = torch.randn(case.N, case.HO, case.K, case.V, device="cuda", dtype=torch.float32) * 0.05 state0_op = state0.detach().clone().requires_grad_(True) state0_ref = state0.detach().double().requires_grad_(True) - with _waive_unsupported(backend, variant): + with waive_unsupported(backend, variant): args = [op_leaves["q"], op_leaves["k"], op_leaves["v"], op_leaves["g"], op_leaves["beta"]] if variant == "gdn2": args.append(op_leaves["w"]) args.append(case.cu) - o, fs = _op(variant)(*args, scale=scale, initial_state=state0_op, output_final_state=True, use_qk_l2norm_in_kernel=l2norm) + o, fs = pinned_op(backend, variant)(*args, scale=scale, initial_state=state0_op, output_final_state=True, use_qk_l2norm_in_kernel=l2norm) dO = torch.randn_like(o) outputs, grad_outputs = [o], [dO] dFS = None @@ -530,84 +550,84 @@ def _assert_bwd_parity(backend, case, *, scale=None, use_initial_state=False, us tol_n = gate_grad_tol else: tol_n = tol - _check(f"d{name}", got, want, tol_n) + check(f"d{name}", got, want, tol_n) @pytest.mark.parametrize("H,HV", HEAD_CONFIGS_SMALL + [(16, 64)]) @pytest.mark.parametrize("T", [64, 128, 251]) -@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16], ids=_DTYPE_IDS.get) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16], ids=DTYPE_IDS.get) @pytest.mark.parametrize("variant", VARIANTS) def test_bwd_parity(backend, variant, dtype, T, H, HV): if dtype == torch.float16 and (T != 128 or (H, HV) != (1, 1)): pytest.skip("fp16 runs one representative backward config") if (H, HV) == (16, 64) and T != 128: pytest.skip("the large GVA config runs one length") - _assert_bwd_parity(backend, _case(variant, dtype, T=T, H=H, HV=HV)) + assert_bwd_parity(backend, make_case(variant, dtype, T=T, H=H, HV=HV)) @pytest.mark.parametrize("H,HK,HV", GQA_CONFIGS) @pytest.mark.parametrize("variant", VARIANTS) def test_bwd_gqa(backend, variant, H, HK, HV): - _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=128, H=H, HK=HK, HV=HV)) + assert_bwd_parity(backend, make_case(variant, torch.bfloat16, T=128, H=H, HK=HK, HV=HV)) @pytest.mark.parametrize("seq_lens", [[64, 192], [31, 63, 93, 123]], ids=["two", "ragged"]) @pytest.mark.parametrize("variant", VARIANTS) def test_bwd_varlen(backend, variant, seq_lens): - _assert_bwd_parity(backend, _case(variant, torch.bfloat16, seq_lens=seq_lens)) + assert_bwd_parity(backend, make_case(variant, torch.bfloat16, seq_lens=seq_lens)) @pytest.mark.parametrize("variant", VARIANTS) def test_bwd_zero_length_sequence(backend, variant): - _assert_bwd_parity(backend, _case(variant, torch.bfloat16, seq_lens=[64, 0, 128])) + assert_bwd_parity(backend, make_case(variant, torch.bfloat16, seq_lens=[64, 0, 128])) @pytest.mark.parametrize("variant", VARIANTS) def test_bwd_initial_state(backend, variant): - _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=128), use_initial_state=True) + assert_bwd_parity(backend, make_case(variant, torch.bfloat16, T=128), use_initial_state=True) @pytest.mark.parametrize("variant", VARIANTS) def test_bwd_d_final_state(backend, variant): - _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=128), use_initial_state=True, use_dfs=True) + assert_bwd_parity(backend, make_case(variant, torch.bfloat16, T=128), use_initial_state=True, use_dfs=True) @pytest.mark.parametrize("variant", VARIANTS) def test_bwd_d_final_state_partial_chunk(backend, variant): - _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=251), use_dfs=True) + assert_bwd_parity(backend, make_case(variant, torch.bfloat16, T=251), use_dfs=True) @pytest.mark.parametrize("variant", VARIANTS) def test_bwd_scale(backend, variant): """A non-default scale must reach the backward path (the engines carry an independent 1/sqrt(K) default that would mask a dropped plumb).""" - _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=128), scale=1.0) + assert_bwd_parity(backend, make_case(variant, torch.bfloat16, T=128), scale=1.0) @pytest.mark.parametrize("variant", VARIANTS) def test_bwd_qk_l2norm(backend, variant): """dQ/dK must include the in-kernel normalization's own backward.""" - _assert_bwd_parity(backend, _case(variant, torch.bfloat16, T=128), l2norm=True) + assert_bwd_parity(backend, make_case(variant, torch.bfloat16, T=128), l2norm=True) def test_bwd_no_decay_gate_grad_floor(backend): """GDN with alpha off: dGate/dBeta are cancelling-reduction noise floors; the data grads stay at full tolerance.""" - _assert_bwd_parity(backend, _case("gdn", torch.bfloat16, T=192, alpha=False), gate_grad_tol=0.3) + assert_bwd_parity(backend, make_case("gdn", torch.bfloat16, T=192, alpha=False), gate_grad_tol=0.3) @pytest.mark.parametrize("variant", VARIANTS) def test_bwd_with_checkpoints(backend, variant): """The checkpoint dump is non-differentiable and must not block backward.""" ckpt = CHUNK[variant] - case = _case(variant, torch.bfloat16, T=4 * ckpt) - q_t = _thd(case.q).detach().clone().requires_grad_(True) - g_t = _thd(case.gates["g"]).detach().clone().requires_grad_(True) - args = [q_t, _thd(case.k), _thd(case.v), g_t, _thd(case.gates["beta"])] + case = make_case(variant, torch.bfloat16, T=4 * ckpt) + q_t = to_thd(case.q).detach().clone().requires_grad_(True) + g_t = to_thd(case.gates["g"]).detach().clone().requires_grad_(True) + args = [q_t, to_thd(case.k), to_thd(case.v), g_t, to_thd(case.gates["beta"])] if variant == "gdn2": - args.append(_thd(case.gates["w"])) - with _waive_unsupported(backend, variant): - o, fs, state_checkpoints = _op(variant)(*args, case.cu, output_final_state=True, checkpoint_every_n_tokens=ckpt) + args.append(to_thd(case.gates["w"])) + with waive_unsupported(backend, variant): + o, fs, state_checkpoints = pinned_op(backend, variant)(*args, case.cu, output_final_state=True, checkpoint_every_n_tokens=ckpt) assert not state_checkpoints.requires_grad (o.sum() + fs.sum()).backward() for name, t in (("q", q_t), ("g", g_t)): @@ -621,22 +641,24 @@ def test_bwd_with_checkpoints(backend, variant): @pytest.mark.parametrize("variant", VARIANTS) def test_checkpoints_match_prefix_final_states(backend, variant): - """state_checkpoints[j] is the state after (j+1)*ckpt tokens, strictly before the end.""" + """state_checkpoints[j] is the state after (j+1)*ckpt tokens, strictly before the + end; rows are a shape-derived capacity bound, valid entries pack first.""" ckpt = CHUNK[variant] T = 5 * ckpt - case = _case(variant, torch.bfloat16, T=T) - o, fs, state_checkpoints = _run_fwd(backend, case, output_final_state=True, checkpoint_every_n_tokens=ckpt) - assert state_checkpoints.shape == ((T - 1) // ckpt, case.HO, case.K, case.V) + case = make_case(variant, torch.bfloat16, T=T) + o, fs, state_checkpoints = run_fwd(backend, case, output_final_state=True, checkpoint_every_n_tokens=ckpt) + valid = (T - 1) // ckpt + assert state_checkpoints.shape == (T // ckpt, case.HO, case.K, case.V) assert state_checkpoints.dtype == case.dtype - for j in (0, state_checkpoints.shape[0] - 1): + for j in (0, valid - 1): n = (j + 1) * ckpt - args = [_thd(case.q)[:n], _thd(case.k)[:n], _thd(case.v)[:n], _thd(case.gates["g"])[:n], _thd(case.gates["beta"])[:n]] + args = [to_thd(case.q)[:n], to_thd(case.k)[:n], to_thd(case.v)[:n], to_thd(case.gates["g"])[:n], to_thd(case.gates["beta"])[:n]] if variant == "gdn2": - args.append(_thd(case.gates["w"])[:n]) + args.append(to_thd(case.gates["w"])[:n]) cu_n = torch.tensor([0, n], dtype=torch.int32, device="cuda") - with _waive_unsupported(backend, variant): - _o, fs_p = _op(variant)(*args, cu_n, output_final_state=True) - _check(f"state_checkpoints[{j}]", state_checkpoints[j], fs_p[0], STATE_TOL[case.dtype]) + with waive_unsupported(backend, variant): + o, fs_p = pinned_op(backend, variant)(*args, cu_n, output_final_state=True) + check(f"state_checkpoints[{j}]", state_checkpoints[j], fs_p[0], STATE_TOL[case.dtype]) @pytest.mark.parametrize("variant", VARIANTS) @@ -645,25 +667,25 @@ def test_checkpoints_varlen(backend, variant): each sequence end); each entry matches its sequence's truncated prefix.""" ckpt = CHUNK[variant] seq_lens = [3 * ckpt + 5, ckpt - 1, 0, 2 * ckpt] - case = _case(variant, torch.bfloat16, seq_lens=seq_lens) - _o, _fs, state_checkpoints = _run_fwd(backend, case, output_final_state=True, checkpoint_every_n_tokens=ckpt) + case = make_case(variant, torch.bfloat16, seq_lens=seq_lens) + o, fs, state_checkpoints = run_fwd(backend, case, output_final_state=True, checkpoint_every_n_tokens=ckpt) counts = [max(sl - 1, 0) // ckpt for sl in seq_lens] # shape[0] is the shape-derived capacity bound; the packed prefix holds # sum(counts) real rows (per-sequence, in order), the tail is uninitialized - assert state_checkpoints.shape[0] == max((sum(seq_lens) - len(seq_lens)) // ckpt, 1) + assert state_checkpoints.shape[0] == max(sum(seq_lens) // ckpt, 1) bounds = case.cu.tolist() base = 0 for n, cnt in enumerate(counts): for j in sorted({0, cnt - 1} if cnt else set()): n0 = bounds[n] ntok = (j + 1) * ckpt - args = [_thd(t)[n0 : n0 + ntok].clone() for t in (case.q, case.k, case.v, case.gates["g"], case.gates["beta"])] + args = [to_thd(t)[n0 : n0 + ntok].clone() for t in (case.q, case.k, case.v, case.gates["g"], case.gates["beta"])] if variant == "gdn2": - args.append(_thd(case.gates["w"])[n0 : n0 + ntok].clone()) + args.append(to_thd(case.gates["w"])[n0 : n0 + ntok].clone()) cu_n = torch.tensor([0, ntok], dtype=torch.int32, device="cuda") - with _waive_unsupported(backend, variant): - _op_, fs_p = _op(variant)(*args, cu_n, output_final_state=True) - _check(f"state_checkpoints[seq {n}][{j}]", state_checkpoints[base + j], fs_p[0], STATE_TOL[case.dtype]) + with waive_unsupported(backend, variant): + o_p, fs_p = pinned_op(backend, variant)(*args, cu_n, output_final_state=True) + check(f"state_checkpoints[seq {n}][{j}]", state_checkpoints[base + j], fs_p[0], STATE_TOL[case.dtype]) base += cnt @@ -672,9 +694,9 @@ def test_checkpoints_varlen(backend, variant): # --------------------------------------------------------------------------- -def _safe_gate_case(variant, T=256, H=2, K=128, V=128, seed=7): - case = _case(variant, torch.bfloat16, T=T, H=H, K=K, V=V, seed=seed) - _seed(seed + 1) +def safe_gate_case(variant, T=256, H=2, K=128, V=128, seed=SEED + 7): + case = make_case(variant, torch.bfloat16, T=T, H=H, K=K, V=V, seed=seed) + set_seed(seed + 1) graw = torch.randn(1, T, case.HO, K, device="cuda", dtype=torch.float32) a_log = torch.zeros(case.HO, dtype=torch.float32, device="cuda") dt_bias = torch.zeros(case.HO, K, dtype=torch.float32, device="cuda") @@ -686,7 +708,7 @@ def test_safe_gate_forward_parity(backend, variant): """Raw logits with a_log = 0 / dt_bias = 0 match the post-activation path fed the host-side transform ``lb * sigmoid(g)``.""" lb = -5.0 - case, graw, a_log, dt_bias = _safe_gate_case(variant) + case, graw, a_log, dt_bias = safe_gate_case(variant) kw = dict(output_final_state=True, use_qk_l2norm_in_kernel=True) raw_kw = dict(kw, safe_gate=True, gate_lower_bound=lb, a_log=a_log, dt_bias=dt_bias) raw_gates = dict(case.gates, g=graw) @@ -697,11 +719,11 @@ def test_safe_gate_forward_parity(backend, variant): eff_beta = braw.float().sigmoid() else: eff_beta = case.gates["beta"] - raw_case = SimpleNamespace(**{**vars(case), "gates": raw_gates}) - eff_case = SimpleNamespace(**{**vars(case), "gates": dict(case.gates, g=lb * torch.sigmoid(graw), beta=eff_beta)}) - o_raw, fs_raw = _run_fwd(backend, raw_case, **raw_kw) - o_eff, fs_eff = _run_fwd(backend, eff_case, **kw) - _check("o", o_raw, o_eff.double(), 2e-2) + raw_case = case.clone(gates=raw_gates) + eff_case = case.clone(gates=dict(case.gates, g=lb * torch.sigmoid(graw), beta=eff_beta)) + o_raw, fs_raw = run_fwd(backend, raw_case, **raw_kw) + o_eff, fs_eff = run_fwd(backend, eff_case, **kw) + check("o", o_raw, o_eff.double(), 2e-2) assert rms_ratio(fs_raw, fs_eff) < 2e-2 @@ -709,19 +731,19 @@ def test_safe_gate_forward_parity(backend, variant): def test_safe_gate_backward_raises(backend, variant): """Raw-logit gate modes are forward-only by contract.""" lb = -5.0 - case, graw, a_log, dt_bias = _safe_gate_case(variant, T=128) + case, graw, a_log, dt_bias = safe_gate_case(variant, T=128) raw_gates = dict(case.gates, g=graw) kw = dict(safe_gate=True, gate_lower_bound=lb, a_log=a_log, dt_bias=dt_bias, use_qk_l2norm_in_kernel=True) if variant == "kda": raw_gates["beta"] = torch.randn(1, case.T, case.HO, device="cuda").to(case.dtype) kw["use_beta_sigmoid_in_kernel"] = True - raw_case = SimpleNamespace(**{**vars(case), "gates": raw_gates}) - g_leaf = _thd(raw_gates["g"]).detach().clone().requires_grad_(True) - args = [_thd(raw_case.q).detach().clone().requires_grad_(True), _thd(raw_case.k), _thd(raw_case.v), g_leaf, _thd(raw_gates["beta"])] + raw_case = case.clone(gates=raw_gates) + g_leaf = to_thd(raw_gates["g"]).detach().clone().requires_grad_(True) + args = [to_thd(raw_case.q).detach().clone().requires_grad_(True), to_thd(raw_case.k), to_thd(raw_case.v), g_leaf, to_thd(raw_gates["beta"])] if variant == "gdn2": - args.append(_thd(raw_gates["w"])) - with _waive_unsupported(backend, variant): - o, _ = _op(variant)(*args, case.cu, **kw) + args.append(to_thd(raw_gates["w"])) + with waive_unsupported(backend, variant): + o, _ = pinned_op(backend, variant)(*args, case.cu, **kw) with pytest.raises(NotImplementedError, match="forward-only"): o.sum().backward() @@ -729,14 +751,14 @@ def test_safe_gate_backward_raises(backend, variant): def test_beta_sigmoid_in_kernel(backend): """KDA: io-dtype Beta logits with the in-kernel sigmoid match the post-activation fp32 path.""" - case = _case("kda", torch.bfloat16, T=256) - _seed(11) + case = make_case("kda", torch.bfloat16, T=256) + set_seed(SEED + 11) braw = torch.randn(1, case.T, case.HO, device="cuda").to(case.dtype) - raw_case = SimpleNamespace(**{**vars(case), "gates": dict(case.gates, beta=braw)}) - eff_case = SimpleNamespace(**{**vars(case), "gates": dict(case.gates, beta=braw.float().sigmoid())}) - o_raw, fs_raw = _run_fwd(backend, raw_case, output_final_state=True, use_beta_sigmoid_in_kernel=True) - o_eff, fs_eff = _run_fwd(backend, eff_case, output_final_state=True) - _check("o", o_raw, o_eff.double(), 2e-2) + raw_case = case.clone(gates=dict(case.gates, beta=braw)) + eff_case = case.clone(gates=dict(case.gates, beta=braw.float().sigmoid())) + o_raw, fs_raw = run_fwd(backend, raw_case, output_final_state=True, use_beta_sigmoid_in_kernel=True) + o_eff, fs_eff = run_fwd(backend, eff_case, output_final_state=True) + check("o", o_raw, o_eff.double(), 2e-2) assert rms_ratio(fs_raw, fs_eff) < 2e-2 @@ -747,11 +769,11 @@ def test_beta_sigmoid_in_kernel(backend): @pytest.mark.parametrize("variant", VARIANTS) def test_torch_compile_forward(backend, variant): - case = _case(variant, torch.bfloat16, T=128) - with _waive_unsupported(backend, variant): - o_eager, _ = _op(variant)(*_op_args(case)) - compiled = torch.compile(_op(variant), fullgraph=True) - o_comp, _ = compiled(*_op_args(case)) + case = make_case(variant, torch.bfloat16, T=128) + with waive_unsupported(backend, variant): + o_eager, _ = pinned_op(backend, variant)(*op_args(case)) + compiled = torch.compile(pinned_op(backend, variant), fullgraph=True) + o_comp, _ = compiled(*op_args(case)) torch.testing.assert_close(o_eager, o_comp) @@ -762,46 +784,46 @@ def test_torch_compile_forward(backend, variant): @pytest.mark.parametrize("variant", VARIANTS) def test_invalid_rank_raises(variant): - case = _case(variant, torch.bfloat16, T=64) + case = make_case(variant, torch.bfloat16, T=64) with pytest.raises(ValueError, match="THD"): - _op(variant)(case.q, _thd(case.k), _thd(case.v), *[_thd(case.gates[n]) for n in case.gates], case.cu) + op(variant)(case.q, to_thd(case.k), to_thd(case.v), *[to_thd(case.gates[n]) for n in case.gates], case.cu) @pytest.mark.parametrize("variant", VARIANTS) def test_invalid_qk_head_mismatch_raises(variant): - case = _case(variant, torch.bfloat16, T=64, H=2) - args = [_thd(case.q), _thd(case.k)[:, :1].contiguous(), _thd(case.v), _thd(case.gates["g"]), _thd(case.gates["beta"])] + case = make_case(variant, torch.bfloat16, T=64, H=2) + args = [to_thd(case.q), to_thd(case.k)[:, :1].contiguous(), to_thd(case.v), to_thd(case.gates["g"]), to_thd(case.gates["beta"])] if variant == "gdn2": - args.append(_thd(case.gates["w"])) + args.append(to_thd(case.gates["w"])) with pytest.raises(ValueError, match="head count"): - _op(variant)(*args, case.cu) + op(variant)(*args, case.cu) @pytest.mark.parametrize("variant", VARIANTS) def test_invalid_gate_dtype_raises(variant): - case = _case(variant, torch.bfloat16, T=64) - args = [_thd(case.q), _thd(case.k), _thd(case.v), _thd(case.gates["g"]).to(torch.bfloat16), _thd(case.gates["beta"])] + case = make_case(variant, torch.bfloat16, T=64) + args = [to_thd(case.q), to_thd(case.k), to_thd(case.v), to_thd(case.gates["g"]).to(torch.bfloat16), to_thd(case.gates["beta"])] if variant == "gdn2": - args.append(_thd(case.gates["w"])) + args.append(to_thd(case.gates["w"])) with pytest.raises(TypeError, match="must be"): - _op(variant)(*args, case.cu) + op(variant)(*args, case.cu) @pytest.mark.parametrize("variant", VARIANTS) def test_invalid_initial_state_count_raises(variant): - case = _case(variant, torch.bfloat16, T=64) + case = make_case(variant, torch.bfloat16, T=64) state0 = torch.zeros(3, case.HO, case.K, case.V, device="cuda", dtype=torch.float32) with pytest.raises(ValueError, match="initial"): - _op(variant)(*_op_args(case), initial_state=state0) + op(variant)(*op_args(case), initial_state=state0) @pytest.mark.parametrize("variant", ["kda", "gdn2"]) def test_invalid_safe_gate_args_raise(variant): - case = _case(variant, torch.bfloat16, T=64) + case = make_case(variant, torch.bfloat16, T=64) with pytest.raises(ValueError, match="safe_gate"): - _op(variant)(*_op_args(case), safe_gate=True) + op(variant)(*op_args(case), safe_gate=True) with pytest.raises(ValueError, match="safe_gate"): - _op(variant)(*_op_args(case), a_log=torch.zeros(case.HO, device="cuda")) + op(variant)(*op_args(case), a_log=torch.zeros(case.HO, device="cuda")) # --------------------------------------------------------------------------- @@ -809,11 +831,11 @@ def test_invalid_safe_gate_args_raise(variant): # --------------------------------------------------------------------------- -def _bits(t): +def bits(t): return t.contiguous().view(torch.uint8) -def _assert_bitwise_runs(launch, repeats=DETERMINISM_REPEATS, label=""): +def assert_bitwise_runs(launch, repeats=DETERMINISM_REPEATS, label=""): """Back-to-back launches (single sync) must match run 0 bit for bit — barrier/fence races are timing-dependent, so there is no tolerance.""" runs = [launch() for _ in range(repeats)] @@ -822,38 +844,38 @@ def _assert_bitwise_runs(launch, repeats=DETERMINISM_REPEATS, label=""): assert torch.isfinite(out.float()).all(), f"{label}: non-finite output in run 0" for r, outs in enumerate(runs[1:], start=1): for i, (a, b) in enumerate(zip(runs[0], outs)): - assert torch.equal(_bits(a), _bits(b)), f"{label}: output {i} differs between run 0 and run {r}" + assert torch.equal(bits(a), bits(b)), f"{label}: output {i} differs between run 0 and run {r}" @pytest.mark.parametrize("backend", ["frost"], indirect=True) @pytest.mark.parametrize("variant", VARIANTS) def test_determinism_fwd(backend, variant): - case = _case(variant, torch.bfloat16, seq_lens=[497, 16, 1, 480, 0, 253]) + case = make_case(variant, torch.bfloat16, seq_lens=[497, 16, 1, 480, 0, 253]) state0 = torch.randn(case.N, case.HO, case.K, case.V, device="cuda", dtype=torch.float32) * 0.05 def launch(): - o, fs = _run_fwd(backend, case, initial_state=state0, output_final_state=True) + o, fs = run_fwd(backend, case, initial_state=state0, output_final_state=True) return o, fs - _assert_bitwise_runs(launch, label=f"{variant} fwd") + assert_bitwise_runs(launch, label=f"{variant} fwd") @pytest.mark.parametrize("backend", ["frost"], indirect=True) @pytest.mark.parametrize("variant", VARIANTS) def test_determinism_bwd(backend, variant): - case = _case(variant, torch.bfloat16, seq_lens=[497, 16, 1, 480, 0, 253]) - leaves = [_thd(case.q).detach().clone().requires_grad_(True), _thd(case.k).detach().clone().requires_grad_(True)] - args = [leaves[0], leaves[1], _thd(case.v), _thd(case.gates["g"]), _thd(case.gates["beta"])] + case = make_case(variant, torch.bfloat16, seq_lens=[497, 16, 1, 480, 0, 253]) + leaves = [to_thd(case.q).detach().clone().requires_grad_(True), to_thd(case.k).detach().clone().requires_grad_(True)] + args = [leaves[0], leaves[1], to_thd(case.v), to_thd(case.gates["g"]), to_thd(case.gates["beta"])] if variant == "gdn2": - args.append(_thd(case.gates["w"])) - with _waive_unsupported(backend, variant): - o, _fs = _op(variant)(*args, case.cu) + args.append(to_thd(case.gates["w"])) + with waive_unsupported(backend, variant): + o, fs = pinned_op(backend, variant)(*args, case.cu) dO = torch.randn_like(o) def launch(): return torch.autograd.grad([o], leaves, [dO], retain_graph=True) - _assert_bitwise_runs(launch, label=f"{variant} bwd") + assert_bitwise_runs(launch, label=f"{variant} bwd") @pytest.mark.parametrize("backend", ["frost"], indirect=True) @@ -861,10 +883,10 @@ def launch(): def test_determinism_two_streams(backend, variant): """Two concurrent instances on separate streams must not perturb each other: every repeat matches its own single-stream baseline.""" - case_a = _case(variant, torch.bfloat16, seq_lens=[497, 16, 1, 480, 0, 253], seed=0) - case_b = _case(variant, torch.bfloat16, B=2, T=512, seed=1) - launch_a = lambda: _run_fwd(backend, case_a, output_final_state=True) # noqa: E731 - launch_b = lambda: _run_fwd(backend, case_b, output_final_state=True) # noqa: E731 + case_a = make_case(variant, torch.bfloat16, seq_lens=[497, 16, 1, 480, 0, 253], seed=SEED) + case_b = make_case(variant, torch.bfloat16, B=2, T=512, seed=SEED + 1) + launch_a = lambda: run_fwd(backend, case_a, output_final_state=True) # noqa: E731 + launch_b = lambda: run_fwd(backend, case_b, output_final_state=True) # noqa: E731 s1, s2 = torch.cuda.Stream(), torch.cuda.Stream() # order the side streams behind the input generation (default stream) torch.cuda.synchronize() @@ -883,7 +905,7 @@ def test_determinism_two_streams(backend, variant): torch.cuda.synchronize() for label, base, outs in (("A", base_a, out_a), ("B", base_b, out_b)): for i, (x, y) in enumerate(zip(base, outs)): - assert torch.equal(_bits(x), _bits(y)), f"stream {label} output {i} differs on concurrent run {r}" + assert torch.equal(bits(x), bits(y)), f"stream {label} output {i} differs on concurrent run {r}" # --------------------------------------------------------------------------- @@ -894,12 +916,12 @@ def test_determinism_two_streams(backend, variant): @pytest.mark.parametrize("backend", ["frost"], indirect=True) @pytest.mark.parametrize("variant", VARIANTS) def test_cuda_graph_replay_fwd(backend, variant): - case = _case(variant, torch.bfloat16, B=2, T=256) + case = make_case(variant, torch.bfloat16, B=2, T=256) def launch(): - return _op(variant)(*_op_args(case), output_final_state=True) + return pinned_op(backend, variant)(*op_args(case), output_final_state=True) - with _waive_unsupported(backend, variant): + with waive_unsupported(backend, variant): eager = launch() warmup = torch.cuda.Stream() warmup.wait_stream(torch.cuda.current_stream()) @@ -914,4 +936,4 @@ def launch(): graph.replay() torch.cuda.synchronize() for i, (a, b) in enumerate(zip(eager, captured)): - assert torch.equal(_bits(a), _bits(b)), f"replayed output {i} differs from eager" + assert torch.equal(bits(a), bits(b)), f"replayed output {i} differs from eager" From 4ef1c8bbd373d2c7d1d2e155505220f1d4f555dd Mon Sep 17 00:00:00 2001 From: Joshua Park Date: Thu, 13 Aug 2026 21:09:26 -0700 Subject: [PATCH 4/9] 4 --- .../benchmark_single_linear_attention.py | 100 +- benchmark/linear_attention/plot_results.py | 142 ++- .../linear_attention/results/gdn/b300/gdn.png | Bin 106305 -> 0 bytes .../results/gdn/b300/gdn_20260806.csv | 16 - .../results/gdn/gb200/gdn_fixed_batch_bw.png | Bin 0 -> 132796 bytes .../gdn/gb200/gdn_fixed_batch_flops.png | Bin 0 -> 118452 bytes .../results/gdn/gb200/gdn_fixed_seq_bw.png | Bin 0 -> 118069 bytes .../results/gdn/gb200/gdn_fixed_seq_flops.png | Bin 0 -> 106703 bytes .../results/gdn/gb300/gdn_fixed_batch_bw.png | Bin 0 -> 133184 bytes .../gdn/gb300/gdn_fixed_batch_flops.png | Bin 0 -> 119449 bytes .../results/gdn/gb300/gdn_fixed_seq_bw.png | Bin 0 -> 118596 bytes .../results/gdn/gb300/gdn_fixed_seq_flops.png | Bin 0 -> 107910 bytes .../results/gdn2/gb200/gdn2_20260813.csv | 41 + .../results/gdn2/gb200/gdn2_b1_bw.png | Bin 0 -> 105374 bytes .../results/gdn2/gb200/gdn2_b1_flops.png | Bin 0 -> 85416 bytes .../results/gdn2/gb200/gdn2_b2_bw.png | Bin 0 -> 100837 bytes .../results/gdn2/gb200/gdn2_b2_flops.png | Bin 0 -> 90061 bytes .../gdn2/gb200/gdn2_fixed_batch_bw.png | Bin 0 -> 100289 bytes .../gdn2/gb200/gdn2_fixed_batch_flops.png | Bin 0 -> 89913 bytes .../results/gdn2/gb200/gdn2_fixed_seq_bw.png | Bin 0 -> 89556 bytes .../gdn2/gb200/gdn2_fixed_seq_flops.png | Bin 0 -> 79705 bytes .../results/gdn2/gb300/gdn2_20260813.csv | 41 + .../results/gdn2/gb300/gdn2_b1_bw.png | Bin 0 -> 105914 bytes .../results/gdn2/gb300/gdn2_b1_flops.png | Bin 0 -> 85617 bytes .../results/gdn2/gb300/gdn2_b2_bw.png | Bin 0 -> 100770 bytes .../results/gdn2/gb300/gdn2_b2_flops.png | Bin 0 -> 90307 bytes .../gdn2/gb300/gdn2_fixed_batch_bw.png | Bin 0 -> 100841 bytes .../gdn2/gb300/gdn2_fixed_batch_flops.png | Bin 0 -> 89658 bytes .../results/gdn2/gb300/gdn2_fixed_seq_bw.png | Bin 0 -> 97872 bytes .../gdn2/gb300/gdn2_fixed_seq_flops.png | Bin 0 -> 86945 bytes .../results/kda/gb200/kda_20260813.csv | 51 + .../results/kda/gb200/kda_b1_bw.png | Bin 0 -> 110709 bytes .../results/kda/gb200/kda_b1_flops.png | Bin 0 -> 91525 bytes .../results/kda/gb200/kda_b2_bw.png | Bin 0 -> 105633 bytes .../results/kda/gb200/kda_b2_flops.png | Bin 0 -> 92041 bytes .../results/kda/gb200/kda_fixed_batch_bw.png | Bin 0 -> 120163 bytes .../kda/gb200/kda_fixed_batch_flops.png | Bin 0 -> 99402 bytes .../results/kda/gb200/kda_fixed_seq_bw.png | Bin 0 -> 111867 bytes .../results/kda/gb200/kda_fixed_seq_flops.png | Bin 0 -> 94530 bytes .../results/kda/gb300/kda_20260813.csv | 51 + .../results/kda/gb300/kda_b1_bw.png | Bin 0 -> 111028 bytes .../results/kda/gb300/kda_b1_flops.png | Bin 0 -> 90192 bytes .../results/kda/gb300/kda_b2_bw.png | Bin 0 -> 106839 bytes .../results/kda/gb300/kda_b2_flops.png | Bin 0 -> 93642 bytes .../results/kda/gb300/kda_fixed_batch_bw.png | Bin 0 -> 122066 bytes .../kda/gb300/kda_fixed_batch_flops.png | Bin 0 -> 101945 bytes .../results/kda/gb300/kda_fixed_seq_bw.png | Bin 0 -> 112621 bytes .../results/kda/gb300/kda_fixed_seq_flops.png | Bin 0 -> 94199 bytes python/cudnn/_pygraph.py | 12 +- python/cudnn/engines/base.py | 6 + python/cudnn/frost/buffers.py | 17 - .../frost/common/head_reduce.py | 10 +- .../linear_attention/frost/common/split_k.py | 352 +++-- .../linear_attention/frost/gdn2_engine.py | 201 +-- .../linear_attention/frost/gdn_engine.py | 220 ++-- .../linear_attention/frost/kda_engine.py | 198 +-- .../frost/kernel/gdn2_bprop_config.py | 10 +- .../frost/kernel/gdn2_bprop_f16.py | 1134 +++++++++-------- .../frost/kernel/gdn2_prefill_config.py | 10 +- .../frost/kernel/gdn2_prefill_f16.py | 728 +++++------ .../frost/kernel/gdn2_recompute_config.py | 8 +- .../frost/kernel/gdn2_recompute_f16.py | 606 +++++---- .../frost/kernel/gdn_bprop_config.py | 4 +- .../frost/kernel/gdn_bprop_f16.py | 457 +++---- .../frost/kernel/gdn_prefill_config.py | 6 +- .../frost/kernel/gdn_prefill_f16.py | 517 ++++---- .../frost/kernel/gdn_recompute_config.py | 4 +- .../frost/kernel/gdn_recompute_f16.py | 350 +++-- .../frost/kernel/kda_bprop_config.py | 6 +- .../frost/kernel/kda_bprop_f16.py | 1099 ++++++++-------- .../frost/kernel/kda_prefill_config.py | 10 +- .../frost/kernel/kda_prefill_f16.py | 551 ++++---- .../frost/kernel/kda_recompute_config.py | 8 +- .../frost/kernel/kda_recompute_f16.py | 542 ++++---- .../cudnn/linear_attention/graph_analyzer.py | 77 +- python/cudnn/linear_attention/ops/gdn.py | 100 +- python/cudnn/linear_attention/ops/gdn2.py | 104 +- python/cudnn/linear_attention/ops/kda.py | 99 +- python/pygraph/variant_pack.cpp | 22 + test/python/linear_attention/test_la.py | 176 +++ 80 files changed, 4335 insertions(+), 3751 deletions(-) delete mode 100644 benchmark/linear_attention/results/gdn/b300/gdn.png delete mode 100644 benchmark/linear_attention/results/gdn/b300/gdn_20260806.csv create mode 100644 benchmark/linear_attention/results/gdn/gb200/gdn_fixed_batch_bw.png create mode 100644 benchmark/linear_attention/results/gdn/gb200/gdn_fixed_batch_flops.png create mode 100644 benchmark/linear_attention/results/gdn/gb200/gdn_fixed_seq_bw.png create mode 100644 benchmark/linear_attention/results/gdn/gb200/gdn_fixed_seq_flops.png create mode 100644 benchmark/linear_attention/results/gdn/gb300/gdn_fixed_batch_bw.png create mode 100644 benchmark/linear_attention/results/gdn/gb300/gdn_fixed_batch_flops.png create mode 100644 benchmark/linear_attention/results/gdn/gb300/gdn_fixed_seq_bw.png create mode 100644 benchmark/linear_attention/results/gdn/gb300/gdn_fixed_seq_flops.png create mode 100644 benchmark/linear_attention/results/gdn2/gb200/gdn2_20260813.csv create mode 100644 benchmark/linear_attention/results/gdn2/gb200/gdn2_b1_bw.png create mode 100644 benchmark/linear_attention/results/gdn2/gb200/gdn2_b1_flops.png create mode 100644 benchmark/linear_attention/results/gdn2/gb200/gdn2_b2_bw.png create mode 100644 benchmark/linear_attention/results/gdn2/gb200/gdn2_b2_flops.png create mode 100644 benchmark/linear_attention/results/gdn2/gb200/gdn2_fixed_batch_bw.png create mode 100644 benchmark/linear_attention/results/gdn2/gb200/gdn2_fixed_batch_flops.png create mode 100644 benchmark/linear_attention/results/gdn2/gb200/gdn2_fixed_seq_bw.png create mode 100644 benchmark/linear_attention/results/gdn2/gb200/gdn2_fixed_seq_flops.png create mode 100644 benchmark/linear_attention/results/gdn2/gb300/gdn2_20260813.csv create mode 100644 benchmark/linear_attention/results/gdn2/gb300/gdn2_b1_bw.png create mode 100644 benchmark/linear_attention/results/gdn2/gb300/gdn2_b1_flops.png create mode 100644 benchmark/linear_attention/results/gdn2/gb300/gdn2_b2_bw.png create mode 100644 benchmark/linear_attention/results/gdn2/gb300/gdn2_b2_flops.png create mode 100644 benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_batch_bw.png create mode 100644 benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_batch_flops.png create mode 100644 benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_seq_bw.png create mode 100644 benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_seq_flops.png create mode 100644 benchmark/linear_attention/results/kda/gb200/kda_20260813.csv create mode 100644 benchmark/linear_attention/results/kda/gb200/kda_b1_bw.png create mode 100644 benchmark/linear_attention/results/kda/gb200/kda_b1_flops.png create mode 100644 benchmark/linear_attention/results/kda/gb200/kda_b2_bw.png create mode 100644 benchmark/linear_attention/results/kda/gb200/kda_b2_flops.png create mode 100644 benchmark/linear_attention/results/kda/gb200/kda_fixed_batch_bw.png create mode 100644 benchmark/linear_attention/results/kda/gb200/kda_fixed_batch_flops.png create mode 100644 benchmark/linear_attention/results/kda/gb200/kda_fixed_seq_bw.png create mode 100644 benchmark/linear_attention/results/kda/gb200/kda_fixed_seq_flops.png create mode 100644 benchmark/linear_attention/results/kda/gb300/kda_20260813.csv create mode 100644 benchmark/linear_attention/results/kda/gb300/kda_b1_bw.png create mode 100644 benchmark/linear_attention/results/kda/gb300/kda_b1_flops.png create mode 100644 benchmark/linear_attention/results/kda/gb300/kda_b2_bw.png create mode 100644 benchmark/linear_attention/results/kda/gb300/kda_b2_flops.png create mode 100644 benchmark/linear_attention/results/kda/gb300/kda_fixed_batch_bw.png create mode 100644 benchmark/linear_attention/results/kda/gb300/kda_fixed_batch_flops.png create mode 100644 benchmark/linear_attention/results/kda/gb300/kda_fixed_seq_bw.png create mode 100644 benchmark/linear_attention/results/kda/gb300/kda_fixed_seq_flops.png diff --git a/benchmark/linear_attention/benchmark_single_linear_attention.py b/benchmark/linear_attention/benchmark_single_linear_attention.py index 15a761535..a95f49373 100644 --- a/benchmark/linear_attention/benchmark_single_linear_attention.py +++ b/benchmark/linear_attention/benchmark_single_linear_attention.py @@ -268,6 +268,7 @@ def run_benchmark( Dict with keys: - time_ms: Median time of the requested pass in milliseconds - tflops: TFLOPS for the requested pass + - bw_tb_per_sec: DRAM bandwidth (TB/s) for the requested pass - max_diff: Maximum difference vs reference - gpu_name: GPU name string - cudnn_version: cuDNN version (if available) @@ -339,7 +340,7 @@ def run_benchmark( raise RuntimeError(f"Benchmark failed with return code {result.returncode}.\n" f"stderr: {result.stderr}\n" f"stdout: {result.stdout}") # Parse CSV output - # Format: case_tag,backend,variant,batch_size,seqlen,num_q_heads,num_kv_heads,head_dim,fwd_time,bwd_time,fwd_tflops,bwd_tflops,max_diff,num_iters + # Format: case_tag,backend,variant,batch_size,seqlen,num_q_heads,num_kv_heads,head_dim,fwd_time,bwd_time,fwd_tflops,bwd_tflops,max_diff,num_iters,fwd_bw,bwd_bw output_line = result.stdout.strip().split("\n")[-1] parts = output_line.split(",") @@ -365,13 +366,16 @@ def run_benchmark( if profile_pass == "fwd": time_ms = float(parts[8]) tflops = float(parts[10]) + bw_tb_per_sec = float(parts[14]) if len(parts) > 14 else 0.0 else: # "bwd" time_ms = float(parts[9]) tflops = float(parts[11]) + bw_tb_per_sec = float(parts[15]) if len(parts) > 15 else 0.0 return { "time_ms": time_ms, "tflops": tflops, + "bw_tb_per_sec": bw_tb_per_sec, "max_diff": float(parts[12]) if len(parts) > 12 else 0.0, "gpu_name": gpu_name, "cudnn_version": cudnn_version, @@ -818,6 +822,63 @@ def tflops_per_sec( ) return f / time / 1e9 if not math.isnan(time) else 0.0 # Assume time is in msec + # Util functions for calculating DRAM bytes moved and bandwidth achieved + def dram_bytes( + batch_size, + seqlen, + head_dim_qk, + head_dim_vo, + num_q_heads, + num_kv_heads, + num_o_heads, + mode="fwd", + ): + assert mode in ["fwd", "bwd"] + io_bytes = 2 + f32_bytes = 4 + tokens = batch_size * seqlen + q_bytes = tokens * num_q_heads * head_dim_qk * io_bytes + k_bytes = tokens * num_q_heads * head_dim_qk * io_bytes + v_bytes = tokens * num_kv_heads * head_dim_vo * io_bytes + o_bytes = tokens * num_o_heads * head_dim_vo * io_bytes + if args.variant == "gdn": + gate_bytes = tokens * num_o_heads * 2 * f32_bytes + elif args.variant == "kda": + gate_bytes = tokens * num_o_heads * (head_dim_qk + 1) * f32_bytes + else: # gdn2 + gate_bytes = tokens * num_o_heads * (head_dim_qk * f32_bytes + (head_dim_qk + head_dim_vo) * io_bytes) + num_chunks = ceil_div(seqlen, _CHUNK_SIZE[args.variant]) + h_bytes = batch_size * num_o_heads * num_chunks * head_dim_qk * head_dim_vo * io_bytes + qkv_bytes = q_bytes + k_bytes + v_bytes + if mode == "fwd": + return qkv_bytes + gate_bytes + o_bytes + recompute_bytes = k_bytes + v_bytes + gate_bytes + h_bytes + return recompute_bytes + 2 * (qkv_bytes + gate_bytes) + o_bytes + h_bytes + + def tb_per_sec( + batch_size, + seqlen, + head_dim_qk, + head_dim_vo, + num_q_heads, + num_kv_heads, + num_o_heads, + time, + mode="fwd", + ): + assert mode in ["fwd", "bwd"] + b = dram_bytes( + batch_size, + seqlen, + head_dim_qk, + head_dim_vo, + num_q_heads, + num_kv_heads, + num_o_heads, + mode, + ) + return b / time / 1e9 if not math.isnan(time) else 0.0 # Assume time is in msec + ## Gate generators per variant. Decays are LOG-space (alpha = exp(g)), ## drawn from ranges the kernels' io-dtype arithmetic is conditioned for. def generate_gates(io_dtype): @@ -1036,6 +1097,7 @@ def generate_gates(io_dtype): np.median(np.array(forward_times[5:])) if len(forward_times) > 5 else (np.median(np.array(forward_times)) if len(forward_times) > 0 else 0.0) ) fwd_tflops = 0.0 + fwd_bw = 0.0 if run_fwd and fwd_median_time > 0: fwd_tflops = tflops_per_sec( args.batch_size, @@ -1046,11 +1108,23 @@ def generate_gates(io_dtype): fwd_median_time, "fwd", ) + fwd_bw = tb_per_sec( + args.batch_size, + args.seqlen, + head_dim_qk, + head_dim_vo, + args.num_q_heads, + args.num_kv_heads, + num_o_heads, + fwd_median_time, + "fwd", + ) bwd_median_time = ( np.median(np.array(backward_times[5:])) if len(backward_times) > 5 else (np.median(np.array(backward_times)) if len(backward_times) > 0 else 0.0) ) bwd_tflops = 0.0 + bwd_bw = 0.0 if run_bwd and bwd_median_time > 0: bwd_tflops = tflops_per_sec( args.batch_size, @@ -1061,6 +1135,17 @@ def generate_gates(io_dtype): bwd_median_time, "bwd", ) + bwd_bw = tb_per_sec( + args.batch_size, + args.seqlen, + head_dim_qk, + head_dim_vo, + args.num_q_heads, + args.num_kv_heads, + num_o_heads, + bwd_median_time, + "bwd", + ) # Compute MMA SOL% using the per-arch FLOPs/clk/SM table and the actual # sampled boost clock observed during the benchmark window. @@ -1077,16 +1162,21 @@ def generate_gates(io_dtype): fwd_sol_str = f", {fwd_tflops / _peak_mma_tflops * 100:.1f}% SOL" if _peak_mma_tflops and fwd_tflops > 0 else "" bwd_sol_str = f", {bwd_tflops / _peak_mma_tflops * 100:.1f}% SOL" if _peak_mma_tflops and bwd_tflops > 0 else "" + backend_tag = f"{args.la_backend}_state_on" if (args.store_on and args.la_backend == "cudnn") else args.la_backend if args.format_output: print( - f"{args.case_tag},{args.la_backend},{args.variant},{args.batch_size},{args.seqlen},{args.num_q_heads},{args.num_kv_heads},{head_dim_qk},{fwd_median_time:.3f},{bwd_median_time:.3f},{fwd_tflops:.0f},{bwd_tflops:.0f},{(np.max(np.array(forward_diffs[5:])) if len(forward_diffs) > 5 else (np.max(np.array(forward_diffs)) if len(forward_diffs) > 0 else 0.0)):.6f},{num_iters}" + f"{args.case_tag},{backend_tag},{args.variant},{args.batch_size},{args.seqlen},{args.num_q_heads},{args.num_kv_heads},{head_dim_qk},{fwd_median_time:.3f},{bwd_median_time:.3f},{fwd_tflops:.0f},{bwd_tflops:.0f},{(np.max(np.array(forward_diffs[5:])) if len(forward_diffs) > 5 else (np.max(np.array(forward_diffs)) if len(forward_diffs) > 0 else 0.0)):.6f},{num_iters},{fwd_bw:.2f},{bwd_bw:.2f}" ) else: if run_fwd and run_bwd: print( - f"{args.la_backend}/{args.variant}:: Median (fwd, bwd) Execution Times: {fwd_median_time:.3f} ms ({fwd_tflops:.0f} TFLOPS{fwd_sol_str}), {bwd_median_time:.3f} ms ({bwd_tflops:.0f} TFLOPS{bwd_sol_str})" + f"{args.la_backend}/{args.variant}:: Median (fwd, bwd) Execution Times: {fwd_median_time:.3f} ms ({fwd_tflops:.0f} TFLOPS{fwd_sol_str}, {fwd_bw:.2f} TB/s), {bwd_median_time:.3f} ms ({bwd_tflops:.0f} TFLOPS{bwd_sol_str}, {bwd_bw:.2f} TB/s)" ) elif run_fwd: - print(f"{args.la_backend}/{args.variant}:: Median (fwd) Execution Time: {fwd_median_time:.3f} ms ({fwd_tflops:.0f} TFLOPS{fwd_sol_str})") + print( + f"{args.la_backend}/{args.variant}:: Median (fwd) Execution Time: {fwd_median_time:.3f} ms ({fwd_tflops:.0f} TFLOPS{fwd_sol_str}, {fwd_bw:.2f} TB/s)" + ) elif run_bwd: - print(f"{args.la_backend}/{args.variant}:: Median (bwd) Execution Time: {bwd_median_time:.3f} ms ({bwd_tflops:.0f} TFLOPS{bwd_sol_str})") + print( + f"{args.la_backend}/{args.variant}:: Median (bwd) Execution Time: {bwd_median_time:.3f} ms ({bwd_tflops:.0f} TFLOPS{bwd_sol_str}, {bwd_bw:.2f} TB/s)" + ) diff --git a/benchmark/linear_attention/plot_results.py b/benchmark/linear_attention/plot_results.py index bed2c0df9..019815db8 100644 --- a/benchmark/linear_attention/plot_results.py +++ b/benchmark/linear_attention/plot_results.py @@ -29,7 +29,8 @@ "fla": {"name": "FLA (Triton)", "color": "#FF8C00", "order": 0}, "flash_qla": {"name": "FlashQLA (TileLang)", "color": "#6495ED", "order": 1}, "flash_kda": {"name": "FlashKDA", "color": "#9370DB", "order": 2}, - "cudnn": {"name": "cuDNN", "color": "#76b900", "order": 3}, + "cudnn": {"name": "cuDNN (default)", "color": "#76b900", "order": 3}, + "cudnn_state_on": {"name": "cuDNN (state on)", "color": "#2f6e00", "order": 4}, } # Backends dropped from every chart (rows may still exist in older CSVs). @@ -55,14 +56,20 @@ "bwd_tflops", "max_diff", "num_iters", + "fwd_bw", + "bwd_bw", ] +# One chart per metric: (fwd column, bwd column, y-axis label, bar label +# format, filename suffix). +METRIC_CONFIG = ( + ("fwd_tflops", "bwd_tflops", "TFLOPS", "%.0f", "_flops"), + ("fwd_bw", "bwd_bw", "DRAM Bandwidth (TB/s)", "%.2f", "_bw"), +) + def get_backend_display_name(backend: str, cudnn_version: Optional[str] = None) -> str: - base_name = BACKEND_CONFIG.get(backend, {}).get("name", backend) - if backend == "cudnn" and cudnn_version: - base_name = f"{base_name} {cudnn_version}" - return base_name + return BACKEND_CONFIG.get(backend, {}).get("name", backend) def generate_charts( @@ -98,64 +105,74 @@ def generate_charts( sub = df[df[group_col] == group_val].copy() sub.sort_values([x_col, "backend_order"], inplace=True) - fwd_df = sub[sub["fwd_tflops"] > 0] - bwd_df = sub[sub["bwd_tflops"] > 0] - has_fwd = not fwd_df.empty - has_bwd = not bwd_df.empty - - if has_fwd and has_bwd: - fig, (ax_fwd, ax_bwd) = plt.subplots(1, 2, figsize=(14, 6), dpi=150) - elif has_fwd: - fig, ax_fwd = plt.subplots(1, 1, figsize=(10, 6), dpi=150) - ax_bwd = None - else: - fig, ax_bwd = plt.subplots(1, 1, figsize=(10, 6), dpi=150) - ax_fwd = None - - heads = sub["num_q_heads"].iloc[0] - head_dim = sub["head_dim"].iloc[0] - gpu_info = f" ({gpu_name})" if gpu_name else "" - group_label = f"Batch = {group_val}" if x_axis == "seqlen" else f"SeqLen = {group_val}" - fig.suptitle( - f"{variant.upper()} Linear Attention (BF16) — {group_label}, Heads = {heads}, d = {head_dim}{gpu_info}", - fontsize=TITLE_FONT_SIZE, - ) - - for ax, pass_df, pass_name, y_col in ( - (ax_fwd, fwd_df, "Forward", "fwd_tflops"), - (ax_bwd, bwd_df, "Backward", "bwd_tflops"), - ): - if ax is None or pass_df.empty: + for fwd_col, bwd_col, y_label, bar_fmt, file_suffix in METRIC_CONFIG: + if fwd_col not in sub.columns or bwd_col not in sub.columns: + continue + fwd_df = sub[sub[fwd_col] > 0] + bwd_df = sub[sub[bwd_col] > 0] + has_fwd = not fwd_df.empty + has_bwd = not bwd_df.empty + if not has_fwd and not has_bwd: continue - hue_order = list(pass_df.sort_values("backend_order")["backend_display"].drop_duplicates()) - sns.barplot( - data=pass_df, - x=x_col, - y=y_col, - hue="backend_display", - hue_order=hue_order, - ax=ax, - palette=palette, - edgecolor="black", - linewidth=0.5, - errorbar=None, + + if has_fwd and has_bwd: + fig, (ax_fwd, ax_bwd) = plt.subplots(1, 2, figsize=(14, 6), dpi=150) + elif has_fwd: + fig, ax_fwd = plt.subplots(1, 1, figsize=(10, 6), dpi=150) + ax_bwd = None + else: + fig, ax_bwd = plt.subplots(1, 1, figsize=(10, 6), dpi=150) + ax_fwd = None + + heads = sub["num_q_heads"].iloc[0] + head_dim = sub["head_dim"].iloc[0] + gpu_info = f" ({gpu_name})" if gpu_name else "" + group_label = f"Batch = {group_val}" if x_axis == "seqlen" else f"Sequence Length = {group_val}" + fig.suptitle( + f"{variant.upper()} Linear Attention (BF16) — {group_label}, Heads = {heads}, d = {head_dim}{gpu_info}", + fontsize=TITLE_FONT_SIZE, ) - ax.set_xlabel(x_label, fontsize=LABEL_FONT_SIZE) - ax.set_ylabel("TFLOPS", fontsize=LABEL_FONT_SIZE) - ax.set_title(pass_name, fontsize=TITLE_FONT_SIZE) - ax.legend(title="Backend", fontsize=LEGEND_FONT_SIZE) - ax.tick_params(axis="x", rotation=45) - for container in ax.containers: - ax.bar_label(container, fmt="%.0f", fontsize=BAR_LABEL_FONT_SIZE) - - plt.tight_layout() - gv = int(group_val) - stem = f"{variant}_b{gv}" if x_axis == "seqlen" else f"{variant}_t{gv}_bsweep" - output_path = output_dir / f"{stem}.png" - plt.savefig(output_path, dpi=150, bbox_inches="tight") - plt.close() - saved_paths.append(output_path) - print(f"Chart saved to {output_path}") + + for ax, pass_df, pass_name, y_col in ( + (ax_fwd, fwd_df, "Forward", fwd_col), + (ax_bwd, bwd_df, "Backward", bwd_col), + ): + if ax is None or pass_df.empty: + continue + hue_order = list(pass_df.sort_values("backend_order")["backend_display"].drop_duplicates()) + sns.barplot( + data=pass_df, + x=x_col, + y=y_col, + hue="backend_display", + hue_order=hue_order, + ax=ax, + palette=palette, + edgecolor="black", + linewidth=0.5, + errorbar=None, + ) + ax.set_xlabel(x_label, fontsize=LABEL_FONT_SIZE) + ax.set_ylabel(y_label, fontsize=LABEL_FONT_SIZE) + ax.set_title(pass_name, fontsize=TITLE_FONT_SIZE) + ax.legend(title="Backend", fontsize=LEGEND_FONT_SIZE, loc="upper left") + ax.tick_params(axis="x", rotation=45) + for container in ax.containers: + ax.bar_label(container, fmt=bar_fmt, fontsize=BAR_LABEL_FONT_SIZE) + + plt.tight_layout() + gv = int(group_val) + if df[group_col].nunique() == 1: + # the sweep pinned the group dimension: fixed-batch (seqlen + # sweep) / fixed-seq (batch sweep) result-tree naming + stem = f"{variant}_fixed_batch" if x_axis == "seqlen" else f"{variant}_fixed_seq" + else: + stem = f"{variant}_b{gv}" if x_axis == "seqlen" else f"{variant}_t{gv}_bsweep" + output_path = output_dir / f"{stem}{file_suffix}.png" + plt.savefig(output_path, dpi=150, bbox_inches="tight") + plt.close() + saved_paths.append(output_path) + print(f"Chart saved to {output_path}") return saved_paths @@ -175,7 +192,8 @@ def main(): batch_sizes = [int(b) for b in args.batch_sizes.split(",")] if args.batch_sizes else None df = pd.read_csv(args.csv) - missing = [c for c in CSV_COLUMNS if c not in df.columns] + # fwd_bw/bwd_bw are newer columns; older CSVs simply skip the BW charts. + missing = [c for c in CSV_COLUMNS if c not in df.columns and c not in ("fwd_bw", "bwd_bw")] if missing: raise ValueError(f"CSV is missing expected columns: {missing}") diff --git a/benchmark/linear_attention/results/gdn/b300/gdn.png b/benchmark/linear_attention/results/gdn/b300/gdn.png deleted file mode 100644 index 43c09ec5bc50256801374a2410357c714afe9bdb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 106305 zcmdqJcRbf`|2O=$%BZL)Nk&6a8BrNUp<$F6k_at|tWahlQqdqPql}OuGLl&`(jZyc zlw|L{uIEALb^Wg2ecylIkL&VyoZs_2%lGqnkK=f~p6hkoR6lxXDKi%{MNvzY4l8O> z6yqj}qPxPl2!FD3Vx=zrOWN*$o}JdEvvv-~HfN|K#&%X0FWFr*GvTv8V`FP}$x>Wc zOiWm88{c_5J1bjh5fO|3`5VHQY|e?;)Y6UPQ<$s{AGf8b)hoz~6rKGrD z+cBiO!9kC`dTLj{Q_u^&eaki2y4UNeC#}BAxk<;Ta0Lrj^-N4|j$F9?xJD-F zfN9^-tu-8L_1Lz>vPb{?JCo%$-aD=5usTQ}cC7u~DDyYLp)bAPPLFkqw-^n43yW|Lym*w*&|5`)_~db^ZVRUz1OdrQ=(_etkq~UDlPMy8!{7 zjoD85RaI%R`Op08U!6#QeXf)zKu~LuX_4RdPOHIksfcCXHdX_z2e{>2x&Qq6)8F5}rPJ!~hKRQ})~;P^`XyNGz`A8^ z%Z-X!Z-tB56qa(4XX5rQo`KiGIr8$B=KbC4F&II&kQnbIQL$)%jx3SWMNvLbwzy9GHyl|78Xx$Z?`-~ zmT$M$h>N;R+NnO;!SLqnS4|-nCZ_k#k36YJ6BiYAkBbxR?&)c1Zl)foK2I||>XLO~ zyuT&ZAS1bCG)Tm#wLDs-e{Af~)vH&-nq;aIPUe4ovU{MlfF=0JlStQ@$@Aq=H*4SN zCK?{VM>peB5^2a`ER1n=7-Mv$l7aax#DV^y#fl8*`Yfi*I{rxPq?kY8@S&^9}EI zZripk#dqh{tt{tC!wP!p((eTX(5tAZ*mc(|=Md7~k}?`zer_pQm?nqF_SCqBt7LPV z9`F|3y?ZrPA*=jwo8XNlYu3}!(N&HbpF0IYJbNFT?)f<-bn;z_ zkDp(aou=w=QxLD4s9mBKB0135 zMMr78IeWx&?_ zheD)m4_ahm@iI{+zY6bU)EZf9u_c_SSXZ+6UiF1%(vb)6XqvkX{rUQVWqPk{A(lw{81}Vd z!@J8n4L^VV`qe8aNY~Fr@xXzbc#BLK1_^2YLdRbCG-NuocX7&nJa>CBD-Yx9)vJxq zpN~y@r7Ey;@6} z$MJRTY-e?%$X7`|&wb9$dmF zu?tn(wjEE7+UGBm>pIc4dgG=|)*Uy-ic3m9U>i`BO$N7_nOO}sYRm!GLmrEkidxi4 zpol$>RrvJak@&gwf`UwPbJOB@fwnJYF4l-fXGR-kU-fEkJ{2NqP0PwFtBAT_JvnTa zVo|5lCEGY+`Ezol>)zwX%qv%}BnuoXl97?IthaeKH+OcDO4=e3D9+JYLLz{J{g#DOXFGsOP@-@yW zGEZcz%ITV(7}CpemRhoO>Am~+HEW)^-5<1d?Q6<4nw&Puw6~CymX1IZVdCgmXEJ9qE0aqn?tGd4CxrCV*^ zU30JfQGqO8^=7rOyt=v*lRT#_EbcIK2=VROwJJ#Xk~0BFo?=@ zm3HXewqwT*iP2-vpJz61q zHqkJ9#U_=9?9|7Zsji@D`M3?67pzT+S!E&Rn$~>pgidFmRJ6o5@F?-d8{7j*x8Bc&zJ)7CzXy_`Be_*taDi)FQTN!VT|0sU0-O%&t0 zb?dDA8utcD8yXsxca7%G&8S6Be~&yAp&6}=n>dkf$>Z0X?^5VE`1@x6$Oy-yM~}?w za6kIBMTxnyFEqpM-Q&M#VZlIYU=a*-{}hiNTqAz=aF0K?lxA}ehpg8^CoNcdPMNfG zAxds`e8YziAD+g?vs}7#3CqU(WnJAlYbUXT3vaI6Y&AFQq8cjA@nrX?c}k z5i8H_7suk)Z`-y6t4i_o>5X{bUaPk2KQ;UDT)(INy~konx7j@r5fP`(pJ$Yjk--JY z8yOjyuH~nrr<42L|@z!!)(D2B!LQFL2n} z&rJ`px~C_$#cD)~UTWM^pJBV$)z$TU)yqek4Jj!SUml4sMiZs$w|Dv7@grtyrM12N z!CR}gr@V;i75-XLp&cUQuxzZiK|x-g_RE(q&42&;Jl=WX#htmM3Lg)`bT`O9<V{$P;IqdNNsUxmtOtv_G!bE^UJxpxhozX zLqS}-+fwnBe(K#vk8Wa5<`)(ojQbWU<3Lf-%7Guq$}T@N(pf3yJZ=SONVk9g{^H_d z_XNw{8+LLP=ayW!a6#|Hi3=Cy@b@VA^bUQE1$Wl&)-FMb-ZqkH)257~s}$#qmROVP zCP!A%m7zZ^@rgM?WuwgDpBE|->x=(XBKvFCu9Z)|LG8CqsFigdS3h&Y@+|k^qeoW( zs+LbCJf0jJ9%h%Z@4j1QbIf7PK;vtGdXDoIntk2%8CqseSFgUa+x#FsJ)Kk5`E7}G z?)aR8SwTU8S+|^|q#k*8=GCdG$;su8K@~5LzafpFGVRCr@5`~LRw^ke3H2lwx%@5T z|DNmS7M*63u}8PmpH2Sb$B&FIE-p0tLgd^G8Tl2ldKoA&laGvnlLG?-eJ5ERt6m=0 zuXd^i4vzl0yU}I1vr?r;y5MJDNRIPFvfX0+QK0dAHQ38xeh2njJ3ROD@~Um$Atj&gGQs6SZHz9--KdJeK%m%UfIgH8d*I zzUO|z{>Q!L6@_Th(XJ*YCWG~A8bT<85)#@#Xc~VzI^y<xQ;2PcHfZE?9Ax23i zUa@*f5w(+*FOID&@>{3%Vvhe?P;0YkxFEks^ zzL|BSB1SpvXgSs6jTSYppS{KQcM5+Jp`%_c&+m!<_qYsC@ZrOU>xG4ncDQH;i*Nef zetBAteR#qG^{8J<@rM^dZTs!{>8GXZY`MiqWct%N_s@i#ooA zbVvdNitlaU+-c6f%d$SUT$qoK0^BXapQ9A|2%miS6nE_7?_YFp!y&d6E3Wr8WD?vX zn}j z@6Ag3d*mU1oz6EWGwt=OKU;R!Br{Wa;$8wFBf?%s7r|7ynH0%a)T{gkCy)N!8^p>ZcqrMQ^!=1tOBzYf*b z7Boy57Ww!v5{?Q?Z)IiW9v!{u_U+r`8RG8O<4zJ15+to#=|e(8X@2)Ob4l53)Oo2# zLxFvG-@3&D%G>*5{zZOZ`cgL_ZuG9TT4lI6#b9|H{zP5k2GD?bN`90IG6QyiagT_<-4<;8JkawnCC&J~@7 zB^WGbx(HQW6f{CLL{cMe8r@JY(|$X3K7?cY=7J9KLOdQeHa2SCOgFo)BuBVIrCD9t zcK7r*4;r$rtTb|+ICCvkcices0@)cZhgIisMM(_c8|64pE~FcK6b$M6o5I3+WbCj_JoZVcu()0onYF`BHHC(3DpfsuNJ3)&qgOzECZ^ zgiGSE(2wfsqT!AoN#>P8+_KJ`e`0TvgsjV6=yUr%VKq%K` zY8N+9+^@o=(-ZCC=0S0e1Fa0^l`p)2{v+|<<)c_q5@zM}fV*VN;P&o!nO+GmxV2h^ zi3u3wdQ41=_CqXvG1J0RKaf8s=bQPE;QKR==BAWu*FCF`U~$IBZZ_!TXfZ0jUzQI~c^ z`7nf=JOBW~1GU>fJ=k3CZ_y|8a%dA!^5mSMAzr~ty-nS}b&j^blD~NIWnuEP!RMjD z94$R`|LN&zzM#(oxTw~;OSyq1tG4O6qq4T5teKm-g1Q_Hk>oE(>>bH;nHqI+D|XY> z$O>Za)E_V6b!!F?B|HT@K5VLaXp3B8t>7|OLtYw^jqP7LzXCACGY|)x}xgU&HdmH!z8p+sfp{Y@$rqir1>f)-shNs$!aA^e+S#H&rU&`mIitE|Yct+KVUVxtsej)qE99nW#jp!kGz zUdE;@xo#ruK(zuTo7a5{+k4_yKxS=W@Q=Zz5DjSELDY}c?b@(Gp(0*ajbcObRu}qw zX}a&t)e&>E&Oafi!}X8SmMc)Dg0?K(!A(7PUxi4V#G~Y*hJ7ZJ3(x z&@(g|^PrvIqh6N=p#qqV@KghfqvAyZ;6YqCzYX@z&yM>1|IzT+4bsz7| z?5jv~8R-lZ-L*?+DhB{IgyK^U*z)Z1w%RSeW19O7So-QSHVlu99Kbd!cyq2as)yHP z;>L8AU+s9sx4uLEJU3B}n45c&w@5KDF)hcN(3KKE=|wGgVGw%jr%Q1u=g*PupIXt` zC`D)+$4Y~~I2q3ESlJP{;+ss$mahhCRsQbo+ zx7V0|F7)RO?~<8H{w#a4FVmqmP8?W*$P9RO?@{Kz)2>_V)JF(?R`Jo_KIdiI1L-U$(Oowfn0+Jv~%P zXkB@Ex%RiOAAIEm1Qt;h-@ZKi^x8OIJ;UZqM-vJfMBRE5Kb8C}O)7CVnr>MDP}e_9tXoZ4q+ zXLqeok9}}T`gu&uFDR#sOO|{DlDhl&u~1+~SJ#G3n;71|e-8@t<(L7M*jwB7WmMje zAL43nve~O9OaQpRN88?o%J0ujmrDuxcEan-=Z7t*&_Z=Ao%&&QV-3JS%ei2JaPBzm zQ|rRDcI~gO#N5Jnm%Fq}*x&5iw0W~BDv^1NOX;l$N0GKM{a_U4zMQFLz*(fH0T=|d zkK1m|C1opmXbRQlZ*A(AUyr+4K5xmktz#&AQl-Pj%{VfWTQc^7_mmdK?p2bhO zDg1z@Qr^@zWqBGmIXr9^y=%f^E*M5xbmlPtgOOv3D% z3DEf>NWn#Tq@q~J&OIWVecM+$W}f~;G{ac8qn9sVj;8tR;8uKfchY(pQ&Uq4PfCCP zjb+U}tb)wYWXtsu(S@nTqrB^QcoLGim5mD$9eLZ%9swHg&VT=Y|Cy<->csEQ6c!T+ zi$Ky-r=Aw_j_n-L;3T?Qa4;hkEPl?fgmn$jA8A-K+W2b$RVFWxr(24tgLaFCa?& zl=Ii{??AFyU!*4?h$`lq=;hnCvNpx+fvJg-adBT)>>ahXX}?>y@^y_|yqv~*pUHbM z11vps{q0&Erxg>m(H|Jkv?M@KIs9v1L#8+ILJRbezjf&%e!OyF@p`_U;s?<<17}0A zXx(Nf788+(z*=}2s*fd;=yO^ZwCy-8dNsALz?b8Dn#Bnec?J;9>juJ$sV`4ZBJE#oEcpM>`_iQAj_c$WTU%TIt%zHhV32Vg zkGvW1^>u1$rJm8fJ#`TWSW@38R=hf~8q&}K6w~OR-)tUi(=m`W{rmUtDVR=#QTTFf zCAt|Qt>sfSWo2xO7A-n_P#_4>J5xE~GOB6c0e0|%2r3PA{-Sx@$NLr06s zg~IF$>ZBd3PJ@?yETfK>5>N}9ZamIDsFGyo!A3hb)ih`Gata!!qmxr4?!{|`%4nMs ze(9w|LxRBNuBz<;dV{SEXO4ogRaAYJJr({$hIvTko_fF!zN?lJ<`su0_kS)eP1M`C zsivl;83I9Bb#-;{zUY`3Z`^(>DwkAaL!x6?%8`$nRyH;z$7brc6noM$97!}ic<@Gm z0pVE2*ym+GerTD&h?(BgCo|>md#N$&ll9FA*6rvr*tK76r_O$T{0#q#f%ALO2D$3N z>8w8XYAt?x@Ggm*aGS~RDW^Z(C-r4^EO)L2AdGNjA_{;+k-iPpqX?=F>gPbCsU|@# zt_faPJWywMba!`?3K3`zie26muk-49MurqY1~X$#?Lr5Sz&CIX_n(<|9r|-8;O;5d znOH5)z_g{(2FAt~Qz;j!kD7*#_S9{yEIpzrZ;d_z1^*?yQ{2Y>^XIp`eDx|i8LzMf z@Id*EP1e?_Zr#)i)jB|cQ{97;^RULJ!c>!%^bgsQzu)N|Y*xF$MvF`NVB|J4@_sb* z{{8Dq*Y2VmWf*l|>F<112c?k%8|fnu&1;Y&lMxTC{jZaf_Jh;o)!Xb{`r70ygM))Z zDiaeGC78(a8ED&jcKQlM$j8oEpJiF0qhl~Y#sL)q?ypG6_RxHC^=G&@l+s8zxB4R| z)d!t+pC65uux!kVvsStbQJU6KhR(}m`!MVN>VPb}#8X zE((OgwsK{p{arc84B%&!O^&!mQn&Ka3&-Ps{r;^Tdy~m)e?!-J^Lx6EqWh4vU(6Uw z!mI$THlM2a<6IJ^ZHE%(JTp?A;hQNM9ToNaLg6haHm+$(OcNOo+uL1&X1KW7*xa#4 z;u{X_+n49`OLWJM>!>hrG$;zdH_x4huDPe@4nZK&f-MTUK|r?Ic7moURUWC1uyiwP+?p)A+ui~Gos#f78?zHb~yaUH1udHN~t`qDZDnk(bt;-u25 zpNFNU?u2&jUzdK#6a*Z~NOV<^CR@thRY%OLUNS)d-s3XK4$HL=nURieQoi%==>^uP2F$Y*`u)*l$hkf?LI<2h|z5S`#rMw%NP3o zQE!>cJH*AUXC^OGUNS@bt)F&W);5TwzE%qsu-@-HZ!3FTUthU>8il##&mWzO4^5#c zY0DI3q$P{Q3}<{QP2D5cmx%`W0Z9Rv6neR?*|ofHK<)0p8auzvUw&e+t)-)b2_gt2 zx(%|jhFy;=eOt2w!Y{N;C0xOd9*4F;LjeTdJ*&KBi`Rs-kn_v-G1TCJwjiTc@XT2C zXSBGX=wtGX^b_`!KYJ?8*SU|>>4Dnjw*3KM8a!~wHtfWCMO z%-xUzodht{%m@t)rT8Yt`Vy_(;1+x+C^*ph3kv%R;8CFwc9$zxufFKn4B-J4eLr<~ z_5#4x%(eC%mX}L{L_{bbD0M;>Xq-WQrA0-DOsC+`GE;ABk|C_BCA2 zh+;zxtEkEK_xG=GGc=vXn_-~99iC~9VhnS>x9b2Q@=ky zvX*8aq@k0hp({5ZJv{aH>d1>)Y3$Zj@FyO20^V$|b~&$Nolp_0p%6cb|Ia~r;%Xho z`=4nZS5)*QGBU-7g1*yG0To%0Vtk}30PA~*!%gE;Ul-@<)teEpkT7gYx2O{U^N&vJ zGc-!w8=iKlNq7Vuga7`62gf@1xK8UKkfLQ)=PhaE>rtOG2hzP2*%N`Pgp*IYT-rvn zUTMYnQGE3c4aZ6buWl~|Lx5sO>wfohTPx%SEg=%_XXYqug@f=71ttbk=x zPH4Of^d_aN-_RSQ2W458ED2~_iA;r1)4G&_XR@Z;(sqY`&FNzQ6+m9{Lc)Oh2ro~& zGb$#A1rZ^deH95OPt0>)=RaQ_ammGVC^~tmwfy5T`H$vdD6{*cM{ACT?P1c@)veET zV3nI0UP+b#{Q6aB+zMFD#DmA)Z?1^bD%Xwy!}o^yQ!9I^;azk}_k7MEBEkwKfS!Rt zsV;Zo&)Szpxja}i8Ge~|9p%A$uIvDN_XrLSHb~cPZ+Kg2R-F)95~{**#K?#j;M)rh zUxHz_FTDLoaBWm&S*YM_92{Oy0fA*1uoi9r$-dBzqos(L2Bp&jyoW@U5KmG-u_Gj( zIP9mVJAWVI5M?j` zXq}Fx+!5OhDV8Yoz`FxK)Z~mIHJqMJ z^&kV{2b9nZ7)G92eSN)ps`)WUZKuFZd~V(H2n*vuF?%{t>p1*ZgSuoire+-4o|WYX z=6fSEQzk1b3+Tdx__GiSJP_~t7B1(H+8GHxbolULb(z$(w013gnDw78YY5^YkOl3k z87&C5G+GcBj7EaRu)ye$S(A5u=<7Y5Gnbu}Wrg=b8+ueuWMQxL0*MsD8X@=LJk^s< zBwG~z8_-R;C9O!AfYuQ3T(z_p>wr_zicj^(k?#=t_4M^WR8+9T!2bX+!N$o6#wltn z-y(a3A&M`-YvNw`n2D=lQR_Cjb_cxXs5Hp2HK`X?!wQPFmxKBO(w11B#jD8M?-z_+9Hm{>uiBW)Z3Kn52r#|g=d?7$`&=11ee%SP5`t*Y~}A@0uS99mI!h9;q10R=eY5|Vjb}GmvP6AzR`EB z+v;o{zLStYU$2O7z9e2S82~542le!@Lq@z)@#!5cfB$YtvuqGTht{4{{~~$!2cllO zc0IMnz^NsR92<9QU1z&=jy+aC^+kn)>pA7JKVnw~Psg1!{u=OS(v%Z<~x- zJ&1M%K}N2EP_LsZEYavAup5O&RN!F#DGe7vRXKKEeAAZR9WPPWsx17trPlY1K?y1! z1u;-^NIUEHr@jEul=1K02{w|+CTF!^HkoSwh2u|NtY>$Awn z$kEhLG1D8hLxT$S6ElClt~EyDfbx0zlrQyia#KO@4|rmVfH}H7CkrMX3BGS=I2k|F zZs*VqrCAXdSef=FDQP*l%vZ5R1?%nt%6&W5U7)l-%MXszhsw%h7DPC{1%z`UD~I`B zrmZWg6IO?IPY#r9fvoK8z)rUpurM(*d_;)jRi*85gWqHNv3_a9CC%QV*odGd)>t|$ zVI2z^A3W692Dj6vPy2=IzIyqxB224x5*{fk=NNam|Bf>yJ`h&29n!qT&wV@mM&Yy@ zh@(%EX_0SAN{YP2oS<{ne7MB&v^*mXmce=Fw0P@lnu z-=dokQkb??=Otvjl=59zaRI_7x!_F_N&pcJ*1~wBB)d!M`R}yC!opl#p;ESsWMyT` za~9&7f4I{Hex6ef-hGJ&C9Di8SWRuM@?|VBQQJSNR33~p4HS-47l{ANaKNpst@p!a zk?0+o7-|Pt+V&woAAfue7#6Au?RNc?vbUqdz=zf-D`=dYNR!6uyyAkzr;&E?IQ*kk zz)!?_D!*D*TlnfG8?Ei1FZ-`vl}4JOEcJr!t6DHf({CYCZ1Q>Vq~YQQmB3H3>Z}m- zym5n=4)TntZ&=QsV0Fx_>IDw4f>)H~I6zAZ7uXw~OZ2E;2QpKw;5)=g#80g$B0o&1 zcNE2SuFlcuHZmt4AjGOaS6u_Rt@CS$W6heI;CB2lCDERM8>Df?c+M}=T=y5u$wG0{ z%f7nPb*g7K#GXUZdp2&Qf3nL$vrY$LOQ@S(p`q)r2cieK?q&N!Ad9qVE730+>kW;1 zeEyx*o=n(lE!aQA%z|{}fu$M=hXe3{tB$lWJT}rqc;sA{oNVy79n6Ic!7)8EV~tGJ z$D*Q0{0{uneGdeV@ICk>VQ9o*WBV;8Dyo^VU57rN6)RV2KnO+?UQAlBs({vusb}AU zShvgd%|%6RM0@4izI`d_r}6O@5=KMohU}r&z^C!b&UW#eD(Oiz;Fs%@q268@5+G$B zg_o!&FuCDilx8jDM=&U@}$J2MuRTfoiJrl1pbp{MZvM_tY88)@~WNHnpp6X z>O}>wkwWSd8%#a|DWIp&B#QibFR0!hHy|8)@iOi_AmrxWUc-tOHnX0ZBvZ1k5e@}i z89}&O*3?7XG+?H+aDp>OXR4@I?tURWHAI4L4=6?vSdLC@w=MyXuu6m zA)br$wMIgS^krsB6b{~cKTlbQh0psnGrr(t^jpmelD&=O?dcDmOcb?YL*#nDD94+A zeudC(n>#ySBpXl~*1EZ_E?>gsc!BSQJJtV+7mc;H-}x_@Ztu^#3D5X1KN`=P7gC}N zS3CdD|66nQ{}+q~{Qq(b|CvXaTp2bfBF;` zn=hydRdgI%nTCR1do3x+`uid*uHj_@^Qj)H88rQClsnXvttcHQ%p#trbVxW!rksK((D$l3!}i=u!)z`Dnt1tyu*)U` z7HWdFYOpAOQ52oP#{b->J_+xl`6|zSD=CrJ)@Daez!NCq7kC0x%j>zhazL}kJnw-X ze=oW*|7}sNz)siV;>iFr+acn_1DB2p8nXAqxXj9>(@@r-dVoaj2t)fbf%|?XG7W5gSXucxcp@zLWk@f+ z8Xdw|1tRy0NJ0xKB@*kz)gY@UUM*HsP+$yPc34f#8|^MF$NVSm|Oaj2|=wuSS20uMt^P2zS#fze+i`n*E8B`R4^05tZdp|snys9dl zO@hSRz4yBl zo91R?D*j-d&u_A)_&6aYCL)_5Oh8evXhrVMKWk{|6!q45Vh6&FH<8DVs|PSZMu1KJ z6dX(-$FPzb6$GnDe9t0m6?)qG{COj^Un`H>xD|~{QuhYOa9Q#Kh)Id#q|)c6mnL5wiCj5NRh%hSO)C0 z4<12+QLZ1vG63Hk=+F&PxKt zFSG>%2Z=rc#K~aO&5V+xfIBAHIV>7NPk}Pa-xec2_u8sOo^ro)XQ}k%PWruj_l}Q` z=OLto6yMdpA$g>50v5*agpf7%M#SdD_9=)RBnwaF;p(D2JyC()!>)M|1T*jdnXif+ z9>dgA8D=K@Ya1X}#qNQ9U)2A#hNy7HPcqAWH zn{yL&7C*l71_s>l$Ao8OF#LpgC#iw7w6sqnBRzmLpD8^M5VB~fukVM?6Myn-3!W4) z{KEEJxelSe8S132&=}+-@O zV*-0(YcQhg^b{u|kgG%tmmp9a=L`}Ag>29%d@b2io5+oMvf=wUsKzPiNlz;ZYiso> zHiRh6r@j;yvmxWXl2U|t8c;%5{78n=nB^Vok)ArFc0pXT%kAf-KSTzZsR-0p@K}PYB84(LsX66+X$(4QtTS9!=b=omHjfm6KN4RXZ95K&LOI??M z>a9YV)TwhtBg9|_mRhap3VBAfgO1dzC;$nq7<8Cqlhh)xH2DZ#M3^I?RZ^ z#6T@U6vyHuo-!%B5Dmo%?ZITjUt|#~1vMh*5mrbLTXpW-IYfbun3)%#{y@Wg4=(~D zQtYw#Rm7Wx4MPm-ytyOsoDp30;CcD|TXYR~M;d2lv)BSZ?nf3<^*ijf%(y>`&x@}~clBykT2V8<+H zegu3T-pxgc$lCWIpJ0ZW;SQ+)}0%B3bAR-Qe1NLjS>_WFVBDH)AcZrbz` z`U42MOWz7+DES(hdt-um_g9s=jY^fjWRn2FL@}w8qvxeB%2I}iKL^- z*Ddj#OE~`Kpt7=hzzKc*7cV>Bp8m7~HVG|-@SzV%Dw+D)f<%;rD9IdPeC$HVC%%|q zj-ucnwh>3976845HL-v2ueFxS&3(jf9l&%HD~3StmOwL#mYM9Tj9-i&B(2_K-JonO zcy4>CEi$NUQE{6MLguuNPu{+^ zsM|%N$zQ*2FZ#(jbhZ1Z8P~bu&owM0CiQUv9X!AmS8)R*e2>j#vA2NA&$O^1GJg%y zH~|4qEuYgj&pkh(9gr)dcNY>9;1*o))E4M@AoMZGxhSgbf=8R@{4`F|9<=`%zIy(} zua9W`hUTAF*H=Z9cwj>+Yi5}%nmsTbMod=1KGZgI0ubu(p6F(mdw$_@ocdbGyd-*+ zgatwH2eP{RAQ?+kVGhL`dM#W|2HYTAleJ`609kf08uvLBW%M!$13bX!<&Epk6_+Mm zh(tvwZ<4wrZgm?fgs)6xz#ej0+Y$5ZM_=XY5-cIJK4xZ*Vq668vl7vmi^~_IcjU8? zl%gncadGIAWb6q0pv-Y`J|xio!8R4d&L4?a3Tz}Y7yz-FHSOG37D=_fgrTt3?1<98 z+xNf5CO;{IAm!u~au1X{$Vlvr1`mZ8WQ@HNnk^0G&|Cj$$iX_pzz;(Xrl`e-RaEkK zH#vLY+5FZs65H?r+wa&}(=W4Q_#f6&qZVj&2%#$Vun*dH*TSg{D3rCq8;!e-ZXkJT zKEKeWgym~&VX>4V(SnMqPnZ-%=7^#opdw-88a5hzSQy}{G#5XWrh`DC= zL|$3hSMy>gNSm~pn9!EicW9%aJ+IQfL8p=E4WDm2_@;Mo0Eg3|9uB9sW~M#w%fJX9 zSJ6_q|Jb^iDG}4;ilCO*C&v94iGQ%t}TUz_*&ns^8{BT><-UfE{d4(aD7zcwxQgjLKom8 zC_;!o4Y`8psApfjOyyx+)5HpF+A^Dsol5OXHLujJx{q0<=Mpj`)DfK=juK*uwBi-N zl@y+kz|wC^I7KNkZr=u;R|xbpl*;<=ZL)Tt7L|`#!=#5!rgFyj=Q1J;VmU|KBMwQl zYKdL_3GhoI(Ag+=q{vMH#?8AR86AB*p}z8w>J6w<|L7!oJugbe^R$HZg8ZoQi2HlO z2}1rYGR^O~H6x!}QNk-;x9R050FdCuk->=WbN6z z*Sf!XKlXWROt|X@tnTj#CuLAie*qDdvwM*jfFROfstEchQnY+xV)$IuGph)1L#@~- zAV8}gx`~DY2VaWBisO|l#u&;cqjCVWi&2*_t;K^Uau9q5qbE-#IL=sND(XFAUAVhK zJo{6orgZQ?Kn*tFZs0_E!l{Biw0!N_CowMnNIpjjhV+PyY<5CZa|L13@Nm{|*g*I0 z-8*CIxT`f-jKofVrG>@#hZmp0Q*>W^*-x zaym)65Oclz(K_%Bh^`tf{q5T~R6s4iUrQ$T+FPVRxF^Qz!)?c}lNc?K0JKVS-U3Po zu~y-NVU%W1=@eW8p^N{z3kKre!$s_e`$R%;NI~DL(xP|AMVf#CG3*fEhc(jD*2X{) zcMuOOu;d6%7636JM)!-vZ@~L_hPP0STL4c$9&Tt?y|p$anZTn*k6t;dOu38XPOpFu zQwX%|k6b65T|=RG?YI>rBLRRd_D_ZN@x3!X?ufxd1=EaIueLuFKKWvY=|05S3F`&_ zYC&|2%oUFIHL*5Z+|PTu-=hq(d(d^2EOKFF5DSs8bb@=Dns{etXEifa1r*THwI3cW z{)Ekj6A5mCnHU2MB6{)>9}qBUXJhjL$^lJmI0OIGTjU21k$<7{_gGD8DrgWwq^%f? zlt*X^Of)JPmY^|W+vl+pNIZsO<(Qm*$2JTS(UKv|K8!HR*UC+)!x?%%f}mI@4JU`pcdkXa_gkau3F zA};SgW2Uig^}mL1ENSzb5k#fp;TkTk2&^-5$VK`6rJV!?!Wvm1%}Nt^on+lZQ9+zk zn#zM%pCIu=-X7(M3)PO7T3ipC@xy(cP1v`hM$VZXllsXW}U$(zr6F*4Ea3 ztRyZ+WTa8k&W7$q_j1x6bOhrwOT`AQ=bNk-Ry6Ty7vlfRE9u zFfMbl>ULlF9A9PEqGcGR7km2lGnsKaz|HjU*DVov^p&PBr!88x@T;-0za6^W7X~vI zerNul8qXG5`hWSO~ zGi}^74Y4p-)O@oD?VNH$!fJcr|5`IKuLH@Q3 zTPu*ujPQSdC^Uc!$!~*1MV-2Ii52}7O!gO$1Ey-oXbys&WE=${i}n2cG}yrh(3shv z9ior6VhrMMW44U2VHN~{u*6*q+ht{Ws8eUpE~2CJkgsE1-u4BT3X_eTL4&hH{wevf zHRQU!Klt26vJw<2LMUVB=CR=N<+80&o|xIgP#&hKicoI?OHj*hL8TJ3n1bwb52K{W z-nGDBES&UwpNdAEzqj) z6R^^69UF%Tf|*pKa9dsy-TXUy+Hct>avGj84Fw>twTYjG!lEa0_Z>}GXha~Vno;9) z9v8lbV@onW)G0(5Cd{?UJlV>d3_Sc||DKU!eG?u&|1F9Kyi6A>7O0G_IVNR4zi5Z= z32Y*&GpH#81x96FDF7uL$#mo{gdk!cD?zJareK#U=hXg#AphAk-%s$i4K76wW0QZ1 zKp+v~iMb8y5CEGPi$=}^3^=tXpq5A#lsZN^Qpq7UL`Q-7j3)l8P|2bTB z2xns;9pIj>M1!xF2j~>VJU<2wm@w%^u1w%)NUX$OeF$Sz9wbSNaI}#v7)8)*$V&Ni z6Zykh@AVVcWcLEWc&G|^z$w=7jz|S^-j8lHf20)i!d-mdeChpl27+r+&TEk0fdY!( zQTiN4B7_+5{vD~RjALSG`*%qhH|Je@ot9RJe&-YJHoJz55W*B70wb17`-`Tmmji{D zTYL?{zfbRYRHC4!#){F68;DvS1z@W%Uiir68q1}SgsM%V!}u7&esqAA`B12DNvqDk z^a!=`LmAEG1(Q9NtJqVS_~35x|ouN_ksC$GORSQ<(x7(gJ^gRGG*Y371Sv_LsFa2=%S*Rqsq`w7);Qq zI|pqDI_S=wf%ka^%WwGVfU|hAvB{HKkA*^ZCUpuPF(}&BR>ptLFE%#CvO%9~H=BGj zz&)9vCPPl}?N!6}WW*{!kipO7Z|49dCj7lOauF0jMck`MJ)Qj;jevOpTQq2KJk@^W zNR?+GGSg5naD**BtE?3wr#{^T?J|LsOrlL#F#reux%Ek(Xh<3ev35MOJT7-cWyY*7 z%tv+!P>$W-G8!a^Xei*B7#|35_=!;Sde=7vIwkar7Ec<(rDZQ>iaN1CuzrYBLY+cT z5~xjE$OsXB#4U{hg}>SZWhp_er=(mq|k*Br9T3itBVu-kboXNhpwk zp1v83^G0^|UTh8)fanU;VzBu`nws|uWoKq)h>`@^-~*Hmtgb>b+Jk#GbGb%I-a#FmRYa?W*##8&;memL2>X4& zTB+=vgJ^TUDQ7S76v-K0K+lA6>Ljfs`@$sw(*PPsoZu`q760x}rI5W2Lfh7eP-G$s z7Ji!yh@@JoDISOiUYoZhKfs$LaG1DqC>Z4XA=*Kvc+Ndli;E|o3CZ4|K<_lKAmAC} z=1-4plV>F7$)IY0lWR@7;XrWHYeeFr_)uR9&`b(^mT$oENH^z4e}$2A9cmCr$KQG36tLQ4 zGNvIVD>4P}R`9dmZh8h#ALNTm)SnjAcYLtdUxe9#zPl?BC@*(A$(RGhhO@cUI|*Bt7Y<^cYs4rdK}KuSe;I$obI;?1$! zRdDzkhMiVm<*tU)ga<57j&6FJnRx*R!u%c%4e(|4Tlc#Zk#uP7?=D}6?tuzU@>HuZ zJWA`1B6m$Q!{1-c&vY^LES;aaIyw)B8}u9UKYWmn1HBnypprm-2o1hJ{^!Xhdg^<; zt_aS}*%&wjrYPm7LTFl?TS1K?nsD?l=P|MKWqg6Zh{#a;4q|@pzQNsmgK#k?UswVaKO4@p6^4HE>9S|Qw# z=S?z%*h0Rz0oUSu^?OZ89{x7Z=_#-UZY&RS7!EXdV_5nmcVOM}fq|2g^AyJ4;I+Qg ze1ezN3iJ>Q%856fbm~1VlviUgb&fT$M+LgcFgZ zA}CMCCSB0hL4WrZM&=L(Dc<0>HDz zcm-RGa`upcHW;QahPxJgauWW-f*dioluLr5Zm-?F z3}uH*4h)A9T?lQLw|@(Bn5D+zyS!;>X&C8qPfQenfruGsW+-vwc%CzlT^WG02~LCE zLej&~b;&s^$g(J#J)WCRd)B%^Sa=yx^q>fpK|P7yfpaCJ-GWdQiDL=Bj+m=tAQoRA zP_lZvz9&l8Z5(fdF!lYkAT$Mp>EKL0D3m1ULL#x)3JsVXb3-~1l!rKHY*aF^jKw8} zQ%KC5@RG<-F8NT>>LB)sp8diZIE&#iFGN3-=4Rnuiz4B08+sf9ITbaf2%z(MI$pA| zwI0*hyoqv$B9eoH1ju<+`su__L;~L?H9RZj8LJ3x7)!UD9Qs+4e1-+Vh#PoZtstzJ z(YT>K2ML$pRj?y3B(N~qEjTjPTbr<27$a`r8-Iin@W;qQa^pS=P(TJSj_{&F3Yp4z zX7f;3R-TcOKPs1oZ{tSA=vTnOwPP{shVa~~K2IXa8H+m;JBif?#kErkf`00!pC zp)EDBj&Pm!2bW@2K-cFn^kNa2yelUl@$gJOW~>$!Ody!w8w z07tIEodIUeN2RZDU8P&U2W9q<8{ElROEe0<=hyGSE-m9Kl+OBbJax_y_`JY-;L(if)4Vp5_EJ zT1*G{kV9CAM%_*QELz6?YtiEz+xCx`fdx{+1oHu~rphXf+v`sA?k=-Oyc%frDEX7_8`exO>A8B(*1v~I-NiCzyB;i$U&jhT7QNOLN7Ygk)_6v(8-c747cr+7iLFo6WaV|}JyaeP zis<+Bmstz)_8AI}0wzC8+ml{&kUW}UDHX)T3}#2;M^HQnZeGGI$%YerL6r)?#ag<$ zzzl}R(l$p#YzX>{(`gqs&lN87WDAzGUQeBZ?L(r`$YQ|2VB+_z^;}=@_S*BldZ|Ye zW-Cx&Za|}Hz9o{Ahm@Cg=a!rw0Q&8c_OBsZ8uS@8xI+3Nml5EAJUCtmK}6Q0-N&Po zq5UI|eZb8AfES)W84p0lkCRLvyTn?ta2%5T`-U9()I66wN5cnKBCl?Q6IpR)+%UEh zYaf^^HPI9CpLME7At%WD@(%!`VDUxC`2g4hr>3%#4P0(9)aM-_lh zObD-m%WSd8Mbkp%=6H;I!;jWH^Yn(@A{a1ki+d+;rM&@zf72}!64o8TEvBq38^pqO67 zo&+$H%|cke9_L=7%>-B){;f&cKs|+8LFyu&?`!7?8>mF=y4!Xq9 zKl;ZL_4^m5xN*{AA+*rfSBB1}s=PZr2J7w=kT=;4q`$)?JA4Lj#{(^joM43(--7ES zn-@i#oS%B}XEFmKXz<|-a>9a)^KZ~lWG7aL7#Aao;-Nf+<;I7_4#ejHJ!=?Ky))i>ZeQOVTl}-O zmKi6Ir=Ed7wq8_J9l+8T5SW$wXAwV!dnw1hNJJEI_s7j0fKPE!MMc#}e?JCjkulL8 zO~-LcWX|Y+u=n0kQKsFtZ<#}z6$5QREhLyRC%{Z$MnDWLHW&a?nrsYu>0%1?>+a9GtL^7my-LPulocV;8ja} zLcN50&*`2xV%=n#Dcw|Py<9{!48scAPKl)IkUEJEd7I{zCLdHSy`4TSw;=GdwtxI| z<4szzt+jSg#&ah$e{ppe+1>+2{}+XLbEQNmjyEf>A`5+E-aoA{%aXJPcZW7Xp0v!I zWB$hkj@1f2ooGLiXWYbm5YAM3E_hySpcZg>y9`SS8SjYvUOXB7o;lN(YzomiN+|&0 zmL3t(gNlXWL=o;{3BI1O*2yVrsAojxDkE>gl?Xc*FtH))ISN$M<@cK#SDW?1e;}F# zBYIt^8T_SdGEsO-cnx%1t4p-IhH&5YFNzPb{a^^_tynRy3Hl&Q7wF;i*NF-u6sNx= z;~vzDo2lSaOwo@^vyrzu$sUlh5ePmOe{hnO2)HLjow2KG0$`(86|WW|R3=KJwHXTl!yW~^69ymppE*jrsW;N z)$gE}G0(Z$Y^&a&H&=rh29(-`;U7b{yPw+i?W@ZJ z*0RVQstTDU&macpWg^$7iaB{eEapS!?_M+C3^DKKIxXNf3fp*;j~1B@5EVp*J>%HO z#@xGC4WB|I_-Ox7BGR9*1Bv?{9vkp0!Oko9;NioLfRRZ>SMg7r^ZbfVojP@znypzK zg##h%qi1h`ml!D-cBSc6X^PMNBLifqWZlvw8(b zFSwC=9NEBdh#d;QYd1+;);d*%ga05$s79`dEw|B5Ha`iGl%eZxQh=8A-7w4rEQEUn%hT%ku+^BO4Xri{WcybrWpZ#qmxp8nSw5ES{*ew%%YOnf?_i^^W+bHz*al;<+h@ zDh%FSYxkx58My|X=03aJm04YPT1+bOn-cZm_Eu&;HYoKGz`hc_40lh;D=1R$ze>vW z{>M$*>92mnimLaqzwWvV0Vn+1wpTYbF{v`=>}E=f_OTcAVZ23lQl;@@=vt6d?GAS5 z6f%xR7&gO;*eTJr`Z}JEHzA8&GUdy@b<5HZo6M+5ITz|adBTJ+^Ie^TB|iwm%k$Do zMp!qcBUc+cR)g-SQu=%)30kxiC`G$0cs)eOF;aV?Uq0NVfTV!QVWU%QDc8l+uyh%< z+_0U^xA9?}b(RsZJ))l8aH~*B6#gPcB8UDlZVNNgd>vO5`6d^}#U<_@7`(OR&>kmt z?}1Adhj!tfX_JcQZeS|lNeBOFuYD8$O5z5`!3KztETRONTiAlJ#A`qco=QU*f)}z+ zVNOv-Xs@G?n!2<&--BUv2T-e}>r^Hzp(R8Z(q-PW-h8HE>87A@F=wvTvVT)du0%a< zxgENkZE%Z~f7w*1O(`ljJ65~LF#OBR%&HLB-2I#P?~lm4#A+E%BLMZ*K7ZF^$dp*j zQqqdLh~4-SOU|KeS1fe4aUHK;-P)pXlVHTc(Q33tI97AdXyX=E!vIP(ifWc&#GPpE z(qTk&OIBc@FRT>n=?2>(VP|b&6Vd8L!+`DgcPwwN#BWs9D2!1NMfmO(7X3gtE?`zA zAqMGyIVGlPy?=>mQ}TO3^2rj1$6^e;2I4BF*Tp-SVVr2R)ez>@v1)826C;X0A3drA z4^GLiHCg^4lOHfnF9{BE)`Uz7N4l{m8j)<0 zoHyZN1J5rtG2RdC?BA}$h}FAO%0(julc?i#`-zN2p$EGHWjd_v$9ECo3&}^>Az5C8 z_UB{4*3%#jgg<}^6#7c51wVq#&26yI#WkkoiEH7_j9-21q(gVm8^Y-fjp8XmUsI6! z^NCI$td8VR)PAB_V-LNb+lu=K{Gr3aWO-FL5g>}OJwK~(ZIoI%Tz-xY4i#~bsMX4} zd2w;EarBxmpJrfhT3edmKR&0|bzTLJCM$F`9?+J{9NA=bV7;h!t9ZST2kgn?u6>4NX`1vThMa?hs?O~ zd(%xak}p!P(l1FzJ(99;sxVcu-}`R&*7s<1C-cgm$U4!Tp#_&b75ioTcSy}%E_WGm9 z8~>||10WE@EgS-G9Ba4vSHn(@)lkA2r{;ly_s%ML_jH}ZgLMcffT<^CQU?!{fl#sa zTejQ@dnPQ643JsR0*VoB5}!&?mGS~gSEh(s>AAEqu0E2w<{k@Z90@1%z9|V5320t~ zss`-17;;j)&`6jakB?XDOoH%Hq@dwqM|l2FWRbpJN>^5+IH+%8xrUyRY_u6 zAE5`Nb3+*HGdd?7>_o<<=#>$k_7|7yJnruI^%I+I7h61?;EYH!^5S<9*g&bU20mST z0Sl!tCv$yzR_*8RkIpVvYF7PGn%7z8HoRVNZUd4ukZ5x@Iy?(P1?FUvM4g7 z(%Q9aQ8pRrH^)R*m9atxUo~pDll*VRJ2`L2;wX<+54n&W2tm|1M?~-qV8t{Yu(CTU z24P!Tt@&Jmi%^^H=Su5hBN#=JGP^sgYa8R<;n!;)@i~xkby;UNInHF>m#$G@#m}kH z=L?1?7#aK_79G{cJK$zP`WJES={ZCc#(}ysZrb_zE6m+@GG<2WW`gtc`_HbR1jXs6 z&8V!OwMk*PSj9-UT`1#@Wz?%5a5SX%(K~ug-^JN(pv(RYN_DyEg-W)f8~PD)_>f`4 z9H@NbFEMr?zT=>DIS-=X;+WQzW%`$O`KF_jlmpFCQ#2z~i{O?*UCe;?ih($!CXrwR zeZn7KvRa8bn}Pv_3PfE2V^}40_WQ8Sa98i~Hx1gf9lyHqSAwW*zh4GHREUQfy)@?| z6{N^!tB6UNOiu2|Ya^!Nj8+0KZ(;3Qv-^)_JAk;jl0?xT!Hc36GHOX4l;(7~#5!gx z+42T#HBV^h!i6TBv$c(ckmDZn6F`-9avDu*DMfgH z*z)nImypjAH`lQtWbkQ0E9L-h)7=SSi~}7(=${U2Cr}*xg}hrXfQo>XTY+@HNxC_N zFd-Tdu5RS-q^lws7pWXQ&EN&k(p{M#MVeThS*wd~yO_Y;T3s4|LO$_;gw>@f^^hZk~=SZ5=teg$5}S?a_`A zB1~fWM$#5H>^hni3^u+s%^Hk`F0fvzmShbT=k}l1i%(Jw~KVKad z=?*(#oWE^H=ffl0-ICc2JPk$Uh_#w;zVz#sf0zcmszQ%h?gOPIL|WR_F{2!1a&!X{ z8oDZrR%_*<3kO#}S=cO!%>xN}iF{FHlyU?m&p7x1n$YVfcLibd0frrE_o8Rdw2*bvfyEbT(AH zIs6JMf3tU;)8-v^ExL~Taf+amWW3MVE_GS{;iY$-v*u#w4J}IwV-C0z$Ma2U z-Vf=#)e^>Q-=;&5@HP>Ea;yaHDuj8~-6GhfVO20z*{T(&P1PJs29j#>@P- zaijt|I1Mo6As_dUDRJ3asrJb2$I=(5yP`U9fr2moPkLpH%t1-5GlDmxUYeZnySCX0 zwPk`7b5cU4}ih;A?3*1eWGuqD@6HaqO%5<65Ub z`(28>e|=dSmQP_yl5&kF6NqSz#BzecIRM3`prF#$yNPkHxP`i3U6_K{`S>VGo1yKn`9o0k3ZL>w|It>{dTYA1`u#5m}*d)KB+@LU%k}JL^@JX7NVtfB-Sr_dGh#CUD(;?8f z?Wc*0$8Ay1&)oa4Udz($+sNsPe^{>UJ_%GoYWt?_DBE+=_~-LZ?Aa%n)}@l!UBJ=|##Jd!z;XbJeto&bqQN0$)1l zwtaGF;E(!c`;PCe(V^ROY zs)Q}s4$5Z#i!Z2=nfeG#c_IXqFk?B7tAS6}BMwtcBRhUZqASz4G7Q(QUAroa{?Oi% zp(BGTWoO@W@4J09XNfA}W)1z<$*wxxnY<$AvRb&XKwiZ+8RBG3LQCpSbEmvaO|3PT)z%7|Gswk-Ghm<|cKOu(Ca6xG5_Myrur zB1nGxbU!7nTiE>pav;L#Z-?hcj`QOnP5CMe|9N7f{JW|jZdC*-IewKXeHn`pJNfg| zYY?6jFHt$Ht5??Rv_BzGa@5`P zK{+yIPS)aFov|ki?HjTEWH9RV!0vN38qG9dKOHO8&Qw!<_zRI7(j|58+erIj#+Ah% zoUFYWH-l-^8uIWdpM>iQEy`HfM-Fx*@uNr>h&WbM5KO8|Eo)suzELPxxS_7NL{Q?< zUJi5!y>MatS4{nO?;HO03nNB4;KwaKK0bqT%%D8wD)F+AtCTS^#;vA={ldY9;5#O) zJku|YQB9ksH>H5YS$syJ?y^2h#mckJ!daTa8jwpP0q~Dt_~;9;V9Rd zizcOmoo80>9;y)|HXJ_utH>Hy3|W5p=+En6t3XRnSPYQ}PL4&aBTb0ZOM$lEO=oJ9dxN>%)%+uoOcuRU|uU=J95NmCc&ji9PioK&8 zc*^;LLA%liBNS`|Stn`oTjAp|m@!H0sjt(8{*PP$nz=gws`_~K0W8Ik5?VCW>bZ=) zk#C0*{7=snX$9$0n5A#nJ!U@(?0P3>3IN?$@P)(3m-<%L_k^L z!_sZ)Wns?kDa~m545HKb`-~ZT{V%n$BvA_}rZ5IGM*7?;52`+0j9#_H(4oeC=?<`WoeO78T+JBp?RAUaUTb{mt16 zdvj^OH3d>9^zHK!HQWtEU}7LgWJ7@L+~P2x8B0;4+U*2}17|Fkh{Gr#jx~NdU(`d? zH=_9oEZM&Yi47BCVg{(c`L#FpER6D3L?%`Ts2!U3r&b&u7d;!G7XN~_;ZqJHl{t42 zsftF9l2r4F2=9L}3=O(m3CIYQqS*KHaH}Dff24pv4_^AhJ>Olqxg@Up;1ej}ZSxbq zJdLiyHWt|&g{S{FyI`V>zJ8{@@N3sxY8Pf!#B}xGmHpwPiGyN5NN-r zMKa0q7a|rplU39wlxK@{QdwpdfMarN|4g0O5Ne{`Uh#5cW)h>kLlF0_GvRhcUP9u->t`FxYHg|x`eP?BG zP2}&FFJI;W(_aln9}&tTbl!Dfxwx0W+HwvF^l!!&T%Ovo6cSC}w|YE#+*oO+v88m# zC-J>Ewb$-G*KbUb3CAg^Ug~T})(>Eqh_IvL^TO>zD#N3(Ox0PoWXr~lK6B4Lo$9r0 z?OG^lIW5B0EUAfD?#rh9NSjr16d17Dho!b|#OIT=h8cBv`1tKsh0GPoEvJS3G<=bP z3{E1(c1i5nr%yAQ2%Av?cfUWJpe&@a?`TL{ipuq@4o9kYV^>I}$GVx!3 zo&C0i?mSXD#)EX=ZT8+H1&ED;Y#oldTn7neeFh2@3i37J;d9Q}<2b3R^Ku$iuvpgX z=h)?kPoLuZ8i_=dry++7ER3*!nSZn%@fdb!IAueqOG6~1API4Sr0Qho=|Q6Drg|iu z>xJ`?8i`*8B*XYat=rIsM^u5kK_!S#fxF@{OLUIwePJ*n{POdq;;6}t>YqaLVHq4q z=LL}jW1~?T0i&E}j+m*6LJzy~yq;ng@nP+SwhFN+R4DM6bV)UQ=hLaq{bW}J$!Kdi z`tx$(d@QDzm;5a~-b1QZd{ae9MFhT2YyDolVD+RV+l>~xIG0KwhwYCj$ z82UCftS*DO0z@g|_V_2)*Ma7gp(n(;65d4-1E}$_Nfq)0I>J&_JzV%E;vVPY)v?Ul z(2hDxj@w42)O{)1p+%vw=&Xd7RzORPMUM~a^-+Ek4J-}^QvS0@xgktUp%SVlW#ilz zeBfxR139}A-#?4{(-4)`WJ=0vKvaB3PN0?Xn4s6%O0Vx2t4|-~&l~G;Jou8J zl3%VJJ$>Wh!`d|1s1k?oT*A}o3V0M%D{O@rqG6XEf?8G%#LRQ8+1hL0Qkg=M#s-;q zM2*USrY@O+m5!)%j+naYN{N>ibn*)omjebimfzp;!02NP6p=M0wFMw zv=tSJapHV>3IYYQvz<;4tfqxUrJu1kQ8oc1{*(EH!cW6T|?Iws{uvgDvd5g4uKT> z?G|P>y#!}SRv$Yl8q%zOixxZZ-jzmQ^#3dE1-z9rk9IHC6qd@P$()0dC}lBp{3 zxMvUsYtsPQB1@)Xj006oX)%E{E(TPTBQoJ79VSvr=x9qBlZDx?%dGm8{0TSc)lhrh zreI4;jV76Q1p%T$fd#DwlfdMMcqCiTBjtdHPCv1#zD^b`eKNGf(yi50Rs(C&rL{AC<8*PJp%tZ2ph|DW6wfa1j-~x1e6%Vntl~HHm4gPRblPw7d67&+ORQuq)8%;H zLi*Aq&mMJVPR~M>PosT`Ye=jI3XM zrSeZIEz}?Ji-Ls26qz!NjlvDi*8v7P8yo10k21J{H zWiQNnCqL=2$#Z(L*ZZs^M{YfO@cqGe<;9~B$D3jFYJnd2%BqqevSg$AEh)~t3g51J zsy)iKjzXc(TXLx1vLG66zTo!zoE#3gdCZz9#J9x>xPE0(*3M5^vGYG_J~N{)2fz72 zxfpX$JA{M|_Wp#VT#Cp2G1INc*By_Yk5%h?4BOuHB%KdGs5;KjJiyvq!buyh1rK|< zZ{KbW1JUsNkLn+&~6F z=H_l+Zt+~_A6|Cla~J}jJk&bf%SOIfk=4SrFmAXD*q&fzkdvG;@XW}A)=0URFY;J% zK+2)cQ@rHNOcvn7DP1Jq^lS54CnUlx_*}QS<3+Jr*aUC5LSabJ{Y0Q|Orrl_A6q7I>YXYffuhIB+E0!-m z;E2+ZnY8P6Pd12M{US`5Ct6+&NI1%JfvXkTu^v4#$TP72$7RUAKCM^#g9i7u0u=ePi z*Pg3auL=juvx(}n7OGYGy=-3qw?Gje!~{`$Sb7?>J1@N8oC!oyWAO%mFjv+nP1o)u zv)+#Q-UVZ_7Uj~4u_0j9S#o!uq@8o9R-f<+dlndM{m24sZ(+Ny*6UuS>vXE;sLs#;7^Hky~^}k8`js2 zg0#cYZ#RGTo?VSORHEYqYKRB@dMANS8K{aUoHVrYVwh%5(UxYKS#|5bxxYVlil^Ry z$=whGt10XPxwz<*ZLe3LiCN5`(Ewa&cqt%#0yMRQjy}?nq}3NUATX3eIv`o|C;~Ez z&d?c-iH?qbU12jPMfnhl`XU86QV<=`f($<1OYWPp!O$8OI=o2>|C9k;Wh%f`tUJe^ za09`ef!xfag`_?g$vvorHV|{ zcubq`C0TEOR zaMW+!ywPwhg+F)8g#mJ`{W|ao(Gl-f5IIg+2_& zx9vu4dy>~`Z)j*ieeFkMjC(Yz$ou1VZ;2CFKK&8}da(dBCWw^<(BGKDr4}w+7*BLq zj19j$Tk@L?)p2(enqNvw{m|pm&E3cU8fT4bS>`%s(&ayXSGk8S#hpw&ZC=;Bnw_sK zJvZZiW8Pv^m70yt+4Tc8cvD(<&%ge?XIgqY$7>HlH}E`NsJFS9>SH=KO?M3I)4O*z zos4arEJ*vzAz$AM`ofdfB`ORPD=Ldp!`iB{^+XfS& zR-Ih2=QTx(yjZJlALpZF=ypAM%y#P6akTv639X56vx~m+0hi#sRuF44%wf%Fs=dsL z>a`=Be$Qm@tACU^>&VA(o=hl7K)Kmur^+d>c%u}K)FC6RIs~M{{QpG(!2ED)o1iJl z&wkvf)+(E|m05E5eMHGI!eB5l+yw{1Quv&VSFcVovvFXV+18IQN;F+ty(?RmFBAPq zEG_GE({A)$)Q_QCWyFV+8%TYuLG$^@Nx>vx*%!jXo{*DRKf#rQKbj=fsa16YaI+EK z^I0ha`BYQ0)2h|h)lj77utGCi)KkX9#*V?(!M(?i6=nt-Zk)iiBuVnL+kQ9i%9@Eh zj@>W@w2|k=wJbRE;{`fZ=d?iupi|v$Gq=+d(;(pSqrcX<`+PCZ$3(Ux6^nSdq1Pvb9Y_c?q z+A=AWSlE`@$76kEqj$$=IOuemVn-`~|BY$qmK)~~@ju?VT^O;wBZJv|C57Q&C47>% z-az-S-)Djyt)ZM$vB%E5hi^z*^K~wwjl0)otaS*T|6(c?cOn#8oFmv&&gp+9Fv0oe=QVQ@6Bt@Bq8JqVr!6hbHzmJkW$@1lwK`IUWW1YGiQ_{sfxp%l%u=MvC zAUl`}Um6OkRc1Y*yPMQ*dEB^i)0>Ju!maOu0H zT$r^h>T+373j{;H{0ubGvsJ?{Qe{&O4{$~C(-*ou;0IZ7HoLF%?S)t`rUHl9FXJuv z?WD=`BXnl1CA=)bQz5JImnVKUBN!sNSURr;*Um{qR~PdGJ6D;`CJi)j4dhl$YQDR* zisJ5~U6YmLKuw8ct|iEh7EOkZSwFioViaoT&<;z}Ds$DIU6xv!A4yRM;BfT~4h@7I<+hS%%? zxq%2$63-qF4-bYdPM}|qm_LbvIiQyhnSVz?jso#T$BE~M0E$1(D>`T}rlpOq)llp^ zIE;70>xW#M3sYhT_;~Hj%&se++dEwQc<0wc{a2hv>leg6@MnHOq8NdVCuv-}mFctB zG27kT+mZ;EgcT8Ywn-5yL(@RLqmK1Y?jJ-h zAz~dD0cpi5HM9BoAuxtKXLLtRjy;h!uWg_eMQRaH)gLXVAE#8Y!HdApPKcXIvrp}M z{cI?Y{g(xiYdc}lK_qQ&Z~tQXvUx&{(4g(XnXbKCwb@F6xe6;*D?sF~WlipSi7tyz zV<`*|pU9d+A!MObLQc9ca|d(0!R(Ji(PCiz=y_}4n-6@IFHa)(@!4IXa}qR5w6mfj z07G0-X7oZJ2dYys;&sOK^UF#Z4-SqO*awnc0zYGySBb0^4;Kg-s!=|y#yy&D-AG^4 zaM<9*hmPW$b!S`OnI{N;jmMnhBTt67`U^SAMf&O)vB&L+h-$*8jN;H5Q^n&!cnYy! z4*2j1d3|r&;dN@$VC6HJ9Iv1K@}(XFv=N`~wGYV*gz*bVuYvf!JNrgXk!G1EP%|xo z$(sB0+(5K$S4f}r@^;LYO~ZFP+MUx&Y7SXevu@odtKLWKf7!4B6?XywxJz|a7Q@7pbH8uk7@AOM+}|$$F%aiHYa+M)fYgke ztU>sqkf6uR&}w6|q-I*7zNjlSkPR$sOKx!zQzeeB_DsA6K$bGOWcKKN6Iqp-CC*Fo z^+-i**;Vhxt)kTB-?{}?vxwa}ol)|wjXO>8p&YpnW+z6)$z7uS>(keCvTiCsP!8)T~~8Ye8BydeN}6 zzML3d%5<7solW2*8`{8U*)+35BF7P!cx59ZO5-P6XOFtUQ9&&5cwqMu%1nwg0Zh`O zQASPbCw_-}&DO5*H?1sLNHctCyFD?}^OPo64j6=`r%WO-`QmQ2Z|}c380z{I4PD;Q}T!YDjF?OTO_ zoTnXjC1jHvJ{1)uN>lHF^0^(;SvDI`j z9)7aeAuNe20^BPkQ^DB3a&kZ1Z7AE>vBPg;t`!b?2+m^hP*C_Fdg$Q zr9wY_%>uIel?Bi3hn4t*UGhUci?zWrToH~F;g^EDM!N5YfFw%smU<@Y!M+bF7{q~!D4qC3G0FJk{1>J zj`?~;cVP_Z>&(sTu2jif;+&)FANJr>n&8M%KS~r0d1E#v8#Qaxh-3b(!yWejNV4CB zar<}$^OQSjzboPcK{)hugs&kIk)c&Q#riE|ro@6fYNs_aCBS*KN zzV-({|KWeaP>TV7{-<4)p&UH(&)1*EhT5C<_ct&Kl0ESE*SVVRvK0URa^yqzf5cY*30uwvszVkI(XS5KI{42*v{}e ze-wDVg<2y;BtmYWz`~_V_rXKpa0yS@3r1^nyFEr}2Wa#}<@Q_q(v1+Q;UkG`D& z7Qwmb7^}&!Ns}h*XJzTs9m7sZ4B3 zVo|#NNH>+A)WMSE`?D`2+wGASCKQMvPSt{az^tcf@(szV#RciP{(g32_eTOyyI60? ze6u@ZN6VgFbT!Rf2hZrPJ3gvmT_gqqD9{2LsVJ6rW9#NaH_nDP5inK+q-#$SXxz2x zhJjEek_Zi8h!!1>neJY&O|7|oI|Gh*KTL<@dm1#^Fdf?>pUB3>=_@z+!dclXU>myk za}ll_C%K)X*K&AmJN%@_GJ_H@TYO~bwFpUftltYM6=`dL&8A3XLd<-csq$=yjReL3 zy?EszSyEFemiMvicdk*vvF{v{n8+^Zv?oSG;g%=W9R(qeQAfg7_NPrT;=D)B@_h%JR!(`*! zFuJh0YTK7!$O#vg-^#qeus7MC9hMAYdrUe%eVaJ>^LWA$$Ac_MvPUG!GK@2E`>s5r zE!1CkV`5aKm*PcPzxG}1Lod@CGWPz%hogkOf;A36YBU?Vv$L0$Sa_M?kpHKEE;dmH zObq0hKO1XOh{GnCVb5V%(u2C-5Y>Io)3L}hN})v?4j5zQ@Iy;^9c?hxn)?d)8|ca{ z6dLPO=0kssU3{o%1?s*_i4tF~^Qpj+;(oTX~{mj=@%#A4CFD!z8!w)4B_78 z_SX6`^1a^M{>?cIk}O6+w}|Zn(V+NCIQiEUQw>vQSzsLx&Id=jk`jCo1k^s%5s^F7v9; z({qBTC4?x)j@Td2nG3RDNrgv#eb*16?528)Hs17L0{tW*hMTVAKOX>ZY^B*sC3Vav zzu_KT$GJS8{Yp4LlN$?Y^6XKP5#NVD;jUek`O?Mmx%zO-q3h~qj z;=?d)0R)!pJE2)vHfAMP*s%Kw=d^xxop&e36vB)uvuevHH=n|+O8}Y`C&(BVoKTP* zS+iawq)6dulR45UUnck^XbLYsD{jyfbpyvy-DW=c3tBIcmO$?pSZ zM^|V?XzP2NA`J-Jy)kLcKMsC*7Z{4SQ1FHZg- ztve%*_lqpKFJK#5tUHimh<$sbu1Vr2Kv!)f3;PKfp<(l{ZTfY%v-SO|nX_g^Uy+UB zIZ!rjO-_|2y0DP0cC8UZch&UFJU&equx2<&1f)ckT^>uaq$U#b@p1!Tto6XPH zqXSg>azovVoYT;P0fli-Mp}-vOoet~8|yPmF*o`AP}r*-%igj#q))TAQyFuI`FYJcb#^Wr`Jmc!It%BWdg<-}=B;?uCu~k~ z8x$VWAN#n|cXC(|i#Y%&Z}O<_tMSI|PSe*JA;maMkWjf#rmpIE<0CyER^L zLmrM}?Wy7+&m4-V!bVG6x^SVu!=SD_8uN+^vmUj`>RPj`@e3LquU|KyRl~3OQS!j9 z{c$bbv^2YI2BkG*$e6;=Jq`9)0emzN9`|S!Kn{;({Hj6o>yRS@R)7AmX{l=K{Cw7M zH9k_$l1jYAT{}Zdw^2QixvRN*@yny%@M^@YCxpokDJ(WW;75}~^M#fI8YD6Wpwn?u z_ZuOs`~<4MLTs_ym6ocuse-tHzHlp_N=lwk^;iSVv+jrpk2{zPKK9P?9eZ-URKY|t zk77!l9F`Ig>ESQs0N~F8QQO1J$BTgI{Q*S6Dyx9d4aA}AA65&ehhC)!!pWd_CqRr) ztjK+LXRjcBHXys60W2A@5}4B)?%i^%$R`-oOQ0qi5}TftWl*i!7Q5Ni09L!>oTh6r z>D?3Z;rUFHNneQ|oI-}X4C!)ljLj2iAi$6uj4pcjd@?cPG4Z1aK`&bV@zQ3(R^xa- z#HJfz*8`bU5rDKZeMhhE?Bc9DEHGPmLgBU>xSFZ9t*Fm+?4$Jgvc%5rfRK}9EoW+{ z8t;D8yEWa!nU@p7Gz}0iBvxOAs5r1r%ln!Ek4LdydR_J{d-B? zp#VzwC9slGzg-BKiwLgm4Wz=HB*3;p><(uoS|rpzgm^be_(K*E*;#*OP@AFdPV^%W znapq(Ac+Q~1#~LhatIsbq|moz?IgNr`eNi{b)9Nlrd6bEMDxzxHq*!5Q!oWu$jdA z(NeA%J<(3kUd!0@E^DYGDixrfat5+b_?-QJ@!>M~TzAWFxiea{ULhybsdB z6+HV|o6E{Jm!X-x!gvVtR2oD)P%BR%j03(1n%xuQ^)jyZf`|GX5-=(rkWvJ~yicy8 z5VvCMH+ZJskTKrXpU0x&Ph!W(&A0oGossZCqI?FhO)kXiY9!suo@)<|JhOjqPpW-~ zS+DI44i|jswtjSbVf>YV3_0@*~Q+5f2W3zWl9M*?|RJUu11` zNOvz=_Ml+l*Y{JFXRcVw0=?@)W<3R_LN_xRATAZTAhL^FbjC zqRRSZt_c6WtW*D6DHYtBNDFJqM zRvb=So?crOyK^|_-9`|7;e~q+9XeD$_eB#uqvd9swznDZZR(hq7g;gC^?PZ5x;uyO z4D400V#C#ozL>xdrM|h9cR6#UWy=~-x1a3yI&hgDwtlJE*0P9q9=9W%YDK|myc`Ph zAI@gAp#sXt%9>?5$(YE)V-7R4!&_!I8nh*J2Sp_UdhRVaK6_t^$#hhBLmRB&pcNd0J!ZnP_2({oh zQsH5b1}IN@V5?Q|;woCE$Bfi4{M=O~JzJ^)gSiY?@dPhCMA_)6Nb94i1Q=QECKdT% zL3+r^{OAmdy2^!P7H}>0hr+f$c{29c1EOp2ifeoJ>|sP$1Ulrw_Ix_-ye+^M5@IgD zF$SO(2N+)AP(fj(5Kym~O4Ply{5Oq7eK`F4PTlXp_>fM{Wp^g?9f^L=l}rL~@uin6 zRUaYPVr8+{Xu@D&+N$?C5hZEe%pNjx=3E~YS8LSo1zV=`R!@kT`YJg~Qwle};g5)+ z^T*NhN-r}CNZnv+TB4#jagoLRZc+g`?;+=2!~9+t2%V*@T}~ZYF-n$E$Sm>m>uc>L z1k*lSOnXV#S~Y`B9r3!0LBqsvsVRj?{0~2(UXhs0N}9Vw+f(@v&m5gI96gX~cg%FP zhsTJ3uAp!c5HTuwnjS*3-Ux+4of%cfOYn+YSr^?M;^QBc$M*IIwz7qUu#jL*#yJ$* zt&D*5CL=?iGn-V}W(HL#w38~HTg)?XcqdhpTVVkGyh+!VhoCO>1{?ZP*U1S(;bOrk zr4u_(C}m+7HlsiM;P4bD5ir?#7O^Dm(ifbeN%beBa7axZsk9Oi?-N}2l3Qez5zrG? zFTRC~aUg;(AK;ZxI?@e;`S2q&Ub@neS5z!F!ZHytuxUXa-;9B0rifQ-;0FD8IvAfl z=B620{7i%JV@osbf33B-cdTc{51(%Lk1(3}<4a|QURwglV-AXKgb6fhtvwds65^2` zJ#a)OySTK_8S-=t0~Ep$!Mr~B&QnfeGs*svm`g?*zpi4HIScwJJTuQNRcJbO?KfBz z&_ZExgr0y8+>L||p>W7vzTh1xf{FZs;yIR%H9ML{O!w#|`6)K)#Y`W{`_cE`r<67y zA`YpOI77u@&#H=wmoE%0Ei&lgyrD`a^TRgTb$DBG`4^hxNr`weo@;x~t5bGRup^A+aa;vU5~!e12Jk8=oIR!pjl2slzJL#lD#)l5VuD)n)ZWcDtNkm z+7F`AeNsDgpkTEXsZTcodi1Bq9{UY&Hr~m}hnsCTMKMQdY;1|VUVZF)I1grkNz+=5 z$CRA5;SO}pFUwPS>T+|RPJLjT*LC(jxxE1dq}hMERaj7*oSHhD22hPV*NanTH(Fjv zVHMhaveyTselEo6kHy#FAAw7j0mX!pJVMSBiq%HVni=IJ+lDZaebV&Xhg-eFdP@Ox z?`XfUmZqzHn7Ktv73H_6!gAAU)v8f1qzXyw`I?Mfv+=#)mw%3(d{#Z@*q4IrVdi?9 zYA8%CQy=wAC^ zLqZ!QJ7zu-d&fZ>^E@H zr@N-L%f`>@ZqcoD^F62TpXT>;tF247dHej%FzoTaP1Zu&9CJ zAyGt5l=jP`kHDoc$-@R8PXu%nVT9Sk0I8 zU-3q?W;2jdEPx}W@WP4GPKHR~JhQI5uA)&@8WBJiOf{gK5xpe4QUp{Yxm0zn!3goT z3*V{pKNT0RNelc}v%vF=Ci!Rs$w_8q-^TM2ViqW(uIHB*3`VT5Pbm-ty~E`dtLbbh z8KN;tn|-H{?XT|DKu4EOGsv9B_#UJDb~1(V7G;6H1jY1iRWxe?((dJRPp| zYsX@yo3y&&iHukot$YKgv*1#6*oJ>ymcn`Bq31@eTeHFUDrGl3%Opjpgp4}U6-^zI znXx>@jCb#jy=nT-H4uPCUg=xXUj^&mBPKG{c%*TI*l8j zk$xoo_pzeHy1uq-di1|Oqg(e$UY>mx&3qodXgILk{Q_e?H>Cr%F!OyCZ&B6~Ju5i0+CiHSXuyZ-B?^ma7X=V)O=6nZevPnfw_%ozj7I%R9JvZepZ zW82r;k#rkK+2lfGxPqMs;&~126&GYI3@=!Q{QDl5T}2fd z47{Ur(SP|xlU0)q{&h8qc!U4@#?^XnZHw>cpG?zf^#5MZ?0@d9H?a!Jukzd(Vvb0= zq$Cv($G=KXr^X$oEAnVV)vQ&^i~bp-qd`#lXO;_2rMztMv*D-ryz9Y{rEm6(xd`e0 zTg1vl@6LooY2`%zD4h-3F|vJ2NGp2O?>$fP?z4S8X4}e+rO9S@q(EO^yG7l$BY@Ms zuID@2&Z7==8AxthnXz}moBaHat5&TNE(?KD0t=>5)22S~fs=3NKMfg7ZEQ=k7%+&` zNB_6T)f0t)M=7BvOd6J^G3BvJ@6@@lB|o>X(L=y5Rj4hV%#V%@5C%Ml%JVNMXPDwP zs(8tO_3BNlUxGT=n0X|TL;(&(ur7t7^qiI7MViFBt6K<)RG1I@yTfU*jO5h=EvN|4 zY&l>4ITjABRaO=jTf@J@to9|WN|zePP!yd9zG6`04T!c8sTB)Wt@HEs2L0@q-W>~d zuLFYS<`V_2GK+~5Bh=hJpAHkx-*%t<_{l5(gn@9l28WyLeTr&KXrwP5O5s!g>9e#W5v&nL3+?@{&wv9PZ%=sk2dEuAmqD&1dQfJ&=9~d9U-6-U zB5ALP9>u*$XKCSx<_d-E$*A6_%T?r>RpGa(`qYh>%=j6(Od2fx*-g$#5hecAV9v1R zNmyeQ_9icHcG^)r7OYysVbi^X8$PDuSh{Gkl|zG1?_0F^(J`_`Bkh}8qk{9WIx`InC{a^h4l|{rUYOUzxCahQPv!$9_HESL$eI$<`?5E89gQqd!l^5eG42$%9u(CqKtzuxnlXWClelFs_;FZ`!~aL#x_3(7f3V~0h2THYe{p!dpVeu1rW)qHOGT$b`wBcts|AdG zh*Gu?WN;`I6T%Mj=fB^t?!^uN(ds{P6sw~m90tD9!Hty8h~#}th%}F6Q$Zy#Y5v-<9b)SX%}-e z>*GNi2~0Z|e1;hib-qjT`O`{YbRi;6(#)zN!h>_Q|AP^HM{_W1hpA1BtcIweHqv`X z$8t^$-{WwqJ)z?X^?&~I_e$qFawYM}Ioh1N6A(!^Bo{rr0H@j}{Hpn&|155ey!&JF zYUrwxc|gVJ+~bVvu!)?~M#&BKt@e)|$IUwLLagayY2L$%!JzcBEhD2zy6*Vfm4vFv z{Qk=o|9a`Z46C5GNQ5oEh`^icF?_T@T&4x)8TA$$42LR~{D4;6rpo@>UBBC2G`X*^@8Fgj45{tVnQwAX+-6lKCC32=AcC-YujTgt>tzyiUR*C*6fNiK zad8WZ`}FG2z#^b=s$`iTgNUcHP52%6rS#)0*1%v1NaC zf!-NNJ$O;2y)yafrJJXg^8aiLX?^5^sbwy3)a$=@`Z;~=jtJl+xj{U-L{w!$@2IR8 zA%e4(e@DOj!tXzCrd$4IpWehY6fxAPU8W{s^xcJB?#n1Cf~@I@%jJ#bYt9bsN^HLX zVvD9&hZHIrA2zPYDAY&A(o>_>z_b{rUR} z&6u~su6kCdJ%HLV$ys?gA|isG_?)AE*Qk0Mw2|OtpZ^hyQvOy15k_etry&g;BHB>Tu`-})LG z8$ZGer06k9oen92(M7~dqfrEK?*HXa<4$`VZ6J$?ai#3Qurw?9#@8*vfO5hyqkAf4 zdRx$zAa@bx6M|&y3nk}&tyvdad3}KBm0khP*(3jWMl)GoC8zVifDH!D`lHh=r8`X{ zdX!);%y9>wFUKL}pT)edp(TRvE%c>=J4b#V1Ss3F>=`8HV$AXhJ^pLUel&K4SQN#Q z(wWTn6SDuj6|tHe?~UMWCIZ$-KBOml=;c!i6)kiWfF6~LUhIj!3JPT$EP(po@BU}E zotH-I^)W5`wumr4it&9y?%?%~8`iH^s!)V4D9%DF2o1`RZUXV&3o{C3l=60Tw85id zYHP-W``xhn*Hei$6Rsq?NM7yUVpXex5HbuY|)gN-FbB#Cg~LRkCH1MKI|CNRr@!87V|03 zf`bit5@@n*7{SotNE+gw5hp33vAe`Rr)8isYc*~2*Q*sLo#MPhQaH=ahxkO5=0{?b zWzfk!G5}Sh6D?I4JTy4ITK!rp-sPYG&vqUS2u$s$ke84PZC_B4DpjHkpo4)(=p>Qy zz_PZO188V_Wn??8$b%=@tJLW{bqb||TBH2Q45rfCJtWe7_^|uQ$;{0WdWCjJ`j$jN z@w%g>oaXc8_e~oQjBF=k6r>{nN)ipN8M9^?s3d1d(^5JSc*TDzEF5|8u=Ww^)efQb zR{d^rP`Fu+H_qe3S+2a~F_4n@n5nC=N(Ro|8yR_v5{`m0B&ZJ}ER_|Z_2FFAw(=}; zsU&4~urFQS#z?b;s2g>(FFgT{qNKhxn^+)YQc z1npDNYiqG*XbrH574BL`>``3qQHuTo)>Jw~edaMjs@BNlBNc~Q1JNyr47r{s@P!`s z{H_zM9JH@wo=d0QSS6(ts=dZp7$82%mc~B?y3ujaFSn$G@$UeUND0@5(Q@wJ1vi9D))E%k<|9>dX;BMO()it4f}V7$w8?K z{FurLxw9IsUbL=WaE^;oMMM-gyp=OpK|TuO?sGcz?7|rBkF{BM%G{lMHLYdjs#UKl zG&Okp)`YHYEf4ejnnbSo`E=dpO1fb8Ca`!!btBT8eumoH?wJzx+Z`EzT~d;+nRf$t_b0AKGrk)bcf8@=x85qRPyd4MX;yhj7+6asEQwH+V*pxG(@EE255a)OPN16+zZ+E}~x zHE{ONAJZUgq^qmz{~iWD_r>+=GRee%gdu?*k!P*C=)3sx9Id=+_&a-CGzgKUx&C&p zF#CygJYXg|sAR5_6q?A6hUa}Vy0-R@;^l}5QC?_qWh9a`CxnJ8zkLo-3al+<+P(*G zxBsXR8f?7v)0^nSa#*SpXoq?fGpH#z4jY8W04EuqmP{fNf5I(#>??o2k5xE`QlJ3R z01=N2eZemXWqO|$EipHdYL&*7xhkGwI(oE+_T4mnm`);1=)iz##(1ahrAR-l?Kq2jA>Z@p*UUZrd$FIzx)f#i~XCDCjvkGbr8qM3e z9uxa<2G#|qS;#G#Q=&7N+HA+2-d{(x+k+dJFHHXOuT(DFtG2Vx2obKgVTA3tK@WPm zb3;N(u?gv<5_WLtRUG(~IefVfN}S@xsY>KbdA8nENsWECv+=~ zps<;o%v{QJ5yQqtGG{gQ+U21SXd~-OCjQTLb)8)ejn)MxGcp4s+BSG7+jT{)QR6Gm z)rRv$L?_3~;YEc@Y`GAdEj+7{R47KcugMmXhZv@-=SWrp=p; zVOKc6;#|gD**g|zc{-kqz>6;e6_zxD!(D0cN(+NVrxC&pj{`I>EY&p|8G0V^)owU* zT{){!6;DAeL%oZ>(w#6?`LXo^nj+!YDC8Tt*BY}DrAlTMdFZErNF&rDr4$7TU_uo8 zyygj3tVm(^$;8CHX;hBwJ(qCr${%hFa17j+N! z6~?8FN?J6IJIZ(srO_jU*8IMtw%4>kSeY)U`mvx2qF^}-UKX=w$|D(tcZ@;}bL{{% zZ65TgOlm1}(8Gwr23$6;WqNx0DfGy6_VUu&U5k=$3 zmXagOK~jEvavAW=EEe$+_ax*_8R~KM^q|}2=rku4zM>GmLFBsBpWVZxT&MkfzUi+_ z7*G}8$JgfC{fcg%pGUe*#=mPU9W1eI(2|HUL*TIht^9tp1Jt#M&AT&$FA`7L@%*{y zppkn-Hx83qp?7q|(s6@$6gX&hl0h_X!@f8t@nA^60695z^07J}*+8I|QPL`hKiSi- zhy10&q7wd+rwMn?L@z-qGI<0L^#O;4+^Jf*vKV48?&e1%nd8~fb1AAi(c-V<^Ym~n zUA*Wj&xpF$v1>8W_ui2{7r0o_a}&cR$(R6xQH`i}ab?35VGI|xCOJ%2nD(02EBkPw z1nnu2@vl#p_ddR+)JUJ>+U3jHf>$m|o^^b{vLp0*m>{3_eWP3VN8D8S&lJI-h9?J$B~a^s6-55=3&{k zUiL8+jX1)@94&8(Xnq$Bl;}QmbY{%M`8t3D5dx7xNlT^YgE|(g1lrTI(eHf+cU=mp z?P1sXPe0a|%uK$63b#uUJAp%y9-sYo_}TBgPL&)?HWR!HsAw3dnPj!UE4 zCR3C3xg)Ma=0_ymp*xavhg($U`i)KD6XR)eVD|B4y1pzeF*ZA}WF0X{tkIAnsQ~YB z=*sN+nQvy^>wnM?P}Cc7-n{{1tV$JjANB!}WX;LjwGmm-tnTb;=HD2EUSDZ&52w*B zlh3}2{j{U1^RO3Vo9!4@Ud7#32Y6TB6QfiCEIblt&7a>{mE-Ugl^I{2-AOUAv)=3V z>3!!%H}%p3bF>{(H|q`)ht}x2eVAjwSzTBd-TB9r=n)JNB^h-cp#4jW3{+hXfpxLT zypw96f-gWSy2j{B`E})4?DpP>8@Dl55uFpVh^aj8)3*z?f2z0l@PJj9#JG5nD{jT{ z2&e*&mZN}kZT`Ea=Ir!*ud$crOoTGgH|iO5L8*NM^HLMe6BJ_#RY-Ah%-Ve%MkE5I zJ=d>a7cPq|K=;T0_6sGaBskBqPf>jrQ2tr zhECk9cZjM_%NR2JnlrLdg4DF1fmp zzgzm|l;2+6hU7Y=(d}g5T|!giBY0VWFo>~>dcwR0bCx} zqwt)OIK^bv|H0mS2UU4~>)+U7G>InmF3}B&Mh%L+5o2M44aABqLTpH)f{F#f#Kb75 z+bxz@qiid70lN~7B6d*h1-pP)A~q2DeXhMN-}9R@^ZxP9yl3W}!_4`fNZI?@&vW1F zUh7)db*)qU0=}0@9)7xk#fc$}Mgtw(ZX0$IU_8t)uubpz6lM~BrEM_4aqs&>XB%f~ z8VK`*?5uShY*A!5!@&iGHyVb4r|FrPm)c+pw~BR9*wZ4_eYk%8x^(65sPCeD$XNr( zqOoj^;0yI2+1a@J$}M`b1|xu#ggd)(31v3i&x#w>U1x*w{0y@a_r(`){pK%asZ>Ij zsT2B-zgs$dGRT6^Bc1J1Zuuji%4Vn%Lbsmk{q8zzi@4s85VM^F!Fdi6OZ4xgo7OHA z&qlz<+7h$lMtfFJI{OyyKd%=roW}lZu?%TmQN#V%8nd6D`G*6{nih{bawH}p!ee;H zHnb{ljX1!m3yG`b7+aa;HP`RJ_T@mz1}R{+7A^7CpR-142#P^;S+_pcNu1Mar3e~F zD_J7i{^kux(=JRw(o*GSpQkwJ=CT0fpHy$K?43B#BcT0+f zJjnHpf!Gc$wYTI@-W@3@9a=W`uyuzmKQr;wgd3_728-LA`?F4S6C?_C`td?B4wfh= zh{sUh!F&hwpV`on=xDZJ2Jp}Lne?)Kte+Szf93mS_ZY1EC3(GH?>>j!sOR?%i?p^UI7M5StMHZzynB<&SL8Fvg~)P6t+>EEGMDv`{4)UgF0-NPSerekf5?da71mI1X}1W!u_Woi z9HZ;l_ht)~=i=}Ic>&k29aMKHU2_s8ff+`#z2 ze=1%2a(fk1Ij7jM>IO->NC)98#nRo|x96t5(s@MClfxVpf?v`O=Hx2;An&2->9@_+ zIpB7Vne;2SlkaT=&Ku4INui6s+&*;hplE@{WbE1IsUjQJm**o2*0T%*4Jy*%?9MX; zFtAc=6adr{zM@-^?`E&M&yf&j5xyzB1K^lX(w!(1@XPJlh*<%%MKo5K^Y32nYzjqZ zJ)?+tliB!95=a6)(uDnq!Er z`JGuFYWg^!1;>`RKG5p*osu+?4p%6POo?o@HkmqYNbJviN0Ss<3Q!tmtz4>OQ=oX1*Zjhaie?1bF%j4Iu~$fj0(K`)BM-1zY} zQz@2NgS-hGq)%u0gO2aw!q}3lc`aFQSWTcd`V4tAQKe<(@22-WzZryW>|uJ+b2=+} zf;xcmsXMdcSK^cUO{&<~!54_==wB%+>JKA3Re!?$PH{W8+xj&+1MFWTJ+mGh{m)ei zl8n*cNAhKcb4uI^90p#} zVqD`R_t#PER;s-Gz{9V$ce`*vwgBm{?clTPMj?OYIcCc>aPI}9`@7TeJJ2%&NsE!* z%9(}EzH+4#_jc5*+7wJOMUsfzVRZ@{BWBd?_DeB%1y^`ya&s>J|)K(N+QC^aT*IXPa+KeB9E?iN!n68<`nWf#YOV<5G9Qh~KtsK4(Z+ zfFCyp8rL}-ilXGg$-}~zXvh*@|CIHsU~vnfOI);_EQC;RG~Y&evlbM=r==|)%BtIL zMhPm?ti&42*U~K1=@=7uzjL921s@P@blEgDMOK$(1No|x^C}68rRkNsRd$P(Eh#%z zHaWelZ|v}S44RTit9Q%+OyIOKIDontKCv@x;fN$;9p8I#MEyVIN=7HvXqFnERB<=~ z2>j$v+d(uayqd1fQdtsrf^ilUf^@3DBz{ww1)NqW`?7_l!y>D{jok`a!@g5ZKFgpx z*w$$+5C?Bpeunl<0$F?uy0IJQwt%&kyNdU=K3S~$U>r${MdG5(#1oOuAD7ba0#86# zJI9?Qg_Z}3XlWAa5G`P225OTt7-g7elR1&)?Bbv4wIH-PL5O=>%_~Pb`J;%{a2{OGg8#0hKLDUC0Aw zrbr;LhZiQ+{k;ZTKW}^mub7%&JF9m@fqWG6INrkj-!W=ZOL;=FvoaT+S~p_B7)+B@ z*AWuxzXVBehjArQsSd$sV^ak{5&p)onCtZ2*_DoZC(^;x*Zo7Dcdv|FCmCD{;Xv>x z8vH}I9zUmemQApJOjzE% z@vJS$PkfRbUc$>A`Pi)yr$L{U*3wk1iFEtwTdbxP=*?NsX8%OTBjbncCx?oQ2QErW z-+?CXyz4EgT~0-feY+7N_fwJpyteiTaD( z;XS8XyFrJMdAKES_x;*jvH;Gee-AVz4UHL7A#o5zACTUA;9Re^$?|&TRQ_%#%wUl| zc+FV&d@xb`NpSyt`V?J`yz1oDF{*z8(n@jeY20$4)o9#!2kekLH(tNF!g{0H%3adD zP}n)u<^xx}fp`=k_s6e#I}6BlfSL3b%5YY;R!s%Vj|1 z87xJy2=!DNLM;edy@G1PpIsvt?_Gbz!|peixZ3}!C||D&toyIz5Q~FUSy4+jo^`7> z_dobgZVR~ZHY_$3P0uo5AoEmXp7?-_dJFCkNiKA|Z4#$0(kjv@( zHqJkspY5+xkOvl}T|r`duX=dH3#J49TP?YzK{KfkAM!{2JO_<-Cf)*@Ly?hRtDDiVJa# zxN7ks)K2A6=GxR3cFUZIWkt(;Gc?1+`WV7yl9K%0L$Wq5HC^fsQ~}>)%S(b{C4}(< z!uHz@0J({Uz9Arrc$CcHejKxK4-;?v$vmd`qCss~{P%Iw|x*cExLyNA;nnXf= zfxJkISv3zOt7Zd0#E@Sqr}E*7K8U&WBUybhyi^uG>qyOjqesp-Vte^CD3dZ(?LRTI zAqUjK^R`ktY=JNu(*0x*q&v57`+7#w3@;?43w?GZcx~j>O1G6C66d}rFa)n?0?r`Q zN)W%;_w@X0f);-`Sdi53=*pU!8n5P#kS{|1)!)n8P(J@{$DV${DG5iqClcljqsCjW z&2&UUUOtv)MQrS5i-kS>{@?YR4>~*49S!cmxwvQmm_Wj%w?89tP;TWlXoBs1k8SL= zwGD-n%YpsS8v)3gMawk{E2xWu}`JAVI46T%lIj~>T#H2CD&oYLfMzWd^TGEd23dy zc5vkZWt)`#rMI^qWO4IsgQ+vqR~9{Zv9OixDxEcKq3<82g=38pLC2oe&GgzwM)qE@ z#My)8{}M8*Pk#OTO*R;&AR!RK8b--pL~i79EKJ~P?%p*N-y25ep3W0f=%K?k+%r=1 zwMAne^oI4UQ<*Ufyi9lc288C*E$-b2Z8WesVW%c&3y;ohxd;`-f$B_OrcdxE@{)h5 zdC{OM&feQsVP)yb}n;QgZFY+P2`RLS{3 z2hNHL(VMB?bSl1n&S@eo$i8Q%nwPnKJv1+5A70AmJC&-ZHA|u0ojj>qEv#j94W~-6 ztzqeNQt?mM-$#;jh+zW|v}5Q2$VPG{#%fD68?|<4)-ULFgm5cOslHbx6Sm?xR5ZxJ zHbMpO28*)Mu`lxmFJA(YbD&&wOFDLRTQ|x2cCO8~;oYfk%HYAeXw$4=QMLsa0LH$_ zz7jLcH>v8gSii&)MlTF>a2rdc@kw^8SrNM&{3bQ?k;Dd!gHC+P)X8@nE zlu7T!0d}jvHqp5_k9x5^PFDt z5Ft;OlEG}jD{0}lqL*O$lSo9-7W4V1!snj|gi}|RxdEKcorjSnbEIGQI8Z!wJn=9J zB76pBS>W&uO?`KLeBVfj&mKsd;AkA*7tLV9pd>i?YYsW5-6 zN$7oM{x)dI(xvztzl~gIW5ruW)84ALcdUtoWY?ZOJ~V9I5@4sPByu;iM$1MYpqMGs z<(J0@H0?28pz$#4` zvp`OCh=7gyY$1AOc$$9?>QK;1$du15kuig=gZ>Z#I-YHh7fdMbI!JS1ghatdrl)*6 z2=j;%_ix+>K-sNInunUs|JQb4y5(PF0X{8%ZZQCmnbZ0^2;Cto$^nAY_o?Nu+D2Jf zca;thy3v(oXOwptYI^PTtzFpj6HA(QQ+t;@fGs_+v`pH;(80(`4LJZcm$FOlTHI+P zw-PhHb?#u8@{lcvNK+dxOHKKV zvu7r+Scn$X^3OSIM6v&3v_`z1TOK}fw&63xYa&l*6gv%b=BMu8@rOobFoO(}uU$pBM zg5#BDrBi>C^cHxp{()KJ&68Mn_tqRCLEoqz%e1PK|0sQfj&ul9g#X^7KQG$GoZdL49F5CDS|=V~i|05kNBHl6M)l0)cxt1L4fKrbHfWgrwFI zszV>tnB=_rxN$EdkJ*-_r8@a_3dg5%#td&eV={Tta|R$9y@;up3%qWUpsY^4BMg45 zwAmaMMHXdj_*Q_Kdx!d1nw@TL{u{uMaNKEh3zOHydlivM)$5&ikeSo#c(zXStd2W& zZbL)@0l2={?2Gt^#QSrUWAu|P>>f=ED3i+>2CRO8f2%PjqX9L$h};fFYWYZ_9=db? zS>$mufcM^dJuXnbL!TSxm(|If(`&@DFW`L48+*K%?QVyawLzqGZek$+_R5uw0HEh!DEQUOaCdw8ex zW*mcpM!>p9MAoVQOV*Z+EqyM7U)cw|?P8;u*}45d%_cE{1W&_hO&o&~E&FfEf#U#@ z$=j*-wS}^9512w|~)o;`8)Xkm$sK8Z|3a zqt-YB7c8ow=wNa9P*0oPBWr#QI#sA};l)=5P0mDo0qKmUa`?iqFgrr{fpaZ%)9veg zRFuabFToWe*m{S#i4{S+#T(WT-Q%G+#fbtZ;yQotkAW)>RLrm_;*CrIHHt=hc;PhC z*2v|^=Z1}H)93w_*&kx~-%O!BMlU*M0#Wdslj6~AA+-Qz>MK`OPHXC(>LE}5E==FM z8+8rm-!~zIG|hIxp(qC1Wfe%_Ug{5N-KLc2lx86(@X3geRl?;!6?#N8J3z|s8BQxNBkiAX?O}~)JNil z3ynSBztqwXFGaqql?r|-7i^A0;k7bEMJDTQR+O}+9@_`cGswbI%_LCX5-G(Eci7{q z*YL`%OE(p!dZX)Bo%_In1NY{#FMAT(ZavIl6l8-I>xYNtyjohNO`8&*?rS??uH!zp z$K8hw`fkx5MXP;Uvr5bd^YYVEpuPAJ*cbwYcqp3sfzWp)(MzF7YzY%+c0ZXL!>B61!n$4o2B&=dFO43#Mfg?lUSnk(S7;jX&#@y=JTg5^MBw&1yX@vV=!|AaIL&3I7UBW+4( zYCS)XnwC`Vr+a3aqgq-X+LN=>YKRvsQzQBkx$(IgHzA=g6%2kKo#=YUZJNMJbav;d`R*|4dF=sO7!r=F-lH9%c4ya)9ZwW^Ss)K?X-NRX%g;~q7Of#l zug6eg7(O}Evt=XB+tg~+sy_^>ZT^*YT|g~X?W$dqGdv)+p%<}4~-MByoyJLXw-KFL8NZEG6Wbqj=V#J0GJ)-H! zFv1#}SRlPprAs?7&wk>IiNV2^2V3oZd$BUux4fx!$e&utkCto+$r)?(;xw3BFu<^m zq`VL86`Xs2c)7^B)w0w4znv4>f&QpyQ z1uYeD2UN-bO_^|1Pb*1qJB3FL^yF)|9G%=GMo;Zq%rUP}YDEAX)~+!%G^` z<>B&ZGA;~DM@)oL1ef5=mmREk)HU7JdbYV0(l&seGK`a*PVVgjFS}5}sCtrmlcxlR zp*TEgLZx-?9NmP9^%G|$+SWkcGg=I0DCH6r#bCyA|bW%7;>ell8SVczl@KgT1mvZId8^AK5*j*GxCXs*|G zdqs>~MdBnlf00R_As|0`aqhi$w zLV)t7K8&ncwY=%@psgYQHs3fSJUlR{h+7nSoZ9qi%EiUeE_O2uqWitPtUK7x zWh(p?*Y%~UR^2QU$rlgr(0z{va@2*<*qn6ZEzp!k-HN{EU<5SqRz5`&2m0~Z9})_^ z=?Q6#{kcl$n+AAo3?YH9<+<{okQ`oUB>;txn{9nIPPqA7Sr-Vp9V!*AZx?=e<)qgI z#yDr~jq=D>>6UmI7I>H!FVjBz+9=HX)~#E2?%kU|^c2h?2crn)D{9%caX;JrRoOFr ze?8dSGMxlfESz5a^UuBE`fa=cTxmnx;{xx-AG(AQ`Yo6kF*!Lot-FT0x`weUR`bC$ zz=9qBqE5N;i&BQC96s|bz46_@mh?-0w0Z8Rtx1cLAD8d2V2bxpxIV|Ai*P^ZtZm5_ zflhyhriWz7oJ*IoS^3DRPt z;mh%=YcBqE+tu5nICBD!V*G@Md*o*=zHQz5Ktg(#)U>qMuMXQ4KYHnK_qv;YdRcX$ zd+%>r?EgASQCb97t6hSn`5tHT){Bkqty<9Ei=LN*Q4MwG+hv_-a9R^;5R|glcZ)BR z@#b~&=2IuZ6gbA*-+=v~x|%F5>aci^1b`LLz|+XFvhjm7s2Edm=yhkI?EwVX9)PBG z%9plGz5W}Rg*PodkH6NqWHBCPocE&S{rI^9xAyXj?Nhnm5#Q4vDwsrrENFh(!QVk z2p{cU|64eH(1Y-Ct*SY!IS^D@r^Uls$v6OgFN`qrQVykK0&V@BKp zC(3_t?Bl2x>-J5;`e5>QjEeXnkXPk?|@b=QCi zS#=g$ml;2fpL_G+gAV{R-*CaPOWacXzr+{4 zlfIlHY(I=nSs%XQe#+0szH7S2a?)e=&Wl4krM--ZI&S+lZr@4c;&B`_u8VFzyj81g z&&C}VJazk~OqF+bWn4hi$gx51-&B8PA5txnd^*CpJsR2WJYpTXO0^d?88v97GlirV zsEST-9`;a@su{Cq?|5F7rNv|L2CLN=JPjS(jk&ygRVZ8UzLJ~jU5A|@+{jj2&Y3zT zNa{Bd{{$l|=o&;=AstyFnm!!@ja7w?SvXB?Od1jp;GpaXOf|929Kp~@-M+7;z1xoD7(p{w z&=zO5fX-`fCrNrEogve+$rqzZQ!|jHoZ;}4CpgW+Xjri%HJ)m)-@CV(5yv$h9ovJe zi)<_o5e~%79<2RvcTH6YU|}v+Gdk!A8hs~9DM_s)bntLS_zI~(xnG;9L&TOCaGgyg zu5M;V{%w{)9TbNJWc`xW!N^I2<~|pf?xEfcc*1*Jz-3`Q#pkMv&Z%#SYFtat15SnSJ~WTLG3)lAsD`U< zG?-QQ!&A3g*LBg}&&J|#!OM3h+@$beOFpL&=s`zlKkA^Re#_@ghQg{8ZS0|*W)RCP zdX&SkJ<%M1YJp)1Pnis4=qffib`<7aktQux6$l5i zC%X1o#GX~M!{G3k)uhtZY5(nk!HqO8aQwaqQx&qwRve5V*2>RRMv)Vy(HfHs7XW+> zxpIr7y|R6Tt%#n|6C$NI$}3Pf5B7#US|o^IA3xrD-)^>6oB7d_8+mdRSLRHY+KX2< zXjE>6CL{mCBg|f75Hhdy)S+ZmHK>=S&oSDD2G}09nJPOz3eD+RvNloP`t1$&M5%U$ zt)vQWALpXkg6dzx!l&O;{N>|_W}z_!L`CVhxLfnELY2V}&!p9hO7Y7`G#rd@CU$VP z&hI375qd9Kw$LN9G7QjSNkS?G1i0lW??l`6=zr_4bt&&2O!cBc-gKX0=MbH5cvruRXvOdX6 zm$q#4{r3(=_=VoIsT+<-_g)GM0Uoe7mG_}^Yk_6G7}HZrV%LNF;Eu@S-E5%4<l zBD1U?@gfIJ>s)^`h=SQd+-Z5fQv7xLW-ICEKVZO{=etu44xM-M(!lE+Rpu<*ULWkb zzf5dC<_lNS4gl2&@=N=VPhYNQhq6yBpwH@}3sfP)`@!ET_cUT|5<>xGK2aOmhCw7o zmZ(@`RfiR;Bw44hk<1)X3E__=ExdTKm-06v9K$YGiba2CjJmQMF&el=zi{isYsI5s zmg@Axwxb?XoWwBUfZa#C)X6sjAVT9JDlUiYOJi63k5TPD?#81dle^cvN$xDxtsV-D z8YE;Fvkx9TXfP7R5>GvU@nSw~mO+CC$(IJM-|IQ|u6jsz8Bn}fu^u=8J)Sse=K+Wh z4`}JhJytHF;0O*$H*S!?$ub+*T%B3z76`=h9yEGUqicK(q!LYT(bQw^hk8ndD|`N{ zc{G&9PV5x9u8HqT*twkFDo#`HiQza>GXO5kV-huM@nPz*5670BpCQdL^2@Si*;vhn zk+Ex|A`6+Zu%6nNDn4PIqqCf8 z>&no=ea@IRxx&DtsL*mk40lKNSGoe>UZR}Pa4}4%^v6Cq_=xwwR?bf9AkE+qzZzZ)f#0$aDf4JzLZTfWm{fuWmilkmC(9HEaVM2 zh^IBx;7fBIWWg|LG-)M$MnmG*XRkf)T|EvWGB@DBm10GMw=tzb_B~=r$HfUbqbNE@ z87baP9cI=mCMLvle7T=HK%3eq z;(&m8U5GTjoHV3HChjeH*(!~W-S}`v%E5H0is7iuF!5>L;B&ObcJbASHvcc|X) z9r(BO(dyxc|FyjRm@V7+eK~COnyD;P%|9kxqb}PhR+7pn;}OcCKB6^+U~x236R-w^ zJfBAwVAK6&6FU=(7Bw5Q{@pAN4(SF-{wc*Fvrilg^a#)T@$_-fmWT5{-nP^g zisJ-sK;2r#KKm!_!hdhs!<{^NS{ak$9*bs=NXknljKa=A0FurxXg=;I9&leco74N> z_PiHA3ZKuFKu)z0T$*DvvLw;mj>5uByPXIe#euRGCttCt1t)0CGZxa;*4%ndbqd4f z5&{vzz3c6L@(U2d^P>%$?s2QTY1r$hRgNST`gp}|QOSNazsAlEvBlw#1j)^eS2W93 zPVNx(6y22?C%JkmY zgX*#ua95>%|M8wpN`XN)`>*iChgS&xd};|(dasQ=`hDJV^W<*Dj$LxdT(TbushS7w8u-~Y3v=$ks7> z_v3+O1-N|p3hVc!6~li2jsN{)yZ@&fwzJv$g)!0j$WTD$K6}P)g#1~X;uYhmr~9A1 zGdfPE(_j@ik>yc&wt8x9_1js-y?X+#yOk}<&6{*`Yb+_q)y9$E5`I|W?mePY&Nrf6 zE{9es!?}C*@QYBmu0m33= z+`|2kn95;ec1vqc9cF{&dHik*#1j`xB>@a|;Nyz!e}soiEzL-_8ePuQ6%5Fj2qdfs zr>hpo2&PbFybT;GKYe@DyApBBG(^6&sRD|(m`ry^MhqNLDVRsJ4N<#;mK5kiEL;te zUM)S>_HEvMCRZ6X8;sln^Y*}Ghq(ZEM)O-7(rmw(puVs?W1>?gTyO^5mO7dlA~Pdv ze{PrT)TW4SAxm;;Fw#)!6n6k5)okGdv^rmjA2vB61`UVtdfuHhbq@VMCuDO2j7LBb zASNrx3XllLkJ-KTE@Jfvs;LN^K#Ilx~#KUh`^L)9nXcrCK0ty8Xa;stU!M%+W#y}QHS{@n9djD!4gN?y*( zoxZEXpMmuldDS^C_a_|qwj7X97+s?XXkb0!xJg$P_ZI#zRgEa3o}^^f?}PIf+fXhz zm?p^r=k}x?CZG%iU*0i8J;g5(H3tfx20}<=tXTyc=?f@7gxxf9U&XUHec_Zx>Y9`E^emgc=T={B14!)Gxy`9&wm?O4yb^ln=vPKB`ZiVli-pBhr-7lCx4a4DAp=FC z2c0o%D`8w*OA1%6|JW*3k{Va9+ykS-!aa+%SfZ;XFpM6v?^)CK6iRyRn%-xSI5W=m z7X!kEOZ&rOEJ&>sGeERf@`yG0QNRMlg24zM8&ii%iY0}67HzSc^3iCvR(Kvem+ivQ zz~E5S;AUR1pzPzcEw$edcreMg%9zX&C|$Co@0Ih7{{}Jm#_8#;>rp52nZ`=C3~Y9m zwS(lgx)Bt0rRX2*_jM4CRXb$1|A{~mL=G94oFZS&arR+?5QBW^KYv@(?`=c}rHC}o{&W2BQ4 zI7G!vE;e5vZ}C>ITvhD};HD8MkO3GtC@3wyKyCJ^WVRGaV#A3)i>kP%(LHN_74`;I zI2?uAvz*1=gY`S!{LO4Xo`q9XFk-02B%w8Pdkp)&^7d2Q;hDJ8EOP)od_JF4-Mu}i7@$K0UYRTABIK zL2B4$6JLy2`LeuyouhMW%%)YkUic-JE1CejpTyp3X|k+U0SZ9 zMqfvqQ6vxO$kCI+7$jpdIP5v}Y_d(FV4hX0Vp_R(iI=Ia5GQKpAKx|5s6XPm{t`4K z<2i+Ae*)xrAcI(tTBF7xep0;32HYLKOcbS`Hq|-0jbno?NhW$r5;#BXX*^)mFLB3L1u`1Wex)9^~U)Q4;B* zs$Kp|>HJ-eaFR%8tdpcibayNlH0y)CEgF_|k4VzD&SX^IeCOX^j=A{7hnpkU5fFfv zmF7N)NC}+?$Lsmkz|+>rj~t3nO4-(mu{WNNsq$mzK8*pPLkvy98=I`3E-7~3wJwk zU~GKU^ziU*1R#3#0Svs;7-n3qHfNTMVJ-HAdafCr5{WAbQ_4q&vgyvzsZZt z5!u`IHnYG7-JfBXvycK(CbKYaBs`w+#~)`{-VR1*XJ^?0stE^;wHDz=eP|RB<`(md z275KxZp~<1cdILGxLoMw4}JhJw4h>gmc)#ph|@K33RRbZ@Jz|%0GCQLr9#YE%`rt2 z;<5YL%Op}8*&I-K728lLxCx%;zsl?hL?Z*N2j;*M0H@&@RoNr#93NR`u$+Phu8NPs zEcYzcyLALMS&&5WbhJj~DufXCba>*CBXFAIw!69V_e`xg1g%%TzA{J=5(SLweCkJl0e z%(VOj7mv%VM;ZdhX3;OD^W`NB+In<&Y_r`F2_?#wz5k$u>2HYqebgw3s|RaGUC(Yi ztNHo8FQlHF5ztGoNE<|&C%K1Ekb43pS~(Kd+7(r`VHk;&*@D)%$%etHo>`A?44^;M z1BQO-$tlcUob=#FHM@h`etb)|@hW+h@dESNBLY|? zfT1BE<|S~~IyqX>f6+fldDTPXtZ%wID0Kzp^u_#*^EAH;1g7!b&+@-@*ckfDk}k7v zPX>}YHLd^oIi7{?q~E&pV04wj57OFDCY3b zs+*VR)DP~`VM6=#1$m~&uQFgeiD#p;K0L2|LAp2l~|?O>U8U?iV-QRzca9B<)^u-5fR?ttMoNDzOWGy;a*P83@EmRtmwjJToT%bar@3IC) zUJsh>#m^PDlosaBNpNn*&jaN8O}kL6@PFgqakgFEM~@LJBmOO$j3(GOA|i1_sVoC| ztZNiE81UMBB9XXCo70rMD*x<<%I4YmSs=F_W^G&Y=;54&^bu1-a;C(@my3d&=+Cjp zfi%xBj324ft#&(2h8x=tKx~}#O;y{%EdQkjP?=6CVI{T8#uP`$p@Mnb^!A{Rt7djifoPnp3QmjuJhzn z=j87tP&&kd3u_7$;fqH3T#}YYd(I5r+DlBkdG!VW-oCfR*#N(+WfG{7rHpDess@Yr zg&bq&WL3$%CjS*_uwY7ZuRe=HCLK}*1;YGl#XZnVu~b?^e%n}wNkq!` z`Tgfy&jSFI(1*HGM``ltBi?12pTCL;Y4&_0_>?7P=D|7E$ul_5(%@2d)vHl7alX_9QUdxP|JR3AxQdCgkO?Wft5ddi zhSt8LW!aqmYNa71>P=pm4R$1+3#aZ{pmVbYz=$mC4ozt{pFzI9v5;&upgpcjskc7R zMP59%mt)CJ(?%$PnsNyai4197bp#-Pa?EDTTOWw2=F;_4&LDjplo zBG<_dfdb}ml#l5iHG+@C^#E;Iq}v)70Z6dDw|UQ=i7U5?5=BEs4Z04IMO)0rEkWcn znggnO1bV>V1&Vk!C3{NDAmAO?727gjB}MnFCs^af#H8G88>Sr=89Dg-BE?l!Q|^;K z{V92>V_GQXh=UQCZ}0}Hy<)x*YDDgP^Y3wa?ZI^)l+)@vnEN1qKkKa5C~2WaWV6k; z3{Op(1Y{?Z5IMy_G3X-F6E4i=5k9T)-O}rfcskR?iF8wZ32s^)(kr1lG}^_suDZ6W zTW#Ghyqjv#Q=;(fAvO>Dsl`Z})WXhW?pwK^t9w(L43_3Ty=uI?nmYCV;5?{Q&~@i7 zUVNqcNq~$cjB1vIOgqqjA~Z$nnINSm(Hw0E_@1lh1sg@5xj@ z;ujf_0Flfa_sJCfj_*0*{)sw^`A^rVAZ}WQ*2L{r5h~)C9*%5l0^bCOvCfsS7 zmMF(ACTKq(Of#+4JMV3-Y}tu;5ljZWhE<6BK zX0;kMI^XCK_6oXoHe*erT#cgKc>o+#+V+y+u_BExa(Cm9pfjhy`U2oU)uL0yD}C|ZB?({<_^pL1E6e@4F8!pT~8WU zy2R&*Y1WFCTv!RPViWS0E;P@*k^Rdm7!KS308Y7vR_@p8NxZC-xS46B@pE9Q?0%p3*|CeoTw(m3dD8(9W2DI)O=))3b{L)NZl={fcK;z{U#$pW`jJQZkL zC4S!J9V1OohW@4@u~$N2=}_M$rlfQy5uV+m#=ci9aUjk**6yW{QuX&K=DPt5hyy~i z%!7=6o}ts9_1h6w&r#F9ima0y2^z;rNQj%6s{#aSLo@-?NR*xu77sfnhRmm7W~kr zh>lEy_7*d%(v%Tb4AbNkEr4c1^XFo~6!2OmWO8aS zsj`ycFcopI?^M3^C1eS0o!ubV8Z06TU^c5)~~$lyuTo~=;RSKf6y+WIx@cq-Jx{`{d0SW zB9KLHck6mBpN)^)DJ}nol;IwUKzeE68Np zcg&fH1%i~%k=1rmTtRG&OxE#JfWmVjWZ>^6MyfJHR%nplrK7IwMS4TYZ)`p)dS}xz zWiKRcj=m?pKfUKhkIXK^!M8x6tbUXrksw&~Dg?ZLQK?DS^l>)fuR6f zpiw3+n@p-wuR8TBK(~IjV!;?c?|i#f-yD+CA)!zApI-Iuu;AnC3ctH5S|}z1k1jvX z%v}0Qirw^GyPG7|`C?j$Tgf9cKU_lmulA9qQ~zDTYWtPcBK&+7KQDCebQEt}x!*@F zwVmD663He1%jE>zjPl)BtMQu+AIaamP7B92Mbb>s;d8GhkF5E1-7hYDAgtTjy=r}G zR^1Pm!+*RR;mChI$rD%zMQgu5h*7QvIr!zsV^qoGdkH31{qSi1>A%Wapqm`u*T=8& z#K@*%$=j3*O53GToauQ_@R(@?E_@p46e4?PqeiBSMDcciKg7C ze?)VWW;toRgPmRB_nnQfyoI1Wv^}H}qw%SUj){+gwvg8vy&xFVmWi|+0K~GII>Le8 zbHQq2={j7zYJH`)#$_niaN{L-q-ZwmmJylI*31TgO9MhXj<%JfUUq|4<97uoWV{A$ z7M@@;A@eHv9z)1m&Wh#Gj%JrDg_ACh|BOKyq4uTB4yiXr+^$}&nsqXI;{N_>-0sW` zw`L;trNt)J*ONTuwo|4|nSNQEJNl}cnILZ@LGeTx_uVB2nGDe>iyme`|Bx&7@dCN8 zKXxSbBBsk9Xc=MWZXK91wV<+avTNWeko6InRJCRcC6hz*$K-!z!+-%dYWURDpV|#8 z_zpk{E8Nve5F?`v2oMS%^s;g>%07cplSr0ETPsQXG4C&YWL1iEx)NrC#IEPxK2*~Y zpfqaj+O>vf(w~x_-`_g(+gG=X3+g_yux+N8t{jtN;Pz=YraGznX};*B zlU;fCi_F^5DZ?{zFp`e(O(-V*pH}w3#iUk=?k_I%zLtuC;)riR?g3Kp$0pLj^Z^W(G`)?4GY=fftm{`-H?GH%2FG~i`>68!J~Vw3-i$Fgz% z2Qg@&3w=i61ttJPmYD<}0}Dni!-0o$~ax z&8#n0DLkR^)dT=enyf!N)veoE)X&MdDvEKb5*jUnn)u|PO?PVuItRMFynD#P@7}z* zLf)d(weWm5XbuOlc^%}ES#_;F47+JRteKiq4xMO(vvr6Kf}zWo2LK_6nZlQjU5(2r z+`~rYs`NETBd*t@kq}^<>pvi?yrXLu!w3~hJx7e{cYviUi)1LLc z)py65tdYw+mSn8^;{NVjzFDQqzb!(q#um7ckbU;-E-Yb<=Gn8mY|BNrEt(qUEEHm} z3wKroJm)f6j$R5DL+`HTiww&rtF~Qv`JANBA#H2fz(+NDp3i#va?X_-Nu>=(cD+q) za-nM9;muDwoO(WO4j*Ph6aI6|M0eXKt(SIJpS0WJ8lyI0ZRNxSoY{G8%Adn*JIPyC8GOz4*3~YZf?Ky~|M#w05n<0Nl`nL^S^c7?p2s9yN-AU-{Pyj* zmDQ<4JH@re2p43;niZCcwy`KJ#a{X}-zi(?#M7tit#Y?k5cb`_jvQGWmdgTzkhvoZ z4Y@^5|8inMy{bMoF0v_~!-mR?Q2y4s8vnyJ;L%My)_cI?3St~w!1cXNjd`FBjFy0+ zo@NUpuC=;KrJ^jFQ6(DBCk%c6$930W0Z_izm%K>4 zNdipiDpfYY9r7^uhdD!XcjNEwwhn2B*OW!wS)Ouy5se5qrxdC`7PT=M`U{Md$Eh~w zfsvor@QoN1^wqj+0)#Z}1nNaxBdoIIMXC@5`LA*>8@4wxM$9*!hrsMuan6z7NqAeJ zw)JDhZ~fGv#Z+KB&Avk4B8UL>)U5|*>vngoH5zH#J6Sp6gay&H7kgW;xPGJeiMRC^ zzG!D|%;nI}`GHLY%AIuh_uGvdg2Q`c5W6sG7lw>jZa$66eO7`uz@by$1c#k;$UC`o>AEnqVstf__hCaNw73KE0 zpx)(wkA|P@6$Oz)>rbHG#sJ=pzEUJjs=M*W!fME^O>P6$!I$1}yMB!AbN6A#!C#~$ zQ1d)Tg86`fCtj%T42d9PFeA_4uX#5$-7ZIZ+$@;59%c;z=8RrTOQqN*gh!nhQQ3QfYzc0;Xj`SD^}RvL4)E{pVy345zGck55`Og_Nrq0L*EtgE~^q zyizSk9nZgPC(Ho3zfRxe6bKKBLrUX@if#CO3xb~AX*3Q@263xonY4WQa^LX$%eIIy z0zE-vsW);k2Z&}l8L#ESJ?4)Zd7;QF#*2*A!c!}Zgkt4O}K)@F3-mznZn3rJ6O?S7A-((eN zL$btH_kzgeV>`O;E>W^%6y%GcRK4Pt(eo9830^)($`LV=cxU^i8fT=oZ*B4t0^k|? z0$F_P*r2ULVJpt6TReDB``J}CvNmh;-s*V@xFub{k0D=|b2?Emxbw098jZFlQV~g& z%Z&ADyPLsh)$7%(r?v(HN56!EmgWer7=1Sja+d~r$S)l3Yj^rY7#8No`6407VeqF{ z$3vfi-=;K)_O4WO89*w1`zMS_5NoOo$%ll?spZHj^{#kVkZwZGv zq>&BYaF$~PEbb06iZu#-thO<(i>W|bcak&*BZO)FM-fN+Yvnb@a>S*TNRZPgV9(R! zNC7nxjA3S0#lzI1`79mOZ=}I$FiNe*t+j{{1hr(sy+4NpKmKm_S8RR?at}E+M}tB( z+gU9&QT9Y|-sB}k+d{e_T4+fCwu-m1k07a>Rk!4pX#JEA4dQE97#5VKIYD%mm^M*a zFuG((8uKFM5E+%zZ(Z|0sV8Sx+VXOaB$jj|(}~*;T}yxh+(4=7g9xDIeSsjKJdpU6 zo$7AB9(;Rd+Ynkr5-3sR^9gJM0__h;Ym|i*c`G^Ew-+t zYxXmCb?sKI`onD7j7JrdFzNeZ+BJa4cL8^R)$ zh(k13p4FqjYE4OKfbK&MYMjXD$pHgq*C_Jor&}_2AHJL~dC%11E>ovX^Jf?7GI4>b ziU{Ch53eera9@G-k%-=MS(`ow7OUGdd<;5%Brbon8v|L*NGJ^2OEBL$eNuOnU+fCe;u zk6E?K1`lN&({`9+5Envy#B+~v-4y%=D-ey`teRiBIdgl@Ld=`T#qRl}pi1>2S;|H6IR%}sQo@;ujavzcHJ)#Ta{Li zVhyLM+&1KmPY>)2?WxmI_H!Kjk?DKg8dJdPJ|v-_iU1lKswtQY%{12S?`in>d@qYJrcWA)~v-rd3=~Fm(V6t*-Ghvc=4~H|f@hT*Dmd z4IHQIVP}X8DWW1vG5Uu?zhB+ZZqe<>kAME`x}inAG|pe~+|}KM#cZ{LU>2Sf;jo0| z$~5bn^1(HEQ5?kOBb7z4snOh#aAAH_uv}Qy2EZ?&Xz&{5e&P5zzUkB>F#)}cT1pqW z@F?$kKKI?*=VjA99dV;L6ckBYIoDEP=fvG8^yFfR>!H+rL;RaFNEzbwDtzzc224as zVhPqII%@PFlc?(KsZ;G_B349}=HJDWS$LSCnyljZP6OM-5X7t?jFreynr$4hX9-Z1 zdpSQD36%pL`13(9x#w1%i#7K7FHDcvP-+{ioo{fc*fE#{(~9a*H1$1e-^sXh>epVFZl0j_!?*4#0zvi5*)T`l>C$x$p$SLOdF#^4|Z-g^T zwt@KsvaO{L?x*2r^mERil}eYs3-NVY9O+XahsmB7J=c3D>ToMmJAb9yhzsh|lbAkv z8ZId&QLZv~C-C(KdJp+}a4?(P@!N0DxYehB*^qd+O@?rD{OCcQKx8?>moN6mmGx>wVh<`)j*qrCr<_l{pbh1cdZgzPg*Q_=(}Mwd&We?|DW2Lkf82 z>13eS>4sd@K~osdu?+4$d=q-$Uz)Fbh*kx#r*3k$XkpW88fi6#O(aOUgub1mQA^_A zz_XHBP)(>`8i%kC(_31yIysFSD%)pjejU;ZN28{ZiXFs}AOk%)5z_Y=;X8oSjq0ct zhzILyR;}{ELpjo6OkH|(xoXLhbN#N_Lj&Q!NkJbl@bSN4rMlsX10~upq3;+Vmy#Pd zDlsSW2DKd@Um%=*cg&%gED2%kq`0#0kU>u7{u?MCWG8T?mmGION^ZKg9D3|glg0s4 zh&${l*Ly{qLZU@U-|pu22Mb(X=_{Qsg?a?XEhC@YCngJ_l*-HO7rhG}&3dv$^P_L7 zFbkj)r%p`3u)Los4CY(f{!qxi;q=+Ft%e%?_r8h`Eb-Hb+=*WYP+!R?CyV9f&)L~! zZ|78Fb6YyTisdwu>#@&N>K%=Z zv}Sims+I8!@(kNX{tY6H)7CIsuv2gxzjUhqxak?=?1~Tm=Z~*6h?1mjZq~l6WRGS@ z^5G$hwk~(j32CbqAgFs}@a^5)b@-62y<~rYko#-n4H|9pklN`0A`3M^{#wRjX#00- zBY}BGuFnsdcIɽ}Nm#FU&?PnH}#dbDlECtJ5}m6bRRq19Ya8o~BO&CX!l-S&Y6=66T)8~OhS$Ylj|PRJCONIBWz;ttI2SDgk*5{;?6Yog>~%S!!AyTN zt;O?m-Q>r|M+t(#!`~e}efso^zi)@$zI<)+rw;aqUUP!VbwUIfqU^qmH5L}2REnHR z#}6HJ?*m*P&vv$M7y0kqg924>KZ=(-LR;9kLr^3b4gZMhYIiY>(@#EjYo1PX2u^r?2y>G z30a(imf=y(rQf2KSIuXMAw#QZJaXZ}1-JTyyaqU%xQ*SAGQ1%Y2u>*WWIF*5L(a?3YhH=Ls(wcYS=qizNxLGabvf#-gWsVnl_8$C@;4 z%8AdaihIi*Uc|#1+38nrjTMgbe*p{j+5X#_l~_Fy@0ar+gXMiA2y%^~JiC}9a9X*< zJ>7Tw6H~Fz%Y7UF;-Q~Wd)~2Dty-710;+y%?ntZlB!rrPOqOdyxL15Cm!VzV=M`&V zf=P82PV1G1Ke7T!Zv29+;*j)uFJnlr^jU-t<)63%I^R1LxTxghQ_Y$;U(91{l5Ps~ z%VlRf0`9jrj~X|wF`MXH$*z`WKm)1|X=;DQBr2t|fD0uNWpWk-7|Mr+hBEwnS*aF9 zQ*R#r=bweB49gjz}#3=Y{ZuPkZo$KqvuO?%^ht?^!Ut#H{d&} zbLR)Um(qIrl$?w)*hwgrkD9N+GlGu=$!cRJYc(#k0eQeZ#l3Hue_t&!?DPA4mtSB5 zW-iz`u~12gkoI_r^!JQZb1_<<^2XJi@{TSpF2BvT+qr|&jmt|(NtuLFEbav<)ul4$ z@R2(Sb#O?=XPHj!Y+d9)&y!1=U!Y#QyX^0R z^wQ$B*XC_fLwi!#q1A!Z4Qr`wZgs#mn%%eXcb0W!ax3rwIsQ zt2(pSc03<)a#0Ey;l;_PMn@H0canWw^4ePWUI{a2&FUOetOq!5Ac?o%fJ0FGAVekB{H zh|Zn|++c7=^L~=h8*$&>LsG~IIDc&@pI%%=M{2esTqjnO*P)=*_^npqDp%QHzB!TK zj}Ff#2dT`yr{Ta!@@`gzxjkgi5a|7e-}!5=4Ub&}m1!Z|?h3w-9yuv-$d`ITh{FXB8nTE79xNwI{OaIV-ubhxtkm07E zKTi2|5(nT;WY{wa*iW;vTu58%vzIz0es5RBtRn9yEpAZ$GfiozGhPjTAZk^d649D zbF*~alc=5sGrVCiabx-~2SFSrn{y_^nf(^zw9DV=4az9b6K%TNHb0|p#4i7DZ&E@{ zuUj86us@?V`ZXAO=C8IsHwvsv1T&8?+)hjT^UmN%=OYShKH{ViBdFs4miko!?+Q{JjPLb zUwlLD7UxX)?4E`o?>v=g^+e*Oy=M+cOeeZ-CF)e7cB_-rFdG!=B%5Lc_7=W0sK#FC zvOiz^zEU;2#_t8pKTJki_r6WLvgm4b12S1=w)-sMgz zmbRd6K)rUXny$b^&jAnJ%<_^0&dbo`RlZqyQbE^$0nq|-elA{owjo4xNW(CqjB*pK zv7UoKt{OTsey(ZcCYET8wC2#=PF7PwQlCQlZY zIQ3c`6BK@NFSu)Rs6Zz`P+crk6mao@KwhfaF_=5(Zk}%OEF#LiaFsrz{R?x3YK-@0 z&VAG|-3jLl$k^n22P9OR?x0!DNz#p|=fELfkEJ=S()YjLeBfO$^;52$QlF|}L_pqk z=dW)@lhr$-V3{&6fAJcvT3u(~mNTl}iF-jHJR29_L)PMd;S^8@*)5wqJ^fgeo9W&g zGBr|%z`zD1lRIL`i_c+VaM7(k8A78Y<>Z-EJ@MRz-VG8nz@8mexqq^3Aymifaq}Kg z5_O^-6yTFWHhwMc*I$r^wyfQRY&h}JUj;0K59Gbwlj+$~Q4?>QQnkE~y45FDiV69J z4*zA2DBcA`YxKwzyX3iW!*uAD{OYS8{BvH;DpF*?^2yoj3PcyI(*NM@h1<6Wt_;{m zkGPYH-j@GnEFE{<+1wEc(f{f0%)@fd-@gBCL)qGeB5f)~rI19Lq_S1AhqPEDWh}`O zLP9E{Frj3b3`%8x^7Z<2=v;`VH@G%_>0)q>+#ftCd?q&az4`l7}9(_Mc)afL0F^K=NY zf)AVts+Aoqz92D|W!x>ws&ys?=M6KK>Fh~i(fh{6(I3oSaAe9>S)4y}=Hx@YqJyN& zC3l{2Z}#9xOL{x5T)A@oU8jtRW=25F|6)7dU^~KKo}ctE-Ea7PS_^Im$t-);MKwtw zQQ2qV(xqcQtZ>ma7iv9j_|dwYi+C3iwW%|=;HKs$0>5ng;UUBW9sblha~c-sBNlEq z*2Q2hX+VE4RiaVKb4HQr6N+%Tp4*5q3}xdxNl7~*KJO%nGn(5BF*bk5#?9&`akZ0f zIZQ5u)GBMO!NboI7x-CS-Tbhm5Q=sz^AhlqW!=J-zJ6rhdP4g_d-=%iLsqXMyuT59 z6>j4m2A=e-yjcRKZ62t9Q*)pgITe?#_<0!7-PY^A!46ru zLT7uISonIx`t^o$htMTa7~8m8Fwg-sMV)VuNKZq(oAb6{V_mGcZ!46>ZC(BBeS4Q( z_+)xrEG}NfxuVu_Sjz#|8MMqD;}mDW@Mt(BL48qim>rgrn<5%6kQ66pBSxRYV~ff4 z+-;E*<7cM<-reVa$8nfknb3BMoYu%yscqX4?@QO@e}Xm@`bNMLBH7KEjHY9r*=sIW zF?WXAG+yh2PW=yqhB zLhS$O#PmM+MAzoe-*#9U6>3&GOs*uuB`Zz?CJ^i)yYUyNrT_pKZ*(6dZN8t{oyCuC z#5}r|mp2!|DP89lCO2Y!NK*YoCpR%T28EAHTwSnX^UjkCV(KqB%squKKvL>}VE8m; zF|kKZhzVc+oIh|r)e4H;orw7 zZCu(RPW=t76yn%F3HJpHO~mpS6;~}}3^|KX3D{P;gojO`IRwd{J_!@2gNe*@MJXYK zu%T=BRZRzkZD!f$@)Td;Xckk)NXg!##x8kUe^3$@Ct5LW)NZ}7k7!bb5r0?u4qpQW zDrPx~#!6zEqUbIyDl}n41D^?S+?CNUqaSoQI8#(v3Y-Kqj@j#O!b{RafA?$=pL-Fx ziPS6ZAgQxK=pTf|3@@Kdu02Ri_2_6v-5EI2xPzdjC>4wg69hI#?ic4j#d`GHyb9{g z41|{Ul;3r!XfsZJ9O2=~AA`r?VZ@e`XK%`bM0=SyD=B@ws_2mmbx8BYS`uhrqdmeE zi=Hj}RyHQ;cThN!22Ze+HoL6eP8vt^WggI$q0Afx!)1b>tX{lITbXnVm{lF%JU&D?Wz)l?!1CO7aZ%zCIS) z<1d#p-2?$aXPzT;W1h^>E5?xr`Z3E=@BwM`9TkI|_+N@@FB+VnFNt+aQBC!mTl0TA z*jsT;1ZlTx9Q^QZgFxFQZzc-Ii+Dg3!iw1w5!+CX2d-)LsDt*k^sYA2m^*y+z_MFE zKEu&_&tw5NT&$qVtO-v0$Bkq=M?rRD07;4nYD*JvoCBXO3_s6kgk>2uXMd3j zM6j)w{{h5Smelvok`$Nubn%%ZG6>Wtw+UgtPCW4mR=bPqA>z?7M+W9z{F!5zDMHJ| zi@{GAhjEr7;<)$1DaQI!IeFcwj^I1PzQ~{|G1pM3V(w{?t3D0KJL3%kr0ix zdgPm)0I>S_=TykBJnxGuO0*W$sC1+~%1Qe}HLx@$D~+Y11kB(+pFRtz#vX2lL6;#u zxzcuqf%avJ$g`(U3GT_s9z41PS@sbIs_c;}{n4;oc=4v=HG;1EnfqFTda{liNdQzQ zJeU~73p2KW7<)*oZ9`Xd*hY$icnUPVQPixp?RA5U?<=C#?h3yKXy9uTw*{xs86Bh|vw)G{_eTqnz22nx zHO5eYFD_PC&wjKm>o2|#AR`AthDrVg~JulmfI}jR&Zycihg3Y(2{Zl8or{E z;y?lINW=i_vg2FPc5GB1tSZg9jj!5@B2u_Zaca)Oe-ST5CDN6q?y3@|V>liekN_f~we;jY}~W(N)}Vcic^ zsz{$Zb?+V|`mC3gUQTJH2GfyH>--SUUFU9I?@tLi$u;`cnPeGxoJVk!1k!IUSFymZ zFK#8-mt~jVL(b5ba7+X$rNlZhYjU<3@eb5OKrS`?6EuH%YuH<0_K#lWiy-C34mTJD*rfnPNF9uD|grS=G?ejex%nswO%Z$YsbCP}|l9EHj>>YJR(j&1i z$nIGal8gCoQQgc249;KrwPB8k(w;G7i@NZbBYmni+<-8Pd-H_K?GnudwTqO*kEG^k zyLVTa=pEj=JuZ`}%r!_PX!(BpyQ4>LSbI|FFRCGqf+AdZU*|-&T z>e6MdXu@WYD1Z-ccNdcR;Ab1^4s`++P$2us?${%+G}FYjnZ2Li{SDL7GB!miJYCTe^NIMEjU9}bGP|3*Q}WXuK(<#&mbGryBu3*- zEnfYN?AzcUo7#W)NQ3h=;^g0ADuW@3%*2p2@_ui<4-%zBMM*~{j{2pSj#ri(|2$c@siOJktTJ+zT^YEMzD#?PRdngg@PRo7i8u{<;SofR zrIb+rrXV(4dpj_RE$A8jyUdGVMxCZ8lKWJiLJ!Yy@5WWJxz4eD(|H1?gdf5dk?L`~9L%4^Tr1U0U zw)H+3=+I(^wU&A4EAB6857xi>Zm}5SO_IN?GJm{GxMf~TrgZN0-5bnRF3Xxt2c!Ah zqJY6dk~I?m41reW~QzLgYk?)A@vxyVgTU-M{<=f#w97wa&Gnz`FjJ#p?&e{AavEAB2^i5o zi3D8pSf5ew^7o*rMSHZ?UW&T*VJM-5BDF=-fS=0n6P+PBYNEs#dsPfPG4>4KP>hTw zr56(e7N9ID&N_i_6KcD2G*IIRrVt+)ysG(Za&V@Qj6uK%5(6}Yj`I{ZYcJjC8+ku$jI~2--L1& z{S_gjGYIgQqaVV18dZ5Wqo{i~FpqL!G+Lq%R7k@1@89qA1L$tI&{JVS#qe6ucJSq$ zK4V1X06>sPOVIkv>R%3>CjdC#5_bSt1%3b`J@adlx;0f zuJ`YZ4^lfs_IuAqiw*rbJ?yiN=EJOu?VCFP`MbN9^04&JKAU|mYhBxrF5k9`#)#xi z$(GJeTg`Vrw%a*d-LFr&LEb)RxyW>%p(!4IzbOw>(NK7)d!eHHmxO0G+~qf_e0Wl9 zQ*3)I=TZ4x3$?vDFaDY5$JLg?fwXmd^%@aiclU7Z)EAYN;mF3!CTF^oAhMnW;If|t zFfuCY3c8MPSmfttyW88_YZ|z3@`TILwqLF;7I99+rPV{A<`pEW#Py+zqcI*M{}`w- zY0{+n#ztp25+p0>r1qDj`R(wz_x_>{NDA3&C;1ypF*?1+4!%(O~ z6*aYW;D9xyC*V$CPgNx!=J{OL*jD>j=l%<{e_y;9>Bs_M1d3N-E3TvCj5v18>Wb>st#Ar}pgo|>)Kv3J zS0Wl3ejmASu>IZ5rIF=}%_nC%;CXfoIh>kWg-3OCc5X#NvXCC^)S&0yxzmAL<&61) z!#vKuu>O#t;!O{k9Z%yg0nz0SXN4U9N0knPEG-wIXdf;+d)ht09XiyLRo` zk$;K-ePrr$M!!46n2UzD|8&jH&Za@V2K^c=z+P75AMG`=wmioUxIyaGt5?99E`{e! zA*K-$EDR35j0cDyPU}HTTKtpao5$WQ%ADozUkRnqI`^#H^8Tr*sWUL&wMD6I!7$bf z+we89dT~jbp&4xwwyV6HubQk{3D)LI!zRhA8_-13g=Oy@Q{R!S?t`LLm?tkR_xoedv}!M0cFXG*%Y#M`hEM36$RK&Hu86?SFdj0v#04f z0b*2~LUchd#tBD$XjAl}uC%`P<;$B8Ulk#4$;qxRE-~b^&zs&Xm_6GVU90K6!7TZg z#6Kd-9}~V`yLZnbzhWN&;?af?uCB3k0aKC+i;H*R(Os7>KNs?ibm}@4Dv}FBo`Z+9 zPhKIqkX76N(>2t;PcRfCSqFWs_CC!r63CnLDjP&pRE*Pj4z2f@-sYSb3Sq@ui;W97fgKsp@$VZ zNWJX{&nV;IgLr10VW|JhN?%5Xx*mdi1CiL$Z@|J;G_&uHC!;G}~+4uis8E zvjRN-HMpZMr-#_s*i`)3OS5#zg#jc=5B5dQCYUH#l!6JlAxOoh`YQBU`qFcBu%m`W9g2q~pV@?xU8-<+HvgnCK`F+mEYVciX!y5=^9lqU6>i(A6 zs10iO%W{+DmmkjIZehw5!bShj0LKr&6v>AV_axpOXV;G*lc8%ry-GpW@Btg=yS$ZC z&>bePfRX59;J;`ZMMgG#V@_-ItksYATFh`UB9s^mLk#(}+FH~P_=$5CdwiH8LT(4+iV)B%xDrPaaS0%8hgpXM|-@_1L6jh#3)uM&B>K1z{_g&pGuZ0kyQSh&g!h;EKQ;{n+$zm&q`TFV&ZmN_6;@Xb`!6|I`>o0Rj*V0nu(4j-p&z#RpKGQ{| zsDxgYk!77>iyyqWuj`qHyhgYq3%VGCgG>o1BTnANgQk$4(blWQ1Jcl143#$hA=<%{ zkF}F1mT867JM-}eHI3_#`RQ~iB*<xg5Wz$~h2YFY;cy<745 zG(zmiHlj_&(Hl-)ZuH6Rg4jL}uXmD;sqyn)zI=HyGt-Ges$k*GpWVNIUlcrl$AlV> zJ=6{_FVcvok)yq4!x-hmjRlk5h?)oC^E}6bMdx?-Be%O(Ts(aIc+H}sq8*E7qLgUj z9fIGL+cUsN5f8dCK0bcuKE+s5l{NA48X!_C%E}ThwmO?g=<1_KnmoZe#{&-j{svTM zZ3AP`e)Q;(>9J=kIxK9jeH9leZUU-AGS7J}joV3t~$fE-39SOAmB zD60TxbQbXvpx2I-n2e1iE?qc~1d@3i=PJfDuAen@haE1gUH8G-jg5_@e2&qVD!@W) z#W4Xt-j6%_@-_&FLd$x#GF-gD~C8Rs01w?fkvKkl59)3l2#Ot}9F6I}MI_6!^LWWt05 zDiGYGA1=t~Ua{)iUm4SVeUmY!V}+W4uH5&XS0oP#2nY})hC;YJ?VReiubh$%POH9X zV3D=Xs806E8avmoR}irwp@sK}rtHj2d8xLx{;!d#!@*W&%$#}XLR5Zvc{g+l3l}cT zB}TmmWwb@xRuu|BSe+cm z?m;Dd#m2;hgFaq)@Ial= z@;dfEer3(=JsSz%lF>F?XSSm*s-_Jf_|MNpwH|U_E!IdFByb`+?`Sqq9@m(jnfbh; zo9SZEf!M^vO~m=j^|&JA`X>3nP&r$xbN@blQ!_S;rRW(m}Fh z=hOsu#*++M(WI=mme`_7FjBsr$^H9dF&~_m-W?x_Bu9aziy*uk>SO_)2!SL=us;lF z|K#G=Z& zh|;P(v75?W00rL@UiS*>^Xc4^!RO_g%R{zuH*Vd&9gMUCRo?T`o;abpiX;$neOuc} z+uui8=M@&Jh>q2Xb|{^%LTt%M9lH&Z>fh~}Tr$00HC~-$P|nQEtR9I|E(epdeHA36 z6*=geic8(Wrkw!0#1oObORZY95~bk~#;H2;wHbIbtXBXU^|5n%DUPsWY&ZlB$lJ=H|hK_3p7}Jdy z(4>r-)XEnxTEUs4Oq=rTOQiK>QeX`7x=7ih>9&(cv4vUFI~b2wAZBNFceZ)QpRRk&uw!gG7eG z-h13CI6?c7Z{r^wQm=A}J!lYUU}!jlKFG`JD`mKZ>F|`=D{YrL7(U!{N9W+-!zsA_ z+5eog5fI$VWe0nhx$RdqCk>Q#jIf^B$pA<_8A!&$+WNuFLCKDbmMn2zv0@NnwGNU% z7c2>pBHM`0j~C*v3d{&cI#<|r;+MEwb{8V@@H9GT%6a>MzK2FMLHU*15>vD2(b^>QbZ{Mya3Dc3Pd-?9&j^Y+A zTxg-KtxeZeHM~Ba)1xY&)59ZEmd^umY}2vhaPpAKmoFzhDU2*n*tV^s(6rV(+}W>R zzX8A3D$_`BGm9v5BXsiyf%kczQI_hjcbY!Cw}nR!10$nk{;~F-^Zf;~MF>5Eo3fCp z_ZJT@`uO$Ql{5uKdb#8R(%2;Ji~PA~nz`GuEZxhrAmGQU7213GQiAjh7jXt$*=^_* zawQ9flZp%P?CdB=*v+E|bRXuj>n6j4E5${R6+;n@GUKW!>l-4^Y4O!irqtHq6M6pQy_k$(Y z$$Dv1&u3$OM)16nGV0;@IzIoyhw2gLsrt+9@Eh0wC+P<5i_JOTNRvi&rjSljo;r1^ z=(n5gr&=wV$jCZKF61@_Bn@|e3igpco#UiPyc534S(Ltt;^tr3s%?nBWol2;ga;42 z$WyjTB1eTZiGl-jB@x$?{BZ=K(L;E9DTYY%Z(wqTaB!pdpFj6GG|R!MdECyOJ3mn4 ze**~UAb6*&1DUU(jm@~S!pNxSj24x3K-2iw_s?IjTBa+(?=yEq*Vc(@8ijDJ&Yf>! z=nUXCM(-VPU~k9c?g!RB$qAcKX1VEuD5|Y~vW7M&7k_-_so70g2aLTxuv!L9)n3wN z;=_u$xb5pPdyiC7+LQ`&FYCZT^+z5dsz(Be1*a!H>p3z(S9(q)>G89_-!EjrjECQS zC0X9IkZ~u*fR$!0`#o}M*~^tX2A!Vf#( z9+{dNu%@jVmcAv~h7$ZxKEYVs3ieeqsnD(kpm3oD!*c?af<_3?HquK%?xbf-n>Lro z<2NZNI(Z!fsTBPi#AhERxIL?Bpd4>RP(Q(LsSm)C=+3ufQ1S{L+C5D2-k*{(w(?2@ zD2UkJkPDx;Zry4d4k@gueBoHJs_9)@Clo_Ud{U z4rs-DWALwYU1Y*f2k!UmX>Q&gCV>C+VsI`ZC+Dn@19%@+QPY z&vl$S09{AO}h958A_A_$c; z$i~f8s#bP?IRwy74c=`{YO3LwKaX4^cUcD=z?WReoqk+WQiUA}@=kDQIR5ed`^zBa zpqkdVcH4{y*mL4UACQS~k`O&BD^-azAqNaj3Oc-`kFjwZ`U5DdaEd(AL%)15%kyCR>wbSi>4 z#N?q z+qPv(I|Sb9(#;^IzRzga^a#n$oE zdV1||oUo`Oy`vG&22C~68~n4DmX`7A!Tc=Nd=sI*s2`X}jUy*ZM#3p?*tM$@xcmMK zvTI=w`Q=qTApr(t1E+SbiB91j+_4Y+a>dU8Xo_(V$oC!2!eY*Nd?r{Q7;+tle4c)9 z4Sk*Yt(l^zr)RoVTStcsJsgy?3RX+(8|}JeB*;_YhD7h->K#O>uMWye(AP?E4S85= zA}-Z{q0GTqzFb!7He}TnZiVk{S6~(=CofM5v@cgwbRRfypm-Le(R+_P$tGq(X}WFt zBhO7AzCIBt3&GJsA=O(;I#_XOh*ZfppUK<0^(U4L-@-@}FEoc)8L$`nqADe!u8xU;q z5`!X(@{4Q4_j0+)np>jZs=kYfE)tWYn&`deGk30-voOc516EB%#bZ+?)1JNWbn_W1 z8$XRKNsw~rvZ`upBSl{~J5>$2|9(TDXbUD2NaFo^+1O|x*kB02RKNa%p0WLuxU{QW#TMbs1Dl`JgfFgzh;%**7;{wdE(QjjgTKMO@?PC=0mk z6byDJbfRY(-X%_wa=W}NSI5f|$%fC-fS;u6ef>shg@cZ(HS`}!b^}<$Sl!d*f zW330={`}!7zBtQYs~xg%Vfeara{R;tD+`NUQa*9CNd%u_8)}(UneEr|?ke$Ge5Lgm zx&L}8YMHgfMprPqKup6$ji%1#s8~lQr?MU`U%~CvnKJ_t5);KH{)Jl;)*c$>{u1?; zk4{#73GD!6^k_*ty{(}?zuP*=ld3&jqFyAcX$W@xKYztK*EhCrIb8qw?VyU8Tgz=~ zdAnb~ZT0bx{hvR_pXbPL{mAo%SLYA^b>j7smyVn}*A%yl0{Lj2F*nVTbTu;a zh?c-#gl8xDGBY=y0cA-NasbOKC`*AO)I~PzuB`MS*c&i#;9NVUUyJmk^`PYqZNNJj z+o>{g)TkN0zH15h_hV#*>7pUcA@PHhG&>$}_kkiPjb?uvM&G}`MAFksE`oIHK!!v{ zmh2XWGvcDCXlrjlnNi8!PIhxioeCXA$Q%g{kb+RLG#ZjY{5nBjTvf_^VO1u^H-g|* z;dB{ZBLT%Ca_nYqc(7>nm@ypzDVIUJWv2j8n_O$T9M*x9jfy0Y=nWipBXkCwwFX#1 zE|HD}Jyb;8P0;ykk6cCwD2b234t6BV820<`*1&VYHFtF2(%PT}G&%OyUs*cadHE6d zGoGBE*bP4_ntybcd;s?AY1o%dj|fEqKt~*ErUqWP{^v{qf zU=|)E0W>olHs!r|Rn(SV;w;aZ2pul%$CL#PY9#w1#qNqzk>up$9+#H3z$-VSiWz9> z2hV|bYSpfN`x!w&+t8s_Q3gi_wHEN!%WFqQGvej_?#3cr&-ylxejg+lrK#V_4a#VV zAaP)zXF4F(M3&%0%;rJ28|%3b+OYn2KP?p^02+MBk;{>VJMpxZ;GyK@7vn2R%9vgd ziP&*J=Fp;V-*vIEDgr9TBwz|U*?ztBb@!X{oUZ7%BA}5uW`*=&_s&^9t}a*%DFL2T zUitTYXRtGSUlLY$5pWC!+Ctv?A%>977ad!Y?+r}S8w!c>MNNq+< z`z>3y@~xYl{lMu?&V6VqT%V?JXuvXn*QKQ?SHfs+Eb)mqkt^zj~Ov>2EY5Zy`=B?Vka|CuxO$VCFm z_XF73Hhw!KRz#AxXii?cth1HV?Y{5wVGM=$&&gKkMDuhEjB zE&=p%x(H{YPjOehYn*9%@4~W_6DPJoLH>5YU0f$;62z*PFE{Zw`T$;n zPRwT}nkG1(GkA=$x_Sg=nuA^mgo@q0g6zfuP%z$7@J1|_`=wPY(Us{A9^8$LOHh?3 z0Xj}M?p4}uUPf{*5JZ~a>hR8ILaLQeaiIxfd}K{pgO1Hyb;?WR=FOXjLQfST3ye%2 z?8Q24TMF8RAoB?x_N#wtlL@*2yMb?rRYDwqj6svT;KVIRr*~l>y`S|bINjy>qLZPJ5yRn{UbHG<^C4IbI8nT#fQ;X-9em1d%Wcc(3`1YP$E z+gq$*G+|n~f2!bhWORwg$Dg;?*Z(b$uKQ1(cvt_q=1tlt(!Dspk#tIwC~fZ)FvZ6Q z+=#77o4Pm!=)<&^q5q=Ai#4PWT0~`iDYtI@E2zj>i58amGd;!QCfZq8TE=oaN1*r2 z2%xHkzOEywxYBxA zU}AX*y}q23G=qEQqGIYwn;49)G}5Nef|H1WU2)1keF6XYK3P`UE8qAnt?nCwzMQkKOPo zB-=6$sm<6+CTn)t>o*+ zCTA&ukYK(C3K&^FF)^_T5kAy;%(FpWTx=we$|@Lnsj0#?bj*)g< z=xrTQY(9LozP>(J@$kL7RilJe$%$%-wXeHOn9vCfR2T_vh;p}X(OfTmj2vEAX8&^8 zT~LC2BCt>o6spyYt)v~>RWNJP{w1D0reKFQ@4duQi!Qxc-)zvFyj5#j>iV06PICs`~UReer0j}BkGxlj0$=`|OoHLo?E-QicJP^6890Rg=x zA;ROFu>L~TfRz!fBVqx0_cI^weSguRJnzEL8K-hgPoE{^lZ#(;|IOKTYhU!9dpx;d z@G9Ivbm(TX^c`Q+9ZoH39Tc|?OE9h_y(!Xz3130|wG>V>Vp)l}E57*~IC6a0mzmzH65ny z>u(&GedR`y-3{~8pbJaCQw8xN^v#=i>$aO0wdU6JMvF@by%SkTdr2SzlJSJ&+5){I z9Sr>ET$aN(lD-W&Irf;%_uQu!kAL#JxLvlTr-EKCWOua71h7c3el%0VXvz%kh9>pg zSn56}vC1pHb((PeiHHp)ktzr>?act8@2r0(?;H{m5?u#uGv01As!s_x@6YtWU*GRw zXT7z=!E#L%Pu83|)t5XYqPg~TQJ{C6%$0r1?uX^*Q z9al?8G86`mW(`NdHez0_&m}uFzP>=nt6fGu@$llz+)ImE@9j@YItrnh$Reop>4xLJ z0&oxv7zjI}XKASd>nLVJp9IU19H2+y{SfoZNCG@y1W0gQsSPiDiq{f)7pAZaS{?^) zlD~qcGEz9uvOoX?9wQJShzCEArn#pA_9>r0s*A+m0ai`?6IMN$QuMj`>a3xd7bt}z zn?WU~2y@y6yh){V=S{rm#hHyWf~!6A;WddoM1F+l6bXjN*g;{1h8r7g2_8-hj6YrS z=M*2;LEkTw+Goz6*9WPt#kXfpUA@e!=ugBO1%+0}!)3e`be+}eNRF(S9W3o!sn zLK%bS9F9g>d}j!L479$Xp#l|UxlnHw(2>4(6w|e zqgt^Sa?5l!Mo(uJ)!af!0D@9kSGSdjOdkRi#S}&A!x<=98cgnm|IX)35Fn0kSsU9n zJnGH?!5P-P#;!M7(GpaG;}AMLY@A1RI|;m!bar*~#e!f_`5TQ{ zizOgalr}|2_x8+2N`-1gXv8XFSt2P0pe|mN4_2RB4M?Z`E%d&3sHpb5_)S)qGq0%< z%kRP^e|Yn{s_H<<78J&Rt{mm$q^hdgB6S&_`U9V!jw#Oi_W8pHRS879Q@kZy&EFrF zdL(H7*X|OhUUI0}hsb$C=Up)|#GJiDqvt?26eO??Bf#JvH0tyVOpsy_&K%i7kAO{H z$0mBzX;EODRkLONO{nrAGPPNEbkWn*l_-%LDguEMCpz-lE#2(95EzlmdhH29LMCK1_)7kD8w7wa z+$(`2`Ek1k(V_PP*Yks>g7(CWuFuM{;qTo67DRy)uX!-J_hLe|V7j@PP04OYuDtKJVNhMpJ zjESFkX_Y-cp0;G|YkXxpKNiBH^moedFPD@5>x1j-4M zEBp@oMWIty@$E1~?`d_U`DFmjLvn$&DgoD63K4-}GsNe!71Hy)c1bF2nDjteNz#WiMq2>Y(XeBZ&?7!eS zFv^#6b0q+W`%6at3Oe+P;wa-c2>xOuH5 z5)yKk$651smDX{HU$|+BZRS_=3|CiI0TYTUO}{-%))N zJ~*ldoG;6TqE%HlLWKd#8;o1H#@JslPGsin+;EwRnX^@>?sO--;))0{4;0`~#OeHH zrVcOt>G$7j`s*ov{hr!!!vDXJ?*IIv{|mJFX|v3s4>p-4mKz^2!a#BzHrna9qtDv^ E17(KEng9R* diff --git a/benchmark/linear_attention/results/gdn/b300/gdn_20260806.csv b/benchmark/linear_attention/results/gdn/b300/gdn_20260806.csv deleted file mode 100644 index 5e0625982..000000000 --- a/benchmark/linear_attention/results/gdn/b300/gdn_20260806.csv +++ /dev/null @@ -1,16 +0,0 @@ -case_tag,backend,variant,batch_size,seqlen,num_q_heads,num_kv_heads,head_dim,fwd_ms,bwd_ms,fwd_tflops,bwd_tflops,max_diff,num_iters -fla_B1_T2048,fla,gdn,1,2048,64,64,128,0.217,0.649,89,89,0.000000,20 -flash_qla_B1_T2048,flash_qla,gdn,1,2048,64,64,128,0.102,0.352,190,165,0.000000,20 -cudnn_B1_T2048,cudnn,gdn,1,2048,64,64,128,0.098,0.279,196,208,0.000000,20 -fla_B1_T4096,fla,gdn,1,4096,64,64,128,0.416,1.260,93,92,0.000000,20 -flash_qla_B1_T4096,flash_qla,gdn,1,4096,64,64,128,0.186,0.677,208,171,0.000000,20 -cudnn_B1_T4096,cudnn,gdn,1,4096,64,64,128,0.129,0.428,301,271,0.000000,20 -fla_B1_T8192,fla,gdn,1,8192,64,64,128,0.796,2.490,97,93,0.000000,20 -flash_qla_B1_T8192,flash_qla,gdn,1,8192,64,64,128,0.347,1.342,223,173,0.000000,20 -cudnn_B1_T8192,cudnn,gdn,1,8192,64,64,128,0.187,0.721,414,321,0.000000,20 -fla_B1_T16384,fla,gdn,1,16384,64,64,128,1.567,4.957,99,94,0.000000,20 -flash_qla_B1_T16384,flash_qla,gdn,1,16384,64,64,128,0.674,2.648,229,175,0.000000,20 -cudnn_B1_T16384,cudnn,gdn,1,16384,64,64,128,0.305,1.309,506,354,0.000000,20 -fla_B1_T32768,fla,gdn,1,32768,64,64,128,3.119,9.976,99,93,0.000000,20 -flash_qla_B1_T32768,flash_qla,gdn,1,32768,64,64,128,1.329,5.294,233,175,0.000000,20 -cudnn_B1_T32768,cudnn,gdn,1,32768,64,64,128,0.529,2.494,585,372,0.000000,20 diff --git a/benchmark/linear_attention/results/gdn/gb200/gdn_fixed_batch_bw.png b/benchmark/linear_attention/results/gdn/gb200/gdn_fixed_batch_bw.png new file mode 100644 index 0000000000000000000000000000000000000000..fc6518e4e2607d9d5b9542470e83052dafd7934f GIT binary patch literal 132796 zcmeFZc{rAB`vrQNN}41^A~Z@ON{KR6l$1!B$xuR+smx@|+(1(1S%aY>B=bw$$MD*`!HT+B5=7_q@>B|>vuIXEyr%veGT)A}F=8}m4&(-r*)+U!N zb_)s%3knPHT(q&dVl6HtWd1+z5WH+@H=l#-3V+C?vzT6}#x@KnTk57DjJbv>2Mh|DlE5~bM5)uRAoK?J%!Cq|X z!c2?OwQk^LCAX2$J!XK1{8@6FrKRPsv9a>%FriheSJSOtz53IqPemU-tQj94=V)y& zbmeyZ^W#j*)>}u<85qzmUAh#19dj~LobPm!;$2~bcIa%nZXj;9 z#VMy^Ws}Cjc4~R!?nVcjS7(`isIon%Q@^)DA(X%S&7Ez(e*G%Cv1$|3u9E!WAMFyV zY1->dOib*?2g9oz-!wF=j*X4w9~vDUJ$K=PH`5ya7?XyijOWiAb5@>kc6x}Kco6**tS*%b5~}WHBnShP|(-TLf3*p z`QiI@qXjizf3&~+ROl*2ovRAj&Uf~)dO=wTH*5c|w-5FieyxqoTahen^hZ18n9QrS zS;M)K!MT5@?%uuYJ13a zDy%oz3A34G+EWB!qLWuN-hiAHbdT8-v z1BZTp|EaBeAdUJpIc65I-+EDZclWN{yAKVuWYs-4Iem2VZ6~w2EjlmA##_UFodn49 zp8o!AHiMs(Yq7{w%1r8+M|U$U+QoTy?mUq=Q{X(cC%by0x2uasKs)Eujevkg_UdHk zw-NhyH+(vNS71BE>@qd<;u80@3g?3^)9M3PADw-|LNT5^d9t~qzv&A?*4@|FcS3CL zSLsTP?9A-!wG?A(wt4j(hr9&yfHgbLvQms2Hf&&E)#$jt$K*C8t)6oww&nNM6Zehm zU22<}*yMsZw{PCO`E$-nN;dgqWJpV-#I=(bo6;iBSy-g0?2|~3{QC8xeq)NVdqF{g zf|Q(`TuE*0p%2+FEqlEw=@iYZhd1xve>gVF6rXRm!Fp3+=SWgg5-ru;RqXM%-c^5i z`mc3yO^u?{yFA;0yO*a_@-Ij_>pgsC)+8vLWc)s=PfPKyjOB{XeEVl6__C%$+O{FtY{Zp1(Gx-l^}69+uyDi?Kb=cC~JSQ@Dwg#MMFOOP4M+j>je^Z(GK` zOZtYd@0$;n7i8?7J^4`}kPH!_AFi-L7)j+;#P~ldM0_DV#WQL|dEF z-NVCjs73UE>n!I|Ho>y{qGlQAtJfkVgsUgcrfK_`I=n>P+XUtJAYnKX>igRcm|Et198x9p=@mM>GW6nK?N8u!Bm< z$~IT0x*)g_J|LA)OFU%T^E@<}e-6#e^Xsmk!(##ctaDrcDwe0<`6 zb*QDgG4=GPeEZ#UfgGVvoY~~;hY#HT5x^nJG(0@ar;@;U`_3I(t`*L5R@Q>5KI!Fc zZ{Bvk=zkyGJ2Bi#QP@wEc@_C&2r?+2~J5T_I(y@ZiR(~Ed#%HYGrp# z{VZHev9Dhre>_a^%9Sh9SWrRT{2Pl`ZaCNfE|PJ@3h%-~*PyhS=dUX4hI<$s#s_^L z?7K`$`K{gQhbWJ+>Z`&IEk#zaoSC$3%e7vN$SFc<8OFmkKz=oys{YoQn{TJhAXDT< zLwDs@mkcF6^5t#!_wRlwDSXp^I$iC4b^59*wE16MMJmf#?MMOH@8cFVvk8nyIuMH`55osy=R%}W$ebj zpUJ+Ua8yZ&OhoN`&myJq*s)`)@Z^&+gR$$GiYnI`OB&-LepzI!@cN*V zk`j$uZKk38@ZZmhOLG6@w(Cc)wzm|lNr}<(Z<%O|s%)EfmazU3aO$zTu>PCdTV%Dh zBNTFzlauYjW{R%UeHd3xQe20-ynI>6)5m8==$k&P;YrFZ1KXaHD}HYiduC>)Y$*R3 zddmI!_3K?feh8LTRz7;-i1b_h_U#FZXW2TD!-)r8xc2|FzEki%ca~_aEW)lzO|{gNzHZ1 zd42usxz_!0sGk=*3!H<-jSURkGYu=_G@f0!*p?$K=``VnHSeyAm*@B+aqWk;V$^|k zh@}ctA8hBje)5i1-DK6>=1^tbxNx`$_W;2w-y1jWTU)Y5h5;R6@fa^M4C zj#9t}l~h&p2THvsPn;-EJRWvsZq_;9@z08#n$JmWX1N5o1rK;Erl*(Qc zzrKIi!Op=^QdhSMd#zDy#OLKp36;~Q>kw;V;^IBnm15RkmSIiLUAh#oXgNm=kk)?5 zwO*k+)HZJ2x=82c)emA??uuJI3JaxZ>F8FmvH9HIa=fIxoZ;QOccugVTV2c(ladS? zl1@-=#U6|MbGTDg{g6fsuy-l9sXssCWjq$`{~;9LHvY`AS8jO4d&A10)K`ycYipS! z#ck_9$^ZG$u5V!x1hCUv6}ltWp=Rl`!2`iEGBViua_Z{rpRM*^GB>ZowhY;!b~4ez zqGLozub`~lpA8wDhK`O7%N&oZ&dAPw_uKCG7dy6&qIWG;PEQ$N?)zl-S>uV}PA zy8mvwb*}7H-D}Z)SW5t=;n8oWk#|agxGXFzIMVe+6pkIc9vr-K!vQ-t;52nDE&qY) z2lwxrI=AJ8u=UF_=oGBA9yzdfFM!CGuU~I;b)5_BFZo$Gw*fa>ifu+x%j-U?uiw7; z_5n3AE?r7T1+3qlC=WrZhh5Z1d3k!(I$f!eYPe(yNMmER|+h}bL;4@ zPS?3uBti}(U@bODNlEpmMoTtGI^BC>=k|E+UEjDRgPYXs1RYh2^-zoyKa=|SQu9;w zOalg>t)NG0={=}sSFEk88^(*8tlMRgxk8L>BG<^^532XZ2AY&89#X*o?y^1(m6Vit ze*SEfXx6%QA4evp~mCudc367Dx|?BzVKY#_>e|aF(-AG?Bd`A|h0)zSM+o zqL|y-+B^|!OINH|!EctUsjhA$uH~jJ$c7y($hzaK;*jQZAg!~Gi`B1O$*6sL{&?2F z0XpWYFd?A`;Ey56-4)K$6PCpvKb}n;>M8fna&cl!QVicnF}FN7*~IA1qh08tL-8C< zRN)m6P)wY~78TJcRpSr6@JLqosEsJG+#?UK9q|zN*j<<9$6n8{1wv?o;Y*`j~n;xoqUc_3vxUUskeNp2>_qal zurHqv&J8y2CJ)YZU_CoKqx0mbvdNv!pKLk<9j{(e(h~#eh4eDalGWiXRt|#i-wX)G zA9NgN1o%8ecb8Tf?duP-tn~exZ&&GLOD}dn5Kc6Xhvt_R+|3u0`9> z)7zVA(V|5_Yy!@@ddwy!46>-##mU0!mYwzIj&|K)wo3iH5D5xRHL%Szg z(&if>rmFt19z_y6()IQASAb|JX=IkHJVRBP_Lk&kQG+tZ>51OpqsNbzVlDS{@S->@ zzIN@J&9{06RB%%D0j+Nj%&>X-tmKY6*y`Ep)r8oTjOHoa_KYvcR zI_|S-lUxx2(x;wiias~}!1U$I7gC*rHXQVOqV=NZ?@0fz@$s|w|8}7T5ObY9z{$zk zH8>am{9p3=H6xlFG5g<}qYgTHlgdz^c>Ln0jM$bO4-XGK*_8;V(SfFwK+T@lughc` zA3KWd3w%dIm3l9yqZ%Kntprw*J9UaV`%<&}e(NuMSthO7J8NocGA=cXpti@z2X9CL zz*kSpxue+Xz!c0aCHL&Y+h1Sa;<2;G;Qmr{Uy0woeY>l#&x_o#Qnci!SB@_#UQQ$_ zMF+77=@uc~;6}U)3MA8XU-<#Q^X6^_=X>rwco47k z!h-ZTva+&QKEGb2{mNm@iTitsk)BVzau_?{tEYt|K}kx!*srleQ${e%|Jt~Qx;pE9 ziQm}VSN{Cap`@{gDe8qqdkKCC%e|QNr$_wPYP{2>rP9*U#GEE}y^EHLP8S}yz4v`Q zzP+Thlr%dbLV6pJx$t-nVL_&+r!%bkPv*PKaG<<<9o|TDbLLM2ws>v6y)`zQ%*I#iUBVpr%%^p~uSeKc}dt#PxX{B#&wMA50|IRxt zX7iP~U}jtknKOj{3=_7SM`Gd@RCkX%+mzx^F8Iw^1@#I7W@n}jG1Gr)X^BHsLbjK? zbV)$=&>_oj^;S4m-bmv7grT^6^0iMjNQFAFGNEeV5T| z=EpiO*P1n@m7{-A&a*Me)Mlm^(9LJZj?2i%I3;oD?AhnJ6=GKh*%9#+g|B~mb`Akk zb5+sa^40C;UI(De;W;$$XIgbpZ^_e$J>(Pez5mEmK!NwbT7pmVOA-i9*|^-@-Tmgx zn-tYN`*gq1z3}kj=H_)xX*!3brKOW>0YL&TyuIIzD80AeI!R1g(Fzt02`Ldk`ID!rmCMOSv5Qn64L$%@>cS#!pv9{pu-SIz2g%e(#`Twn`1K zPN`7gwEHIcpswNGN`ua3U*X+Atvw<E^rT=I+lnZ}n!`qHwWYUT*8wtrNqQ+*v_V zJMaKj&s|0WLdos{JELH`nPmxyKoT0rHyl$cn^oIiUR!zV)-Ap>4^^YeHEOL!4xD_j zcR6;WS7>OcLO5W8__ZI)@F|=J9awQ|3^!l>xudxCP_(2o^O`jbW@cub;x>%fUkbMM z4Gm|aDg%BG%=Gm2F}S+UUCMg%wUXP_)7SS{cIy#&dBdJEU*ts}yeiH+{XTu+rxrtb zSjYFzeEo_Nz7Z(3Sbx-r|CG#xu%<9(^>mpmiVIMOFUyua<742RWSHr@0SbGvdn zVu;V48U5CPH-eTqQ^k6E6xCm?ch+@gOpV6vRHDL~Cx&Z1gB!wo=aL`J+T8yhU}>h= z!z=1hfL69ZgM;60chZoF{E!0htZ#Jr*rjy_@~!E4Ge-j+9iPK(9c+?;iV_Dnu*;Rf0@@rs}!9)KF)k zUO?Yq^RwobYc7R)td(~gqF=v$ed*_EW_o?}ySPn+nre&g&FXzdfBGBx)>}CWzV}ON zitOQhoSrUpzF5S=7a-4$0*IYuE(*w)*Q0F2gUl83k<5$2)I~ zO8nOBC_B#~GK;uVWn8}et>rQx4wP*LiU;k5>EqR`tO?^IHSP>cRnvS%rT{vE2WrB1 zGj#Q;Tn<-uT8pTE-!Q(;dC7l>ncef8wh2ji!*drdT;LeksguVFK+4~meByyOHuM*M z@3&xfQ^b~sI(E4$d;vS?I z%1t?5&SdZ<8nS!4jaO_q=-`PzZhu=wQQNm~&#~%bBG~`aC-sD+B){`4zVuePoTS8~ z)T&Mjo_WZM$CTJEW6HdCt#?DR65T{+{c2_Q^B;Zk?EoE$NN2f%(`v9&G78UwCQHb{z8 z4ZKwB?d>g}1L?MU_l24wY_v3+KhyYE5L)4G?0WTFt1TC5?!{)Oqnq=abDbJDkE+n9 zMSLw=zT6AF9ymB(PfvQ(WQzN}$GW+>m^{!|5_>h)#WwV5&mvEIk(P_3EjFaBr!F%NqzY1C5>^-+8)uDe z+_b3%ozlq1G3o zk}o#N1)P$2Ca*gkITf;((1k2iQElz6h5|p&^Pe4^9DukKG-W^CXJr5@|P(j2l{Vt-`#o9^@}%uFdYFd-fxml$r3}!y=2-SF0!WS%-p<>bEBNS7 zfAa4EI~^UJIxyasUR1vwK^x*HY*h4be%LRWIK{@bw^k@gio{Km8*nIA@j+$+Mffy>BkD>UO=sVL- zfnkU9!8+Rox;D54AG@WNrY+;ewC2k9#_g8JsIK0u@2IWABk!dQyNa%ZkF33SDj^6x zx?kawaOWtH`)kp~^UOwR(1ytBYNs0|4D~*=yKJP$cUC^L@dhY#6JF`8Fs@4mbD+=y;}juDHASo{+y{P|2u(9{21+W2iIh(Qq#+K zf-=k*_9GW4?w^{c_r>gBq+Pab8R&9PAQWFc)z6E|4LMj@J!4{G1O|+q4(zUaGZN1B z{6vRx2%FI~AkwZqdk&kLK0LcS%IJJ;jLfG3X9@7gUSN_;MPEmEigb(t0&1bvNol-w zbo19wnVT%ya;}5s*|dMZ#&d%a+7 zZXVrCP>g9~3eEB3$A_IZFf+U1jiv|W&mS4qtU2-YPFlxLd-KWBZ;UHfR$l9L_StLR z@>M)S`_0?j1N+in4n}xs%%Q>@qas*c3iI(b*^#D zO!&q7-X;T(Oza2jc7aO*5$+FUY4+F=37J4W{oyMJH>P74Ns`TneUAc&nyKPeR0B`SJER+d&!JLl#%6`H?DgDZQ&n3{7R z*diN#%C+7WCFL4~t!nW5^P9hG;!g4mAV45TGbHiczC#g)YuyS%II6z>J#l^CbkL>BOAK`KpVL)v_;X4A1@S zEL3|Ca&uU?e($>DyuDme_HRCX5Y?OfFnR!NOBbH-2Etr4z$Hy1Qw*dJW?QWfX65vU73@++E4B=V)Jbc?r+H z{@-o+KUn)!Pb|@63XhD}%DzO%{~oLu;bEs|W-Mo??TPRL>We%%D0_jIA`6$Ic(hu_ zrs?^E?7onQjh~;5jlXQe0d|q_Wrh_2#Sm#kp12Nt+(~iYQO18HW$GfAtippYDk{2E znebLr?#|W|8HSZBz_9irbT48%WdFIdxeB7XJd>KszIH6&KnQ^0+zo_zT5w zk$b!JY41YY1+%mUOhk1$v#qS~S5Pv$E`8cXsHTBz^ge5wZq5AdkN(iHeH_he{*pbb zAM7=!{biYveLDFh$3S9oa%p=zS5vxPB@%h)k8=&FneNqb;tXbnwMB zZ8boAV1}NqLUAkxK6jBD?VL!aLAf6;w+Mv9NJ3#i!^0iA@AFjS<(7jY-L!rC5`BGr zj)Cn#dK+(D%s(V$I5jAEC(ggg!s4RC2GFfc)YsM*=@na!iAX4!jJP~J`-CG+J9p!c zN4ti9|0XEWs?DXubIJb6dXYy$#UM2Kwm#0y%UfHO;7M43k-XCp9=jhzb^{k^w(IC+ znVwO7qIpJ<{vf!4$?+k7Xvl_8J)7@yCB^6+x3?8+{9q<56fJ+kW^DG|QgmhIP;pj5 zyarnC#nyfE+v$O&{XTP}7m_jUqH-w2!Y1{b6Oxk+fwvj%hQz!bQ#iSy$zklf@7$lU zu`>@;A;4Mw{e3=k(!NeIzD;{x;dM5@7b1;y z_i&tL%Lw?`{$mmM%PJ~*@IWcIb-ORbfPoGFCN3L%8;z^$?0!nKd&N+=2+PI?J2(n8wDhICYKVBzxbk(LGR#K3Rs;s$9my}Tqj{!8Um65TaGtpzOlwcp%JyH`Q(roKgRode;HLJ-SaWSzX&@yP@h}2my zH|rQG4w~->9wHTkX4<|*GCuv=ya2`5G*9bPg+L6Xq$wWgn<_ch{XrFZE-o&zNZx47 zz*L7+VUuqgADI|7%o+m9@`4H>KUfNp7K~-S!m9x1SB`&9n(L9`PDw+(Tn*%dU%gy7 zlQd>&yv@S)|Bi0olPzWUD~6ILG}ZQz(NPHh4h{|_khke`wbQbL@$ubdzN>qFLKo@D ztcjM=ODq7R09Bz(-1uF@o2zXa8m}C(@P&K#uI=gVefGEzq$V2Xr$)zk0ZrRFQj|B* zxDno2vDNIf($+gmpz+XBu~2_yLE^VK01~?+saFjYH8!eJJe&vYPW4Jc2%x7*(0y33 z#`7@Bq9I_Q-0{8j)D0C3Kz6i zTPfzPiuZ0lny}rDVo%yjN){;53!8*56!eOJ`jHd;YUZ6I=n9!B#*ZIAx<^MxD@1}} z?M8de-#Riiqwb*sKp2=hK)bxwp8OSX?9lQ3s!TI7yL3@^2*u=Dyq#h7LT#w{1x$|6toHOY9_jjMdc`)M( zdS%U9_eq$J(8mMB0f*5Q6Jo3p7musU6+Sl5an< z2!Y2N)iSSN?~f*EBM*GLS8;T zMyPaP|LCZqii$G(9NCEJ2?=d* za1bq-JCNR{En661O`E{e8xgQD7rorUiD0iF{)HcH`-jI;-araeP0|ISdXUtB{{F=j zpKiVw!BGeNg`xPHYz=}5YZSbCY@Lh z(L+1uicG%8$b-uT#R&Kh6`%~4adUH1(uFQFzrPgO@Fl!gON*r<|3wg04e}6~5y+=jh`ci255pk1%LZHq>Ko@T7kWsqt z<>|@lm&gTc$osbg?_djqq)FIOvgC;27}(6Cf^TrTH~^pW{Z%nww$<}&w&RL@pq8tm@d7s^4Yv<$HXhLKy5|`pfI$QJ(V8VC zvEc*+1PGl?40ro3ca$Nw3}bcU;GH8^R`d4lt$X)w;Cg#N!1eU1zPxG<&F0b2?qt_~ ztEn7X;y1~-+$l*YD$=O{x59P8|L!{rbaZstP+ATR4KB?!U+HB3AH&Iu{qu&C=#^jy zVo}lE!NA)YzfG0`PA>`Be%cdIBIWsev0mv5Qm`PFU{iVm#F+*hc@WM5k#X;(PceBC(h3^(tWmQfTjoq$nQ75=5&Pz^q^8+_zx#ytQ#hmUhDx zp)h3S*#A6R*!2E(nQkn}yy9B%qo*g%^6nk?oDoNmw zwCUyUEeDWa2maVyxK;_hM;q`=ssSix@& zfyWaBCL?leqn*p!caF1w4YGhDA1aK`o#O#jET6dCSmBf{n#5ttmq-~we{Qvgv#=5^ z_U*t)5TyW!rmYfUVvOL^S>AU?M;!t?#v3B2quJ|Nj__t&vZU^NrhovVxiNu9*Wdl- zNPlKV)!WW}D_5?3(n`yWSWVpP4cxsHl9_2Mma4n;a^ae|xHv1P0C1vwo40IP0vM2d z3Z#i*a3Xlf5@ZR_(NUXFNvo-Ag6SKY;5pDA{FG@b^#GWi38nAtXy$A4DWSf9=9i=N zywMgkn@DpZ3x2r=CQWdzqK2bjjI(qUW7Hxy!KN!(HtZuk~L|^o;=wZ z5gDlrTrObv`cu&}2MY5-KhTleoXL4$!> z2(QYKA$Ju){t)BW{pO7?qVJwwIQ!(Y%1LG?cTA?BRa~@;J;v54y5rdWb+)^=7_oA52ZKKD1>%OL!>5+UshNFgiJ94e z;?xidTz-B&45WEZxgI$&2y{jY#6l4>XB~+dD93Ji zBLK&8pa?XKgdT^No*FD%a!-WgQ&E zAt+w~?TqI~5$t4c;TMFtkA-hnxgY^FU-IX$7;;?o6jXKG;wQK|KpQiGf$WRpsHt_yIy5e zsrM515Q~pM2QLDudMQAaj?dwQEEEth6x{MboHED^4|yfEPT-Z((rkzapAM(|H-HK^_w__gz?|6NfF_By!T)4yYZW0fu{VgpYiD} z5E%dUqyPVH^N#QTT9N-RteaOs8Xch~CpFP7a6#KA(2fXQ16TgtspJmS7&+v6Vqb>1 z@m-uVQb9?n62d(##V%^f3WJ_4C)&MSO9v%KN2b(;p;a!)_X!#4H^4Lu_% zKYHG*v)~|Ml;N$pLARKwDR?owt*l%O9}9mgOmb+2_07zdVxvRI%kA>0K8()Ma^jbM z0JqczZ~`A~#l%B_UkwcnsUnc1r$AY9$>UQJ))^Cub<*#k!zd$+&Gip0fK$Wg865oe zC9{h^lT9_H(DD@+18!%SS?9M?8WLQs@%~^OMo$Ox=hbSxs`F6v!?mR`Pvgm_kLG^z4xN_ODGBh&q><0nX z%L;~-B0-^Fxe*j32L?p{!i7aJFRMOOT>{-$<(nbW1}@4C4Grvi@hUC=v83{1)Zq{? ze@GQND`L=rIjjeug%HwX-#-x758Q#9D}`a9)p-mvnLCmzsHw$J!sL;n>+)>PEFq-H!}`QZ^2J(F4kqblSHZ^(M1W1&K#Fu`VCtlRjh zxj7bED!YggBP3&~YM&V^h06GCIexbbmIDPd z!-}%9bFXjmiUHUF*`5ii1i%6~+=eNJ6qTeCHGT=iPE6*00PxOyk4G=9{=y#9 zPYj$f7z2A56QTvi*z< z?CW&$04Y$^*2alx2{GI6tdJl5(JvRjetmREXO%|zG77x8yw3_Qh-LJ>M06V&;S*-uckqM34YZxngA{i#gtkg z0fLMfZH$(dR$?}~*NYz>u0dZUn(oYj-%VLV@K9` z(8fo=oYPQvXlqup0L=Lyjl%OsP1r3ihx>{UF(5&hq4o=TFIyK^ujk}+00{LuGIksw z7Jna}7P+7NSFb{Ev#`=JB5M(cdKX#>^xgh_M2JNmfDF|IhpvT?#?#FJ!v6@2MCUEr zwi%)=S1)uqh}P^nR4!R1C81W~V0K|ztZIsS=0l!b@VRv(moqXly1}eSwmXa^51*SiBrPZ2ehd;m3IUbILUuIbJ=5q73nmX$ zQk^iY@$1hYA65QxERDylP4a$*OPOeBXcU&v-fzPwAQI6bc-HUlyBvf~5f5XwL2gfy z7wCq3d>o#^n(D^D-A;enukrW6>VL{lXkTSirYgs%<$Y)2LbkQ-mfw(7} zv49@#8DD&-&+rho(nUrjfYA@)4x=d)Pc=9C&&(4<)SzU6pZAQnqA=08!4muUxidc2 zAR4WD(%hQLcukm%M8ef2k;ma82Glefu!PRBR`J9M zP||3Bo^p*KZV}>-Eu-VNeF#pTZDiV_u3Gz2==KVxnA*6^59WVum)`m>mr>B*^j1%vJN8 z=(i7#!X~s4w>!0lQDWEZc{dLPC^rm*m_}@u5k6jmdU0i50W@AzYlL=7^0giLCn))G1k13}26i;L!xbZT;J4#`1Q zL-$^e$ic*@M@R^(rL*FuO`Blj>jDFWvUk?xfkq8D2RaHfBp2JRp(O9RI;ak*(d>Z5 zzv}9~IaGlZ*F|m7LQfMAPZFb-b;WVU?)}HLf^E!%I zhFiB*xdlhZo<4n=h64HX#&Qt!Y{iRC7k{|ei0Oj(NhL;je_2MH13L#}^;%<_(c$6i zJd78cpC(RdNcOT26D0O+)F(-ypo8(4r8JFLA}jo^_Qhqrd-v|8JlPGOC>fzLtp?s; zTEEb1QDkHVNJlQoHcHZ&-^DlmDE%qvOAE#To0W=j$sj{v_k;}>VUkVJbAp7W$K!m)HDltOT6a| zMG0BJN;l^A?c3L+E@Y@tv6t!l59Ze?hkaQ{LklK=fvmzKHKhoKSdi#Fh2S#qtmem- z1_u0zQ9L}H7^@X#hT3vhW88&x3N-kBcySRVi&9mYDD_tkqH*!@a1ni|2+^qVGprdD zu1kto!1pN`S8Qy`fM#LWSw@mLvAhB8mH+-(Ftq|a8XW}y!T|b%PbY5=@Hu8B2{;2y zOu5w?4x6oY=IRK_`JY7n!lrlM%iPqukaFK6#ejXy!{H$tfyOjhz!W8w}s` zJ+t(NKt#0!zxqGn*TyGnc513l7aB0jLn>Fep#(gfPn2UQAJh=0{M&*+7W1 z%zPj)Onw!Vh3~t(VP304y>t=)WM)bn)Ncfzi$ z;a4pJfrDDS99V&V@#0cobzrE)zbrWo00PL<0X8nT^j5mVVr4zpA_9Pot|6&+8K)u7 z#<=&slMKXkaibA)-L!BY0}o}0zEMwl%eC}7-|dxPyO4!&w-$iVD%u;1;Uk2V7``t$ zaI;CnE4+bQ;M`=Th*pucWC;uz@@Of7dgy1b1z>iHV;~_hu^YgBf5+ng?19q)-Ha4+ zNzWV+?7MndmVGY>SsXLGJHuc*A(gwZWc!1LkYY$=0q_aKd;q9(K~qoo9m@0*k>j+~ z^NV$~P#wwS1;}gC;!zKKm*wuiYHqUo-A#>k*lhZx^XB(@O=xh$vqlvmYwY=m=7*v% z`xbBX=Jx&;H54nDX;y&=LsgX7dWU!?4OHlV6<)uNt=R*Fq0q`Ik~WlLQdI^I3*OPk zce2aXgB`;^^Ue}+6f?4>L4QcMiI*xMku;eKV)WQt!YXnRy);2%K?9Oq0(|z zE}`&fmWTM=8_EkPZ{M-V3aA7djj~xa;uxqeFlI`Jh)F4;q?i0f3^NqX4+!0G7q*B}xgA%Lm9L^FIU& z;J6q}FL8mPEd7GN1^wYE0MF0_XD`zK;68F3#`wuP7S7GCM>FDy>mxmJQIRxhTtF(i zM@H)0tsJfvDwO6AHNZ~6xG5)6Cxq-6)k+AQ&<)7|By3`wTwMB~MOBlL`zK&bBw5AS zu*To8Ne)*hj06C!pBqEZZqd=)7>lC^GeUZO=rUw-2;j&ne z;eT}-&36%EE9Cp0OP`p}SB2C~szbx3cXoCrD+2-Ju!BQh)v_Sm*3xqXPUKz~u5DOqudwKJEDh7Sar>E!r#mqL5w6<+k<6D2S+4Q_1NE6A3 za>LaWS5@(A4Z)v%1@bLnZYbjSK_6a<)NMY4M@|MoFjj(T9n-rr+Yr~7TBO{dfu4gt z$ZrcIz7*X!?8mW(hz};S5at0w6p%A40Fa5*IVUHF#0g3Zp~Laj%7BcVl@TBrh>(DN zc2gmd(AZ>(9eD5mWb$%bit+HtE6%;Ex0epReb#v#@&E|78RvL-Aes-s(={j$N~yJV zJt!rtiBDFRDCuD!aBSq~XF?CRh*FG}iF1ejp=TJGIurCISSZno3-| zD2DO+rQWcH#<$F}M!y212Rc1*&-xPNFw*2u(tz`4?z++Q1g#|J4xkWJjTkn4VHa_L zq<%8$fbWePjF!y`YHDh)jYQ@h&BV2jNzuL41t^&|UJEYle`$~WD@17YX&RYz{qI+Z z^FkN9aPtc*;8Z-n1^(+t8>`sqk-U_z`^o6N*GKY?$O}PQO#Q^p-{~)8#1j_(D zGvRUWF>Pc-KPeBB-fuC@h0Bk7AAJ~IRFjO;PK7YRKwv3i6l{w9m(*w|s9VKm)#!i7a$)H6ey`a6tI`XB;B~^Ha)sFG z|D2rUTnAj0iBg53BMR){J7sP%I|kfN)(K=CUnpiL&!JVO+>lPb?A89a(n3uCqmF#_ z55=BCeu@L~zbEPJ5HyvGazp_tE<&L=yPdR)R2R1Y{~)irup0}mYKcxt6^h0QB-nyq#7_RZkP`2O`Iz}n z`Ud4`0(eFkxT7&$PfE?XoE!##Vb#XFJF>x8mz9-i5BV>w=-1;V;VewTGR!=T?x3b8 zzvCzyfuE-HpBqs&8Y~fiZaBbbfx)J>s zY)l|0KHBmf)_a8z1ko7}<0_J0-U^@jJAG;bfe)eRjvz#EhgPxfqapSU6ft7{fP!() z#1(auj)>H$JJl{-v$3f${~CV#jS=ScJruWAlkffbzrGjaNBB~2RemqzMO-~jq)8sg zxKSXET|x9u(2kfyA=9Lg59eRaArQjR_!HxeAs3nTApYvRGomO^#5^=TA&kA?g=)3(I;|T z4&M6lpB0u^Fv5{fDB#C6d20*GL(j#L2k)bU$t+88v5a{u4$ta>rvyhRh{C780)o5h zqqEB~V*i&@bRl!r_51!$jT${;(vhEA@f4h5JnY6A<0MSdc$F5GCV?SswRFg0hydjL zp;VPEIzDk(ASqzBLk}kRKiS-Q7|j6auw59qCvO3Y2Km~BTvCUvKwhKk3{_~I&!XqY z+d^CtRJIH3+*t?v*I-N5@g`oDg)6%i1hc&M`8K2&_}nRAB|>r`7o`vELfxIeaYcog z_ffcW8=F030u9q<=WvQn-in342g!2_a2dWZVu&WV5KVOQz|HwzzEQwIe!Cv5AsPvK z8OQ)k8>CzUmaN`NQAn7C2<4JLx^VA@^9=O$d4tx&O8`3?ShW}cT=*f#RCC&)&GV^g z(iCt}(h-}N_^rq%!0t**L9Bn&n==2Ee1!lZNCSri!^|FUy7LCsHi22@x~8{{jjYg~ zig0x_gjI)w?SuKLg{3;Pik|oV04xoga6nQr?;7^4TWZ--!Eq=QN@11&+M&C@|5B_U zu}70}LW}|s%aiE%2!@wQwMGK*@bz7SQBGgb?fPITa0EgF|6S6tV?qax8#48ua}XL7 zLM=rteFKgwVgfKHt}`ee<=6seCM|7ky+AX9FdqNWP83hL;Nrj;e{IWsR&h804Q?!a zaAe~m=>YLLTQYl-oBnFu2>K1i{9fQ}jsbEM7LHG2Cl-lImmXWS&*3VGu+ zCW;0UQY?JuWWo+Cdsff@HW7x5R}BsgS;E9&IR)5Ej@u%Ob;OUA^kZ=Rc!I%rxa`zc zJe`fVVqj3hA`x(aGpcC8hvFatGS&<{sr+&d1K=^o?rbGWBq{@@$Q&`-cxGT?KI)FX z!Z9TR8c*p#ytv_ONxuPWoMP)^^-Nlr;CAiV;|Ze?dd!sU4p8~zFdOXG&xbzSAJ=Q5 zVeUH%iwbGL5MkY*U`a4(YioymCxaR2U{1HJg*lV_CB#mm3IqlQs{92TK$N>g`s<)e z|4kI4xE#VS><%1)199TV%b(<+Cmiv#0V$3`o@RV*(aup^T&yt0jY(3H+vaBea07QJ zLx%~KB!?7`kv)@EfF?3;&{p8Qp2z|(rlzLi!E=xX_(gk;C0(cUA1-w0cTEQ8W7ew< zx;r^a06smFS7?L`$$-?dWrtxbQLTcp$_u2}IrIl)14%lIqp}cA=Z2mpOI_F|061LJ zy@QT>DbjP)HcWNC5=WWsF&+;fbqElR*!6I9MnY9?KZ->q{wtJ)9=#(ObtS|3;2RKQ z_bPHgYk(&py<#!ibkrPg6h*2C3O-bb(;Q)hc*X!JBrePGK@F09`XaVKJ;d=FoKh~@ zZQ~2-HxD#F1eii)0mu*>G;nPCAWAXiAcWug+(HsumzMP$XA|dyc@5$AOJg z39v<`ym3~WbYcPCP6Q!9s(55YBDfX&{j&a_63#M!BOnaBOT3V%j>&8|8DyD6KPm75 zj578C{)a}0!~!|n4P+IV?gj2M9&S*gDNT;KW$z}&ZZy>3@RCdK@ntrlt5&!GYTX*62TezMs32e)e}=w zn~-1dVA=OuZN{j@v#E^LB+Y+QJ$#rRvcCr!@y~hbAW37P=ksYk=R=0r0ap?Esn_2dleO!jrQ3h~#3sZ!|1A#7;SlYk_ zcmO%?$byq&84?HsMF0hHUxUo%0ggpv$l)wQkrP%5ii(GUC590)vCy)>*Q{F2LTpp$ zH+Z3s!wN2swwz`4YC?KqUqh&0g??Yz(dPp+XZWTl@GaN|Jc5EN@cey{U1(`($;f8r z5469+<}G`uBJi!NI5`7QO0Y1b1&uJdMUo|77~zU=l`$B!B0dIS9a4i6*VT*Uxz0(F zLohC5b`-28&WhoM+dal=WDIU-Laid=@_1HkYC;p?EDUvpob%TPme?d4TqdS56lT!e zc4JBA6^(W}Dgy^}m6eyrf=wXj)a0#>LsARBz(o$p`uzIVr##yoWNSi~BBux#{6&Vm zf)O^1QtO`oMVbh58VZ^}AIQJBhvf>_gs387I|fs9qwr-;4q>GJNyz0xY=YWFaEqTa zGxNs?STRZ-bqcdnmp*0CKxi*VrW4&T&;Kgx@3#7Ul(bABc#^?;;ta4=pzsmsaY#R= z#yd8}3CW|;g5|~?>P0E65^cIjS(uPiPPMqT@d%l;$LwG%7!NU*>74o+8e&+Wx?nR< znT)W2&LANnK?oc};%Fhqv65K8GU#J0Mq$Y^$UKBN0I?Kep@~jO4qN>cL}(1k>m#8m zTsyuU-dDO#r;)%3M-bG+8;G68PR@3PC(&)TQV8I0SMuR2 zl~<6hjwK_s8OCD?R01$NVK{@VJPqE=ZuILag8!hM5f;+Go*eUo146K+iM#=m`1L1G z%w;&4Zk<(8Q6WRKXaYAPK4FPf)=EcQ;%H4?GIVX3gb7-nY1BpouJWv@o4ki`YGNI& zuha_?CwGZrLmDkw1k9=p8&*J~g|UJOROhLx$;FZ@#A_bxTZt?1LL8${F4G{>IOL7F z+JYYop~h3^(1U86T>3y6^dG6-z*A%vlo%*gWN9g+7~1*s9l>xx!#%_w3N3sk0vKXw z5y%>HdIBWw)WLY{d;V6Seg;Zm>~dQU4fVAlIr!eqn{}WDDRtt)*Eq3z&wFL z#v~7=@COfeY}w3bwD;;Ro3E!rC*kVUE+DSE-{qK0tb^f#a7HAh0$LKZ0)8!P41N)q z2HXi?!W~0;mPd3E<2_XbqX0R|h8%i?zh-1&k|kD5go~T}k&yj}*$@E@7TiQBeDV4i zr_0vGG&rel@2MLjzL1k47@1rIT~0pADIYsq<96A;@?I3BX?q;b<* z@XLQr>4dR-*l!0M;xzJh z-%$kxoX2Nn^>l(9q)Yr;knV^2M{5o^O)#SeCLRm0f@#@727^4k%UQo)l(3+#%y*|u zD={Gi2!Eq#8zv!(k<~I>=UkxnJZdZi6;1|4aA44l<7Pgdp68H&5r!*CU}G=(`9|Sa zWFW~9b`UZ}Qc;jU-)=$zz={PhT8oK2;Lp^^u^JoKot4c8*d*Q8Xh8dQtoro~NCPb#Iv@ho7Nm7EoR|+hlE6@6+ec zRqxyLIhp0c2!Ln7Y~?Xo zShiG?%;?=P1xQWYtG)T*(jwvvC zeHUt_r1ajYbsyOm%DScyzVY}&Ne4{M#YE~{;0uTS<+^%#&LF(+r64TgMtKWSBgwb` z&PgKsl&}pTxH0SEQq0Y>Y7O5BLxK_!E!Uqt+l_{)ypQuk`xVUZ4xR&^YYx%}fBtK9 z^st?s7`RDqsD=EYNOvF8bRG;@`VO87;F`vfd;I5dM98O?*DePbX%pUGQkt&J{Itw`1;8aCg^evu&Sm5O5q|55SHa}_|R`*hmAv|Bw}q3;X6S& zWn!pZc$k`)_*;fUYZT93wY0pBqr-5VTQEY191eq==Y?LD>Vi2?p?4E%5Rn;hpcT^& zG`+@37-0t={OP&LYACLYsh)aF2RHv8RJ{jW&i(twec3Wo_EsSo$qJEGB+1@XN@Y{Y zs$?r!5sEU3jI2af3oRwdRum26u24w3=Y7Wi_dHLp*Z=pr|MyMTb$zeT=N!jzoX5!@ zPaaVfO_UWCBN=x2yW7nqOr?D$Z%n}r7F??a0+=XTBvAtJ;^H<&@_oqN7!W(12?~;w zKNXi8Y4LSHaI8=d9{&?QB7;`iJRBn&SiRwrYHmse-nO zsQ2#|_%7^1&CN+0yz!T&c#`p)_v$o8V8=9G6j5DXqqw-(%^G(inL?E&NQ`o5Dc&@%BNb??C<>Y+@wx>Iu38kWfvpZ z7`^)=TF{!w$T-hP4630HWmw08=%D2&GjznbOq`f^Y}9fp-MFWNUQ22z@xdj}Z$Z0Z zQBnGvseNt5NQwsg$Fyc@vEDt15!=;@GbsaP2VSDXmb57OP>dT2~ z#WduVP!nc^xst8EWkhPnFS=fHU*1o0sJ0^hR~%3oc&cjJu=PJi_SMmuGVc~OFHYIp z6eGzqQ&V3v0K9{!w)9;x>{ZNNq8|b^zCmLb5OF1W2}ONc5tUo$%<9tu`E(pdS?E=2 zZ)ev`Q6Ic>`iA*3gsV`d47_;beRA?7z=IiSRSf9YR*Xi^6jCwFxpDCcL6ZEh8rqzk zap8kqUgvdG1V?fxg=8+2L6p01z5!;uoV||wUsFL(YiR8%^Tiq}?y;e?EvmT~fltM& zxRd_;3r$HF_usvnFO$CF0&q4TzCZcbV34VU+MRY&Rff1L*ddsi8E&G;^cM7xX#WkZM#=j>8e^ zxCwfu1=o@cCc;!vLrKE~T*_c%wb7sSJJiJ@H=@F7J?@oGt>Ee84G?JLkDv|`%?DUf9h|4)9jj>DrmsiXrjAaJoIdB32#`4kT!c%( zPxvGu;bR98xEEqyv5QIKfAah&uDVxjt-4J+?%~7-KmaoKK9W`f&F(&vA5K$X};WmIuhoq;_vh71`} zLm?E{R2*GnZjR89TE1t3Hv+qUoSk5?EAU*)ABosc_CUY zhVAR1?4r+I@h=zPfflzRlMUo5_t+kHzJJKeFGza0*O@$2aLoR6_0<+#k()Zu@)(l0z&wH}&Bdw^I81j_@yB`v-6rqQ zcPSN>)m^>GrT;aL)!7@?HaQ-4a9qo31W?}rVsQ+h3szwVmV}dxq$a?7p^2nRLd!z_ zuAtGP^&r5Cwz=obOUmEF=&%e(A3z@sG8vlce(2HUK;PbGi6bI@cp#cJC=Q#67=k>apZ$MXEb@9UdAIqj?IA6oS_3 zFySY3kN2<*{aC57g`!Fx>-yp!U5FR>$z0nyxRDx@o`YRF6tv75e(BUKw}`T%iJJ$i zXE0WhTRvu(6jCle;Cw)wG8Is;j+-Oh?we~1w;5M}PIyeHRE)%amAM}sLfz#A_car> z4htpPHc_V2u7~rH+8%1^y?Nwa&T~PLqsG#vS`Ef`@fkVcK&??}>J4Mo*Zs$->RXX1 zA3&(QOds;|0x*~mA4}$;zfx}BzL5gC^*CB4fi)Pa6+H<_QQDew>VY-ScCT}$R0STo z#;oDhVZL0V2$+pqB;XaItGd|w&Apj%-MXPb^(bg_&iiUcIzM{;R`r5co2!e0VGM6jX3|B_C4(3I zr=P<%5gz!N`iTG`f+NU>+gufC9cJ7EjkADcZ5z2b>7uXZh}A=#)TaL1*OOwa3A6=- z6GN-etaXUL{3Pm|}*4Z1qag!$ms3vl@-RSFUD8xDs6eE*n<^O~X zS_(>lZG;(-G9llmm9Tvq3qUJuH^7$tbW<<$!G6P=^=N)@+TZ;{*5kM$0y_y}O{X{zkT?kG_E0EC^)WPLyxPBT5_f3NgE5|h_8Fc|vj!-vh_sEd2e zx^*L6M9}&DgZ@6Dzp5Jl(Wc!F;X$1{v{4JdsA^LFpJarm-azNy9BvCsa*%bx+f2vk zd2pHjH!in$Fk0Q)yPCN+@M^q6r@(joXDXzV2rJFQLE3s@0V?hhq<@F^8Kh@MxZNO7 zK95@SmUu4JR-^WVmFZSrjBms{^;!qQNraDC-?ZnK7XAfY?lhX+zpyv|vfdTgGml#= zr-A*Kl@@ot_2k%6Hb)by9gFT%&@PBBrn0hZ6$=1zHR@739%lc+iZ<=STsmCHo$NLH zaGxD+*JhPFQ<#=ydKepf4_Km7S@OA)B8p(1!Yx_;$Br>3pfeTzAACuWN!AtD3H*Hq zU457o`ryEDpX19P$z5X)z?6AM3bqwvzrULlA)6HD{&r$hihO9SYL>o!kw8wHHknl$ z&3n&nF0K;w|04GR$?EN-@r1mfq<9U5>eUe z_n_`{PD{UzuW=#5;Pt{qoxjL*@YF1aPI|2eukOqoNa}U)0XOqgO?jT=>lj|)#((R` zz2brOCRYgBdchZ;f1{8tWWtDs@u6#Y|5&I1CM;UfWdG0u%lahVs#ZDTBp18Yd zW;p9)%q%S{vqA%_(65}5{HWd!QeYOZH#7AUQ!F5r^Qka^2$K)ZcEAwee#E8T56r)= zO*PYN*E4LrPe_lA2#3W}k4z z5mp1{liJqUQOr5vfUUT1#%a9J)zD-J!_&eVrxD;;q>x>6ys=_4Y|ug+cLeFDM4H*T z>@Sn8_P$glwuw9xu%Yu42T8AT@#ujAr)Esu+PS{>2v>dJQUv9DLH+LoYUX-Vp6?W6 z1+uq70Z@53cpDJc!}RoyRL!$eudEy!OU#ks7;22uxAVE+Z$fOF;&N=>dAyQBnp#$c zG-O!EKxSPRoirHI0#8oL(g(P)@FXF8=Nwr9lkx-0BZ;Sco*7yYFKvj}HK^@S9l~b! zi5%VinUQ}zqhAdsWDcqRK3rXzjaR}ky|s>IZJ>&}I4!Uu%=Vrf#D80dy*EUB>Km z>)-51k7;AOMu_$UbTH`l#EBDs{wlV=NFd>Ihsx z5%G|IazzCofqWIrRqVu`RGx#3$Tk8zz}h{ag51x6s->Vy)qIlb z3A{`(tj_lV=QI)oS2oCiT28tUDc*c!E@37Gf`Bv`tz7$NnTfHn%AX+|LWI}`E2Kn^ z;Vb+>m|fnfla!bgnGu86o_hT7VJ*-awCv@IS+ia*?>%cZkO`=)jZ>aDNwPAIGgU4; zG$t<=U=p`s?Et^z9Y}gtC^v7e@uc|L+Wfh8c^{s)X{QaCB)5?9Svhtb@B!cY;}F5M ziKDIwUh{P-tgcztv17)J5oiX2^YN?TW-V?|N6Fj|Te)Ocf=qWl_S2hXRYCj9{DIK6 zGJ8cg740)4$9{<5@8ooRVrxlslS@8!9i>Txyp>bdk|u?+RbX-U9B{w=0c?1Z^8phXr}gXJ2vnUs~57A11) z0gIJ|%!V*H>tcpf9Q4r;xsQzZ6Z-<-Wx;Nyxd&69C^|Q>m(L>~X$5vRtC_^N3$ul* zlf^*@psvhtqT{4CrxMnlS0?SfL^&B8heZ*N>(Zr5%5B<+oiwU~abb_5FopB=3E`q- zzS&|qa~#gAmqeOP(g0@@!78_=4ou6;x94c!xe9u0lxYNj7zOLJg`{%+UAe-!y`JdM z@sjkuoTaX1#H{9agTlz|-DS{LCuC1%W~MBx5M&)CJOHe=xT?ZbGc4Ss7aFM}W;d^2 zpZecPDG3YHZNr_nb*?W=I&YU5*%Mitt$0fi5-eB4;vVx!C^!4w5_^1jio|=RM6^!c>|sZ zBi{>_%##}P#nD~6>InjqK2f0{-j0DylxB3?xHyxvafXI9ii_i7M<6g-h)7AC`r{0%%WH!( zJqHaMq-9laNQX&p<;bwnuv;B&v+V8Jg*`O!rCR+{cK-bN_WRo@I<4w9Ij>#(+c)z1v9=#tuq&nm22wo}ceLEbLCwlWd?w_d9r3q)Qa8aJD z`w{Y=vi@+X6O&6!sM)b450Tf9=lXz`&162Pb>Y&yQ@o}?xX+5{dSZefxGH{m<%k>ym$sapVftP(a6KR)wga>pyJKA`1(Q z`9to(ywAv-wQd-k6$qcT6Zi}Q1cx=t{~j-cllQof{)BV^5RHA1cvI$hTTd0}v;f^( z`?v$Mtm+;6)?kvcal*?tqeuwVN98ir3-9A>n{h-&aSpsQzX^;aqA>9VL-!^MD!~p2 zZ^G9oDsOFHhFKt4qsAy@FSDN-=Fl~k5FT; zBVXF#{|>I)uzvk1>XD`2eOYJrkSbjgkIuHaew2N+6!^#;`mUtT1kchD<?eWL%is^~j zilQ_ibKjJHL;K-4>AF}dxgMCS5xR)v%#%oK&` z`8Me7pkXk(9)1 zB4W+%niOjl`7!ohWLlTvQKTb^0Zn=-M)Cz-!z4l!A}qvPnu{vOit)8%|y<5#2!G(`(3jIK@ciObjfgpURn`^btkFmp=9jvLy# zj)m4}J$&N;X(^B!iV{tfy|My{tu3F0&f>$DD3mnIM+FZE2$D-l#4Ggc92E z*RM+5HqqRRzOapW%L#a9oG)yEJbxazY%!H+O6(CjiN{iJvx)YmpDfJin791WWS0|5 zW1UA`4+);S(Imu*h=9H<<)T^aoLc=38lM%pql~bLYki&-<*HcO*6#B2{sBSS=zc}) z8tbS5=5nQ9ybM1;9bLE_2?+;3X1pX#Y!aYn)hJ9L@QvMTVi0?u)|L3;Chnv>G`_7V zb9-usxR^2jS3YkESS0dS+H`GYt2DFvwQgJukOoY0}V6hNt8?FCsj z`{vRXLNz00;07pL-AWJ^9? zeI=z)R6@|QeU6YK>cFkb5Gs8V!=s$Lw&?vw2Dw~pH|11AS8sTX`aqk{p@ikQZVrUXA7nt=>(ArJA+?j&09 zsJMYu`e=L$etaCG7)dkta?^U2f3+R4x>fVx!^55Zri^R_4^bO*fJMv`U!TtQq4>wh zF_YHmZj+plvq??|-5O|*QkIQ%Li-c7G|eNNK__a}igDcd@^a$w8AGOQ8jpVb^yx0$ zHqIS&@oHrs(*+04Hk>^$uQzJO!x73|5*kOOJF15yU!4A9@Y>163#NlFyYfUDTerOW#b4kjWwx)FQkj2#|2b4O2yVM0wv1KJ*!5Ayn!lP&$~?wcg*-Y)vdl$j6+XC$do?LcH@=0oBFb{bC#1QVurmCqkvy*P2w>xIhAzMA0dL*<}pvr^xws1c2FwWA17NYD!+haP$g z!IMxqiBdu?4R6VV^I|>6R4Nhk2yy?Vti0PqJ(+Gps?io$7u~w5KE-MB6r%ZHACK-+W>}cBqDo?3n~{__+^^u9X!=}lbCq` zjFnM$89gBMiZ~_He$t%7)iP}fp%4WYXusT9AdCwyjsp#$uGkOq z!vHYA=4pC8Z@qPW;Mxvb5Bd8$s{eOMAa#Pgr^2aN=R9Kv>L;0W$MvJST71KY1olw~ z-y=dk~AH|EJ0_5evLuh9xZfkunbck1s&nf`AOs4KR!j4=g{)BXud@2y=V8FzyOH}{1ERMmkKWulhFQVp=We&qOwaTheY*ML zkCW!jpL=QYjmhuAZ-?H#SEqiB(W^ZUn0(rB|Ii&hjVVJ@e~Mu(hKx zcpd6u#Jt5E1R0~xRr5=Qqf(5O#=G6O5bA1_1JV%gNHo=x3tyys-m3p5B&EvUa zSW`+RBu&T2X#LPwZ#NxhkU7e-xGHv5cv96Y*+vp}Ir_)6{7V+>kY#;LX& z+Ogm!p2FqPiQ#(eJ8++2IyS(QY(JG*C*)C*=SALcXax$s3m+Zi+kVJjn>=78${(FS30(9R_d?f zjN2X0J=#GA^#l$zEVB;x@bI{x*(F#;hkok@gBOiDuvGGVq*o|-(DXi72&m>sT6@xjkVSEG< z*72Hk8q9W%&@K5d=qfIlVu?}EvVTP5j>EhuKgHtDK(B_H+7TUj>n7u(E`Mp5uCja9 z&`ISLD=+kwZYup=(}mxp#5_a%P1Vlj72QKZ9GuUz{tnz8_uAZ*`(WDxbex)m9#|Mok0x zIQje7O;?&W68e6cpSb^gb(9)4I>W#O|KBd~`RECci;rl{&;s)`B@iuP#S~qclHWlI ziAD#{`EM5N70p~yzgK~-0qbDZ3Rnp%GoUx4vTS?Uuwk&f*s%@5C{<-zWgnuN`7A<}kc9rn`4 zJqP6*923d760r6e`XeD2+7I98g|2_zuu8t3vpj#?KI(2J^c!w)nRdo#6q?5Xm-mOB zIW;q+%@m>u zYPEo0FerUcMs*)H%ohy$V@^&Bt9R6$(%~@ohtzG~^$vojiJuu*`o}|)^mT5|n=>bc zkPXN$_#vR?X{v+Zh%4ZaqN-u|HW?YCkB-&O7(=DH+qNJ`I7AML+u(1@!(9}AU<-oI zc`vE}lYH6z3LMa&RUZ>FF|r z#@{-^F*<~(aB&us)uR65ic1T@F|ZK!bI1@G;F~rAIR!C9mb?@*b!oSHwcGE3Zbq4w zD{2i2ovGv(TV-qVrM5gW{nR_#z<^||1N)#3zyjS}E6Y<>hSM5)5X3=GfimlHMp|c? z-Cg5OA$efnvYFRylUJ_orK#ygJ#|QZToXV)i?yqAv$LaV38a*u*gQw4B7OR+*8?%^ zqBs`~0U9M6)Disa82?~O%g>O+cIlsF>m}X6Hj?Hcz*pTO-L`v0!9poMHPB={dO^$Z zXE+LJ!aoeCIMHiPKmC$yRr^nmm%Kti=9KsT-ce9Bj65HatnQ#?p@fM^OVf+Z9RgHK z02i;MfRuRpxcK9`+Nk*Z{{{}B4_BDpQ`uOUt+sB9GTJO=EFzku9&Ne=Vya_VSqodlL*aN0B(STlJ zFVdUmESC@2y1nYZ_xE%Ok2bL7c3Q^5h7nHA@j#ILa@1Q0MbCfx85>N@$>IRF{JnQP z?{x`|4G3R1u88EM%JGK=0vwcG;&L$H0WrJFnC<6)@CK!0E$!~BxIuNIP(Xlu9lT&p zE|N)5ctzo-X=R1|U;6q=5=CQDx5KP1H%1w~0baFEgXVy)C&A!!sIvRnK6%VNerzxL zLE2(#%4eI;a^oor&n&AfFQKllCDL>lys?~Ts_`95av#vLK0^gbv=u`Ts<+R)(3;#= zdjW)w#bEew3N%IdOt2Fq@FNkaBsHi>9vqEmG_t&Kf|sR}0?KydpKlWhCnW@DV%xY& zS9)-5VX(w=b1G4R$5fJkD?_x!=xhhVg4zyrrF)LVO(^mGRccz$YBEqL`OwEm)! zM5fB??*!%g!RsE2AwTpH-thVJLy|-=8)KBHI`t3VxW=oJv}+BZj*#Zdmb%ps?baFp z+8eGiz2V=h>~m)31BTN|Q|+#rHfp_hEM5o3d?IL;MtIgr-a!(5g!3~kXlCze8H7&i zisduCR)ZMxCehyv!0@2G-r7B=84IDZbWrhr$k}vYq*q1rs-6s9EJ{4G*R`@F&Of*6 zDo2O<;vq4;hS>1%=@&8M(Q;x-0W+d9XEe!7u;@D(VGg`-G_N0juxrB~*KbQ6U6kPE zPU$hX{r8tO-55rT;2YAfDtDkGv@FuUY!-fu@x+~w#R`G8KxD+7$z+n2)Mgx?gI&hd z6}1Yf;0aA3e&v~%fd1y2>}_aT@ySyhfm?jgsD z@~&_0EabV*5<_`6$zxRT1J-fy` z<>ckHKb!hqh1KD$D8SX`Twj!(a9FGuo;Uu(v#fERO@aqen(K zm-y!`@$NCL8|354(&6@PqGzpoX5NArGRSA)1s=#`ilXOJcPn@wy2DR5D`{?EqSn2> z-L_d26*+R%55tXb|=;w&p1y$SI|UrsZK;yzi*Q%YPMg1{!u0um+btH{pW{(n@kuqe@@& z#McMV)+b_e1nmZZIUZ=fgx!O>tIRllGNayuL`2^_#WG^!q+`?q%qm%}owQW$ckr7h z9hlD92AFf4Lc%+|Be~v+lnC4_^_ObLj!zicc;q{zKaLkRI^qpZe**2$T~jlDdS8XX zoljJTs-UMhHPHR)I`#cHQC#G3HZchrI`&Bif;875hUe9LAJ5T*v?6zf{;UfVg4kS( zVFIHA8;XCNep{>U)rd)+LF!>c*MUu1rC#sg=6b)tP{W*|o?2qA6O!tQClk zWoEQ{;O-*CPXy1F&s$~VD<`>zn-fA>L&`CM5p;t3Lc|L;PQAzKHp9kkl97?y*FR*Z zTB~WNT01l(*K$$=BaYAjw?bFih5xfD(|C}WQ-iEr8SB42-coj;jrCdMHi?TXU$UBc znGqBit5IHN&L;eR=F1oTgft8EceYer;K zWvi5}(!#j8Y4ny;3Y2aIJzz7h5x#N9i9W}^=?2XRisg(D@|#e`Eh0E-OgOR!BhU03 zepYAr;)r2gQs1e4ZgHJmw?WJJwp$WlW7)!*@NNqR3bZi(_{T;io-3)Ss0eu66jh#g zsKFWs2M?V%SUG4ilb-!(B&o-LYySDFqtu{*nudn>vKkDTNzSJLv7OR>GjfFxL<&4Y ztI!n>=sl$hXvabHP%`t=bNhDsu=l6NBmeIPjyF^~_=Qltp9wSSu2n^6*RP4Gi53C) z@c^|`sQ9G{qd%n-s^lPN>-dWM7Vf;I z=;G`+#z!>g-|h?Oc9zKvo!I!pJzM>_zF0j}wOw$tl11pS)lEL{31TR<+QmJj=yi)t zVdMG5Jm?Qlj5b#v@=CYyv%XPB9sP(RxKB4ZSX5@wOCZgu@_brojQsWII7Ox{7egVD z41wp7PMeCiK<-RKV$g*&hS7rOclL}8cpEm3kcvY7*7|n4ut!-< z<2(5BzLJZ4tOMNYkgc`Xm)+S$p*3#Fe&0@R`#v?PE(0?6JC5$&xv<-kXJ=|WKKhWOmjg%k+Ml?>RfoeoTK;EVD8p)M=8@3_jLIyROG{`dY8yB~WN>MyB z=Py_gFwAGw1^#0{#2*`r@*WWimZbRCq0-H!tCc#5gnAtULDNZv?$)qu4gNNbxmDsw z9_wC*5=k&Go*ZP)rMID~RfmUdu$O;Dlxs71wT17MWEB#q71|gozO58SpbYB7>K!z) z;Ew^+J-kHhiz;3^#q1uotu#VHZw7L3h;zrmql5=#U{cnPWO|uXO31E4Q5{nY1yXiE zJMHGWdeZeTZm3F`H-UsSFt}QKU)T&inkYFx5@={MGQWK@rSH@lKg61dc3KQ?XTX;fK zC;|riIdPw1p`No2MGA3Ipzy%JQ%IphaGmL#YFRFIv{RLDw~1{7C!I9q4fKHOxbc}9 zvOD7{0tg9vuGn^8`K&phZ^4fr7K9BDxU?7FK7Wp*nlrn1dF$cBhhx%o(cYdEP>_8( z2YKf*YXn{~1dnB|l@;T7yk^vkDn49_nG<~+@OO96&2#AV0`It?KBGl%6;iV6&EmAs z-uxA*bbjTh9i60KvRxV+EQ+4rtbC0s;4Rv%8%-}%pE;vEphtK2zT3OMbvW8%TI{bg z<+);B4%q+vJ8>0$(StLz`p1V+9I`gJyMW(F-z1pWX1dR5+%qWkXqp#Bp3Z^tGT9x! zyebk&aN(ow=#5ta3;QoP!W!*;6pFGkLP!?K&c_r}<9+&d6+LB4OxFa*ySHx}y#3zH z>`MB9;ZAEwP(`Uq*Z7S>DN=13G5}+l_ZoR4s;ftY+=*rS5&t=6(L!{{8Nc^Ss(x8s zf69_*RB$m2cJcu`Gq?@s{x)wYZwGfdiyN;`P^~d2^_b)!nti^)xWq&c9v1S*Xxf)C zv}_YAa$g3GW8~96nrdjdSNDOeVvxZ@N-QOm4E`u~kbIorS{e5EPwa+RBVa>D)`|FA zTo*3<^7?8ghP1@%f66`#&AoWP$G`RH&53OQnAIuEZr;2!gDGAIygoNUJ8xU+Ke!uM zDbQweUU(t}6r)4g2yktQ{n8xD35OR2{@nwJ_QQTgI9;dA`-Px_1I2knm^0$fjz7|? zje_z`e$frTkFJKg=WvEe6zGycJSK(&JY+^~KXGju6a1Vcm3+sxjdm{nev-4XpEFL5 z*=IMe?q=yY$pKP>qWL(gZ62GHl(f9vPU^>$(_`vM;1Lhb=SLb->~SGgI(FO#%7vet zPS44}-mM$e#eDGsBl>SEHgY)vZqz8xsH*71jEW`bzl;@OzXr_ohyYqpR8*H%!=34y z8U{}JT#Zk--Ehj(39ZNZoQHbO>N99n`eXP}L^%h)x)lESA?Ca(PX>k$J;%;CSsg3Y z4<^VTKmnaga*jH40)OSc)=qdQvp*=Z4ShS=|IOeIGsjMta3bfjVR|FOTlLKNfM}W( zW_jvEW=$b*E zA4iSuzd)_Ps%7)$cqOzWT3azE7f8Q&?yygdiLTuYH9lh7Nar+Z|C|$V z-i(c0kXU9@U$GNdVbzL&f{0~W#w-KJ;%hudcP()$p`dtnk5LZwI_<;3Pb^T#5V4%9 zPOI$pjt%(wJ<*7Gltty9&5@*8}=tIuT7O-mmlTVLU z2)+uR)^Aa~A`0&14#yu%tN2t#9Ir~P(-;Yq0h4i?k6vB1rDh$0La;>1T^6_OTjxrH zO@R$V*Ty6aEhujn)p}ue@w;=K@*-QHZPk1p;q zf?O5rNU5YUpedUenwHwUn96Ur1g|F4xCyX8ol!e62l!B)I*u3v5(cSs@_;V&KC#iA zbwL`9>Tdh0)Mzs;Jg$L>05k)q{L3kV;9el*a%F7tsqqjQGb^)FmY9TC=k!iN3WQj< z^?sk&rb!(%`iQ#$a5p8lJ*h=ANG0}g%HnK7FJlrPK71I7je&GKq=CTEJ18#}_HAu+ zV%QM6M~TJUtf@R7+Lt?m&h)eH7xSyA=o5dPwukx}eWnIoPNx@T#s9tAlCoIR7a;FJ zU{IgbNkl-3>NdmHuO5A-hxD(7qoz(7twM$MbJY7r-5j~m=Rjta{OU@KrZZFW8@qLG z*m@m;;Az4r<*H0_q6nJEx1JPwb@l4iPeWyav6@;ZPcS8ev=j72oUtT|D4O&~PEY=O z|LonU%tBhx7$+0pLf%OXA6^(>9s}j3_we%{K7F#$G7Y|d-QTsUGO&`XtCP0UdI zJp#Qw=MAb{w=AjYCn`eQb zD2wXWtGBDzc%DI~ClL~2F%X1A$?xwgj2maDRQ*fk8ztTNyq(FDTZt8cx~aM8-Fx?@ zogQ6>rY!-;>MaaD)QK2D(UI;V?ayuAJ@8z})%kZugFRaL9d-8JGgf_UfX?aXZkX9L z=-D%9yD2M!_&Hdg$NKjtzTRbuvh2q`hnTFRL7o%2X*iWAKxE@y_#*Q~dt@TNJw!Y` z2m~s#HNp8^w`@cm7UtXI3XxSX={)dc83NXTBhr*S4USAex*op_8e5{!CW08NkG zDuglB759p69W~;meq3AK{{7|gCK(m&_9~ZmZXb|xp07uKg59k5*ZlYK+?+cDD99vq zjba+@tgbeTbnYX4%j$Z<=*-cKvUy#Pb$Z#Q`VSi|zo7X~A8MdKG&EGZ)&53Jn(RBe z9?e0W&OmnsD*tR+v&WqN_fpN8m3*LB-N9FD==3-_c|T~4Vlqx6Ap$yC(Px8DJ-Sn7Hdl9p5t3h24~e3gF+zLKan-PY9n_r!+oSzfYjq0L1S}6vweo`y}_(!2|Q?S{t24i;X zSH1p=bE+L|4kfJwugh`JSloosLc}-x!}#~0BCA@q3oEJ@DDGDVdN_ z?*kG#3GMR%w(i@vl+){ZrpK!EFk#B+`($^>*nRCEflqj|)P~B-hWJ9+)ivQZl}&$k zDum2XQ3yeS-2x*1MT}ly_i&Kpi;}*d^s|I8!u$G3vsw*L>R^xqfPYNSh}w{ElTKJF z_Yi2&*ZbVRrB~2X;jDjz8aVv7w|8}o#vjA>@v(oUXUE~r29;nz~c*NXyK7otY^5(YS&3u0QK_!OjGZW=ld0OzY0xeSJ=^z3n*pDMvC<z z3TVi1(EEMBH^2GOnv#Q@^Nnc@g=x67=lhx=`zLiz>kQ8UAwQM%9%;^v?*g<;CkV>j?YdXPR zL;yg9a+h;Gs}Fd~r|5k4+-&^j&D^f>lSWTm95y~%wJExD$kbib%h&0*kOp1A8N};& zB^1uV5w4r)Xe{hnxXh0Y`w!v|O4dy~qFBu|Yea35_kFVarOsWhzkl?oN2(4HXAwjs zh1D;YGk+t=!R#g9qI~IGmwf9?!H)>~AOYw4>W`z@;rWP4tev$9P>Mq2-M#cD@hVG+ z<9u((aXWr)J$}I&huR`9;)wrfLlf`CD`LwNo*lMZ|-#)dy-jjH*)dyixw4Cy~;u+@M@QZxA(+ET5jg{Fdkp9qGSjfHJ(_IBO$v;+4Mo1iF{4L5{m*?}&kH<05@2$) zPX}(v5a5O2!F9u&UFjfAgkyGa5bJpf_x`!ZnQ!JZvunH#R;9wxkv=yc4VuUT@&6lTJz^rP@N=)DY zuINOsV3DQp`$CK70|SXAvj~p>oR4+*Nj1rTu_0Hpb>V^kdb>71FG7_}2A&m`Dd)|> z9f?b(-m25Jq1QS4uWf(jz1!tr`vheB#T+fK^L3+cd#c8t3~yZ$(#k46>g2?%2`x2O zE!Fl7YCcfwrS;<<1;boBF0yOH+k7`itlR5Qry%7yMi6iylIohJAs&*}mp2+iH%EgE z8r1apvPhS*wE}0X&GY^msqTuNxkfczPUz*Q*IEjJx9H?i|6MD-`8Oi|p)4v)FE7o# z>Rj~JmixLNpwW&VPT(#2!ig@Hsb*0YeU*t-K|g2Usa>3Lq(;>Y3T*v~U(zkNf??24)ezd0_UAM+fqI=+F<;?5WB2$Mh6JX8VK93t1hja-ZU`~7OK2~b9sv=~-xV&x7 zn`4J_i%a5~-B|qIH*(!&>#xV^3|Kt%=#oRWH4k(*Um03ermF}qUHkdV9hdgQZ|(2o z+SX!sL2^omz#_kTgZh7n+q8W4J@+oPHic^LOj&&9`|Dc8E)O~(4OC&8MRXaUtz)_9 zm{SxjyUMKMQgF0gqWNGEIx;IMJ@_87xH7`T(aSYxu4z&FJ^p08zdRjP6_p26$Vr(; z|Ef{MGO*%qbx{(3?;M%{ZJ=!#BxaIQRuc+>g~$$wb7833NasQT2(wKW+WK)|9gq@+ z_`B(nQH)@zx{LX1s>mjJ`chSnhvI~xI!7-fUk#_chIi4&^6v)^RlPs{JT*1-#kagX z6DmF*MvqC~gD&^|akn*A0%vg>ys;wf2$BT38$c^D*q<6O<0iMk%xMAXEDPM@41bmk zDy*FBL?)KisvG7Tc8%+jkdW{T6lKG}t5xhrQRA#4f$9C~>YegrKogNGF*cD2r6o5f zJL&jQpsCow9Or>J(#bF^lADo`*P;dfQCMNWdNmsjwg9?wb5Hzn{2Pi2xA6E|fN(5= zkzjPrN0d?UujITqs*?0aL(Fy89RMH}5S7cHG5b>0AS<;xfI$4@zq`N1wW((xm$_{R zT$!-D`J?eUA5G6{wJ3H>SUjy$yW>ZnJ6HLsgP>O?+qIhpQrt7A=i#%tul?R!D_V78 zxJyxNYrR|FZJzJ!{i~{A9wM5(jA9P4&$f^W^G0>Y?yofOb5$B!jO%li- zKXG92M1H~r+bjql-ssV;AR%2CwC|@ zEDmbWvX>DBe47!Q!Y*W1PR#7r?HO7kBC*G_6MPd)sL4Hg%jg8Mpy1sNwO8HcLQkS3 zK>#`+?byf2Nw?w_Q8^Mho|0xY<3NoQ02=*=?ND9#(aZI3*WOP` z+DoYz?&!PU2lf|rlohE~${4@`A-@Pr%~w|3u#MKvT_+-)uQCsl5f>Cw1(Z(DO|T z*F7x`Y3bEtzWb`Xzq9hbI<$<@-;g;Vf1YD(?>p^6Dvu>Zj$3E{>z)42{S`{f6a)QK}sNgL4qXJO&Se63AK+}_zt`^T%g zAKI;w9d6_|bfEIjnGHNe7sUPczY)1){u;HT<1bNL?s4g7^VkSGC~?1l;_@=I%j(z7 ze!fXrPXIhG;Ced%XX(PP})Z+829EJ%_9Stz8t!bH}8b!x29ARH9-D5jK6y?aM~2}D!g5> z)91Dk6_xG!n1?-XHBoQ%^Qt6=Zn?8oMJJ^0KCNE&Q15XweSOWYTyMQv@4s+olc{G+ zwrty0t87xIs@SzCxwhjU39Hh*1p<@; zNHF-`WYoFlY1%wyUwxOPEC(k;8T^$p>?rfvx ziYs+Sl3~N66U-jGv+$-%9E}mMT4Y7h@t6WDy;#0Jh8h|F{rI{l@j%M)u}S~+xA?J+ zK)bp$J8_6=y>ayv56lcIGBXwrsH`e;Y8MjvJ4o02x6^_>4Gt%;4f?@|dw&e|R>2df zTAQ3fTTG2~pHA^{DrYK@SPM*T-$=c;1N{Yo9yMm(yqM}b%5&Wq3Q;pc);=JOe%KY! zIz_>M<0Ed#o3?Q>WZc1%ad{cMC$tRSP^du+k|i#yhOv=#Ft9QzG}tTS4=7%Xq2wgcP{)3n=m-y-FAw6zIH2pCG=o&yxZP09KX#DK7l<@3fcy*Eq+NOPn4ygj;YzP`$IF;vBf=ZBA zZb+433)+=ZlqguGsVTCl#1}JR8i5ouru_wT3xk&+P9xc6(wKarZOx_tei!L_5+cwi zbn+T|`%L1GE$tk}AiDq*=)R)JVV3Vse2b*s8f=$Y8zfEaM}r1X}UAOaB`=RT% ze@=dUYs#g`ciP_{b-#R&O~2%m_3Uy#t*HB}gGsDOo5!W$ea7BAHsk#Kn`$FARjGnB ziJDK_L(TN04Ih{>IUe3p`X3tqKyou&)?VJafD#xjOaMb7S$V`gw z^=aCmJKgI*v_b1A<&`D}PS#J1aN3`;F2f}beW{F^5j>^Z#2Lto5P7^*5Z7uAkIE<_ zH50i8dnWb5{ou`X&+n9&=oG~!0xY1v90;<); zy^&!)q7_$R4z)7^_Le!iw1=;+)K%_eMUBhqOGmGlYD3qqjN6-Axn@bmA+-(0&#$d@ zphbD&Yxun2$$|72tKzk{-BwjLx-h!MBU)wzxQ#~o*q<$q{5ay;lB|}-t*#iA&Ggfp zK6r#gK4V%>SzL4d2CL=cO;chv3_O8n z``cU~6CD`ipYvLb+>Nb23oMk;o-v=0N0G4`mpi3>#HJ#q;-r__fP$|s+ zPWIx6)`CJl@bCn-=qAzH3B=0@72^)+pocp8n=W+^Ysg?fh2C?kBi#pu+GTX>rr+kl zUX$gW;?ZtFI8FGnCcBnr>k+x9lJAAJD-G+u=k@!bgtBvYw%zvlmj2A5uXpJ18^6+r z2Rbo4;`G~0uaA}GGd4c0{r2t=!A4A-OfOv8@}|nKb;pjCvC9Uv-P^j+&epe9?9t!p zd8(`#gGkd+3fR}<9bF$o>g@<-=wzanH&9cmBMPf zXO_dsP1maG9@x$s`(An}@t%M3J!cdBU&&=@PdY@0O$pP`LM>Qa_P(Es(#gq)Uo*DTV$mql&+FMXkj93hT+xd+lY z7!HK@&({nif>N(ZM=|YqWwYvO+Jme0i|%au@}$ktVP?AB0#V+NxC z8(0)+rrTZlQ9{ChH!FI%H_Ncx81A2#L+}u(4N>c7cq@%+8pbV$u((~wi1>-&(N~F5 zU5jf^nJP?^%cm?jhOdl5yWEHqrPKSs019#x4c$_85V`GnT`q{SL3Tb^>S|&n|b47SedWWU{>+L$&-w$FO-{x zVQSr|eyiqw{mTof)T;Rg)(gY#zIFo7cGCUS)V4V5)K2R141hq000nH`U_HGlP6IDz=UtuV@2WmKsUo#HD^6a z8cmj~U~sV^NJr-mOK;fx)0j5aBk63-A1E=hK7JgZ{-MgQY zNm2RH82|L$mOFGT7JXYQL@e;~9@!~>oybf@ZKN-p<4%h8?9B011 z306l^?Vp6W*6?{xav(Y=_6bp@K^ORD-vSC&!4iA6)0gR&e$?o8$wQg zpMc_hHt>hML|j~4uTT7Xs0N!lb>$8+t8{=-U7&}yxw%}V-k{K%g0^lgb}NXG0U^Pj zAhl$=rrVmKe~*s)iBC=ZkHKIF@4%F?f+og`D*kDNi&LYzwJ5{eQP1z~7i-a1HtJBC z>z+{H?Scm7#%Wm}+`wE%!B4Whkt=RK?YAKN+@lca)AHMj=CB7(|mu~;N-;IjOZ_s zeV#&VAqeZhWj1SvK@tj(Oo={i)WhS^){lcIM#O7}V$S_}@&1Watjyc==g%#Kr&sdZ zSADyqJ3ngKRmO|ZMuKxqWZZ{E=b1cT76cw6fr<7;FdR`{wzvg~M+4Z?*a^u`X3yWZ zEUg>Qe5Ld|=TgCCbtcM%7It9wfb=hBVS~xseS05e1mB5^(BAsG1$uK)e(~VvtvNw! zcpWq_(A3YEBgKi`|J9o%AygLC+H}&iSMRb<301fEYPk$UpwCATNLEHJQRMLaJy(QC-!UtT(yHl#j>M}$wY||J8F?L z=suAil^y_?@;eu1{+x!a=SQU7pCy_B4!NHLk6731{B;((}_OAIlv=tc*3ML<{A_7 zG`@MEk7;_`I~>9@AbTP6I}Ok1`nY3ZqCAWXeG+ zYGn5N6iKz^j#CQ4{Q=8d84sd4v<7-!^w`^z;O4-jC=^5jE!fd_@XlJ`kU~4bu}y~V zp^mrREo)k*InfXWhlT=^4zdZ_5!NFXQZgPju4*@v&cyiaCw|Apqmk;{9_;wFSsL{! zX8z)0BMLZCy_)#|Tm^0-L`ZWlRJQL3*Aq<4U&An3-UR$ZK=j^T2Au41z08*U&6wja zinf@V=k2wHwNNcETfGwwx-T&ebUj(_iBE~{BQK5 zXE8P6%g5U!%+#Ftw!zs)N7OxfjJp3aioV=ExQ?FhvUP8%5sA$;$@zPq+^{s}MqG0t+3Sxb1B-xWD%EQpNQ zq%%;8>DSbzd(G~7fN8z?+mQ9 z5YwIl)KWm`w4|RF#Rd>Yk|pBHmdMM*4=Vr8-(ZyeP*wmjKsS==MuynGJ}M)+gdM%J zZPu(Wi6=*lApF@dP|4kT?>}PzKXfavpc&fR_cJ!$Ts*#3LFtGjlbP%-voxK8YhjAf0=SJvNq z?A7Nqvy3UXS1+WU5J-kppDnWyK=CPNX#~VJIKonVSko2+7ICY^7?dPFIzP2aF9R-a z2(yU{=0qhv*hZqe#z>9kym=@k8gC|6#BGe3UVRldj5wHQtfaF61eo7kr*H54R_z#g zv_zRHLjWA{5H8Z5*jwfse^}jO?21Gb3j$)RW#pEfpI>46q)o|=N)BAW@XDB0=E7Wy z_RC-r)@VPa|9wl6vRZlMpz5;A*Vt8Rf1!Z zB1Th3n$%JHQ8!~gx@OK?lvOQ&U!=Et!XgFgNsCVmGM|N^58ii+Ssl$4BxYGV0LBwNE}_%e z_?HdrKR>BRe^SE8MCa1u8$=Jt4KgJEOG5hkHso_Vt5|Cj4JW6v#SPm49*#jc$iMC` zKon)Q;H$VOc|mi{Zhnu5fh5C|2{$GZmHr>v-aM}6{Qv(xc43%_?7NdCOCnp8Wkivs zQnD}EvWzW6rN%ObqB@qcB%)G;q{dR&Dn*u*D1^#VB&Cp~>;61tzVp4V-|cq&aow(N zxA}Z#IGyu8@AvEVTpo|d^YP-yWe297DNXN@2^&C!@5xj8l-v z5*Wpc_Wd+&wb%k;yzRzNRS_xQrb*vkwu+SK<@T%VEE~M>+7FjT56-O|+{Z<=eXh3K z#7j%Ng-klhc;cN_wnLphdg>ln+n#CRdo5EtZ=108{qFRY-*TojY&Yaqc+kDf<~P3F zvLeCz^!}sX%MPQ$A0Y5h?p2{NjG`5*J=iy;ytF|$wE2AFtlqP$cQYvDgQc27=Rl9WrrTa6LY zkRd}dn}gX7NsgR_G%n*{h{y1p#>Jc;btc~%ZSJDwF+EG?mRor_YBVKc#2Zonh^t!Q zvKBpl^VU-kSA`fe&sc`CB*tkF3i#Q?C&6K*-5_`2-BcbwVi#TZS#)J6hnMZUM?uiz zZqHQ_;-}z66CQ3WZ_7H<8-IUf%s)IK^If&-)UlmEfBud}KB2L{CT3(9WJGt_IjR%# zj`m1an$A+Q@=P<1W5filE_M5#5o?Ow97knZ#u%)_=KFK}plxq$`mDQ8vmmoIaF}1w zob}w%%3IHC*5lEz&;5Oybsx~kJp4&1H>Jf^M$Mxs`vz&uyWPwbS2;{-w7s zsfcR4Lf1|ix_@Qb?Gyn~7f2R?O=j^(ENLBAoAPcJqiMfALN%LnE%b5s}5c2eWP zh`0qGUVv064h&Y4ex|W1Cv&kQ^clFaBOTw;jdO(g@5I)MAm}T#6T&1EUOl`R?6V5pe3+)<#57 zV`fwJm|u%G0;ICJ?__X3xK_Y&Grf+_{Ur6%N4TBQID#390-FdQ$O)e|gBhDgP@va+ z%-=idw0~>bbiu7E^{dfzHrhg>p@6*fd&Wt>&3WE>b_TyNqDlWH+}X@Bk=JGt-M%N4fI zht>)HZdYsRkBaDZ_GfP7Ppfu8dtKk|CW?Vh_niiBc@&p0xqYK)KQc$fd|&_3?C_s( zW;bny6op8E2GB^6Zu`0SJyd>Trvw%r(RIB2_`^;Y1OBDlbZ>uXtr-SbdY?%mIVPOu z?*^rd+6}3sBmn6})0>PiuRgE!jPENa)+oUD@A%uW1Gj&EV>?5;nw;w?UOU7T@w2eJ zd~Nn@Q0BnrB$m6>lovLZUkYBvZ=$)M;&d98I_1=qzTB$2-1>2j_2Z9o-$8zNFsk%0 zOT+zwJ6WtA!;Kz&E;1<95U-B~X-%h;Ry@ydB{o5nj$a($yN4>EA_hc&m~3Ym80WUK zbeF7!Doi{xsW_IceGgHA02aem9R|6{84DzzSJ!YQ46j7o2G*mM{dhnrQy_nO8~?73 zK^=?cyyKKX#-XV{U@1TNWVFMgf?*L?@3>+~4=lJqt+K;IUG546;C#ubGg(N=Dxxv8 zSMh+)1)TOQw~HpTsb8mFwp0$+DJqVgs(W@$e&^~1Px~$D`@N#H?tlT_6%|)b4>TC0 zxuUG$nWDt~zFj>0R(yY}Vfkd&vdaV4y^fl4b-@BY}J`C;qu;`$zvQ4D{3Ly zeL>ALo0gF{%X~K z=@IIlJw2!GW9m$te-Yx}APo4&HmFyxk70T{P0dYo{g5$&Qj0MfrIO8_C-7cL(9FGC z_H0j7VmS<-Vx4aB{42vbI9g;BNPwZT!SxfqS_&uhqtm6Ox3>jx)vR03f zLuk3_TCIkd_%p2%o184YH-WP4t8cbDs_8!W;}fJK0&E*WlMIj~Cjfwpb%}p>?%X*! zLJrboi%dIGU&v4loqQ<~HuwI~&2eXPPs**JnHuL(yQ+e{BeV+yK6yHTwa0aeBRhFX zV*2k&TF@jR_@ zXomj6&+hA=My+f-dBEfy-aSSgTTo*6AgaUQ$PfO`x9{%hF~j|)hacASCfyTIFHHw)LNr zocwM8_8$fUtgd^d`8p(E3@L~@l3y5o@CMr&dU%5Wd5GCRMJ<9>1HQf}&DLMupGw>Z zO+m#MmkJO(Lxr?$NbxyHz>9#-RN6h>hTs>T{*~57PP!Msf#-%7hhCd9&T|2WH-z4~ z&dov(FlYY!IEvJ+9LQ&^2ROeUw;md#*XT3xE!sV3^#WGv!dv}1)7j^sdSxEs5Tzey zH<`c(`M=jd5o*Jf;4$|C5Lxv5y_Z~Oo{WIG5XkF{%bd#9SMzJ{*9~Q>kw-BI8Hf(| z8hT)KLLA_{IInT(uQ%8+D9V_+sz)u{hw{AZmp6Q+exWNCEnBt|WQ*s?ck2?rSBqA? z)vLp;j}Q@Mf{Y<*>eUkNkR@&%;zWb4GHHiq%l@Ggp8d7vg2NBZ9|Qh)x$o)T>FZai zhFmPYrHcOc`AFKreo|f_`SjFFxxJ}=*}Tt5K3@h0O*s52?aq_qWBv9;m9B0~jO!F% zJ}>XfrRIwL;q&JHIG29NFw2ctg!)2SSAVN|5-JgFq@&1VwxTo7Q%>h(^4n4{F64F1 z@~s;r-{~g0Z$9qh!^yLv3%^(P*(*-DhppEi;>ASP0@>8?H@AM72QG`OtrjiOYeu)w zOZ5!2t?C5$OkZhuQmIAn3jfRbj_Espy$VcCJ*Yv6CI{4sH~PxEgIC>)0@WTl2u1ES zQi+0cWBcdV&t~_I-QW`YbpbaP700I6o_mCcVYzk< zTG&UgUj6zEU>t56J|rwGV$556Mei-G_CGfAUl zSh?-E!8*_8#y?senZ0E4sZ)*by?OTkHgRaWDqss(EQef~qH_?(3GtFZRVOonkWtJs zV8$|!W_Cd~>GlpQis``-d;GV6yxoT}5e*>@5@N{3fELAi16TIP19H@t4Zdg2g@q-L zbJi4FDrn}$an|j(t0!;Zw||l^UP|NDbxdb5CoF>3+M7EZwKPqYX=!A)^xwhYao0DNN$$!%U^hp*W!macUBtS7k8ff@d>Hc|nZSPOKZIY1b+uTlw9*!DCh z*#Gw@=Z1h$W-!<}k<_m!>nt<$!@Y5S19n6qgiVwUEd14+cVwy07y8oNqoj7hBh>Z*ea zJL>V<+wRaVa7(T?_-xan@x8L8?!7gcyhH}VNXJh$MOp3A>i_#H`2Wb0{=1wi|7Cp_ z;cx%wzm=nec_blbIs_ z)+jFSRTD<=yS7f{B^6aW>%6*ZB9$q`ckECZ{b#Fjzl19yuN0hs$aX4p0zh5KqU zDe4@@Wl;q9kxw=vJA~(4ymDsMWvQ4r7OPO|MOx-h=?nc{e)#YsLl1kjg=9;Nz|5~~ zbPf;wrl2km-;T{l6lMOCp{ar*O(UC(n13JRB5^4qad2*QT^27Ygb@s*gOvOgr^jU| z&PMF~QM#+5X$OXov+_07<43=F**}MSsJ~{H{`AtUlo5GnZcVS14 z6^m)G6!7e>0A?5!Vp~9IeV|@rfCFRN6Ej=cH-J$TBR?)j`!w>j9^>he#`Ojx{GtdD zjRBu$$%r?89j&u{fEfG2^g%n!x;$YKGR6yh25VwNv*5}vWzu@ev}xidByuGMi8qvLBAP)?JEX8Hg;1m>0L4oMSp z@^OrQda8XjxZgA`uu+-__{d-2{WmHxwMP3p3jNf^B=do@f9# z{nvyLrw}O{9OeRgG|--&8G2OGwdBZkz4#dvXXtumJ`a{A-X*JQ-KOz-)vtm-^~{?E z*P@WOVvtqrC2j&~YmD?guRgKL)P#0fW()AW`UK7{%pzdWe4R_q50RZAU(n(LnNs!y zVg+CvPEo0l){XD+nnqMsq^GCnm-T}>HXV#Vl?I=Gxl~`*+thz?(+Edg@%U0ByfoP->5!3srd)qjN-r}uBacy+x3j8 zE>V@yM)Q^ot~qr?3F2dIl5p7Ya(WoQa+U!*0!C7Xvy+nuNrAG&PR#snKioGIaE!lB zsEp!8pV0;nLQl?UZ{Vh<)EdT51|Ze~N-}3akrA!Bb?dh0PQk`!q`nbR4v$}sGb_+` z@w#vUaGza1oAVageuFcNfQ5o+D2Wl<#0UA0Bz+)OYR{>j)%lq#7v;$ zMI&|I9xL-%pgGm(V8;Odpm)x|(}jS-&`~3pB!;~L=>7#m0&9xCdCEBJ17CBG?ySQy z4?X*;>!Z)UoQuu3$Bg+PxPBoBGiEnVS(Yk6NW0125 zU_pE$dsP#a3c|^$Jq;UDsI`ku&^UqIUbA)UBpgJtChd{KhUulu7`SuwF1rI4BQ6RH zl6-~(ooDr1URhCXpF$EAg@uq*AQ|Grb}ZS1xYv!tG$1fK(21DGBYy<$28nkg88K%1`sA6I>(8E*&zL2rc>Qg1i2C@_+WcDtSdns{cSa{8Rzhf$ zY#B$ShY%B60wuENjrUr(#qJ`HfI@b_!3BdQ(Mu>uq2p@R6Z(E9~aJnR=y*@}FI922TyL)qug7x=5iq zqx23`vqXU4#HQ?>g|OOo)`o1mxo82GlYHh*ZE0E=3Nrn0eNQI(+3al5XWrj-MLL}^ zaS0-)Eca3$+LBO47dafTNM;Q|m6c>uz0Wi7(BeWCa=40s$4n*=9)__2GiP1zxKh?H z+M`)dg3v*W1La1LXng@PkS%}~Zs8{T)N3fEBY9;Cz9lkq3s&XcvY$?T^|k8_nW!1v z9fVvAy2unVa=|W4z%iV#reBgC?KnV`QRSH^Py~@Df{0{8j5#w_9kQ#+Z9R=fjegN_7o?C4#U0Oj^9EyDu@6)roN0yk z(w2)8e={vs?<^p)_zS7GPTpRLh0p!03#@?lMAYTG5BzL+)C*oJXAE1x8zfK$<0r74 zlDql~&Treid2_A}r5wLmmzl{<{5G}Ht)<3q5tQ{t6Qf{FkLPcyaf(Vi6h8$oLq^VQ z`jo||v~RgUTqO*)jvU8qj=71u%OFY!`q16??f&!m33U=4K74pyHQa3DkJ5$V&u2`Z zj*&U_7Ru8xpv~FZL%lAj59p9nF0O7FZ#;K6?{{tNDVIU(13tw*FW6VFaaEOF`F}k` zcX%~)*rjn_Vhw(Cj%(U<@w~;}mvLJ15Ad&9`a(PY%AtS#EdF<|xLjNBbo0_r5{_6M@pZ|PXPQG@U)-mIe9skFnM*X+S?PJ`#kJ`L1yqQ-qP_GztS5r?Ax{;qcatgKno|63Fz zay(L&@Bv`9qhWjmn{_Wo@j8x@Nn}(AP6cg7yxrk$A=hHm|KGA-Hiwr-JUi_%hn|z% zCAK0XLgD$FZ3Pn@oCr9`;eBb?3&O|hsC}OOZhp}w29|W}Mg=ukK6I$sZwn~&lk?fYmH*|V$^tb;O zVzv+yG$(|}JJW8`2ejv8Sqe6ABocU`XMkLO)sON#)l1Z82Ut6F+sAU(QKHKX9t2oc z6Zq*I+l9XOHDVu9-?ZSgTI6?}U?@YKq*pxJk8leZI2)R#cjNl?<~$mJDjMpxv>1pM zZ%9b|;&N0{Q#I-#?#~GM33bgu5 zQgtN(=!P543Z9oNWb9ig^?$1Hw|#1jFZzja=)-=ROC9D zR$m+*7IxI7;zxnedM_JnDK(g^od?XcW$RY4vJ$Ns4c@W*=a9P;g_;4*RA$gB38(Uf zIi@O-p-t$w7`I|$0ixT#ux|e>&~fgv-S^YKPp~=R6+GiZrA=Icm8$0$2VahbPzbhR zUV)7l1Zgc10Um2z!9<=|rBwVV(?xR1=`wM%&i(ves;*l26~AAH(T&3GpD%~{FKbEg z9H7TTUb#-Vmzo85a^oerwzP9+Z~!^O%OnQ*a!iE%q~fWMMIw?nK5HLq9BK=s-U z#czj$4 z^TBEKIM6DCtw^~(4~9LkKYNhbupQTCi;1$#IZ%eFw@m2n_q|RQ{rQ!l(X$cd&+Q8d zxX?Vk5sEU3HafT9)vNiR9qG)(;|x_|U?jsfq8FtRi5MY!En#$z#Aez;@SzOw+4v9s zgmiGakqqL@69f z#2HeLGQn-`Kn`I3wMmmEwtgUkH7cGA{q~!MB_uM}W9;+g?CPrG`n7u|@?{kH{8W zpqbsanN{JKBb8xgj6BK9@fDrN6&4pf_v0d055lD|^DvAxd^n!P)ku%?sW~FOq8A@5}7LN`1VY)4u?XY~pwq~ZWF5}}$ z^e7^y#STY&bqQ7Z)sY`q=d1%lP|4V!j@?@|0(YCrsNzc z-B?ML+7YjH`>vQQ6}cA?_#@rSsLLa+GaFcnQ_Vn^A5l}vBpeHS?6;Sly9$6jd?R`1 z4TwV!7QXc)lgFrx<%D=7s42g^I0+0=eVDkoiNt8+uwe@3#zk-Y{xLpE@sw$l ztL}DA$LXpS)Ft^{wLbg|ibZomrv1~4Tf4K+fqoG$pDRXj(Up^Ul47t_sQLnmc z&XN|J^5ZKLPV$dgKH%6fGMfxFiCxl8u{ERxn2VLff_%yi?)sfmckbS8*r-t>j2E9j zGnQRt;=czrQY>*jJQ^{_bCe2@EB#GQlX-gK8e3h?+)~VPF4>!Y0f=1WKr+}wB)hY# z1l*LGD~#bHIZjIK{YiDS?Q$QLO{MAuLYWu{o3-~dIY5z-9lv?%wQjn8p|J+i0WmSBKl~frttVr5 zvcPo1mMvZCK&3&j-xW52LQxd`ae5qN702_F=@9Vc1)!#ZwoqvTv;mX94K1of!GH4- z;zvRkFdB*uR^6Z8-rnhbzIH5dyEgmkU|R#m9y{CXbkuk~u6y$fxZq}G6~#yTL|s}H zDmT$-NLFZ9G`l{)A?JY12v1UW_!kSZEpP!_UPz76qi+uMy^W}Z5`SN>8MnBJ7dg5Z z9PuVX55iUoKocNo%lXq8TqKljFKl;Jrm%RfK>J&H-gEU^CL(;}-Q`^BmA9;_?(fV? z94p0w6d8p2>`FY<$M~_ZA?|C*42F2j?A>cseV$Icga4i5ocHGL*@>MjFFzgkRzIi4 zp2ya;s)aU?#aMLIkwcEy!hNt*MiqO9n+uNI0v|gK`U6g5lM{6aW3kL?%NJ)A#QgZS z1b*wLBtnrcJ7*Hwi;!{{r~AKLLvS4h&^q%;5aas9{l00Xa`*T#DdrBvv9zZ|Z-~fs z>w+wVVqynMEHGs`ZvuUVkW(_)NLZj8s*e~$40fRDyL_}txOr8b-bZ(E@9@;;ez$}= zCMVT2fXN7llm1)4{`8pM!NhP3^wwJ@AZV=oxd`I<=-dj38aZ*2NN2-54@YZMF1qCFR^R}d=ZlX zJVt}b%GIZD{X1b>C%x14s!wepk1gl-h2{M>*J;?=gc?rZBhe;<7!gN}GB(vz&;iI+ zqfZn>n4l}jp!kg}dr*~aFb770fSau=n*wt>E0*~0Ou8Xens~mJhJu6eSa5z1x`c{2;8M;F=GqC;38L^dr!eJseZr|=c zGYj_>^Ru(BIc(RKkL0J|(X?gH|E)qQ|E#0tZU@~l4DJix!OlGpafWOHC^iv-BJz)C zEuRNzReh2&i=td!hGz6&2Gt>tF9@X*$2U};HS2lJq@PV)*4U_K+LDxj?vfE)TRtJ< zl_Ed10bZjyILH_2XRhOdDkK^58M5=K;e!a4u9J)#Qvu z%gdFo`rB#xOy2gPrMKbWk*&F3iP>SaT14GK=fzb^ z-L$&sJkjj(s!kwD4cZQQd|u}|eXoqBP>S`PX84n~RRyxp6)c(V3?}aU%wD^h8Q1OS zm)(wNU^W9&f<@g5V{y^Y98yfF!cY<}e@uHaK)l>3rM80m#;T8al_gVDO{CAk7(fC5 zb^>*%I<1!|_T_!3n8D;2&n4E<-+5vQEszKEM(lkJ-;k=+7gZy2c`K1X(4>o@srXS6ip42eCK##uMTme~ zKSySmVxtqxASS~xgBKOC{DV?S#--Tqail8FkQ)e;-Gv^K;ZqNGogDVT zY_+DrGPzA z2#G5h$Q7el8m%dqAOm|}Ll;PF5>lTUb2MC3cHc_J94uetu)|S+=EBGF!3_WUp-Br{ z{7{D_cbz;cHe__Df%sO^px0>D%zMlDY3e)@A?XW7{sy^-v?Tt~GKxls=$Y{Y8R=1A zIMBH3srx9kC_EM?uS5o+oav@Hb1xDVzVv9C%(|slmL({vDHNt7hfk_ocQ!2rSoIFJ zb+}zhXX;IHH<81>FPl|*trNyNFf}F95UgtP#DcBS)9NZv95E-GqXH0gk#44ss5J4_ zJO(P-8}%kPq1M&PHd2pK{cfOHdb^%naj9669g};(WP}_>L`>$?o-N!J3+@Uu;Gork zQ>H;czGWlqi2?hcO}3Nt#<$bX{IqY&obs6QIVcV&w;p2{1dKJGUQlMV$RROjn#jH4 zNBP@r2D<4*aws^YKVyDY2GA%$_TCRxZ>}h7%BXQ9P;nnX7)c(CS^JbqPW(j~W?pH~ zOZ97Q5;q05n5ptdGUz}Yj(c3e^Oie6^gXzdG|)I;7L#7Ys@LKR#Ydvt$T4&b>aHCd zi!UNON2)$fk@NbsTh)R$>G&EKXCR2&%;?LNE^5leDb(Ra-vv)}zk9>Kfbkw-eZZKl zLY^cMM<5HX%M;t7yL^;8rMQAuhzSb5JGxRWW#~~*2Z2MK**O?S2KTwUe17)e`KGfA zt@nr~pSBTgUhgOG3R$6o3&MoU_$Usd!+DruiKc))DcmdNm5KiX{gzU}@~(0m#F>0Y zCbry1+3xua8I9Ui%mEPFs!vZQ??&%FL}<%OLGg1D;$3*3gN+`;Rq-Kj*o-nzjI+#r zJYP~)R*v|YSe%2-m%37BkEx1}BYuiY0c_3eB?sHr?S zI3|EGqKQl_fjzwBw*om)+bdSgwZSm^aIP`M0dfAt<$@U7gbLe&nCy^V6%`&s8(A7x z`!cLn?4N!6F8r{y{>r0)cKh}n)7J|v-o3w5_wFfuMt8n2y4!+_t6sF-zR$7X&-#-N z{k`JFmu*Au4XCbV_-2Ko-IpgjpX5iKc~=s1`q5>NcFR}!J}SHPSEEP7l2p)bWg*M9 zrGfnk81nhY+}Zf9Zz#`Ou6cWZIDvdIgl$N{5)Lh%x&D$g{&|K^~Q8j+gKngt*q70R7i( z-~#PAPI^3}6^~%n_mVE2#W+txu3!np7QxI)lY(^IdsIu96X6Mt1#|FJ?rc5C0`NAK zQ2lj)mZJT$R)ba@=Xl*7xi4Gk6WAXi^I;}86baeIOP4~>j@~*r;0%J|(?D16Foc;% zJefp%Xtl7BLXjlkGp85kk@<^CN~{2j%?5X?cm&~BoIJXnwq5u?|2!g%Ya8~kZ_Su& z@EOR~A`Y&bS#-7UH}Ir~8~}a?nkn7y4qws^&@IF!%pXCGzmHFIq!$lU+i`!{tKA6P zs^Zj*?G68SbF+j3(rVoJ7{{;Nj4%EARXOO=OW$>w=JX(>E!~HwrdVS3cob8+5Q>|C z;b~`U4PITxr}vyq$G6pg<&pW|)0Z!kIpdK&w;%1?)-7AQcI|4ywtGZhsFVA0ih)9* z(5((r?hMu%*r^rx@=@~aTj^acRZtX)x-qVf!nHx-BK^gvUR1~VV!1qA%JMzn2@k2M z%oo3Ppx?=uvJYaVU`omw zqxXZS`VDC6RbHMBkmLIG>k~0%N?qQ&xfeP z^qsL@US6u>s563D(FUtuyDM`k|0c`YU}~=Cj7yVP#n#iOADYS8OOsg1O&+r2>5d>o1)xcmXEq5BL56tg43xw!}yk&k%&>Jm3gaGvq69eF#+K2JUz0 zp%x0ozIDKFGp=l$f!H7ow%3(6swOd)MR;^(6!Bm_`Gaa&b-prJf#JG>zOa9^X%V@jH^Aa_?&ucK`3hOn5 z#4z=eeXtY+iz{Rc14~@Ia6!uBz-NsRMFitNJjtZOBV3k<;0IHapuZ+)dNp-(P7;xYm!LpubgIdd0;CkiCcLA;$&^#Y_|6qC7I< z@oiV88;x^vio!Xu7`39!;C>3lsjTEn!vWuOUVL=9{i{b>LyM{N=H?SJl8z#nF+Cm5 z4kJXRnBVsh!<|8c2cJY5&N7UKpgXm{^X%U!M1t-9`M5_arDFdx#b*;in zNWSz-R=)Sag9mBtEZ$@Hn}#+>Y1m`DR091Nwq5o-W(GyE-F_X#3k~*?U)Jf<#BjLu__58J_m9ol zd0gJuNS{>z>)J|X!jmV1P0J+Fu*?@4?EZxp5BtW^%*!Np5>$$cizh*KT9#~GGGfFg zl$pJt>sZrCsekCV)N0u>gj<^!7iUcLe@KJ8^v-C-;dJN`eV$$DjLX)9b`!mL>hZ)U z=UB^WV|AmCGdJ}v{GQR+V`hH_^=-K`H}BPx3l|1X3vz8{26*E47~74)d6(Nkt#p}I z=5z=ZQ~7$!-wlk-t8>TpV$t8Nefwbq`v)}U_INa#s*rCG8@lAqiTFq-zx&Yk_0%`q z`|*(8X8jk;n|GXU$)2y=V&|y*^zhND;s;}O6KD6fDc|oJM7Ou?bHVd7c)OhBgpI&4 zr8i9pg;d*XfU&JYr&@_QSKvlI++9#wypepd;OGuL($nVbFlrl12mcq?$DI{O6-2Ob3o>OO8#Qv?PB1z_ZR-Mx5f9gCEnhg#IpRG;lO^9Scpji6ImxU zbm%|onE6&WhrFVIy3>NH+sxR|{i;{j>xn0NSnv6GA#QooxN3^4ZXalpFRgmD1E#z< z)3IQG28eDjacHaxuDJ;YXgM+PZzsU0H@wi*JI4gQVAilE~v!bQW;!w{BoMma0;kn$^9e4Sx!!YdY9L{3nY3P zMY<4oP48OPa@_-1G5O`omz)m%kJ1jYkq)O|;G7c(@niHk=J#z4?YME@-X8M+pH*6l z?%+J%rp>?=!?@zT#UKAZgm`)49#{K0An!6k#BMka?xM_f+rB*+fGCx7+)KZGTg(H* zUB7>R>s8pJEqi*w9Qjd9Pm4Mhdptdl-1hF+M1Rhe_wQ3~UcR!j(OmrwCsd&+!#e+7 ztznOD&fk}@mpr%QOZ4RY?rYh?b|z*JNsnPraJ+d163XzMEgl3Ay~?4F3EMkzFVVh! zV=I@i&4U~!z%}jdGNJ9V^mE4GCaOL2ws}&i4I9IQ3?dh5FB!U__=Hrw@D7U*`i8*n zh1}F{xs!v*J@O|kU;b+~6F($~s_9SC(-Si@Gwolm8k3csEyaoiHZXwcDkD1VL4yWK z&JT=a18|^-)4p-79UAgx4~b6?LE^{Bp$i=Q8AMyry?=jexY}rn5*_FsNo_D+7N`OC zfGw57HR@Ofy0aY?Ui!ioPsRS+p6&H4+Ho@68+A+wAG_3TbZq^kyP3m3H{2CD{&&?p z{i4?cd*~WAePPn%<-AD?Dl{&w%>GNei1?!L)L=wZH?jJKYqLFZ@8mkY=Q^=R8S_0w zXjw#Qa!Jycb?9Q9#ST-4x)z_|Hr|U~`{KpN@3)(>N?F#EpkpR6YrcNNhFi(W<_JGh zKy-<5dTVkY7*jTH%xd;$H-v_v?A3#(%BS3{rZ7Fwci}FGz+*&e!~P3E*yNkZE@km3 zb7?m{0MJ9%Wv(C1I)4a%#NLtRo{|w_|4UK+D1FqIvzpKE9FZ)fXlE7=yTatR2qA+mA!cJA``+JzX6CA6XcB|+70{m z2Djhc5nMvc6KIy;snk0>xaCi$_K4HO}+N7ay5UZX0js?*{Fgdq? zd*gh-d8GK4@yM{Dlq(8U?&gOP`ddN-hXW68XWAD=xcJIqv~ zHy2xyHf_0Kgx>&v*W%?g3uCN*F=S|_0_9a0aTTFz|WE6>s+nBnDdD9 zaB`AWN*iy~{{wOWYlaZ?p)5=v@Uk@=u_c9_OT~{2a*DXe&WLfE!a~`rv^qU~3O!UL zrD1G=!lW;U;>~C?e^1+4%@v9l#`D*jUnd6i^s1wHQ42**5Z(1#?v?eVNqdaGz3tRy zh{Hs#WxSqIpFS3p;pU+E`XI{BuDHx9h$%ZwkKNr=7Fp}ot#Mo*YUuEB7$6{=+rjhP zgY!;3qONK-Zu=uDReM{5wUKvUS13E1IKCOF)1kwkwM>uimnxIgCwGC{R`mG7=!H5J zWoBQv+O9@MGr@2i77sPn81`));)G>*7R|`-@vD|r05DMk$zP6jdtC-WasmJcvsi1| zY$;Cq(OfXqnOZNsdV)Jepr}j93ubIzv$Ubm&;6EVOai*G0I{&AHnV&Vdo1-dO$CeV z_O)xnDQPdcRKx^#eMw$97aqRbcZR{YxSIP9Brlruvg6@PPTj7j1(TWWyOy7-OYJ!j zq^D0$9}{t`>W3`-W>al}o(TDA?1gd5NMQq<+tL@1aOYlHyJn5-Q+rl`ed%k1KDwKI z=bi2Q*kGrN>G?I)9j-C-GE}{m8ny`ms2w^54(gK?@^pM3rcwV+LgO#T`Sknasg3G9 zvo|do)TWB>=JMnFg!drp2;&trTX`;&p{vUW7gB=jIU6~BJqr+cN@6?=SqR%XB)%(r z`;L61D0@6S?)80Bvwb$*cgmCHS3*%_*#VDDy|Ch_66xcZz-75XX8Zl|7&*a?!1vo& zRVGEVfEoL~kftXhW+@@(+`e~jLUM94Ga&(!LWShrlcMnIt=&LHSW6T`o{G1 zAdfR`vf9Qj8Zl9OS`a~KBqi3>zkbZ{Y;QQ-wdeE;A8oTMaPNyo7L*55%XHImaK6QG zUph0RE-1CTnRp(T{sFq8iIQC}%^b_{2-7=Y%){Y$ypmAcI2_0Pkv@w1eoG%u&TVk}`o+<;rkrd~r@qEks3lU^gC0co|Pb5Bh)J+P` zP|~x17hJkaTT68ECoV~)tL}2Y7mclVXm;bZpBtWA=kPu3kX_=UoUgSNiU;*~Xet!B z*O>#*2Tg)C9uW6Kkd!C~bK%^1L(D4;?8pqHtpjqSSSe#A`uWq3GN5B6{S;8@gCae} zi>=;1J|_TzEg#&dzI6|0+N4jK*x8hVHH}qX!c*_2r)Q4Z*pi@b@vM;|xF0d*mv%mVJc#ESs$nGdLoiU~^qhBGI$c}LFZ7CH2fI$N7vV>?Jy z#{YF^aps|1L?p5&#F4WNjsf$%gLFEd;NaSi66w~XM_;zpMp&5evq()K;b z`W9YX!=$2Pr%rzB^V>5riSXid`syy7J1=JP1}g3laqA5A^;G)-;urNU;?c|$rWcR$ zWv4BD+KdT-E%FM0(vvjuh)fP5^ODB=1o-xZsC5MPmYg%s2O!{n?Ow-*dA~_?JE1AP zx4UKES;-eZZpar##Q$S|fvH)JldZxbd0A%E6aeoBVi&g|{PRjiiD;XRQX+hPeG#9A zB0f8i!OEV*fi$KTfOG=k^CforvJ|WY`9jY^*d> z@He~LbXuT+mQZE$&S55yat3&-%C~{kD}YGDiM1QwJEKHOtN8I{D7@!XxORQVAy-xI zqmI>UR+#!~>~E3dYsus4#?{+o-Kj;>AX7uqO|x7>SfIb z@J@G%6}eNqWM0N$IWaB`=4p9`RtJ6kNnDd@WZgOn8|w{DPf(P7*#FPGI!PsPeAJzW z{iH8R%=B5adiCyWvrYORK6cFZ+-LWsJ9oyZ8!5-b{eE1TJ9lm>=XOgrptMK^^s^|r zk`gYl9@@66sT5{$H(h?aaQX7>XU_~x4`T7#x{aSxuk{r0C0w!;3Qb!yI(vM}mYOqS znjAf%Q8s@e$ND7ibo}+0#_+vMec85(nZ28qJvw`Fc`yET2!UreZzCYJb9u?5z|VxP z?o?-&sBcu^YdN> z2SGo{cWvGJN5P{)+ME5EuV;qM-3nK6f{yX_-Miyu5zv)Ozi@J>M+E)8-nd5%CFlXK zEhaeRL6kE3v_I!cS#}gB{d~g^MStnz-sREYd*=YNd>R|+znGT{IdP&ZcQfgH_f5)Qn)ehkf$pJ+!3jt8;^vdN~No&9Ig>pM%1c5ztAZXL&> zN#QYAA^m7#d)II18CcK7uuc=6nJfEcvy*|ew)xjoxZar*Qk>JXli`pdIWJ#&oiZeF zV&5`r#}@{RP5@ywO>Ct2XkUA9;%x83k=}25tUK;YK^hD?YkGPX$UTQ^e%fxVv3>jY z9XogS)0%4}($%=w*tehup2QR`syldj&si}~7vJ;Tuig2_^XGMN!VO#5O)D`_P~LA; zjQVtdT=G~*eQ(+;&_n{{Db^iG&y)92v+Qw;tJk1GD$IXL#*#CJ{r`aaTD5w03Sq2^ zDUIAAWiVPy>6wDRwvYJF7aVrrMG~RVGhLDyu%b11ZM#<4uU`i~n&~)>LPOwqXyGFa z@QhJGHim*{EE1Sw;EE&Af&M7a3|9^rG7|_nm1rFlPX(}7B+EV(*rR9Fi3-mB0483Qn<}#1v`7AJ3*Dyfv-o(@yIsqXD-tHn*z0&Ih#x?K+ty z*ZHyI1)vA0+fEEHkv2bst(5Aj_+$Qe0Y!kVZZR)PR@27(mJ(n*rNkpz2Q5&2@YQHG zlwC>l{iuOGH{S@?b!5nUWlf)7S{pgpuq>?DQxWW%Q2bw2uoBC3o>G|1lJmgj`lLT) zosIiaLHA6fqe3u#Ft1P+i^A^4d;cx;Thlr=QVi=Hw?A8u6*U$Y|q?SCRm(gbi~S6=#5TrBZG+!mz>KYGuOXq6R|Yi0#(g~&0WHFfr2fe zn3b;iYTzxpBH@m^+KH(7tbXRgTtjqWd@IQagY4u*3eVTFo_Kc94wjfSh!3k)n>L}eulwf` zAha|!JDSRF5UR(HW}FDWNoGpM`5DbXz(G59X66j;=tGM7GsqtDsFb7x;4UBTP+wD- z3zSe)RP+E9XyAzxCl87TuoJ!c`%sTrh>xpx2doNXy=XL;|0>7#n&|zdWt|+g2;3+yN<@#obsfY z1sGI*V~PnHa{PQMH=HcZjwy@n`Km_s>P>$A zb#_?mWdff_69f?yh^hp_bH1=5%5DT&ovG(fFj5$R21ye1jvYH5pwOA8-%{J~=_VIc z7~fuOZnTH9z^572>!JK_2Xru^$9(gDkv_Q^SF^guYDcm>pSDGvNlg{-W-iOFpZ994Tt9W8aYp;%$90H%2Fz=5K5=W z@r~u96gHk#GY3FxPb5uRLH#ai0C~zjJOBi~=h}yt=38@3W2&!TQ+Pd~!DOG0P@HbW zn1%23Du@{ zk6SGDjj!-t^sc7*Zxt5WzEPDyyz;*leOgJ!_3w|X)x5{qtthncnCQ>C9s~Yw0c<2y z6_W51bdtCB_kGiM=RcrD0&;ugy8wFYGY+sirE+_z-|NLzuMTdS6v(uP_AJwR&U8OQ zEvwrm>W1L9pXgpQK=y3Q+O+PIGGFs)g5dUflb9lk*QrdKU`%-r)U z84v~K5(+3NVTZY=1{rHiV0USmT7b;@m1gqGA#7GY)B^o@39Lp4x1S;+-2Q<2ZK6D( zTT%%xKt(0nSIYcB*UAfMa5u|X17ykqj=Mqa&9GjRKvltdrfoQIQ~C`;Vkg)M(nit# zf?13Ls@{?A?C^Ay!c36C5Iqh+I5@WqEz7Iy>`zI9)bH%MpV z;Rxv#YzV{+rlc;eq}WMqi^2fxL{j4(CBmAQ7eCcH9LEz9mI=~OHalhCUJfT%`d#=j zJc*1)k3I<=Cq*i`d3Uky)bl0;S~X?N(1$g!B)!20N(({jE#7k;dKyVjv@)Wk;89Fy zY8kn7%FgC`*E8%YA7=MHC%e*N7jG};29u)?ChhNN3_6O%fG-49drXXeX**2Wuj*U4Zy!lq}8HoJOVp5lUg8^bt__l=Nrq$k@HYSmO~91OQ01+ z2AZ#fk&PtGzS!EhMT^qZh|NSumY_~t3{BvY6tl>|k3~4iH)kxeiq(Tv42wm@36;-w%`*cK7F{HV)mkZOCjk`Z$ ztr&P-TPlB|aP8@1wa4>7r^cs_VvjtPFKn5OpQ2BW0YoWgJ9rUJc<=|eq><6rVIusH z&Rob|!io;_2`b%ff1dyNeCTRDJv{&elU1#6Tq*x@uawRwCnx7PP=bl7*YpcVSV7t? zhJr%y5r&l+^3*Mi0v0T(qrQ&z&b?U@+QJ(OMFne)bZ0L)`9`YD64j*)^73Cb^~4*&w1 zAYCGq2PF2D&%S!4jp2q#9D}u!yR$bZA#UN$%AMF^vu{A11`QipuUK(5U9XaeM$@~W zmjy5?HAFA1QW0Rq3I-)H7tmH<1!XScxfEh=$*t~Zx^f>u1d#E@wY^-K8EsOyJEC48 zbsp5JjM;r^yA|yDsE;H^!duz-!z~kfvwQ;)J)J-hF2z~v)`!2lSiyP<^QRJ>6~jqy z$)r|nA}f|DLCnmhyu@1&8Az#MelLVOH+4uo`IbZtcu(!kSFmj7fqN1W!8*8 z=x2IPseF8t13+_-htW3^l@#cf)5a{RLtk!zkU^Q&P+p-(XwbuA&sm&&CPf$$e%eSW zyw9g)g1rz@#J(;|?s;|ErVK`v$(hOhBhZi9ERQbPhji`L+qWkWjf%o4@CT*p64+TP zs$4SX?EJ*1P_66Tvqp)`J4yS@y zmVs+Ff%$}xxkIL3Jcke5VddS~y%W;Yr$AL5^=TUvW9tU(t9-Uv=51#8u)>+e}s&(5he1~R|;V~mUz2sA6f4bkD-KE9*29e{l zXML^gvTmmXA52+Nhic#<(Y%NEuj(r>S-n6@n8r@{K&qEheorVSkRrVISPid?H`Ly9rbWqoKix<%!Kf_+np$H=9jaB&{J7)Rb zC3Pc`g0T}sXhlN27!=y-`(=-as#s^_)=V2&+Xxj6bySnGH%a?D<-Rnp+>bBfNY$Ni zCr4mEiZ`5~_d5AN^{Ol}&xvAFhAEY~D63LPc=p)q!7P*frEFAj z>&D>AKm*Y=FPQoJSHxbZv271H5Lu;WbXpO37zxJe2!%hdBRu=Qi4(yr*O zbYLVo(eSjaJ;M#fXV+BrB3G;Th_Srh3B*v;y=foSU!#4E)vH!L5Cz1SOvigdAj?np zb6>|-KJ3c>R;aYrjZy#Wyy-YFS;HeqhQoE%r%lDFs&2LYYVz|@s$A7^P{z(n#_n`^ zuFl+Nqd}!k(!Z%CP+B_2MfoZW3u7W3H(&dhRCv#zZ69w@V^Y_MZj)y9v2s~(VL_St zLx;uBvo;@LV`CGNq5kg{zuViN7ZgjRe=2*7B}9Quk!3;M<2UHXl$*c57Eu#&UK!Mw zGHvnqG9&eGdg|@B+OvWq%qB(1vfqD=SSZ3dfz{2c_j{ulR zs?8Bh_JcYnCd5Bbw_SE0zij<4(yeLF z2cYIleLJde(%JTP#x`Qkc%`6CG1>60a$ML-Z+#7?Rf zLG&879(Y7xDZVb1^9$Q-QBhJ#qhpMY77dBzEOanvPY=I8EC?U`c!a};tDi)eOInil zrQf3aFD_=`e*h$%rd-Lg>svBEQ59FTy+-i)Rj6zVvQ@#K3~m9OzN2ZQDPlF2Z|VScQJbVF|QTZ8v~k*OrGs3eE#8v|=*gJeF#| zv@w_jZbUb**COgBur{=+WJY@Ly~n%CKwG-EC5OrFhPI#g@D1_^`p1XZlv3{Wv5M>C z;l@b|9_X@Fv7<(f0>$W-_(8+9Cu38;1Jpl4=%vpHi)Go8Gtr$9`B8$Mq(BvdDkMw3 zHB9aO?q{O>C(m4Oj0QTH*d>i>O2#$Dt} zgw52kKIJ3IMYiA+$Ega%?D6au37q@EFDhtF`W`fMRx_2kFCuU%)PKCS3t8n3+1KQ~K zcmxkRW>>nSmb*_EM(6FTNtyRy#4(AxVfGAlW9R!MOAFp$?qPKbN@g$0^BH?6q^k+ zWuA92%$`DEoCr<`jX2M%P|8Y#nK!GaS9`sLnO?ly3S-4Lc8H~uc#PJn=h_oTORZj6 z!Kiqb`qrD-8{Fgpt2YN^4~6a`CM^SxEmz;QVXp<~0y=t!A}g7Hf`t<#>nKcm{g+M%@|a<#7F2nb#w{~>F6gxM zVB{Tm)LyP@ECl{BBdF8UOWaiG*)pe4`dJal~G1Sr5PgMn14~}2^*&+FH`rw4j zYsqPLCWu3RL|v)NcMlSEBUdI1EVu3ndhW5Jd*n{S>kG-StZ`*=yCCe=yWW6z*2HBY zjZ7a<^+f=Th0ctg$~F$l9s3trs3m4l2M=J*>NEj5g)7V>X6mm$g(UJ85KpBPWm9RX zq20nh6IIbiVuNUx&TXXKXvXwODIszk4fq8&R*?Tg0;|ON@%Cx+U+gTj|-6mKPV)RbP%M-6e@1o z*TN_%)i2psKGBd>8lBWTVw9uCV3*@4ImU@T@4c$81ip?LZ*W|DbvkRa9U+lei zRF!Af?u*6}W1NK;T_h4VNCx zN1s|B-et;$BTH#kwP~>4*EbWb^eCng<9I!ov`a|(thLVl{LY^J-fDI5}^E)Mwj5|ly^7N!XB z6YW?nWC95m7c%np+Aqgq?JOt!l5DX)ZS9}dCv2$-&G$=o!D|nscu!sA0jfX-rcX&r zcGzz8d$IA-gi3|+;mr^l1}Zfp0{ouoYBy$sBOiL#g=L!XD~Kv2@=Z+P6b|%+@b7|C zzX=a6Cu;ecru(qe$?jJZKKDSVDZ+PTB*LCUD+<*A+VbrH;ZO#dbDFh=r{IfxE7A>%{oWD&9Ubko9 z>A9R0iy!WR#6f0u`HyC5lA$}aTSgHpzTov1alVA8s0jt`4$rPUnB9)v5_L|#p*EI( zwTwNlFGAjNfsh+_EVa}L9md5AvPxhKMXGz?-$X*^ zzc3G!mJbKWHld(DqvXExzQ1vg&3=A%ko$`(-4`xevhriG!+-57`P)+@Vuf?K;XQ_( z94D;=iWlmQ6mU{8$*IaVpp4#zYNM~?cLUA{*2%e1NV~APxyKO!4AjEyXMw5IqI=O- zE7`xCw=|{FK{*>XX)-hSUtwf&Y*4!*xrKB=1T5T(+L)V96k-wG3#htFPoF(oNOGjq zG9czWxkws_dxAuHaixP4&m! zBk4oU)l*6lh7^tD{f$_~=QtA0N!A5-S{zo(8}H|HE5i0g67h2~L%PqjWvUD~Q{eqr zh~CT+wb39UnD_B)9DT`u+yZq|*eUl?0*nYLsLInRtDb>UzM25SKK|ry(KPdB5pNr( z;8xnq6Esywc^GG^<}`|+A!MXZXZbjQ5ET~|e){aAG=%)?jQi~WR`{4?yL2_I-`T9D zQ{Qvir5ePsiISVH#OG2~+!f?4{l2wQ)v`qN@9mqYDHySa^&w9~lpp_4_Q@}xu;`DnA%h{2q9Kx^ zZrqLi`Fg42%!I;L@D_wbuf<(jBcuEZSTaPCmm+v2Rft37s;EET{&Y2;UWqD>0(BAtw)H-_n_vN^ljko+qUEX_x7-Vo=N%#C34#ei-V(g6mlhq$ray{u$oD zSn;qOQ~UkEl)hhK_TF0g5EW`r)X28_oI;~%hV3!XV$G8c-M8x9p4!cj9ZVZcp_@YI zq-q3|r1dSzG*ogs=&pQ&QJW0_>Jmv0T>*83jloM!iMz&4{u2#k+9_4th9E(9?yKn(ex>QI{^p0Tk?LXL!a3i3Lc{PL%h!I+ZUISCo_o557bk zW#7gAcU`q{ln>Qud%X zKekkxQYCEAT=@6YFj_C_bV&BU73>|M+n@oOmu`eNTVNX}VrM3qr?em;on}yeN5UO6 ze|w5xH~Zd+Xi7FoXpvc5zjtqf5Co~@qDxg)>M9lhi+p>o4BAtt^O+&J3c<_jf?jkB z*s~7)a)mkxbrU#8k!9NP1TYIYS$?|%51dA(Mny9f@-R{CS-Lbi;D?R~F&`&@IN0=t zdzN7;Bw~E8tjt}zcLTPz%E`%b6a<^r;HeM4T9CwTu{T zDHqUYmFJSDAwBPE`yvBAO4kHv?j^busrC*4d5rV)P-`GUi{SyVSOh^`{gGB)3}?bD z2qn_V1{DZ6WD?F|B%1Ra>_Bwvu|ra+QG_~ie8#ra?5UjscsPYaNaTBn zlEl#u*<^6uzUAYqE=$@9QY^E23rKY&tBM9skv~qZW%(^wy!xOy%-ymA1&%E*)#xEP zW72jCy`+NZlX<3%er1&z+;O=XZ1QXR<G6!1ueCAfhR{a zg_=~QqD2%r)a=`T{w`m62=%!fZ^CTFbg`oN!34rp?9zVn*E$Isq{jg)+L?2zDkYLW zCzSD>&acVzJ6W7l8BvaEc<}E>U7@=v`^mlf*574JnBwS2MQXH&|oY_1iIHC&Vc% zb;P!Xr97LL#CTB+qZC3qaq1syZsNT5r(;?M;8theIINnC7c^->pGIsAdEz4gEu7I5 z1isRlykkK%HIfp8T>da^9}ah5bq?^QG&7{h4+I#yBzJoV>M(G&-BAzFoM74tWd*Sv)UZHonjH8K|Y{sbdxJA9>uXGY}DYr(F3ER})ZVVPFEg8fy!pCw0 z%!)XFR|;PbRYHEBKIWh88ntdcfAYTVyS|B+tlBAvobpj$Euly3jBcg;i}&%Fpw&WP zenlaIE>q4Ip`H8GtW{a@^8Piti*&hM`5@pqlz+W5aGFUsf91*vVyJo>qF zqd;?_MB;_ioE&_s+n6<(@TfH^&VNI45I}A9$4x``ldK1-8mA!ic!|S7%@NsM%|4aC z9oDgUpl^p7oQ=(^K9~O~y>#?i?_|vTq(MX7wio81bZ3l`NGTJemjCwfd^A@&hrJ{5 zmj2j_nYnEr2TF&vgt0Or}4-qlL9yErHD#jWD3mm@|tK52NJH}ES5y*vttiMSN6ePEx zUzGoJWb4d_PLGU4#YCH2N+;Qk5nt%O3EzYh_zqSF7%qJ}{o)kmD%DCfI5TDL<4&Kd?!V;jx7TJ`ll z-|wsK@U9c<11?V>e<7u)q*&~?`tvF7za#LN2#Ct~>ZT5e8E~YmFeiKuo)Fm5M+LNjYOxa2jV@j96D3Bfgc>5-k zEo;Rg_mjQQm5bbFve%cTg4qD~E-&6!1Mhph1vvvBcwSQe(q2jv zdU1}DAP_AmRy)GTytSpH@)aK}{jz=%Njq1YVec!XpggtV%k4Pi3e4=+tJ^ z56p(#rjX?|V!5;uaG4&~-0{^*>D2x8uaJm5u35I1r^nng)V8;QdjuT-$%w zvGY$q^Lu&e!K7iGj%uk0I_b5k(Ov=rsfF{Zihu^L3FY0ZHC?8h@1&;Hag4ND{ z;{F+Fl57bMHuU+%PG~?wo+J3;h4Jl_d)V&1=0NQz!p)N>Ph7h-RjE0b z?vkd;RBEY48KF3eRtIwB|4<9MO-;}dv9p*V#Ux6vjEqf=0{Uc5Af_V275k5*?b_I| zcwD^UKuyb*ntC!|dJbzWEBFWH+g-acEjcH5AZy~z`Bf`gOxH%as4Gvj9Dn79_=i>F zPS%j>rTAMy{-7$7JVn@*7gnzl$|il8F`TOh!n6#^W{EUXmSSXs!oQ9B6StFzk(#hm zF=TmtLZJc;=-?c%Z>=;e$QJ|2G{hqeyz&)B8$#Ovs_ZM(3Ubml!b}q?BmsAWpWn&E zZ?Au&pWj`{G=Tvzh_HbJpmc-WSo5O3rSY(zfGjTSPXd!`kG?1^pn(5;I5}j)P4eceIzR|^6EY(zy!=``+u_s39;9 zy^D@2)nFElM3jON>%i4^iis0{tX^;ygT5Q8L9l{^F39#sA^9cCm)C#=m$KYRl}5oo zmh&0;shmH(Zff~z!PilD3DYC}^1uE?-q#TvLwcG}FOB}=H>TEr4oofn_O#77G-IY* zh>;!$+Ir}sBHSa}3?~E~g03lrrBNPJ4zqN>ZJPY2IsbpZuNQDg8_fD5EHQ=*`H>pmP#q2=;5B-yLM?3Ezj6m zS#?|X4ZFFLoNQx##;LTVurda=Qc>CtB>?DLJ;j4oo?F#Ftm;%rJhcCwVB9i9Pt%Vu zxII^SGYK1As0xf?eld^HW+JZjUI+d21EMbB)JDQN%|}s#p^sxSS@a9Cq6J5?YxyZZ znlSXn*P^@!4<1x^Q~uri##IbntInv&Q(Hd#FCX{^(#-$-ncAqK{(t@rsoWu%q~y zNLVDdxrUFG^H}9G{MD|hJ4SO9yh_`@PG0Gu6)kBB?P{d_*CV~KZj;s#Bu(VJ4n%q(gg&x*y3pg=jnQ7?MOP|6Hq>2Glyag-fq+>+oE10CN)A$7 z$hAw8y0C|KJ8$7cDP7NJ!C~9W888cihT#90oNqH)oYO#Pg5T;$2&4vTkiKB!N zpRoZ}Oz|MAk-{CKY80g(fot}+FQ_>#%J5>3KX>ouzY8$UpvghPdKS49ycqeFtsz>z znrk=OF_LHEZhi>(uK7W*JV z3Q0;tb@bwC%gKO`oPOF{+xTZdL6 zDB{A>H#F03r#Z1!_Jc^-IO*s3>{Y&TK!=7{`LXk6o}meoWSyh@t@m+fEP8m5ov`SU zKUddM5>(uuQ@dWh#>=;9pHQ}c&V>i`%`NY);1_WYG)(n;)UIo82ZTey2kUC`7w}~jN?^%-ZR2@|x{CXH!uP3-eIhLgX zAVl{x?8$ZdCz0-HkFq(YhZz3thr!66Xw_C7mA^_ZX62 z2Ub~N96ylp`l#|yug;AcAG83JNRAE?1;~RNLf0i*0QnB(=Z`(FihAE=A9o8B*ii7P zA8K(Bl_N-xAUT||sTBkQCe$2cVe{fghL|shi+&ry4CUA@Q6}@p6_gvWu~wl^-t3)Q91xunoPNxl9DoX zobrPF`x9tNp^(^UC>ZxiqJ|WMb!^#4yN+E4khNxPo3dKkoUeee9B2(mM{YuR#eK4` z(V%u>E(W?JK36QdFpf!dP&PX#LJlPv{lm-1Pv22#*e8T_aYo9#8NuslRf zfh?^tff>aEgRIg@esuexqafPBPwJmlDND+<1?Lucc{%W;c&vS7d>XC~d&%NO>>*Nh zkhGaT=E?&fP)nr8Y#bYafJ~`st1P;NwnJ4IK@gwKou07Gn2RxnfB6CfwemHkUaSj` zQ0_&pQJyn^eFOkVzf~WP_y6$dJI5PPuEPO%V5I70}oTsq9 zuvhD-mf~DecP=BlPsYq;)U9d@K5z-SCR&`kg+@^MrNm3hF}VPWagnxq7RB9#m$UWxXio@{9V#<$M~7_G|3b~RAPtWbV~s`Xjxv|v4-O`Qv4`jpMKky&<#K%?vfr?;Nv=B zv8VE^{BN=mhl}nR95T-N$`5kD>)M2FbU;ZW(2yNDx%|1Q>D&gE+6UWI2 zVv2-4qZyFmesj13gYxYlCs!0%EoEa-8y#P+F^3!2)UZQ;bx)D0IurD z0V23KPhiF-CW|fg0^n(+F%Qpl>*sp1H0BD~+jwY< z_kK!64Vaxax9mYTcg0b_^6Uwc{2a972O|Ms=JnCG?29f06;tne|glK2<)c`o@ zK;$wTVM1tK<+nOrrG~_jGWZ8wvHK7K6$h}C>zr&BPz$UjDF8#QKnN)cgGCFvd9C?3 zc+6-4!LtyvewvsszH=-IzoH9l&c&OQk51V58|jgp9+U;bXA+K>kW|=(ji3S$#O9D& z2!$P#tt(8bZv^K~iLYOIQz_eM3TbX< zejO^9Kt0k3%E@0X{9bd91;^ZJVZvrks-nvNKTN~Krg3M?=s2)t@|uWzrSc$tjd#5dXZ$K2 zQaOGAm_w43VDMDWO6O`0e;@xyAcK!B zku-7d;%@YjhgN}!ci3!SIRdg}=pJEtY!#5GEG3fJL?^I9PD#ipy21{rp3 z6dQfW%NJK1XaWfDC^^${IY8)kNSP+GaXM}$qleV_yOE$=n|}_#rn&p7M+;Zg{Gs6y zcWt(oaE+LkjrEiM8>^F~^#U?WaOdoct7><-8iEvi7RP-@PowTk?K4W>!FwV*JULje zU`JqJpbGo@+&)^53)9B%LYh=T9y*~nZf4bStpU$*TWG^uC$&Ya_F)-yUlG36OPV~y z+T#oJhYCeO?uTSW3_;k)oFcQg?ke6c{Y<99j$}6qRmyP%DU1-q$jv55aw3E+pds0h z)XSsE;8hW5F-}-3Rh;+m$0RBemru-p7(<_4q!6sDZBnHXUalu6C;we{9UE(Z3HXq_^EE)z+67PA-_bZorc;vprK2=mUIUse*LDu68;QCvagu@yXyTp_)W z7PX!FQ)46RY#bZJUy7H5*@7)BCVOmypT2r^2;gcq%UaqY61UmLvsfH*29H||;XvWJ z=DIjLQ%AH{pMmyrC`pK89REF5yqk-{b|5&Gf7&eJAwz(ZMoR?n@Qi0w=h$mv<c! zl_Z5HqtSrH=;Y+`DJ_K3?!>s=tt?`H%d|%OuE9sCFKAR)z3n0D!f>in9bZgt+D} zFaE47r-Wa5;dh3$UZo5gh~%|~Q;r#IoPefHtdUw-uAaz~Pp2?S?>Accm8sL*pVvFW zdJ@MRHpOkJT03#BvB77Pn~IoHL>6qXV6!*@mp%kSmsOP^wlq3Xxs6%mHMObS*sFBB z%CT6G6BvHHQDX>SRGV=gWzGcn)DVja@CV#>87bd;qzC!zSDZf8asaTtVhoa=rvwh8 z7TKce%wVtkReH%s@<{M*@wip02N<+!1$~EebR!| z-P(p;=wi@V(Q45Fju8^-E4^=GV{>-B4DwWoBs5|JL+wIU1teR4SJ9`3U8wx@;ltIN zU6x+|IZBI#_%8<JK@v5fb81lCvAmex}BjRf2)KkzmBlt|AX_fkuBcm{tlA!Pn>Tp zfSQ>%Y+9|*TCJaN>*vtqwZ=K?QyzYs55X6N6MfC_>KY)39z!; z((CK#WOKGO5%U14^&x~X@I%&0WTZQO@sI6F4^+HAr+1kjd1VAgH};WgFa| zZ&^&n=1KBF5FDK6?5P~@5Rb&%hHoz9s--pcuVYGH*Jkn5aBW2QHc98M8&rhW%QQL6P)0Z+@=^`ip`?5_%uBjw;}v~-3aZ4xqZX)O%I5$dCAH(yLT_Q zX+y{q&oz}BnThB;RLqe$2%;ewJxC`E9cTAEOsM-QA1T2* zsVSK!jn^v32b>vN)m%t$6pulTxOp`E)H=>`gEqmb-#5FTf+j!}Ddwg%;?8>j7Wo$r zSd8e3{y)6Hai1)2F6i>9G1j{=*^Pa~N6$YKyR|-)9?XKmL&s z!KOpOW83eIzz;Ud?FBpUc}n@CcnMb>XvXeUtr3q6=%KAN!srmvFI``LA=KVUc>hoMi0iyx;h*;p+>11+tM_O%S zS3&YZXtE^^ZzaR4Syjswq@i^XHZIX3QXD7TDLF|HttI-Rpb<~}?H{F5Z0+^gUhuoZ zi-PkAV?{UWXCLuF!L!thn)yCb_wxB_8X9 z?xw3F6qF^c#9ux)cBZH=RYv2=E>eI9%i^Ud@o7*8`o2f_EEZY9k>`gXqcj*YWJrP3 zoFGQRjw0z8TX|7=99IXJ&b7Y;N-X+I5cY$RM1+$_-2Zq#|H0|S!_bbU@oGd>P_tI8 zYb0LS_4lK3AaEXd1?a=X#ta&eKT|1j=h_qa1LGJ>kXQZ}#xQ9#fkO_J3B%zF`Jxo< zLdk%y8T=8-SfrTzz|?tep!7TjbkI!Os4Mgfv_%lCY{)fWs`S9h0Y$mUx2W}^VYT4F zGeS|gh}K6CM6u7sBrx4U+qlz-GDJQJK9<|Sg+bYxr)LQD0+(qXH27L1vO)lUfv3PI zB&(B&OG0Hqz>w^GTDvF5xn}Q^QGVCqujRlkLw^Bf=aJvSG?NEqF{iw8` z!9+kUb80oyR(r~G^)DM4)JAlD8Wo&7xQrI#F5rU;img&ZB^iEQi*hSJt8?{;>Fcte zw#tklwe_O8m%T74@2-9wD6rs_cO=%=l9ONbHT_2^(k6de_}Wp52ub@MwIAea_&TUf z3MjlZc)w%^4_oE;KWt6yPnx!OxAsSQ*PfdJ9}M#_zs$Z9=78FP224g(qpIqMRU33w zh~R$ZY#mKOD2Qj27w=k(KY*WwhS9=R+uTupV?ff|L5=iZqgSLB9gTh9vGY_YU;|&u z9pM_KX08UqS@kBxK<%vk2IZd#8JuCizYyau8vN-?R*_z-(|McXRiW0<;S)Pk#~Y%B zWBo^hqtgo*prpD5AEs5@)4^^Nw!y8iLwgrG;S zUq^vUT9|~9;d!d$!nFi3m_=vV_Y5MxVng_PKygH*8 z4-9$%{WB~$qD1W-($obLDGu~VN}T74q^IgW66)UipPJ|WC-&=4=%2b#F%<>h_F zE0^FX2X!2frD%wk?51Z$1U5v^d<}Qm(Ti$@h$MkAL$IL_8zc#_3SLu6t_>yu;5!*h zImU(j!?ApwTq3;y5polqzZ>kP2hd|k+gaK-Joo7^P{EfXK{Ab!ygoc$C%H%gRI+<= z#K`dv4rZfS{i&$8Fm&)p-on%;Sf0_Gj|WoT>e%Vh>858X6SKsdhqFP79poipO&>y% z0T0^|bC}x%JT4@)#;-6?B^KGTgsG79)gXbVWw2qPB^b`tQQw;X9v9JwiD#C$$bxMD ziT9d$FLyX}7}Q!FbH|ifTmb|@rQxb6=Nn&PK_Bapmk=whX6e`R`MtM zKgl(#O}+xd*)Dp(wxI3B#|U)KQ*om%lfEhvOfi~@w5;tvUVwALfbRCarIXPA72a%8K5Ff3Nc-IQFF2Y=P{fLS$ z0_fvCgSDigTIT6t(&ZYxq+09^3T%;eV%(%si`bHIfqn9h@VrkP2B{FE@l7y|6QEe^ z^iW{q=4gcP9x~0_v;X=of+rD&07z7uG~hvVWgez`NnSKB&!XZhC!7p4RDX@96+K?8 z&Ui**A%S2{CJo0vchQO9J(!Ahiuhmh=y+#+DPp_ac`u;57|j%T5(ddoP7Xb!qjR(> zD@~~Tj3&=0NK3%A*r_#^)n`n3DouADg8s(JlFhYAEsjcv6}Gk&(R6U{pA zT){E1WK?JdT9Me%%Vy*NtBDmXz9hrv@ifPPzKjxz4r7~m!E$rrQWQ5+$048g5)LxI zX)H@7dBi z115q#tvtHy;bZ$o^I9i+?fjw6-hYFmwy81L(|$m;1L^={EBny(hV8zu?x44!#}Ll| zw^l4ASqfDcv=ytc5z2V4fYT4YZvF8gYpVUDrEW zHYiAV&!o_`UUh%^NuWwG#XtLoRX&S;jQpG^{ONnFyHY=KG@=DwtVlS<6qATy8I!DM z$!$#sQ>G zXLX4IFsHjUvFw~gHJ~vyS;?js-F2Q6!?Tb>ZBlx+kjaHV*`({_QLInB<)2wRjas&x zyDYwZ!Nw3U1qI)t9S6dl@BiUO^<58Ug?2o6=y!xJ)eMC2aGM>oH?`n$n0 z9=|Y996G|t~qzC|QlG1MvJEXjzoxSmYhLe9F2os`&;5eYjay6Hq! zp+2Sij^@mDKfK8#?V7W0hH5osne5cUrF4G8TTonXk_?V{uN5O(#63Bl&EFC71m$8W zwoxOOy$g&SL<)5_U;6u$e-V@u?$0=UGa(riT_ew#{@{4cUHZs51q2pl#k|9u8I5!1 z_L53fvxGM&Gx~;YZt0$E%QFBAj9p=TpS@059^0Y>rVXBD&hhQ-O_QuPtlso?iX8Nu zN_$av=ElTxJi8y}pgBxiX9^UcO?UhJZV-p93v2XR@tKcX#!Qyo83FHMPDO0CJZ4(W z9FeD^j^Qr)5}MO(u-poc!al}J66&d|@Tac+K*hsWY$ydOCMim4@3(PB;+{SWpR~Ux zw2BmHikPa#T@F=Ut(3}RGku*4#CHWmZfEE8d7!qE41IY&!P z+*FAop==AG*UCW$Cc1U%iyDbxUX@=(HzQyoQvu}~M4m#eKzZr-q3JFse+!0e z!^Z!RJtv6Y96F(T>jnQNe3Jj%jOoN~+<@YqkVao8Y?z+RZbc4tHRkiV_j%F|5jzWw zuK(2k>_04mEX4bk)|tYW=`#gp=h-)4)Vb&Kz|p0I<(m7x z{L%XqywuI&=qNG+Kw2DD4jiZF9oe5HBqNj_n6O8kNP+HvF3&gO3siW>fkM>V=K#Q`zVmEu!^7t>@<+ZtK_ z`|odqs~2694pGc>e?ygB$V5>!b*%g2k3pfy=Ud;}-hCKh-I3@JeN0-7(9azA^k4PF za_pmD#{L}w(zzx1Dr(JR(s`GMB7UhNqEc7x3F?N*xG?yj`ZHl-Na1{9uTTuR9h~L= zDoXI36V?RZrg628s9X6?6c7PgHyiX&pZYj!+8x zVdQalWcmhg1UTmq)t=K%lBt7$kAlLCJ|TkPJ`3B8-PYo*h^!9KG-b$USq zo*aI8)R!$?CXHt?Zob>}cLU1Wy=cs4uA64isnd;|wvYN=@u;nbIoh9C5I1$1gqOm; zEn;MobsiRJ1(Q*dCd?hM5|i9C4r2DaqZA>)qq4@tBqK7Frz-e^HU>dB8hX#!mUF|!m`3ozEj53y2 zVAtZhqpJpWagru~IH}X;GG~$4s1!ZCFooRDk&HJEC&IA?-=XO}h`_Tjjtv0~3qJMy zZ9xJda?lmfWD_uqi1DIyF>`XqL6Z#XzM}D@GfzrhkJMS`68~Mo#&8y(RG9Kc8)AdF z;>c^d{=QD{ApaP%sQik~vQXyQ{Hw-5hw9#{&M@TI@*+o)NY25&;%Jx1p390SqOIyR zemU7#bf;{7!i};}WG~{L%6hVNpVFaC+Z#}p#A<-dU4-=9mfbvN$Pjy9n@-q2Jj6Vw zxPl3E6&EVM*a2;y=r{!|0#EvJJL>vaXX@Ujtl9=)SkiYAszSIVX={9(@1m^2VOg*AErii;AyfxvUP%(ea&c6UV z36^?FareihS|6VV+6jj*@nemCkACFWZj;D&1V2{SpPx%XH4(?Vkwl-?%U^RG_uzFi z;~_8RGqEU(5?Mw+{+8Y@O-41c?}QpvnEOnuw^;=htf#~?Ke`3MG?>FdE_w2#zbGA+ zGAe=td0Sb%_yiA43V24YK zUyA09;wVagA|8N_6(tWXlTOIxGN5M07D-EsKWFz31JF zn-k7no<$kv38vcMd8qf;mQWnPQjc`R5!j#V;a#>%Z)Pu(WW#c`b$`ZhX7Pw`7}v2H zg|1#^G>UzYF?RS)oZ9Hgk0y7}?=(4ntVW2tk)0TV!|u3493G)*rn`j*P=GqXJ>LQ* zA*WOZTvNTBL8Yu#MH&N{bF@1r+tQM}NvIK5yDl>Ie!&t3ZVKI8*eu@4>e9uxn^}&& z92SC*gN#H#ni2}1Xd`J~2*jS2ORyNn#h#eFhnWu2$s$G4S1f#FXgTQkRR*iDtQ=+X zFg91v!%Q#Uth+vitbP=o9E1qU;MmcS%qax=N(H)+#E|VO?BSfb^}EjM?s|3V-RFRX zOkv-+&~0U#z0efV?ca;JK`u`g?`Pr8F(XG-5 zqUmg-#S(SFPQ^rW(|eEKXa3SNtkNQHUay@R1G}%;Wf)z=kZ8QZ49{s)ux7?vu^J|L=%w~qWCuAzD)ExjD`niGx;Hle-WTPi19EQ}Ag*;X8otjC-a{G@La-=mLVQ8!G)io*FN zZJF>)xQjpwrE%9F@-(RYdW>dAzlL+d z-#KHi)5nIYPrP5`K-4T@bd$*1ImR{R3r-i)Xi^HyWcfZyd`I%GsAB>pkj@LVG}@gi z9+RD6lK25_kT@l@7v8WCz}VXQU%aXh%&bU;akP|da+b|KJE+(cYmV?iqmEHlNn$0G z%%WU2ypwzk#bd#v*lNz(K*+?|K|YGqUEgA29R{?G01F5J&{#)OBl^kWY=Ub9-3Ipw zni{mQ?%utd8zUkHP38qJFIi49o?_6h##$(R>N>XXaM(U1t*@^UQrsR)T@^(56WSCy zjM2UolJ&ATIs5#Ri|5x>t(94g>^%4B+_XKsCB5bM$<~xfhq=d%#_vv@;RvFmFGH=< zWPcOc#bK|kaqrR|QrMg;Pq&duY}5hkMr6^p-~>*vbyR90R}1zhe2?U&F5(e>4nu7B zl-U$BCs#O6)R3bbpz#>o!pW-P$M+?Nd*^xlN*N?U0lH8dl8EFpZf4A}7leXT?%3wj zq@$wjCr!OqA%4BH@*m8Ibt72H`@ejvrpGQmaI6@un9jIyklZDLu|A&fA1sR=t$YP_ zyI?>4p0m4!C<-(({qEff^UobPndf;uW|Dy6>?5a(RW0OHpVBYaNrhSUO5|B2w)veWKN?5-%t~mwAY2A)|KnKboh3%73LZOL z0hh_*{CvIn69pqnd+)3XmQp)P6RARgd@)}qoqUwtpxTo^fk1>Y{;v2zq}&?&H;vNz zDTc#piCWfFBFk=|5DPLMZA@q7>(MVwNYh|x)$?iP;XopDhlnM96my90^jwaitDA9$ zm!0!lPn>u&o?*5q!j0DVyvQAkLb}x6B?Vn`Mn8*Z2m>Q!2A~i{NAID4m=&oSJgJ0+ z+`E8PD?B}=zewe2rT=@VN>2sys+wk7WBnQ;o8;14Ic35*;5ZTIv52=&WsX*-lm8tD>E6_= z`%sE{jS7vwOky(G{EWUuB$p-XEm~NjoRhwfVE=0zb@{0*Qmjbf>5>X*RM_!iAZ&2or`pkAAO!-3w0{dWc0GTj#bq|Ekg|jN zEu9_N3BIbZIT$hjOY~vE^SE{l_s)XeDv2`(y50JWs>|emLTps}<}>MD3POj+@E8HK zz)WPEJcD{CYlNnV8|U~)L#z6GT2_)z^k884tS?feOP)c3*Ap3D+W- zQj~OR<#TLupIyc%2Z?bv>s_{`;*b3`Vey8t%}wg$Yw2n_ zU5yE_zY4F9edQb_5i%}!U_+QGbO!n)y%RPdt;{DDLm#R2Px!3tQ1h5fBjMzqagYpUDk<7Pi}?UfD;pGjc?+k86~T!`OBkYTjy{WHwpl zyG;Aj@$0Rut@e2=YT0#PVvm4Zs{VgJ-%<~izRKRbuT~eS{I%6VnZ z#TTvmyV8~Is&E_aU-YT*4N6r0J)x~t6|Y45pX*V5RsE*?&w}qY|79rLul`jit@dYH z{t^0*9=gt>pTEH!+W)++&Hr{^{phrKu7Zn>NBA=mMiB&;E+P>jL-%-cUUD|7)5MS6 zk?;YAEBB$^xieO~0CX0tr3yaBtS%YCq=a=2U|Xr7I9IR`_|oTn6I6b5ypqSaZr=Rk z@G|9TmYO>s&_1ZriE;wc{oGF|a06i!xEg#(GI22yA`wS_>hWZ?%+IA`KJ-(;!wVO_ zyQzGPcj%_7=UsZ=Bxx32;~EL00^V5f>3>d2V0N&`XF1uo((bEBTm=|ro>bHVbR_>% z59M~O-h6xIw{~+Pi&_Ky!Dxbc(JMeLxL#w_>C{rMeYLY2Jf@xMYv%o75&2Uy7ox+V z@P7Rb4XwagH^lXqG_ay#Rj&3&2UM3CIjRVlQi~S)d>#I89H>E9?oVs+ZkdP3PB05?McL#g~~E{v^>Z)P*+w7&7Bln^l*+8!Z_8xoTg-=LwRoY zob{qCu&}=X@{k3R-Ks&=EB@I&<$K^DldRgcYY&_AsSj~EcExTJm!XYtMG^}WLNtEu zeJNX?26QwTiuug~-)Bdo|5P630Eb5JYTYjt0?F*j5d%;E#0Q0t9db+C-bLQ$J<5C zNJ`iEa)pYR@V1@0gZ?fsPs%!)K3n9CM)w!YjdLGNs+<>KZvT*Qw*|Kc@;b4p6QrI# zM_HqO%C%YPaJAwfEl48y_^`-Iib9ndRQgr-N>SDe=rxX4fatw)k2CFL>_LF1Un@g^L$U=R&0x z&o*|it8W_Vos7N}J$7DRp12%8Sjet%q!lN&#%JHK?PUXegiJ?}-ZW9-$GV0T(kL{`%ygft{vaRGJIK2e&q9TY? zNlN??4dEV>MaqZxkL>^TgNtl%C=a)9Kd7YwH2zF2zFxQ}m%7 zr{Gj1Y*M28byhOZNGwdm2o1bxxRm$A_6BbnDl5sV{Qj<12^q}Y(5S?5C94v$dcAnH3~{DGvLWn)N&QkH!l^r)rF@|8u7+D@5XSNcgJ_z*aTRHcw} zg9LLFg`8;AM1Z)kScLrJkGcAYQ97~?F`MHiu3^X~Kn2P0A=;hw%(WzQO&HJ5pK?o` zPN~VLBcmqUUe@iu3#Y>L%K7=FoluA$;`UyGEZ(}WJrJz=fIbUd7Qd+&DpD+VEsuS{ zqt;+fR9`&%(kuvvbI4@e5djlOi%J+qEzL!t5=9@A7g zp=)_M*I+600yB_?9HuatKxw7unYcIYxpCG7a3LH_V1iBOp>-I20PrOh*%w}=_?U`W zryyq{upv8r_T-7EqdR2(^^b4!&(zQP@;&9rUM5r&N?(*jJX-0puBN>pbLDn@eW>+pw4rm1jGEZ9KNn>2Hg_A=XM1aRp3to{^^ z*LL)bOb@r4Kc*NMmzXt+?mZw;AXmSi!f*P`UE{c7C(a?z<-E21w6<2-Zwjfa?q@Vm zBdvTwWn#oUie56;!ls3$GjB!_D-L1`T-HqMl%V~UuAQ39R8FvO*BWB?zkMF1z107| zWsm>ouD>l&9i9KLR|@|(@A~oc*E%701ofvm(aPLoWKvUbF#*s=Q%g4(qIJ|tI4#(v zbkKo5w~}qp{1&ORl-?V@Dj)3;qF)dyV;n^7E;jd*{t_WQP=wSMwnbVUQlGm38j7Q| zscE|LPAYa1k&g7hoG9`a&(?mMd5PxryPu?o%q-dncvmu6MF<0O`jQNY!TRjH_|vd$ zijHf)-Fvafng!n?N4~T7*V=LnlfKx)7?e(-i~)j~w`{!j8{)s$ytbA((lYX96rS$V z5bDyU{@adx`~mThp+gxdenDi{W(O>lJA5Q+yJKIMF`vU}Ie~fwpC4Xhw8hx_h~A!8wnlDY^eZQG+S+6ePPWCTjRVja+K z$nlTdjamalRv7f{!?K6#t}OS0C1E_}e7pdSbl(!Q!x+FM21qi38rg*Yz)u||r+Y@P zHW29}ndhgg`T3Wc?nlmUJl}I!&fHt~C&jPtF*N+}x{mY0;(a7qhuAs$ ze6BY&dSt!U-0`$wz*?2sR5Pe}@FA!P$Mm;)bu!7@QzPG^SU2Cp&evs3wb;mzMV#9zQL)%&NqTh#K9C`H=HFlBvPh>5Z1bq zIstN7zI$kL{?p>}`_)P;>^hj7uT&+y*YmlRYY@;hd4+S!Lf!l?>TY@Yo6Gc;`7DgG z@qKS_(ZkP-uhvgG+J$Qx?7YKl!HTel#VQ_7pbDW)Vs3{+UAk0uD|;|yE_4h^&OIML zRK8FXk{(=>L+uG zz2Aoz1de^^rTc7!U*gQ723B+PEso05(i`LZ{*+6+<=(-)vhbj(1)HpfT8a{t8)|^8jPEt#$|({A zjbhBziVfdf#JY&K*KgC_+<+%CnuzY5wPG^>Iy`PEmykhfK0P)k6F$S)eDNlSdMzne z*|!6tU+7Mo|&{lR$cBeLV+0`mBLl}=@ zPa_99_@G9`7CQIW^PM~TEjaPMD$Y z6Uj6M7f^J`w{KBj=F}BSX4C{SYx&K)cbktsAVHS_sZ#NSfOWBn*0|8uua*t z5d*csPJ3$qPY3b3{t1Z2?5&KPX*xPBU-JNO)|2B{oykjM7JS>8>#FH2I0C9>W>!O| zs}2yP8uxPr$L0i%pn3d}0rR_B=|1Wv#vH_5S~xYC)1IC4Sf`#)n($0;1KE+;ld+ep z>1Kk?i3u^g?N-iSoqEDnm0@f$?^(J4#M7*tw$(i9iB?~_ufk>*$ehyUX1zKK*5j5h zqjf5OB}2DGra;gfZ@c-2^oh2^5rR+=P&rXRfQ+Cy~-q7|b-m zCrt0Pa|GZ>8i>xb#H%{yU1dZj%v>6k$uZ{e8grRnOwsZ>^s4uV_i&FGl<})@=b22R zSbXDA0Hm?heb#{ZvxFAeAQZ+OU-eQ`?+Z5r+ogwcJ-zWX7Pf#UsLP~&B08j1v|!Pw z0iQQAUV-6_VEU@FZv+tKNnB&BVqm)!M~^sEpFE2ZE~Aio@pQsCHfHmtg{&-C4w!_Q znRy#=lo^gMjIb+cNQH3$f&Shv95}eeCjqLvAy_3{EbE=UqNQ^IBX!u$;k81Bfmqn{ zQ{p*DKZ!hSbE5{O6EW0^k$_2=hk?;Z!QD7Bru*9ST6eY5^M2uNGE_`#c#cmg4%uU< zoW_YA3#^}>2wG>Z`#w>WMQ)IS1V|7w2F?JvD1i-Rf&vdMrRjc<8F6|QN;GfaN=B?l zTTJk*G;%?3Ne^)lMg{Xr(!BwQOQETYqC>v8oc>ZF%YpV%aO~w3)8t8w1;3WbjDj`E z$Y0J@jh<~EbiKA9z^wT5_?XeU9IXp>c-sO+eDfz9Y!=?MRRjD0z6+6 z`ins_WCilVv989GzZcdz7>mH5d~~dH^$;^M4Np3#}*W48J!EH~`(IcqBI1 zU&B9hh{eF*c0+!UY*F@_N^$f@O~XMO4tFvw#lCu3KF?bePQ0DC#V>4GxHs90^?u=< zOS6t80rw$TO!8n~7}J#kKFSbVV_5UUR_tr+ktdelKs{`h)pd;Vlvdgoc^7>iwj%XM|tNdY5%6xiDN@!*=a1sROFtdJae-EU9OV zFXKQGI}LCO*^xq)rm^*k+#Z%xNbIjQTi&F#$mY5WDk1)$PQw!x&qVGiygBAT2{K!|Uqey52CVlsl|!x^+` ztN#KNL^6%WvoQK(IpF4Oz+pbf!<}3;i%MaO#R_+2liF+zDld&FBeqlh0us?e%ww_v;N~Ky;I1_^ zx}}$R;oB7FG52RYy-fP20{5~Z4#7+lI<9P8=-vi{dEf`@JCd}_8P`QE3Aa$ApbE?3 zF8V>)s;s5%d;hI+!)QsPH+e2&9%9 zt}L!5B%EbKx!$fKe2pTU9k{ z)+~Tbo+))NVi)mABXBLWj!%f)u){2y0M@fo+R<^d(gxwu%lg_NNUZp=LHD4?Bw~c_ zzt#%L)s4y=&{++{2R#>a7{hw(&JdDyP`BGL;eWLR`7?8`*zSYHF&L zDTwPLr&9@V5uR=M3(d7lbZht)0j{|51bEw5yZH;sCZ5?1cudpBu(y{LHn-Ba0R_~d zws#0h$d~{^4vX1DGaO)i1j)#70RK%Yx?aC5MiCfUWLf#yg`WNS52VPePpp80d)$K~)rF0`&`No}Rp>&|XSEuPOkIbQjcGKK|_Y`#u zXrn1_6|Uh z$g@GE@~B{(`4|e|(Px&pvArL}eCE*wP_Y)V5=NY1Ve~mw0lMZg7m=s zkWe^&-a}07r1wIDkQC3W`jC2wdkCF}ponQgL8xD{31p%>QEfj&(0g%3^j|6HIWv|o zYOJ>9v$&+Zv4u>I-Iv;;sY@mWj#KT6i(fGE0%M@a2lof!`iT;-dtn@tVPuKSTwB}x zXywgTKXOuv{+FdwF42R)!f9fRksv*osmta+g%#q&Sx1+CTD19w55GE84_QH%R5*3a z&TAN5%nyi$Ok@BARk19R5olY8yR+u)e|N`bcfYv(Z0H>(LuC{RKe=yS&-4gK1Cd9O zH_vABv7WidpTz2M*PDEqkh%K0^sf9*{T|ML-3yCWn7Ir4Kc~~Hab5PEJLjJapJ4oQ z6mXeQUVe?1kI6jfbZro}fJk`gocJEyO^_i`m~oD6c2XW4?VdThF`S(uuKFYKCv!~v z++~)GxcyPe7rhcTHGY&1oG!WX%VUKa&mo+xuNwT=Q*Xdr85Dz8u8bv0dv*W*{T>WM zco4tbkz-Q4Pw0|XusVj?ygEzmK~AhHo4h8mGjU$DFg??gph(m zWB}+Il05tpMF|Jta;I4^nuWO0qhb6>?~okuC5`Ahjm=6!1%2dji)4w)7=)-rRLvqN z=F7+63{1~AlA>Fv!R~dE(l3|PwD&TYcq`eG6k*~{&SfcS2#RXXz45-S&!k`8=_o_6 zJUl#(_l66`LO4F>ZWoFsaR;ckxw%I_iL*43gK)(qGTR8pjf)ZHr@Fr7N&wOr4qVYI z{oR%#Hv2QES(IvD!JjhY>?3gR(}dKEQ?_^68T8!Ru9B8<@sJbk|HFc3GEP-#n72syL}*R4((2n-e_m)jA-wlGghSHzait9p=Jt&v7T>JkA57|uQI~@9Kvm* zF^@C^@)dS5Y-6c{)DEonxFrP+G76Meh7;#h-^H~BCiXRvF zz1{Qx@)$$t`^Zm?Z0onDjred!@Aa%6B!hIm_eCG6t8$W&8c-}Hdk}`Kc)y?&d)Blh z;KN^kjq9jC9H|RHZh*;9Aq{i*$V6dT=^9vYcbl?oHtW$p?}c1-2SQ042CvLu~jKfndto@;tM>~$~*mv*6_{|8!H!gV*r~3^ycGL;nFnV*I zoXD+DojrK);9%GQZwb0Bl)x>j5}%nh5<4*loS`ibn_rL!E_-A$eBIoZmaEOP@ZQUR{ z%u;P<37YkrtD_!?t&T4T^U`@A4wignZSjQIFGbIrfGEu4q(_1KoY!x={>bkJx`ZNf z*{P~#EGv$67o2XVuG%Km)M<1#cUJRkJCHCB)u>T={{>5tIA5bijj<0SCG+4k+`ZI# zKokRQr;=K@sG7mPb0oJ7Y(K2tocNQo(k?+rPV!tyemNWw^>FY=^l#?^Tvz;W)9>TkF*LkR@#X@vnakA=b=+n%YBi zgYz!h=R>w`0}|O0#Ji~8hso)JX{(Iu?f#HsyZ@2^L#nFO9jajXsiWZ*4J}L>%?+lK z#LT)Qo-Fb2a_3}SaU&4Rv#vNp-}h#pMvsni?#)qc2b$YX(|O#p#m{dXO@eT8I0x*- zH~AzL=tt)vgQ{zGuo{Bdw$0RGFw8rkOKa*foO3e;$())5!#9};o_9We`Y?Y`b^l;u zA^9K(f$v)rQHR8=f{rkoI5BU8F)Wab8ntVib#vBHGhX%Fa30SM%!1x#C$6RPY0d`j zSWSw=Fm!WSGmAKk8uamO^BXOVX{zlY4m&uwCOo^muKvlS`D};Rns~I`^R}Jx@4w13 zzqt4}RWnMoSk8@^yM8FOB91y$x1|J%UUv&j=>`bKoJ}u4?wWadRW|)}mF~LHgcF_W z)WPaIdi-=%GvWeWm6nm8!JCur+_@7^11xQGUo>SEC(9nLt=?kxQsIluY z1y2%lP_pNqnB|hw)CD|z_NMoMSHV=yL+6OEePK!)U0wgocnI}bj&pes?lRM=2Y6Sv zq^1oTHgsh@MV|oxY6gzaPln1yDJj4|%E##fo47l*yigBgQENqhX7y*O$}f09gd0An zy7ySH;%qBZim?rj&X{8a?f#n>an@l?7ehDBY0^?E{7}p8MK0WOf*@?0hpk&O3DmIr^ zo$DQto9nLEf>Me-@dbYxrB^gnrT^&kHYR`0*`CF26Vd8ov=S~iF84k}Mkmu}gUL>I zf`j+z?EYvW4kYs|qDvpP7?srYE!A`|F>I{ePai*i+(F0BtgXYN98rQ>nSwSs_*T#) z>;wo1Sk%8n!59Dae?r}^p{*s=+2q3A;4A8q%r5YQb$n?sB43uAISxfxHmkw+eH+s zsVs%0E=CPD!vqdlL6_K691kW}_BNS@SD0*1IH1>17xp!Ac$6Xv|DhTvZ;Owg0lwNv zezhIVyb|I7s9LDQ2zIvX8~$J2y^CAT>G%Ji!=N#PDNN2`7s;TAqGEDtD^iq1j53PI zu^dL^G)^UUMTrcO(t)HLqB4_`Q%)6uecq=iqV0pEu#xLiu}$pBZn}laFRpwa zSM-b*i0IBHA3||91Qp>h(8;r@R?-zNCGq0MvNW3%!`_?TM30yuOd6m%Ub%7@fM#kf zz(br_6@dRj!tcPCMfc~1DH4(CQS_T$k~pIC%umbUUrX5!g9$>XAK?LT+tM&TKK>;g zq_=+jb(F>@u5OR4A-+LE(HO8a^aVn%CM8`8|u#{4gXipjo<3e zO9B*A@QZWX(%LkT(j?Q;Q0{iDh)-Pud3BH5b&3?<)h{8reuv&uzw^x$_Q=+f9rEhw z6uN=#Y%y5^V8p9eZgKx09iF4(3{NmLvm$;&##_!vUecPWy(W>;`77?xr0uaY9F)jI z=OFKQ#Nd)&J1f;!y~VU8%hHgbnc`e4`)ybZi(P$qr$WDMdrKusvf+_rp1R;_4!VEb z-|~_?QnZtM=3*uNwj^r9am|{HO;o+}0nfoDw*=QZiC@X3uSCj(Zv&1bziOhoKyoeT zc)HEWW2A9&>6%e+<}+WE0#v#vLI&+xN={1iTf%Y86W45L<3D5P5ir$ zABR%^@5&pxpgf$Uvd@^SYT=_PAdG-}?`WL!xF0pgdD>8qyc^0pO>*KXG8|ki0MdkC zvWBqnQ%qr=h)0)*sOBxO57Roky}3pTc&MY5H;?ksMbdd0qaw%K(vWR@_R%<q_5~vaw?5%EDEP-@8rYJXU}Y|SV}nyW#FHb$}LSx;av)VxYK;Za#mos zLVyaIj2b-*;JUVEwKm;J&7@~)`Z)fLbpFB*d~X`%sd>?uOSU$F2E^lhXq2unuJpVr zl(;~tcQI~fBf>y-i1dws$ccZLRIPvkGCrB_Rw;xpMcFcNxmKm+4zkvNSPfxY970h) zb93kbb@Ep>2r=(~LEeYMHG zzj^y|@b#-Sdqbz7#c-Fd~OfAn#d_Az-C}R7)w| zH0|WAHCW#A7PpKlR@e~|Vl+5gzzHKs^P~mqj^~35=g-Sz9|yaC0TP7EVXmz^0|x{O zmo6n4?u{YHl1zH7WvG@sJd*th;il)sg$Sj$Njoz>Y@Q4f!*)4i@v4TNM?jSjm zxlF)dPL9%!>+PUdzx3XuK%?lnn-N_VK2s4ipw} ze%h1*Xk#Y;8ri{p#*PUih{~v6PDn86=y8BqVK$vEu1X+@j9Uk0HL&#Q#NWvOj~{4V zWz1gBBVlum!%F73^cphtQ_OmWtE7gH0Q7xWxXQnC<&D6e(bE|qPAvIHI>LF%!u)>w zT5XdeRzS80KxCMZa>vrQf57_+5w_OH$DS78<}@T~*s80ufPo*X>cuzHdy!~ZgezZT zoP)+)c>kd*+e_-fV8#O#3ZKotu9p)|H)E+vm@pYc4Xeb@i(iz=Hcd_M^1L+Mk8P^e8w1 z4XsvK~Y_R3evu%-P3n-o_m-UX@mX=%l=n9dAB`v!){g^>5?eBj)`%$rH0 zu~yav-W%M;U~~gZ%k&ove9vYdVZ5NYRIn1X*8fG|ztq)uNpbWS^`C!cQgIL|8adl@ z1B>(oNr21EWKQI{^XEG$$>WA){$@SiPm2b7~Myk9&vkXsAI?+LdWSrlbO~8p<{Lu1DWuHCgDF zH_CV7LQ^yD#I+v7*P6j!m)ZKZ^k1O>Uo8BgKR^VHp%nm$^usX^l83D9Z{ zl;EeJ)cvLvFoP};vgm~PS72I;hV83jgqor%w7AG}!YGwya6%#QoVyowvH_Q^p~@N{ z>L$A$sOd4zL_vLRH&Qg1G}d`HSdOE)^$ zibtUJJb{G72)twG$1yXZd2dhvXqL72HQuS$^K!^m{e4P9aU%2@dz|kJMUoOiy6@y_ zM!`9cR@bHL>NL}L_JjlzSklsvGJKaq8dgz(cuwhWA6Pc;KER$el!8)9QOaZC-q`)M z#-Otc-7{Ba2x$czgbhgA_#tiD-C88M(`|p< zKFL~L-4=rCPemxn=9M*HsW2u&N6IcTda}Ia>Kq82-R+(}|3#i)Aem$D?1cr%zh%~u z#Z95W)xVcpR8*u@U1jwMTh_4Mhnwc~FLQZEC2o+14_)Deb9%@=RulG}iAm)Xxw|HU z58fa-(Cjt0IC{w9Cyy`m^zbtvF<<>GV>7Scg(oI$+2Z6k{_oZqF`%_hHXh$TWtJiQ z<#>Ut++zsSl(cm|)!d-_c`GqD*);MP-Nn63Fq}m9JH1jeT4*PP7%z2Ubhz!2y$5GO zu=f5!OUw|sl|hs+!(w`?ZMy@n5*rqiU_xH>9gz~e%?Ed&^Y^}D=b_%AU)2q1YGWX5 z7?5BnH#DR#hvH5dOok_2<<{J-H@PsobsA80A71JYvz}sSx16lD&m`+@ydVX?A>Ja9 zNO+~pQg3I{ZX_@`m`z*r&)QCfP>vJnHo3t?J&`M068eFNXo#$uJRJ9u)Rd@JnC+=u? zDu1nl$0;jq5a-DCE?tN7+3@OJb=r-*8#yrsv4)sNk)8S_+@#<|MQK81uB6b;1x>6d z!}m`zJe@I+)gJJbGR~EJ$nm*!HHm32YaxSZXl;C%P(SF&p7ug}Nrv)2iS?C!X<&fp z37G1*$33e(ntKs~aw;?jf%<&qauM~?H>9tnG}7;M7ztl3jglPrd)%c5+x7H)bp=c? z2(6RyE+NaM{-P;ea-wzRTnY(T2He$joO(t8ce)r7h%bVeIX!!(;hI9-d4iq3W7{?v zR~5^aEXb{P{&9ODC2k3-E@76yes?Ph>7|k!OvbygwxTuW~Y&eXQwrM`D1E5{u(Sv5V+Bm2o_x8Y4}Tk*6B{P|IEU^P=0eASGDL-bO2C zV2#8WH@hav%N#2qep22IpLat`L>^)t1ItLx`RTzrsfEVv>KaWR{YXb4-C`6G7f6A6 z;U@sE=^#eLus6rR<76I~xVLB?Z4o{bkCYXhTrEuvMKnA3T8P0>yc#lUM)scWPuqb4L6fz}K?&40J!+%k#TP_3q z)1#!3ndB&v=EPP=8#b$ErJ=8*oh@A3cijLvkjtWJ1U}Bjbjq_jm=bv=PXujM8aI4A zm$8`@nopUc-a)sxoy%Ta?7xfG+VJ?C*!b*cjj33-92h=jY0iP^>OMf;Y^R_(Ct;w zlsp~8&_5}T3QF;AZ0qe1`C^WR>)erC(PmLm$CEsWV~^P~L;rWXB6=|mmUJwSdiD8z z^W==p$kSp7vM2BX;rMpzWY}&oKRyJakVMJ~RGzf};zKEBwdnCVTQ*)ojP#X?I`H=4 zgr>J(&!L!|kqs4u8%;K9srX6xfL5;irjsLAtg+wCEi_~iePoiz#( z(WEwd*m-1!XlM#L4}LrC$&)8_9Cb}hVciCMJ~o!YnIh#|gE2n(j*Lfn1Q6(osYJb_ zxnpm3QqJe4rSlaJ z>sP$KeL22R3QQg?C@0JfL3lpYN?aWmSZ9##-r2FU)65UsG}{))Qa|vo7W;D!&{&U^ zu=KLdWOsOi16uzlX={y?jZ!uVYRf}cU;X2#1DiK*4k3CMb}pw|SeCEU}sd6X@ucsr^U-|1hwSajH<<1-`aW&j=*f?L@=2dww;5 zZe(GN)j}!iduI#r!z(F9WvHq4L!IBF?Zv%vFmY}k(aU=g{OBU7Qxu^;butt_^I@(H zl*D%mU+}w?8tIy-P1c+-IUQm?WhIVFq#jVI6k4Ik)b5?Y{A~!398?b*&Dzn?(IUKQ zyFT&`1%J=H5D<`~xxOI{3Ek=!!P8=&gJ9eQ4D8yKl!{SE?u`^To7c^=D7AiSJwnof zne!5bu;!j^$XFhfo`_<75Rt%eXxML$jOyv5es(|W)SUm|agc(v9KZ9kg+&xdOYE4g%1A^sk~l`ry0j*5>nF(h@Txv)8NW-$tq24OOC)R51OWe5`>IEv-V|N# zJeRK32DSJ>N!dV8_k7ZsMT$@=<4bnnUquBt^`- z8jdU0qnee=iar#T7D_HX+!z9RBGlh%%3l9Q2_PrZL#hBte09?5zI-X@wwyGN_}{6# zUYd#`u@e*vZik2K13UR<$XKJiuKU8m3VLg5(z3shW8&&xLDnx8S2GU9$-aaeSo!aC zwSbNgri_7Hpb)d_{oKAp}e=4-V4Q($b1Erw#-QyF#`- zFwfi})=@^Sx++3}u`*5ew6wIO9U;-7c_-9Z&V?)slG4mBi!-86ilAh>san-y+qQ#g zBnY7)^Ll(dfIxz-mK<FI$V8rE&xo4b{q3Hb_4X#;M6rSQ zBbV5n?w^}z?4(|G)L$)X&iJ=vt%jtu>lXH4;q#g;#>_`+^<5t#@RPH2uc6ss)|qilhnnz_`0BPCv*R=7N*exb6;Km2 zVNl2x7NSO-h+4(P%$y=clKgoD&v^)UUs6`=`svqg!^jo~gVPE=5@2op`IEo6?}X8l z8zKrBV$6Paph@o)+LNHs7b-&GbH>JaJUDeYnjzbvU`ui!VFt&a$@syw`+$@#H9@~2 zO;~VbZ$TUgz4L0I9XSu#yL)b`IpfBSbc2r>D^Q3yn&*vqPd8Bg)si35L#36dceeQG zJ2?wMjwg?v!pB5Nh^UaxGEnCfTa1_3j@uc1A7g z@-5}G199eEO^p!`(SROYLm-$TT!cdig#me4B$Sd43!L6NTV5t!lm?=y%Io(d=y@dN zw|C_y1pkA$n-X7kr(LfTuYLDB2OBx=<&C<-4F&(hC4r=Onhb zQf`T99!a;d>hp$k59k`^%^aoXn@FtUJTf3cP)P3=NLL$8G~hym(Y!lJM-UHRS`xEb{+w|%^KyT;(<)#64e;KR(p~>0!a+%Um<}ReWni0 zEl#Co@7H`QB5)rk0$y1jw50v7xL8=)@u;QA2}aWbUdtd^iF#z_Os&(az6o=Wz&XkD z8rF-BQWgW|pTW|%!@yZHX%oH8M;43pKI2=Po+l0{RYc!L`U7k?AL{KGhvk~qQT|J; zV(;MxQUlzo2bU)VHf)1Oz70z%n|msySmRs}dN}a%1S3oGu5d!uNm(*h9*8h28-HkB z#%C#GYKPZa4z~1-BFc%vE_V#c=fiOIG5izIyb3T+ zgbLCeFNKVx!lbWM0aGDIz3X8t_KPNhA12R9wExJZUd}VbU?fO#E19gt#}fKXhJqs# zaa=^@7ES*3H$S%k=t6|!H53V%j+MwsF^5@V)^Ol-Xw(P&^laWf$4Uq4QmY*KTP$y( zj0H{BABjZ^8Cj?Dl0LiJ2hj{gA(P93#hpg*bb{c=SUL(~Y#f^YkHviON2~1}z=CVye9cf3=YlJ=tMnQ=JS*8WW!M#f< zO49Nh2^ih1Gf2HB`E^}o2pP5;wgnNrYb>yEsekUk zrb~4$&7LAIbuL7ww0g&nN+)uClwrpyh)U^!NxJqc7C@8*Rw~-oL39St`*nBvoTfGv zByP5Z%D4`i-xfhidfVJ@(Y$%z*{N&AOJMH+=Mxxy>i49B>zTmbU(>0CrTFQ<76%Q_ zhh(jIyWX$I$XUde!-TstkXmkAeliUn@0??$g5zTY93f|Z7+i@rJ}ZzAHh$b)ixcV^ zKlkWy$Wc>+Av;NrAHP}I)r0|0;wvMQkewX*?+$_>c=+4wfl8UA5)3{e!&fMDHHHr< z4y6@%J~)PSDDy@mIlIz%BXt}Uc;^ZW!v&{}Be%EG*5gFx_Bph+SLsuYqiCq|3E#PN z-=u7At`uc3ElM2-jO>Zwy=N9jBOy1<+uU65WqB>ZE{10V^bnc=YNx-^cIIQ}B7xUP^v=hlQRT0R9blv?% ztzGrFnY!?$-eze`=)m7gCT`4prI@1tk2$x#A+<2Cjl@QZdM|1udexroQU7_A%PXQ5 zHI3u!2tm4H z*VLKACS}!sTJueCDulM&UHu6}j%SxP^F6oY(Wcs>%06T@_LP)DagxXl*B_!i>+6>d ze^XdV+@Mw14tGN%^n@PzTdC2FP3($G=iUghCEnlXZj|XkK++Ql{X6ppK=L?E+=skI zCjdo1&jo4a32-8(>>z`i-2}^4wXgWSrM3ZRRH8`N!R4aKC=KF|e37;-!^{WHIVWM& z$#tiZFd+-bo}jMO0w6Q|X12Yya`R?V3=i&xfVKTvTfX5OD3qMgvsRYkLMq*+vL=)b z^wBi6Lh&rB6YfKatK_N=as_0BUX^|u$g9_M$kB;@lLnQU@YtT3B z?o(;mSh%myhXzDJl~nhc8S*h_^bvuX<;Icw5~M^(W>K?aRC~iUfX1}1X__)6^8_&V za{p$RN_kJ91LmFu=qC}IBmB&-%C`?YXCx=>(;j_u!DQ3EzTYWOI(Uec3Jh&-{yb7! zLBJIWxdSzh@k_+HO5kh`&Bi61RF(o-1hw)zl*o#;Z^KS1C+BNknf}&ZMMw@hBQ)1c`BJM zT0J*w;vQII zE67@|g^-l@x!p-^DP6`z5B&>8+({`5PmC8kY^mA%&s063=r2Qs^d*G?gy6#&abZ~f zGgz(aVgZ-Ip1DFfa<#fb7r^HLQ@L$zk+Bnq1YU&_nIeOv+XNFz=P=m5;d;Zh*9j9S zWxS}2fdDnzwfjSKe&m^R6rBjM?$3Kz#tLsRz33Rvz3xHfS1wNj-rMJI*C@_uynRae%G&(4PoCGub1zr39N z-gIo+@uxfe_p*y+XE8xR?MWiMvjYt;(myOuJ#P1H@^ZJW&5du6TVV_t=>C^4MJIRo zBnsT`>)xfSc zJy!Lif|UMe>7c^a@S5lBkAOeU()C6~_U>u|xZ^m!-x^>ncDGu3Pf3+X$4Z5>APS-y z&N;|y2q4X_q1gDNbd70_0APTQWA!C9BJH`t`~eNhYnf##{TbxW()S`K3p{r$NK-=h zmB^ojYk-ePo-zMEwm;(UhA#HO-Ok%@k>LnBinEM6l0$))Beu@iHy5*Ww6tfEi?(M| zwaHy8ECO1#xmz~B`iY360X&TEXx=4V-sw@eooMQG@$Obyrni2Iy8FHe2Y(wsjTSsv zC=U9{WphV8?(cYN6tJ2D;!arp_3~5L1KAwV&{T+4N|>-VU5&L#Cj?l8-d6tedPARk zr{;{VC%+W1o_fb)8t(gpg$$F+l=$+ZygY`SpujoJN<|+43)n}gP^c~0#ziZJ4tX9L z8dav^h9_|qxeSutSK$WZxOwuy6-1r!zoEo3HyFWK#hWT$V-m{5)Gu#G2*X9S#P3O3 z!4+eyqicDW1N(u3tPBN(*uqLPHQ`!k@`q86>6g{x2EX84u^(c0{3Vp04pOR0xJP7m zg1Q@v{|KpN2xkQ~3FyBdMsBT{Xz_3LmWbvGbvA{}yCR(&Z9%(4rWSh>zYYb^43)JG zkmR47U3@rfB1jNr5|Q*4iz7SSm%PpJz6MP!f8=|QQSiwO;o-EN5z*%5GL&X20cWqc zZ!$KNc6_BB3Ac1e)17jC!3^<_V&c)}=4x%0qqP!KX=18EQG7o0ag$j?kV425IHFAi zWax+yUA1okGXqQTp3v@zdx>!&<%o=B=LXE_*V~{; zP7Ft%7NWJ%I%@BYL8lNO=#7G4$i*s>lE8zlCZalNX@V-9uCF_jD_L%6E`Ft@TJfMX zWT_&fVk3Tit zIOa3IX_`PfXftC!f)D2c`VEord7wqtZH=bF+TG0t7+fwYnhm9}kBy3dL- zrK!yT^nw6OdF4W_QZS0`fh4{}*}m{EstB<2t5_g_1`Du(O;u&=sXe1+9%yv(t}>M> zlJxwzxF!hK#FSNPsGgfzkjyH?`BMIO8nBeo_FbJiOo*&FOA2#9#TI~sE2E-5~d_GLF{2m~2d0Fjm-@Yv%<0U#OFJ2VR>=qSqcCW!WptnG3c#z`O zQn%x|QHJAo;^@&$mo4{}pGmLo%iSdwFN!_<$Rh$s4dGR+VFg#nJ^iN@*R_17(gNj2 zFwLhFe<3W_S?YSJ^x4}{sQWvEB7zB2zB#o!ePO27^EcZ3jPEO3J{CWB6p^j*i1z7? zrb1xNSFdcj@>zasu~O_Jz~v;+t;FR?=CePXldiP<|2wx|cI_|U{=cvP+pIaOnv$mM zVZq4FDwrB3U!!`7#+L%^UvY{Gr#g{s1oT|T05)+Ucd;h=HS!%H-+G zy^ZCCeH)CqQrWbEmyXpR$^RqRAy^9F!^B(a?Q)o_cbd}A>V{1dwXVB@IaW)2Nu{KI zN7fn&-YMe`Whj%xMoGs2<)i4;yVd4@_Si~%B1B%pnX-~GRYVJh+RAuhceJVKT2csw zC2t1tNy0nn%QC7_-65jrGSpf~zH5WIG`T3FNfbmRo%ru}=d?)z&XTBDsid|d_&RwJ zAnXwCO=akk+ZJqV+nsD24B1jDYeh7sg*I#7kQl@IYKl>=JRZUkAt3@rLO=5(IK3PM zsHZ5hd_B4#BsKE_#x%k$4o@m7g|m+W7{|eXQ(6Qhq`^pwB_!BmPWuAh1UBd*S9eZb zI~}-p8evfiFVYQKFqS3^XnT%uE=4dGdNAY;5LPtU>lE}t+xM+c{49wtQsF2*E1@e` z5^-|Wnzw~Bz!?MbQo$Vvvoer(U46~8OYg7gGb(uxSE3+WykOSnza)JjxXwUnFbyEU z9@bxAFm@O}DT;}Rt(r4<|2&J`3T`K7+Is3t7@l+{^^l@*f;L$fVovFuQBr1-IM_EG-RV{`J z2|RhSa&T7O(eZ6L0pe6c;W3RmBbdV^4sjG&b>bCLKTK+G+5|+IB>;w8a$L;uAd^|w z)26SLG?CDJoK%o*Eh~8;?mjqcW5oXBJq~?8oEDi{Vy7dYnlvtwtW(C`-AVqs?KHZ1 z`S7C1hwP9ez`w|pbXKX+w{vT?8Y@%ws;IgJzj3;-wTG($?{;&+%IAEy$b zm(PZ$mIHBUT)ATyMiNA2Nv_ zFShA)Ke716NkzDNz+;V$O-tiG(uGoc+t^{ec=2M7b$XE|hY&>K*-g(P*SE{pVSZ&I zS;keo29EpMgo$tuEDE}g!q|G z{O2l|gZadKk%7tdAkafRL@faZZwvbZc)-W;H0VBpLo4X9ln*jS4Lk)%QI9DWY_nvK z0-QxST2_)Ul2M7(1U1D3!T1HH$REnFGRy=&bWlZ5!e7&JtuML0uQK|%+ze_Nhh)lK zO~a#;whWh>YssF)_PI@~n6CxIRwma6UpGA`&Hg5dWcK&6@c9SzsZ`M(b+Su3W z&NfxMi1tl7wEKBLtLFW@qQ@QXuXB6CLtwZ;?Ne_5FfPQTvgP33|5>-T?o<|&rr%TH zb}RJN+Yk3l%&NaGqGjF4ifEXqQmJG*q1dTY&|>tF%0+1o0^dF~J_h&dT zi9TdR%2Zq1!xS`q4Ge5O&o=g^p|Ik-UeA;igc=6?UsvpPw*cLeRm3aeuu%KmOsvha8UZqDMP+{76YTfs3-StLgM< zLvb!MFff2`+fCtFdhiSNeqSRadtMwJ;T=i;X3auJx|W%av$>FKR##$qgM#V714re1 zlVQdv;BT9#sYTgU6mI37EUMt+>lzv5R(5&Yap#o}Wx3ZYJAMPE&^k4GLl(Ujv^V(q z{Z#4LsKgN5E3Foy<3nET{NtNhis3pL85to*j`T`13V%a_`6NM3tnwuRxUJ(WyUQgf zZso3+U`%C|;MhNs=#g3B-SU8_lu>Y4E0x<<-ne(T7_H#Pu(Xwxh^QZX7c6>`AYurm z?(pG{fR3*%zR7#^8LqP`A|gUd`4S0W>yU~C%Nw^?yd=}1pxGhR(}xl_0V4`9H^tHM z^}!j_r_X1$iieA5>p~;J^W2uK^tw#+S{UcziQVJb{^l zr^##?vy$KxCps4M=XdVh8K;__y7qk0en1bt<-_YcPZAgbMszK`du4Iz^%sV81ehPx zFy;MC^jfcb;Sqz5rf!);k+9iv;lfx$Lqp~~M{V5LRQkF2r7}h#m@NX_&1essyR(Pg zMF-knxu7WZ(i`s-l1u)?I>hh?5~QE{P6A=+(h(`|N#{F$)XE!&=NoGUp_w0GOz-SIeRwrU(@8iNFBQVo^PF-=i!#h{h zUuuhm;J)iGaCSK;WPrcwZ3*m$QDR=z2Y4D(bKRJ3vA^Sq|L^HZ0!52j=5zP!=CUAsGXK`zS5P%NIe4{vaS)IDT4?Yni0 z>*9H?CkuZqv2D{#^`-%uuY104ovxjGPMhzmk&rKS<2Mfq z-Jt#ZPu#d+E$&^a_8mIBxTH6G>ePs+sHp3y0d6fZdfDZArOHhnTjz3R4M=E<4+Vh)9e$0PIpJGKpUzVG_s!>JY~x26NHxAIBYOLAK7 zx4Qt%HE4Tg-udf6h4+=Ltm_rt9$>WVPQEHB5&x%kee6dl^P|9r_CYL9W51#bOA z7G6H(6n6o0j2Fw!524HI`mHoR)wB7Uq1-qehOM!a0H-f`HBA~bGj;9ibT1yLDgvf! zOaI0hEZQpun2mUi8y9_U!C_%Bm}2R#2>_Epj)073kr2j30Y@9yTv*sU0W=# zT6O;+gX>5fmR+eHeK4k9c;Y4i?#(OKzcm@rYK3`;cZ!H;Tg=R&)GJ=|eK%LZUQ&|U zikOU75Na^$^l6zBHrdH(KZ=l^N)dOjyb0ELsln!eu`PLl;E(J+`^l64p}rI1u@2Jx zqDQ#T$LUI9VBi_#0b=k-XV5C>=<2rk+HuUF zha2C4GFYi(h@zOgai2{cJ9dk7?Fs<|0Qjl4MOMwV4wfJp!3Pe+atR7igb?B`_&6KP z3wuSX*6BpYdzv<;wnv9^CD;=trn0xV{A9Xtzkvhu`R*Fp>KA|tHOaq{kzuJ4|H{QF zljQuQkKA;xbSi61VKqdC($?YpzCN|XtVcM zd&^El=oiaVEM=(}PRf?v4_ib9v>iOD3*edr@g=IY?zqHs^oZ zhaifwmOiBPoY&`Hg=1_8nZGs@4+`5w{v@^eeW!Wx?go7MD{!v)=i+AE-7KhRE^}|u z4ptRyu%vhtjaNmfW`F$6=hE!ftzvL8_#R7N?^`(%gEBG-t{roaWy_W|y+t82S+XC# z@-ykEI_cgZqrnb&O*(!LKdrP!M%)jdrY~ov^gU!`zRStEyDn*%HfxqA+U*WY#Bt0T zd(rtE9REZv$Xkv5Y;8W96tZC{jP&)ZYv1tjeXe||`eT{>D>RU)M~VswIJEN7MiT~f z0M83`ZEker=uyO`bJOdnUx%q~K)(89?n5qDwKjt+qp=Ng9<_qGhd?-KwFwCcM%Snw zwO4?PX*U-o4AYCJDV0hYflS}Xam)!2pNt{xv?JLBM|tVf@kka0YZ47($08#74j3@0 z&DPNXI+J-0zzRn>p2&Xw9Z|F(Ni2i`&G&j(^D^$MuwFvF&66po|ZTdoGz$HQt9u0{n}md zed(@V^WpVj?)84y`N^FPz9YP~58Zf1@yqj_SL$)*NNB+-{M+MOh$?J~><1D-o%%wj zdq=s`#b(tDY$XP?!su4xLp|rsr6iIow7xThHv+LD5_F@UkKtem^dl%c)l6ONa^tR| zixc{W({(}`3R<_GLagxO_j*v>T)TDa<8-mrAe>C38gk&u?Am6l=lK^MFQJ{bPp%2T z*$%1TNoq7Z>4;!w(0YCpDY594n&dYR{Ov)GC!s?=vB*?d0NDQ5yGFdd%_phvzI@fC zW%fGe=Cec=1rsfkN(h}I`4T|b#pQ_LQs8L0=Wb20F*`>4k6l~SDqoZX9XfP4Mz&ur zg$7Sk(U7pu;A;hloRq}z*^FQpioU~L^+VF})+EqA8c*Z=IN zPK}~$m#M}pI5GAP4)VLjITA=if9DkeEc3jr8=uT%tJH6G9>VfUTiep536wi$*p_0L zNZBC3ImgDiz@Uj*gYC!bAcbn978z#tnA~Z-8DalaAk~A3`&81SA=Zwi%nTpb6%v?g5$`m0(iIfaw zj*?lS#M%z-=l#~b-tWKf`(v%;_dIpS<+{%EIQL`Uw|(2T{Rq@Nw09{pFEd3^OZTbo z)}|=N^%OAVHb1Tbj z;*yf$lA8q2+1p#&$w^39{MReQt!&Rq*tXD4;a!-l)sNdz6gLO?Kh4?b25pLlqW0}p z)^)!Bm%*x>P(v6>3 zR$MjE(t4DvzE?>}>EToPVSyrf#gCtT6U4ud%lfZ$!s;#6K9V*2+5@75Lh|M%il4!6>f8>{cDYHeM0wl2yyB7#f!;N6FV z*8STuEI-Q?xc^yE=sKwuuNL%-|4>w3etzf2$0z$gq;d1^uzWOVebskzxc$_JRJ|=b zc2r(px$PPMvR>onEHm}Phbv>0uXJ>D#Ne%#uPOGQ9Bkzd4i0Wv=06rtnK{+9O|8De zjs>5s^+g@e<{3X7xsIfDLP9i6=?1Fj&WU^}a%Wk-e0jKp(dx}7o_RfZ@Sw%^aI`{s ztg1ift$ph_A|oS9+S>SfYi{o?^IK|i>eS6sO9Yy>yZ^R7cI;U4n_{}c2DKa>6K#fsmXo1o`}p{XM7*!BzkKJ;O1#-sX^WPUh6e3HxyBEpnY`Q28;^9n z)TlArx-x2~i}yg&leJs7a!_OTL$AuqUzU}rUQalYeg0f)&f}Ipx*@A~S^xTR{Jxw6 z^Dft4wI7~ez2_|xJgxPqsQ_<&y)S!3-f4)U`1ts?oO>6e zay(?_zLt~0nV(amr}$)T>8LHIDwc$c>ZNL$iY^&*oqGE~UefdxqlLfDI$Ku>-K6BV zdk(s#R!S;*wr}0K)p~NsfNt^PuAc)-^{CM7ZGL(-&uMtIsi|q`{rlG&686WRd2{#Nmjc-X2M#pe z5=$<&qt7&=5yZW7OBn8Uc9yBFt$i1xO#96I{bG*5hwrLA57liZdti^+{O$;9D{#^2 z|5ErOCO&>Gr8%ur?BQ-Q{`HNDnwp>XbeTVE+UDColWcpdOS{Ui`S$mp%g%JuZwgVI zjkNe-RvpexF?!C7?oH2?A9!w9Oy_gs2lbk}M43zM+0aF@qyr@w&h zCY^MBSsJgA;o`X!>IxPOQWfVvq%u)H7Ogp4TBqqRv^+CkM460sm5aVVnQQr=`>*oh z!-sj~9Up~;hllscQH;ATO=ybr4GoFge{0wyay;vy_LsK%@|l^LrFk~pbh~!#O1QLS z=~7?zwQ9+m@X@8aR%q4fQ1x}mN1v>|D{XN#%dBS2 z`t|fr3=6$qypUh5;1=A=~nyElFjn@g;Skmk;A}8lkbfw$;UEHFIzf!NL~22ghS%-`}q!eA8t3OW_o*C+5kMCxh>d zGh6LHd-`;M)u^wZpJ{831tSyF<$L$;RXDgkQVZf`o}2vjt73Y*FE&2@%5#T-IOQvg zKjm6QpIJ3IS6Wt9pLuHUPS06BIXSs=?M1tIr7tqBUAvZ!p1u>8FXQk%ygDn0ce~1M zaRckw8J8bz`O9(ZH6NZ|i^mp!xC{R%`!hLga{9DSW23HxziM>oOdE?{XMUvO^z@$< zoSZ(}&c3B6TOhwjT@@#vVqjtIaAJ}j7Dz6;+*2pC?zJ;xV)lEeq|gB;gAq!w{G)!f>= zy#20uQ~HCBfq}HGMs98eBzTzB)YKMHLgM0wgbZ%7Ga&K#c+O7m714_Gp_I04*}}GS z*RF^$v&U%{o3pSnYa=Dk=PIvSyFT>m-R&2IDYmb*ch*MUyNCV0c5m?MlVA7qZS2(f zHD0HP@4URikoM;0`__g#ckYySyu4CAP~X}r-{g?oJvg}ezP!_+fRGSwlqp2C?RIUI zGiMUt#qD9ihE364--*JZ;nL91;PXD|aN6TjbF+UJjhMW;bNIM2U*$}PI@5GtSJxVf z?YYyi`0B&K;h~<iDoM@xC9rCo^1(Ykr|`qY$jETVA{rEFwm zbS&rMosrqUt`RQ7pI>Z{k(EW#>MZjYOX!;%)>r;tWrCW`Y>r?j?@1{kC%eRk z*PW=(x4D_3QV~ zS?=cM?syxui$zRK?7&^AeL`=_%4mnrzGN054S`t<3JvECZzu^uLEZEekbTQmE? z7QOiVEFEoa>*+DG0_V|1`}Xb2$jR}0`gGfmA3w4-bFW%;4QXpFG8A$w0Es_}#EEC; zXbcPtBFq$m<#+AkN63}qDY0{N`~MnjjW>AV$UD5Yqim+*%aVYLv1 zURSQr%GvkFIQE2?)!afZ=fAResW-lf$gv*n+Ed{1m$#y#;#|wKji~SueHLM;9sGbg z*APW%{F{|UBf79-oku!oI5;>o6A!GEviiaY;3PWUx549JLQ)cT6+h)=Qs&p2zZ!Qo zGu0jPsn}DI>iG6P32$p2HBcj3{PSXak%3#;_{6vOqV1wmTec_xMKpe_&$TiN-*lXV zRY2htiuYJ{-LG&WN$s@qMmwM6A+H%DsvpqfYrt})OgkYHZNW14;EF$9kxb*o(MqJAj zlELWA^tgUGZO@F8c<~=D>d2#GG!%f?<}+{ar-YulaN%0^g{C;G_M$xZKhokxMN2kp zNLsX+)lGH*r_igl7Oo-b?CR6Yv<%D?9kM+Vrgpb2u21Fs`P9;^wqTizOFz#`&f$T7 z_{g1d#)JpKp%EdPU@P8&ct~-ht*xs&-2X22O0`QA4WnMFE_>VO&wAB4?cs*TaXdyoQq<7{`|Shv15_iv&WyFF_ZUylAdmv zK4!+n$+>Q#zh2eM2Jt9d?Dy%h( z~PectQ$OB;Y;h6rdI>p)1)&!UQue7wZzJ`UDQfg7X zCEWfD=4zNR|G8A3Q{?`KneqV=XYm$vyRmkC&rids+vDP5M~@zj1q9PD!+MG8zi`~# zy5k3S4*mZ9`y&RVw{Pb{c8cmQ+PIAQyP*hUi!3+2kZ0wnPH1bfm27i{$(3X6NGn_o zr?=iQHZ?^epZz@`5z+g;p@9xpLgSTT(_Nwe#0Xh9G?b0xUrO*gpX}D7KmYvsBeq#A zpx92f%CdMdJ$<0LxVX4z&He;!8>xIi4z6?4j}-j>fk|WE9BhPrkO&ouuIEdVz}TD{0ZGP)d(aWOw}d zas9^n{apa;KqZ{Nel%Z+i4k;mb{4tr#>T_L2xLnT1M*NtcD8S9tdNR|iuF*NEP8`D zB(S_oKSfbw6SrC_Dk>t7SrJP-GS-5rdTEQ@-T&%`+z17JLnFC}LY1PY&b8%9@=Bef zuvi4YAsnyQk_V+1@8|Es-QGV;l!t=sAiL z3knJ-H<=ulJ7Ome3EvE~`gP~tJ??uu#!5>|N6=hRO1NZ-LO)!I-wKV43_$&gcmMNC zHrQ|xg$}kPaQTLgZ{Ln5OJ2AsTv`9-&C<@!PSP=@>ZW)zbBPKfixm|W<++Rt{~B%& zdHq^l|G7OAMSsGA3)rE_!bN%^S^>kb=T$AO-S}4@D~kZNKo5^P{C08@-D}!`PiielCw5?5F=-`?#N*TLTY-E)Tve2O0W zLY~Uub>H_pTMCiABC9%d#CL<@;g+2jI#9L{rx~N_CITj~L zkyf0YivOf^ylsz~TAat<-_~|^s?%e=<-IgCH0M5L2q7i=qa{2A+|AsJO|aHc{aC*3 zdXoF!seAnSWg#Ok-(4Lc_9@S1EdZ}@qO6Qe$M2s5=bEz;w9783sLbQur)TTBlA=Al z{r&x`^Wx*;G#{VXP97!bgtz0rqIbG*NJ>g(WMr@vmr!2y@q169YFpdbyex1T=j=ic zuuaZs==z&`JJxR8$OK?s32Yzn{iBq?NUW9{y5oSWS5>e{Y&uKnb)Og-9JsBK5V_N3 zYtF@HZ=~9IEV9PZR}>o`-*wbCexS*xDz(KkV=Nwj26x=Yhf5e0aSpsac)c0;i9LxX z#o&dsxM4x^mfd53^zTx1HsCrIuT{IQ>AI^|+T)->zAe4iS5@_5ATHF+6?4&HVV9GW zMaUgtn~|+E`*+ePT-@M9w_Mg%Ny(RO`F7ib6*r&@W%tYlUO;HsAXoVAL@c+vPx6zf zK(a}Fy#Gt=_o|y40)m6pz!|Y@dK-5ux?ASL$4x()pE4Xdazw)%MY+HE>4u#q-(op(_6y2*<#eY{|6-ADw^O0yru@~v7s;$+P3qH>E@n_f9uYO2Ny{QL|*!8{j z4Gi3uUQ=FCVTuj$?*02RB$CEM-8zw3FD!d|j4q|!Xm@ond@D~O?~vA9@MhJ!EVKRh zW$hNPSh2$Ncf$VL@6a+mOiWC@?uteTOo&CxvzPd`@ws!!p;7nm-%skh8>es^^&|PQ zon&#P*7BC;MJ)R@G?JDT2s zg%B|++Np0~VE2LM#37Uu&_c9RBh4zGCyt_9s1r-=d1ho(;7rq@T6y_LRN3I*AlDeT zE^5d>t+qp5GXQdDr@%DZAuPxLO& z?ai|t%FD|M1OUKgyZu1cPEE+u=0Ooy5AZq>y~sM7DK=#4m~df%IL{I}*L{Z$tpIqZqe>B{@u9*3Zwpb(L~9a3I-u>;ly7^Q*cxZ~ zdoc>tXx>ie5hb%`)U>NE=>{*3#EkYmbekSSuZQi>{PiZQ|NYO0-`w5qOCEh%@K&?x z9v(YOzuYg+%x^~KYQ>{_jK!ueWmTLJtc}`r5@QNU_MeG}`4GIr;U?J=u!m zd|a<=YO8>q>8aBG`uKG!-x+Fq<_s^|M`)^V6 zRP^w{{?0UV&mYr(Z?g@Qeo}rxPoF_wrtt?@u}ax zQ)hNQJy*Xbp|7)aR{r*TkH4dp0`whax>n6$o+}XZV(-R|yD$5PO!HkkPiLo$4t3DKUjD^Y~rd>EN*(F7aN z`1s^evl9>G94Nw~GuK1|c-)pST0~U@txC!ndQv>gi_%_+tbxWpb_)kB^)Ml!6F(%N zzpJ}DEGkMo5lrsu(_0JNra4eHkEFgp_l{fpcya9qMi!Pb^jOr4)%V4+)32ErmhqJU z^}j=Cxjh!!FlMfS{T%NYU~uZxDf@wsOYiNlsz8a?TQ=8<0FN+3Wzoz&uZPtU<*)M= z6sV8G0)i90T)ng1ZG5T5ZSf#*C8eJ~^FKbBd!$}1d zi_nHyBlEXCcVHn!9FH)|J^nXRooIjVf%algiP@p@Oe7Ft^KnV<8IX?I1F^b0` z>idT@f86QUl;G0mhR+=buWBb~^llUuwCbBgmzU`|=b?+5QW?g~tmD}9xD5>UC~|$u z0kLFP7t1<%CnqP$3m5^wBF?rDS-k@3RyR@a@d++Ka~b=7Hnot|kF5n>hmi2=GE7FR zHJ`NdidaCtoXqg}dx6@qWq^BjblYhCI|-3v zSSY8juWws0t5@H%Qgv2#&5OEiXfbtNFG3TDw>=%=JtaRWdGOZ-Bt#1DA=vu-`t|FS z8_)EHZQGV#x^#(NTOqCPTfQ%nS8_(?ygCxbdwf-Umb8@TMevF0NE34s(RD8b*9i-Y zmfv`QpzOr{bel5K(D=nLp>O5=S!Ea zPAeYa8Qrj9!?kgl8u^vI4*`A8uNpx$yoUE&z5mvx%L{5t_<$Vq-GchnQ(GEG@so6^ z?bRruHd7-eZ7-ZyUpNkGw{8>~C=^c`7(eBkU5F@Df<$6xAg)*`xo#cZa^VABAS9ia zrp67ej#v%^PR*@yl#^f{xHOEX6ZOl<5XgHOUPg^{)@(U-s(a1J$x5kwOV-splXG-b zJcfH+b`7JUqNmQw`_gzle!O|prcG?z+)3fm4&QYNK*u#-!MBTke;0dpp=lYp0!S%# zGhLbw5J6+E5{e6Y$^$8eE_~sE|wCcD# zXWE+R^SjZD#HuQL)SQioUg(`9mM+M1DPccZZg+GTXHP3c-7`>#c2W-YG>Q2(^AP#7 zVx+UIv#ZP7$YYwGcJY$ecBeeFrt&fkmmgm>GdtDI8dUvS-ia!KL{YBRbd^gi4s3Ep zX6EITloXk)SNOINqxY0!f2Bgyuv{*katXjCjhpj@gL+re1sUHpHSI-cgJKr+(9+`R z`P10c#0@Af`?c=5;?wGy(M^nZ9lwMnahuIRUI0s^bm`J_wY5>8qu<4Tdf_C2-Z%uA z>q6seqfc2yKU>fW^$!e)^@gdhgW4fK^!bH9`2IE9x9fGX?96{;6!c1ty^7x+aT^Cr|Y710B#g`}dEG%h>$lku!R@0Bh#^))^i3F1cMRq&sEI}l#Tfe>&Em-p* zE=?Ye<;%+@xNOIz2apLfkUv4en;?WE72GeG_$G6W#h!I=a3Dk{0>NMC5P%?x9TOEP zd1~>Ibp4dZE%(C0w9Xk>R=YSm3xaE=tbcyjLr1Z_bw(b=yzvmzn0pI)l%OKU%&aWY zXUE1=-kTrl`F1z>n5O2Y&6`Jm|GWS(HAm_Dg@%VrWLXiYyZ7vg)d#%RwlYK`5m4Ov z=~MEXvllLi&Ca4SM?B!j8d`f${#2Ejd)ZUL>~r;Tn&0)D_;R5<;+F}J2Y4*HmpYzl zx+lFT?|0p;jXPS|1ZEe=TsXK2+%R5=7`#HDgSgi75{j*z-LDHp92?;{d9f7A| zB))=SI{$WGU%}0GoevIEH)@7fe};}QH8s_Ny_)AXy%jV9J@y+}EhI+r;1G+q)BZSV zMem<3{~?2neYtg?~;r0biXm#}Vq5p6?Fz?T$tdh1GWZ*KxMBBp!dOJ9!@ zxclFp_r%Yi^-s?p&U^m63uw)G{OiFmkCiXn|2W#He{3sAxNz#!m5n-y#sG;^e<$0) z03??(Yl-y)&1$D`LfBSG(1?%{|1;95YO}i8$n&pW?>$qrjAD^WPcu(dE~5l^FSt(h z$4VQWIU|hX==Z{DxC;~*vsH%q?^{{I(@V-C~Pj}9Wu_-1BGg5zdeu_94A+U?JJj=+{3 z6Hjl|4N7fnk#QPV{cMB4k*16Z`c7I`HKdSG^;0+P$Y%8Ys`^K5NUwtg_cebee520T zM1t8NBkx_Fv-&?WJ_4(lHg6PCoE)3E4;*Uk7{ATO&+q(%qdQQt7PQ99wf$%rC5=lc zg1_cw|5&A~`EUGgml8AN+akx!GA5H6|9IeqAnC^-nxHE4(Lc zdr#oqkCpLg6e6W9c(-obmgzG7b zo?+a@nVIoA8Mmn|+B!Ofq9LvR;QcuIZ@9d!lf|TZ|+FLX&LZ?CQ#`?v5YRjKp?_XJggRaL=@3idj{Ra;+q0TwG zxCn`eFk-2WRCjlE#o25+{u4$FXpW`OJ4b;t^&>Wax7%1esbUd*&DVvm_v-IR&Tt~t zKtJnhT!hTsm`r;NY7vc_%n)&AK&#aKIzBR@`8s;l-JPdCwX~QLwkPuxgX2(JAgVeq zkz;>8w-rn3izpas0$oy8U(XJ(L5ilNZI61w&ZI*RRDdeyDNgzm3TYAx*iWQ=-cnQj z5zv#4Lbs;szpNDSwef4A%Xp$@+-@csuPvYle4jz;zxER`t8F$7QMVJhntkAZk^sdZ z5}Fx6>-UE;2#5*g0j->X=S=LVW1q04-p40K5)>Qh9>S$PXWeMLI(m8@=FDKbLuAX( z&)2?t?buF3ctE$>Z32r*^Oysa6v$r62{y|FcB6xNa7j5ku5MpE{ zX9EHleJ8*H27EhsS4t~&NAYN14=@WZ<=s2YP9Bd(NHbVYh*#&wXVAZQO*eyB-}xan z5Wi`&i){Sh>B7Ro@(U0OWHp+4u%H`1=0aZ2{t1}9ml8a1XX|>S(F-~rA>FW3h)`^R zh(2I2Ha-NeaUEs*tAGDhJn&5}MH|hQ*6TANKYcMn#XqMy8eV>=`G;xcOv`?j`Yj z`)@alPoG|Qt3BT?N#!?|#s>^(L}romsEV5~s9FPQ*FW3y_a02+$)I4i|Qamv7w~+ z7SI0P4@OA)&JJA46dWP1^03*s8d$nq)+_$eW1}N30@J?`6|~h^n3&k-c?=bR3cZgH z+Q)38%NFcSTAk|iHZIwz={`_rVVBqMzg6=7{c_~Oy-<4U!9N^3aYAKHm>{_A9adjB z{b93_R=X*D5bP=}SRmGTsZjhLG+tsQfKu6|UxWsz9NVcLz3|TRUGO|TGAdq;>he~{ zMK7!3@!L{g7B~`gj0behdR~X++4U_WtXqmsk}1j{`fD}No?+qPKA^?B%dc^d;C|6C z!oaYYA}mzV^w(9yEkuAiTJU$NdSZ;;UU+b%P~-A|8;|nX<_K0LSrRTD;c8bq2Q}Cj zm@*(Jh|527u@|0vCmLQrL0UK=K));nrd8d$mjQ{8v*yB$!}w2P2|!NsuM8202hC3K zD<2#vrl#It7>VSFyT$vJ<8d=j3?4X`yN4f^Fp{9r(o~@D?d<6A!WzVxmR}v2oU8=J z9(wE6Vmmv#A8-kkbbZuCayGWG2tusny?Al0FJ4ZOvyEeTmxLnU;w#cBsnH>fFiXdY~R+b7F z+w#9@)jmwiIo0UM3Ic0Onm4S2fI=Q635y?tgQM7Wgm?$(Oh-ZJWZibA_Vv$S=D)9- zSXu@n5{Sw*GCKMaStWq*)nHaH7Z&c)xV0$|nVpV~t^F*x zS2jD{e?K!eF^Sc!K>axO>|7Y}f@V)Pwds%JMI2zHQnxBV!|g=O>4}zwafCnE>7w0c zg?d*tE;okt(Kj3>Fa(wv@H0=c+>mvAQF5+Mj)V*g*?p~LfAskA2SYKUY{O2W_9rXv zbzLFY1R030!%+RJ>)O%h9-yV;n(aczO76j!I~=C(84s5lsE?mx~(BwJs2Ngz_EIfi?o6Yqi~!(6kD zgNaxgvQF^i<951A7OKLuY$Js z|6+g%@%LwdtHK+2>C~$LcF}>+@$oVQIWd1ld;H!v_#5cDb8v7eL{pMRPFLL^nfm<4 zM}bJ&shgep-trSEkho%=J=={4dzIULOWc6@&H+C(ZyRfe)AqGB=R?GUDFEEXwnh}g zcw59^DcT3b-!)`6<)cTrDM7|X7IWR3}_4YDGLQR1{6#j4adArF)A1ERpTCDv>N278R01l= zdYlVs0HR+RK8vMyX;+*!TO~UMa)9Vl1($v@FJ8PD50L}>d;rqUSyxYbThBKPNJ4lj z*q1)AIl+zK1BXu9eSsGu>9@Ofd+g+&x7rFuxO}aPCaxc#I(7vG1>QG->F4?H>gR?C zC|1CGM^VUqnl6WL)eIK$y>558o$W?UF(9cIOffI9Lks-Vp8KQHV(m#;Mw#6U!4%fa z5Okh#$aBp@H@ifiOsY6s?a~jg8L-(DJT^Z021Oa@uhH2c^-KTl8u&v|2S(A*g5xZM zGZ)20g;;uu++2d1Vy_C<@uXshbAr`VJ#b(NdQW3Z%VqNN@~|4|_x?2!FZS$Xwk=)+ zFFPHS!2n={vc^WvrMyx~D1uU}^EW@~RlNNld*^>l>U)e~C^!ap1+=(_Qf;HRhtT{J zp+EKLlVId8Rm5$p5WD#!tPrWZ;F1DS3eDyDetVDV&)ANxj6XLzPXUWe{>SB>%t#^~ z76*_%O2)>52$RQyp;v=oV(ma7n^_a2AUno>oY*R@tgPUVG};n%%}83|6z67zTw*5d zjf%>lT`<=F=725S7?6#8c=49kYoM;84K733F86c)-Gj9{H9Ip6xiKk6HCBSJu3TbO zwB*%g3--h@zXi<%@A;oEt=M<Qb4=wy3cX9pZ@)W5o67|R% zgb0L{kqaF9J4cy+A}5-2VPqf%b`ki53v^ppFNx8 z+uvvh-vNLnjn@N?FNI|wM=1)@|NLx=sN=#mq}=+$1?Q-HH$kHP*W^n5S+W7)gE$pD z4P$FCnzWLNibTe^`4Y-BPsoSefGT#n^_v&}e@0*wBp~V%w_y2P&FQT`xTHb&2c=#^ zt>{L*L%;ss^dW9_B4y!TwYNGA>7zwDFTVtDPQFCfpoe+|zpP#&(K#b`Sgl4T@ug*0 zW(>k)&niA_QTK+0t7qPgrV-9yc3xg)n7*K!alL)}7Vglacjll(=2>?x0u+%nDWyAo z=1dp79OQC{ACs7t0pyX$lZ}cMeoqcR(XE8XiA_Kt1j{5^@J5IjY9L8dpgPq6i#QvI zR0#GF9NQW>IbK}j?!rqyN$&)x)h7%4mi)Epn>TOTpmiX8o1dQ_ilvhOGJa!YJj(25%Q1qIvq@X}Ah3|t{d|!vHB{oq>PfTtQN*g^Ja%>C|JXBWY z^vR_#5Nn;9&xh5w(OlHtmtBrJbVXNGPkkGVB19#nXIY`NCQRhvXFwN7VzAFI#ng}& z>g5^{5%mMnvUb8`eg=z4d{`O}IB>_3c3%%LdQ-4Jp;s!cuGSE$L%%x;%YtvdB>bw9 zXKHyMz^z%kR@K}U$b1BZrHF&+JE=1+NyIU43_~MaG}@`2p!r09ty{?GTBAo{w@onH zaOi#shyib~B`KS_dwP5zdB;Mf5`9lr6d`R)eoHYe7fRUPU_NCuLffxO!s-C_qYO|) z6WlC3daEho_a!bFbP9xc=Bg&dAMTNcdUpPUMv9IK%D>0#^jWF|rW5AVtp@t~)_;FH zP$dwCGwi=E)5V+vV7bO6G-Dp8AV_4n__x?|vC&0Q(;MKI4r z-o3j9_G?0(Ld+qeF|_~E=4RO@Nm?&ZuGn7wuzQgp!6X+kZ%0Q95DyqkFn}0RA+8UI ze+4S=xyOcuyyoWSFlD2Yyr=z}AR0Visf(Z3U_kQ;3R()sJu#SrdSi#sq=s!oItAoD zjo6ET@Gw6hSG&zxFn2(jcd7)8Ncl3R z3m+f71o>7@ng}4r62GOZGAvr9@ht{2S(GXvExm&D?^stf0V&~@78aWiEgk*r)E-#V zyOa^#Q#o2hcno9TZG4I6ho{)m!b>J}Znri!lkcGW(um(t0Eag52w*R5as71>g(bO- zp}G7%C?{y(Bz>SjR0551saX?qB5?&qM)E{BEMM3+=4DVsGMqcd2={uqbKMCV3&*py(H zqC|p%ldqyrx-`*$86P?Eqj@Z}_ok&?oL%U&Ac_oGARbR*P%sFzxqN&h!i`y({Cy=5$zJE_;d*XHvV+gbqG0lBUH()_kkaWi;j#TvY2>#<2 z5TKqmPeUuSfkF_bUXS)Qrs$Zi1U{zmdItl2gu|ILXPCjL!ZQu;i?;g7OjF-ueHemC zl1@kd!n6hMRAiHrwKW?>Y~`R~zkD&AzsW^Z=ZoQ@nhzd?Bm?;X5&&{g`YDA=6UmW% zP>Mvy9~l;&sC^2|t6#o%p&;mn3LlQTf9j-HLPCNy*u0%jPGN_E&&A3z*T4}bggb{1 z897S-Lo-B3{5QNn%4~F?CbM~z5OK;8J=e)$Q4q=Po6vpf5VZ{?j1ZmWI_=Imoziz4%PSX zU;ci{k|m*c?_TYzi!Q6EP)oV!DEZ>W3t9@}s@JimDYe@mK-Avd&a(B+fkDTABA6?I zW4(qk=7X4P;36f~H zw%6;|jG&=v?S5cGRDkg#2^LKHN_c8??jS%}$)1G88hcVGxL?Fdm3X+4Y-?MVky}Nd zODUnwfRvlp3|(Dh8%25HP@-F{%nu7G)fr(OjC-NfAdj>T$t zQAQ_`BU0ZqC2Osvl+K?Qg)^BLpnUBSyV(M%m**Tr85tRs7Idvwel#ws%GgdLW4-&4 z>4<07`S0)SDH9{TBp;;P?%PHlRW{7N7`uz8d3tRc&wx^>y_^!9pDzop8@BMs!QT)N z$k$bksvGOM60Kf;2go5TE^Lg17`gcQxt`Epm{^IBPF&^4{@DW%M|MHlMf?*>fB;2^ zm0!I9_sJ|-U9_TqaPTt7E+N31r|%U*142uc|7dUtB1R2eJjN4g2{BVzs&e27tn3X^ z94?=KJ%ZOfG3&sI6TBdsX->Dl4K?5e{IDsR?@z-wz&X%03>2YzPvXLfk6UfOe@PF0 znw#qnTMJlyJdw+Q9HIp;1?QYlBk_qY&x3-38tw<3l~`6CdxE1 z28FJ^Z=aU@4P{5knq0{j?s5C%{(%xUgw{V0t-$a3_a`^e#R#0S#acuJ5s_P|5Y ziPDVG7tGSoOue(|MCpKVL`?Hp{6UgWTB9PKJNgwoD@2{)+Ph$qcF~4@Hady6;w5GY z%@kZ{%PP%m2FxVN=VDmU{CBT;iHtV-v&l#-qR@FX zZmQhv$e6_Y4?()gfpFpx#n$!LkC*VL2H`4Vfe6lz%gf0jY8PQQlly^A$p)ntUiKJJ zYt(GhMhr*OqDkUPQk%+qAf(uu&V*+%I{Bkz<);etN97M}Z|@{6!qU_HColu|=lpop zdsG!N9iHS+B`-i=Av{y?dt*LrvFMRUj1(cWk;jP@4>%&#{YodQmh+!qn<--~w5a?# zD1Vr%ls#;-Xs%TPkKp_`1}OsUG&mOS-<=tW(DN`jNY7#-)W;w#ix!77ctI^p(KiGd zq)MPvjL#g`hvEU78Nto4J_ZvHGeG7W>0-CtbgZnb3H3ZrXNM4p_@)r%B{8DAX~XSk zizXj%iJyLj8OWUsjI#xPDk4 z(}KYu9ceT#ZhC$9EUE_Mk|nywgm2%yo62@i6nbQp&UM?dyrD0J*9hFLeTwRvJ+OhS zAlj^8Snt-(8Uo^vI!&LoaEP``&`xP_yaIAh1-F92($)9304nKAm=(7;{y28}^l9Uc zlFKr-JpnB-$M~J?JC!6j8psY?J7w=Rh(t zP|4Z}R~z(iU`)TOpOJ`jCqgh1fz~IgPlhH2lBH>PE$G|PuWwfQPJj;VBx$ZmZXu5a zTx~!<{i?1T?7y|WJ?HzLcz)92!8Xa<+wlIqq{obNrbWjoygQk1p^U*8SvxrB-5G|I z-u2r5J1Q{@tn@J3DZwsM^k+yONlnwt&?Siuz8g|jFup?0BdFf_y+z1t#^OaHqMJ-| zpkk4_=tY{NIf$&UzO5&TRcx0918MxD)8YK=1`@%(D8*lyPQ!-8i?;yr3uJZ^2_CgI z#O-lSbMs0BIJ8S77A;=-SZo+PBxP-F4ysp+gNgxS;!(gTi&c9ta%vYYouurEi3#h$ z7UEoCL8dAP+~x9%t6d73|6+R)KdN9QRQG@MHna!49v&WKRtUdlY;GQi(-?L>zOt~2 z$uY!m2d!!Cc|+r=-IIa1W4%0c%b=Xwvwh+7RPC zP&H|LFlMBiK8YH78515+3hq~duR78C>?VmADj_)fwnA5Kh%B$rG{e#U?P9Jn=1-u) z5~DvtXN!HmCQ9{^m0NcU8KFbd%DH%a_ujo^wt`4ah+&o0XV0DmP}V)U;{nu za!5!>_GFu_x6HJPwo#_7y7@|35sAh+b-V^$eNLg-T6yUYs7#bIkL=f$K7Rd?4AN z1x*X@15akU;Rdys!GObh;QcaUlU_v z3d)gv|2rC3dOkgO5Hl?H2tr)IW3@u|ni*BaRvHO4F1qiqzN*4==%r*e6+xcC^$+GVsdhFaABD;38#+yn}vz^ zS*7SY@cp|foF!zA=sm8(4~3V8TD+7OR1ibL*x0YjxVLLM$5!;@T*z8_+}GE)69fWS z4{u*zdcyaB{OZQ0o?;=KqZ%CM+zW`iScRU4^+k=X{Y{^SczwT>^arMYd&`Z`R8kGOLZRhFdc^ z{MHYBOa+lEkY-z!{gd8`l<+TP*9J$PaM~PxWC47I<@DqZdaYg0NjsLyheKWBdes+J zrhh+9py9XhSK>8prI~*zyZL^P)@2K=(!w_u{`~)P2#_yyj+i!fIvTQrmc&Wiz?9(b zHT8}>3xU~e+Da4VnDN<*T$B2CM~$i7I2&gzKnCPO+;#u8Gw07UBPn64o%_$q&|j1n zY*0Xh7A~T-2*;;fPESwIDIu)ACp@)a06h=!#L>l!y#8OEVN1DYUIBVmh_)b8@DTm)VYh6WU=V_k{l1rQ$> zA8%q|0kLOc5vhzpuVfiaE@IhLbagqQSz`c!93EidKfm`LI{XM9VFDXKN5P0mY)>`y zZ0`94`k9&e;-Bk;g|q%Sc8P1%{EY!VLsz7w$VNs#_s;=Li;}EFe$gqD>P=;6T~6i+ zHS)*iWusHZ8_wTYCN3pUa{G4r5ff2{e;$9Pj!Ne3pI5G7g;uN;TB9^7ce-rhj3+Jz ze3+=Az5Q%cPxU{c_0LCSxAZ-F7S&~+sA!pA_eA;~*H{x6%G zkF(S-1U)v+0|j?p5arq@EYF<_fVBpM;Yti@u7!Ohn~i_|W%Yf7H~BY`S$>1(_Lsp3 zVTKgUzh?&&58@tyI!b^gTy`77Wj$tO$k;xrL~XR902&?;l?;SJ0tHQclIY{7#Z}8y z@rbef))J8*5zC>_GofcDua&fHlfleU9ARh7aAAw7V|ms>WaLE3vW*8)G#LbbhyD;`dN-g= zmD9?GDD=f(w3XR#c90LsZ49oO@&eZ^h|_>-*#GG?!g9cGow&FI1Bhsl@|;K4V2*J9 z1TMhPmSth{d!_&Bc$K3@s1sczNJt_QYz;xrGUbZ$2{=!SacK0V213#4>EZy=G!%UO zr;DmJM+uxl`2c*U#XLhLA_${qaf2E2Zz>Xv9T(xD8&-8%_EueO?WS7Iz~o3Vy;M$| zo#X>9jD|vKd(^wITl?)I;6Cd6wdBr$#H?LCIxa$gZvF_z2%Hyu>r@HGX{1EE;Xa|r zDL{CV5e==`7Z^Iq{12z--tF6}I|CFw>O>%t6l9N?kup|baUakQ@R?~nG2FGuuz&}p z`TT$zj>+(26$mCCQ;;vjk3fzof#-$MJ4Q<%Og?!&j~+d;?ygvBSPs_x$HQ~lG`**OMnDzO&flI!7< z2WfJSQ=Y_USPq6mOYxpxzkVfk(Q7=DFKdrwK_90(lWiH_xh7@#uBQkrUucZdw@NCdlC(#OR4Pi^IjkWgq$JOZakS zwvif%VyGuE7|~#k9v*Hl4k3aFJ`X+y5rgMaWRMt~w%7+doa2Q>CB87&qcHU#2m6@` zwj10G#xK3;$xuc9EIP$l0PHQ&(temsBFzit`_FSK&TqkuqDRy3YrCrH>V`)S4G*(o zu!)Co@Oj!rz|S284q#S(woqX`;Zu&gdcWbcOK z0Bv^wAPcb=Z;_N_ukrEsr=`d#4_R-q{Z#;c0LufeUAvNyU=~$~%6uh!Vv-jT3bI$tggmrvX5F>^c{TSL&%38jFWK!(dYV0x?Ul5yuu;{kt!QHXNZOWQY!HtIynusY{ zj7w5pu#Q)R2t0WwE)?z81u6Ad)@dfysm<2z7;a49!5U$}Ccs zDdvAyT0G#+XyI3LlL%;98tit!;%j(-y^WF;P9no2;seWhHNWWZoCy% z{4**$UgZeLkpr&Kd!lMW9lya>x+W~3GLek*J2_Q?Iv_JX2{Q<2;*uTx(Y);+xMOK) zNpcA)2)dQ@P;v-MB#wiCwQ3ah)ehitVgi;YN1o8i$$ghdLt-Xti!t&&pihv~mu9Ns zAQw>x!XeO*Fh{%!yHf>L>`xh_6zk)(ATsWhx&t>vq(nSyLM;Jd5cv;;pY!aD>@op` zO<#io17E^EiE;BSn84VH;KdI1$I)G6&MrZ1Ia$=WE{zY?ez4E%h4tp$6C=g`Z{K1u zA`QBHhqTp~N4|`K$$9x6f1iJ{METZC)!ir~FTWNlG0b)q5ZbxM2Jj1@))2i?IGr#I z+&KI^L_i=30_8JFs<#ZS7|~s@VY(myS75yfN=6tV?DzJva2yB5Dgh0T?ydrUNbCN0 zLYb=GXvq;mtRUbne#)RxR#sL&cyJHE8R2;`d~J-w-ts)>inI&SH)E%kV8WeDjpA57 zLfN37#O8TY8h#tk6DfxnV)0G1HXXW&^F?<$CUsXg5$CaXq)UT39tW+A zNI1mjnH3D;gPbD=`BnUfHOs3FvfAD=ZyeA>;-ESKmKB?naKj|up`LeRlB5i5!4}l% zQPmqzKB;8B;6sNL`ut{s#tOo#4tJL_a#7FA8L&gp^IlTbx-b(rfVa%u+HO1k#EeCZul( zP)>$maZQ9rgIYj_Zhf=RHIs%LiodA%y{%@($`j~e+YJhaS@jdG5*cuU-EDNvTd)JBY)C_@a2DtWi!62Hd9V}r z2(ZR2@5sT$O(t|&b0v@n7?6aCMFAQP9ktVKiVY1o$YGB09XK8e?X@qcxv4|!QzL~P zp+c(0XU-gX*cy5?`gD>HqMf3kN8>0UHFBES0cf`?@9wY)26x_#T}WyfI`Cs(yYJoA zhc=CuzA_v;-gNWc)`tdYdcajJ_hiMmJFM(Px_f5Px`GT)M~be+^bUkjBJzR@dsQ#l zlU1}Hj+_KK3cIvUPak=cxIr-W8S(abK13rLidbQxBJ!5_D`t)5oA%5trlS*yz~u5C z=mX)mu>A!}{Oj3!AfEzs*|Q3~2i`-`+5)KGc5yYS_F7?*SR;S21N-*9L>Nk5{IrG4 zApm?Uw->pO!UKN_?m==i2hq}TCF9#xvC~i(!`SRGyVwgO?c^LCG6@eH)B$meP)HUm z^HY-~SpcP>jSFwMJ$k%#8Ey=GnjhrLjrQvAUV+KUz#t|%(USj+=C2JYL<*x~8qb6p zv?^m}k5g{l>kqgOP>Ub(24et<;FE_Jf!p?}3kmr1e`o&;<8Z^PrBAD|on9S^#7*_L zWQ*Y_QXJys4G4krR5enTpSgH(6VC4-r|Q+~w(j0@P@&rM#0g$Plo2QeT7UpR;!uV8 zJhUC>&0>dd+_;hX0V`Y;5_ve54C6&hQNCV5 zujBmUCcNgxw(Z;ZVCQ6D=i+=xa!QE|(W;AFCoLg9bs|l`9;78Jr}Qz0*5$?J=XGt( zqrZNsfes)CG@+qLu3c*-|GLF8#=MnSW%NE)&3D0~%g{b8k&uuW-7t`>34enUe#nkP zYoNi9%M#ri4O0on;w3zq;U>#_199dQDY;1cown$Muz9XP-76i#mW~G- zcV2M^=iYyICDCFiGNzv}fgC&<6G4V)2+a(fN#-GX3!ET9(NJzbKOF7KDhQA5+NWSz zf@1Q@4C6myFm#O!=C+q&ei|P{2y8({%$EjLUjnwGlz#k(OF{W5g$smq$*9n;mTf*> z)!x>UWp?dTWX$)z?p4~BRZYtgU0ZkMwyKe}e`rBAeEoELWh?eZ7j_pAMc!4?YH+tuAq?p{8M`ccViNP=^c#(!_($peSu3 zn9rnT0*F&cR8)hHIl5mbGi9dq-|DQ3nOYLL~TJo=_}!^s-i-w0%8&rx6+UL&0*@Vu52@_<|KT5iGOZGF3u}LZ9w~CJTorV3G;W%! z;biuCoOV(W&HcQj=(@l=# zXqd)CQm4ZO(BhO5Xo6=!$O3|2VSAu);|Y`l_W`38+xXEGZF)rJfmM}|pCf%8)CY#6HU#GmhJcI;C+z1Y0u50i;a|Zac}pl9dbkJGiyR0=1bpahdFbeHwxZz2dYric`x!HlBVfQI zVBAr9!*7-K8#Z)92%kbpD!m*%= zE4re46>Bp2Gm3bhou6-Sb=3y)lI%*#x(gU5E~jK-!!_TV)0U2G3&$&r(+* zE{1*PTxm`LA&*%GCb%~3Kcpyg62ImJ@r`l36G5V!mnTnV1c#2UdbNOjnIB-L21d~C zQNk`&AB-e#XJ*E{+o+i_nUX+IB#!oB1b3VjB?#%bfId(w;zrEWh*(lENFFE3Pja5l zzZc0PxFt?~d!SscaTo_~&;t5;F>O{b%ml#DSjM#Qh5qNlgt>Er2f)9Ie4&pWE(v_p z(LlltW@rp0*Yy4O?7TS(1!UpHY9k4n4!~St{j{JrK|Zxh zCj|}|t*Du(UE&EB-iE{X_-p|I#WoX~9)CLhK4-!GPVPMvwrgJn!kA55KZIoRjP?S$-C@%0@!6wbtxYRtM$BSD z6+-RIc@HXJ`LS#kE>uUiw=D&mXLTN${S8%mF59&$9hFwpPC<*h>&}Xb@1B*6e04kw zyoeMHu=aqeSO4bc%xN?$i%Uw{s)uXe;1*N z{Ql*&j-s{vr+`FGy|x(*tZ+0aw;yN0%7}XP1f!$o5;z~3tK^*g`yIvUbYT!?Mq=77 z5(`9t0?PmkP6#!$`n;zfS>T0cZ6hmMKEyX00|MGJBop^lhCTo)DD;HF0t$SH`W%6g z3a##pIfY2G#HXowSHli8Q=4e(1ZHB$|1IjBe;CEv6-oST%q`f6c+_-P7jHnEIdiK} zdqKaoZqa$jCa^-jlZ}(pZmyMhK;YdpDW)uy5c3yqd(6{yJ#pd$0`;*n@-IP&fBDU~ z-zqZCrgX3Z(}_%x&cF4@uoStbau1{(V{lc3uQU=xMMYv3IsJ*2ubG@EnY0xO9JvWS zYU4F))->;C)P)@Nxc88XL3_CLW4XZbT1tX-f~ngv=LLSX)z`O;%t->l+JMRnh)8G) zxTRXphR0h@CqEgmR6s+fx6AVnI)UN0m^0_4$5TsQaD9(Jy%9LKFOX4inwf^cLbK+~ zISQl%c)DTj)`qx%v&{J_x~I(x+y{0F z6j(<=C81omKK)J8|Gh}_^+grcyXgVaXSVM32%fr3TU+CxabU~ZhPU*i_m>fw@ z0@M-h@4h8u%W<5wNF1*ew~?9XBA*bEv_c}UVieJzU@j#BIRu{{B@-pXHb&9&I#v8= zq2Y9K!zA5`s`(lXhV1gJ2loH1Rq;1O|$nwgGLpAYqt$9Oh7R7h8&rI2C>!1sXpjlbTPI02ruNWo1U^ zgQJh_=Vn9w_&SD)4+UN>D|fB@gMWTBWX}vL(P-}-rW+5dch}OWx3G!c?Nv^HM=t*F zhp=tBHZWIx*n+)A%V5#P)FiuvP)=#BN@W9ZE*?lE%I=fOq(cR1%OTCu>;3KP)Xekw z-+$McH^TeM-STz8--@OUmRV?w`6iBZ{UlQO)0c!j>o9VyA+JpXSK_l>1j(YdJ{h6Y zf1$t^gCcF?oBkVNrBqy8+%%=&e$YmSP_hX93K{m)?d@W?Cg^t4((*cB~jTFqow!xCp3zUlK z0v^_81ecWK;{8MFY&zEM3E?_eDT)H^#fv^Yk9fGax^96>$|#Xmy)RgpDftIPQHW#7 z>};UGkWj3)Xi;$bNb(2}>^mb4zVbX(9VzM%C2i9sAf|W3MxyE#bVKiHCz=yOHWRya za`z?MKf|qv7j3QPth7*_R>wd2lIoVWq|-CEMojv8{J#!u9OX*; zPImPcKI9SJnL+`cv!nCSmI{`SG*^fZ$VXvs6!7^c&9XInYWIA7x*2KJq+IjZ_Njm80($VC6X^ftgm3=R2~R~Hri zMeikZD_Vu31wvxBql4~|qPiMW5y>3cFwvb8_{6ohMMOkO_R3lH4K!M-)cH*Y|7iXP zJZL;s!Cd976NNBiSli zX09;0(z*&bpFUO>>I)IZpYG~9Vml3VUKC1?Hoh7{Nz&7cCQkd&x{AE^GFkxFEuaL5 zT)c0w(G~a7MdGj>d|>zXVB-71q{V|e%&yy3)p1-pnfAx60?*Z57y`7=Yp!oxKchv+P_{~c=8g4RHhm6!LQci~OQ)S)>XsK(zQvxA2A~&sJdNNegv!iEotr!pen6l&X3ttBNwSxn;;O(8wf;uY>QBla;hk&;f>k4s-Uk}3}-UrAU z;?NuwV<+$95|JKa(4Kl=6SVV_D?Z!)>CO$GcDQr~jK+E9dAPl!d%UShoCN=wR=Yi- z!oudq+@mgPlV=PU#z5Ct(#?^hBb%=YqpzW}qvEQoV7!SBvK#qS9fgS0=y&8*HC^dQ zO(a@riYFh}Uc{BEZoidbo(-Y>bU9O^!y*$(cUO%5*1L4`0pK>YwZsbHuXt0kheuot zLt07$N9wl0F}h~)zyb)v7cbO$9;u7boDFhl?wPd1yi@8*6Y5#!EaWbk-tXMLeIo@| zY`Rr|hdYry=AJ!&JX|~A7IOiZ3O)L#B?02c*L^5Vcyp6y<&z45qHN(16v7sNEqv3B zX_wn%j7VAqLB)_}+6IAd(Nl{2(xekj+mso#B4^{TK_X?8HhakH*w{LNJ5vsxW$FNE z(3V%XxlLEF&CUej)!#ogp%-+c~m6;43CUAaxG#(T4(PI2~QJJv%Bf0Lj zKEPb=8w+E8w&Y&V_K`hlC6j8)9ZE&z8{H0CsE7pUCQBh!s527hmEE`@n`O)?Nu7t*uk)P9HFFBJt$Zamy+w?rrQ*ECG(94vFwdEDN zwry+LX`DrQbLFi=^GixyU%{I(iFrP%QVY* zhxD$SJ;Vpxt4_T^r+$(*C6)_^Otdf1J_{*l%WIKnsmMZFj{4ecm-vv10od_8MjKEl z$0PoUyRwJKXa@!$MKg!+!0GamBLF!a2G}U7Dhj$G47vH>={V!B$Nepi|6?<)t5eUT zmsPjd)%keAO|@NYx*PDJ%`>gHN|+KaaF|o)Z5yXcoyNtN%xK)wGbDWLUYIY@E@&Hc zrqSu0%Xr1pYt!!Vu-|_C=!0%)0ewqv_FyIG!V)(DQDxBshOikKZBZ!yku;FaWK7E1 z)d-4G_G%rSj$cw6S=dw{JwwrfWLq)aYM;{0pR8h+jD zY^d?e1@^wxQ;zeV?ow@?UzFOZd6!|WWzdmoM-U~Nn@U&=$~q~ju<(o11CRmx>jw$G z4PR1mGLj<%)Luk{6zP#T-vX_a{&C_T&kv4`Xv;RDEC&q@|Mz+KrSnQWP=$9|X(9Pe zrgs^kfWe%*%3>A^a9&(-l!@Y+y==-aT1qVjlzx=sAd^|wm<;%K`#3#~@Fn7R=xjw` z;85yvI(oj}Yr}RG(InG`2{(@INHfn6W8?WuF>M*@`*87qOZC;_?dL8X-LU-2&P9nX zJc-E&;=KKn?HliQ+ZcXp#)iiCnHh+B%Kvie;3PL^Qn$qaAuh&K@wSUffAjXIN#Gci;2=AjeFB^TWMwC!Y$3H#2G5|6!uqa4+NV zYkwO#)I35r9%0afNBc(2AK#)`JUawlPKHyIu^y)SC{qt6gvhqDdbF1Gn1(YyO4-Bt z+(eNjCLl8FOcMj;TbEW(dU_(TDd(n5>(<>TY03Q**is?eH^6(mS8^;m12#$dYMwaE`Imuca%g%JYq!NiM`Q#A2z(A z+WSR}XU*&`AeP+DXZ$1O&+#c%w7v9IV6!?iKSmRPhAeyLh1pFf$Xaid+~{ ztu5n^7LN7!7N#UZ5qY>;9J0KFY(tt}SsTck$u5C5SiqT+l@6fuOzm#r`!NsL@csMu zVob$tl+}`v&1+M$$}U^I?yaZ9qJ@p+xatz+ky$JtSN~Sxo95+pvxd(NJ^f8XQ}gcQ z$KvlL+o2AJhYM>gPGvOTa_Wfe$f@_sRuqL7$c$((;g`?d`70FUhvv1f0AAcIS=A=| z1wZuOC+-YhNQ6SSB|r3)<*&E%fC)PDw9@$4;#C419`(ZL_uHKaS*GipZ7lVWJUHUe z+F##hURG^;;RCa^C_{FT25kVlXsH+P3TcKM^HPm~;>TAl7IigdN*>s?t<*;q`mDor=P&mgX16iFoBLI)*MA;ObUMRn@ctR5>d@U!zJ&J1KD>P8*GHD@+Zq{{N6<+`0$TKV=6tQ_o#HGl40o?ySm<HME@$rpTjfq2d@Rsp9FaPKh!EVN7<4GJ#)9b{#xKsm0IHP~bxv3gryyN+)q2;IARXZKe=% zP4o={c=8B*9F_YF_Q7lEh?M3nqh12_bb9zZQ17~Zeuj`Zy5c+%?P ziK#DR6X%~;nhlE}zV|%Y-u+I~-->>RA7{z*aY*qAfcv%j53k8MIGEMmgvR?NH-D!gi*Y4|<#4@r>nb|&I94;(8RR#l%AulhSIeP7q>M#QX zoxTCLI)+Yrxjs5|y5vb{qG9@D(od;pKrjlPX|Wrq6L1&L^TEU9Up754 zh)G%*v>QoZFYL}2&R((N!c>O_f9$M};BR5!7g&5nw=4cs;QV?w{vs~V!WDvPO|zJD z{ckPTPtVp%MfY+zqaN|;Pu-^kIyNeq(sSf;o$~T(%74W(*HK}|V6EgWitQz}G`cwP zDMT4c`OtEhTTyrUq#((Mc(#;?b_qLTNvBLRwmSYp4A+E;0HCO?c&0a&57(L0!XRxc zlwc7hsZA)aL{otMo)&oBnJb}`1u~T;zrnckoW;fWwl10e?d==N1Tn$nD{evu>YXy5 zX#{X2pcu{ckKew1YoHhe8JsS&X7bB(JGELChz8eH=K3t&i{KR;Ba&`r^U0P*PdG8q zo@ZP4`qe%B3KyuWNXQT0MKpgkFmxvHouC%L0Ht#qO<6z_(8)U5Z9R~^LLvWyA%6)c z*vO8?69d6XT>CX3zLX7Wu3%DTUby4V3PS}^oZW!Z?2p`+a@oJCC;eKQWgP!6VXZZ6 zCUwWD2~G@u4Vyj2RhDou-w0IR&NT1{Cl(i0VI3`&zyykUgCNhvEyNf^pf0%Wrlr>T zPiPy#7Iaz`{MO`uJubkYBo`jv=?R6TahcqpZ?~Ug38*077v7rh(Gw>U#pLI%1F%KK z$|RfeT_u2xPqKX$+GHFNru5RJ(XGWg01Hnor_o9(>(Q`tB_>vOq!$vlhUH+8+Op|S zUYV%&=x$^&)GSUJ(0^lvOwfsR7we`j46Xy94n`6*0Et6ir;fa;hXvlf)IBuxj^Ak? z`Gx7u(_I=I=8>_JN{F7vyA-(oC>T8yLtfcD_&CjHv4cY+en-9>p`-7E@{vR6i7LVL zF*hB@a?szE;$etD!J=cTtJX-~#(VJhrp&+C8u2fVLzW*gMU16om_Es=Mgqt@$JPGF zKWCBd_*ye(XR>2J26Exdk^9fIx-859_@{FSU!Te?(9?!M7d>2KJu zJ(~XWpI<&Jc2^vk9vO|PM#+$(1nLhctrJsX;LZ0fp{m7@s17##7kAoGIY~V%Yd+~r zPp6hKu_-07EcF%-0ljphzseTYKc8qWy(gbpCvUKxo^)n{iEYjkWsr!M&{xX9NyKUH zp-}ncU~cJpz{n$ja)5yUE)J2#g7>DpUljp1*eZ`M_=RKbfc25|l<-Vf;Z)^Cu!{U) z@37$rgrBiB@yfKpA3l7jDc*UGS3bfs$MfnUVGDW2mU+`#m85?XJxIW1w1SKXP1f%_ z2KB&S_$YHC=FQdcHOpd@oN$c}ygA>w>B4ZTE>f6?oEemgtKILy)%435{`j&MaN2EN z2HETRmwHFC?WNLA;U@H-RCve&1_WhM;+P%V<=N+|F=ck{pkW=a^20x9V=b1@NWtqg zDmt8a?o_Knv5b7R!Nh#rIL(()?M9lY8n|vk6hrPg&RS{^Dic(3x_@Pj>ND^9^p|fv zE)0s&@cyUVeB#8KspI{W(%vE2B?l)>SH|IkYvOgX-;4*3{d4CY+GaMeK}5H2Qpo@L zX1Myjo!M?@*L^8)cMUT?S#I-kbg>eMBLu1doCx)?SZmOeF&@?=Vf(o8;>C+s?pD|{ z!L@CW`nvBcyUeKDHYB_FlTK27!)N~G&y9XgeWxN&NK7$k4AY%n))C<8 z-G>irXLS4E(~Rp*E+6-DfpQ?T%7U)biDbC;W=Z*SV0ST!#!j(czn_$#8)1KRobpqS4i~KZvHrCYu?q{8tL^kdTlo zcY6<@E^k+m+NSGX2lgPAP3foNKW?}* zmUrxs4vM=9Qzqa7(k)6kY4cuw9`KY<%kdZ)3kZi@@P2Y}?KL{WRs1v^kmJ+xDHmGYN*` z_}JtR`aa{vpU_xl-)21c2)@_TOe zoqrsIm-UdZ(mR~il||??XAZSo`1;i=sIOCcsp$I&lnO7gr9dPSE260`;lrh6hW(=0 z-h^{g!KqIKald~a)1XcxU;F+t}y24k?S$HcHi21w!3!Srr$Sge|94w)3(KvXLddb z&ushXK6E;8VDTs2c+(v%*4Xx+Vz+aiowv%Gaf4=zYQ4q1s%XL5NxkcC-}uA*O#JK> z>CX$k7&LEEc0Z;-K_dD*EYa`Y;z1YCtJCI*e~Q1SYX6Pk3Mdd|2@iu&PhY&>X4drh z=IU-$PfIJSr|^<eat=_muFwCu`G2MJ@;Ms>2NG@F`HC1_gw>msKP+Nj4bAckY{<0ksr=^r5@{foo4ufd4OWaCZ#5@oQa$-qE+foEH z;6bMHtaA6vIL2xOj!r3E^+N_BfDX=t_-#}3f4Ep}TtBA`=eEu=P-*r&dPLOfitA04 zctdfgtiMHVM8J}PDgz~Thl0DSiGwav#?1bTH?^*PQftbJaY1?u#nusQGo1G?pdq>2 z<;}B?Qf5$vdgMQ2+6&3L3$$V8n!*u697McgxMMTh#d$~#Ln97|g`UeAo-vB5M*RH8 zse$uzAb(}yEfANUebSx?7Y%!s-pJ9Q9(){ZIeQv92Vk)zIFz=1{AvP&Z~4jIZ&lhW zDUB`Kx%6xG+iuI2ohi-Pp=&zuq?0ZeWNPJMAG#8RpX(yMk00TP$~W_UP5_W9e z+Nj7relrlY%#iYEO_AMA8&pL*#Uh{vG>s12D?+C)$wys(EhrYE?w!ny)SY~3@xx)~ zoYvQEfQV{)*~)eM*0$?^Z%yINL}uVbC(ok4-IpJ=@wMOX?cJk%b>o{3ZnplWM%BA@ z?^wPb8~U+&w}jvRcRuQ^dO)qD<~b{DMkV8!Jb~SoMinA;LtL#ev{E@lb4_>%BH*4_ zywccWOp?lQ5{XU=yZ(q8yaPG2J}B!s}UPDp5-wra#U%f|OS0Uf}n)hrJ!T zX&pdZET?3iy8!$v=1Mrun4%Os69P4~ZCjGgSd%8_nslCHuX11BEn^iP;8dVSH+(43;Jw`UkF)|q_(Nd)4geLC$;wkt<2#^(A2T_E<_n@P^9 zl;bW3QTP&()q3~d2Cg%e>tWwNtZ`u2^1u;mhERlc=U1{x6Wq@i&>+Yvn`EQ@m{*9@ zf}7|^{8!V;CuDF*LP0h~yU);p-^xk;$#E#r@4qn^Hf#@Cv+eEr{uwzY4&<$ZhCqvn zY5f(sV)Kgp)*HY|{n`aFRwSSSW^MuU4s52&sb5@tapkI2;oL9$^8e{QI#*?pU9&5$ z=JTyIRt80c)C~P^9_sw1yP;t)I=4WEHUCzdALG$v`Ate;XS%g5oGMyF@60zqVS!MX z+uQfwMaO9J5-T^Mhp?;7!ny9^;*4oW-P5E zMUlnSPXx0B5@Vn3t^I`2r_L2jG%Gv1QATy2WiL?V(*lLK)T@A70K4_yN~I4gcH!oBl-f?>}4_e;J! zg`?$8j)LF?8MByCC1?8s(*sZ3z;6cx@NZZs4ui z!tnXn(nppf^N5F+JFJ3rC1Xg_3Xs6)#sUZo`4@;M16F@s>{G?s__rKBO*OSR+l=3> zqxD1vK--laq)n3Te#$yCb^yWq4PH7UE4k5gC>7*V)3AG!Sc|Me02nI#iycalJtJG% z1~M!YvM~Sga7zM*N2P;%<6*OiDy%p^@52WRqF6`d=iuhY@i|YvewNlL5`##28lYuM z;NqAPF_ET@k2p$-OC`6Wm*? z=^{lNP~B&uh7tm9TRunhukSBuLH?aP>H-WQ8m$x<;?IukCHscCveAQgGjT zXsPiG9dpDPg5YX-70N}R2U%%;V5W)aOHg8UC%*6nVBl3`J7On?XnOME=MG`wXRE+m z-ja(CwXw6}#7dr)BLB$Na_O*vJ6sy9TY>fQMo3n;CB-Q6sc7jKC7AS=UQW$aGP@D1 zhJM#_rkw<6HaR?IGlJ3RiX|JF1Il3E2*&|+l(ApJg~&+n($ZEfS{z5n%@3Jec_2|n zafpT^2zCm==LOs11tvtN=RuM$VOU*C9~g4Pdd@5&Kp_}n(-jAz`k~QVKyoHc{Ze!L zdX}s}#vUKFs?Usw4(QA*2P8atG;q>=(gYISPB5$s(C6It-`WfVkSO-5REUt>=^~$c znBA^UgSR753oLXWS)NtTs3KQMJX?_m7ukQIb@e5+s|U{cbhkcD?*!K=cNn~bW7cgy z0e;Gd0pu0*#sH%~@0W9%r$o$(v=14m&KodAXeAIiu`E6j!* zMdiD@^kVa-O|#Ej&CF(T;Ua3hc8)|2d&Dr*UozDSNM;ODFHi%3jDkLWHwm!_zTiMX zGwLe8cdo%al3ZDz#>Jpq=+W^UexszXgF5aXk9i;S10>S}?(;bZ+o!2D1alSOmPpE0 zTnfhX%A{hX4y+%#%z@pk+pxCOPsq#+1wD-GnDY8{NLl8Z^h!@~R_9f#9C#)9b{uuR zvWeSR?t=$GeoQjQV-z@8s=pEm_!wV3zvpI$CM{cPQ1YLQSyzD%i1@e4DVxg>?NmwjI~mOzF(b)r;lIN1%Z_uR3 zKFuDbg5+E}LG?BF$#7N=OURsd*whFDzB)p zGQ7>YGH+uaW7~q3@LK*MH)`Lrg^s$}fvk@oJBPm*p#|@dMpv91I>Lak+>4Y4ve3xw z!d|%qtT9oSD@UQiIOu*37klI{yOkLNOzns-Yq)Ve1sg|Rr8JVtq8_fLM~z(wJKcGa z$=5@vbuR)5bCQ=7ua*PK8@&ioEV7SxH7*IUu|2E`CVF!O-hTSTU2&)xWcE+4%*)SD zK!oFOID=tsuF67KK;=HO_7D=Wf0KbLnR&S$zc#G;mp=WAHzm!i%3zX^F{eCff8T>9 zZ{k8bczaIm34D_rRT~SFQXYa!LPG^b1kz4Hgxmqm(_*g~_#aJ@lCp9frGoUu&z}d8 z<{#Z!w~p&No)`|#i_qouDq6QI5z1Oc*k3OUboGA`( zMAv$uhNB3?n{VFqyil@9)1?tN*PePV|4w87Wcdmd{#{0_9+dXILz5_6VX1c)lC;Gt zi8K$sj#H^c^kxNhz3yMQ0-H5)Vo*Un7~IVc0smfIUs;+n$3`t1v+mDJPoqf2osXo)#naKut9 zK86x9knJW#*RAVCbIOO<`Ka1zq~_6wM2WrmEo%TD>$*}oE(HqZb&yULIU9aIQMCjE zdwb?U6wHx%os0Rg9U;dQh>=Y9f)M+VvXE7ohoN*N7jQq~UOIs8D4aF=g{pV&?nS{a zN~mFubkunpTN&99JiLueP)kWGFV#xn!k|vvl0@1L3_8rWhwgWKL9vq*G%(y`blte> zixW-}GF%|LcbFcHUl9Z_XE{LQO5ZC*zcDInM>MZup4zTKlh)P0sH_I3rhK1IaYHfo z5DtKj%*pz~$dj3#W%HY#GC9)L9uR+jsf)F(4JfY}kqN=-QIU&(%Kl zz&=njyFr~&6WNoT#2f7M;}^)hG`LGLHc>m24)INXTUcmG*<8V-4qWx*;O{iPz9ihz zTh+f>VaBbt+W@AF!ssN;ar9^Q@z@ZkxE4BTTg!^<_j&OG&~u6nr9LwU@GyCV(I%k- zIq+N~4gJJ$GV@L%6;j$S!yFeZSdg#M$*(6*v)vqf1yFXJt#$7xQbsZ*Ha-l&6|)#c zJL1xxPbFJ;+UYpe;QwaXm!@urzJU36I$5W zPMKFg=>Su-ndy4kl?1rb9ezNXnQr;3{{-zyEwAF4Sf`(6*oLuXSqr$vc$)9Sz+E zP&FR~HDFxbnHe-zV%?(yd;j9{>8NUwO3uyi%wH7&+n3;`Bo;22>#9AMzq%#_C|`XM z%eb^yNA@O+QZeJ`xK^^S0CvE2DBRN{9nL+2&jq(GA(yg61X)YebE!p{Pil<8;%6 zw|o+L1qBvPIEbhkkF_R}xjxI~bCWyWtP~my~naf3TgF+f9e2+~1-J;bI z^og!(*qZNs=u--)XK7#_+N#Gr8?&o?hNamlhB5b2`|@ZVaY-9Z#fwZuh(bza@hX|6 zY*@+uisG{^Dc{VQ$QP3>H&|-~78-cV$y6%3mtYxQFsbmY$P1`M$0MK<7^fexfkBE1 za8uCZmcbPQ891{UgI6AKq_jp56MYhh^1wD-(_0q`meUaWUFqmn5VERIs7+Z>k=#(x z-7G^g$~+bUWn-+39DMtf{Vl{+z65OlB0_q3`P_eU^ihs@{)oDcj*gs7K`NZn=72{G zd#Oa^2GlcQC4=}K@e1=fatws;znZyPrmGU7oE<6OrO)M}JE!@MTqoW}((~b&jxgip0AC$-ZR<1KvTK!-u@@=fK-n~p*;F<#teQxry}Q{OnCSGUQ)-K*RK)T&39OxT9)q!L%4v#$GBch2@MW z`MSgm=Y%D^UO3hZ!%%t|QX0k)Dq`ug7UMfYk7mPIkh66F-xv=!!4h&jZ`LLR>y_s| zyc&%*g)g-(f5sXx6kyMT1@*F$_cy0zbl^0K004XxzBP#|LZ7ihpt%zX(0B3t;NpGC zjp{|z0NwRLhkPLM9-t5?z_lAU<^&HttiPGNE&vD@M$~?@fF*h2GEmvP^APc!NLUks zY++YfRTS|`V4jDcyW$@=fy;gWbZf)aUmLq!xNr=%&VB7sN;zh*y$EX9oEnhylXyz^ zKZ734y|tDb_5stV$HQE@5PRa@ym{ksEA)Xbl00O*RHAccQIqhKLhbK_GuNdIbEz|i zo5II0AZQd@|77mgjOrpe#9*ihohTTNQ|O{@*#(&8Q?ZknC4rf@+9zR~)1)5MNX*&= zfWll5{pRb}^ij>*_qVRUZQv>~tHlj$+P5Axyp8Wu-&kBgB0}O0_WPdvx%~UU9p2tD zj;c`Bp5C}~=M>eUJ4}1&ugKRZPq;iRvj4PNU|!}DPUkOh#6()$w9&VCQR2<%yPJBA zE?oc$s;f)0yDoVMSS~0bgI7RSX&%uWZg|^Xpk~$IBHoy{iQGjCMy>eH;E~dt8|y4PKudi7IZ$D_s$HJoKQ};%-m40rx5p@*{hkJzMD$0jf#p1v;JuA z=vxh)a{m+^g#(v-?hV%#+-HSgMifOI4P5)+2(-;|D0~IvRv)3zQ4V4LhWcKJ4ScBD ztd{+uRZ~uwA3d_^sn?k^pF6Lc7PjA`Bw6#xq&i;wjhjW9{j3Lz3WYt3C&<_Om>&RyJ`Xy!^my&p@%u1n)gD*Wje3AC0W11(Wq6@|XBdcB4HN_nMN&CZOQw|- zp%D?SZ>Zd%;p)3$?L42h;6b9F6jYM$wuqqoD%#zjOh4!AQt&S#(UqzDYkuByLsZna zB`sY_CqaR+Nj?sUFVJ-yHcsYkfw;~-ctxH8mo0hq&ZT$Fu2GoM1+47zL>#Nh_}VFP-X% z3X0*Ni3CN&Q4Y*NN$H-KClKVHc@b#Q`DLxjt^D?js!b@mLx;LA4YySRuo0sw${3tl zEBIPre-B}?w7l4k1$3ct zrz>S;FA1j4bLal6cNFM4q;-+$Z+gy&a3%~Y)U&zmRYzhKEus*QMpfl@oNtMz^a<+d zb^g?^#VVq;2d%Qv`W`ysdkU;L{X82u)2Zm<&{KMZ$jzF`Y`$-`^z3Rk3P5hnUz5ucOzY0-TL!e9WHBb30t^#W1F=fx{j$qA!b-) zYmp46Gv>^(uZ!2zu( zO^8FKvVqbxgJwv$DbJgxM~kC0S6at)NVmgI0El2WlwX2=SA(dcO9bswFp5%;&B01*jmfmJc(XchxzS912&tl3-`oZY?fl}DVEuud^MwKn|Isc~U0t`DNgWH}>*&R^ zFQo#QK}KdHPy$nzq7^<%0Sem0;D)A82pwd?+)cD)&0IWSko)XE3aHVa1_?CIz`Scc z>9JV~yo*wG4{`y9d+u?ZLfsvK+Jr8Uk_R$Xx_bnVmbjQH=pAB7HSoPb)xW;tgL@%K zr%#{zhOTr>!@%B|7Mhrr(ziAeo;w6;NVRF|+tUQO;9+cQJ$}3o7yBMB4!o^{Ln1s` zpKRO0HGqAgxh{Ny9lT;adYWO56vcqhR>Sb1<-jN2a{p1N(V_AF6(dTw4LoA|=h@1GpjRNTDoY3v-fLb*&UG1{t`;@aBJ z*DGf68+*Tl+rB z5g5+E>ioTZ=L|_J1|g6|kF=iSDBA$Lj;^b&k|_W$P8mY2I)=|@>Jln|N&x0Hf1xZ&7p`J_l$mciv%)Yb@ z*hZ9)f-9o&l_H4JWl;6c_vfnf^FDoA*7X(zhk{KcBc7j&s?B)O!`a+)a`t6yXs$)1 z4>ke%ibFE(zUG`GVX??QQ=NG%O4`y}cDML43UM&wa}OAq z)aEpgg7P`FGOIRS&Dy%XZQLoO@3G*?cjT+^z@?#~ygJJ5s$^#6fe}3ElBG>RFtd=DWuF!tM3JoN z<|rx}>S3X!kMlE9f)_uT;}q#uY>Ud}J`XLmV7F&R&Mh7F%Ax1N<6{;TPi$?TG4JN; zeSD5t9ys~Ec-6H_mt_yzy(u|P0)(F54;?Zk+Rd3C#qP1EY3&91rpKHV^R@TsW8~a{ zf1_UV61hsNk(C4fNDF%QYBVuLre$!9ndVn$KCP|NNAioJI8F7_>ziSEc@|TS&t5oJ z?^;&K>t@yYJ3D@V?C+@2qG7!uS1iA$4R}`h=GV7JI{geiyM5WOK5*ikL5EE~F6wij zdX8U0VA;pIle!qhb1S81Cs3WBQ^Ig)4`_E_vvJu2nK6=gPvSbsKnM+iusysU5v%du zZ9>~`OU--l;aI)Snl-NN-Xvz!vIu4`4jri{oFg8<~ZThd4O~n z6wuUT|K2!7{=(+?g5_L-SlfYA@WJJ!GdX$zpHzA%03*$K*%&q^ApNEKD)nz@{``A) zSMR&4-grLEzIOBG2+`gAowA)*R449vQ%J406gdxGg-(q;XEk;IhP|6i#*Gsm1tfwA zuQf6QLBp+k6~FYr-8ZgX!`ATbFgqc;8BU3*$nQ+pxI+cjce_JPax)H1q{P#jd`Q9g z$?Eq7lNNswhMCPkdVpAJyZ8BZF!T6w$mVd}%FCYZ2d)iGKHezPRG~ zpAEnL+>w!I5&75iF5kPpPJ5%ds&s97n!?y*+(?%lFRxbReaKrH{q4ydqJ5^}DpA$& zlZEQ9YW*g74<;TQ-~$FVeU1qimy z#@JDyJ8eHrqfEk-@Al9)9{#z1{@5EwV$Kgk!Po@2WFkdj+nf0g&rzl-xLlrRoiqRDn~3;T z5L|!*wDk$#<@=6(9+8z?Kb>X`&387HPmWLj7+Lx(60Us%+t=Lsp(dj65JIIajZah$ z_^#;d_rJ{tyMCGlV-I9wcH|fBq1e@9Pd?r+-QO<|7(eIBm(X|LUOs!4HSHfo$n}+3 zzgpdB?UAFwfodSCH7-N^yl=NRH5sQl&d07*H$(f6VK4jcQXgXaZT27fo&Vks5R_oB z%Er#W=&KzSo<3$^6s%W<#emLuwbxsPOWQ=y->{{!#vJs)NQ(bcY%641IExRG1vjjL z4h#-Q_CbTk+k9d3-B3Y1If0nc$u1Ki?Wu`!f(o|n*dfNCQiH>Eg|TO000V6f39O*s z3kj$``(brYTd2_cgt%*ezxRtTM!zOA(EOp;xdZmg2nD4G9yrUs6ciLBQt=B3X5t}^ zq8*y!xfe)xE8nP6J|Vv z3h;d^+$_O%pqsa$F2UeOOoRL?JH#*wxqh=@ya%Lnc95q++fktM_cf{HV4=S?k zNpsFE|26BTN8hLCUf*6kXvO>Ng+tC(`$s%COb#kd4mLk)sP9a$FPp!{t#a7Co-wOa z|Ljs-P(NjVM0e^h7l-uHhg&!Ho)&Y-q{(spYJ0xb|gBoGb|b^yXF= zE%imj$Mw&_RgA`2C4(VgQC~1_&yBG`&yI6jLDZFogF5Y$^aRxs?pAB^;;vguwQSIR zVZZNOhn&(pBd$QqukUky>+G1tTmT%)Wx4yx0qP5xiyzF2FD>2L%HnZrX}Ri${re`$ zB|NIOg6WU*o`dJF`0^b{?Ud-;Z`WGnzMQ)QHr4KamJP)opg>YfUCq+id+fsve*N@yz_< zqSSXdphH9+q3{%KFNDK^pM7zY1QSC+E2A9p8Sy4uxFH9-t*iM__4p&YBMH_(M~i?( z&F`hs1k;?Ve_TDa?ITDsEat?IZJ~2?dSbT=rRPQv?>is&nSO+FhnmL!d9!+d{pGPD zY`^YNo$8-Ird_Fu3ehvso+2gCq7hnZh0l)n9kF_qUCE{}t@+W~8%K7s2>4PsCBk@$ zPpR6FEf;+TmhQiQ3_b_WsWqEg`RT(4miY`=bvRXQpQ+BxpQbSXZfY!L6L5oKw>pHxo)Bj>7 z{s)_~*Qc=Su;D+duk{e^XJb;teP-JX8zG$qiELfY}dJaMfJ`>K@&Rc zOmMJvS$3;@h0nU?N^h$ZL@hP_hUG4&n4Xazg>UB(!%+46qs09F$04h1TKo^bq|8ez zA?~eytNAZ)HHsVl)jDA4=-D1&8zD}zkH?q&@bL&63CWQ0Wwl!Bh0B*I*gjGO%o^OP zbLWErMkf_ceHMUrswIr_}&9L?PLx<@qMXyQVg1X_VNMBq0(D?sAP^w z6sLuopOIc;>aNxz<5leGbLebDRmKeN(r~~0eSPQb2PU~$P}7KJH1=Gf=4ZNC>(;SH zTQmEI`uk7Y;K+EEd8fOtU;i>W-m4boDmMEtL&<6Q#@Is}YN!I=LB?U9ythT?1CTxr zSn{j?(!MYM!)(<28~p>ew*{cLGd=3tB`Uf}Ojp~_)BD($e$VC2YESqt5<@D9)RO=! zxr5uu;VdQ_YIsG4VSKi^T0Iqb|ReQ+`syu1k5wkp*RP&&Xt498xM96(nYlBLY8|xL` zEbEFKz?SSL5HM(ru^p90y9B15Y94w;_3u$Gw}PTOOjn(n zpQ#_6aX(iqww^=1AnyprC`Gxybui!dA@~j4V-Nz2Xk8fMuU}pt#4Bv}rPtK!^x{4?4$y@Y_#$3)`E@((MU!LyLIBPpiEsD}vYUIeEYs)e zXHXNtQXr(@EMz8x@00NSxqd8|R;eC=t#w-upSG%cHs_~$duuh^b^nG*t#tE=dG<)! zEsn#+_(&e{1H`5xexcswXk1vF9yCxBwMQN^FLspC=EE(872u-@m%)cU(YtEmv}0y% z5I(1pOD6Dyevcw;GdloJTuya8D6_^I3cH6Cp0(J)nXIADw_;LcT~&o9Q01fycYe46cw0A7B&V@{-ch zU35HCj&3n|C4n1YADp^uU^-jwQ_$6<(<90O0qam3yQO*l_ns!(ks%MpN5u^1DZC8U z<_BtdSDYgv`g71uF{d9c$fgJ$c`IF1Riz_FO zn|+5x=9EU#5CCzf=8Rn0-0ebcE=;;URXIfA^fR;fQHZK9>G_^Z*~dCMkowLK)CC2j zC0xh-&m?FB2b;NO2>0f(+)&ioGS>wR&>bknIpJ$rnFH9;s~f9!(~*oA$kt)`<^HXp zB{M|#5QGSV0K|e7^Ik4hz4Q1$GsYuEHK=h0C&p!#Q4sB;E*MeT1*}dEAAK+^d{_}} z+3ZUguLa&;M%s>qEAl;LbMFVb5l~ikI(qvD^lkt9@zORy5oj}a~9p3(fjV4 zKXmH+_1Bv>6RO|0SF8;&+WI&p%rNWGF{@surk}q5+F(`35w^kZeW*!S2psiVB~)0x>1Zr4%wjVo@0g3!XcnZdGi;$&op z*+f(jTaF$*8t=vzAbo8nF#KklkJUL>`Mm{5ode<}=NObW%Hf67fnt%#L!)I6L?MH| zf0PU+pqBuQ*)`2=k6#D#P_V{5NTtk3_qr7 zZY1S8K`0NIeoY_s6QBnLN-jL0WeEq&{RIb6ixxXLuRVUM}d-uU}$6$ z74x$_5UoF4dT{tkZJBqb6cJ*Uy&-bW2mrc4p%ctH?leUx`uy_J_6){#r=LXdsR!YV z{H3$Q(&k?DDsZdMAyQV%UtV*@P!Z5UR*Rgei-GG7!07C(ELmO{#NuuR$xMB~fB|VE z?6LhLRwvJ%*B#L%>J!->vVsi+K0|o>(x=hr*&CHzJO+P7XHLbdM2+2J(#sPghdjDe zbz|<0)vC8_P5w&Gy7MZiv}196Z-wDS_o)$^x+zSyS4=&=df}<*=XR`e?XTz(5_~_A zf0*#L=jQUCw-;1?ylPvWxzW5+y{ts{$QZ}2W;-j8Fc@qT5E6u|f2 z(tBt3w*JUW%@HOo%s>3h3z%;^sFTn}oEM)8HwY7nAF!Mbl=1CSeuS%;`h6ki2uRL1 zlo5wg|1u8g(2ye?6)j(LYh)lrZw5*W2sDy*n2XmY<5L7A4apio*~ku8)KgE*ix-#J}B94)F0 zKTuygU-?U(bBoCMS{G`**&^?s*}wYwl+!prJhr5lkKJd3%ZFE$Re20uVK{zf z>bm?6bE;k??CrLwh_E*RSNGDf?b%nCg|WirS&_xJD9cS~vtZ?hoO64@q7*`hG1&ss zuoLb${)-m!_4UV)66(RReSN{c4uD>?r%aTIHl z1-yP`aY~?WFOa9YHgHqvD!r1GC@>HOmLuTJS=K<9mFfBpv`h+yNqnw)vKs-#t1lCd zYVP+TM5GhpCl;~S%j`J|on9bo_x z)S4dOpF6Zm84r8ugQVfSpfPrc04MzY_oa$5wHY*xh&*+;$?Xz>njD#p>OK9R))3VB zh}?D!bcF;_+@CY8wsKDTr~|-8BhIN^>io)Pz_G=h|c0oP7jUT%$q;^G@W~ITYuBy_g>Yyxj75w-g4b| z(IJgc6xQ*UeQe>iMSt%seSpH`3~ngn|NbKW$%xYjr;TkZbFHvw&bG`8ES~b9SfoL} zp<%V1Yw~7V)MzqT)5wR%a#a9%jIC51*2U1DW5Yj@keu`D{0+!due^y9ENs=>S?&Qi z$lWPF9#RTqII41Jse+!#mKP#wSLh93J{6~11v}Skjvsc%KYM-k59mTc0vQ}0B5@q8 zg+w2uc7ZumsjVB z}ByWYYq+>PjL~h?x{%Z8C^*7(WSvsJ_pFeUw&fIn(FJ15^sGK6NO;C8_uh`m(RTMd0w2 zr{c@dS08~E77>58cgeHPhsur0>nEYmfOC=26j%lcT}J}r7*C5|Q^_T05}LEp0q2rr zdzsUUJ>JZ2cjBtjkiD&*R4;|RS2*wgwq`bvy;$17OAH`rPRzKb<&9EC9SlW$tDIjD z=kI_=>SLNliM4>yLqSVr5C)8NWa+OVWW_2x32DeJ$rq`KWX!$CUS%c!*@6)^B6(s^ zrOYz~Kkmz9(M8Y7^2qq_N(y9a+g4UK`PF*N8ld_t&Zom(j$@fZ2mKXl@B$O$r}n;Cw$tg$A>)(Tm60n|ewL}` z-j2ET{MS?K{QJ#wKW|wXRFtzSy~~;py9Z)ux$sMN5FA#TN9BX{z0B{mxSjT-C8!@V zFa${MeVR5$&GXrK=0y~9;_j1H%(#-BUXUtqxjel{dbWPh;b!VOz$R#<|~S@bAgw#Vyps zYui@evYmM?nyQu5m!G3$^?C)0aw!h@EQuA9E0^&5_;&82|MaFL)LimuZya51mqv}Y z)-Pz;rHiWD=|9yoPtEq;-SGRO`$Lrif2A0no%>}xS?qo*N!JJC40kKc*|M0A1Gt5`8pYn?ecj*kvQ8%$2g{Dwa5t`peEKH2Y`P;29YA zdsTQB0a{7=C|-N> z=c|Q%B>syGo?=871j->(?J5-zIIZ5=%5t@^Uh29Ow*Mb{?*dhG{{R1OV=x%Z7?*J$ zY}b%mh-ij!*%65fQMn{UMk!LNaT)j9P9@#kk}kRtO3ENqsu6{dTbFT3lu5bNdAxRJ z#{AFvt+UoS>zsAYTBr5>&KT9+pZ)o~->=v6^?W{GuXho*9RA6?JvH5fcnu;Z1uM0&D+XQH<4J_$Rg!$JH=d3KlgF37t`3? zv13O`cH*;IJuv1qQ@5M*S2o&z6x`;pp{ib=hQ~Xf!=CpQEtL4_7p*A8HE%5^j443G z+ZcgP!BMoVK@>V&M9vp=c=z!nAxVkEtu+g4C<^AYpRr}{D{xQ_?M`^1vwaI8xyG>Z!&vPdI<^jceDJts2y8{Qj4B1#Rr=Z`!vj!+(_D)y)&O zxoz2TJ#KstTBX{IT(i=@Wna88b1h;9V%mJuncW3vwBOxM=zjCOUOD8GN-hyDXO zU_CEMW9?NoS#51*u=%)0O%%NY*8wuJp`>A)GT_&m_umXn3Vta^(PX5jShF5d2IS7$ zT4%HQ@#?b7fPpL@85by{B&29&{ZkB!`W3Xc!Z-ebG@{X}Lw7Gn#%(3E$~a#<6zxmT zu!jKIbygcc{H5Z>OhEJ)=2tM9+)nL10j5)oE_79NuM)L@KFD#~zyFk_6TylK=#DdEg?YGFHfVKj7Ddc3>}gIBXEg0P_p@p zZS9-4l_5ZYUFo@`d1V5s%vG#}V!|X{Zkhatj94H5pb@vj2=O!D7ugjq4f1!;9W8CW zj|og!*h}HEXkM*nTG<8}Wj!}FhR{O=Iu0B{XaCotaX}tnC6jI!e{ie_Z4~t&4hIGo zh!KNMmv2kYpmWJ#V60^Feiof%X!nOf0zw=oCzS;GNxwdJ0wNi4lZoSfaw#|K8_YcEXY^~DUvf~kQ@QsBZuNDVw z{{hAj6HZ%v9HOC08H8E(yTnNN@#gg3lU(m`C>VWPFYcahh89Mzk>}&8QxGPUp%{`$_>+MO1iCBzSj26pesa*v$V+x3wQSTsno2XT^77B2KZov>RpC(7?-`W0_ zSs}h!<;#{XmHB7P&fzibp|E}|Lqt-}J{!$+;7a0)+;9kktPbA=fhBG*k_V!s&sOO; zSHb^F2O?E%N=}X!&q)Tde5((nmR=>?N+ihstzX=}eOubxz{#@; zy#M(ARl!>(KNxbc`H9mPut;7r7T2o3a?LUt_j)icg8;^vh9;uiQQ!Aqf5Km3I;biYX znuOAQjxxCr?yi;jur)3&B6gO)#cLtT7)DAXl5TmL@04vjwv>({3;qtTpa5Y0TD((xh|%dkV8i) z*J1bS;dr9I_Tx3%=akXplX;j!h4GRC7A#>NVK56D46a?<7@Y=6y^#JKH7N=pH#*4K zO+!|w*4Cf)B5zUD9&UvA=%tz_0o7sr+y$)3;yfT?4V$vKS(||yggr%o7AxW7eR~;P zT=%iI+va~8oqFc`tvY;t-=D3%(`GJk*N&`Eo*gzzF z)^M+6&&4XZSa`oT0Z+FapYwREQhVFKy-SDQpV+GW-!}vOE4kJ6)v;|Qf-{Nv%AnI) zyN6V_tJ<*TmHYf_;mR2|So*18V|1<6ktLl1uY?a-wYybH^+=<#b6`nt7sF^b*=1A=AQ@iCr8-RS|L3eFe0~xt>DN?nTG_O z^($_Y6P!j(uC0Gw+~QVN4;yc(<(Rt63`GaF>CAG!B7B$UpPbY|i zyr*STW;vy7OGc@oHC-Irw=){*OBpBpsroo@XoJ~0P8?<=G96Ou*se22bl5XAkT?Oq z{rB^`G|8dC^51UH+i_-@(n;03_QKqDg z82CMa#56{&&_P@|dj6;TmH*{GEp>Zt1*R@IL@9IowsKTUdnTkxw?2T8)~>_w`N-~~ zp^M=Yb;EMP_D?jK-@ig~SzhC>zOsUC;8Yd8>uzXvkEtDnFGm$3+_iy+lD_!?ZFgYr z@dWJ_@BH+kou~c#5G`zBavyq58Fdi;nMUXyx3xhPLQFA@0JGm~dLP9vA~XF+>c22- zjwwK|Xa9XumN);WHSenJahyjAX|&r?aTjANQh=|{J!EO{&&^PNb~AFR z{iI*(ds<+jNG6^vk`A?`T*J|IecB#iyB($ z6vXN$%XW?I`0cm26<(9EN=$rWCjTchmNo#IMC`FZ7TN0)mkDTVR#nmb_BSiP&q z$#tJPBqz)Y`Nw16e4KtXI`<|^Cld?_$!XB(Iq~)>_j*)7YiwjKbVUM@2Bcp3GNth= z4_r1?GYsbay`Vn7hiOnjV&7BHkh}GC7Hvrq#$O#V>MXdf&&=i?!t1v?!Fd4+hvbl& z0Dn21n6EuUxBRos3*zS4+WI|oYCwQPZWx3I-56*m_oB`*riOqEzOAdzk!`c0Z|26L7hC{yHCI% z44V(Z;|sNt6ftr2@!K>0-J`90u>jGG`s7DZqmD$k-JItu^@(50ebh z#N3a%)Qxkn{b-dEFAio{>rrNZ-!bK1A7FJ|-&`MGaLw0~4q-Xvl_BdBsQ46Q+z|Lf zK5|*+h*D_^6=@quLm|gOQZNp2H$j0FVGmK!)*@HSQ0t$r)DP}58V^*mkVV9PyK$&5 z;}1}Sl6HT(&8iNx^Bm!J6qRBE+R2;-nJI#nDpN728s(@J1-(P*q_f*hLj9&5^BRc6*zr?z;KB83{u^RrO=|az% zcPBH1({P0(Mw&>WR>odXUs^E$HR6hZd`tZILJ>mgvWWDMft(8HV&P-xVZ(7=26qp% z?jVC^Z6Zh_@^TCzG1h;5Yagp9a4vctbaiW740u;O?^Rp!7V$f40triU1kfl0-ms-Iq-l3`E4RWg=_{aqfb{U~mp%-zY3Whr0BX^b@?D58ST zFIElu01YFo;+fq{i*UrsD{I_hpSR>R#F-;_bHYj zzFUL{Wlb881Ibog_C>@M>jIHU`+ai@Em=L6aJ9?9ZUlToQ-o|6$kXZKuSi z$eN5-bny-%0y#@r2hG8nGEJ=9sdlRz9Sg&wk`CcjBvx$_u9UO<2H;}3Y50jH*L(zDu0=suw4jhayUmeKS-Zgt;2k?J*$~{wwo+0eHUcY zy)T?R#3>=;nwSE{AEgtQn;ROCP@CR0|D3RnW43l;3ZxubNtFUIr;s)TH(W0WZ5s$2 z{El=WCF;a6qJqW10vs2#YQlbomyd(19}$g{9ca7hNzPRdIPg$gxE!+)Ps zb2-jM#&Nk~pvg_xersz$+;`uowo&lQk@MnIqK?6C-3d~Yx;fP7;1`hR4u?u;rb%Sb zqBzy5SQX#E=s@(z5ICBdO_N5~u6xZtoxiL1<(J<;@4`?$W&Al^E;4Q$m@OxWCRILe zOwSMpOUsNfnNkS}CXtX3C8xI@U)!+DmS+g@*msJVS#;jBZ!RZhEwt07=|uV}X{~}T zX;<4rG4D z{C8DVE8f1GL)&@C}|*IN|dSb8f&bY{k; zNu(-&HemwM*fwNtD@wSM7TJSAe#)N&yuh5P??NR==emrfV$|0a+)gt0js$aV`^s7s zzc!-u5nfyR{}~0P@JvP1K0wQ@w6dR1bd-6ra4g7iYEgY)UKF@WmJSGHQayL8Rr=#? ziC=$i^i*!)hN-`#b50J{NnP^x&6@>;($K$WYH0S<)j{!R&&uJNBHTa&FtKIDJ8|3M zA&{}WG$HQ_m~)|1H)#t5MTaa*>+<@u3E=Z>OO^(1-a2@|Ul+}DCwI;Gdz!_kZF{xd z1$^82>o$ap4DWFM&YdVLU(*^LP*pU-xFqH%VfO(#5Dxs!kTIa|T3y>`8hztr)}xa%426BYhx+D7e7RChb> zR?)f3A;u_n4Nf_?aNhP;)82l=gtq^>N>kOX^xyw;;bVS|n8X!~n>f)W!=qyVXG>*I zv8VT__x;B`Khf;le}S=FTjC=btxEYS0U!7$D~%?8)c0)!A0X6T z;z!m=pFD78D(9Um#0OUMp}ZMcpe{b|`dFZM07S(j{PUO5F-~8|?*IEM_sz)I8R_X<-{;yu1E&YPKVcT!QW=A9+0E)5pTt2YbX48KMs0hl=r<$rlNS4F= z2qg=kYhPE0$%<@xAvLvEeg!>UF+>$qqo|vc$iW3(2faQozzcSmF;{Wiu8tX^?h@(9 z>RWDzK}+T_$#WWU8Nx(7JdIoTI|6bH4TzP|@IU@c%s-bdyl!>=0?oiH zCg@Nx_V2{-WNX-Jp6;N`?`e`b2+(10*ywuVgRc2sKNjun*N|alND$h|V{A>*EDxe? zGU9B_`|qvEQQ~s|Y|gwyo(Mq5cv4c5flP4i%1B}`%$sGNBNuhVKsLuZj(UHHJt2;n+ zZ+NP8@991BpOEPQVcp`gS>4Wsnl)3qHI?UEdJ!g->kD#iQheR7v0)jP=t`@)VJPQZ9Mi)7WRDJ9d#KISS4 zt`-{Niq{aXW#c}JaT5m8%OYo#HEP}O>+|eQnMaBi0O;IAodsUW6n@{MPF;n}j>YJz z+S*#Bh3az=YwI-vDy6K9jiRRcRg5g}kh;BvVL)Z zj6oJ^_|;brvB$+eHc9pC-o5*851)5_6ZmM$j^lABs`RJs>3-C0b$Z@ndG^DPHxMg= z=1V<^I-qFX7YgRh@rg4gu1Bga1yW1gKui+-24Bo34{+b8Iha)*yy|DY(rS(XpjzM!05d=k#^WE_IA))XqFQZqO0m zsFTxwVBV5?|7V+KLL(mBE(*X{e6?l8SyK>R>WD*!4h)Xs_C&{#QzC${zw=fY?>`KB zGB$fF1(CUHX=!N|!1O99TjVOzzBC%IEX@yqa>f%e_CeCxjE{oQ%9EM=C(<>O`HC2! zO!L^2;5iJ8-n@CnuFt}f>LA!+1?*F#WpdN=v=u1Gy^kTpK42u{EMy;E%yU(~^Cp60 zw_X;?zRsK*2I`mxxKF^Ln^6zml7Il-sPv_Uk^Cs;xuFs`k&y?N|+s4veJ+G=l!%VchQpbsln z0LG9ZQvz4H;I<$k$k5>ySZXMqK)m%h2SJ6GqVHu&c^&Vx~dyC z7Hv$@9~oyW!V(Zx=1CD`+j-#dku4?u1er<)lP-rP@)HfLSPe4%Tl!a9J;83`Pa;l! z^IVf4e9$cZju~CLG}i`*!@~M33PKO0x43NeT^jy;8_kpk_N(DPfA+(#|B2ft=-ILj zsTC;HOFv6MRm~WmymNLwW2>ZnCvzmWIY zFQ^ro@It;&(0<4JMs;raPc-gNZB%vkfWB={s9>xqBH6r!8!ySz`0Ok3sfAxjJD~DLzzFs4r06UlmYi}e z(iE(;6cnIFbKxN>Dv zox1fE{oy`~xBh+I^6x!E1w>xvmo^~~@?(SmW56skPyLT*dTJn#%FT!-zMLoo!I7KT z=i5CkoUz@0KX}nM-1Y@9aa$@z*cW_7kG$+>c%qcW%i*zP$QYh_x5)q8Fy|AbqAk$i zwAU&)m#r^slyKL0W=EbtEqTTvk@xlT0WF+L@&)zp$;*YcEFpC!k*Rz5M}W(QjT>jS zm$_l&=TPtPMAxV{1`WmLNqvLx;tz|{J?||MT2cA#-Q~L$kmZy^(v|?8?e(|dqPY`$ zhWg0~H4ha(pi|;kDK}`4wZt8s>Pvj$MGQZm@bS`E^!_b^^RuUuQi}6*_#N-TXt4sxs z_;O+eW*YF#I+8>-eI8VOyN@_|kOc=U&Q@y8`8DzDsHIacm|3n(5v~D~2?qvfFPPNM zuSetY`z9Tot`VSr?cuI3)eEJ*Aag$NlBu1WNj{Wgs-VH3in~Dnr!LV9t<UoN+M+ zutf{KM&gopLn!m`^D|fH&4-)xVlPc8AEoZzq{Hx=A4*AaV(fxT-Cs~K11_~Y^@#Qh z2TPT?Me3mq*k1a(eOL{VyRYD!1G7;SdZEL&Wx+%?Y}V}bpML6nm!x?y|4GB+2PDQv zMSC0Rc8P1G9ma`OOW6Oa_WN1-=Swy0_Kt;d0=A>B&6vHn_LH@%X zx3X}iC9ZzG;54Qte?f6io*aH}r8>cu9gk#HhgP(qkajX=n0=)T?eI{Xm*hMQVu>%_ zj8*qxS_@YoCko2Iahw7{1x}pYOAfq5N)$K_W+?)OUrJAZZf*J7kwa#&J?Ap!4=juw z4|m|nv3GGi)TyNNL>&9LyTzD7?yK`nixw?P_lZ+w!9X~$h53=4YsFC2x45Q${Lv$R z*Z>Z+#a_W~7=usxHZuxg)C46XfYxO$u}SwSf#Tc`y7TE!a&@bbJT_3ziWj#B2tN;% z)|1gk$b`S=> zgFQzoW)QXx(qIO3C@8e>m@rUa4X{;1AaI-j4nXz2Q5+)#POW>Mdm8SCHEPuc#^MS6FufBYL>E#GD7r-ejJ!i%$NfBZg15Zw)*JxWcms{rm)pODOTGdM> zkJ1yOv7D1xLVD0Ch_?4IwbfK1t-=RzX0-3=ooh@rXP}U(XR&3w%7{1ukw=fQ?=V3M zaY%!P-iePy4pCd##PvPk|EAJp36M4rI@D=SmztKeywy&y>_oeQG|1u~%myDTu-!{ly2v-gi$_aUN! z0>FH`4`G3f4M*(U@Xj_RvjabhR6xYP%uyH!#cE=i@^oGC_`?a{5I$iR?mu=eG6h*PBF+H@>1ZbMGH)*)Q+XIeCT6@bp1)a{LdSbD7>Y zgAwF1MuRRe>Gwn7Yt9X3;h;$l2Ee;$()6Aq!=cyd&O^3sEJZ(Mv7Ed{aL0SGVZ#PF zm0FJL;0Dq*tjJpWDVnL$!64VD3GYx3(1831y=;=HO9Xjb6iAvuD+gw`QnkV+1W!Kg z-b>0n0>iYG2_ zX1X2Yw5M1cAQE&AhEkm2O(^5Kdg)Qe6*y`G?K80IoLSSj88%FubI4|8a$oX!bl z%ACzfvadlu!<=61i|>5~IZH$Yey7m4epBoV^|&IaL?KryK+# z6L^MA*;mWveoiYFGj8AL2i6c=fihy=gqaXSza<&(Z6ln+VzR67ccW` z{)cI|s4gFMWW2?^Zeqb(c|i$RORre}l#(use`i|^qt6Idok1+G;IPWc_zq}Iv;~N+ z4u3I^)PfsD&T54jxP(C+LRL=D6jX_>B<6DKp6GORN|0UkDkxi? zLK76$y65lSz5C%{AHA8+&cI8*AS>j@$k_vI>OEFxH^)i;DIg(+W*(@(F&N`uAo|1T ztCmO`j5LX{A$TZ%=)FX&lF=G_T2H#W_v$u83k?oZ>2Piseub0|Ygq`Qp~1TVqh;=+ z9Kk}8`|?K;e(LVdb&2*xoFp`taf2d#Q-RenqGBL$T_&GkO-Vwcl1QZkRcST2t(@6F z5;}n2qG1Ilsdm7~t*7D3pSEBKBzB{t3lqBZGKMdrO4@y>f+q~7yD>eDqp||MTb=Gz zXPHS^@9e{4C76Zw`kU_`RUUT@4*sH{s_n;1eA1tdf10$t;rb@SMvO4$OyIyfdS~4J zst?3wPDTC?(zr}ilfiUCHgXgNBa^>6k9Z;d{kU&rTHEY1zx+z!BaMIxU^`=YHl?g4 zXyQwK9&9Oe;nnn45<&RC3TC7 zUYAzcdDTzyTxN4Zae{`uSe#ALtYW#!iIH>8fj0 zLcP7K{Gj)QB?D9Rs(N4VT=%!Zu|vH69$>M3Yx9EOvNvxEkpIPjfYBx!whSI}2SP0Z z@I2@#F$qYaC?qC7Gc97q0213Kj=sf+5&OM1&=otZ)|?%h4~p@lsmyK@BOY<#7h3#a zn|T7U!p$vzu#@wv6JsE{f@u}vi9PCO)A!KFD=3@Feqw=}uz%3f+x-C$;~AyOFn~kx z-w2L|DaIGR=z0yeo=4@kt@)_;F9y+7NgqX7g4kRnSz$iuMs+@$Nx>B4-YiVKGX^wi5jq|NJoe<7nllw0K`$|pCttVbG`FkM*iDydtj_T4EobD?BVmp1 z3^^&C4Gxx2U^EQUm$fP7dpReNpyeLQ>7Wy4%S+kDs|8e^}7ge9BG0G*AZeCh630Wq_>d>M3-_6^uzG~aV zX?@h?bsk-|*84f^z{>9z7QMfiRb_2qwCB;>g%5}2?vHbQ>0;?(c{gtU!U~6%F1Ln1 zfUpU7&7ZfeFXN%E7TS%V29cvJbYGr%61F{=ik1o4t7*sHD>32Nu~T}XO`|LBU_HD+ z0Y#MU&jHd?`<;7}4VlzKWq+~B&;opzS>xI*+uLNw?H};cWOQ_SF||#f5ey_Ke{mvf zVS3-w*Sh;HRrA`~r|&zNd$oJ5P}pugyld2OBJUgfVxoXIxz?>@RAkb-XD0VsDHPMU z8GWIM%LZnaVWeZWdVZ>9b(zn-wQujI-mi%m*R@4!$ zpz4Z#^&YOq$}z6l`I=G&BV?|)(!qyG7c$xY>QO(tS@BCot6iU)KS(2Gi1N?_Q_tcd z_kOt;!MPb;<)ry^dBa519IQ=CPUS5L2Lc`HKDnMp+mnE=*{kk5R?+z%C<-g%=CZ7P z=6Lr#b-RlSIKJE8Aye1uF7coo$#-I5advk0?6rU8RNZ}0?G0%kOou{zMV_tP(ww!u z72J*v2ftJ-3BG0F(zoVAvHqaB7dFU+a3tK*fp-P*Iy2g_{6b^x>1}E$^x@o zqchg$MCNcUcV-YY&L65sUvfM8ky*b^vyF|73%4CEytBa5FgCTjU&OOq#j&n-iLWp3 z-MhD7iu=21B;}w7ABu+FameCOPnS}e;-QP77JT?#jstmm=S1PH9kV^~oYpSMf-L}p4&xjfAyu7@EQKOIEJ$C5i&DaWsqJ_8S;Wg~xzx%t% z!#b<%=@ctIu)e9F#;~OhcbA+YfM~>cAShcKrI>mifq1RF$i#3a1#uo=-;OGobBd5mRK$ykdwkZBD>fqF1AB+8S zR^8W0Eq0&CEX9FSM*cOu;+{iRj(t-jQ&aDq0YlPm_VTn%XsJ-lIKH8|;+P4YIezFL zM<_MC=9@2#89VCZn;UzHzoSy7&ziL#L|W{V{K4Am9r8>t*ncsbukbb@=3GLZ13#vv zYZ*_MQ{|2vH4mM4ziCO?>+#dR{BFiF&dc?j?U144OhqJ@_=4g|x7gny8|^=B8M;D1 zcw|Cv)~O+iQaNklRGDKpdKG6zo_V}g@9jN@Uj6!IzD;^7($|4BpXlfz?@Md??)CC2 z%yO?TA3AHS4QD1S2B8!d!%hXCIWvr5B%STMJK z;E831r|T%rw~eUyP|;YSV*I3<`6%x-&->AD?Zn8JLDq06^0hIYY5Lc%4LV+>dUv=s z;L7w+$d)xt3I6R@T>6#?VbkPP(Gf4tbZy4pJsZ{1BX0EAs$%_fI~RK&nic!bajf&e zC0Dm|3mbQf&in9clRo1%*4jVWF?1oUsxBX`oGBCq-sW~Hb>f@siFa(H&e_xQYQu3M z5PkFdCH#3v6urX2LP}$I0|Nv544qpAg!+s&HvW5n6Gex%^$UA6Xnmd>Omb)0>H7mI zW&u>)sivcvFRbnjj!WmmoFLlB8GP|CV$+Q+ECOlCIvH(v(Y}Sx&55dW?{w~u3yzJ= zi+jvsus<4ar`42sAi-1x9I52rZ+($k{c{tBm4CnBwq2(ll>qL|o} z1z2bLNC=y&o#p#Tdqr5tv0=S>NA{hv((LBSx)B?zD^4e~cxvAEs9;P=`Pl~_Bg@J; zYC51(^{Y*Cd1@s$biUkVrOc!+gOVxckr>N?XWi2E&%{ll^9&OB(juS!043+VfASN&B zbjRA~i&ae(3J>C&&Y_bhPnv#YSLVD)Zmu}6bHtlJ#(>PC?;&D_9y@k*&-jkMN{JZi z1*~q}g56_cFE$-f5P9~cuvq1fw`m`x1Zp#UD7BOM@t#sI6ghbZj)10<>6j@g+4V^cnl9?-{7fo z6egRZ+eXqG+5W`!aqWUBh>)kO3r~qu3aK~{jHgsl9uFc8P72i1<&J0I=?f%y%BV@5 zl|ziqz94KnlLStXLGgwDkdk5&bmA#5jRbmn(j1Oo;7r3=-!Oh)mdTDE~a%iroyi7eANX1>SeNg5Ewy#X< zy+_H9A^&`W{JC_s;|CV}GyT zPNrQ}aSee%9}0SKh``(?K^<=Pvy9LybEizu<{p2{d*_%t)5hfVmu#k6-xw$kjBC-d z$JmsyI`=QvL>)TXPN8@fjpb4fV3@i5++W#%L0OaNS65-YrcHlSRXTz!zij95XU8Hq z3UM)OUcwxZ*~D#20=L-OZ`mmofgJdLLU2TUfqLSSvXfSET3T8Y`Fq3DQ$tyLN6pz5 zGrE4&jeq^9gIi9Y$*s-;%%tsrT~abIDr^acCXT)iWG5@Js%K*Ovb1)KI#-c~zhfUv zLNH0)bb{6KY5z-tRG1Fn&hrNKxVTuZYHcA7&68Ex@z`jhj&IU2`ERN@Gson!RHHB| z&e@q<#_u2S4&N~`{-1R2VRH1$5etUrM?D>)R$5!fmAaSnQ;vJsPzr_X#`pV`q*Xy<4f>Qs=&W=lrd(v01C-Px58kT=U`Nj|YAL3DkahtmAop`f4<~C(BRs z)$^;(DEmpYKCE^m_mZd1}kvj+VBP) zS6c<*Z`3}HwPDs|-NL<)M*Tpp6UYYkfI*uVw`0Oj*5~qa>;JVRL9W!pWEoKN4o4wt z>C5c~6rlEHj%hY>uS7Gdwai^^Zw~i%gjQMM#0(Ow?*mlEs1w!PDGL7Yuf2QwylvTp z$dztc^X^u}*_R126iwp1wPvr&&T!`bO*1xDbCZ8yr_EF^kYp%{o}piQrHSQ}J31Ha zX}|j6uUd2J^7(DQ)9UtP2m6$kW(BR2E5`z3JYgt{Nzz#ZuCzaK&8d^?tDoE|P6QPS zu0|W1uA9#(*@9x|hnR$0+x}|jO$Fe{!e@H*cu;N4fFosPWy;k0?UowpR^Oh#OR^A& z+{OCL!^V(;nb*g63Q%$t?HSic-kR0?-rG_*P zf%*cWIQG6`cjnP=lIwYxe-8Ysro)5@EEDeq0gk!fPT==)7m>5qdKsPPbMJoe@#QMP zgW*euvbL0|YZy&;@1xW_DmB)H>ET7;=t0ZkpU@>S0VHI$s1q+?FTR@{}ks+>B|wZu(;?gQXgRq7Ns<3GLGCQGJ+n{H5s^x<%lmMvR~ zslNPb*Ifk<*EfqS8(TxPRx6o#I{T#?;}iB4lzX)fn^yY>4t{xTKI(wieW$y9;XAJJ zB2xS=xHV^$p}IXs?8H-jo_Lw?+^ukw;FEI|d0LWpWv@qHc6l1?qgH}6+UUEvM#dNI zzvp;s&K>?g2ze^))}ysCjqbPpX?@M}ZB{>ViCR}oj~0cy-m&eNw6q?JMsciU16xHh ziapCfshUiUA*i?Oxb0VmKacPnn!md16us9!2d;j+MJd@Mk~=wR|KMKw@2>AsD+Rp< zy1evVTXCny(0jkEHyg@p+U|gkmK@E=&A-SckMHQL)^IxLJwS_eB%7UwOOG``3h9g=E1kohgyLM+y zs#~;dsVVuEgdbNcOOxK)Io)@>CBZ|$Sv>-<$E?GNiFz8>60DXPml$2ZK`1v+3gq;FWNMk%{58*msn(=ZK&nciHgZPx ziSX*U_RVz`RNwLKSNvQ-!TTcNd8B~A*o+}}rLxt1wBft34_v1;8%v57wGtGu3Yv{9 zRDCAR2^bdsXb@{5cYy>}fVv##XnhMTy|M=W8{aOd^Qhdp&y8|%`DO1^A z<>8hed$*pma%$YpDk_(LERh8oltzr8&&3o&_in1^VESvR= zPcM+tU$Tp!5Gqys3d9n5Fpvy3c=^na*RH9X;V+v?$XBL*JHGu?YgcSeRfXE_zCgIy z?jQe>%A+IujH37lNUY>RFjwt=IbSDdq+W#Mko?t8mQwg=oh+fUFe@ydI=+L6$4(YX z(mjGu*&kz?1vB?Q{a%AgNjGxIpgCFir*6;N2JO>C7OOq0vXD-)iR4|=cV}1@8SDa| z5f9*&B%u2(ja7NQ=A-qKkC6{42ER4Rq)6fdt7Rw>Lr$cm__(ogHC^1eA=9Q$-wQl9 zYr}GYyv{cv(l>Jve7n z^&$|Z!4hF%4(ybgU)^U_%4zMF9bDg@eQ@Nd=E;{>rnB>ZX!RvxJd&6=TM%C5<)jz+Kh2yuQ&<_)VgUmY1B>$oQbp+U`ir2L)xwA|DP0Bs zy;}o9Py_ap>KB~I7!qzgNH&WK?mFCI(Tw>m|1JmILytWLqInE|vCXA{fCR7C1>eXRnI{bY6FsD_%)J9GG|m;S8QV#>92NZdWsChK++0%FI|{C(t#;$w$* z&N_PeoFv2<4%0{M{pHcd)h?SAXmf&32X%v6^Vc9~d(|7+kWbg6(5{ zDu8t-(P=4{+pBl)uwU)uwwi?I9o(P{rSU`3t3M&iGJX!rjuF!jO;!kmInY#WbN$Fi zQTOeLr0MXDCY8VO9gmztmCuLDZIrLdB2!h*xvZM1*Sp;P38?+d?c%M4jJ(IQ2X~pb zJWKdhJUatq>vpB1%ToA~J;zVw_1@IpegYpQX1`ryzTY!?NpjL$4EREpBBAU}i$VP> zW*F=>CMozPe=J#^Gg9g_qfH}WZ>5eko;7QhZUvCH5T@;#JUI+qH?3CjLqB#laS7hg zSUa&l9wwO_rP)v5f>aIQ6r#7BKxuMT#Sb5_>kT~QZM`9{m&T;Dihp|4uMA>+0F1eV zxo->)5l$OYCVZ(Z*a&BK^FHkL>ILWMW>{!P zV2z5Eg&AG(X)D_0tXVsT#Hv1W?ARh-D&4H|rU0BAUAeit-lYkI{V*eQ_Y~BI6w;_sW`0)86f;V~e(*u;B2HA!*bq{&04t zp``!==I25vV!QxN7iwp`PW5J2P+M2F6ywazZ0&Aj`VO{bn4cNBC`2Srwei z)m2I50Su6IjC(Vsq->M^htBRKw503zgs%mhg#K|{HbuRY22#OC4_O15G>C#@DYXJ^ zx@LuyBrXSD&dlv4$wz3zdu!p3i(#l;a@k(Qg^-6MFAjbEAZp$OwTyE(azu&#S9$sU zeKQA5q_1;>_-a%+E_V|3hTG`7WQvt}h2$ECaYC<#3EB!&S9ypTd|JPn7W(kJat ztXRm*I-${lteOK5Q=xM036yS&dp=r~{C-Y9sFxS;fTJ7UG!V!y0t0eS0sl^;NB9O0 zV0mH?Qb-nr)lW@FzWL*em8xYm$YYSr_wf9$vLid^Z7`u4YNcC#ZBTi6P>mjDAXn<`xDP(Wzq7KWc0^DGu^;s^9|L*kK8fFS&9&jr9(H9479oo7iRw@Y|> zRH^~t_?_-9^I=-$5vWvQHf6K%AO?j$`$h3%-@lOD7QmMokOpoLSNxsw7Sr!zQRoUd z5qVM`*n@Yur@cdKz=CD8e(3WYCbQzo)_3~w16;}MfDnb&QYI}(@X!dN+xniG@hCwe zKeyiuLO9_Irv^E10bDY!LASc{i37F&Lc*W{EN1!rQn#V1fv|7>&PBdbv#6EAAj)zP z(ZTC|CGR|tfd_JNI`&O(_9H`PA+HA!=d=a03g06O>W+PS-y6`zBFtQ)4{k|uG^->Zu zY5mEEHzE3S4kn;A4t04MZzF*dFf>@SU4q;oJ|LpRk@IsdZhfB<<>m#ZiXOQ6)wMxc z1t?Y%(0x3mbf4tYZw@W(Sz=|PKvVIbVTtMmEa~`n&%(9Ve;sj7vWNu?%B zilP-yQ>}!v;;`^Y15&K)oy5G1x{A0gLWWk2GY|jj+)W+41q^9TAAv>u1B)h0(mB0~ z`l>Q?=Yf*-+<@mHvWcIzNeYFw4o2qj3Yu6;#}*=qAyjl&bx-VLX#jDx5~hS{p$-zt zUQk9!OsbRco7V~QwrTa+x9)LhqA+JgX1U7|g>|27AJpi2TgR&+fMcL4<`qS9a)6W! zap%|3S=3_9Z!=)t&+;G>;Wu=j9Xl-5y#(doMo7dB3kNd+Zn`mtdWq~d>f9m7v!Mft zQXgI4-+hpI8kizaLrTqlXJ5p&M_CkcD%@qY;y|n3q(Nd&Iql6wbWUc0WIwhHYe)AA zP%k_&_mMzdDY7YhhC=OWO;3Nd<3m=>hbh(XZY3{-H#cOniJ3tb3ugxY;@-^s3lNX4dXfB}?xW;(O%FX}_Z2e3Gxxd}rP0Yos8TTZj^jHrLOY1R#d3 z-sKedSwJd(xmR#@8A37ouejXCnS^gVJ25R^j|8W<>9M57+8wr0sS=!R{YAr?TWqOV zh%{@~552t;1PB;h+&>zmjkD;zhv&zKL`N!0tT%!o40{At;WY*mj! zfN=(zfI&JI#vfE zu&sP?aw-Lkk*%9X_oi=}~L?~r-^ zdar(=6{FXL|NKK`k|6X6~!_H!oLZ8&)(ns|axwA=0B0JiXxEHWYCN5sQ7>1bfX_+mk;&PDg`)PCt@=t~dhW4$O|Q_mIW;>h=A@-1Ce1Z0PdT@qqJHH@ z+j}E;OAnp#aibe*Xi2dsY}8Ec)aZeZO?m#Ksk}1rRs~L*GE%4AvXfJVNuh)q4&I9V zS}7(3h!8f0FbJZdq6-Lj)sNdO3th-^5^?&6hYj?PeK=Rw)Q3l$_Wb0Ko_+rs+t+Al zya0ZM?dc^On-6F9IeK9s5j+ih<_Ux@*JT+s)ulDvRYUR?9e&ont>V~5=7R-daVVt5 zOCvSJ!eq+k$^h)liqDX2($4P8!7>Muc{V#osB~s=P;e4b7AF05e0ea5;=%j;xAwr{ zsF%~`_BH#DrsOfabeq`_*A@C!W6#uH+fGnXG_O!d^^vjM{rz2$I4$EpvIQJ?pxsrJ zoMr)I>fTfV!bMY-xu*bN*vRhbOUs4TC#nl0NMpE#?kH9p?y;mYW-*^LtIW}jPJqrU zJmNVyP1PgdP4L~>pci3>lUF7Ied6E0Jlpts$B1aC_(=f+b&J1lAMjYc+~LBC3d={! zO*fStHJ1O%Y?TyAB5R5JNzT6#x*i`@7by}1r7qkxvSI?%i32Ae6pN1x+cT$M?rJ-# zUG73_Yind)q{6VCefu7N+DhRO^~?3Ktz~MJMl^!$*A_ZchGYqy2K`n=eOPwsyD0_4 zCIe`2;RU1S?D&?Cc&wdi>JwAR`E;uIP1HU&7+<23B&0&S(_({RJ53ac-(>^o_cDEhFg*MpDh#hvFB!!<_36w)3OkfL`x82QsAzv z7a-&$?$C)GQ*&#gQ@X4jM06{Y_RSAW-j8Nu;aun?bZR_X);?3W_z29VN-gP9^#8(B z65TJiTkg-_oIaYj(VEy|^K5UbuFbFI_NxMji2#xG`>kb!$>5D8g865HoSn4aE`$A_{# zXLF?*MGoT1q1 zh&ujWlpiq1QCMMNJtZgZZtGE#OApS6=aw}F>JPa4G6CoN!8_u!0D>8kFsE9-XTQxj z#cGVpWMvPuI`1JcK)o%WS2|j0Wip7yys(_&#}8myh`(a8T7#|llZ9># zVZs`Aq6(+cxmoXptswuH1Xm<}{jOa?(SF-9su+oy^6`bM<~@s`YO_Qjz%`(&LFB=I z@F4?06xD2hu)w~ENI58~KH zqlkTg{mT+=K+y41a&%7R%N?kDvAXOZ-dFPx6qUw~ zPQe=pvwDa!vjuwav^4f1SWh33ETmKw>K2@A5*q(_tJXjQ42_uGL}Y^5ikE_8mtL71 zDHgki#`Tl+ z+52bP+V?X0M(&Ze7&y3=5h}?hX|G9&a*UUNcX-}RP%yg(55!~5B%Eso*HQCK<3g`jv7f6Poxw6&~-Vr16@&>vsK`no1|7%4o3 z<`J7;S_xF!er2dxY5J6Sctm<$Rup=suS(A_YT|#gn(V#1tO;ryXCOIRqE39;sc+gi*COp0!WNCKUuqDxd=?OFu}Kx6d4> znL{49mP=`5vQv<8;meSSh(2_d9E&YUyXoo$F6g)FL%OMv;#r^udYUC;xSej=y?;H$ zj~T-ue!+@9USTxb-{}2PtRXY*i+lmWio|v{Y|YGWqPdDPC|AL|>XUWTSp-E;UAv%yHi%Eso#B&IUEfRnx%sW_6n~$Q489D6c9N<2m zAx$i?RDVUnM3Pz~1&Ik{z*oLrtdKd8kNiXx609RMp~#s#b&K}NhB7(j3_un)^%2@} z?!bIFkWKrovXuBED?)si;fEc${-nzS$;1(i(7f8~wL~d&w-s(x>6W=hHvGi-XGT%q zjF`h)jRLWeRZo6%b_%9B?0u$@afk0ib3VQh!*x#pE~Q|IGC^B%brPaWk zD<}G~+v{zX7{rg1m5VD$h;n&N0fJH@3qy|K+EXbBuNqpZ5WsP~xj`jH5H)4caMtvk z`ypJLe4V$ynQAmz9f6Zx5V1f}_a}MCp2FeuUfo2iD7kJ!)j%fs%k?t(bh28Q0f9&Z zm)tzYFi9;&0!d4(V1cy#$T@u^qYBB`9w{e0&b{`H6y8ypzcrsR2p5Kp_zmp2MIkH? zm71VxnzZvUT9%0F#2?FGC>WI**Rn<=?uZNuJDS}6u5d1VJxu^aQ2mN|W}1I_KG$H@E=#u)O=C{QkMtm)S1b%-icn*Xg=3O z#@H*_cH@=gee0EsdbJm#S2kOGRgoX1@pNl9A=G3M2v%m@&t)8fm#`xA08LUSQt!b! zhu?F~vY-gtWbZ%r5IsUS{6MxPd@uf+_icHvCLN#hm9f?e#`G^1?+ z4?LY48UQ734FeO}KA9M0&mPw}!7%imNNHt$ga$@&Q4w+dWJ9aO7~cE6EkRh=)E!jY zae?v#)Bg?FACuinCD=Mn;f-l#6fdoriFQP=7(fba000A@a1|;woxeRbTLLvwj{6Zw zp8_vwr8^VFW-pABt6wrsU?jO>zn$E}W1S`V1`OAcdq}-}PD8PpjEwOXFl()N6i%bH zHjlV64=1+m>E3yeB~~q0V;2(+GlVu35zuTku+~aPk|>}eo?F@*h9`r1A7>{=Q!4y|v%x zBmR8uS&QYcR~nqn$A}7Z<9i-D8s%Gpy_7MAr-kl0YZHQh=SrXe`{yA zPc#Afvx(UpFM1!{7nk7Jw1=h87=p0$++bw1SfcRtzV0nrt4G{V=l0vlQ?oKTOHnEe zkoKTamilTp3F2eZ6;o9u^2LtcoU~NF$h-^@!+X zvSk`~VwUi6@>}S2vzYh2hfr@DEM=0_vU$N$6MwY+g=lgP5vjV>x^C!GG=Irg$|W9 zRhEXhd?kN-RZ)H^C99mgCmjhrRASj%lEW{Y&%5-=xE+P1QuwErSlYyuSA{7!`q~i7 zFB{_&i@moNUb<&$P2;u^OEb)>9_KW#x#e^G0I#5k3;5(vHwSZyPv#!nkuLn9rWnT0 zk-iig%vohBI#uGmkFEpqsoFlh`Og~T{EYb-LHha{GYjid<3eKLO`Dzg;q*+Jr$?&F zf@(h4#FvYYP8}BBDq@mm85nxkm<{6se(dm;azf*AmR^!UnZvpw%3bjhSBJ(|!pG_@ z4pFaosobfPM`_;fQgH~)_48B5u1}o%@xh3<&TR0d%7b2AAIF` zJHF~Bd!6AC+~&%0juh2bDV7^s$o`Zs+rz`#7(g*+(4pgi+-7~laHyf^9N2S&?v;KF zp}h&dKx0kGL;)cdV=)AZ&c=g%pb5V6avpJW2*6g>80LpGX#dNPg~jMMg5H2385lM8 z)>H-62=U~=k~Rs|;*F(|SC+I&)0c#2M>M>$bERvw#Zv7?A!K@19<_WOJFL}Bny7?+ z6ZQ(x)P@plX0+8$!i0%NFWf$nU3{Y^p^^j2#9M&TQ+f)-KE=q@p|V&!m8~sZS5PFF zmHb_>AK3X@bLn>01Ncdi5=-qIf-oeJA7f29EvSR=7+r(o^>N;Dks{esiTiv$Stdvs z6y+)mePQ&)rs>>b2a3DTFZ|$!sBDT(;pbEouTyoa>GzOEec>JKY0v62?X9>bRAmUP zCgFd7;>vq%TN`Lh4r1h^8Wc^YuoASUR?>oIKsfgay~{oH)OgD!xMj=!lCSf{&vI7Q z4bKzHE#*}qnu&@5S6WwKHhq=iV)Zt^1@jHa9GHw$;V-)STzw0s%)UXtoiKu)^Y!=z}ZwHad7r^r*1GQNI)~qIUWBXx3Ye<{FG>oeBePM-k-CMU(YX}+Qr;CH4p=OkH z;gG0o`Ufz7hg^HV6nrRbGyn04>^&^yH2APh??X~=dE?Jn$CLM99y0y%uR~{b;psi_ z9RK_&PWtb{`@b|k{~NV5E#!aqEX?HpAD2$^pY8>y{XC>bTv};8Gz5><13}=A^PW-a zB2<`IPdeAQc~l!NMvH)%u7_ZVUqfk4QB@FnBXr%lg%3$811La+e2*mRB6c46%jRVu z=wwGuTfkH{6Bh-1WCdYcXoLPlzDp?~dMIq_*0Us-raMOdO;-1|4_rD3H_K21ZLA0+pAF%f`FF88X>+4Y?~5svADD}i6(Z?@B z@N&+?29N;CxEU+$CyprQCweoslULzDtJhuFB7>^HHNIggZ| zh%GArqE1?nA>8x!mPo>En_jyWg2hcp*1lwI1U1Le(h#JN7d?P`;s#dr45<#t$42Rn z1kJ>JQ1!tYA`cHC&-7o2a-}A+AL_0rHZ?iLe;vVAd9Y_IVs)v%n^%wP-!4MMz7T@F zp^m`I*`%G?^QpspIsjZ-{6hhLq2=|K&7u`RHWjdOQKD~AYJq_Rq7)Ti#SN_u8n$22 zrM!jpzX4KXGSJd66H0l670b{dvRHuli^zC{j{F;_ZW;nbmh`PP7q6`p`CI2((eMXS z_VY|yEk)vI!q2bf5_|x}>NH89sLLiLD`{TNVQl5R+$;g29Z(3f@yg2*d%=XE1Um&P zc;6@GC*}m1p}4(r0-9QRazLg8URJc}&50m|5b0<@1mtpF6(a-t#!JqZp#%)lxE*@N znBbW+)`F;^={Fqb7)i5e$LpBjQIh)ZpwpA^pucWxwb@iX+%Nw&+*_$hS$NXw?oi^$ z%JI(5k2eo`;AK9pO}8badyU-TN)wZTIkVY z@=r6SP1Own^90VIqS%^p9^=+mEE5^iA*r0y2RYX^bFONgwv2w z^&^zzM2B%!5wcDCPEF9ep5=hof&|%+B`|!>Nh^9{bl|C3r_ML#4U9Mqs9(u4eiaAF z0wa?Bh)O^mGuPUq*qrS%2#H87!j`Q!SPvrmVKz5iiU`6H>`=D=R#+zKs5=Q=N#RFx zZ2BK3{aH@?j{1ZdpA(8t#gU37bVW0DH_$C&86#4zf=f*x-KUNZuw-YlRmO0R8eFf0 zMY74`1P!0!88zE;=WlSP&`=+QFTMy343(y&wW8`a>}^U3q&KI@Sx${Lxv9`&k8TG9 z&;ZIHi6&3cXepwINMsU-|4;}X<_)kYHW&#B~53V8JU;dmQiih-u=s?%OUY}4nD-m3A1VFcNH0U`G9Auv6?7<+En?CD zm*ec-?0T9d{1-^p*$O&VMh8$L+?6<#^!?(V<+>Z#F7xiLCPO#+Zx)?Uie?B-RxblD z=UT|fKLIHY zI`Ksz>Og-(jXI#ebOcYbV4OLZga<|y4aqzK8!d8a%wI}7+-%{ZE`%LoyzaY`LW4Bs z6zFj@NYuID+CxY>$pcMx1~57D0bTiXsep))0XX^9=Uvsb(N`56iq_xEv2rDyU-?Am zK6!s;J^3{!frW}oJbIp{V}Xb#xfOatgVO35tl~-caf2N$uW0p3f0wfGIw);lj6m29 z%!ABK%V2M!AfiK}0}2a7$B~;X6DlJLA?2ROW}t-6IaPO3@&elDxjImmm2`^lrS&0O z5xOa-cq<9Wyh#!4ltWge-y#!SS?HSS-DNm%XVUMF&T)E6_q~L-^C8JyiGL)Dnv{`5 zKmv=@QK@PXM4+IFMZ}vE|Hm2bfSl^8I_u|B`gusBV~M(hbP$uscgGps!h`gs!3?lQ zH_$O62R#m2A87ItR(XNlLG|0VcfoWku-oLBaWeloOGNd90Hu;MDwul8#>)Xz0ghb>Zk~gUC=o7 z_2}=?+pH%{7>Q>i*W@PMGtFq-t!QMF=NDT1Y>Kt}DV(EUdhbe6u+Ze1?1DK~Dk)>^ zBL~o~t5DfSH3VcEZvi#rEFDGmdkfFqU3j1iX-gu55ExJ5ZW%&%mSX5DhKuACz7^d6 zQv-v-%eFM~R1%a=_G%2^N0@&wg1>MEP|>sj*dF5{R9H>%|1*knQh5^gM5b=S^f}22 znq|@-qE_1c(3#ur({Z_}Fd|>t{o%>Zb3>H+=#$dBVyFyB);S_m1l3g-nehNQf$XYp zvFG(TMtqRFswkA_#;WD|09}&Rp{EJagwMGlQcpv{sYen^86EOcza6Lgb3H{WnR_JU zp>b-Y;^a&(w7pT(KPne0W{Q$T#_rGIcT-=Z)XA}UXgPiDo{v8DRqniDamLRQFB>SE z-@ih0I+v?ae^2nrR;1cD>kKUI^KjD#cW4(X(R&)x)4<=UFg3OxJ$B4vKM8bNG8D@JJRp6 zBKhg*sl@3$mfZbl{Z2Fyq?~y>@OTr4qJ!dL=G5EYwjQGGH5qA8?_Y56UkYKOfXkoDRjUb=DusYHpUHpI~DNk2zE zJ(_VsB^B{c7^ky0gvXCW@>j~Q%pMxqlC*f6dQ9Wh?<9UC!7xpSgMP+|<}88cc*n9l zr{)1!T_5JWq_Xz6Rt1!f;MK$nkVl>4yFG3`M%a}$J$ZUqB9i3*b@?e`I z?m4pq(?Ozml0pS`9~kGF>SU#$=#vYPSTDAKK8@OeD=iHp(c`WzJ#eG@xSZ&pUt30B zB~=ATKAlR^7DQvh9*5w6Pl7LpqI7oYw#%2O8I~Dpt@;xOqmna<{oDjh&&m!CVNInX z!^7wT=Zp(5!RxB3@-wm{xg=hSTZk=cNF&d@^O+XgYz*^~U~ zF_JPzp9#BSKg#U45ZUCm*S!?~>9FGP=)R}`8shRUX8V`!TY(^CsBx!tqq?-8EOJ`I zVR27U4YJq=tH`EC+0BtN2f?$yaQjt0N)M3Stn6gOy1d zy|@^fCF)S=)%`P)Ig^=wF0~uu?OLd$Vxtnj>)}7^mMxB1VY|Xr|bmqfgPIkG$Wgdc~$6)fl3bGYQNg z8DqCm>FWg-NZdC%>Ig!L?^NLqL^#}5{Dc2#6FO-+>v2gKXhg@$ysTxk?}{Os(G<6f zYn&iF)H^y!@`_C_@wwDuQ5{Z8MLMQ=t;1-hkVM{Yan4-{I*h0>n!})IInN3tC%X-q z?jn+Uj_yNJ=0bS19kl`(voPdNu41Sk1%wiTx*=i zu5dP=_Tl8Ue`r@B-DKR_9!bB( zX=^Pmocu_$efxb0VuPxYHZ4(F)EUx9sl@733yygOIxO-%kpY#eqlCl7eJ8;Mx#JJ;jC+gXj?-XLWsJ4o z7nvX!e2%7iaI&gQJIYV0)WMu3_uqd&vqA{EAPjCiDP!iL zC~(5DZUCBcmYTzR@9xjq5A?!(YtNT3l0wk9xP54zoH2yXv&1NyljG2oMss|=MEKC) zMRH@Gma(+;GpAEWB4om5VIzvA!tCC=P0P?w`_WV&!UZ`)7f6Z#r}&e$q3Sb3z(5A# zzR_VvNjIF%VeuL;@!7mnEN2T1mIJT$C+->5ZKXhg&NPtemSnaTes1}?l9)+JW?E?b z4^3Jlk8bSInT9@`K^n+xxy#weKNk!?^@f%=DxoT*xh=OiQWk@`vH9_kG~UN?qApYB z9+=q2fJW! zweh~iOYZW@-IYi^w1`wJup7MaBrUXvQ31r`dgXD-Vdx9r0#|6H^?=QWYFuo@M?Og( zM!g8wCbEfs(@x|kjtH( zJ^`+zq`)v_!x%Cam7(=TJvSl;z4Aj;CP@VMV`F6%_;OpF{?94Ye{H<^M|^Uxt-S2f z8u)4pNiLJWd~lb4Nit6ABc1A|F4{U0=ZCilBK`I}M~YCu;w}(>aNm~S&&ky_n@SpZ zxPP%c8X(P>*SGDD7l4`PtqYSTWNUcb%<4O$5=PDQzc*?vWAl7zv05xZl^UT6BGd<} z6-w*0_EZTfx{-VNEs~A_LYglg4reeF?Ax4y$8_}Y{4P)vMTyyBqLG#7hifG?GvtlJ zWANDG8C!LX4N6a=s7h0=Sa{Lwu}A)qUG$A3|NR@&(87gPBmJyK;)<(E-jXprW(2=I zuRm`HQwbMQVM~^HQVS9w=19ZREt+7-{MS(r=lXI00({x|nXy+D8^C(|#rMA9VPu?|h)GSAMK8}fA^ z0LoR@r$qa^4%G4iTEzg+HMkX$^U?UkoY`6Ley*SxI(&|3M!@n@$Xg zZ^#;`ps+a$pHw`FJ}LPv@F%}yi<>$haXp;5jy9i1D06`KA*mN>t4Zc+vcir~<1h@#(e)u$a8!=!FBu{2}wT}?3ue+AqBQ0 z9hz|%+_MsfTtiZMxTL#PvLGfpv-XdZqWs^tgn_N3B`22d+>xP4ZfDl2Vxtpafdh-9 z{ZzO0pC4qBtnvBna5sxza{(~wMpgJ+=EiCh*^EnBOezy8u%Xy>QDDTffRm@SA zbYT#e(_11MWOT6he@92skK&{9$;#X#*NF_VQj{5Gbb$1b7Mi@o08y04rpK9gNw1K> zP|*W%G*F981A9i4amr-w5$cTOB*RgJjlgS`+xZre8$oZVkuV^d#@&pIQgGMfYBDXx zU_@D!HZ&vI68}kshps6%;f(FW71OLF4j2m>#8_!1GT=PY%|1Z*jpxkY1w=_Me9FdX z`0gPb$I;ZY`~UO1kt+^`IwsU2Fxc+Mp`x8kZ)90Rqo)gI{>)8VKZ>682V}gn2FI9< zY>eoNG>WgM5b!TGY)HDqJi0eU&s6(4uYk3`Tt%_^?z+F?tiZaXIK3{A3yxx z>-@j7&ZHUht3{7bbmo68WkB*RgnA&Srv+~tx}}>Yv7rb(ONb~LGSETzVgeylM7R_J zxiNIcoY;JBD(NW$!XpvqNFEL!vhNc+hXTOX_aQ}?6<0tEq1JaGY4g0-=Ra zht)*Re5)1rd&Mj3NZUlF9=sg~I3?Pf1K&d40KkNXtlfo5{O@$1QX zIO9$Zll~Ti!}R-yt{g*?5y0-Gc7sHbg34di06)Y5?S5)@0dH^Qm=pU&jPQA?+@NG5 ze&x!86}VTGMy|wg+Es%hs@9(5ql=_@umtu+rJF|8aBRYr!E&fhw@M0a~Ft5i|PbjgF zu9^guATWy_1OwEd{v4bLx!5oY|GVYqtDK5q7RFGJm6+0Tb{AHO%zizcdHaqXyHcEO zex*EBy6>UY`4-7Ft>-H-`eCWthrK(Cx9TkTpm1(o`uvI3lQZLsVr8NyHi}5>WUxC zx#1Q9z>+2v!We$@!+GoD1Qyds3tT(W`UWlay`|q^9@|ZAI?dvyOCef6H&*bBt{a}` zHoQN|BE2IZx3!K+xT}}{7r){^Kg+QzoQl%UIJ>T+mIB+WHS=;)K!5<5TyW7?`yl%WS|5e`$|Cnc-cZz>g z*elUp{3p++*Kg97E&Heb^LE)bftAISUmLvpRXvHIKz6YWO7%gD(&x5+6e z@G&oT+INFqnl*P?MQM@Trl0FlMozY_37H>|`?x-|Rf+z_bnCa0f2(@wYb@$$w8O>IRu*XId` z9B!_ggxgoI6tDC6D3v?w@4s!TKY|xtHc98l&7OzfH731=WTmI0)Z^J$#s$o*;YPZm zmGc{tQIavCkb!n%;$Kh4=+D=JjKtc%7ulyqO+7BMbeoGyIM6^I2K+*JbhJ?VC&+fJ zO5CS2$rvphbZ;YFM2A1a<`qGHz>|4^U8|4&`F-l_*~>*J`zE5DJ6v46LRndP;A{-8 z)xsa`{S&@RN|lP^Qsq6LJ0rwbc_zc9f7$ZtOu+Eoe?|hCYN($NARhH7$^QLL!aesv zZqqf44iE`{`Swi%eYf!B$%g?xax*w<0@l{n>oWgYx@N(+@bF5FH+z2cb=4!TgaMLM ze1=^}m!{Fp>5EscY;j!kcNL_YOTibSG3OdTe!Njhdwcsf%r1D3STghUzkWGZq7ahq zarlYQhzLI1+AN>)3DT^3ye{?t9<4p1owu*A22k|^T-t~gX^v3+^OFa+CFB0QTI`kYl1H_j z)ipGLO-#aVrV}?MsXXQ0@w(da8sV`T8VW*W?iSkRj$8zvmF%w^huK@UADcs5)E8NpSI>2BqmAs z7vx6cyX_d#=D7U1pm z;GczO%s7UbNA|cl58&P``|aBggA|fdR<`L9j!+4wvxK?0M%hoT%6w#DDT>6iDU>?U zABX1vX^#Xjd36T|XTF zMZio(Bx!^C0%4Obdmn+s(tFRII#|BH008~4R$7>Ciax&%NM>|w>|x+K4)`hvd*cAq zF2^Y1ZK0Njzj4tZ960he1fa^l&oLW(4Syc`(H5#cLkm=KV6zV@fx| zURz*N>AZ6%-?C-P+MYdo4gMfu$$|ch$;ncH^`>!gana{MLO+Zd12rrbKknMBL*+Ff z`Snuuro`|ULgM1b;aH!uaN#7o;IF*k4GsX2w#P8eY03eMO-xO9;Ue3RdYNG<#I~`* zw}2Nuf?GADes*&NAkA*b800N-rcIiu91cy&OE?XusRXZPwhJii#NT+db6pbYj{PZsrThRYx4T zFa70UbpTm;Bk4=7ou!gNCv90(nKNe& z1j#S0T*7_QR(utcHh zmy+Ly$++zED;knye7_kUi?)_m!)x;hx0McjdNviX?C3yj%=_OX81*uqpI<$0p1C*f z^K}XoS-WPhiocWMwx*?fgc`r{X1cEfWg&WX+vBn(Z^i#KMs6X|u*DT54fNg;BAq zxmoTOF)G(BZBpHjA3J8!vd%!fXx!FKj%L{w3+QtqOW(m;ko&W`ww4z&VCiS)>mhew z^bYKh1rEIPEc0@j@+`6uYmU5s@1nV3L(d#(*Qov z=kcaexaE_Vf3o(E_4h#0Fy&kOp0~G82jqYB;zdcY)2T<(G-JEcA|-MAiLv7^d}Qx8 zcZClhIg*N%#ebT}EpkG$vJ_zE;qvwMjT{6%mI|&Zp#M!oqKF9lgl^otTOA!UpbXPu zv*L=U%uS0d4m*7K{n~}b@oCR*-YhHI+4&gVq&={}{GuXF%s4cn66lJ#0P5d^Nl8hG z&2PPsnS(Si--e4pKsHaDASfuf5j3gykI%N{0bjYHN2&uuD~MZYDbHaP{-+BX%RC z6>9J#3C)@ng_g}{*j75oYp3ddjNVUk2bVnq1|6})WOZrHliXSv9Dx@)BHW>%nl)j<1Q@W!A}(!?iijwDi)6=a z?OGJCXldw{gs1P*8QQ-tPB#p&@Joo`ndIWdDHwGCKiOCaqZ;?lV3{_5evShaj!ShJ zHFb6F@`{RGaAcMTeMhZpRt<%cak{RU^1hL2N&EjOIZu1aJ6oDEcx^|=7DjuhpG~)& znBb^#Nrmx?#lRn=7=qA{j>yTahA~rF^-nb0p+Pj|l7h=jRQMFax&TAho|Oj=ne_Po z8uVDQY?&zY>fO7{tJgI*PZ1K5Ld3nF9|F3ZTMhNA&tfs-hHu}$i!nZLA53?$-o8EU z`i&d>4BHJ)Fi2k~cHPXGGr86D4GhwupD@AcVxON0qbIhKd%fWQ3+bQ7L|-&ue+WZgmuzhu`d&4T>= zM*)I#kzqk;spaRr^A{{&qa>sc?gu{Cq|N7E|jeN1Z^P|Bm>2Tg||_l z^M>{wGk&o=;EO#|jdB4>Q?3Nn{WEv$*a5+-s=b?A_JHr6Jz6Lo80O3(yB0l*)Q|Gk zCG!ak*tQq@fT@1;$jI>*;_71f1ua5MD~(G%ZKq9}CcjCUx!UK5+@3Mg-rOt#>Z^I9 za;ks)`SZt6kVJYQQ`KM&Y_dgmfpg=`$&0p1Fpr>}!UPv_*YBehY)Pcs&{jA5u8O^@ z-W?P*PK-}TNbt1AlPcKtT5I*1H7IsO5jFUbt;Ql^e|%A4J+yfYgAltBKsoYUX#Y$^ zn1H=#q9DhKLPU85X7I9U<|K$2#_;g)q+I-wsrRmC`(XDio-w?<=@(q{P;a|Dzp)~; z_0Tw^TXX`Z8Aba|U0mcDm&uAk?78>u-9yh*jRAUHfk{PzB`+kjF)e=z#HYvE_s_gW znK?^YuIcC=6hW9pH4&6s4YsAQxcG&2t4H2a zS4Za-bRFGq-i$%ffoQ6}6Gd7L9)r;I=@CdjkO56#7GvDu?s}b@{^J5h4K@3Apv=I{ zA!5)cH5Y&X*e-hO)-7#qZMr=B%a@lL5K)Vwqv#p-`55YRGd6ZIt`&cUhN!0llsEsl zapMpLM1anPp$UYr__FcOaSToct4a)6+EWa0$Xfr;QcIWiU;Ggf5z#bl6|E!EY5{H# zhjw}bo^$gCvs6JG1XEnneG41!3-C0TPp@CS;*QfzKOvcxk#YOkGjWtKq3P+eD3W)v zSz;*ospD5VoB00u;iE?#VR`iS_Ev*PdxeT1&26LugEF}rSr)nIaAag;fA`l50eyLE z=j$d%HNZ=PPHY^$Oez}w=_;~L_V%IRk}2oRor~#%n9FqnZ448tB`ap^SrGVbN9;n| z8OUSm$Zp06>=5_sZeGR5tn}<0@9gZXy#J2`3YM6 zvryOnld4c4A31c0XP)_$!${theh=%5QpX_XGNYE!$tGF(Xd5w~#qQlu#EW|50HYLj zUT7MHA#JP!%X$Tm*1^HbxYilB z@Py!sKtZVT6cOHzpsh&jb#rt)giP3t?5Vr{HvYB!&Ycm|^WfVJ4i2vN|15#ZIP>VW z*(@tHH3l@E>3r*w39iL6&<3JX-+W`A*GWml&9n=AbpJXcE_Zd~@7?NMy&HwtMNq&C zFa;%;v1yX3$;UTg!jq0wD^{3){P=N9D1B?zAaa@x^2wOc>({S)8*E|u!H;k`=R5oT zd-;V67dlX6NJE+Bg9MY$+L?>Mbhg7WieUuBeot-w!-B<&xQBm%B&_&Qf{cJ&JsB3_X2#IA$-IDm8p~n(g~xa(JN! zkO@{XhJpAFL1Go_H!IuaI6&Ckkf5#R=FWp zZOWvXGziCiFigYg7b=|EmKF;J@W3X+4RS$pe0+SkdQ^a&Scow|GaPx~{eOBr@%P9` zY{7D*g7*Avhr-LZW-5L9k+UUtik+QZ$VHJ;7BB@)LG+HIHTOB7)f!@3_WG|N#DnuVtof66dsX2Pu+Cu*R{#+=u=VJc;rmus8Qyd)~ zxuj4-7?f@TF;d&!z8Rv9+|F?ql><$7R^isM5rkZ&z!{oeanaGqC-_#mw;6%#A7}~r z0E<3D?m|`x;2-(X9e94S6xN(RMu{93d4K8Kt6BjrDFl4W7w%3LXCe3vU>~{yAVC`s z;LejLW{ftl5Lra4qm7~RzNpTpGPEtA(79IlysOJ@y3w-b%VQy0x7{fqC`c2I6};{O zY>vG(mU;E*(}Jw5tl`y?sT02UJl@D?Lp{L8W@imz?23G=XM~9C8}xHqe&8EV6%fEE z7Nf*#xTDkQ(T|5o>{8F?B3NV+Ti*!b=H`ywKXr70wiGKDHjUZy;&dlWoXGBH-|s7G zL?s@#_d9Y43u9OnfM)fmsLhrRx={4P90h|YB&+%YqUhe!Av9ohS7c^p!r-s@#I=}- zM#Y@}CCcX_9=!mgV3HO-0zo)y5RJHbne%-7vrTz`@y?sOULP^Ly47~)BCGu5vuDrB zS2+)@7h|vd`t|FKS!U^xuj?Q{IB^IRjHdLQImI(;s;kxTi0>U7!}4wBpa1F8CpL-* z)*!~$8!$vFHGewYl94NQK)!TKfkPM*4RoW*clIqDr#QC?nDH@w)E~!0m~n7+mP14F zbk~O{QF>rzc{WPe6Y-B8Ju00t7sibUe49Eo6$b297|Q(*v)38FWt~~Hh@rb0dOBc{ z$(?W_>#1vdnQx}nFC1OcggZDn$s$CWpat4|bC1U)S-Bv8>{^RNpGPaF8KEX;Z&`#* zs~qq}%61m+L1jnm!n4^Lv(lQ(bjwRJGct}Lz5yXH)7;uPw?6|Rx&WwAEG|=#pY`^) zC8{0w&g;gwI#L>KO3arp-sztJ2~S4OkKF6Ockf=@mG1J#=Rpy3z%ryL54{x zzgun}2HK-ob?I>>o=-3=Z>$FkG|Os%3D;s#(UTN4XyQ?S^cFJP@oRfJkPqeq zUONr@8VfAe3jQ!0(2Jjat5A~NMHuo~{JS2Ez<}|G=Qm5ua;%IsWom98R>D)e3 zIlu_AwuLGxDl<&GMA>rzd{1RelGP6Eo~0ELPVo*gI#%KOgNF~<&|?Q_>kjP0Ak}%y z8-PHq9b57BaVjbZK4j>|j>iuNq2!%N$LI9Q*P=`ymQ>5SL&O}9+WgAR`>_gh_Wv2m zY_4x?T$DLS)ae=O3*Jz`O6rckaFXYs1${bgwY=Ho=(=^oSZOw@oLHQQ`#4fdgZm@H z<(Hj2d9ok*{8;#QtF!qyqJcyG>EY#n@(0lnpP?T`+rdUZ5p>@$_FPdW59Uy(6+eIe z+!ldT7L^2Y5H{PCR-?ox>3UPSq~K}MX`O-&2-|R}cek2?QF15~4H=xvKty=B(dSjP z;5r>AHldx@YhCGZXFNr6tn_gtfXh!^clr9B#%hM&?PW`LC2Yn)(o>uO>Z7!75KW-s zvNaWDHs(e~(arBrA1@6Wa2W!h>V!TZ;kkbO6RF|xWjAiT0zv|eTn!KOWpD3s)I8l7 zUx8V#e1HD@^16QWCQXvO9`b7?L#WhQ)%_|M+jj`fQ18%$@S@{UQI>zY?%a{FeFiX) zVE|F6mHDvI^&O(ApDtRdg0nwuS+#PdVe6jnAIzyLLDaj0mlI0^K*#}#ni;=A56eOR zy$uY>m%+h?Q=OW9_a8kHnit2<2u+)I*}W=wsL)4pmU4j53fCv}3kZz}2nevpb7w-a z;_rf8Qr!znO2#s5U~Q-ujv~jtLTxDzs}$7CljP*&4u^%Mz(q(PF_WwrAu?^1rvw#c zs6wh)EZs%#)HDMxWM?Y^3d7WOP&<<(Rx{vCP~W6t4_OtviUCAEjL$x2_G~UKEv@&@ zZ?2*@K!KX#PKAp$JHgC{^5-1+MMBL+jV^Ox&0%j$*Wr=^(rsE&q2#FEVTr$$ojrSf zlukt9TqUK=oGJ=|WkdT+>|&hwDa@;vFK0_jqoB3G9QDTtmmHq;_sc=_EbR{D919pfxpHfR?jnidh^DOdCTMp zZ*b$CI^W8ucfuOYvUZ5nRGqRCQPu0Zx?Vx~zlFIAI3-SQ8j9*PjA-XpLoUgD_e3)` z2~v1%#7&tsYqqoxzCFRW(+*ap5|7Ajs58E`j>57>=js56S4-q z`vDNMwx&j-%pV5isyp@d$HTP)0JadU5tu~Vxqw)>ESDMV#pWX7`gQ> zwr}h{REx3LSbPjTxRo6C8(v*YX@Ab2FWp6X9STP@$Z3!CQ7 zvuD<2e8nXt7A=}+IZvsF@$)eV2vLdX1Z)AH#|dVmsGJKWoUo~>sZ-gJqGGAX{L@ZE zMn29J)QO#10pJK%cjk`dEHceFfwKPiwQIgN6g~hai}RDg@ZBq!cQnsYse{2oEP&I? zOK`r9U%GUOiAM3GgOI~SBTQwkt;2te-RTz*t`b|6v|!^&iKaG{ZMVm;e|V#!TG8S# zm!$|CJZ_vA$W}7vIBeUNl9MShb*h}SG*A#{qR{6yz{Jn6E=#1B;$p+m7JEa4h zOcvyi<1ff;G;=Kkn9b+`0mNuAchSl}mL!IaTMefnRwJv&z^3v*e>z$rkOVv0Jkf?? z?Oss07dB@ywj6O54c|nZEcNv#(5SAGiPIJ{*vMg9a_bOR3wAu5 zK~P^qL&NHz0eL+4aSV7TG@N4@w8`%R_N4wOh8Y%j67r)RZ!7T-2T*kqC=2|Kd^mlF zPMs2FlJP{Dt)rgW7tSA-UnC^lSso~gv$GNXNa1sIUKn&F`1wV}abVTNd7-1f1N~8v zp|O{X%XlW?;7OnK@bK}3vmh7Lps`=kK{%)POLY#G6}$5i%}sv!wRzv5sBpz`bP17N z1*7K3uC(2`7+U zMTsljy?Z&N&&woKf~NsKXTI$3%hc3#d7Bq7fUV{ecMz7i2)RO-%+f{$ls4ESMcbR`c$P^ z_7F|#%g>)r;x-WnWY8HN_~&{w(x6s0p1gy9pd5to;}oELQu6({<0A5QWq^$_C(0LO zW_4$$1cqPTLT6R{LjT6~>)qghgoNprp?C`TGol25Hwxt#kV0OZ2?7Pvk1m_d>V#zY z04M|EI`BP1j~^FkxoQ_heMB{^dK7#yM^}tK13*u2q(u-G21-Z5iJW2>G))x=()IEA&8fsh(OZoC1cY3`6CWS1TLo5&Vdl@Dzq~}0y#r48sc7tt zvrJ^4>g8xNx(h5fSoLZN+X=VM9wPPr3q35Wb2q1f|G* z$cdLXU&}Cw%?HjH5EwXHR+blG_Ha_ttel*jZj=`&jo-gE3tt+GZGr4ubL$otavqTp z*xPY?bx|5qQ@$ECH$3x)V94Uyw{IVPD0dH!xYubFNlTY5RR{1e;ImNkTv^a}*9^fp z^BgN-=;7-Mh1Z##;U z0p;}=K!!cXq7eS#4vxi6ir6RTDmCmqf)05n?jbXWiLt$Ef0foK2a9OG;g>ymp^X)z z{ZBI0)Ku5dNXJ_D19=Ham}GnepF9U*A}P3)qPx4h{ns=g&&T=IDLt2GuSDr*g4&nL zd;l#KtpzYo&SugO*JjB}p7N|vk}blIOq(%-pDDO@?KE26v|d%;xB6UC=!PG)RbneS z>6|WFaUjCyI9ie{+nXhv*6Tj{2lUAR>UxH8FQzth-)J_$|HT2P77nbqQv+jftW!{) zT~*@aOm)q`2S{(rf>q5Se;`=pJ%$C12LgWv!s~EE1tuD?f8HmcZ1DuVPuG)TO~mO< zN06MpwHE--{&%}VuU7p5$$|dVFX6m=FLXik!2QIF$m*v38oDUf)YN2B;=Y+n3SaJb zC||_xIp8u;OQ(<6tXi_fIOA%g3tvppA2g8ZW6 z-`9L-PCw2N5ZPWHcnzuaE@;L(FJ5eA?jm?Me}eRWhEw@V2q)q(0D@HbalaC>DX0Wa zQ=TdT>KiFoYLRD?RTKiQb-s48aynkbt%imNDXhhKl?S-Dtk>LY6*6Y>JqmdI}$RBv*mmN&%1l~`UOM94rIC??! zYlM7M?~|<44A{@4-5y|E2xQ}u!i6wCUtety7N!PLWxxTAJufgemd1F*(Vh;f-IUu< zk4Q$GJlW#13DqH(F-5^4#0wVTkn>cU*8YN9KW^_WH1XTZ7SEQE(F_g_j@>Uv4f2|J zt=MPKL@lWK+C^wl=mdcZ9D)m;gAyQ&+OO|&G}M#}a_-<4(cIdViIN%h*^eKM-lKqg z^@vMbqJ_I7lJty>j3!;=+F3&nq&dU)p?j!3U7m}&+=~Sw(FxrsJ&8-$X54BVx!`ZG zQ?K5>oy&kbP|@+{!F-9$0EbQi{2~C>R~ih|3a=MHVUe&Qv)BNh^_tCDQZyN7KYjK2 zb3@iU`9^JViJ!n$n~Ni7scwZd#|ivuGq^L{0do!F5yMN3sFTU!M!yPP>Qh)MwvkV& zzv%So=eDgs7=9P~FLv+QH*enPN5MTPYnf+*dz!Pu!!yx?of?n%daJ1e4a}${|B_sQK zEblyFVIlS|^quvPm@L6)RC3GIpg#IugPrm6{rhmd>#ZkGmX9+PsZa%_0^+Y5Ad0uY ze`^Qob#9!g8uWmVGB?elV*1i=L;`vAm4Km!ZqBekOGC{d!|fTZQ+$D>YQ4I-qb0A>r1h|no*1kM=& zFq*wxKh+)UvK|7DmwkOvD6FcRnwD@dJDA|c0$Kt6JQ6>~P65C8U zRd5Fe)lhqbHXlG@-h^LIb2f{Tl+o37-XC0HCZVrA@CQZxBjo)^k3N&L`P=V_bXJC} zmAfPoVuW=#SXx?IR|1xlxL%yVdc>vfo}LKMk<^l6kS-5nkGg6&TrZa3MR}7n?IWL1 zFm52F9r_0pr3#qAqgEb zYt}4$KUJ3}mxYf_U33z^F$EQ&wyv(p4zbxR0U%18AhkF7C64!cWFEuX)E}J0{ls;} zzTe+1!Sr>bRJ}iNZywNYM$O*dep31z)(t#zghlR zYQpnFUw;V;4;4de&;+1hq_<-5sh;Deo+HRFq>%LbbitUjcqk!`VNG7bk$(38&PyK@ zx`0IM0I@Iq=;+{Z5b{8Ke3i4yoF_eTeHit7^&Yh$V>H6FQ^0X6KX!0r*#ajD)Y8-( z19UM0aN8wNK4+B!3+u*h^7Qm1DKTS@JL>@ER<^uOGcdiQ@KK?(|LfNSQBi_8C8-cz z{UbV{b{>)sB4@Wm)wo^sxkfBGBx+!02_hvyBG{wy;$$`JZ3oYsnF^VN0OU~A+ugn` zeYD}A{9Y(H!U1$pTSh`q3X+@OSh|H)>H)E@8c5 z;XrZ!I7p&0DYbJxSYaqZZvjid5jmH)czv#gBkEu9+A^pu_(8lWRxVnEmuz+zV$3$2-=9e7D7jB< z+mHOnncF{z#P{)8sUTB7N#0)l2l5FsC5#xP$_N5KQB}q50Xd7laSG|Y&Lt@Grj|m>9fKM|zO^hY zEDUTlmlV?d-b_c7;NgfcK4_q2X>oV?r2Z%k`0Btu|NZONQ^<(!lFra_q?%;iL3ilc zRr&h>ih?9)`e;V#jlKSmeYGRlu9d8L{P^^bkD9Sg$ZAtXMNjWm-H(Yn*-eXJ#iu5Q zxdZtuV!13nr|vHVv)Ts_^qISWLVSmphre*e=)@VV0r=~RyqC*yo1IzRW5kVxNPMj> zqa0wit-W1m_wL<>wcx4&1DpY3s?b^nLtFo7Z|l|lnT`Pdbs(aO{f&6dMs2gk&~V@n z0LL+*;2iEDyUGBJx30%9w<1PJPA=1Z)*)?6e5c&ttv}vc5qwD6tmbZ74q*5&jjMEeqyj zGK0Jzh8Cr{rd_DwBJ{e1sUT@BU7#Nx85swrK)xzR4+Qm#7cX!-`dUDjlNpTOkXo0( zW&9RXO6G9H2!-h4p|08gr`&4GmSbgQWm*Reb~KNDWCg;g7I7(d?|D>4PQO+oMMnY5 z8qb`=(HKWr1ua+HZ5;qD@Nb9G8MLvniB0+q7}aYX$!^AkLN#7xx?egKQ%7|Iy`Dgi z0JnQQ=J##lu*J9x%Vl7bwstf!hTDXFgCMo>-f1V~fcW&h+aCUcQWCcs3Umd)Uw4ZZ z&z6_hzG9Y@YSW8htEZ^tA9|i+lC>R3bkJ&n)DzfxI#3$&k^ti_L9&`}TJ83WG0gzc z{MQ(&s~?1AAfA^mHZx5u5q`RRC;$pW5)fkqs3{k+v+EK(8~y$_lGY~3k9b1~Cq^Jr z#6ra5DRAU{uh&Y-c?1I&8eC(jKZ_e+pCv)dP{Z*v7}$QaF7yrFmUc7HpmaaG{rmT` zC$?O$Ln9iGu=DoObRnmUsKR3x*=Om5zE^VkIGAxK^~N&P8->yjZhSkA5yTeN7A z4oVrP?Wyy>ef=uTyz1_j!j`ry$pjz+(3Lk-0(A7nljbO6sjKPhyPADnGj`L@U%wQ? z+MzCLaj`CN*pR_H7B!Yho#cyzFgE}?)BhOGWlbd61Ez&WBZHI|FP@F_N4)styBQ(0 zrl9Wzq7&tP!Bgcj(dhlOVMfuq08op9sni1!0fgl79F;Gb zO37(}T@FDP489{0g6a?7mWLO)c2%w;8wj9NM;y5s9`p;)k~TqQi-L4IlHdEsnSvzs zYwE)WyKJ64g|YvKyet^e?7M^ptL^%(IT%N90I1@;XF!_hZyfL`(XSnXPL8ZE`bjF1 zkjF5g=)~Yi;6ZnO5Uv=!iVI~@$VL8B7XF_**8{swG07qg`E^LRi@e$kc>(2BV_3vu zOh+l5>4lgrB#bL1tu0SQ$R|P4Sc4O|Wcytqj8Cw-v;gHPq*9kHa%VtXkn1-JE<-}e zXAD}`^{|aRg0=?&rP*?+&}rclcz=6qU~^+PA8=&?Wrd-x29@K80(UW>SJbB>A|kO% zu0hUBX-#mZvRZ*d?W4+&y}-Rb4C#k2j~I_QD-1y+YVTVhzey8w6mmLnMPpGKba&e$ zVDjOH(upYcmY=+aTTux3$S*FgMbt+#9GRdZmg=rxoH~XCXE?;r%$c|ELyLx4MA45+j(4GaM>UIOqDkDJI)Y<+cf6a(HBwo>lwy3DmALaYmy7{)!(m=6E-QaVC^g284g!mpQLjcRT z@M-QRH9>PI?3aXJ|W9qJC@8LvO2)BrNOVBf#$)ry5CGPMhJ5U>~EI%YS-@FL=YecQWHt61%gyHbar2W zynawxVB7+&y6YUZTA>ZQ4W}H#98cCrxTu-9R0} zbP@T+Eci5-j^G2uUt)typt%mMZxVFkFro?dEFlZU#>%W8flmz8H}=6>Q0x z`fH*U$8md0D#jE}Z+LZUufKG281h^-^qa)z0b_4b!Lb@?*BCkp)c}}aZwzK*)UfqP z8v>?`fh1N9{dYbF@uJ=0!dY(e-ID6>?CN?9hzbvb=I|ugYm#8Rf0V+5W^gQ8Q_T3~ z!;juV^f889L^-g340e@Br52fL znfs=jP0N9Hx6Kq5K0rHhx%jC(0K{Ml$3YTjQj#t1rXQV!j+7!rm>AX~^%E@vCr(j;lbwqXQI}l*UehS);7?FC+sh-ps(+ zAQ=h3lP_Ps+|A#FHET&)hpJH%$!y_k;U0ao7RF#}BMT0pt+7X0$%G;c^91}(vFI3A zt{DiJL6LOrkO|*Li$|Opz~mH2?ST)FI}O435Tq6W*LWdGBp&j07mxAQxRrbIVJK4$ z{wV~NksS3+*gOu@WdIK$Bmx~=Dk7qgFyK>z#;k}J&!HNMLbYWvP$d z)2tzYwpn!TSY^>I9&86EYcz5ggn0mKI2)np-B9@MTiAr~)`f-3KO5}pibB!CZWLeX zuBNRWuJTIQ$r|t(`rXX{@Pwd4y^ZdRta2oqvlTv%-l<=+1}SGi_{{r5SBz4)(RA`N z)0Vpk!HG42ISXG2|851RrhJ}%c=SSS2=Ao6i$M!($wNTmx6lfM+5;j)Zv2$lu z@p~x)s7h`mtpjDRg~#$0NY%B)2l{3HcDs7#oP-%D4@smQxeqTD1vBomW@gh~lm|1( z{RM-nJrEK6j|WsqQ`P?csQ4@{d7&yKDK`w>$s=T|`3dkB+9Dn=FE3)z!8Zb4Tz{hy z|Gj)(w&OVT5X1r#poeW^TR%r~8#tYvq;pBuZ#v82IHLf@R3il@eGM;sh;_t4z8?5( zE<>OK*(#)@&o18?Qy8lOSm+Cih;^B=tQlxmNeqtzvH?d1qtbZkGOy>cHs~B!20O4% zzYE97-$!a(>VaIpgrE&Fc$3z%l^tY_zhP1~6#p0v9RPL+J40QV#=_f(XJ9 zfdNtU4k(tmbO|UaR?G~NVhP{`hGDxopkyI6Kcon#ZZ76Q;PT@O1~@QR&uGfBKD}7! zNiZzB%Nl5{NE;7fgAuB+TtTmyYj%(4bgajYAFqK_lh6eS4kb~4Z8S4`V*TS+U)%aE z>u*C1Zohqd?B`oES!)S&=(wPo@M69;sjf~+C~+h0u9K5t&4hZAsm3Q-&M!iODO4vl zjg5lPs1g8*YT8Apx}T1dSHj~?F+nogb=`ib#L4Z`9>d@(7=MhL78kEoew!Z6&|pZ9fbW3 z_et@CR_?p5*&dH`3nveR$Phj~~Nt1LW`;T_giJ?)-y5FX`c@L#Oj3aUw}z!(dt=Hug&@vBm4&zBwR z55liMX_iXPgB-oUXd(7C|+mYzbeXhdWb4dYLjzy8r&)@^1mHHsDhDUnXs=cjE8%K|riNCHlJeX4`W#}6% zLdHQ?^zmcP$Uc?8{xq#5)Jr`Il1)c_;m_of%E-v*2Xw2asd-pp?H>KvdM7p8qlNL>!sccoI z^V@4ZxA8u-B~|jR!x!ujv_P7%g|W-c{>rJP!(awtUvU?2`P~r4hcYk*{Z{79!G{kXtoGC@GlUfF zuSl&W=N!;WMGpvxCNqWjh5AuTpi_-3;F%)xX4X96h4NZ~mXwD9C^OwL6Jz^OgjJ)H zL0YTB^a+}zkhyR4w?CjTe%c|S0GsS_h|w{{C!Q9rc#cLt62#Oitj-;vVNC!VNT&() zH$)$O)~z=L`1x;R_e4S*mV&tZ{z;Z-t9rVP92g=D<#n8~`UbFhakFys^1?p*Zs>Jk^Ky%Y+{EZB9?fx@ zzk1HTS9^u8wx_L>3jxi{(bB$HR}pY>;$Xy3H+NMi@^%iK(4SXS&1!0d_sSKUn6ijw zMJqZ5VU0LGcp_O?nuzA&_^!0*O)@5wpNO`cPd=xa7B-c% zS`V1YIN_e6l(8}-FY|VZ@18_EYM%Z4A$(|L>%ehg#OqIa#e~a;y2|>%KwSmul4+kl z`{M238}O;=284BLQD%R}m?(}e@jKC}q@<&OvMp!!)kL>PTJZNo2!9(w>QsJ)VT+yu z=Z=)xiZLq^713^FGvV zf|V6kj4M|fuUJueFp{|kpJ*@8NS*bqzcjEAbre)n^9C+6FfzL3Huh+}z-PkVt-O7n z)<{vHKwMY5Cg{WTl#6iAzrV2H_30!4(aSO^%;Js=5nh`3Ho)ae(%5Pb;`q1wUr)13 zATtv=Pq{PE_#d)k|Yv*DL+4+;K%ly%JRj!{4C`mq@0krPV5Dn&C(nRcjHUJ)NO58+ZxINT4U} zG1OqR`^N_imz6_36%TJ!jF4#I_zSBII^BKzFaPwDAz4oe}8~=DCO?&_TAJvooCnaJ^82ekhCxKWDoz- z=YO{-(w^`i{}_M%XqW%7I@12B(C|Ozp7h6AY5)B%PEGZQ*#Lr9X&O{c(E8*X}&os!T9fT)T0Z3o>i*DQ>pop zq6tX-A&a>G9mZy+0|s>GC~{C#6F25gD$wAnF3|BqW})BC`X&^QCG8`;87vS9xC9N> z{=7Ow3uQo^LT+U`bO!dpguw|d*JzI5!L4t4c`_^BgqHVWte8^y`+|Fm zMp~Dci>?E@$;SBjK~$k(R4nH)QQ--I5zlGbg{cRBrVzOim;wK4^07JIhtl62QNk2m zr(*#8kogcToFiSRKrUj}LSnjFtC=@{B*k$RX)y1KiV+5NgC-Zw4$g(W%{(WM zQx}RjOHPcOl+RJ^^Jk%;=Txqkb!^VC#gI(Zk%wXibllIEZH0M8pdV!w75k5=_U_pu zmP7H7pa9Rpkn%_|hbG0-)~KDHzKqGQ^_1;7p&5 zA*U8+FIo{s!EC5N1lf?}E2Chg2Ifq7Vq#)ZYoB))9eh@+kFjZarK_7;c75q5o}dSi zcB#cwC)##dI_yIszO12%ia&~Yy}W8J3}_vB6=G5PF02C>icNGsLRYjSHa51Xjqcw} zf%5l`wKcSzevfsmpy!rc`>^8?TTX>`TeD~bir_s54m8bZ#y!--)>xUiz0cdjDXRK6 zLu?a%>wBu1pCD9>z*w<`2L2TNUw#2M~@Ol zLn{7MOw4kr+CI44xNL~q0UDMEJc&yThfQ!|V(orfma|a_VUsR@-&A1F7AQSThjQuQ zR?(p1uVO`iCW9D;asDQ5WOX*4zEx+Hw)_j3$~ZA zh9xSp&}>yrV=!offoH%g&*rJWI&eVhONeww857yAdE6bFs#O@=2yshj3bxO9V~H-9 z^^f*S8NuGf^tUU}P~7ZT!+a_(pr6cz!^=2rBm<-9p|f?FAqe0Jq_R`g@XbaLgJ9=5 z0_O;k3_P9Yx|}5~o@1Q)D9jupR5HDjo&^vumvA8V9duvmiy+eviHS~oe9`=F@r1rd0#U#@aj-^nJ32Ya2o>S9c?Flsuv#md^cDUJW> zf(b?8{0skm%g6gpY9d55xAVP@PfS24;vo1?!#3_CIr37#m^ykbU!j9Y3_+gLbRcg zk(udkP7v1r=+S%W;*M3evy)X(u{~4vPJe`l02#|Gz0JGC!oniPPhcrPIG~-?xNsyU zy*OU=xZp1zd%B?PJgh`f@O%3)0!vq-81xUlQ|LKVvC=ml;QYgNCVQ`hU>-8NWGuQk z%o`w;6gx2D9$;Z%FyBfOwIJ{ha;K>gZf#4-f#998h6J5q&A?1>(X@ut)!- z(qwlhP}b_q>x1k|?!N@kzX0Uy{`G=H@fM4Dp@e|-%iv0~5KN(KYci9Fzj_Muh#uVK z-)e99*udD^xpSupl@8l~*pr6i;>>x6yl$i1sNhXTyf%C5R(-N<;nRn|{Z_l^$@9~H z?$i~y^a&3Y1`P^@-U7xjsfR(hnKGvz?9?gr&Dvho8CM-b)DmCcgU?job^tV-TuPlFHv_+*6xCs zzIf5m?#=2zm&uAmcz)aeaHZ;4W>)y}jI5S+wQ*@()30rd+M3ttX1tT7yw~=DS6ypx zU(Ep=QRU;G4|^9W&P8CjWZu)W@JDR6*RGX{HPyF7*D6RexpeH>>eus(QoepVy)%aXV76#7kT&T)tzcSk%@8`i8%@z@HCt%&)L=4IH}6RR_y z)^6?IZ;yJ#j>|ztH537Qmm8W7mee2VxR{#bQuFZ8mBL*XT9&wGGIlC)$up;FEY3xk zE3vyQTGrjQ`mrQs5Vpeccbf`oQ;XXv-A1dPejhvL><)K*Iu!F{F{gFDj)cir898p8 zzS4X6JOfa2j*>&AcwhY*w+JbiBl`yVyT<=1zN6!)pJG=A{kq4PRXn|zxMruVH{7@k zC$Ny&*#C%|8**ll<%0z=yXpG%6wz%Gd(~9|v^B4k)hf8V`li=`=6K(`e)+P2C^wSM z`gYVL3Fg+J@LQM>CF=o0c9f{D-a)u`L{ll}64DFLL5W;Y`DJW`g)e~Rz0|5@R|K*I zveS}+77aPE+T#tsDk&kyADV6Tv^j$n@9#%bvLM?lr!rYD(%i&!_i3#`uXDc4CCEUZ zf)j~Yjznro-7UisR;!4?nOjvB70BDRx+=VPT9K(coJIK-vN55-$ex7rXv(On>IKn9 zPl4IDgTM7Uq&3i9;gt*5&&KUn^9R5jX5yR4odq*W_PEjqMe9eEAC&yG&w>(HQ6z^8 z6+@2E0W30}AAGNWm|e^?bFLFMqY15!RL3$thMr=D44-Bs7drNir5D1MQj}`Fh_`WQwXEH2*a z5()1NeoZnPW8N>NK%cim1n@CH$a&Ld~QVRacaT2 z-!m6I0cqb1$p{z8jbPt{z|J_k>iT*o$~BIEDj*xw*ZRHpdV*zwf9D2yPz1O*u|=Fu z(ET8BY4yQekQ#4!)i~b4_QSKjygB_PI`+OirNmaBqEo}Tixl|C86%2Up$j$)*U?cxZoKp4LsFA%}52L%8GR06Cwm2&zua2dg>uOd2|#Ne4w@zF)}itqzE;{om|U%^WT%QM$_cp^*C zyv#(L)63KpI3oGnxu3t+z}bp^n2%3R>*0hFDCXLc*>c|__Uld;x#JZ75$m~22u^-( z?nk7}Y*@GZo^+<(wsS#`cSklhHZEm<3tI&GuR;bx*9X?9BLf$*e)*Fp`eHBBiY%iC zlTls^U`8LX6`wP4R)^Lnb9SCKgZ(Xq73 z#^L~Mh5kixC*H(WrP-Dc*A~?O?spqMj>VRZ#W}pq*7Uk3NEBI1q?$q>arKT9M>0TLa}P za@waS#)>h7E5Zc;f#~u8AQJgBq1ZJPAcyK){0o6RyKwG8z@dj>!ZG3VhU%qPk2oxL zcc9tE&OgDAm&I{lGRc(4PNniZjhLC;#)NO;4rk(#zaK}4uL zu!)hlbqOd!Lyq;@CWb4F3^vnfm-Bk}Kut04(|Cg5|dnShK1 zlZ7l2Q2DX%z4ZAjnz{LQy}EW_t~Z^8%#Ux5jM)m$ed6jWw97ze;9<+rTF}ffdMh?; zUBz!eCi1+GlN}mm3?ZEt#f+kd*Z3QG(|J`eUYkh0&z1IHLXT49;`y1~2y3gx8kK;Ix^SX2VsM3mM?8_*M zT5(AEfI3CdZ>Fe4_Vi2eFJhA{r|_Gk&B4<)8kaBF*y&ptQAhM`EH7QQxny#l=ckc7#x^#V){;ASn*aM1LYJ*B?zC!HG=WcHuvFHvrl|GIO*7wq7xjxx&{a97qk%%qA0d;2g z?=N1m_lU8|uJ+{*2h4e@^$GuiIF(w$Fdxf4=NM`pppL+yD8)f9jP6?dt#hXo?`0wzunGRhw2~WavzdT&JYC}s|xI*liG@%T;0gEon zVJ|VW8lK&|nME#qb&ZaW4$tM?Saf1&XlSB6P>-TmwjOm$NJzlPy=<8d$~OQ0cEybw zH(13jIQ#ng7`aL>_71fb*1fmeX<8L~L)K}ERgx3S6~Xw$jpKCVWR>hwouwgfLcTp1 z@w!qLw!bd(y13$f)*HCRc6~RA=UQ(rymJ`cCM_dV{!&XMHRa94Z$EpgU$7e)?q*xI zZU?=Rl9H%N1zmg5qe`vpL`m6`ug>jv9B;JKt~~qU+Km^dl74nqK4+3Q?5;bcqH@#I zbLlf_`9O+It)7WP0~0*6`2__(E20k%j*lx{y0l}5prBgv z`GfD?y$d@YF3%QlOhx5W!kM&|_ICHKE@LX^*B3vv6l1l_8g6cGpL^?fe!>!b`SOKd zErPBg)okhY>(h+I1$lWb1zsHVjEn~?EqAFNInw&;m)kl)?J#BEjSdNNF9LRG(MGGs z)n07Howc^NFFt(u@YkmjBB$Dl4z;$n@}GRkO0|?nD#fZt-@Uo;{;qwaU}WYsF*n%igBd@$K4^d3gu+N=URe zzPGzJGa;<)$9*Ok7e|s~*c56fkBcuI~SywtlBx*4L9}cj)NR zb+`x?_hoATIE<`2L)U%Y(X8tcx|wMZ@6#h2?}9~3oj z&5dYP z@;HsH3AQ#}W&1DANT?yVjMBPxHD|kI^pYA-v}&)@A`%>Co~yVEkVIFruq>jmj~O3} znJrnmbm?cq-5;E1+1Cr{>^pv(3A^PcRy4Pyu^We;Qv&&@9p$X>AD(6~1xl=iD9J?(Cnq;nB zzpfw@s1T?56MOdBNZ0E4lL@9>Wy6n<;6Hx(!Z6xXEn>qT9HY5S*zUIu^63c8@!rXl_FT`{m{=ccir+KYk4UZj|!cD!hE{ zHn%%0Th9;u%s-uCy!vWiUHG`epC=(wcIPJsTbbl@K7Xc1c6ImjW5FX~VrIVS>PnG| zvaqnYHrPVLE^2(P>w$phk*9lCBOYm~H9OC8;@uvZnKDgTPWR^LrdQ!h^N-2Rx}{xi zKHpoDz}|lBne-j3=clh<8IL>`y^)$Kb~HpXdVDHSNOzm=#Ug)!TSB@SBqzRen%-0T zOy(Y*Nn3ky;NB}g-3ltRNnv{ad@GexQ?o8HF_An?c?E^m-@m;Bcj}!lc`WwMx^rb| zsN5u_HZI@T*Z6+@n+qlKNVB%T?Faz6C1R6APyV2b z+O$=MMk7&2;!01IK*HohKfnA$-S=)GAt6+l@3GL;OV=!O-+Q$$i2qdLQ~y)nD-BF~ zQOAQQ9t6*3{C_nLD}>c)yDty_`clC%C?q6A>l!3seIdgrE6S9a+pm= zd8fcAu2!Y9Z6JI8S+rz|{b*s>ir(7V`btXV{jb^|Ff*mA1al4@c5Xc8g?9G9IpFo;&$*i)Od^K7mx<;P<%EX|4 zW~ia_V9SjM#qG!w09cxyMac$*Q4=+Zx-Zn`Jh`2j)^5{edVjT_SwKL*)a1^cI~AYl zGKxCsy!{6!=@}SSzkU0b*46ULmGY^mQIr<vAD zWxMPoOHKhwEMLvdofPnlrB=84G6M?>i`SoSfQO3Dtk~71kA_WpS!EnjN~H;|oupAB zm^Z>vKuBoiqNOX{F8&%Uvb5QGTJU${drw^MIbba3fHfm9|B00WT5UovT`uOFWJweZSvUd!GZL1a)qX=zUE z|3X)qMI%2HxHoOytdC+%xvu6@6oMCB`31S;s-s&Dt*l|pqFwL(1-ylTzuQhD?itB)teKiw4)CP*7Q17 z<`)(&#p*4kF4euFL1sy*p?^au5XiCMI{Vw_9-ufMZIi=;iSzUK}%h zc2!SLue-Y+Pd$9!4O-J~V6GhG(JNCUCREO?rK^s|sIsL>*RzcpBjXeN_v%$n%#kPO zE?kHSXhPsIvWwEY%#N-P*s15uz_MlE7QaK?2(&zWr`XRW6d1)zopk2sM?-FcHOx(R za;rYwD_or15&d<;lWhPr@0@2P0j0dfFE{1n<aG~UHiHK=*1b^61#D;91qPh4#(oY^8(8wlKs$`IrWOqSc&$hBq5 z7Ql||8fkfYM$WxV$mXx!zAZ>Kt8Mx9D@rU$M@R`#+S1c=S3lRoXxqBAW@cupLAz)S z3hwXM(cwgP^8!>l++E+81l(8m!Cr(qhYiBssc~- z!MQzYx8Y}|8SFzsf}CeZPbJtBFg(0iVeB@6J{3mQ@h2+$1TL07Eq|{gFMqS3@?0?2 zl@`Dd^*NOMo|nBvx_t_CKVo%#`UcS|Jf=?z{0| z-qdJsYT{1wAIR4cz&oZ!;7KpX=hW)cE{Bd7TAe>1ar@pqzm8NH$B9>pa|oNBr%5MA zGwlDwG)-3A_3~QVR_L=HE6Z+QQc{xdBR3ao%hB-Gd^>d;`{54usaZ&1n&af~&!)`l zS`F(S-B`4A83iQ1TD$EUAg*!!>*yK>;0eB?Pxtyp4)yi|yI1%XgKtXy>Hg!NAs@~Z z5DYKf++?TIk=Q1!)tq6wJy%lX&n9cXEl21E zTjOkW#|D~rR@;HLGSlVZ$k(VQhXJ3GWOrV_2bL+9ObhuxEHfti_^5FieFOc#^^u0)%E z{vFGXz>NU&lgEZEymZ#f|3UTmkcMw??>*Uy_p-kVMu!Z*MAuOV{&bYsb{u+-XK=aZ zy@k7c_-LBDZ0$?@{ctbi^5vD+L$-DIpF-iHezxX%Y8T7d4<9H3w|K@W7#e^trg-KM zyQ-JCfX>yLekv1r#(yW5tRpK0;I}yCa`V{%D`b_jDLRJ>q82|`ZF{P2`1q_UeJbH+ z(FBTW@a>t*waMYl$gQ`LQVq=-?Kg{wtra};HV@ZSmTr9!;=*!p+`ZNvjI^}0udWT9 z*G$yW>h(KH_R+g5Z9HzSxDe5;=I74w0NmXy4%|udECHPmN}{Y#`zMg-)xa53Oski9 zdwcV%N73KBb?Zv&r^TQo`k&kwkTF@MuD&dNr>m>G8t53jAQDxO|9BWh@Kd+bD}%z7 zR1P>|^j2*|fDabPV0x^-t31+Edk&Re9}lZ8?K1ZUIp_NxD>zK0M^8o^U|6%`^zkIe zZ9s{})$waTWM%Bae|SFIxscX ze=p+TJ)##NI~D%v{6=cMG-!24M@MkOI;Rhglgvk+>^XS)H0ymYuc7Kwdf~Vf`V}kk zefg9Id#bnF^wzAvRaQ=}ak8P;^WhLIfdqGVn{0qy6eN^glHV%)Yq z8)SI=z6?%{nmv`dzAijG{8e)D{p0XJD+FUwq=OO(&~5Ck4>N1P^U8; z#6UQFySF}mV=Xi_J)K)!y%x}yEFOd|Zxo%j(cap(`}hX-I-i6D zTTM+(S4H%4l|Uh{SFd()$~de6Bv3#RfXd;HyD>>1Xy@8!&sy!bm+nA|;RQ4T&&xqh zNj558M>HMlj;~*a>a4ZF8Op)%fiw&NSS+tzJ;{;40||^sJyaM-C7XATpBAdfQp$ga zmionKDSX$ECRwnCB~_9)Z{DOFgI39A7ndg654P-&iiwGUV#R+Ve$Cyr+daVIt~oe- zYHsGn&QUynej{pLVsaDE@djM&B1-e+>D6E-H{;?2KzrOs6ap(s8k|iveI&YU9|&UW z&!4oY7jABDEeL#q1Mr(L)H}`}8WP`7d=4EtM9E4sBx-IWEC}!B#ZExFPyi*{%{jT4 z>L?AlhrOr&qz2jow`KM!oYmo$k(Q|j=8H6a4yq5^f9!}exGGs@OyCbwBu)x~IQW6p$o0>N8^70Ojjh&n}f7Cc^A|WN! zhTyievnxc*3Q^v%tI@4r0U zEDVulW~w)7VyKW?*m1%#LE%@2_%e4|BqmbyK{$B?eZ0LDfh(>|kD0$boh%>O+6!?dbfyPvtI9*#2}l))XoZW53j~A}kRN?3!2Pa_bg9|&R2?!fb@TVfCVNF;xeGto z|Jh~Jbv!{lZ0)ke5s{H(5d#FzT(FOsM0*NXx68DO z|D|^7vS!CWUNr?x;H*D;ld=l{V5^#Z0PR)8IahH>ZaShLHt@E~V5F;@U`k|(D;;0i zQYVP`VbfQ)8a0en%ybQ^;fv8N%4CxQMN$;9Wcl8Xlsi5?r@pUJiDpLrBddn=zam}7 zo7-hZxRFDIyLKoU#HiPnPnjEcqHbW1_}gXMqxn+z{W=r~K|P2GL!F^6J)9faCdy9l z3@i>m&Tlz6+|C(#mM5(`QI`Wcr3dg4NS8ZySva&lW5COvnuL2;k_&Z*c%?djV&g{% zSVC*4vRJ^?cuxWFt+mY)s66-Z#K<;=xR4JGqe}M0$(N^ZKnG|A-eMI#Plxr0NqyGR za_*_Ly?Vl%BVgnXV|{mspzz{k!l##7Zz{M<^`7AUyt{VoI{f`4q*B#q(h?d*zpNxy za!I-+WzVq#{OqrPZA3((y83!vQPDN?v*X-g?8md6oj4_J_$$Brtv;c&p;{vDNW<*V z#88`R^}xWuPTfkY_XhrejKI>BC6QClVvbz(DS@uBu(40S3Tv?9rQ36<&Xs5<*l377 zX6TL4Bgn@;v2aVM`%5w)paUI@afd z^D(P4inX3UvSeeTqLgdh$R_;?kg!(3=dA$C>z+2Vlk)Pkw2Kz~BMX6+EUQCVjOi^g z&PKLfG*Na(R_5r@qhPhZ!=0LwDVib7njs1cD01^6pDNO(h{Bp_vd35f`Ah!kZ^}GF zz(ZTA)AZF;N3Gx`t$}*ZzC{z$(`E?-P0HV1rI2^#%$eoX`MH@%)6rkG-5k_yJa|D^7TUW)F<#WHxG{gRnL%E3lfp+wzl=ixG%@}Oy`kr8EovOiK1R0P2h7t= zr*5HPap&E;y$MEG-9+op*PPN0){d-L#VzaK6o|ZSA~-$Lt)UMzOWj_({p2s>8Mf~F zH_s^^h_ikcvejji-I;gS32qzv-T1=#E}c`4>oWz>UMZ(s1VNMo!gwb|+6lr}`%Vjk zLX=}+dHuf7QTk?P+s|e>23t5GFRZ2VupM^a7nZww`}PCxZXZ3s%?NCmpO7N8n{)e~ z-ZS}j3uM(;&dMEf~7v5h`G!WXAlF)b}&cA1|&B`BEo{PJhS?J5P`(2AnCZjRm7PM^Q@8fb(0KkOMA8L0|R0(PZ!W!-Zr zs8o9Ev1e6N)4#EGDnGyYCiVjB@e_PKxM_>i*bCF1*Vy1uX99cP=N4+5F-&vRCW1|l zXY=LmBbOBrwuZ`_3(dXZ5EQWPd8|63xBksi^0B9z&>LdpmX$su9L3eoV=mpWMnIho zOY#ZI+AFJe74WkC<_$Mz;!h==)egijT#!@quwT`npHZ$%v?Pa6=LTCoN_tjYu8di6 zca6Z!@qy+xFgW%~9(vvuWPMVEp;`b#GRae}Bb)vJcC_~N#HY@Y{+ES<+(x$?q+=RN z+s6+w#yPMQ6p5u5zC8;`ur>PCSN}%&(4iYZLk+iAaD%sqdvnWq13;J_8K_H+EMSS= zhUFkKDDbHDrj8V(S&F)F;ew=+aP0ALT0D_^A{R?rh_)$g;Gy{J$6dJ{I~b{v4Z%4- zesDHBjfXV6MrTX<+8~9Nn!^0tY_iL|Grm@-b0{gnJH^V>pN^`qVWx-#lVVo85;DMv zd;K$$f4b_D`9K$-L)gsx`6egX`zAU0Bk~>-JNsh;;f`p)gP)Kj-(4RUTFJTB6$Q2p zvIXRhgV<~Yz@a~xlb^qMe&&x)!>-bLiS;{m-cc>Ne!~y|sDtEu8igHTvSisvvw(qtp~B7`iHLd}c_g+ppJvgL*#7AhbS=ZJO%n;kotvGm1Epja zw|M!+`)g6J_5RWNFm{HPmXEi(DxdrKXCWksq<}VXUQNOo;cJ8EynMenPqZ#2h@PyO zy>78!y%YDS{_S23i*o3WPimE_#|8I+Y4fkCu-%#2Y)3-2CP^7Cct>qCuc*1p{z=F5A9DlJ zO%4BTythl8JbAKWzOu5g*wb{cS!*yOuY^(5~zNrUNIt?{5Gc(xcBS-6M~S+aU1A9%YnZRBaTyQk<`P5+B<;qR8+@N6d;Hv zc8~I;T)>&37v6kMms{3?k#$xr2Vf;SKcS1l1_iN=I#p@jPXn%Y89?PNiXdUeuOLTrf{LTADnhn89($Y@ss{%cggZL2z|pIjn8Ir9mQz8p zPLgvIZL3h%$&P~1QIs5HYl57J%@k->noi1IQ}USH|1h> zUDZq@4(x>|#3L!m4rZy@>mT-5_G3}eGPBymk~i;eJhkT@^{pK^w6Gh#*A}L9MkEgC zC@L!QpMJ&jv#qUi)~Oa1JN4+TZd+INa@!JSdFAZ#thp&Oh|z+L-J3N>8k6D&bkJ_^ zu6dqO<21u1_`bbTWjdIdjqS@{GP5&z!Ftjzg+lKR@`l1X1nL|zDVr7E*Vg7|5W-xs z=fV2zCwJt|fOnJj8quL4P@HQ>F<~oRWvZ+dd|0aAs|eWcOww)>DLaxxWE@6Z)#KbA zJzC(c$a>+>Ln^#Jpr_2k!$Sd{n>oeA#Asy+guMm~qqsY?2~(_*hT=}xE@9L!0OY1e zAa(`Gxj1`FK2q6?d)9}l{#N?^^rjb@1D6%WYc;z~d^}UEwmVmsjNez3Dpw*Tz`8?q z$Jr08$O8qhjf$%}c8j~Ege)gU5&YG(WE@ff1k4psbV+^(vyKxB?TL|Z;8;6V_tyN3 zhdATwPxbW@-^IOQxqt$48w{9)dskH)i;&JcS75ysghoZl9blhenN?QuP(Rhx)v0~# zG#mNso#xt80nT_)s&J1JC@*5(2B5xuKhjx9nKCroMD_?&gbrgyFu+?OAzXwJLC{c7 zxJ8h7^5te((lDb&GW11pH-u?n(|mtxp8J4T!Ai90$Ad?a?ga2LX$VFfTuw+h zKGg$4I@QwbGM^7Qr8;Xn`re0Jp4%lx2LkNx0{2kW|18&!ajMaRzJo%4L4err7X8HZ)jPD(dL>j1MmJ z8z@mp_X|KlCQ=@TW7j!}q;^(+!M0Rn2f?W9-4z3!3>yY)<=x7RAe>7mG(ZI!mxQH2 zSn)=qX9d`60ch|mq+y-=y9T1UOl^TDU!|ped!^d2bpWOU3Jqzn*PqB3+M!XB&NF*a zO0Bk>6NDb)$*1H^QN%+7VdD9dd6X_fa;y`E&ZHhIMn4xZ(|ux#TqC1erYBqD$f@Yg zu?KD38n*mJvB{_O;(fLJRW?KATG1pB4rQytuW$3=002Uiq>6c4FL~OTX6K6Md?&T* z*_KGjben6OgfD|MQ7Xz>P;>(&Hyd{9XmZtNSWLb6b)b{=5quvY6!ee{0355$Ox+zu zd)n)!Mtwt8ivrc(1^c}i{}oW4qE5tXmP@!l#C}7YalvL&Sh)3YaNF zSAPGHNEm2GI9-{abMhK3YPaHlRAlol|9)vM#5SV8zq9|dB~d5+CUVzDh&ePgG(j@g z_kjOyaq=lUbvAUl$#}YK=n50L^`yV{J?(NX5FDE7)sn+L@s&A~e>$VLzC-vV?2%VE z*UzE@#i~Ns|{PjOnztHjF0g%o|cZiZ?q( z+G(L91Nnf4>i8NQIoe@kYPt=stA_?drhg#55+fQC8>`3#)eL(yd+tAYz|{9WZB?X_ zPe4HLv}UsGq^9^vB#>8@t)HY_&P%-S)$m#}JTYX@9e2`P)2kKywq9HBLfp^dIB%hN z{lXg~oueh&Q3;5h?!$)<6`AP|uTI;_$gE$#eyi8I2leCK8thBnBZXAVpAfHf?F+dV z-_{4Q0xP~w!3pTy3l;X13e>ahj|9UG&gy>t_=R-Z@I^&* zKtw&hE_Fqh4+2upF^F+C0ilQ|^qRLfvU12eUQ5h>T#0yz_YQ`H-SxqK3ce&SBdc^y zWU__lh4oCB=5`&K6_UZfLZ*W_xn!_4U&e)H>}Q|-ih8NTZ8bgB1rB(mx2z-gV2)C!c!LagdiL} zDgrES*5(M*L*|0a_oDsVc32xGU;w**4QDP9W<1=R(Ii-tvxiS1vt!_AJ4a zp!jjN%a&FTb{7;B#A-bEu5rNaF+{5~r0XQ^`Sj)aM$~F_uGqX=9*%9%Y;bxMVy$Lt zErII4zdz!7?lnja@(Z=T9zRvoOTFOWe&8f>M0OhV6f{(Ac6V*r_0kn1*cN z-L_4ohv#`w=qZ>YhEN$7QCX8690`06>3MhQf1?M0?vvTDOi->uTpIvE>c`ty#zHvGcq*{}E7kim_KWJ6f=R#FmEdDks6eN5U% zuRmmY4RYV#8wrEoV^NchUW#SeE-nd^MAT;&Gevb+N-`Wc>XY8upHWepObpkWH#fe1 zqbFtCv&rl9C~TE2P?pL+M?T(DT2-}1&uR38cfhg3wz6ra)hbDKcMxX(xc)RyV_C4B zX5TP}#Lu%hl8Yu==VB)G0cOX5S1Rt9J4BHOq*JMlH+HUSI%(U2Hgd zLAuiJwX(Ad9A2~iq$|zkk!|mKM4p9Le z=-h$I&{4t;BgP<)&r`i21l&Wme<8RXZc&OSYpQ3v@56@?0Uh1l0_aFbkIsPx1$#^J zH8nM<&q7@&*_924ivK)TrD7P+nSmk*s&RO#g9H8!iU;b#!{5Trup33Y!4x^yKcKLv?e(KrfS=c5!88YDkq4K+0Z9Ka`PQA_|h>+lx| zb{9n`?$Ktq0|Ns~9bqD7qTIZ^yh^3PZXy;$7>L_tdp$u{Q%%Y=y& zE+4@31CEX#w5LenaoPns8ZwGqj%9=W^Yhgi4@*c{`ta$YE4&F(Yf1;yoiz)R-mBNI znIUPIc7ybvNj2Tf#lfv79tuZf=5s0!I)!M@*PPF0jOT zOrpmjTC+XRb6mytKE$2_#83E3ZQG*?VgA74e8@1qty8iczkdq>Rq>Pmn3z=hu zn3xbb{RF8Q`ebS8Q9ePb!VBoxET+_w&dQ{=%hL}HwkoJ)*qC(7OiqY&V-L#<1w#C+ zidAE;?FOzovTb_E?kRMZp>J~Y#xFHC_a_eyUDS1kDM(VF2{J)^^yaNw??I~8Rv8}~ zE7fd_Z!?5oE&0c=^Yqu!=?Q@^Uq?5;2UL6~>)eNJv4%PT8dX^WoF)T6&pUgxA=@Pz zwMqNWlEsVf;0l&gpUTVC_+%mpyZ#s6*Z^HApsVrc=9}#kBr>br_JT3tAJbUVKphoy{!p;am4-8xnFW_=WJp6KV6MV?%0nzX9 z{N0hRe}es?!BMQv)t#SE3~*^Z+YJYsnE_f5@HdIVZgdj@X&Cuty)0=UXyQ%AwrPdA z0Nq#9NCDNyq72}I#7w7_YI*`vB@G0>1ruP1)Kb74;z>jeN@srGbBK?4z%734&7A?y zaF2}SL-+0#coealMxY5SCbo6RELgO=#Mk&=omSYEd1dSa zYLBfaXdb4*972E5;W);A_e>z7#%r}GA`wF4qNh@JD?#dU!*#Fp7v1-={{6fPbbtxl zVm9HiVk_C`L!!oIlx~KN+IBN@b1$Tca6}1Ibk1!j;@top!vT@r*$r&KZdnX4zuTy& zh3Knnxp5lLDGCc*lH6ZpMvVQSszhHBy;R3!+gsxfgC-1?1izXHnFOnP;9*DLvrOC3 zwzWXnLG`9zxl$2MC^#Bc`ENh9lGF{yiCtiw(Y=QpQn+q$Z%1Dm^moPe>(~35v$C;9tsd1JvFf<7JbVMP*#llkKhDeVz z{PK`+F&OcqzWsu*X3Uk|jt8_C`EVpHG&FQ%90NvF5PBpUaQ)h~bD)5f zmBrrKm!C_MCD5(Z`EYGDyse7hXQa=*Mo@bVg!{er!cqSi^jj`=R2U+2=r7 z-(4Hph~S}vWJFjQQ4O{Bpg9d~#3MME4MO03Je!=4A88=$ymOhCl_o7(bWu)nXz^sT z14$7rn5etph8~cc&=YkdY>HQ7pX$k5h?L5PZ?W_Q3sg`bKByB%T zSLn^9prW#D+o{Bt){_2x0ZV07{njxv)57rRHE@UUjLEy<*AEg;ztJz$7$P4;5R=(4 z#b~GCCr|E?C+*}Q*nfLbcP1)oJ``W#q_ZuE_@>Mt?|3a<8|D@JH&W_1Ws{&jFXy$g zi=taAE-tPH@i8PB&qtGyOHwo8KJp{_ttY?lj6a2(n~(1zosksTA^%2SX2kA6@IP4l zjf&P}79V*gt>y4=yM%;RgEy~&hjHU?V9I#&B-u$fri}suhwyxr)@;*M?vzc?eg77o zh1=TcecqyBm+yCb`SZ9j!SlQt*iyR}D|8>2>RpXhx{UfwCi$m(ugFtO!22vj+SyR8 z(ZiOBwb^krR5rwdON&k5BPvhQsM+;Tvi(Y8 zs43lA$st4MdvQgD`bb~BM*p7{Yfgdv^t>QedEnA+7&5AHBJ~DFg6+EBcjMQ?qssB-*EQL=y=6P`(+ zqLIopG04FBqL&eR2)F+AiTE*A4h}Q_%MB@U?`(SmYQ}d72^}5jsXpA9jt+D_T4o1r zY;3lS#p>th=ih;HuM)X3%1P_YnR^(AGK@Rxq^R=wcXCkmxUgJt>FDOIiL2@}WqV*M zIYBYuGD~+2fIH{kXA_B1JCTwJPku7da#A`Q22TLmH($3AzRaDkuJq|*p zJr0l%a!ent^25i{pBr7cumny^2+p(wKBuyrVqZ)rY4E$w&UmMU(>CPZ2uz1pm7I0RVqb};c-Ax(oXPM z2)$$;tM2>vNYozVs@T;KfKmeN2~!W1^BEkp!~g*+7K~tHV%&wU*aDZ0ab=8G1B=>J zUdS^@9W8(>bks<1?J6RVym$((i-fzBB8_=6c=NXZRHDv)6g48<8FTCIYj(`>+$sir zoe_=;ID1p(t0(qYYv#99{zs6|9u#HY0V%)h4{nj(eyh{Us*@u?q> zK#A!FdQY^J=eu}sMDNe{16jqI_ql!i!|uKHWv);;awH!*RcmkWy}G*7@CAHCHwryq z)wBg)8RS6pAXy=6`>5tWYvzBX^5ebTDkRoIgb!)aV6n3!1ZxA!w!AfQ0TD?m>Xz<4>9i0GF{ zFZ`OZ4@#}L<-R=Kpd0D~ThKe3Ze}vX78JyakjTZ2BGVjB%fdWDE1sxB0ye5X#8DWA zV@6pk3z4?NG`ES#-b$5lni54kor427Sma{N6LFEkm?FZXVeuuUxuSxZvGcl_(QEq|r2K#&OOe0}I|Xv+Qd%MhlYR+z{z0rT7|NLNBF`;|AkfAb4?>p3N?=z#bS z0C|jhi^KB-PBl0>dhoyjI)n&m%OY6AiTJqTdAz}p0r)Xm;si&6vL;$rc|84*M3Ks- zljX>UdP7WWaMh{7?VZ{4&2$nTA^5$RfgK5&gdaC@d}=}AVE<=MwEUtuswIrSav))@ z(11B4pE_>dvSkTwmW=qJp+#<1!E>ZuB}s!;?doRa3C!Wh)1wk@YAh8wkbbA|XJ7(w zlI1u^@y-MPbhgjBemn8@zaMr!((Op zo4}x;+o8@=4CqfxOm>E@;97}QQ1iIIf{Vf*@JEa9up$#T4LtJ-o)42S=c$hkLVC*K z3cL}QFtk^Q9y75)D0s^2?ZsE@fj5D`Ud)_cn;sKfm^rPu*o!Y_m9W}GS_x1NIsTbB zy)v}~`4c}8ItASvWNq=&bvWPmUz5I8T3`?a@WF%2D!JzGi{U8yuUGti4;JUt=YL)H z!nH5_;`HqQ&Xn)}r$>EC@sR5}-Y!zYEBz&Q^SbS_VIqIDMOCkRbJ6FlJ>0ZD zC^(%~2SU>bcLnH!bv&_(hmdCSQa$@?*K%@m6%5iWz!p~1nj z7mZ;sL)F17jw(9Lvkg0A>fu6DzWkyge4gD`CC_>OGgcq5(+^|G=(}y56?EHYY|Ibz z4{PEIOj#2bCL}9Q42;w8ZN@(QwqhZ-IVqCt1GSD-lJTESS9B4WiA-!$k95JAFeNRi$)<*vj zmH#eS6d!6P(tGU51a>ks4Bw@pL4Pj1$A|1;y)%z z0Inz4TLQ_Js3@e{YG!tiI7!efC6hB~y#eMeN}VC}I^AkB6$VE#v3DqjkT(zk0SbHs zfKq+a7^wJv8+Rua9!n{GeT)%r#!c_U6#d5bRcHu8hF`sAjT;hhB#IpS_x%WQfrY~q zB6gCvCfY89!X;>9o0_1ZN_-RnLb}Tg3`UcPl>sv?!murp1}GTO8ieP=3QtD1o9O8n zfaSh=^QN-P|AD{18w%4#US1li3^V&)1)ivGn0e7PFD-_`px;#%PRt9;EG%MgpI;mL zN$zjSO3ql{m}6h#k1_yG=ArdUyy2;m1e+1}UDc7lJ7z?J*rRGZ*T94_*l5?>iuSH{ z@gBejaQ%w|6Pt0>tSdJAaaXm^t~A@P4= zd^{geU zeFKAA7}M_f{PqQ>)Kx~{9daH3ks>ktvR^3hFc&j|d7IZJE%@>rN2^Z}se&|80eiyG z_+k6K9{FtM{=%stvV4HkWc^ZOKvY-Y1{YDp+12x4m&=?Z@jSpK01m?k*~1OpXYy=` z|3DbVaIYz}T*y%jWbheWsa6@7MAFg8iJ;~hf&JCHh(JX!BvcWE(@lTp(Vi3Nvl1~K z4ZDDV?{#%85IRAA+=-hFCRYkJI3z;8#oW zAA|abuw4R|z4n;UUu2^;7k~p6Kr;`yEaxR7e3(N*%3p+AAe#k6E=jEh!xZocAsaqc z)ZVv`lC=AMAN{m*aB-8~e04?k?ARHSd(kH+6YVh9MeU(~1kZA;EmK)L%t6@Ki@BXbIzj)e(A^0DSo+=mlQ#nDzi-(_=vv7&#;bIfQ!9YKEGq9*niXcoXhTve&#k z!kywFu>0){hQA*4VCLdY=`b{H$lyp;RRIuP(UbuXT#vjSVO$;|AAVf98{fgQ(5Kj) z%8iwF#}CEx`$xK8chFAVj7{xVtgR~_cS7!ZI1CSfxW>@mRb9>Y2^(JIk_c&2kYTQd zfj?i5rylVa&`>m=1o?x@CywGF7R@+GY;+;&HbxiedEGFPu?Evu=r>`fz*KUx#tK}H znIhww(MH%^n5JTYUY-oq1P*16J*o*h5QQ=J3v{oVA7}Qx(t?m%h?!N|&M+Wjz}V7~ z%n)k-8Ts9K#vuv4#o)LJAl>0b-u?g^BW!Hq7EeoknKdIf5d(x~B4I<#8C6lW3L8xxRrc%Guax6NLdesnj0>&o00TLP5&kL6Q(?D2!dQpoP%A0FaWGs) zA5tS`#QhYVM+OIB)&)KZaQ^c=u|VMC}-KCK%|S>S)0=uceFg9+glL;}+ik@I=< z#L%i;iFKA;ya2bb&T`Ju;4Hot&k$vY#fMIC(WI!^o~MXk2So*pQH^8ckO5$dn~qSR zdQ=f|jy~=j6|D{V9@nG*o{A=+C-xXs)eZ`Scc%eb>rzZu)+}30QXP7NF;C>o zpVO_ap8uX=vPeowO7q1@;*-yVUxbX}VN_ayo`(E;g7%4otd|zowJ{=JrlnZDfy_6C z*jF6p1NL0=7kiR%O~t9~M~@zrI+|2g1gMqQ{E@KHEyUK%Jb-m ztRY!b*kB2c?LnuM7HS6MgMA5EVD{%)a&D+M5@7>V+JJ^wPdW-7QZ6h83VCXQ|Jkqc z=Bk%NyZny+11RF+)T5UXEC|{6BP!tjf@>cc z)EjZW$hG0}FgbZ=A9UY0oM#|!VUdVRCq8~c(|jceZN*h%GoNb{gS@zlsG|(7V@o%^ z&geIZuHAs=x^&i>tsJB1&F^9 zagDJWqo|k|m2N@p6xZbBsak*N(2FRfL3gyFBO!dV_fA9mfgbJ~efq@r*n~xj+THyk zf5Rc-*og+uMQFnC+Nz+p^UOxWnPgZrjuq=+j8qoys8nxMtrd$1mj!_)r%4bz2#Yl) zHXp#sfoe z;~4#0==9~!k4{G&CYii`8$Jq12pHz6?V%?z-~8wsVkNmT{)Vfi1+cgYnjw0SICe>w z1ELf%x<;oT19IfGxfuzHc)GNuHXTq>;^XJ%KYWFDKFtddvpRU|=FN+d@r#Bf|DpTC z+p9HwDEMD_X6Mlfq-0e;hV^6iDkS^S2@PJtJCKS>tS$lS!Hp||U?Ia~Ks$VmGx1~e z1b3)0Q-ngHr-qKG5{`eFnVqFEeye17@%w1>vFisMOmI)3vJu3fzyn-NU24wK zy_kixLQA1EE#lyKvLoRI>J@w@@{oKm#zcn9kya0dY;>KU{^)}CHMqnX^I(+*2@cy8 z?iet=%D?9Bj(mn0y|z4LYYb_Hi+#iJGTM1h+HLWq$xr*K1?Pz#X)n*mH~-3T_F0qz z{FNef8F&+s!{e3%1Jawd%>dQA%xZRkOJV9r-88LCu{yD(we<${Nb2~2|8XD}BB5eI z@{!&t3X`m+8Yijx?^m7TexiRY2fmWp@ghpDV21U(U8PQnwFpwL`^gikH4(}p)xcs zV7mz9AMvdOEXMNJ=heg+jj8-kfJo)=yyCnPop+Wrpi`79I9!a>u}E3cK1P=3K?7q- zN)QSF8R%8P8BCjhS&cgT!LLH9Bsu(mm`F+U5azJsn6Jgvvq1F*0=Wq#8!nAEoV{>e zmSIer5J2z*C14D-*M^74Um+-9A7hGH1{&(?4=exXgpCU5%;h8HRs}$c{E55T3oRG+ zWTZOstb(3AIc$%?1usNTF31OXP(FwOsX6ewwgRUTUmu8VLV3DgJK;g7Gvx3i?4u<_ zVg!tUzr!6Id17KB2cLk&_K1u$@c2H2L`963I0@w=eEXOh+=bq2sx;gmIALrVlouw{ zKlG!~-P|uo9C2jeh9nj>4(s#ZM@f&6*xHt_+mVB2Ix)NuU{6#=+&YATJBY!JnD;9t z$2;v7{ku2ix%OI8nC1Nl#^nuQ7GHAA*wmDmJPm+K>dzj-1t-HWM{z4kqw(@uI|FuY zFbfZuZ4XVRYQe~aDH<~HNVpim2-;{!zq0N;3RWz7CWIL>h#MI^RPbQjk74n-sotFe zB?SdbAcq&Ayx0HhAVFCPI8HI7Qxgn-m_4x`qvC6vu>cAn2LR!5k#;WG2!sT6lO{N- zae0b~M#AXhCr`*Z5@Zeu)r%ZZhVc$%!DFDjFkBtVOfv#`A$@3om0jpMqh^@iAA$a3 zHZeI#QzpCY=DmB%5y(JJ+uG+(Urp21gQd+=M~GFzSiNl7tG|B;c>RX- zF(UAg!Q5aoQcd~RT#P4Uq=kmM&n>qB8~&T-Bw9rzu7SX4DL~*QFci2Weg97i&In_# z55VNk>={}rRx^%yVG^SgV?qGMbQG{IBZ^72Rx@B<3&v_8HAUk57dYltJ`D*W@CZ4T z3`)Vb01mt2HK#S;9_i6zu?RImU*-V%6DSZjAe3-G2Xnu7?_Q1Kiht_gOdbwe!j93d zL`FOZ#)u3Tflqqy#Za;r79YT_TR1iVop}9=7nj21Nw^gfE~#tOIT#LKrKD787b0rN z67<+LmMaTXZI<>oW+m3(I z;lFD?u@P7)NQ4vxg26KRPgDc^ z?_Zbf&sUeV&v5T01ENcBEVA$Oe{cjT0yha+aX(J8!qJ^m=K-h4S!Ot|Pt=D0@2zSv z*}tkQj8QHDz94$%#*H_*m~S)gfOiDg6eci_mF^3#;;%%I zfx6f8-r(o{=kC=?amYqJmi|Uw-T@qb16#pLsHG2z>%ZONPl}F;;^F6~2Y-MW>sDrF zX1K!N4OZlj(-knb53kopjE(`_d#t?EbuVwvLf4yc()stYIKGs-=f+gwzXBJu#M|Sxrq@+a zZ6(u+f+hOKjmW>(3j*Zj?!K4^<|JcNOsuT8@o;jNpe;*&Op)n^-7%gWH#P^Gm!qPhy4Ds0gX5f4(m#RX zdrW%+{2Mp~Ap)Wb`ru69c0|L3uVfW;bM{fV{)onhokARqr26RV(_pp;4CvrRB}N=* z1+U3+0C~toi^(w)Vu}AGD&H-5{4Ke+n4oT&nqcP@Vn*)xn+rz>T|yrmVqz`|;7x=} zi0xUNW(HHF5@;w+yCPC$I4tWZLzPmAa`Q7R2!3*6A<)fMK^NR0W~RxI6md8K^~vMR zpI7GJ_t5iJ_2AGtH?)y(^dgyWz~N?^IJFROhc-ldRm6&r&?^V64NztY+*5?1KxfSZ zl^yjCU3gmjTUj|Isi|RD+4*2dhu1ux%pw2!MF&ipDvhFhBPl7V;WbXoIuDx`@$7+1 z(m<<99RYg?gU`~Giw-(+3wkmzE1W)a28Db>cflB-)_D+Wvd}oauN)IbR1V-!82~Zx zNL;%MW(AxvyH6AqKmp<}Rt&-YivTY@f7^$%ZQoh7FT*N$VucPBV|Iibj)f9oQjf<< zT*YKWyK%wleOuOuOsNo%k3sebm-#uZA{WTZK#KbW^#HMw(Qnqd6MUO-CsHB2cR?Z- zS5oJ4TxlK@&kwLYMIP-=bJB)L_7NjLmUA=qq#^@^n|6~cB7-M2`+1;Jli0v|;t;-7 z2#k;`47|H=C?^?VK!omv;)G^4MFDMeHD$ino<0bu0*eIkD*|d6piQr?w0teBm7oDX zafcPJQQ0mvW~>E;&Bt!5dh3Km4wANiJ!0qRD2=Y!BJj|W{_m$Q_8ssT2IRYhvzlR| zalcro$EqsPMpm4L<$(jE!{D8SNj5_}GCyqpEpm(|&7viw4FGMs;=?1j_16OF6H1ye zO8lXK+4&HY0R5NZ08LE{)DshU$XcWp45gBXjF6yqDvlv>c1Bg0u}gOV{`G>2O8S;RD|9gHj0AM2~9Tye|9A494%IUQ<) z%w4Ee#x|_V+ht`p-~_&pcr<13uE7$iEG|zvQE1`up+yGf;)+`UqsFC^QTH%>K^-{u z)q$o2wE>U=(ikCK7a+p@nwo3S9HIl0Ch|Q_EAbtcb(!OYC7+`1vx_Z5ER!{Xff^Jf zZ#A1hb2@a|x$*4zaD+Um1h6lY#v&AX;}REgBw?J!QfT3(N5# zvsZm^6#ixw{8#xn2Io)0HA$S4;ElQ^Eu58!7oEqMLA0(DJui|D*x00KpG7-@W*<(eQ8Uv8V*3QH z2j55>*K+J4ybLxrvHwBVn}FrGwr&5nS%hSYN`_>n%!OElN<@Z;l~j}=AybrDN|6SX z5Di$%kWd*aQyLVBWG)(nkYtMF`yFLH-?#n$YkRlnS?{x~y6@{c&tus4{n(FN7cO2j zDO(?Z(#T6ye22r$7)+_Bpr3xK99KF%$;DrC&HaJIfoR*CjV6N_mfdJJ!e0NkcxDFo zo3~u=p85E-R~B?Hjo#UDUG;GI&&_5!Ex-z;<6ZlZ6W;2t zd>g3f!1#Sfo}3XK7_%D?K+sJOKApx(NFnmXhgSOti`j-p!WQ08bf6nKF?l}7RPBv3 z&e&{9K1x!%$BA3XAUxylpbZW`%W(qFk4Q`?g8}R&u z@e1!nX6{6xhG6p62B=QORlJo4ddxO6hX)ta2S~=bcu7=KFnAg6O@SR%2;h#k^2Vb_ zYD7H%Fa8fk{V(*^VC~J@w?7@di+kjeJoisMCSIa@2L)y3q(vB3kTOzIO+P@&#eD0oDNi5zg?+wK?;-N0PLG)9_MaEUKz@wHg@5fYeznCiMdYQ|(K}jK73{cR| z==TWL5}*zEcv@+u6iB!g;FGd(~1`jR#D>ecxqrCFx?~=alG=6P|eDm zJHUz!BF+EKF^&6{ON{St{9W>(nVZ`5=8D#Rt$l>r&&ip-_TM+KRqbx*)*cBKrEq(F zcXyU!eXT`cmlS%FxwZg=4G3IQ$6GcjUW2XE&aU&9TOfvw{F0QCvM2(7n$8b%s`VX# z51AZT${&G48n3A&LR5)QJjBO$nz_A>5WEHm|5>g^zr$dR`P3+nE-VYT?!3gYKh=XU z=#)+}Uqp#}hFdO1dJI1k&$nE@ysqiEiTV>P)wh`G+<*cQI+8GHa^G;_I^9~|uMF}K zS-61*Ee@|#6S2f7x`zQvj{o~a4LM2Bo3GxmyLy%A9@{OzX_2mADfwj7f8fA61f4hM#E&B&(SAAP*013Sxg?~r&Fk6kSvcu+yV%wBM$0ur<*}2 zN@w_ISbu`)BJ7i;n!z0n9qkKhaWPgys<$!NY23D;W|%Q z$}mLQKI(%fI@l>mKFT_ldi{EBvBLq46kUSZL4#J1h2qjof!657=wpQd#GRPIjjE)X z88dZi;Q3t~@D3=h_=$s32U&b*j2Hz2C7JX@S*JgKB|8D6up-sSfc%rGTGA%bx71QF z<+OTNl9@i!6m0?YZsIx=nRi<&PLD}eagZ8uuS9>0riO zS^tZ%udj>W@1v!-j_w%g#WmWx7f9|XNTnR(c>$ROmLfhed;eDiC-Ehazc7)C7rYe- z66l8NDMaN;-1L!jMY}&_<=d7r>db(`+dp-ecfdl6L!yPhbDp}<7LUn$0*EG4YKyBM zk8Nyk=j}~3Wq3oZESxT)I1|+^$M8A{9Mqs)$r}RlB|WN$=AylaT-L+h0_$SIxD^{| zIm5YIsGo15L66y63VC?xi-Qp^211ChTzQkM_5wTPy@rILU3;q`nb0OOXic5w6pa6Y zNb31Sb-phh{gQ(Ko ziA>IcJi#j*TWOkzgF&?$Ka|6uPLT&xTeYf(>rVd!4yRf_d(Y3`s8vN?b8<#>ZFrnX z=|fQwFMe{~B)lnDix?gYrq6WdCZ+~QclK53^nA$Wl_efTz;FbJqHv-_-%d4l11uh@ zcP#mAnRIs}HdrrmeXz3k3F5KuJYzwwhj9obiBZ%XCcjID&j%#2aL^ zGlA11O-b`R-y;Xk*g>MWm>xXnQqe-q~ZB zzJWnLMK^XJ{iT@V{LQ76k?tQsq*#~=-UC~Km}C2vEp>WKJ-$(-UU*hM&h5*!+#CBG zY@G63s2A##+VuQ2m~B|*6Jk7oc-LRgAs?h0Mjg1ehyj_tlF{h<?iZY32RuCY@m)j=i@P25q4ognP1C8#$ zgZfHtnGE4eJ;`Bu$?az5g8E7-J`siRCg!ipleth1eKNBw%B;}AGUPC}1~XuL5W1vK zR0@m|T@{Q(3WK;R;H`Yn@!|i})@r{HMQ7lE<(ql2LPZerz@usrzoz_5-2Gq34^BP4 z=v$kUWPXK$@V;{ZjXzPTjsisNK#^-tuWk+>4^+c^7W;CYpP2uRc@2hmJ7fZ$xOndq z0#z}YMX9)O@VR^EIA@qEsK4h<`0tutLgF}_agS}IYDHQsc&$4COD0g>wg$?#Hlq>jgt`WJ=FXRukqK?nieb^u zZqJhg{d#&cr+ce#oa5vq1!Ky!5t0t zh_B*QC$>9*edicH{el{#A+2Dm$brK_78DE>DZ99oE0^5=lI?lAc}q{vL>5TY?qX`t zVSskV-6M|OmdsIhuDzw=;6{G`@&im@5_Pr%5Fp+<%&VPoZhjLE^WlLT=ubd8efUR= z*eQYCBvYi=ELzl9abfAp+QJTlCxCez!BSB+Z7g2A_|x)p{dQI>{^Wi>Ni(}+e`MUZ zRxa;+HFd~{COuQh5U{2NwLylA9{KRl`100W~$VxI82cmxdibPWzs zE~b#qyzZ;y!N^MSD3RSGv^(*SgBJhaJ!wDeDK47Q8$R40OY;tUfO|jFrmYu-tBaD8 zx>x+ev*KM{7F;B9S;Z`@N2x8|Z5Wgu8_@R{2nu*hU4?A&>9eF&&Ea-GUDHyD>YRp9 zrkkzCIa>T`J#A<0ps&9n;H{3i$-%(LCv?1DvBv)hsxb3p@k=;*ib%s-2IbiX#bNb? z`y_H=@iaX(rM5y+4xu%L@F8UY$wd}V5DdqfnJG(E$dqi+t*Hy4}1>3Zl$hsO=>`1DiOyPCF| zv*Q{?aDp4?i+VztV=bN>B3UJMh?;;9)y}}E24VkUCW3Fww zC|8u@v_Uc~2OErEt`gmuY&e; zK1IBv$~l?bO{oXqnn)9tvI#&IVL(wC(fLc%UeNH5hB)kG4|+KH0hNHgYgo-Cba{66 zRPLTQv3%FP`GN->(T@UiGh+z*2RZdXtFgE(HaLxW?cjmKGvhk{KkN7m8_SuzbDrie z9I*Gz-oE&*dnmW{zdyI;BxJSumjXagUJ0lWRPyFz+z^Av>b(iH44Y&{kPT;)*-Amx zI+uKznZe<+Pkx8{=X#3=ejK|vkh zI#y}5BoK=T41R2qtv;ab4HR5}%CAqI6&sOY9X~bhfqus; z|G$lgVH!DDt5d(4I{bj|$gYY#ckTv1@cgTR{rcVdcCgtBKal+YKF**2{xOXD!+p8v zHuj-bL&UvRT6X=VRMHhf{klUwof7t}u3eaXFX751|ws79xOk2Wqx6Fjj zSkDm?DJ%JJ_uidIIsweZi?E;OocVvqGahCNb?)578TSPqI$?QQNiOK4|Kg1#QIPsFgw#Ejy1)Sviw&1#wo6 z6&{uEB5g(X{7)EGnuQu_QgMl?eE(K+rjza3_qC3{$|*R*kDNV6tNJv$e@~bm7Ex!^ z=tCrt)%J@t?a*yV_Tgni3|gj~*f@Bm!Cg9O(YBmA z*aR#qd~fE{r?W~YDlN)6d5!v#`1|n+eu(1bJ@hEYlhN)v;3o=&q&k|Hg|6#=z8@@Z z^&o@`)3IJU7ITNk6~+d}h3NunQ>YJn6R1!CBws~u=pkJ<(cSgz#6J;afLl{1Wxnh< z*prx5gK$Eal#V#!nB}p$9qZ!uwh@+#HE2z!JjSZ6$;e0JM91@7k5jnPL}HO7&;qi;U&->e1K(O%$aWR1=E- zpLBYtc6B3fP4XZpdz-Om@eXowwI!{iNOkpHX`i-BsS66IByc5W>r`JEH_l40b* zwKq|!_=P{+H;rP&1K$j?*j5m|niZR!!uoocHBBw_sJMOfpzaEVzgxCtLAm{6?JZuL ztUVSU$D-x=WxoRle8g3pF?!kT0&euVhdQ(jEle0eFW9D zKrIDXV5`Vj70`uuG3H_w+)Ka(o|6{v$abpa6A!)>rnTs0b)H$yrZ>q~Vy)A#BxXvkpo6^Gx`tw`V!ul zt1kq~L&8gEG%4yVunA2pQr|nD*ZyXQ5a>}D(GP{gXli*pOce@&rAhbo6{6k*mJ!O9 z@l+u>fe^MoQSs)~$e)(25OfyoAxYs~c zi|C%~6uG@QB&5)+ijlAv5rgVKc%K`cMNb}V-bP)R%T3qG&WHr8CNvJE?Tw;prul56gLv4D6W$XWy8ro4^BPlGiOd- zuRk9Zq3Thme{XWXY*>*He7b}#Z!30Yf>$YINN#>c&?bJ{@#l9R$}kIU^GGP1{s~JI z)R^B$1-^X5YeHB{m4JMsZPqk9N0jDX`YD~Et?;Wb| zay0PJbm)c1rb<`0$2L@0JyoOOQfzcmrhKLO>AN(duIWCq#aQ#IlQLl<|GI<2vFV}s z*@DqZcm`l!QG&I0Jq3~khwk;ec0JZ>&-fC8fMO%~*@B2^r)Rh2a#_qe>NIsbfBGdV zPOB1g6BEDKfSN$`+9m(>>!8j~%KvUx=l5s7ZQP{E7}VQhVFQB{F^gTw-K&+eqbJ7b z$p{;HYtk%hRM1qyrcNq7KpH7buXB~@z1yusTW(UlI}1k-TE}kYVo=0h;}Vq}B)Ug2 zi`vj>?*HIB5APxcxY^dvzcbK1OgftGmO<&K*c}vf+^;pe`kG&feeaiMk2!9;bn0%D znENB0P-)3uKP=?Ivup2~5O()4jU!dXDi7_#$t&PS%-q%S*b}w$F+BdGTT&;gGqGG( z@ki<8`Kg}Z%h3d1SwiRDDeO;|EW9-L#L7Vv>otYq*_91i0F#+9GiqH0CN1&(0gjf%y$$(LXIMxyy9Ol-;L)45 zALa>4KRaQW^Ww@QZ$R8{KyyIrL=^IPLCec^G{9(g)W zkITyp)&eIx@uB3|%SJWY9juHR6+mZ*ni-PQ-ZepV5#YXH3?W5-Mh%uX8fTh1@1I6x zV@`?-d`Szz%FrU!QXFhjUvZb!BGP@J+Aso$jd`VqL|263EVdL+G5A(})ysp#1|6-0 z%n9;Y!Z_I9UvAg3VJ#zhy*hyasIslj2T`>QFMwwcrL)^#w33ZsT6jY5{`%i#wwhzA zE_F=KAP-)z4aq_b|9M{L_cB7#O|pnKIRoGBzrzB9;$d~`wd1!ZA5)o3S8`Rh?tL$p-<5|yx zs>{7*_t_pkdfC*FsK+e~`|2CC+&=qIOf5YDc4*Vz?CJadc&!DL@nU8z>VMw#${f*j z$Y8>#h`dx;`*U*6mG<2Id-@{^!Jh}V>q}%3vxr5vuYe(oHKWr^zOLW~2+uq4GERk$ zBuB_%N&W{}jwE^{0yr-;@}U91su#sT%slBts5_zocb==8S4S(OIT4vhcFqMcrfk@1 zU2%TG%IY{uS*1x*^QyXn!xC54AU3V>Lz77dK|F*P7B}k96FM z!x)8IZ3x8O^5BSw@*T7LWIkwKPkAouTBRI=j#%(|(sI#Qh~ygn<>B2UZf8I1cdbUT zOHqGb1@8@5)pEPvvZ@8j(;^z3)NH>4JzGr3hR}e|>+f6S+NqDO3TFuzN{sVPPg?`n z>#i80FH_+JgA^zXsX=fqNwZZS{b7=O*W>*oRO~-hmLDA1t)APen{GLIO5HXPPz8?l z;bq9QJ{8GT-fv`g55qwa0r=!FB}3E7v(zPjE8q9>?Dow~DbJ-ww!0kJP?p0NyWv*^ z#1$DmQ4X}Ooxy>{Q@XVjISj1W+}D$84raGdKFXZ{Ec1}xAV^ZyG~MOJ5IBc7>+)*n zYB6V9DF7*`h(QCz7>P*%@hYn>gTovD0)dku;K(bo2twq6(g5^CdX>5x6{Dd(AhlKT z-pZ+RvVjaenr``%qr9i!41``U?e5${wFV6u#OkRuckd)q@n5TH4_3wu!)PF?yRUQz zK!nF93_lReD-MhQ9|l`Tq?*3pgHRKu_q}vB_Gwr(_Nc^Kdule@; z=kj_(yG)Ax8}gO&lOFQ}!zf=6_g7ZGTzdn)LofbCwQ@8a5?qygn7;IJaVI^!=0ZDy zFe+;{s`=$b{Xh)RF1(=bxoC0a?ci5P#+FB}A1$ykNB-cO<>&{*K_fTbHQ>Y9OcSUh z`~a7{*7bbdxEizh^!w4}((mmufBP0hz4O%Wlw|7Z_h?;?^_WS~3!`$5Py6>l&dJzp z$L>4YMMi%8cjmf>x!Ji%O*eP?d7%7f=PL)IIxCqN+s23brnqKb{7`b~L*`1=l^;r~ zo;_?;_9Q{M{%F$;13Gp3)HLgB6O&XMU7G>zrVQ|S?S4#Eh(;#P0*TC|vyXq$s<%}jr9c*QFd#cNeSA;#N(BD$-LhqLShAmn zhNfm)T!%nbL^af1b655IGTz~OzgSJEREJ7pPzUlOg@y!UPR&G0+&MbgGoB}`+eZW1 zZ++z&!(X58*NsU)u*wi{isJzP{tgofxUCi?(A)3Uvm49LZV|PfWRlKnEBo>#5;d>V zjx_szGTS4>9Dw%VCC_a5(PaxmI*zW7Iy#jJt?zgNZ#wRt8rs`C3glmKyW`8W_c4Ex z1sVuvCsdh*fkt!Jacau-&#sxL^D*aOVAC!h%E7}fEPFpc?6(tTfl0$IU9_8Txwg@A zy7Q9f@rS~e+24AX7IVT<^~+6L)PYomjopuT$*vGw6?FoHJ^POzBblJw~%2uvgg!DjOJu5>sYl(M%wH3J z%#80rP3iyy39Ye}s!%?{r>4-SuABCu&e_oIPL>8x_~Oihh#a|Xpedh{f6g<9GY zZj=bH6zb~g)CuE|f*O;)d@dp7mN`YlGX7AIxTUsqxWYRVKR7_99c}uyw;SrglWWnk zWfEaL&9C$Ji_dP|+QA`ep`vmX)HY%%OKn|frWDl+gJJm{c7PFC>&s>=22 zY^bW}N_}kpx$Yf6h}8W9dve@&S^fg@S&pqV>6PtgL-?9XhQ(AY;*Kw!xc#j_uTEoj z*f=;;tV>w@xZm>PI}?xrC^c;O0#E@A%7S`*#!nC|som=~XfTEGK1Qq`;j~849sezN z-K(`s8Vp!)Z!6j^U7jpati>>fbNS0J?%HL8l#hrUTObbH$o>=FE0xQToZjsKD^sa~ zm#tcLl=y=(DCu!2Gq6I5G0n|vV$f@Os^ zo=Fn3RlAsKuZi3Q#{f-lx4!hni;*DHsUd@-#w3vN&axMmus#l2N0#hfTvkvYaw#c2 zz2}`W>A&%TA*gOiv$y3XagHd%wOoetA3*ij;Ozqgt4FH*efS#_TM!xfmpm`MrrsVv z=HSEnkrskW+$)Oq$vRP7Ut6MoAmWSx_6_DG`$VcAn-G%BD@UhLhuS85(veXJ?XL~H zCNuf`+he#ry;M~XedV6Xpbw)OLB}J+49Nl+9NH3d0w4XA0|G(FfvRmz^tu^PX8F?w z_hA=O7>SH1`SRiy@X!T$iNkraVF;d4M#)ab#GfCAVh?|!?m-t6K`b6>;<-OipZNm? zNK_<@;;XeB%?KhByoRWtc^{8K*ZHYM-1)OXy)YeA!AK+@O*F$FTj^NJxUhOQR~2U+ zGVf3zIyE0nausX8#|i7=ZtB;J;1s?e>~Kc|kWeALgLotmc>oK8;)b|!h=E||Q=x#I z?4WSNHPoE{udS}l#xWblhWl}=T*wWqtCY{e1WB}UL(v{`{bX8ptcBr5pvF0FW%b}_ z=OLyCp6mRV-3}igd%#(dk}~svKrJb-vGUV+@AWi?CSct1?sIc<*@Gvn&>l(n%L`N0zVw`W{3_Q~ zsY#OuTbflz%QPbsrBAcDf_i()T~@7ng^A9FC6^ugexIPz0)X`>p_@Z+2oU`o+MM-O z6%KNQ024A+w&^uvq_MH*7>n!l_2c;l(sgjf(TN7Ztbf87S{L1H7kx)txAwDPp7ScZ=}y#H=r!ym7=y+@ic4(V1JZS3YdA(PNO z9OawPzj;ZK0W6bdpYPo_QJyFQhF*6Xj6PKK>Q(E4Vy%`fjmSt+-SYF#5BbWM@5jiy z82?>$2z4Jzh+lw+lt5YM1zG`lM%-rwEH}t+4IY;t{+&1-J9$ zujs9Wkw1Ad?CW!MeQJm{Y^Z!*;R5xnmS@)2nUD^zE-A^(yMK>qJ-qj{2W`5=JH_1O zW#gygoE`kYQtO5e?MRg@q~v!5CK*afDlz_l5s=$r;k5aua?_>~D1f>A`AGQbrjz7v zpU^SldceneQkEly;$mD4bBv zu`{eFY{5uatKus=RlfW>4i-a&TF&RRzz74};|@t!!P@%95(c+4GqfSS6tZlNWD;^k zAY$#fjT<+rxhR*pJH7z;XVhj3p0Z~tZ_sz85WF*b{`{MDbXW{}J2jpXX*z}~#`g0b z$}Xv3%75~H?3|rLU+C9oD$4tkk{+qB{6`#L^j!^b4>#e!R?Dx{|5_;VV! zTzrRfTo(+NAxf?fa|rA*zBRJv*2NVAQ)+Tu<#~(E!z`&9?d+pdQn+AJ#)GI9rtfP; zZsW%&q?A;kd)B5*hp5}bvz@{aBr=J1CG|EyyxYE;_L}?1|9-nPeBWRn>w9{A5Vy4s zXjvvHAF1$s>rTx(aN}Qf?Hd-l#1sA9kdd3e%{BwKT~%xH-~m-tKb;x0!P3fe|8B+! z%s2l~#}*oIgpQz`;BdP+P_zz;%2J26@86_tcI5E)?|OJ-_Mp}ZvW(B|Z$$yB3-j%=VwY^BW;6C)x>GB+_Qd60ESDs%`H$F8 zRs5CdnL~x;ja3Gw?Cw5Ar~IvNmW#4ZfEQDxVgQ(YIf@`gp1#W6lBo|45!qZ+s^njq z-NZ3rH_weY_WD8dLkI0r1G@UUF3NPwyrVyCf=i9>W|}!^=N7nvAH{!=#10x#m!m%@ zyHbPuAFHVEN}DKk7bk8W;VVQ1EBBW-f1QJtN#;(a?&}+RvueA_z-uE&E^TzW7$)j8MQ30E@k7+FSCPKgwh&Q`{&{J4w}>5gR;`pgco)m^hBai_U+y|VK# zueFkb&-2DV#mW*3`onyhL9)}Uiz}f$&8bn(#8&0yJJ3`Q=-SP2?PpI=qZ#M1Ya73G za~S=U>3-G*fe6VkR_j^K1r8xT2S<&kzfU7b34O=k${2RiwQHOBY{7BvK6~%#dmunb zZa3?5H~F;*`~G``MrQzIr(K=+&tf z9ed1_wS~_vulA)Rx_5NKqUPUF-1NGGLOFQN&2}0ZW{!@ICwmT&A_Xzg%@bN1i9P4m zl-Z(%SrYAZo#m*!q@>2hYGDCNylT()FZW(})n??Mx7__NdCM%=4&mohh&wNvGA!19 z6W?E#YL2mzkz7L|1?Z^7K;v_1-iW`7d$Z7%;zjZpTr3?&oA5hSP}D(QH(M*05tYsl z34JZYJhTPHNH^e;hutwG4mL3A;-tOedZ&(Iaf>l^-&%3NOO|>oD9juuSH&m!wkiAi zHQ8~1pd8+r7<3$vnbQ5b%klr&ShuUh*?5!ohHl1`zB4I?Pv+SjS_!H{|K~VadS#~T z6vwS%dLT521R0f!2pfy#*`Z=X{cfCrOOIK43lq(TT$P_I7eKAHMf0ejc6Xy>7Y(G; zI{f}Fuj8o>UqQ0VXeLbQO)CeP(Nx-Rr`v8P8l8hE<#pNmqL#vbUYYND{UoL*kA+dZ zkJO%htm8d-pENo3!d|IS0gR(^U^-j0)u297EYV+GqZ2@M%AOa#0o|;K}WW`>Lql)~iO8v`3V*v21P<^g6BV*uMMT6uVSw zt-k<<%!{6L80du1r3YG^`Fw;m$6Ut<`8jZpR}rPPyn zvo1RjphB_Mg|OQ9*|=!V72s^pzF}rPAbPISxWQQ^t*w0AyUn4lv7trCqY4sI<}H@V zdomCaDz;mwq5E2&J(^)-Mr6$dHZcmdBu=h;h>feHmePn#KustB?7YvC#eywE-sUC@ zrRmV#P*rJv=RhzufQ=8f<-JNRbdQhZHGIIU_JD~!hhl=ppL=q%^pCKXY z^uP=RCe4oG-P7PNXMBj@bCF>y-iJ1^iHES8ujPG8C2Z!*nZa|)vW|>y!06L$;+>ra z&t4mYphKN~f6UCiv@$(Dq3cy>n_jVLX2+Y2sN79G8@=#B5ARRRQ}w?y2-BO6Oq%dz z^|Okrg{FR6vrM^*5YfT(uy^5Xh#8~sf)P|1p4A3)>(yF7fLE+l3Pa;Tn}9icLp)!0 zdV7!e=~`X@+%i@o3bIG}_{`v#&_L@`^b>ux(LTy9GQF?ud^kG;OY)MFJ>cSqJMD_M zZekFFsb8A&eaK56+dlO=76}%^8aT#?(ug9e0~5asXo`^Y7*#IoGPi-u1aRdz=&AI4 z6X^KV1}+S|Sj8!_)x6At81$e8+qKreV1l-gGM0@kK79KCI)*YnNY@^T$UFKo6-b=9 zN>~362FcJ{vh0X-o@AhR7Lz71!%@Pa#r4D~>OFA)!wI^>+xk2Bz&u2OWAxqvkA@Oz zW(6J`vg~?gb|_;k9NxLy1%AVJNIk(iXeyr4RjQK5Mun0YT6r7#GADL-EB%6#Y zquuGj^hBIM*|bQVL8KXqfO8kufH{LxMhDl?S#m(`0kJ~+{6!-SD=Xy|n@hLQ$R+PT zzX6%?jHq*flc{`S-c=JG+0HKqaHH(Phn9Cl-7&Xcur$JyL7+4HOCB7byoGObjxP=S ziEspG6i)`LDAsT*uj}I5oCEvagi}aJ87JjdlKO4f}E3Wg$CDO zURYhXYiBwXt4ZsN0N&O;sHTti-{r6BAC#q1k$mVf8z=Pk*4;IMW~A4l3+K+AvtzWu z!re=4{3EMA36e5GKs(j?kDn~?)?n8JFd>3 zG)5CY=3HpEo4Zl{`qdToxJALq?&Ew4MU6STSd#IYUQ!>?{nVL!qVzrXwSW3P%8zu; z8ncQ2xTbF=fD0#E9=qM#+vx)yGkXO%2K`2uc%3IJY7bu4F^VDT1l^ov?UX&Z67>0) zg*6eJCN_4Nc^XZfH^5*ta`^HIUx?f@hS{(MY9zgp>}sU!=uFgLK#^TTBdH#7^)93? z&OSA3A7>?~xHgI#rACeNx##?E5A!b>(n9be^1_r~U`={#SwXUtDPTglj~KySJ-)Q^ zgnk)r6<$aK>m(U`<+I*HOo7@YmD@LyaSo<3?08tv!Xb3}!5b#RAISHJzGE_c=r@K( zZ_ok+Q}c1ErATLGs23mRBQY%{$WAz76n9?h)y7u$)2bz=mV zIhLzf$Ef!k2PdbCEj+H-T%-I5Uf--YIiCRIo5C{&lDo;1 zEr>C9v5*)^D-hks{d=FrO`9e^c(4adZa>pW!RaID>4>d2MGsy%eBBf8s);+OmllXC zeRz1dt1Xru1uTp51Phsb#^x_seS3psvV#{w5E4(=R}XxkxGK0Taui5VM?iGVLI&-m zjW~MOv+!?H)h2H3@T$~}JDn=`Ha2hSVceqNX#?PbLtzgan7na)c;3HqK+4CjvYmvW z?m*3r+PiPs9Re4H6TmaDV;5M^Uei?=aO=admI(~Ou=pkW12h54#dWWsa`kUq2#9C? z@W|TUBf4Ny(3FHESBcEB>3HTjDpIM|$g_rV39r=r$9`VDIn3$rGiTSkC$Gv@Ny16eE6{1JejrPqKBST;C50Z`9*7lbqfX6=^bj&ITgRenH(KLeN27pWh*f{oqQBjju!eL0$ zr1|lzyh3t6$eTFF$Uf}pST6YSndgq)yya(3FVzkiMpw!f+&p!zPB(K0N8?3enfiV0k7*Rp17Ey)MbR}_&=xc)G#WJpsd|TnV-%i020wfFa364>n1%A29zXP(o|-ZmFn>t?xK15wg&n$Z0Fp&YD0&SH^7a!dhp%qy$P+hlWs{#4hfE!N zacNVda~@;eOnoTW4yR6<7epm$4*B9%!_W3rRW-6^y`G9+OKQ+2;#_@iE{8|Er_jT( zy`Eia9(3z}UoY?AF}y6Q&fVq&@wLhYGz$SI(^$KK+4SVmTspVZ-x=kPcR z4!qlN|MBC;;&;j*RLH#}{7mCv@19@YM;A5jw%;1pXv*~=#hqldl8J+4LWs!Hgk+{M z;79KyAI`}Bo5?3uVMRa6{maN&i_gioY;K|huP|$H?Kg%4>*PdbC2R<Or*;KH)6SLtX5LTG9j;|rdHMKz7shCp{h_8Pa69>D zfRv51mZ$li3j>XuUndyi?*}i(g&Y^PnW| zAlsun<2AX-DS#*i!*bDu2=Cy1mN0gYq%os}HJkc5r_L;PTKRKI;mY#qzfKLqq>Z|% zo?z}=>=D~K|8caFugAS?6MgpV_2guo+%GLGER8I3W(5SQsH%!yLY9#z$j$%HagOpQP^XtWB8l=`z~&0;K7G(8LMMF^SIT_Z-W zZy=w_+j2gv+nmdmt)nu2Ozi1DDtE^oRTzWoWSKWiM`ZeB;Iz{QmAE761#t4c+~u6SwOIY0&QTo7b<6 zDdI)^Hs{|7v*|jL2{09~8>$AgIsH50;s(>RnlF87`8Xw=Esk0+fwJ*YdQt$+T0&b_ zRhFAGVQ0c7^E0u#ZaLE=sZjDBBSwG!@#Cm*hFlTwUqFx_GQRZ*;2dkb7$CqO3P&>{ts1dSzZ& zL84DWP9pP^91+}|Lg|kw{%x|7ZFCW0s=*DJk_B$3;AD9+UAgPg35&t^WSbB_yI-D< zZ`AAZATzHywJWB5E<4Ni0W($#b;p`}%xv2Fc^L758)b#9hi{b~aOCo%B!x81(|nwG z?VgLyoQ*YoqkG8K^o_zOcR|$TT^9L5tbkt#a^y#&8rZaJ{YC0$qGL?`EKMg&jrSxf z1EZmF5Z7#3vMxOsI0D9RfBAS<2-E;9$q0?eedF)=1r5p+ww%@u(tnWG!rz0iVJFuO z(wlxp{tnbBI(7j*EFiis>FLv&OU)%IVYG|#URwVQ7iA#CqJ(d`cEB09O8SDpf1*lE zcASE1wE~}|kTs!Q^_&h3lyRo|bwoO{Ld*HmqgSeExXr0bPr4}6b51=u!;em{i=}7v z>GwCveI2n?us{yzy+rqQa5ekyy^P-@>@k>z@xDHq()UozLic96J$w8(m_i}N(UOit z@=$3xmr0aD(73m_$-Q>XoHeVQo4TA@Y0U`tP`_QNb3UFSm;zX*f;W@LxxBxoyq2T9ZeF(gpcZq#{O; zO$)r=oe#e=rEdmUz^IiDJP=Ns!nG?VpO`?`(2qz^9vZqH_&1}|Y19$>f1ZMlqGr@{ zejt~Ozif5YzVj$$f{OxFss%7z8PHUSw3>HK9pRakgG?dQ@Mo{w;)OWV2&KufhNH;K z55gfY)ctE)LRLP}u*HZO6asFqv&S&o|{qsjtKYgWenxTns0M; z)q*NgG}=9InwYk^mB`}(Vqfqa`)WKTl2o8GN!!!24QEMscV0=k!xffyEOv6zcl&;( zA~P!rKwok9n7J1J6EdCH*O!U^!cfwVtl5xL{VYEHa19Sog{bu*5OTlP{|`*?US;Rw%x+54{$xtF`5tP3}{@MBY)oO6T;A$0{l+%7~Bs zk-+g^SJr&l7c|KHDfSIC9?7iH4Xc!f{q)}<6fiEdHi98CUl@cN0kCJS7o0R^qBk?+e>d#>&Dn`7OV|Er_GABpo)63Z{;ie&U^_yA6PZ8y zFL>sBzU1!yCcjVGQ>G4EUl~Mcu$&5@c-JWqM?ol=dKA?;duE$S6-Tazw^5?Y?7~w( zKo;~{XFCQ>^z*X90`Cr@S{GFZ8g=a>fH zc?XE|c2DJDQNB1}e1zR7%NnOuZ8(>ZkcLQ>=x_v-4}homVnI6yA8KGxR&?MWpz_*H zOtp1#n((`7a)2uFoi`s`a~KZ20ycuwynygw^9Yyf@7~)1NeY(V8*t-)~v zBF2|aeDHQp+o8)MF7DbnA^Y?5`z}gEO>@mSy?vK2U7GaI!g`n7bAoxDA6nhhzjJC?1=t-{i}ex9705m$wQ{oMgq2K6ujB&NTEUPbML}3 zJMZPFxacm+Ok>4dstBr}@AE&C!toU%5g=t^Yc}m{=+M*p9`Im(JuUkrQf|z!bXYUZ zNm2e~*a)X9{c7E$ z-{6q<-Y4SMqwl^$3c``N5Fq5tkX7)R9TvJ)!lq1NWrO5i)wzjT==EyxXUwLq3rGum zF(}KV4ltb8xjRs(GufG#=T_cC2F&7rOfXaTykzp58>Uu-i+~LXItES$D1@~DszoA5 z9apeNyBs1C9r#Eja9ppZGD%M!hg2*mDjKt9&G*JXS$W=CL3NzMSYkk$eN{&IzVq>r zUi1N8Szn36ifF?len<=;!z55p!LOgIv8;ki2P-jt#kxONVjUn={)Z0};cHYM7L1*Ae&>b* zS6Dy#c4eIJ{lt}R7H*gnFk-&Bj<;oNI*=Kc7uIn%)IzVD#K&G&+^7DQO^v#Q`KzLJ zY&(G~+q`Yt1=cG&4FZIbl@{yP{Zva4G#qr^dQDZaSMIJUe`s&#*ul#qw_;r`j^-Ic z`jnCQG!&rIg+lm($#ie~Rz020$a7uKZH_#Qm!t~4TXVAe49ZH^f)mrfV}QPdhPk~s z`H4Eq`j^hp2`esN_&R}^neMcz+{mG_RRk;_^O0LLJ#!wFa%wID-|#OUg*%}DVq<4I z1!TVFEF#Iq#421 zmEW~L?re>g-EVcs_U#SGlI)yGp4rvlpQ^|D9{d4S4UGdYT$W(yTmx>kaX8356LFJN zkj2Sg&pP$9PuOKb!giW`LU)lfFzzwsX-MjSi_3P9_;ZGlJaBI+DSE&tfq@*Dbd;o=( z4zu_g)bO?w)>Hk)x>j|H4%51ISEG7ZW?GI{wdtdeq_H|+%jlYB9Lo|^{SU;@&NNT@BI%hqiEiXYBr5iv2s zk!3j|Z2&n)))3N0)h&^FtV3t5M&5#Nx5#64(#WM^Nly9lpNwYLCWHm|W?X!F;(^B2fpK2&XVywyP)7+!c%f0cQ%}LisoX(0C;O zT&YPDAr8uZxaFUvu0C+!0FG}z#(()UW9_^h(Wcmd8vcCOU%&?rxR|sO+>41OwS*z0 zmC?oF5>?JbWFS1S`P5BiPpoyxcf^8Ea7Xts%@(#332$+m*b@*vIO6Ip`S;4(L0WDf zYYEH(JK4I3f6?wvM!ne=HG~jbUAy#*Tbe0Dol-KYe3`Vxa=Q7~(xIsVDdxSNSe~Bl zxBG?4`nF$snjfm`Qt)HKj}2dXKaM|VmA%edYv3ZwV^%wRD8C&XR5ZRy=XI_-=~*JVVS+7|wW=udc7Ahc8A3rl1!Q2K^|G1~Y>OQA;J}YM zdhM>NEHL%kr?>uFZb3HPhw!*SI+#Uvq1_cRCNUwXMCv9n1d3Pv3tS6XkkJA2PEvRt zSP)8yM;o3*e<_2@*jPBmf3t&`8wx{he0PoNlcN;sU^bnS8`TGYg&b9u9~UuxUsKGi zaFn@9qp=`2{^G^l)p_pYfP_*hNr`W0H`{nx#fn(&6^~VfLPT8^rZfO;?2=S0wG={9 z(g4~(B1?@3(qBN!LCMjU8|jk5yx!`n2 zt4=1pV_UsaWs27a%r&KAhtq-|fgg|F(Zh=M9Mc+w#*(PJWA4Ny9gMZJ?n8AF z3&lk^UAcYg?oComtv#jT#BX_}U!Pb?X)61AXIxs*9ppbLD@)J1?AP}K-^{<8yqI>r zg1@6i*%fVN1XXikZBe_wpMHxl+O&MYzfTXO_=f7*RZWdQi}t{M@QMy`&h;!}XS8bc zb8r6TS|b#b5^ZNDYE?BU$yt1Oc3F?}&d-BwZ#FK-d%ZH=BO+bxdKNDxF0k<)J7gA=4A4&Y^Tsc&_YO8(xP>MuFVN+~?TOCiUxBKR&y( zWNgN&Od+;{nF@?3w11xs;p7A=nP= zHdA|&_VzZ_{FJLEpWDXS*&GG7ja|t1_x{!cM@M*-y|1H$W%ivreIjr8#rdo+opPuA z^n$z<2oUB|t;wo*e)AzRv%j=4dwPrk>~#d~+WceD^GgG{A716(k=I!VRsQO<-_AJx z(doaM)ZgN|F6p~lg7hpOKje0+S| zUIsBK0IGcFy=G|McCQ!f#xkf>Wg2y=;76~5{f(KhP)MZWo8bv@d9>=}O$A_GN90iO<$On!0kw`Xa_0clA9`a-nP zbjqBV-dwJ~>4X;I3xvFt3d7E)HdR zuFYM$0L}TLFQTwy3H^48Y8k2%d!>Gi$b=^X?$R(Z79d^=viy+pG=uNl9ci0Fz(hJT zQJ*6>kxh3*jnPDjfOTsKHlO^l^2_KN>UJIEg(Ydtz{@&y=&+FwsYm==kDgyfu9#44 zkzGn)#irHtUfo~*koK>@12rk1$G_3dZOCv?0Z+5L|F9agEb36=WAFHpZgKl}2fE%H zWt2M+u7{p-H=}SJdb~M|oLPiTD5Y;6zXHTT@phHaB%(0|K>#RL4|sGt;|4NZA(5Vw zr$BhXTKn!5^}F!VqcSLx?E+obtt*ECIOm*aNUXSqqP4|=zz+V|=bMbIjCYJ|Q1gt> zy?&YiOJ_@`qhYYZlinUFvL^Uo=&TKdY*?kbXzh{i?~03Ceew&HzE7581o(tW^H^h< zKP|wZZoM<=zkq-#2LdiI2;xU(Pw}Oa7SR$MI1Pn>q^8{7fK|fP z_`d^g6=k*8LT1o()s!!F>`@c!Kkeu?d=`AIL%(1Vqq2!w{5Hjm58Jsd#E+Ygj}rRa z$Z441?f^sg;~|n7TZiZJJt}a+q<5O3x@)9%nW&P`^ZWzA#b1($qjP#9sX|~C$teKY z+X_V@nhKe`P)s~D%nJaIv}Q=ix5e-RnZ7R#Uf1HKT!+F|m`W5-D2t{TdB%MdJAz?x zMksd2ZvI)q)C%w`T#6rq003|@y5Qeoc}<(=w|e`%sxd~^F1&i74uIrB(izHB15!a- z;5+#@{f$V$K?f%xuuw3_e_M2DcMos6n2V|dd2>Uw&BBwK7&i4wk~#^Gl|tUJ6o)u! zz{}faM~afdoziVGC=RK|uo)Z~!f&l253s6vCCC-I=R1J4~0ZDaXy`EJu>WQ$EG zx>r_%4KRcy>RprUL2j_81LH2cmg@*$1)?mHAmE^zCwxV(Wwjn}OlagiFM{|f@>lK( ztM2G31|raL^4qPLoPO`=yH+2?0MS z54JVu?b~fKu9R^AB<)t(d8CSv77aQ7RMXf!>5M&roV% znO2s@LHufEKLD|tV=BRF3cPgghTrZy)oYMQRIzmxur6#_+DBm|e6~kYpM`h%Gr78XTXcEqdM)6}g7tZ44HbqZRG)I}da!ofmN zjN0~*3N5JRLEX8vOB24eXL3hdTe~iMvApNK)6~?IqRPIXh)jShuF`}m8Y(GWv+pO{ z;>)+5NG<6ATF7*N{~SAh4$t&fST9gy+~&(Jx=-h9!BHffRRe}E{Bmwt!B_n^h%OnJp(jzh_ zS}rnf9Hjd3_si+9M}>75t>` z;zzpPpNRYA!oy&G>9PSqnLu%(_2EM@XlDQ{wSTc)U>v|-7%`F{GAMGi>#2J6>m&9I zv5b8pK0ThE5yMUe?%v8}9~-VW#tzk#z!+ofJO|0AwJ4|Rq`Gg1f{DVz48RYHWWoU_ znV6<1YG1I1M6`x?$$Uh~_!zHru=)yOV+C=jMZc*E1R*j6iIiaV=BLo+{uJgjWwMce z9GzzO$Q!OFFk$jR{&$vL45#N@nFt=FhJG6iq2I$dS>X+!6Kmr^u#MwzmSzg`^zl?} zICTsps(Z)$hSsdQ$_;+0ORoNnC~*ebAdqL@FBj-($n(Ro=I~s- zfuU(pEkw6xU2tzc5;h6FqdQ(vjk>Y==)*TZ*+KuFl&@ea;6;AEFaY4Nhq-N!o+Kam zv}1`#aY>B5b{&L{qG!Ahc>*^2$-*hWA8!Mjp=-EXgdC1GbXCoGv1#-_C38Ydr}fl6 z{{FMJ~isry`$Ivwd82Iv-4=3W=m#YuCOv|2l(vAGZgHi z!@{)hARKbT`IxT}Y0%Lh9CH%Z%%1NgwhGjhNsoStD4ZYXU3Uw$dAfAS%X2&g*n?J!#K#>)uVFvF39THO)44cftw}6ycU$7guyxk zpQ5&I@A!8XjC0a4$&Me+Z!%^|k)u`p25)54FlXn;OXjyXekmz4Z>1Ppr}9bnk1;<6 zPo;VZql}%550vN^4zFSe!G?Su{ne5kmq@uZk zThvsca-(^;mIb}fScPJP)gX?M929+>D$=bk%soPcY|q_PYeZ{Miu(9=y>WX2zHNoH z!k8;PSOG-Mt-&Z?t^0W;-aDOM1w4VUvh_FoRow~^veW!rA*teQ1T$nCNqag+LPAi?dtlmhVKS85fCv*utHrtc zqmNz7@tk~MeAd|XK2x2``fK+*_akX9HmE~f<_@Flf;`a+AOfs}8z8}#6AfMxlZZNw zSWB5{KjJK~trbs}e6sA42d=6Z8+x2IBg{4@xa@jjzOWy8mT#o*hNrW^vvbOH?E|UD zIcTFHZ6Q=QZk_XzlLepH)>F>SuNV1XY?CfK!?q_pzfM1p+r+ry;?<6dN$2KwjkMTZ zWzT2d{WG4%R8cJZUSFq6m%0kVUjvsLLrOHq>rLuAcu3-R#VPml8qpJCvhy}8X0M*S zIF5Q)q0c*MI){_H9AeO`moJx4gv2)*woL+)MOAPQK1zsI%t%=e_8{S`5D(6tJv+i5 zZ}K2|B8?TL%Vfk9&4pnbZPih^gSB3WWA}MWC6J_RTC5uzJ0)!JV4)uo&*$_fxOziG zq@Ot#Dl#Ta=iMUQ#x)lMSR!j%T<0|-a14#(yAE;695{#bD5@#YmTf~S*yJHyY*myn zgSy)FkrW#Gi{#skROq_jmVjf*lqs;oFWkA)05={OITr^vty>)RE{X^UOl%-h-kP6y za3grG%5a_onb0WQty~}eFq|We)K9~Grk$#Bn7b=*fQiC@ z$J$VxP9~`nB7Vb!$uB_@PWhC~T6cmKpDv0}Mxjv2iH%<3fr)PiUn}IOts(yJi5!X7 z*KI=CI5DRc@#q>$zc24?^K9lNXFV%R*_9D`%huqiYxdv;v!2vbOm99ttMf$7fcO25 zS?`_Suaix;RTdw2<`~_W(9yYD_bKrWPtJWB?wb8~m+eW)ti?ah&F<|PW%&BV`Qfiq z!fH~^6>CehS$fG$8vd>5)B+VY=C0}MkkhuP@M`m%{_u9h_zP);xR{h7CA526DGX3W zLnPw*yOTUd5L{(?n2?02z1Y&Wex(yFq{QEwXxx)Dm!bsi#Px`gwkY4O^DFvETCTr= zi8ln34UuH{gBZ-1r{(a-kqz>J_?1?m&Lh9ELQHUMLX313Ih!ed$Wh?tev?sU%oM5x z2B1G-lfvnl+cCZjwnJL1ogr`h`qT)EG?S_@G5%C~n4oMcWSo1^+dCE#JJN*_lU=8N z{R5bP>EUIwZV&523G6L|uWUh|01R@zvr(N{5P+@50Vqe*0^0VY4lDzb(-smUy3e2g z3fTJSZoAzUHy;Jc?I%BZ$JN6QUBVtR5GjG+uegx0Tl%e>KC!4XfEH~xu4D>=D7d6< zFIiEpeZXA0rQIpnK~f+sSQ#DtdvHaIc}F{3Ol@^i@t4Qw;i;JqTmDGxQPlj_=yNqZ zoj+TaEt}k|U`N9Dpt>!4e#&kz`2HUEbIrfF=e}~S@%h_^Yv!#CW|$1VVEk#&AH*h+ zloA95K@6jJ`ceGAM%-wT1Ub-iw%z3PrV{O#s|nR#{oj_6{q=)A$Kol%H5yRXzQYM} znmy-bc6PN3fEaq@ATe*%?3_)8%O;owO`1ax1VNPJ)7E3yuwM8$Wxwg4>y?ubUhzo; zhn7_a)gmcr6lVksoJn=qsY}~hTSFHkh0|5%e#Mr!F+j12C(Ku$@GJL0c+UYb$#PgFHI)K7G; z5s$u#ean6}-6>O%C$_DDy$!A|;14y@C<>qdi{dQ5B)-p(a zuzz3i)L)a>Z=4vmW@fdmahsfXILpAin)N?KK3vr{c+e`t{aKxdp04A1$l~F(1I3$dTt(dZz5U-rTq5fDIMCs%mBR3)RFO_rgosJgydoJ{=S~!JCrDHHMi( z#rKMd^5_iT!VWlo%AE}peDs`OYWT+}Qp|@bx`ZxBST<~w59Bl@T=y$sBh7TLI z+wRWAi|x|`^Gcsi@}Kaz%@Wd()yKz@g&nfox6<2gWvR)SRsrm#`3ym5@rImOnt%ma zrbK!Sz*#~E)M6kp_&aP(KFZ%n?+Bfc z^Lu77%mge|^V{ZEbc{Ue0%%InSk1ZC?A1|Cr}$j2j4duCj(ZP_`}vgiE&Hl9_4)2< zpFgIpTJ){>)7zypbDFk&W#E6?i6eXK%$&}JH@+uDY#dc_u5Zis+hOfMkq_zWm^r1{ zmt+&X@gazA>zeioYgT@Iy*ZUwAk}8BwYND-HXV@DLLFyhRW-wsZ0^?Dv;7%N(%X+u z-oAO$)Vbn1S{8sYrWoAp#pmfMA{#;;|99B3K69Uul3UR`PgK&V;Ai9eLWhVh`4_*&AGLprzqqqi}yCE)I45 z{#njdJ2HtzQM10*9{2#wuG6b%!qECIDh-WS6h^q{t%9=1j~=xZ=~*KDLP)DqdjGU1 zeFqM>PD{|HNpXQ;Zvk{AECcOCH>cI~K|~+wj0bC#1tHR_WY^X{oH?m22u$ts6g3nl z?X}}#j2ju{W-ZwtxTWWjJ73>to!&UBVB)o$inuiAGOf8;r&hRWP7GXLJ)l}ymy^T2 z9NMl}+hpqHO!Fq4rUbq3cdkm~i(l8NMIe-_8T1srgq49_YGXK_FZ|t~C8mH=5QqEE zEqnXz>+^$MyY{M-o+--l{keHX;&CaDv7^5sw}jx-3I15oYn$M6R zk8hwClYlH5ZeHLRGfAIh+STE40T&6q(kMMLKU;bLffVSRap9bFk9nTpHj&0Pky*=L zyx~gxA;!?*y91AQxIny8J8|*#@LQiMiklrie7mB;vAI>h{jbtfpA775;o$hwx3ul3 z`mKA1&UqZa-dcC8ozMZ=_L*nAa$TE6PyB0rUG~vVYXH#{ZZFVd3Q)g3yF!{2a>7K! zMM~ooNyU%fKGFRcq)ilVk_p4Z%PQSh2Hx1jN#~CMz2ENa@&c3J0|FV+)lNWaL7qrt zu3`SA*(2bf>+h_RIR&XhBrhU@Mb;+tUtH?-cRl;|ee}b{pY9dVE}?*c$$cV?8%HhQ z>*^P@1&+{PJ?22E|B2djvYDR+{Yk!$cspl4a(+@dBO(`l2v(lP@m*vAw(s>&rdx=6 z({nsO5H3MG+Cc??rP!RtC{dc*5MFdc^|2}CC1;jf|61FLq=?12a1R9OK~GPVDgYv& zyAQ{SAtAWZO;+{I4g=CkETA={7ShN2Ro|jfGRD>^iAGVEwHC6Ft0H*f~2qFVINb#liLz zJ}g*w?EORe*F(CLC&lVcKIwM*^UUL?{7U+FJKQbAFRsb@9liIM7LT0$_|*C@v1dLx zoVJ3h(H59I-0jZT&-rsspEI7aIKnjh#^-)X`Y-b59U6Z%Z0U-k>=^}DhyY<*>&Mk< z*@_630COb9PGoB_d|HG3c1n$dJ52m5@a8G#Ysf#`2=)1ESY`4@*PclSGA)%vR>zq}%HxW+wQCB@`X)&j)^FTCYshsdMBCn{m$=GqMO8bU==q~#<% zE_A+G{(Bj%Xi!kSQGA+Qy*p$c+^i6@{}2-4Dl5V7S#H2P@n^;M4(~US7(&xO?Vt9rqv4gP)%A3+}Ren?MaG=VtwR0+WC&1 zm(ykU#r=Vyr~mp`b<@Rkk>}}rNHl8YXb@ZeRh(l^k9kK_@>wH;43=g(=i%tws|}w# zyGEgdn*_l^=buKlx}+9{ACw|UMz0l z*qbFFgtO|k((p0(7Mu4CHAWc&q}TekTPzi5Dd^I35|R8ZyLe%dC&|gb4fC-S3Y`C3 z-An)c;&w;`P-6wyE!rZK3b)S#x)*dP&x+jn*8aX((S5CDQgIUUKFz$8r|3$34cOXu z|M40QVHJay{-Sa0?vRlVSVa}{6ZEJejG?mZ+shAIoxNI*@Yn5_nf_bn{EW8?vs`+) zyX5RU`F+5W2(9w0n`cduz7}=!Y3?We$U{48E8EkYja<3|y?29S~Ex{2K0cm4Yzb>%@J&@i@kiIyupw_(Q zc(VNqL~K!h;t~*C;IHEb!Ct~8?^5>(OEoMa;kx>y|L@+pFgVQ|7rKE{MG;4 z%QRAM+_>@MEtk|}ow2ckOnk|~q#YI&6R{ay}?x=?u_Luja}d4m)UtktweT1u^H6+;(vxR5Ulse;e-ynl&7CH6Nn^Qui3 zkjR%v9|T3du17nnhJ}Z962>Me-M>oYOTj$EmQb!7>g84HR{A+nxq{vMGURp*he+QF zom%Ioi;vT5ZMycKv?Dq)VtJ^Gv-5BY-N`gn=mgVe-$8GP??ClqPkwTNq9s0Pmfy?P zxZ0sObiJqdTFMUGp6=4Cu;A%+Tt&kH1J2S`_ju;7Eb^hX)uql6fz2wzqDJ1f+a0|O zVZ}8jk6M1QP3Qah#l?MEnj>C|s@wX61kZmU_jy(9ZPnZDfB!O3jg!{x|KmqlTckq& zkDvelW&g)=&{F^Zp>;cAPPga3KT2=5r0D+2@CaPq2)wWf%|Sa$n547~ zo^CHBNMC?ZC@Egu3L6|obwz6W9=pH`tiz3KcSC&+TLN)It}CUtQjV8Gq@SS+nxY?# zWOM{jJ$b(F?e7F$-9e-TLqr(^y_nlD4{u!`K72SJLJbNTOgdjFy>e3;ZVzA}prU{jL59;gnt|>jiPWdb7NMNx=bv&ko83`s( zYNcCVK{zM%2jb(Zl4CAfyf_6}WIUV;MHvkLF5)fi@~N_ecE8HccPljN+qVT~zC*G_7s_Ox@GQvMAnGCi^ zkor4l@P3Rj%*A%;K3v>|`}gmMiz@)HtrkF5QApq^q8Ruk;cMd>`dH)$?yHO>*OBTc z(qtED4?-ldJrx!GJhFrUT1o;7`jNM>1e>S@H*DE5g_wC8sVa>0YtYo8(p!_qvM?$S zlX>;#JQo^L!+U?Tij zqRB~&>?OW5&0LbW0z_XvMg5aJAcBPuehn8RTs1*n+y+q=VMABLngR&PuO57xlLO`*ef8^3b_dNi>yaA(K$1jH zIwLpI!@Y&)o9jE4!QlUkOkTG>0#y;`f2#niA-ZFXVphPhk_SQJP=5#wMUN2yK&P&z zhyx49hA^n=C+0>7SWM2~`q6`A2%dS5(<^0$U)6@rWd~7zQxt#QDV>INGq;r7) zVf7S8)gs5ez^UY>s*R&FSM)wV7LfN-ytRDpp<(*O@Yh1|pTugCfe4H(-O_*kC6soE z3o-piAHF+enW@1U67=Sv1f}dI;Hn1q@Q++3JE$NhwB>SATzh3`{a<$zwR~ zX7vSv${LECSx=~r?N9>}cA;=jfcC+eemB-0`|1dKqad~lU~}E)BPpYgTfov<@~~J{|$k&I{c$AGD&VBw;|60XuhD|Je~grK8f0pAS$`? zQ_WV%kuEaJ^}W4gv=HY0`I%Q6dVOrGB@4o~S8>}BvLVO6!=*7}KKkhH7azCDaO?TE ziKqXgGt;4H%h3~6TG=s6MbcHUSTC5TH_WiRzoS{>4&rQ zAoDs=cQD=zb9Z7#nc(U4NZDw7Avrqt>RqMapmA2~f0uHTp&2YUnbHcS{hB^F0$n8* zRNNasc#9(YJ9yuuR?XgU;HObX0wH;`-|1Iy={@m(r#7pjj2?u=h%3p?&X&35Xv>MR zs7yp)gUbK{2vyk+pOeIjv@5oWzXpI4tQcpQ!vn`c2T)aWaxMAZrNKYs-vj>(LK7{e zpeXv+Az!gnWZNFu*Xq#|Y(bpy>fv*QHOUVaeR({6UF!kqQ?l4NNC#ZPPm%@HX8Ze% zAFc+};8#qa6uCgShw7i&jqHY19YV)oM^x4oq$GuTo#A-PwalA$2O`gO_gf&D&~~Z; zk6pS_vrFasit7LGASE^xY)ZN@5XK^oSTk`4)WB-vD(8YYX~m6?ZYH3!*La~T%1@$o z{Ni7u##Sc^P;v#IEiHO;%URKJ3}yc^WxVaPismU;asJ$ic1?C_iGNB%*X9M=3XaKtLl}(GUZb5h73xsBirSGpbR|nyZ?BnKq|&!twoSGuD-yx;V8* z#p7$&a#ob@>g(Piz39}0asMpzlmF}z6G#LCh2(tw)j}=>XrQ)YOy>C`4JcoW0d*SG z#{T1r7Y*Z-mE813jgGpY@l&~IEL$BlJeF}-A(Y_4S=pGxE9Wi^UEs}ja~!(-|KVor zWEAFV-vKM9Qwa9uFu4CHb!SVq%`PRambm9q^zT*@gmBthhyNQzcsm0st>dDRws6vV z!X#%<cimPHIhfL6RP(u==O<#ue4839ETRJ({^PT6LK z(va2=x^~q+%Jb4whu zSEz<0tI1p?yxYCNOVW3NaYq|t29z6-U_Y$3pDgzYO~bikIAB|uNe|a*KWt8hH*669 zl>Qx8>W_Dn4MjIbMFGaGsfA<|ae>qLiQL%`-1Z~k?ny`AUK z&TIFi;+az$$e`@i)d_PUq4EY7QfGM3^;S>5dYI*vS~I6cA9uuwAI23aokE-Oh75(YmAHld2k8XlW#&}-7bcv09AvHC1O0)8}SR5qf>Wh3P zrTaNXyP0l7Es)BTImwU%{>62Dn_pCHZ=y&aXptC)SxMUIe^Utv*NBvQ+Po z*ReeNKGLP4o+afDn_NU5(D)Dz>5+s4S19l@b2f}{7R=U^0B^)u>e5Vx4S8KSy6EWG z|0a0ynXgli@{yvIOSMe2?XqJG38G{tvYv2y+R=pxVfRH|-r8l$su3?!bAA+_N+=EE z%Pge{l?PjWJ}ZGQmZ@%@=7nVHkU)8&R>5@kW7=?ZdbCBaF0LPrVz^RIX*&9*v8BrK;5@IDB_0S zX!MF`y<9oq-$+JQG1(4VZU(`;vE7q{PujyM+0W5rKiRb?tXVKHJ zAG<#}apU=yt_ItjV#A0F19bx^w2$1^2ruMeOC1NMs7$Zm`Yeg_C$>F>$YReHb&noi z9i(E%SoN=Ns{Fs*%pI8XsfN$TF@}On*Ev>b%2IS-=XGSNFSd9;Y;t)gvFO05W>5|@not|Z%bB7ao$RsFfT%Lq}GdUskC}sM0oShDk(oRhH zp-ashnd(ceNRUsdbxq?#`N64(tH# zQ+yQb;(?+9mCMj8e>7T?M2IY7fjNgFYangUzc<$QFZ*C9oh}r+$6}|{5>t<;sHBK~ zmnR`WL>n46T|>tH3TrxaDA-98wq2qH`=4yOnc?=`Iau1wn>T+u(8Tyz#C&u_Z2W;7 zSlo9P+2-%4v0{7sgr0;62?{TEXl4D@H(;KxK?Bjz@0&Iz^uVtkNbHPE1`{r^LW=ox zJ`Te*(iSoeH!9nKmXG}p8jf%h`VP7DpCC!#ccMWrlOyR@xkld6gIxBe<*;Wb)UjVacE} zI(%fNmdN8Gp6=3W+3H05|5%xVcrLpqb3bH4>06_0f^E9hey21<8=AYj`%4m@oddm9 z0)U;8L)wPz<@K?URWd;7DDW6*_dTeUeV_1st9WjSs#0UsrTY-bpx?=8Aa|XnBIiWu zx`^ETB~d;HhQi8Tf}E6wv8xk*|L^6*fMwnXqJV@kg1|64<|_z>AT2oVsk`5`Mbb~Y zX8=_!{$l$S&1y*}L)X6@BnH=Oy|fnEbuxzZ2SGQC_^Xp6p>N6)rK%SUA5@+QRpS+r zk<$%Eb?z{)^Z!g^@_?C$?TJ7EAco8i2y{j&7Egn|;P0rSA(VK*#Q^>OE_nkNS9|YW zzGk(_J@j?GTyI_?eWnhU?DmFp#}-(|0Q4f_!t1v4Xl{$KJGlWscvEk;32eI|5PpSs z#ekk61|z$uFCn>=9*Qn_pQHK|hR}@Y_4)~60;aj>NMr4_L02;N5pJ6F#>inXm;an|3vc1q~ zn0FZn2+@OrkXH#-&6Ihw^YcfxdlvcUh0L6*CBdVOx2{RCFNj#sqG*F)yQ1pJ>Jp4) zpZ3%>-)>C2Oh!?24D=%1 zZp_+#5j>1Pj}vWk;(@;KSBT1A5#S1+i7~$Q9UY%n-rb$d8nb6RhSj`XchiC96E#e$ z630hiP#8h40&>67*N8rBbV@GZgsLPXu16s-l9D}$1=rP*5kTg+^xy#t8hMY#3}p2M z7^R=T4IRg5thDtjZSM$022^!Jf*+h5F&i@16R$=J6$HWg2;5Slq!Ij*sgRnW-5m@J zRIrB&A5RZEiXU@nv+Y;ZrVJ$o+{i!OZ)X!#Ad_-8{1)vtyjVL*8qPPpzDIu{dM`3_ znBK`=p>d!sG(bxaB+E9vTNn)_piT_#UM7(;E`&tT7{HOyVeKSa!ow*9b$RlnCtP!| zt{sWX8gDYRQYp2iqRxlPw0lSt0gOt4mnP!De1Y<+-ka7UpI^+(jaZrV0B2`o!bj?v zgnA9gk*bF4G!fBVL3u$JgwD$qi@dt@5&&(Pauo=yT%`}6EJp1!inbv9B2*uI8rLce z2uy)4F|E*Im<$zC-!jYc{|WX=fT=-F;2h%X<5Tr+Wee7=H=Hi*3E zIbhVXEC5=?khJyvew0qiA2IzqcKgYcoc#PiqHtkWphB|j;sr?Ew<=mV0-ICT0EpFk zyQN{JZ`8Otqt~&&KTP(kC|^iaeFazM96Ozyh|qyhur`f3?Lhrgv<@Xk|KH&|RhRIS zaT@}89K^4M-4-rb@St3yk+)v=T_H!vb~s_vvld>Xtq|loh;{_{MecM(gFZ{{#DNo7 zR3HJ|fzTxQ?nu(!?t>b7pM=l69(e_sM=EVCZwDVq+ZI5_bI!4M4;z4UOC}~N4mgv& zB-q%Fn~71wX50QkGUT=lP$Om1;w@4=#jmy*^fl{I^d~%(j@0xuvoFy^54GVl z-T=x9bV0OU2oNk{IJo@pmug#Y3U%$JdD`=jppVu)r_Kq!m-fTrUsJ>~7S*^vYg~t- z173$Bs)Nr_TU$e-$!6b6T>vzqk*;oJ-zQSb$9?x)_;CtjqVUP%MKO}-(`(zf*RAh! ze6VQ>88|%}0OYh!NheNPvl6U`LUJ3rl&insB;`0$J&{H`+C-rJb>a6dj28994@!e^ z%Td4fZ_X+D+1NeV@C}iz5V=;#ibTM8z4_CHiwmBfz6`xgly4D^mL~m0BvPO#hAsDn zF5E2Y;FHgtax&02-cHcYdGTI0vvuEIS$b)- zKDJ?WCCFi3lsKnVV>6Gc>hC7KJIVJpG_B3_0-`9L+u2832RgLrC0tx-9**n67?=P4 ztWlMEHp>=T2t;D`(S{M%9Aec_4U6E->ypV#S2jkR6> zwmj-)yl?3_ZT&BY^)#0~>*efZIAXnbmo<-k*F1YPLjUMjjeSS2t*!Ta)k}2?+Md6? zDd6a>3jvD;yB%$I{OkFzKcC$xdfmhK@MA*-KaU=f%Jf3kYR=$m?@+qi+JYw5`Q zE+iR`8s$dHFyGVjA(iLOrfb)%Nh4tVj5G_Q(I>CqSYg=!0JeG1hEB)b* zw!xO)9Z!P)>pXrEx#yDl8m@uy#%zO(j7G8+7Iwe$+|c--(a`K^!%#w z)^;PlfAtGcwjSs;4lRxGw0&qF)s%Txo*hO8?IUanbGtfxo|g(_PN4qtyB>Z ze9crcYwyE;_5BB|ANaTdVAsx9uiDh)%gYyT?rN7laOEKNZx#32M6c4IS+c5Uz0y`K83XETL+w1a~~o!@`Ik26R>EtElnHc$t0 z^xxN3za;3(q(LE)rompCh1Fz)=2Do%2k0W&UWVDxGex0!O+u)8( zCSyUXXj8DMzWPR9l>F)5@g&mX`AuEC@kCKk*AKBGIIGwXaQ2-9Db4mz42s)UPsW z-{t2OBQQM~$$=_|C*>!%2cL)4t+9;e|0q)F5e)LZOEwnEKlbEkn=VehOAg+PXv-zK zrhG_1ybC1g8w|6j`W%4@K}Q3Fd+FMhJD^GOt6&YtEm3^?Ok5sUXf5kT>d9bN8#2n1 zBiFCdl)=68oFR1&;YMK@Y@|bIFqw`qaJktr&B|+yCbULjasEu&I=AaPuS5!_NsAT- z!Th73s-Vjp96P$Y`lWgSS9P_KJ}@QY#*uE3djZ!t2wAZceZv^%{UCj2;1}^DV+(%_ z2>KLHv}&T>yCt5LS8fBrzqQe#No0i&3Ci~n{LBQT%DcOF89~qClUY8a!7lGoHE_yeF>JM(XDNguy9?y6x`NSWi!LqDqV0!)bZ6q069%00Z zZq3#2Qud5quCY7k)vItwCTK?}WR?J#Z7Eniy2YJHR|-yhKtUgv_5~!%&Zd?60)n8|P=@Ks zepYVuO=x5VsS9{H`bv}4DPCeUV^rJ^b&2Gew7n6oVdUD1J z%-o_i{e3C&s!Y!{2bI z;E~0XNHB)SUS1~>TE~94c!1wNjJ2Cc2-Wi>q9fpRY+O@l|XzdH=SFiK)qsbD70OQ+$>6Z|b=wy6Q9x__| zotb}&vC}r>93jBQv5rX&EMXdHmks@5l#a%T1?C=HrcaN=y3U~I=0vyF>Kj%z4zw`a z3i&?bdaFhl=yS?Lm^e+;<{9QqY34BA&x}dxQOcS%YjTK4d1CKTSt&Pj+df98t3%=> zn!&ax0X@fpycSli!?ArpP|@{NPx<^IlS7DUTlf8I2nkMzGIiI0i3C#eJQ@!woJWV& z{J;LXOX%u~BPH-i!D3KKd@!IOaUXtF3<#cO)2>~+2$J`~iHQP$?nJfH;Q6i+;}Iif z60@r!`0^+a)U_KnOu#VpMuHBUE_Two3&NsO_*zvYb>k_qmh zz66#dM((rZw#M^WVg0@UK~6sYr+MNDD!_;Mwww$J9|?4DKV6%4nKW+6k|oDZp0qgj zy-K@&vRSC{C{4*SdYoMRIc=1I=6Y3|wogjqNf69bKLPnCI`rgeh@Hq<{MDXhe>^ju zGT77Hb9agr6hh1(vrWKiIeM!Dl_8lA$mKlR4fWCTG2Aw$PJ;#(3l`0u9RpK;T2@Eu zru)bWo3v`x*))?PsFWfGoV`BpVrQF!duY9z0oq{d`-ka!oCYUSMW-incuLdxfB!v! zb#UpWIi8HnqutBeZk0ig-c`!zO&d1cL-!#CV|!ebghyfh{D3rMCxCaP3JR3-lT+3S zqahBFTboz%@tGaNr4NLMhpV#xR1X4o0fvrfe1z3%AWV^%zk!FE-dvFOFxzA?7=T`Gb*wg1jQUz2b&R1Bove(Djq}n z_2@Ba$*2N}#F&L5+9~n23qso&{0`mQqK6rLfVWB0X zCCK8!A=@TDf9{CgoIij5JyYpcre;UQKZ`AJ@d;UaN~Q59F-rOOziJ^il(`t_4e{WE^$XOy{c!8+NU zh`|Hunr0rKzER?LuwvzNb8RVYT=gyR$M6_G3HvBkz(JSeFB`+l>lESl_gn7Z97V;& zJ(IHHX9nZGAArciLNVd<(`e;?{qp5rj%z0!osnYuNxk@+Os>12#18IUmhu@z_ol(b-(Th^~{--E!indf{@ zQe_I4@KW^|Q=7{|PNWVdA&fk2O?{{3@Iz)t+r>-fr}I6l!d8~-LcWrV0_O#e4vj}f z8lQgtQv$fc7Lo&6j$nz^$%WQE4l9`&dHCwJYv@H;kp+aoGTWm(Q%zyw(`=Q7L#TNZ z^$jS+-+?@df%c?0;BhuAEKDL-F!Lzt(#MYb{xH4*b`ym(F$n)=W?~%zvdPDW=6t=$ zNlwR(^ac@=Y>N1(aO?z=SYmDmZ;KbtBh^XfAxjsRh%CRdIfE9JjuR%#U)wKYQNbK^ zN^va+$UF?d=}p==#Eyf#&!_>q;4 zEut2w#~c4IczFjhjWro0QoS+rAGT$H*;(QSE6$`mb0k1x!|G(=5Kh)O?fA1%2RhyOcJR9HU|P5Dj( zbdxr1=Kh53{Il~N;!vw@ZJw(?*Tm!1n2r96o*W3i`0CX8k}KOHE_=*Pb}`t$_Pz!$ z*bTRJIi>tgw4R%8+_*Q6*%!c8``G>3KI=)ViK|=_z6C@$_0ojuFx!6QK)u2Th6=A{ z!vFyS8AfFR;4%TsVm$oWmUUMeU$;N!GoC=i^zO&p+(Qeqys!WGPS7XOD&l@IxWgM1 z7gA<%kv$%n);N2m!tdz%zIKvNeKI3;)hzHNe$f_|f=^Cw{(TQ3sY{O*WM8_X^pjIf zwKQ!;Na#Wb`?Efe)AuWeM2+ZJh4&KfiZHjSigMZ>1dE1?^&ndfVYgM!E_ik}lC1Em zo838|1VS{I`r5*yn4IW!lm`|IkZ?0A%K;*QB}k;|=SubBDW>)Y%pA|Y=NKF{E-QYW zvJb@<83ucw6D#UtVxb9hz4yx+8EjkdAznH)eIIUJ8z@Zv)YsSFZs59>NBf7V&!-nl1j9h;YZmcm?Q2-fNrc1P1a5UxEQ#qdwSy00qd@T0B@*LP$QxTBy z2n;fcU{aFXI|Ik}gcM%b=9$c&+1$;3GGUO(T=D&YUB|EGQI6}rpd7}?W_V_#3}#ZTi-2_$#W$;zfW?CI za1+`gTvpIhvdP^2U#rvTT^n@pljhpMKx<+Q+0$Iy%_mRFR>XCsq>3O<9QoI_;kCEi zlFeK{dW8l};4@+m;Y7y~B|wTfSW#Z!f2Vb;=FKBPfNY5oLpi!iJx(CsqC(O+j`MD+ zVlu^HihZN-To^&q$&xEM&`NNUcE@t73{gv26ICT3#F?V@?V`tTUPs2i-qE zPhw+4b16Rgu_t>GEPIrJZuKJcLP&6ez)F}I0=3|X#NS@*2Ur00W|w`i;u@t_G_C@; zX!~IQ^CGKs`plU;dU`rJB%V4g!<75`|4bA~%VC5>0S_+Y5F4dxMTj&p?CDM|jF|dOEhpcmU_p%N5E|(K+b-=&vIi`T@ z+vkb=3U%mYfIQFx@BQ9#LckaX9*03@2UzU)(4b1hP!S^(X9Qv`Wx@>T0&GYSWLe}US>ix)*}jDHKc(N)f)#nQ4g?yxiXQKqFz*; zJjy7^#fmd!n|#M3KSU>im6#>q4!IkO(1%HHQ^N*3(mG@V)Z#G1h`r)7sVjTNWQXK3 zRL|U)7U3iGws+g1%0+`E-+*UyI`#2mTLO%9KHC#=5^;U8lin6uZRHg^`I+T4a*j1{{`q|F7AP>@sE&zCYQ;Hsk4Sg zcvbj0JGB}7HH$0_^Bsa^RptV^v5+P42%iEL%>?W&LU!ZjH~dWAP_b!3$T+~mp9TM6 z7K!W9pVUAOQp$n-3-V4jGx_xV{Ses0rvBgnNQh{v%nXc!oYSo|B)ba)}-Hv%y(f!Drw7mV)>cc&qoK(^Hz7#?XN5a+Y5>P-n(ypu$z~aS*OBJE)SdmT z37jqAQhCyrxEQZd*d|a)DG@fr5rkKO?&5jg+K~NEHiEY)L@M7&Ksoq44BV{1#?sq8=h^=irMViKxcNE$I}R6csPJoJs=U`5k6Ah~h`gend_PGeYV&*Q{Z!8@@6 z$Pv}ZeA#cCI*cgo(@(Vv7nPq35$)4016bzzbVKLql=+JFAbc zYSJ0o@c;~3pa#2>V|mjZt+Q~vu>(;&FjWLtS@Mit^5^fgRyNVniUEz?(bmzLI25O0 ziWSxk32>qG{N~NBi5Z6-PI&53IB9ycV@MA|W|q;6^@|(G-P;chrbOw+(+H94=#=I@FK zQQ^17c}qP3XW8-M%;TI49PfHDafIB%KzgheEQmktk7m4P(qEH6U z{{R`CgsSKdnSQhzu8AQ1f8&J(@3xNGw7p@-Cne4?$Y9dviNkT%a-iyI^&MA`SiAu! z!=W@29DN4m7R^CE@gjgEZuwM*SKZzJM6T3ZhHUr1_{&kEV3YUcWE@~#iIsmSbD3;0 z&KU$`;~`Q3UIRxwfMv=Q$Xv{r|Nf-LAKt&WVcS1D=Q~kQKm6STBuoT422t(a>v~dK z$1ft>&0$>MyfUVdqt*MY>vorx9vyVTKU(9;xJ z+Qva819om8;D}TKB|M8T+2mFG#leFecMbHm;?$&3f)T7-G9{XizTJ6ysET6tO_#O7 z7e5y6YihN9cE4kJH}*$Ymv~8K%WnV7S(?eyt5dhG4bq$fIUHowq^N~#(H}KeRXmxG zKK^craKOFafreU}n5?FZNZu=Nww+=zX3RP`G>7=L)INORWjN%0XA`}XM3PeoD_1$F zz5_EgOdzXdfNaRpzPS_j{MFy+6ehO+WC|}IhNU}uy6fbA+S$rr+Ze)%cD0`+ z`8!fExsO%!L>~zVW+cF70#<$3yHW(CCaX4x!wlXlEo53;5Cm ztE?J|&J1+0!kZBr>cU+fOMLhcou&w+%u)vGfQ!s`cNg^F)vhxFbz;6d1Lt$bkDb~= zsBo_M_JGM}|Bht~m6)|(MM|wR<0eDzCBwvyyhE&1T{4?`A>i$8Pdq2+h3C?$0U5yr zefGj0D0om(YTQB_1|%>wfc% z2`$s`hCD@55}j|~zuy3*B5DaC)$g5GQ=x3UY?sTCo{!F~i@fsqj3UA^EiEncm}?*e zvk2nmca%sLee5k3h~Yx0j}7{*di9QcXwLalr}{FJcf8cDxk%8S+L`S}co%79b9o(U zq15a*KSFr3ER`M!dhKfsun5pC*L*7h+_OOgxVZQc5|&W`0gB&#`wbq5?RN;Z4;kw% z7tRSWJ5g43YHRV7T*N+Vo%w9w`&6w2E@H3blbp-eAonG+={BCU9|juro^SqsZMfW zqQC)iG6*H5f-Yk5Gx4D99x_@&0B3Rh(6#IUe=Uh46JTM%2hjFO<>?9z+EmZV7+gOc zyVV;-?eT}F*s`{DYSp^S#oCYnt#3Qn0dsthg?;_%mFm{Qno5b%3LQ4G!3a@~*VpR1 zhk~iHMZv3A8U@b_NK(|xmQsgE+X^>Z!g`HXN$yMEO+Joctg1odD#`{x`KVeaYYkDg z37gumX;W=vv5Ws`6qC!ojIuj)5>@@yy>P z$6`mz>9mShn%99Ms{H-|=N0fE`Pnmj2Cj(zRO*4w5ffzKB{UZQUBWJVNSrSK$D*gj z09T)(*qEpQX_}V;*$ovkFLuu?jh|Myl}RSO^8%g z2v4(~{tYXeb?cXPiZ9Fu#hwt?G+>Tg?Bkl>NNBC8i%EPh5IYen8;#Q`cyZZ9q@xe=^;$KyKfI3_y5dJZ$*5|mOMAC>4XM;o9r z;pT6jw)bEBJ)X4Fvm|dk7~fd7HqqI{rS4#zHt>aR(HQXRcd4-n+DfqRG}nK=h86Bt z+$49O%a;k*2t_YKidP_OoDJ{!Pum38B;U>&`lbGc6WuP~n(?|&suCu)wS@hW7%rJ9 z!Ynk+nZ#F+`#~^}Oxx!CBJUL|R^$o>#?z2&>P_m7i5R2_AjBA;CS&>hJ-m2c-Ly1xeCq!6`5Fw4bd zA5L~(kP6!%0d$gZzqf-gQA`4M!D|koy%qlxJE40(h8^2uJRLL5HNVD_Hy+{zU={65 z*(d_K#1l7i_ZCn>LM~1nNAYqznqI^KV-VDN+v;xRjr6I!5qN&Z5-M~ha^NM4a!zqY zH>dVs4$!Qe6XZ{B9C%d(DN#PAW&41Ihhekt5uimlDA3b_f3yi`E~(|i!}fM$_Ujiu z{nl8nmSZBR5YXP8r$*q`a0TosV+R_Z2R0OQE%iXqbvN?d%=3%S5)kfsaI)%)-f7ep zr|-7<>?mX;s<}Fuq43X3-ao#7@1FYFhje&9+nzrSnZ?-Uq<6%_>Fm|U)isExdIJgA zpZaU4N($kmdip=F@DbD{{7h98ddc-l=x-svu_TksAYB7GKkIU))r)Uzox@UUr-iiW z^XFa_%&xb%m)&?MhQ0H?v>E4hpj43rKO_hYn|dDWI17S{CK(0!jeKUOzI`VF_NV-< zMxwbqRRdl19^XTJa)={AetHh$6U%-ats*}?$j0t$U|>aFIDw=i1e{l_xNs>j)DP%$ z1BH-lWI#TpX6YvTCTzG;$cq<`vh9cy042KM>3!154|lUYl=@AV-ECf z`CiHCw+mGbIiY3i`R3mErL~82(rDA>Fh%8e6cxmogLG)D)3b_L4Z*}^S)P14EV<)8 z9be+hqpTx0Fw^1Qv%dBm9Kh-KJ&gu||CkR`RMIErqKnWo@N-n2|j@$6c_aNz<*S=qf(ytrghMQqW{;@>6XG1YGS%gS92gd?jwHB- zSzg0I14zEE&$h4Rp_x?C>T>!K)aA4<1ccOZM`LAF)F@>N(YsS{er%5oRDZ;gB|MBg zk`D9u?yx-|a*FwUlbfptAHTXY7(onCjIQT4^{d_a<;a>O3zx7M=y!Rw8h;vH1)v>@$Tnv&w z-8KKv9yOu|ijHjYicE6dCaqg{H5I~B?=7Us{PZ~nMU)(SoC=bF+uYH!EfCqc^gq&M zeIa;)d7y86#n9+c<uMx3 zhz?gi&J;NB9w^=Wy~(HbHj)H|0k`T;3$n6pVg`%;)QG^wX6nCbS91EVmrFkqF zy()+64?cBFn9-oLFW@;Wr!BZs7(P{-SW)mADzNtj;-V3g8@sRi^#s?W_x^gDZ&g?X z_o(aIfbLK=xIyKVV&nt7upohp(cg%oOlUB%l8!a|YWTQeFfSX&uUNS}AjPn2`FB9) zh|$C0j6OOzO#Q;3rz}=b1fhO3W_J4^-xCNX#DT?A#=ilB7{3{erk7Qh-_JOKHc@5HjY+8HBWKn$+h*vTfw%tSM5UzK2mk*A!MO*V{X?6q34ie|25ri~~ z^JhJTbU{QR#0hYq!Fuz4MKAKHF*m}4HNgBd2L4`T`Xum&A}$cZ*Fn1BCm(fM0W&AT z^=7lCuy{olCZ{)u#Y1-CoA>Wyl74<4Erk$J)YNieQNu10EG4Ppr=3(j*!-%O61RLh zBQ?0$v)_-l=i2vGyx+KC!-)uGgcap#J6+a-&UnJDn8*L=e=^y0W^F4;h8_p!$X~ zGegR#pjw;hBqeE{$T=T#`^nOJlvPsT4Z)kLo#}ApKR2;gX5pNdo45%@3=(~dxYU#s zQ!0da5_58gEL8#=-W_(xe8?jo z2NSbjT1eP9p5Rn+2jM<4q{3Rtb^Nb9F+iL&o>cE~UsYE?eb^qLLx@QMVDhv|5^ar^dIo{*5wskUjG z`j?l43N;C8nW;*AmT>#{%ZbOH34Y3tRZR>M+>7r|r*Ehlcbp_3n%IYV*pdpAhB>1( zq9oR4i=)82>mGG2rKymX*_Y$b{w+LaTafDhh`1|LE?x|;Hda1V$eCaQ6G0RLEO`A8Lo^K@|Na*EM zVqy5)C%@}y=xCsKd?_|e;D*?9b%Qzxr-dCXWMrA@^t2G4M|r07dh+5Pm~YI=7ue>U(dF(LMpd-W@jK)(Nr>#S&X z%%|gCjy9hZECqDVjhWai8?s?=MInetNR<%!+w6~mRa1uO#o(FV%;p+eAq;yd(@QuH zQb2jeMtF|?EFQR$>-4XsieS?FLtygiLZ{K9gTUAaNf3q3c|y%d$#(@cn)T?&Ucb_p zH94kvgRh@Cd$#kyfm7h33A0zSewI0!>Y5N#;*k*)@Jd1jDas?K_xQ<^eE!Dbz}cZ@ ztX&)U>yBV9_eorP(eZ1#(PF7@(f$>f=$oVq*EUyrl-Dlg8^BLIR{=?8z$BEfLw#W{ zyG1g3X?x(rN>YbIndIwzK}Xy!$2AJ$Dd{-8xNfFGIfQ(Ia}| zNp2Ll6n=FBre8`S<@_P!xYGSrQMx*Z1IOxk$e^RK_gap{I_%7-HK&znfBfE+|L`ZSDjMmd3Hdv8F9(yp#qxX|({(!+<^%*pBek z^d3m6AQpJkJnlmhLXzzWD$9>veSz_Edj5!3eP&BIB`s9cY)AWOt*-R(GnbLu@^NCT z=k-V2VKnzc&MDP$*e(ab@{9j$7Nkil#-lUKtD`O9Lc9r?h~%m&)YI34%iJDDxf%@` z>a*yW&^E&QWwB(^_aak7+%~VyxN=`t8ytKn;d8K(csZIFUF_YEAp!WBIaBobEW-J2 z#H;iJo0CVXK9@epCa9mlOCVeBLqTq`Z@+f7+s3#XJlcON*yz}s0V!Z4r!j@B4I3`^%+@A*`tyVK5|`*oFge*B+g6lKWKzX{lyMc0%$}h&061EL8+> z+bp8(?a&-BcLyNKG^$MFIv+p2FzDQ}eX%!t^wjiTG<>pij}|rz1(!-5;nD+24hcZF zpwniY;QNBp(jtgMe00X9^4sdFe&qM0?%msefD>8p8E7k3-77&9D-lQ*Lo6g_;T%)x zwhOIHY*gFW=5Pz@)T`HfUVnrH9z3Nzc`v7=W4m@kaKLm$s=wgTUG|;u+loUQlaac7 zWUj(0aU%a@NX`WPNSy~0=xGl!I(C#Qy;N)2saTRu5Pm$?byR=Skqv@XvDc|ys+7n9 zgWf*T8uqEAB*A@!`V|+NVJ|=?)UFaStx8v^{*^*cCX|r5SxR{h8$fy)3J4Q>Q~kIV zebgOQqk2|8{P>GR5gf1MzwovXT-Jyl-E#Jk?U_r?Xqw(zkQpI*E#k zg|SHbHI6S99q-jk$hYv$q@mY)6GXTAzAxhFhL5`!u^-Nw$huTM9S~^%38#_iInqf+ zo#BdG+EQ)ilV4lwA0K!Ye~@Pt7(${d&et_EC~4?4KVz?2)9mQ8!rB?*;`Sa{!EI}9 ztCt0E0$#0#Im~SLVU2GA6y8NhN855ETQ#|>H3TI%HVJTV2(D5xq>T=!-uBHcz`ONH z%%NWU9}BJUVU-+XrRqGO)T@*eAu1by<9Vu@`tU05)Xu{Vr?KdWO$$w5*X&V05X!*>_nyGkHp)MpH0>*F;0=w75_ z2PXoTch}m+`tZ}{KudNG8j>!S>8X_;pu^ufPIy6|LaeN_e1orx7hlg*kSLn$2)f-z zJ!XAp;bXJx2SRoa(B*mG-{Xi;*(}MjR8mk5D9s0YT!T^A@IN7e7*dF5IhZ3ma(tT? zJ+alHdUD+_lpE5W~qsR zZ&pjTMH-odn~x_EPSe!1z%g0LI-Y^=hzrVNrBE<^*KkJo5wA-%9u-AbdKEh=hMQH} z@jeOW3wkfd4;*N&?Vz9r;lY#lPGgP>s8&4~_IqFo>Q7{UuVz(O3yPo)6}Lri3xNA9 zr&rIOS-g!XX4VxPxQU;usbJ95SSnS3x5W}5!4B)Q8)Bep3^RZ0I`tiUZX@T18dD}2 zs#XKb*SALdSn1{p8<*qtaN@kWcao1d6bSVku@MP;OVXE)XgG}h*RA(jh)oni3)Zgf zw>w)hS_%a(no^_b2kXvoXCD+g)kw6xQ~<9uO2Uy72&vdSMw9RRfvL0OGpe6A3Q?CH z{aBx!XS|`uU`JCWQHF$Cop)d^yxM)wAJV4A~oi0Bya|%qO@?kkgh&1FtGd9 z8OxSkuJ=arJSx!@&4PNfcgC-N{qFDQc%hAM2fAh3ao8~Y5-3W~L1ZCUT@F=If8y=U zEo?1!?%q9Ik+aJ6CWz-+@C+@LjUMY}Ns3M>^^n*`Uf6BTd6X=z$n0ojbOyP)HQjE+ z{nK%r$45%%Vlo~@)^8qHI=D5-l4$KgLAl*_?nPb_jCjUuIQXl4*;{GpbW_La}X50J9wPkjB79=Hfc}s{&oE1PuMQ z;4epm?2w3b2E{(h@;G^B+PmK>pBW^N(5ZW+7k~cEe}7>wJ&{toLh+f<#3$ zf6tz`)lbO)EY3Tzjp7M!OZzC6XN$(ld&iG9`=8G#=iEcW2+p4{p8GdD@6iCCZ=$hd zx|AFFmU{!*RIk~)``!^>A8{IaZw^%b%odKuftcCYZ0Xbj29-(x0@OnQmU|T=a)%He z%$uETc5gw#DlR5aaTtO2b^c2O9|9qP3DhtfDRYrh#p0X66atr=eLkD!$g>iqBTM8? zju_dq=W?P8c14Y2PBx<=rzzK+@3-~%#NZ^6wxRlnGbV`_CsntI>VO}Yc8uJhex{nH zkcb{Jg){fhe39OiIz8~qP*KG5p>J91z!JzkrdO9Qdayvxi9>v(^w0+c`oNnRdYa}l z6X%>+pq38EOyAcNho0+nkcw?133qiyr78EO;hKB!eB;KA+l~MGAGx%<{p3j;(d!s&5K9^p zdbL{#%|QdID9BnOwLU0#X3mYKgrxBsZgR|@F0(&sMK`Thj)cR{IZV+M_yC=3ZZSIe z#Y99BJ+=>9O=yjzAN>23Yv{`8KRjY5E5foHPp+N@G}1{FEhQ8%Z!vvxg}%{=n|J5I zgDBdRlu^~uj)CE)K?93;o!i9b#%N&?ATC<8I;ym>Z{M!SaGe7EK+^{$jUIS%$!ZY! zP3O2NmPt!jnYjd@TMF8id`-Jf}u61V) znbR~dbN{UcMU>ie5=rw*=ouH9`w_oayz6#fQdw z7HiWq=nU?Vkw@n_F6H%zEr$LH1ipy&SXa=;kv_ndqegzreV?`<4QB%3(x+5_V2C4+nc&x9|%~r7 z$hF+rUDm$b+LWjzOO_NP#Kg<^0zsDP`WKkms6s@_wT>d8y2*5B`bV}z<{bp&4g9-N z?_E4jNG6pZ=9arz$@%l4?`RIeWL+vQB;Pvby0wquH|3nW(FPa9A-mu2AKQ4W;~I*T zsN+I<;>iw7W$yzMiF(D=(9wcjm68c z<3oU8ok6Cm&}h(7+)63p+F82O&+Sb|pc|3lKf44uDx%5L=W}-!w1l%YlVTQsUH@j( z2GCm=!9Qpsm2&aJ*sZ#6EACNi=1w)+E?-@5*MDA+`WB!E$^31 zh~|-`@)gN9KNnyMR0;uU?cfnX!5>rx!@`^`;Vbt1{J3nHA-!fSO*yu2oK1W@Rg6ds zo?tp$mpql#3nn|aJwh1qM&&Vi-DrhtP>rTzdjiYc-cyc_F6Y$-v;q!|uTkvM>x!Wf zxPez6=gYv`n3!VC+6s9)LwgwUcf}FpOhvcmb7@CYYIUo5o7GW#{nEB*C3Ru%od<94 zMlN})Y`>M?^=o6E_r2p-uw;~f(tyC_^0iFtQaJFdGX3EQTY8KdHEQb1+&t1moov=X z{O*qE^AHEzTfDax(mLCzpUpJW9S@>9;ZL8|guLGmANHQm&c?MkTfO73_HXyWa;$PK zgB=({O>?FU2yPx4bEKN%NAa`PKF(+3;t|8Zr#yEg`zc}1qYK!WD$d>9w#VVl*?MRF zv2O&x6MAph9B1p_`Ddt;#VHK>#JZ@cs8(-qJz6k+oQ-R6Ki<+N_zpUf`449Hf#IZ{ zXY8)w-#FEsEd_%3emHGoTu8WKg zILPBZw33m1qA7WV{OlZ$(LrD9JV8^91x&t_0+LbuHrz@9u-}1~m0Up0>g3g$FyT$$ zB$efvHxDJD`wA)3DD{$?6$~l81GTT(2_#t97WTHyxxCd+l#>87K2AHWmuI+vaS*v# zAU9PrtJX6KS5U*;L~*?T!@G@xjRgA{x^wdBKc6i+wBWzG{CxLso+cTdVu0|LNL(vi z$N$~Xie+Su54<)x`1d?sj^BL11a@DTiH+6Kq7JM%l^&AGOnkJMAQ2^h_}FAPD%ehq zYqp9lz}IU$Av+d=;lG{7I<-JNWHJL^|N0V5%R6bQNNh}Inqix>-_vbqUaLoSb;roI z>Uq(^RkEOt7-yx<5R$u!A9xFPv?NRcou&|_Ua4)ZZDCu`+SRq?!Xs>f$*&8G= z@%<;vz&^JsuUDhRBB6yaU6Hmq_aw+#Si{?cQaka(lWf?1!a-uCoL=sohXkBPwVcH< znnA`Wd|qWJN0QHbUPDU`T7`WY_gE~fBLj!X9z+eV`au{z)$l+Le5d3>!~8+J5J=27KYGSBt{@Mz{;sx zN59Y0pyC#NuK@`FDNhH^)l{a55By0)5Xv=9zgkLb8H~B8mPiv?E!d)UqEwagC!&J^ z>blh3j&5J*4V}QHmv#Jd)JR=`BgDOI7I%jB7C@eCm`ZoeqgoumfsCe(-{&R<8nnYf zQ8HKM3ul_XY0pOrA(7k^xUo81shx0(j?69l=6Jx7!-xIwYZ0_V=!k@ZtOg8aucl>Q z-)&r?eMwr?T4%r+YkB1wJ!cMTiM2${rcw&v?7pO#>(vgz}e%IpPRGVB;jU1 z#dw`lxj|$S{I0yT|LcE!UD)jZ2iKglH0)Uof3R9d;|#20np9j(EP+q$Eb4|JRN4KP z6=}b55y+dT2ULsTI2X5|aApy%h9$;KW@Ibf!cAJ4Q@C}kfS{)Xd zVvSt3V8Pkknti?rnpru1WzWUMz>rL49lu)~jkj<;fzT@Q)xWJ?DxbG*$@41@ubvhr z{xXki;m<$+%xIB5GHdwo$2GA`tYZq*SMQC^(GxKq(&ko&jVQ414@a1i?pxo&KC_Jo zGFHCLyE7ylxAwejHvCqrHBa1MT4VZFNwhhz_jR4%QKI3nRjR=@(IL$`G=)2<3fpW~whS5OGaz{MR9T+RZ&b0hQCZ(}}4l#E~TTLDZ}{i!X% z>ge%v#{UWqJASLvK{iZW=cCnUhlF_2cvpfg0UXJuWx;|49RiG-abt8hilt>wt(OrT zcE*3+(1huTrjiDwv;ZeaCk|{NPuZP@4g5cU>>oy6a?gaW5JPd{Pn?p?3M9E|Y#^V- z*63n=^}BstOSYp87s}ZW`cs3BGt0F-cfLcNT9p5wYhgvISsOpr--xaoHVm6XdHY#L zy~^w>3VX_M0G1R=zXgd_;+OTMnYrtZcbWyZpOP!t z_JJ?0pWSG70m0S3{^XIrKl~p{fxNzn_5XyL%G6kKgIgRwaR}^^8okOH|Np{%5)5SX zhr8`eCzYh1uz7@aFWWn8X&BZCHea{B4Iq4`-nA}?A^@Omxd9W`1-yOx7OdvSt5c14 za+&2j91a;LEICZ&pAYm2vBn>lYWVC+=ej4&QJLTxf9uQJ^&4J=Ay`$Q!LuiH`-v^T z-!mSnO+1i86SD{AFRYpM1e^!x&=&x4gjBbtBvogwSXY#B6ked9>qI}m;#hq_j3lOL zb`Kro?rYl^pA>pfZYmrJ9>8QW50Fn8E21lZl`zA{=v1;jKV^H8ON+UoK9agUBCe6{ zN@q=H(@kv`V^LN&-r2ReSpULdNs2s&4NDM$OeNzP?a`Voc3zwl`u7)h8l7X#H^kE? zi(b4aMnsRDqoUsjQe}e(_%~pbq846w(@8s%7Uf;%H1eRvWB-gL%Cv+;ER%szEG|*L zU^0wN6NOQ9Q$YZou@3i|umFTBn2VkfMRv6x1^i({dD zLi|84yb8;H;*qt1HkU8BHa*12(~K1Bc)vw zK;`(`A>R+~^{w%q2^DzQve%@0bMokScr^BAgLVe1Smfm zad#Hi*md9JN&12`1MUyE?o}7d*DY+}q*>s-aGXi$p%l$BihXszw+{)1TizOLFbs;Fx3eLmOM8nxnAf)pfFw!S>##A&ZW zc*URddY}C3y5roeKTS4yx3oplC^jY~2?h&K@AQ1%%jN{B59L(}9z250qZ=8&W52Zf z->+tTGdw5D4|FnsvQcM|DH%(gM(_KkSfXuY0mv_qIxEaXigomer*);?vG}f@COb%+ z$JFxy2Xio^AQ63u&9V-^oyyVg-)IzGMMBI;29>FpRI5YUiO>r`U)$XMFh!OVAjR>#XP7$yS$3hoVHXR)vy~s=dJ`>+4M4 zLz%-@C@@3~+1pq?Py0U)FxHm(+U%`w0Ma&qy%vVAjX zvb(+Y>g#h3YT#NHJmYF~c<2Ku#EG>?x(CVkS}8{(WCj=0PyqD@)?5(>df!8E+6_e95Jha#OM!UTfPr`>!Ls$Qr&v1lTR7Z8Uc~uHOXq(LXQYH-C*=3F-Ap+ z1{f^%54{%r=x2OLpO&%&{PnAf1eexvJ$HCe;CoRL8Duy_Q>Lpn5nP%txZCiWmdMFo{VilNbn8 zJf2ZzV#&NkJ4|QmtgLP@9om!SMBzq%X z35ce8yKR{LLL`+|)91ZH1@pfLtgPavHuJq@l+3mfB~RTucsjEK3Rx`=?6$ti$?4%_ zn91Atd2X3uhjwi>IG8xj^wlviS+;oj3G3lfy3v@&OH|hJR4{A2zGesTW>=`Z^sL;Z zyrq~#z9Qxvl~3PiPR8Zm$i3j*&qDb2#{w7Qz%DrKzDV~ODjndxcYGP02708TQzchB zGuIlEw;72Fe(U5BVgskqedm!Kx)Q{L5qFqW5hKzNOeWSqr0B7A0NB6_92}&bFYaZ1 zpq6l4s(~}x(kVz|9i19NXp>Kb)LXJ6g-etMj!OlXM$1@6gme!QN4;NZDx-ad?pa6l z9eEilz&aQc0YC?G*k@tT5bFTjRaOP6%UQESy1z&y)9OH_+zoFK$|r~~m8rDyQfilt zvK`0U{;X)`5Fh9nM8i)N`AeKym5KCeh3XCId&r-kF; zjxlOW6Ui8aI9c3h@f5g{N%$rS_lR?Q=gxsV>t$m~SHp^lCO4KWr9F)e=<UlS0m{?x?QRqCxL~nrU8@`@_{=@KfU;!nrNe|FwI%J4PxC0) z4mwc9#Z*;ni2aEStBB5*wyRH0%3FE%f{#E)Es4K`Hca-V=leF(bXZFy2jPY7*0A7FImWq51t+J!*9ppv zOS0~ygei;+FYxqDdH5_Ro0_o(4*lhqwspG>+<^xM*6O~;`rNi&mg@=PlLQx=4DLVx4cs45Xwh1A*>qBj2+IH5?I2B-v!PS?aHgU7>d^v`#dk6`sx{3ro;Y>$BPv5ELPwki;7%Y63uoDwz}3^B8b-0a z#1N)Fkhg%4Lm6!lmer?_aWM}sdE`jruI(NG{4ng&feewBB*`^Z#WRNH{QaQ0SlT9m z+uOy9BOt|Xp%=!N^O#4^;KE?{8#*$nGN)!^ViO*e%jmJ&)9oT0+RiN_z8@`yXxJK+ z9t;JVA^WI6ndY4*gzvmOEO?+~05nDG0}GL?H5>c4@?WZij6+{E!`*gL`}@yxjB_RM zD^d@Z^P!-*RoSt-&(qvOKsIx1yQ!vy80_)M^HebaH9h3S0dAE7IZz$LQ>2xaeA;XH<_eAW0XM*Q0U{8dhlKNnhi^eIlR617da^<}V!E*(H`TMKJ- zaKmp*`trvFYH1)~1y%w$rI>Mr@Y98u=!k{|y3pnD6BRy|TQS2>Vz}Y{AbOZF*FI-_ zlIsdlV&vK`v>KZ)8d-9Pl@A`?+!2OuIj?ZY7qcSlOVq6N#nHytH@^-9PouJ~n9>w! zw@1$F-s+M1=rEtFBkNHRWuM0a7(84ybnm#;lXdMbkBXwR<2|S_&A2!;&jqYu=0SUL zfn?Dr?NuiS<2EIHO>(JZiI=pqXhs2)pC}N!b}O6ikvfPHJ^($(PMYeC$2_ir+Y2Y= z@ltLTKTbyK^ZF$;MHv?gGHf&|b9desc+PW`fxo3(Gq2V7qG@me3M&!sSa^P@YgI6E zl&#hZ#{QwfCTpf|5{{*w7`-@?nKJAwAZ`uJunOEp^RI6yqX=bgEJ|_Y-4?g~48l-j zAo9H<USvrpiYui578^3d6ID(_7}%L6zKkA zq`Tetc8qt}+A2RBCEx{86#1;~ob+Z}@mf8xy-V1vdxRyCMQ#!SpmzG1NtS#v$mNS= zwRo$y4Ne<~lLJmCi$c5G-<&CQ6*yI5dA^y*6Q;Lqkb6Yn%>~ zR|iPsMA7~0duHgDJY=R@U{`+l)wdCICX6WSQHy6uDd3dYj3i^`wq~`5xme2-kA0P& zX0X}b96X;_N4i`B-nq%vu%Ow0`Aw;nJbrU3=S7WPrM&)eR)zk*t(Fl`GRyZK)6C^Z zcjI0MZ#{OalRQsNwDZ3ymFwFRS?D^qL!bQzYu&hOvz*5Lkh|thKKFSJzZ!SSPP@Uf7-;>bHcksx#1%4r50S_z*eM!YK_&>R@ag@z z*I{(hMUP?!!kTM`{&?>BAf|DO*+2lrKdi$a)(6t4M$@Vlz}ZS!qgrs+NXQG1ZEs9} zf@`1vGAcWWBDpL3G5=a!YtED2BHl>rOX(ou5UGxZp$cpt8T}8rvc0oZpY#sU?na}C zYO?fCqh`un!?*{lmd3{ZJD94AOf=)1#||c&(geHb0jMs>INYF=N=2oaLP9eMVbXYd zqPUZRG10D~6=QUr68PS}*rV{z2flnNk0vn-_K@nbw|0i&*MX+%s*`5fBh3lgl5PN2 zf$!BUHleq~Af#I&3XH6A*uSn8MK{!WM@lb)IfTaw#K`pG>x^>pUH&}BavPUf^%#XF z4IsNN1v`)0tW$Y%VSIiT$nJK?9@t}u|L)${?`2FeiLAdo%=*lNi&d6<-AcA#7R2|G z7|OfNEW1;JT)lbDB&SmO$5`94xl{780Gq+p_&jUIr*R%siBRs0&SNS)Cr@B$vynan z@C+gVsh$v_S9l@(fi2$F_%pEL`>$d;hnKoGB)44i2*MoPiX)*pawUjF8|tmuB1sYC z>tj&703XD|)UK5d+308Fb$&cKOmS&$B-{@I@ca&c8ZcGn^}s)@ps$T3jTD+uBt<0w zP5%20Y3$vbH!GNrFK?8nrV<@gfenCEw)dXDcyW%fBYhV#zGDZb)U2gj=NcV&)VhR% zO$*Uj<~}~dSDxAct^XkVn||(74JmtY*zEUrEB4?Ar>u9AS{PZ2{kS}(hE4F{|GBfL zere6-_@ObmYESGc+qV4&t{ky#`3DJ!5jy<5j*t7r3I+ssJ@Y@TiER3~{SWg7Lu_MR z0c>cKtVjR(W3n||P2T?xd-MA*V*NknsqZhz_`9oJo@BxLQx}VU-t)Bn{eQsl|NGJZ z|G(j(8yO|-3gVXYIPr|OBE8Ozu&tpSJg$DmkG?B11R2gDv06B9-aNsRBtAZyM;MDH zwOm2N9h|>p)`fpSWQ;8Eq|${OQf>vb3h^tHSs~DJyTiptq1=6yZRvm;9~$z0gJPcb{Ln(G>=CTfCyz z!66p%b2kg56{+PEN4Y+8+e#{HK$nD7 z9>xv#sN69x*7$kRb26czcSxCxLHzTl;{7ZNH(nLEV%Yneniw(dD_Yw{pUj&zt6iP! zeb&)U16QmvUbsJF?)xhZm#M8e72u@klYIc`CRzQA7dW=;A<7z%;`igh@F7nn(_O7 zCG)zYp&OYweB#id+v_$B$BZ~J@p+*T2P;q?{37y^vd}d8|I+`6z9VPwtGnb^Ii#1h z8ZzGQMs5=1d&h{}@<*>%?($`eA5CLUmGJcZdgFk0b9&U=>>1N;;l?!sV&cLbXW6xW zQap(nbW1w>#F;+zKNDJF=HusY);7DJQNMYMjvbF=ybiqcY}lrZo$=3iT;8=|#EO@b zt{(fV?%*nt-AQ(>c{`WDl!s&#`*+v*SF%CajeGY7Jh5*2yx%M8djUU&fBpn#)jafP zSZKINYrbz}jtdl7e^Xvosn-RcoU-EIZxP%&q07|B{rmTuE|>r9XKQq_zF%PDF?rk1 z`uF1hc^-HF&rWuqum0n}4Q}mwp;HSGaJ&ZEH$z-$iLBaX{gO+RK7H<1`*2~IyQOC$ zxp}Z|ac`p&Ompt&+$XrZ5J(n37)Tu}!deuiR}7p#C09-V&TNST&-!!GHp$z_i>3R9 z36_GaDl#D<;pkj3O)cl&f?7M6iH0(N0CpCj<4?jjYtEeZF?q#53N7Uc=&tqRyhvovS&z-b-zDb>MN&LAhz&5*D-R8I&oGlWKERF8S-%c2t z%2ePqR|`qM#H&>HW{#+BnHO!6pHu4?lv#p}>22w{Rjf5n#oD#wTitdeFV7y&7f~Zz z6SH`L0A>EP;?@UDq*jra15G{(Aivi|TP#}Cx25g2ZQJlP*gU4Bb+Pf!gjE!#8^L(?_Wx`c-&; z>(5h0rx&Wdd)512C2GhX{%oyJsd6>z;^OG1j~@A#a;uVW(SUwW{@DP*M{NrgsyAs$ zPAIVIPuKUpP3wnFG>$H#EG^b=qmwy^g=WAAY8z->rB8I}0f?&dX65UhFX<~#^=Y1) z*#b`tE|EqzGT+54mnMgq8Rh)`m9>_1r6(`v6=;ii`c|t_Q&&WrS;lMistZwvhHH~by|DpSbSRXRp2O# zR)Gsl9HXrobvSJsSPI|~ElhZlkj@vK3n?Gs?hC&PPx=OiCNnfPsHe+DJ;BrkEQRHP z>9J?b4R7b&fBNA4W#&<&@ewO?qNc+llPRBo`hePaHY&S;eBod+X?8DY|Gk}U!~1Ob z5VGt3s(vD3s&FQal0K~i!xF859-4W?L}0E{V_ZMQ?zejU?llAy7;T$+RSyaZQdtj9 z(}}OL)i>FWcvQ8fd!fKJ@Y^18Y2STZR1H#w0kFZYSxo9$1Nn9wwH);t&zy$jkLi0U z=*3%St7wpoOmXDNBeTI*J8Y6^zgIpL;p~G+)_jx18Yu1l%FbRSM?&Z1fHADWN7wf3 z8a!mksY~+2;RfoP^JvkG{Q?Gi1Wy$+gPeliu~Itp2q_%i*r-MdwhK{tI#II(+nK0N@``TBu1ly24Kr ziSAOs>$b8HTRoS)Usg3#vBdy0$zdJ=*tlmo)auhgw0{oo7Y7T~KE^Wz!4s8%w83On z0~1Q{MNRFN&$%RLgZ6?~$gxI~R?+H9dcza0-r@Ae|CD(ypA45jc&%QSVr7|!xsAK? z7?wTbn>&-TbQq%hNY4SE8FgOEHk;fN{;Ai*iMu8Eee@##8GnXpN^F5Sl;`$geaVBD z<)|j$R)&X+MZds))u^XIn`Bc;Z(jK=J)cE`Dt~P6obcOk=OqncyXuZesTgu9x50tj z2W#iqIfq9AJc$ee$s=sgUViHl6mK8`sm;SsLe5^9Kl96fxMFpGMXXJ7cQCdl>y~&p z;n8NAaKfKx*7o%u$tX#>;lippVnoW-oD4Rht%LnlYYH`TO*Z&Kb=V98hR~V!;N=kw ze_%;>Nou;^4!CtKFssRb?8uP|puTyln*{K@vf&TNuy~1mqYpLmu|MSI1Vu@XLeDns z1JwhYuGx74Z`s}_*e&;`;Q~hhC6`>C|7m1F@FPu(@bOis;4hCrF5C)rM%hHX-1#|M-i;VJR({zI*p2h5J9&`0mrwavI2vkP7hkKJc zDO{4IqU!_dvsTDz28`X&rcE1vni23@daQZPGjN3C%hvBmtVyQ~GkV{g|9)dS2Ac?u z5B@q=`m>y!QI7`~Nw0PEPV@FttIgXz=tl zZ-~z^&#-$(+3Br(jnGlaELxR!?xEU~D0*0uy`-4EHsPtWZ2k!0b$6{5S1_FePXow|XCf4^`E-9I-=HS?GjzJ=#fznUSI*tLJfuV0B>U&LcD5bR)>>kDq>vgQ zK5w;xQj+pMJvd+flf|c%cVwl(9Bj~e0ty5=ah1)vKkaM6J&-SsQUvhHcvMR%Ywp|F z-`+X${#=bm48~uD!5QQdUZS^_*+qcX*p-==VxWApqt^vC-Cv4C25H~8l7(}98rCMU zA!W)+>(>u@46L}bYfD?UKm~*y$FbWZ+M{-8xY!k>5hi~M|6Onu2c_S;b0;k7$v!F} z4vy%qabMAjGE9SQ3ISM0bX?iACRHQ)gd3Z0t3^OBqq6@Q)-Hu3o&@DCHHKB;_s5rNtp3AtcIc z(SOQT3t92dHO~k{GMzTGTQGzp+1TwYbfCv_HD-8+p(a{%?V#zOgMUo8bG<{7EeC7* zNuCe}m3Kk+JP#|xQuwQe_WulaS&{T#e*22n`QMh zjaKm^angwv4Ml0h&WW$L2+8L^g*5S;zAIGMX1}5KXAA3kyZ8W!vy6B9mVHX*$^upz z7+tq~Xz|{YUU-VrhsW1(mT_cI`3`>(lG^3GoPCw2jN3_CTJ5)QF8l&AE|T#yw{$|-v}pyn=9bNW_-0|R#^X1={CPH%iM5Y`_O3;p)D!3Ko$FAA3cfSrcp`u>lwpT~XOT;=47^Ih$V)*;Yd1 zYR~iet>dvuN5*--k!D%|G~? z`bi|2B!h33=Hv?)j+q#hwzm)efC?KiK9;1Cr!HyY(o0gyNf$y`@YJa$sw)nSd<`Y5 z9(yL+l!Pj9vDjB%^-X7RcZJfWmu;B*xe`rIpt^5Tv&Y@bpE`@YP|bL>8|{%@gh79n zs~daB)q)?G#$hG)vSTaDg@YQkcFIIL>>eZ+((n;^)6=fS|IkeC^0)5m2={FxxWKg{ z$`nJ0^W@Sxb>mp=^WL_2h>lrDy+{y|)7d9CWdWW2so)oEzDHa}yAcx!r^wQCyT%jMZH^ru9NWt#IJPsX1L0T0C(c* zRcBC;z;LlHs0A92`zsPqohyPZAs4#0hy`XIxqo7@JK=HtdBu3reWhWE!zJYB;b)PT zkv;Zb49rYD3F2WAoT4&Em=@te3D` z=>o?jqa#`q``U9=-w*j^GPMXbV#?s7p|A_3z6WtO*T0U1^Eu6xsqWPtS!9ZgHW6W- zJ#A}~DSIZ>j-A;_!VNII5JV&nn_v4refq5FBMWJY75UJv4{T~*lY^vsg$f5Q^Y*GYB>y4{ts;|8f{$&Pbh<3-~!LaSn1K;U+xPdov;(?>-_L zfbjyrbXhY?&<_0Tw%~cbhyk!(t1&sUYN)+ggA#$p7mVJ`7vY5ZsrXh~=IBVj4xCNj zyV|*cPa>4O!r}Qr1)Ho(HU5NTKn`;jEMYwvk=>Fkr@_ZxM+U{e(GtGs@z@U;aW8*Cfy+`XuuHo912q$-S%<6|JOlm<641}IQ0tKQBL_1dsyz= zxUmWks_rbFbpBnX$*#&l^(9Jyl`+6)c6P@3Gg$)xqGV_Wlec(u`M~0osqMEGbwy{X z%9lEU;LtSt0^s=|mg>wgT_3>8Q7a&ERsPTjCi4v9mgQh#?Z{eJ`a@yRiOV;=eKU@! zwxa>xm(69&pQiUQVr!M@{=^>5LVFjh-MgnWWN8ub+EE5BNrLO%r1RyqsDIc{@f2Dc z#w@j|Th+(M=WOoK-0IP1>=+#}Fyaf>>eO00C%%|5b#JX|O)h7Bb$Y^=3sxPkSi8qA zHH_`A(haviMJ~{e?NM)6Kmtos*cnopusa{UTMI1;+s_!v%VkNlmU@s&bf!3FV#|5} znC@#P-dM_2;e~WOwk92Hus-*~Cf8J6FV=!6gpiSoE3r$M}Cp~Lq zph?GEjjw^IR?H}H=7*h&un_9Ma86cWO?9jNkWJAc?d8RRD<3AFY1#$kf#AN5Buftk02Z#ovMA4QpXF#|%IhMs_FZM$N& znRno7__2+tXR}l6d|dN;f3>nD&z7VpxYI|$_3u;2%OFSaCQ080RlM-j619GrECJ9O>OdY_kuP93z*!Y6l+$kCeCt!nzC>z z=lda-D9v!5GpDb06$3`k$e*lz-O*i+JG$iAPTQmwBvQowNY7OrLwW77Mf^xZWe(oq zEqB)OvO{x=4i61c)Zq~jk7p2);~0Wsa|K+IDve}T)#W#$mP@-k#}~S(9@!EP4|V1x zS*OH~!^3|)Ns+U=-~N$q?qdT+Y^bFB!k@@5NU6F{kBf_wEd{^gaB&|j?p8xo96g>$ z48&;moJWk)jU5EKyn|w9NJ!F{_XvfJeW7o|LOa&LGf!s2c6N3mpvzHvWgWxOee}tQGewq>KNY3J5GbV#I(}IlF zaAR8El^eaoR2atWCLK_kjiWutk%F=ARNoi*{bn{fGXI?EUwA}nO7^oaezfa4?oR7~ z#UI*8TS~OB^(szE2H86R)NcI!6=O~21`QgB?ufv)O53(aTOG~9orkzDaant&+>Le{ z(=60$d%rT?OIO4-&wf}X?8{|y{Xe{?cfr*Ksw;COk)puuFa-CUKRDY*{rzgV+*ne(>;(I(f`C;^1mmas~S|^<7>DUJ(ZXT1*yBF^!{nM$m?94M? zyeTK4pE;HN{(^9y!BFp1LiSyHtgl|Z+K`<;Ju`Fev#h@sM11s5&(3AQk&d=r*_qD@ ztJoIg7BqS5)6Q_(Es@$2G#VG1_Wb?V2}y-0PxOo`XHOV6&O3XWzDeGnS3!@%`t&Eq zw@@jQVEOq^cy*tOzH%uIn+5EQ;Tf#ohV(5tJzb}p`xC4D&iH-%xG&o!YymKgqAMqp zR~8FXD_*EA1n(O4?{t${N^qX9gPBu9j5U-u&8jwN;AIg+PAb!ZZ86E;pq}P^DZlAQ zzmLohuqnaStYTHaEnT{_UI=jh{fqL=CSMtR|7QTaN9dTCns-=u9$bY%9N-X{gSJ>% z(AM{IEi}z8bqZVMA@LANMOez9t|U19`#s;+p$0P@s&J&7&A_2j{tQYNC5p$Tk&ml+ z(IA24eW%EeTi8CF#!Zr@*!Ks}D$C&7KHDA_)v25J8vScgt11wV&0g zMX8zd=5^crZ8@E_Nyad~yf?7#@MSyKivK=^eSS_l@!hz8AH#Vsb*aP$g;*i57Pgxo zMp<=E#!p@D{rK-57BJGiR{z1G`agAT-}3;hpH;jsO;YQBMjU3jYubqzcyctPc$CZr zDrm7&&*f@CratebQVp2Qf=(m~ozM~Y{_c0^la5KRF zYzsGFanwGdpnjLKZt<-hoak}E!8SB|rMKfOZ*G}q)f?C(=Ruk_z9q1HbE> z3s~Y{7BI8$WfK||rm8e1LAi|I^Dgq9lhSwljvcbDYUk?OiI{;%WZJGCo<^lA3osH9xj6p=#+;PwmE|rutY#iWJL8%qu_4_zRh%-vBUVG^l_iqNOa&_6 z`m=u(Gj80m`gu*9;KJ+ZXlQ&bx!D@%Ko4+jUB4IXT21Zb2mo#m9^76@ak!2pFIZtM zflba)kS_I4GP*FK=iWz)l5?pj-@U$H{`H$T8T6lp2nv6uP3O>PFNCP^cNccEdlJ}* zwy0jhLsmYC7}Or6!8g4F<$)KLm^iY>ax1TXY4P_rT^=)<51(A>GGaSk_IUGK3OCL+ z*SJ*UF7D6O$!h$k^&j>l-!D_^-)sD<$Xx?8o;{Po z4DDDAV0}8)7@W35Hk$;OCN9r!B_Wgo(jd7)zWc0rjVg#^)e@qQY$!25R zS2p!pW+0(9f4k4PTD30)iR6i(56sZ3%S=)=Ww6Uw1P+HnF!`!+g4+1D&}3i~otpql zGAJCRFFT0%1U`+cMM7wrgFZv7XI^GASc92J*WD}N@`P zs3XBOLJ!Q&wM<)@CUdy)4h%MT2yUn;Iwt9DYbt2ZWQH?8%` zjn;E&>X@NzBTJ0Rr;V!QvitIe6;NW4D*m-2ph2k+1%Vk zT$}1}*0UFR+i&qJJj;!}Jj!cvu`!Ol=k6F1Hut!tvq#p70(1w$%k{UE8+LU=;q#M@ z|MRc*<9GkL<6f5;0Zsd--gc}#{OX3TUfcgfhf$GFCC{oodX`&dN%E+iqDk!lEe5Q4 z2S=tVu$f`;Pn&<;!-=uqWi|@y>3wr#6~}c)Ga`7q?f1G=a_;@y?);>7yz75`Dxa}( zY{<>zedY~axSn@H6!C)Np9Vz+(F;$jCbSCm(cQpkZ*~^W7cU*d&X6n z95TvxB>#IyYET*k3b@pKtkg)0;Ub$Qhjq(MK>-<}MDywEE5x7+~!zpp0i@t%+D4?=Q|5 zATJ5^%{`P&ad_;=y@w~tMLSz_!8(g81TCwD6iSo_n%bZVRXPXJ3Sk2ZXcXJ%GiwB} zaf|A?V~ar@$C1G;t^Ai?_GwTn&S-ww7@MfMg~lUHOQ&%o*6bz2F zt@~*q-BhquNs+t(1nM8l-E87b!u@9T7TV2C$@Td*N-OWBWulH8Zh=a!_TbMDRF z8i0W2r+<#e0_pxkXs2OJQ-duN+sg_Zx%Sss0n!TTKCngIHC z{4XN&`y|);G@599l_^Qv^G}cKcJ}>q~{svyJQZSszEy4J1^icJ$d*jPKz4 zk-Qiv0|q>)8(e=Z20pNo+-bvuyCsBE3T%Y0r_s)1s=DTDVVkfKwmvMIi?r4Keu5u` zx>19YL>*@N9=qg_ued-cQ5%HUaH7QZ`aHCfXeNjg4MiLFE^;l_2(w)^>^m?TMWz&* zA_A-71%m{bNZcj8Rw0*4IuD38oXYkcC~D=Yqmgo|1+@C8vv^rTl&n}}$c$d5L!Ery zM}W}8dX+?-?;&RmIqUS$hj9=_Q|iMFPT|0lb@jVwSB;>kdTFNgGB3; zXH2V^cImZj6Sn{Q>!-5g;P`A1Fm~9mKPWgfDWkgQC*9L=&pEkwPFWiN1mPivXW5+K zZly8^)9U&IaRtQF*_;Z&hs&xQu&7|e$iBmTvM=|9r=$0l=*R6OM-G zt#S$`-vGG%vzy@kEY;0}TFr?#aAe(R&HUD}Z}4=y4kevP-521G^Wq<<&;9Ohv^n9u zcj69Nia`$sd^-W=M%|~0j^L!ir2-az+MOorac){9oj^j%=`^2t2NFxNKp&y~$3eFD?hB+meid;WfvLTa@?SFm zfhX)-ZTfu00XlTgi$yYdWIe0Td*!?7?Vr*aATlhXoh%q$5`Fo_0r#x|%!?v3O0V@* z!V)qo3AVLf+qW;=q>_D?%Si!?MFQX!36w@^YxHmdbol-*0E}D4%ybD~!gNkinIK<2 z0+YgZ$Z^WF!ZU(=5@ylMk%tVAaq)%?l@Ao~wS^&-1{C8-(^cCDVx8ewqzK4@6{gEu z{q&@6zS#z7FnDlR(j(&1iasCJP2$O9uqZ`xqd~leN8iqJEhu*q$c*@_Dh1>pTKAstyU*ElepDJ6BSRueFYj` zU)Jp1&&BII+#1cP0^>C*;?X`d@ObOYPw!vh<5yd!dKSiVoZR6NS{hlSLWrBM>jVIJ zz*b$jV#Nw9*)~B#2+yG{rKuc-uWzj9w|zQ`Qpy)pyG|Urc||H=H8-oWTZM{TltQ)1 zU|5mOM%N~@Rd1i;ZBP%V(#)HIVjJgKt50X1_k0sA&Hb@&IivlVjl*_C1iKm-T0RY( zL7W_q7YIE7J;$G6OqNK{V1Hb13KR`}2${oGtP|4nB-%bu_l~CBzjtpvX|J%u4el$q z{kSmrZs53aX1RjK!~1L6V2Mkef+eql{>3o|QUfmOmVBR4==0APK|#!8VJnodGCbIQ z3pajTBV%|#VND_>y_79^jLVddWK#m`q2Kh+p4Fdb7kUrfQ#_Pd8J%h37Cc&w$en6? zEKM*E5t(L!tC(u9)W>@rMi6*xdI1-K@FyA_o>C3AX!gI$u`AW)73ZSl{ zSq28tR|nM>S4QMK{ITngdEEUiMTk5U#D99wXur#V%*;e5_`+#pvL1A%U*RckKFcC9 zD1Q^VInZc{^Qas==Lzvlqc;IHj`9;h zogSh2lcEX)EP1&~q}Tc;Z~#w~N(Y!o&8;L{A!D%r!Odel4lm+g%Jv5CkO*6+CKt?K z_vg0fXO-Wx7q9PFX(`|ptluVz8?YgXkym*7uNx|c1=n8;8CsUh*C)MP!7ee)WJT@) zql}h8+2h1Bmd8P_u77lSNQHVh$j)HWPr#)%8T$@Q1AD3p84gv>;H2q=@lH0^-jii9 zMF-`+6K_P{UW^EqEK@qq%CR1tO246(DX6n-woG@fTUJgesYFwgnNm^$896=5?nFZn z2ozbkuwePtyV={U1#FL>PFeE#x&S5`o?Z-j=~9|NGe+YEIeih6A2PksR_wI4O%c>8 zZ7r6As_Omf|M%@CB6)*F$S%CNd+rHqj)WRS}o{CI$Suq z6UID53d-05jncUXuD9>|H8g`x zViaDo6m5TD_qar6+nwFg)!q|}t1jeV7(eaMoo*c#7dK?hURt%?w>GekbGp*|+08Xm zvWC{_%j7}B)q@b;Qw;ixW$(GPxeafHEfPTXswgL=6M-c8$dNW2&FL&XRPla$vM&Yb z=%n;zWaFSq<^soRTti7`*60g1Nv$VWc64F)dU2s*ZvjDi%UVvHT{elE1s4;4-F1!B z>_SJ&xQJJjgJ-w2w?8MRX1>(K!MV57OHlNWW<$(Q|0MEj2tCw^%f@V+P@+BdBJ1#z z83k=!U2x74Y$Fp!B<)X>K{*076P+lDF4P#-pkw&3s_F^F=P7?hGnHYT9P=E^0O1lg zE{rMYasH6w3^`1b!E1fUG;h>{PV#Ugsquv8kVSPa6H~&j(~$}*Due}(x@$f#4Tx7r zG0Br{Zp+p)vNAhPk8~=0`NGAEOOt|Vd1z!QO{_u5sloL-Q}8O-)d(Aw`)*R=zA~g! z-J1QK`SaHs^Lt1qWN`ivT*`fv^J*bdAi21ZFV`#WLwVli)B88K&2bsf`L~~Xo$VKM zZPLqL%v)sWOZ%mk2Tz^87+V6RDI4-r`;iko>a6B~R@%Xo9{;=5rVLz=>Ntal20M5o zY<5{*uRD>y#wPd)Z2n8T#>Pk2G?{)MeCj5TTQo_>VDEUY-fSe zyOHm=EMyXJvB^yRr-p0l&aV5{h#hnH)Us}}2WJJ#2v2`i-f*!Yxe5uM+m9g?_UQgq zj_z|1%A_(}+Ho1IgT=7UlTH-yICtp2y<{Ah_P)*bQOn|!pKoq%Qp2X?OoBZ(szady z8XEPcUzB98A&r zonDnPdxAzmm3#U7caTHo11gaM^&kIkG_4A|kjreZ$Xw^Y^4WN}c^sI=)6%Tq5u-+> z1;p<`;5nu1%j=Faewi|jT=)cmg;V^u0pnaYVlE_~A-^8|8Zt+OL}G&k1vDfE%o~Ab zxLIt^nwb-oYEw+=u>9^x;DWSLhe&T6%s?tk+~{z&)Rg;QBkBEpGI0a9o8&%qq*K?h zBuwTb9<2XG&Zc;^+XWPUNaG{YEw@OeEC|*@(V9-&uHmx*iwj2X%&k?zsg&yyn6pZF zlp}}h?1Jkk^1QcwU(lZ%IIr@oqT#f;;hgpkm~&d}hAb?N-j|ApnFyGLwh^O}6%k?P zWGBd0!mf)!i%rM$tH#mXrfd2Vs+CbI^B{wronFK~Acw$%6_xItQv9uc zU5szn_I!zQMNCeA54~}7#a5R#0|EkKIIJ~0YUW3_n$v+DQIzaiu3lYHdtGR>(@!Bc z2~(Vbp(e56{d#|s>7+gwH@Vs9qYu;;K~Q0dq#^ad+D@+jqI5_b4HZ`F*vZLB1m)T3 zAz@m8%DIRUb@x;X4L1v_l zVI7(@tu2MnECY3PDNzuDsGAl`hesCTy`!Cdf6b>Sd*PZqNiMGz{n-|YcM5c?u-ZQ~ zHG62@HzO=%Jc5>GFTY(G1t#U&12h&LW4hRc9_||Bb!t@G3R5_lzBpf^@F)5CFMl0c zAfpSbgx+}YpbDHG2!fr!V!n%OWV7@Dk&L|5pTt3rSlljU=sZ+tt~*61tlrJmw31q# zR7plr%;LEbeP9|!N5>%g;PoilC?N=~ zC5n+9OUhluMrr9))!yWo;prtlHQ6T`(l91wUz+r1U;BK=PAbo%l##jH%$(j6o;dO$ z9Z?VZ{ZJxv&b>1$<5@&iu_M>u!>p$rQ{4-lFM}sO{BMHZFDEZOM=U^H7{iV&9#irR zr1#Wy^n+oJFLwE8%Nz=XD6$euE56A!;>0=DJsrVC`k9hQc$wtBT`+azE~@1!qBo4Y ztz*Wy%-a3>-ZHV)WiXElPffI0XjHxGG`ApGk$#ruXJ-O~v^fY#mbN*U|JW?$rx?e4 z>FToR8HZT_;EeU-G5p!e@1R@^*T*aLkQe9!sKrUlGzK&PZ&@4w6ZrTo0P#PF(n=6Z zLP#i0XStS^<$b1R4Rsj;>;o%~B5VVcD)%~-SA5I1k>-nM*$Lxq?zJ1dyVb%A$@%bs0yeI0df0^Y z7aRmdxvWf?;KG?^NA^!R5*%BNCCo4;FGd`_DqFau2G)uRtO;o>rce_;`4xC0-@+vJ zKcS5}cI3zrfhhxsP>dtlOM?E>6AzG?av7?LSxc&@mG#*?EKBC>FX$a8Wtj`D0oUI+ z88kFHK*pnJQId+bKejxt0~vAo>{MGNIvf+R-L1&?u}>q4w{JM|Ug3FDPxkrsR|A8O zkSrsM^rt#oLNp3?_y*R4ITf@^UEoPfnDfdrL5>_tvZkq)4hA2GHgAVXa$X=$%7W0< zBgV-oKRBL&ZXGLA?vtXRkUxG_X%sb&(ka#!yPi|Y0Nq?ZsWa?`i#ZkiuzsKYHY!}U zbI>~wD@f~4gx{?&J-qSg!$}hj@+$JwNH7n0Idd|I+wvAjZ+$7w%z9raK_nIDd z%PJ`j?&`@1LaN!{vshxr7YZ!Sm+z|aFJHMHJC=8wJkD{M#HuRh0#;Vk{>0K$KJbcd zO}@jN_8Q+b)F0SC_WSD-1%|(WUA;H(*)_BGBip6HE0U}ZVZP-^wiZ&aAdYKiZ^ag! z5-CBdvADwP4g4JRU~ZmgLNrZy9}d!_Je|7z&xLSoKrWedV*h`4oezGXC>WO#d~#+U zqs92^|JS>!HGX1?P)&qGdOhdQ5T#k{J`%zoldJ|C`=3Vx;qd?9s|`#lMN6qFX_)MC z;^58HwNsDX_eS))j+js6nN;wZktl}%_MEiQ`hXKW99j_i=g_1#1KN{RXN>qas&#_*ZNTEC12or_pGm?b$^lq`lz4L3qS}RQ^fFe) z@+C`&Oi*__fIVsLAo7Lr@E1qnHF-Z`By2%lCFXzCx-Q8P61%wfcrzR1519V~0)a6_ zX;9{xll>%yNE%-DEC3)VC_SFSAQXDQx&4;{>((30XR8L`U;WZ4L`sq9b=%+$M*ObJC4U6Y&cy{juX zw!SrdM7Rqs**?6V28+lm;=C|!O4(GmW1?@MH7?V0GMQ%S`Ci? z0{Xk(+q#y)TMtvm^$7YZV&0io7PiZNEqyGDLj!ou_?G`3Ml4ANl9a<~*#y)|uM3}Q z8$|dR&|n@eH6$jn@`Sg-aLQZAa@W&)@;z@X9GCG*_5oD=p}A=mg8HS68a6yemztZH z0-f~ak={%*L)M63Ak0KxI{ER-CupwEUI6^l&E^ujfekGLDbcJ{VVdM2<$SJ#CkQpb z>W(?!t%<|uAjmG5G}$m0h5@$pW4j2kiP!;+C%#IE9>Sh?@73=WTA8AIayfv7%FF5p z^aSmsq=S6Ug2&^#h1u1{APBPG~p;6;^_!|7s3SR+tfDCzm$6Hfj3!U(o~Kw)jA%p-blfSZ0C z8ZtD$0sVxmPC~gkV2HqEZkPHFAjYHNR`s2A?IlHpNO@~Ymzm3)U1kd^SpcMRA9aO; zv}jT;C%GEm+*UL}n>XwoD~c%=)VxNjEwb}ov;5toDM*CrvJk4^>@)Zz-Jbg6vj!xqOmHvXmG$DZ*(S1pA7#f@LluC$5u6nlx;vE{ANcB@d&~A{Srm z%NHajCQ9T4#&fOyJGrTX8k-bH*AH|n71HMHqUj>Klc~xE#BkK5w?`U9uEeyb|jI2I*0s|W~C56tpv%noEiJzqHq1#1sQsVAn`1L~LJXU`j1G&K)gqRwV zwgLMl7^FvH+A8R6QA{rvy@Jzf1G`2Pe*v?AnwHo}bg85RMP{voMTSzf_r6*qZ7i98 zrHsF?Ub%v5br>-@Y3TLdwRdMu6={xq+K*z0s>Yn%>;xs8!;T{f$RDvdHYTER7_`5= zix)y&034AT(1Womm}_b3z#g)5`9E;sOCG;0Xk*{2-bZYwz9?^*{P>o>J*e)5loI92 zMLcdq6L4sl?{cbb?>CHIn`4>7S>QIRJ$P;kvfe7m+i6hw?m5HR!_P>K>Kr+GblD56 zIZJ@qhp>Z=5WG#hvN@kiGbOO!s1YOTzPaBA7q-D*?9X7P4!DhH$6!7LPIuA!d#Lc9 zN%!YPofwG2PL~Pnvy^k^&ON}XWH+ie%TXF5*Z5?J3%)C@%y7Ra@@cgd9L|4p`kUS0fNji2kSseq zx=ixl*|HfxDS6Xn2p(+DAroPE=voHkmZ9F50}r@fo7SzDh73(vGrhCr8AsyeTe}$= zRd46cou$Z({n95>hORZO{q480mZRLKlJYq^IltmB-|;T3?|)?DGX8hjGx*jaG&;In zxejhd@Cn84N!h99)`ql6?59MMz#)b%KtuhjAAiRpHm6d6Ti(7y6e@iWH4B557p%D? z1DevMpYNOOOD}=?tt+?&>;Q{%H`YZG1M2A32T-7gW3VLsuta_Y$3=A3FqBxjCI{eE ztxBaG9OHE+MEFur35y`&%j%QR1^!Ndf8qaC-1){tUB_{pX08WOxD2MWl(`e)ff;&b zqB#G+=Anql5K~e&(iGa_Ld8_FG&3wYc91C&xPyUS9*Rb#hR$4Enl1kA2$L9piViSr zaOlDtz21kb2R+G0`Gh9|$M4Vg^ZC60eUPNne`(`~sR4{Aaqyz=Wm0acgipDI@_E)cNE=ANn?dtEJ z?)y!9`NW_%is)B&!b}dFxA>2pWBl@d7EvCuH*?!If#14L^1X2r>&eZDQ>I=L_ggWz zq(OvVbv#mPvst}w1?(NeZCGY@Sav>+e!tWZ)tDLpj;wSHsKaU3!f(AP}$i zyw2FHC3^mL>#4(Vf}2DUbpo^mMSRIWF4)THR=aGa~B>D zDpVY;O+E{^dUt@Q=ZCiAXWZfHr1W9`iy6jrqxu3i z>;`n;sD93C8-{@>yPK^>#l*?9q9?HA0JpaS(R{3L_w~tCb#!>5WO?G=!6D7JMFf;= zb-Ue1x@Jz9>}45s){5ROz6WS=Cbga#n+%2xZaLLNJ9|m%eYGHyhoBL}A}}y;!|GID zwP*{gITi60srp(2Q`U7VB86QOS~ZPRLXHpYBQMk+-kBk-*Ru4Oz25xz+-*==jt!;3Y-Z zGFcr`Ao8j6MCy2KOGs8k=kljoYUY&E>0D!zj1tHC`%CPe(H6M);jv@oMrjz~qF2r) z&nAXjnFM2EU&V=uSy z7)tR+YZkfv_AxS{Ku!KMbYsRkW>Bb`!i8E~?_fPpK;%^K@V#1Uj<(OsiK&O2#>UP; z78KRZkm2Ua6U%jU3jz`P_#O(M+Iz9WG{>=4if?gbGQAgB#-Vx*IIDK`y~um2JPoTN zm6V=NTuK9~KCOb{3$7e<`My!%{ft7H?!Z+b=sz+rTSOyC_;~PA^&Y-zQxRa3@!*}I zAs0dsYtcZvSwl`sX^257C3X_0h)n#2?i3=#4cy6pZ{i^zq-{t8bq|VMvl%p5`=m7R z0opEsDU@D8xjUFG5?KnTrJV@*J?xU7Yp&!Z5~8fwXpU=I_6ldN1D-@kIjyr|+tdO` z96@KwIbyS9TUplmSXeiZO%E3`P46Zhy13YLSHU0ao8suzsOaJLD8#ztrI@#XDr#t-Ch z%*aiK?ZjP`s>X^Qah|!D^LQg{UR%@S!FvLdFA5TYjBJ}y> zX5(lFWf#oHun^kP4Rfd-xAxbzolmuG?|dxj`0@p$07=f@vI;Q@+uBlq@8XQ{96kyH z60arNyP5zvLRMqMREZB(-^4+kliS~V;m$x&$9U)zxkPU8=~A7b#`;EZq#Ye8A{r6l zJmYTj94=eROadX9XAmi^nTE8)Dre<7))r2gUBN!-^G;vc%>Wcu`aXxT9242FEE~Oa z&duc23A=?j(2iZrEnJx<29h5^|^=ZnOztx_p z-R-+|D+z6ghCeK%*&PdgrXWA}5sHnX2Te{gU`^1Ro2v%EThGe? z1vbrPS}hip`sNcGPFu5*Nu?10ByLXFJX37*1TD0|q_J7Q1ayWjNp|)71Gi$Qmd*mG z%&VXHq2+hysGivO^!Vk)YMT4!-vipj9xNI3KWO6q{v`j375)Fd`QGF&Rvrr6XAbpk O-8023i>_R_e$QXWB+_>P literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn/gb200/gdn_fixed_seq_flops.png b/benchmark/linear_attention/results/gdn/gb200/gdn_fixed_seq_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..ffe2f4e9942dd76446e714f0d9643b2747f876db GIT binary patch literal 106703 zcmdqJc{G=8+ctb6rIOGrl%$Y|QiejvoViG(hzyyB%v6*HG?0`bb1FoJ$W)3#3Pq7J z51FSzhIc#E^L+39uIIn+`{P}obzj%LR+oOib2yHD-=_VzqN%R9Vj0^qilSC1E6Hn9 z6ax=M(YZ4$#-E5ztXIbQytp>_%Ifh&SkDDoB#9U;tXH@e|vG$?i>{P+Ri5p>BgqB(-V~6hTY~R zA3yF2;f@n(F7p0k-(Ah|DMe?AQNHuLKKsm$?}eUUo}F1~)sPmHX7r-VMQo~Uw6k(# zZq`#RN@7h{SC=~n6N^^b)oa&E+umHGq1wxWE(Qf1=2VnF_VTx|Z_{%J?h6+$3Y%B0 zU%k^{PkQN;)d7!j21(a`nQW`iA0O+b7cgDM!{L<^;qrgmx0yUA=bLx$2I9Z8jkK2s zUcc@i9nHCHo#6F+m%ccw&yQ}rJIz*k=+L24?{4w%SDqHxaQ*sq%?C0xs|5|$i&}k> znHufn3Qv~4@hRD5qW|lv&yV#Q?)m=tHQ+WgDa?Q5(IP#)2DP)PKl44td(N|oTJ`7- zmIZBmsF%)3ZI+O@w~uYN{jdIGkq122QY~$57tG(^OT6#fT^%+2`?n&VS52(k<+I(D zs`+JKihX_0Hs!1+y|khfpCE1OFJAuc-O_MDqYEJ+Dkt{rdHbZA{4UIkQt&)yt}g-6MVJ#pYH&5E2(-0XudWjc zu1`H)T3^4nG_$6r=DF(t8(uLBD=V#>oSc}L*aOSDq^FsgwU)ETvQNIG{m^@R4}7TOOFtG<+t8r+@&dh9 zhRLBlHjS&dZQIscc0;G3`SQw5ugc4pV4LvjK4iYP$MOQMuESu;-+vMAu4SMp*KNFq zMdgk_%2+;GucQ8C+g5bb($c;TWa0MD$jG1`j2wxRUn+d2ntf!jSu%U{;K73im5JWR zjvc$Q$!GH5ey>SQYk`>h&js#d&Za|OGUBPt`K|-^dI!{Bq~UG;G#s3s=+n;Y>FH5R z|9!l@r-yeS9d~GUx7fNNjn;3yr0dz>j)i5IZY)hB!??(@D_psIcYRPNwhRxR_?GdB3DGj!vu6disYmjfn3_H- z@D+(mPTo!h1}tA^e)nA4>npT=yarhoX>V?Cd{tVS=zZ_|@51ze;^Jb9x}=~o(+aFy zfn%BS-TA@n;&IwOHF1aX*1Jy^z5XpM*$^K2T4(b$WVzV(p5>(iS8Z2Mxc~e8 z{e32lS(b4!=V=Y3pQWazzUg>(`wdFbp8D7V^Tgl$nh*T+3*0iwM#Wv4y=SMhEq)Xg zmPs_`*gO`Ui{zdEAU)4O#o7HR>P@?dg~2P`GQI-MAT)j8NAaeMi&w}xJMW=NzJKRb zS6Amf`eZ$-7cCu~Ls!*?0*{Gh%hw8&pvbj!b|y}w+!}0tp~}sakd)L`dU>VW)F=;G zKODC(NBrltm2X{W+-g*stGta4;mP)pLTM{LC&c3~D z#oJCtY(c{!uMI0VNy&FsgpEmLc_k`_aN673%S1`Ij7o&6>F}<;EAF(7Qku6iH8F`B z4b7L0mC@H;j#>6q7i* zcEz)o$g@MSw(1PwJa83d`Q_WU47juliHSVHo21jmrhVsU&-F@}+S!F*6~27+ingez zNX+HuY81Q6*g`FDZ*p5{%%3RqSR*ARMNu0!Zp^a!ybMK|Vfpflxw-pJ6u6!5tPF3* z?~8d)?_p$Q%*?;vv(cfqmM6>Z2PbOkE@9z=_xIXpViQq*#l_P54;&ybUoCpScxirI zT-+{?af|gEHgNIrExm9t&D_8>O-S6q`|aR_*19E3>?L74^fYsAjk0W-j#^7?+O)~* zck?-h6)WheFO7|H%D1=i9?RUyxMBqhJNxx#XFjx{_37t3^X;|$7ASX_>E+k2$4tF< z8f0m1Xyy?VWP17XrI^P!FTQoSPD7e&@7}d|0Xlr)tNu*P{`qb3HIs)^%8S9<+uQl) zr=@Fl9DQAvtl`k}kt=e)J8P;z_A9ijla`hd?VTu>!R}*Sr^AH`n7z2W45X(r=6YM2-YL|aP)yyA`xoqH*U0MKisOyylNG95UUoRIm`O>4~31j zwQmiNjl=EdPL@pdf1VJyGkV5#_#fL*JJQ2?<60+$iptKy6+t zpud@8J3AkjQ>K&myTJ*r?P($Tc*FtqL&_@W2EW}@uzs46!P1m#*Dxu`FDiP3UCK+J zJTr>fy8ek)H0~ciHeq=wVn=HM>jatkOm(iCnQRYv?$phMo)$be`{#^h zv-rS7UxR+J&jIRW+4T+l@jUx|X4d0&E*BL^0jSYa&)vtikur%F1bDI*%TzW+J5j_? z!^(;u==IX^T-#PGZh#Znc!eO%$4670D|xnVZ2_Vs6_%h5{O0>9D|7RtjH+aZnW zE+G-8ldK-`?bY~FF8e>fZ18YakSB}M{1?Aa%`fEBKz^WoN3QGVEAFTa03 z!K2}-aG>$&DZ16m=&P!#p1KYgpt`0cU@vUI%P2>o9x31Cd$Rva2KKPF2bQ0(P2-;I z=QJhg4XSqpUZPr!OpciLH@{d1VpHk(Bu2+PLE|FEB}OSt^>$ErGo zmmYET&^ufmP{W&tTJg*qH(o0#k9TFar8+^j!n~WcZ{!9)j};0>M#poAUxLQj-V0$fe6RZ z4&|uIH|6Dz53IQJm1PMoNuYDnKHJFpeDqzxOH3U6R3Ns%HI(D*{85txf51xB&bava zBehJMqy#7iZf@?8{;#`ho;#Rk=c!IiOuPXbsWU2nd2z|1ubF0|)!qPYIwLbvPT3+O zBO`IZHU{09*_oNF^D|>1WfO@(e^5Z;(AD|PB%OO6jLm5^<~b#gpLRrj#e)j#%5E<9 zEjF0Lg0+rHKRT7~Gn-#MReSZy6&C54;ftjBRa%>vZ1I_$&B^~3)V8hSRt&DHpJ|J^9TO9?`VaQco-XA-Q$DzA#=fem+g0ek ztR>~iwb>)X!#94ua2qkX)Hkm?KQ~KHfe8etg?~Pz9uj&Z%v2{Zq(OAYjvYarYKIS# z*OMKr$>Y!L)Ls4AjcK1vqweJNw2H6qw11&&Ecw&0XtlX<%_;nnxQz??RN>Lznp?XZ+!keqIxr;@0wuk3#pE>i4mQ#?horIc`VMXF69^R z`|4Bc@&-SB(hVx!>6~YdVxw+w#?9?{b)T+^T`|9}D#g5#bN@&EIjmvrXgvOIi>WNY zLwafi<=V`ok5D)%q2LACW~vqog@!^imIF?67CeK}GO~2M%HqpYF2WrN7T)hJ(p-RR zDE~FkG}88FIV#X%(1)t}eCco$29yEzbmL+Lpc&101p@zDKq45HEqjIiOSoo`h6FB> z$+YbHd1l(2oSfRoEP}=CTPQ)p=hXJ?+wrIfGSbciYUppuU3v8AQPeyIQ2LW*W&t;D zFh`0zElxjC`07iBDOY%RRmfU-# z9`cS!bG{2<57@{_w~AH5`0q-d6LcN?b_IJT7SPs8qi~JT3HGF6`|686; zpUzA~kg*<6k)Ld=Uq*(o)u)FyQ!-;91gt^xqD~K4mPaNm?5WaA67Pk+Jv}8ymEeI7gPE)r(BGxkUTs<7UQ2TlmBZf2sd9YP`>*2bHc9(j zM{OkNVAtNg3h3TiDCvt9FK#P2Pq$ve*&;{!{vL714hD~3{Z|qawt~!%R>phe(Q2?y z2f$CP`6Zw?6ll4 zv#)j)WQd@v{o3wmnMiD|%4L`r+VM5L=qC4wTg;&QO0%fP2U=DS#j zjbqiA;N1~wh$WSpk@5S@wRMw&FNRtM2h&D<2xZA{eSYQxr{yv_qufL5&?DsC+{Bix z6$nH}d-OUNT_DT4feD?b1acSYhC%xdifk-S13g$tJv{RGPNQ(LvY}yQg^wohYE;(} z^3?kJmggA;8~e_RHgDV=Q$Ladd9lZtTWK8^7nioFJFxmL%i66^4D%q@$?_NjZjB5T z&s%Y9*)aJ&P_yi{~GR7m=Q!a@SOu_)s~=1i(jbgogUx*?HUqw2wdLi0|ygy2f0) z5PZ}r;QHrYQ<-HR-~{$iF~X|%Dl04Zd@qy?=8(Ju?rSnV-rLKRo&FAw7lb=9(dJoB zP8sN_$Pop>a8rCDd8GEhK13Zs+fjr9#e&iV%*C~5&uZ-IH_- zp?MROi3;6WY}GqymU+#>6@ZL@(!E_;J{omfUzB&@T0bTo-UwCqk-$1xq6DqqXUl#^ z;MfHK)p+?UtlHD~rjg?L*_G5TuSt7UXa%$B2gUPq8v&*3*m51$PvAFAq5fI*o;r1^ zHcK&LB=18202duaBombLEXz6tn?^2(!GwUJJ*3q;wPL&JXPD3fxTa;$DcQao{PXbI zhuP?koa<>T^Sr&))Yb2#Eyd;3-5Y`npv`$$XWQw|$qi2?C*2k;S<>E96F)iJN{3Im zyz_X@>HP24fAv9xrt#{RkKqK+!46iLa(d)tP@%rGqM8dmQ+`UDGvpI-QWCZa2&l0o zIess2HyE>N$~jyWC8;-%ZCST1%r_@FlgWLx02XjWtqIWx-E?aAnQ8_~RzpJr#j=&+rU|P@Zw^Ls8#n)04!CGPltl}6E3l*9tc}A`$P8lX>&K1PKozb zRq{$ofo1#A)o%UlYsyWpcG$|pvlKXoMrQSnqiQI2y@q>thswpC=}NSmzSPftKVOsc zc#KcTh6PmvZ6w}7Wha-vMN>{{eY=5pVQ5b4z_XuDG0Q93tI!Tt^J&Xd%ve{eL6Qhf z(~0o0MuB7N12bnvv0qEQ2J+FZb@ari7lq`%ecY#GATH`HXP&lF+}&HP4;@hZ0Kl@r zSj4@1SUDg_24kcSwhz6)-Z{QAU9jN^5z@njjLkjnJAX~DNU4PAd*9ZmVu&e5-R5TV zsAb-Z7t34PhjPc0a=FlQDpc7QJjBtchYxvs=V6p#_5LX00M0sce6XX#ANAOL$EB=z zj)Wk^RY1<*6CA}eqsz%|#w&lMcKn9%L{E)e`{<)1k7*|-L*1aqlAcO2%eDadQK(;w zEjUzuym>% zFw36Al7>?uwqftt_0ZWF_U+q87SdM(Jq2Q2b#$EW>FrIq{$)4?m8uN@3f4?y**NTl zlml9ibb$<@ZEfDVb*rL@dL?QVb@KG-1eetp9Xl&x!HLA&N4KKCT?8h3)cG!Tvb406 zMyC7bQZ+R-A~~iAz-uTi^!(k@+j|8s2snz4Xj2|D){nSTBiVRp{PsjGqP`WluL684 z1KB%bA_-l!G20rBAp=whdhq z0C(Z_>(@lX6$6TcfOZMr{D`qVGRkQLu#A$D;`>qPc`m3E@AVohk4nOy)!ybD;*N_- z%2On>R&*N!x$8bYx&nBC7*xR6sdp5c&KzvyngHY1gn**hEgfQH6GX><(qW;OTe0O#DYtY55)GAX< z$ACqM6o!&{5f!-ig)EE|YNBRaXm=X`W_spO9bAvQ^(lE3cXnQTac;;2-2oIQwq8P{ zkL;@ti#wGl{J8>HM#4B09-=6GDfHa8_v{z4>>dWG{CYPJ* z8j`=o^-XZ?OAo0SI_~#t_V&Gd{6@cP|9GG4|00Z~C2duy8p?Z~PbbMjV60bEHCA); z{{8GHPMjb*J6tw@6nL|r`t@eTW##2`GM5^bbv|USm=QlBl?q5tGg+)vROD~z15DA= zRJve|hYmrEkeT*vyl)#bJ@;qm2-|zmnTX!j8`)QagX`Ca>;*S_jm|^lUjcZh5dA7{ ziaA~b5U5uu&5{$?(Y2j@I=`3>sk zyhhsSsm;8+46+PwPm6?k)T-|u;p_Yp@vE?kPa{T(et4woR*pJ7rJ1bG0j8;4F`x47 zsKPQ$V-oY3quB=6KI!S6T zZ*a*0{{{n{#bRqgIh92x(MmNC&Hix0$a~O+#W@YWMf9z%DC? zp!n3HW=mLJ)tGE!HT~yDN91~I6GglFzGQG8O+T>$pNR4v57&upjL(igp-?OK8fcBV zTj}N<5Gig9Y*fTDCBznK5Mff4;lG`!j(iCoYCk*e(%U8PTK%RfdWkskz23chM{N$G zZr=RhJ?Cu;V0Q>~2lq($$0Lq%FyAkkX3K#EN$TZ^LH07R-}g1^Ar}OIK$`%7x{b62 zs4T%!-C=^L2+-9ks>iw10;(ioS#WuY2X~%sNtZ=I9v5&tKzhxiBRgJ~mzTbNz3A}$ zy@zm}S`hHouU}75(3LKu_&)N|BiiJg`jDZjkPR@2lCttkPABc6GI)PuQ<#2i*(2E6U2ig7^5rFZc5HPV5ASfP}%Tt*t%U;Uu{Id%4{qKp%qJ zJ*yIbM?D6#cSGnUzj#Q(xyKI+Q^_~Efrg(O&(L>nYWx2E`-!Fkp1Yirle72EyDFcZ z`cF4gN>Mr4?nIP$2zS_aDy!C&@m>FKu?4;z@Q-iT)YZvhM?1{TxJOF+N@8x^={ z=Io?MhGDU*p}|@@Wd~* z+|c#T4H~N5-9or_=6AlHVv zLnISyai6)VGe`>@o92vt{J~o=p}9NXYj47=Awd*aPNz_NVWnHJPbq%2Y=JUSyXqS} zqh}QiD~^Tf44EpBx>qOS>HT(~&$Mx9MXz_aqod<17}E9wjiLaH9dCtivp~|#$Rx1}O)>dHfD(fXi?sV6B?~cJ+1{<=S${%E@Iob*o=z z7uPmO>(hCY-=XteOi5k64C0^kpH-D#jba@4+yKN^*2uPsJIi>^-u`XPQ4o5M*>qBjW7_DqikJ=H z1XglgbR|U4_F^X)u0z&S&|FS}uhj-+fPQotJQ3gdqC9EzZiJXaT>brIbghF)I#;YZ&X^azcwrb-f&Z##A@c2X}2>7DvEq^w?1F~OCa9)gN_JIP#W zNKy~h8&Pr~{-15mU&F0%{p}}@O#T6f)Wb)AJ2Hs z1PKnlqoL}a7S5-$O_jS1CIX8$)4#r03vr5c5%7Xj<2^OceP$)Vhn>6&E5whS57Bt3 z4jWUa69JYhL^8x~nd}TV9{sE{@HH_#Jp^mX6c>e1iA|CO52QL(je5)b_ejcNgOp{Y zk*%#QkPS)uSJ!xUVKy9ebNK-Ashmb%X?0^ONE!$epXfJnp4|D)* zV{QI<7!a0c4SnWj#MNV@Qg6*WS=K+04_abrIMZDg<>rcPXO7v(1CWzWHYR#CtlyfcOFLua%d>z>C;_7b=+Q) z!v}wEPOh5IBRtA?{?8_g>>)PAtt_&{LDhd|zM{iX91LOjkc*OLMRi6>zd*vk3+Tz~ z_PDA5V_t(OR*KfX%k$USpuQtME8jBG(a}}i6{bV-Nd}5BSmvD+^`$X?3?CX7OCex) z1$Tjv!g{*JjGC2*lyDiXtaCMa`s`Vy{g*lVJQg-4sy-0iw-@JDqGIhf$T=I)*U*MdZh3mm;UCSJ&y2Cm zgo5<5fhPq{6zJ~^acmR=tab)@QLIc(+#eGa73Fd@3iN|lgIJt#UKOaYAjwOqIK(E~ z5S%p_L(SLy^ymnDC#1aiEs^}A^`kcD*fu}P@Pe`==G=1wQGzF-=OEBqP1V-B%uG52 zxww0Js=6Z%DL@4xsqnL?H6rl%?f=0DP3;8TOc*6wRe5Y|Y}Mk$ixCHeQUa{-S-%{( z0XX||g@~u8Cyfj)TnkJ0al+K(Bm?Aa%R(rqRnFfwp$|G6iG`kJ63sTCp{(XJ?)&@nWPZ|tqSkc zT)fO@c6upg0@2jU63}EV<)4_Ci15cf6$2;&_{8(nX6VPfkf5CQ%v~?=oj;L(2@mX$ zbrHmYWmFszQ3hj^Q&UUfhyIm?2G-XnG8yrG2!j@4e)l(MCXux$X7ce1xfJk)JHLpC zn^pe|yPQTwOhUq3a<0H0k5dbIU}3xO`%xtx2_ujb@>k9Ss@)8sKmAs8lv54jP!fBk z3T+oJTBJ0@N5Na-&6arPID@EZAT9ker5NFuXk!&*IZd!(wrty`VBm^A$`=j)Ewyi# zh=|JXF+pi@-?@UU!+;=qp1zO|wiErXQyh=ylFQ(aV&jT=SbvOTF$X-4+7!xqd8LPz z2Zn`(xlQ~GGvGw_kmI6bp5fW=1>&U>-W5$v$BDE_{sAyNlV31uFw%SG={8W$)s#Hc z z@VHDsU94b;dJ1?=9}C|Ev0dfrIg~ekOH`S*7eht2Ywf;0znPLBR{jo{HmmDhb;jK& z7_YY?`cz%hSLQi*&wNTRo`X|=X~o8*`$JGVUsYA9O-FUKwsL9f=!kZC~i z${Twpx`wd(K-hfBO%qU1Qxebo&VT%bBuCvmJs&4TN&B9dX6M*vYnT?QmzkZthI)+) z4)3)gS=wnsF>Ml$?>tVDX4ntA+=fq*jzoMdP?C>G0zf-tR8UYLy4TFijQ!N883Whm z+wW?VurH>MiwC^|h|_Iz zfB^%Bu3KSXU|@o~>-qX@I}jSl3`2vaN5OtgGO5_0V3=Q;n%Y6I>e))+NYGH;)8p?l z2ByD}6eM;oQEG7ea`1CbSz2B}3nB>io~psN+ou3vc=ew?P?_qFyoTT!bm$e#tgIIR zsp0_$cy*E(FI>3r)OX(J^_w@!4Y$WFBo<2_DR6TDY-5GrK0G`u1Ea+ro^7bWF;)~5 zg@9#iHiuUt!A?{K)FpbPNEntbJs%Nqtlo8S9SO9r7qj0CCC&jCPa^K%Y-mGr(DrbH zi61&}Cnwt9?!=ZVMHfpbo}HadoKM%!knLmx_Y?c_<;%lRV{k#}s`O{hoI&c2$=uxB z4KoSq(fh9udVK%BDb_7pdTR0PuZ?hsSGKgD82~cR;#@vGJ$+Zy`cC60K(q;DCCbrO zZ1&09>I`u>m64L?m=ILuH7?o@o1GDc!Q`Jm#Bl*#y!pLO<8zu3F@MN(1_=|9Uq`*V z92?68yDo%ChC7jDy@soP1=$8G)4#Cr0GRA0D93o_>lXZ3IA^kEr_aw=0Z55?P3|HW zJyi6E3ET5Rd^|U_e-iawv}h4UnOEJtIe26ESXh*)u;h@b&z-1+3W6MK)!TDFye~sb z!qV6X^S?oijgcZO0PamW`A?t~51>PlWHdrY%dkP=bS=c6hg)c>qYqqz0!D38EUqp4 zYmC+YkYzx(YbUn)%w$H^cjy<7LMyNI4PKyMPE=(QY5=byNkjglPXeLXv_NnLU{?V0 z%vQgAt9i7f+_Dz1{uB~_zYg3ao!$6d-w77NI5sRrSRPDsU0GQfzlK7hH>9n=g%@3C zZK=qvG-PIJ$)^y)xs!9h+XK-K`fmZy@}V6gITprP4lZzg&wpGqP-zvce`wRbf%=XEX-4~#Hr1YSB-t94nZNaXii4XZ#6VOO}_Bvn~Smk3vaUe zGq)l75_jyGIa^;wZtgnBarOFwxr!>xvVrh+5m<+mE*q<9Q5El{av)w0kLC>?WNj+< z4sIR{?dT>?SbMyZ8gOj{D;8+TFjXW{YN{C>Ev-KyM0hx;*jwsGx|d*Av0{TC-QaMj z$vyOYw&dgPIA&RrfJ7GM#LnV?fB+;eZdb>TE2wA179-ax_=+6>u%39tMt##&Muo-2sOKO z>yX%$dvl}I@-tO;xhMvJ%T(?SiQAJER5U9pGWh(NeP>24@8Q>J)N6ee@CGfcZdOg3 zceM!wbXj9#<5Vv!OhN}mi!P(@FT}_`cq6F`ey|mFNhCvY>?i_3>koKj?fEIw7!Hu5 zU*zR}=jUg$1pj`{Ysb7djXnGCv6Og0 zj;0(Hrox5c*uW?-56Xu2iDwuGFmDkP(`Tz`tbcQvwvv4aU+z%m( zH2s}F^8zh4_5*$;C5yPm(M70l3Ss<5hVmjzj35eMzIgFseE}MfEVw`QKs!;z-o@p!i%wn8@r|8IOTk_4L{CQun_j9gEFd8g zaGGqX0Xnb9-_rmDv=63~KV|{zgn*e4EgvoUX?FG@ENK#Y0W`K07(;e15=uE5Iu`Zj zM|$Z@aQ}Mx`bv>af(&o(;GpHcMJ4=s-54T#Gk=DPDN0XIPxq0oa-V2@4BjdWDzxP*7fyj}Jl6Ac&QbbN~AL`|PT9`=^3rV;4+3W@bMWMxt|( zWD?Xyl1JBlsCh8iL&Q^%pJ)4aQUITy8{+h5CK+WSrsHit@SUF{8D->v1Mn;!g{ls% zp9ytL@YkSU1XHVg3DROWDm%%`0uiY}Op0V_BOA~Wp=tR9lhk;@R(pRRQ;FW zbHhC~++cxZn2U(4klG;<$7X*^)(}ZnZ0g1c$rjRFP;W6)$A&ctk7@(F%iFfzh{at< zPfu6xtN(QjvZ|_^ySwH7GUtrHgJRhqP97#s*8ejprn=A0Jos=TT=sTEdr4EqYazwF z>-~L*r}D`kwwg@=RFXgzd~=ck@$xFzWA`HI_b!Z4=%pFHL=zoG#ll)dD0pGqEj869 zzbV2v02$r7x>Yc_E}TD4gAZjuYfkOJw+Oy@0ilL?W^)AWbX?zC=3N_dQOg|3*%y z9}RrVTLqE*P*aYL17gnwOxum&cqt&(Muz)*eSLwjx?PlvQ^zHEn9cO?P6T$sMlXT- zUpug0F7&>efKL;Gq5ecuKy<Zoh*wYyz-$Aj84CT*-gKA<7#x|Wuc1n|QhecyMZ$N(PQ zYuML>{D<-%;s5tt%*l3yY~ZKDEVAgo27Zf<{59@zr@Y}X;OQ`XgjH45r@3(V_?Og< zzhTjZ%l!ARJlW_5n*U=yxTfLC|22cZ|5r~X|Nr6Qdb(XGvKD^Cfrqu@A8aQM;<7ii z{#zE_THn5XV_v_W6d5r|$!-@hcYc2UWf(J{q15i}Sqh)YeItIiX8T{Eo1)148PpJMdJ69Aipj#|8uvx?Z~p+kS?&}k;W@Ex3@+sh3E^j{TbOI&&h#Ft?VHJ-kk zj&PAa6mS$JDLGx-=eX#pg|F8hLwqr*sn?I=zh>Nv1lO)zOAM*;Ty!urGs4XYlfXri zI5F5dWccXIH(7D7;g=S+h|+rv>DfgaH{LH`B7>9UN9s13*hxef*y`5qH1NlmdJCun zagP~V)BxmA!TTOSTt}HD^cj5f)7^OlZArKQ_9O{PCnO}a zfC)YK{4Mys$om?^j4dccc+iM0EyhTA?9p=k?cupuPZBHWv}8)8Tf$WG;loPMh(qx9 zj?_wdP5MDofc?Y(6hcN<0B8ueE-fk9Tq&H3_*})Nb?Yvp%o5xUrAQVE&BwSy%h$89 znVUD@uE|el3P|NCkU5g`MJ6<>3!}LuX!2yh5quD-rbY1ZWZ-?e7-*b6y%livGW^{w zNcZCNWDyo=ZE3mDdpABV4z9BbWWLhBzJ59G4<7Snu=aQay!oe~U}j+Sg_8L@zrG|} zVt%rHC7c+N>Hu%_%E@j1hs0c6FX0VABMU(!j))wA!NFy)#x>zc@XdoPl0YM=tmtqS zpB}Pc*e{~j8)F~ic^M0?w3#Y||0K1D3m}r4F=EE^{?n(`L~MnkLLZ@)bjuau1)25) zQIW@e>^l9H8Cf-1l^7ATWkg@W@+4Pn6~zj%P}r`*BaAA4E z7cX2O8Aqr|+~VTLY&@|vpL$GKWveshmp$nZC29=DqRIHD)?;5Jo!EYD{rl*(=JXzF za%G!gOzSm<>>coNL1y>%*##p*_5eZ!TOFS1FqX9m`CGn^_aXiao8TZwgUeN6WH$+a zM%9mq0Je$2EOB!$rggHN292y_~q-@msM4a(6z;U zX2dZ0UxUbin}-Ls98aAvT?u&FM2P zIAG9}u(4mi?#%hZqWlYN4R;m-lkq}L3qJee7Ah!ei{=x3;qkr(a=^kxL&5OILrLmU zS0SR)QY5+r`Ro;@;P3>AkqS^L@=XYL>HZf4D80wFJO%wiWh5Ao1ms$-^FSAqw-L5_h2p?Kny8Q9v{t;B3AnM|`8zXWbXHpC0pfme_Z5nX@muqk1k z6QCx_;3e>F_?&dn(b4?eVq{tzGUg$9dH=HfD+~2Fa8wp)7j&NOv#FZA%+M2Y`R#aT zAM5IfxI|bC z9sgND@lyxhPY?1PfMaA#6oa|TaUz3u{%C9fym`VS&{nkXp~@pq5g8fTf-wZR84sOH zG;`za+T^ed~0WPFG-k}-pE7tD5%F{P%h{>;_Y)wP}z;HGMC6j9{G|Bm@C z^n5E0gq~E0uRn~~4@E`+_H=#5T6lV@bQf|7BpM0=cInb3es_TUkC^F6&4BTK5?z#E zZU_GS6$XSzD?^plf{e1?eRL6u3YmlSo8NeJq;C8AP*T@`9!RZ%gCU2b2|~?L!5~eN zsYh2=S1Yr+Yf~h@>4$d~2ZxkxjB$g=BFNkUvauiYoBylK+ztq!M_Y=^6hPny^@Zqd zHOL1bzp!L}?vD@-rr1-@0wRW&(u%i%yjGr^SE4 zt5^qVP!ZH`VOAN*L$zBWUl{mlX1q=VbKA=Mi0_>^50-Y^ zx(KtJ6vfKQ%8UvR!15@esj10&R%^wKaqDiq)KHtcOFkvxx}N6dmXpY%k@VlTS<|fz zTP>`2V=cxbC7@g^Ub%85$%(;Yz}P#K0l54mLx)9}+821`%6ZvXKSRMK!z1R>6r!_- z`$}0he1(>SF_lFa1wk^~1f$ixU3M5z|HOBY&qh*?NLN`FT5tkeQAseTeXABk{L$VZ z`J;GX2sl^@+=d9NmwtjqsW^(Oa1THr(y|14@~UEDG;w|%V4HPOmZfDSucc?c?7NpQ zX^>x)hqguTA7~>C%W58}mlHojAE@O%d!}>0&H)Aeu9(9X>H&sIklNyd^}G~;x0F*S zA-7GK@Z^_xx$6sajFvkP=t>brjpbm#aQP5{Fo;Z#F<}BcD#nqpN4)%L+rgW zhk-$IBF&+ztIuP0?eAG1WI~hD%oEKkD>>&P?sH{YZxARv%1uZZ~} zm~*C5W^|>=W+L%e0QTq3g-h^Qu<;>7+qyQaH)iizK;e_bOps!-fmLe&jq73R5QzmVcoj4RaS*c_)e`Cg2{7$PEKBqt1K|8oT@tjfNw zI_>Xj4STg?1-Gf)^#fpi)!fX1g7zBi>>~mYsZ-XNqeR}l!1Ff~yoE#PglHyA02YvN zXeW!+mm&__Q=fW&kLau90miRiU5((4sLpGoN1eA}1ZDY=a_$}J614{#W9jXh^Akm5 zMBc-?Ks7jK5S}@99xVa@fCai4a@P;gIVry_hpv(VdIF4q*s`Qv4tKm;Rp9++E%g+_ zQxFF$`3uNpCW@V(ZxSN8mwVmV-nEgp{@%X4F)f$>`1~Zo)bm(;^wg)+<9h9wv%=X$ zv1OfD%8Mzq2BFI0HyE^rM|&OBn3TUC-oH7}Bg1WdIs+MSV^;CPNwdY_;p+y!7hXe( z9aTLgQrk7hh?or8O^O}^jJ=739xVAwijTEn#tmLRRg!f0{+=!^-qldh=_oR}h6+mV z*Xbo-`BU@vY}BJ5ddo4m$r~_cu6Hac^u~<`toOG;Lax-lu2S)wNK|-NR#ne1!=Ku> z9gh${6(QIQg0scIp0)E~#o5`j)d)p>ZnXWFW+aK`1)88#{QH?KW}4t4trs>~gqlny zJh2oNt$oZq&ttr*(c*f^{9&G{mX~zsVssQ4d|l12tCSesfA&RMsDlkMJ&0DZjqx^} zAdPp&!bxXw0{3^FqNPs4_5$V!Q!zlnCDcLk+#p4fFiPhH)S6|`jp!*d7v|@ITE0RpH7?M6ve)YreOGL5%`$Vva%)PK~n z$BE2$k9&TD8VC>dR&8iJ2dOw{BmAAWZrvi;=@gq59SA?b!Ye6)$YC0OygP&}OYPgs zjd*Ef3NKzUL;>|5*^+D6n^ski`$I`w22~nVkUD&CPt9G$*c?9pV=NfxpQy`((_((+0U*mm-4s?r zq4DsxfI^f0fAZvmvI&x?#=a{96$UK4iWyZpc&q`4QW7waV~hwqKhz*y(FXo*5=@ zmXSa&vLp(C#pLM#E|ZB96cYv}rs`1%+_U4xf6S}5x3?7n#9d5)lRz)_oyr&=tskxt zS(s>UZaOkx2(Y?n!PUf(WBk2D3C6i&c$_?)4X2Uv#peNHEdlyo4$Fxz6^8I6W-?m) z`y)Hmmr^7sPfnX4*a9hJR1$$iWLX)H=bU|s`1*gJOE?4^9JFH5dTSW!WX2JK{K8jvcQ3(QJD<_E+p#bS zt#r00oB>oM?-9si7yL$q4;O~oTqGnVNhprm6LR53j?qEE@ObhU+%sPc=UHA)+5)@)BIxJBi)o+op&);L-Gjwy{Ab%C5sa? zyEK@dCpmdC#*bz35r~$t?{B%>QnBKn*0v{U!^VyAh}R>9m{t+G;vSKf5C9`aj9*C9 zD57FoNDyO${1q9&U*JDC*VZokdsY+{bu5%BWIp1sHflqIEisnIwQCna+zSXXW;8b! z^8(P7mY`Yky@z3gRl1mhDjx@r&Z;wPfDRa;cJ&?cyn+0)at7oK0SsJWN7CUhd`dUo zL?kOvHVl-}wzQ8&AjnE;@3Q~l9Lta2IEeNAzyI*w|L?W--+%mn_ZgT0#5(YU)_^nu zrqfaim;(M@d66(oMx19PXJKJ+;<9Y-$tIe=5Uw+UifgZD)= zIh4}-4W~30e{ub5NXWbMys%k<2bH~foOEn7wo*!#D>wTfQm(8U#ON1Pdh6;ugW z3JV9v`-Pz~oXIi1Vk(-M7X6AUQJ|0Wl5yuuN=T4lxLIEF_tr87{CJCnckiYfcHm!v z3nTZNp(9%q{_FC!=;#xi3bOKkKi%J6qF<;B6)VY;D1O^H-_TK!L$~l}|HEmnul{}e zKTwJPTR3W#cU?1%n8N!6*{|%=bk4z~GKv*U0lsuOVvFHt`sXYHS2*QDALmk>xE%Kk zJrQ}4Yq8N8?0sgnw1dZ536yM2R`#+c(~q)%gX=0)x85lkZes%sMh45 zmFqwh2;IEz+x1uHDN+IQG*11ObHhmrWb6W-S`6Oa-ybchTi{?|nK*Ti1+GIGA`@d0 zpse_bOJ|bg`KZSuF1|sQfq@HaZ#jAkk^Ly5sBXan58?j%t6Sm<+B!P6W^ZKR8Xjzx zgn}3cf(PD6jy4^WkpJI5uM#RU6yw(rGqI5TFj9-U+VyW$;3;Yl`xl##45?EjAnnfn z_gZTjE=RHUJl>AeWagU?!b)gHvKmlI0m&tDlp%s8u}}@}?z7_rjLE9+?e4yeb`^^& z{^z#W6nT5HV-8f7A*xNnZFqCl`}bPtF~syY;ACUF3c??Y>&DT3#F<8`$?%$TL@#*w z+0@pS3Ec}>HzM)j>HsI;p>B52nVhoVux*D;5u;Mk#=tJ zVYgy6AWvOXG4zLc$-eV*|IP_Kh2S7i*}}Po6XcMyzbg>XW795&HhB_`Lz;u+>Bwvj zSbb2RF}z`D{!4K>05K?_Xps{^T&~6}R2?N{>2$iy6+4lhbDJLLBjzmr7C8q6-WI<* z2w37#@E%XnB60K+Ik%zo?OP=tZv=Np@79%IK(-PC#f!G9M-rd91rCSpSu%q}hEhnj z5|Y+IWJ_o$L?YyIY6)Oo+O2u0l4LrC9L)fUy#!}jrB;)PT5zA*A8={Dx9k8oQ;EY3QqhCz^yvBUkwQyc|a);%(DVn5jDj zZI*w4wZf7YW`nTroEP?pVrn%uf|$5?D<2QXH#hs+IBW(d484RZrkmpl z$&H*yi4#~OZD-{3c4BvuoYrw4s9L@AaK9iAC?d2TT7>+Pmse`2s$L$TbyjS)dd6n zN+i2U+5+}nD<*-+h#cEmWCAV%AGYEwHLb_Lv4X|0E69Kz@|^ODitRw{yxIv2q26 zyHo%7!j0q%AADW{(^5j6U}2K^CKQ@h*h}PCLP*F&kb{vynA&rfzB>km@F&Q`66Sx% zQ^a9`gj!?gqc$9g7#&5(hWNZ-d?b>y-*=9E=tnUb3?*kzfgqZVaBQyFfWA(^CN{=J zq*j!L62VDNVv>Ut3y9+*$On-_#9@NaQcxBSC4K|j!oflDdn@=!VeVu;=$;0fFAo2( z)q%Uc0q-L=D9-x)c_z5Ol9_{JCzbH$5$YCpc2c#A6eMtp8LLb07=&tH8ky^CqAJNA zM7&s-n79f?P=m*U|FChNI;p=M&6#G`sLjw#k}}ee+W%i{y$4**{rmoZ88@8EVrB$I;X2*mpC%7@JD?)_czu~)0oM_ z-YszGv{NgpGD`v_TcymyrH1f2L1wZo9cRq&MLfx!g}FNIor!iz%b{@6;grqp4=at0 ze%8-zdZ`E&o97TiHJEjHlUtFF4`QX~tuuOtRepaNJ-L z!ckKkDf3m7l)5!kfT^0DeABRTW4Z4Cl)f1&4ghna?YlD_Iin1MK%tQ41W1v~B!g1S zC-RrjYt3sznl3Lde)sMuH01JoHVE-v<-KiSMBJ}iVt37TN;?&me=6$9lbVtxzKJwF z5bU>~JZa4_N@Ow%PNSAWvL!61?+ExKI;4lKY})PD2)Piz&3?SH%l45x_UI8xn7m#B z!9eFV%fC&nA48u{4AysE40}c=c;@3`DkjB@+@=B&g11Yieez`R^l-*H8v^r;GDGf$ z&xIHx6ikcixcXzx%;iHS=af88xvRkb2eE&$2`xD&CoLc%RvpZG3fL_ITQ z2uz!3&xy)7@wY-X>wBim;Oh!$y$GdZxAz+6R(wiQghl5BHGMT-1?_SW8lZkpMQQIPKei@wccq`WmG zIHYquW!m|S$)iKSiXA$15V-krq@+b@%NbH%>1s>@`lryN5v9BmX!HMaz?4@b1d zzke2W4upP~Uf}jNSCxJRV--bp*4i%H;S5Ni zBCUn`Z^k`cthZz`0DN+0J7dFTG!3U0-WbL#eTmL!DT*r8(RVnvT=_D#$*bxO)pf+Z zS|Kx!vN?fFcS5su1uom$jM@?*f z(hTLR7zx+RI&;%Gp3{q49kJDYusUzB#)+)8K&wYGPmc1<@_iHzFl2O((MqO)lqiZ& z6)ena(3#NqG{$it`u|@#ce=|ZDnv`=d|CQiTp@)Fs)6dot=s6z<;z%mEE5~?EIfGb*O9L!bKljxp@XE9^JP{(d*xp=jpTJ!w5P~)9 zZsZ#Wa{G!pLjYAOXB4`kPUfs>LH3=|8hj3Xx)HW|gj>=3U_VuhPKPmbSE)p(lC#g& zRPQ~8(8@K1V+ZOcd>a-@p)}+@tXT=Hpyp`C2j> z`M5&ySHo6SaCUe__IsiH1v=i8$Y^jHNj5G?Q5XjRJbP1Zz z{Y2>{J_7>uVmKV6=cszZm^~vIOXtAgs;fe7Bf2V?Yo(q+VLPL zYGzY6;pI-9ri#E$JVO8)?aP-mR9zs8Jt!c(e{X^jCHtxmpX$r`qktAsojCiTRX7t!+C} ztTW(?!BEX05LEQmUrel^A#tbFm0TyZ8EH|PPzlL0iH0hrRl~8@w18RjpjUwKrPw`w zswu7^V!h7m-cTiozXSte60c>B2>);wRPGpJUb^gw+xA6562y%yW>eC0`un$&o zmOfuAVLYTN4qc8safzOtJ&}G$(w5uA7(^Y#MFt26fzu+ajK%hmLQ*_Bxi813bK9>i zh*>{v`4IX3e}cN0lPTg50*l&T(fQQIb{Nu8&uKCAByxCE@PdC!n3t*fykQKq?4rRI zQ4i#XE;wCnWM`W&5*)l}ryY=ZYsRa=-^XF{`VWgu6tbt_B9?Dtc=5duJ67~N z%!cfRtPrKURHM9Y#N}X-`5-%1rN=?TCc{kPTu2D!tJjvTHnV;iWX}Zaa;pCIT~vRF zGBFhcO-}^zY=rRia7*sGrwZa$Vf7~)j|DZScbDuV z)i44V$e)o$tn}zcrxuP&k{E+BWhvOQXz{5M7Sl(zFUDL-O`r(f|JvgvqI5$dES zdphd2p^v}UKMI;(r233Kib0JO>{urq;R1<(an||yZae~cs0`8PZ=ov{=7zLNDCm^^ zVhD$K8*r#hJc?T=$CBZMTO2IuwCF#(=N8vshbet{p2owo%cT||g=qO8tNO>6C^{$^ zn@74J&VcC2&rN!=ujcWmU^~Xpu0(%Y;iGFKoEcD+>Yqcj4WL?}2m(}6znfKCVNF5P z7luuS027HF=|Ezy@R2Yy0y8iu{cu9VcJ1`m_s1HDT-gwfY=>d18xXIhA(jS9SS$!s zPQ^~}N*N9p#RvC9{I8+|M0!0=JU8Yz1zk@C>R!P$0Y zoQICv1&(0iJ0YpK+)q;W_n9}V;%4*OqE`gv_-SgUE_eC>vo@+1w*APQi4nKrcyyyj z3=KV;Y5_0fTi8b}B<7Zbx8NVK+V6g;pQzflqQ1eRra?ZE%)VYuUYT2~^Iv~Jyb!1E zg9i`3vn)e@Nc8-s!-wCfQMw_bZTasu(gS%FI_#+|a8mFww-{2Ay3!43-8iEMG z{j0HeaxMSBm-V@@Z;M4c2$g4Gp2OKzDCliwPjs-egGk!`bsgn5(!?#|uQU0GPVeBa z10@~*kDrPSnynh*#AtOrh0vR=-n5(6{gJC4~+D z7bWnu9TXFD5d-82A3@X$!QZ)W(?AO6>(vyk}@sje~qGNbF4(1q!$z*f@xqzpu6Lp3?M;yQSx$Wy<4 zdfux~F8T-p;Qn_l+LZm#ZA+2--%o)u&eByScbW+hh_QF>&yfM-tbB_(!5(^oDqf(; z6Jx~XfqC5{FdTkOMGYQNFw(-bB_%lx-y8Z8$2PL6fuD>l3(Bw`(2w@V&+~1$r-P?# zz7kB43)(maOPZjBKfKPO&U<}JT0_b27=85aBNY@kw5OM5tfjMFU;X<=VkDlmtJtai^jy7u z{ak)=#C=}A!`iaf)k_~rBbo#85=4iR8#=rXsH1~wD8c7L-ciUa`Uw!4x2*{H{t1Ug~+I-pl%W8vHf~vd>O_P6O{c0XoF17r4z$sUmTCc2}{abuA|Gb zV|mu^d=(0b)PhD(9UrBm3bs5fpOq%!O6=FDZJ?w0{~Qqj;c%0OnJ4ZSq0pRuYCAZN z$g|-hgyUa*YC8?M$Y`akaPjg?<5bj& z%S^#pamMOTZccxwDT-*Z0YJ_zE^B^%e)$!{Y6z9sdR(a*JBtA26%?Y5JTPJK0Vf=Z{}z5j9^ZWgju|;@?1BWNNN-cK+4jd2A?0eKfVG zXygA2B^BeG`mCrw^X!b|jF*`JO&tE36TkzM_xg@24gr@YTcX5vBih1V8_%T>7%URL z3f-YYRS+eMU)h@7>;ri7GvuSxU$Rd`t_JPJpM`$A-nx@Onzw1c_t39CT^>Da_A{bo z!&l!nzs|1ccwl;2a^IsL9%lL6>f*W3@yQXdAW)1W0HC766Av=}@$LL?lB+Aeo*9(7 z^*qBGjlf`K5f#40;_lRcahyUfsJIK1WHJJU&L=I}2ngO)a@32|j?a_NF1{-xHM_cH_>0f5hXsx4ap zWm=vLpZ@p{!qB3NPn`kLy@O-QHms@&c52mdS{iCaSxlg17TQX#@A8YF3arJKSnLaZ>+Fb@$MQG&4boZlb;!={uUNv4BgoK)%E5^UIy9yo$Gxg0~!1fTn4>*oq2lmsIG6nI(u_6hf2(}FkDk?e$ zw|@BO(Y7sHM)95uqfw=)@)r7&a~YADA#N!Ui#_H3hYs+x1s5R-%1+hG*T5`m4wH^s zx9(G8xsS)R=Z>5ufGN`xB;6Tj#9P+n*c~|cM`3r^$vW>&%V9%@T11CFgeesp6xx+@ zy@w1Ku$8VhXvF68m*!9JHi@PYVR=ZD9xZiT(#L7q@@HH%($AsF&gNOOt39KX)Vt<2~z@e2L(StnvTpW5gmEhRJD>$mPQ^l1@>UBgK zB8O$#d8QL9mZLbig{+uF^yEkWb`OsxGV{(QKSJj^w-_6mcp7?KAM22_`eePC+4Rxr zcS)NmzAmo&yzm*Lh~$HV2Uri$o?>V;iu#C{7lhS1iuYL;f(u0RCqJkKi0UZ^<_zvGsUIUE?!u;)E%<`#@1KELrfZ!F01_%S3?=gmv~r z6P_y>sbq5F>vk&F0YWs>$L)9QH?SBy2s;1xpulc_9;S>dX4JO?I~Q>i`APtnVN^E# zExlGQ>k4_5I2^9rt{EN?QAgpiqCQo&0V4D^?4Ik>h!r1<_epCE$Y8nb@aP6I3dEI5 znfu_+R|tMJL1~ej?~GF8&`M%O`|+cM(-3x@)*`H%LP!{fFG&^&xC9-m8>%R0~Ee)B(kHBIW1=RhrKZW?a*^tXoLK+&?YhAU~{P1i+zQX zGd~2mAZiK(TC4MZz_r`q^eE$Jn8et`0_(_D3$eL`Upu9hj#0$So%X%iNxSyL(=B}L zvju}$>>&jyXx;t-n(TPG?PiSrgR!3ty_cQcj*3VS95`bkPQ<a_PL0#-8}V(hTwU zl46c}Nv?wcId?5CHO@K{W_%0vBu7D}gdRQr6POIHp(8CjAD~yulhV zLN>F*EYI^h3D(1*(Wc4v(R$CL+uSq&b}~SR9FCSerfK)_sM-YN5_PwjRA#+)3SX_G zQA4s7gWt<4oYIwJkDTgv?`oX&zC)@P?U$+`43E@b@T>d8&qFi|I@{{r-AvwuZKy9I z3DDht7P|U$1N)vz*p+}wSas}}*4BMR{p?d!`crL={AE#ARq8ylN8PGt$8P4-Kt8_1 z7gc^g_(z0z!6hi@#q9E9`Zu8s?(och8Mddx7l=G*RZwfB{bxY%cA$CgsI-Vnmu{1Q zn=`bKbu`}(f)?5EK^ej>J4!cbPV-B{*bv0LN1vB;-uMs7_E;GhWv549CL`)9ez5)S zH4gu?vbdoc(8=j-bUDF*BFmP*eQw-X8Pl{|lB{x~dHzpmSYfL^C1?+Wls3lI47E~D z*6~=zlX3_AMbr$uT$UNujHyh~vp&(-Ia_=%vbyG<=8}tosYYh$ETuAUId1gm+T`Zr z1ytDa;2=^S`+_Mk7C#E4*N%Zqsbd){cuQ>dt>zHPnw`wgBM3*)e{6>3(7rvF|M8;- zLEOeTYfye{^V)cb%FM)!0~Z!tZ^P4ki(p0LxyAONfN>9>$e0>BnkueA&wE*PQDU<6 zav1_H(M3_}g>c?`EVveF^qVfTCG9@U{B!HwfxK6@{~9UPkPC;v-bsIt57=xlg5ypQ zCr!+wS&~_oYDu;pfj=s-I#Gs|DJ^>CTFU2LEhE{)$`G0-hMi^PTENK7tWCyPuQ7u& zvV>}8**F7(#dPC90@c=T0kFVf7tT(1n(Nt4*@X3$k zDL4$~#*+I>&=fvI5qGot_qhRoE#z*A9$FM6LKx$-7BPP(pJ_r{nN&t*Xh)dyc!&Nz zfruzWQ{qm>pOUVavFs;hh?&|^G>M+vLD!b+Ie1ak@tr6@iG(V0wqWD=GZM6+J4NX7 zxGaeUSWGiZx|UZ!Nl~HvMfW*3%WMCDKOfrQt7R!pbEuYO()D4qqGq@r0Yz;E%&AXU z&CA*`1*0;+qCsJLuqR1l>HP>c5~1so1vrw61Q(~GzV+*0IoY>{y=F%%D zg5Ng!I@Z%yrbh{!!Vf7B%`^p8k>P2mI2xo{Py#&)rc9b-Kmo}sVMWmTa8Nt3h!XNb zQ1vXclF6y(39p{IB(q)dgSj+iNbu17a| z*`b?%>&ZX_f4sJWZz{`$-q869Tm+sf+vsty?aBuwKe`|e$!RqcfL(TnYB9MQ^RyM1 zaK9_lN3#%$j$}6ltcbs5^WozoG9kjdo-xk5$f$!*fC$!~Ubbl)c=)gw#`C7K@OsY0 z#kDX$5Jj8ixyChV42@1_Fs!MSxqzYQh<4JtMOH!0DqOg!%R*1l?)3b5X|DQke*crJ zRh;v=S<)>n>I`gwx%EU#Q;I7=yHM&hR2*N{QP#(CF2vIsbd>;c-6&dk6-ouXklq$4 z1#x#;F0wtfop$^UK}_6?08|Ao(hTGMc#z!Y`XdLdkCUSsU!j%)j8a-8;Sz|Evfa+) z!ktj@qLH7Fm{TaOsAL3lW*)MJRAsXXNR1q4+3VS-yd?Dbxcs6`8xLiW86p#Ws}v>SrJa8~{dC05KAI;kvUe}bhYr=hojvYd_BbLf#>>5!MBb)` zsK}*ECsHSIRwSqK1KR-dv^QGt`PyCuOab{Rpj?vl23v+VL z+;|*&Zt&Wz4KCMnv#{E<`G;lfxHg8ndN1@WPFdsckXHGuD9HcrGwT=Mb%v~5Jiy;< zkK47={bru^U32@@GU5D#db_I7m%UXn97a2`YmI&3SpQTjmWZZM3i%wb&TrnTRXp_U z#W}r|cC#kc93klZtN(OOyA+AkWjnb&VVGNwnj3#+y6wbPOg>M?F;a>FM*9RtZ1gBw1j`i%o@}O*Olg!krV!z56DLicKLiO<$0!a z?U@vz((!(Fr@2?wd>+obv>B0QOeV$q?NjX2G9QOj(BszP3GZK1OT_n_dEQsNPhKT! zOKBLn^7$bYKifbbDcZ-QdZcpNZd|zN@n0efG5_!g;DwNZjoEz;e=@Nk#O#gtjLOl6 zCaK@IatO&=RPR{EuEU2HgqV>UCZZbNvj$PL8qz`qrTuPFyB2kpf#*PN7_gH}eLDRt zI0zTrdsH|>iv_T>oQ{1%RxK=kXqrA3e^bKMmB~)S(|O+HD?66u3yPuCzNMaNGT!k% z>=(C;5eg6 zTRt0Q9e}E8SwjlUIv1bM>iT}eC)z0wE%mci?}}qt2{)hED)>O>*7bNtqZw14Igcj8n`KHYq)lG`JaoyR_R$EDxzt5K85kY5a*;Bz zm-_`WMg%tRRNdnl*KgUX@+^<5O#9MJwOygJriE8qFXi9kMih@Nti&S0%)6(Lawom@ z3ccuivUXWOO4>zkauWTEDUx)4p_pC@?i6>;dkahJLw^m^(UH}{#clK5+#H{9A$1%I zy8g)7Z=tiv_rpS+2G^XlrlE-D6mrw{EXr)#Yu5ckzWW|_(0B5~V)`O80wUMNu7zSi z!=dm#tnmRPtJ#slJ)_s%v(7#aEW$Kqy%x4v1yIaNDHiZweQ4+}PFAP<9*B4$K%0Mz zTRO-D$H0IxNp3|&<71tEzHyGZO03$&J;yTBb*cFTO-AUDdnjlxj}f*?bCjt~_%XSF z^b!El*X8yjkWE>-bgM=zgAz)p7<8M4M8rvtazy$b2j}9;bCkv z!A+_YTL#B|4$VOxGPdO0lT*Vhw7*-$NpQaooy0xQ%!&TWyVsF-X9h5pkTO>r5O?^&y^iF{O!rGj5Q4%_5V; z)ee*xTD*ugLvOY<)1NS-ns{b`5{u@}g>-re%=N{F;;{8eQ~dgG;af@Lr+^@B@$sju z?Es@ixU*kixIin`@e^EE1-EgSk$s-SHn{e>>=3Xv9GIu`qbjS zUxxa$>i~Jos~_Sek9gK)I$+zjIwkpyo^jsu3JVkQd;T?A>%!c*{<^Fjn;8=uIz@9+ zRq1#X5~90rr|)#N{NzENibvpwz{?o_%u~KROf`8sP_FI1GOgo15&rw zeR*W{^~NeDcdc^U4D1lJF>l9jD@L&OdT~RsO_^6c9AG)Xy$eV_s?@E_Xp2S$kDji8 znVEjVl07Zr0`JQC&h;f#&M#XS#id3P5Vm-QOmfoA+^6bGUPt=qY6&@(0JDS$x7AcV_n?SP_+(qcTJ zSU@Z`BaKRmoYR|Gl=~G=GcvrmiDEw;&{`Gy&U9qFrRC+_S;#>Fvz5?la(sf;JwMB$GHlPkvwEg;yMNO%-?}BlJdL9l8 zyc;}Xb7<}|i0eHB^nHVEX3B9mfuzM(85!Fc5$Ll6O&6Q`Z*%hW8| zdtU^*4}seDb{c-@v~z5o+O`XiW-4sV49JTv<5V_HjI#xk(Ajm%MJaetKw2m5v;GkHWkzk@(ghjD3+FVwIX z%*sCiI+Xk7zlO)G{n(y!)RlY8sKDGSx0pNbiP=u(TJ#dCgi7?)B_IhJORH2>?X*;G zEuQq_<<{d%3_sPPF>CKjib3%Tu=2l&+lEld33{{L`f zUTpk498GDft))KB#+8`$pWvgeWkrO3ol9hdzIRct0flucmBU(qLXzlaqUs8^G1H{ zL?W52vfx_FHmJ>;tJimaWETF^_M^<0BP`u@<;s=8TNL8r#<=xX9;{0U!>IB9(#sy> z0O(7We|YjG2kRj=LVS2l05g6YJcM{hi3JMj$nsGvEuvFtLR(m@r<49bz7yV0N5zI% z;U1*}H+-M*0;rWAC^^k1{oa)4F+?38|Mr4{U7$=uPMV3MR$NH)`4#r4&#rYv1 zW2s<6ni$zocx!M7<9zyE`c+j0+YIVHfK%qmJOac|GS2%pkA&(Y)UxmA*=znBBL?1} z%8$^3+5`k|s&J!VtoZ)9bKX_d$3I?eYxV;}R<2SbV$tLWj(o%&RDn{M+2`LK1=ECC z@30R*z(eNR&Py%T(79;-K&z~RWoEThhq1IRM z@Yn-w+{JfE0CPa`bi}ivN@{4tLO={aG2BIO@#<9^b-Pm2rd9+TOs1Zld?|V>!Ey>rU`r7Ju>x;v#$Q9;oj}vXR z%aIEdA{bZk7-wI2Uuh%)5a|fPUL}IS&lz+@NL4Zf*{u4V+gGKQD$j zP7KXsEPv$}B2W^-h%G=Fb7zmGb{fW|=+?jgKJH?jDc_p?0>ug`zrqd){mKub=WPz)nCFl6P^Ph;m&-2_-%XS%*P$_(hjv;0#cBPD`mgUq$f>p4SJRdWIGsA=TV z7ASA?si|hAh#wnQ)&(f4NcB{Q`pOa&ZUpi9p5 z#<4NJecrow?fMjwPZ(JfX+k*Y%&X+ozi?ucDyE{~8|6zj{Bfi{2C(y>QkUN-{KXe; zhs8cohPzj5N@3H$z(727M~9lbU-Mr~MXxm3$WO;X{rvg!ncLkct4az3R$W^4QlIb| z{I~4(MH1+L#9Td*_QIHhDl`5v0mnsRdjJx~W%v~`SJ;L6p!EGYZf3^9dnDKw>+_n# zR=n>O%V!C=hQqdqRqTt>j7iG$$tUTYSi9XE7ia|ojrSOM@E^3ydrj;s{?2Y?k}206 zGYFNuCX;G99p#7_y4QySnIV}-kj+!DtV#1$-N4+vC6eDl_kOX8AhkvoDZa_=2NH`w zU+y4M+M8e|MGMx@!A!-m-E%FG>zJjlhWa?=PoP2C>-s8YoSpG5+EA&)!tpni*B8F$ zk^%m5xF~n?X<-o2?j+#k<>wo4GeWh9^s7;H&Rt}B$>KGs<%<4F4iBV^7MJIedUpfk zuOptni~vWmTeHUF+li4*#Bp(YEY3F|12~czlo~YHHQPj1n1a(8#%@Srl0u?0U+PL~ znQO4~w{`>^83xJl9v4(Ze*+)Ybmg_M7?^Pa9b)KCe+ zhRFiiTs`mRgwm3dSA0Zkstc`7BS?~p%7u%})M=;Ie#RW{MaWc1^nqo=tY z;(;b0{`T)0G?0d zx=2_vx7S@izbo4g7kGpplNOD*F^s~lBr>Qe2aI668 zGlNf&=l76Pl23+GtJ3T5pxf4he4!1n4RiSCpT%HInU<9c)y65OB_(OhjoRqs#Nas) z-fo7V_l?-RN!nV94i}Ay?GC_?LpF(7G_}cOz~D18&$Yo~epl)FI8tb?p0-Nk{FMFf zoXUCkXde|U`kNML4t6uKr0#E6J!wvP=(+I#JICw7hl;#YX}407CMOUSNs&pf@d^fl0u4}3QapEZAID$LiT=<0mc%K? z%8yshpoW*5V0Zb5^##K(`N?&fulV$eC){ptpXCd?9(AbCeJ`Tb0x-E__2~%p0t8Sn zBDsWX&n#UQboOynMdqSCor@ec^)@<9vw6whh^mtv(*L)8KP7cuKPeU#w3iODG zJZ^>LS)0yT#jW=H0#Y7T@w^aSBb4+y*z1XUcLm~B7vK&$iB`n;NjT347R1?bLe`ks z&ofw?@&d8?JsM{5>sh0Dg90FNLzaP=ncBOsg*OiIAH0F-azkf{sSdf;>c-QsIpM0( zL!raBfWDK&rqZ~cL;>(kEtSTZPyN(uL%1k4Y80Wr_nMEawTAoAP7FV+?|ewAl4AD4 zrZUb0oWk*MyP`l=GxaB|7pZsW+1>%)qw)5tn>1aG>up;T;C8AYJ^1B@@XnpND+RR3 zX7p9S;V!)DDqORHV3s$0qI$*by5vp7qJ9!!&_p}BFkW<}-&*`1Hc0^T)*klrVMjI-rs(Qb7<_Ten)|&$@NP>+c$^uZ} z?d2P4%#leoRL(9SDGhVOL_(&Ni+-p2h&nWGl^Aep1>qDU^7j$)6W5N$$;zo_3MZ3 z(nX_}$-CUYz zx-pp-^LtY})&!E6+GmDo);hw2nf}ZBn<-B$WQm63959Rx2*cxuH%ugFmE8>G>tiwpU3?7V_H z5`g5;iS#t>dJiHbb9i|i>&zRyeMZw*av7V^o<>ccua?O%ySu;FLF3W2D` z^W~*Y4w;qYB-aHk>rt+Y41%?M!rr=$tZR%^U!?cZ(V;#m&9LN44ckyQY@qsgg@JM} zx)kD^oIy*d07UPo`8W`dsu3w?36E1~+NOxd%v9B5z(M^&&V zVr2aFmt0>NRf6kjZqAmPa-MB-x&ym!(@D_9L%mJ-{BlPr6Fg^GYX{Ih>|M8rc1EmZ z(UZ@;hgJDq=J@^Jmj0})Y}vm3a^vQ119+b~<{X-_R5rGsy0qqw8Y7^-#SIV`K`sg% z3@Sy_Vzh|6GnVGNcHShLKj~`4QSw#t^Q{a@Ek@LR?F;10!#yNFp*bh1`&_mK_p{B7 zNnHK%ArASq`$JCMZq9lZ9Z?}5t34I z)s6C5d&<$RT2Pq?6;EaU`s2qH(^rMh!JYgB4Qil{!%m)1A9N=mF3t3^&pu8{a-}cz z!TV7`8HOx%a!k^XNC2Y}LIfCe+R`;hF<%k;N0BS;Yn#85zgr;dWwKC|XMw1nv8IkEhRe2;va2 zrg#<_Y9rQHi$;;a*?sfxDi%}@n3nYWje9>?&fD4fYYXV`69{*P>gozOPHQ!p*h);A z7TM@AUp8U=x{-9ZXruID+VDpll4vfTyJ)3Z2{nqg$h_k`P>JN=5hLIX@Ko}hw$-KO7(Yr%ki6}jyQS@YO_J%Og=5!icE;v|D#+& zWap5eTgvy|r}BFgxokQXAX36`OPW&phAy~%06&u2A|wKJZ~pPT0nFLsS6JGdq{ZQ4 zc@w{6%drU8^{Yz8;Dxbx#dn~Cyn+H@*pt1w)rX3+CLrQ9IE~h(Zs)AU{V!T1D@#Wx zU5NgVS9S0dpEPHV3I&rj*NPsuf8xwmG+5>_oBHy;!p~P!Bwn+>6R;}Q4HY2myoFTX z=T|kUSBKu{@wwTaT!;t^;g|;*1@;$Op1!*VvDCbkUMr3&NN2;>9u$8kxMz?ydufgU z#!?V?O7D$`Co*aTqXc`-jiJ1gu?h;lwh+rk+-(83XktzRCgA;}H7&2Q>|Vd0jN=Ld z&coh~m>`3%Xny@-45^ep=?Tcm-_0l8b{YicH7nM6IxS-t7VM zT^`lq!&|4E1J8>5sMNS#otK*``_O;OhHl%ONH>aw2t@F5sS)XR{CN8^oi$?#0sMin zy5l(i5}3H9d%rZ54ihQLrJ+}>`nUIR`kFDmacBKN*oefdLsa$rPw&c^Qo4*5{!V(j zCo|H4I?OuqlU0syY?hpJ1F07!5Q!pSY^C-*Slw!22PgpfLYHGzrMEQ zA5V_lQd?d|5(eQgJKVtz{SUtue0FT{Ddqtwi84>xk}Ku1qF8t5No3lt5fEO2ORYK~ z!}|eybb|pUOeFnqk_vS-rZP=_`seM zrmgEsh-wZ0D9yw1T4&aL>H@66SslLa+;G5dw8x=eYL!?_+)1ZFJhqjD&6PEUJmrac zL&^IzcZ7fYTtu<-mu7%0)Y`6GLoH|~cnFJ+62bU|Ufn0Q3CK&-K7}_)8kPD_a)Bvl zk3S(&SV5O)`UX&8tE7J+e`zfoG5g#s4^Z61=;veTOy4m8D=Pcc)YNOWZ$Bz4N|yx* z=nyP<6p#+;C&s(e70E~{OJka^XsQZ|&l{Uw$TII0apj71nGlKos2?sZf6`M=Pp=19 z9Z~V};96bDG`!S~d#jTE>zJp$SiQiUy3%eRM(Q`6Wrh+&MvinztrDQS^vjos^Iw~H zKdJKI{Ko69VK^Lh08K5Tm&xqYpGRg4RHk?=5RE(rE_A5hukZUWBmKV* zT-^WUp7c6xB6f_q`C>{$aD6q$w>SDKv*Y`@+&c0TdslI9P;R#@Ab)v(m8U3{nHbP> z`mqn0n3z4o(obSMNyfUPPWjI68CD(qJH(;>T+OcgA+fu?EgK7Ee`)WXrGsKKeHu<< z^ll<&6j6c9rce?OH%-JH=r(X2i<6Bnbr^9%V0_*~+fw=qSh||FWi8N$!2EC;m1kWM zH7@ z(&WLjQ7KETtz}wCU@|Hukwa?t(v3L{3W5##vco<0TxX4wQA3UB|yrY5qbPK;gEoHuURkTd_YKBW`EfZW}c zrd2B-m1Y)03=bAl@GW$Jkw8sNuZ(Qau;CkEYffpr)x6V@XSp3fcWz`@5mQ*D1tvq< z`J*8>zK)Xn0Y|{1{Rs-ToOSG}6?Y&Bm?FiRKARrC_$C86*%UG4*6;&ii*5#y-{)O< zVGq28m=ZPnK{(P_?^5>(&xDIxmWiGgCuA*AfR#7(q%l`swCH7ir$aTZ9BqvCs#Tsh zepMTPAHJzdMM1>3F*D*Zk&m6SUnAF%htLb42NKhfNoleK(>5*HfcG&eJKG;*;%1JI z<5*(g9l&W}ZrAfFO5&4Lj+~Zm(XLEQm(7*xjLY)Vp}Jl*B>}Y;P^WpHb5oDj=94v} zU7|FXGIu&9B+Q2QhNr5^Q;dw7dkwm`bkIYLxDfRpel~{2+);Cq`t(cvZggzfC#eq& z_ka`J6&MM#LG8mVd1M&!oAcdr(=fMNd_eyFxZBAw!56`w{H$Tf&Tk z^*a((DJF3%n-sX>lySf~kmVIoejt)H8qwU`zq#o&&*Cxi@T;|o*W+>0}>w`d2=} zF=mq(8Oyw=`I+=J?wA*iY#ya9F~9WtcWb257ZTRaP(8E7Tlo(uQVR&R-bEVp*!Q7wrtnJ|JfL~#22@M9-c7jr6sRcK2B_B#wfeu2emV0 z-5zPFVLRsB>#VmvjuOFt*qXD(F|x`fEe%tE7l(W=qF18@J| zYBYgNvLXdwZVIrjoxT0;HeWc>kFXn~VMY(2Mty42>OM*l*@1FLmkyFdVnqvEqsQT7IT5HCuu zFdyLWUAb(+6RJ}KYLB0Ud7q6vl5blK5U)H6Ay3|m=u%%0MG>4)g5^Qe0j2^~CfvK{ zj;CCA|M5IU{CICz`l(BKCC?z9$e0iJk9PI$jf_z^riBX^;(8>rtO^LxiF7b_>((U! zLz7-37#a{5nBDE(j*t08W&`|7(%$Vr*R;E`H0|Aq^wX*rcFi)HP*5Xl+St-*u$pO- z*YjMTBNJy&HyCG18A3X*7i19~eeK#C8YJqBrO&SohHv3C$E!RNj~=+Y>9SrEXgQ6r z0z34oQ9`^GgOYlis{ESi|LAh7$Y#~5bjI6$&^9V}s2otazJ*M5is?R*)sr0S%J-Km*Soj4?zW7Qs+||^e;KjO?e0CFt@hse zV^u!JrM=kKdqnc!qosNv7$m*CPoE}BjR$%n9Ssn+*7*rjndc{6F|0c^#uNWPYXZ$G z^v>>_1&wQ-kAPssV;^XyfNSxAv@J-ZsgT5zqwi1f!#1CCpLbPUpBbkn4Ve*ff zh@-aF$Y`I1HK0?21~LwZV#^EWrUV+oj;o`P%>v*CK=OOq%!BG&darZh3F4y!Ysx5r zFaoYYI%xf&FfA8YzIIe@`6ZN!(}zxx5YstqC80+qN$Coi<&hebKA^rRGBUd>;XSj$ z`1X3Jj<8Yz(GT#%`OuRwd6i6pdGow}9QBHKY5O)Q)Nt zZSdaNo6%+-g4cuXhGWysS?kMtO#1m+<4|n2_wEyCA03%BP`~rgJ?CbhuYK`@=l<2p zvO2nSTA+V&#?gkpJ!fo*j%cI!r#xk*jp>=utxNhW++Wz_#gf9W*_kst1!dKb<8%45 zOn`mFoS1#Y1i46rhoFs{pq)w0eWm_e~x93!MET4XT?Ft2L z4~wgY^Poq4vjGc6QZ$E0YHP4V%_VV+rZ7zYgj!HcSb**wuv7W=`Q;S2TS)s{v+u&V6l$vzhD}%^ZSd9p6JpK3Tgqy zqY>CR3?gOfbDu2xAh7NV&`)|o_LsZJG=#bQsFe%&N)32FXm|kRK9I~`fIl8^h&;}9%w9I^s+HT6+-<_h znP7f^RUuChpdKP$m|jTF>!zhOjZZI`3P(1B-qc>_~~N z8era3kveyfRpN(a#iXFp;q@nf$d7Y79FujliQh9T{d6y9L+-4$|<;>;r#e_$sFcxJU&w}~zD`YW~XnRW-pj$j$N>u2?RSUroD=X_v zDO|u6xJbMCTlc*+m5CyBPWV%BGrZRXYb=Q%d3Y1c1f(2d&{+k1}8dEWG(j;HVBM7p+n2ASTV>akD zLO)zG?|OG{#N8_lyYj*ef+tgZ+PMw}eRP)5WMurl3S!S^6n^d4Ik==6Ux(pT8MO6eT>`2sc`bVVMq*1RoAMTWX-nZn#)u1-zfeC{~{X74>EQvNqMD4^;!7GI=q}?3=w5eb5_L!QE z)qIEgpon6>GAT$;=S9|W@|`_y1THE?P1k;YHt}ae%QLf%^*rBherek49gFU%H}3e& zBEWFlEtiZ>$uHW~%d1|nXvB>-w;yG_?&Lo(qw3?ynCI92axbjzbV0xH_a&FS@lyt{ zNrtFx04|OM(~_2eg!lB&P-P!@J!PgDi2UO~z)?RO;nrZqzE!$bM0dq8BOlKR>U80G zb!Rj}k&Ye4I@GVJ*)+F#E327?$vPYo&doCO5!dsV_{x(={*@xI*knt(kDp51wLM1z z0_H_tqaU(Ig(wdh69vjKs&(UFwU%N3_2R_~%%hp~TXo}67yaB$27T;)r0ITs>Wj$- za*EWUS3>Fc>x_&H>%v@vF`quH+^sbi*;;c#OCnNOu$@%eZa(u_9j#>Jt21Zj`QFE-J z!5Y^avnTs^sh8hJWoWC~E?2Is+f$Ax8=C4L($)G4nfv@y6d-g8xR=gh=cxciFi%Dj zT`LgFb~bv_m`;hQXxg~(aU#Yaw%`s8gK$_Bz=~3%Ae0>S#H*cZh;Cv#)i%9`NDFyV z6WdpcCTCFlWSZj0Ym_SvJlE?FkGBo^*2H5&(|;qMvt$=B9?ZZOioBHy9%s$<@Wt$i1~_89UaE+HXd*g9sqCXQi$m~n31@Rg`=GoUEE^w2<4MTnoSytR&Q#q zdW}_@e~xY0(ueL&p7v+Y29zXZ1&6QrT<^g7pB|Gt$D|)iFZT>y39yP}m{b?;qudQ=(Em9Bd$2LQ@_5_qc9*Qr;kv!IA?HBzjMKx`7TkBkY zs{`0;ESG%!_h3Ii%9L7FHF0LM-fbGy+g1zRL_itL zuYt|q^E%5AtXa4W*ai&$I~XGsb?}0EV&bY@08m>562V^@X%3qR6yFq6L4Di4m#@Q* zNfaO2j`T3}f(QzlAzE2X;8N0mLD9-XA@U--D2Fv`ED0H4>pCxz3@1$@Wg{z`7+ACl zq3Mj{T{>QiJXSpZFqrn?dk42EL^+LXUB>L;T#e-H9_BzWU^3@ALZ0GtZ>? z#|;EhOBlLt@t{QgUvnsRBo)(k+NQ{(>T+J z4XPKRI_Zq;A)lq)f_(__=o+Ojr6*L)55^n-NqPdiaqb-$E)-*8cs%7|M^RSswFaWs zI?!)X8X`a)Ajb1tjRE zTb-Sdm()VLbg)g^QDP&=_7*p~*?a6Xz)zT+99yoRxikFZ| z&^r3*gJTSpmTk|jdve0WOI3cx{j|RU#tj7P04z2VnK4u=NUu<&nc7_Dq?a#)p}V|v z9IUwY@kA5z8`C!f9cJ#SqF2g*F9st>6l@zY0^X&i?Q=yQYG-Ho;HTOkZizNUjb={6 zdZe8Kt|)}DGImzb(-J9}z>>YwD2p%zXLK*|(IaV7q)Au6Hj5mYhqMuJ9_!M2^32~; z&(3pnR)|y3p3WnJz;t$S7riv6t8-oxKxp0p`iB--q8Y+%T?S%kCoJyn6VnJ2WGYoU6&$qw?@G#iDY;&fWJ%f-CnuhIOFLPE> z42a4au_zHLiTdvcOtAUKbwv5h=sF0Hn1I^LY#O`#ETH#G+Xv8}aL-n3P1DfQ;?ouJ zrNjRA-3yV z6x@nuq29Jzg^pVOlMHTzd5kj;yjM5zNqWSuY<}i9)=*d1mcj{H{y>@S&d4Z@{@Bu? z(-e31(yp;B{g`JtARslfsCXG7YOUCvKr)NzSJ}8X?1hqgzk6RcX;55v8yOhXr5yPh z^v>=K*x$!@{K@#WA1)wi|6iLy;Oah8BqS1q(_D^-tpXco@cZqUZGX&=qn&%5tDXZ??0 zf2z*z*=y`4eQ9lX*pm{9<112s!vR58mwtMcFxI(Kk1w0*HBR&`b&NBM`sZy{<+5*I zT-2J~$SPQ~Px)ohk?+@HoHR>s{2uVKsNb-4&2OITWEc)K*5KOe)}W!GdC>0{oJ?x% zD)n<~H-5fW_>=BjERMJ}Izkc2k*m~uXlo^T*xc@>voPst-!${~MFayuZFqDy7byL~ z?^Vxa6zfy_T()#-S?kT;E$1!3p>*KNXG3usm;eLTW515wymsvy?_A$;$oaK$$=%=G zXqh`l#z6=_NFPehsr=zk^jy7_A1ZWpuUkZ)K6o%GAz=pzrTh2(COhas9l#1K{1vD} ziqWZv4C$h#4PptjEW2j^;%TAdj=B7aZoWG?$xv^3O+W%2J0mJ6VO(HijXoVcz8=LI zbEofy_p8BbN57pn+^f~-i9Zv9m6Z+Dj#A~`1}gXg9~x!}T3dSlJ1qv;m%&aMtAi2# z(Q{`F&}-H$cAjy}Clr$WN%=8p?i`_^TGswL%N%&{qck1Xt+NJY%E(!GZM7RJwrMAf zT62BP``lz257OclWQI%tQ@g!-{n{(vyscpmYH4!T{?i+1AYyLZh{o#Jdryspd(C?o zsD+uBY~6L&<^#H^-g6^&ahP7`=hq#{q({i*h&jDie|qLm!Z8b_eWvh5m+#DU`U<&kDXn3=3$6Y3bhC(X5b~4wiGrGG}zYJ zmSTV&37uL5b@o_M0s%-X#Ku`hJLMzcWHl{jeQ8l91Bw6rOn4*MKPbHuHEjyLD_Dm% zWg*#FN2LA$6txy>2S{?3WJns!cyrHJ^Nuib6weB#@2rvXaU5^} zX#wkS3()}%Usz5(dkQ3y=ciT)PY7!sMRa>1{;xlMLQNGgRG6Cl!BL3$NvonPR1C#N z9F887(8&s95Xl?94~?TrM zH_z@EwiKisSXcBJAdEPx?cqcu+)o`a- zMirIf3vuC!Z;7eq2`BB{*Q`9v0I0B3;KyFYw1}WC%t)W2H>^EK+(DUPJ3$} zWkq6sCzlwNKNIB$2YkwtbWgQa_tA_FZYuk4Zi7rO!*m7(X&XMu8)y;INPVIzy2%#i z_wi%HU0Q?M_sW^;6v5z@c>rnsI%N5Vr!Aloag8*sVQ^_HaCRVHsLsWo5N&5D)vpZbWQ)AfB)lQ zv)}rndoG^%hq}QdM?R-$2N}#_^-3TzJwe%{R=w=>=X1Wixj9mw)nNYY)1&iU4{L+D z#$j}G4{4M3ByKk@`nB#$n_dUQx~k>>OfpQ-U)Y*TS4w=Usc1g0eX~Cu=K~|9GeOz5 z>S{mgoyfI}+c!y5y=H0~2p~}P(|QHg8@*5@Mri&$e8V7VOu#=mrCZ5RPamWIaR)|_ z&IuH=#f3aI^+Hp^h%9AduDa{A(hC8Yp~(6}X%CsNrZze=+URPx_okW!O*IX5A>1hA z-s;?GY2MJ$#{UnUR{nh&A#>rBL%O?6u}1?B#`7;Pb;wH#G(G6fUYyit$&zSAIf>Ms z&)0XP*iCQ9J`Pq*Z{A#O5O(RuH9=eD1M}R5(8MPXuUS>-l@&x$(O_GP#^c5e==i+s z_Hemyba7%j0*xV)HH7BNnSdX%Lu_1bdL={oCx8)%6kno~z=889HU#YeQP@tCA%ibA z&WvM-I#XCYq5*^MBd(+isRSq{+z`r#ap$hHJdb{-JNswBa?o^4Xt~d73zO-?p^6GcXcl+=0jl8q8{@QQJ<=;z8`~9xDMfsLSA01>!NLqe|4wZPA zaD|;r{8K!olR`!-kq2#}sd>oxjkE!i(1`|ZYz(7?58?z7jWLXfRzLC#L*WCd2dsp= z1v%XI`e(YenOV?ayu)Pc+_`f$k=G)u(@L|>me0T0X{+mab*EnYi#*;>nRPYyW|88N zMs?%3evdX=7Cd~y+V%{;<%7@KS7xpl*QB$}$KvGWi*%Mg_|c~>(y}3Z(^EvJ^_$w~EL%2vbf8Axn`ql(Hn-SSrgEred?+yll*cI=)hU%36qxDi#6vea7i( z;VCVn0(gDMiC8Xe7+j7G7z6z+Lj=*dxlGp(PW zVjFBm`81E289aY27*-ObhBY=;&q61JR*MM9f3*?`2%NVxymAJHP*<}ySF)tEnj#Fn*~*!R7W*0qL(IWn_ewK;!o|;40_89|7^BH2+M&YM3JQ!eM*T5A)6i5mY(HCUS*mK!^ zHc}nn!4zc4q(3N#r|7>)Q>oKG-#~e6Fz2BWRv#9KIl)uAvlir-?}Wm?`*r&;k=w+m z;U$@;EHcuRrKYJVW;NB3G#-Ol8IlTQ4(DNyQk2hQ-(;=G+;8y8A<$PSK$?o_wEZ%0 z;NM}?lT)QMZQcl6i45Hy_K~x&VXov55N1@<3(zB1kx@CYtzm#Fy7mcfvD)n`v14Z! zp5#H{&EroxyUk1iXxWpOG7bdg{Rb81F}jZY2Q8U#XbpQkjSUg4<3L{=NK(DM@An!M z%g*|kc(kNdI>@b%>CACoA<>R9I!t|5!x&Hw_x>qV8Z5G#2j8$UDhnPflCbP|mjhn6 zZlb9c@8scF`iM9+1&OSDvGf9}k@@UQWuJqTz%sT6SmE z>GuwtnA?Gxv?96Nfz3VKHwxFWa> z%*wanI3jW%bJBZh?#g#Uf#$K33KE$k6`UtnKRZY$_de4pP8Bp8gEkd`jcTJ%#3Z@H zLr)yl9s#*g#`C=Pr)A!|R|w9^iJ-35L2x;4AA##H2=_%HQ=|6|-k;o6IkUy(;`J`C&rj~!b^7(DV${MM zb#~gMsZ;lxO#FPsa0x(e;@SmGPAyCRaHVdSH!sGwy}}guX}!P`YaW+<4!GbsaK^3B z-%oiwZ9ZeHCK1Lt0Q@y9v2K4c?A^7RW|PnL8`YR($bfD2O^7#NIm73pDv zzUZ&(?X4KbY&5cywrR}IwQHA8Ev1ZEe4XWAsoSLMRZgBh-5qvMDkYu-?lRud4>n#o z@2B1Jv8v$wF*wb3V8SC|@RPT75#PEe)FnBrjN?}>KV|dCFg{y8mA)5~ouNE9>2+k$ z64-CP%I143#P=4p2KtIc_tx?tn2$;u>u{5kC@mBvH{k{lmpjw_lcZ3J9J4YNpNdb|s_hb?f7qSG(0euZAu@;+l|lv~!U5#wl+m zGq%usWqS(_{1{pCzVo~f^IpGnfBACLvZAyx=_X!>K2aTB2b**7>AyryJsol{HHPbM zTKf;^L%5d4#@|@fUBYD`-IYfdlmS(N+Np@Qw5&t?~1E;$C`Q>*?gyK|v@{f&&v zZ^6}ukhROLXy^8(Fxet~150QU)tS}Bz~oMm#32dCL1qpf4(I9g?jm4|Y}js;%;|g85i5Z-eGn7h#vHkFybv{6+DMH_dXtf`SyPMdqjwPt^+H*rM!2V*6qyK zp~2m*oHoElqNXt<#S_oY%suV zPdtQ{N0ws~y69Dl+xG|M#vCYix%Tsi*O@oZ_OEgG)6Ln-clYrr+5FJ)OURp`dmm#} z-6{z0CS0jLdCm;!lwhWnB)y(Bt4j&GphvA5HEIyW_ir7DjF@TU z`=?ePa&WF7x~AR!P)8jJwD!|^Dp7}7OhK(>2yxJ$KG z>(p~DGD09cgxpYy2e+c#+nc(nTgt%Zm9aCrHWd<`I!vlj=?Yn8xC73dp-g&QTB5lq_)lbqw>bg2Kpbe z(k2%?yEeD!J?7Db!aQP9fBH#${hiPc6(<$9)m>jpN7N=AT$jjVlKW6#dr2 z$Y>WRjUND+Tbq2gJ`NYYbO#8}Bc05v3=o|Nb7lsZ_qeq|0Yl68lv;-Ihg%1~NE!qV z>^W`Jjxt8$(qp)9;#%%h8h|tQ#>;<^d?Ke2w>Edvh1b&foiyn@oZ65A!+Qj$(>#wn zGHN(;Tjl_4pI#ojDZ|2V9j7|byz&uUv+VcQl)2~piA^ik`|%=pacCTd{~Q;*TE=4* zEs}I2C?~YEw9d!BdIX8lAaI{z_o63;8LNMfoYp<` zrjC)`{eu%HcrJXMz7kU&DOP6<*~q=J8xy%ng@YPeBaxM+XS^b)N*6}?i*?J+u?~Ag z;7ttv{a$%_8KTHK+jjHO?UnY=h&UJT(0(KVS?VG7Zf7Z-=}ysRh_|Si(zu*P#vq5F ziOm8iaUYJx5DozSR<{o=nl?2BJ@IN^vo1mEG1vhFhd-q6C6^5*t1rmZ9LV9~gq!rD zG7ZOJr(Y;-6Hsua&qp>YXY(?wMqxspT?dqPn<@d*peeIHA}{WaV1{8z(fbp@p88n~ z*vD#+DI6Jkq$Yod%|7)scFTK_+4OW+A5{-nL6{R(u53GSGA-(84utxX_?Fmb5VFd@ z=?SC{<|$F=|I7xYkm@14Bdlq5Y0LgMTzej4G?GFubVE62x238ntP~zaj@%!%!&;mG zED^bJp!_w_X#Dh(LI&1^4?`O)Cbf*;-VnJr<3B5i!<$O+1KWgE2zyPA&!d5Q%;pel zATckX`y8;u6>L}rtEA_AH(<}DOY%WbdqYp}d&FM-(meLJ(_?xX9dn=mw4m{-Rfh|n zuHWbQcY))*#9gQ6TSm;Tb7ga2(uY~6xuN@=H(s%c7_~L1&5ES2XKFa79P073lfU+h zg2vw4-d=d~>frFeu4x`{RUIJ42i^$0)TGKfzkw^3b7$5uzDx-tqpStlAqeMAhY=t0Rbq4??tYX17-l95~ib-jHCM!7Xrdh=I5qf_(RM*cesM z3*;}Fqhz_-8529kuyg^@TduxP|M@9fMAO_2?$7Udw-TSw6E+5ybZNn$3!SY%gZnS9 zpkPROI7a3sd-vWBrq};X7(#1#rCzyzLC^?RLMFcH;{V6Cl7lvZMqGAMMO)ISGLJL@ z)?eH_gz=IK8{gYw=~BL0%X|l*5d+60W8{;#Zyj0`u_|=P%vq$!xLv8lhx@35BDrL$#~tW&!XRa^Dp#DQeR&EGl_NV*XfDw6VcY#lgSpYg91pWd9i*GO;Io-fr3=9S!OHvd_{mE`1ir#uzKqxBL2 zG{po2Fr8`7C)wG*BbW(VI$IhdbP_M`pF#%nS`7Hfcfttazy+_kyqAZhvN%hm>cb|3-R}t`~(f&Bx;^oWik-xdy4&Uh(T&3dh+?OkGI`K+{d(@nbBXJ&UB-}`-jso`gLWA!@T zzOU(qHw$hpooIj7zu11ojyHu(cJ~98bQTlt&Ux0S#GZpq;V}E<8TJj z$`_K#)^Le$$r;)&q~3FpRT#}PfBiUQEc`}0HZ!}d{YOhpW0F&5ZMDkuXB=kFY72XIUOKYmPi*M-S6MG8(F-Elqmmo10>xfr^lS2=5>_Ja?%3cwvC|6GqH%+Yfo z&xhpxiVZvY9SS&+i2@NIqd0mZ1TCEXgC|7Qi%bl})>0Y)Qq@JH$$%dG8nd8+E! z7Nh)RxBXW>#&cvPqL+r46&llzSG@Tu0$DWxJi0Rz#u3DC;q~M72_MMUGEn<^il+Km zKi2v6qoVp-H`QNWZ-0YIR^n;mS+lGRew}&afSlGEqjpp8y&-FC&NC)Pu3LZthcV7w zm5)6hVqb66k~Z#3mokyX1Wx&Fk50XRAzvD!la9w%`@es1u3GD_GxkHHCHGf5!jfCS zZb=7(-0>TT+h~xgX4o2Z4-88tPK|b&I!L+0=!``~c_VEWd}?(p;dvFwX;j|CF|gtb zJZG}%3J`x60$~0Hdr7$V&+&s1SbKY*ob7(d_(y%{{DCO2M4mSI|DvFfDAt6stbb~Q7=gk z0?CpCe3{Y^uSdYUv1nlE1@?idQNmVRec|okZvP1!gpTU!24WzL4M}#|hZSX?+aPC< zA|MhsD9WZvbD7%~5}!c#i6UYa8uI`7KI^@@ovQIx%(dUqtyRGSaNt%;;vz1=}qGVE?V87be*SG0C@pTi=EsV$^^cqK*%fML0;fzIfw?{;2=` z{skuCBh936=EFppjJ=|C`cPeR?h5CTi#{^_IP7XQDSx9sJZ0Du04=g+b%f^Tj72!; z4Wy3exK+2;pJcM~hgXPBSWjIkw9$suMrh%(gCP{k328Mc2F2bZ(9n5mLwe$=uWmaT zj{EGM()9a-*8IzsjEUm@Ml%{~z&D{ofMKfvu{{k~BRymd?TMo7#MHlziQhb>)~qZY z9i7gS%A<{rH*WOr-Lmqg*P%<7dXss|o{JDj;LQ0XFR!^nBn%zlzkU1AG2JwSUXPMz z_|p14W|VE?#*HI(U!IChkzx89cl}>$G4Gu+;G^kZ&5$*y_Mw|oKyf`77eLXgKnu~> zL8d@3_m^?rF_TPxSS_aG?vPwX27yP2>(z7cdMLI{DCfl7Rm=$EUQ;(Oe0g^Zg`>uZ zKR_=TcGCV`5=kp(5#`>bEhv9+_nc+j8m@#&0cC0}svNP-PLC%``+x-a@iX$yG$nEl z(WIhVxBU%w_}6dR^d~Ql%Z-ayo9ovw)(v z8%MwJi5DsMUN9C7sl~upEiayL3L}HiJtEkkkGza2PI{WN@_ZS56R*3!)YQ+TdF0x2HH_SlyF%UTwx8are3SWRq zQoOpHiin6<)M=d@MQ*SoauEdnG-gT^)M3y5^F`F`K*5ZcfBiIL)6jdmPlXd_M8Gm` zrY$3q%myhS*lP5ZHK24r0hiDhX1G9yKDProonVXn-rwzKp(%C+ti4#5jMr-&{$1s%-oh9F+kqGT z8YO2GNUs|`pl9>VFc^P5J7@s@02y+M&Joo|Fl7MM>=)bG_y?NUTMO0?=(?YJFeOh; zy9Yg=W<47EXy?r?urRW|2u_^GVodvmC*w@3di`#gZB^Aox8sle7XAiF_kf)U=)KOJ z8%Sq=k zaL;L|IK@<}4O7uU>=5y@L&|9f`2XQU*!-E}xOl6Te62!A!1*8pP4gW-e~~Ge{A{o$ z(dRH|$5C<(ktQyPZMxx{&8Orp1p1b}=Xz0-%Oo&UQ&e$nVZOzNk|mkg{2p0|Av>{A zMz)1p`Uct_@QX8q_XZ8i#UzqW)PWD+9DN3qtY;(2DTxZWhYG_h=}YO!=W~kQP9cHB z$3H(?&}U2((ER}8??b63ZZVWoXRh8R>TStjNPj=8Y+2!rUUGT?m=lVMhM?+XWCIxaRi;BE^_6wBXn(_$hcZf5LAH9zO+UPpvrf#}lk>y9 z4ojmGLhHSkOqw2ptU$)8VgP`Jw*&~wJhdP+ZU7J)+dUP|>QQoPB3gX6n5(ns*G1r+ z$;q6WG5?$ccwvh3865uYaf{LT|Ji2X-w>*(Tr6U5c2N+v_oKk^diA`C*NKN7wHS$^ zwG9qgV`7$8!Ru=Bvf7Y)J>$&pUCr1*&5Yc-QWVtKORbC%*EV*okn!}Z;+}(HO_jYY zH|8!|PMS0c2v?4L;{Gg-_1vWiJ%7-u4g35>CeQG36Fgh-0<2nc2izC#IG|Cr!ClNX zu+yG7oPjx*WQQY$087&01Baf=yRGWJiN2sBEo~VircKuadu(K%U&#c@UL`RJ`IflJ z;rdeg`~z?=y2&dSFRms;*tI!_I!??l0YNNL^vjVC@E@qED@{g?BPcM+-BhOV$=f1( zL$+->f45Y%3Da5SQF}&6Epe4~FJ@xoO%abXMY@tKQu*zC88J?JabFS4?eF%+KVs!|0+C_h;iLysu zAhXyQt|t{7r0KoqxXkm;+V2iYw3&JW7Sd@Nq5lA|TDRlCqWvfjRC!a*jQ+P}bGE0` zVQF4O*bi%SC2KaHfO?<#e}UAX`Ljs{FBv9h3*h83jF}tz-sR_UknZ+<_ZCnUinp<# zI_w-FCa1bOM30q|FmOqXPdt=T?D+f|_g9)%@z<4#E9qZMuy=)&V(mzQACV%9#uC8EWV17NJxvVOA zE?4#c;fbZPtcy`ayx{Z|o-V*M$y0&;^cM$Dn@?;<5TDM`+P%a-M|`p)>XmPdE&i8^ zzetMIyuxmRahuTmHtRKeLPFNg8`Bk=u51rAfB7;fverz49rhi*Kji&e%f-C3d^dJ* zq#v$eGGQ2fc6uYg#y0chAjB%iKlmnvd zXI6AfN%_X#zd%72sOhFfF6%;030+90O6Ib)CKAB;{WJkg%jRg6LZ^HLV&rUcg@3^GW4i}7KR zN-+@}pvwtu9)ap^_lmGLo`C;MUm6@*7DUw8WH*PeEim>oUvSZ~SrEVim z^rQ>8)QMmHzQijQs_A}#s43_wTUc0#R~s_s$VFnr2aPSVW?&&XP=GcTx^kl2hW{H( zq7)}Ysi3I1+f#)QmdB47DitqxT*k_1nn#hYL~!QhuGUv>LP_ zVo^Zdkj?ZR5cduWT&b&sJT*oT%GR60H$@t+_jfV8koI%uWb=^!+kG7*eH_}X-`Rc& zRxC=cEdYreQDjb^gdK(h@=qkS3Q_#x zRf3`~e3i9T1PvCi@5h}u;qw8s&GlIO60*41m*EE?q7xCKoEkaIBxh?^hUJjAAyPQT zNVFs|^z~uLJmN-Bf-`VXnUY#m755X|{g|M=-0y!Md6LU4WJuKde==zhgY;d>s~T;V zDZp@Bp(5@%5HDk~_m_Z)RCXh!w_q#MzN79#?A5D}g^Ei7L2n1@O2D}^mSM(sA3o?1 z4%5>bH)zl$Y|rsI6)~&gTT?$pm3=^gM+nqt6AYGqF8kc~DX~u~B;4=rkL!u~zqrB? zmn+HSD}0`5$;yrEmEBMZXS#xG-es+rzHVz;S@+*Zq}&4h2eI3L;ca-BhvSx04bwpG zj{_xW&n4a5+~k)nISipRmfgv^afsYQ%x2)=5kiB?L0|T)AY+Tt3Sp?6wM2V5jTivI z@!GcTM3bBYfUr5&x2}8SG6tQc`s$A#z7lCQPw6LUnYO?~%x0!w4l4fd0Mw`@b6{NK zHb@&HHxWw#kJ$+nV;*!0GboQp_;T12Omc7>IY6d&2;vMSc(qs6R4`^Dhdv4G1!WsZ z_dXx@AL_VCPn~%aLtqE4sW>NbMYpU2yN+ya*LvWSL<|4-TLWMGTrs6eDqOk$^{O+B zwP3=|5?;ep)mBPO#Vur$#Bj$_sRY>_vY0yA)483&gTS9QL0Wn_^#Nu4eto;MthdgS z&bpSCwLFixlo!3BwfLzU88xMw7TE%E@5uX>3|*~C`MY($rRy3M`XMP%E`RS$XHfo9 zT9S_0+eeG$LHOue?p5G$w<)@mHX9*BT<7fFx7;YRJm?MJr>Wl)u`-e=0Ai6GNOezo zW#;c5<>or;Ut?;7YmYv>tA1L>ZJ!}l-qLu8mjmL6Cs-&Uh2r$PN^)52GEQztnTivXP!;R@WG%%E{-GCiT zE^l0Kn#$PvV?>{O(Jad-P=&tw=U?@l{Cwps5xSpbcCzb}IykW}4dSEyXwKgW^1|FC z556SaVwiHYpoC;+x(LmQdhm2)Z1Hdg^OUi8?#>k*8Kg)Lspcg|Ie|X0ZxWe>urJbG zZ`$!kgD?~EnU=Ewh(>Za7&D1OT;w=dIEUa71&O_Q#Ol#NFJbm)`U>5Ku4WQh#@iYW&{i01JpDif?igyx%+=DgkrGrnBmO-bG7{cP)D%8Q5J z0IvE1u5l|?tZ@0vRK{G|RT++Tv4i-mgm=c{POVO_51IW#R#t`JrBh{Y_LVxKA0~|< z3+Ooum4JBB$;hY;dq$~Wj@dbpAa}hwTl}OJe{F1%$tIUN$?}+SCz9V zI3KkltTG%xRk?uE){l=k4d*`!e-~s7L5EoT?tQ+u7y=6Lc}K!{G-B8-!&y8Vq(-6q zQqNgpn9g(L5SLK~o2^Vz8~l*km6miLJCHmwvy#JBm2qF3w}u+; zvhGJ4KNG7p=tU~a5Cy1}9IOG91WF8~!99Z^OqF7qm4&-Hp0sX@7M3YGHD6 zt#g|Prmv?YxrowT>`8!5s#q-V1#!mVbK6Fo240Xucko^gYd9bMkG*muWYN94VDTlujkkqm4) zeYzhPdCa*NL$RV7)wiqkh+X~&+6!n!G+qGwC_knl&N-52XrgS)RG(=}iJv?l*)iah z7M7k&UMXaj8A!lGTTII|Yhhda1Fr@iK9eXQ>=4SXRrEm%@OZ{YW+afZE(g%Xvi0IN zMz@rDUJe!HaKEwFrkdH;|HVBJjAZH*M%-wrpp3oe-3)o2Rxu%Gns=a7-d3p}!wtw`-uWMKhTy$8Za7-0w}oMKo~@CVU&^3map0f5n8QKD=F0q0@o( zzo<5_OB$?Z33Xlm#SJLm_t3XFEmaGv)UVlDOD9kyJ=woChyz39Ce%h!B=ZciIuuk4 zT0~4HGg2`flgXVKp5RUDdGCKHT?^t<%q)IWTM>Mf)kW$<@{Jp?c<$Us;D`Hwv6vFi zJyrC>Z~T@#s8jz9reM{?vbl+tlH0|0&xklQ{Jmy4JvF|6ZkX z1a*2s`liK`XUt(!eOUqH>-GIp567mL)!zi1XEn6JP`mLoL$bb1VG0D=UnQ%iru(g zS?zn|Nf0C75HvMniYNUn=N%}}eqRUEKk4@FH@ZEo4jqzoUQ16(<=LpC&3a0FY`gTu znh-Ei-7>UX*dTVtPM@n+T;Vp{KyQK2i+_{TB4pH#TpGY>_bYd46@O{kG>JMv-0caR z0^X~h{Gz!$Q!66&IX9+%8FXvAx_X@_l)v4MkB8#~wUpxn(qqJRi^PAEt&p!h%L^Grsoq2d4(@Yn}Ti2A{Td9OBC`# zBOKdsljGX>qU{U}3bt;xGf{>)9jLHa-O@w`h>5N;Kuj5CzHlHoL>J(nCsaom?yLqW zJ$q@ChMi1`$a(T)YM~ofMadx!!7iEH0q#Lvs{O|it%KGlbvp39FSS1}WJ}$;I=C(f zvO%|8MUIXX^*QBmcA5ow$2oId!x`=h#ojI|lR-8y+6pdM4uBS7aQgTfng_4yPi7dx zHKDH5=2&N9CVq}}6#@sdWHaLzb;7P;$FPpSsbjwvIt7(!9OMK>ey_5epP`Tube3)Z zUb=0G(-f7dp%!!War$m5XBF$|4X*%L!fXJ{#5{mMeh#y>7kn3TY&Bo?Hq8z*tdoz; zU$x$J-1j>pRe1qXHU^PCPJiuhlHtxeyoNT2YS)_NXC?Fwm_VcC@n;hiOj$KZ97d1k+gCC+_^Htr4V&4jej)eNn+#0>J=7>;;Z^2vVe36 zd`KZ{T<7`!tXvbskC&Juo4$5697uQOeH7gPD?C+n-ocq=4sXt8hj%V3^BLHuhUFhs z<~~ZF>+ssv?umWaH0Qs40-s&ivp!~JeJnT0s@4c|{Tn-~EC~#5^s#4q?<#`^4H@*I z&zG-npMM(qsB+zP?@sqSNn;cnnRZ(H#8 z?Ja8h2+*3Wyk8Z(LQ=S~MOiLK>{eYt9-2XIY|CgE|F96zrn-o}WHhTH4ZNwv!di;- zAss5o?|Ch>fn1|o>zF*27CH^V@Vaz2509BVtI@A(^ApBHnS|Jb%jsbzXgjuTI|%$Z zg*of@c@am<)YEFLt-*<87<22UAps;S(1UzcsS?RU7OsMsT-^Lix)3{QNoXhp+pQw1j@OMq;H4oMCR2aQ@bd}Hr!WL z^V>nj%a$cXjW@;QHp$5_E545WsV)u|1s^^v-yg>&wkY6e-fo$*e^&U~{$i&2rANP9 ze`b=|?Dd?`z|2MkdWT%btgflU4bXdz9E@t&AFk>kL0jkZ$Lqn*5KjSjbe?#2ncTxM zfQSb1p2liNgB^}Ncg}k7$B^l@t*UsLRNTa2W`_Tig{SxTd0G4T>t|)kzyrbO{oUTNLLT9ip@Fc)YfWULl1G zpji-1OnY=Ni{3n($XhLcLVWPLbF{nZ;_CgzpLo}0!|~Z%+@mmUrP2vg9oBO|$mb(f z5N}<}$QT2Dxcp^8ijzP4k?~ZeigaU19*c!aFM5%3ar(!#yAoY{c2f2>eKT@k=hfX7 zBs}=FO_QkHH|ep9S}1&n>y&J&p-_D6F)Pc4qaa3c1DTu;VbeO3DAXaBE}5J`dJ;Im zGoCh48{Ftqaj_Gj%)+pq=5ZemX>tCM55r95nFy=+>*+^UK%O;+@)q%hbGxfG5-Cm@k#kU6tGu;g7{WgZ&_t)EJn zW|QDHPxW#2Pi>aHU5Ph!@m=nZyzV{(2{=q!PCado)~-29Tv~GV|qBY1$?KQ~K(id1=&qq59JtN~QzSQM~|Gp<#Z6Sk}tOQ!kk(?y6 zEZkj{6*Fxk9v6q8Ji<*jG-#IASD|n-%bXdyM*1c)l<&fUtv+4velz6c-~aqGiO`=x z0N9G%TH4`>dYk zbCKNjPoF-i)btHs?|!1%&qSjyFSiDxMva2t+qP+_gLfhBIN0KEL7zEI58qC(42yYZ zLPJAsORA3j%1@6DTS!3}zFx{AAp z$5lszT-v%jAjORuH2BRgIRpu$6L7@*`SYzzOx6(Ef^t0g&U8T5Zf?cDSE*|}^fWUT z1O%PckIQ<_U%X_=7)#4r4R4sI()`?E1Ydvr@!{aW*G@iTC4@U}v>d!}VJz0mM@TC9 zunvGcs(BB%q#n2!pq+H4?ejoAvj}RMvT6j7f(=L$I;2y|J^M$DsH4z&#X&>+Fl{ka zgIFJ8{{96Ad2hWi`4)QaG_0d`_ntSHg_6!n;sD|S*Lr&T>|}>26k6T0;ul4rR*T?i z=dtVaZtbHwYaW95Vj_dJeHXvn&e5tJ(3)4PUOgOzbo1`hHz7lhB%tdoofwN9Oh>d3 zE2}zo1PHc~GyoaQ34oJ`Oc}hX^p@=dCt*2C6CF;s)DbfbhQHMzeANhw+McJoC=_q! zX8z{i6(=A*WhOkd2Z%?T_}Ku*>JTc8Jf=lsH=S_4JrpPB6cU3V@ta%rF&NQI_0kc7b?eUNZQ04-#2IO-NQP+!aswoskJM`L^ao*5+%+11?`e}bJ=q0)O zM;^#hIB~O@UBJ^bE0R!y0G3jK_ zz$XbujvS!|`ka(8ywZ3CJ*%Xwx4FSN%!4`K_~>XCh2s3rF$+6fAcX)qcAOX=i&?PW zu*3a5-XChvu;J5Juk@MRb1Y;6ODzrq$*w)g$06a>6}~-zeD9z;T*Ukg+wu_6RMG^j zlb(xAO7b98Ac42AO%0{%);@J5oBhBmDH8Q%5gDG!Y`jPAfo1Oa2O=Q$azMrNG5Q7BCC1r5-DN0)q$(OKPR6=ARajAQE@k&kmfWW zu9i6c#$iN)4R4Hgz;?KI=hsxm{Z8%!Ut-ty)i+oR(&bXXG79{zlE3g-MF*+<7F+ zIE|X72yM|-B$St3K9!W@p%mRi!Z8TV%dEd1GqBs1-+F zEg3xcuirC_+*V(wFiPv;GoPuFMu}~Xudb|c95#Ky{+F0GEXy=&dvjI%;vU;4SC;EK zuY)z?A=$eOz>cPKYt*#qAyB{w?%P$r_C8Y`@Nw|%ZnzTR@qpi7BETuM&&(A_tefYbrX`ndo4YX1Fk9B2F_xSp1E&aT%%o{la$2)GzX}E?jmU+4RO`~9hK<0S< z9RODiCfvw5AF?=P33CKI%?cILT zM5v5~7wn6qnChTZDl;?ARjH|dPoZ>`PUqKG5@L0)s^?sUZUSd|1vj5LX2OJBN;%Fb z>-3T%+94E#(;9c4fb;f)n5vDx&Y~JohX5Xozan9e;<5}8^Ef6|Y&hl%!cEc&UwJwe=$2fKn z8E?VIj~4GDm~d5xkQuMC*2v5L%4W5z7rnYSMop&B!c8oF{uw40#{NBaX@1lRz3ecG z4TYN(57U+`X+zPS7Bzm1K=1}{o<9$L9<+2MVb6vKVhDqc;eyWTKN&)Y0FD58nQoJR zz9Dv*=b&f~N<#z>BToStR|M~2rbdB#fLn0w+BI|5Y98tuDGSR!1RjU z1}Tor)U?r&s9%$~^S}vdSP|%8{Xzz`RL3%TnigqiO)A7dVx8P)NrC*xas4?prSV`< zHSRLXtnGzC$7hpY6-kzM2L8U|IKd``zVWw0=mDu7y<31U=qYv&&PtdAUn z%6YIE1)^yz!KowX&6^|K_k0%T{6&l6s@8{5JHL3gJ*T!=XR1r(aCo7+hkr|t#6 zdbM8+&Yu=$J&b7G5q1%wnaKXyO)~|Y&)YIB3R(dL| zYB+q#wZz0>e6}s^#CuBgLx`1nCUhcahdT7!*|=abImf+kq7feQ)>%;ueCwtifKTQtRBFyl5-*^+Hm$n;=S~e zrQ;emXX&c=T}c1&m*=ux7@Q&pFaKP(>s5XQwSI(e7E6Zweo%EPq(Mb zo}t*?!sF6xH?>WO=`T0-44gzs;u{^|eqr^Ow$j&ZoU zC}91;&0hU;Zjw3{v#Uj~@7p1Y^rK;c@3ILVT8A!egxUs~sN?gNiHV7Yo)t}{<1d+= z{kdG7!4B?N`gH1MNqFXJ5%ev3acdCE`Pj72 z=N*b(W?EAwSeu%zmmJBi&IKKtd28=VWiHYLaavkLzen`KB7T}CT<+~oJSydXG8DW; z3P*C!5Slz0O)n1uT$m!Lz6RuI0T>d1R32-(CR4|3D zs8mrX-2ugVnwQs+X8J}@|M3V3{1K?LsvNG3udFU+E+h_bbl@X%#zfOeJ96)cBO}%C z5x?hiWn-xQZ8@05lhR%KGj33ORk(HNpLk*%mTX3q$4KKrshe2~_X>{7CG#?ByL5an z{^{GWv^d*m{(=S85Lw`y`>AGu&v5geHWTe zKbATF42ygc*ls#iTTWiyOuwa-t*p2peY=>PeTCGD8GOe!?weWV;XLEq2hBAfukfmE zEKtn(>e416gf=D7-UjlrVO6tNEx=U#l~R?1scRZ9_^al)hn69rxkfAP}ps9xmZf=w%3WaMDXYNJM*)^cEua*jMAvdkTTN@27 zr9{{_{)F1(KLwaVG1)=$dy(rceA=Bh{}E+6YjSMi`;1$Smb9(-2;XQ8z&5thp4P)g zI?V8WS$pyE+1X|0v5PA5rsC^B%@-PdJ~Xl7cNAK(8z$Q>^JrP|t$qEbsc*je_P_Z< zua|#VYO2ACA2|B6=1nU8OL4yAkFcIX5utg7kN~KOLW=4Snl%Wk;+#-u~#3qB^oa9sm-VOtgA2W#~^ z+DVCEKucN2X81^O0jIw9;K4)zYuS7O-#&83eQsVX=2~!+5mbq)XJ9Z4{B*NDJi(6o zOEmcsB~%|L%|UfBI53D>vn(CnH<&PUm0ZX;1q#PK5}0UK2gG*n-Fqx%BL_LmDd*)& zJxbBIS@%X0NX#?JuXX+^!?g{l#|dLfdIR>me)4^w98O)|3%!M;#6trlxPXzUL)g^^ zo=S-Z^8D*uE=*8Ul19vl6BY(1IF&$^${{(wp*vJrAu@T7RM^byo=XitPmgz(oAlsB zvE6+Z4At>daUw$e(vOExB=1EPG84cjIS;ZUoXRbaR%$e*07|8(7cZQmCY~NGb&8mw zXvzyZyGG7GYvcLaiC&v$&*34%4NjnMR#^c@_-*Rtbjw(s`o&oTz$kn_2~MoGLD}NC z0Ljf(5)mSR07s+#_WfW8tdVfNr<=`Zd3gW%=V+>E^wCj}|K@6p755Rlm0>Lm9M{X` z@m+IK4am=dDH4H0w{LGMp$oXz`LO23XoZ?ucX-UYQNLbM^5`7;13i^=FY0$FqD`MH z8zOg4F@M#%GmGoDY88c!^bmspL}hb>ZLN>ALcKaGGiC^otQvs?P&AEEC|!UPIxCn3 z%i+T}ae1@CSmfR)cl3E^r5IHJt_LU^>yNXc&K-^Ftx^5@dqr0grdiBhyObmLt)Wsk zS9A1TBLvUQd^T@RBYE1=xNOz4Gf;_(gtm&}diXU>dq5rW=t6XD(w~{Z@<-<@Q4o@x zI@xWGo74|~R9ZQ82xFDwuOO-rz!=|t)0;+x!`02fQz*Aas0F$*Ow)W@U%wDte;K{B z4YL{|2srijv}JT8fCXm9OZO|QRjZ~lD={Ca`RSw}##Fk{qA^=ClRO^qj>#EQv*j;? z{l6Y*1ryo9$Y=tZq1*IHV6FWgob?Sq6s6UOZ+-yn`+$eY??PJ+yd z<}w5`8nzAjw7wwFgaHo>Yt?BXA6;t5Z>cSfxr+A!k}$ibis@`qR+{CJ2@|O$o zchZfnuFh`rwNp7BVzE%ejVKKK3}5+csTfyDqcGIgR0}j&TiJrr&olj?N~hDA$tMa5 zDiC=IpfWG5TH8tZ_HK`>afX_ta+E1*KR>@`H}@P7{LpSGia*(L_>)*t>fC<%i5(IF zYMbn2Xr=}%JV=QFV5pTG&q7wI<#fXgPUkwI)a~21FW{fNa+J;wbuaw=_ej)>qIrG~ za10QYe_)%v6-pF!6qrvcKq%!+I41ESMc*QXuhfC}Pgx7XEzgD8CvNtGl6|LZeWBg~ z)?60fzl_RETSY`bTb{wQ>Vdfgt-Udo?ZKnz{%ln1#n&%h zTx_YiHmzgk3pzYr3GyX-(`N!E$opw}LNdVrd3ovm1${**otYJz+b%=uDJuA5X7z~S z+>={0Uxzp!npim!K64|i=x-|NH5xw`2Cn|{Q+W7C0y-*(ZB#9m*<6CpSfS09ts zpl#*(DDx8QZz1sU_!IkxU}kFO#U?jf`gY1jO01Bvg_9H4B0BH+{eI>)FhgY*DoqU4 zcrvK~KFyjgu<@_I2DIK*9=Po8%hgfT3GgKUpZ(GU*$z7~!U*@aC%-=@a5J~_n@aVk zuTy5sflWh$0fP^aUD}7?GaG<9wu4HL-rF{sD#LiB`kCb_`eovu4r;z@b!&ot39cjj z!`-`g4SPvVrBK9Z>PcGGzAL^aOw%KYfK1$`S&J&zMYyygffSK?@_Fd-^RoXCj0Peu zf(2RfV9!#RvsFg;*jMDC1$aAeZZptcEu?22gPL^Vrv$&g zvAx^x;9Du2*~UqJrxtoXT(hRC{-m=L%YP$4=Xc=NN@v7b-_kh(J8P>T zQZvA%;rco=8{g-_?=`7~G$19O;y@0oY(chMON2~k>$G1-C z%mhP~ivKUt%hB$yu4L5lku`fP!a5UY-yHw(t!k=(^s9(RTEB80z^E?)edS%DXJh0#WoCANc1Sw?|}TC_Qs^G+x~6{#L2! zzDP5BT3gi1X$tkFDkug=P3NmM`t{d?lB$J>JIoY_6GosN8T(Qfv2IR$$-Hr61Z=J? zcWx_&mv{;0|09igDJe7pUly&TGB8>=Qf=nnVITS5(R798CY|q_sx7nHhhS^_r|mj% z;slX(i+l@HwEj}qNIDQ6Rv0ho*m(TQU-sa5y?sQ;Urc!yWAC$NUze_3w=SPeyp6N< zgl}Ti>5`Ygz5*5@x4T2|>9G33UMeQLE67BSWY{kq@z1it%{1{KQ#L)EQ%!iz0sOZ9 z?3z7eIdUh;{!B^OGt@wlC^E*LE>9klhfu4br3s^q4+}4S)Ba>7qa7fDV^XGXM*O5q?h@I4)`om+Lr-`#qw;!zHgL>>l8mRd9Op;K8dn zN-~T&DHnA2s=OK3x94dp; z2M!f7hvYFmVy`e-Ng>)1MpbSjSrsm8uV8uV_b8HP67hTGw`!_4bNP=d}rQrSh;W4|vA~6mpg@>N}{UK-^B#Q2xewsLqX{_eVp0%AO%?bRzl^T(?oSLWQ7lxPy=vP$kof0W>24i5Rsf zy@l0zBX49kX!7K}K+BShX@Ui%+s0B?=2BWnq({rCp8ASDBj?$(0LSa39u5IL?x9)2 z1mKo5jCQ8f?y=O5;mO<_hTgiTzhlSa_QPknX(m66iI7H0_tY z8`9z*DQaR?HPAMD0Y}$&S@BU>5j_1|DX=$glr+}^4H@65%jI!&BqDHahUxoZ@K%HU zi{wl2%qu(hJ=<2BYfGo_kK$+Z*}#Hbu%B!=B&$#F-XR}vSL$OX1&KT%v}f=_qqwKi zK|w7K7*z_h94YmCzZe`EY|y|T9eRy7PkKV8lzRkrfZv!2H4zDB=N)VXOX&oJE1EWI z?AYx>qs_W|bdX?4eC$FTbFVEUm$Bpa0dizL8ST4aLi|D%^EL1QTaq26nY86#O9b=D z=~0^Slwd6!aZ3%^aeu%#Zfg_)$*)lV)FXdGO{8g5rxMfYGu zeColFuHfgK6}=mTzlh{~@5y4{nuwmw(Hf^pOZ8+K-%P;tIC$&T{mRjZbam%_Ye=$T z#M5~9!h<27D^0dNZH$zIm*QQUNTi3%hh2U6F8_ zbn&4idX-g7tJB%hWSqDhn$?5fq>5{l(TX_9@b%qh+{%rIqH&z25sivmDIDNoxiT8&3IDf{o>!5S5?O zg(T6Qfrrh_x(^j_SXK^{5%uaySKV@0i1YvMKIWcl)hKxPE(M)rqsEQ3Yt&em*gE;d zjGL{w0qvW$=rPm(k9utmV-6)WJVZjeYctuE_w-h-q5jH+`n3|oGm^7RQ#qm2kT2p% z`zv_ze50Q<>pLkhwWV1x1962&^lJHXQC4Ds8j; z+f_3E$lTST-^J2B+pZtE^?`ve2~FmCU*IXbP^FzE4h`Idg`FKlPZlV>c`d zeOZ@)veM&*o;--8Q<8+#8ASN89CfyZ9CUT$M$Kh8&U3D-&0wL493U0js7lSUbVkR? zSmTk026K?Z;0~3Fx0w`{9+~-|FzN5f`Ln-T?6>P%;&KR61HneUT$3CPXb$eh*V%ae z+=kl+Z@-uHCYg2SgZQ|jo#jx_XupG+hbNu3^eBmm&o|5j z={&m z&@LLZPPTt)BeRp7R;LVuG`CpR*=fsfLzF@v$Op)KrY8NUp1RQ@_rS*DlTIU6oH!VD z&$zG0+pG(#GpYnt{Z_YlhUD6Y)=2DO?qRHvCjDvcZo27%K~}%J@pV+{ zBMRfIBLb<0Cl59%*1l`IKrR-r=Ae8_V7xo+K&^B%Ce?_lF1Wi5~p4{g!E z4Kh1`{$ZEWnPIn*96t*|(Of5zp3xbkk@{4r+6s|8#^7P4;LW~eLaWR*~6c0 zWn!}n*+mp?nVELv;H)!CPf+i7R`G+|#jWqz70p!|r*r1|oKG+Wq8pO!MKTfQ4D?|W z$3My90jJVMi=<}TH}s^K==r+5NDf&CGN%k<6s{wlg(Aa2`Y@C5p9T4;mNQ*RBWJj5 z)N($3%nw# zQk=7*P?jEoMLj;dq2_8rf*01(HdSAkY)K;~VQt-E#?K-1j$sVZmPg#B{~nE?Lg6@{ z^JVhxMhBxYD!!%BvOwxcdWu6XSLJkhsL#FOn_}Tc5R#}!qL?^zV0qz|R9Q?oaKQjq zxS1ZM`EW18t8f1U87DRS{|jK2Zg0(+HKF6?80<(b5wzRuhOja;{-O%PeKU-#AK{d; zUmbGy?%1N-+letBB<#~=e!Wzr@ZGJ{(HT(WQVDv(OcM?w&` z4Mmz2FDyN6#mLkI4Yc5UoBZsAPF%!zN&%E$>?7eR4+_IUrpkMG=Cm?7BjaOOzUDa- zqfdhj$^;Bjb5q+WP)i9nUc+5^7g_C`fFckZGSmCw1Zsp-%S=s~(qr;@FUKN^?R2mO z$2oJ3kYvJ;c_T@DYXbGXGiM%DFbRH7aq-&N^eV=q-$sn37Kn-wo)F#(9Zog5jSV3RviN8?f zk+UUN8<@GhRV5=sv;y!PY30&$XAj4u`qFO+53h6bg@+(EXqBq8? zlrY$x;9D(hmf|Bf@r;KpzRl5_7QM={B}CbpnwrLcs2Q?O_%wKif}$c@Dp5JIV}?_I zg<~~}N`D_8pR*hRqRfUUrv_COMf++Vww6l;3Y``0untG1J&s?X(mGm&{B%#R%K3AR zbgfMIh%_>ds6^$UoaX&KVx>S*G1|BvS;Rt)JP@!1GvPZRD2>pSsY<9I2{qkk{d0Dr z;ALM+^1C2pGi5KGcsi?+E@AW-Mkx^hi!&+nh58RVB(7zGTM9huz#cqtz0!`$`~-dW zboX6=oV0%tuM86vlhs#RRp>nJW@WU2LO;ur>zP-+njEYCIR{3ufL>6?eF7m5 z0jDm)cXbZ>5YCDk`}aMff0XFkg`Xk27&NU--*D{CQa3kKb?rY4pI%c$u;JsEm&QvW zFO^VU^LmS`Fw1CuGu_LO^DAd^SQL21el%WA6x1EZ9^I=meEVd(?s%MG8S(-dYy5)e zekJ1Zi->g7(Cl{k0uIm- z0H233BMIXWZ)K%^&`>dcDY`y+f}*Kr<4JxNb)e%7<{l4GxyHF>rW%X}JZyis9Brh{ z@FKK3?qzC3NG<}94P3+17u9pWhF&Q9{6`d3WyGr@5!&mF}uL2i=N-|Tlo zB(em^nZys)jnM28_NX5_pl@Tt-GL08wQNSx25trD>ko26&9-}cCsp77?W5rfRRI=7 zUzdFOhnBtd>9ySzj_u3(2eqXF;9b2pY}ld{{6yYw{G2n|D$b(Y3qrY{{A{i!kP9ng zeulPv*Y4eoIWBw3M}^Ty;NLiU5H#X~(+D{ZyU5*r=jhS`tb%Gr*x+F|?w`Rvi6Wk8Vy`o8VVn|gWK{`4ul?brF zmJUD-)7hCa1vkdkHPYU*$aeh0Uww!3wcis}1};2{NQor;wB3#KB2n9v%3@Y26P^G`=2Q1{GJM}G(oXE5fob(bymR@rQ^#ISs^+&y8rsS zpF1gX6ry4y%84qQ?#+c3Yv%2Po2O-LClBYcVq_Qt1z->N0>?!!@1|Hu%n9cR_$ZhC8!$JZ}hc)V{|zjBV_WtS0ebS9jfOEb85 zE2JT?|91QSm5c)}1$;bTcQ;jkphz*(~n6xeic%PTEiu-xuQ>%@w^=Z#9B3Lrcj^R0>_L$ zQC`}+Y8Adb{ly>FGZHbHE#vnoAnAI0M&dM)fE7ptwbVhk4}4SzoUaIs^SQ(75T1sN zy?fwkXT9t&GZRU6cT>NG1BJr0b78`Q5B~5o`}vY8wQ$39h`OK=Q1@vRY;nkCWYu#7 zu~kDPiA%s~5y91TZPcCif3s0kw`)M8}e>3j; z%3sX?^z9Rh!)782JtC!INgw#tK-cR~5wd=fFrsfW5DaQ2Ic(Zj&<$UuV9MYizEUl? z(8h}N37k_&WW0f=N(HH;6-LVJ55!61JLX1U%V8Sg!gzE5NDl65;p7<$+w51bb|19- zD;NW6*b9GMhi3geOJaF6kz_iN;Efo4o90SVI!)|I>vDe5bT&nhMd{~4AVg7^@$$WV zz`Vl+hyn`V72$y^YE(~sf|^_?{U=YJG-}i+1Sa_f00RT3aq6lyh$%EpmeSFLufWru1?mC94L zjywZ-fIT0!Hl0z7gvM(gdByeI-xf8vW<6APBdk)pURWKC-n$bMS#~!72=sHm9 zMOt~MSN=>ej zJ2(JL@YVqwZx=~e#u*z4GTEyXNX!2a`q|~6;~tHE-duT)kv5>}B*yDkTpF-$Y^#zE0Ln>j;R4UhdUn&0u=A!E1lO6n%#4OD)4 zh=fupIQKYqW>`**Y&dZ0tJUvn5{~B4YInf%noo-8r70?uzYTK9-79}z3 z!B&+lpthGwii?Yt{90iLx|8AazD()Yq5MNLqrZLV2RTRT4BkKC6Hgx(VN9=HCmdZ` zZ(94u9|zqw*T^nAEW@1g>Ql2{c_HKF+6l^+zWX+wT<`y?)x`-|=qkmJw@WA6aHXm* z{FHZLy(@SqZ2x|b7FWd9OsfhG8aYN7qnmJ`w|{3&Cnr0*ovH*tXd!M_$^}^AkBf2K zpbVp}_%20j`+eo`OwVwXwj@4bAVu#83GWl>E{PF)ry)a!%954Eqsh3!BcfB0og)6G2<0dw3@beU3bKaTh0b5pJb&lks%#MxUjUN|ypknM*Do zas1xknH6Fj-3n~i%e8&qzF7s&nWY@KkaHXy)rnMcsnmNyFbY_}5cwA6)()cv%K!>< zC%axY*Gv=uGKPMW9LMa23D4t8E`J%n;*I*PF*jON_@JB}A(@#X4`)D9`el#ZGBeG~ zw_$xVm0Q#23SBxSCTt$dNCw9xLQWZmF2<-#Uad!pxR3S{-Eh>#4 zb=1B!>zjGqkXd}Lxuf;LO-g4_p>GW@NYB8=WXD;{R&zy`!qSw|DPti(M1N zLKK3!5wJu>u>?`AmRp5j9sjN3J8jbT>(XnAWg7D5o}n& zF5J)Do1BxJ-@RkJ?>okO|2i3`BoWqLd#&$xe&>AV^E~smwi6(Cq{#njXYKq66w2!U zhZxrmofC%F+o5kXf2jGNpv(*o`k<$d9B?sAp*4u!+XsswR2|fk_We zIltZSXYzhHu<{_P&cyhj-dJM29*@}g~t45Oxn!=IP-CO^?hY-~G z8=ke`={#$jijzussol1wXv2!;@=+Vk=4yAb_S1r%t>y`}``-gH{77gSSIqfOL6i1( z0h#In6~bjc5K4p2}k#ycFF#$DlnRZ?VO&)!ke#;^VmdO`7jO+ z8`xfC%prI<(o9mC01ydk3qJs$sXdB;r+uXF0E<5;c-r68h|SYPjKO_5N{H68>GF zYo>i>4%d^VWqEDLUkP&LEiNWC{bL{mb(FMT>29zf$1bZ`N2x*=CXkUh6}na6^6&@e zdj)0yk9m?pcF{z6dO{h;+sKy-YWwS755zPPfE0DGwVHEcHy5-KZr7ChhHVi>z+pHk zl|{?|jw3e5N8)r?FfVVb_SeVh{*~^#d5W5kIPhIEXY3EoSRt+PrHz?1+(jN{kkMpH zSoyo^zYG>l-B%rv^0#ET;Wd`z*QO;PPtq*S(~n=-MbG+IDpf?uer$O`_9%6#^_PE+ z$d$OqIVKG`fnE6IXraNoYCj|>t!L!OPOiJc*+Aa68_0(mY^A_C#}_nX;F|Df=uN0f zhga#o zK+(ps!S#6z*>v@aA3oee>OKAvW7xW>2x|6g{P_5AIu{~h;RWvfC*<%GB|w$~#3>-M zj_~)Xf-zrkE&-|Opb~{coHP#vNu&uVZW|8?H~$IEEYk8EwO_-{u%KhO_>Ibw7M>>^ z*&Gs6ojlq;7Ukc5b^goR4(KnlC;V;6TmHDQ>)`t2jGlK+CQFM4R}R_s8M&YVBeJT> zHocsL?Tly-;(MF@Lv3IE8u-8Q<0G0H3aH90=FzpPAf1ygoXMd@C6PZVRger-b4jP) z(9jEK+6Iu`_9xXVS96s)+c~D9E+SOIMWq&vJrSzT_kfrwd{43#QBIR9O^E-Xi3Q+} zzgK6N+Pj!DDWb8VP_>{1NL)_D-G`P&+4`pwCr+%upF*QgX&08^7ReE?SqcjkEHU3| zF17X|XO@hA{eBgv#(u1u_{R3+BJ=({SI5Tk-d!PmlLk8L2SSl>KD{&*JxMGt`~bBH zEozO)3rO{T{uRyaMQX1oEe~FMWrf=f+re-RckSAxR0%Fh8D~G?9nuFztYJB&B4w6_ zuympH`p^HvbS&B;D5A~)PZ?%f4SMyJ6!9J!CMII0%4vC|{+$cPQRaz;h0rbCHXx{t z(plv*2K1T`iW2|dfBBr~qH8S{Rcm12h$D^gJDd(zA1(58*c;@Z_}f1#LM^I&r>d{= zUwB0EUr6Y`kPz9!|4ZEPt;VOUdKX#6!pL`^AG1M;s^tKGlaX75HY@P+t6!MOO5C2- z!o*&A(M#w9pruYQ?!t{m>qozB+&@-y-Q-s4DDeON6v`R7TPzki!K@X|FQ_>NW#RV| zg*Zrvi>7z$RHhr6Z}RV5Jw6|z`#gN#g&s>JSsMQVz*n?RAJNBEQ1~%n*GGt6@qWZ= zQBzC#dK*{n4$gJAU8XDK$8V3q`2s@1=-w}$K8=Q|H3NbiO}iK}=0o%@ja)Re)YjcC z@v=|HYwT}zV)}`N@1A$1Eh7Yyiys2+EEpkOTEMVGJRxyYvZtg8BmatFiV*P7Mi{iq zgFV~K_vGtrAOnW(wblE3>RfT?6GtIVTd_zGM?tt}VKEIGHM&UweSlQMZA_u!uauDI z?=~7JJ%(R&`NAe6-4QpCh&a&}ATbpDZRU-Rq3&oI?4<3AA)!i5T2^to>dR=- zptu-X5@R4E3B^R5JCZzG^wTQdwQtzX2KBcS1r7skXX*+h!f%W0LY2j5nE95BT=J>x zSnnsztWxOq3LY?g_YG4gYH2`JD+JeoSMs91p-85fgns@ zP=ruJ1RoijAvchth#3Gb8wVi)RE>FK>HhBCud4!2+j;%;lM%QYrePE(q6C3uu^wGM z{oI*3okBY@*eBy%^1Q%fwN*IOHKvcQD5@;>?rySyTl>B4Jr*%1oKzvDiAGH& zb-HjVRartG!NbP+RNnHRe}?E8i;U?JpiYuwVS2lN*YTmXSozj)@zu2AG^Pu}wSyMW zL3?kJ7ZG_q#1#baRp0=sV09D)$Q4wEA}}KZZ2|Kd!JyO&e4uE$1XhEdbtO^sx%8t% z+~j~g?o8hyXNEw498SULK!cA(3E~>f8#qY5BzTQzJ_R4eI`*bmw9zYORoSTNC##E{ zxjfbKn=S*p1)2RjFEFSp9ZE3^bc=y(hcD0l=2vS)4uf0{tE@e04$K>GzgE3uiIR*; zWg+rz-pxT+a-ev(T$;a zM8P=c1Z}LzHs~swG-;x`L^l*U>C2()9T^h{6T5u9SGA{I|F_NnwIyK>(vMOHz0Eg+ zTY172W74!ki`3iO7^MCMDimbF2|O%~U1Aa&N*zT&wdfsw1W9@-6d!WNBf$*sl}NXJ z$=gV<&^pOnjody>h-TuRA zt&&<1zf5{=1SEl6sNp^gHA8G5k&`x{h(H<>hgrLa0km z0*3El6^PqnL^8Z4P9+mWrc>g8J030Gsu|p&Fqd;LHp`sF5XxE==wq=^iD1F)Zk?lOGo1eu6n$njNb$g|*TKq5T?z~S|nryLxy1S(pegQ5OSRVVXtUPwBHIVdCrVQie> zH6|c>!UuFeTUIR}5Y5&kZjE}o`sFWrFBER#9Fx`A?uWaj zR^7A!kU{glz!2E17(T9yd?#TlZ-HXnF(wJAq_VN^Zv-s#R_+|*)ve0#mq*RU+d+<% zh{38mz>3Hy>CmBrABDKEk)*3V=oiad`2_`>Plmc+8NzLwK0tcaG!bN-F~L~4(IVCw zmU5v8ymyxfB$0{%Ww97R#)ot$!zf;%OyJ|wB zfkO&v1H_qK)?J~HDIT>w?dm|rjva!EEPw@DGc` ze^gW6?9nOx?xb$jn6Mpwri@&1uzsgyd&LOi|LCdjYGFFLt3aW$B1UisBOR_Toq-kmA$NRD94YH?g z?;#NgZpust+HRsJdRp+)_EB&5UVBi2`5r>AE@)2$eHKWRSwp|CKc?uv9>T6QErJ`| zSFCB%d$0b52h6QJ`<|{s_Lw)uK>H?3@iFXQFe)m6L<@A&RrsTl(FqC-CK566JAP9f zh(r@7a$U*zI5OAjndf%=Qu9DXmQA-A9D1U(+_s=ZS5_MyJazzg>R3O2e808FMPpoj^^Njmy`5@6U z6M_T)Q)N+{a}HX_qCy3h#c8yc&+~$)-1~Xkep~;xvwu?ip5FZ+^I-4iYhnvoN0Ie3 z`Kz0kJXk)l%R1CLPzGh7RKbzX3QPBLgERgxMRZD;Y1=`y5ZoneOxOlkk0aE7U+Spi$B!$$*xNbiBPCe^oINru#quEiN*f}v9T){{?okpZ!AsOw=0<%= zF>>OAd+r)($NtSOhN4}7SgQQX6bU1C*0W-Iek<6|g9cJn4X(3=vkQbMu!z%|}!s830kRhQ_b zn4k!3<5yeUNpT6okWO4%M316Qma3A6zER7h%pMb#1G7imoy1XDQgd;D6n~knRb3my zT%=D{k=gUQhtM|3Eas;1W#LRpgVar&U&qk36X`ar~v0Nfo5jWaXVO>0B8w_ zh+umgs00I}!*Pi1>FlY$rA0%xj&I*Kts4D7{IBla+d&f+i-7%{xy~o%9pKsRp?Nn* zL_gBa0ZP9T5e2R9+Wy>SjT{cUT8M)i1B=(stMYz_+k;Qnj$f3x6;(8-JD9Bh=juU0s|6uVbsaf3OhxrO@3$ipvSN&XraEetwRC z!!z^E=2Vqh?9m(PZKx4P%2VtOkH|8KE`BE!Fm6D6vgAuVXX&U&*Gna(vIs+IFlA{dh@`ve&?nTBjAIr;ed$}O$Gp^qZH+H?_baf^jEl$ zkHvY1brxfgjq|D+y{Mbky}##Djw_9o5F{|!7>T9#0IWxp)5MU7BD>2v3k!>`J8P>Xa{3YRr45GGIFD!B)mIMQ zW*$~Hqnut#g9lb-9>ma{0QXN|LP}i`_388^Nh8!<+Raws5fJn6h4M_x{|MWP^jE~~ z^tmDW7MVZ&^kIpw$$D!5#P6AODZ83A9gYr4TOj(u6l%8bH77m;r>J+(*y z_LP)0EW{B@Uk_wu4MW^1Vwk;sg%7I}u&{y^q_=Q*;CspXHffL+h$AFCccTV1dU;i0O*|R>im_x zA*{a>5O>sZ6s=e6lms|hS8_H?oxsI#6qdKH$(9#!820vY)ORQUFY>>5+CF(dV-`GmuJ{9|HwktZ{KbTb|a%u^j2o^-9J(GEVyE5OF)SDe_kUIB} zSy1NBr`O4cD|FY|)(mR956fmbMFv#9pN5OyP|4sWoK)f3f~Qv=d4*(-Qf5*1-Gsm) zA=rBs4lHf1fD!k zjex_CfxHi)#PXpP8AjXl-yaIiepyn_=jY7%hY~LopIlu%X6N8Ku%@HgS?rlG+BBqHd-zs`uOotV zq$k6pS6$L?n`5Ao4lITJ>pwp79%cX~!UY6*#*EiPiTGCfOrj-LU zTm&~AkiYmn8smKU$fj>!^zkH^h-#KpO1!1-R902~FptH_v}4V?qvyn0%{+2y*}myw zFI_WqmTC6y9ZvT6i`?+IgUv6^uoj9X-SmKS{lnb*E2Na9NrWzTVEq4i&OmnJS3zaC zY-D6)JdS&+->R=csKSL}1|4bAtn=;v6~8r)vy1Y|s%CUj_EOFdENElKv9~S;CfW5~ zv@w+~6DbKK$J@w?29*f?-fVmZ;ao#=Z{L2f;0rvtHBafUiqj3L*pHM^nfV=;Roz?b zTwEK>VUM+rBL&@Gq5wWMv=e6BU*oJI{TY0MG^-A_7K`=6dNcS|xo9!O>WsMdlqi-% zSpZ7WOAL3&$Qg-L${5^Jy>i?P`C-ZZq^kyd`jK$#u;jkUPbi<#Z_QzVjYMvM6hl}j z_qin<%JM;H`;=ebXE=SkDoMRAq-XT4Avc4X0wpLJ_MO# zSeG`|^PQ`CDsXp1mhF89w_LUO3|{G1ao|9PHStGm(SXlgQ1h90R5fVSD12VkreN9M zlwhM<)ciN@t}wWwfA{54@VV2m3pvXiwsf=GwE~+5;0h_u7Qn?kSYpR#IkatBuU_cJ zjd@|Wt$t_&NnRc;K(P=lk!%Ub!)iBF2}&mkDCzj`<$vQjrJwN09C4THDpe%UCfC|D z`U*rxbs{ikkd(yIf9G+CH!FD$CwL3sNm4aWzy|4VZrQR$q~ao{(1-%W!& z4=1H*@cny8-s{qWNV`;0U`OU|pFM~bRb%6&znY8C+_!mtB}GCWaw?iJY# zdrV*m@Y3CU8Y$}p;l@o5SW#D)s2IWt+SvO^>wjN6)UkNegEk~RP!)4 zQ;P+r7`%v3-)$*EG>wz^z5^XQOX`S~@%y1(RVluI#WA@+)39m&B42t^c7|q4~h$-L53}}~04j*qVvQ!12bEiwMBKtP-Ie~MJw3gFa`w!>xuPq@ec=hML!0I!f*QLO*bc^8 z=k4>`zk7d%4G=MqI*$JTam3n>^H)gJ@7w=uKhz|?rCa?qzXsVyblmx?{ovI)GuC(dx?Yr} zRlBgz;>yZs||(Q@zW@PBo{STjLGZ2ut{ zX)@?<%~ge>Z|yyQkfGh- zLE~wUbS0G+6Ip1Av?T2DI29WP5)!eU6&IO;9-5r5d!5dPMQ5SmL1fLiL>#P=!dMbNIG52+M-z(~lo*lG@UKlC=aKNW<`pe>fUsr#=rMkGRES5S|2y4>2 zC4HLaW3Rc5%t0o9_9jsfAXGXo;#8dU281|Ni8!R?yk450vaRQKwLVI5S753{^HNe^ z?lXU)CIz*eJA_MUvS9%@fy?5z&IUJ*a@oGdYSgH%e|T#yac1#&-@4;{YX|1iW=(mj z`FF;%x~!Vg+Wi;f7{2m8$=*;~1JN!B$vd8s(z$EbQ4nOwWU^+y$OEHmyN*HU>7+ey z-%duKTs7qp+>e9f&pNieGVyk&Nz=bwb?rh_^n^W|S9aVurS*x0AA&DC`F}q&GRn(N z*XNzTeY@34m8o)%>-8~o7u`&hcVo`=&29Lg@5a_olO0>9C0`hKz2T*W&%5XP9_}7e zajx$*!%GWiwYW6*`yRQz(cMFQO{y+^#8goX1TS2;@VVjG;Bx%G@H~+zmZ0ZHyi3oY zJ0}k7QVmf&2AurPx9;cuC${dYd}BPY#ick`?We^4&ug!a+vc)z?iQQiilR=z4<<~F zDRo*J@+M`gd3xomzA$9>i>4pr?52r$`bIfz`S>JSA1-d z`QzjlIGI4982d0jpigAj-A)_L%+F5Rzp&J%f&CP{rz>a5<@=ksX)a%9nnhE-w^RP5 zg;VrOn@zLm)8MK0Nf@Ukhj*KHo9Aq39e(^%n-&fMo6YCI%!o?a+$VICdG}fa*S-F= z>c!O!=X%Q{+NKk)eMCn>eEAw}R{CkbM!4p4?5DJmtB-y?W6RyLq8fWv6uv}@qTCY| zRXqPr_eO=uMW09X1)Ql|d46KX$Kv``qik>X$hDi$J=d;n_iOV1a}OtH7Y*|9iH|C} z+QusQ%efWXM;L$K`l)e~)}@;N=kMR?8zt1zahF}M+&VCDK-wG8?krq*f^jS7tvtf30-x)rq4W=S$rFiD2ed)pWZui)!OzeoIaWVtIE_GysqhHT|eL@D6_R1fPDv}U`JDl=Fo4>**&5uzjA<@?^~?a z{|ZdHr26|novbzOJhb!UAUpgnrtxIeAai1QP3PozhDsZoe)+ASMuQZ_^T{l+&E^4z zfSUG!HWXaLG6;h@O92zJFySVmxie3o4kA3iYA-X>ZHhLVKVtEOzy zyYG60!kRYIBb?u5-GR)=FxLZ2k8&ST{nc%3Y_i4l6V-1g`lpmV|2rd_Gnx_3E9hZ6BurD(xld z8WLHKaDG5S6CQAz_weEOCxV&eAS$Xr2(?+S z?wn1?QG~xw&maOvHa~nmyyz#nr`2rxoTein-3GCL6^00ahx>?{wvFV5krVVX|LWcx zU0q%LkZ#apG2vtrr#3p)JwI~d?NbdUAqepu& zsA(LysDBzn`9!+JqLztrv#e~xRWK5m`RNlf)n3fTZr;4chuLB&%VgN_h=`8Roxnn_ zcqU%X&UO&fM2_C8A@%xR0Amxm0+75lf13-!P?0}v9d~H?O#m%T3wU6{0+A9HL!$G~ ztb&G!FDa+#QQA8{1)O9JYCLt<1%5s+x8g-p5%Hi>)d{Qy>%GIc%x&7zAbrx|)VG^8 z=eW!oAh2X*XNv|=qs{R1GwWk;&wvA8Jm|?>1ZtDvdV(et#DzAD2@h)Tg_cr0@xwQ7 zK8WYJ6OFq*=FLK1nD*>hQRI8wM5!DqwsiD21cBu4IT z&eYHEUpXiH>=EsLP>{mOW$1cIJ@}@=@R3L$4Hr7)bVQx$DIBE5?cqPvU=j>G@eO6F znZ1vXY3GhR(mx=F6Gw50BD68rtXn6NfE*ZcsBs^`$n0U>{@&NCrg(sJ#5>1w`(o8* ztSlO0Y`DY9di3aHOMH^R5m;|-LP~(#AZ;I+GAu)%u%Es`Y9K+-(9m!OPtZNB_UpAs z%#;V%Ukk|u4edr+-dJC8NvMGgy*(ll#SPsixgwhsy9s4R#_GURyFD&D|LCk1<|sZ3 z(^ER)q7IPV`H_cW%@JAx;>&}b46L#WW4@K#CI`J(SjmVVS z{o*$4AbIR;rk$ID+8fc20l*5I=k;mTSl0!^Fka9+&EVGURwvienMi9|^fxlUh5ef` zH79~>gxDs0h z-A|{42Q?R{%>w4Y1}M;jv?tCYF(T#)tf~}CYngUov~B)i@XlWCOhT*TOPW;nZ50s_ zu~+^rZZR?`FUrMmC3I@Z803gB>>R(FXf;Fd$RGR;&iu|9KhHJwjeVeS$8zL6!ILxN z)nfQQ%D4%w*66bXM8xveCVF72=J65AiO7TjtFHz(C{_tO9WgenPw{3GM|aAmj!+%8 z_FrJns*-bv{^cQ(0;%gn{Kaa$8q%aF5&EyJLk=s^|0-{H`gZh|zILG7QbvhLinaRX z&GxD>EFApYyG7@C`uHf`9TZk0?eon{)h$bHezCUBFx;faiX$kYrS0{e0YSUcr^aa} zf&{#Xh-&*=eSGJ{j)sbD~>0?ljSBRKSQ+^Dk5A0Cwl5M*&K+%A}}gU>ezkz#q@(4 z6c;V2-NcNZqq^YBv|d+eCBq<)9%V{PNO8yl2e^MiB@g}DJ;Y!?TN%yN={JoWKO=hT*9MbI$={?0Ok?c(^!=8nuB4~`0ZzW3R8|Hb3*Jut z9#vfIrg?A9p~6a$QPaX%qHY%z>Qo;qPVcED{+&CD9F<(mk$t=rJ8s|QHIvK7hBe5F zt-nZ>)W>x^j#!e6q*mm*gam)pH*@zo;^x?~g+}^OF3r>MTRZaQ3#&>7>FK@-N-@R# zNCY2}S=?Fpw5z87(hK>V_mwjD4OckB$F-Y1a>rDN&w=@T58qXw|$+AJ$)+gE-wAp6lt0J&)m(T@Lob6NBAq=%{|x>*Zgmj=_ys3BZ6aghwie|9lQ#+ zo_?L)VV&@#)KO5CWER;%j5HG_2C(1RT{iYfs=A8wZrrMLm4G^oNyXM#ylow=v(xJR zo1#HZ?b?y92~q9f*YCPOwxQitP|}3sx&^&1_N^qI{x1DitXvtnrDb{w$C|VUKfXI% z6-#4GCOFG3PkKs3rb#sJk8hTpe)ywrT{%fu-=u^yJO5hJE8qxhhUso@`@IX|KVLf> zx`T!I`1-{al3VcbXg=PA^Nm^m4F93`4t-fL_0M%HS1|DK5ihA%PyIP#KY#fmgL7g- zUn0`<{8ixomd6p)D@SepN&jw z?hwCF<1Ye`aHVN|dRh+||C^LGic?n{l|ke@VBpKV2-dUX-jTZ0(mF#ID=tnzE-kjb zg?CMqJNAEg@V2o|GuxXC|76NQFPC;_;Rcagin<%!jZxu*nsW}zRiqjsRhHA0&gc!? zWe$@SWpr-@O?y>8Q=Nry3PmJvZJR}OP*6}|%jW2nVqsX>k>-XfWxN`-AGa1MC5Tvd z0c=SgifP*<+io=iCikcur&nXmdQyjiB<~%eLAyi%FTRlkwVfOl^7xwJ3drKl@Jw{? ztW*qBVZ4fbW-kOCH}gCstk84bCM~L6YvoxEdnsxUvb-fZfIC$VpX>mP-(yRPcqC@9 zSQ*lF>ACPF#d!#}(uMVLH7qrgh6*F(sp!aIQS~mc%7|r%*5~)ZHP?i*nB|FYRHh8B zX?wFRIXrKS`lH}lp^XwfN8TpYx zT;hF6LY%*qJK754|5C)hmGuZ>GSp5?P5_n$x2rKM7ciylt-(GXhr7&Ug6T))IGR7X zg``Ua$znP~(kOFXn1OJR5#GP0#@IcQx=aiV=XteJ10qoZD)TO`C@-HU-T-7n;p{PT zsB+4~jN7+U4er-6E)npVqOC!F<18^0CJDA~#`5LR8kL1~i1m_Qh}y<=>4)isZ`U=o zC=V4HA?^3yw=_DQAYOT5R?7WUaPCL5MW{$D53Q-66?^xCNRvPRC@<%ts zGX)gnz(&`FCZjLbR&=)+K3sEZlPfr|UGF(NdxdnE(SM&%E508@S zgJVTA32;q)#dDL)Haq&x=agXjGH*$t*mR%$E!nP+UkiJzsXwf#wx?&g?1V#)b`};B z$&7E3Ie1Lfb^<3yn73#yOa_Lz?8m&c?$-6qmD86B3{^F=(Z%MLPrO$wmRWx z^@y#<+)53GvPDM&ZEe~K0D<%}K~gPZ4lI>NRBGxdxRx>ukL_q}ZEf1m#Ams^7Op(= zaLmYY6z{^kjkf1N72tnoQo`cAJ$e7PaoIwU!$Bh6O!?5T{cu8R{)3 z`|T5n^dW!-&X3b}UnXy6og>v)HCWr(NJsf)N;ERzQf>db6mq@J$dq{b@>kGcIjLo2 z3Y60VyO(|J{@J-b`}XhtIA_`!Yi~&Ny9&iGJF2axwEx`pXYIGI{?j@2thIhz+JFAP z{>%5a^*~rPzDrBuXYSEPf-{h$k;2V%)?(YC2|z2Cr^XqZm}DGngpqT^#i@3j*E0ZZ zjYrB?wapaMmzKX77Ltf>1Zp}A^uj#DThBJt5lDn;TZAb=(Dp6-_-ZXF!>L0yE)tSILa9$nb9!LFZP=vbe~=@Gowy zSMkXdacCF>Z=>20cU-Zbz8dmqj%0Hf0EwDVM45mSa{u)DWIi_$OwbGPEP1eJbO=yM z$;*lTSV+m#SJ+y)Ao%;q_c!a^*hxu^vv&-Y41j?DtrLFHSj750xYSr4AyM2BvgpHy z52yQ&Q79UiwMvjp{f5PYF6~j|Pup&AnO-Cmc#>UsHm(U-TMQ9Gr54CnlIv@Zj6x9* z39w?6wYIPK=ly5TV%YWEznxg?N^wCEx}P4M`v~7nOf7!I84XaQ=6!TSrBIm60z62@ zvG93v>By1w6lYSGZD5OAh9F*<{O%nGhWO?S_$ni__zh9bQCmPlTauKv(Zu0W(&8yh zLp7S%4QhQf@&N&?znDPnJHp#&xg~0}{Gh}N} zzahN2T%?~qTeT#+gwG3}8?Zm*5Ug3Vrp%xR@O+GVek{w*kLgz_UQHo={&sPfv`y65 zo=z`ZG$7 zC4*JVth|)L=e9yjcW(LH#aUjN&_M6beLfi>XMyWGhP?1{I!MDC8kKAoIqW&}CY07t zobis4c&1c|jDoXVW?8B1x#6;F-AJTCu#TFy#0ePlC0f0n2pgFsIdt2$o`#0Qc;Y!E zJkLVtxEjN49TCztF4q0!2%wd0j?*c}0v=K=N&%0iiG%81$n`SN4)%x%7^u|loTv|< zJ=?PH^8z~_SK)RC2Zx+Qudz%z%6PsRtKEqr*^ve{>rm>UCRa4)kfkec0NH0LRa*}~GT z0r3{II1{DqRrbOu+7UqSCU-9aIu>BjE759nQQ>}+2Pkn9Jg5Kk>(SgoVdY}PrhdRE zg>`#NSYY*gz-Sju2kgOze~eTpit9RY#O&Dgku0z5)_d~3JcrR1d+=)T;1(A?Ik4~9 zAx5Q2v=IMIZvC=n;uHAAc?2URTR?-1fw7li<2aw}$yzN%ccFgxsww*sX{)1T`~k)U z>ye3znG$LABfgeMaQLbfe++5*aMKsjdqbKO*FZxfBkYU^Q#U!0Wyw&xr!UTe)ks=0 z3J8r&;wXGa7O#vfBd8{y8jZ51%ym-kSWXLxRy>?rm-Vw^SFe|1mH!IJLyQYos|A!5 zuWe0Rpi%44r=^-La^{<_8nvE$Y~dgtkC{)g6%TFDz%S)t6KhOTqo^Dpn22=H#MheG z112YPvqjG-e)olm(5NZ-MQ$c4%gb+<{y@^DmZVW&UB0MfAELQMk86w(m}nef^}}n} zI_)j${=s4lA!Q9%j`ZM|E4^3pGybUk*uzy68M&xz{AdZuLwf`LJ)m5nd6rTn{Q`{u8F8bgFoCV+hek;19}sKjr~3=TF3_Vnj|_I=c^ga?73W zBp*|!(+XP^#r%)guT3QZ=WuD0e5#ZiZKS%yJD(k}nC-esISp{NKtBvrOdKKN>wK%c2CS*PT1m&~NG!XjXWQMmZ zs4@7Ct(XH`wgkLodU%)dS*;nuar+gKq>Vb5CsFeLIJv=Y(1GDBr#Iz4C=}frO2TJ5 zl8XBJp-2+G!$hbIZ0u<{JFuMNE!_FhCp@5;lhR{G+l<^k05UOweZs&hxU$ki960cF z|8BSQq0D%v$;=FpKV$rN8RwB%kt%7qn0-*#2`n$bp-A*d7DVyErK9`DZZ^2>Y}g*l zJ87R#Mwh+a+Nfz$fZ-gD3WUZcpmfruq&Vlzarz|9)FY^d?JKP#kJ(@#M_GPJW56BZ zUvnT|2G!NH6*D(Y30OkUK;UE)aw2L1fsBK5a64tJ8qjzpUjuTop^Ejqi$@u@?^(aA zdi{a1Wc*Wp|J%_(w36?zNp&#um12?`#WqHYQ6T2+`#!0%;0s8^hJ8)VobkMfGeM~W z0S<$ZKlAw+2u_mK5v(vW`QG*9{-Z~a_8$m$w$56~ctVvjf{aHlogGBiqLnJUd-|?F zYK|9G<)SO3szhad`Bkebr)aah>jI}~yzO6aNgTYgK|lTLA0Gw|A3l8Bi^pJk>}F45 zfJm98A`-|zf*NyXeU@yGic*RN9u{&!P!}9FV1U>C^W7VPhDnM~(E*elKBcvPo@tys zU4?O%E-il(kt}2bXOby(6b;PIurab?-`z)+AsXvM(9Mad3(eBrmAYTkE4q4e*}SK< zzM%o|ZBO@L0DEyd5#?_3%?o1cDaOLj$-ZcmiJ#U<0bG@+ykFR-knNqmF$fftHu zDDJshO1YA6g5X?#;iGn{ET$P7YR=qn69>_s(;I)naWBEHtnn+c{4Fgtp*~S^RGLkj#)O+*>7V^A$s@^*6}%HiL! z10+@Ye;Y4O&F~+UTXdE)`$gXX+jzH(#fvOpwl+PXvtM=?fZL?MEsaTD&R8B*7J`v-X6W5YS{g3ZvCMw% zcju|UZ(V6DD3_q{qx*dQ*yj(1CMGJn1!3VS$;$=CqXEF}BkvF_{*pf&AZ6omc%dCAhH__&?~3^t2F^jYzrXBZ$VSM!K%T? z#~d-VMcOBKi|1@fN4nCtlQ}Oe2sCSggTrxRl8lY^Nhgw1co;XaQ;K}>w(f_AWEg*d zF&|{0f7ja7FmQfjn-R(w2o7SCUH_jAmr6E}}PqX)XK#ojF z6XWlGu1|r)%Q-4S-GB@(KJS6pvAYlxE!=_npd4~8_X(zT@@0AmEz_dsOSLC2NK?kE zMCN#9hq|#3i(Y+V==S`y7t56$`u5F#K9??2QoD?C^l!@Y>F)ZbeUcCHPtEZ(!^2|M zq9tVZFWze{_~xwGw{78*kQcr-=uDp7NiBT42|t_%%j)Wct*1+bel_ebS~kLd@5KGX_<%Ou_9$ zo2I%j6RQJf92h#d9#qQIEs<^tkW847q1?3PLRkUg7Y4CSAkobO11t4j;Aga9U#WjU zKpuFY^p~V06j=|9IY%lfyr*Px45i(dlx|x5<`)tmTSBII$-vqh2ac8GON~7Si%s&P ziZivErn7;Y4tT#qeA9t~qge?3?j+C{UkG1H=G7JYHw^^x?MnCBmZWwpffn(tL{*BQz)HI_jB;TYnK#h~3LO@Ud>EV6zhTgD!LTK zxPhYRoEewkUv^I}KZgG&ZKrBrr+4|RoujF(Id$07&idkIyNed51s%+7uUK5Zv~#P3 z#8wG$!Si-@)>ovrZL|8H-`IND_S`nxM!E@n3X}Ez<5wtZB^UWWnsYI=x)cpX9Z*r! zcZ&S|-O*r0{q&xpt3MBC3f_q2$ci<{JUy7T@^=OA?%d?84HZ1QBi2Whbv|g|RyDiq zAHcfiG((o4SU`YpDy{IV!4K0*2S0?^JDR2hx_G2t*k-PI@0>%`hvk)^X;l7? zpYlWMO*9;w|MgQ-=^w>)nEP;wI@>8BPD0G$Dv2&+{>x8wWl$3=G-<0Q@jJ8XcE?L{y)Hq1$?{5w< zUpXkDv2@UoJQ!wmVMg!RenIk|<<>JXG3B}Z5i2PzGI{v8yXs7&8s6apFcll zIM7z=C+Rd4Te8Dng%2vcxdO#rSiQmU?*r_inN0m%d9Mda#G}!-o$}L#>KlUnfNyUtQg+q@+Yq z32HZ$Zm!-v@~+6`kLCJJo6K5^z0Nmenud;A+fRiW0 zaqvgn(I+R~R8^G(uG|*(VFNomyN|E0X!z%v8n1izR$101d*xa+mDJYi_Dk1&87^$h zxnNdygHxm0e8(zpZ*S}V#@z+(v#Tj)``-mu$Ju`T_;Ff1I5_xxeR@>mFa5zEugHtx zlXvHHn;H}S^rGs1dqaq-PHj&CK3rE<*Shl!XI_3j_tjbRw{LH6vKj3@OnKhEz5F`6 zz)9g>p#nSmK2$fam38tg^<$@_&VGpWAL**Tv;WWrMR(SgLN}R;pj8R|)*7mc8NPjC zU$c_^+3n?dx<T+HCb6%`wt&h#go@b(&W#3@uH-xOvNg{ zeJ1iMZirmjn>YK{Z`>HKfWg@KyK6OF@vr&}n}KF26*aYzckhjZptr$zAOZ|;G zwAAZw-|~xp&D7!Am^W_R*gie;-La+1u%fZ?1SR;pF(=Jngscf^X=!GP-qCSfH{h|; z;TSn%YT;YVT^{VQ8hItx`|~H$@{NZ$@T!=Nmr`+9FU(zAjq(KT27efD-@Er%mEkS3 zp8lriC1w808TRelm*}#LmDN|mtRh+5ar&bC^r4Kf4P~+W-J3YP<8*udTIjnT=n>TM( zN!!`k`EagNW3;icQEmJ%Ga%7fTU+ZhKRf85lweb99?vsfe92~JvgWD$} z4fAaioy1dIj-=>rXqkSMoxRJnuP#Nb=)spKhR5=pN6-3)&)*$br~n!)Nta0}tu6j2 zKOeN=V1&ZP+ls=Wv5ME0oc~cEC3Y-52!Aen_3yHkTaAieT^-+UQM>-JNzvN5xw-8- zcIX~&$+q}hw&^a4PUx*$o~^B?nhTtlZ!;+@`)9+!O!rxr-wj!GuU!6A?JeAW;p?f! zY>UjCoGoAbw+8TRIdN*Zy*%^j(^>}+^B;w7)3@&5XXfYUe{*KX?;o$$q#b?YU07I{ ztGrq7kycD_US1x%ZQu3G%C>{L2|Ts7fq`KBf|Ike zg;C|SWqtaSq31Ul~P9DBd+|5-ztCT3cHyC@IlyH7N{I4_Rx9I>gAvhTk=b z8<^eos*;hFwY=Di&IX@DDdgLBeR_0wOTekkYqrnZBeQ^Alvm8s4B8BX_et!^Et_hY5T1^~~gG zcomDcCstnTo6vQX=imOO1T=#iH*b~~&Cdnmvf>Wj*-9G5>Rsoxe|O1OO!HtN#d9Tf zX>yX*Y5oJXt~}{%eYa9TVB5^h=tu9+&`?UjX?!3`>BYu%Y!4qkWL&f7 zB1cvD=1mmGvSrJBQ1-;Ss*R6zRfRK9B@@FPUoKtSqoyJFKIeRm5+(R9dY@rtW+n&4 z=*KSb;n}++N!=2^i*qxRw3I1oyX9!-c($E`L+r7toi#Url(m@UU=F& zI#5r2j%A$WKL60?_wS2wxG7pH&gD;E3HE6C$eA-|7&mD~Y8zIfxH1TyE4#;j+~#*> zeCvKD)tsl4gx*gSs-mT!FxmNe8T*IM>OM65Nol zj}LvK9h%ax==8Rt8C`RJ2L}ffw;w-F-UtfP-+%k#Vb> zTQXT#ec1gYr7+PEBEO=xq@;xY>eZ_>o~?a-skZ#8TVh@Yv(Fa#qS+^BTR$YVJ6;xV z=5$+$FXf4CQS$loau=7>%W*&ynbvJe%ht;B?sFLvshc`<=+JiOku%vB>e-$cK3CJ# z<*Eqg58R{`ooqZ<;>&`1x$I&~A^)?p@4au|){FaU-2C5|hS>t0HP3Z&2Ol5b_OtJJ z&zPIH)g~TfWM)=6abh+8@l%32`_1ku&4UL^6A#`Q>U_H#%OX}IT&%Y_f9LQ1{!#Jq zzx{^Ic{X%yZEa4|f4ATgTie^cBeoc(MwR}Z&D*(ir%hKCx2)>~2X>2cWiWqDq1)A) zE$al&xm$a9chPQ9uHssl?LodCK5+~XvfAz0;x=B%RXXhEJ*kgXy@7662 z31fBjrSh|5w<{*~orl{Q(v9-Hv0h#m7gN~7y>%(K8$;zIwwVOKdv|D=fE*wG(dkbg z_lGDMzvN^QvV)yRyU%}rVITKg_0s$MyKe*rDi^uW)&)lhhHJF-^!UAW926E3q6rr} z;+>YZ*~rL9zk8d;%>Al&@5*pt3ouB`Zr_U+FYd}XZv8v(orQ9m7!ta__wvc!DcjAK z`wX97;*J@TwQ2XMN@8YRwv0589DCr9*Uim*;{)GI|NN>C9E-kV&hRb7%gYO;Lrq16 z{>+Y5sYjm#IP^D?#jfETX#2s>&#wcU^hQ7cO066obIl9;3kHUUJuml_yA_^qN;{S@ zG&|l>T35FMJGA(YiF-=_E#@b&^|txme2#+$H8m@Keoh*mnY@C#F8%gx6~DBd_Eq-V zhH7d{TwPs<078HROBx$jW7z`=ZaH-L@a`X%ti*yZ#6|82DUbhM`Sg)U%w;@5T;$QK zTV|$yW!SWpF#P`ZY^i!MpVy&{*MP?EMx(XdxWPyjhl%QG=U5)`9pUn)N0q1$@R;$! zTWBuw5CG2Dq8)VqhuORPyIHiNWxZ5{Uz0+G#*+HJ>EVIp6B84sP*I#_CTzUgX?9IV z{`&STsc|HH^RWuDgwMR;z}9aC%su}lHC!pW#bchI`jl=g<2pY(&TR1|^++lHiFNPZ zy<`>M*=nrh$HpI!BRll7`8i<}+-t1H14t_stG>>GMaAbB)&4{jXhpePXg;3kDtab@z6jEBm$o=9u<1vj}-bu~3m$Ss- zv5AfT`t^%N`T8;)F}>QJz})=&d?soE5qS$T3Uuyg@_iMtvOBN3A7xbu&^Ns5YqE|q zT9swAR)c_~wL)fgwtoI4z&KL?@X5lx=887PEDVY=mGOcrtmNGpnWrWrV(fo?_5V{| zUM~9Z?U@~~0h=ko$0u?$%qN3_mZ26X;J=y_6#^!Au3aZCe&|zuJ?_B7W7giQkDZB0 z=fqJI=}9SopSJ}c+V*uM-)-6W7Sb#eyGnnxAHsjo z*LeE$sdal96IG0>-|cc*U-L`kQ-5z+OWQC$NcaXN=r-W zDDCH$jO_-RS9N!HJB{^n1`EgsN69$0N(;REbl?RBvD3X%IJSVAsMzcQ5q=Z&2I@zQ3fQ z%qoK!s9jc#D+r@FbxP3TcY_xL3y(XYk`DYySjXF$7XMC2?6XBH6~5CPqZ!dxqiI_^#}mZ zD>{1ZrRF>@kWtcdcx4>Kf%T5ITGja6xwDcG?8lGA?>=~t;4wd!`RtkU{{4)y`}P3} zvI5yI0U}Zdd=1#(et2?Xf_n4qx$nKrCp0(S_SgD*+`u60-lh?3`JvX=bQCU_j-qI2 zX!6`=rT_WopHpUL1XXs+xG+)K7r$|#C6(e)9^7@-s`iY$i?g#35GYt%`MDUk>#0Z5 zNJ~Jq9{TgA9LQsM>d%Epd5^UybPBIOCK0&hHvJdq%U4BM5tZi?DjIrPfzvQ8uF)?k zDXFnvBxO|f0UMvBXM%c2{PAaJ@9wd>pIaH?F|a;5n#|2~%$Ca%K_Xaoe;gRYd; z)^d#xwp5~s*MPqz?y#*-H!dA&D-~j-CA|VKtQ5t2czBqRg{2ggk}S-bKlLUX{^W88 zwiobN{xT4+a-Y#xRa2v-z?`ru;%;pW-M`&s>^#BgYxZ6$OV&xyPSoI*Fy^2aH_Tmj z@7_ItH*G1Uqwc%%D9OD`>h>nBt=h9F_6)%($>W>-{rwHHPBT!kct9Ew%`fZ^ z&Uk=+9{EbJ`W~ys*4~-~=E=YW*Gn-aZY(IjIyR<@D$Z_NI=aw*{-LE9=Cbl7@7}xj zy1blW@8uspMMWOb&eLN*ub~6BVZEZcrZ*@Y|N092pLNY1vKlU4yja%tF-Bg!qvB@h z-Mcq(F4Uh!g<|dkrW1A8Z}7@xcj5fpRbRCc+*s_Do(}}le9bTk!2=AmKo`q%=wA&A zomiJiS{A|1bt!sx_g-FG@aol(idigPNonbJU~2$;k+Hv5U0j6s?BU@S)93IgraZCe zOsB?v4h;Ht(G?$Mr)$8I>+#H&tp-7#tX z`O_bWIKY18`ig>wiHZK7J?qkrvG;t4(&H7L#Omm6%-I~DkRSoPmeZrBh*FN4KyzNs zXACbZ^UPbW;X!Pj0_V{sJs)G#&z~20_x^p>h5D`fpz|HlwmT&xUPnq@*%d4=kH^1a z{xS;D$yY9IV?RGLJFn+rDx7F%A!`J6$MV-dj>*gbgNF_+!_qXmXcLhOY=9+fipv2- z)YTXr9#)Kzb8GGIzX`^cU{d5xF+0c|NYq`nY6m@3j?CQLikbxVny2OmePzFW`=&81 z@%$fYlY)YR)}EeYu{-kLqk4iiGHfM4k_b>PE-u!Nj>lwfY8|NhRSaX)I~eVu@~pDm$y_h>~TDB0I(KSlA^x9bUgO$-pF{1M%VWk_|pKG)tmd&G1_<*bIC-L5?s8z{jZBX(@d z6uqQdVPQUGQ{Znj#(x3)(A@fLBbYHgbbt2+vCeJj=re~F`^*N-w2oZ_u7sOLM@P4R ziavSrq~$1jwuV80Q&#OS^cdzz=AEo1>fM$P!!`=Ud-Q&av#g&7mu04WKx8iPJrnF)b}o2cUjNT@seVte z;rA8n40uuO;mU;-<>gEu&$JYQIAYZ&3=J>r8q3@`sTnCLL`i)^qQ9{H>nG=T28?*NIHt z>dDa_mh6Fl60Y!(KuiSQ0ykd6FrY7;=N*ZnX0yop}zghZ5^-5O!fvVp_?tl6^8D(gr*|OcL zao6xZ7Dh(x2U=p`%wR6hB@ZhP97B!tMR`@KjD5RL4SR=y0wKHID2v7-R`gN)WP!m? z_dH4Yp2c_fyv04ucJ&_c;ruH-%nBd$FFS{NtORh_{J)`z@45xaJOvd|WUd8p+p4g| zeVP@k_Q(mj3tz?UetmtiUm|StF-=h(dsIJP)Q*mZEVH}v9&#>Yy*jaNgE>T{O$J=5 z+I8;SIjR`MVnu|!`wTDIIsh^{LwI7zl3`8=1noU%rJOF zS!Lxq$Z&^l?_YoG?HMPCm-qKrrRqBZ(bwc(Irn~#?o9yL$bDCNisui17u+NuWeS0{ zA=he+w8L+MhxT3Sa)x>7TPSnNk4cJJ6h#Agm(-sEig zz-z85fvBPhItglGql5@^Z$7F5MIPdzC$^eMmuSj_nVA`l;Kxl2SJIB8@u&sy3e9a{ zq~+q`vIcmVn4Bca0(O;6w3dF6iTb7T*)R)``H6WM$AMFSe}6N`u?#P{eG>(y1U1we z((&Wt&z3wp_wnfQV~HB!W%E!|S6+xrd?NYR%)o$mW@d(C?b_h5uoV!&62*q6m-I{7WuFi zp>UD4STSj=yXW$cAk-&X3WS(#<;s=elc!IgZtd>A4*9qgj2XaQ9pz;RMHd^A8$4PS z|FyNPEp%`)BVgk=@5mshvs+BQ<6v~78^kH6@O4*LRQ5!Mhja66JGqPkc`n5>v4*HZ zWJlm2?j(t~TefUrrA~oNI|czSL;TxBLDC8b^V!-%HZWQy_2+{Ps{=q4R$8HFWy zU8d&dzM^`MR^$b@Z@iqn)b_gc9AH}b^%K!M&#|%bOZzlEzYOHfZz;2(v+!$|qkCb- z!`$aspukb;Ue(;@^4EQRO=}+>7#bPj`lxuM&NyLYIUnD$*7@sETZ8_QZn=t$oIL(A zlWX~M2}r<{l>Cm>Yu4Ope9@Bh*7+qGSxyvJfXkL`ziq~Uf7?lEH;FEz0BWx@bBnd& zIyEd{2$TX2TSHaXPdmyQ<1r@%WiSK+PCVq2$vNv}O$&j8s;_t~rwzpW-0M^<25*jKW*)%GrSM@NivzB2Gimw1M?A*7t=473fGFdqm8de3=z# z!ZDUAT3T9h%8EMGJx9Y96-_PHMb9uLbb9HXPoK1NCV#{{FqVsPw7E5ZD)#5h!#{mr zO7R?_OnIpW@|dCp`d)gsjdgJE*9*dq*IEL&H}4W#H`q)=RYz=DzQ)+f%Blu}ZNs;3 zuhHB1j-LiE=b%?~l^$~&{OD+`?Jal;?g3>ReM0X`>}E#Vu|I!8BOtWebK-3G9e#u+-8nl0fU|x3c7?NNAE~X2?mP#Jq5<9-*8Me9 z?88>;JS?ha>n;FR#=^1*Q-{!x4VmXQy=SeQ>kYsfUxA@c+xdojSGQg{yrS#p_kwqK zB@!AuRH`q?Z``nfFc3)3gn^)HX&1S>8Jsu~P=5vTTBMxYiT>SU@*`$hB2iJ<_Ju2- z3%F0vKqnRDcfM3 zN%jZ#gZCtC`Jc5gqyM|5NC4)b_o0oYkkqqaA?7%5Gk)O%sh*Ak0MGa&MDu1c>nffk zQR*Q+4v&76?*PWV_>2p`#Xv4I1KU%%nQjP!=iM3M2$Y;0^W zHwB@9aV}qe4VbQNda!8a=dKvz0|ySkUDS0>r{M_sJg(j&Vma9yV7p*{jivmDasd6% z1N?!uwqh?jYjSIIp}dp{a5InW`h|bV!ZI#!o#Y~P2rj9Hn97q|?EBKN!BKoI_0{B- z*=T4mA5Ax6qV7sv*?2VLY^8m^|M%~@ZoJF;6Y~%_!lSrsi`4HsRQyvt>vPV=5>L~=F71;1i7^B%<9Dug08ZOi2&T;*Isl> zhoBG&NlU-b_Xt9vPm^4KK{2{E=h?GTH0mtq8bscN0CDkvZ!-6Yng=dC%d9dUYV7Je zuh0M`Lw>`BdpvR=i?lhdSZ4xXJR>U_p$DYBzY`U86OX0i>l4E~{Bp_ii}&RcA0Lq0 zv}qHbeF~q_^Ia?c_+J2Oz<>3>IiI>eQJ1teN@KQC1^=p|swyNaTVT*vz3=gh8KtzP=Z@F?j0e@U0q1TN80`kRAeUo8M-*a$N3Z$$EuEYCwaq5qRkDAk`(pii z6M$B%2{zQz)p2ogh+fp6(F?fws3~3(5HIuG$F=ZR^-@N!H;1lMC!k8g!@{zqa34Gh zS`Ux;(p`Z!4WoJaYkCrC}{@49=@Y{83Jbf zbk3dY37PX(wx8BHr}!W2IudrlSts}<+z{eFqF}s6k#1{kjSYj`sN7*-d*#X% z<=Yz%6OasF`LTle!D$1rz8vkQ%F_o1)bne9Y$`WL|KpW+FOW1Z#UkC^M;%R2W2K~~ zXZyaS_5OJEvdR^967wXB((S83&?8O9dTYp5AqFj6ojCL~hLgt>xx3uEK*xx80-)gC z{I7rs;(Z#QKY_g%UL}BNhlGR#<%!+Pb78VsE^)Ev3opl>xFMp2Ylo^!`QVW;K`A@- zpuWETn8>m8NJ`=E?h6r0jd-19F41yssZU)kEmJd1OP6G^(b3Tn)TS|MTTi%ixcw7@ zEclktqL?ZwD&SIrR0tXW>ael0d?e(oHLoSd9S zr57+Kr66hBd85(ZuJ@B7(iJBQ^0XzGy#*m$EF>=+eY>DddFSNh@XixjS9{IdyKL<2 z`}?9_4C^LFuguLj(|Ddbe?B>ry`>(o2c`zGtDK?2ZCD!vM6pGm|33V)@Vsag?XCwc zEiERpj@T(r>@=~ebst|wVYmGCv!$pA&s^HxGPJ$^QT#w9VXTx7;3XrLncn`Z(|?~h ziKo6Sv~>9ZLDU3~&(M$s0RjzY=!-tkj5crC_H#7xap>Sj@YA&Sqo_Kj2q2W;O|gtg zP0mU0lDa(J-?V999wc1kA$Qk_lVKfOMMV$5Er4GF)}s4jo0P{~!KpK6)<x_5!#mB~6&L3n=I&?4oZ?s-`UY_J@ zaA-=f?(uQGpUqWORa}TdtgjfY-d422=_=fP@QIZOFKnSWfc$)qq#Y&VpV%k>7!wgD zc#na7D7om!_aHOngOZlwwk<~9W9!N4AgII0KP1XxiSzuL|8EocQ-Sl9@L!^q&iNs2 z|Lp<0zKf9;fU#ik!cKJS$$}dI&GDwC*Gc|Ee9IOjeu(HkF*OySyaqoBP~|Lt89h78 zDt7cSBL#4zIfK02g(&OWKp@VBa4s#O%*97MWCVh_;?}(z@LuCp6LDdY!gKSXV;dp zS3vm8y7+AeA^UKnuM0Bf+5O}iXwIjTk&!`MNo<*vVPvP?-m%}`mOpy*XtG;C)@eI2 z1q$4zkMdQlrd~MoGn<&0JT`pJ27sDu{BjQ-G!d3#+@}tZ9f9pO0|0m)g_T64(4YzQ zMTXAH-=FD6f%7%&JOm0l5E#JT*qnqPt55+Lp}VlIxOHJ2>nKR){3%gzc`O{`qzH?1?`*khK!7DR4igy z6L4Syw^U3_IQfQ30=ksX&x5|V7gQ;HxLv(<>sID22vosvLOF}(rZ&6J&pHBxEx}&$ zO;6vt)%eAFx!05U%8mz@3w2g0ySPZhnWd=G>#UojSwF{@3{=7h>Hwmvz^eHat4Ozu zPm%*}r!rPKj1W%rE>w@e(dr_t&sXMJUtc3ng**e`kOaJ$c8)KEQ9%P3LaiHSa?#zk ze*MEkhx)Y_8i5Rqd7`wyt@FYiVOKJnD{JRTlZo#1lRf~To4k)d{#Lhpce@%Zs$ z?R;DF;n^eRV6@EZXrbQH6wUr!^XSk$Zy5YU=H7SpuL`_*Bpv+-u7-e5c`jjHeX9H2 zJ%%50hX*APFtGmn`#5bBp-ja5mI;P{rdqBzB{1iw@=q0fB_RJ z?eVkBm@C9mff+h4m|~3AQ{lR_Jw*7uNR(V z8_~>mU*G{tz7AR(knm#t`t`9!`L;x+mP%FCsiqTXSu{KRKieiTlKcky_5YQDCic;*_T&#IhD@XF zSJ*xDL3uv!8r<{8?l%^_P+?5I2}IcvfDm7xw;b~>EziA(GAd-<*^R)S7PMp{-lqf+ zIOQ~a+7xfyG9!aQswZaO)r~+7%I2=*?{5hyhgBgJV@`DOX$KO$$wcZ@k4_X5 zZO?r4=+V17%mb9x!uT;NbX^6G&Aan#P?Zb93|SD2cvr15#m8CJgV?a*Z&2ym!M~8o z^t*YJ6@kTTz$0l}93hMvLG?ioL?c$gLM}_{!1h=IJP&bJ^xa_Efo5lf;FtkY;fcJ} zM#Z^V6$S4UIFQsp_nBdJ_|dc!y<#!S6k?K|D5hXX;R!6~?`v({xN!-vwGuQ|q!>-D ztZqWLD#0r5GWrDgr;cnV;=F92S>g5rNHe0^sX$3P5*kw_nma)cEq^j3A;q;}Tna@mXwILecxbV2+ic|o`oBx(V_TdixR zAK}Xs3g0Efi%f;iNPc!yF+pk}Ns5C-ya%;)jJ^4x+G<_*P z=4O2yk&;u%0!CC$?czmoBm+*Fn$jTuhR}0r!CWt0x%&9RVA@4Sl%AD zcfK)O6ivz(+eiTrqe+O<;xP~$>E|ebFb~uEt$UYOm<@j}zZz6qzWABmW{cLfaJ(+* zp(M!(IOc@t86Z}2|E;PZ2ml=@a5F+TvW4N zFuu0Cqk~RBKp?RT`5AETcU3`h#6$8?&>AEzHf;CJhBrw% zH9|l)f`hN8q=SY+RxJq61FRq~!$O=QWQUQ6tAHDM3)vtrOoLpv`MKHuY)9nw zPJ_wgRdant3q(&Rw*fbKWtbp)_dYlmgQBWnEYtB(DZ~IfRu{`r!i`ED_}R%kWw_`1x-_4Jv^@ z*rmrgS-y+{b5epd1?z4Jb`sLraWE1XnV5iSHbG*knB42Xu%COc5gGOFU0zBdk4SSJp;K!>#pj5Fiq+pame32?&N+m;>m$vVt z2W-a-9NEc3K59{1sEZ_GPtU**2ZfZFy!GD@n;^a#q*RSA=sqHI^ANFcJ3ddI zJjweF+X_L8Wc?H_GT$NIw6zC$F?b}TcmT&Q!`Cl?PLicu0u))!DJ(zzdnZwvVL_62 zPk06*Pu z3AW15&s>8?z4^UOB?Kv^leFP=Tyi4i^|!k&B*Rs7*evf3ca47i_(zfG6R z$B;l+w{IUmF38&6p8L+RRDC9(WRhwF@#6j(jwOv3cN+Nb?!K#gmay?L^7BX0R>~rT z0IlJ5O${eL3Mli%-M`3pl89`ENzu{a0j@v9|B%T7ho8W>56hU`!=xk@C~>rukf2~}v4lyX>+yFTcyBvCVwsf5vhawD zXuB>JB9(Frh_aKp3{X>8unmq){4nMi*|QXt#kUKF9VZ5apj`vMGwYv2;lO|pQcRl} zy#>*$jwEZxc|$qae#-M!M8u<~M%X;~WjvTNbEy18AxFg12TuoiaSW}9gry;qgmf7A zuUEVN^ZxwjirDVhmfFYXhyzS9?%A_PNkb!`N*3`XuJJD@6f7$Q5I42x*iYt->8ZQ2 z&QBXdxjGZp6HW~ZK-sD-(LUE+=AQ>nN;WDOiP-ynqDaLO8?6iRpCFPgFJ0f)+bg(x zH#Zrj+q+jCoU#TQEreF#?b|uP8J7%sly2>gej&6MMj;9>w6~>XmPAeMx}^N0xHut9 zH4edTgLp3z{(f^eUa%fX z@&rX8$BF$y!XtQ&uc|`Xy&@gF(Kbq&nhg1BYAP$2A}0xhu-$R8Sx*+$nV^WsesL4N zrf5UZt1rRyLYL3+O52HmhU#~(+Ixv_Zg$)x+yYP6{NLmSbqaIQh>y8PJ7acj>)efP zgN(5O18qNLc6KO?Vg@QUH{8#M4+YuYkFo<*sgSZ3vK} zf~*4>!gHte_wxMy{&FRHx|+Veest5_Teo6uoli<&E@&C`34^RTfBFzN2Sg)tlhMC6 zEdorjEkG~KpZ9k1@bLvfj3}wDW9jOeEAA=<5D6;ekewGijUDwRK%m zACp%z$m9(G0PH~ug^kouNoFN9k8%%;DzZVPA(J!c*Q!U4@*s%+338tA6!3N+nOuAP zk|^NT)}9!AM){RnK*fEDdTNaryW!Ag{XL95pvPJ6o>Fgz>B89$_s7 z&O0}1Mked;5foIIF_qgp8S=AFO{|UpX`=oV&CjhtNdLON71Av^cw_rjgG>Fnp6kYd%|K19y!Cc*G`>{Xy(wj20Ek&_#9x=_<`PCALc`$*3 zg*&sYmA%o~*{PIq?car*{T+V$j0Zlie7?HqLfUjRD@DZf`~pnBZr`=bN04y>zCgf= z7LXkY#{nb2+8_b>lNUp@Rezjp?dZ5h%Ch$FFi3BzBrAX(_8NPbqH=R{ne~oRs!a}u z-*!j?P3!9E9qE3Xv3v>I%kf2-Nna2dV~Qg3DZcKAlcmz)5Cx<>TIyY-)U!{&B+xog zNoXiIfef(?2DhdM0kRat!U-KgBiDJO2eH{Ex{6GUYusD{2xO3XN-?g3eqp2+Xpr(m z?bc1O| z`A;QFia~LLw;1M%WN&#xo?3zqD}-`?a76_Ude_nSdtvk?RJo35GJ*IK`au2ZU)I+s zC2fSE#B$d@zxd4?xq_7z{8(;@2e!?NlVvx54rYra#!`d% zq`Wb)PR5gaYZF;8Qb5AF5WHw9Ow`@#aSgstrbmIMFb^U;7xe(cmgq5MV+eXEsHoD~_bmRYF6;*v@0)mpqUqPDdkG zKy*@IMBY6Yc`+YGdJb}A#b_uf{mNK(;nMxpD4onC$z;>9gghuRfW;PdJZY0_}tWYM@f#uMa=>Ulcvd0CJ(I&$4h|#K|vchHI#Vj7$l3Yyo z4)PXeFjyFuEz`eS`3b@^=G-cQ2?Gkqt(pS&0v^dGBZ9B=0z??+0bGMHfqq=1x3j<30Vt*9ZiR|-UsNJS```DYs1UY77K-d&*76#e5|EMxz zI}!7X*lL`*=M|dWM9ch~pZn~TsCfOCFTP-2&CSi$NO*xuy{@i4cnH&gg2WomzV!Ws z`73gVm~`v@5ETfZ*8z-#au=!;4THY;!w1a{tK%T*K)FMowd754?Z_M*UHv)>SB3G_ zC9y6me4uoJ)w2UPA+yVE-?SSe>%u5*r;w#{nw!a&bfH@qej~1qy48A2i~d^gooW+` zJL*Zd`~h?hA!j>NCV15f}B(_CP;|Z6mDNuUQTi|E5@o}rJPb z)A<+q1rn%2BCbPs+@~kG`3D71-FmX-k!@-E1P=>pC{ZJ;O@ z7xRqcK2}Q??_2-tikyDWbsNIs@%NbBw~$%3_~wOmfE7_pvia3g;01yvvXCf6Mutgp zMHmu3ZdCiU2`^G^i(EVf<| zKgcn;$|AcV+Ll0f6%5@k$f)F1eeTSeSWLiU^zLb%IAk3(G)YYH6QBxBj$2eMmw)l* zbdKxQ<hj6h#aVXeUd%!{;W=75+R{kP56m2%OtZOX$(XF_ppz z(pb(XwfLxN{BHiw0Z_dO8yRT<@NST_3({P^smd@!A%s$tqoX64$%UniAj@kk!kW}0 zo7ZpLi1SGjB$fZ=4^yp+vkByr-Rg4D=OAR7A(Meoi?~M??}Jl^ zZJ*Urg+2&=iX7SeOKLVD2cCTK42lsNKKn;h5f%tLkWU}&|AXKt9Yv`iP7?`~J6CX;y(HW>&)D11;fJ$O}P zZ+Gn8?F$7LO*osMdEsifYayDEQ1r~uN0a3qat`bQ*RDM+euJB)u=2s!>JfHan3lzxexf#4cAg!>a~U4e5q z1pd1T?wcn0?C3=80;lr>Z~#X$wX`G$)>!$ej=g=09262WJ9g~gK!O~TW#?rU9;mQb zEV!WX)~#%iGZh>ha;s#470@40cHcYlI!x{-)2m{i0Xk?X0%L0gHX%rao{dxtBgi7A zopGuVjF5kqwFQLbs#^~qVg!Sg9|r=Q2)^F?*&M&SdGlr;KfiUNqW-Kb3wJFb2wfV| z3t_L8qZl`!p@<{AcArbrMyA#$4_y4nZ00BW|4*D(X$_rn_fN%!^-r?J%CR_5=MZ?~nL1(;< zuwrrslG%{2)iBt}CLs6t*2dd!Gc7)ET#!07E4 z!erpO#vbv7-R$NB&FfZF)RU17^Fk8B!nAmVy!y$in{|6}O)bR0Y&^ zr8N$sz;vmK&|zRHOcz26{U;)R-D2evdoZzIZea;y2kUm~$pF6wC$aryH$8^rsT+R>HcqxRxPmS|;^xUsSN=M9~lhWD&?fdPiSQOSO30^qx;4bnHd>91sSo>R;^k^#`~~anJI&p4qNcvv-yPn zebg3#&@>w^2{O{+e%l5IL(uz`H{K@W8ubo07B_p$G6#)V_GrLMICB0-e8gkdQ=ZUW z&r69dyge~dq{)B&{0V`3vKyyhblGCh!!jCHRT0>W7lz3IK7i^a#OxN7lRNoRapCK0 zHmMq~QB@pa@s9lz9SZpvDoHzYRvAtxbPI!{@_1+*WjxF7r!F~Y4I>yBZY35MIlqde zM4xQJ6a@&K0vd%W!h!@Tfm4AJFGWUmS3nogBgwpFWl6v~*|t>+kq;=oHOO^fx{{6f zm^kx*xMR>q5?xN@US#rD@kXuUlXc<%k^??dMQ*7kNlPGyUjo`Vv57E^%?a(5m|hKx z3;T6UXf@O}(0_8aU5+zEF*5v5jzGYvMZ7!CFe0B(K1S#wP8gBJo?FSj z4-BKT5bE?Gqfy%FMp8xvPMkoc^b~?81ZR`8aNf*g=#|WikXcQl$m*x+A4vItEQu*r z4|BCfRIEY{w2KLx(dyOVlepOhbJ4Bu+&g-Xu-tw-UxtUL2eO}kl=*)R1T_4 z3O4fKWMP<91mQ0gqb1S}vS@*JuY(~Ws}%PsPXY;J^7zr`@Mx4>T?-_O%)$RSq09hU zq&{{;kQ9;*4ZiC-UKLz?A}Jue$sEOvM~4%`ZsMQGX(sOO?wAKqg8;Dc&|Qk1J;b}P zf8jBm;O6G`g-~+}7dqMf9g^n@$3ZsIx{&|VdvwSPN6&frX{?WD*HcpRM*T%V@a);^ z$mkOCMGm;wbE!G`mI($5NkPM47TG2^IS}}<1EY~7V~t<*gPOBBOv-V94-bDSMQlWv zz1IL{a1|sX4ZY@>uAf&kq%aAmp;M3XEC7nHA!gAa)P_aGyX!m$b`#;>L?VT~Pr?c~ zFMtN|Jop$2UCWPdJ6Yg~Q~#i);OtCt02`m2D=(Q{8y~NP-&aGQuFW1@Pej7C<~*Cl{!HIDrh;mVLQp zEtFkmEhX}PFwUp5h6E1rdOV@a#N)imhUb?Bh=lzJJ$VhVj0);o*lKp9Ac9G+hjc7y z*UJG45kE8rO)yG^Oce)CJcrmtlJ|J(rEs*3+>tw$bsnKd0V&5*0Oez*2s0zx16d1b zScm!@7LZo>ZakM_q?*GA#&?s$xKP&De-vQ!@9h8`_k+aMfhvYN%C%sr!aF$n=E9jX zK42=PNF6{>uYixtt;RNi5lo25a}xVN;v5S%~H%wB3piT{*uN>j3r155vE&QTXuh z^tX0esep`5jSpsgYO6--5}_b6XSU)|p%C7w*j7aSNrDA|4pRiPA$cz1Vs%wj&_-60 zEh6GF;so)yAm8+mg8{J?#+Zq zM4Bg7DP|)$)sK^1ieEglQCW_j_zI1Y(thKCj0aVW+;bTlQ$GAg z&5nMu`h{zA@eYLnN+GX^TYtlQAnGVAMP()xH5K#F{~$0lRx-Lrj-|pPP=i2ieKB4=kyA1AK@POTtP!8IU9$EcPK#?#o@_x6@IiR26(slCt5>h8T*)vw^ht6+ z2RbYhYD^_=s61sY$u0pBf^@b+3q(qi9YcoKFh>{B-MxaByw+&2KypeFQO{xH0@xK} zP>~#(L`UH~Q*wL+8Qit}1(=EBuY90Mc}ALY(LhQ}SI8p13nXRSh4+I84{Al07SFp) zuB4~YP1*Hv3i5Y&#F2EP;}-XTlqiupoSn%o*dv^zq4*p=A845&gbH#-Bf$=6n(ZSv ziU-OJv9GPZVLzOLR3Pa*vK~=Uqt$I(I;+;lOyPjUaI(&ca84ZP&r&B}IxI}$qo&q< z4ZZr|C~z>UF)z-*n2Ctw>F%k-eYlC0TA-sk-@q7{5JerH z!+fL=(JespmZ450&g{L^T#U&S5FQ#E%YKYqkUESp5K(J4l}4gAg2j-$2JoXdviVgy zA5pT&P9PaFEXNAWY}FJkoZLv32eb2g9500bnQT2G)Z(6}Aqz0w z{uC68JoEG4o^HUH3!&n8F2fc~AjgR=*xwX|hbQuBDosCy7VH=|N~TcZCM4@d18UPz z7({wBRChh~m`>qLb$!C6yXD@O))4d`noB%!M2~T4YCzdU&q9WL*_u6v_-MpsWDI{+ z=3}8?v93bAMt8bYfYZ%IKWCxBVZ@f4J4RCB$gJVMaq9Mn|2j}+g6qMoVZDidn)=?{ zJcQ^-E06(+C832r4pX0tnuEgo{@HgyjMXUoI{JL=;*+=7fs>~x)k>s2%8 zN;pM;Z-WNt5D-zjoe{&6ko|^`t3P4JnoRFS%4y0yeIUq4c|s|Q19~JbInI$IC%~Z? zQSgEh_`s>nRZA1A|9Q&KuS3JIWC^WjiJunQBy|c=G!j07eHfQ?XXk!$>~pR4O^Td8 zOG)vs9$E9g)r}k^f){`cpcl*}a^wW_&k<7rb^*B{;squT5NkdqkDTg54n_^(GF-?f z=-7}cR#X)3lPF#jaFQ~w^k`DWP-%tdJ~4K_`%{P*dZer~7iMv}s^8*BJ}4Xi7ggs0 zmvbMs{fq29LPkc4geY5t>||tS6(usdsZiZ%kkLehjEb^X8X~eALXw$PX(FUTrRe>h zaX-)d_AFqas#fZWNV{Q&I~3A5u{K4i7ebLDT6Tjgv7coOr}jVlH0u$csNX{U_cGT*nB1Fo zMfy|e-MG=AlsZyU&z#bucFtNuNx9?$FDmB4;%q}ArCgYH@~4O46M%>dxM1;X)R-(l zuii?pqhd(LR2r`MhxcgHI3BloMBHdtb<+NgX+22yu9P* zCO@sGr}3!nRNgD>m<+m7b<|R1J#9Sl;rrSnr-Uw6aejBGb z8$}fqXjfqn7zIV}olP>4lSo5+Y~XPTn}j*(=&8LJZFcoo4%1{HK6@k$sZHpD$H`y> zKb>lx#JCvIJ7XP&e(C$@9^}iz*^Ys#l4hJTQ)&zf_1UjaR>S7sx)9)6xc04O8#M6l zFQx3&H-2r!YgJBThlS#jEA^Wq6U;z*O$q8^gD7rP|e*!@e@~g zr|^<$y-jWa9AcjL7b&12){GtyyBhhe!i~S@-L)4Ka91s@2E?!Nq%NNH;ep4|`-m2B zG=mXX0@Z`=)v-pUgJh_t^cuionG`)JSI40R26R~SDNRT0UD2rYd|aW6qa9G36tkkN z2^!`oSUP$yu_CON340hmC6|81c5M+3p~7{(>V6Bfs4o%-0lo#2;+sj+O+9mijI`%C zg%Z_SQ{LJ*4sN*1uYXsi$F^k_thNHuh*o1-hYvE`F3lRlYc)|p`E(f4f+%{6cUz+x z=Hf3)mW{$(6KHe#_D^>f zuU_3pakoUA*(tw0EO`Rz0RaIgAH2TiWRF^fXl?M}KjDb>oH=u9xXIN2x9em(Bv+^tHG7qQ z|K1Ai!mQ_rhfJ6T#`U3Q;kKzYQYK3A@pER*_l0{5JH2k=mB%G1k8Y3Q@{dnF;Eud= z8J+BO(XspF1DM7m=ic-D5{*5e*2&%#%uVPN686x8!ICO}gpO(sJz5)F^&Ri5!Tp{y zPwvcL5AiztLD+d~gD==hV>~89ktWxdCb!$OPCP|X2{7tQ7_i&;{Jp({c&& zA!so*GCW%r{pD81T5dm?6|;_(lM<5q)Sv?s-jHWh`k{2a0tx+#LH6nes02hu#~p~HAOGXT#o?j$<;v?Cr+P~`@+9+`Yil1_K7ly=@Xy7O9levb z?XT+A53P7jKx~1D{_5Cir;a@8*muQKO<}eP2%DH%<%4bGvdavd;hmcK15r8%aYFJo zprsK_ILAypG_yQffdlLzs0B2az?|e7 z)<7VcxPEeVV;KXMu7)n9@4$guk)}(OCG2?4m?_OktL|J|oK&V)>cLPVwgdZUrw{;D z95{F+)foD1v@S3e-S@^|T|l-PaC(PF)Ks;4hx$fF4c60BQUHzlKc=`IJZFM9QFIyV zF+tB^is(9*Z$oTk)a}W76f&R-k!GlQb1~bJ&+4?$xV{b&-apMJ_+m5x?d(_jmR`rWK zN=bL_4sStbqkY;9FtiM!g0g_7&dQJ$hCCO414=F2@_uUp48hK<-dxrpmuv>^*y~U^ zX5_w&BRN#w&Es7gy1?BM^=N;qotx?K-VL1eeszcC;L-+&Ab>n>RW*d z*!IoShL&T-#y5^$nv2XM>wH-fwjLd!HhC=HF~u z)T#mdKbD^qZ^=;-&v)?$IVRN@^LV?X=X*{7*pQ1j5@x5p<~rQqJz5S5=HM(b!vUt-lOH`wtso( zZ#8xs9#a|qZ~x!FD^_nYZZ=Z~o{jMfbStx{U-TEgYj_XPRf=F4ydYxqet5yN_-JUu z_a7%h-D)5DUJcY6?&VV(Y^$ZfadXp5t!F1I0TRVo3}mV1-48iPE8RuQ;Yb%1cWUX$ zr_K-Vyli<&EwzUFP|IVJLqA0=`@5_sweBi<4Z#T72YF@v>XSo}J(hnW12}3g{>~hf zqlE1siBx9#Kyx@BX4fr22nk!l!G=E>MQFJoVei-X7ov}$t7?cn7ud@PjcfJCs5MzO zb&TzQ4`CyaRY$==`tPtZmQzX^1&P~?|L^vW!yDMym6)ITOT0XFVz!QITnx&&rJd=j zGjxGrP-qMR=sR&?7;Qh`5;7J6@@S$7Atx`d8SJFC>+h^yNYiPc4Khz5;x=$+&FFtI z_Iiil3vm9^>q|`uJcw|YHPnl{?DiwrlJQ*|iZ#N3onOc^>Q$P)Ov8b3)$ z7`I?OeZ0(D%Dl3GqpL6)CE<_awwLfCp(^$!%=b_; z;L)L?RHu>V+H0s((QJ+c-(c{wfixR{wAu-G|1vMm2->N3uH25`wQBWx9b?yBj3IaC z6G^KlGSC`r$u0J44haxv$w04+;LOzOy|FjSkLf%9P6#^F(R>p7$}MOv*T<2&ba;D; zn_|;I(T<+pBy^~2d2SmY6`h*&+W;ZbzkzQsdhDgQL86PA+C7b4y&B0>J_<@>{DKLy zi@8=vd)kx;!~l~$nrstiNz~XPQW)TvX=VVnGU_kbt)wDYUU`%_=opxp-FO=94gS37 zk1{Bv>8=f9+)MY(!kxB5M86?J+VX(Sf07BQK-$3fR~tBERN?^?F9HLJL8yfUM1;Q< zaDbgM2E^D&)>r?$P5rSlPstk(pd33??5(5Y>#Wf7|BX>oK`t8~=A- z{Ggg9kJL5U-S5VgVRim9=SaFFU1KEI#v1a2ijwUsQ-}dkzPpmHqygH-}$5c z`FYqszR;o4g-TFkqHOe9KA*%UQ%6*MttL5(>mNwD_p$T0(PQH2GfXnZMFguJC(DT8 zX|uu_C@Hz?mPVX0ZN;y6y_zI!?1^rm6Y$BLUhCpe<7Jm2T$ zov9RY|0M^c{w$3tHJkN$RoO2l1`QjHH!O4W-;%z2?1$dvC^N-pLhFgB=4`HKiwUy8 zzo4)EPnVJs%CHs`RW*&l)tffShlaFdQ-Ne+{M#f3`QG#GG@>I}eAYY{76*vLA1A$Y zmsg>7mpp=ea6x8S14Cjr1%VWDT%e}IhX>_xK|ARZxL+8jmOX#lV5sH$Bdzu;n~dm! zlaC)-n*SDZ!PyMLt$M0{C{K5P>Jz36>!u9{qcqMSMXx&fLOgX0t@Jj2{ zn*U5skx7UWUE2xcmqIvDE6p<5yKC1brMgWH)6|>KpD=T#vW)Z@1r3u4Y-TIAFzM7d z7O!wfDMl5YJi~T)c>LahzcC54f4jL&cIg(Uo$Rbs(?0QQ1p_C0RDRDZ-^d%g5H`D& zUs%Y+cH7SRwlcUA+F+f9ElZ6UQaJr-6$NM5&O$(}cJ_wSW7{we&O4Q8O%I+r_&AsA zHaVBdPoaPYyq%mpxl4+qS+1AZvo^RdQ$u^!K{>ZZggM(}J?dim`hA8c7_7hwLo7Bw zy@_cFj?jD0Itn!7q{peX*M3)j)ud~iQL;cT)k%MAw7YzD2sbb>+xz)k&!dJR&y0(i<2e`6`q*Cy?&b#w9UPJeRfc1 zET;ct>UY99BC<@i@Hj_8_oAGtM;^C1>Q@@iOeQs$0N%h3Yu-QVBE2&BRBc5slM4!2 zI|4vLP}m42)DfMbc|~XbujZ{rtT!yX`Tm39;jm*{w?tThK*)*#8f*+;BnMDfs}f(! z_fAIgOG(F+epbyeds=f=vkVYN2e2TSEES%CU!jAeGFroT*r&??gp5>SzaoB)=Aol$ zg1%C8cz+-6{8QQK$ERC(6bf@Id51!M=LB`bf(|gKu-0BwH8-i~AyM2yLqn(g(AbMF zO%dWwW*)BIeEoOUwNtlfRZUxadzqgtVyVVE0fOx9UtFGXTc|;xOLVKPtO7|>a?)gG z6pr!M+qbHG*E<-*SBuZ2yq~$3CbA6triKDQNsR0jLZ?}hkpM2IUFI5WG!U30sWV^pj&Qb-XHf+ppXkBD5*)U<#9bZ?pszW-+(7@=Cgmb zI2HdMy5OvnnkYW%C`j=tJ!hYF!})?nwMLFnPdzpQ!3kgAd`Z-N@1M*Xx3rg<)*+6m zP(U({EfE=7B8#r3-5#)9t)tUVmDN7lS%!av&k*BaR6~aX1Ccm4Q4D5p6-tE~3gE#t zc6RQ-;-}|?HDLmc$;uv$;&G{Sp|`6>Q|MUf9I?vx6#BJi&q<2|xe|%|rXn2kqO4MD zN?So{;v5}OwxA|e?S%~$|GQ0v96L84E%-~3c7wV@>Yxl`-zKoe>2sZYFw2*fswKX zucEEJ`B-=0D})}NcVfqmrgG~1`W_RFr8#uy>4O)uv$Cqag<(`g7B7q*-Dgn;7mV*HU0c0UVpEG7Csx#RIRg=lFw&mr z*W%<%oMW^a8=FvViK4vr`2y**S{cQ_N3|4tLK|5b{2pue zdnTKk4ovR|bRZxZ^GtE#A073X0zritE}Q~4616?McE$ClxzH+v1s&Jl8+59uXl%-h zEd?!6V5`_n?0~B(GO7go)_mGKnhYhS5vXei415X|>2(K2m|j19okeseOM265olgiRsOMY6PGtjOzQM%lgqOxt+sC4 zR+rJWMj{8jpU5VlD{mEd9RoU-M%CZ4Tu+GWMvWU6QDJHIszOyFst#s<1beU92V7VD%M)ZHX!#?xKR{mf_c^6Y1fgH+6H)I z8;awt>|p|;w;-X*CM&cf3eBiw`TWL3n3HJDBJCRv|J_hXI%q6+tO^aBc{%9Ws6&$N z^~Ye?s2EA=K4Oly*U}^Ra(}RCNHG!?bI)TLhG2-HXRjI0>dl}k80P$QrsXdtp|aNh z9Rzl+CrR_zXEEnRx-(9O5kGo^Bh%rC;1WP5XYMxSXw`Ehqirs#%85rC1}KSz9(w+8 z>rr2RMe`|?P*!DA=o_v>dk9H?()Kt7~a`J(fx3~LDErAB& zEm2}YU{yNN)Btdf;(7rx*TcCMA;n?hsX~NCd<0Q)DZUoxP9pN(N=|Oft0aF^8+Z-< zEhwRJ5z6SFonx+E4dr%g1s@XDO0W@mI<(Tp^T!hc6bfW!JjBkM6{a>vcl~*rc zRw!ipI%o-M_S;;F_;0rX0ct6>^yv1d=kw67EoB${k7&m)hH8s=O4h$ISp4+)6J{o% zo9oe6Yo~Vl^8K6bf%i`@stU~sPn|U*Q_)OWd9%O2zuF~LzKJ*xKU&vp&r#SgnXsX; z5+sbWSujx%&u3}Y7mFzpL@Fd+Q6cgK+Ps?J4L7N~byz4N3LS-jB(g9MJVK^=kN>FB6%}x#7}V+l>DYtg|ctYxS>W)NMd3vR&c*Uv$Gme~j=&Z%E|^M<^(^h-;$uJD2C~6U|VWJ-UoFRJdZQj&_7U0p4|1j6z9hg|=S%N`pKkVzR zK3^**_(N9{dmLWxb#DvQ9{20?`NcI9H13|_!9jgMww$(qJlYEx117^2k*Ex?%QG>b zV+kcJqBFigUHT2NccO3WM`{+?StHe<${@E}(CZNF@M-Y^`8&*VjV-46+*B+iXTLA0 zmSVG`*(liX^)2G?%xG1nzGdHwaPuIFv-aXKt0p5-goMSk%9Ny9B?HiuVB5uBtPu$4 zrkyRyU%G0Q&+6;tb&%P_0rD7A!+tBw=p<{QK%=Gn28r*;Cm2K7DlXg5r`A=ouPm6PRXg=INJXeoZ*aELqXOSmA0s+K{LFDFP;1W0_=H0gYT_-lm_&f z96fM$)+4Lh7m+*=1(*BxDLq*u=EKL2A>1u-M*xC)`do({mCRU9TkJAqmQV?z20)6v zBHyo){Nuk})q2d(ete*@P)!c3hU)CRb(fog3L1`=iDAD|~*RM%HzO z1m0$n(kJ-zsV3Dam)TyJi+MroWDrnhqgSlZf)jg|cM$t`QuAi)KrT+5;pE_OSj4Cq z{P&Vhr6Tc%^Sa0X5&U?qH`RU|e)wXsL|=xjAfc^mM~xPqC_ zjtem@DTb{P)35%K{*8$rrIu5A=5t*U`KSz7p)5=fv81NGA-6^<9-5~bIX+Q>{!)J` zDVUM;Aw;$Qu>9X@wT|8IYjD$^1+rM@UY|lxoX>5(JZ64nMo8BNR(E&-5_XV+tkCl} ze}bZeGnECB@npSbr(TmEPE^x62-+gmfrEpCEQ=6Qhm5xst?9`JeV7 z+R#LAU_1Ea1|vM0wW__t{`1{}t6F>XqgDNflSGIWV;QWK1#~Pc5C^$wpeACDOa5oQ zZ>L_-s>)#zYkp17wYYlR??+OhP8ChD7HLB7(@LU&jg<#7=c+3n}G(w5m9gQ^2$9A>kCv#@f z>%`X3u0iy7r>R4ycyVaEaW;&*h+3C$a^mwee9BIu1p>=)j0+5$9J;U;#wSnVk2u5Tms*u;*I#eCO!pZ!N_hzPrO(8thTPZ&?6 zTx3wGx!-e2JS8^VG=paJ?jx(H#Usk{gfzI}PiiWJWs+$z>S<95ePWF?`x`{{=XR*z z6E|hjG7CxLCyhjq%3VMAxi8(GZ(~XkBfq329xkfx1H4x%0>7 zUu%=@TJJf788#DjE6Z+z3Kwkrh~i4}lET*IvO|XK&GVZei+qqDwCQ0g-(HnciK6tj z&@ZwF`pcWSmip`&hnCXpbsyI%&{9PvS`FMS^r5!|^OXir0DWcx3^$16jX(MtEmTK_ zbnP*^kM)hNg8g^e*7Rqz7Ip_B!RI^EmHQjKXh|d>p9)wFj%oOT{7E`@XDjBjY0mPu z`qiylSGI$iU~fRiwP!SCuH|)6>Tvr-FNXGC#)oBD1gwa{jbid0@cUokt*mDNYs&@ z)a&sTp7Y6Xlx5Pf^52RIUBMu|mul-AM}c1mnLnrZIcu0kjU^{k}gSK;bsnC{hc~SHXDDlMMw88#I_+ zPN~Ny){relLjQ{oGF*;$C-Z7X(}w=;mSrkItPp&0PN2Y4wqbh0XpgRR4BhV84`xK@T#z)Zf2(S1p&|f_v-w zReZYkaOm!Ldn1@;D?X=c0^E=X)Sdv?};OxqAX|kR8bGY zEQx9e1@%^f_(1}7w^sw0nEUUKdYdHMimkwJfv71&R|y=l$mSDgOhE;(7cAXR-}ztN zr4Py;I~XTYw;6I7qFS{y>ef$wE^?)uq6Ty9k znYHb~iO)M1+@N)@yHEw???ZR=RcQpmc94<8gL5Vp7Zok$_)u0KM4-7_d*M(QC@A2v z-BvF&|2wv7YP%Mw-w)XgoLN(pJ8`d008eqwc7V3;MHhUnzbSuZV&KpLya36N{=Mfv z&nlxZlgV($smqC)6R)6kK+A4)HgVG4UP;8??6OJr#L-ijHk+uz+sNNs9JaSoX4OUwr zcow&|HWqH4n?n}fr}o4wiO9k zH>`n!mhi*D{syO8Ft(b+@JrSzr#Vos5hQH%00+}fdE8xT-dEV33DZZ^aqkkIA=rkW z{efGH7V&p^p?SV>JFeKrGHch~Ggt*7Rtg^?Yq)GF>oqrGGkw5~sLu;+8Gwcx0Db476vRuxY@j{UHaxg>Ya=syccdxfw`rz2vZX80-GEU_L!zHPZSa+NR4|8#sY`0CBVD16_F(hH7mGZ3#ImcJ@< z9?GrcUxZS)&5j2()&@Pub#JOKChbM{>My$KR)evW>FA+|baNr!W@U3YCy>$D9^HzZ zKHyT~)1~Jq8{g&R_#ZBCXb-sAR)51_TNGlF>EXI_uNGc++B4H)EwXMB|4lSw zpjazo6HKqH_2fS@@86C`-VlJpH#`H8qDWuP_S|^=Jq22a5iW-)K3d|99CnC{NJn@C z-kDC@oj1T=?sz7FuKqFF9b$0`KwTpdnuT1PTL8O(J&pFL7e&#uB#&^EVR%92j$7)9 za~UuDw%#27oqG2Fz>Mv@-cgko8*4)7P}76;HM&2kHZBM{zax)12SByn$o%f`f++wFTKx8F4k7Z4Sfur0{iJRHPyvS5~VwP}W@Zy*#TJ<3F6slG*2GI{(3obNoE^IP9i+(M_!~ z^}c&Xr=2hF&S; zMICF4Rby+!wiX~MH%o4Nw#je8I;n}+W`$ZW{|N*qU*<80P?%jpx50x0C^aPBeFeq4 zzBeH3Amt^2S&oowJme8|TW?q$B$Lv4dtgs@O?C{hdL$P4v1>PwqT1@1#}!N|*wfZ` z+&}4;$Cc`g55G9+klDf1W=SXiDL4J#y;r%%bQ9-0fy!+0^lhPGxRcptbQp$+&)xE; zw)JU0Xgd&r4dm56moKIXR{hB{%Ft2|c$IRgDf2G;TPb8At{r$p*SPui?eJF>^KZB~ zeT^}?hN_C;54a@`J46F~_o$|^pQ{84u* zTwr|r>ci&`JL)-Yi<^>t?)Al$ZBPW&Vj!RiVstDyHhX(din`vVOBa;RyU=#n=0D^r zE;sjW-=V_;ek8hyqlC)TXbT~CSZt+@Agn85;k@-A-?)R^28sX^F_ZchWMX^ix^reX zf)LONz6~RibadkGv)YDa%1R~>`|}TPwHyu@ppI@mCnY%J^lu{hJL(mNf~Q?xvlz3- zR)uz;+kW%}H6VPa@^Lh$9ac1HvNyN@bDN!A$2Td?NxH-D9m3Hi8g$3rPbYyXg+%~) zpr!uIBk$tn3n>T4!Qbg$8W|og9w9~neK+znSxDZYaAo!IQS*uS(qF{5vuqPJbNyi% zUi}dE@xewUoNL4XPJtP-v9s%n{?6XM)FeA-h}~9_n%{txvm=(J{A2Tp-;b5c0W8Nv zcI)jl9}Oz~H0)J(s~6{0(Gz76u{}!H@mTAF28c9EK0c2{1XR3@9Jp24zx(dpD86b) za#Qd;>8VQ4@kpHVw%~Up-oJmpa%it!K0wg((c;T0lvt%@p9vxU(L9fqR8L^YmXXRpC6&_`nwWG4OuH7_m`q@(tcgb ztPW0Km@pm|vog{UC+{L{>M)SH?+10wae|F}OtrCI0o}F0ro{R*Yu!T>zjmbh%<`OM z$dMWW_;RvF@JNi<6^N;)&6+jUPC5<_7g*}XZ52bq-re?4Vv5Mgkrz?BfwGjRVkM0a z$3CO|x*~n@?r@KVWFF1CQpcQ#A2PPOR#GqLdbWn+Qp%)6L=(A_x=0$wlIL3>LB$>x z@xgBMcrQk5l6WY*riNsbiyq7Rw=3rY3MPYOPVxouSofbdgYAMzVoYW}m^$K@^0~B* zH=jS>3*ahNr%N9mnn)3uwQe#{Fbk-CxeyW*g$MSa)g-fM;EG3mg7 zGx23448=5RNakZE92|JAJmA`|s>u!8`QmfGDkPrGoXs1z>%R!lZ$o8KVxQ!7+&l8v zR`%|ZwQh1VaiOjcBM{1n^=z{e(s{4wec~Cwp5*S}5m>E0h}{zh>>#BC5|JfH&e^%{ zjaaOOvT_{l_xqQTC&s?v+Ar+1?-oHCj^ecP_`#XEg=!Pnalv0aObd-)N?>e_S zmnYtOY<;w=UHFiK-4o+5nVNthb2-tg98AwGzhYF))E_e{!(je*b1j5EK6Ae620(>a zN)gs-943h6+}q<#445~|N_MCz&deDyLDT)#&GDU_pZ0m@Map#ooFU&Qkv8*zQb@}nBEwB72 zzmvMGgBw809_|aNvTeydFOI7POmAw|WP&HK!v6X#yD9oIogfV`0Py0Vo)?yW>HM){ z73I6Hy|LKy#^8NHwO2DJ?EPT+>h*=`lgS&&zer)p9FI`n3@&^+zPUetnll>-3F(s` ze(_>Fvxvk`Z-;>cpIh%VP>#a=T~WM%FD_EDiRXqso{1T zZ`hP4X3nJ=#$wQiMaHk^R@$G6AGDbIlR;xQGnexC1v?cfb9B*n>jL_F*+1aAL?Gv+ zQF+LRNeUXS$9>lgJJ2ki(KYi;(qtW-8l27{0>y^fPe757T(KT#O&6mEs-afd<)P`3 zg6itoOF*aMk7RPagEfkhZ8iA#H=#Y}Uf(c;s#k;c7$BdwW6d*2^h^+4Oxyk_K7I@T z#dz;MWM$wjYnNY$6JXI)3S-0$-}`!zEblED-RSFv`&%f1bOFrmI)$d*rPoJQ;XUv- zt#qDKCP;@3<1d&DB+*>;ZRe{KRAO0V-jDP^G>ao!KFB8ch*<;!3Py(pr+zdsMtvzb zBmIsAQX~kP3#51}M`lCQc~Y_imOvmDu`!>)k8mH`!UNf#zX^$rI2c+CzmeCZ`W>5g zI_KYc^(-pmkXcvfTTmj34Kn-!<&ZD%5YI(HRoU2$1c0Aw^0rPc6F?}$tD0=F*r%lu zXVg3T#1Y28(QNA8*#7U(&MQ77rs3W9z+CsvJZ^Ugba~z}qE8!dM+==!mTM;ddb zN^wrB6UtuE=zZNVb`w>|BHjJOs+%RA2f2aqJ<|z=(oEx?+yxg%8Xhr|%dB$QXZ&05 zRI~V$l#~T-HB&ByD>S)^Q7*CRf|gijJ&?!%>e;Mc-;GF`YRL)<$i4t=wRC7v2NZ zs?YvPR`0zT1kM6(V-kl$UBIju7T&mXQv>F#^Esf$lK^3pBIrz51u=jDu^eBq@!YHf`G3n6cx=*|IFm&TOUi%ZM#f8`61tJGXdOUhGD#;uERa<#bvW)F!qR z2?do!GQpf#?p3Rg_G_zU%f=(rJ-7WL!YI*CveJDNb-;tq^uWtF)_ED0$}3%_GKs(_ zd&1=#*{+v1m4y-A6>;;T{^()VlJ0|+meX*d**YKJ+8ybxB_TVnBALT#Ll42rlsZsW zvf0?$W+9s#v^rl!R629Y9z#GYA-Ll~N$2g_-Q)I# zsP?3W=axkNyh3!nhbISfg$LuKMdG5Nsj2xe#{H!ECSINSzg8O2j!KW-RNEunEuhq# zSj_nOM8pkr(UF(Kx1TaW`M8gc`0=IHWo||uB=VUaz;Os58$EG+DF_-I;Y3~G2l|$)gMl@ zF~z?3rdl55F8wH1E2$-gfrc>@ciHRGVSQ8i2U4stX=?tdAG_4t9Fpa> zVp3y>exw{tX^1K-I@Lic--(?Zl{5s0Eddket(VsiPF*q|#PP?c{qZZBp}*jo`Hbrd zB7d{wb8F#QZEbZEL8oTt9@LoBni_fc;b%H#QP!^q7Uw0eG~THQ)3J3Zn+Uo)R6qngpdsAz1}}sF|2fyZ^^l|CH6W5zW_f#sO(EI_Xv2& zDlu$9i~(o)?w+1)A2Qz~1k=J`=m0Y_imYhoM{eMt1In^JmK0|#mo$0G)j zm4@XX*1K^w*_CFUS{UzS0$C$FJa!(AaR&e-`nTaC61TwaFhzZ8l}K!?-=s-9A;>Ue5i zalh;*e0B`dM5@v#t17Olg=!=WD&cG*sBp^-rV`1TJ5K4TF=5x7BL=m}SO&;eL=Xn; z{vc@hLA39Ym|WW3K&36b$|Ix;CARJ&mlHT+clq-bX&1T>m~!}+Iq>-~ahsazm$?B; z>CjJL%!XoYan$hoi1N)h33UC4`yDX()6YAO z7rKd~x^;*CPY*yYJ46KxB~1Zc`S2nvI3@%8Pr(v}%pnP?sjF|M_1oRYOdp&-=&)TC zSCye?VInv+QwY%Ek&%MSNDBoV6=&k(3eK-m7%VT4WHFIPfCP(GWSmgc0e(RLARiMU zHmQ;{Q17O_(s%t02T&8Nm7iDiMW z_l{4)%-Ez=>BrzR{Qm@=armW6`Ou0GJJJF@0Uouh;=fIB_&hga$va-@5@H#=e9Evk zqJ3okb%{<@vrZ77Jy?2hn5T*!(VIvfNrUaxx_5(S)?efogCi%D?fgTk#07h`*wr?) zi|)&ytMmzZ-E+;3^MIiR0?b%Oc!6;~1_Ba^WMYGhCPZ0#K>z|!FuNKfsjRt_Y9K_i zKNwtZdgRi{6jT=BRc3tkBraf7No%c{9$&wHwfY?F{G%=92f=R$G_g1D_X3AkzbsQp z>}v3>iYi!3l^C7Tb(q<;E<8y&EcEbkKRdF?%}c6A1pJ2nn$1_rA~?dqaZ@8@dVp*`c(q~>Ko5Q6?aT2d~(Bi}CJ!7=9@2`(TNE-gPB zB5#(=`Yx>|AZrp4tUS}bseseeh4rUQnWCUDl)31V^;@f2Jvp=qE`=i+%ALmV#)Btp z3+{t{YdjDpjHId0i{*V)%wt`lKSq!xgynXsct69QX6WdYzynk(Eje!-;0gP6kBtrH zHq}hYpUDJa3P-0cnvHRx&H3!!ecr)43G|1<50fdwP($r+*oqQ&BEP8DmlqZ*lw@#Z zJnf1Qq0(p&;<+Q?EH}_ya#!I7K*z&S@+f+>^4aOJlm?<5k{SkFIp<;ELC6clxKlVF z7Qcr(Iyub%w^7MIV0vfw50GI&$BUKpjPDkgK`d-zYh+3eFNi}drk*8RpDPPf@;OS7U}C0hg#}%JYB)-|qsP@rpPFz9yoIhm{@)E;K&qbnJ+{x^+`1Y{ zquWSzs6S}lR}hQt^E#q@?8e1GQSHT%R-e~#Vk$rS6v0o9D~De}u>fo{k`RU(=jo)x zm0vQyd;T0W^49Y&X=&X8U6lfSx;2^X`t9NG_9I)2pI#YT5UIV|OTVyK-|@wW!my$C zhxfGDRiDgw-ZW-&s+G~xVaQW6Dkj!;1MSj9m(Q)c1Bi%6ie*S{F*m6nm;h~^vUPjU z(sC(j2;REo)b&_QH+SOKZ5;-qCz$o6oNXz)9Rj0Z$L@IuO@uj zcU`x6`Z#?5Jecxr5@ga4$bz1UH80p-I5F~f&7{+PWkIeW>Q~k-7j6)@(!%w~lSsH*x_l~V%%jEI^@_DJbhejRXee$Px%0cjJO1&I#Qz^Je?EOe(hWoUWIrwKy_Y02RIt+h3PoyRn zS3C=$=9T{tg{6)W`Qj91LQQBOL zi@fNpA0M)5;h&$_eA27y@)Kp@pCFA?pI}TDAaE$0g1D#9$?jIaV8O;yru0fc;vEQV z-hWi}Z2g#B2aq4MQJ9%!%-P-0@r7vw$!WQJFdC5{lR)<{sTW6mzA#F(om9b3XlF{I z8ymmY9JM_~m zLg*eoW1=$SUxI$Qy1FvpP0ZF%P{xD+7GjzSAD6;Mq<`s#7zP->mtUj^ESmf6Gd4ER zT?~t!xv$Ci@rh%8C1vi@PBPM#y}?up;=Q_NO=r+r39R(I=y)3unb4{LJE&L;)zjkx zADq*%B@OD{3pA$GRTi_Vsg6g22mjQQ+UA&QIPW92GKI* z0M`HwHg}VnnO}fG-TpE6%NGj6!}k{COMq$qt<50|emOoX^L*m_*Y~V;EG-_$Y?=;o zA5#8U{-({^wd#CH<=*rBffX0)?k6kMrGpDsS=LttK9dIB`v=P+aRZuw;xV}hmI9!t ziEtd?&Rp8MojL_ya@^m#zkRbnd@vlp$Vd+eiDX+2Af74NlM>_B7XD`ZiQAI`r)BDB zwCZ>1jVe&a$vlvAA3Np#zE+WwV(PPe_tJ!wZwj^-od)l}14JK>gRBF{Jv_8RObYXf zac~wsXt?ytb7b|@2dtRj^aYhjwf*~=p8RN>M3T4}6Kh+=6}M<8+t$BE{CwY)ezS~Gu>H(F z&rklo@EKyDhe5;dQxud_-b%Z0$O4HBj(#f4ays<>b5+x~63NB)SQ>f4Ma+hBU6MSV zr!g51&ks9-}+W`A!3u)hphUWB6 z3Jfbes6U0irXQTbn*}&?1we^VvFNKJnqOdgHE5nZi*CDnB!ukQ?BgkvC*PnZ&I~=H zql=vOYJMEnZN{5?$K5)n&p6ha9AYkX#NCy&k&1| zPva(8{LvGvT|YxDur&A6=U`W#w{PE)!a8^EEDJY$t$Vjn?*L!k@w*&?TjvZw%(`nU~c3>V$@OTx^-^*98-tg_8 zv79mZy5fx`GakE>#@ulEWjb_eWm%ClTgWS7_c~N3n$_;^?n=DpWBkOTc7q_k>EPOa zZn#+b8FIC?<(8up?CS0Xv43D!SEPPi>Z6FaBo0IVK8jp6l~Zh?a~vg;keLjg$~f$q zhr4&`^dZc(L%Q`UTCH$5Rm-~%AC{-WAc@GyX5Bj9=b*;Hba*oGaQY9grSAFoN8{qz z=N825;>BjM^@e1Ego0QzJTDvKXahMvH38Il2U%dAa}aUl$+`K1;M5*a*!SK}`YPrM z(Y3Zc6MTK@-2efE8ZkRTGQVS$$l`k=`T$Yo0cG$d(kqqgj%3!TUV{ea{EIxx?smEN zEcH!Az>}*;LL zvQfw3`m4?KZnR6fRpmEEAc44ecO~&T@3II9Nax|&XL>w$wAtcAc$onJ#5Y)_?*dUK zK;X|nst`cA>QvsdI&n;}wPfb%+BN6c;OYBb+*#E3E9LhLNPK!0(vny|RTCQ=U<)D; z2)r9 z>cP?!gLW^hMy*%>;A<-7k!>tSX57jv z(rD;E=u(@93%%NPdY=2@NLQ+{Q_pk=El*B7I8Kz{d9lxGG?l|;_^U%Nez1EG3E^aV zcMbyVK``fUe@2DOee>Y~k&<1v|3ueNJCmX6cn0jDBD~XhbmqXQpsz1Jmd>jte9}9< zU3`CjKOCnxuA{GSD2Ju*PPvjNpcvl%lZ@=di0jvnsk#Sr|HOOmql!~d!fj*JL!YPz zhXDUE!1Z0z3pAB>6+C_+j~Q_~L2g6^7>N0~ zD2S%&mv3h3GZG2`1>_{CogvK5)2Sefe9Rdy39hOrF*nsP^zWOfJ^QQ*0g!q60qKP? zmBB*v5+hKubi|=b1+xs0aEzAhFpg<+e zs8Fod#lq_rSX+V1tk%r z>`gV7KcbktXO-2bNPW4n@ut{TqTvx88z)v^enugbK%7~Mw>VtwW&y2u9R<$Z9kYu- z?Cii{yhA)Lt1mI~M_~`3hB-9f^M{raIfP+yP29iy~ zS3J`a#)3HjxBMF|sTEO{CR+e)zr-yPs@gI(TV}C5JmOwZrkL=1;P{Smqyf=_s34y( z1cmb9$GC^(5i#zZ@nQ}V*x!}G-rvnEjr@V{xg!>b^wO?b^!f1k#Iak6>n85e(lAXA z+F%k>Gc=*`pwjS2C;jBoj3Jw<&X?h^yzHOSuWL?JZnOMQ`K-;^dDpil?&z4<)}rM2 zxw8u|1{Jz|nkc3UKYj=+DTYoejp}S|T6JTAX@5F*2&N>WnoLJ?NEuStPdC>bGH`SK zmc621mKAom<(l6or1)pyp?Zs4Qu^ChH}kvtpVAza?uAaeC1R`Y!5fO7r>K4(Tju>& zhTC?C?a`PleQl0=E>#NgY|P4(gM<2T2jxYGvFm*OjCM?Z9fj>(a?ig$Mz-GP+EM|v z&o3jvGM#5A+_CBz3=BX=eSRn#_=9#q?vODetfmsG!8o0Jzt-l5Lk<6$v%N&$4ycKP z0T=5|iyc7$)30{4(L@ko)ZtugpeOACa|=!Qy)3(PEsz^F&?YL#qhqE!YSXJv_y;@W+zU* ztK#rm;}>5vJ5=mbpwB1g6iC*pH-QcDys9C300RGc#-*l|eSPMXpHyDH^AT zDk6$JETxm4UUp8-W~}g0S>?@Y0aMm8JMH9;fS{ZSySp`}Pam+li{CU_V0BuVFl8*1 zP6Tr?A+y)8GsHmAbFjT^rXqv;0Q`n;2Ew z_GF}&YI~pz8bSBu#klkCZAfhP;rS;Qy8h!mpQltw4;BuI@o{t zZPF`)$YLpgYe_2X2=s89Wp5A;tpSje;bh{)XrB88I0~w&+4*_7xwVJ6rto_nFf=4P zR(OE8Zuwx6C5ruQVdfNE`c|LQP<^a{N`Ch2*~bN@II{FZe6lG0o8z%G$He1VIC22@pRSlkA+H z=`febr;i_OFUr#VJ|mXh`7!J4%UGYF4AnSzp8tyiaexb*3CaAv2Yw}H7AZW|N{6GAXu5~L?gFlBT2_V3@m<$%xVv>t(+ z?f$Sne$$Mb&bRu44;CoN^68Q;WNl+=svk);|JW}c!);`6y_dBsIcj4m zN>0v=%$rp4E-nFQ?~p;n~JAXUxNyvUVdXz0bOCSY*`!YFS2O7 zNI!#ehmYA1Asd*gzQghisWe2MO7%Hkb;NZGc|pb@wK%&@%WwF&w6%H~#kivDB%t5K z?Zblm*{rvz(^+lU&o))vc7LeK$apfC}W_d+u6o6jt}AXasiSJy4`DXBxDlbSH5 zoCP9uu&i&mEu)Q}p=u8T%f#zG9tuc#i{M6OdwqI`YMjWipLx}BCfGS)G&l&!pZU!i01-0l3>{aS6hZm2G+j6QhPk&)Hep9$2rB}YaG38dUxg

#YC&oNl|!9ZhaA+OluT^y;*Jx8fWzTLuwiy|Nov9J(dy(%~54W5rt! z5g**?Lx9m!tRs?!zi;22q{-cIcY{;NYWF}&9gn$UcCaXOZw|H5RAnMTnp4k%= z9^9v8rmbo&qGuoj`9|bU+O0o&=$E(G5+9GfkEBJdvsuT?@;eRM>K(67KgiOXTymUDv=a(eV5UXGF9zn)!htX#3cbstGXzlofG6v-LSrN0 z3z6lUI>y)jd&;7w)5rIB>+;xNb40?N4R^~a!z>}P;97SLcUmtvFZ??oO=!$iR3_yt zfKTq(T!2&2K>Uo89fS{BmG_u_RA#beT8bcN2ZC#Ise^+tpd4hEUF`Yp*DWwp9EY?y z#5txase7;#cl^Af;$k%@DZuM};9_DFVRxg3QXLtLpBY8PuonGIC{}@pw3u zyZiffM?S~wp-!r;NAEgf%_MdV-~!#5H$Ygvi55uQ2iak|ZJ+6DFfl2RDemH6SD3C* zqy#%?;(RQRghFvF&0DW-7V7^*uV1um^=@^SU4}N*>lW`;sKq=~MLr^lfTk{)bjg6w z0FY=)#!*mAq`z6sB!n=sB$Modg8TGO7O{1=OFdKaMus$~pYr&3j9~?!`pm}WewGU; zm0fJZbJ=>w|EN>O&F`HRx!pR(_Jda7E9Z%RZjbw)edd$6Be<{i!!f!`j8=aclzF1l z<#qi6rf<>OHNr`!&NtOYBlp`LE6aG~a(3ahpvPN!K0g1fjk4CE;46(+j=$d}T3RTd zc7qP}xe~RE13>I!KxZQO534aG^_<*^B7L%)vpe*4v{SDzK8 zdb8H{9*|w{ynWSKO?`Pd1m!R7)eBhOG16y#bMQq%QtgqPn9$*GJ6%+-NG?bXc5yxU z_-IM)$J~O|zm_knW+fs9hoxG@S@lPM?{hAx$ML~^5+bpjaj0sk+ijo>-*NvWbm5T@ z_j1HQynZ4K-R`zcHVSR$Mi&6`XpF1U*E}5rEAdO%f+Uknx{N1wz18xK%>*3 zKL5jCw!ChgP_V?Kzs~W6eOXZj+N8dIXmOL&{DtXlCKg3D`5Alv;E@!kEQgoZPRBYQ z(3@~M;7jkub=)0ak6u~$*5c~;U#nx=6ke1MN28#@+*2Miporm{&FM40^Y5x)m9}l$ z<_|b%)U8pS(Pv)|Oz3bY?up(wsJ6;+U-noy^|3gZ{T|Z+X{!k^NDQw3*kIa?)1*Gu zxT=n3+AG!euaB0jR_soVJMv*i`b$@(30V`og6*B3=T4SS*}d>*{;yg2O`@Df>(VzQ zKD+-S+3a@r*7N7T9y8Zv`0;~7S9Wii<<)XdEYRL$#Kt|~s>EiSlzrk^w&{}3g`J}( zZYhg%2sC#JGryn9blOoM!`0V1|5aHJ`8NY0+t6hO#SSk6(za_XFA7GPTOQjpO~Z zZ&w(!$oFhi9i*E3#T~a+oNf8SuxT~0x2ymACOB*UpFipKUF5&tfjhcbf5k9N2$9G`6{x;a|S#zaQu~$e?oWdJ|g4<`E5fN5KY(rI7 zqfHV(lXo80zuQ1stz0UGrG%}bn3)O@a#EN<@~qAp{$pK&!+fft@%&(_Hf369nYxre z6Cw`BC{1Dm)2&6Ux&53GvYj>Piw2+ zNREc2C&2y*`?~a|+420jTYL1plH?8MuW8Uwd>sdfg1iYNV=Gd;>yfP{fjfodUST3v z0z1v2l%Y`pMEESg9p_x&e|D53$q0o(d2V|AC9G=wMvZ)`sT*rLw}$YU%K%|K1OEt( z>VH7wVCY?;;y8@6Zn{o>4Oc2Zzh?;UetgZ*Yms9I@-G zqeQ&Qrk!U(s;B9Tb{$(McpW=4#D3tro^@lltE_$^>@N`z8I9S0C!o&}5iB4A`kLIV zL6mJ`H$TIPM&kw#>sN~tu_VX=Yn{V_D$9W z0kIq76%P8A&rIp$M`fjxLCLMq*CAWX$LXKKlT-MI{hjz28Yq$&Jm#QY83S|TA%j-n z&JtZ&x9FV>)+0Fq7c*we^Py0{7g=2 zcG({uUc3G4xSzJqzujvt4HvNhQGkL}B~xN}t_SR!wCc}5n%BU&G9@pDAmZL!>ouarONDt{$np7MiPdmon0`HWSKQdrb;^k>9qft1C zPDQH;)QZOn4$gVpJ;ZJ)N4|}9 z`|X+YyPP$-;?;BC`K$xC_r#W$O#2+FMZUH9es93IMir`V0G4Zt%S!fi^xoHNT|siCr-M{Dlv`~5;az)rG)$f{uDA*ruOsm#}itIVA~rWKoX=L?h`L;lBTwyu6yoPalzabRXz+FL{ zn6G=V*{3{=E`1vF7*?OVPd#*eX)d$hG5b2iMC=~8uwBhTtB#E>c&NA>bJaLvt&Kw0 z^5K}54UYAgf299Q+a)@QnzN!e#|(G9wc$;y%kI{`ull+^R6ApF)+EB~z@GME;0EmI z!4s606~VNJ-TmzdKRtQjX!VKRLXC(sCgBihM^sq!nx4@E z>I@6k!+m3-&yCU5VP;}GlpwLKxcnGiHG{<#O+MkyEx%_w>{Gw{9_sfosRFQWyj~uP zZu;ReHUs()QY0eT6T&8$OJ$i3+?%CqaWtWz2fI8DZhu&-SY0&0jj&{Z;T+UhP=$~H5z~+;^ zow5(U+811R$6=Syp*9JZ*Zfvxagki{?B|V=$w9;Ri<=IMXNL6|rU{J5c8sY}=X!Hb>f8i} z641;izq+jSi#xWbMb0_4@F{^$=g_+jXrn$RX=j%^?T0LT+U6%}PtT$=2 zN@m9!-3<`!Uh3YqwI19beSU6a9pIb2G2e=dBUZfLtSNC-s1kZJp_I9RN-1+jdfbOe zk|muyPv4l%HG*zz6Ne2dhCGdqW>MT_$yIaAoGrtR#4Gg_u2G!ai1&qzoM=x_5F}~~ zwo5mylTcb*tQ)5C*D0v*yukQ?EBvca>)Ja_?CO%!ZxjBQ+q(5JpnhAWfI?@B19}M9 zvTyy`)K7{QNC+WBh|r&3Y~u6y<}FJ`)K2xVNO4N?(Yi;8&(ve z(Qt3ew#}3B1Hq=)U$Bq4QkD|Mnk4=vmnl+b2Y(_6KYaKwHLr>Irtmp=y8XX=`J(es z70|oJf2d&nCx-QlXOB5I8`G7bOY0xvXjoGx)h1@`-3gIEiqC? zUZc{mEz@?=o0Rk4v}N@m@}8qaMDe)bT+MBDl_(^T0eu2~FZIzFIWVg%rON-K?a#w< z&fhnF{I-vMWZy$%r|goos7TTxAyg<^$WkiIFif)4oh>91X(2_{7F)8XENx^dS&~$u zMExFDWoF)=-|;<;??2y;V`kp(ncVklc|Nb@JkRTV0wOHmF>3kinx9>R?y`nJZtm4m zA(wMnxtx%qo4&TX*l{beAOO?C32hOGbzx(e=&Jp9zI$U-Ef#xI8Z>SkpO~oeplXcX zD=QaT2YAF29FM!WklhG84p)Umq4^p0mC4 z(I{N0+kMu#X1hbo=yUo$Ncr|DBh$NEZr%Fzk51lbnRUyj{RH(o6wnW1wCgzic(?qZ z#sk008is^YQHa6CIX(xk8j&+zd zeMT;_ba}j|(JzkSQU%BGxak*E0xtHW)YSu9ImDbSvHq!R>{7QykEv=;N5bt(F`K=l zb*Pu8h62zk8OoPKV*LrlqL?))^-VQBqfJoTF>4LEHjSa&3t znEBflk(T3thd2}V{7_*a8NP0lQ`7Kq-r#`)2d0ISG3VOY*z9O*5qjfyz}&$%qty+& zHD#_<=5NerS8AC5Rx?8LNSOdY7aqCYd3nMKmpjZ%5u#onbo|?WkYgS3@e;U_KPiBr zd7R;`eJA>TWQpq?=-bd^+Wj}5uO9vQJaVV@<*)gM=N)eQs;jS0@U^wd?=;wPexH}u zN<2cJM;vU>x;*S%X5#e=9!>nITVB0u)Z$YVf7mFHA%`cye3GY~$GUeGK3= zkXg@j#rN9FjLc}qIu$Wff8SODhzLCukg$0ZzkRz4$aB`MGbgTzbpu;7^1G|d6}27G z2pKGgR}?43z^e!1IKeT8_dXQYZ4EzcELK=3eDiy3W-%%UESN056Oii+fV8E&e9hUK zH4e3G1U_k($@FRiDIxo$o#%YAlMahD{MQLzK)AW{VNrE)l2DNGj_SCpw1}5c;L)Qu z=o*i86#*x-groRA@l)`rC)W)MC`DGYUgo`TjxoptF>Wz<2; zoHz~c#j~aTTgk*=$tu)!(?M>*6E)8_Y&Z0G=GH;Q*P%5aUWsSvC4s!CWok#GG^1QDNDVpBi+5OIC1lBzrH$Xt|Pp5dg z&dc*J+TLe85nRl(-X({1B*UH=+CoL;8qkP!^eNJijO2-Lmg65aub|gsP~IO9BYem9 zyXgKD9Exo4XN994{d2*?7&jMa&2i-Ni>q?>T>J?*FQ^8z zRgFpduMr^&o`AB`__7h=5;5~kkCM#g2(nqH-gl5X>& z_LtNFmP3{xlGb{!8IYOr+@)RP;rd%9FX>_Vqx{?Y=@Fcuv3XUq2avgg~;_ zq{QH6Fd)+^G}x&@#a08Wi6bN?k=_zO91$yHU4M(ElKJeC=3vM)RKh=Yclf&kg!3BG zFSSM2kIaZ7p*>Ar?J%k=_Gcb59YKIA z_O9S4YM&lBcsjJ{7_v9p#h7NBiNa1&L;^R+L>PP1V@YcSssyUvXreROw}MizO{KN= zcrk#xTb}^IQg1}L^)t>ZV5LcCy=d+o`46Osqmc{l?eh})8}1njq2P{zQ^7ZSnXodW9>kb2H z0i8Iyz5L@Dan{;UiztN z(eKyUPMn`~44(eN<;w$T-t4};<+g@|EfZIDObX~=5QnNh-QApN#}2uN+Q|Vm?6)nX zHMr0so{&6w;Ee?4H+pYXX)`#;==tWzVQvKtriIOn?{ZA*VvQAExUJt&K6hDlbiQw! znM2om(X{;d;n#BJ(BZ>0?QiH_0?_pmD#43^np? zsCNs8CiJTagYNM+oB2(+?(HdVH=K@i@ZiCFMy@Y#ieXRUg)vwj7GY~=mn28xik!e^ z3#fL}dNa>yy6GEroyUiqslO6;RLy${n_?5?Jfu~F{EUwItKdoEs5p%s^&SA4O(OQA z6zB+AI{D*Dc1T)R{}3I)6dLd+UKRT&($x(b5vx>tZ2fl>E8+5OyVnoD zOvw8fTTN}bT3o{AkvRp&-xc=F^Ir3_uq@cI&h6F5HnD8Q{F8lWZJa8HNKwJY5iCpS zOETR|l7h{8VSw~`{~0$iv0iE7cMr5LoQ+=Tps3~ycm0YkZQ{p)l|fV4TnGjV2pd|m zRt+}anA}w#ucd0zYeMQ8eKf83J+(%+&vidSBc^fgEkrY~tN`bBPTqKP{@`xCXW$R} zQ+lY?2h245*sSuCbyRd=!|{b^H*}dQczuKbd1NT-z{&>BT<@H2-F{6Y7we!_%hY~q zt0`s>bc5T0IoJb0-IlYjqK*Yl%?icsg66W8AO2WYXlqab|ffdeCn$bC%|&sM1T z!W>am(5-`?1NCL*Oqx}7QC0uQ^4-H{iK`t^<(@cf3Wi=0T9z+&Uo(P|mGt3$+x9iP z>{;x3KR);jE>RRoe?_}JbC^DQ^yqE5Y~?OQ%lrrbwTR^#)P_YEeynM=n?W3O$s!S^ z-)bsTy@kMZB2HR$P%h&tk zGy}9F-B4%@j2%gd#x&UiQdGdiy*r?mr#BB|(kL;FXegjO*o!#U%djB}{y!r?AUh*; zp!=CxxV<%0H_Om{+qRmA%AfzBYOX|{Z>X$VZ>{OWfy|Ewdv|uH&#HJ4PxZg@)=)RJ z1U@}ZcYg2Suty*KH|-4~SunC9GfiQguJ21tN%8EgTkp^a=WC3pel9ACV?F}k%pt2M zH*ueiCZEEN!d~6px{_B@R!5-7Rnr&BxzY$XGJ=*}o8W;;WS$8{cL=_VAq-^p{U zvixY06Yag`6o6#Smk_OCT{CzN)fjGY>633Qlr3TttTlKRZ?+$rsPlS01sRkBwB(>kz=!xU4Eu_@#J?pagM0PN~yzt>e}34ae#&NCj%6~l1Er>AS( z%oNkJ&#q54m0x1$fCz;9$J8jN<&KvV?KtIW69WJfJc>9C5@4{pnEF{IcfE0oA`h5W zg?bqF_1?rQ?zhsL7ky!*;XT?YLQ>@`zIV{qmvocKy2V>X{c7O5H>-xz&!4hd?YS$a zbja05TD1Y@lqo)HJ@2=#fOReUO>ZH_sr%^BO%SOwG5rS%4|agN{|gLOP6AhOp5;=F zr4*sSN$!{fo~G}2#1ej}V7kQS*G^nEC~nNUxRx~j!|-k-!f@7TZE4nC*h_XS|DoWvP@ zvv=jZ{IBa#b}YRyd9+$MEB+C4s_f?pWNU#Dcx<8=kpYXu@3VPV{bGv$qz3cEHV^*Ic z+jX(J`p=JYpnYj==e%5IYgol{28NhsfP9~9bDrx0H;^(}K`k~D^D_oz88Iqxv+;nA+}s%7Fp1Y49AuBEx>ndAP}aOPi?&rz@7I1*fkB|H#1tB0rO zjzcCei~^RWVM|ia9x1Mh$n;#lzYnA!+Q9VM6r6yC2Zsq!Ft5`Eo}+!S z0FrM2W&`dk+$tYz-eqa+DO)|x(Jiygr4w+SE zWLDMObit{I%u;0pql6dD3ol!Y9ye{?EbhNdE#7nE2ugjvxtg37j_&F$Yn?M(GJc30 z*0Egl*R)mA74vLUZ(pB#RsRqm^rbbSH_PiNGbs_Hd4Z1Ia2UwEJDf}Y|3X1^** zHV@^fv|G4u5ES4sx)4qA3|U5J9!ClB;^C93#i?dw*<(z`BI?^f4qOa}FR=3}N$BYI zr3rr}PRyj++QGRVQRgaWHgwcp$$7Z^*g+`vf$Ngf2IOfA2*VKRP#Z?rMa`@b)LlGC zcMQSJh8J*?=1k7dM$1~16Wgk?Nbs^X_|LTX`?x@yJTt{Z+D}OSE!KTRopF~))Q4J9h z!%&EAt(d$kz@Eb6E>I{5BLWCDKXECdcj;x^$V&~bK=O;8DfQB-UdWPgt@+e&wom2L z&@1paY*e+JvV03+Q{`a;;3*h{?L1;cJMY`Jp9+cAj{v|tmcs?UqWK|6jQ2=|yCplv zi;fxr^wBtGl6wQGg}W3Cins0XZei6EQTF>ssdh^%9=k_mFLKHcZqi~~{~K)s zPXl@JHoAxSp-a*FGS;}WLgXt`qGI98!++k6DXm!+brrBv(h8*_QqlIJMvjFEq@ z!#>yJNrBZ$Cd*0r<47pC4{2+0HE)Q_<{ew+kbWxz1h+VB*s$qO5`eLgZ-jhj)IHX~ zpdwg1#Bo`2)GZ-W#>Qr(?;Ulf9iv2G)-*X?$i<8|B)KkTA)z%N-i#IM?qB=V3m@DN17?*F}f5_OdLkkEZ2Bo&(h;lnwb*DOA6?LVuv@)Div z6ftAKcV^=LPD3w-R_I1M^Yt2EU&vt{>wjo)iDZ076ak1cba_D4Ask)$6NT)??VQ zQgXwpNFo8;8!KIJ?hDMp1j*n!r3w*2HkA)+uDXDfgBPdFDdR+lI;aiKSs0E{)4)?( zXA%3t@8V(A<7I5vxm53b2NB@`{m1kE9zMk+;^sWjk8)Hg8X%+gx@b$pnK$HNH zE(si?YB)Ne=XBy=@5UB78kcyFF^*ejbN|L&#@^*H;2dSd_NM>5Sq+%*2>UQi!pu z&_?tG+(yio;`P&-d0nvoCr-#hb#~ymrTsL>Jm8jM&4yqdb%$fX~RM z_HW3Pmv`7(%%o^lqLF_@4WZ}6!&oeIUQt5J*9m8www^ORKrnaHzlaG2izY$l$X0#W zr7H)m8ngcaKwf<5`nN?c&kGn~3rD0uGFdXm)_X@sj@1AIt@atK7%GSE=ArvaoLIDC z8%KxSB2(6w|?+E>7!;aLZu^JM1q z57shwBijg`|Awrn^dbMbsaq~``elo-ySsbElb6L&B5nNoJ%yBWIC-VJ6470=zjh~d z*q}6$l8g8rb@iieXv@~E^=Z@KwR$oaGY`m2k&>Qn!mPYPxWJ_uQ6d{+zDBHHMD|B= z$^7u)4!4h&-f?i6>saQlq ztY~$reWwHOJ)%tM6~QLt<TwKnXqzA#0rL#&)h#MM%*=gz&ryA$9h7F5eIdf)w*QNgxwY|P2p*g+J8b}Sd znGouR-;E(e9zj*ST&VC5d{e6gwV}OeeUFJ0jVL0qhh_9%#M$8NKwqg)rhHH$C z0K$~jMR!Wv)plK-bH#38x7BT0kf2nw{!%mir&5uORbT~ECR67$cXCcM)x1q^IyC@x z8@B-bs$Jj}a?iyjC0_9KQbAMN-{j>f1mwl=UJ~f(IakjwABICyL*$;7XU`l=3DO(! zZ*3!~Afqk#EvVCS&WJ>@r*cyD6(Eo(>u3b4ArYumq1w!+aYN8_S8ROu59-DGwV@Jh zk&kY62gjj{1R>Eovml1=e^zx)N?O;|gUi)B-@8#fxs$u^c;&JG5wklxh%^a6vH)Gb z9Kzw~0-Fk*VM~^m3X_~&{UMrNMqui&^Tob_=lz!KCvy{|xdYSm{^kgUdjR=*M4%Vh z^KQFLGTL#s^w!fxA3rVuU>qfA>bzTd>NpRC1#7 zCZ`eTO*pQQS!IvyhF=1Irv`TjpHvJXEfgBhul5|Q>41K{v4rwO8;f_Q4!c}bpX3gQ z+)Fyvnt;!HJD{2x1~fY)b~VCl2B9fM9|RY}+PWP3p`&78w8KMh>O3){{pUyTHHB-e zUHZ8`YyfpO=NC@Uf>6VSV|T04^a?X7hccg`%RBSA@z<}{*dPv9sICcPP2d=$K1Qad zaq3h>Ka@9^F07&=tN2?BLh)u!M&Pq|p2;JDOH@nTUl zgCn?f8RijnYfgWQ(05)!Y^bfS$LBcs{&)D{#$p8Xps!mv^kVLUmfy0o@QOOjDrVB) z#xrw{cPT5*Y%CQ69^$eH$Z44*9tX`;O$PjToA3;xzWsOIbgsaJ*uZj@2J@~e3(BdH zW!piw_DMsSF^SM5jXJ!wh!iP)Krl*EmSKAQ&HYSm?L-Aqj#W?4!CkwdJAuBYv0 z0}v6~GpYOzG6g|lBsN;KY6Uk>Ra~L6+y5?9QF&6c*2u_WJz1Mbb+jh29 zQ-=EK0vNlqIqM{S9@z#ybThk6l)fDGvvs>+lCyOM8I;jz$%}s%M!_LDK&P9@mw9h; zbN!4up)k@+YtaxMs*|Rs3dDt7ChO}emI1Wv*AD1g$dgkx{NHP{?%BJzA7ejuZq^Gc zoY0)T{<}^xnpO^`q}W^Vc=RfXvaE&%Fe+la#tH(luSK0DdRI`G**&s>o~3fBEHlw+ zd|9h(Ue$YWR-779w86Uf7o4?+hMUO0_WGt5(Z>MusV>M~<{)27>}k(lyGxNj2Basm zc_EKSI^W*YBW_ripCx#UYd_6dq2-Foi6i+K=k)(}{C3HI{oTK5CJAG88`(nzo7K|& zH&HvBL94=slgb=!;1!)A8G0`|S`&eG}MjBp@gXpY%q~&vms7c zMPDk}CyS-xM;vV`&sy5-(9o+Ty`V+@@TtGV`v82y#bD_2D8x=nj*@ zQqn9DG;eDyQX2&T)%oEW9Dt$_zYW60kn?m-KA=U_`%`gDyI1KsBWuAMb1vW~@!cR? zo?m#63$Wk>Vlgix+O&x0nTlbF%(j8@ICP#Tch|&A?alTjFXV__inOU5k(3 z!gt05J(?~aPYUwix#0C2h2lJkcY3sa*tDtP zbssM$QTx1F50Q#cN@3yWMFA@$qloRWu~f;~2Pjda zXn_2SVqt^8sw##I5S?5pycmM2;$=xaUv?cAg+B5M+4&Ymzht(_)A!MASRs`;jpgX~-obCDy6 zsHSXzY+xvXf5cl#RN0^;170jDpN8RxG=RKAIXi&2GJtbt#LJuEnx?OM>Xaq0D>0=T z0A;#8znDTF8`&cS1VLaRyH&sz$h>l#22UiDC7N9(iJac6pWBp5Hd5`P%Q62=*wM*x zaN=_KWKV6O4CujRRe}JVhM|$6e#)|@Gk1yMlxR8DwU3-66wG^y`dH?&4r=(3^A*tjsaVSt21)h~vy4_<8`dX!EP(P*@Vpa( zHTbORhguG#LDSVjpnAb6-(PojMF1h;TqN@?FZw$@&bvj~EdJM!6eBYa4rk5ox0!7(Ufu5s)1p1IeE|96*WRPPV*UpPAoN=*H zWx%@enbq3wa@~pBh8ol6RJ*`znc2_is3;`d>L@rVEGKA*b7KMMC=dqhjEiqj)v>fu zXGE9EGMhbY2R0iNz^+A`bW~IqmL^@fa>dTkg5pzJ2|3<;O3>7<_(LtVTHf>}P(XG` z0IyTYl_Ao(!g)*jG|M8;l|s{{Wo1?>*b_a~c&s&8RR6G8H=piRTi1$8b z1kILA;tf`7g|LQTpS>DHVnekWiemJUvo##6o@o>PzOb2BnHs4uq$K20XW4Mwj3Rb$ z8kzhUn37sFNFS+gihZ;T!TNyI}yk0Ju?k%jZt%1GE?vD`FkWp#H+82~-fY zN>|CcJlvPr*|kT!I5n5Q;SXH!yUFDSvOiQJA7@Qar`Npw7zs_*<<7q8p zW(dGarI%q7Q31mXSZf~w#3FvpPh^B^5{0o%dvD;{brgfiyZtN6FXgv14;5gf#&ebz z4x*LBOX4a~gBAZTl9n5~2RQsu#Y zKen#5rM&0tm=SUnwArMFnsBis5PQ5gnDnZ9r_%dJN2hEjNBOe50Eix8JomDsXI=SD z0@86-f*eCfq6|kH58mx{>lcr7E5s>Rx32LF^&e}RZxZbIwtp2q^3E??RfI~ZM{XRASjsaAqEA4efe+rYsqWY_( zcvSh1jYyUl$aH-kSUD-&#LIRL$s4XEf*O^m{_X`ML}r^{(ka{Z_xAOT`|wN0pd1nX z)xYKj=7Bh(@NKJ|?Xd(hJoxKyY-L>>>BjHuaG%QSX#2wa{8wyKcTbT*@N;7Vq$|~} zH2YWi6SxHL7b^)YNNSw48bTz`+5l(L#D6^)zvr12tHr4tT5Y<2xi^R8>x?Y=4FQSPQBTq zE1;=3sYyzH8H@uU{vH+E#p>chRgNMC`A@{*4q2bTK!B{B(1l9FEAw)=C?$JQ+*kal z4M14bg>I6^AfpCSD@ZMhh?uP7!N76XMU#n5A0>_S8J0Hv)TAFLE)1Hs$0b17WzB{A zOL}~&wsh0=lAo_oGxj@tW!RQCnd>euzwG;MRdhyj(b_eI^H=yLmtLY4_J@lLBINhD zIXlY9xJK;;54qC&ql}G>v&&>be9(f2rj9sFk23E@ro!k8izpaf_=;uAJQ@3@ehb2u z`6tR?qo{t}x{c?#PfS|Jz`ZsqY(U+8YC$C1o-;3;M(?=9`-fM4?~ESUml+d6%&9Mm zFb%V#!nM4#GDx6$Lmy3DNJ?V>Zotq1P~iq)Q3YEG2PeI>6!LScHvi`dk$cs-9@RSFJADb zFbvU!543{$%&p(Hx7Qk+`jx@i>9o5MC2Qe1z&`d2U;WYUNmR8(^tu-rus(sGWw&%c z^jF%h#Z$n#FRd*KMpPdLnepzgmR6o>V*Rle50WCDY6#G~<;4}p&~!e=%VL{rgA~s= z-MZzt+DS`g7G^bTwrl=h*ry($H(N;4AZd$L)SbCz&c7ZFDl!#x#tg~YqT>}0P!Z9e z&{9}^(b%zn0&g7Qq3Oo{DDaRZ^ponY;LDfCy~3jW0oH=3LsmFC-u-yCnqu^Dx=%CO z@~H%kH#s>W+;b3o22(12BF$FIup79Ll>eQw?EZ`P$h>tl_a%`=Va?d2)UCzI`VXK#Y;hnQW_Hzy71P4pB!K5wqwz`-&x0 zRv0>pcMMtY!H+mXgML~g&;R$fZA0Ya!DolB{c-};X#klbgxMMfTzw(6a2(a<&6p!q zYhBX)bSICU4o*t4P7W*YF=1{ML;w`(`(t~p03{AQMt&g62RPx}B=QSCw#MKf!Hgm? zI@GrPsC6?rCnkxQcbs=)PjS64{oF}H{TF5w7px9eC{(endv_QBJD8n?{K5;T(;htX zli2aOf8+FTGnVyKUW`igT#iniPB}4&>ouc8{gO1XpZ)6_?rV#lOk47^`{?~{RBJtr z{Wl>d@m?H@i7U`#7%iwBC0)IG^xfEqZ|0_}Li+NFR-o6c_L_Yzly<+)tV^-=DLTma6h391O z?S~ZAksLTXOiN3>tqHV`&V(T#xRyldr);Cgyf2+_I9YJJOVZ+A5B=jT*jP9{!4WQB zKGpp@*)WVc>25-Tk#d)(r>fTF^5|aWfF^M_Z{CTE(^Z~DyBv*nqu}e;J!+U^j_1`q zB~&w_cD}*d&M8wjH}Cr60Av@BDU219!+P`as;*Fs-h7$83!VMu)vF`)W$K4^OC;**N{*i7pESNL2LPE8E(6_S(%C+-!Tgsh-m=9 zN}q%^smKkY(3F%Hi85aY=(od&hqSSSKqF=Z)DO4|+QED5hr-T`2sC_sS(MuC`1lFb z5E5+C&MgVKaAD}Mp+iql5)<5xqZXRRIjkGFtzu@u{Kbyx^-`{LFU3(|#C*V1 zXJ_a41fofgV`x&JoSHR*inXAqXi#}c89wP>Fl(|0b6&Axg~SV>l;ea$J*C*pT)cdF z5zXGEbw4iwFAQS1iGIAV~-G0rgRfGI`b=RjUO@=d} zW$nQs4JP!R1Il05Q_bPvfn=Aw=s^0}p2UBKDUtvI2La0wm4oosbbr0K_jG^Ia6+lE z&n^u`iWQlQIqgugcwlj@TpM}O1YrMjG+*kLvkmV$jUK5p~8Ls+s+t z5XhYXD(R~)$-hDEjr^f|Ke_Dw>xWdcJx*0<4iGdI44egV44hV9$)*cMe8VcX2mV-? zb~cD^D&$vePdR;rXEbnqnd|O;v#scfCkxMK_^N*;x!%ow-ZWr> zR(CmrKF*1y)v*0=Q#L}R<&ImH9}jgXu%**^h)2drs;SiU^t%}uR-}X{*kkzP3h)jM zAaObb#^5MMuTX^?2KZ<0U`m<+*D0pNkELIOG*&Vt4+A3x!JkGUzX2mBGOj%q((#ra zy|XDP!Hnp;y_LpTfS4)d^0P=@Z8Hns+zmW^`n1j{F)coSNR6y-xc1dwElrUsY;QMg zD*G%MF8iSI&YD;1fo28~S9|cT_*wh;C11_#7TtHp!E_qQ0e9L>wmMx(5ecvZGdcejt z#t^r*b`wv{*|TRC<-NdiUZ15hs2r1l8A(U`&0|S|d-v0yN?0C*R`TM73!L9IlWiUf z1#!@B?`c|z6Vy0s^P<0LzVT69vFx1@JckXb44|XC?>B0;@-dYtX|or#FCYn`?CbaP zceYf4oe4u^GGn+lQRkfW==XgFOpq95_5-%iluo}{LMKflk(qz>MPX{7Z-hcJ_B}gaLdk7NXWNt-+q5XV6{07 z#~(FD5VcEIG&3&;733GwU7culo^29WAP_=2=1aez)f-b z5@V?$bX1ZI7z{XzN9U8U*|T{c)415FQ4SDoQp`NLSwr(08#xkU&I}GN(eDMZ1O=4U zR(3K;;J#M-UEPhxLJK4V?(e41g-e^v6Fii$Hm8riGB7ZZx&AOhtvxoOTX*a*2d%@$ z*8iT<2vslMZnxAoFJDMZqdL`DLasL_k(orr-@k7NQWI{zb@YDX6yhA))ot8w#Kp~9 zT~Du$hlfgsY5jL_?+7uK9ipo>Y1OK`K4p`Me=!%e2zew`nddQ+8WeO|gg_Gt8dCtj zhOIT#%<|!lpMqj~o?54Rl!MAkiAE~E^-IOA!kD7^zQb_Y-N3EWD0i&fip=6}+&DoX zm(|Y9#mW6zigeNJA{9Ntlz=@T04>rK?hMuEsAxF3p2VJus$-Od9pXa(^v}~qiYz^1 zvl->p{badM3g1=uv#MvT!TOAtgHg@ch%==xYF~KYzVWT!$Xc3tD?U5GRw)BW1a>eT zfL>7n6XFNpA!5j)ONJRWi*m|Wi*H7KXG$X{Rk6e@bQNi3V-zo2qwRPzEnr{YS=;V> zH1(`uF>{yrZ|hGbligcF*EDE-*|&2cN#$kXS6dq?6fdXC>iuieCXT-|A!Hb=EL&v2 z3+z{?PMt_3X0ylwZLxTe76c4`adqumH@?-KCz4$rrXHV>F*3D!w)*of+C4X~OGsI>Q;eS6oPw}LAIo=<$yO$m$0pm-W85y66awrAaZz{D z(`U#L@|SMbSo}vn`~1+Cxywf^`uMTEZQje`FIO5^4=niRvU=C5JC;=JD^4Khc*k_; zV~8(NoBl)vqg10g803&KQz-?cw$ta)(hQcwfP|Pp{5~Pl#@UtvR(YwhKP-mWtF{Nz_9vgbhnc_;%mxLWT6iqrwG$ZCZqC#gJGQ)S>a|K`{YwXS(-_ z)_!mxQxP#d@2@Tk=t&Vy-cdT{#bn&fH+h{dcAxH@m4_U8Y`%|;`nR#If+2dU9 zb%jwrelNW=HIshC8L|2zIq9VjJWguo>LgBzh+D2qr7xXzVc8g}r`4Bk-dzg2_RTi= zx2X&3Y&iIV3T-WYXVEjq1SxYL^HRI&qjxc;E?3j11xaS4>IpNraRYC1q=yVZ90o<0?NUKudW%2}5v_K22c)%&EbC8ec{V288Eo`1Zp4!2_4 zjl_jM)qeX3*`V7lTnPO(cr&Q-?AM*g;rJ3nabvWt2W*b=T-FZf;#j zN`C?HkuN`itU|W!-7T%%^F?(UHF`h$9600@8a3rvUdsy_y93=OA3Q}}-MbUPE6$n+D^~PBns{Q+k z#V%r->f19Hy3eI{e|oEi;&X$F!km4b;F=<#B22AiMWgSRwWd>>~oxjvPyyRl*daduwQE&Y*>Fdp7OMEo7s%4FC_DZ3= zZTqVeVm+LoKGU~mF&;xP&|OXK#wP z2zBL}%0s%rXzczCEm>wY->x6?(05tz{u1d>erfkZhRo!kD> zYBTf+G2CfA-(^!)f1PgAdhn-LZic@#y!UY+cm8?X?Kb6YYAZ%~m(B82)rWI(er;S| zf*){*P)`S{XJ`?8p3_7TtqrD)nzLy2 z*G3U*mT`M(`s2IA58)R6G4F};zD@*_yPYR$(VxV zF7-V*IT}FTpF;vpaI~z{u7d4zZuf%Ek60}W#&ElbEc^f7tl6%BT(AheZ%(J5G4wHL z`D-0X(Q)ftj^DU`eH?-j(`Cy}!{Ib<+Vm(zaSj{sskB59^Ck+0OJ*H>XXM+0Z{L!M zG54hWehOYu#IKDE?>?7Zg?aK{c*!_MBJjKalY3Gq($82LUt?82ay4S2EOd~zLbYuv zW$6?8tW~*bW=tF#48A$1KMlIHjo)Ft)bt&q@;8e%NdQT5*8WlJV*n8Yp}}sm?djTJ z|4nHDI5BI#*Up+`fx~2hqSZkhwOFDT^6@3_$lgi|Do+30=`PU{zxlI40EE)`(tJEx zS&n4XR!qqI^zT1~o1I2o)4X}}W7Ol_hOg_=6+y`4bK1w+H(N2h&X$0QrU2MeDdJ3` z-V(Mjmps7wJr2Hi>|#f`O;3$b^IZ?w%>!T9XfDs_MUlZ|$f+u>hFOIEMcPHJbkvva9;j+=(n7_wBGz6Qm3<(&l* z6^N>NPg8lXkMWTM@X%2XM1Vm#GLhMI&%Jy13Sb1d84Q`?Q{c)jUZ9bpUt4S3{J{M3 zd9l8A$al(Di|XX(=gXo&7Iu>8Vu$!`aB6<%ks}4NqV(xua`QvxsELA-V*Fu;c%vuv z@z&*J<;N6u?g(a2ZWa* zQt%>zqtt2OatqiZIYHk9OObwu`v~CEebUin`|~=4x>-wooRyt@oVF}C^~?m|x+knL zsgYs6pYPeZ`zSodXht`ZxdBs%WwoDFj^^M{gII+Ok5BmU{POvW4uo5IT)d5=*bfLI zXY^5OKZR|(tttwIPqzJndeeeUZ~vJ8$VQkZddjWeaNY|-E^mHY7I11Z-TPS-1^P)O zY*YGs+f0cAtk(Ok?P(CXh#blo_t&2?qC=$rq$E*xXg^^&{=tr3Cz)fx>L-|;nEuj! zMGWg?5h|=Zexug=uTPi;Aw^%WltX?dqwiA&knyDA%)UTXZJ@kpmwNwxu6soU+^}t& zadmLwO4LOkXU};D2J8bm_8d$%oSi^PWu);5v@)no7(uFDY=Hr zd3*1Vb9(>_o4p8LuYv(4fnmg;2hX45Mm0vs3O?(tYwp>6&1%`b8xCIc_uqtNTnHh& zh5B*~wq(n^Ud}?@dv&sAN^4E6?dvt>t)})<49j50*m5{&TD5!sw}eutZnQAn1w>RU zI{#+d?`(6<9lSX31?O=YyTKnb;&|wFW@aD#T6_?zO+&&yi;=U*gXQ@`%NTy+n3N?c zr^gV2cad+9e=PRyt~p5n_F3DezT31h;!MKuz^ym6qr&PQ`NPMzwtP~SUW+9GKPA>f zaSH&&o(e+anL}}jm(uNaCZ~XNXVO7eXOD^WLmAOy?lUTL+=wz;?--^ z=qb_F49iX$_pxlbC&&*fvr^inRUtwvN+Tte7UbQy~ezPetcM~d+Xdi=95cYKVx*r!x z=53)V#a7%{@YOf3Un_lf%KK9&l5W4&{LJo+(`SPCF>y0o=qlPN%h+|%65t43YT@K|b80lev>Ed78a0Z# z+DN`{^y>MMFFq0=L)eBPAU$;&Dn&4;sMJ}Lt!%y>!XuB*h2iF!LRf3!svZx zO6@H{gaOibBpt!`X%GXsImz=rzTM>46{kBNh+2xZ(qTlrClCakYt^CR-`hIE`qE5U z&kwBlT*cyhea|$l_)k@T8`e~}BF0z!-z(iJHqt-8F?bB|<=>yvbtOXn^RvNFBEi2u zi~sbO`2Oo>PSa+jjema*EIa+=3J@soXfJWI8>e$Dogu+dJZoOuJB!DiA@H~Tq zU-Z9Ig@*#R*^ZrG@e5ZhL#=lG-m+yNWzr(Z7x+mhYbd~q|2V?M-`^iE6`9)4bzhE0 zSJk(B-k%=beJ)FZ@VA$MLvWRx2Z=qy1wfG>PeT!}(9(H(czAGi(kb12(D!{^ZxYnq zO&=?MX~Q7-7{Co>06FdiZ}Mq}kuogso}NlXBScdw&Q&GF#Q}>B(xU~Wp;LBex75;4`Vs&}D}uHU0!DaR1H` zdqQq-&a@RsMozs%Va^vI+j=v17+mm^#j%>4EsF zzp-DU-D4P_EJ`53pF1y+>v9T?&#BECKB`)=Mm8+iz@>m3hMhRk9qwy}i`@WvVD4E@ zd1p_b5S6Y5x7feZ?7{d*(E-h2M_p+DK)khY_uY!@{c6vPHH^+Mumha0LlWhpWa&y> z%U>glGK6j**GS(Z806B~@Z6}EM3=#f;f#QV`#F@Fy0C0o#j_h>%q&ya+sAV%BA!Au z=k)g>O5%sG1c&L5QWh_$#-ObmyImj*WmJAN*dtm&D`dkDp{!{%?hf4k9Lxg@?%CGV z>h_K(O^CHuT~~jACVCv><}!;s=kBqZtsQaZj0w&`Dv=$Buha{R{<>cf1`vEbrEn#5 zq?P-?_5w9i5TR3I4!Crue*MD!eWSjSb=nPDxJz1ZYMJ+J6VN)ip;g1&m50X(CgFjyR8Iw@b~>?cT1VqsySj6v%qbe~jBUq3w$ zp@us_A+%!Es!07Zc~GARa`l{AVamQ*#b5U(SH`XsPu(#6QR{NPmrD;oEhxGV%3VVs z)!R2o2Pw+;CSQvCh#A$fk_>SDMZ6wd&G$pfnnJ^_++M@tE>kF903b0c)=FLHeLH}T z@jVQJ8C{_AOCAxP{bB+bnDyq(F;>M1wj!0l{*NC+t@c(#hWWHUPceeszj$^f%hVYy zE!fu-`agYZ0C?dgW&~F5iN&`_%m^n&gqO471vTkv5|B#v0Tk0@!A@juR)oaPwBJ>=M)d0DW(MrZW2bT;+kXwWjxK>_Lxmf<0`B3QnGx_%_>L z8pWHSDlwQkY@@_e`Xx>%rZT^8|B;S$U zyY;CAIMhM7N0V}sh_mzhw@Ml~e$JdZ5xGRz$Dq|k$h>lB01VBHuOS5TUSeclr`F3C zQme>1km^&qLsZ@RQ(4h;^})Q9UXTPJJxEC$lz(m{+%H1HBkh4&5P@ClWKNiP0OO=32U#!ofW}HG!`+4Rd)B8VNMM-=8}!v3SyQy3v#Ok65*&GrnmF96jGBI4S!r+ zI1@y%f&-e}To+2`4kbFPQ3_4d+h(m=osbp*iZa_Tetnr}5`~d;T2Ch^I>EwK)IO0m z@r@`K-cgbmBO;N#Pa&25qvA2XEWgBLjRaSgm1VPvo>UQ0CnRsz=YfpDDQ$OO6FqTH z&~SV#x4?Y`5=V|XZ`t&i;k!_^+t;sO|6o(2JemD{D5ax{ia+#Rv(lEF?h|klP}n@( zr(*Xm&dpEzW8=8Z{bp3$v7*3```5er>p8_BM7$D8aXDX$vPbDk$T$3s>prYy#~#%3w-#!JS^&U*S%m#HaKA zMPWoss6kVo1N1NG{7N`oa`iistI?VVqgQl@{pr1%4r=<(- zUw^jeS=v<+4#Vty^`qL!Z@>o5LEFR1KOUSL&Qo2)xYGMWc}SDw3PH!qeoBUpSWc`l zf-h5nqZPd&QvIjEm)>7SeS798Z2|9m-=~9rte>dY|Mnd9iS@k6^Bx9FupcfdbWG6R8<493n?Tl|g8xp7 z`+G%j(q~fC=WF%dPf_?&;I_^S7QDTu z2SLV+iKrZ6(sj8|^X2nLb!dAZUW7rkZSNhi0m6g!8`Nqa$Dc$?*e`aePB9seg zU@%e6^6Y9LPZ15|bgis-H4pOHf;XPi?%C5o@EdP$Z_NzVuvol|dEvGP=bya9!KvMq75BRM%7e5Tji}&IlRV*bn>JU|+u5QE-iU`uUo^NQBfdVa z_)S%h0d2N_z+VA2eh^)sPOZ915sd$}dGkU_!~Gm<+-+WERu7H)pj#Ic%Ux{CvXb4%m;dJcaA!K zr78)f`ceH-lJ?R6jv><^VwA78i#bu0lcz&BH|}@}(X*a%ZPAB$xxb$5+sw?PoMvPO zoQ?`2SW04cSg*l@&FeI3H-*MhAZse=5b~{$FKit@r#&}D)^UC7yGHh2&k-YLvN(i% z^}c~XdRAUk;f`NVB=rhK{VF8BCiNB0zs?dgRg83jWPLe+qgmJ zf4j&g9y=T`<@8c}FVh!EdpHZR0vA?f^%hwosn?8B%#DwAPn(+g;#=iz{CJz{MS7j! zh815k!Udu>h=0+Y8diVC!XKq0EB?KwHia2R@9w6Nm49%75f~t-R^2yXa{u=iv7QL} z_ZJ223T>*ET)7xm^wnC_*a={HQRB+ByW(%kw`IL6mr=s+AuYRt3%(Z(CPx9Bb$mOS zs>;HpUgc#N=B9*?gGF1I8gWd}K_I3Ds|~82Q19U}orZ~!k6JJi%2Y(>z`SOl%mT?F z6AwW|@bpkc7@MPuU^*H1@Me$*1U@7VU);MSNA1o1Pr?x#ALU4Zn8n z+82^z2%m&z;&1<>N6pU6Hzb&ioRx3izVWcxER>a?7y`!1WPyx!i1I)*TC_HA*cAko z_ufV$6B*Lb)S6ZCxpO*EmZ<8(qfMz&vm+i8aELJRNQ555EyIIDE3|XBZ6o5o$e-GP zgLRO%G>S3nm<%vIJh~dGc(Ul=9bGNsub<(}RgrTfBliy5Ir6(6a1s+do1Am!paB+W zTH0;6_j{9DR$PU}l@D57f6T^BMAZC{_(AB`?@$Iktys9`Yv{;CsVBoO!gDf0aLd1# zzE6;BX})BnfOnRn>8ARG$;hU&*pt1654Qvcnmn!Il5RhFVg`Jh9a%$CUsaoL;cmFv z;P36*pU7)Roh^4kknp^1t0Z55oSwwOftR0dy^!m?@?r4OXD8?#gD~O}bdH1~5Q8Yw z8O+e1=1M=Z3QHqdJ-pbbB65ehXEBa%UApW_Q_3xxz~?_)4j!v`W$DtOAZz77&Q&xt zC)p3``URgqKg5?Ply4U8yZ4#OjJ79Yt?^jq>4mG1`bBAaZt-*$$h50huP&roR-U(& zHxqh_SSZqX*NHAuMwUwx{6>YzS)%?mZCeXx@$BLm zepe3QyZzwP?Y6nv73=w})Yrq;dbDVYdJ_Hx)|RRhmTN5@8?)BQ6xmP z@cBdwBKWt+p?N7Iidw}4#s#h#k+qn{=P`Ht29OQ~G1hi-dz86VMv910Az`b{>BRv` zM5V?^of!u{h%T5ud~c%N*{>_fQY&^9JZrFS(VT+kK{1 zQZ~_lVMf$NI{afY_SZx8Z+CZNMDL>WF^wB_vSUIA@JfM$d6fd=$R?t!5_}5b&fSj{ z8&|Pm_w=8U*LkioGLh}rNkkqzqeaLn8Ei6@a7ULV;SQAY=^?U;tta5-dvXA{$=-ko zwQ5nhKE@as-A|@%g*aKwo-Yt?-#>`$WCmxx8vbrylQ{!}J9n*mpKGu(fIfXThxd$y@E z{pc=^Xt^TBwM4KPhpTEg-S%O`8=O;=PBW5B^(n6faojXi2G7~ zlJwH&m(&R2XpCfAkGlZtYWftIER)N!D}svyi!xKqP_SScI)T#73ye@G_(gR-qsI@D>7}z^OPbDxV#tlJ!t7O z3y>VWZ_7LL@kwM)d2fYH%sC(F<8KvNakS>*sk?rMuVbt3J3sda(t-PKqA{d;!i8;# z28GhCnh0N$nY#yciZG4VK6DWNG5TAEOF?0w2||qK$W3sZoy@3f5E(P{poZsQVC%=2 z4xQk|eSkk6czy`iB042Cd#({?LEp?(s3GRTWI-T}Po!~gM$=<7>gPb4jtvq<{Hr=* z0_BX!=-S`q;Bw;SndL9%#)VZvGR3u8M0 zYbSx!p?9={@pMlE^^h?_drBf^=q6ustSEI<)g{`Kh9i*=l92UxKDhD!WA8nnsy@4J zUlNUpDVksjcGNA39edYU*)}2wid{ibu|`D{6%v!!P+@~zF;Nr|uz;emR}iBjC@31y zh*1HtW5J;3H}}T(%{y*6=Zt&q_{O;RI7Z&@v#|HSJkNU8nrqIvCSLsId!B-t^T@Q% z;)mu3@)cxM>r%x%C7*hu2NJQZxCo*+9bWP?pBH2W&ivPG`u%S+W4mod71z_}&0_}K zt?syU;O-McLPOTR0szStKZj}+0I2o$;uS88ZFY92qr=bxHY+kBZ&jEEQ}pnQYu-8w zeml>iP-4x+C|Drt|Ks*7kT9tpm3ND?&Gx%<|Nb`MuP)Mz5c-Gt7tY|=u9KEqmW(8Y zLxK^+4HXC}e_%jdc}~LMO#tE%E`|OT{J^0IYL24IunW9}cXDCKkB?cFjOwKKrwC;J zA#!Cz6TMU4k@RjS$D!+#>yk+S_9*N|Hp7T{CmYlMXG8%U(%9J#I}IvJ9S-4S&M3`# zPA%KW$Xk_=>-6gnvV;Fgg>HJs^Jv6kKi}bHI)}V{SG@J_Y^NEJk`?L6`TIelfvH;E z&(AOFg3Hl)b0u&yep-rHk$>W|M%y^w+MqO1oKVyle~#zY^F6KL_dkYltOyYivy$R3 z|3ceG10Q{3&WO(3jXv`9IW4Dc^p(=9mREyPASab5u(+90B_&zWw zEk175)zqr-r+r>nji^GQv}f#l5>a|D$5f8+X6PNY{DxC}Y=3kRWH|iP!FGk~qN1Y0 zdmJr;dAAdNfR(k)=tHMcBS2rM5lj=8BQ9j=-GKo_vYCW1j=zI(aB!#$ge#%<_I=HzaKHo643KphCZfFT$4?=3A4fvf6NIYCOL5>NQzwtxLQr*Y z(%i~ZL4y2Xf(c*uuSKS-+wrF6vZ z#OoPeD>g>=S69&)HgpJEhG~KO<96lk^ab1^&dHr@G4O@= z)Ye)WsV#(Dfs6oPSIHx0?|im{G9H~4I`>ANEJ0tO(NGTwF44^ufXh(&+%*)^dJi>_ zp$-mhYWeb|eEFoCM;aJ8i2HJ7Z@346DxMYCG%{^qtzz&W{O~6ye!w~k^xj;xn#66cspK;q5ury=1Th}}Q71+xM4XXKcKpG_N6RD0791UoX za@sYx(b^0_1eKvB{VQ%5rf{yb*7c59MM35I4o(YGyCngL*KVNblag#v0dqwr4qK=! zF045bktXLQ;Avr>o*q?q+7FENLDT)A=yz`$`?4 zmXaY7h*8U^$p+F-njDc_v^R|+tr#Mlb%U|&3|DqW^Wis-`VXh>y+qB1dpGXI&H3X7 zX@%TYCm#ql|HD{Zc39%rW@|e_yW(8j(DOy@1|OXSELWqEpjH^L7q$X+6;I*v!mXJ!sEHAV`rymdDx%ly)YHhQ)ixJKZ7S9UJ-s1Z7Et{ z+8!%X+b4SOJiPM$%{%t?3_aSEOC2gXxX78YPbdS`!xC^vdk$JnxJ?nF`)%lm#x^)K zH3O3zq`Zha4@px4CO1kcL%SjZPs-@_4Xiv%(#V~i)35|(z@i;DXWiRSyZrd}F=C{i znN)-vN`5m4!nhJrd*wGZ$(Z;RY2<8 z%SiwS9Y7b7t;Yi~nLTZR<89slPZ7llZXD{ZSseYjyfG2Mj*LwhqcXqf*XoSuI@Lkm z3E-)$7esX-OS9?c#cci9YMm$%^_+XJYS?|W8KUeIFKxD(sN$d{dG<_Y&XhP$HOe%x zV0-DK^38I2@eyg<`Q}tQ1)Jo|4$A!L*`q=`*RCA8VQzZHbi02t$pdPmf_LT$Bvz`O zKqsRp6Y12#D}Qn-DZ&=a=~0OYk0nJmf!ghj_#mME@e7&XyePG7&4!=Xq?~yobn{Gs z;shU)1Y?fy@w&>egEDzdkOdLnWe50iF0B1vvs4VY>gKhR=F1P5@Bl}NbPz$P zYCA#t>ogA9aK!wi&LoLKQ!fAhocxxVz9+ABsk!bTWMLnw0&#~4-oQb=qvlMQ5DncS zeD!J@`3cZnHo8#m5zTUE9h&qlI@3=r4Hn-LBw5t~MYwfo)b}pv-l9BVT`nd-RH|!K z?kc9xxbNHIEkT5mKxmgp*mR+C={W3Ep1*ybzMxZjXh;8f#!)x)8~DLpQ;Styr(cm^M$*l> z>!-H)>C>kvruszYm=~|-(Els)McVSov>g`!$?Kq@&ohi;E@b&~r`X-}I^thEKT|?| zP7a2{Zwqphr2pwbM05q0Jtr$ERqD7DJ-b}=jF6>{eD6g9K}wydk4ls{d;WX_;{?%Q=;pbNk8T{)Suoh|(>QTxX(tFX6aa+8i*zRvXwmV2k zi;SxrW~2LCh!e-UtG*I(qV$>{QMC{+_d@7^eAC|OPZHW3r#+Z`9s6Vf8@sXiFHu=o z?;XHtBvl5fN`td++$G69jN}x(bs9-hC~3K87R4l7zfItDYp!$gLgAqJcEIz4UqLf? zL|bR)9V4et$gsTk!Myj*+;^U#=}Sw0SZ(^D?>tDwkI4S z)5M&66QDk9n87^ORq!<>;ovllBpCDjtUjxe&ElIVH77hgp7071#ueNAfepmaCz5$K zwkb;^QEQ5-)D+I=kQEFjh`x1sV}_61R~0PnXlUCj}}@*r)WWx$Zl9_i&3 zA!?t!V^Gz(uM1x-$-~s5UYf@R)b=LXXVhx*qB>mW0*Y$u)3$jka(KKXw-aeTH-5*2 z2S*d)^&Pai*JVYyMr}pUbmqp5p<1cZsc5*5`KY&ityRD|uaFJa6qj@zr)$$|=dzW8 zCJY(0xJ2p+KzNgNxv$$Lku8X3Mf4Rnz*w-Zxwjpdy{Me5)fLN^FL!-5w>&1EP^(#U zy;9wv$!d$)XFjtKkvpa9FjeCX3ZEIUfu?NON?2$8x!HMI&M$L&lO^qfx=Xzb@6sFU zGP@uh`kC0T>eA$dk+d?GA`-iJqOD`9s!vdjAq;1`h}58nf_dyi`h{umBes#wwsY9Q z;aVZ|EAxHtDi>Al=3{ApNrZ%Wa{!N;bQB93QEfz^qZAaByEj782}LpWN@%U_{j_YI zpzUu)jT&Vzk~8*kNT9|s0o}wzEc+TZ(t~6GPUjU)J~*?GqpkYO9B{1MHQHV2J~R#^ zcJ?6#GY+LNU7)rBnApTN@Y;s38Qup(Dq!Z5i6cRS|D~LOC zGQiql$dL8K|81NPImks*kRYtP>+pGrE{o^-_$<$9+1t+Lz_fGO|3|v@k^`8n%zIZj zlN%P*L_8fsQfB5bs@S6S32X-Xe zBp3G6PkfV+F}QHK+kw;nLD25>Zg5K`Tm>S!Jm2UO*7$0VOy8_8CjPNs5$6gDlMwq^ zmS`lY#Wp=uAIy27yj!0W}HKy%o>z|Ly24ES^Xy{+eoAT*v!9vwOv56g{V&WS{3fk*-K<* z8uCU)4NVw#&HVJ+#W@dZ*$Uw1^_i~TJ1j{|z`4<+TIf9-m4vLhUqJG&&Gj4%*=P#$ zd)Ru(-PY)aDLx}W?sKUvsO&a}%}$2>Ap<4ppN+#g<7*jthrOQrB+#^ePLHGZVsiks z%Fcp~2)AD~u258Qs>I#mMN-Zll2(aL*E4pGg*#y7*?}d3C9b>aIc=VvCa%puTU>1{ zylK8=kg?m%3t-0dxErQXzC zF9Mh(h$&Nuyp2dHd(!ev=kYv;5hu3_3hFdyfuZJ0ox%7m<8q&lSC9Jpn>k7NV^7(h z@SOf15llIjZbYrz-#}_d$sP`Z;rlAi*AvGb8d7}dco+goCQ`G0MbE(|J0`j#zaIi34WFR1gk4L>@M-sbm^R=0D|)osxt zeb6QI!w#5mn{B1#1LkzHYaY+pjl=S}Kvzy~w;K~Jk=e6hRM}cE@VklBPxeOjQ&nk- zXpkD8;~G7%A9L2{qwEkqFU1l+j$P8B=KtaZ@te$9_p1DLe%$~V0;}*K;Jj{V{PeTW zT(G1O6Iq}M#j%*qQfGTkN;TipvAXpe_5?W%#BM!$=mil=Wail6^_?<;I~Zs^ zfHR{xx>mMvw-WQe;ha|kv-wlY`Q`ox5)d&yc{Vi2jIt_|gk##>-jI3DauloaL^7Ku z)3MaAhtK(y-jwHecP&@oT>k4?*q44|DhrDck0Gw29+VNX6|w%spP>X2D~Q_nZp$jL zsV8S&w!9Egh}1|lNVcKpg+*lyX_%x>u_7X3^I@l@U6!=k%a{ryzW#=a<}dY}sxz092eNJlan-@>#Rw3R zOzk68Gr3R8#7du+csA%mZ_6X{^yen=tM6kHXJXo5(1G(C=gys5=JcD_uZ<>xLW6&O z{PHVbkTuyu{G0YBYw!Gb2PfI?w#3>_eMe30_e09qP|03aUw>Pek(6ZL@9(T!fT>>Z z*-%4R3*o1{KYDlM?NRtmll%@@p0ibKBa8~nw@~yyCMm)G{cBgr1rZCvmmMGB=wjJ8 zzCM4$lug?4RrqDIF2(A*5=CT`q91B8@=4#4FwXaMuWJ4i+mB@;bl^%U`akiUI+kc|`G%{-|L5*=)L!)Lw+N4j*_ zX9~U4dR`VtpvhVmmx+X3E>5s2hLXgD9v`tQVVC(MY!@&sVxLfQ;zT`LTU*zckTkEg zs%QS&5#B5rY?C{Y`H6l191=z(1Rn_zg*DNy^uA-cmRPxejDy1{=DfIMpk;U?4H76Q zZ3_A%PoKr+Pd)bYgZfo&0d8IaV$vzCyoxeLxeN`l=g4G*p>aOMb>CsR%5_XtaDkU9 z)`<8?q}P|}$ETq{73Gsgy~w-48RIOAsnTv7Citku<1mC*bCnC(LM+V{e2ftG4lOuo z8;A$#Yc1=_vneR>W&d|?h7Sc!rZ{$?<|$XT>JF}PJD|KM?qrfH{>3W`Cg(a=>{>=7 zrVxX0i-a2doTe~BwmJH@lB;<;S>8)*1=`5~pnVAammt;4<1?3sB6B0fmki6BwyH-r z^TbhHB?M{|naeeZu-sBc5J`!UVD8FjBDu8lvYa8&Nyp322e6V;h8kFEps;TR){s2M zot|Fw1T&DrQL}8N1!7LMX3k`v$d?o8hlQCoP`9KslJM=#k|=;iRm(_zl*X|v(oGaS zvh=kU_J?N|s3xhjCxyzX4M0q8bm;L9_+RWG4Lx)EbgeiS%N4qfncD9J%S%s_^F=0S z)Rj4@TbAe_v!>l^_Ap;$P#=iS%Ez^a+;$9!q~2E&!3gDG5otg{M*nLKab}P){A`eW zta*{`o4k`nHgLU-`LoWvl?)k>;S8J+s!aV8(giy|zc=si*V8Jh@=b8X>7Z&l(ZIkR z^XCGb?oN1ad1#deqLstjaAVFZSswDN;zxEj|JAYdXy?!o;JC*Wq2 zvcup5UawQha#Q`d(*3iZZEgP4{Clg`->R8<*auqBikX(Wqmy0it9^dxPN(01ZSe0X zPgpN1p&x(#`{Mznc(=bGhOPtg5_t05{r3@KpZ~jg^5Q_HCga=&6>~`s8bVZn7n9z>k%ig;R0YXhphxDXK8cxAu&XYY8nXwb72 zY9oCtoQoVzMU=%aH7gcS^Z3dlKJ0`M)CW4(%QqSp%|q;E!gXf{Nn0*e_7E2PdV(cO$6hql7(^ zoSZx!4zkj(sHK)wm6~{a6IG$zSh$2jw+xX~)T;0l{8rS#Nus#roOm$S&z;t-1{U_W#QLBwP(yT*O*U2{F#snyV#5GqG-=yliN+V-O=jLg3K_O49qC?a zxj3cSU=o`YzmbfFE>60mnV$=aJU*?MkOtCmZ6+>sp!Yz_&M9^(A%QT_OHD9Nwz$7O z`$8>q2zYQR811u@Tt}or&@44{V5x5+sXK!YOA+bE_V-=BAKfB)r9prqv92ozCPiJb zEb<>@xDb^0c3)Psytflh;KHE!=pQre@E~@(8(CT!V_tP`Qj48^P}U_6=(e}>lrKpy zbYh}B62^lRCqJRm<3N+*%=wH z9rl~qeXFG>zS4_e>%r%yij2gQk#A|v)!m!HVK)Os-K_kKu%1g*m%i4LiFwv!outB( zmxuNF4Q&TBI`V3hP;~Kw%pEnEP7)TA?)1L&@JYXwZ1XkiEf?&*gd?MBQ=89Ke%!sV zAqrS{JkFLF5-m0;<&{;^u-VzQh)!wmN$(SmAy)y@yNu#QB$2{<9pCjsQK0V^P@{5H z^m7Pl=lV8x1rcR!)iTy)@QIxztLbEJbofId?vLqaxe65`$p8c%;o!AgmktDb?K?}W zCg=-n0i7jC#zwW8-UI(#<~2#-gs!~q$!D)`%`Ytzb}7ItR3mcj9QtG2bwe`nPcvWp;8u+b~5?PeYcpOeg= z-Q+;B8chc-QI-o0j1oQYu?8dIOJ%axvevZ!#e@MqGgQnaq!gfC%qOh#2SX%-XMFvwHRFKzft<r)q84JA(PR7lfI3ho0ct|vkN2Exm5Ys}o6$<#FJ1>i4?Of^Q%=X_}aU@Y5@9V@$t$44n z-KKNUbj`at_vHrMVe5m?h%lxs&=3jdsDL&F3dq$JvMp&}J-pJYbNCDi!W6bkdxTel z3yI{<$uY-+_)2dqV@?>#z12CdT)D~f%SB;&k!CJ{51gR^D>X2hSY_}^dxfyDD&@}# z1f(hGE}$x7B0e`i^XOBza{@@!dYLl)-_6y^zWV||N3Afj%uhf0AAeishv$_p%}8Dy}3wJI1>4>NZ+xbKD%d#`Me6q&v0*ZjBMCoB+yLS z^a3-9HD-=eS%di^9)I2jpq9Ar%CyvJ-~3eJ&3}F!$^p)rfnWK8s~SK$j^m^YCE3wHpWaN|@Yic^o1o2%gPOzg26E^S`{O zIbt7+tG#7Lv21_ukID5zqN2QizH2##o9M;cg=D96Dz#`q!OOkuu`QPjUlDo;s>7vE z#2{|$Bn+#44=lDu6{Q^;arGu|%_26la@&Z~`Z4r;kq+t1xEo@+uN{RNDkdR?Act|oru-c3K& zE?0cu=7G1~)dl(G%<;&dTJ*YZ9QE&bI{*970%U(d@HQc#l=ySA_ODCy!5Td8at~Ntwq>5soCJ3Jg*>1}+=cbB8VMWQ%pTAL`5 z1H{%PX|6S%@vWDoI{*Tg{f*>SP3^mwKQZo|90L3jzi$QDik5WvjT z?53w!x&|Z6=J)AFMl$f`bNtYHnxr+x+`-VgLqC)HmoSCzbC@Z)BiCP>5?K^KUl)dtHXVxRV!`#2oV9 zys#v=TeA|NxT(6O;}I0{H_3wR;Yme74x&9jvo=K1no|9RT21GU_&xKRTEa*A7+@+o z4$O!2vriX&SffhF`sA6Pa)^tY$v8y48dHx)sN_xqdt|bC}LT@Arwo62HIS_>aHxMfkfIxIB-K*|oL=*g%h| z|AB>e`mTP5BY*Cu_wMdp$MVr9A28P~&HG~#xwxq<%ztLPd0I@a{R)OqKB z4n6%}YWnlPzuL;`zXwA%|zXN-HWn7`A$C1ExiIGZnAxWXz3fcbrI5lViaBG>pF++ZH7EnM+ zD%R?BOE%T1?PHnkK3Q6^N~ACj=S$=^ zO8S}qGsqrJGr#`bn`oU1Hrb4J6t#>o73TSfC%z{2hQ}d6y!!&kG8-f9OCVuV(||&p z-!#ko5oe57?ako;do&I{=)}Dsa}1ZRUM7MBWxJd=Gp_@R#56I}gREMA^uDLyL^etZ z#3>-YPWU@4+IyF^AkbC+v~Ui7xRF0;hfDqsusf9-79`x-QY>+Jo@mZzEe-WTlEJGBzk zUnLdnb2$v^iYiEx*$W87(qy1<3y-aVdu&(UwG7#QzGptPI=S5q%CMjvR##8R&Xm^g zvl$o_8zO1ve&^$}(op2u+9Yv3Qe~(K${<1lTav!XksNCp6W-vO&c-6I2i?=8Mt>C9 zPy8H9trNRp!tl^35LIyF9Y+1^re-Q!3-mK>Z)9Fen{5@Tv1(}q66KkNnbW2ldoRXPGIO~tpjf##3e(I zUmISyTV{2%7!QDazkT~=GTp*6nDp~(-F`O%;{odp*UQ&J-`1T=hEup4I+xi+4YEOY z)kj>o1k1{z;UH}X#1`)x`gS<^VXg$bl#WtV*Xv-(uy&OLKW}z)1j=}OEdRDJLvH5?gJtwP6ERMJBpd>&CA7F|l%$DjLqw zb+_donE&@yQspLXzmL8JNM$mVF1@_0N7a{jE5C`%LN#HyJ`eXL=n4@v|MQc5dIr%M zG#GK+v`=XFFfJ~uV}*ZwXyoy}B@1pg{6vOc4tJ;C4DoUCePCt~MeLbx$*(Q{y7gB( z3wz1F?f8g`WNl0z+Ym1EE4AJXkud!$tg~3pSpIoNSz40F&t;dOSkJ^=<2!m=<;Z1; zZ1U+U)5H=c%s*iJX4cll*6QbQM^rLXjRj`hQ<_;?A|oapC31HtxL?zlD#E$LqJLal zP_b5?F#IIB(fpLVxhz(riPh!x_5lbobIw|RCS=w} zPMHQ+bZ)Lz6^mI6om_(@?=pYTv8J}{idHbgKvA*mN8ht4SG^~h*kbz>`-D?bx?wpu z(hTMCiJL)SiyM?_dAjERZC5XLV!t0qULm@zBYFMsL%1%FFYiYZxgBUEyU#zc1gkog zzAgeStvZeG3iW{w9o*^Q4I73oo+{Z7qpVFY*w75RJ3XfgR@mypKm&~B7&cz$G+UF) zK!ssf`i=Sn3Mcc)EHgQv*WS&AM4L^sR=GG7q_k)3*VfhEA6i!5g;TG{X-fP&Mh3|l z7wAd0-bo3hvcnmd({i{h>7||m6qjy8pq(hTVj-acxWV7vobzlc;-+B6ONFb9pUt@` zm9ea7kdcTMWG<}dT#wgG)Kbg0kA2hX?e*uKZJ(E4{5;xm+z0n7Oo5r2gr=0rcI`&atQE zrTJdKRA{&+DRJ+-Q3n~1rN>Ll#-rQ3;tJI^Ztk-tw7=C`sv0c!C2*^Yfwo*O)X$J$ zin=-FS|fzZ5A!7`qDGfM-4rf!3F<+qgVg)h^QE@kKJ?%VkBvP)t{lS|D^{Ye8stX) zgSu)!s8JJ+V!M0r3b;ZrMH^1&wT#rYuE6!^6&j-0mQ#i%E((%%`{bKVQ=l)ZaqDQ7Q*8#m7cBHf)I zu=&&xx1yM2+Mt#xXr-jq06d$eW5LI}`}*Y_Nopw5>Ux}WNh?-z9LTKN%S=(PVa&OC z^r-9rcorRBTm{91xYS2TsjClIGpxFA80^i24HUW9J&54!(bsC>X5cur17r?F*HjT* z1bglSH_HW8{r(nsN?;qCrNLp}P4jy-A&bdc+o`p3X;+qNTiSMqU6fj5d#!na}Pd z)kLYMkC8a|&J&%Uyyq?a^MRv9&7fZ?)iNmlcz9joZyq_!h|#=V&#Z2ruo2P38%|!q z&3A8W8-?7Pp=>aUlxq==!6zo2p3h4Xg}H#|pVhx~t4l1D%aH%{;>dGW86-9^XX7yc z$L6tWCbbM>(T5{mzuL2n-;nmV>O{X!Tr0l@GUkx239Z6tQooWbm4T1ngWdg{mL1Jc zAME7Hdl0{i0c?alpTAg?FcUI8mTzQ=qcHrLb7-i^NJR1@*eY`6ynJvbNloMPpWc&v z)6lKbBYWp?-v_?J2Z4mE?@ z;EV6BW_fV=rrEaD#j#~IYPyeWIz@(x7kZ)3rt7y0O{5Z zDc1m6@cDE(Px<|QEM(!J!?$i`wP(s3Nx+;3zQdsg2<~Hz4<4vO$Nyd7zGBHA`W`~jTNkv`C#(H<|Hs?%x`7;)&2qV; zpXN>FnZ5?{lyQ4Dv$EnkR1-!AS`D*$<34-@m$Z_d(Gq95jVRQa;cA1e;6(_vP&xjyKKIsqR|3RTuQmB4o*kggwM0 z)cM+X=7Oggn_O*J6nF8SWpCv~d{0KZO!<;8UALo!LK{ z#r4;j;4V<%-Ixk{8BgcY=T`SiJ$ZBiywCBl+0z{~K9%4%X4N@cBmLBVFaX3xl8+?P z@^o<6nvlBS8MpW*`LkC$F^@3LUOjdto7q@;=V(K>!C&MYwQ~xSokx#eZI*bq54y;0 zZe`ivO?kQJ;zOjc)X>!qT3-NhmFOn%+QX^5)lplD>b`r99NdhW8U>}z(WC)3Y;O%p zHieUxD_*25Te3D>jg-K`4m+w&`I%bGo9?yL&ms)bb`p!!Z7#7oy3(M2ob?fu5w!ls zq!g^99j2xShh`L~tacXV$`*2$HU<3y5=C57At)dlI|LVGw9O!`d^@Oc01V-<$Y`l}ozkW5|OGPv6QcvAhgzulqp@ID7l%!obxaR0XO zxT^`zvRFh3J&WqiWuNy23v%hq-#92pAzolmfCqCKkS-|-!i^MRKRQdlvS zJ%&=Oci9kT}`?XCyX0Hytb@%R=nBujJH|M>H;R#uO==)dQoh@1Vv1zC1wJI^@#ZHF; zK&p{6>4avU`IIbanhyJ0z9qZ+rgq!`#nZNf=1Ec_r?Uj5n&LRF3pv4UF76OG;M*Pa zP@9umWT;)+gbvjPly22aGm>c&$^C|4sQJN@C!Viqq!ifO3(3!<(M`}W4}3FcNc%5u z{yQUk_3D+ZfF085AJo6r;Vf}W{T7eBM^nj5y5RuKM6-;MS4)2O*)*R|9Jks$I@X2E zY5QV0{=*hZug^B*?{wK6pH=)Jg zLkT5MEqMI+@mP-+M_D#sXSsCF%*>R>a=j&Zh`E(8lt{1vXz$=Nhth07!s@Yqk@=)# zCT2Z-S!Y7NldWG&3#gmUM0nI=p*%{p zF01$b$X9K)Z2$yJ)C;B{VUI@ zbon6YOxO0t+&}*e-Zl@Z$Glb9^eFZq}~Wr>3^i(4puXiCzzbu6|z| zX6mOhAQ%)sVU5T!B5l!vxkN6AqjI&FLg++!!x#?KM=^@S&0rMSQZ{K~yy5J#id-Br zkgUbFK-;h0x8%Y!ku(YY;{i;u(Ax>GatpY^eO9bPhy2561rA;*GGCHyl2I1Pu42zBo6M-Wmw>hD;=AX&@8W(K z;^{_DUOhz32v9qdjC~QRT^^71mY;cjXTdg{Y|` zf`Y!(?6K``F>4_=jGFKWQ%6p@+(etrlI)v3M`#}Q1;O3toptZts-TGCr*f!qlwFBs zq7m4zcQYbfEU>B)PsSgZ!;Kd`@Q1y%g0b=8y$9$ zYrt(qNhUjbvODo-nR1qAOwn>i(%ZXIJTsfKSAqpT^P@#1oM`)SwLO_D>)pwwwh!L7 zBMo)jr5$~cbX(_459lGAyZp=hgx+rF(Lf-%>9L%Wd0+hGo0Dc<(ug3L0MX!*q83W; zcvNYn7FF@JPOzbXy*cSngfXN$ZOG-9e224SHD#IHB5HzuFsS-`g1SK)im;!a{Hbkq zF@hl4#8!Glp|c921Px(wTX{^g8w4Mk!io3eIHj5-)FEpn3?bNE%|U)|*R-Pasp^8f zN~TDvz45d?&#rAA1w6>NJhHh?NN$j@Wh7~p9I1`7Uo|Tb8-*Ak5(FhAtgR%f{H@f;z=AK8qu0$Ee)Sh{Tvn7<#JUwM`zRRmEj3ORE(fvCz@eMhU zxRc@z);%~i>u3D9ep;2AG@p%dBYkaFAY0{yCjF0A!T3+Q*f{nle z6|zHyRK2_Q|p?BPzY#rP`PYm7Ezq(V;<~i_=Mkb`f4A z3gA{e$cE|0^CYs&W<6G*v{Jhr30_Z=rmIoqd82KS^^D>Qp*NdGNy%9~I|YjQz@oUe zV)Xg7u3tAC;`?B?dNnY(L@bS}&qLS9hd>9>Gzn>ulo)4h$~@X#%!z`qf69xmN`Lxk z?7=4$Lx@&T2@@x+D4zZ27rWRVX8)n7Qw3k~R2rxa{-jEq$6X2V410LeKdVrb0(N&` z)?v;SJ}wjk1$AA-`X@c}o3o&FyWnjOg^!+8{pzb?LtW@?eo3UCXcd<1vF- z?Lk-R>s}EjQk6YT`X~NHC}F3{yPP{Wi6dDTg<8{XIKcB|Ce1KOK@IQTzC1cMub8?2 zJNEEfs3Odi9_anuCDenYIZ-pRTH;}-k@K7f6dn71Y;A8_zQwu*gH8lEbZ>Dz>t*Y; z6q2N+lUH5-63JQV4zG^Q2GpO7wL~XOmqiG_-Dt`!`%uaO-t6vS3x57N2h2S_b0=M> zBKPHkS+tF65xTtd*VW;iN$1t(oh3cvQT@5PjMeQkW8$>1gbp^J^bWG4D$GG?wp<+8 zPm0?Q1$Wm??hjynn1SCufAJBrR2^C&tIDEpIIu$-4sAmw+KuM=3d6)32V7H6PONOO zhsu>;Lv+ok(w2w{Tj5(6=zGp$N3@wd_S({yJWq0=#+jD3)vac%zH{<>t2^|-9&8O7 zW&e5UXL7m}ruXO~6;iU<&8x%DAy-#*7&d0xE^9bu6oRNIidTg6 z?B_JmYx%^Eu|rV|qnV#XEq!N=$46lr$&^n#RF15*D+go%tlX`6BwbSJV?qnHgk%~+!^kY?Ma0Iw zbz=mnLCB?MU_V$U4~vxzhgQt>-H3ur#8sGRk$`bML+qAPGOQ@xy0#@A-a0zazWSQh zuuAq+iY3f5U;S^7<%ijELYde+-J;g<5$a*cOpc{{xzglAUrlx>=A{0Z4lUftb^uK0 z0JMv@fv_3}iDlHdJu&llFz{EaC06A}5Nv-kCGFOvk;xbed7_wW`snuDj;$>>ue2H#WFL>8jD3e%g&1 zU?@abkNzx|`YYbr8f=KmhBJ(gz0hM06jxp3Sba}U7PvE#kZ&uHN2q}g!(wxdM4cQoyC6cx4K2-ttSXTK- zxh9YM(^uNT3W|a+CJ$ICSvVuNag3O%+GWon^N?Z9jJR~0&Ei-K6ne2QmMQ3BKSKi3 z5GR(rP7XByk0*bz&`2tl#_q6@ZYY9kIb(&a6gkS5?L8`q5)kK;iZ#C+uEw6~C7@O$ z*4V=Bspnl(7kc|MHDmzYwydUIZHHmlIi4xbC=ERW^TBJDyQSlD2|J0TqQ$yCjEMOD z#>)Pd)fdn-m`z&E?b;7uha5v4%Z0@ZF=Ql$&Eg65ymJ#C`mE zyJD;y4F}h>4i);UM!C-PkUa{PG*|w*ch9TJG<@MyIp(mL_PmwOM_4R?(u(tAG&|HQ zZ@7iMsB5{Eq$}YB=%p^EK9rx*SPc1*ghhV=RS71$p4!2QRkbzvoNj(+U(i%kQvAms zZ~eE~ctrn|_kq{688gx_BP#DPltr<*eJy6$AL!Ac^=eK^+UNC#c@z)p%F89U+fe1_ z4fKY+76f=MezNRd$t(-G&MLW{$=b9AtiYZ6ray;R8g9O7*PA>W_|L)cO0R%j5#Oul zY7PJ*%~l%h*Yo#5zZ8{jW~{aA(1P+*VB`ycHIWtFh6@;K-t1x1R7#CnyVGpk+U3T%WSx zLMh8>WBwOG3xh&~x~d-k_v3p1v!p}a*}?)--{020uCalC|EsS3_dm3yseM`;!_X})`P)*Cf zZdKe^1o4=Ef2{g@)WP!CrM@WwM=Zac@(GXS-@gL71pU{2_5RlHA43lf*OROp|Gw2r@etrn*rgY+6+8CYX1;qvucC(<;5m*4luXE2e+W^x6BNyY%9fMX z%hgvl(x}Jqpm|d!UkDoqdAmWgP)XjdH{JYErZ0*fX{Xp`jM+4%s^dT6m{rsvO1!#* z_H0{Lwf%NNOg|2Xxtr#3ET7+QVbz}sUJ_TdFcwahBPr;hOyeo1ucy&tkKzzBg_A_J zuG!A~RbiFhHzQ0NXq_2Nl7Qok*)?h>2BT)jig~w>kVQwT2hFL%me#;S$_Wh(=`i@B z<>tHm>*ntf^GxA%g{vnNKq7I^3v`Ihm@xIBAI&qLowfU{)lAfJV5D?_2CpxuLlZgh zFBIe70@`I&wzRDm4K+FS{N&S5Q)x$QVn+p!{?P0tSfL8?+cXcN>`pS!4(SCox-FJE z!#913dCztFaR0P>%M9u&IHdPjQOSeJ5AEsnbclDNNyZ!O1xnqYZvq}@FfsFxI|aI| zkTmN?zSK<2ns>lV*$D6r2|rrv9r04gC0&J413HYoXe-AGK4!@@b$%to^N88$7E`fk z!x+8-2Q$&Gd2Ew=^2CWvd*`kf0L97aGn#kh3e+cdu^1jSJKZpPsCxYh#F(i`W@w6I0LF|UcozReWbI6 zQB|t^igtWPnoaKJv-H(eu5_LIb^r}L8D?v6rZ`)-LoGY?*s~f*cD3j{bl3M0fN_Bw z+dl2FfCuUufu_{5TUY=c`W+S57o@B5S53?>-q998RFDBKO3Q?TZBMuPB-ico^XH~; z*4gP^{+6}zb;Ap#ZjCF1i*5>s_jbKO+KrU{9@J5r(vsP7 zCRHuIuV2zlu4e?f@H_O}e0QC`3pRvUD`6xt_&qJHc|ww^{4?nsZ(?zI9J5r1rD8LVUqH>?T0_2^L}BW=pr(GxY1kfO2l)UU38 z`Uy>{l%VQeRP!_S@Xw?VEqi!TXL|fAxWB~sdi3Y_G^q0S{FpfR>n+V!7?3z^s*jaZ zO}w5lxlN%*?$NvVC#6couk3e@K5Nr=exO;a3=nj3*Sz@$S?Ja*t$X@cGPyOTl7nwf z!t{U5_PV+;G^n5C$HCF>#%(OhW|=&35qYl-q=L)m)k3#605L4b!c zUIG%1$TX~gHx;c1;EgM68cdzDZ^ixN5!=3=GR?;-xdtIj@V81Q0!fn1W{#OUyf24% zg8f?6=tRqFD?r2U;}3cXWBBRnnIj$s3;veHE)bhj)Nm^MA2$#bZO9J)v5urcEV8QC z8?sQz`zJ@|gY;pRybD{W${YXuNAmc1NYB`!Rb%;_fK&hO$>>l1Kby(@7bcPa`>U<2 z{{K=i{Lj5>b8yHZ@D|X2`gdNc?hH$dzpzVph4Wh%5jpY#Vj8h6!AbsurKWu_s$qofaHM9a} zJEq+Mk3@``;6f!LhlxrpCS0WHY5Jv5uw%kgDFZp2NI&P@c`|imUA4}JdCIBLfn}%LcMgjj66)C5WT;@QzORPs zt`Xm4SpVLrSK!Oq$9R2+t?+=H}tcD#D z3)G!`_;3eF+^doegKbdW^OPbWLdh#9B?rmlRzPPtdVt3-FKnMM(|=i5+gpe4B^eT4 zZanoWv*r``HwNo0U#I{4XoqDp7Cm`uuz55!(R?B4wG!ThIQE|Q@ucMEJKM?u)L?{> zwmW{&_9eeJvODr-$=-a++3aDr7uslF@M8|Ly2GTht|2e>#3)Gm$yxLQ)Y3*ZnD6J z!-wB|m0kay=utsi`@iXkbyc*}hYMq&<^cuk}R z)9vToIvjb^7I!_5>$ZP(Z9n(KyGFiM)o~Z!;P}AAXFon!_UifmM+NI|{efFrM@36+{mVb4^SN|%k*>A1d21Eq6$v@dF1laHNwY$6^lfmErUGYa2igl|i zSx0hv%uRmz;>Aix6hFK)S34AEq{iUKJxCHX5*-x1a?D%Wk!Jq7W@nYmv}~p+JGMP~ zL_M%%_dLWB^j8mbG;f};uZ>OYkK;*7s=3M_Z^uD}&{+ViB8eOIXoLOx6I6&H3WnY# zi;vA|cm@wQk;K=EuoPdG%W-rDTc~B!77yzl6b_1#&EQItfkHRw;c6-E#TdXc*QSvr zCI+0+gb6s3D-7!WF}r71PWk!+e_>atF~kS>B)%sa(73r>6h*>jPEzTp{GVD+E?U1z z5f7{Yq#%R`lXzE<@xu@$eG>V>?li0}vMAqwK#=d;r%AOXpw>V@6=SgNG^i$e!B@&b zfnNT2xEkO?&Si+odl%K1$DUmn6XQ=s`IG!EFlB0>UiJ=_Z;H;`i7T4v& z%448DfFP{{fr|i2Xw|mzv0v%^Lwj}>KapZ?hEJ=KZ9Q#_P63|mL4A7;!BkBm zI%b=IWLmeaY^x#7=BejkZ5yh5dTWXV{~UL!*7lCO)SdpzR3FFWa_bxtTDAS;*cy$m zgS6wJzR00UegTAAwP{q_c`Yb$>L9@^65K{| zrA?DUm;wv2$z4Q`kj=#^KT=u&-NImG;iiFJNIg)y$494*vX^w1jE~<_Py9c+MqAj| zt?gs)eO9m5t>0RVJW*U70Pu8%utaWk+PDL}ur6F*HGkqx7P~D<9^*ZHq3X)8k_)I> zvZUimdd^~0ODa#()PvD3*&kYbGodW2TOBVxmvN81C$J$qh1>@yBgGiv$`)BFa2FuFT@ zkmL-pN`zabGJ3?&N9jubB6Ui?hk^U@pZZ6pMa2* z4iawRk?b7EMO`4(P&Oy3zuH`qHbw1#FCmsKZ$NPzfI*GCuu(uM`cSPC9O0yZoNCjf;ClE6N~pGxxed{p5=_r%=yyy!!ZD? zIiA$7gTxETF*J`gEEsA;ytQ@sf?kJtt8S2ku5e6fP$$R9g+;+1CO@Q)u7@a+2ew-W z4}d>-!!(d(yfR^V-_@vb!pJ|pOEgKPQCMr{%$XRu^ryMLlC+4UqrGx+1F~do1CS}iT2?*wjT;BCaUKNXW^dBu z`FhpAF-eaC-%9UGk|>_#bVx~FKfNA8lPNidCT}kjy^*m4j)3d){e6Lb`Vlem;a1v{ z;7e6N9AE#X{o@ClI`qbYDQ5I<2F3@#lO8f9QF>Ku2=?oia>@}ARa_>F3D<4#6Cw%0_JU=fI@u1acVOz=i4ye>)p`Scy_hJ=I{kkH7b+5 zq@G+COC%EWQ=Wg47-^QJn!>NLN)t9S7kyRCNmK*9LlKKNRQ?_Atic`(w|bXi>uX>@ zN4arrx5olQw_d$%cu>6FUwMuLD9UDjx93>MlI+=U2~s*QIF=(QZnx2$5+j6Gok$zo zA&B&DXtgkoBvg**rb55-Le4oldzthj@Fe5}2nu=;0;Np9B2InQE^6Cv^Es&oKNF2s zvoJMC6reCif-nlu(Qz5gIANHqnz^QWF{cL*-KetrL$G?_cI@>&0+GYyfP3BH^Xg!_ zc=4i5nKB}xh?fWQ|5x>DhSqq8P;9R-yvcoy-Qej#{g5+W&tW79Y!WcyPsf2B_IWj8 zzs$&z=IV#3cvck#2@Yk@0vtu2E*})SLMNQ6PY`kF&IeJ@%@wc+ENxtmZ zgv_*gg3TqI_G%`v=@UWh6UXmEo|Xk~-LOU(Q@tU#B8ME9o+MV&?)2qV+NL`;Av^DN zY@5|R-^aJUfAaWoV-3_ zZnB5_(i(rOv=@~12~2`>Y(fqH-Kfh}rnS&Fk{z3%@7~Iixr^3$cZV+5_j-GAY>wDl z$RQDE(9O}Gd3b8u_Yv@)-VJ>A0e9Il!Nvt(w`S7kRoh+Nk*BC-So3*>CaXDvYj^#u z-C@3dEm=?Epy71mLP&jz2YxaojCfShIEGi|mcc^r%^SJ#$8lia>{E5Vk7!B$Tr+7u zX3sSFSF_0~tUJ$hA(6X!qtcr8e(}McNO|G)d9AmZ@$?j@Wy2qT`l(wz@2k6hl@Blt zJTjEGsf7C#C=6ho^@YpgLM(`h2bVr)ahFMF9-QUL zVIMU!cLwAhW?8Jg_~!>s^;@;N`fvt2Pt%hit`3$nZ&FC@6*YU50;}k(D+EA}j)yet z0z1zzwS8&ZF}sI;OmuToc0>&RRl|bpUABf&%xc~Y|L+-ix8vW?)KeD6m4Q;t{IFZ8 zQyamuw{v}_a8iT&R4WBC^hW|hVRCB6L`ZCG7R{*$Ti?xu2XZ$vm$m>E?bv6K_DQA| zrF|3h;1Ik$@%h)ab%bg`COhWv3#jod_}hVx18OsVXFjgR$>n!i z^LiG%hrEYN_b^m~Q?x$~g1t}c+%9b_>G%T9#;<%>4*NYNhyKA9j`li)93lFg1Zus| zNN7Fi`nI|G2tcb;Y4Z4Tpo3h>_|q~Iqk?Jo$jm@p5}Li5{D8;QjX_85-`wYrYDW;Kc-8k_iIO@&Fxfi>RjyE~3UkIs{JMpjhi;E)mby7kIAh*=`! z6T_2X^G`J(i)ZIwVk>V>6LTIX5$V~UWBz%D?kYUTMd=UeoR|#j&X>3Yd5 z_wM*8OJ4jwkDl}^S(1BBgNypS>$5Kdno8Ucv^7A_VdRM)Z5{=`DkuJPCMFRmqXbOy)jlN?Qcphm zsGooBA<2myP1Zrt#N=VBX~@AD*}cK0=3!eJ;`e|0df2zTcx`4`&qG05`TX(8nWVGQ$`c3786AqeK^=uCko$$gq;4`I zxDFJZv1HYHLr<|RB=ObOBOalrzh86t(w;pRUaQoqZuR3fo1xERWYWFV@@l?rls~Q< zeNse=p9rh+TV-HCm}4ix$~F78(e>0BsI!lD&vwDF7-)%1M(ES9^n?{#ti0Zkzq-=K zmAxzd;YH0qNMzEEUMPHlQHR2A?im+i?uDyM*2SYl&~p~%5CI<47^8ak4yrnDBYArI zr=K_+z7mLPgS3yrWL3}5I)5HpNF=8669yq)G|1wV2jQ~8fLP7R7itiJ+N^TFabxeC z*WW=dPi!^wF@waM5^P*!2BkeL*~NDEex#ZtiV>tGbwQr=u+WHXY1KY(3ii0(nYMMS zkECv*xLFi`qG8ZLi4f|dL?b$fQ)uKs{*$fjs6#!hZEVKIglbNBHtl1RLw7m$%8P?4 zDTi0D86TZDmUujn2*Zp}(I5e&(0Npn)_PpidL-#5;SCc_u&4;jUD@5 zZ}p+073`2B!Jgz0oU$C!YslLYuKKV%yPHTBmbL~CcRhmVZU_y+nn}-duQd9#^T%P1 z1iP5R!_YN7WWABMdp(NeIBI~XJPM=p6pSPGRwIjHI&)*Vws-l?W@G`~O|Lok*86tv zbdCb34@rB>lfjfoF8r4}1FFk;e%hn`5duPY({C{C*I**O*Z(SqYnjLO>GX334SIA5 zu%{UUu}jT*%(tI*or4HHTN6kd~H4UgMSQ<9Q~zfg%t7I;R|)5Eu5@-S+6!dtqQ;Gq}oopcCvF zJCgqNgd11q(g=TbvD+pIGrdhCX!W|iW+O*e8bIQ7mz*X(2Y_}06`IG;^_0ISaGr7B z`Q~|cR-nrVRimJ7umIP!h;ctWd#`H;YD;c*EO{aF)%DIrf*NLO6 z57eM&<4+_-YXB2lH6CSUbcGnw=jYEWoFk2WcJD?weTrqmR}QoNFfOgP@!w%)V7S=hshKl^|P4 z*RwaQ;Q_5CV6<=HK-~XqtHhesj$!$*6}o=0Sl zC#t4xU!iN~4^3;SZ?4Q<+yE?K#|{uneSdi_zA2{Uaqx|z{^qPqH#|S;ubGI})>fzF!PwnITHbZNbs40-BfuU!UUool z=GF2^X@gzAsQI&J&00gz(Jd+&B6$99>)QMI94uFc+IO;D@6{uS))W7@abqb3T;%Yh z9qJiuQ^b1@_CyCcR{qmwNTU*#CEkZtQFl^o)E-og<09Sb1CX`xn;uR!r|#`gmZFYG zJ%nF*f=!vcw~ka6fLx|BD}D^==>K-?48=zFLJJayma+%`4dJz`%9apmU~^{y5V$S( zY0RE`Kydke(AMVhr+y^ltm8DT;8FERKQOkFa5`>K9-OL-xP9F2V$0*`#O03r>0Rb8 zO%hhC3c2(9s^r<@8-r}zN#|e;uJO|>%(6s8Qp=#p=&%#q{E#J5R%6Ah&Jm(aHaDS7)&z; zQ)4Fkb}BK9N@b~-CY+P(2@#{B%~q&HmKlbKa%gCgku7Z$*+Lm>sSKqkm3^t~O3`vZ zuj6~)KivPpef!CSmUBMm{l1ph>vg@Z%ucR305~Cv$+SYRVe(X z2h@$mW}$k$OwW%C(F+KqUO#%}E&TNs<92%<2M1PM<&g^D|U)7yCD0Y_vlIKEU&9(X!D~FYa zw%=H$Q}kawQ}0)hJyhdw)!fRhdzFNBQRux-@7xB0XBf_P=gIDq|P3t_F1;fOvo0;Q(Ed%d}pa3R>@U7 z=ZWK|;tX<2_5Bx^p>Rvw>hhJP0H|q0|AkHS2w7gcI8-6_!tU$8wcHN>a7Af?CT}1< zA;+>AiFQxgDo#AEAt@&sHW@t#Io(1rg`Ae^SlAVD^>XH2wlI9N&5ha6qz*sO1tspk zj;-OPQ>A8McSPiN#xWFm<&=cKx4AJE<+b3Z?)Tr%N!q{NV$m%3zv^f9Hfokt#*Nyd zI7TIy{tm+4X^|s)Vw!h9qfCvB2?%qene#+_>n#}F+~%X4J!nIV0s;d;B?pnM#(1~d(4RJ^uDR65-7!-G$6yiJNiuQRea*SOx-pP zAVWt_*!yVq%&!za(#?8!cqCRszKY2nfTxQ_ZBJLXT_Ho$#2oynwvs!ay$M+B?!)dV zLSad`3^R+aRN+bf5H*fi#9;nlLq&5)J!wfwR~@3nFh=i(4eznFD7MlkDhXwRy6(8=dC7KI?N%XlTS z#R!mSW8nq{-W4JS-urK>m^)Mqdnv+R5M+x$ZC+L|_m>1W-Iqp%igyCD?1B-9LcXw( zj}XXWjY9qt(Q8U!>sGDU=VDKHd^{V+u)V-v*#JfiJprMKS+Z`gnfD{^oe!G9sq&jE z|45RP?IP3_UT83-zXmWlNVJ?@6e8;E54CmXJyOKJKt_>8G0!OyGHDfiK0CS=u&PB| z^c_p$-sYWcrtak$BBo5R6UFCd(||k47^w_1foFRWwgj?}QB^MbsLBmoNYcqAXERh~P0b`Y^& zEI9;E6P>JdHe z&&;csc%y1fT6b(UQj#?So~tGSyy`0a?yK0hr)+1~TI#c@uW>(MhQ)Jz4>1D3Plkr@~!l-Qj+Uke$Mw@FXVFZwMFy@S0?n39*gT;}(FW zKw*qiI5k9L#{@qh1lzbFKi&5csVp$X6{kIbl#Gy$p|5hJ-8O?35}!E=h0~>5ia24G z9o3OQezMk5Ml2!j_R$)?046@}y`Ay44nI$uKu4?IulKC2u)aKjenNjCQ~BdkTg%|M zb?a!9m4UU5(tDrf(JP4p|@U^XhkOj0DREh7lixtF1$yN1vV4;*CZ~(f`=H z@;y{Z0yLK}$J6j}EOLFSJc1o^&}b}Gn#6@tC*bS4FXgA)b=S2 zI&+?$w^xG!%md`oye}5u%GV6IlCtpFU3+M2`)SjrMSt2*yPQ)07J{_`GQ3NBpd#?8#(Uy@uAWw-==|9O#3^RQi9v+I#< zXjES3a6EIU6m!|UnV40_nTw#9vr63qF4c+1)H^E&%a*mVtz+y9*3T{Y?(QL#Jo zYkK%o;;e3{-r1s8%ZKG)?1@%EQG+~Pa3=6ZY+z4My+wgq{k82dL}NMf!75GoMHbBI zm--o)5r$%&+LZ2b%?dp<~(jR&+?yvnZN@lbJ$}UKTVP zu3Q(#+OR`7aCA0yS+~tinm&@d8;j0!K)=S~-$osUb2y4}dGldp9}b_8*^FS~ksBq%0Bg>ph{e06&RPz8gwAENUk*S_(uQtQGf8!V zg5I-KmdYmS=6w}#yvaa;Vb^5_M|Y77DWJ&y#@#F!8kY{%`9=|o9TpCl13Rs~(x6t# z-^aFd=YIs5b1EfE1PDjD5V28%SE$ta>+>@?WenMe+2j zIw8wX`9>(VlgFn`UCJ`ttsU`$&4OJuE2Qt71DDwT*`eXhEW7hFx%!?Vieso%J*p}7 z&1juYsB2>vjuJ)*Ma}lGHHs&|u5!cf%r`#Eqfk}x8|Q%<#Y-wQ z4H%a`C3wvxHw|Wx6=8a4nQo0rN0;f9Ij!1u@t{OG z!T@eUI;5MQ&AQOhu>GoWw4fB%;N5zauYo^MF)f^XW)=fidhg$iDMmIN6}Gqejh`!A z-jaxo3un_P+ntgILo7g?8z^H!(EA!;zcL-UkT$4BoP$clQw zjKqKJJ>B?VlbJnciIS%j@Z4KTSO1hM-Z2V~mR;0o(v62zmpRa~^o37h$F$oIX+NAof_?$$gDbJ7SqlLObM~vY+XFKQ zkgzZJ5HBHDFtM}s@2(%@!l#tYZWe7sYgD-IE7y?Ip!aWUYRpO6p{0dG6^ct$IOue- z3f!?g)%ITmkImI#)DADC#^s%VfTOy6hq1U}vIi^;y@tly^2MP&I)m+ctl~Gzz@UiT z+49xRHA<$>&>`Jq+NywQeTa=K$&Iw{cBFHgb>CX1$%a^fNvVg-WEr7oV2~c9xWFgjQxo5T{>dskK+F(7d-Omb_?)rp9^MQDB3DXUdaoBU zhl4Y+Zqh2q+d$AvjZ&S^r~Q3%elLrr9y&sA}#0nx% zymfGFg6L>K{z9v8iGuZU+M_#S(%Xldf2p8YK{UUE2R%2O3TXy|_WyxL?CUDI;5n6q z^4h?)?&b{s3VR~jRbkjXsyPvbgg==GX0iDl?e8CYUS1jgjlzZcf4f55cv4OXjwN%9 zRL|&qN075sac|ZHblCYe2INijfNBwxIfG>;$EaE=T5<^rkW;u>%mUICZ5Xz3lZdQE z6vf?3+CA#3x+%*7FzHEOw@%(6mGn^XFkH9fAH|hZQX^4EHpvUVJuakEfG=;pknH4Y zuMFEysc8hl;Y4sOdaV_s;?z2ziNsh9^5=4KGQ(@KL{48N+?i6Tv|0c4r6?J=LM$uR zpzx)7-s#kTGUywoU>Rx;C#&)UVRuG48?Hjg=(>tN9F`iF{_deQkRPKjSPoM!yv^U2F({ zV#kD@)o|&l5ofv1NkY<0HD?}L_7QXb@P)f0I3QvOw> zd4}ex${s|H*`+hz4Ghy0`GGW>7?{>y^Fu;%la5c8Jl#$Dr-!lsn#cXU($=91{P*Exf1fDgaD<0n&Y&@atOjjL@gv4>UG-sN4O+s93Sa zlAXd(YN5u)4=6q-K9pKi(DorHp2We!jk+h1sBYW%ceZtl zTx@vkA0Vd?jf#Ocz@2i9sa6aHx3@f2jkp(oDysPIcN_TOrS|1KN{e{LZMZv=lp?jUz5&n9c6j}pKWVe8i5Z6fb=11?l-o2F zf~lH>7>{Ixc?9po1?;S+DeMCfF*;#$^%fUNAFd>gN~8(4EuyG@^K}O~DM2Rs2Nf!19%(IF947(RvT8=W*wTqrv`_-~q6o>{!60SRv%Ft@caDZQd z>`m1<9QB75${sN}(L?ePh>S{)Sp=a`T0LgI7|1}@W$(zM2~mu=goF$W?R;t|kJp+` zj%4e%fKSmAyIR_3QG`+N7vBy=o2N8^%NryRp1%$VDQ#SA?zYPxBF;!zg=F$eA6*a0 z?dVLHdI2Q3&5aqBxin@v@9=mY0)a00yIEmFeSJ*r=O*s4MugXukvDIABwpXYwzk$7 zDPZp`N^4Qt!G`1>s(6OP%$TBBI6zG7E-l%Sc??IPfzr_7o>pwu!1a4naFuQ{M3Il+ zy!GhKwb+N~gvVuRsSXnE=OIHs$a5;(k^r(;lGaXJ+nD++ZL8Fbq*x28MiUKv>dCgT zCF5Fc+{e+3BrHY3%2=5gKM?G(*H*>$UuIk9b7d#f4OU)Ufs9>yb@LHjSVnYHX##{h z5B-S&Y-5)pzL(@V-)?UxhkuGY~s7R_g+Hqct&l#i6+rd7f5IjN|b4SLP zw!5zs&b#c8;L)<*t!(+Tu9d|rDWrTV3@a81y0Lh%d~je+xZmrdxZ|mCAR^H*(h&9quU&Wq5-{L^Gv*&8t%HTpz^j^4>_T27FK99WwXM zT=~kN6KBBDDWHcu_<7A}vQdl?lILx1OhwNpH+qH`&x6VnFy#jfQ^qw&Cuva@@Fzs_ z5M;B1%F4=a)LT6VTQH-ApohUV<)N`nk4E$W7wq~jycxS*Iv1$V^VRKrF)f6T-XzR5 zI^6LMZ`$en2!o6nz@0fkDUk!`{NY?mE8jih_kn21w+oXxzc1oKX_-$3HV@O2LQu-= zr+Ao#j@8uc`0*W$QE&O=LzW@~~+|+}geKH{mi^gqk zXi3usSIg4#e%o4WHTJKV(+U_=D4e^u*~4Q}_|>1dmzt8Pz15B7hh)Xed7;()yI74p-L$QbaJfnoe0Ar6)%z`@V2A#)#<^f{ zb%*L0d{O(0Y^I{w7Z!RufwACileHLZ&1Tjv2&%mdzgieuK+ zHa~5&pky4#Pc|4d=)<;PZ6rU-kRctzzuOeSoGB}=AX_@RuWNfLOHP;~f?j5HLINki z2z_+~o^+nT6EK4-j$dl&+HF93=jt8(HB|TCB}};&9AFQ${TGbluio>g84PS5tB#Nz;XX+Py5WvcG)t`3$X?Vz{w{Lee9M2)&(S zHyuTyxjIMs(W$E-PYB5F-W^(Ucq9g5wJ3vC^$mjB>)#PdP$cJ4pPIbEs&f!t+r3TA+@V9@=(OwCNBFWlXAXnR9S z$-fnN$lpv}$@q`hxP=(Y9K{rCK6mQiSu}3PvS$T6XeoGr_>jwb#oJ4Iy;Wik zJmp7plq5>o3h6kVN4VEp1hjl$SBgRl^e8s1b2pFm6;9dig@~mn-$-J-m5~RhrMC*0;2RT9eUj# z$b`Zt`a&7UgrBlgi^vsB(oQYIJYrLWKGKOi0aCsl5a`+QU26@$FS++CV`;KpAP7v? z?}?}^#5awGG$9fAR&3WO4iBMy8)EIzqub>{{)+)1K&hqx*__BG+?cV}L`H{*`e6`Q zV*Df{GaT(e&LYzL3i)?Wi97EGc_2I?mBQ>AoDP)_^=_BRLRDJTi8QYTU ztHu|Rs*`2FQL#B>w9gYnirV1>YTZOmXeF(P2=!~5#mot(x~`0as|#peWEVZk_(B-ntCj8hTE@&QxOYK_GhpWzq!uqikB~D_xuV&U{o~Thh!?+9uK*=7;paS@ zdbFoVBKT&i-=tI+;TOoy`y#`T(h8lXH*ZP^GCW2t841ObV;LFdT!g z1fn}aLH2Pv_PU@XWt#>~l*vc+1+<7PN%rUS*f^Ap25FGb>UVX}kr z7I5|s`~$=XoUS<~n>sd$pruqp$EHzQGQD(f`kdzDUqVCxSE$om{>>l?Tz(Z?%xYU1 zgQCcm6K|#7gu_BpB1M1ItJR|Ukwy)j0)wCi?(2auM?2pb zqXxHtpiBhrEe?MTRZ8`q8Z~;j zrCQBRPk&OW|3Wl}jED){Jaxi^2{s#AS(>7rWW?=PK`ap9n1T3+IgVU7iC50bQDc9+ zI_UdmijkB)pPugvRanh*SxSzQ5k0}VcZT(lb>-o-3IATaC}+YQ{LT!AYk5h#jy&Td zizl=k2HWwO74v=O`z-LhF0fq=j#S8ZBZK)YCk3v(SVrX*we|l?c-<)c#Bcom~16M)+VZ2emoLcU(aXOS2 z>JScS@q(8}*%?Ka87qU9Xl1%Z{HLYxBM3_XsvZe+Fj3a`Fvb`^-n#BmarRznc>(a{ z3P-b-x@^_w1wjx6UYu}B(Z2hZaGb>hRaiC*;8v)Cdh7ryl=kS_*2uWe|qwWiT<}q|KZj_fpi$z#n}%KSo`|! z&&(J4L57$_H4W!*W@TL}i;f|U;d;^{2P|o=wCiT!KTz%>VsI+7&~W>X`S&iM>{l2S z|JingODap;B_)9kO%zIC=E#q-&cV<%q_n!8dX@cgOCb1+cu6=duiLg`jp9-H_FYZm zlyTfuki6P!hf>#E8bFFr%akmjBHHce^zj9xHqwa!d1vc^v)dkvT?bMDnf?a0sj7fc z5_CI@fe~gld?&5T+%E1^Fg^bePcIkK7mSn!AF^#Jo~<~@%kcHF>#t-Vk4Pvr>-R1< zE?4dlRvG0xiRAn(UcB5%5T0@Tn4qeZrt>jP{x&vT#pC3IWFIZdV@(G7^}~A3VRA+A z1k;NxVy45;$!2hJ}%lpAXtTSsE_;}L|xtlTp4$$x%xL>_r*od_0*qAZ5-CH%kdb2GszKxi>QLLE3 z6-d${bx+`xc8l8ero6IlwV~p}%aWo_pOvv~!ffG&Gs$PT$w0|JkLTcx=<*W#! zD;%X-Lw0fLoEs=??dhQr5j~V)NYv)%gH|j7&{{&_kiTy<3@PI1gJYNYZ5$Cjzx2#) zjsC{~z1mZB2BGssd(Ai!AP6z1Bqz$^hQJjfFIJieEdW)A4T8G}G!3 z!XR5pRy=+S<~9gH%|J%wjRAm+HNZn-3oxAS-1&){zNu z&1m`VVMR_o_kugg81yc>Et$)_S)+A~`D-xpYCJ237@72V6=JNRWKa?uK%hg(maW#` zEUuV9u2%wQ@-fPNLEQKpVAgM$6xEbF;pDv0;z>^fnE*H>R?fm1p8RD6`i|qYi!jhH zx4BwSbI8V&fPlgM6k3Zn3U0~b+FJmHagcpS!PIvH1)vIa76GQC^7j{k3U?s)LXuIp zq8UQY&zQ7qEAdPX(=)h!%K{E~DVk`0g+hr3(*Cq!i*Ai`nYj zG2V)ypb}F-8F45)1S^w3VXW@9&{+*KCuM)}DnkSQcM_^~;S+Gj*x5lx-4Rq-0W@$8 zt~78guXLXb8gcZ2kH6Js{;!_r7zj?B^yIntg7bW3eZsh{Z8k=cV-tv;l5tq#_6r^U zRK7W6vdbR|DM4hT6{!k0&Ao`~CHL`ZzHyV8S`wU$-HP1E*%>!mcr?tl4{!_WOnAc(>$x$xsbyqCN8Xj$y(p zmK^ktJ$g3F>a?VFKp6<*bHHuslxRG5cWv9=U4ADGr0l1bT@>_?6qX-#4GpwZ7XU2_ zXnWa?qNz&~Kvg5f4-0*Q7<@P`r)aH)f1B~X4Na3H_&zLwy^W2D@IN77*D_6flc%Lp zVq!ARA6rV1sEIx}TWLSS_cm8wibX>3$!rnW3j>00>`p3fL~HRvJn+Q@;;6u2Qv<)M zaFe!tGO)WJz+1w?v@+*V@E_n_zTkiT{rd2WM6|!P>#%wL3v67zobTrX^;3#2I2oBQ zCOK456-0V@4qO_2u9aB1gYkgZpF4mkamGsvdUWB%2nGIEJTuoSHM>3^-h8B^D@_xM zvAVjtA;xz1v&iRJm;l_mIJYCWI!R)OP(?St!ckiWO@C&~O<}GrcHlh1b|V z^9k1eA7|g)dOhDf1|@oei1b_;1Nlfu9pvqIyS9nutL9NC zv{8`%A+Qr&puCnDXm?>}@_A%5!e+v#!RjsZo2LBT+%?3g1Z*AM zwO6An!dxa|!RZ!j0;krRXCsJK2%zFPciA|SaVKQ+0x0`FqGBZgOJ3XWn~Z6gNow5D zD|h#@^~&vwue^F6WL%|$TDsG!Y^>3a(9TwJ%f)}mEyQQiI^`?atbNoGA}SY6@Ue>; zPssYK^jxx09HZf$PBOMyjs8#EfRII;ThiP%^(XwA5n~J--coDvU`Kh6%_QAR8xNw` z37lOLrn~@pJ5dJ?WnN_^|Wjv7ajO8qOpVe-{yqam`e?FZqx1jHU<|5X7}i>IBwQ9VY9< zQ#FWf36Ri~e?~M5BGIM{Xm?6FRT4V{X*fCyfxwsmcn%D-o!2@ zCPvPIDE09R8RcEh`cwIJcT#>6`Ko|&q(kwrlQIif$EDt1J`ZxU5d{|8Nu@dvG^`(* z04_WYGKSkO*D=}xbyx34zJk%+^x-fvk67D${L=aS70gVK z6^R4lVa!v;+iuYdlqtsDmSFaBuA!zxihb&(9m7|z5_>z$UIfTsJKv%r>w|UcR==lq zr|OIua)S0&UIM(CAmfBQ{ne{iQ+Yp^avmn;$I92g(_qSGJ5pdjJ`4e}IV<^GheuER zf7~N=7(!Ww)I}UG(vH1GyuQwLH-hQQ-)`&q=5SL``3X+nCEK*+| z>xE2Oj+zuQT=c)9c6iylSFaqp_)e~$>3bq+t2?Ql(utqg3aAkBg=ErmjnfPO@uNQ4 zr-*81l-iRTC5s|2!ysFSLTDWE80A&O^eFiJl}2kM0@*I6XK?L zefP@(eF0@bTqj=H&+6wc#*-tF z@y{!yKUJEPe=0l~Vti$(pI^@`qJGEF?T}4E6beARQl9t534X~WY$8MEe zCu3iI`}J$#g=<&+xxB^*SVf*Er3+84{OkP3tn6$PEMjKPyyqHY`q9=V zi`}9f77_`2wDI-zU3z!!PEXS4rqIwLesTQl`iDQs2E8d$r@BwqDCeNjrnJKI5jgGr_?`}LTwZ+1P$yI*o-2(16 zo8ssgpOiFK_TCGtz`fzdIfu>s>0dA=RHxPl4j!z)Y&V^1Pyb|zxExlZV-Ts8EE>LE zTwlL(uDg4}nKMocPtFJr54RgP?(eIQZh{!McsgL_sR>)QY>~cFV8^r*Cn`j`)H~EN zmPGkg&X(ZdZN{TIcI>E@d7W&%M@=xYORo*lq=cXHfJkL-M&ds@?E|-N-8#+;wnViB z3l{ushHfX$&dzR!S{zEB16qk@Nru(Yey{X5J-7YBD&we9|e}r_G1}*taMPJc> za;$Cv8pOjK$~w5W=oQPmX*(b16*REaT*^VZs=zMf$(37_TANF-`P=5E_cTTBp!Q%E&Q{!i5VUckBxI=ke zp1*^WQ#>*6pW>}s+sfh&ro#lXDJU#F`0$|%D9z!62S?Gsy}Mxrj%#9Sdb_kV2@tIU zUd3zCqN#J|9$1*s;CBwjzS(Ol2Jbt%?g*nLX!f}kF}d9kvY1sx#(Rpf`qjhOES!~- z)8^Bo;t<)QLZL)4hy+3XAfFe|!7Wg4iuU;*@ zKE&eQ;y=Y$0^p+2xcob4wx7R_T+LeA-dQu3A@Dbx$>)`%JNcH1ogbvmuy=b!XVd;9z3_k3F6q9&c_eOD~-keg56DB` zMV8;WUe;Q;o-D~l zRNefF^MIY(+!$-pb2tVF-aG3a12~N-T!7wL_K|~(JkA$YnyFN(W)}Lj3WZ`j9Zp#R`j9n2*rWch&I`j7V&Z*kbw>GHSjH**ID+86F$qU_!)_rH32^iy4Glr(7MMqI zXnPzf@~&&x;pyr5t1EFNDK<8?{)VkutY%eB4L@9D2&S8ZDb6X@4c(qaYHYq2KL~og zqC6kgtKijIJov;C+u55B{U3Ew$}IIge#jF~b|*hW>fwM%nmn=kE+VTHKx2 zbmm#i3Y*5Rd4_Uf9!#K@oHOra$ufj{{d)KQjUq`jo8gg>eF=3aU)ixzT#Ur;bGce^ zq^RGKq6B{SGJpTiM{c}-o9n%3(axgE+jDn1AAJ7Y<9ZokHz=>W-3im_4LzOJ!;ckJ zhAYz2(uD8yJw5Xbo#I6v{mRV?UF}CS;gV2LUU3@L79_~0J;92!zqxVY?yj!w+UvG4HXj};Jlofx48cNIy&n2P0HD`sqt$vW9~!NP@q7rZY< z05^_?`^&4?U0L?)N~6cCpvfP#Qhn6V!}*5bl<^UygWT8u2}JTXU*wprvpt8ai}H*2A*lGe>QwW zQt0(3Y6Z#7>3muKV0n}GS5LLE`Qui8ej1QsQ`y=@T_dBTt1^bXxnAoP=rpXca&}W{ z{XXACD_SQdCA|xyJ0^L%uTQ!`e5emf zOtwWcvTv?v`4oo}68g=*EALP1!IGM?3$m4@YuBWA?|j2|?%b2V0!sx)J}Pl{1U9(% zPtj}?4A;+;)wCrKcNz4f-H8dcpQ=huj;D%PW! zJ2U!NS(TbZE&BB=IFLZPl2t*cj zD%)QjC@Y9w%a$&UJ8{C16nqoNvxW5qpA_Herz_$UcErRapEzM&y7GN4bF;hI71lTO z>b_5(0>#vqsi7ig|I*us@SX{|i8%N6ty{4|!NFf};L>HY*lb>)EYO+jJacBW;ze4~ ztoO5vGz(>Fz1{>^iqowuFU)9ox#-ig;Z28=lLttmK6`d1sM%zdDtP}dHEms7U8THP zer{H&UWh!V9+f-#85oOs_DkzA!SFc5(L;#mNF68f~&70*V5x!+da&~TRTF$eQ zVGL5N7daDeTF~k2>(}!c2GAQcoTs#S=N|lNyGP2V$KfieD0kZTN8J7u6 zj=4v0l$%eISeif9!V?V>AIDA#Ez+`-KZN(7z>33%J{8C&E#INLDT^nM8bzBuR6#7` zNFO+KC?PfVR%+f!|2G6#6~$msHrUM(_TrqnFQaja>QCaIk$>lo-h_`)<@Hx+5N1_c zx0I%(_3AYay9VfTaed}MpGB@rRP61jaAoO?B4i03HoaGOkQ*KY>vxmT!H(0i=$oazJ0k!Mm|UP z@7M3T(u8A~vT|H`{Zk7Vu)@M0H4Y%F@(4V>)lcub@3A`6l+I_|(KY07V&Wj|K+fgm zW@TwVf8cvMB>%Hccchrts{xI0^kEezT#Rnte zyyGd*#We%(z%{HQ)%5G%f2!UuWWcgZlcW>rZZx zXxoWXuSMz7?U>KR!mjBX7>KJe_;lP*yr)_DCIx$ts z_Th00%FjDPK>Lnxc5v81F@hA$+!IFo3oVbW~W5%@bdidy(OoWwE z!Et3L^G%~dyCuzTsF-RstMc{Ip|TQL)8}v~IuN!ilheoDuzVjvBkw+4m)N|=*~ceE zC`+V}*Pi$q85oR`iiy_a&+|{;xIC#)O`4=`+s!f#3oZ`^I5utBB5zq}iSlZXz+!3AgZScO#;y9Ckh)4>T|WLfN*jo87brA9?O`TBV6|C(O+qPTEp|dXDpof>#u7ClR9;3 zFlAOncE2M9h~}#4U@K0PEUbLJQESZ+V?6CY_=}PM|Nir)X@gV{|L4z$X^Z=4postZ jmvt>S{QvUb^{r!TB7V4jZ&il8H`T~-w#hcX|M9;7M!uVq literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn/gb300/gdn_fixed_batch_flops.png b/benchmark/linear_attention/results/gdn/gb300/gdn_fixed_batch_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..21217836170568f000384784af56e11c060f2cf3 GIT binary patch literal 119449 zcmeGEc{rDU_dN`sh9XG{l}bc}M3N#hr3eWTB4d)NOd)fkq6~$oNXBTAibxScQ7ME_ z=8$A4^K`ElUElkA{f_6)=a1*PkGtczK6Mpu=lL4;UTf{Ocd(YG$}*<4OcX^eJD{qp zLs5+U6h(KIaWVcza)M1CzwLJ2XW)AH^a)pYb7w2+pt-A~{b^Tw8w(ycD`yv*)2Adv z#l=O%MR-oSx;nb-787&$pRW)-?QAXP+(rgZlbwF8B z*CX!Nx3k6vC*~Li>mBqHli2z5-(K@sV$~9ywXCdQpvi2Tt z`gY}kARU{-8AfB%vpw!r&-=a`Js-JH|7WOYWB1dzvAoLO<|?0rDivPF0E+j&e|NpYYy9=-1)70736;yNgP)S7vlV#bJ z&u&tGml?eBl$7(Gz9f3=#T~c)#zvOSMjvnQx!m;pbh1nL$MnN@!*WW5Smk^s1G_3? zl23k4irryzEvF<=L&P|j_1(L7JvDciuV26Z85fJszRqsPUAuNkNlV+vtgjG|3tzN4 zQ+${q(}*nn6S6gT`gzEo3k1Z)#dDoH76+U^KQz>#vS!`7lCrYJ%hv4-eD&&8qnod< zuaw8&o#LIr%Cwf2mh4-PvhUovGhOfgDtiY9iO{yn0s+l9Mi&>C?2#A|!+^j*`e(LZ zm=50D@wrt{Z@{iNaLMDxk3XeX8eKS)AeZPh_B&lWd4E`+O=Oz*k#sGZOp{l|*S8vNvu}9pePx@w^1gkE z7NsE(YN;MGliAf?@4kKGQdCq_>vwf^HM6x1aqg-}JP;`$X!0_*(I>sHLp@qpPpPHA zCpxF{_;%T|!}PmOe^)+vQjj{{m(9k(5u_d?8k`j+X1d|vt(|&aw|_Qd@o6T=g|cnd zymR=jreUMLV&u9iOU=#^ukW?B>_bCCQqG->&CSiuujT?eYcme#dX0%vuZxOkd?x?C zi=26DD)IAsOz$4w=`|OZb1$YeZthre;>3xwD-WAQB$PZq-4dpFezA>AocGl)E{VG* zMvm6Lsh^bn(K&ESSGe?**Vy3Xh}}!CF_r+H*L8KPu$JfIF9Xfch^r6x7*1o>W z>FI)N)~+r7{(S@a*U->|VPVVOPOse?efLm8`|QjVHdaANiB?2JL@UqPYRAb>R9RV> zZBy5e7HK~}KUqB3>6U_h_Vyz0KYY-B<#}d@Z7t)A7cXw)F!+D^q-C7zNP~Y5%E%D1 zt$PqGaPX!A@6}GjsnfkLg7C?LEsVUkK9Isf?sxJS2#J#Odc4B)A&Ee2^QgYpY)lSnzCy? z_2H%@EmidSvv}py^!GPqWlVVF_73b;r$$qhptcglGySJZK2PV+tsS9R-V?*LlzHX# ztrP5qzi)0c51hy)6=tfu&pF$e0_RJMg2U{mCeS1w_Ot0afA_k>GNO#K)b5Oe+2pxrifXpZlyYT%<|`AkJcMVzBg zt*bT^m6h$c;@-V`H#xIG;oR~+dZZ-s%Ocb@G}w@$Hx~v6Ur@b^76cpv@|EehsWqI&fni(U4H#q;r*>$%lqmcmV|N1e8LgX=rBS| zD+}ii*GX0D`sviIJTo~W?*7}@z|hb}CW;oP%tuvK)juqZ<@k0^w}J1Wg|m}OCnqNx zSsp6f*k(?Pa5%0f`i+&HJrE)3bwh(xO@{B5BM;IYpFh(T<``&S%ED5H$UwdTOK00$ z9wV%GFJ;hAOiV0J#+?O=n11BpT5^o{c#WQPbDSIKZ+d?6>l6Ol@_sB_TwJ*?UL+lP zcy!zL?Mh~5JfgND#%msX7b z?Pa@i<;vj1A8Ut;drhy##VyrqvlF*=9v<7eZ5!)d&3IZ0p)~pIpC68nj^r->ZqB1S zefsoz88>E}uyQWn(*USO2K3x8`m`zPs4% zJzf`(0j&EP57pRRLEyTB=r}k!itiCMUi0ze$8>`%UX6nXSFBwdf`2KhuV3vo`l|>* zU}&tx&k_Mxc&hSUOCDZ}hlhvya!>F4`SWyYF`~;9Sy|Vtxg=rzk!Qya_L7UsRXjYT zd?&hAr;Y~j+`-O}AV=LXe0t(!!((!21_uh~+VNCm^-eYCp2K13_MAFBv=9`kY@~$( zaHP^-8|6ACdemz?J@J8_o0~g&Bq=#LsKsyY7}j_t#i*^V&D3@Cy%Z}qx7Y*ILK8vb z+{cu{HY}Q`X~6?sF(1?G*RSJlH#|1C4Od(z!5{JCXMLs^#p*jdVV#JOK})saYIEjH zY_L`vzd18pDMm{r;&E;Jj=Mc~RZ|Q3^!&8x&`eKPS9F(L?a=V>%8zlm0g*i)*Xy zJ>3HoPfuUp(xt0K*^%Z`p8TGf8e6n_^=cabZH}!9sl(0VGVYUqZ8q-Mp=&j@oLi1j z(yor(+95*Tz4u)vC&gG9Bc?C7l!9cmKd^XYlyVVk2@;i8&1|gI+inPZEt_&bFxaZ zO-Jf36^H2?T7T%#?R@7lW( z(e1)^s|vGEcNCGcb3I3dZ_AzyKsb#OI=r~QF}vpE!(aG+#;-g%#7tf)czeq}c=+&h z(!Qk)O-OJ{)2{nl0 z*#=Byk%vI!JJr7{v@c^ANtk>tPRzw~=VdlQv{ zv@%Wh>^Rjd)l|;=+;ha%dwP7uu@|n|GDb?aw!)`c3PJ%0cm)I&mxgfc#qP9rcBVY> z)oicVb#~s4XIo6I*=DAYGxhn)m(nYo5|7;nOep_^1a4flnEu)7N*j5j=chOZ29Xt0 zD)O0@F*AS4vQ)|T$<514RK3dU@waEisc$c$lgnT7R95`y>F79bU3GiorcLzQZ!q(# zUstY4JE;A_)h6|Y^w}x~28Ir#E|0O_Tz^MK%uk;VFAH6ltQ@@Z#)d1FA&8uyg+IqE4#Ex^6CXbO5MX-6$Q`44;SOm(o+G+$;moj@%oFXfXkOJJ5CH+C2HK* zu4b3FYU|M*C0}WeZ@=-_d&0Ts{dLWpCWqA`26u)!_;%S;Z=&)4bWd-KA}f8{_g7xDl%l@AKI4)l+V|wo z{H$fJAN=y=%LkoTe*tcSkizdBYtsMrXJetuB$9b?=ex*7$BrFK9TgQ7RYDBO3EQXX z>z?*P^1>cj*}xEvE%$!i2e2*fC=Yj$^gW2B(v%H9{_OW-0eAJO@$pOI@lsB_Mm{Hb z?@RFO-aoRnL}s7lmux$I1UICU^&`m#^M+&}8DuF59((buqW+2Z%#heXK6R^qDK!`CeVrhkFQf=97PW5jLgVf5$b-bb{hMc5G8}{oTxo zJ5q#IMpQk&en4Jc)^Bz_V8zATw>M0V_AmAF@}kETZ^*K|6dbHr=sVNe)06HzdJ&&k z1inOR0Tn(w`I&ZK2zwFkvQzt;CBGW8m!3O!F4wj9TJd{^Ow+GaBGetng2?W zaO&x(3(TCf6i6%!z}OOGpI^W}IJ!qpA}Fnky?N8(>l3R>;o&OJPkqx$-oGr%*!SY? zw2{u~nVHYeYz_wo1u2@Ea^w9}a22n{#$Lkrwj+r5e0{okbf6`3RI~s|s_nzAU5>7< z%3ubGckf=5asSod3Np8@6cvMoqy}lM)fJ)i({Hx)XqnZMd!S16RTaP|_-F$WR%9R1-;oNi- zknv&)IGhH-Hz+q(s-dCb^_MS_HDA@KJ6LWhw_j@jZsP8W+eHdgb$oqlJv7>Hw8N&F z-m3EYzIWI7SFo|sVMRPVJuQ$&7?&+8Exo*Y<5gAa_{_}TzrBZd*wr!Ph(xGva&&T1 zL^#90ksWA$ab_tVK;LWq*%9B*X><%Mtv#QU(~X{s0&-{eJ3isx+&%p3`lup{PD?hz;!7ZnGU=n(}+v5k7rBCnWb13?NjDDwD|q% zS~)p6oFE2&5x=i->wt;dIy(dN-3N**E4P%qh~Iri2}jLoX7bGYSn<8B=+DHBUsoHybOS7`F44$4?R`<~8?x#~Y9TrN(Ppm4pSyiwHDGuXE-NEDK zy7jNc7hJ@3zGTJ1!jgOTkHA_<+ms!TNxA|HY{z28O1lU^%t_h)`|;y#L|<4fW}*y; z|HNk++oqzdOb`BDDz=R>1GbC>-D&>zP28?dBuJ|1na%l~4vl6VWufFwKq~oM=;xPO zAo-9XWm(7=_Q~AbT<~xj^T!1FaD?8Fu8fR~H8O4@_@=!v5V z6UMt@OxZ1FN9s+7%DMeoan1+F##T>#RJ+1?0IV{GCgW(fFxbzXZTp^NW{MrnwhaQj zB!JDf<&a^h>0vAb-JJ>cYT&ciFVFsYGpC-agFD#<>>z$FdN1p#Z2Tn^&4+ID z$jYwA+00vI?OXZoj}-DaiAG0`98t4^Hlzqf<=9)zA1CER3qVB^kv(=8hzhwN_|hdt zf-wMD_W{y#{7gwn5!@5r+x2Ct3z;<6XUgq*W6Ca{NmuYO>%3#fk8i@|SiWb^9{aCP z_VsULL>IwN4#>;PGw>a1-$$Zug1p}m z&M`!ljf}K7q5+pzixopWNR-{UaidYa>J$yDuCO#ZM6u zjXc+09Vl4lbF(vorUkOVC|3ZEd3Nqxg+(sGtuV8+Jday+T!`kVChp0tU8iYPr|c6? zP2B(WHncCAYz$&9RsFpWqL-A<(89*k9kqA*`#Fo?z5p zGj;vZyz)03ccj~99;v(z_3Z8m_0H(*(YWWf~qEuEPFUTCfgEk`PS1Dq-UflszRn~bnQ;ZkJ> zc9c=KlEbT2E|aphN=5?%f1O<>EmMKE@4N1ayXDvI?c<}D!!6_Xpu(zwUXk_0``C??jruDq zQnHU2Nkn6lsA{=s-Y;!LBb`gIY-7uJO@e4D9YXd(Ei1hy zU7OFfzmcOnjH^Vfa5?}oww2AKmje+g?&;l6d-DGC=M@woO+o#@;ui}Gd5zi=^bXFY9~(?VUAuP85qVhFXOjK%=g+E|nyV@|ty}`72%tdwt|nJ%!Pc!? z2~YgMq7w>=5GU`q;q~10QxTC>agR^8$k{jLEPM3mk#wW}iO$L>P?US|7SFX3l}@%4 z?Adv$`OshT?!goCiNS?_js@+Lu}L{3 z?a|-gE<+Hv+I?8jq@u@X%h>pk<@D4*p>U;hpYiQ4_3;-kGElEeOBdlI^zvGu+F8s_ zPY735L~mgL8Cx&w!3vRAPilFmvTxya=MLla5A_zB2`UFqcFe@FYLbd{olcRt#xSj zzCmy-oiweuJkGXq<(o5cT?h8xfWUsOk!BjOy5nwwZ}IfRA9E|KAjllY^TL}2xSugp zr|q6a5ROhe3KqVbg==>pG%lcDF5AeJ)d_vuz5INIGQ*w&n-ZDldQ(^DS!Mux6kQyV z5Qg@ohrDpnJ%OCHxYe&l(^|#kRZCO0UChIBnS2XNOL5y;LCVH-`zm2w!Ise({)6A2 zbLTVDirO8zG;5^3 z(@;xSiLf#pLosTRwY7CaV`DLLSe&6nQNSWrdc~JAf7MMNxyS1T@R&hSqpE)&JuG== z3nMM~C80Y$Q=`GAx}A$AEPr-%fGs3w^xgH7t*QL@=f=7n2fts9>9(E*Sp`8=aCUyO zU*h)hG{3pohF4y@KE}&jg!q%r)me3@cfix3CT$r$Gy13Mrj=v`phjPW#3bi8`(SX< zZR1HXnVL+iN5OH;uWnt1y!T@=<|OBya3?3HlXVX`QOco%qP9PDZ6YutDX@9SjSUYI zN?3b4c{`uq)*qX`9xxghYerz$@zQ5%mHf92v248~ORg z>36fVM*MD$k2_H|nc5SpZ{7U3GkL4@p4GNf*|9ap;&wy(yHdBXZ8@Ld_ajag=kfD2T z_|;+s?|5Hn85PZ@yD^aq(u%YoY)u(LMZ8{+J9%wdz5Lh5XAv>~3f0#6 zGe}8Kca%<{C`UUTgK6Q?H^zh+UWWI~Yux&{2Zz4AC9Trx8 z7_ofvZd#hf+lY-s@hWrR-s8FH5#JPY^Zl2He?ef)V>`K+sKPuBk8=>cZTnJFQq0U)#hPe|N`OdA$ssMUZM%j_E}Z{(S$)!wR=4)Elk()GOZ?7<7%=O%+mE|EvUGLf|?o= zkX`KQ152b|nOqes9qsxh0%>!FU!haSg^Co~8yXkDr-Z*$g^u=ibJjU`FU z(%DcOP6Xr{5t=$QJ`;jHKKNx6hJBQj{ z3FNpf`j6P3SKh7pX#c_O;LkJQnIu^fr40-opKRwWn76;iQu{7`AQa!_sJGsadb!cP z^X&@d{rf{k{v5_bIYQgOVeeA&!!|F9xGrVo;1IvX=SSext1EW9{knv6*#WR1d{=1| zJq@e^iz7xi9C=P3GJRQFMWy`oY}np2Rmsz`slV5AKu&3kYP|U}f$(j^M z?6ytZ($26Oz;L6RLG)u(277uK$(N!CP$3r(5D*AFdwI@p?039}Na4)rEuERIMmdZt zgOwk+AV!Fk#}g@=i3(H-Vp@W5SMh8XDi%GZlx0yGn3A%|$qPk&kncCaY!ddF2VOy* zE_(M46-a3F2T{_$QQ?U)X)(mlC=&(&0RhP1xOui?ZSC#66f0pi57#VGWQEd;;%tun z*QCdLoSyfK=v@#U1xb|2eJQ?<3|OX`c4pCoM0AY5u%RPrfbsytL+_X zFNN&Px@y&8_4EU zq*4rv56X?lavJYnlRI^W6k2hnI~(QnIAuMy5eEsuN7Lg&208YeN1vUfS+{Q8oAUAw z+)7k#7I}Mnx3;$j03Og$_V)Ill2w+km-Ai&RQ(!tfX}&3mW%=Z01(AZP3ur`B!%|k zsHb@c3S9RhB$!)RT(g-$)--E;W^?Jv752MXi~S*+;9#)9Zc&Ojtev65kMjFo6%__@ zpBhXeJ>ef22ax>yz)*s?ru6O~*@rlbvMoK}645{LY=gtYN;n5$>!q*Vcel3n6!{6I zk&vyP>eQthh)CE}4k0cU=>uU{%71^N_~u8#b5}X{kHN6nQRYN29)VJ0Wu! zWZQBiB_(O$^tujw=W=s%!!mA#Rv3np7$th_sIlp+`ptrFAofepMmCCwFvIe70mVwG zeP$pn@`O70&a6fF+K*VNbx&6aT!o&Vz7>uIRSgYgluSOOIwkAJ>>|AD zW03UKNTNuT2MBNWYate%6@2>@Sm}X^i>j-2yuI}}C2gaMuTZEL8az3^1mWU5B6AYJ z9ub0sb<_QE6Wbv#LKA18bTdu3A_Wds3=i3OhLU;(QI)XG1foJFJ10Lk%?8(=--uT~u+CP=^f%XX|fMc z1!~q#`vyuzMkdu~tiNgXMHa5&EX(q3P``;$2-kt#`V6!VMFjLLtIAc;Pmgv!+cVgI zY(bfm-nw}o$xuF$vU~Off#x&fytc^IzjOcoeb<*aYuV@(QP1MQ z;%?qB7b6d29xNL72cba)xklu@bo_yAK!l)n`ijhFiP3|AGymv|M5)yr7@}g=8smv~8Oz z1}H!*&+8UqC!IUa62Am)(4*Qigy>iq_eYOm&dBbrfAF%~^pz(+5u~lWVLMX-QHgGJ zOry6#eP%WHa|8w}FK+30Aw|*rdG3lOgPwN(i4Uqcfp~<=-GgFP8*wC_nmIT;9DM~P zp7*8qgidn#lRIzUpFf!(Bqd`j zul8HkcCOtEPq+3sc)%hVnSAE(RD|JTgpELGPpNXwlkUGk>&P-f@VcAh03NLSaaD{7 zq^)h9e@{jye1KUc5V57xX6kg;f;+2Fm40LRDH=YMoG171Q&H(a{Q{+2C0$*1=4H#w zZETi6yCaFeUxem=S$+dZkjH@*G<>otE-p^%g=?>drD{t(?x8u->=P5zU|@G*&d_Dz z>1Sz#Q}lS3qh4A3iV2M+CMG7NHUZIP+llvUf_BG<9xKL;)8WD|xJS32bWk zpg`ce8{2cB^o5X{Z1mQWPfjT3FMr z-n@Cw*^TC4-@BlT7dwy{$+EzWL#m(G_|=o(Xb`RsFv)e?i1Ov?33UqtDx z6*+*&L$JJkeqyC=U|?|E!*|h>+1tw*8DC>(Sl6y)LRLeRUki99Twne%d`w-D*H4ZM z-Y2|x^qCV?w?m=C7QcJ9wNhm{B{g&s>JTUIWvWk8hVkrAMRLPJB#rnL5f zNd)1{Z`H}1wYh7Dzp&$LVKpKV_#X4i?&%Z;#E zb3-@%zisXWVm;&1IrkZcuQ)lMo2DtB^;P8f)N!QX%7whvBU4u2J{DTPX5#X~;~C!l z*K7cnKLc`K1GGR`&_0MUR|lh}jFA2L#Th%6Up4xhckf;c=CL28&SzXqKHcL-rwfMG z^34q{1P?E%!=%3czkhWt;c{S5Sa|vW`EQGW06$4b|Ld3bZL|5>{Qvr^6$ z>+%2C1^HCtMv=qOk__TZP&|qaTx)mQr}sOOx(t352><^58v&QtHjR%R%jOC@QVQ2U zjK=@dHPN^XU)QsAL20A>0m5*39%+ArJXQibcsSo=j>GqtoJ7UGaf5R~kA<<_4h-+{cwnOD=W%@?1@ zOKwHXEQ8hc%^ccKKpF_;09YZ}5idP7*tQp{fUUR=NzRQ6)rKgI1UW9S&3d;r|E^cm zA>y%spFBgv=-J5?0N+6Xk>((uYvjB=W*5UvqHHQTf3vh75epIEH)I)RIQF8p=(9k5 zP(YT^t1CRyw+g0_<#u*_Cvp&^tfnxmX81iH%(YM9(WFwWh7DO;9I=@t%HW7@$64o_v$| zbDNs|4-z6G{J}+vT3WafX^A2J&yNr0W@Zu57q63Jfc?4;we|vN9N&Ts)3$FBJ^kzQ zat2hdNrM2&fe7A|ii%3;DEc;niB60SR6TG&5$TN7f^}g+trRaWDx1%#Vn!8~xdWJyRFDv^V>>L}-Q88Rv{ zZ9jj80R4J+c(mfDLDDaQ&yP50vt~ieNcnL2^0e8c#}^9tU$02Zba5M$X!@w3iQ#qz?&21vsIM< z&4jN|Hr$`Oz8Cmh##u=QmnSwm^uR$^2*6JvMmqcq;t|Ge|CF@vvPd~I1v(Fl@46Hn zAsD?r!|Mu~PyBzxOb4h)%f=RIFZc7`4WF<>pEkK!95v(e@0pf8K0P`nLkKobZ82mdoC_KrbTHvx+N)F5)%syp6q$v2J_vel5p=p49thg1+S6Uc_77J>DPWp7)CLbPulwH2+d~Y6qb*RbNBBS~eDS;=?)>?? zcQ@mOF2iRQ?L(Zp_+_&fy>kHFn`cc?AcAx4_U+qnj+}e+Nc1|oAa|enDVW`ljfbx& z&G|+U#DP0dra>6qVB3b8P8SMAg(LVLx-EMyF>#~YuZDA=xq>53{f&~TUsvd%X+;SH z13&1TX8i6p#2i80^aumdLL1`RMi|-)J)ranc|qQ*ufKlf1TiPoL5`K%Iy+cQKF=!X1sf9E4SI(RySdLesvM@Puy zko%!$6Z4rMH)sr{K>U7B|4N}y!Gyy?OOXmxrT83#HH&!o_A(wk#mUL~2p{7UcJBEF zhN*L{jGKL$B@K~WS-9om`hDyXxhNi-GZphG998;YW$4f-GhR=XLw_VZ*wAnxNE2Cn z>AYTTKG<3mpew9r%NMN{R2;N)DHRBJT}V^H9xwRD!72qF7(U%k@er$5;zN)JN&%~- zaJ^mE{a0@|aWpBpx<0EIh0h*ug~)K^tOk0B;3#T$89{s%DMiwa;$A?&g(Guo57O6ba)H0 zV>`kF%7HsTn6$Di_KiImVy4M4t9Tr9%b&Fw5$}%MhTob`xR(UwInY0b{!HzLRovXm zK~PBG-sAaK0T$#emu^jXdhh+>!-YeYuNe83ut>xLinV+>Dz%Ta=1|#TrKj1yfB%|2 zo~}8TEqN}Rao#ULq#$)P19X8^K<@E8+4xL|bkmY1TZ!Ej<*ZP4H;Q(}`mQIh>M=LF zhk{vJ&#J|6&*)!V_=||0_e3JYCH@YyaHA`!u$puG@k>&X{HVkzE_90PY=@l_5|?br zZZXpWcE3!$`_~&i5#kn8s5^2Z4x^tqPAQLcFfae|dhQw%x>WuZ(k|1{4WA3Z-7Ntc zA)3VNb96($zsU@h6x>T8{j2RTaeQ!H5(Hcm-6Rn*fz70<>t5^Lg~9pp^XKJIeI%OY z|D94_A)S;<#ly>bzrnu+08oMwFi{0b(n*}gNrVWu5+=Asun@R$lcMu{Qwxj#GqEX} zJBaMIr^DX9C(U}(6D|rd1w@= z-oElHRt)u*%qItt7?7Hl@^m;|xKe=)W};nyAUO~pf(us@pY5x^R=rJcGxGD59w9G0 ze)@DT8miz<4kRV8<|3XsIKV$x=mflMuB$s@bFdumZ3(Xe<-d|&?GCK<_>S{1|4D6| zhg1Rr_u}92B*%7@$9}d|tD=X4wE<>KO3+)Jd)>OFl&$rWPoy>5|Cu2xpi`uc&xAohc=&R3ej!br zL#kJRJdYqwRj*tP-+ws%<_=_S&W~lW-z*Ou`X;s4t;~e10PnvQK%1n&^d7it5fg;_ z-Z88xu#e;bnk$EQ1jk8Ff}fhZbxPZ0w#!Tr3e=CPgylRAwk0?TO1Km@emJ)uj>Ns5 z3)9x4-S64Uy&4DvzmftTLFlBa<&2P6k`dkU$Maqu9%z+rMd^`LAfp-~8!1j($nBYk z>HT>^xb{up5;k%SNikb~dXNr|lS_Dtdk+UtP*^ZZwV{lf4(9>Nk=j_O*c&%!#!YpK zA^o?a+eI{VuuE#MX4o+{)Q#E)I71~IufUwdw7V-xmORw5l!T*$*iYNOca=ofv$MPP z(C1y`awPOCs0IOk5#&Rp4(c@q51`}-OW}pW!oo(GHKhIy3P8Pvrlu>f3ATUSqm4#R zv~ym@D81CyL5E&ZAGL}691xYYumsQEj*S(Z^@WZ?DS!{?`U)+l02%c1ULoUB{t!N_ zr=I4br|w^+l}}ol6K%L}{N`q2%zY-VFyj8{+zb_;8NeUlClF(pzcO)Yru12VrXJ)Jc)x5VAnK_&?VOQX zHKH_rJ{?7>Xn=Qzdp2(_eP_0O;B?yHa z6`&2`=sO@S*wi+g>Jw&817P= zl6GvR07tI%OVQi`5@>JLK$aP5D`5y42A(HowGW@=7P7&m;TrJs(yETp(NRZd=arS6 zPq}2h$5{bjnff7oiaU3zI@!0XBdL@8Ktn+;DnqQ(t;_eFc_H;DaX0Srt(^|ULKt!F zS^#M_A)Y`({o7H4XbmE{1^PM=YH-EwpJf~U3OBGOVZNaOXSKg9r5lO`n~- zn){NTg!2HM^>G7zGIrSr+{~50dSJ`PAuYiG)$4ij5R5#u3m{CtrHuv zeG5EtS!eo=5Q`iDEFdo(CO$yr4@FC58;a%_8$jw;2z*x``uvl|hV$j@jE?*3e)E)(ZlaAmj&OQFshX z>xl^luW^PT`4*wof^*4!0nb~ zTIkC(0;IE8U0ofvv1xa#+P6mmc@x23$NFI*;U|9H6Tc9RrT}E2z%5L)2{d457@>Hq0q;qO;;X zMItTMn}$L@QNem~RD8h9iKizH%6becJjkrT!(s8xLChs}Tgb{5hVX?D07B}xx|3~ske@*x~>@8oe>pwio0jGraP4PQ$d!v_x z9UWR1AOWiiCw$@IdyzGk^|b)Ba0rlT zErj$DPlZ7pYP}-IUa+GY@OJ+44PR4+`?E7Zmy8nleBV02Jp8AHL7iAh`iZtDBe2&GnBJ|9fSy`p#6aqbv5+~N^ zd{EG$pLGu}gQXx@nSqNzVNpP}ObglBzB~?M4XGC3sX>tmox8c~Gz+Fa0Ml@iHfPv> zK0$5Jm?C)qzbcl(N^~J6CZ=LLzRKX}(IgyXCiR`^uyZ`xb(t3;k=zt4g2mk&5Z7F*XBzphs zY)SMp5zWfsBI(-2ua+CwgV@+u%ze;pO4M2`<@Qg>Z@gBMMfWjEi$alYuW+=j4D-U< zPqKh>;P32Ym@DYZSr%4G)yR3b+v^)EnW%|lEMT&qoj1*Yy7J#wPOh%CZj323VO$Gw z@oG+|>o0$a|F0GY$jce>G1Ag2eu75Y!jJhx!OBVf+ZG;f`*H5xdZAac;vwv(By0`7V;6= z=U#WME1(7bMYnX@9a?P~;`Lpy6x+(MFD%mvEDh5I@H-G)UO~Z2mJIU?G8Znv@4X0s z4y8sWQmqo!W11gv65Nx&DO!#6f^OpHCn^GZB^z1L(1u@(6!|b>SA^k*7)@}IAOi+Q zM@Q$B%>TIR-2(Vn6r3pvU2@xY?5K6#q1+;d9{_X>hP?)s?FH2wqV{<_OXFQ27LY4i z3L?Tqw5iSmWW?UeUF;qKJUsmTizumd1zqZG%SFh-h)R{87Z%3pmw);l40@r7$cdu+ zxBszGTCJ&btx33=rK*=0wY<5V&4^E)a1ZP|hsf^TbL*VD8s>Lk#eahI{1=e_2!{WE zxOMdSvf)n<-2ehf5bM|4;Wj>K4)X^_C>{9EHm&ga1J+N*)qs*?@k0UCSpYwvBz$ob zoL}<494I^aj6j)yN|5w7IWsdHwcO$lAC>`VR=dS698~2Lezbn*O1qIDjk=#Dpd1`J zsW~MJAa_y#wYUO9tOa4I#J7W=YXO$@pH0HMM!?DgSZ#)ua55ANc`~yzf;s@dPOZGfyj4HzFogmupCpTPJX$^ zaU|1Z8RaoOo^_oV07$Ju4}@{v@t<7rTxN1o7X>PB)T{SIVy2gpCM24I##IFuDzZ(fpI(RfSRQ9#Oqn{XP5z6QB; z5&SX%m?%Xu4^-`q{QUbexG_Kp3`bEQk|=-Z+a^X;Vi-Vzed#x6(lsjveoBhl080z zp`oJ%@cp14Ct)Yc`AbYZz(ql%EJH+NR7A;a93~E7dPu4}92(h?>M9;*!plXhim*1) zgH&l3c%LP$2xvB5TguBw>pL@|j}uy(X=>W`X9k7sPrzOn(L}nR&^3nk_GM(QOl{Tw zoa+ma76?a%sDo>(jJ|hq>yPW9Tt`o}U`7eqTyUHs5Hz?#_%KZBq4xCr?xv)p$WPV* z0N_mm@>|8){3oMCFs;q(s7BB>?L5*!q9jLacJh>A#Kt2f#a)<~SVq7?8NkbVfg zd&FS?(_}ip)rG9BKr_G#%9yd{INW)F91Dt+-pP0XKo_PF7#za!jU6bR5#5{go=mRa zagud*qMLhY4qq}3Lo{mGL4YX5Z{MnlG{N7D-tjvTHfJ!l!#Lk<85Fp;+uY{QuESLC z-2{T23Gs(4PEH)U@;<>>Myw%t8=^Lnc^}RB?#nSA;Q|W(o_~AJVJrpl=>r9lmisKs z*_!@abBItWVhiZs%a`Q63Vryn8jyKy6xqd~VXQ$SXdhD(v4sC?V9WKJlY?J*$}YiR1HYQ{4kymSiJoOsBM{W_z0gly)a2zQ z)Kje2EHKuL9{`B3UI;p1Lfk$edZNJ)!USzh8RU~l^)TjdM3EmHuN<>1LeRF!i}DRR zsf);%8K;i@Th||`gYRiFjTj(#uL2li*wz{}!}>#O7to0H$LSURlBu7$fpB>MKHiNR zNobey8Wkas6Wa&;FI-!Q#ev`-3KfvNH<{Z54K+4`%&Ecr2|pts31-gi^pM6>z!wSV z7T-vcThI4bG7x!4v>2b9(E+>TWAh;K>8q!jn|VBFDhX~ z_Jnb6Wgk=mGw^JvoTQ%s&dV#LegRg2Cp)rs2bp2TE3Ar&i9eweApiStW?@vjj5=pB zSRLh$(}TW17Wi)&(9T2LrjX_=asH|50>?NBYZi1OCnA}^6h!7>;O##DFkhi9mv{D0 zn$#&|Ewd9RE`V3@U~w>|>@_X}`_I1K-b9=ua`thTLm@(tS~yX?VCf)6v{Zg>oTnh1 zqt~B4u@+7bFC`Tz;0w$n09jP;TDG$HmEYV3s9vNyfs}p`W?z?LUC-CXErny3jw52ml20}8(%cUDax zvJIWv<1lYKb-ult>Fnt>>6?D^b{pb1@dT5?2pWjUWJ?Sxcp!C~G(=%Q5Hq*jdcYwf zNaE~rTnk&eU4Z}7E-@%}=xHJ1l+KNqnAMAyt=*(qWPrCKbJ;K{2>qCMZhQwF1FQ;x zsgCw63=_K?Nkd0^Zfn6o83 zr|gs*4>6O4lD@kfc z(9y(D?{~5IchV-5u9M1ysG^OTKqS-&#~4I-0fAtgS~#$hIaZr&7RU8e&`N%YQN)pW zg-d1?G$7<9Kur$c$==Nv(zXGPIRrB`pLLv^ZKz)NH0Md+>=HjKf+7pLuFmr?GQBe# zK34_S-wI)AA0AH&-dVB>)*s`vZ|Yqmpl>r7L_+ck!u&LbTya#A8fQ<>CAic{J)cZp zDt`YN$Q4PAgeHQ9$YsTOuPaMrwR#KH_|&OWFqJ7mO(QcTvVLLRh&Bymocj=m%VW56 z31lq5i+fDgPgZu*w*=uLJ~Ph`5o#yYDqdh5;F~o zxDrZAN~CQ9IeW8uRy-WgEEEQG8ynU?mVqKzOmt8^rw_R#C5H!)C*+~kj&j-j{c#hWZ``a$s!5?3Ycg zB$J)!?dwa1C>o2feSw9G%pw~)wTMD*Ur=Nu8=}EQgnS|XO&EJ1oU*xassc_P^(=Jw zE{23iCSana4f3qtc+9)w&?fi6Y=F)iB@0?=)8@@vvLz)Y^>W!XG-u0q;ZM!0;6$V# ztVdNyJ@K@kM{5%qK&_NNseljz($l9UNC$6Z!Y{K6@tXXp*avqXnVHiBW=LijV8)VA z^9=>%7POMH)jlv}!=1PQI7&w4k#!Xz1vXR#9S2(#Ads zj-WLgjq8u?>P3kVNWAEv15TJX_>t-0qMImGV|ByO7kBCD^ZVEz@Co55 z@m;$ZC?YS~%n>IiOa^k^sBNde%aUmm2te3!@V53=bI*TEyd8siNUkCP5@8|w`Za2H>;{=5 z$Qj0ksJx?cNUh>fPB~z)u=r%^HIsIx7CzBmDWMfR-Qx1hZd;C}dNRSn?qu|Tn zL%KoiE(7c)#TVkpLV={VB8$~32*fc=v$1{zs@zJ`8fW;ai75f|zZ_u`0LM?nOOZ`L zoF;Wwe2nlW!PUB^J~~PON4|42?x>YBfCB!9q5(Z+cw>efCpr8GM5I*#)JX|D?FhV_ zJ@r;|ov;CFJ(Pm^hrJWEdjPSO;VuWL5n$PWZ)#99VYd@Z0ES`aaLbXtm=&lABfS&b zG*cJy%w5SQ8TyaScmB$v@@0pQaS> z0A@Be!LXtbWJC}>O!iwu6ITHtAR;Hh^=5z+g*wC#%G}V?PyTs)NwpF|V5svQ8#T91 z2BRO93=GzgUJlTjV)Vn30@ z5iyY1_6a4&aDtU9R}MmtCq5u_CrJNHAeUZHthcwf4H@cESlE6XOX9D>Oyw022r!Xs z0jZ!PkmH_iMgZ#9=CJ4!3m8Sb7No0*bXJ4s4_SkPHOh+cR$oJkAeA;S=17k}N}wlH z%E1I%)G}}&NP)s6XiyC6rm`8Rwq3ib2B+30P1hyvWgsJpam>=;Fd%(MH~>!n7isSu*K_~B{lBx4O*WY+TSO$X zsf@@fD=HZk3JIm`GD1btA|p;EMYb{;BH5*)PzjYNR7ltTk@Ngr*Y|h*d3C#ed`{%u zYdoKiaU93vcrHVkC`Ngm;c9uPB6yUoBKpB<%_R4!%##72Xd#BEs4&`7RX=;bXxWr< zqxiVEx)joku!85PGLWQBCaeoYg<@;xJp@L>>=F{{JmZ1%rE@^Anz2wd z6-@wGw%@vStNrDqe_)2{iT=^-(2*lO%PT6xo)a6@<(@xE!*}7Xps@P+jG@2I%*xoV z19^Y}I~}HnHwhD;U^FKy?J_PJGW$03dPi4R*GQVYb<8M9kMWZhK8A?XLnd#Sv!41A zAz9#C`{Y2q*~j{^^dvF&?AVS$MMgpWr1Q2DueC z$o7_e|1CV}gLaP=?c3|!^TTX;=|D%V^(9u@9qDvun;8@NTKBiurWo0J=$g}`+hG9Q zp#6}kMf$*eC#SliVa4*8QnelpMRs4gEHGUR3NBx9f#%w>dGn|piT21lBV(>bVjr`+ zrHVpqce>m9cFgu2!9P?5S40lQ5MoHW>X*Q`8>zPqRgZELXWF!Hlx zLrT2F11zqDto)7`BUsketGYDbB3=^4i8gv_X0dzIuB(t4P`eaJw1#j1(I>68hKDtFJ)PVZrJM zu`j+Wg9lF_bc*zlp})zCg8`&vf}aii1b<70ZKSt{KEEZX5H;0a3-5k4+aP&is?9!W z8(jK3gd41pS|l+AM4AjhvDPpIRtSqlft**kSxK-O>BwMHcT$nTFz#Tc_11$2^+6ML zKTBXVSiv<(i^gpFE`po|&zW%TZ5o5hj-b_wjnJyNgE+wtn%&|N1nzu)jcJfQ!K!w;y6S~Gx zuy9C$f00f)O;}yUyNvB0lQ?$)m_{MT5!k;T-Q{Vc7)=lN)19s}`wyfQaXy>~E#-50 z72qx5@3> zku^gil-Y|ut>k&J^2AlKo`*f!Olj}#+_@v}%u?DCyPjSNINb@P=*p_MHAFKjh{3XF zb$~KvpDu-wkv5zDkPtG8DA?jY!1=0gAEt?Jlz=-9acW6v6L*DkW=^*P8Q% z@K27f>2@V$ymhW&O~A9MMtv1-6lXOQVmHr}9Grut?U(^sqYNQl8p<^s4JpHY)R2_w#+1ZQp-% zUAC=7^`K6Rd5mPmajFM)WHn`d{Q@6(Oa_+WV9GS9GwlETUM~Yq%{&D;}L?<1#Gg z>?9-fH7HO-nF2U1L$b6IJh2+=)zLtoVsPfu+swpPmFaN>Noi(!C{-(~j{>=Aix#ze z^*U=Co#Z}jYwkpN=%-8Xy9#I$l8oteW2*k?868@-+=0K=FAE&R}DpsEt@qIp=G&-`bGl7jl6hJqyb z;)jiuRV^%k>3M7~|6LKM@CxR5iCQu-Ou+i~?P=c%D2xL>KHI>R-PiIYccD2GAeX9s z&aT4-gVg4aoE>rW)y?lO!Z#RH_3PiiDHP@M2lZR@^!a#elB4zbT-#kNC7-#1+(B0Y z*G%cr1ONEu_=vQqT8Z+TU@TV$Mw)Ulp!QY)|`- zs^$i}%ao5OD}XiYqxAqRyUpx@2wm~aRVbh(dYrrdp}4p)L|Bt1ZopzT+H1Om412O!8WB7M1Bz}ORgU`Cp|`l1h` zA>_wy28g+f1tOliw;GT5?S^NKY;#eADXg4Y^(WQb!WAsfvBGx1N1%~LGFXCdgK0ix zgj_nIOQ}=JhD=F8A2=h$S+cHT zB>On7N6QP(cF0dW-RW^!S|VvI-e#6Uu8A&PX2B&ka(l`OV2<$^ zGOp2Dnwfnl4`jfeAis^nZVe*o&RJW7esoGZyFaF(-J9rh)a$-~a>IbV9&$eAv`zXON zLJ>ur;ua@SJfR9k^zwe=jj>~EPeXG;4ORoK%WwK)7_u7GadC*#mre{9)uj-X#t7mi z$+2B^h=czWHCeq(0a94y-wM-Jt6^clE_cH^6!AOh6(F#VO+#cmZBqrCQ6U}{Vros0 zu8%|j85uRR0))Wg&{4b?uGFeoedrK?#=pYV$DmbQnHo(TKl0 zWvov&LIS2%>Wk|GlRvi!;E{HjI>J7MX+tY=HNv)i`SRr!2}=-&$XECnp!;aL#DvCr z1i}0(mGaUY*l&lKfMx2A*;+$j9A3`*I)zSe?rapW13lfqVF4!UbC;;gMQ z)5m-s>bBl zWtG>IX{6AoVA5<&1=WSuaT&1~n=au0_Vh34nrqTGOFXSpr;dxlwv0)OS`Wh|Q-9Lz&u9xbYY5qmvANT19=sULoGU3jrZ!wMgDFtQRg+(Bekh4d^Ae z(%5{hNezXF_<4nCS7P`XU^zRe^YWI-(wDEN=XgXpOoVNQ`IL$tUO?QA}Kx*)UL?L3Ase5ZELCQYZE`}yf#y-7@~ z+VV;Nw8cF0z6GWb7Sck%H0%^)m7{fN8N~ZkA!k@9swNIB_Zk)n^J)6(P@qnm zX_u3?UTf2wTl`RP5SGd7l1_P(f716DRmA@-e@a((Km^tnAPasFzB0?~(4Vd4;Ey7J1&&+xyqLT#9=p%!>K{y8DIpvPRV>IM z&fqVFNZgwNd)tvEHS47YNGL}z0CF<}2E5uD(JqsW`v^vhd&Cq93@M-`XZ*0P)gP6~ zIP2Ak3|D=7t-R;8MeV>E1N*8^tex7h>XuS3HXk6M!r<4iUbCSQ6t|c|oO$Uh9;90d9Pn)CQwQUnG(sTlzF9IH8N0hHwy?Xj-ZBF)p zxVHpHsF%6BMXyuOgHh5D8Fn%TxGPh9lzI5_s?5E}9srs$m4n#^Q?QlJ>6%nQd42!Z zk_~1lwLjk_Bw`wB2FM;oca8d@14c$Irvy?IG}ph2fyi!i_0XyYzJWRwMK&Gv1eT(M z6@=%c)z)t=Nd?l-96zy*{VZdwNT|T5k{nF#utgptVwOQ;#hW*8j%qpLQ8__aM_R4h zfKBm5A5|AUJxfrO%csBHm5KIR)(&G@eYkwE0jb=3o1Ry69lxiEuV&|-_efq}9`uVn z3cgiOh0$r3JKz_HjqCF;b9Q#zG?E@p=|s0ZMWnMc5tKI`Ih!RS`)rgq;+9iyEEm4sGt2;}7FRTRd>rTyfIg*Y7$^fRM2C?Lq%KMtqwr2cw-ckM+dc zL)?(4O2z|Vi?l=fLi7We)V*tUe`Lmz*x*e+RZt~w^Y?EjuGJv-U2q(t1r^&Vq1{jq zDR1;M*qwTKXNuW9C~g$fl*c{8Q~TQQD$`aX2d{mQBZB+XsRo))XO;>?MQi* zGzPo@GUfB`pJ|3#x<_tB_>glsdy@<_3EaggXFg|eNtL4m;FWhB_JI&}BYN*ns*BEK z8a>J)9&tg;naY|DBS~?tNy)k+*ApveoYr84iA)Q-w8;JwqaZlGW;<#nKibKstBvNgYqM?iqmp zA~+YzGj^n~ZFohWo0drg!BF%VihmBkqz^w2_X_6SfH^+FqoCw=Q+~)vRATiOcd|F- zl?UOoh5{L2pA-$<6|aWKu^}Vp+({z*zCrf``76~hMb-4fcAYErEiEi?!uQ0`Dg>{Q z*2A36KYck;$aoyRziBuz=Y!#H&ncVT~>30L}5X~b!x6f%k+A|Kx6aZ0{= z4+ z>S8vyx~L=MJ=63kXuZTM6BtS~DGaHQE)*vk8cBzx;FOxWP#Dy!$5_@Dt}+&m@?T_U za_cH8Dl+zrBI)SGM!RsFpL8`@Q^t zkKT%Q0-%uj3J=21_MvPH8Gx6Dn>~BdG8Xe2(OE$FSip?D`$2%WU@~>->ub?>UbjXa zher0;5N(lF2n+_>Fk#VaBgMO;nJ750-VX@3{437w`GJn(31naI?4(8t*uL`0vh1&H z%=~5T?jPZ8$pAyWOiV$$Yg8_YXE93z9Jn&8y+T~FdYi|zhno|(?g4hi`T3z=O&b3b zFm!D_L;)>Up5j>0!B3LF%nE2ANSp2G zm`5^XY@T%lu>#}OV!3v(*&0^XKDGgjnne3_{V5?g!U6IcE#?v6@-Kk ziPb~mFB+od;*o+EfBaon26sPPkMnDGdL-iX!-v22t4jJbskX!1OSYqkf?|`WkdrjW zd$yOuF43TMe)fZP|IWuf{RtiP_QsU}pZ6U)LI)} ze!*nN7$zoL)1c4zbX^buDfxRRj+5mffQD3aZS|%O${c5nq}CoWf;5t^W|7}S?8C)8 zIlV8yQ%&OYh7HTg>QO^k+pLc#Rv#nrTHw=t$uh-M8u&bzG zx1KzC@{S|Omc%Fis9xsGhIdFE?`UC+GQGXL6erZASmNtRnivRP9pKoBseG1cJ@ctM zfTY0i1q{7^sE_}mjb6&shX-iKza{lIvb*YmE&v74k2dp2|K!cKN`T(0e|{4Sy=LT85+!-AmU0yL%uf(Bnr2fQQU(4F&f6V{X z?w#1^gUvvOvO&d0{dPqc$jk|+FVzCRWRCnD^#?(33pJ_R#WquNV;=;fAeK`;=KS%g zzLQpRCn(}2iY`KI*Rwc)lstsduZBbzWGyr8Mu5A=gQdU8&)>O=L&u=}e&40DUG7@zd;$0irl3<}=lBS#rqlP~kGU{&ahESZs{rJvc;6}_|M zJ2;?Tz1My@N$P$F6uiFmvLnd}jXLN%yED?UeVerGk;AT20fX@pq_5kSjq>porOTHp z!MPYb$4&%r7aJFtA8DW8awrdc-W~7+&Sago^fKhlZ}`M|4?^TwPy77fr`?;ewV(gR zFeSL2>Af;F%x}ot(c|Y_vQ>d9Gd6y7qwaJoYwJ3sW|93IY{=(}p%8RMoL+k{M}gl+ zsoU4~yMVa9n&t~#{WnN<`MCfERBgU}2Tt+#ON1~Jg$4eSJ1PY)hUSHN$mq4~mI92;Ai`Huyn_f5JeC z-d&(J@(mK8TCmI_nneWQ6<*m*LxcNM?df|N)KwzI%FMC>j;k=y7SqOM8zHu(rxjik z{;6jsq<9ml?WCZ?s==|NOl?aQK|H9QpN)PQI|HIp=22-zZh&%qHkBm>+h8;cbebZww0AMZF$KFre;6J^!;UYm~rorAc z$iau9^M6uk9&y{|i_BpNMYO>}lB!H9YxK}EfV4Ci&D<70KfgPdFjo;5R%aJuPHc2PY1{H{!x< zR8+50X&wtRFB|7Y)y3~2K`xSF%ie!%m)i&rq@0CNVU%gyu_C0B(zysp0pK9d0iik? z?E$Ytul28u)tNT-CyQ(#INb-!_O?YA{5=icbFlJk(6=RI$k*vM|DS1;h4Y0=APe*64dER7yJ z4l(aDIm>(Oj4$muNr!`zroCiE?Lb28UJWHv0Lh7>oJG|@l;4on;r6=I7I+(-q^!N= zw652OPlf|l?l(4>UMs9ki4w)_^5UW0?(f=CKBp?xJim>L#>p*PEtP8+gXlH^c&DOHf%TvSV$z1vb**Y`gltL?d>sm)Ez=DwSM@7Kyt^Tua%i~X^- z?bO!I0=L)~^}1JU*sB(04;TD0`P0iiVzpbu+WRiGzeWf7N0gG?SDo6n(hXd?-?maW zw0h!3U!>(Qna%^+exYn9D#{l)j*3v;nh#InLz>XK!4kg@Z*6Eg9EP(NC2{JBWfcEAA_)#@}okScWzNo1zkCC*R1zXlL?5_G|WB9@7!>?0rYIb$_ z@oLFRDr3~g_c?&5p1>>VOfDf%a<0hpH4bkjH~RPMz?B|7M)AC|h7bnh7iz8g^}|kq zcm5u{YbX1o2DIlfjD3N<(tlqDuhuwk%kQ%_*#xrjI_YwSm++@9O{NFvBey|`+lz8=>KUCbQ>2*0{z87yr`6%nMFgrGw zQGoqjCo}?kF#zYI05|ymeUhOH54g3oAy6$$;U@B&dOTHpBQYeQM;LK4!!%%ofD!8{w=I!^~D`}lBMLX zo=QN5R{K!DO{T5R1(quC-`CLzmv;iwNIHqPFf@t%@?{m6LJBpDYVVDK)QW+iH`t-t z6;BOYcEa0 zkOIIbBPOC0a#9d7p@l${NuJvJNZ09L-NhHNUrc!yLzjqoSMLChpnB^9CeXaMsaA?( zEN*rfT!_<|N82(~NQBG%wBRODR!QpdwS*=FuUz{0s1lBGxePHG&4{Hoc}kNgsIlyv zThtzw2fH0gva(=7GJ~>}R;%0k4*kZ^D}O}DNm4pVH2l$M!dS1{Rs#<7+1dEm>~ooV z&D)td->7@lsaty0zUD03^$Ql{MN3I8`F}|uQ~|hbB{1j-K;irv^eTV}&aG2C+nVEXC{12~h+1qqKT$bkYS7g4X&6>%n^GVbSUKW&( z9mr$WV2KzQZl=&z|Fo##@hA5vFqU!34`fm(<(GoWs@p6x3u@ZEAXFxlIu6|5iI{#9 z;hoXjSg>$?x=*{4^|pEo+%WO+`LavfZ<5u6An(VV9o}|A%i^$Y^RDd=ix<_DDrY+8 z`+n#2+enRWCkCRw4S^FP;O^uk$~T`r-3R?kI^4f2sX6CnS=NPReEdeBGT#rmZsdBg z;^RHVV`^ZRqVIY{769?dw3X`UF=PC6SHtu0Th_;!X~%a=t^n0#Q*{Et@Q7J3nqp)U zg|Zwx7S`Hi7oDIs($ARkclBABJFyA6Ii}eG^#T45w=K9jrEJ-P*3<8mRavV}`sA#3 zAwBE_-XKjQ&(3x!>ZGo|jX8>GM@McpHBSXJnd{xmBn0N;AU)V>3IRy7hs5Lb>+AZV zyOP3|Ffb=GA$c`y@1FL$C1)&A<*7DlB8+A^dmA^2SzZ>u=*=kJ8WMt)g9?ubk747& zQD>{n)e1`s2Kt9ooLll_$}Fa!0{b!4xM00~V@zx~qFX;e>nD;JEJ=-HOWF^N0_$%3Euu z$3`{_i3v0Av8nv6+;BWcO^Za0$&y>%^OfjP4C$Zox11Q|IH8stn#+DZ1`tf`ewZEZ zrSG!AUk7*-1JRsMpO%-Nzsnl|j;u#mzx+oZDe@>vlQ28EO=eEd;HgOP+_kon7^uZKf{C|JOwN00tn-UC?PN3@0xdH3dnRk7BBYj9m#L35PA>)Zh#Neqck`u~{l zi}?kHqS}(B#fxR#x_#?j(R8e4ld^qA9q2O7_7g_XP?8OqTe9L+qP|L)lV4bvOj*K| z;VGg83sf^HQ?o{DXgf4`B5p%3J!ws~CzIr8`i<#c5%rCOw#gi)pkIuf8O;c$Vszjz zJ(h=@a!Ji>c?H*gJ4(OthQT2%afwZUoR~)}EP4=fer*VM|cT9by zTKr@;k0p~>qa0OwrLOQedY&s*7AhKCI)bI+sc@Vgj0PXUs;F&D11e^ zD92iyc-_%z##Cw+T+_V*Y##?&5-v|>&SKi3& z_QFoc0;p}fr`k70V4lOAAMZq&`U80FE`joESy|%UyIoTv0rT$DBTD3AJUP`;s}}_^ z#w5}6nPM>T?W@KElT>ZE%Utqp?&MP^YOO!n$1Dmf|KNa*Oxbo7q?X`KjU;4EiXAoV1MW780>6($v<|QBDS9s1fJf#3LiC?N}u!p^;BVug^GS zt)PHoy6k-d=VI;8tc#D2kJaQAbaValTZ#zU5`kSoGn3KT*=5jkc^HqV^x?&1_8$7T zyt;R`NAKRr?>7m(B9sC`*eF)~fF&snhE;UY>(t4Osc{VJZnMX)ZmJwgun!?;>R9SK zeG$e;VlzGokBASgM{F7fRC)1|gap=r#LJW`_$*25Oar)t!de(z!_sS*5Fy9QVN_fL zYoCEjAG>kc>FV7?!$>bjA+zk+;DG~uz@}RM9+P19`19T+7i*3p-Jkqx$x6^qX<|X75 z7EU7Kicc%|a-2AsdI_ow0_PBytvPf>b;&%6}jHVB9Bxci8)To1E6td!?SCc zDk*H9j+rN5^pm9*t-ol-gX|6A^eZ-o0-D#8xdP^#!kk)Z|FPrFm|#`^n4quA(=~)! z=x%zx6A>|5-%bnl6B%k8qZ2)Fqzy?J&~|U+rYeSP*qS@JMzieF*4Zv)Y79aSrx>d# zR+!FT<6Bi- z@~tVfLI~>y_^+mMY9sg-&QPCpBQk#2Bt{W_sB3nEJPu?#&r#I?225bO<{@z^%=oAO zL+b(7$xFR`7Dk$akL7_P!T!lyF~HiO{v%H5gYZ!`D;a-I_x8V1#X5b8)f2;RNh~J} zany$m3!pACVKrl5K85_;Y|t#!)E1R}F`YQWp}2^2x>3GFPkxQw!w+9*U5DIGLx&z= z%RFOoN?W&kcYOuy;bKlq{#~B4FZKHtO4@Et?U#5o>n4mQ+K*0jCWcOfCBBW#nrE1) zIXSoXRSv61h8vl0UArt8N>9x}pD@gJQ*6~iHazS9UuQLi)CXV6whMHxu43tgd3;vl4hPl;cqmYl)#OhYA> zY>9+BGt82hoO7n^egEs7HP#m0vWV0I+~)F^yuKYt6+m|0PZ&pDNv^L)(A@K4Fz*+u za9ETb&~%c;$y4yrk52s|B>^?maPkUhIz4!X;98lPnMd|3j8i8OQW9wtkQJDDC4$#= zK}@iaO^x_bQhnUj)uM+BpP{)O~jzBtvod_aHAHd!Yz~@rV zQQkoB8wD)szyVSj@1x1SEbMvc73Ulpx^}sO>OEkgEt6zKNX=#X=&>!)S@0px-&eA1 zmB}$ygw`4Om*4Jh5#-?XgiJXK#8zC`C^dMGrmnm;Y{XZEB<|X^D};a|n~uAcCLE~* z2di}vt~gS+xHL9ali*GbwWM_8P>58La91!i9lOfP%ACDj-y3>Vt(e%Ox;gFgBaICQ z`Q>*)pE!gHb~6o24s3xaXr@ddPD|Ji8V zwluHGHLVfv^3X!t{sr|6=j%zO2g0LCaMZC;CPpn}N?+#DP!X|Z7@xiP(PjX3df8iY zSqRjV6g@CcCQQLg|8V8I=?-jg%nEWgjA%x$-AFQ|Sd_mVD*%G>Xi)iByWz8>W=d%uSZp5TEs^02B{knC#*UN^^!5QqF z35}%p$#G;|5pfngWa};_2)YU`MX&n)dU4F%g<6l1*U4d6w-=uOvDcl*^>>l zNUQa`SGPN(U9i_b&@`Aj&=2nW3$gTTie9nl15HpJR3%}t4_bRv3qYV8s%YF}iNkAmCOU*ka*(kH6x>E4KGi@afzI@)MK^=e< zIY?Gak_Pn^20JOm1|AIAP1B}6X}m^|x+y@5?1&RN^(JWu z#z(|Uxa#-?ctB$rSG_KuJi)@7yKbZc$Xv!3A~Qx5l`%*T`0+HOS`>e8;!(=b#ng5U zx89ZsBis9yD?U|)kK9}XYPGE!bZu#TD|0#93Pw74WARPj69qA4SsM|Y?WQArN?I9` z$?RIIPN{d|pHZkw=Amh9c6J3lmGieJ6D9==>3=)4JFwd%I$2u%=%B?FU8o&*QhlAL zn=c3%=fDN*1p>%aS0DK*_CefzKW$AZC9MXe+xo62OFR9FdmgVI^j{Ikx7#t)@l>O; z(z}@Kumv}-tplis?%V|Fy~xbTprBK&+`l_-7?{Rf5k^8PvFCt5Z`>bB1zb+@mHGdG4d&>pe*WLzmsrAvR3O^cPmGzxD&e)I?q zHjj$v^*7q}n1G6p?6d2kZy^`FgiF*ZV!`hpzR?jwqmj7od_R2e1gq{SpW*!d#cU)v zon}=sTzii_QjuNt2e4&sNMa!Shtm5wuPx;#`Disera_&W3|r==1h#W3QHTLXovoXp zLq6dADc-m#7Xpo(ZTz3T!~gJcI8{3h+*FiU(hiB`CV%4$RW08Ka>1YZPdqe zv38#K6ew^bA6}bkGKj3h0$BAvxs8n(oXZxQeke4BQPC=PaQT~Oych?u>;8mBVXVQgn7=^JVHp3)do37APUT={7bjQ^ zh@=7nGC8)rc6Fn0AZ;wz`5BsMv}C7%*Ab@3fy7^*Oz0?@Ae-x?wl_sZli6TNM+Wtx zd6v3{Yu`V@^^dxF+QMotZ>i%cIX=>I@r_+sui#-(6o(jfG|%r_k0V&@vqtd3gDH7H zDRN$4eGAf&@Z`w>9?e%4@*y@Uv)aDzKRF8@#dunk+nBGdJ{ujM>YEU8}t1 za9|6s)e*z!K*1G8LsEjd)CVlX;UoYhji_p+sY34(UIgP)JY?O@-XJP;S3;{ej!7Zr|<}?>&VoW(qSMAF^Xd&|WN|SHP1^wod$!QW*9ju78GsXHHE0 zUY%aDW*8YcO+GyNNkm~Z*X)1VvQE~<;^F?9FSQrnk0ZDG!7@{g(Ms>+-CAn3eo2bs z3=AbXMBz69KtK{Q>|5dkz{Aq?sFUCKj3g4_HP3MkQ^N|#DT7kK)?)7fS04dckVp$v z8wZ4Idfl-Ph`kjFJ95Q5F5m%wc$gp*W|RBS-drED`Iz>%wx;p4xEDvxOBkBduntwg0ME$)Js7?lBD17&juxr5TRo2I=&ks@x;x}4@h2PCFi@?%-|jE^U?XTz7wx_0$y(c$ zCH1?)$wB=tMwF7iDXKVM+Ls>q0w!m2l>G4=?~{?zqgX#|OSgUp4D4Qjt~O1~31HRK zNWg|kfUguDI%#`XPG&MFkRMvYGKk_|U`lnH+E+A{G=Y{N<@PDs0|pF8`IQebHjSyA zTKhH{1nUo{v3N)1QULpDq`a0%*oi!Ep|S{No->Q{@)ifq=IMtQPv1jo1S?*6(|lh< zGD{zzKm>BX0Xq&~mp|c-sN1t9WG}u^pgE$Yow_F;AM2oEh_c+|T(}(PhWoVm?K34MtZb*3sAF4>T*vj+ME2tdG!p9E`p}n6Tmj0m4;% zzgVR)E>53flo8!i`D^;MZ$CX@h32WH^+&9^F~!QtDlAv$K(2vD@ao4)<_E?4l$*J} z9+9u|+S};08RtGEqd`>+?pOl!N=2B-B|fm7UPp&qIT;B z5xMs6RrB@gHg3EGRSl~ofhO?_i$>V!M@wVO*`eR{50JQHYQ!KmI9Q| zDowMm$(DYxzrCpNO=xHoQq!_F#cb3rB|90|miKpI_Tj$;M8AO4Mkb9Yb4BAB7SnR~ z_AxsfH&y%Y?D@`JH%Jt*o8CvZ4{YCKL0EY&-@|AMYG!RH=tC4pf@kxilb}r(S5Tdz zESIwR@;|+C)vZ{`^64g9+;3cx%m^}5fz}N>i=hIo;Ww6c$>;9{lo&;s_8vQ7ocH~^ z21Y+WXIpGeynLJ$9Wl@34%OS?;Tv_}9z{}$YBPZFxgUOJiUwKuJ;L?^S72C8@&3K8 zf3I$I_iEnyZ!hxq*9CRE38(yje%-5oteTRl4tD?kvc7ZsA~iJcFCP(;MAf~MfthXbqtEn+`hRP+#PtFZm|f|CS-=l{&CCa9RV zwpovx5fGQMbUSc`qDux8gQ>}-6n#!x1!G-`BdU3W)&a2YyW4**Vj zbCEuZ!=wox`0Dj*;4&+cQ(@Lke_CdltkRp%r~1~8p4c`!D9Ne|^UNRxML)g!>@?nG zHjnnM9qj=i$vY%>JZx|t)ToEZP?nLV?~0zG4KiH|HW5eUW{!N=>fmw-d~~)?=?liD zlvMF=2iU$muU@t>%W~LmGJfg9!Pz1{0x(!wIr=)Gvc10ZVJ@JqLy1uVW-pGA(Zi^aTkowZi`|8*z5|e1z2@EbpWaZtOd>~wtG(QHb;-U8 zN+$r#-Sj9=>1q*p?_U^9$6)%Mg%Z#8@?b|y58lS$nlDhGX2m9(;?I@MADHhb%5>B? zlU>>&$E+d$h%1W%;H4c+)(m!*lYL1{T?M?k;dFp{Lg>)BD{qdYO~F&b?e)7BavcY% zj_0)dOH1}j`>6l0{_(AW7FxGYq@?xte>h~w%O38oLhc}J5OJRyEefc0ywLCPbm*W4 z&=zYV-PLiS0Y>H-N);|NYXQxahPTRv^!5cq4U1HzpdlQH%Y0a7zUKBMs}1kR1?Klp zsP>(tsE%Aedc>(=*7k3+;Wm>ImE32!F*Ke3puzWOZxSkU^TxLrQZLW*H3w|Wg;a*q zO3L`m+Rq1~$FRYN8L6q)9qs(>gJ#K5B3-)tI38VdZvV1E-ccU&L2(w5q7WmL#1Y$j zaLN+Kvgg-7?sTg0Ej9r&QD+`LS!aEr4HH47oyI!+j02n+j>Fsmlp@_W#(0<=cRFA> z>@2^NfiEcV#eMW}$!yhko*Y}x%tHFbV(bannK?`bqvK;Z}3*pW-0bkT>C%_mI> zt~$aYR#Y;yeLa&p)ZYd%b+EwFsO z`0X;UXZf)}#fInRT_tE4hg~D~PXlcPQ~dJd$0@2q+w2%MH>`Evd?>fN02h>C>a7Lh zXHgW)5AiSR4uC!xAm%FGLm`O@arELHbQVER^5Fv@&<@o1p((_hEvQwta2k%6#^knw zAW(J~a5RxZobkAD`d8Dc-^Cqw$6KhT@o>W}(@sy@N_bsO`*PyO>yqYzSTG-W?YycC z(%ks<`MPuE_p}?8KQhHa#qi%~{RsE0J&W${n8wIDSk6TBK}_1nseQ@|`6MAnXp{Xn z^)I_gM40?7nroDP8i~!Cuxqr;(^v!$dhv7+%?2%C#rVTBvj%CE)ByLUw}PZK&Ne=F zJ9BT3xnTp={&O?2s*cO;gAZmaOe5C}3d~qj`{>-zED0$V6%Vx5y?0ien-^ozr`Ms_ z27h8&EH%w{Ib|HU_;udfWtIwBaCfTZX^f?qa5WU?;@r!wclq?2cOv8Z_vkfQey0$w zP!I<&1E>(VnuUfEAaz9IZ@DmG0&_lmFaW{`%NaUkh)_AaSHIk{dDY2Q08RJTYd^p8 zqxOFaAcBL19F&DZR*7e<+$^h$*895KPdKqa&ztuCO_3A%3A#lV26MLyLoOLlS`qB} zi{b-?^S+re(mP*XF&Tv_#QuA*YH5_{3v2AxUNQw|o6VZ}J$LNfsoJ4K3Pnq}B~0(r zIn={zVQ$SpfBVe6H0fJw+R2klOye_lfM|CCv9LS^Sj1+N>T($X4~Ow|_(G6!0IsF* zy9dK2CexgvCa5sb&vbx?g5avk((EgYlUk}(gn;zv=VWhZ=nsTl=t-U`Q zl)3w!%87jfO#NhNNYEJv8u@>-$bls-TDBZd+b(Ss4RkI@o0!F65kcYZR=#hvK?+AY zq@*w*u$Pa`eAO>rh(W<{vw}4VFLB~a3<~GIx@h6M%Om9zUw70yvVO&|QxkMSOH>D)1w{7b*T zRW{$`H%xn)C4<|CfArBSMZfq3xWIv#dr_<*>6O_lQqR|vBH$LB82BScX^p{Y%SoM_ zX4swxTRSN1I!`MH5A4d-$&UbWl!jd$+^})u%lh}4=J3qk5mQo?tC{?0{p=Ev?(YCSltufh@8MoX`&&O+6-FWTtw{9(_q?B zgka<__ZYskxw$FXyF=CM9&cmqE9WL}Y?)T$fO-%YBuEymlZ@oUF3Ulja(s`>>xvji z{)U8|o9yR}h~sOLX0ZA{3#vC&uG!?nTAM}e$KwV*L4>JEV)j3|yAwzP)tQMGA!|Iq5wnaSdx?KuqX>a1x2Li z7NKUqR?eqt&Q9KV=x}(aZku47@_DxyZ;SWIxdk_um*!3ZKoFdT?Y9IGTEOs)hXCo! z)`dIG`72nxcVQ00z`Td!_*C?(%=j9vM%--A$|BMq`Vi^$Ou5+tz3x-BI=b4uzI`$H z)atjd;??)IvwZz2ds%jrb_=7Kp-!8lBz~{I(5l&nK|c4~LnG{Mv&uEoS6j>32|9T+ zU+Ubc@q9k#r~*BBbdP;zJ0JhcKopEJRI!+y7Ga*aP%F-_D7bZ!Zr`_TF9lB{&TOSt zYnkgO>P-B$qG8g zRcLI1KNKEDK~U(mJ7L=wbyib-s|MwZ&jFx8%IVcop>~GJB7fL;_eeQqJf$O@Is#Z+UzUdrg(nj5j)S+bsmi5VmUDXr8Kn#vv9^%fNtzT1 zg}Ykyn*yB7*&4Zgi^r~pogltjH>g9bWwJE~qz}hScUU`V2J*KUG$3;%TdxkNUzhH_ zE8pwk*|l%~U_WF>`p<@}tWn)f&e-+SFeRm9b%|YK4OTHdAhG-Jxz0Aq#s#f>PoM5C z0vm#_>WzFk=z)6{dmoPudb%#??v$G3XKePLJ5*ook4ol`6$>k_ja&2Kx6)jl-$}|3 z^|~$h-I_OX)`8fO`WwC%jQ0#km;Q)y>+I}z=Ajk+)|Gv0-g}<7DtohI=cY9hJt+TFeF3e>-Z8aZxxyX@8ys6kdhdS3>lmuFvSmm{=;0)TSS^i3EEg4c<#rddGg zGXebE0G>uX8xo<%W4_d0KUJ--$$ew~!)Vf9mEYJi+z0aWDPOX5p*a+hsg0kop zs_7j=MA^B0E!E@sCg{a<@n)k160wbS3! zXR>fH9g~9L2@l~{_#4FI_rYQXSmHZ2ej3E*$kB2PJ>V!O9?J5a!=r=ogC&YsvVA;) zj2(VTS?6qdWm~gtkm!3-KgC0U(^DZlD1(%!oz#K<>3qIZMkS zTMuLG{owc1J2 zO#O%C>tHx4_%Z|iNY{=oyuC#%GLDVeX@Y+@dME$WW!?K~cze+Hmq5;R+gJ_ZHC?ZN zarzybJGM2ZUyScN;{6imm+RRDU`V?}Yd|OMHNtfhVg(-7{4hIa&qYJCjF|-LxLlru z6=nC=O~<4Gv>CsNg&RMg&Wetq@M8NfLN304(QYxwy zL^V`cvX)Vgo{4@f3NA=re>$5|`l&>{&y?z6Hp(ny`A0-HwoVe#=w>ba>E;lt zt@V2E9iUHBlt*zZV75-)@SmHKf5bX|9aTETr>98}Er9p%C63m8>c*wVw{iRS@pXB& z@0gcS_PtK-_6arpGaxyz!Yc9pCAB$EPj1xI@T~HPitnj?zHI8yHGkYf;wx>>c|UHw z?CGgRpWl7N72jJ{)W?s8(1+q~O(Am<2>feGFUkZVb`d{LXKImLHv#BF*hTTkTBxP| z(dFl5UQ-iJwT_mU-}FH9TmSJV`2_{_Z-FygzVLf!UC`2zVrfrw-u&d$LdHX4%SWp2 zk544e&tC9z`XfLsdg{^`${Gv()bYpr-?pov+BQEn2Ug+bs|K2(#?Ct$cT53?ms1Pi zM>Z}0m7AQCjm|u0Z=XoZVaD(dKQ~p*h4l`Iy-Twwb_67lgCA;n=jUO6e`+9YGMnt# z>T9hUc_x`BuMe8^q~)d$PJ!Z%6%Y`VHA=nd|HW?<{LwZ35H!Q2QMj7A`Mk1QPo7k* zYcR^9__gXaU(Ys=o;>;L_bVTw8hLkhP7jf-y`iG zN9Z;gSmA5EYPTP!Jq8EfUO4sd#T(QMSvl(enEtB%?CnBp{Lil&P5#yR|NR{K^*}3X z$?B{A-@lanzwi}*r_=uThjr=FZ<|-Mu_o?3AZZmh?mQ5}(V8Odp)<5%#_aFcf`6#N zp2fIqQK4G3LdE^JD@!IGD{tqh^+>yI z(>ey*X7#YPU&PuFp*SF|z|o>pqYn8i1r26`H9M+hl9)v@t8E-k3bJfum9X$ir_R zziRha$5^Nu=PGfAm-NzSZyW6$oRK8{(lEx@!T3*hK~x}`WB-*gi)imXu1yh~oBCRp zQGF~v4Q5php3HcToK!CJ{HUoS*A4mmVtWLoHhT9yKfhIe#wcQwXl~Fx9mjlF=ev;X z;3k%kFyspDX=#RF*7Ya=xeZiu&DsZ)tm}3DGADFB4mIv*of2RO?<)gPeBEnjY^V~U zvbAZ0@mmAWtyurQDeB7v5~G|)A-SGnma&6dbJ8Hl^?!dE!-$ggvj~dA+y}nn4HwmnqwSD+qaQj!aw83_#=3+OuI4RB4xX9Pt?caVIa!xinir_H0$@hUF5(Q7 zfwpQ@%Pens_tDIM=%2`Ng>duq-*^-Bo>t#b>Ki066o{g861^JLdDwSSjVNW2(Uiis zYjgS*-;7(w1k)Kim^dA)4q^>T?SUYUkfHzgMmi3&~xACn%!(Fx4KJBMSK0MU_ z#e=jOkB?4s={fVvt-#E6t%q9#tnJN|P*mPMqrjihNq)U1ge15(Mc+b~#gn`Z-X^M< zpLd^0*(pPeNYO=v%uyv%=;Fh_(-27SfeVB5F9K)B#7x$S4ZxqzHsQ4kbI)l7Op~o7n?Z%#O(!6;bw1?&Q`Pj*^cqe{~hR6o% z%0&=UJ?dF0Wqp#>yJyBDCVxAe^s#g$b^0-;r6Vty5916E0A~wINrCFXjHI~ao|H#2 zRS>3ETS_>IJs4AnW=HKfJxQuK{E7<>Cypc`s8b!DzfQsX)6)IF*hS?2Ea7ZG^n=$L60aRv@aB=xR^7_#ysKP*jU%FCjdGS6f9y?*+~Ke6Bx|9Cl_#bd7`m%(|s{cCZ31U}}`n%wAJZ8;tsswQj%L(1$Ot zZ*&bwP?X!>v4UZLz;Z9Hq$JXr^j%zz2!C`ES1w@onx z&T!4L7*LcmXa7BfxHJP+L=Q199Wf#{)YCUWzdzwu6I!J-X)|H33A6>h2~!(Uhs?2S zb7P+GF)#(Uu{$DSWmq3gB}Sr_mR_SNVwTRGZdC#@h;g&_a(oYSIa~Pm7npV}c-EeV zOV}GISdA>JJuLPos93T;O!_=nl6?3!vRB#D3OZ_OiY8Idk~cE< z1j4x~2q$-tEw{@5{_k4&ejcCdn4h0|CG6{H%xrOZ9+0YB1_(6Zl{M`VfRjdQ(?&q# z`p`Z3u2sJ}^52`nLcisBaq~o}IwRR(mMIOTscu?}K>*;cL?~dXR(n4?zm>y7B8(C) zI(LT@?1bPWje*OCA5OU%t4{%>VG=9{)7`#`+7BMU|~Od=peZ4XO-r z$_SU#R9v~Ln%YBJ8d*j+){AbIk()T{jY-(6PKCQB1vX!qO@?X8_fdJ<-p`D-5lJz6 z)Tka2W*1Hk>Z~Lkxv&Vaj6SEnq(!ZDzQg2dYwRvvB4mzn*zn;W(=UWhzDk9i+JBeF z0RZLO3q>F)q62b}#_}7K_Dxq8w@b^|OaqYrXu+GO<1&?TXtL&36u1Kw=Ds0mu zoh$z&zKOYe#i62SsBs|U`NN~D-c?MrpIAJeO3m)R{~#Oaxd~frlwr8LBAJft;ME#* z5$$s1Pe`^@jbl0Jt(E@8v<>Qo1~!X3hfo|yl@)kmc((@1*7H^#7C&j15Sl|iJ7{4wz!xsb7>dNSxty(q90 zFV3EkV%hnKFoK54XOV^xO$EZ$x{9|&qibQJN@WLrS@v(Sp!m^r8XkSi#;BPvoT`x(}iv9)4%&G&b#buWjnw8`~>R4d# z0Co}BCYuGB?xSk|oh1WZx?iI$xgEPSr4W})E0TN18oHM3;3i_(eG zx)s7in+ke;)fVlBrH9{MsD)M0L(|F<1Rby7qTGAaX*ZIy_UUOucSb&*n$3H0P~L3 z;vIYMXPWP`2A_-G`mWpVufGn>EPQ<-*k$7%>wNR;e#3{mulscKVR^^c^`BmPy9UVd zeXCne*2HcrwWQ{+M%{YqC?+URV7^sOPJ#K3GSm|Oy4kCTcNiW;L~5ExuWgD~90bC= zi8jjBC2tRZ&y`KC_vD`_2n72s?}f^s(%iP~<%)4mXe4mo@jp@UI>yDHgk^$}b=HPn ze?se!Fe^TtjSzEokZ`d;g4`kUIy?|DI_^{r_^%cc8nou`9F;O z$yjGocm^g5ZSQkF!6Q?#pls_V%BAeF_=BXhXq)M*SN%H%&fd z>JqU)rQ7V!?R8XCK-?>THF|k*;?g}K@h(@_mbRphgy233{P8s<92#E6xTX2>VM8jv z@k?t&;ksSMD-!<#?fGMbtbvr3DJMW&H&Ey~4=_W&M>prDXHx9l_ntGVu{cO zbJTCqGTKRAkf{Uu%01D8-nd5FrZ;1`#-RGZdrsFy3NRiH;K6~(6|@Jsu77S!HkoG{=qZy#Dt8;`U_*(iaZ!D@T`X_c^d1OfxuMZMXl$-kV4D zoWKA7FO2cV3^R>w?99s&5h7A#=T*p>BH2X?sJZV!5^LOS3=Y>);|J@wJC8g*ww zFTXxsNgG-C$0r*btmxp}Ojj{1;`-pn2ZkK1^fy|!&F)}U_=5D{)Q%mEt{j~?Y~Ih$ zZtb9N5@x53-DnRyPL?afelw-L3VCWC2l!;><`7gFkk%FI_$UeM)!ViESR~>XX_H%{ z{?>+H%G!ACo9_%~d@7h}=)%jq2AEll{>zAygDgJ4`Z5B+;EMbo?9PK3g=(1wuIlt)Ol2~ah-d#N zLl^C0v+Z%x4wRPI{!6XdlgU&}#SA}1--$d^a{s&(Jda61ys7#u2ok(_wJIP@7sDpg zc=?!R0j)^&rJ^o!n7d`Ej1*-CjU13DvaR@(5H3VP<{;d!nlHjDX^C{qI+au{R5Rs0 zs`Cvl{x|^cQ0`$jv<4j}8~_Uz&5=P&BSt9eTW3UXkhXoAJj6)`LJ%{Y=v_E6SOc}i zurJt*7fa_|6RNv=l&3tnGN46*!E<(j^i#0Wpz4^FK1bRr>EIFnhAMFIr^)k@Fik>I9A+RFl^X~mlcU~=D&7zdN%LOGt1-V zQ4Ml;1SGz7IM0N}gF_F0m=ZGSm**FZ=T<+c(>El`|JDK%EvMo>7uRj7zvz~GyDpXX z7sd~pC)$>_o?CwU(4WN53QWDSsIL>5InpLA6%nP$i#W>Uu}r6=^Ap3>x8isQS?{8g zOJuZTxkIhM3&KJ0ho~T zw#_`;psL4FGplzM5R<}-Dd1{e?1du+N7x|gP~2}=nY%V?`h%z!39fgd2X)z5&$-9z zuQ{yZeY5WCLqkKyRku6(TT$Tf(E?1tn(ybo`MSN=w9Y;!EaM;5sb-`}o3f^f!vo@C z4VIVn&5rztd!+JqZ$)LY&s^#IqHmWJExtY&g`~ZDMU#s%nPmmld7YIIOLHkZT0+!- zfmOMYqTvze8j)Jg6t~LVcZH3dL|Zk=h3nIr>M8%4p2gg^qt;yGk2Xtr^xy+4QPWS@ zt-8=}>3e2Bynk_852=zL!ieZf=VGrW7@`$eWc!-$ry#+4uFy=~@>@x)&aV7sXKSN+ zf{*-GsQjt+9+i|NV?B{M?-#DB{-l~Wlj|~aiJW;Na8GsT$!auy;6<^){f6}tHhPV* zZZ`S!rgj6ZyU=8Fx!iK3df0yp0hgW!mT!En>Qn$czeznM@+LK4oYT2l+IcS$yXY95 zTX5#NdbSyp8AdTTQaDonQqzA36OaLa_BUlF{q1;Zx~ur*HI02{UpwP)4JxMla9gFkEm z2sl<(t-V-XwFZ84Jbz?Be>&EnwA)1soEiK#?o;=ktQWj=`k7we%q<}_3t=l=rG_#h z3=)3Yw#-J>YumZLlu`Z6D^Tb@i)pm`)_pJ#r4by`lD{eNH3eQ;?1mA9B&`hn!mx`7 zFpQIEVOK#}EaQQ2BgiCG@Yu_}AGRn*!_7TwGi;kLI24=tycfMRfqTRV>wyfAql-fZ zkCnDD+2%v{5Y0k=eMA1N2YW7D6<&m#j`w6f#xDKkG!DoyEbe5gL6h6n?6trD-{tY{ zTl$q%ph8eJg}Q121Jd~p@Zex&lmOAzw)~Htm7*U}baU@jES8q1=GJB9WtK6zPC@W%Q#61m#8Ap#s0JJv`N2cj6E?Vh#Y&^< z6_{L+?EPlX`uo~6zwn%v(%sK{cCQwKltr<|a@8KkEP%5&2W-NV52p;0Ie=vDx(|6# z@m_kxd2{==6m$ZgMEinx{^!5>_ve1^*+l|7-|t1T^NWNZz6Hy{7ywfaaiVkoC?*93 zhMroC)ob%&4@{{7I=0Jsvtgni*tobFc8&MSZ-|8&i!CM$04%BFl-&;Ql+F0FL`MX$TMKzWAFzB|KK^s|`ie>N~13tP|9 zFOQ==oy`W8eh5Hkh(puS3De$YbuKgU*s-IK--`Ddq3offsl~>a&}-2j(&1|g&pQU{ z>eX6iV_lba!%s3|#Kz@4dQ-$sH`HLcgV{S0Iks z?OB!L^v+&uEr+GZj4ryz0YffxxYggKx@FGLgE^X?yv1U1>BE%oyng%6vE5v)T9ecu zk+t=!tC2+?78fu6yn-if%<7~4h#q7`d+OoLK1`j!qHvS44LE8nZ`l5W92nwGLlde@ z^UIhWU@YW#YKCo0o~<93=+~jJ_$QgVFmq;U4RgO`%kIoZ4_ebM(`EUndSg_%pJqJ# zv&Rqge4JUSTmfM+2HiZ-Tl)u!CYk-mmvZE)PF`Bl%0Lk*Gm!94MEm5PTka8Y@xwMY zO0Ac4pUxa!`ou}^6PehzE!xE6@*bz>W<3@*++4JYxtD{$(x?yj7u_GCxu_)(^Cw*a z?Y^gRxq^Q<(MC*rEMQD!b$I}_a5lH+>Y|JL(?Tl^^WYMiO~1ntAQMy0g+GFQ8UZ^T z5S!)^gZRL#Omz`8m#HnoM0cF{`v-ZOY{@r(MhQ~r98LA@gKXAZ-t@}4X$ zjQ9-d8^vfNafM~10VDAAr8CWya6l~uQKT1S78oEEJI04WzJqJ@n!o?Q zY}<41hlPdLZ35S7{-Lb@j3LR~YIf*X9~UO6H=%sUp>xs&iVnhNXD&rWtPHaM^2;Ut zf?8F-+6OHI&%~~{_;y*Rx^=Q9{FEi|Le}K{|BY?ug|IZ zcmDrv8F%!WdhOM>2H~g%toZVPd)*CGn z58^eTVE6(7en>a^rQ)LJ7vT|*cd0=73U*2?!}z7m9)KkCCUkqOsCiix#=v9Fi1w%d zhghG;ipjHIiSbQLNr+A~87Y`sxt_QG62iVnUQ;A$7ef1k_mYQe(``m(@w(%ogWt2_qzS^#k>l2dY29$ zAxED;=JzqTwzF-Gm+gB`Z_TrQ`RdhWlIe#J42;40%!xSmcR!Jix%Ew?sDzgDFfL1X z2^t`b?RW|tJlW40?wnb($^$s+rx(J+T;5NMZ+;Cvv-7v#zKTyu(oX%brwn4z=4p*H zvtwCj=F5%g$hZ$bGR^gFoWfuX?sa@C(J^dcxcZvhNWB4VnJ;=_~1*vGdzgM(YA+m&%O2TGD{#pU3F+w($#X zF66)doCsXJaQ#29(6kxazvyMpbA!Bm)<$zm>6XOqrSR}Fc*~ul{CajFbt7#SO;)Lo zA-L$HK-Q9TEgnCM!o04qJMUI)p#4|=c%U>f+0|cN(`Y&@!YXEESN+KMtG;)7wN2}X z&kr0qV|ZLb8=Rp|4Dsd%OCDBG3Ml9;)J&&*k()+}mLs;JX8|dhDXD1RzJ2`k0RMm1 zAyb=LCE4r$u)1L2{a}|lV>{$l`;F^$ynHzh=D#9Xz)+&O{SN#0F)TkX_A9i{ylAO` z5k_s`u4Hy6#y*aLOe0FmXqVBPB?hxDvn11QYY>l&x;ez0p~FA6iNE~k>=(VNj6E*Y z(ICO^UGj)9+c5M;B8L!%Y$yBTMMU5YF#TLEc=q4)*VT;3-%dHcJ|aRF%5zK91E3Fs z8}vA(MeqphSQ4{(*!VKlB*@hFdFYw|xG)el;5p*!C#+T4QpCDag5tkY^Ffxlku z4=pq;)SrVz&A;LT{x4=<&#YlEEi|znl2#cMbBMl^*uwnu*}g~EWejND5BQZT)$fCJ zd@_w!+7~I!-cxT7FTM`$e#V0^D60pCHhacfY(#Ddp*=MJh z<^~XvsWV9AxWEZOLIW~1wF|_&G*~m=@Q(BJh`ti4zQivbbrW#EU!`?TESoqy;21Qc zJ`teWgByJ2F^?O;EmKH zAt|sdYq~7C&RJC(M`+2ZvE-+#=z$zBF^8wZDJHc9#pq#cmTV;y9is{<;OU62y$rk$ zRGqCSBT(oVUR7MaYUV#XLK9W;`>}Qg+6}8MUAflQ)|~TrEqfiWhgp-Bfh}SzbQI+7 zq`TMY7pJBCV*Ig@>MnGyinAJECw;NtK$-O_ZN;oXnGz{aR3v}G{VjaN1P>P$$e8^D z`%&GW{P6`cXwW#n%<)k!Z~hI8AFRSVD>FX;J#30L;G3`AVYykpj&Ef=4jp3hLlES$ zjy~s^ZHxXCGsZ+_ws7L3)Et4VMf259KPaPrf<4BT0 zDjeui+2?9=RTJuIy3&~JBIC_q_hjrNspl%LA!+C$vea&m8Ac{d)96-?FMJ}_JdP+^ zr1tz7nL>iMo@bUi#%w9P1cW%ngMj(l?e(ixWW@ zR(DXUj0*JAhl)E>B?k${${?0~&Vd0mm*xA3z2}#5E}5 zaa5d-pXtsugIS$VCnxw0y9SRFx7$zN9N4!?Jv)jUqgl0fJnaJ6=az@QdH*YDI*Q{x z0q`E#_6xZ_X2;1e&GRY6Q<1Z4GEdD)*$&TtTZ*Mhrg|}(yH|^Eh$|1KH%HW` zpVEup%xI$oNQg*!n z*NPlQQ4yLdCe3ZG`VG#zH|v_y{!H;3@LuNA%O;u5owU!ypJ`%tK=xtU5pq>}#4KZR z0i{8^|Bh)wy2Ydd$EVPd#+A4Jncc_zYj>87`nYmf?RYW?=1Eh32i{T=*==#onecHH z#K~AryB3cbM*w1gY2GYijFaqrYEEeo?NMG<_86F?Un(U%-wp;08&^U5ES2c?afjBU zm;#PgpZdi^$;_@Jci15atpXU@I9$@g4KJmQLYN|er^P%$`;wJJB6Kb6ec@ebgYWc{+COwOhAtHy(5dTJ*0(>DfO3^dOA1G_R6l|HQXT%S4z1>u(sbScdYh zpI7nmqcgmZOc`==T7A&bLn-5if7G|PSnnWm;nMxY2_zJk)QMY_+rPwWY7QH#K{lEP z;@LkFbZ)>@J|NL4A8O(TC=z}S=dlRn-5d0XXDp*epqvyiID7EyaqClYX(u+stzQ3v z)Bfk_alHoKG2GYXP)D7yRqubjaCOY7|5_6|JQ}WQTy@o_B!Op@nxy>X3`uTSYZwf4cyafPHo1PiMAvM22c}CFA+m1uwIIE=`y9t zlCvTjMSZ_@202PbksM5R`XY$)e6Qc<3qNPgC(Kpeq_=VN=*TlL$V0X_2XG`)FZQ5Y>2Wi zNeHJOaJA!c%P?Jg;(iM$f!UKDS=({=jkuDqG;F&DY#wA-`tN2dF{ z?Wmsg@2A5l=(>Kv7aHPGkVQ(tFE&O#HJ<`N+GcoB3i<(eTz)yfdjXp&om4RJ=O>CG z&}K|v>nwu^2^+d@!^*CkES_<9!* z$@t)%L=q+rx5P7|VKx9vhux~Herwf#2SfjO4oz{z3}?^C6b7i-4j2gN3f}}N5=GSn zp>9`XndGPJv}AGPyP0%>KWP2s-tb)IvSy87cb-iGW$7Ziq`>Xha&lVHZ%XrMDu=Vz zAHoyNrUzF@aYhCUs1Qk+sjj=aPwUB)xv{JrY3qZVSrM=|Ztkx{U#A8S9UaDEY%`0FX?w=M(FQ0aAuP6TWImDL z2+s;ZD)0n}ySry4@7L5BceZ1<`8$$&4p_k0WY(t)tAPF{ZOFsNZd-?Qhlh~cMLH)n zSzUD9%Tr7zX)5Hn46$^gdXmmCQCS@TYXAKFUUopboc!nM`!8&&6^yr#5ioktQkyw_ z;K73jz==Djg>gW*WQIt0LiYv&?Tiv+q~B4G&VsIy$>eoQk+r~VxWIyE|N2<<0h+S9 zSiIo4F(!S+y&+F^hK)%x)RDV$srx1P%>Q=bzhkyMibUt2FBZIKfA24Yth|M?eV=X5 zng^HI>c&5n?9V4x_B9(Z(qd2Lt8dqA)|gMywtl(FA1wQKl+)BWX`cVv>0D3Ve>S4? zH0Q&%KSV;OTV0%6cCk38X$3sBxa8r(AqZJ{t)(fS=d^HPkCs_s9P=6iq>rcbuN=3#e(vCOFLe$i~?cTpW+|iL7JIL z$Ur7vfE(4%WHD*g#AL=~t9*1@aa2>dX?!E+DY=*!=(8#TPx>6}5>m~>R$cn-kI&cH z?eJfPy1cfGCS51EBhXu{9Gd8>+m7DC0uAy(O_zpn7+A+0=_t$>E6YsLRw}Q|t04tX z)%)^x)8R(MZbvEB4*@Fbxtg<6#*k8^S%xN(-_h^!zFj~10tB}at2WM@=+RH)rNee@ z`eKKuj)IDv0SclIu^nE$efxDhe;dR%JqPdpR-8DKPtG2S{WwwB39+P6`|t1gxl`YO zIj@@K!~c!x6V6aT?5)mKl`b;A4y~Y>&3Q7jG}4G?K!E}~3XUt5GaW>90PEdDFXM;J z?X%aqu!#m7kFf5F%eLlPJ9bo$V9%0#X3S!&o*ek?%s?@TNy{7^PZi6kZ3>cF(giJ^ zHsR4_D-#6q_-ofTapTUUPnuF23J+j^y@WAQM0p!VR_LznS#znn>M!<(`WR_ilyVdvCrVA0 z%QY(9vnm6CFE7^!?X2WZQRsePj7Kv+^rXX1`bzuw~oHUKM$*-Zs(%u@Oq4 zhW~ndT8$6a88D#bvo>EVjxO@}=LdedCq7?jme&{0PqPq|Ng$&4tT<`q#sm0}cK*lI zD~@OAAn#7LqVeGFbZPa4aD52@O^1eVcYBwlFD{_wLR1tr{SmsT=Yq(;Iu`}!vScSws{LM@24`UnAp4vw#uE`6cQ@EhA1 zXJ$KSm{IIPvPN?9H?x!zWrhPPVOWHC59updl-`*`PS|ljG-)z|wCwN{;0JkpDtpjS zYvV6-A`iu72W7FmZY;+l%~4RL(>NqE@{fI>7?nBYPS6s}mrBAzQh|F!+Oq})chX9} z^n(^x$&T2z{F~tv$6~ccyI+<4NY&uKMjm<=J>i0Mrk~zIFG{VG456d9cnH!Bl=92T zCA3J4!H&qLZ;T9JfB)QW3hq~og5QNtZq4{{N*_dkGJv3*oI-||;o9NLg@)l%={w#n z-Iu*hPIdYBEBEZ5K;C}Bdjww|;U7M^-=A;zu~*SbGR+7@v)=X{p*2OcnJcx1&`zc} z%hcdQPv3!R@bpAdi^B%ZP|%JYwfA{>d8L#9 zekH~aCP0qJsLsOfB!|s#UAmoTd)hF)CY3Bu>nuli5CVkzCDjayzKDMW3hxv=h_DFI zDes>}rO|y8yno?OS~G6uX2VupBeOB12IWa#dc2_K=-cUsQ6uwhZ?{zPF(B*f*IL=&EU)C6YO9d}`VKXS3IMPba za?5mf3794nyytt#a&nXuB^f(@02Voj8#zQ3q)65JBqv6fLww|BsL7>Whl#XyB`sLG zhfp0`GrE$3?ja|eA?1YlO|A)3{yG;#IEW^SrpF!TUeylS>YavEieft*V6-d#BG!5oF zTUb5|9qtjJtOKs9%t)Fv{YYEC+xp>cY_1h`C&8|K`)YVyU==6jw037cX|L3vjUbUf z=%SGN{95;~S6P;foTL^*HgA3z9Q)vp&-7?1ShRMv zQSUbj2#$l~8gSr;1)FutrojIHgf&Nop}|ozdR5HZ44H5|^t3TemW*LoMPF%s4j!Vp zy?|z@z&F}Xp58Lp1mB5K^!~$#=Y^#$W1~F2H`Lr(H{#l*Q9YV9WW{x$q_4j)7w?}p z4kz(o$vwkD{zyQVcLMt)BUDbMK30v_EVW=Ue!x|o?%4=5;|0y;NQat2M*3sFT>L4) zmEb5igoRdjHi^{g?6O~8aP$UMzaxwosT3vQ(Av&Qt)1BBZxImb6T-_9G{&4uzF4&`>77H=pJD>fTcx zE^BKtCzFdAp2V2QF87+k4P1s3tax&Gr}e@+9k=RZ3=kR;m?$w1?%^?&fNBlBCd_E&z_Q26q)rdC34Pl99uFXpEB|kM%fZy z1w_k1UcKxvNVHqXA~jm8+CuQDG<1T2>Dx$W9V?{UciFO&<8m0*!{^nL6T`&|1`Scf zI;4X2zzh?~CNF)g)V{%Q!Dkzg3z2fS3egHz0nitVD?j%U_YBS07L%}?pUJTg@wk%g zS4@?Ofsg^BDDQ}pb?S18<9uw(mmXj>_Trg9;#FDk5Z4nMJn7{@->D@tM#=Ywpbxf& z?3BA>$rCsWlJ~oh=?HeDm(XrziOP1+5Hbu)^14TZiTTHQVA+vH%GXa9(0U(n77O4b zwxqPuNyjxc^3qAesI4eeYt6=GqCdk&CIxB_ z#6>~^uKmZwwQJT@z8WpEjVNze65eB{2uqkO`2laYSo$T1@d2yg0E+S1j1X^6QF6?S5;dU1V2qEHR<-k6`Z$S9dAL7z75)f0`ZDMZp;6;p6ucRvxTvd8KPUt79m#NATW?`|IUP=@1k#hJlh=Q)%2 z-B&g5Lz3KidGu|SKYE7XuWQWzNo8uY|C5Cp9tBdAc75b3-!{_PkU zyu6XD$q$;!66X**w-fM;X}9}{JAF9r;i2l5GzQ9C92zqiMO2Hxz#IH}2vz2BW;GMk z=M(WnH~~6KPurUDJ4QX}t5Ez|p&BuCim7ruwTrTE-zH3{2Pc@20R^^m+?7;L9m!Cy z*&sZ9abs0geOxN$P40&#&@O;iRNq$dUJiBel%%1(?B4j3_6-o@9;##%yUWzU0LMT1 ztFS2dR&3t1NlqyNJhGr=!ZdwW@vGZaa0(}%lfdtD?5T`?r_j3VI10{BmjXmgshP&p z!l&DU;W#W{vQ<^mGmpOa; z^=*HSxc$S13!UHIF&wqw!tXu1>Ce6P>z4J~ejcH161ewWW9!2C>t@fp^G!jyrCsLS z+;^YWzI~T;*CwzuDq^j1X!x1w0c)Pe+Mh2yqf(R1@*jKXrY`WE=|hpf5c6y5)-C~i zcw?RSmS^W-HcexUo4-fTb6wF&Q@-tgZx@E8V12WOg%d3;E#FwiV#ELVEXr&LvoR%y zXRpfJPMX~B&$k)y2VP3)TIZSXhUF9!>v%Rpqk&Hjgyd~+XFM2kWrej%!EPfCKaO@9 z-^;jOo1Z7>5k#x67HvPr(5Mb>3Z{bx}hEIx=^9hjS!RSp?h&pokj!N zC;6A`03OZfJR47vX~wFy&6-|nYhxKTz`<^*>6+P&(IbAJ7?CQ4$=o5^Y`4~7= zZ)MB|Ui^xS<#=JzC~>@ko(})8{&wXaZ2gNa<{6*fP+M{GwdhX2)N8w^PGh5&^|I_Q zRumTJ#Jp&n+MjSw`dHWD?uJc;?sXa^TCXYHJ+A-#zR*6&ZC2j%l!gi*)>^eej7^Kd zuB&(%J~&!Cx>e@2bAEDg^HDfLRpC9K{7d1HH@cB+R_-t=OB}hj+snhug!kiFPaPD{ zYtFE$w=tKdZqf>HDcpOHx03A;b6Oo;_1RbsRU5W-ABKo2acgZiyIl_@3KEveALo8rJt9Q0I}~yZx&466m#>1 zM|PWsG-#nS=39m0rCU!xguipDfk8=Fl#D(je|8{1roo>CD6{EQ;LnQYlVr{K5T`qH zHm8*TSbZt{^eq!hNeUC+y)6HLW^+gSzMv?t4YO%K-QQnkUHk5mak zVPRq2{*z1hS;!mu))M>Yx_Yq=M(k=c`^QuFoRSNU`YpL3TY{m1ItcL*=f1sqWzXi3 zDG>>^++%$bCtN``m}G4oV8;)sm&FR`qQce?EO&^JdpICy7jxicq@Qeu&~UeW&(Ufn zqSf+@JL_*N6qPa8N-yrdcI}$Bw)XK-eT!e^>BjJr8>jm2-d%9dIV~l&DBxq%j(1QL zI$@c!51!B4Vd#`Gb($QzX=xW8U3O>Q&Z#N)c_8vynevk3?d?53w!kNOve}^B^HZO@ zbPqc@Y)RZJj;Bj=O9jn@R=p@>`<>YBTm){iCw3cM%x z;he;Z)Hl!X8}jmQEDql9gPrOqyIECOX}chB?b{@#{sef9AE+(6CPO!j@G&L&NMVP~ zttTxHt9!hbVvKs*wi`{Fb^%L9l$wWF9$!%YTg%?ld(}FiSof)9md%AXnaPY5+1;?` z+4hX$cmWlZ%Ni*BV|Zh+5j#JREjoJobl+*#E(;_^ye`So1?SzEx$^#Q3_(WZ=e49b z9~i36DxUI$WShG2SEleLIK}bqlEeG1^VIVVo}}+0;%*}0(1)74lS=>o&*@k# z2P}knY%Z=7V%hMqX1*eD%1@ z@?hkdR+pg|9x~phlK}|9@_qlcJ7H>A9oA*mr z`VkUHXjr5P(XRb7U~nek5aPnf#@}(&c%xUpqoqdZn|DZw|9p{bOIW{fSc#LDx zV`s}1nKu{5p0i1~yQGf7{m7li<;5#INXp^xEbr5{8MYV8{T$9%0?xf?Ju-$0P|+40 z1;Z%I@Z;rqJy|Oamk`X-h`v+aoJHo8zg~WOvYDuS1~1~uopKyv7VsTrYAdItP7-RT zc72raw3fx$XANpQKfnKp1Su!KEasERAiKH#BKk4L)Pd&Vl$2EDWD*1dbw{S`R{5pJLCX!fd#Dx5R1TTktAs>PNLF%yr(xxTvl zssYa?ZBlcS0gt!*w9aXqN~Pj-3xDuz^|L(96IhtgW@Xshv;0t*=AV(LR;z7e=2au`H%OiKHjj0Bx?M6nk88MKEaqps~B=nhhS^Th-kNE7r8C;rcXOlt1SlU zBunxveY+PNWSwa-Z##Z$9fJIdw%7ujG_TI}86=ro9w5SV5FF0utNBkS#Yu~nz!jN( zi|O$wiy9C!F5{l$7L*rh^xVs6R^C`>O7)aMium<=`q{n8d2o&qP7Wp(oh!*&Eb!_N zS(hZ5hn=}=tD4kf*)FBr08o8u(*f^~yC@Wj2BW*ya=$i->WM>m-YJuIPqW?&TWySD zm4?e{mxO1I+L)M0x=|)cHo$5H>(mpcT++_WI*Q6kO`Go4?lvAj&#avAoAZwNr0CiH zzUcB-KK14At+?ly09j?w{lwY1ZQUwg*~SdrTSDAVg&p5NV@d3SiwPB)3u)GqH_)uR z+$P&R>hZSj=G(&wkb*6^P1%J?X(A6T^y|A=4Y{|Fmt$;MoqOqIZN+GdlsXE<$(QxzE#Oa@B#4OxWJAJCk&t@+`MK^qEPo<9Q{C}_UAozwuVoQrVrq)4o-6^px%H|!OMr8dod&Ib zJw_eJwxG*?M$b!?Wg(}k-%c*)JvF&L_jRkriZSLe3KD;axl3g465b28_kWOhJUC#V z63{SS0H&Z_99kx5@Vj~N9pMfiriIi~D10<=kmFF&l`0n(m*;2mcF3VY42_?s&fjYj zki;GdXtm_*oQuC~2;9eCII!%Qw<3P7&9$b}ZnBTcHRp)+ZBQ|GFIt?%_3c`f>WO6%QClh&-g&NmM0%D31K<)N7`PS+=K zgq50e=@y9_wG_viO4@Rq=n(U&Iy9{GYlUL=Uh|0)J$oM>eY7Z}UC`P2ug+TavnxDp zbw6sJLku5u^7k1=Ro%Hrh5G_)N7Z5zlVscT+j%FkIsN%foo_snu4nTL`HH_aN`AKH ztLuN*BsE_L+VhiZzTDS;XY%=%iUyhs)%@50tKWR+^qE$Qb$b~Gn@`1eleilxsWPQs zR@lBZj5q5uJKo*4q<{8&FMINlO1<)dOBcb|8TUwd0US26?$5rB6p9I^atxisO(_8I z&P0MumA|(~X0{4BZF?C5;E?3P@MPM-H)S z=hGh6VYSFODD3@JPPuE}6~xzRd{fnBzc&}$Qx*B=s8$8+kiARSB6MKV@uh>hm&EaA zmbrY2$8?t9cqT|HRZza^K-VVCn@c*7>Q)ZeWK6G9z#pK@I%R&jm7O74bp?@n5pg0WG|FR z#WM}JqVkjKuRP7ID$2)@S8wr}w2@8&AMI5q?ydruj34l5V;Cjv@vyL)jMxn+{JK$V z={D>yH2^Du(Y#Nmqc|lU#As z_IcK&s^uqQ3-?(8_XQ;Myzu%~YnOb_3WBHYsd?PUwPa?h_E}f?UYvWBSLPY-g(OgF zmIgGl%hR_|iMA!pPHWa|rz6q#6)%$|P4TgIY9+Oir7WjsC$Bcjc1rG1WXbX79}6kc zQaA`r4Zi$&VoU?Vhy@Cl91rZzdRxdMh-HuAeY+_y!4k+7nz^PWDdqDr@!(YKsJ>KvffS-JZE)}_ zi}kIYZ}x_IU!V{BA!kLsyXR~s;2>nC)((;~79l0uDpATg$h2mA7!%yLh2cbXljWd zLonR_>H>9SWTf(VgI{$Q{qu^;Xi650o!V+?oG0X2QM>B*qoueE26L&(e*eL2N{ZaH z?C|IueVK(St3hp0>(|&v8`0TyK6tTqEk>N+?S;7cl2y>w49XIJauyIvnK>4Nqn8rb z%osH^DtKqF;RBrPA*bpdujpNU;qXQdaS37tB1V)n>rYlsU<{~obCC1+r4 z7nS+Uh=QnVbD#}##5WCpT!Qco#a{CvE# zZrsZ&u69BZDplAv?4|&%6v^tAo3#X6FVC9Uzq!@2%SO&pqjT2PUfO1rDnZyxjGRxE zIED~BF0iGK3B}&@XH_?nlkGmbHREB`U-5WrmxcG2XN7-QUv;U?&wmV!CB~JpW4ov% zXJ{D1@|${gul>;rD=t_2?1jX7PDW5VukiK@6$s0+7y9oWK_SBy4dk%V%lJAfH8NJ# zv6`LPuTiIa4E9&4Vf`ZRz6YC>v;F(g{~lX*qdSw0|Bxsk6#}^42TN|;QL}GNcmGXW z$eHoYLN8^8p4J-6jWyS@1Vd>bIj|52L?w+&uq(yhbLTt*x64+Jc@;ej)W!A zvBKD?B8vjnn77J>;BsR0R}mcD@ubODCxspxpxv!!NU466#|-EWYIq!rDm6jHAs{w3 ztC19+4z4lxS0uAj?0znNh)2zg){8P&rh*NP2PcwSgodEZl1Gd}lI*%6i)|YV6_gG^ zZ1nn%;Zh4r3BP~EU2Ukt7sEm|dvN6TVfCG3CWcLmcwppAh+m=>4ktqY0kwfLiOwC{ zG7WUldbs&S8xU&|KTn3GL_G}&HTRogk=2MvNc(=;Y2@^a?Mi~_GXQIwEGq>w+ zU5ciro=-cly!VV%Y$YjfWp9&U7e?82-v+-{5`vRh|wdXUx`~m_~`4c zmKvFDx05)2^7XNaUFX2B-slz@rp7f;#7YWp=>{7i0%ZQQOCK_fW;9_mY9Cc3yCdtb z4~75Sr=;1=4Z6v^Uln*qsX9Y6aj~^B4TSqS;0<$WhN^GA^Zky$6MyRnT_Y745$kpX6%7oZogBWd`eSYpdnE}vWH*t% z^{R)zl;1wPg6~$C?j+!dYQPfc$&y;#Gd5ox@(LobpS%sD5=zxSl}RKG^YmcN&R39Q zCs!4@Rdu0Pd_k&F`OT15B3qzO9Y3>z=SqiZ|69DKkeNgy9^JIDN|E_Ao9$m&~*>fFN^sNkhJ&>&ce67sc=^ zphb_*3poYl8HDXqa1syt;=aL>OHO{ZhQK2GQB0px=P_r&vF++72L-kmJlgn!?I~Fo z2X@)UO&-Sis(tt*jy~wh_b++QfxfK?_9`{u&X@#YvE5HfBA>AZ1qH;Zc>-d>qBEyU znsiDZ*M9*p#zb9NZ02?Yv~L${DxW;T_Vnao0O_{CHf0iV*jHe^$|)CF*dS%tB*Ls&y0=#XRd}zFu{qyC&5E70H?B+$%Zoej4!gBnr;86!R5)gW- zFq?$B6w;G7(?L+n$!CtG{7d2E(62!!xbDs>fLr09Wv~2)Pn$C3cD^=$=H6*f&4#t_ zKC?wVQu3anY2Q+Zf4m&4L?TGy<;3&~Xv=q6+?1o<14`=9OI69A3W0zDENaXLS(Xjv z9o(oT36eSJFPo=?6N)Pce$B6^%b$A00-1XEf3PY~cOQr?&xzWu$vL_D1Mapzf0SJ$ zU+(QD9p0~{e`odvxDmTwY>QUvs`%~M#ZJVlf<5zgSRT0hcR~2ppI`1a`!al+^)ePUyamu*&7POm(LYdLUW_3bySVF8?N*&!4S5kZ&?& z2{*HRq0^RUQxjqq>o#C0>JSM3kea?w$x@nq6)IW_5**XZD$qb1EL~U->c!LBA+_4Q6P{u*d}yCksH;@ zVaofXkDA{h>xK?m!m?5uM0^T7%vB&ojHPf?leao^td}*kUGpV}b$ZKlG1a|*)#7y$ zg;7XKKconW98$;H#?Nm z=Y&P24iuFqGH1#8g8r8(#{F`Bq2p^J#8FI!QjKR=ASzCpK7F55mK4M(J!0;ux$XXx zO>3YI0;fNd)L~&^8>jmig5(yq8;LR!`w{BmzG*X1BjR#8Q6JJy94U2b^B+Xg1Toj` zx1=?MR5D#?QBvDueW7VSQpBZ&lhB(n1n}ttDpPiP> z7WARzmkdew*;#WRm zO^^9GItzuE80$Zvw}`Bg-#8}Yz+G1ujYy#65f}!*_3$!;lI~?0!}!D#=b!JRU=;6VzqNqXV^anAO5F)LFo4m1iO>&~ zQBO@wOm0E?3#E|ln9wHutmC}%TTfZEqy~;ophYf~kMw++;JAhKCP+ci3@?N}gAQGI1)44~*9;A1gfUMY4kr_BC-1vG-9$*}D!O!gSL zILU8|m|G&qf2l$Qw&UpvfD*oV*EhoWD1@=p2N`*UvEG!mb`Tyt-c7JM zRaRtNZ9ox`4KdgUE`t*S`paFnUGvOSDRjIb1*Y-$Q(Fg_Q z5&Mo(F}biZ%0V54lnbFL>2tqd1MU}J$E;6Jr&&5gXRLGjbT6&3$iQ-o_UL$9)tix5&Gw#8DJ2Tqfyd+LK9zVQthdw^0u;q4L0p}hlFR;(F8L?`CTbxIVOOoef!^Eo?;guc z-8ypV^XLuzGA;6Kvi=6crQRH+v0v%@yx|9D2?R<04u-W5_PEX|Z(hLW8A$Oj=qTh= z>S0;0nt(gxEb@ZX4R(J_h%;bdgrUqS)aO5CPXy&qm5~UEUHAs>}>PH6NXXKU|zn!^x=;5qf}bv zWK^*XRBD_IXF8jGtI(;1@7@oWa4z+gsY6Wdh8aL+Kd2}pSvyF9dw7k5B(jQ@64Fl~ zrBHTK;xOGIMf4shD*o3Q$TCl!j|f@(Bio~#otKZq6@D$McGurxW9;{=8T7DdSq;)^ z+wa_38ORvP>*}KF384{aWsDqE!*r5%Vl`KIy{gRjQV>IHBzy`fjj#6nnHlRhg;1vp zZz-Ii|BAbdWmZKQ^-UqMF>Jc?-NnHLVpw3OK45pYW_8Dwp2<-wVG~@Xz7?)hl;p`2 zncSeENIwqywUSQ)%mq`O`Y^9p*zTkJYzfa|sHx8XAQ4t}FBs^nR;1Y3+7kd`>U=Y| z50fRA3Y8#2vn1grO`E1fEg^?w^O|I^mpmuo6QEuc7GwA0`7r>u$`N`+gvb;Mzx_gX zoXJ_gNyJGM`9+Nez5b(}RXI0SwP-Ck_!gya?Vsxz&-#~UVzDg)eW%O*pC@TbJ=#i- zQyz!_S8-x3OmI5ZPRQx`mp;7>y}Lk=frS%w4i%JPzSysX6!|;d=i*5c-8BlBP=p&L zRY(}gutK77Yr+mY2dsHgn*vY*BwG~OQez*Zu<8V*WPMT3y^Y#U_}+08X8{M^$J%sFc!1&9cuX{HBixFK3S z<3|Q|4?jIgoB;gDo-9GFF|)T${@b6jtw*VL&|~#2Z||OMU!)flSeKX{hOH6qZN(8}U%s>%pMeI;$Jor2Z@(O2`v81Pj%f~rW-HIS6%!4^qJg2dA z#JZQ1wr}&u%RX-v4QIjR%KOm^4p?m~v|JqQ66tX)Me{AVFwCUdEP69GOz_<$O=Tf$ zY4)_rpY@T7Qn{CySza$o_*fUBI;W#o{3E>FWycolDbcFz7 z)^D0f=+G5L*f}HgjqxZ+G7|<*94Y|F9yi`T>arW(lq{Yq+<-RRM)`%A>B3v#0SLcq z;pq4(EuiK;HqrQ$@bk0xc`s~a4L}hQ8~udJ_c*43|;02RYC3IY*gQWH6ZTdvSc9q^MV<{I`3mDiG-C3`*V z`tkU*K9`0JQ)_$(Li>NBGw_K`+SNmSSDJpj_GqbN{m}}Y25>sdXh={aL7h7@L-v+! z;_h+oBZB}A5|k!!beK|wg6j-rSlHO@{cuG6Kv*n6$S$N6vCs*dW$@6w&Mg>vk+1P< z-vrCYf{rm9J)_QCd9`S!~lv9WjP`Rp$C0 z7qg?dK1Kdj-Qq1%boRpP@iWDLZ?pFYSa2e+Oqw`xj@_DOd;iO%Bp++4IQW(O$VJdy z71-eS%d!M&LG+jk#no9w`*5-tA*R}SU@(w!N<^8W0fd1YVxxfunAZN6ks<#WgDk?% zr~~^sPe1XzBJbVNnu@mBg=qFl%TC6EdeW{sHvMN0*xVdLIos9ftHg$@=+QDcPv&2WXinEH;W=Nidp9EiM=F zikyK!vX9O%m#o4xj+qXhfc&)Ng@zPNzB{4yF%(wAY0#Gn|O@DJ1+q-ZDBm%XP zF;}UOjOxQX26j{Rq4!4|RT@v}d~7?}T&A!tgh)L=;pL9T!WQ!NzLRYe6V5qUwC-b0 zZq*@yH(B}Gp<-~%nz?eOlo&7==G%R}y@QA)v6RK>#4nepH2tXC4q7cbfD}#b@$_Zao2| zBbs%_b^1BOpTVt3m8@a5A8u%TVxJ*N3R8WeJ~^WqVn-M`-I3AQCRH2^7s0M z)Y=n>*Gqho5+2qqOW_&=ka=w`Iz!RA9Gs4=ZhbbjsnmNq6C7yFY33wuhxkK+0C`|h z;;;&{Ug!DKjsbHo<_TfRjt`&|))OtcWgu1{jewH`_M6eP=NPpVi(LX%&6U?C&t+<@ z4H_?pqAE=|>Z#_IN6ttFR@kZ>9r~X<$+=g01UhSV5SVako-oQ{3S#i(e8O8m@CizH zm6}FC^dE~AyM29aNZ20t+sQ|ud;3UPK*q3wIgJFd>YkP~N5;yOnOz=6P<8ZCh5pC@rp{4pE-oEG8vOzt9acdO8UEG8vv6Tl1``9cnu)OC15TlN)@r*pt zGNd*YQtey)JMBleSw2NQrLL0MOnq|mkX3A?=~az7(Q{eKPLUo84XcJ5cICt5D_;F< zbBw}bn1e)35K$a?1DPtujA>hadfP z+#Wk|gwajmedERrv4Vt~>jAB;z_=rN<2D|nHiD_D z;NHurB?@>BsZ?k!1?v1cwt05^tR5u@N?0SRWMcmVO#Jn(I4xQ|Rpw9+MKDEIaptYSW+RR?@ZdNp3VC4#{MZyR8$gQqv-C2arqa)9 z1mcWk-*GzZ*=M9wG0{nkbmf&@}ybDl8n~8bSt;|76i_dq-V zz=C8~PFCqT!9haJdlz#5eJS*b_|g(shS_VOu#OBGmW5kc2$_2Mzz|AEjd+H4fcdM# zCn33t14!zS&~;lc^|H0-B|43`Zv=Tvt?RqpeqTS!){~y$D^E8sxSd;F_0E5Jrlq9kkh;|ONT4!y zA+&b=B>x#d%VJQe0U>h&N(a}2^?)DkxvvoL|9+^royBueukES>pX-645P|Q>F3ttJ zr!AB?LEFZeldI?cSWfZ%lct}?puk&hM(P2zQWixfg;8sMKry3;VB$_n<%T^zNxHkl zDyu3hnN>Ydvk#G~-zUC*g3}gj5#AvYBT(DRCl?hZO ztsQ^=elHn}2HOe!9zgmE`JPSNLmwV(B_Dr-DCOgAbGgTr~2lW{gQPuuhr-lWcrp=35I6gUJkYT2HW);c5 z5&WpPikfHOMvNJ9U+&K1#;}TCR{@rzOKKS(KO5Ay(~$+y_eZIN;_j8oUut$lR@@4C zD((yZ{Yd^tL+X}W`|2X-tSXF@Zn~%{`dK-;KXD zUgLANc#1I5-X_3hp;#$EXVd)b;J<};6--keeu3PiAsN_HQ|FO@D-iDd1^W;}8y=BR z=~Gwp;^QaTP(q6L0GiYXKD?{=s6=YJL z3pZq~mi@XR7-Oe~lk8)E=VTadi_&EjlF1@T+gs`Qm7Oni+0rHj_{P(jm(IDU!X#fToJ%T^%`n`m}AOq*Zkok-a<|Zt%@N z-|t=ZTGSKwBUkwwd!>a_q$V*?f9~JJ2eIm6v~vOzeyk4fYZpbOleW?oFWTa)6Zg{w z%A3;}j~7|~{ey^VLOco!A^HgzCuQZ#i4*^HP4Mt2vKj?rYtdE2_mes&CktdfdHW-j zotK8$<#;^NS#X}&2J{nPUgmbbUFD_19kk0H8`aOa5OQ_ji}#I75AYWi9)^aR;9wdp zu$o$gZU|nP=#hH4%cX2qQ)+c6-(QX5p;p?3of`G;cN8rG5O884ijx`nO1$pX5ATYm zpXC4D)ygF?FZr9DRnhY_jj0M9?8W?C_anEYu}GSSQDIWhNyVxDesq?w4f5E|CmX?l)^KGLxl<)iB5W@DvFY(vtg2Z8k3s6?7l=r}i0iA7EN zB&FwhJavSa5QJxx7Cd-rrX#(?~4dyZ~7j*2{9>+NK`d zs5Ah;>&^Xri-@!uvngD}w3Jfjp2R{m6&hUs?0T zl>f&J^#7UuiT}Bc!=l@;`rXJ>mhkGM`1!1%Ns}WVo9%GA-a99N_)A7KXTXZu^d#W@CJAw3i-WJoid|71f&RkKa z;bVl{RQya6-kjy&J8^qp72ZX?)&a-wFr@Bt;ttMuD&-c$fPt9q)IpGnd-WZrO9PTX zIysoxnbITcYz?={rw?0c!ii|sZ@DvxTnyWFMH@IT6{xh6MkV(=Pul?NSAcECQ!$AO zVPd)X$nG*e&8G7GD*=j{IyDKb*({Bt%RUd=i{kK!cOO>=>e1;4Ro1a>=a=E?W|;l%9qZ;gB#n41{F+| zt5QFHTy1!Ph7vg_RGzSi%^3MSib2S2jvfB5tH8z|FMnja&BZ zsuBl)w~tTi-h;N!k;2BS zA?KYgMpGxE%`6uoT^cQ5tqN*qRPG_{xLwSn^M4$(f!CE)*4&6 zlDb*M`;5ZE=kq6+o4?)ghE+384R!Q?*n9J6tlzI~_(N1u2}Mb$FCrvLkujMmLm5+& zqzsi*sEkciC@z|b%tJ*{N=P9}N~vfv(||HmNSpW!_BKK5}Od!GRz9)JH$uv*-)ivpBIQJ{BHqbVFp(RgQy<>mGLLga~zCYo5= z??j;w8abSr&WZO!o~rTvb#$-Azdl5&K|tI@Mp3aAu>CA@rY5qnAr;2?`|kyLK+B8X zjK|rp`2^;4WzazYtjjRG!@lW1AEwms_bFsqC2^8I0k`W!?CQ{Xf#_r!auAKRAOUF* z7@rzel-*v1qLKr#^!Fk<>^80BA?*nK-Dhe$AV2+YG`y$Y7&y8ta4W8Y^e!x7>d?tp zYKtB(a2jyCB09ZbJI3jg&bWjZ1>6T8Mcq5>Ar>ew*|u@??$v92CT$pHfVE~1DsjRG zgeq16yO8t`N=oYVhJD+J947@N=;wX$%sDP>b2EK-u+;9fLnL*x-NJ3Tm12)XJhD$g zzX*sgK|0QNc!&LRo;vTSIvM-jJGSNEl%)?z+(JL+B5L2mKED`i~-^2@{kD^=A`4? zS3Xb`4N+65=wJIf&egPyT|Thyp*-QgA^Lw8EdK{rWEQT3RN!?)%FV+Zh{ub`5b~?X zx5Sh@VuXLF1)nXRK}sfgM1GlM7%_=PtV5d`3LC{w&x6HPKow`c^2cYvp`cQGjs&DP zp4V5AGDnmh_If})rNYxx;gASKP5w0C1x&%$c^cz>`V-sfesvI(*W>FQNLTELfKPKh zu2Aq0F~M5p2WI!Cg6SYkNu>~MFYSQw>se`x_9J*-90!;0KpU4vKPea>p-y9lA@!q^ zlG6v%xJ`{VoR@4j8saaRLx>7Ot}K=@=h&Db&4%ncYC30s`8#xWQ zpdk~~!y;J)LO32IPREX*k#*$jA(n``=`j@Hw0!lN;Dog(7atkrT2f<|0%h;1_2v3mKIMO#*au>xM z8|s#fcJCmY6%n3a@*XWD;1JygYjF+Au?v(eNFV~ijsJEB+U_p?cR&rf|QiMiB(P}r4poYSnd= zX6?koNFlxqh|8nU=cg-K6i7BjnQ2RECt~|Dbhl7jQM4p^si?%D+LH)eB0`BEhMD9_ z`;+EQIb?L0EiOYo21X?4uTWfQP@>uKFxKo4%QLK zcCz67AAvvauy+D4hc+`QND$xVeGzb{ZS#m^6YB6`|0*Ez`vb3n@F37|nEYzF=>;(M zR~z^5LpLc=-{J>yO0c$R7h#4%e!9aUkrBvo%HT5mwMwwQI!FTEL4{nK=Ed?lf{xfd zT99$-G<;|=k#~`F=LT%({*tkg5}G7KZ+$F#pG4oiWTbsohcbEifh*Hh$JP+wYL@;qw8US_AezfW{ZOA~SDTt5Bx<&2+d7 z+Q0dU$7qRDr?E0H?iZ|lw#x?F5{Q1~PtOWxMF;A5@|Jh?g&~H; z+65GaO&okP)D&WjfkMZ~SnH3bB_e^c!A9oSFOsV*ocMI4n3*ENa$b zX@sBk^y=Tj08RuP@!zPA@Aqap2#F>(x&EcmD1L!--c8fsPv0G!I1GpkSnL-fS1%DTf z12RJADTy+e6EPQqvOKAe6^SO5u8nNoWH~{UGsi7NOVacm6fRbdUof4V28Y1jn6VqN zmOX2WdJHPSXN|-p9wPS=&FX+XC;uA}Ez~A=AjtKGgv#+OP(&(0klQlFglEi1!IzPJ zI!2OeKT@R089_tH2vDPp(O_F&G8Mf@2;Fl6D~3jL9q*BBa-*6O)lY(?F>#hOri@C5 z?I^dY*^X@H)I-T0*MtFKGz*N~0J$BGWAgeIQl@}A$a)S5M~@NJY~?z~^^;yjJezgY z%|`buNj3{c?IC2xp~ftN;(!~pSo=`7Z^3&pi^e!Znzxm?ue){-)6W;~xu+iI36rt? z?}0RJOmdqoM3;~GBkajLWZyxfH`#YEmt2Ki_0pkAg(zBMBx%Y{y%OfxS$6wK%J+~J zkLD+R3J8GhhI-BW;p-_VSU2v^8~BKuEdkXaO%p`eotf~sXxsZBD0L_X4Vgx>OYXn4 zDhV0j*!EMGU*_?bDjD_{$RSyXL>#!Y7}$f_Cp*<&p(P1{RVY)%E*;Iszb^OB_PSvS zmsI|r(PDVF3w3O-?aYB7q&-x806~!2?audJ(3xd=j9xJOg`fDI;B)4~t=)Ya#N*N9pAs$pvV~SSA28_IFtIG;X{L!>$ep>3Z^-~p z&5We)f@3enL317<9`5U~>S3qEXc2d4W7cmJu1$A%yW#VO&gsWqEq@eR-n}Q@FK3N9 z#kT~Q(eS zxl@Z4+0;nM5;pE-f8>oNAC7;tI8SK;n3#O?w9rVFw&9gxw_pO|_9HQ~D`SGY#D%|8 z>?IQy>fJcF<^Q3%Y82oIBo2XacUqUm)jx>%KoAcT>@uCPsm{99fI{uZYa#;(8@5Ai z@cIq@n^*SBDw!FpKq$>p+vhm`;vIJr z(_`*;E-rY!#U^=W?^X9ZQ{v_wZ}oWh`GW3OJfLf;#ZLsU?h(@B5RKcS%TsFg5!mW6 zpjWUB_6ulUU}qKZdGn2_*ML7+zz*%pKtf@I{a&91s||0+L}h}D}SotU04;& zT>l~=f97CWovZi0oc$Nq7;X40&nuMn%D*N5N!d-+@jDOgu(LZP045*%j}EAI#Y-hN zl0D$2pI90kKYI@R-4?tMq66s`Nz|uyDVl|6y>1NaECo{)Fu(k1lE##umgKwc0g*1* zUn2Y(-1`~QQmCOA{)h8K@lp&VVus+E|EG*JcKE0D4+u{4E~q(?m^f}>$7^A=636Pvd3yzm-?xBhJ5%G0x{()!<`1s5IR8Pl_1qH9$8*zDibgdRk8~BOuN?Y zlq5CS(rC3d5WHxrZRQJlNWE0lm0*n4xU@E;uk1wDfqY+C>8n8O85)v{&s^13{a&Oqx}DkyE1zOM*g`p3TKd*JAzb~5Ts$mg9;1HmY` zxO?FA*b6X?30>J{$JnmlV;!B0ffIE19cRbh^#-1%qL!w2P-H=e+6BLsa2l0BmQpIw ztUct=jt$kL6N7IYH;>*g`-+0h4MF)aIiw`%rID$b+SjO~ngTi{GYcX2$Y6Ov*WnPB0~!o!}XHgzfg-Qfm|}Y z{na4pQfzC;LxiH+Oke--y8hDICBC+Qhr#+(&PJUJmF$1wN`uQUT9hewkSB&@cnsKsfS`R&yHIfCO7#uv`moaTlK3!ako6?m>jk zPSnefNn>NT2NTbYNT^AL6FgkDUZYf?VSQ4YO3vZ!&i86|9RC0rrX6JmEVaN5G#PO! zhG)>9sg0fHLxQR(SlW1)+XiJfk4M$jR`|2z&?Bec(RB6X=cX2VAhJ0$MTYNp z(hMO(pI-g3551p##x!ZJ1s*$r7cyrN<4L2e;*r!>OP2yp(0C~0U09!K2#VBqO=sAU)fKgM{qHeG zzNe$uckIW=@Y|r;iypvf1pg6!+kr-L3f5`Srq1(s?!N$wL+a0y#ba327^{HcNFw!s z7NvQ5G9m8lZh7c1(2V7$1O-6H2akLg=@h$#}kr5HxSrxPp=!i zyxDNc5%G~F#)ko3RQ^)`vE(fD#*(Ie0p}18jeHo3ht$!5LWhQGxM#WkEdS3N4=BN) z%Kb!mkeiZbR9->9%ojXh93-cgGcegS388&Cy)E)uQy*R9TcWo?q016ujfb_4t+&|4 z-3l0I4KSn(P)#qz+J)!_f>5|>cK9<$Nnri_Qm@Q+d7Hr( zkwR@PH(JI=RR@tnw(nv_P(D6}DUN!KQgH(xL4nZlR`}<%8a5qQ*o8fi1~5hg7Gfl+ zv?ZmB$Sd}+RYKYT{j0HcJ^F7n2}4(3Po3SAgh|Saeq30-7S#bwQ6mD7ybv_&!M2_Z zRWxdm9wXZ}?Qqm#5ShIb3WI5M=EhMVS(?P0sY3Q%Vre94FQOy<6vbU!z$5CWz`wOhmtT!MLh}jzf|HD=ert+oX-l308}1dBvb9MTNl#c)%Q$)fhK@CG{!1+EVO zgr`9S(QW8Gc!={hs6&eS4Jb)rNyLE8S%$t@Q_?>GrR1o6(t4DzB#qYvoZm>R3zU`V zrRefHEFLnh5hOahe2+V&dYZQk!iG3yTJCfu)H;P;;9cldXe;AYGy`u#?IC1>WKUDm zM*gLDKt4M{nN)RoC5XH@M`SC_Q_9bUhs@oDdWNiJ<{FclM<#iiu+ay%Xgd)d(fDF^ zOCT{t{_$>qr|8UO>BQopQ^y=NDdc~nZW=t-{_GuiZ`6ZMkr`#Uw|7E$HM*Y23BVrH z2lqsvz zA6JUs?T5t=VK8WbHvx3SQta~Xvh7oZGC@=K#Nx%$Q_JL>*cDshT1t z8l6Pc_#gdvsO35Wb|I+SEL{y6FHgcQD>kDhR&H8}FvL2vjoz=_gIOq} zT?fFp)S-kIzwiO3Q}!5zF9nPp0~Xqeq)74gNQb|qONMH8bf~|=Iu(U?`0&;#adv+l zQ9|hjsLiR3gLebz-=^w`hg=_{2wDtd4WWMHK3JgKz)9Kz1R|qcyntR)GVM~M9QDPi zwVqhqhd$uU1*~gS;saEyA_fDl#SSzv)KZ%~hFQPp*mh5YM#WzM@u{uqgzQD_NAT{y z;JY#e%cn!wOXf@zRyf*4F$2{4Plbz1f*?QRkU*T$7?j)0;zM4Dfm2nRf+g9D1Mz^G zDjs6(6GB;vUGEh*nB~-tSa&tE1VEE`kv38$i0meV3WcYf3pnMlR5)Lr_rKJw4sYK- z7wr8^-#R@W(2Shn)I$@vyM|h`5Q2h=eW0Zdn{S^n^>|s!Uds>M0h8|GlYmASDDN7$ zHF>#jJu7Z{{azTZw=I^dOZ;IAzpp8O???8nWW~L<(52pB@FnDPrs&s)%8axTS`d^S z#B=T;8Q$0dw#@!DZ{9eFObcW7Oi>pXQ?uH0q@mJH^K93uOu_0?E{dmGVJ1BSYc%o2!+A zB@%AU>qHel-MM5wcPL3{^zam9lrl_TH`^FX4n8_crYH1bCf2VywCL(hT zb9g=jihm>I8z%X0vigSS3qi@&zbtnW&l__6eQ^uARaRAE5Jr!!P zh;Bpva{4zTgzJ{HL&&_+#z?nA2R+Dw(>4}5p4Y11g`xr`Gd`V4V;|_}n&di+OvNF| z_Uu7n;3W|5DNVx}L3X`=w|ShkC?j|)>5puP29|994n)-&NgC*jH1_G=%_+))CR@=R zX9M1v2$zpIszRfm667DVA=|!2Uk~%hSqt50iZaI0gav3xvqy5UN5$id(tyUvrilUi ze}00KFF|WmcOSiMf01f7#)UiWgd+=l^O0|uRr8h?a*xU(eXvfZ_f)5_n^4x6)Xlkol%VbbSbZ(55E&^4M#O=^P&ihen&DhHg8r&^gUrr2VRVu=p%Sa*C-!H*kCMpJ%l5 z^MB+zE`SrA{{6>jz8y>F&vDTI@qd@!OX9*ITL1o2$0H>5@1I1cd#%1E6Y}pLg+K2O z)#v$-kNNY5WI=h8|2U@j??+zTXvs1N(W0lmOa6YtU1v*O5&8AErvsbn zIyzp+^8!zEW~rA5u-Z`^6#rRK^zWB{wBEF79!_4&#kA9iZXB^jXTg2%{lF(DC#RW0 zC+^{=PEo74`JdJC8sRXe&Rw{0DhSUCVC8GjXx3(~<>BDq*jk@C11A9v&WZW^Oa{oH zXm@k_95XYsML0wZN07}y1!7oyU}z`KRRfMT{lNd{Jn683BXUv)9>CrwF|X^@RdMtV zCwun3%$c3JHiOg2$qDBwilTtKF1AMC&sCq&($m*>#^*EduV$j{(l8|lYm;t^e>)DnqTaEyAb1>*VPK@&V`*4%p z01)UPLR8L|El)B`dv)x=J9pHXCv*P1{%5C;>+@vX+BdjBKtRAE)oj*udAmjS`%#5( zFgTPqBy96ujX%$9Ufp%f!{uN^mU^b=Yi{dl!fCQ+otV56t!i3bn#xVX9Jn#XHoh+V~*XAU<%;AlGo1B0ErcmGN*niq-#8cUTD z_}6BdFMr_{D-&vXwe?imC6~A=sd;`Qef*wZCg)j5OQ=e4NZYt?K4ru(9Mc5Pg(%{68+uM6*yOn!S-I<+cZ6n&rRPNh{KTfIK*XUZF8$Y8`tu!~@#@_qFIML7BV)56n zLlwgqV&J@a^Py|kuGI?n@M2b*dG`7{LJ-2sFnByBf8^)SOL&^;I7#mc4h?wL>fIAu z5@i+hJV-oX_5b7R99;jWZ~w1<4S3@HJHuzfk%WOyYL0$>s!VBFnXbp3kPr^DRjc^L z#BdPuwJ0H}WW5xtdk-JV+*i}qSdFq$dr0-R#nOag!L9p#ew~X|x6!SCTZ~%5`HK<7 zg2|Vz6_49mo#~>P72y-?qPafrwdw@bm(LHnXwJd^lh2A6%wGP4BXaP@!^~FECOh*i zd9^1YwaPCePMp}dWy|-E6NYsIJbNUzjid<$?Cg=~|MtUj`L`WA&TYY_Pf`B`1ZW{XRI@r8B_pKy4C-8)TCa2#_jFR%KB@&f5^zT0` zUV_`vAL?sAg@d|_0!Kw~@Spk1Vh{w34uZ~Nb&iw@UTmIFJu1>4u)$G25`#bY#cD2Zk;8B zQ{Q(&HDgLakQ@g)`B6>kUqSH^y!uaZiYktMoqfuAT)y$8B1UC)Py^FWlE4_o(~ii(PTW`_UzZNwWTXlcU5#l_W2 z*R5Lz)H?-5;!79hzc)9!$x?SY&g>t+Qw?+Pf2)WmC zb5|ffId6Tazu-??>3UT0zervA#pC`UaC9TRf4;;?VPw($eE`;$SK4Q5#)d4`WZeAh+z?wW|z+W^;je0t_XS zMBRV?8k(uPQv$L3Dc}c+%kl8s>FerpLUPLJ_0vnJ!Q1o3k>_o-3F7F@dw_GOEpbLv zK;^vQY#da65Sf1gHxE#Y{uRfy6l{W&z0 zhUQ4 z2K^Tm5I|0zITME9{8?T4nVS;Mh6tO8UbskzL7&y9o>~oGjUcE+-9+8PJAZz4F4{LZ z3EWTxY@Uox;lT?sX}i3=IT(l=de`0ghf>e zBG#fLFu<|Mj~|<^47!36CRs2S)E~+Nh!2-SaBA-DVJqGu#w`c^vb?MS*ozN0fD?}nr=|10tD}GDHB{jk ztK5K*y=Ts!Zv__G`u3I*Q~L1XMB4Tc_q==Fs%fT4*5wFDnMmN+=39F{3r!VOEiL1! zwgg^Q3i^G3DTOPyxveevMnhvGC$qphlMAEs+0-J^y1cJt<)qCTD z*hv^Krk`RGOHq-25znUH#VW}hElINu9X;9#c2pM;U`Zp)I|mVq60uR}#9vJM8~Z3L z^(<`&o2j|8^C&=RE1pqxtFfn-SFH6-RbL&o^>tIG;0@q#<1KiqG9d17$d=X=A0HoJ z`1SVI)BVQoY)G+6L|+YUTP(m6hYoRRrq(o|mKq4*6<)|N7?M6!P*BiyKd>~qOu0=> zO>vC$RCp;?;s|OBoXCzOw#LajTv=XT-ps!TRxQo1fwL~V*JmbzYkhso`=5gC?~dWg z^FMj=#18{Vi(3Rz)}dAj938Z!S@outR~*$M_U1cGv9-0uJ^zEx5$6wAj<|k(@soH> zUQ^RY(|HAvC&DR;tI1jpQ|iYPeHbfeo5yE@g)pIMwEz9Z3l}Ifziw>YbT5>9pOJ~4 z+N!#@E3i~m3ZP2Nsf?Vn5*cNZ@WK-iyzak$F9tJ9>tzYqb^AuXD=;+ZY0S=k{%P$LErgk{g@L4g>z`U3c+v{jL4n0E*ypxAlli7jHGrp+a0^(_@kn!(a7%FR-@hMc z$n!!;?Kqmx>VEzDwanpV#^(+%k zh1W@`s?NmfNMDmS6?1IXfXHuQR~TUi^w#q42vuqg$tVdg%~_`LZ2`) ztpGsqU$wHqq%MC1%Vpa1>0C@$cz8%!+I*bp9)dDG5r(UoI5EEZv|?0eZ9*&jAkC95 z-B7f4DXlL|9Pa#3pgjc{5y&*9)^dP_tAs3 zdS`3jyjjOQgbK#;N=!^dPMXKCt|Ak6DM?vY+`4mzm#Mm_eiMno9#J>-#77KQGu71% zt@Fhj=G4K)m(tSO5S0-wbLUOI$LYR1sgNk^QdiAZQc~h)ta2Pw{4vp_?gudUCgf6P zGvNC^vsJ9X=H}*2+qW-dHh*g=$D!j0DZI=q)t#3rK0L=#lM9OZ2ZX?kvoi0HnE%23 z8|MPWf5Hdmo0;h)i=${3L{N$NVS$*-!U@}QuQY{(gmw)NY|~7YG%2}uZIP@^_8I0R za_1u?8#!dl8O$cc0K8}0VZdq=6d2sWkWFo(GJK@3uymZVlT$uxug$ho5>?+j850{V z;x*3?cs2s1-LHp%%46HMaEz}NV-n;H8lEF3G7L8QeYhp45;$}2Tnki!(1?hLy4tEL z9yCL8FphWju2QrQS5CsY@mTb$k9Fg*Fbpb{h;9Ism9XyF53W)(z-o2|jC=cyDJqFX z7kEtH2iy!AZ{xLE_*3k0)%aOb0Htv%q|w6;ITgsfmbZ2~Gdf7Sivossr`M1Di1UTo z#JtxYH-eXRR`^0#UjEOYKR3T=Ebuyx{aS|bgWoR0f%25EC};tER`X@?0#f5g7XX6dySKq>vBR~q3-M_CQX1gAoM?|5@gb5QceZ@fhTUedE zva*%S;n{eTxYA1ywEc$qq;O(M(t)03IMNF1Pc8seG^=)%U-$2!T$zw!BVb59psM`U z1~wU1+}J*bSXU^prU?lhIW|LXfp~SG%*Td(0E@RH%~;OhK|&G|obpmS8gK51LUb7C zr|nOB5|Q84c!ZC8{CLh_gIrEa8v$`~KIBf^vvSRdWWcq-W<7+P=ZkRCg}i+pmm9>J zFKXdhHxV9PP%d`8yR)pLHX-Wa!-ty#2m>H5MJU}pFtn?ETqDX((}pN zr9T6}JagvECDbz_hl3bAqtsvPn(+^;vHxxbjcMtwT=~4nTu<*%MbQd8M|d*AfV!OF zB}I9Babc1`fDLXuqfh6cv)Z-`?V1y?V9Z=&(wpnC3AcX-T;audloDQHpnQ zaY1#moH)V!a#|5;l2V2+W{HG< zv2*_LbSe+<29Ai)BSwbsqf$@vDM^!|$%)2Yq?#hU+H;WyUZwAr(d1C=BKNuLc^pqH`l z_b)Z9jfvR!bfJVGVV-L0?jC&>0Hz4MC-!xPhw1#6tP_<45YV+{$K#j4$zJ#xH>g36jqjL@kqISkraWr)XE>G<3G@RyI zfY~k+ncTcQ0mdrXh~NKfQ^?C=5D1$*Jmi=*1ag}KZ5;2qz$ROUdC<^swl+CM-VOC& z_4VQ}-QDF44fd0#Ah&{}wLqzxJrICA#!__J@KJl~Q;G11h*;FLXg!ZWyyRhMuMYSb z=vC?K=^5b$xq`7nGOSXLu@}A{E+6Dd|a%)$M!Xz29mqBF3|LV}PM2 zpN=OslM?{mtBGLw7bhFSiixj?aScVwBp_x)I>4;z>S-WX#9 z*Mr?pL;ELyGhD&bWh``lkN$9S9YIOK@~GhCo8qRU!v>nb&6BeG2p?}ZLJe(i_ymhLyR7W#;zR7xQJg#7J1lYg9iBGI> zrF;hklrnW&wNWBa+>}>PD90}^u)QJ=3qn;sj7Wih{M zSXEtJ&iBN1iIBXQFAKo&CL}FNVYlY|!VopM$qm^5+rv{tlL3#OMx;gTDbLwEQGwbB|+MP&x7ny&^sk#Y_t)kcIxR_$(vy^&Fu>P4uDg#MYXiIr{oT3 zTH5se{N}u!ySR1cTIUsH6m@uPbhk5Ws;&5)_v4+NpUw8dK(0-jH%sCRgi#b(X4SWR`m_LjO~R*sMEU;O>B^myy)a{uAHclG8)TU) z*$1km^nSzp`0!1Xl&>!HN!dUakphfL!-E@-I@7{f*Z)}yBS#&=7Re6|eyFXg7 z}eU`%9b zU7diO?Uh4mX>O@)(!*4Nu#o|B0yj9(*Vi}OsYHwc zS2y~YZxw3=1_>~i9HYje#cRUTJXcdHTgs0M8zPD{H#G$Vbm2Ts5xlH%%yd~RE?7S+ zkV~K5+~H*B;P9zkG)!i{kGJk0Xr-s;p*z#(E)|*ZpO7 zIP7ek3@nn|+S=MFQL(Wzh=WITZ30kEhXCEdHGf#o*yT6U7D@!`0@!>xK+vslbQFc~ zM=T}kw-y+hcJA3zj+FtGCmaR(CuHs0+s>=cT)bs68W%%=vtMLd^Wr4m;QV|Q$g7cX zB1L})rY<80S;*yAtev3k&eNujmcY7lI3^&~J$ zJk{4j``fB6@8O=4l>G(1uY|tzcr0}aQUL#s%tN$bv(#te&{WDhoH~e*DV;C$STT>! zFEht3lf-6g_}+Ys$Jhxey57YkX~6WH@S3c8kw^Qq5B#(s$^P5Js|i8B?HMC*qXVos znN|t}91>HfP8I1L!__JvLWJK%eGiPLA)1VJVb(06_ zGvxztIAJCPirV;`$_Gvhg`}Pe%L39puBnO9JU)aqQZ2q^+_-TI^NkwnKxf8!W4gHJ zw}FAR3w3j{vsrHQP;nZe&g2gM^6rjpZAN(ARRqK8qt?KtbUM2cWQ5pC6-iBA<&5Ba zuV1S+^v?~?yNtMIg0d#|!ysf%;%iVoRmG)8MMm1h|KtDbr?;k*w8{FNJFj8flP$RG zv|LSG72xt84hkWMi!i8GgA)Bm`7BetfkMgA?P_rPSA2)xctOxnQjm zRYy$f>FKE}^l?*Kw#+s@O?1W#sd@9TQP+r}sp6fwW}37~0*Xjkha2h4Nz??#)6&u~ zhRI^>f>>snqM~BT4l53dSp#RzboCdavCqnLoQ~}B&v+(z#l{K8XNriJfUOd%69{;V z&6t86hgXrEI|^`>WRt}mEWe@1&gSX{1!gl;nW{)7P$?V|NXCznUkuFEDh}C|OlaM} zAocZjl%aTgEr^8G6w@x@`k7$7K}d}pD0<5grS`<1mbr}W1z?hvn*&wr3<$Ru?kWJU zRI;`rZBp?CVaPIXvoWTs<{W`2!$6W!eSTdqAQK;xh?*iSBBC5R;Wa1&DE<>skfRP_ zs0?KQm&Skny<=rvoughuNLUk|^R3+<1wgsPb_2Sz+L%5FizR=dC78pB zjALkIUh>J4({Q*q7r4k$Y&4Zd?2bjCT9{INBg^$NIOy{&-=YJ52=b@sZ{D0-@!|DY zrd7%$B(M}E-)r8j2B7Y_OGgUX6*?AG*s)`E{u`1`U^2)Y_*%eHK^s9*SaINh2K4F> z;MaK?8ltpOs0zme(fW6ivzTrO41z3>r4;FFR-W>1_v`2j+MH#Bb4uNyHr4r8`& z*Dc$Y?Gt$-?>ycENi1+W)VnUX6ZVdy4mgcANWKP$F;{q~f0_|0D)HwN2rLEU%iyimMzN{bM-3=rPGHQgKcH)-BS34QhG;<+a8aFb9d7L>ky zD~ezd3N_*7Ri`L8a+p#$jl5B9;Sf81x&}y%WQPFTXsLE2tOyMMB#VA@=U&%edM@$@23vB$~knFu`CJiP5`k zjwPUk8kezr%A3Vv0C(ZMzEfdshg7>YZ}#nmGjB(0(-0KfdI%MIEp($4~Y9y}!o zDig30FxzY`SUt+`(2>FDAJJNUSVkXw1dg^-Mu7;8l$)2A7rEJ?Fm;KFN;!rCMc_tV z(9unTZF6huLAt1l3Iny(btAF>KW!K@qS&d+I)|@G-nYgc&~hhc;Sm3V_I_Sa;E@O% z7;nM@3}I4|oclygxq4yr6~jr>Wr9f`LN?xsR1Xf6rRAP40<3@Z?p;mr@*MdN5%ctq zNvgj@?BF2GbHt%0YKp9f@dBfXlO}O8@{YF_+2^)HPOt>$kB~r;4{qMv=76Y}*p=x^ zcJ}C)58SaB_pB30^RoP69NGzxPz^#uWw^KDx*_urs8tFw9@5jz?Ft~BkQczHVy_T6 z@TqlonAH;JGOnCf(O@KYL(qNCcVEDTEU2B;x$82k{?L&lCg8sxU?-qT95t@|g!5OK zlL+SrQg`d!9&C<&ie!uI3qk`38sR@Cdkmk1;0z_fWd!JmWq3|fQ}B#T*2Jk(VC2I5Z}`?@T;trQb8ASuR}+)G+;n zd40O&N(=(J^ehMvH5&Y1!!_rF*t-)M_|G%z0+IH~MIT$pm6R<3VF!IA=UVJ~k>1XA za^R@32^e;^i z@35cji?vb=Pf8Asp@Uzg!2H}2WB=Xv@iG_xqh)z+J*EfvbM1l5MFYM#b4{gmCtijE$Ou!D&Mj#R)_@DYsJ58iC7wGd+r5 zTwD=%o@FJoA7z`F#<--Y<55wkQEABeOqiu$297_aVuCe*(Q+fx0slQ+PggKu`T6;t z?~CjaH`?$aw3s*~z;Brp)MA;&4jn-o^9uAqP#3B76(6$jP{eHr;uSz4+YaDnx&6rt zY3R7O&c_iKdI)N%W-mZjs@o@^7^;S4DCgIHaR0tr9!Q&eU2?=R? za&aA!(3zgHLtv`Ww;4xuL4z;YK5(!DAF-9Ce0$y|S!HYnyb|#;77s&zleWW;^*D-A z`QC+r6swFj6#K?|z4VgJL5xHb!^J#=bZ_-`3=g2Lz-wKK>fCMZTswC3J`OY2YXJ4r z%@yqD?=R)SJ8|Mk$jiJ8m>vk_SHi-;zlZfD3iArcsDrMt+^+le+c)EvIK;!dy&@iQ z+sM!Fmcf2(^~VX?PK8yKV6-5#$J&1juk(cOpgMnJ>;mh$^=GC{nTS>zT&CR_n5tS%L%Bg;F3&}Q4V07(SCb$mU>C?axh(~8irhlhEESsrJQ)YeAL5E7C`4Q}4=k`)%FrzbjR4!6#+ zAf%{&Vm&O8$=fZJurNfHsZMis#jEFMqLE0~j!3?v=D;_byV3BPmR|2poBM+1ew)*9 zGql~e_PKlau9S?7%)7%p-rn9+Q$d*(f{NYlt7T~uck-l!lxf^_CJ&Y5=5Dac{6Htj zfyc0=4Ky?~+JNXp0$_evfIJZmy|fB-(a-o}c^5vvy6T}|WE4H1S^(%9G+>9l6^$Te zVqtOeN{iLo9 zNg$Pps_o(eb?Gp2=+L1{SU@Bw;kkta>U!KL2^bLqvs`uJmflApVd0~oqgw!c>33ko zJq0j9Vk$1-NN%nYh}B4ZK)`?<>{O&BN+(MXYfzyK8lbecmJcO<^>_So67+32{)`H0 z(c*g)!N7HkS<+-vBO@IE!=2w+^hvuVm_xi>E3Wr7w(i?^?*>mshwaM4Zf*e|!}bC| zqs|EcQ3t!(F+3fz!T&^`m$DihF8;8`!?ZCOA`xyaS@p-Bp+47Jw{Oo|zFY)^VaU;= zM-9|Jm|NO(zy?Fk4)7_USGc|g7H44vl9LpPj8j}(9BMuC0Hgu{hO)Xk<;9x>1M-9w zH(UY^Mx8$ioViM)O}1SK;@jK?f+VJK@+&aUS6NQ_*?j-OPcujwgklOnI1UA(NH-jd zGJqloU|&I;y>4juC@Y(C+TjC7nAXu`J~yng6A)ackHCmF5h&KLuZ<^8(J9{kTbS1Q z$`cI(!;AP|xD4JLc=7AlR{arS3Vo#KbTC>ROm9wedCk!9v&>W6WGB;CZ7 z6HxnfUaoB65^-HN@}svBe^r?zKu$*R;pDo+;(P#!m4Di_U?g_J(Ri|zc=H2i&dGG6 z(zu3Em0*7yJXUZrI2(5w16jGq%*;T~>DY~;1m$JRDglfMoP$$n#e+RNBdY7V%-Zy6 zCZ`uh0QVpuN|y)XjFxiX8V&~T979k5f8Rw@6X{}T88dB&n{O^gs}sE7J#@)~O&F(f z;OC`~gc*?vT3#QQ!#DNZRYGz@=na5fBtai5zKWW;E~-sNND4eT3IGSu?b=aA41# zK5H4Sl?9rz4E(pt8;t4t+EUJYtFRC`mY1BF@X~}WTz|U>DT4?3e6)<+H3@3O0AK93NqOF|0KEdJE3a?v5Ql-1wW$Oy|4Rq}Olhou%sN;CW-|0~ zhf7L=j*S8zMgczG`|v>=v|?>GjEee*Jr+fN-Vt8na%NO);;O9fW<dY?;y^cK}0j}6&o%# zZq}?^?m7XCq$>0W!`=;ym7*E7vA{EQ2##&OGF8WE*bZ8-$&3fO152a2rWSOx2M3*_yqgsD|(bKArowEydjI{oAOL+ z(}RgP?0PCd`%j>KRF0PiRS#(E+uQ5uabaLy2r|!Q5H?kl1VP7OtsjC(tp(jrJAHia zW3S#2`;6aU&?(_DY2rj3Jn4;YZWP%sLoc7HzW0makLNaLp5ET$@#X#9*&td>LPRWW zX3Uz!gWQZ6e0<0X_b4NsJc&K8uBPUU>F8AjmF4BlU{M+HgZ68%5fCO1QG@^kLFQPi zqAf#Llj{Y@mAl>`xz;!=F9djmP&)JUCmNWdi!dtjaxV)1*zX7dkB}Hy1*a9AOu+4M z2gB076i$M!%Q%54OQ`~8#5R=uRl|o-JK?o+2gCXC5CMb4;B7y?e-}bJ%)YM>|bFdy;8&qbaB$bhS(S_vE*>S~k9DlA3dtyrbc;4DFQ&078h z24^(TkOKJ@$ZT<%r^p~^k?XmC;{Nj&yPj2dwk5CC5`+aji@f$`W78OLu1%k=FgQvLqn5^tq8;`+l3rH zXa4-*-Jl#fbx6NM&$fWD@ae=>3`)>t!VVr>c_eB@vK$cL8m+4%U$fA`g2fyY)gum} zQUBQVW|jR*%aXXN`+zI|Az;Q>fo{P-j?fdLYF6kS;R>di8x2~(ktEf$-@k&%U&nwJ z@F|9j5n`7VDil7ql>^>jR_o9t;=OaH5yV;8pfJVs(y!cwFs+ep4A4b7Gh+nSYH;Ux zy$p|MNr#z}L@1z)*VAhQOR>q>c`jPfM7Wn`7@yN|f&K+xQDkvt8+eA(&8=TPlsF-e za0i2P`i2sEcsD98ljh`%j0_mNZSk_CVUMlu{oU2o6(!b*gvzM{uX{r3Z#_Nqe;@68 zpAIZT?*|LxCa@}p1U6=UmpCMPC?HSI{6hXu!>(I>)*LV4X>HZCfCnH}xPmb;ZWZ{# zSa;{pgoFe#P0acXKix^H;fCI1qY45B-lxi&#Ny`WHb%k09SpSg2+L1jZ!^4IY??eS zwUz7#D50nrFtKVro05_OLzfbUBeL~WaM3m>$IVPmA0M3+X zC~`(fjBJD7(p8nl=fp9`CO=Z}d1onJU>&N~`9O)BIv56*sWcbPkyT1dmat$C*!J!R z*s$u^*?c9)Uy6!O#gCI0OPFY5pOYsOe%zoNi&p4uhI2zCw$)hpJeV=l{-byF&bopX zP5l!~;4B^qKZi6&u=>q{+ow9#13nH8o!or9V{PD|?jf+eQyHLhxkq*#f;w zLRLNvKV8kEN2W{|f&iHV8sf1ZQI!ibyAV>A%*jINMNQL=i@bfzR8Q>zwtQ^a7ck2n zHx!^))^i?2P?qhVDt@J`9&>G{k4X!9z zmu3RbDQW(Yh=?gfV7EQw2R4=ULb*a-L{yZCRvAsby-~21v_SSf1P!sm8^yv=%)aJE zcvjsvAVVpyAx)Px7s2)(f~Ys_yti)Mf({Jj()s?8W{gOi#!nh97N9P0TC5i0!+ijG zw02GfV+GGCId7lcx1W~ci8n(gO~_N;$d{V|V$pX*y%#{K?o%WwIR5^LCk7H zuSMPR-!y~l+v7@LPa{-*Z7@g-hnWvpvOPn}uot3f@|CLWyKgJ?So~;egs&P8YP#)G zv=)ZKj<+BthX3PBv^&g&MFia#V|O;`LZ2rJ2_Y;YReef$-ZA*V+BKg65wg0mqv|kz zH>dX^w2I_hlha{F2M!bw3G6sGB=2JK3~y~m=k4IUs%-8U4-dm_#@(U17pxFw49-*hc*(s zaD0kB8pqqac_3LDz@m&^)r0Yx`^!FjaB5YBS3{8fv>{b7!TA0KZ!jX{2#mXzk(N~b z`y|o6dCm2a(BHlrs^IE;;X3_^0Ah*0LbKY{EFoZYUF4=Uj^zhm_2?z@B7>287ZXk< zBDd!Rz&ts0=jbj`IB(EgK^6xl80_Ijzz5n|)UH9DLhx(@u~nc0Z16&yO!uzNClR0^ z#xaD4LHoA=*KK{CItAt6cvvG&h--^sL6LT2i5?AH3n%#{L22z0MZ$wzc}7msqM-jn z;X$xZgs`B$>4FLNM%zF7zMa7nFx0is1lEvY&@Tdi4lkD45gY`up-(Wy00FwHCU79IGx6Ws1Bl%z}pkXi!W{VwxNr0XRx;*63}jMz6eN#VmE1#KY!@3)=SmQ$+pYo1cWU|r2Y!!8Y2pLv8{FS4^DxLAlVN(Pe|BC zLO!tN?j^YjL>(TtiO&S&MKVXNzJ6*uScJTy7Yl}?rTO4LjrBq-qACn;a{?H*i=F?} zabv+z&)mEuPj%cYtZ=bo@2@$J!|oX1kCM5MmXYHmd?J>2eJIf;atTEim0?xKo`Qp6 zsf(asOo_Usmtc*!NEUHM2QZVe8&iq`)7jlU^Vza(Evg_Q*DMwY$V^rjx$$go_tYXM z?qsY2V$pZciU`}a2y5*1HJdaC=nIZwuZ;ru%$e_);eKJ{Y z!2KSE3IurEg15FvS-A}DYk&OjwjVF5gMZLR!xghd~Q)<~(!LNSX}K&kBfTcuwOOyW#_Ch&{ag_s~fjYp8}=m;p8QjAD?|;>DD| za-oY*Q!I9~0n$&jLRBnQnB!)fbc+?@mf>{@D17R_Q*pv3CwUCxrj4E%+kmCZD}aZ}9cx03ZWr67k`&jGAHO#qdHx;*XE36u zr_J*(g~-U^k}Z!3fGKJQF#x1;udIv{{J>hVc8Z4)=f%SE4g$nFBM_fa_gx6u4*W+5 zvh4iNKXA~m#{Tbf0lcA>oWjhqqv|*M-kj9PUid?)y%w#f2nEa>k1| z2rfHbP{8#BWN@Ylu1^te95>o{58rf8T0TmbbCKN@Lu65%i1!%N#)c}*e_o?n#N6LxI9 z8JKfg=DyWAQdd^My(2LX6T4=i&`iNJ zkZnSEi&%B_ZPpYX%cU=olPxpWlD!C| zyD;-Vnmf<1sLpMVZ;CC(D8{IuNYRK$jA*ceIO+rtQ3N|GqTq=FHqc-+BN!Vt6a^~= zL{N!hnFz7OiVZx7kOYDldlZm}3Rs4Aerqu2(>>2~pL;*t%LkK~VP?DR_x3qei@nqc6hAJpSw&Bqth?aB}VAT(l5>zK$ld*ji z7ta!8L21SMU^-~YiNjORjHhC;RssNGQ=Xp>EB7_91X{_E?l{n!NPo+>|Yo7Zw6lMz|HT3B=>N9u9 zFYe&boh@(14?gd-tY+f9UTdADuFtxIk)^=38gybR`*08Ca#og{n)Epp`ZhqJZsi9A z25pwPHt#v-MKq9*@L0W(KukL3w_H01@FeB=V1|Q%C56ZMx@?}ub>W>$K6IOa;I+_) zUJ7Y(1cKSlE!x|wPPx_28)E1+>J?dg-TiAbdQv(FOMdQ=iAK(DxSW^g4bTE0f8dn8 z6_ph-njVyKX8zNXy)#*LCM|ZEbQ2Xb>g*mqW9Q!eIVG4z_mrOYPdZ&s*s4%z4hsXAu=6vNlW zv+wJXBS#LH=Q3#&fI2mIQ@u+7=D}_b%?y+ii`v*txnyf>JcVlbamVf^&Z9SIk5X7Y z1l>?Bxb__#1@UZzI+FuVJ0gp{71q&uj}H=cv5F(NR#cQS;v>U(Ix>e?faheXUjV>- z0r@J&Zi^iT#75MOXkNqt6!4dQLRqm6b;^Qq~1X{$p!^nM} z0PN#l#Nd+@5#shgh8P>7QYox$+saOVi<+yr@JAcvMRW^w%xriQ7QBNs{VWPq1F~_q z3#cdlF64Q{JtTbr@78|M5{I^*O$0w{&%jOuwcODss4e*VLEQN%>o#q2rJr8#^rpGA zeLO62N6VW0=bx)M`sUlp_6(B@){U3+#SlO;#Q{yjyP{1$Lj|75^b#2uS}wo2bB~<( z3qcQ?J?oVhD+V{>#PpFW*DPI zUe1mosJ;Ts-U}d=s`<;3rxBmn^o$aU7ejffmp$W05knLG0rie=Wry+@Y-IS zMzXQxnMkdc3Rj%0Luk9igGANDyzdK>sh%_yBb=U`Yak4>UPFkHn;@(v=bO3{kZ+|kX@_= zd#M{>x^qvipN_ecWwMEq5Rkcr+`o?z!>pkGl>Djp1ZQj=<64czv3cvrZe!s3jq&bY zq>H-P@CnU9#6>pDx-m}w31*N>qL>97_D)~hRH9>OtChv)4VN(K#jB6jQi7KHV__p5 zWN)Pf_ZNUE>${a5A-nYfECXL0zqE2qON&b=VK+5fn`hIvZ(HFO!pr$xEL)-v7P>@+ zF(XTfkWHQ@in3zD^9VRL`56h@B5psyELhA--p~!uH2R|em(g=xMDnhnNmMSNV;P<7{>g`C4sJQNtb`fE z8z~2Qk~pOzS+HkN5V%9%;bCs1v?5|7AtAvdyCE(sX6oX_i~Se?FpE0|gmz`(A34!u z%E?wJ&>mnHPpt`iF1kq}Fafj=NlF&vTO);?&gPd_(RJ-h9Th4T6R%ORV|R$+I@x|Y zrV6qvIS%WRd>YEzjV(jhYS&8~R&p^A_h|NiJEztw^PyY)Gi$NKsvfRKfJik*x6#ftk92Pl|dy zQQb`Ay3YeBz2T;W;EP?c!ie9v*1K=ts|+&yr$!4)e{CkUHXT!yIaNGA-HHy9(?;VB zBs^dnx&gU)lt&pbiv*siz>axo?`U_m`ks~kH$t>@u_*G7k?V38;TWUF7!zvjxqS_> zMl(lH#tX1dxa0LG^h8{|7dqmk6~}rC-`_t`qB)EB=qV^F?^BL!ljr?hA$W^*Vr%@HNIbrG0@lboTMiu{UKzo&*?_=khq2(r zX0EGWN0t_9-TTGGO{wR8FDmkHaD$uEL(%%YC@cITxgH>ve-|hx_1YnDJ8Ibc#t_|XyD><(QRmY*p=`}N} zjl`26bCwM)Yg>=?jG_Cit(m&;w_g;d&*?zN&g7D_U9LzT2?zy!VZ|^ z7a*A#Jfc0B|K|Br297*tMV=D>-f_?p)GfUZdZ)x3g}Ms3Hp=s(hWn{oIJ}}UlmPIL zXTdpY!HFkknFJObX@T>Z5q%*ySE7O37ns(;KU`pAMqHr+L6AV18rW*kIDFzm@uoTG zf_8hlwFZ}tbx*GHT3(W!Pm9in$2on|ntE39h(JLNpd)ZJw1Vnzd%5&jKxS*9lUak7 zr=-{d6t01?3Cxa*E-!rg81$UPF)3-}hQi!Dz*5dVRjD0xF0QV2gSVnS6nQ?S8X(8= zJ$v*4cXTR&nmrt9(18tM_0FB97;bO<{7opb_EG*Mpn$F#QAxA2N+alhH$jABQM>PU zWR+7CnWTFH3$@A0W*B$b=Xbs7KwO$~OE;4eP)OQh3v=@xym$WX^E?)B=C*+)SlGDN zoC>m1*-U#8lr7Q+14}y4_?S6>Vwm{|fT3T>J$;X6f9u(0AW&nzHRos+&${oEpU{gN zszBvq_Sfaa58lq^ap0|vf_WSIrdefUP+%KaN{exN6T(rRx7t6RT7DheW_}%IPiwu; z(b^hEz3meNQPP^;9MZ68OJ2m`SH8rO6ym+vvA>qh%{#7%VoB8;gC1m7*`sIA>_jtD zH7FVH;8o}0<)(wT^ovPBSpG|M^Q4R{iv=4jYH8eA0n0O6vdvPpOpR1 zlaJ)+1sxm=C?Gsri8U<>C=Ie4Cy{2LAmFeH6U4CEfM!CCuvN(BEEup%05)($;G;{n z@7N)EK5&2{c zP|Szit8SCiKa&y~qF$~U5bY|Gk-L`g&~?B5Trv5cIFaz2pKGYCZ?uq6A~4$Utnt-(-7EwYBP8xMjS1H?PE%!roST zy1BIi`ZKTuHuFxmNga5M#FW&d71K;JV|P^^j*&L8!4`146RHcMLzp=!3kpD`qiR2n`L{=k8-;;MVI;dk!bK{SjbKImNKlZJjNF-Jm9!3lSrLA@y$qX* z9?cq^{%PuHMFry#6CBUOYo?fDG}6vt0voCrKHCq6W4mJ84LTgN8Tp<{9!#IfE+Nbb zO{ASfNzXJCV#)Wc8RP2;2K{3=dN?4iCBoUBv{mat-5&ryC=rYUp}4W!ye&O+6x^dJ z91|DR z6y~D_v{V)oUB>DJ;GcAB02_aZ$_=ytqMEUOPeU?0i;CRu>cw$i!l zQDwC57bp>@2M5QMX%14X(&SiLgH#y32rswY4Z+Rd*etDO(1&lUxbBwTa`w`BrD#O~ zCpW&?FX=XsLI*LK58n)h>^QXo?9XQRPjBbXRTurdDmnR6IV&i5X<%p*SO~ZL!7Sp% ztmvzAtOm)_;;hd@P4o!BAjo<}sb(akG=r5&c%4Y`*pVZibU4Wd3sESkIWVwhoBAmU z9xlj>m56)_JC(Nr=b&+AXUU$w=gx6+0Cia9^S^$z7~@SAWBN zSy@$@W?EhKAcMQ@rYW}dabv3A9r)5GR8~&5i04ENBmx4oxA#*-X$r!5DL=ohFp(&1 zm4Gr$@ij~x*M1)2@}s6(@+l*{8iM(NYA<|C_8a2Kb&I?p&W%Xw8@UO@Ti@cJ8s4eO3pvB(7!ZE=3#S(5!4 zQC6Qi$IgYlFI~ObSRuLchQryt*@vfCEUiV=v|o4I`ql8QPGaZwt{ZURb%jksgHV=2 z$s?hT5syQgz#8d_t8bwdoK?0pZ>j5?ipHwa&?fbJ`dy!uzQJvmqnqtd=R!1PRbgen zN0$cKq$ch3igUEIPCOjhaOGb!6O{+0lBFFb%;VWIFd`*_CTh>jhOXKAk5(ouSxqj~XPX1Ps*=4Q0A9HlUF&l8 zfK$lhrmQc~hc#3IK5s_MdG3clFEgDj#zkJ6I$kmuXD`|F&#c%jj9n1_;)P!>Pl~TV z;;}W29Lk{Dod)GkKk7dZaN~4C&uz|8r&j#`HE`n(nHVCLA%%+ZvMRc>s~|i|1f-E& zHNrSUMU3(s{_+7u`cPg)&WTWkkTD8-petOJHizs^MG+X3^=}PpmN8is@Jam5_Wa=C z^CGwRn7kSSu(N|h8}=F@WLW7AEQ;R{a3hie*HDp*5rGH4PaN(_TDbJ!!YyT{t9QnZ z=m6m&bCJS2}8gwb*3w#HOX_Tp&eUHoGx(e3mjN+0~d);S@zP1 z`rVGAc{v?c9tBz^O$DstYVc^ZGNOBjjM7V{&zw2)Y9QPJ0#iU8V|Jg_=x(eC1*0a# zTG|nSIib(AJfS4oi71$$VQm^*vWOG-909dw(Co?D zfcBs7znt%YAhzubsH4-<;EGJl%+@hBth;u7vb8r$%BP*F>XOFbCor`9?I?4PKM2FSVM=$CLigQt4u8d4H?6$0V&s)_>&EU%GBd>FXvPq}BUKAEquy`X| z-0a$&k&!D`t!fBMy!sk5zz8;}v`+VEmX+(`)HX1qAwZwOLY#2 z*{rg5czCFDM=XlxqG#*tetPYK+dokY8>+DM^m+YFtu9Ssah^F#x1syKSLZ?uW}hkI zwDCFbiEvOnc^mcFiDhMR4});xJlPtW{15EQvvq|Ks1Ro)Fv1fh-45fbu&Yf^IhAJ4TBgb<{Kl85f4RGMfy;>7bD7YkX;u9&7rFmxA z$$Sp-@3Q*)RKT4yN9m=16Li=@EhCo~iN%JE8`GX|y&G(rHY?X! z$WhK7YC#;Nd={wTBcgAv@Wxf!Ms&apHRa@d1Fmt!p+k0{ck&wEYA=HE9fQN2w|zG0 zRp!w#Kh2#g!5|b?tI4VmoeXrq3<2_gKi)Z_ezMOOa>l%P)e)KWy|O@$)vDDxFVelH zkeEjQC+sYV9}15|j=&IPL8LM0lQ?E*vj1SdwH$!KGnZ*e(Gz=0fol=1y!5fZA0Veu zX1Pof*$Y4?q|D|~jhSMLVY)3{x{6ogT5rK;;NCWY#=L-ZmU|H$Y$_=nnBCeTzgPq7 zO)8#WVBk75N7Ff_4`jNy_06UARn~JOE@rO)N3-7g&#g4{|^pfu-%XW|B z8d|(WN@Osvfw#^s4HFe24sh>I;qObWn6`k%g0e8)r47+svoaS=5fqpFr^KVO^ZXLl z#oAgobbhixwu!-yd9>m=5m1fEVuit<8$As5nj{2}`?U|2MHkh;3aHgfw50UOUc!2O znSM|57twuvTU%8iJO=pA4Iy2!#!gwWhKj0Nhv6O`m!-K`^lW_UU$l%*+}rc*Zisqcg4UagJ)le9aaW2b3#&d%|)jR|%1wB4nPR(2Onjkz66Y-~-f zEJX!{g$0ELxG&h*U9y!F60-QOpAfXNIWJ_>ym$()vh32KleQGax{CZmb3VLAgQB6R z!;11+mmmLZbJSVWIJ0}$`GKHsLIU@Mfw7QIrbki!=c>l5MUN6PAH8BbbL3|m+j=u& z?F@>a->B^BCjOgOxsCQq{(LgY+#bf&Klrj{$@sipqe106iStjI!ou35ZNE#1@OV=z z|MN?I_q`qmO7TCx#_YdFEZOv*UzIC21j+s9k10xA^;+fopI>|jFR3m1&$n3FP0ysv z{hzNEwc&;Qvj6SXf_V4xdH(0C`PwX@+rsjnFQ*>R@qhNim9;W?khjfhta>1nd1dq* zReDTzZpoQ5U3vk&#dZsGlU#$X1zv^r{ZZzPX$73BajnlBnXfZ(DuhYee@s1<-^#S@ z!_?>p*^$9EDV1;;&UNe76>u`GR1dg$>sDEoNfkw{-m2n}n`KXumZI3bW<_S;O_Wr}5HVxY{i-SzsHBIh25z2xrn-d)C{Xh|oJe~k%z^-6JS zYKnW;E|zq|5+6@bnwx93Uv(Pqi^M;--=5X&KXCBineNK_W>sfJ*i}?ij7&{$Y&vi= z;@ta(2|AgZzWvHseb}}6M7lwdTC@^lSoh>$+pW@4IjuyErE}BcRi$5&tdh@vc#?23 zJI-J{&9EeZmXU*^WL&07m;3h)kBxc9$8Vz|?E4!oUAlBAdv<(`Mgp&}Xi`3HMzVsnERe)oh6yU)A43l*g+nf-OdC-44)wnF>m z`Ic=8XU}e4v0{ahx%qM_DJijwpILtY>aH3X7E1^S&k10zF;r1mA!zX27q{Zot5>Bj zUoKj^YXB$rt5?pS;B? zU)jvr!DM814CB-T87U zUW~_mezsbrR>R5dj2P7UO@E6&gU7H`G^j>2~MI^N&NF&-5ieW3H@ zwQGrqiS$fN4bIM2>PNaN4#X{X@9W zyZ7&Ni|4T_SKEGnw{rXT?YjL|U!I>7Gp$h!2?@DS=Q{S|j)Cw+d~1!Mo=$YJo**k8 z#X4bOVcq^P30oyG^9B_*4h}Q_wz@hkl_&d7zA7(2H~sr?pr@xN#qBo#yT`6d72l>< zUf*0>m3i3)TOwxcaoo7;kU6$6cXOUa@^}cpR>Eza0-M+`7GZm>6;kJ|O6G#S?!*rq zarqyMY8p!+3Xg4($0d9FquD=`wbNf=<$A})##Vp$sc?# z?A-Yp70|9+*zSj#dd#7fVdEt;gHoNHop-($KX>`9VfW*sy!o8NKy$C3tW;n6seDz6 zWo~BjnC*H}|E|!)TcLAA?Hd^GEMt?j<3lx~C_eo{v4^`Y7*3r!_2#U|KKJ%zej(IwA0w_!OQA=Yc4O$OJDW!>g?+B(oQ!xjbisX&tkpX+_*NX zwRGE*px#pt&3M)BM5#`!ynv<4*HKiw<)Ml&DaT85)Anz=ZTRjh>lHbzW!|_?u0H9w ztyIpwSW+p~A{X^Fq&{kiJR72s`0BBQEhRUt%_=G?n)|HozVa5dypFD}#j>)p_$yi_ zCZ?yhJxmO1_#(8|hfj|Q&UJJcZIHC1r_|z9*zvHJY&-Vk)y(A3c(ZIrm;Qqyr}39> z?`@-@#LXKNv(LRZGBLS^kN={!cJ<)k;N{Wp{^emIzejs2yqB%cxbSI{fZkJmSBFUnyG&i8=P@!eq7EKBsFk2b6C4~&&&*tbD+pb1Yic-RD2Ph+48_NypEo?7i;u`{Go?9$KFI^yw3u=(!bWeW=JwUc@A`2j3q( z;_#R&%gL>-;tub@j)4Jx(lGU(IY?GrK|?wgE`xhQM}7YMIpSpYIlhxwoBjO#SFT#M z=-9Di-abCYc-o)QtOGcue1dnLGP6iOo_xZrN&oBDuPm!pUA?#M*d|Fy4%zuBUgI-o zGB0&1_!mBN9P-f4K5yYpyC^IvYo}T`8u~3d_Ep5DNn8Kso7GixR&XY_5 z0RgCOG%FYx$sN;5(_=@gE6+Sr(eeE|?Y3>(Nb@^!;zW~y`?oiDbe>+~nHXrfjvE?v zB2`y1+!5<5O7B5S^d^E{>?E8gQtaqdnzBq@<8kB$A5m6b^~_;l754mUO02T?p0!-0 znlD=Q4QxndliW7x;KH5t`r=q#YZ$I6m_IAHv)yxSk7>>3UZ%@*baXUw??Qxm-%GX| z$9W4ieTqJ~lu|Etxe{hFR&c}>Fr#K>B)w$5Qfqzl5<0qDUUbYi9|##LS1CUb42x_u zdFj1uk;g7k(baTJ8$RlmmzC{*eq~gtzYuGjK2&9^%F&~_=QO8Qq2nGpxv8%{dD|0N z_q1F+)YnUnj;!DqPmuCOTTr%0*dXPga?|SlLvfCA_xa!F$yH>U zHwxZ){5T~~kcUUUQY9kU&A}+nJN@A0%TF`%^43x4|6UDJsqMwCJ6hY?#?3x7HKBZB z?*s>|KBxR9B;~Oo7S)@8(P$a8yVz0ZyOpRGF&7yd8XAm;z7*-YAKbo%!FqPen!Fp0 z+{Mos^3f_zJt_HC?Kjb`8K{@7t$IK)ra=;yha;+2?8c&B1z@p?QXFW`%?(zc3*IFn zQgc~W|M_M0NrQL z1Ah5d_2AgU(R$nr^RQ!G>lDZfK6RUujEagP`)38)F1Ziqe?$X?VXMftWo!Qsf_IpinXyp! zN2)~tI9{@`sgd|FjP=2?W{o#c%Zs;fmvPCslwx<`J4>hpgCfbtj~|PPocr-iD2Z)h zW=OWRz?Kf}{y<}zes1|93TyPKO*g|PUfw&h?rwD{Cs(8O)Jt$(I@k5VmvQ}WilSxS z__%jBW%TRY+YniI8DnGPOAGUJ0=f5f6E$MToJxAX_VirGE7DL6>4t$auG1HG#`g~@ zP<+~H8%Tc>5)%5^TCf@by2@@}!RxB3Bb5o+Hr>i$qx|dH*&|Vkdh;H9nv7CbR%XB| zwVoW*r?|pfc3U(9#VR;))u$VB;c-@XY>Ycf?}^?}*4W6ZuCAVulcR|4Vq|5-T%ebx zF0*OVqW;EoItu%JF}MpUNrL)?-U$hOYxuS3cb>|djrwjnIXU@Y@XHfCbnM;whDQ${ z8o#)@6ewURMH)%$kq0Ms+`FH0@KoMKPmM(Hf&z)%maX)wSFg?;*Dtht5h3q2{6TiX z2;G5A)@|QekQ_+*ZrN-)o0|1Y^1Jv+iITJ9v$r{WD#rGKs%_l35!+J%k25!TrGimM zsi&{M|F!$~*QOa1-DFtHe_`j22UVn~qr;v4oN=1{=7HDva9tIF6+b>juUx!iuRK!Hl*f>h*6xu;2@Tx`; zGnY(4s+8O8Q;Hj{^%a_);=Qd$#l8!Qi;1mSzE0qf?Rq@PwRk4Al9Q&OIj?KukBRr8 zR^LKL?|WR8XY}H#F@U&k{sT!%cXxO7d`tb@LZFPgncwP7CGI75=WP@euHp{f|DFpX zoT2m^?WFIq)rs+^4g(LrbZgwYabqQ@%xWbiB@UDK@7}!D|m;tDt75M&6y~XOC54Fa(-;W3g#OEH%hPCshjh$Srx1AHJ*8vxSMMc+EJnH z@9FV=$*Nx42ZaieELi)CJVHdztHw2d)K(JN;4tAqJjTz)}S6i}kJLQd7Z| zrVf>$_?x-P41OuvalgQMGVMoeLAH`UI^yH7uxzoCsh)2=J=Um9j_?Oe^TKuoCE?piYbx+n62*~&S zqdoC6t>5yn@lCn19vN!jq@C0i(8KwR_G z1(5i}cThUAbKT5#@7}%0<8zU-Bke>0THB9N<2WKeRo-Lbl379V85Zv&6va^*Trt%# z;=CdOSI-@|R_0pYkkD z)g@}u#l^)zf-&|9_X2QPOwlrJ;A{KTT;lG25%1lSXR#kzOns$x@(D)lr&1^@BcQTk z7e2~$rRAg5@fsGpINjI0aqHIps^Fd0;3SkBVFW1Pix)4}D|TT=T`b4SANl@%eTq(| zLVxkor%yZ3()kRY@133)xCLg$!_7@q1acizQ)3}GfR1@%>AQCV`)8s8L1iKj-`ncz z=jR<6$s=js=bdys8TWtD!GI0&moDupbQrja7Tf>)mTdG$e}7Hv5wO5r;^GGYsYJCV zHKB?ffQaIe(l!v%rZuYWIP`^6By`1xaM=pdPFJqV-ZT9&O)f1nyPx*6*t3fl-D{`>1|f^p|4S%Qk9hRsLeDdT8 zML9b=hnZ-V3_+6+KKq8Hs3o|nTT-+!c8jE>Ze>V)g#6;LlaqGK(4@hMC^_>y+WN_B zSY453^+~!dgb`o7c#(WIAd2LzbL-=H3} z=gZE{i>E^JMD~bTec9mV=5~4P$4YExx*a=qq;@L@@p+Cm4%JX z7a9ct={1s|lN+R+)^JMNeGK8?k&s}A(pG`>E$ld`w}MkrtAFW*YYLt}eq6wZkAV6V z`egx&rVBfbl2L7o5CIw`I~dPF33cG2g+;7A7-JchdnceB*htjrG9M1jjTD#A_kK-P zlX5?H0&cOSt$%(NCXB{n4j5V3td1{5Hx-O^)`=K0@s^G^V&HzrSD|Gd&n42q@!^+Z% zv1j2MZJWU~5~Z1yV^;zG5xkC4&-0*aq)MnH=E~{swYz1+!TX8?m-O-q2&fd&tE0LC zb1XG)NCBAB+R+$uc=hbuWV@v1=EYo6{oOVaYZ#WIKn4xeF8`JT21LN<@bGXc0QDs- zTf2egwRkKCA&{%(n$_FUZ9m&xv=?%>DZ>UR54pFN%gf7Os83o8*7qcJW){-IijDi& zu#qp>*}cNG*2yH}F$OSkN_JG<-@ztqyad21DK`v^fiOOcjH4PFtDMGuctW5!jawcf zVSDaj;zVxKUu9h+*9u(fWz6AwB3cIpA9Q_;-2bCCp1HNPm1r%{oM^y`_bVzcQ4N#4 zHavW(E?Juh`jx};F$cwO9~^&Dw{?6JHMnd=E*BdcUs$b;lTpyKXHps(8Z`pZ(@)}# zo(O!8f)a(D4GlZI*@kBc!SDdBM!-QKd#yJ?){Yqm+bbU`Szw~rBy2YCzW6yJ*vu90 zOi*w2)8nRfP(*x7=BJOGaaC0G78_7kav7X_2|1Rc(25}OFp|C}Ja9WSl-+HrlaBQ8 z9_JS=E$a!`yU8k)W=|bbecRWc5cpy4;kTTzYGw&nHx*UY;Ob?J9AY0&4gvybuv=)V zpKXZmNtv3u?CtH{@$KzBqKH*K5DLU4#5I*;F_7g*m7+7R#P$I8<%U*V=EGPYeY-{rdYan#0%_+7Uo^-I z`|>PW;!X&P)^ZD+%2P>N&Bm6Hn+k-uoJS_kH$N+D74-_zKwb0OzyFMbg>^?;t<Hc)5 z{KP8kKl`|(bO8N_rZ=A1_s40Qxw=*sL2gt&`kWA`Zgjv+lrPVIpM;&-N8QyROahZ9 z=USqx+Qs=bIvUy?Z3kI%?X_=uY8~BB2wi3!IPW449nkr_^6)1crJd-hD0Dj0V9nU- zWBuw0YCM@%?J{ASn}5{Oj;VY~85&~*r@z~rs-6CX;>L1#g~$FX?>ZH7<{P_>;oEqV|-ZGFl(D#$j2oLp}huohu?wfo8{W_;X%5^$N zYwNQ(D{%0%vF2i@ab@#R4-cw$?t$cSx4z!>IY$p}dR5WOr)45?Z)4k+F9zw8-gyV1 zFpEByzW@2Nj8Z?6%-WEFJE4(dk*bK!x{a)Q7QmZ|EC;N z2=t$un{T2F7WJR==3nciw6QtP>P=;4B6EG_c<&?|LV&L=#=_=5FO4FALtH?ET z+OFR_Alh$R!m3qq^2VwyiU5mCsm>oi_|8>7-o8~OBx3hm1Nv4@-ksaGxsy($@>9MG z^E1aR^i_|w->z>|=PIywQ{IWoeDmfvkJN|2#HDm}M+VS7dj)`ZWN748tlzD2J``}z zY|b^`_%(xn*7W0SY?&QUdu!XSgSlR)6Bid>i!$~}!vBt0UltUI*rvR`XJzr4r_KU- z5`nC=qTR~$s1?eAV$h?I?PO*BdW?(hT>v4FtQvsa0oAHb=L_z{3ND#_M}%Xl9d1i* zv>-?j0LK`t3&Q!9@<6Wi-4^l&_1Wi-rB*>1XM&9T72AUERGt7OM^9=M4G{%WMDM%x zH>46b2!6*pTuVa;tvtOaT4ooNTi=^E2{7&urfv$lM%14E{vwr<@jnau{AzY(qDlC5nSG-YAeX*+)riM8(V&Zr+@p+Jf& zz$>Ghvz|WPOXxmY)B%v+&aYpu7CBG$?VHcdy`$@3a+u;i3dI^OlN@;KC3OAV&$y!N z&@FjExxo+0!0nT>i?BnxP|KSX4aHSKuYPsOe?E9%wX{+#)S4+=&m ziuLY%k_St0w|&UDgtl>*(#LaY^8+b_lnoAd)p>}MXi7TGGSw5GXmz2cycEZz4DKR>X)3n1G+{|T2W9(jXdi4z@ib76ym7SxmOS-*oHH z()#mW{Hse}c+nBN%5)A*Qy2_*!@kR;qQ2;O;Uz9|Zw>&Kh&$+;k)zLpHM6EA`*`2AVk!H+HFyf~nXRBYxXxPLo?6>rjYLH(3L-as zf5%R|W9NyVhevhNPUQ=NYf?N*&>%X8h63TGpZWUg23Xx5)R~lEdB&-bJ2Y~IJq*WB z&>#Q(Gbvmbb{WmAVQ{+8>4#kCBhwRqqXXR%uqtX6G!5-$>d2)_zMmLs5AU>lQwZEI zZ@#Y(78g6V^b5Sj2sDs0&`z3G9aI&DUfN_DBKbPYOx3uft8N8MG#cU(95Pio&bmxv zA^BnaM6F#n*%au3(1vQox||p0iyFE`LRC06-#G~eu^$xS-Me?Yj7lm0aW7iNm%yFA zkT*I(!+H%u-rj_^Ofz*^So1!=Zp-LzFixrz%Y)n7LuIR`c~7lsO`TTy`MySxXZze{ z*-9&J9Vw(;OL0j#wY9d!{0gs3oH|}{ryD9`p?!)4z*XjzQy_}-B8w)ftA~Q=rxKNHMQ4ctNqQ9IwS?mz}{26 zR?W`M(NOT^OrHdSLyyaq=26nAtzSXLAk`R9Bq zLWucN-NDp+n7IPLr-%=Jd3Gw(Zsj&9F|#^)aO>A#T$iSXPZNuS@G#h?)}!B#5?c%# z(+BXz;Q8gqM#tq;$ZiV`ho7Hm3iJX%m7`6nf}7T53xY2adRr?niiky+*QzY1iw5vU z$SdRJYW#S{9hdTWkF_a`&o^#OwI_9EVSZBfXLIgyVz>a>eMIpl9$UNXWLWw6^kSJw zXP)u%Ws~jh>ASgUrlvUI^o^tX?YFZNa~Nz*+|4I1?s_#gb_;}y(WbAoSv~f36+Lc8 zZEhAr`jC;0a}(;Dt~N0;@^Jb6i$_pU@O`@@6fLdwVj&F_-&*}10eqhIsetiMoyL=@ zd$+(SyagtbwDXzMIRErm?_w@4uCe~cV`na9^hs$0SUwy{thNt#J6HwEq~RX-+<7wK zPr8Qh*xxWtr7d(tq*Bgo-m-;f?_Qnr1;>Q=W`6%RX^HTz_)`p`c`{bDu*+Of@0=4%$7QPR|cOK-+0(4(}fA5o-Gd zdVx^Hu04CSPd%yc*_^GFZso+8BmfhKlz?fXoT*uS;AFK+PPhaRa(_G^m$uRiYq&OtD&IAQnTM^9ycK z@XBnL&O=-8MS?Svgo{$Zh8u)2%>EuFrXa}#=s$h~F(G=$>n95|*Kgm>Qny9qVtQ;# z7G(1M*YDW`A_Z@LYG$ZJJm3uYCeho`+g`&v$Z#0Y1>ojBbw=lz!_J#)clcm2c0yAz zTf7GO6+a-PXxpA@Dxz^|G`r|IBd`#paEM#zvx5C@>d1HRl#P!-0DK??eeLQxeeRqO zq@-wkRz9@|%FoY_qW(mSz7!=-=<`5oo0dwQ)Ya|BR`G#yMQ#t2sHvWCcOslqlyb1Z z4I~QAH0PLVY`Ai~?*nc9$i#MdJ)c`zx?qlg`o`t#Be?xK^Fc`FX)e_!6=3%SJjDY% z1s3EK+cq^fe+B(FoiFbHoXIQc@RM&Ytbs}xgTyyGQo(h}+8XJMC7@nRtgNg>{b=ND z>vvz2gXQr@jliCR%5kTGl~0YLV&q#234Wo(Xs?|EmP(}aIXmXO;Sali`478C*NcGg z^Yrq_CS+_vb_f$egHnV4AtvRav0lby?}-#0PY`8eM66Iu44IUse#ffjN&=vw2F9G) zHXh~(QhptlLUc4Q=_KQ-Nhoh!sA)~BGuENTzknn}QP^p(q3P8KxI=WAg z6;Q$-eY*GfHX;2L>P0@i`eWLuA9{nj!ZjHAl+&}d4U^spVBt2V8-5s%deHriQ@jWNy>_`x+A08nPI z;NJNm_l2s7%eD$&rH1ZvYmxi`uBNSiBAq;*i$Wp@xj8vgzd(;ERwMMgvvAQQL0cpt zKy)jj9Re4k_It7l85p79k&-ngtzZ~FQ*?V;qOm8NXT0Ynw#y~d?2ynIx+EyV}ah2 z)RTWxdaM5=0@a_?!MfCg#BuD&z8Bb)^^)E;5A*Z)Shg24QtX8v&!IGvdjWXrbL-Y} z(C6!q8HBYzH#NOL#0%M(3vGqs#S8P!Mg9D17)Zp6+T}cPF_kr5Q$u4I&G_Oi?MH3T z9C?L>SAvYQNjup+G$ElY$OS*!OSr15t7~jrhxi-fANReKx3Lic`WbE-(y8v7&}x#~ zDC5G{R-fdbB>EiK_GGhxQsURDG*7{#oG!WM(P=k9T3Xr!&3LBgMMbsGpWby^uy2!Q zUB7;Z$?`Wp`p34!z7RZ~6g1{u|5^+k`#`<51Jo=iUf50-c0MrRpY~HT@qmRZ2W@Z! zS-EqkTAH@(U>5_Y5AVF?mQ+w_o8+RjMnFf6*F=WeAmaigk|K-3*>KVwt4b&PoWI=* z>#5;RiFM`lIOM_dy+ExwHL;VSn%6!8?d=`ifaVotf6V8|@%c>P_i&dxt_w`|mx=9#~$9B!Q zvsFvK|8(K3{Z(Li?`KdbFlRWI1vIlncCt$FI7z=oT7uNPU1oI&D=fBUYj8EP$Exwg zEVw^>dsJL!9HfCjSg^$;^#zo}Kk*I7H$tcSddrb1?WnprKJ=x(?K3SLe6$|;BA4kT)0w{*B(X7d~6VtqYcxmNlHen(#CQNhAPT z%`TJ5AOWuSrkz+%BBNIf;lfjLa&i<9dX8!!k?l=A9w*KAtXjR=3xXvlH4I_U{QaR# z-ld4yWu7jhMPLeeqWc!d?TEYg&uEJ49BOpDhBq+f45D0Igk(?)NaKgUGd;u4~2=z^+PP7OXA zx`aLT?xFZ{&|V2QK~)u%Ad%#=-=DQ$RmCpA8`Zba2XKt>J1*t$vlJS-SYL9Szv%AW zN`RDE=yyqzw$NV3TZD-Q*2+!R+^+1HM%{-vqvK3r}x374;|oibujj z(NkZ;-RD2)8vYZ>0jWD|f}B&Lx}ZHVV;h=-uV25e6?f>)rbQkJslkBRFxYr@MO#T!ff`d5}W(04ZrSzz5AG5XTC;i=ID^`2~D-L#=G z|8aJ9HpLCJAM#z;t*4sPdc5y=t_kiQOj8NlxN0ZphHEHANGc+nU8SCjVa-Ax_qT_? zwvzI`fB*jQ`1tWtlhfnor>(4nDBnd(8GP}s=gnt(uS^bx*)fewOf;Od1U6peLDHAi zBJct>!TMU{k&&BQzt?BnbiZNJWF=Nv?M}edQ>QH~c2d5GJNQDC$Q=iJyZ6Kk6uAFl zPky`GYDy81K6`_^@;fE3#NAI>P?; zCna{D%DKQp`Tn66tO~?jy`>SWd^r7WeE4Cw(`#%B^e7e>rZszyrNbFan(1|XjN3H= zS=#KP=9cly;okH51~~w`+rOarC#90ynCtY|odyU_()%rv1MaoE%^W=+nzG^6$t#>C zo7w*+Ay)TP9Gm-9B?S8y_4Q#lE|C|eQEdBRkRgRgwF^7fb8zqzF^-}~?%IF6H!K^Z zB4pR;t2u_yypx-qV9gY9 zppDM3l6r+(UMq0$e`<9CNF8n1dx;`?HA=@vc&?;-0jPSWrtU;!VHj%#UU4U4W-JPF zTkpkQc>usPx9xpZq8cAkh5iU)!h`Aptf6GS+`vVZ81>a9hTeRJC2^s>SXxhCem(*3C{yQI%t2+m2uoE)wg{G`6z;evCwzf-XXoIo zW;(3euUP!_3=E5?*|FMVrg7dMQvOzy(X9UpQ=V}`@%KVRr4z9n#1ZNqGSX52f}XI8 zqCxO3kN2%bnqfKKnpnkKl!N5Rum~VYYl%A-{Og*rWr$fz<_`=G24EZZTJbDFKF@l7 z_6lyJa!c60D_OclcNYW_o$aU`AIpb0f8r>m-BVpFusn^jOkz4kA>b_81tuyvEi2GH= zvm_IdP_5m`p{vHOfHEJZzmpxhN(-v%0a&Vb)1BvGF$sT`|;k(9#&SJb5|gSPWx z_iRISFljs>KINRU@2y*_;S;k^uW;|j)Ba?8RnrKdl!ZnSWnTi}JT9+fI%u_mF9Ht2 z;6EhG81}(5RX6t)XmwqmiFmC?v2fh|?eg?I@Vqu}*svidPqWa@lw?~|$897vud*mS z#=ro@f?~NAAIF@%UcawbnqNCj0CltWBClxZDyS!MjT56A}_uRD`plPpSc|9N5`)Gi_2`w`cHOG3zy!myM0uS}8i}(Jo*e zuc6b&;_t}q0r)?bItXqP^QA+-84G+rM~3zL_wUA- zKdSjXs|f&6J7fQG;y1{y87Yye-KUV`4Yu#m zD}y5LhMv)+=RszJ-^4_Dj!z@T_cO5n!|=nn@ONQS+GGr)&Z;Kn2DE&ihQ?ixb;f=+ zY4$i93O=|6UmGUcj}`l(3wGcyu;Y{}C2vokHZj>kP?$IU2UY_* zBJZwUYK1d}YLN;FCsH|}U?z_WrVwXLaIy}46$55}Mu-r7MR*gb_hX_ybrvw5>EUO7 zJm~=0kVuJCgz|m&9r8JT_llUTPM?<4cwU>A72j(w+xO%NFgN#gFeO|%+XhY)Vwg)^xSEjffT*O z#O**20c0)$z|kr99!V5K&mgHbJQw(qASaPzwk;>;6=J*@=Ra)p+bGR}aS^TL2Mxkg z+mFY2AHz7;3q+9suYq{Tt$KTR%O_;LFeH+2-{04_6cwF@5DJR1{eXl{tPu_u}kbeZcyy9jO2zf zq6b9?D_9AUETePh;u_E2O>(<@c`x`8<{1~)Ax4dMU-NvR*`N%?lR!lXPM3fG+JfDs zhFZ9H@7~h#@~!XZ-X+8RB3T5?j*y%(c~Njg`{JS6Xh~J<_(oCK22PT0S@ zmGx&@&rA8-hrnr)IV6b|pp+k+R){=f1e3|#-Tk@S90vqSk~IVh!3XjZ3)HO7oZ1dK z5BPxO;;^7EmGj5wPDocHl&W(plfhQu1dx}q<2zC>=5 zJqhWA=<$1KWl@S`Ah!b>Cqe&N-j^0$G_mQ{e@AkQ&A==UV$%?J|7l_2cZa=KDxuqw zkUpLoH^PN@hSdYNi*-#ImANaEP(gG~p2X_G1B;co?8CQhL}xY+LwRf%cq9m~Io~S0 zrPE~`u^0~wBt(L45CPco`X3`cF2`yqX`}cIF9AZC37UWOX#Ml&&uye0?=wN~#@>Gk zgLW^s=a1zM*uDTGrAAd&)3hLt9s#99&1@qmK#&4F*szbkXf zB0E370-8Qb=~d_jkw|zJaMoz=0P+WRenm!kv7@^G8Rh*t#`x!*6cr`>YWF~lB43~K zZ-)LmLT#f1(5W7&uoMeUHh(|)faGgBf|=nMaT*?8~EUySLG|M?Y_{#P0O`@(9HXzlARauuaybWR9_LHf0}XfPXHJ-q^CCo{OE5$EcaknLd9EFZU0l*-}vr{9m&nzsj z-%AGiKy{_1km={>101L#Yt&?gC>ii~?Oe2YaVDl1iz+7u+aCSbu>!?BK=n@J>vJwP ztwNAN9>hv!zZO39*v?LJ3P4o)8+5kX4R)n8{~jmMz7_f#!D7>==#k~2Cplgeyz7`= z$3Sxc?$^uaW=U3FjAurrGyX9XTn~WBUmq%-5zH_9XIDk-D~F^C#i*{(5*7TCgM$(} zLf*xut0?{MUpjEJek7Owak<)`;^oWQ+Bjdlc)_ATri+NRfNAKI+~?upTmc&-79BZq zMCH+LMwD?BKN3d)M+WX<)X~u?Mb`veDkXKj`7_8kX-mAkykLPa{ShxBY+fr>cP$j> zJ03?iRZy+H^j1Q2hGt^P-9b8aX&;%Dz#hCgX8*Doj z{T|xSa^Mbmpma4P-UNnb{_`XOq<%aRwS2Pg%2hzkv4IxW<4Ia~Dk1rlVp<+r2(eE& z#4bc+a^Y$WH~rc5(@F#e0Z_h#ta<}YI@cb)ZpgkXA{Y@{w15A2`f`YgIyo0slH&$2 zM}u_l8c3$M8<;UV1LK~MW;G(I$u9#Q2@x@2fM-iXA;ehs!$qgiZu`D{`|7?wcR?o8 zhl%qhj9(CY`7`cBxWNS)BiYQ+KX&7!I3H3i%h^QE(vyM42BkZj4rEw&Df#E!!1Kaa zF|7hU_0j3xRa~Om?8reFPy5Z=0FKT8rAyMW5=u=j11@mRe4N$i3?9sTkkD$8F&4m1 z;C(U{79^<80fG&v_0;9J2;wn;2*&!~`i;$^A-kUpq0nxD@5h7yz%>MR&{wie+K~ha z{AWwjGclE8fsJ&Q(V4meJ;cf{LoQ2-^f&Zv|DJPTM^4g0xW8A{{@o$b#47u1Byqif z<_oKvc$37SKrn1EnOh~=MD9GvhX5idB99HppN!D~zT-jfBmkWnM$&g`W+obFQ$rg1 zpQ+rU;gJg0KCVIBbucl0DgqKn%`d>14oqk9>E`gk@Ag8-r4+Or51&Ld;85deiY36t zsg6=HtfCaz;>d8zL5c#;`-njjqTBp4OZyR^qB{97S^8KmVNk$Ge46?QK;#yMN|YlCT4`6{hgp})-0e%LG@*TG-}!eoB_{9 z9f>nYsl2Bq^t(~ibn-1Z$&m`6Jdxlic-A!@-uQrrLN}uz-GT_m18tlU$kCfVy_Ws#99ilnn+XqEvIm0`7W-+N2BJQp>_dWB6V3~SvF zYhXPfQ2hlVP#%=ptAT;c5KZGVK|A|EFs?s)_N+N#$?*!JcUglS0cf8=g|C@LP2$FU z4CskhQBhIObD&FvOs*mN0Ek?dcmMWnVgMQ;m;)b{>mw49Z@Pc2sMv=@6XwLCk)grZ z??J=@V(ei&>_<9s<;IO^-A!bM9$%F$P^vO8UriJpjE0i2eB5t^t@k>ySrkAt#rq^B zBwn<)b3weU)tUVIdb!=t&nxj-Wr*{SphzXAF5sOp0nUg!pJG_D5t{hjX>Kjdi`LW9 z|FA*WH^;n@3C|AMyi!O!gqEjjx3vyWjhuOYd1MiZ12mZS2eR$EoKZfF3819c5AM4j z)31hChyJ=BJhZN1DP*SfDx}sj&Qvh{Ndpqg7??3;rj=pXQ%1p(t)-T!p+@Fm!T+n9CFk;+HJo zM!Nm?ali+~BIP(36nsEQi2(y`$YsBVX1jzEzW8}pt`iAfQQY`U4yLVIB%i_;7kXn$Mq)sculJLOQ$Q9T=y5?}8#K?-DtmNav*$ z@2EmdLv-N@|I<^&zXt`qJv%i&Gi2Xjg)P^RroSUrHmIr_N|;xAx)6?|_>^UGgmgCm z_SdKd>Ij8W3lCI<$LkmqF|rM1y#rB9lD&t)8GA+OsEP`S4v|?7goiooYj6HWzKHUG z8u6pm0R!T~7;69n`q66jcdbS}e*>!*4dVEzbzEG5WX=lu?ZQm^LI={auviejRWM%< z8GHEGua|&sE72n%R#La)+)$U5z;jVCDVN#QFq~pQp%p2|ososjK0P@Uh-fHeBUgK#RaGJQB(!EiKLCEr*U8(>|LRf0;~km6Vid$x2N~=K>(~zFQL67f5c+ z9b~6dO~?^FsMy;5lE6ViH2)h8dM|UPjmKPgEW`;PtRuNDWylF%e)07^1 zf)8lr$>4;6fx#hVWwHrRBXfbH35d}`wlmZc+`8q4hK5wL6b*3QEiShy8rTpjwp{Hc z?&>KKW(* zNejr}aF8(&I5{NhPa-N1pJ2Pvd9z^RkBwvu8Vt@&O6MZ1*`E^K7npE7Ik;NHG!@i% zdext>>Zd*5Ax6|J?A*E2KKMwHpgreW3*H6I9SHj1Qm(;i#$hC!V5|4CxN|Jqbf+Dm zi{Rc=z|!dK?!JaZ8Zcml+uXF-01297kUyx|;nF2Jq~MHBpQb^rqDo;Gjlea9l#wbN zyEfa5tJX<%?BuaO4NY+@7FCmhz&Ubru}sU+ z>^XGSp(^O$!PBYX273|5XaKMbc!l5FK3)%9yMzV*dgIPA5GOs?cgt)?O{ zW4KFLm}os1Z{Cy&5Gb*NvA!e@=|N_S^aL{Ek5NgEmX^#jbeP!>OVl|&lVCw9nVCwT z|9iQLhDmjM=urEWy;c#+Q9r0X$8;elG&j`XK z#z=9f@vW)JS<#H+2^cU;vL92BP}5=EdSY&ZO#eZ>UOJpvVvOQTo)6+lG4sC1=(e0S zX&*PSvDt=TFpCXVRM|6Qx z>@k8xee@1!kB^c+2hpv_Ky$U!YPb?B$uR>s7ofO-W*Hs=JtO0Lt;eMc16`;}_Q8i$ zRm)K;M2Z;ytt!gm^}H0C<&lNSkrA3)_z5(Btnf(WbL+L_psH#qF(Xw@eLw47(@<*E-oK;lzatg!(Kc(?M^_z zLbBVv{%L6@ie#V*TM>JM@x-D)W<*ckv;xXN;9x7HOe8+HkDs~_-KPx?>J>;4O#-tr zN!cOPd;=B!G@d3!0b}KE-{Fxm@T^$Ny5Q+kZn%Etp<}eCNk_K-3zrxlf-6wuTnUhb zJI!q@3iL1u*!EsJ+?H*Gd;N~!NDT)MCF~F3ZTH-jgk(^l*Yk`kIzPb;hk<-f!KY|n+lPo%(AoG<2)`&$b)G3coM{oA3D+RrA87V22IQl1odxX zv=c)nKrUzCgd;9Xy9IZ(^!4kd9)t7r`g3dL8Yc7&Q58FQL!~8g<)`1PYmR*?j!rPH)RLl>*Mw9P=iU4+u-PHkr6B>ZO zNxkqXYAKsf8Br=3mSsx7j6D5z1q?<7v zh-f=j^Q<>vx?-lQ+-3&y1+3!1p9o*|2xMo52bYrf1)~J$kguUz z#Qtg~iyBprB%U#)!3=%bR39HP!?s5ilvoTko`eC&ROOr7JoL+!y}%>mP2HjQbQjS_ zQ3d|%ct=-j%zz?09%JiTVAGv}R05_xk6RR=ERf@p;0t%*AHo8Xlaq6b5p=-(8|_0e z)2Lj1JQOkx{QX_%XH;);)E!__!hKWFUCjGNP8O5&GdW`h$EJ|%1mKasN+9@_0)lAN zz)1wqHxSDk-%p&4&ng_6hP1O0C)^On7T(5o=fxC(CIpXWNv(pubmOeTsWc)ePaDJ1 zA=AIqAJrLajXjAAhwkA?@S=)RfAoAwNoYj6=2t-LHb~ek!2-Zsf^OjeCVEcekR8ec zj@D^dxEv-hTm@(6fnps9lJtctJSW zzx7`4#e<1jF99^5abP#dLxUqV2e5@qU12Az>Y=(Z4K3oT<9DG&e(!jnl%1WQ1y zauBgrtlA5B3P^@0KfQa!mHAl-grmS|pR`1>e6INGIVc*YrA!F#(p2C; zs0>E+;}+79$d`!@3^;t#R%*rc!hipG(my}05zpBu=oiJRZb*xGkROHY9$49LC6mXU{D$>*7MAGZ>WAAbLA)rqI=ifT4o@cbM zNPY0151>BL72z$Qfur~4ro|yCdf*gLa*NbAmdenQ zTORfEwaT)S!@$hM6apIq%tus5;+svtDED+Lf*X)tZbIpT<@*xGUH<;R532I^UkHqD zL~>U+ZKi;8)u%l-Tvzr*mNY z^AJC=9;~l`g07Ai`(`7i*e*CB{O2!BWWO|oA6B3@t)#WJ9n}l)@zn}*z@d*U3gpNB zd`+)bAOqMGye&RvqrcBoed23`w~F$dm$&9_mTnm|)!v1lyZ)%vQ9_6N+IJ{9S#Vn=U^6`AU7C+%Uk7eCAE~Z^kib;MOsefK$)AS;Uy#&Cb zX(&P=0m*sM%Jh%GDuaWC$4(M`K!Ki~=Qgx(K#0>gj*jv`EUW|J-hvH0e>VxhAJQ?m zr&+yOi0BM`KsVnK=c3*~PPf~^_wUC+qDVR7Lp7QxQR#7d4>>i4Dn-oBMuPvJdplhN z#U&cn|7T1)Qrxg%$$(y4#NSPJ2quHt6E)13=az)yh(oNE7w8C3%*h!lSeiK8Z8J`< z>c6-R4G7vJ3E&ZuzD7`w4M4jBJ%*fBr!7EEZ$&r|LwRJ_)btAce#qP83}A9DEKb`a zTy_`Eu7LV#o>L1g5pz9c3IIHKGa=J{~I2~m2p&K}0utc*CK z(y3w`F{?7{4lurQ1Pf!j;gA3$E#g0sLXWe2@aj=;e#yv`H;b7C_9~W6Bt$JFl*eF(j3IzZavuxa|61Y0PGlzXY}=2j&5>AwG;VL_tm<27+Xjfx&gm zz=*8S5gjoY$E=j$#5qJ-YqAieFv1FElX1EHFmp9onyzn2_=2E(sIg?u+CEs5|2{cb zSv6dS5xTbuj_V|%LW8X9^csL$1x&3HBM2GO3v*9Y6%DTg+hX$72&3r~)JR%6!G#j% zOF$nn?GuR@8X*TGlQT9iO^=yks3aj3d6jmO|H`EyiA4hZ(CmrQ1{1&6?+CpoIXDKX zYRJ3N0UM=nq5#rRIEb^icpGck2L{g|D0vA zocIM;Fi8V5*h#y{S(8Jk{xBDbOag3jhkC7f!s2nGf@ z?OE8OS&W=ns5^v-C2-_Tz&P02uZUxb186>E(2<=xw*4ITZ4_j36e|1=$^&r^Gc4OA zS8UkJg7vveOpA+46R0aiuV57h^^`FB3F+@TwCo7ncXAIv=;|vE9ykyQE{Wrxm2f-+ z83f16cVJ@@{SL4XImyT8{K)A&ILgf*po^@?)LLUiHONQ>Za+D96xQ6~(Gw1Rb<40` zNM63dVGoiI|D4AI$Z!oa9Y|vYkUTo$37oLRhVyQL0Z3GcWU(O0<2Tvh-Q`)eKlv0N zMykt?Tf>-$CG^U1u$2r>)Cges#8BX5))HuRa>(LioMR2rD2pgESx!U<(5(MO)qB8m z-M{VspLR;6Ee%CVJ1UwQMl`SXx|-6~)~2PRj7USGjI@VD6AdCIp-B-ECD~cY|9PtG zd;fpe<9Gk=@AvMC&*%Mqz0PqQ$9bGtDPhAI+L6i6jygK|8@|_VYF!ti?|6A@ESuZ;U4W`DAz~DkrEyd;m z3BM%ahh7RwH0a5$-LG&?eaVFSub)Y4vPcY^ptCiU=*f(MFI4!k^zcGjBX^b3K}P>E z!ImbLiIKQvC5*^0MIiFO{DO?J;C&qsstO4MjGy}DOP8a_30-@CVOXvK4@;stPnC`) z%E7^HDOqCyQm8GZd7We+F7RLQL$QMbrSm3&NySc^Zp*u_p+MqykehmjCzkc`o<1S? z0ROR5mo8s62nwBK}ckKDYAAtHMf)H!!kXDfGbrkuG%`AVlZu92jX_Xsi!BWXwf{Ewqk9U<6 z#3r4Ol@+b>hY9&rl%WQ3F*9_>9fl3PO7Tm#IyPkOr(J|NF$IdhLCoa6g6=OuJmkZd zjlT=pTrExnG(OePpvl;5{a)5K-_4^=Qs?`>4}(DxpH%vn-N}~BV;d_}xQ^53B!^mW z$2}GT``RV~7Bd>??Cj7c3Jb*_c<4{0^$5B_5!hN04tgIHoROQ`r+DJ|^W}XNVn4I& zl{>(gq9e-9YXO&eKWZ81zid4{Jr#$)3@VxPq-61rS>QAxo8=f?iT4(jr6a!Dtb@hxg8#LdKIy-qIz zx;Pj?wDsLuu(%z`iUU2PM|ws^9R<|oUZ58+NpNxn>Mi~sM8C`m3?8h;J5WpFsNH&U zgV0!W*X%O#fTKa&=!<5C^|b<+aAB?YqthCcm(tT_|rse=!ldD z(Dm(dPWcEd-{YwMfC2k>Q$^c%+lB7{16^pu$ZMc%Kn4M7DLH#|W`+^3Nho^+({Km#-+puD$DvIAMRVe?y zX>Q8IbE~(}sRA)vzH%jS_*$v7kZYA2+ub-0!XS!8^vN114+wfsFd{w(J5xhJpWp~L z{E0X*IEmK-G2v>nrFu$P#>JE)3J6I;7XveAC5PNg{BWrIMfKuQ^|f0Z?4QKwOZ?2~ z6l#tZ^w^3sZ<6W66^c!pHqB0Ykri59HD*}xrcPk+grqRYrghaTfCEgE>UXNC5C8`8E`4u1f z`i(a+C}fDWFEiA5+jl^a8x?#)3MA8=&)r7c88hJI@Zxss0NNg-s~3^xmTlW)68`Da zr}1-DPXGR-*o7@*O$GSB?oDf015mO@6umwQVQX3l`B+|A^$oz5UV^*aeoJw&F+ z!F@zyz{7f_?OUEx#e@?<3i>u(*>cX?O|;ST4{{rQNrVzk=VcUEKA$*Yf(i9iJ|hGS z^$t%;6T;cY0c}zAG)jWHc{_c}cRMd{Sz`ebO2H*LjSH8R>RgCqSSZM8%j(3N+_G9~ z@z0jzT_Bx4c>K5)_iF^7DF8?}= z^%mc*($Ji0tt>J0$z5tPCaO9{29I}ZLjZZqSv;FtTYLt|^^%&dWE7V@pJ-}20u_&( za$>f?;Dhcqk=o0z?cWwo24=8GTS!L18R(;8$8&t&sjpxrhm-203rD!`Ls>C9sU0S7 zbSEeW>WJD)%5j!TNI^z~m7~Duk=u(CsL&h!#YXv5k&t|TRuC;Fd>?+BXrI3)hqT2W zXf)ZK6D&VZtnBDmpT!r(`xrMw8{L%PEt9=e4dTU1UZ8}UzI3UQT=BT$QDATaWkIz{ z`6Pc`h(dmZ3|%PH>uGT|I&fVSyEtSWfX3W@bhN=!4TCwqLpzwBlj_;;!Y6S927dFV zrn__ffKkyP4s9weD3toKqI+ct26a2thAKHgZbV#_M)(#-uUB33FA%_LDQNCAEM`*0 zJme9`zy(fFsni&Q%_(VA$;!iV#Hj%&%W9T$H(xNlgi1J^vi>C}2$4WzzKukANM;#k z!fvDqZ51z%3w#SbSAa(@()bHbxJULQsK%gDO|e#NfVv_?Hx&``Gk*-OfzAIW?2bN)KV5&{?LqOq9qREw1ONFyN|&~QT`J!ZwiKh!R-uFh>sSt*&4)Z5@wdYyxE z@X0(f)kjTvh!{60CPlNTxRlm6|Ifx@<{nF9*A(#myWh)uYum`59i~=O+;!_?ji0s?di)8n6&9+ zbaEXZqUF`ASF;j>dN6&ui&NrD-6S}%D0_H_GA|+HVEEGPs_d9LxNzdb8_k0AH90jf;(JCA!iNI88@*O*nny;m;?wR@o@k+c80<> ztK0zNwHpE*N63BrO4FPPtjj@N_iNmCV+jo1A=~ayTgi$YBRH$Rv~od3%MtiVoz6|D z$8nOX9yT_D#z|@ue4yXZWpm@6kS8hf_p?_=paUvzdgj|NHmc}iSP2}Nl zAs4`bF*Pg{({=n^+hK&oK#9memwOyUw=99Te@W?>%`*;a(xs8h@TaZmUwN^0Es^iZOa*0qtjOE z+M8uM4HRnxXo=iIw3q_RVyCfagbQGJ+|n_W*!QElZ<%ICjFv2nDJu4P%5;f>Wl4Uhc3*GMV2?^o8LR_dh-0*gK`gCju`y`zCgH97ikg!UQ=&gk?C*KtOt)l-hh=`s2iUp~HK zz@+X+WpV+XtUcdVMs^^y;$Q757oi@iF2VdLp&x;-P^t~wn5803C!iN&i9^S(yWzY$ z>)FGS&JyZf?p7n}e;1ZzXzv?U1BaBPh09Re_3hg?DyOgZA(6BaN#yBJF>d!y6z-X$ z5_fK*I@spK!M$;b-p{D62o^;X*{*&2K%Mm(K5LqUH5bI!+T(wq`+A2_-XrWMM>PGd zt8vRPZo!oM+M~R}J{#Nhc_-+xcp!k)QJq-(ClWOu5XyyNhG{%LM)c{zjZ{!?vhM?P(&sAV{7^#PZIQ8^0jLr7TH;?*qEv7nyr zt20S~3fXS8Z0@z$ziBGQt%2qG?L4k=)aPFlgdF)R@0My(qufX8cfmyk&Y;`u8Pt+h zFba-!{M~s$LpcCUcYvji4-DaRuAIwHY>xecLBcwiKmLh3-W6>kcj5pbHs4@&@G5)M zh_WPMWf=+_p;ARxis|Cu%|9F{k{-T%*>_4#M$5l3fDbg^jbok8v3$p_GN(0T!PDBM zkIGv;yZR$)#HSEQ1?|T>q%BS03Ft_Y&VKr<6}^UI?~7tZER0c2MZ)Fp0x6L_MD8p4 zT>HOmcIAt1qo*XQ6K(2{0ghz+WCvLj1#Yb&Z;Am)>L^hCJ;Js7<4<7Bpx$qfj;u~% zNDAjsX6)k^7u_1yMcl;EyKr*OOovoy#bXoQ6tnmZ6|!-S11gL@tA6?$=45F7|Gp#z z&yokkd)b5`)~~@6yfJ#>#5H1=h+P~Cq%&g!hrM=dAXPRG|J0N|)ucw#NhW7IjOb|l z04b}exvtcz&q6HpXy~8)Ebu1VJ<59*06Wk3l=shqX8#>cq-ReM1RO>m%HRCfvk+do zmO`cjew?wTE*;hTxM)dmc{$+d=$KfJ52QjN#$T-lR=?V&)<>wh`mzE*#@R%&3fcT- zJz_#}jTg#VG#XuwAfj!g*d=Zgw?ppR{&!G;&m@X3AG2>_FJfzheKn`}kK2tA>g-K> zdDV}RlQK?>9`x|BBsK(*ltdcDLlV^;IvT|T(ySCP3PJhSue~Ov4xrphi#2P+TQ`lX zGc#n{0GVnQ*#PX4FV*RG39J|qp~OOW(HLZM_r86`{)x|e{4Y@0g{sn;#l4b?Rlp@qrU3RAv01!sVGu z{p)fP2MEY0+7UEoFBvH1lYRalX;P;gYKTBwU!gKe&^sgwlb3{%1=4m_Rh*vs8=-|U zj@dFtj-#DqT*;4UTe49L;gv0dW3jv!>xu~n z#@~^ME9RpV4<|E%5ogJ+iBD=PgbT0c}#`ro_gDVockU`@g1Rkq#gsUJ3AwU z-khXoW?n{k72lVjkxDEh7|GXW)8sql$m5exA+X-)2}rq^C8(+uSpLz)yR|_W`SQ*W zZeJvY>**zxC;}I7sNc7r9$Xq2@8pX_wa#bu@+NE@;zUgNzQHzLzglT%&YzeM{{M#&>Na_eq`8Ahlp}{>s^>2dy zK~!1FLC0t0e_Ps^6>!>fZwwsuSY!Qw#zT^P>f7Y&PVuaZNJEUy-aM?!xt!i(Gn(>{ zdRo6gM&(6St1{c*m6Z;ZDf%+%B!$@J3UD;rof7sF*eo-PrxOcc!&X>sV3I;c z!V_;W{)ZesHBfmBzKQmCu_l(>M@Xsane>On^y%~X?aN3}1XH9fYrks2>eW}L_eKEC z{J^OCtU^?$g+*^a1XE2x2}8`qO#|cj>JAhZ(9=ZyF}SM8Rb+9Z<(=T?j>OrAa4)aE ztaEg764x`DwW?WlsShbiUt*Gl`nx$aX6b*Q;^SZ4$(3y2g=*g~@ak@4zG>~;4jYmx zd1h;?GAH-#ALSX>ty_cLD+;{~FHBuhKA|q3iGH$A(xPqjz0#+t@BG!2m=b}+I{*o9 zNH)v-8WVlRXQ76I0HBtigTRCs{C3C>rckrOY7O#3&&J&wMvWC)JxfKV_4~5&t2u7I z&)yj9M@5}V-W7LcDYCdzV!6HT_4qMk9`SqRS#&dB$Sv%j)|L)Q#vNvSfX#e=cc=Sn znoxkMg>>zp-dq37l^PX|k81cn=jPKfPwuU9)Vx)zG_j3Z)B?A$&L6&=H+{yw8d>Hk zHtwv5gm817J;wDWY-&113a!=3{kMjQ=YLXgBTg|kYK`QKqypV2c2SPYk{iV?f{%3C zGB`y}dx_LX{ww&zl$E)3_+nlnU>(2Y6Md-4sAV6QfdAi&K}wTKc$#g_S?RI*!|mND z7nF1VO7*QP-$kC0Ni^v+X+95f)o-6MlYpkR-!mEl$yvR4<#fd&cZN`=_hlAy4z zv8OwG2sEWwTk+GfTRO?Gt)IdEiT#Gm!y1G{6A9-aNt7z(+lMcJ4YEsB+5>(ej;J?H z9C&~7qGUMF+ge70HGH1Sy_)x^6@Rq#$E5qVS6*BwGUzI-+X}EfX+#C36+I%7sexPa zI?~3tuAU2cL)8i&yL8JvVM{q>lC&tG&ro)X&`$9Q%ud)CrVV7liNtPNTM_BG1;-)G zTCoz$PG+p&>m5aCM(Pr|ugv$69>!kKfX1eEZ#jPi2PX0-`3(Y80O>}Kp%q!m>bX)959OjdbFvIDA+-3Bs>&oDZSLj$;jqyB3!Q8@W3Svqd+2ZIZ~ym6V|p`MY^ z$K}!WU+b+zHO@QrMu8{u_P|R{ln4)D6wMpICl{KNMTEdY+IPy^x;mElIM5N<)ps_! zGP11GGsA8*Y@6K<8KOJ3P410`C=X83{q&#QB=Oxlkp%ZPG(vG7xY*SYLC>=+qm=b4zEGT# zG(+vMV#oJGiZemZ-XNxmzd|aF=;jQc$1i4Itd8sOew?K1>^NxAWBRx&!&ol^TLXMM zrdw+(9X-9e3cyW^?xG@Ed~x0iQg|K3m4qImas^ABKF7b;B}2LatTi>S=WlfL2Tbq+eLlmI%Wppt=6$QCAK20~`WLO|{$#@GbKC z;*sZfBu0waB8WF++VY#8Cr)S|{ZscwThiT}-bX5AXbUzwzi0`lzS3jx2l=V&XRzWPH0(bo462P?Uf zh>O&+vE2hxd9$&e7QVG2Lw0xSbgx|XW^o_l(B7yq>N4|NW7l6PnOj#5mUS!eBB_Jk z45{X!k=iyFX799t^~PXX+pKuNFP#LD)b3DA1-xO<@AEkJ=qVoHg_F}*hDWe9Gt2HL zAdKRSUWC{U$v>5+g;3x_%nV-T{^D{&2)n=gX{>_9gG8uW zgrbV)6Tco&uxt>%<17sdswVfAAt_o6loQIwZ|1kQpaH-x!5_2I|_i2=Fp-4&JM+y0~45LM7fwEy=%yw;fO%} zbrb#*#9PN;c`AjSr?oQZBg?tO6)&(o2S*W=Z%`LaKiZ}%*2+}4yMd!5CGtEJ>VbVH zP*0-RWoe9&|E=vzPYY~YjXV7JHIQ639y`fy&7ve;^6cDy;Mq(VgRC)P>46#w*$yLG zbw;Q~(0j##2QS`U5F*vo#qQlBgTt?i37-zz*F+V;LlgUHaU>H^oj;K=t7>m4teN@F zGH#1FXdhM2;%y%I<4FZswBygqYI^qPl_DKD$+;`l4GE#C)P%$~Uud1#*{x7#!Onz> zFDGGDzH){*iBpQ0lLHY}jAk3jNuq;^a|cK1ulN(3yocilxSzRdUxdn%dS&u}izx=% zdzne6>|0Cl3Hq+ohP(imO@hg4+IfoE+&4FO}N}4!Gi?uA3gd~ zuY2ry5-kz%?ziNve=wNxF4m2#l?lkk=pc%_pR>GJOF_3|LW)C`_OySJ)N7#9wy<01 z^|E{Flg@~Mb-`fODr+t>3=&oem4LHuv zN>*D!0LyA=ntQCoe7G-k(eW!i63aWt#3e}eBGg@jORDucc3B%-?TfC~-1!)(5Bov> zwK+Yx`%y{g3dtR_ce?dx(ZF-K-|*Hr9{aYbv;HyV!di?rF>3C0%kZfl73N;}Z_ZcP zhsGI)&c<$cm3L+k#^HxdHBbBL)3@&{o5=O6|M9DTxlM=NGU9P)=lml#Z4%-$BMtW7 zM=I_E1u8NHjv2NPt3=MjFJFjQj5&hIs}B@LEK)ao@sD-a^TRq1UtDm%&~t=#onM=* zOD|SKi<4fIw+hW%T&ME=50>8=J zr2VJJavL4s+^GV8VbB6RV>-%LqM~^WZWl+;_|SM?^$9dZBwiE|0Khl+*5~8VoHB~8 zn4G@Ve+b$&2-(pnfcN!799EgHT>0#e>{lat!-?$8kM!XhZxeW_y2&Z>N8gEK;Z|< zCAvXqCXTJ_lE#+q=$itlr$@*I{35}0nC6ZPS~)}UfQCZWbjtPyv3KJ4*nzoF&Sh@+ zfg0Xr`u~A%@_>rkaM_hvwcA`S0WGfBqC;9dCJzK1)7RCV{Os`c^s0c(DGMjHU(+@+ zY--aQEt^hi@jc{Fub=bG9ktZ9+YeA%`Q2<>&j2koM-%g0we7jRg5RzDez1MZ0b>q) zENL?D=kb>RjA;6@V#C~j*JVC`zI93K7_ULG?(Y2#&=t7$o;z5Q#%F|J-4Xf`F|BpOsMO@k5U=`pTTO3H4FR zyVVTyEN0OS0nI&{>i7AK`PRnN-8Eo&WV_^IiuL5Q)=it*ce~>D|NKO+vt~#30~2sb zAA@|zcms1yTP@q}eJFi#DOOZ5(V(u+;47r_X8;IY#xd@QVB)3i7}+*;(EfkFa}D4- z0`(~V+Ee#_UCB*bEGsGckZ>3R*~CPXAIH=3bwd@)D1gYRa11y~fe>L{#8=hv_4_Yb zOBQe`!OC+a6v_dpXrSF$6sNH5;w7SxS#`1qf7#g3GJ!UJ)ZkY58ClXkYsZsl|Mg7C3bN@CH7ct`6^4*mx( z;9KaelDUbU5h4n-t<;`mVn+Yi2G)h{&2PNNF{VNGkuF4Y+3{D^Ax4Z<9ZjZq5bQyE zT4Ar)>_Qk3b}|}bUh_V7(9^N_a`gfF(ybd&FLs{&MydUS1JvrudmkP?qa)GP!lLE5 zV5HzEM!H6u3kCwsC3c~dtIf_G`g9^I{%k`E%O!x<0V_&a35;XjA+lf_-NhnWm^s_z z#bdT`cO=`KMK7xJ?d{vQ$N8-;gRNFjwWwS=p$M(~IfsH|ynM07jh7%g*I6+GEKjPpn?w&4sozSu`9=DVRi~+^+Cs4@b;T!#B{L2yYnpKtM2JqC4XEEJwpO^C@)f8_+qd{~R?EL(D}jkHjtWgT|m&JI0$?kI^-nT{Yie-b4RY?+yc9<-K2e zy2e_%U{ebn#CNfLZu!hw+&=0sS9pLtfUvF`2Wn_A)q9$-w4lFQ!1e2P8}D;5L|V*P zQ1@qU-I_CVw8Ww*t<;HO+w~N6_jZcDvzGH`>*Hs;rCF<1CQm?jZ|~F}nZ9;x?hDVP zTWS55v%qsLNJ86;m~H4arc55SXanUv>e(2|7NM+R2~qxgq3B^ueI#13=ieY4PJt5Q zUK^-dEVilD%NH#z@37v0e{`I3IFP?^M(sI2v>j{(_eHc6vpm7#$iKDV6xK{nCMXk4 z3~3z}?FiKuJve`tyYt<%xa_KxhrTuO^NJ=bB4jUAdLldXZF zK+vF8X>)B$Utl?6$HQ31lNY1|y#m#HL_)$5!DxXyJzlTDE0sHMLG!8eEsUxX^xuQ9 zzFusRB7$l95tc^ML+s8Ax1W)z3dL^<@zZTi@a0)$@V^)4hZztH0|qbM$f78TTpUXa z{&KEg?B-tZ(OitZYeLKROB=FsrN_%o_I9jrKCDQ6y6ajOb)#vm=FZ_=gd3SD<%cim zoA>GW+-(yF9XdPh(fyeH*Z;BkF-x)cp-N`gHhib1=?%Yzi=8W-x+~vOZ0raOO18s? z2~EH4=l7??04WeomFuf2Lv(qNq-0k-Dow7A^`Z|pJX>&$!msB}*Zh=~rTVqf^5vNt zFjI~N*5Y?`Ds4iv$kc%~AzBvFpDVQ&=SjAJZ7=bDuETHEHH*;!j5tc~C zcp$s;rpeZ-k%WU;!K>O3vYe(=VQV7#!&jGQ9U-kq%MSjhmj07Asvi$si`q=y?4p1&W{z?A45aF(;XWVwao z(V+384&*1dH`sB40Hau7VPOpA0>)b(pw4+}3FSA&ox{;ryLO$}0Vc{P=WS%<0oodn zjB#WqO^Y2iWf{l8)ow7cL10+a{nqvC;zEq1yAZxtB(WG@H_KkfCIL^0gd7S&yetY_ zbjppL)KB<`%MehV3j1#P;kFAQfyHqTRbPj}SBN6($&>w@C|BOa%+nx8o-+6#7EXK` znS|EU`x5q+S%*$bU!3|Jd79srO=&>%I-lqb3Y(7ZrNvqOFZ2pL~p>RR@p~| zGQe3Y@T(OsAj(y%A+);Qr4oEyST^^d>**cBj`Da?$(9oA=m|`Qc!?Pfc~{o%#4TG) z#FsxJ`gnY^i1qSh{y}Huo1R))99?v0Ci@+}=5Ni@_BON+f8DEB$&U{+7Ew3cZ#`zr z;Vq&BlxN)Wmro&~;UU?D;4DTYQ$he{b*#Qk6{8=TD2%xLuoPe`L67)&N7VO=kZ+*A zT#tIIK@05%^!ekPDgS<>qQ?(WJ$%=sykjE9+#(65(Vbjm{SHnL&IOLV(WL`rv+puwwsJ zn(pl4TF~j&*LZ|#vJi|_S*An~^8x+{@UYMrOAT*&w0lbwOP`O=w^>M_1*Z-RsFYpD zX+tB)myJM^qc~Qj-veXk%_$L$DdR}E#xQ!)iES?Vg@w{0PbGFe$yBaizi-mKG#Ct| zC#QI^0X(LJ>CL#MzGQcyn&56uL!hRSf~ky1yULUp{jxXH^~x9({-lgZZ2kFNc6puU z?>My8?>)H^x=EK6jnr$!pZ3~?583;0SM1)=Ey^D?J!Ev#cTb$+Tx)}Ems%dCKwo5N zrc^MSnI4y8g&WqcJ%eiMnq|qiI$>NCiuQBlREBBxLvaOeFIkO_{QMNlpAN5r%x#Y>EmOqUaivwy2!w=T z9hz(f_4+uTiaT3+9lF$@7HOc8KCN+D_nti+AaBKyo`I){l$pEpKJTGcPW}4Tj2mbE zA?1>i3ZH%Bw^#O@sZ*p9ck2oK{}~hjUr^*IF3%1U3@#8CiiGC%sYSr(or#fBP)_zA zXNF=CBK`{}f3esXDpLp6{ec3ld-l7$;eHa*Ey4c81~qK9G5^ zY(G&T6jFeQb(`(AUjq??AnThzmoUCqxvrsGk5P~gEui+}LCL?)+-RA+v1fO#xF1gj zDBcUwD~(}pDms#^6e0jqCKU^!E^!50el091E^1&-~9 zKq3=3>O+SPRmchzksY(&;4_X`5sx12ArZw94VR@$)kVC3tcYH9)}*OR>EeY-t&X)) zYX4-XYV*OEK9)sw_W3Z>%z@Ra?$lN9V~J#K(t~M$giDcziKmVbJ__J}SKgI6yG)a8&A4g1LUM^Sc)sR679%n^#F=i3&s_iF>nV9d~H$CG$y-9eH;mz$Hv%IO&J2 zy_y7oPc-WO)0%6LjZVxh)n^$Yi7GJmD`)95l@3Kf;!X8)C<4(V45PwyDEQSh_dOJY zdHZ{RtM$go$;mEXR2P-Md;rihXzeEjIPOy5c3{2(M8cEPf8HY#BN$wg)QTlar}zX$ z-z1&9x-#c&Q8^Dg{LLG^Hyf0R`2#MsZA(Bo-=AiF4mTbcNRz^K!dJmVBe zqfA2Ux3Y>e6IfPibf}zk+VA{Ts>2W0Y0ryyoOFupOeAbAdu%ABJMQp^ zpw42e-%~EM`LLf9q~a|BR~kt}B;p^Uu*1!iZO|v5Rd@TJD-*^LNN88=+#)(OCqekp@O2)0%mL#h+y)lhRSl&Smksr} zB4)uw1sWegjPm~>icS)8Bwa&O2I^@%Qxe3S@*loy0Lgf>GZ!afop%< z%XE1|02DwMg~S$DJkLl{<0u%$>AQhYjYSN;9K#~cKvzT;JQ@eGz~tanv!wF75If`A z+#_0JP3U2tQJnOD&hqQ;l~G>9h6peYB__=<=>U!9Gg`MhaDbFJ z{XYKFq;X?CkNL799{xj09*qC;8NLxuP}^(gw`9NRbz^C)6B=M5oy&AByBA#VbsIMH zyYgFACz1{gO-mzAhkgF-S~Y8m!X1roN3jV+SY~c6SbTZm>?i|3@0V3<42Tr&wu{o@U3Rbn>WY6GZM0?U~8@=qb)OU*r-tksH6W~ogf+q z6mljwyFC9n+&0gmipr(ziJz_A;zRI7Y2UxMyHpstpFshYP>qq{+tnXNM;|14MckHc zzXH9Jt47dvjLQJHUCJx+kJTMD&fpwPBQ|foA8zG`%^CUYEyqORGk*4;Xz$V_fq3rD z^bT*OU(&&~!C8J)SAV{ZHIh0bEykmz?BU(|_R ztuE=r(EiX7t|yh7Zcqj-TSU<{?$p4zsb2`zAt52N_ShaebZB|D5gsC-fF{@j;#vN1 zS^tyMyV&O4QIW>ieeYgFKabM5{E`xOtRJSnH6A>jI)YM9C4C*E6(=c$rEv(fYz}$; zhzcQPV5Ex6xV&%e45v1^WuzabIB8z})y(Qwzc1=2nvrXx`bbJU_>G6J~~+gdb}J1*7BH@2dD)j^HT7M9)87n%{+=tEyZV=)&Ts z0bm{-{zvWRCSq!?lD+tkFte#yqIj3WfXy0J)5v=6Z1C5Fm>tX^aPA@*trZCJ^o|0v z%P*3UT~aYV;q_hMIk{QMnp2fagX#j}ap2^rGcRrKmrg)jgsMw$eI(P$b!&lgbtcI4 z9HhZMJ83OBrmTM%$<-Fz9i*l1qZyluii-Nb-dViA>e3u7qptKzM77QnE-edP0h&qW zpVG5ajbt0ARL+M>wC0zlK0br0jW1kEVs*W19d0s_o3AL$C>mubih+Z(Lg*Jy%&nc2 zm>7R`?hp>-RvGt-U1@14TIxG&dM3*_Z~R3?PZ@K7ADcjcRxPO@bg}5CO-^Sc7%8sy z_H=`T6zdVu(a|?QFWgZ^zZr1r7Ho>nqz*f+p6z>U|Bd3Oz`|{!Y4}uHGHj|A(-ADm zprEn8dBf1wu2)y)?Nu`i{~&s%@ki*1@iH1W5%o)H>2>MAwq2W-J($Lwg0xmR#Sojw z5M6kZSOU3nrjeV!zk5ijJAqr%jn#P$6(8A01T1rKlan_lZ^+H{YU=NI zHvPWrZ0)U?KzSp;r$v~;N7tatpo}}gwBU8iB)usil<_i68;2kOGQ`-_v_3g5oEh5q zBV{_TnUW(uJq2`*fY*{qFre`-yf#sIGYq8(f*2ukhjfF)7{-S>Aj?sULR>WY?>c;r{UDJ9Jz+8P-L7I0g-zXfpsfzySI~3VM?9 z{vpdCg;{g4iuO}G^2g`TMidcZq$*vAY}ui5W2#nk2rRDNH8A3MzoiqP{GKA~*@Alt zz^818Wvs@5|25kE#C4m>Z5)d8N`LHPT%5;oZ#lOOAhmn|A;2qiQ2>F)J>eFPD@Lyr z&1X74E3gZW)L(!-0QKC_2(o#@aiHgR7JWSixV(i=-NvH4GHJ2JinI&5|H(WEyiW9MHvMAc!^y-Fce*Ux(rA zF8WlP7XWAL}NerD+XWB^ynP z1}m}nX0ZTjF!!=;Gm0L`&1&A;P~dMDuLpJ)zjG5fH9hdmJS;%S-iIT2-X{iIU3@=K zObVQxC%T!lYs!zSA}uYH#VFW1Y0(HmlSKxGYn($=&?HuUOl8GY~Bpy{H zV6;Fls7htBk>WElA)(*o$&rXjWT7}w=Fm^;YA~Q_nQG^ZLHkJu9;;i7j!aBEDgW_H zBZVLcoEzEeDgGfO8|efocSTfl%F}aq-s_Uz<=K5eRHMLj%{&JWkb^-DPRXjgiy~2k z5DCEts#?DOuovi7dM2K_{1iS$0pO58`84CR5D&VgE5m6P48DiT7fqpzeq6rdd)hS{ zYXDlP3<4$AU9Lm7uBYc*9(-3elJT{c62ir7qJD}i)k}l=DOcGvB7)DuT4P&HJ~zSg znB7`RIv(iiO>b%! z3(pp$@13)a&lqZPSG#g17@a+@X$lkgGOXh0IDSvWWcGwmx^~EjKcL=^J`J!)weJE2 z;Y0Um2_QBS6>ysX)19%uv&Zl3DllW`g?LBcoi{f2{neSXMu*IuF{*yU0lH^Hn(!f& zwppv3p0{~?kKEH=R1Mnk*O!2WJE#pf2mv2b_?yMdeN~ldC7SW2$#xP|X%HCe<8wcu z&yo{CKfv-Vdm5-CygK|ql_*b1CHys4y4i*O)$UM=LlY@J5%Pvwm%CBP+FY;tZAlp+ zF`Wx5Jicw$6#{T^UkYhN!-7M;sHTk$-@ND1mG%MG#|=$Ua;U3$?n_wdnXu-jkK%Ge z?4lx5%}YyuTV|>jIS%&XM6JvG*^7+WylC{1I;ldLGJuVC!KuJ(f5}&%s8miEz^=x~ zbW>C4IhSY~^xEJQ(Q+*{29v!MQb{COx12NO-O{!Ty~!Skxi&QEzKlgOG#h||1CH}C z>DwaMdLQcPV1!04&hZo@%aLlziCwR>co*EJXaG9yi0~Fb2RDVl!8~rM(6~MDOTQqFJKB{EwljhVdAm ztXj3I^z)I4D1wBJWPZ$FchI{Mrt z6B|iE+1Wyxim}IfrlP1I`q(m+AhW^=1vmJ?u3PA!Wc_EtN@^r17-z;j(cQazb4DO3 z;poborL)Jg((>XGD~7ej1wnvLFy8zjil+Acz}HRp$nr6c$JngCM1^Laq(EH=MDkI= z^hyDq7$_IJbsBM}KXG^G=t+FGsCzWTP~#kj&aGOtnmMzxSIFA=k?$?|7PK7<(ICF_F8=lVJ579D5|&PAuGZ}2OK+n^ zmgeKV_v~{WCxn4r=k@{)^MfSXfU4-NNfr>o=Q3$#E?4{!tY8_HpG^4!tfY|y7e71Z zi0V0T!NZ!ktyFrA?j!_OwYJ3ZJi{Ob4IZoojn&}0a@iJy}mtlWE&eei8 za`~2+m^hgM4~|OB5lr2PVi$aD2~#9O_HZiGK7TH%{&W`ZFKh7Qun-bpSD5QPV*bL+uYG(ud)&oNiyg{#C0FWZ>2{)O!;eiBy_5h2GkhIp|p_K;vq5jVN z_vAXJjY`D0oe+qN{4=_KsY@R~G4MCQZ5_&iR>VqmNil|$B)mdnm&EH+!sDcSlCGW|A}TgH@y)U z_!v&VC=m>YeRlAu7Tt-99$%IAl4<&Jn;9^QqSX+#k5)tiF2JvJs_gq5i>f=hVrF$s z!vDuf7Gta1>oXbyNAtBZ`D|4|HpExmg zOMB=NN7rW6KdYIyUCU+~l!B3~><#Ph-1Etb_!miVNU@TrSpCmBg8uTvtEtln4w`f< zF3|~oW`@9A$?rB(-KFRaZED~3*k4VG19ObEE0r2H9Gi8k)6+{#YVbGAvidfAPT@w4 zt&dW4@$(Z+cL71br`t0r&wQo`PIocAo8?@gq7oZLVhcDr3+gNKP ziC$6&gb>3g-oz9IHiw5`%3YxTFebq>37X45su$tE*S_K#G;3|2dhb$b?7_~x$BeP- z%6VN}6ty5RLEqa(ciUpq{#8+Lhyc*V{C$3ECvNiRD3`rN28kz>B)*osXLs$2n}LD8 zw8(2db{PyhvJg~6AS9FGs!Qm_#n1LlPO_Eynsg+o!bMO~<^g2@|I~Rvt?-<~&E%{m z+=^B5Qz072YPHvRZemZ&#r zZh94?x-;j@=|^vQYjtx^RyH+;V4xIhG!%WOB3hF{ECiq{YaZr+xz}K&jqEwSP9Wqd}o=a#v>_bpJ=`J33`cukW-}0x2 zb*z2L40;p<#QkjaM@Qk+Wu)MRb1k@u{-Zv z4mjX@430?%ODJwvy}Q1fc>v`@2@eOBBu z-x?G8>~LGgaow@X%}ph?=amQ1i5k0KtR8o!Fw`r^s`Q$tudgQDg(jFbMoYUW-4n2) z0=ZqdX*b>NHh83GJX#UXaJAM99sO_O{cluFzc>cU7+k-x94-z);lODZ4b)ISV&vxA**=b^cY#7^dAuJpM*HY#Hy{c~ zSXZxJJwhamEj?j3Ha$DU9--|QwGP1pC_C`I4h7;9cYh%Y#))&vbd`L7|&PCG*nbvgy;s#%|ClFf7Hbph@|9Q5- zQT?N>W2T%udGwo(O;Kk)x%r3Dj58N5?c+1a^UIcuK4W>JrJQqs-Y0D%e^N4_gcb;e zfh53=BQOFdlPkE&3;6a65ocp|wja257udPUw!|u~<*SR+4oL|DSPMx10#a=zMr9zSH9zqIacp*`xPss{ z#a7_!{^1XZ7vnSdY(mIVrc7e$p9it%RV2tyG+UX*0ENl6csx4%3H0*ofkeVKtpW-> zN70D0zHHbEWRxBorf$$5yy9VdEw5T^Bgac`}{+xBc14i=xPoXZzoa6 zKk6NP<4i&OMT0S{L)1JY94a^j z^$`|`g=L>U*Gwa+EW?{Tj(q83wITTX+}MLwhnBzd>(I~tca0IDHMhRETH8<&Qc?0L zJi_qQ?Jahxn)N)&OS8jQPbu@#Oi~@XL*emeO-9ng;o;|-@Tsci?Jct`$i4YBuTE0Q zm8JAkQcscMH;^{pI$hy7f_ol^tXr4)qCEraFT)Xy0c@Ks_2%pBV>8l|+aU+w9sxTAhV zOi-}Ap!AavZ@elW?j>c7^hkmaF}k<6-hqi@IQQn%Cx701b{JEUsL-`gLOA^2JNII2 z!^qO1o?(Xs-DayGMD^0A-a!#qS0h6)Baa{iNCzXAU%|MChKztI_3m4=7 z9Sjwfz~G{1g<3yh8JM4P$F}Ol2BYxeZ+erP=DL4TtuFao^|Z81-9+$+n-+D+w%kmo z+f9F8B$*~472_K(j`rJmgU`LJ>Xo}ibaK6_A>CyepHPiqm7jTyd9TPp}>=NKgIusq>A3Y zVd{|)g;n7fQ|d=Ld!3nc;F|V)AGLiOj{7ZeO536Td9>pGnvXW-6(5KE#A4L=wc62` z==SZ~gG_&3dCUF-`+opbpblPMz7NC<;<8_sG;SMO)iU2}`5#3>=jBrfhEFjVP;A~@ zGB8MA%`sqgbC>=U9Q&7Li`t9670$3#x=ng~-A8NU!vN4GZv_pPz(x_^2kZbiG>POR zmSVm(^PvSv9fKG=gosI_Q>NrT87ynLY|PbJ7pagjQt%kf~L#}o!gp2Bu8eb|ON-WSjvhWVFc<&Sp(dl`|@#ax5x-w+%H zj1%465FIBGGmj|UE}4=n!mh!{l$6s@`IO{Ql=p6yV0Ja&YZv;NOZkR7|Mu+_s*wd3 zX{8uISe+RC*Sp+cHNP$bi6ztX%d!Ok?ikJ6$q(((~8?L2J-+_ zR#2}r-=3t0JRWwSXY`=Dgu1)duie|%(+jYO9bDe1{N$zM*WxcOulFN)YvG6JCqW(o z*KdBR(9w+Y>eu$?&(upg2DjDAP7WV@Ve+|0=>8v{D=8?sQ>)#O&EmvX$hQ){6Uoy} z`6Bs9cXZ2AX?^|64&6SuMcZ(La&=BK^Kfm}c?`X51kwr2L+)3Daf29wo%NoQ1--?N z%P%(L20}|Fi|<**`g3J=;|@blppn#p`co}YKdwh-m_f3c)Mc&_MjJB02};(ADiF>6 z%E8Apj1Owh7-qNSQTpqyMy&i)xv+YpwQRqfGiMI**fO|v`p%s`Q!q_u%wp!fO;=`> zk$>MI4rN5@;HIt5PGAtD3c$d{jJVU}FHf^p2B%0(1PGri=>-3yN2k%l!kRrP-+^M! zg7hP34@6y9@DDnN{GUI+fNa~Zs|v70n_|dG8H7#A%4~~*DAV*MjZ}v&>d1zwQ$v~? z+*))TADY9TqV#_|nxt37)|%;l)bihZPcJ8Dm!JRVLOcGoUqQM5xD_@8fpAGQIuAex zTLIa3;wV-nyEyk#&@V2V{p2|~K2r>5Afwj2{xf~v84<6k8!CwhFmth>4sC{R>B}*B z!U3dlJDF4CJ;)PVd>_%fafmYcUu?4Vpxy|}Z--7tSX$w~Xr7ojdBkbGesf3VH4Bcn z)Fz-kop_KY>9ltQ%|(4xpEqly5d2;+50bAaa%jbbNT!S2q-j8JbIQQ_WYVpek~P@A z^aRgXjLXZ1dz+nU9CUF_PdL`5s^whC)S_eOLW{Z%^(hLmR7wfdV27aPjfY%CKpju# zC6FC8fS3}pFiMTSMGL(NjvYECxf3^%KKEhkPO#c~$)-9N8GU~I62==MVm#Q3m|0Qi z3AzhQc`|tRP53>Cj-^2A(2k<}k@5*CgPDs)L|d`LmG4ENrQlY3g8|@IpX!)HkvWjZKCPbJU#mI+#oM1KbLDG=5_q#z2Cw8 zLs>=Ex?l57uDLbSd;OrAziO3Q{F+LnxU;#$U~h%khE%q*G>)z6tAl7KDFkbBUJ*o=fzr8nz@(1!Eiz+p*1TQ?X_{bic~h^C$Z)8~JU(#kSq{XO%;Seac*(UV3Bu=r6JJjE?FC zNpDSveFE4F32r>3H_;H-$&9*37$OQ@+%fASexdemEW<77C0Wy;xfWs_wWRQ$>hSH_ zf0nA|8`tBeq|16L8PjY3rA!gnT{WFpa$w&+mpIK@O@0{Llvi4>fA%T5eeU*v(Aze1 zfI)oYmTj7c*z@56#iKr3%$rS_T6sV5^x3m#-&YXQ<7UCl;??+xqLTBJ?^x{HJYeu+ zk0Abny-4b1J8n|lH+*IzPNC`wLGxp3D_2i<>lR-9g#GH!k;uSbI z;D%GhCbQhxGtawp%dwUrb#~!+KXz&2V3XVYMfTO@8I=j`3_loKHBxKUYDr=T9sjqb zadV&jz7;m(+SBy-E`vorL6w-wA*MEjJ{$-%{BwEF-6yIlaH=bRqSMyV1mMKyV{(46 zwQBy(MQ3PGa*GlN^EvO{kwWeI#;=om&%1AT+l~@XhElncbTd_gO~oBwq>qImQ%#p@f(>%L*&;`#tG1c9GaF?we91Fk+pf+ zZ{Hj;Q_r2fF?Yt)01-n;L}tC^|9)Domus^$7v9sJe=k3~hx9SI_UTfy?(k06Qnpqn zP?s%d0DSupFS*d2y&s!4gF+C0?&h7h{;6z|hgk=<-n#e1>H(dU3jO}fN!yuD_O?_k z`ZB)9uDHCt6IAP66Vqv0 zsIa@NX%q10W=BPS-8*KC2#Zif)I#%6=VV`0hGhzwEgC&KQuQs-V-VJOaaY$9u*U$T z@?Ubh8}U`P#u`)>O`)Pvxv&GxEREk`;*~6@QY}G7rURMMHfJ3@_rQp5e?O`EKa(HP zF$%-1fV8K4)mm%BpEa97b`_x!imL*ki$zQ$fk&va0s}s8QZVBLUAUf)F$#c6B!rE@ zqh?%WWo6}8bQx6e>~`8Od?Xuj$nSZf`^DWmHwvj5*SS7rmKLWM;l_HbzbHeW@dsrn zs@Tz~MiJ>aEsXB9Ghfvu8XBkK>L`dO`KK&vhl-hO`IdSHXiR7dhC@AN`pzzD&TVAS zaHr{@j>J4h=-ME)GNpjPKnc7LtmG{JBlS2!DaNYJM3}}Ovx3J!0hyio7ndbri(-O5 z8O66uDaj)>f9_@Z{ze^$4ATrUiyyB>C;)Jk1eBRGi>Wnc@ykk`eE!VRG;OMQX50AF zTvdhwKv&F3*mx9lk*Q9Nf@2T~WsC$1?sSHq1TiBH$tJUQ^4510Jnaf(({dNp;>+s` z4e3bgJYbGng!Md($M4^}!w-pu%oe86rE~X-wzU-yiPe!W=i8XSueQKOtIW?<0IJ7L zhQDO`(xB|~5gE8MGD`MFS1Nu~qGh67SkC@xw_>^~YApR8g?|byg~*y{HU8!n=pi4X z^H7F$lOyG#=Ef3^-`(5XN6<$kCU0^`d1jDQ0y0Yhf;QOsfc9Tv&Y-v1wu-s{H0LIG zntJHl)x4FJV8&FCLIthQ$D3sd4HkXpNO?pTLuqygnhr#45i)4zc6-069Pc^!_4i^1 zw`2xNte9hBdYM=oU4E5!k#gCEJ2^A*k5`np9VezwS+ZlGg;pE}^{YksOC&KC1e z)3lrDkrixnc`BG2szt`B?519$p%gukWOF)wsduxBUC<% z`l|)q59Eh6lZ{P*n{s9+tmIvaK#Z!~{IxUK8~?U9NP>{fzzzs*JK;|Lh^Jp#lrT|d zt&SZtvw3Lt=Z3hH^nTh!3)|z<@IYd*L`+EFwHwF=c$qH)BOL5)X!Z(m-*_MgY{4gt zIsm4(Yew@r0$qR-F^Yfq!go{+ek@9*7OWwLTOySjJdabJkWKwvD|OCb(4_{#c|sex zN$hr&c^hZmO*$i0{6`t3Cu8^5YHoD-WKkE08RwffM)5F88GY+UB?jFn_6$&1RgTu4 zP-xKwSsm;SJ0D=;hADp98V`yj?u`90XXhp&3&2E0llT1J?CSkmW_sPcIXt8v=Iq>V z{pb`0G5 zw^mAFUqhSeL8?(d;uHu)1Lz)`U-_ePTl4(KkjUb)Ssd9>@g-z)0475o5&mL?<|Cyq zdNpz5oMA0`&35CY7ACC;3jh>UrF?E3qjO><0uL}g1(K)KtSmdGP!US{(n!vnGuo@* z;IhJq*7J?VG!R!b8OZ_j@ng7Y&Fe0MF{oSsjOL)KqcPpQxUev@!h87SbMtyd&*ql)rww}p^!3UdORQv2w7*}N5jeM!kkUGtulS6Jg|E zG{aep*3+`szn-~w@1A$pH}!jab)NIIdz(K-RurK@kae->c#ggFuC6$uh8*rZV~qIo zsMr*GBS2mf3Gky3yB1^|XE{I)?FJQCAFzA4;6dg?%?8$~cE&Z&+Nd^KyDe1{F^11p z*N5c#KB1+LF0Y~+6S*xNRUycnseX{=^nb(~?Q5W*r(0hOGQ}g7NHC?sc{~jnHn2^F zvBF|`cH_$Sx@wL)%D<=SiZYngL!ZMZ_UsBRQKSTxiEo=^u#|muR1}i@9OOCrqW4UTRYLb z=~JbqRpYf=9b2%_{U80Mj%z+XCU48QJ?B(J>{L=O>75gF?Wi}-$q(yNL*_+!kZp!- z+vxBj0i;0~U)lkV{=N5a+E5jWLjgf%-KPgNqK{b@>h4{)yznB=5&pAXM!_l2QW5Mh zMWjL@iFw7c_czXVYvaSoq_R3d^WvV|R%D2KWC;VPb!QcWNyoa3SCgWNoI6gxFy$2i z^+Mp{r_IE;X6DQj%a_NUSan()ZjszX@w*`1&Ot#jJQ=lWJ=CzQ@O#OJ#ke4r!hhYS zO)|vGVd*;gG=7 z7zUOsZv8}2C#*%7tNR4FCHBsM&|JXB`YAge1IndjzT^cnmYYL)D4YPe{)sCm7xz~K zcy5?tUD94`@AX#d!${K}Oym_ac6R5&F9L%M5%WP$&K(=FC4Ud-jSH89XzFrij=quU zw++h4$;pBb&}|;Z@qn1LWVU-AN0p$thm4ze^Tr}u5s>qkRV_m-m_Fl&JcNrRo%A90 z>m@~d)1rTV;pw4nQ~L1c;I|&(XJal2!9lXumI*#{gC^<$3v-sNPPx0+y218>*3ZWl z&z^=;Q`(E1d*f6K8wwMadb(k1k5R|8r=5Bdqo(Nl%OiLEhQ&Cr-M)Kg^4Zy1{H2B} zz6OdPA44LaSk1X({j6;d*SO2G`Y~Ir>U*Fut3+P#;cv4BkTU!%78^j&Pd5IdNR)y_ z@*`xh4a*gYIb$Fp#KMP{sFxkfF=Y#t8yfe1ScM_B{QuZ{^SGMx_y7M`W{iEN7(3yF zFqVX|C4`PGEw+#)OA6VSk{Wxq>Le73inIwWw#r`0S}MepAxlW2L{Y!{bz)}T@Avoi zyZ!$7{q?*3x_#dBnc;NKdA(lG=XE`<$K!Eb&zbLgRmx;-yp^XM_EIrG_?VR&Pq{(I zu`pL_>7%H=vsYeUmKNqGA!}r=3cU77lu0!u3^xNJ@sbC5zmQ^{N8`EQh&J#UoHjv|@X?Lk^1nyx7 zml89kjIaP>h)3F@73rskH==1v2wQgU5Qb)sTArPr#MKMKLqA(PjGxjRh-F|0>QJSq zxBzi9H^qA!&L8GO3_>3oN3_C6>BY4l2gZbp5jAybQ1U2F@rWwd%goKNTOy5j4O%8c|U*eM~0X7=3Ea+`ehM z{44XN@;B9z!_}KL+h^M&Nc7m{aO6u!j(LpzUhWGP|CJ-z(SqwDpTS zPmg?%4N?%fR1NAs$yjR467LhspQJrM&rlHYC$#YvrI0dk2|Z=p?HL_in8!tAtZ|UQ z$8YjjN=(8`6nH(lo5Mw{FZ#$3q-y}5Z^FlqGR-c@ ze^?TKqkG(0^)&GRJWvk1`Un+7_blJGT^6(vrk$Y>!}=s0pL4V6{{78;44t+<*jl}$ zV57sh-6+?g_d+j~Tps4x+hw8Qg0bz*TgANcb(oPE{%r2)BTHw$sjqE&=hNNZ9~VC@ z9lfc}m)(h@o9VB~)w!^xLc1F9G#C=@3p;|g=s@nlnitl^}|F?Ltsn;N6DP=jnqO>gCF1Za;--TTaJSvz$IBo*ZK`-2# zq&D-Jri1L$gV`!kC4t6nTEg6dT77hNN&b7av=ody(ulo?k-JVXA}%p{=7zY*MDu4< zmQZI6;gkr1zs?3_yz=brU@g7u^xfgNT{my2c3|q3RNL*#ym~%azAUWkmMe{DL|yM> zxXeFq>+CjXOB{Y$A97%gR)$|Pp-`&}b+_Dc>RNA7$Woeq$REfzq}qq`L&_^?{AmUC zBYi$|p3w&}9pXV|voJQNE(i|&NTY)G_SF3dQ8f}5ALWsAP=rFuZs!aDucUn*qCTYK zHyyd@=ZxQK>3p$z^Ev9pe#6z9E)A{ilX7?NqHT*SxI^qFRgGO2YXJSaPH<5T zLj>5PsK^O{kWK?-nhJMGrnym^Nyh}9T*^+Lg|Qqb=?;@Iw5-HbnJ~%f5AWrJ^WB>_ zTrit8qQ3v&{(TV&Iz%oAAW3Z8v*(_D*>!?SP|y#nod5)h9B4%FEm?7^_s#AS6UkmZ z@O;4zy#D^s`{2MJIWKv0B`5kzcnhDXRpxP)-3|37`*zIel-Qxq$B+UGtDgCCV4Huh z@FBOR^f^9y&gC;b8vVYasCw6FacgWC(EsNJTv*B4!Byy=_^GY|=KT31(%fS6i(hlY zQ#z+AE=`D(e(9TojJMeB6gMKpjB!WzRjdBiv2$wd+4b&hL<{BfHrbc4JKxJhSD&=2 zQAYPOC(1*&45+$|$xI6luy8D3b?(O2>Hf=}UJ6qBQ&Np*3WwZ&=2o6zA|SKbrsJ9_ z%73^AE+~o;t#`X6LaXt(G#O!4`8ng4`;7a4w%{lPD9xKY$3kL@cLAGmRK>(c|L}vHc)*|> z=u@j%s|ODmB9j8ZTR%*vOeVezGTorTn%DZxwXqq( zEv7j=I{$fCn_~l}9k4-nS^e~pvE@wtq<2)nGHxGF;04-dY|xd3Xez{R>#Dl5h(NCV(umS(88;5U^_#=3Dls4v*yE&ybt+su%Ha_j>GRWI9NCw*LS+zHrH(?O}%-U5zP?yzcpWV z9h4|X0Yf~-Qr#Y3@@xdP#~8*32Zn2(*l{^*X&sp2t;M~TiqOYH7FL{S+cwJ)zL^sO zwo{vME?P+A;ud>)QS3_PB;$EjKSCC5bG4RP(8Sh#R~B4&)zH34-*^4$eOIDHw1;@3721xNsuy`Q$DM$K0!Ke1F3E`j}b`ZkRp~_jCQFqWzCD)NX;6 zY6EuFLRgt4&=HXACFf7S&3OSQksdaoASQpBxDvk4f-YoM(6+tPBaN@9j*H0*!t3(-dZg2ZQk#0kjq5wC`h`?PHje`02^Eu5wzm zO@G;`r}C8Pt@UN~SC2dPsL!X8&*%5|U7TeZ>-569i(f^O_*p)MuWmc#?%~;2 z9((pLcD7sbh6<0V6+N>SQ##eFf!n+I_1w}&L?0efef%zh;<92Lq;-g$d-S()8h*i0 zVWrpFuSs2xv1R7XY)W(nDe1QfBz-Xv(HDRtRhcwH&GmPdQ|(>f&92>hL6$K&ObQuX1SpAK{1Fo1(%O#5xBUt1MgEp*8m>uOl|z3q*v?$g8Fa@ zSC}K82RE&0{!94sexyv?py`MpD6Jzxug!59A)*-z)C_nbjXIdoSf|0bnknk`0%uN= z>r7->;-{pA%F#{9cU>E`|bOVeq7eglc-4pMMidPZ;s&rzejems2jJ+exMq@WwpY zD^PIK@g?&*3!O?sS#olevWROh43&1=$6#uQ)RwJ!+U zBW;Th7!8gm5cui2~y^bMlEQ9w}muYhjzUeAW9kH zh%)WjZ{*7j#14*5y3Kc5ylk0FWW?UU(9NoF#ae+f)KXfl@XSghEVFPuSe(r4nZC7WUw9ro zIW_j@W+u*oc&z z%wcmK$h$dI!e$Gv{+#hhW_2+5>j;U$;d@!R_SeMw{q4#<-R{>e;Bq~8HmUO?{84oy zVoE?$_;POui8Y%JF&8JPutlOpz$Xme>~)l_=_bqujD7V!E~CoFwEMyB zubdr6U*>4Jb>;T3EYg<7b{e0_*t?yo|8d1^S^MG{{*$r|AI{oat*)TBkyZ*aoJ{nR z)=Mftq~hbIHZ_%gF0C#~wN0ED?P33#b}Djq62Uf+(;weu-==hGK4hkL;emsT?1y*m ztbMy6E_Hft`jvp=4fWcpu!8?Dhf&&lv?K)OAGSMN=XGLkA#q)odQ#L^ncNLCF^_#F z>Z|+|Y@sgCT`1L$so9No5|-VjX|%5E)v!%F?#^Y;- z3#8hL{!Q;4=kYD(ub2~YA~ti^&CR!_cT*7lwSQMn*qEgKOHR_TZ#LcnzhjGX)-iem zHBNpBY8LT3H*hXacbbj0`qt50hUdiV9<8g9QK>eU8N%@Y)70T`Cs=CXRH3xrxdn`)XYWn9Zr(`huqUSsvwIVtx zQ1i6b{6sMe2NCwXk)alQMfBZuzcoM3c#C_{!`6HD*fvf2tEoY>`tC0o^Y1rpJvI8< zW54z=?aQSku?&i{`_lXq-P-xxerVfGixU~yxj6p6YwBwh%Vt-e;G?-9%|8kUYx?8< z`2#Mr>#qNp^)G+-|G(_tcj*7aHzZs$m-OF}!SgiItx#d8c{92-7N0HjF?gU?RxkD- z?0dO5t);~ZfG~5x*YgiwkPzm;9DXV*BWJe&W31n}@oHJ)TQ!Dvl931h`Y}->9oGlc z%en1hF&D7P`=@5+;3F2L95 zbI$(iAk%{!^PfK-#?gJ^x~41vrKM86M_Wx zH(&4I(ffX-0}G;MI9i67C&gef9Ym%;VehuKM&Tc6Bc$>5Uq9-gXP{2_tq=5jKV{w- z?gkk8ZI-t=`OOt(BI-T=xO)QjSvVlDmmD-Khj(!AWyW>kK#%$!Sr zfqW@j(|C3=Z3R%1{q(3|^Zqj;by!O@m7&3ZXLTM0kb%DE zE-5zL*`1bi@ODys2Ik+sc(HX>xpKw-d?Pn&n(TgD-}=zyZrbF0V2R`ZysPnVOqAZ< zvdOANx7!~Y;yKa0azM*Jn^Rcn{0e3_iOfz3 zCCtT3O6u)pJsQ`EwoP3}dFbOhr;4PKeTWq?|o5ge4>j9i_uztRn?0 zDghk)UX_0i1yZGi)vWo8rcz!0gAU8E`>WUR2;YtjK&4x-;szxoZ7WrnKUjKd%eToR zxla#!R}K!50RbNv^e+7`oR&ZjD89s&fh*{`t#S~vfSB3Z!J~vv8u0B_xwX&?(2<2` zqE5#-g$;Tn3^C)bWUeg+E?hnhXnD0sPeLeF5IMHIUuFE6pvtK)6Ri@~rX~XihfZ1x z+XMOXYvA6>6{yvNszWR;OjzN7SB)2)Mo4i5Qw36g`gNFaUQSCR${;=HrfgWO%8GWb zJP8kIuHqZ~MeQuLKav(w22Uwzdbca5>$|wPD0qXG&k8Hr^)^ngg!-Q+j`lZ{=06lG zQ{FF`B9o@1ow4Pt7Aen#&*QGi92g-TcQSZIM!5*h4{kTW(lM`MY=jN@Zp08@h8s15)S_yIIi-EMEAj5tkjXDZ$?vKvi_$hHxZEA7nExlC{W zBh_}zv?1yeqrb7HTG^*xKQD#|;N%uQNbGrtLVt0eXlrUc%$Yfr3iRdm9o%y`7||8| z$F8QC=DB}AXg6{%xRVkG2eO*lkQ5{VNmv1nHk;*B&c(f7zY z;+P;gQQQ=M??&vMe(X(g(2{p?fSU1WN_tjbZ5sV|m{vBC&MVkB88%FcxA;tCYQk`D z__ZGoGNxos{@IlUW5}sKzb_7$bcXdxBC0AwG&#kmjz9yMRf_z~hLDq^3gyT$ems@j zlr$#v%K$`x#ps89jPdH~iJ?eP1d}0z^-)mVjb6^5U&a?Et#T+5awgOxZ134gtqa`$ z#uT2)r%14~>9M9mp@}PuV*DrMdSNBF#Mk6bi=2 zXh9w_{=PcL%qm!qyC$_#x@Vp9-v zu)rwUGDjGqcjDDu5DrGfZcL5Fq_H6~va5o{IP?DT8TdLr9dNHK?3H?@=mJn#HSi++ z1uGAQru|b+)7#W`Fjo?xxNARLfEl6LRSqcFkDrKaCq78DiMP z?`J_asNVF@l^XwwmiUBC03mW#lZ?W^P6u?_ad>voIZICZ+<}>BU%LcrZ{P?DRU#iK zz=(1DoZOLgy|T65SRIO~ga1!Uce+!Ugh#E7U2ZDroS?&&1vT6I6x$m=Yu0+AqY!Nkhv*eZxPp73!M zL?8UqzPuPpYngoEQc-S}Y09t_R+e*c_YqIDzqYOZJwbo}{|0vGG69k4f|Y zJcf^S;QDUEhV4H6S1+nq2%XdFr@wDv63cqv z3}c4m1wie!{e60{0Yn>MN&99IX5u6NGO2np`eb5C7Ts;#YSzB{v98K{_J5u`sMSi~ z18+%PQX84%>gCJpMe>N=&{LTSzj}dFJQN{t6&A$LA(T;m1O!^u!5O|K6cPm4zI% z@O>eGT-^B0lPHt4aEZ%^q;^>#a;Z3+q@ZQy^ha^xB%f#d7&wmXm$Co6-lz|M1+1V}8#YNJBvZVFVJ0u&Q zU@~i0&n)1IQ?qR*Q;0s(pqC!!<~Dd%D1J-o&&A|0YOxzg1=6KM^X!qnL_j4SZDejH z&i?>RZJ58}=QeF3{{eYiXRAL581Kn~(h|Y#fd{V5QSS|ik7k+Y>0;o>mgitw80(fm z668=R+x~dh%#!SNb$T}uc{ovK1X`sWL0Hy&_ci;T3$oJRB+dgjfS&q8{HK(XN@b!c zCrv5f;W=scBAS*Zy8dQFLeUDQ&zSMc`D#C}9?80#i}dp4<3oqk%y|q{!Q|G4ik9l> z4z?HW_uQ;kL4;_rX-QTDBa7VtRW)D=Q~H!0|6RA`wuN1-M$%g7_$Q+gD9xo^2`z7@ zr3>jE&iN!w1r&?66@bFq*~If$OSlyKjuo{69cgR#Fy}Eeb7l#ZobPXc* zj`ypOio~W3;J%Cwk5}q1PNpOcjb1Lwn>3G{oO81@;nJ^eDDodQ-CVc!H+gpc!!HS? zuR2K)hXf|FDoaNuB+HO|%5|IBvo{j7UllvY^tW)V}6 zK`t#@Upx+ywE>5`OG7a|drSiugcg4yy3Y9nCsAM60WTj!y(p_QRhcPsF@+@I%t2P? zXZzQbJ-%Ql8Q)5N7>%4JXcT2=>&XNLhxNEkR4C}bkbzSFj+?n{Z48>`(F958fSu4c z%an=KfDLgppdfqT`=SfJ#k__asGDTenAm6OaHrvz$dRm-Ih?6#i4fflQ>@v=xy#b+={nXajxyUNV; zv_etgbv@*~9C~pCN>EiHK0YSLHRv*>u`BMhCG=s*WC^wjIOq~&4XHLhvbCFuR&ai^ ze~MJYHbiU$6WYRlK=7LFSoO;|=I6|$MNQ1d_?6ULDNkYuUyfT45MoWZm)Y?&ar`At zcH&Ee4r3n=*fA{kCTC6G1m?q$A83%ss%;PBUIL2GWcarJw#$L414$2KFDA4I*dNsxJKloN0LU?ZiHKiCM5 zIUaPupTu-Eq?$~+w3vi}w(YR((-~E9I2lk`%wR-&Q z+3Z&`cty-RR21Ap&wI6y;;4lw#$D;7+nWM~Wr9F1^Wvme;_J>+NVF7$bWq(y`d2|q z;)Xj&v1#@Z3(xAWw?+m~fRtrDMZh8*4_)2)Y{pWBu5ydXe!jXo2yu*kFpBD)s(00b zKUoAKC1{W-s%ubR#(gtmqXm9=s)(7TRXcvqj_(hNtS>q>V3Uk<)iN7X9%auha;}VQ zm62HQm{QNcDd~m;b(mulfnHk&$RRdVmBWu1b>~TW#~w%|v4~Zf>l}5o^;TmL(Yt#s zyCPNxWz-k@C-!2S^a}jrPn!B(n~dJJZOPCL8yaZnD8>Lc=&GvOd*LJ5pd_6wU>Z;Q zFr>|oQK2NGk!$vw8@A!Q^^-;qw###>3=-*aoY;iurnA3_rJv}tq@vszm01SAJSYTF zq~eRm(a-wD3b}!V=$pT-(zw*dVGeEg3@oDWm1CthWAq}$2se!7=E8#qeGbvE_xY$Z zUEjov-_2KF*&jjI9Y{sSM;m=wk`1RABKp<@WcpBFl z-rC-sqN1aNC}vY*Wh66^qM-9I&(mqzPv4&qlyLgRkP@F`6NklX4_Wt9O@00N*MTBq zYWbWRGeS}}-}wn*>0d3ZCR<=#=j5Lq067_y#)lcDV3+YIGMd#ChP4j2Do;MEK$->FxjqnZ-}Jh>)tqFc-c}NuqON zWhye9!!+S%>ueEund(^kMw zPxA9e1M?XGV6wj~lyVu%t;7e;Dq_bmgzr(m?c+7yHuv%pSW+aCNt|pC@pCyXO@COv z>}J-jnD)iP0Z?Rw+GJsi3Op9xN=X`Jx*Zd!a*Tubu0=F&HE!S5`0B-rf8bZ)OWrkkd*e_WhCovt2Y!e*j%yWFwgTKB(wtU7Mxhl!V600sIS zPRBBtHNG4y%-p4QY)1ItyXOoe)+ni7%3a?4xpK|T7ZnwSFOu6-^Q<*!`~8{Y!?Oaf zShuiqXklrwFZ77jILnv}tAc&#F*R`i> z&ppeGi+o+2aO~d0(QRM!ja)jl6hmYTWY`Zs{%8k4XsSPJ3tM2uPAa4gpwt`p=;t46 zWaP@gtxmARKUA;&k|*j_^z!Zi0Nhx#D#(=IzrMR%KNPIR*+Hfc9tH;tDN*pFQo7lZ-*)j=TH zSIkS@6&M)EfnGi77x(^c@_179sgFe_HK4Wv#pmnx72aI6=qo|uHKBCv%r%tV@}}02T?ew;c(z`=C%zg7Fd}a3P;pzs>nuW19?FbuZ zT`Cel+!0h^_OP72ZvFcG{Tu1BB8I6gQLt3%cK`youYrYC*%V)&Ogz?CI*S%&e~YQxo@a>s>&s!+23?k6!w|xHtnwhR+FQ z$cBmf(&fv#D%h5okLjy6?j6-`pbnkPI+y_V zjt?87`70gohiFn@j&u~#h>0f#qlU(gsag4JgMV$L%QTg7;OYs)z-KH16xx^cKHg*5 zhkLftyv$@H^sqd|c@$!HsR0{7ut$Ik^I+Ry{dY&)t-PF&4OfTO1ihxQse3YfF~&A^ zu+aLX@L_htInD`m$@jVDhQYHFdgXxjNhguOffSv%N$d=?!rD5 z?Avj>ZQs77ZqDtz^1QpI-R5Kh%CAtau4DiHQ&2GX;RcN)FWWPQ559dDBtQ@?K=!~} zeVky{Dr$_|AHK)jE`&-9?bZ|?1W`B5vH zifU@Nvn7K(@5)%EU8|bQy)%g+PQ052g&k@ZxQVgkk!)6M!9!}*sk4i-U}>p13<6H! z%&a~Znw$N24=JJHY@CE4hPV^@j)F6M%g}v~oyu7| znNZ`sZ{I!yod9SH-4dJo`#G@^XT)LX-VHlY{NPz0@rb_gJ$%qan5&wOp*xs#_l zo?ST(oM{RF=KSGiZwYl|Cy!ZQSVYAzg!Z61JH{yCSBb#PEkU~G6Z8_V>A}_4U-Zy& z*R1A{@h5-P{SpZVez~n{0#~_t866H1t1!8mBv)paRiRj2hp=EI9$Q-OV)AInscgrQ zJZDuREkRT1V?V(h<%qp|8$07Iw_}I^^{rCHof}6PybO#A>KaK+J`RJ*-HhFD>DJ&I z9bsbsdi2JZpI*n&t*X-xCTASx?R=Oj8cGA%?f`5!1p;>XR#&TmHpQSvk z_0vyxsPDa@!Hf$fWfE-DnYN@dhk_{Pf!c8oQCj`sPQ7)?X|d&aLa$|zIIu#>&n=Y= zJ55W7`nDf>BPv6a`}mADpT1w<3h}bB21pN?XNJY-8ozD;w=3Tc~rG(>7ROb z+kYCQ&|QgQXP4VAby~J88W0);n*`B5WXZ~vr3FC)u&1vK5Q8EkzGgp z_Dr=n#+;#-;7Us>A#Vaiqn7m*X|=Yk9Ki|JyU*wxBI(F0Wp_XY+H*U&8~3PNqEXuz zEIA2EI2B5`=h`o;ws~nVrcNKuYo5xol!pj@J|s~I4%$?|7pRsKoJ{UeUgTGIm@*=b zfJa@|<10O}rjJ@0M&vy_6-i-A*c>hoj5LwK+)GIj2r?mDJO_aJFuyT`FLV9YtuX-d zJG{sI1APyIEW_wubNuw~5Mr?ZQ5_ED4W$UNI=?nv83H?+;?>?=a-_=LaE;8ujK^Qi z2{w1p1EZNe?lY>L%6+Q`+h(R=7E3?QkpUUuV4F}MF1x&?~MT_o` znAu#qg2kccdsMW1AMmt_k<|?Efa-avgRejYEr-i3&VM)o{%icAMaL!Zv(7>(#U1I< z>IpVijX2&ygJ%^Xo{SNuaz4h}zCu>rU1S?pQt@kR993hGI(tn+*s$j*3eGtrXL3kH zl&2^QWMX-;Ub}X~_?+EJ){O)8+a6Aiis+raY6#@h?u*m!uqM*!tQrSqAtafz=5p6d)60{5QM~U z2tA`Ay~Y#5DTVqrsk3QMQ_bot+1h_e?iOVThJoj??uMn;ru2MUP~fkV00>nFk%z7N z#sm!?_ZnR*ldSaY$&&zZP|D4lH#^#BvQL^n=7edy|2Tq5!$ysQVcK7k$|W@@=K*#c zUmRFTb<&CJb7oSAJYEiY50~x9p{HBj+`2bvB}x%D#)|<1lJl1iTEB1KWM9KeG8dCq zm{D1!BmJn{v!nBY07!5V0RE$l+^hMPJMae`-S#m(t6b2crs`ht=hw)jq#b>K|NSKr zkklM!Xg@PWN=pYd%g(EAKNirKN&VEIMT>}Q*G#y_10ML3TY_5BWps}~#3uWsbYHIF zR-q3^OT^&D#z3tgjajnCCNRs{dKDqAQ)lb1tK#;G(P!yHm-DwF(${@T>YCefg3tMI z_`k{Lzr5eZc=SUoE8mALnm12poEtb|3};Rqx2-3!&Zx8pdWF=&JWe)>1v9myIl1-2 zts2l{DQewqpYLj59SijsGnxkDMVSB{T8!#;?egUzYI?X0`f$NGB}}{BCKs{pVs<(I z-f4v0fXCZZhY8+5=>8p`sJ zQCwbiQ6bGcKHbr~h$45S8sShnzYz3>*4(P)w^DQMT}Etici*F=gzFcDD_;6SiTM63BmRSNgbAjx1PUP`StP<9ok)uUhFgPoutCl9Gf0~Btz-;W;JKAOY zSmBR9{ip&mouJ2yG6U=$?qeM~L*GZKJEguyxx;9k5Ywdt7CNoz>S4Zpm`!MzQ=?|h z>|cX*TpKb=%2a?d!!U|chqoHgGgfl)_U)tYJb(WD$lkqsHA{c1!MU7~SdJwJ{CQ^S zaGYnU6kp3qi{`a3Tr*L^p%e!MugSrcHHe}a-stzY_KoC`guJ?*k&%(G1Q@A!vPgBGh1D;f4{l+`~&>@aU%j)!-fqHb4JoA%(FgcoWX8VHA9p&)Y6){ zFmghqUjhSNrJSVW0VrO|u#pVD{|ku^9m5*x#uX+u>zeMjA7G@b>PQaSdRr4gO}>sg zvZ~3OT^$F8?Emw+#e~QWN1zI!5ZZmts!-*-M%;fe@?*UpCkM}%vpwpKEjW9+lM4nR z^~PHaJM>Tr07mc~ai=7?z5h|-59vx^jkUmru+CjZd@6|T5&N^27Dr-yoiHrW3});e z0#Gz#syJ@SilGuG=Pi`oxZ>rC%i`ab5)X0(u89~oJb zSwHtuG|Ylt!KABIX$cb~k^c7aV~?Xcf2~fe<{F*yBtSTLtG4cYVi~O_q3XFkv;bh^ zHA#Ea`x?&XXzQM{oX9Lff_V4WtlJRxi(~ZfANDgHGjiSwueJo61KdJK)*)fe!e<5+$EVXaJ5Tw6P@OQU2z_n8-h#h{+SEKGRY67aSm}9Y4W^ z&SSl&_zygcI^94Cu{3-YS_yKy= z_SM!>hA`TK^-LL>rCre7XO!e<2|CcBaf=3esNcM4F{|M5Ub_Tyi$AI>`fq~~2%)L< zvobH_{2R@?@(QiPzsDson#edx{|M2k=FT~COaQ6<2t`ry z?b`)D-`;U(P1p_#q$8X}osc3`1SW6bg;a=;ANCOG>L3o9xf1JENxvx+9y?|3Nz?wT zj#Eq?gw`Hd;><=f=d^<&P^#{vW$(h*@-oLs5giV>RFrpJb6X?^^QFY_j=)B%YwKLy{ibY3jTj+;bw7=5Qv z%a+H0^VcXEC_>IUm&KyY*}dhgfU%60gWpl+RbJh3E3->1YU@vadwv_idG{Gap%|v@Jz@+QwV5Bmr7sCBMyZGrQA%w6vGOS`6rWA9k52U%H!ujw5b-dcI^n!_rd3k=;IMU>rW8XjbFNm9%%(0sFr?Z2BaSEYT zjmY;db}pDD1U`R=xj20+ALKvr(I?H5Tyg(FUrjLjwrtR-i5RKRDKg{U-6!mhK6&jD@KwF5ielBX`w#kE%f5(7X>XGq&2xmeonvNpiAIw2 zBcq!=z$IPOxSf}Z7J;#FAe~h|D5h6qm_|CmP-=fRp(*)z0z1pEu?q)LG23Y<2VHoE zg@rMRZ}0RIOtOAktA71GN-42K6n{IMo4*-?_BF8i9*kcRdo_$#K``Cbeu|71!9vVq z*9vgOB@lqPu9E4cK@4r$Yr`ue@OktAt-3CUT^WOx9BO`e>(HmhTrc;stECaPi~xe* z>xfg?=HwW-kujQTElp_!govSifwkxts5nb)5#bqW0>D z5Chc~!Ux9RiRU-&0^G(?-N!iOWUF})z2r==WHvdtf$iZ5TV%=n7)%hIlUI2q2N7?D z>jcsMA;6I<5Vp2=Qrw4I{iI@%vK|xPk@P@?!fz(+7s5R3LWPz@B6l34kN>BySHWc__&h3U9TQDwsq8V3eL1XHT6WzOaXAoi#x>XpR9JYk1ty{NZQn?xp z&>ZPsP2q6~hrltyYD#Kh>Ud`=%ey#eI2WQn>$s!Y7hL1Z;hle|QDfgqi!Ix?Q$=n| zN5@yG&z~oca#y6T_o-1u(SIC|rTD>If1gM$G@QipTAnWSg7($OsprO1>7-+%&{fF< zCQ3_zy78ny;W9sKGkvyXjOSd%bZ62^Dj6UcjnN91!Ta~`(}X_e9){wDO4-*tvU&4z zhh%ATU}ma^TpGK09;t`?8anNKLPFWweqqlfR?Cin>S0>`mytVuAqc>HN>0&lTxb0C zx<;jNQGslpggXN;$L#X*x~si1*m3+oZn1|iH+V32bUBA$D*UsS{3o4iXzAZi;yokK+n+7n#BW`Vyd5g z{rX6Ps5ubsOg{GuddXx-a4mo`p;~JSL)AMN8kV~aEI?oir=`Jn+x38e0O3B@W><@B z4DC!!V2YcjlsQ$iyEQJNObP?zcO)YseMk(Y?QY1onvL}uA}NVbEgM9L1L~5!?ki7h zf(Lw!c0!##mP9e_Q#nX6l=PDLMxQlQ%Yex z4R~>OC6ZB!5bey`Fb6IWylhg()Kup_etZx&7wvz) zp5pQBt5>h)1-ctDy?O@}wvW<=n0J&Q9zrQ2MFzER7;hvxkly~^Tz5VG>#sw&Qf!d` zWDs`wr>FJv@1GbVl^0QTIWp~ohY#UCr$C~r<>sN>=#gy4-@ZCaKabLO0tA~FGW+!D z6aTI;l*D*GwgL4GmGvu$rz}|mY>)9}Z_nNCIgBD&J-K!Y2jf(-V`zfj{_cv$L$4+#$~~bZ zO^2TGZ(M?J(U#L~MJ2LkzVU(I7Z0Ynz!I`X%J>F18XJv(02>Qp%zy=1IvLb5bLPz4 zw{QL0BR^uY-ZxsZT|aMNW+ouCtjXgEF4dW*~#&>075T8zawwy5&C;E zG)))`PD@z*D)Sule+GWTXfiNQaHN_r87$%gY2@&#y7Shh+NOi~MQEczbSRuc! zVlxM2lk>Pz>o)GDW9CF<+7o~<_hYq?p``chwPgz=h4aUz1J14D_K13dZ*svxnz4?E z?<0bME9o`2b zv_U{qX)4?(10Q*oza_7exxl_Tu0p*LS(h=pWN8JfW0y=e(ejuhIub;Ia7!^Wuh#lp zS{jNckeQg4>O!i*dvF0arV(;T+^0B7EaJth8s_UszRXh}wsqUKyQ~!LhRW(SYJ|}! zTB~+#tPn50QhzjP)Tn)4>b{N0km}8whi`pR?{P-Ssz!UTD2(H+GRYpfFJIcm6t-y5 z0x-}?<@n@0-Eo}-#;|09yPp4IPN*HD{~R->2sltl(aM#3R)34ig6E7|vDUKo zxc~_dz!1xGNAAH{23 z*C@iH^m1p7sPN5dTJ2A>ekPKdOz<_u~5=m#``Jiw=gug2ph)Rv$E2k`TvO}`1 z!7FOjQhf6Qs*yLxAU}=SI%^y~7kWfWrGy%s(?%6=Cy{dtQs;d*`hTB$n({qW8l zmsl9z--6JX37Yk5Tmq^H6&Sa_PZTzUViX1R)v-t=gy@Go%D~*N>DwoyD_r+HAp%h- z>Dn>Nt(VI{93PUt)DS$8G}F99?dr?9zL?X-0jFl)+W(;WG<;IV53`ok`Pp2%-?`Kb424Rrncd+Htm~+DUWnTB7dmA?a zKpde?c=r5xJEk))J(oOdM3KW68@7d{X{Cz2dm_7DGkNX0i2 z-=OnvR?#N1+c^aFW#-U=zNmQ#%|u&bVbBy+zc z+pFW_<3*N$ENHh{N<@e@zu(KFN;NBdgM3j7pHYSS_Xm_347c;DWCq{QX=pB*8RB<< zNO{y{H4XpxtfT`cRlfSCBgs*vf8-|Kc2vxBi;9Y(XfK?yxZ)YlpgFCJzn>kX&R+9j z^j8)Q_|6a6Dw3R&JiXgd3UJ)Ye&Ry{(u5gSbDfYwT@nG7Ep__}rQE&1&tn%4ci{3b z-?CXvN?+IDt-S}7cD!roRY%n-#$LOyUEaG#&49CUW*ZyGa9OG4#g{=TsO21*$BLDZ zMHUk^Woa4M5wN$r{lZBVuvcdaqgonSrJ-k1fR=77V2asEG>-kVa!5qS_&R^4rL(9( zXj&acQ2#Yre_+<}9{2n1sMvC9c~z6mYs){+1)7-xeLPtXXTFL|?OLT8w-#q~0F-t0 z5-BYK1YYZhq@8yhAOs#ymyObWSYY>z7J*QQxk40_?(W1rB1#KFiCS_ja4R`#qvMa649+v_=xZJVqaF;FOF| zR`x4y!d7?FDbr*@OrYOaV2p z$8IrPdF-A=B%9*u(E#x2;7~9(Hr_CC#W}lc)NLBXn|H!n)3n=kS20K+n@w~=!*HFor|GEy zF;P)otkj6RP#m664@iT92*w?)<4)sgkusC&UUw_CQdnL6>aMvn<`(EU)fg^pwZV~u zd)>JblN?+mGCo?BMj%M%yg>W<#3qk78s1w;Q}X0swaJRErc>j!M$D+H(8-W-j`Wf| z!<0u#?*y>@+rLzmyFwG8hmz{{Ue*vEM9`L1cLO8!^Lr$Fe7by~m$rkt-tG9-UdhI0 z@>Z+kT0VaLdgP!=QmbW}yICyOJvcuH$TH(8 zPcJ^4Pa$lg22ae*Kcr!Yj5-Px@VTt4)9=4e#BUwD9I}NU9q;3gW`ohn1s@3dlM*<2 z*2#|zD*w76%Dp*uoT6-}f;>pH7XTVB2e$D zI#2V#u64!XE7?`;OnrP7{02FzOHd{bw%fWIg$Y()ojV3t#ZOSxUmNOj=y?Ag?eG+x zd7}A4qb%tVf8iL>irC1VG|tC5X0j=~@-OXXnVAjdOT>I9g^U5t6D!|Nar^46%u2RP z^J0bT1h0&3S|fIM4gtqoH5W{$P^9eyt%mR|#GEx$p1&|>GgAxes`{+`a(diM&7T-$ zEyj=80#2MP64jy7nHOhe{^ipqj58`V$4ZSbtrVE&zby21_44xC(L-|&6neT>w`oDt zw^gNp8vcIO6$~6;b@jI|t25w4N?~2fz}p$WQY(`?OwV>Km_4-las42+ zzZtciaFOi3&R3Vh?ePPS01}0C}}Ybeq)|a%)Gn9va_>=E#?@LNJ{>rsJx+5 z&&BFmRIZ6;dgH_IYu~Gmp+kC+#mn)&q+}0%5RvcU67yiyq_h%8+odLzzcl&8(GG{; zq};OTzB@#C3qv^*n9PW=>2J#*4&XnB-1Hwfjp`(N*7xUPZEdYORuh|k@2qnxBe8f! z8J}xguc?6vo-N;A3w*JW9T@CW84e~#MSXN-G8%ehQ_hW9x30R1lOY1dHF5g%@AA0+ z_6bwU;FH`%Q@+=loBl}D6kA5*nN(b47KVc~w`aOG!Rh`f2Y+PC^%SubzZK=GN zc{yN;<@i3uN>;C)%HhT}U(N3*^4!@&uyM*fkc;Z$G|bFuYLU75-xq3s`vFEXM{8c} zT6FEz(3((TUdkO!dbCR}$I;aO`zK688S_+1c>jPO zAlCw@r0tnKd)Bcyl)@>9Y}^Sm57w`AoGY`sH8wyywg0~TuH3vi+PSnSm3FgqhPvcG zh#C%gqj`7-`W^=MSWC&ovotJy*%tex=9`*t;i|Y{L3A!3rKS)^i&$I^4n$hR7UwVC zvcEs#?XXU_!%ii+JeBp%hKj! zPd3ms0fqJ+4lB!#N4?LVKcgYt&YY+DZ&%9;{~iCdzp?+CzgOr?nrOL;F6P}YJ6}s$ zaQj1L2uSmT%Du)-gtHoz{;~hGiug(kZ(LUM&S&~?KW=AK#y{&8P{6VP(f_q#2Dn#b zSMFNt1roV`gn-Jgp?Iv4giF^amD8Q9%bKzZeWO9>;e;Qp>|6RZHNf;8dk&fjsy zWOtWb8qrHblm|rv40#yC0G@uhPh^kN`7)&=v#`Vh207BPUAx50d6gH@U`}zs2Eg}A zK#@?WvQwpe1d;f5w~;3r2+lo);;cJD;$bI@WklJ`G)j%ezErfQV{vWcb zWu?fLST!zDvr{J{6}gHGOmNPFPf90QjypD`56^HE6p9zzF)Ex`JH!heJEoE5Mh5*X z3sX%|1S@JV{enWe8N?{fnvbMn$02LMQoVy%ptiBQ(2g9h2YVu06;}hju6>W$pl3dpFvnMq7X3F8y{YgWa5E0{T<*?9ay zT=k{mR_f8-96CP27g8~jI}pa2>vgLVS<&)=h=#vQ$Z#d`aU@jIIM4v|_Xkj;TRHXB z$vA_YEOnEt7ik&MZ-*s-$C{4t+yMx+Eva$=KrZ=1!W`1z5LPoVH@Bc5l4}Q&JxFZw zEp-6(Egu6nq9fyQIcdhwX7!)z-()=Z&kzCLq7w(SSLE71$C3_7g zuFzs*v&68f`57FAorrMo148L%&;WqyyHf+h!GlXzWVSpGogi*dS()5MLH;zm91_`= z@HkQK6!YyEk`yB<6!6=$UcH+3YniQgkf9AKHE;-Dx7Ztze{UY>H$lz6+jbedDpPX_ zAAG6V@cM;Q3yCRR$VNGGvwb4tmxl9)_i&s_KY)04zX?&wJhpR3tLUS$o%s?kDNiS` zk>q^G!KNG1r8u>F7#o2Zpa=_zZ0{7$wwv+Qhk+gR3=GmUkyhoW+40hI`{^F=P)-HA ztl5;yqlm%(r4n+1h0_%es}KsAel3r?v09I^rG)yI7)IeZQqA0mZ~}r#Ed(wwek6ka z0W|s{8Xrp;)|p5axE(z4?E$h>-^Ixj1n&5py3IW0K+Jc6xCo4?B1{_y?=J%?QYem; zdSqD2X!>W*S#nI}&Ha5W=sQ8QJTHv_EH(}=w8e_{ikv>eu&KExfl++8Td5@x-f!4f zg%~-9pFl??DCx`BuNgF)BZpp1tOQK1yFBK%iob>T9Wwe5!o{^&`Vw74B^PZTpROeR z2D*>Ef5CW-XHBIR5n$;#7CA5IpnCt|g9Vg_S6mKOI8f}GLxo(T>x<%?oSwQqF88*( zxvBKC@7?QtF$M(*<$c!N65FVF@SqrGp4;PqMN#nC*3u&Z4N#*iarx5sHFW6Ed8jww zK5wypQUu%a&7KF$mxrc!XE*ZW*T+MK0Ooc|H!Y1bJ97QBOcuimIX+*Pg>~2O{sg4I z>LmkjYgXCyrkFjs4MDK7hTUZ&V;%_3C`z)PUWG_ljX=h70txl z-PHisq_*e{;_eV38Gr8IWo9=5%9SfuY{^#7hi>Nk`j)Zk(@!t9$%vdb4g7QJW4e5G z*SU>xa0@^s;=rTy@7@KO3@urj$B?{k8<&9X=eAZ?>EXBe6)rAK?_+OZ3NG zMq9W2&1WTp!!>9{$6MA7{U)K@5L;9ToW!>+o*vl0zlcHPDo4vTZ(qL-6L3cGlFA>^ zC5Ay=OV^Obd(uxcnt@`I{HulB19&5z^we05cYa<4uD&_+C*dd;Z_<3n3%{xlcR0l9 z6Cz*=S3$_1(9GQ zL~GcfK~Y*s*{4rO0Y~F+U|QlQXguI_d-FbN0m*tJG=HL#e`Zx-z)<_DEInu_`7_sL zm-uL;XXS$gaH+sZnzBT3;5_Q(NIX);Nn<4YO`HHDkNhaV_N9JcY6q|6S+5PWmhOWI6{Z{?>6!#is_yBhK#N4WA+ef zNh);dF#{2e6QhAMWbqjhT)Dt70>ux2gPb0TEZ9&@&{t|f5_J|qdr z*Gto7M?m}iFzWY?{hY$ir#sC%Q9rO}>x26v?i$tFvUO`3s1c3w*o6xhjD?96M$-WI zgtRhXMxB(>{u@iMKVw2Di=({?c5apWD^hIDC-Mo3Oq+kPa!b|>aa_zU;c6v7ChNI45(iKT}#*mtAE>u|qkWmA}0g5h-}5sJ_>r0YnLr})oS&PAT%p4X{c zHv>^ldVf*e!;t%V!(Vr-rmsspKhnuep~0AnaCIlm1?GAGzgf3-9gsw-C??|OiGl0? zV(&epsyw^yU+f`?(O6@NqHa{M_pTU&VPiol_JX}28f&m1sA(1ywg z^tgpJ7A4LBh+G?x^~unKg=a9C^=&H9^NcCW)O~_0G3OE|+IGqr1g9s5)~{UZ*cX$G+v-z;LtM%d@%;Hv4uqDS+moX7 zdFc1cgTLocR?kZfy2|^5g=pe_MY^v{gBs1RqVIn zJ)ON8W>W2*mXDGyn$!Yj=AEcZ9>=ON^~>@EIuQJy+hhMS5Xt>9hq-hevL+;|Z74)` zym`(A2Rf@rh1uU4oY{*2AdHK$sJD7^Osf0h*yKr*3?w*$$wA8-zTAf`MrqR z1^wVU;?bJR*Fp5iuXk}Itbyk(4C1W+D2OqpgaZ{)?TZkvUS-e#I$Pa(s@d3f-&@s? z<9p&N>{Cb&uiFUNt+qG)w3rz19L4*1gv@X19UOco3+c1?t@_+)_ZzAGWRE}CrsCtZ<}E)Y&PybJh^I* z$K3;Lc?p&I*o-}0X+nb*-v_Tr$9Ih@C7(!y2&Qz4%g>s^S6En%dHt5M^pLPER5 zg6iuxhnm9G!mA18tKRUNmKpP{$lC3!ZvK%!)94sVcR&d;bnoF$?&PuwTqq=gN@zD{ zIx78jR8&2pLb|mG=o-)l+3HbUuE*@Gomm9`^(_`|_ z?lFaiJ$QJ2)!Ob`*XTn?ixc<;`duB1Wl+;rqujZlMkVWm#z}lsFoMN)6EO1BmTNob zn-w_utAR#k#1xttpc{Wtj}Q&_dB(H2Nh^OMd1pqK`A7IR%c9tA5npBPS-*wC6M`UC z6It#la5}nU!+xNnlpnNu7Dd>0lh{ZgiQY7g4!Z3A(+$@KH4l(W(XD!S%FkxMj6Ymn zXSTv$gGfHJ|ETLbUnIW5TFeO|#v+q&4-)!zbw`4HQmcL(8+3&L98|W6yYfZ!b-ew7 zPhY)waURyiz1!aD_Esn=F<;ESl~b=Ys$*lf{ded_WLS7C*bf=jjF;B;wwJyQfvm>U?Y*eX(9cw-L{X_);IyYPBJs$Hft|9s!;S zH)Wv?A6XurGz)K?h@!M5d39%t*D}M1q06&F+HwE)Pj(2L;a_avWRHBfHCJE_bt;{c z@(QsfF14!1r0iw~##3fOf7$!J^OAYm4=nsd`bOhPp?B@v8AZzxhFu%W9&3*@@Ijj; zt!zF7@X9-Td;f`gwgMF^Yi=z88o({FXshCE>sC6fD2yGEi5s1OeN-Z|f`Ys_$xr!8 z&rQu?N$(yQx)~y>T*8E1iooDL;CpB1REP+oOlZ%7{ON#fI{NOh=<6Y;!!Csl%VYCtAGoY2W3{H-B9VfIyg%}2 z2*g-mZvTW+#^Q5whh7kvrDmB#H(H&vqJh8mqTv#MfnzP~UJEMr5R)8rI9|+0KLA}| zmEk;Qb-`{&)ad4(#rkBk`KOK+>_L^NR|A925jU_>A(5FD>#szka4M| zn#AUcPQ+XDjgFMbcl*d-9!@NYtk4C#=2 z|4HTg9H;eJ-l64IG1`l!+BQHCaWB|@smzRAL?gF&y?lLclA{pd%gZo?cb^Q?(cTO_ zlv6kewnqIi_gr94B=riZjd1XnpMS>N--9D54Df9YIo=~;t_OgU1VA_(x{9)f5y)#8 zaLN$3-0`T`!Vnq$Ewo5nro)-MIEgES>bh`XdpHZ*p_Ybp_|mww%RiHU35Ex^6~qn! zvG$G!Wa~x|RJ=xy*E1h%&Eqs&_Lq{bYFCIWnwY{k`lJsZ!PgXc=o&tv!LYe)Sl|&} zNBX6%eeH(_OnJ?`q`fkZWVlJ<_0S4!@(((-xHAQhYq4iYTR=UBay?D4G!l!83K8#t z#!HsxbpgdgTlRgfqH|B2X_@}vW~ukJdlb!m;!4wkHZBdpAw8)72VnsOCpapqD*N*| zTtE6DW5$e;??IC{MkO6M?(m~(+r%FvJ}Q;`h&#mnyE9N;KO&%YhPy2GKB3~wRgXM@ zUO0WV6X(w}Dj0UlfbFAa7jsxC^0bVFtLBPs3$EZrn5gom%$NBApsTm;Nlvr8>0ok4 zDXe1d73g;2%+1zszM31`@zANE_ul5W*%+%%2qiHbR7C-KEAgy{aub496ud}nU%anv zypGi0mu)X_9qiJSD#I$p@pyrF2pq1`dnUIVM2mEPf9WCTJ z4Vx`Zn>B)!M#8<>b;JJIEC}LFhw*(yNYENy)_kJ7`t}%77D?X$Z;=#5dXC1svq1}G zpcN|2JM#p`shtr&EZ-DTZtvF>%{;WHo|sugaL&MpTnoHQ^!r405Z5n|kL?A^P!8qPQ3 z{d-D7g(B3{qUo7jHtq{lbkb5%!ye>fz{rg}Ke&k{&F+LRgTVBj{Hti$A6`LJ!Gg8F z3LV&C8(3gc6lBw@IISazrE9LY$(6PIUrM9Bm$Kdk{DWceHoF9aH0Dj7H@gsMqwmN8 z9gH^|`1;tRXRauyU1-}}q=pbrQ?Ne#XjK*z7TY(T{3k0;}9e zUo7%Vdp9?u5> zVTHEn9Y5ttj|f1hQ9}_3Isi6z zCN!OXf_)aYF!O~qfh+|__0-?c>P z&H9k+H9T(Bu3nygq?!Gxum&f41$Sb-OFRPbc@D^!YOvF;J$rnB46g2-yhG*=GG?L1 zF5PNs+>r#MFm zQxEc4nW0c=JDN{WyRnWd<1)lTSn@@Ya{ok>yDNYG{rAtz?%~ZRp7PE5*b;r>g`0u}!!gkeuh! zwKs437=QKr(bk<-tj72)T@Ht@l1dMy#Zl`b3T}QS#sd;n_InaQ)5&x#>smHSTIg?I zfSe!WSz2}f#y_+3-@LQx3Si5mZ-Pr>GT4a$CZX|-TDH8oau*9RhKyE9Ir)Y1mTJwm(p_xM4&j7>T=J&f zjh?ir#6#R|YfXPV6C97hTFg&l8V*CZ0XrXzL3koUw^mlCPQiS+rTcMr1Khz~X77{U>jDAmBefUNIFgJ32$ zdlt7~-7f2`l!J3&wn$eExshzW11ajFf6-5(XW}Bq$)#1^2Yaw$~9jb>R7Gnf2eFAphh=y zd?M_MZ`xT^bG1v?>Hjl2hVA_h^O+0{u*UjH(N}!ugM(97d z7ed4J4XYS8<`Si{1$~}Img^g}=6I$c3->@m_|j+&8`{L%#;-O0!Tz>9O21PmZ%DU2 zX^o6WMyN&_dQIU3>g&pyQr3Pv=l(Bx(L(9?a;K60+XTOa{BsHVo`mvMI{u;8M`Eur zCc_Ymh=?J7JxDHBn|fKF{K3bT0q4juV!+cgNl&*AIQA8C2ZVL$p(@47UdUc12a}V# zk~Ll6^>3qvEETVsxh4Bf>6d0S$!!7)ijy<|U{5?iV$w=@ONb__JQ!zum)*k4Y#=dG zxM#D4?H1eJx|Chotc$N$7{H7m)0;aLX9fbB7vAY?C|7c;EpKu@WgEHqpJTM z+*uY|}eZi2f1HQ);heRZQjzCRqyu>SeBYD78p zA4r9lzYpiNi_(%1TFm$gmj*1gs_=3Cwgv#n`cL~8L|8I<8RDB@?Y#790TMCLn%CPj zU$AU_<*y1B+X|V)XfJ+{3(P8Hg+glsrmlu~PU@`5;)?!Rzo4?lk2*PO(w{t`#LmER zS?n_K`Y6C^(I-*3K!_v{t>-u8VEhDR1KRE zq$kJ^FFFpO+top`LeR$5tY3K;qM@nP{0#Xi%B2-m-XxQu!V}qM=V4q+;>Kb%43(AE znWSO#Brme6J;qr3yxx_>SEe|V`hWJV+sj!vT1OyhI2_!(ev_VlsH1B)Itw($G7f(N z_{|ne>~FWlwexqh%JuU~uIanU141|#7^JI3P(P$=V}H;Jhx9FbxSlx*lQ#Nhpd#T6 zgplI3`t%YFh<>ce4Cp0-Ghs^P5g1p|1oIN7clp1cAk#HuBFQ2-^A4&xI-a;YB$Yjl zh^5GOl>H~~ZZ`IZ%SIOcPIUF+(eclhTtjlTojpr#ohE@IDHJl->Ueh{_8!W9Gshpd zo@v95kt$)>{4Kn+rPuVZNhy5@=9p%AK_sFzT$9KKG(Hru{t2YK&aYy@*Tlo8x~;@V ziX{ebjPtjco$)5T2VsOWg&m>p8PJRj!(u$6-L;}+fvM_JdS)qaL5qE8S-@A^+SbZD z*ngD#HWGcd0wzgvF!sY@af{SL{(`USZq&q=Frxvm?g-;x+le@?dB=&y#+Tc@BV*o# zdfA)^nc+_Xofg@uiZsHJq;8-}B#@B=n$(fmbwHUtJU#kMwa8Ir&?T!EQ#56a3_Uq( z9L9j#`reRR_kR69w z)`X5Ufk9YjZs@yj1G(78T}R*D${lo!CF8}ME1xw+ed!eDz5U4-U*=O7Ui~saz4BsPYv%pSkcy2P{%+`7YZaYa_qcvCS zItt>E>r<`azaRjd{iGWK6fITWm@U@kguC6I2Oe@9yGG28A=)Z1!|Y5MTipIX9oF{8 zoZ(XmcK&zbk1e+**4SCV$qj$IJIm}(WVi^dxu8kCG-jeKG~L)a`q0rO#$5zAypm%M z6?l~_;*NngpIt3h!|>hO)eHMr&<^b!C%>hx_j^+{8@gs(<_O{_>H4h)<74$}|mwxOV# zs=78F|0qQDJEAgKc8~zJ$=KMI0L^Xrx}D!czAf4- zYe&^PW@e`UNep^OPLv4jTjaG&^Tpp1#^3#*@ed;gbgAL`S)oG7B&-zK4MpH*V|u~m z2>}YBqv63xAN<>Izcq;bzOEX-rjIDZ!H7zI;7Y6xD$U{^d zN6AMTTri5I%lr3XxWYyoPS&MtP#z()SLh_l*jY|Ta)Ma6OPv}+vr$L5$B!icK6H2L zo}XJlNGFR*{^8O^sDlJm6n!89_VGz|j9c0id}&q@HIfS#{*$hN{i9|--VW480Od4j zPCVar4QjZT0_KA0FZ)_DxBG8+sv_#>eEc&148i>n;-WbDzN3DpwGkbY~KzrP8G7GU?gBh zagCM;s56f!G`*-~F@PPc@|}irl6YqD!$5Mn0}+eY*PgMCeR6UXccVZyI-hl?eyR z=5fMjul2~JwdhGrSZ0Vdb^ zeeL2*>>2dw6A7^99f+64(f$bbekk;K<;pVnps%K$rpbX*wTpv+d{p1Wp)03<5Bk67F$;Q9{V6X1%~TaH@|lcCwVd+q(s=r$y?CH?J2f z)dp-2g8%94va{=QzLE6;zPz!|$6a^JALG>=PmG>_iYZF+l4lFGlk+2R9hvFCr=yZL zEQ6q8G80bblaDZ~JVvcjNExMDPT7gjq2Jsr+|uP`sz}S6?_@%={OZ@d9cwJ3>Lu~a zW7K{a4Ee|i~bMD`E+yLnlJ@;sb@d-GoGx|iOHE0zc^a~ z9+IWVhznWuAre_kH}+0W*5gz1LuhKTIe>G;b_1Qn6%<9y>$Id*H}k3S93nc@+O1Y$ zaQL^VuE7N46U{LbUV1V+7rK9I`Z%88`oeLA^I#K0hjv+$Gq}!^9>UT3EFe-Z*{--c zCL##~R-7%K)fh*r^QpbF;?i^{p`t^$O9Kw%SGXTsXU0VOw2AEBv#@FC4 zK9K1KMSZVqahC44@sEFsCd!0@esrUvBzdK(Xn4lJ4xd-=Qq7g~|39o2J~ci3kMFAP zOL1^4qJj*?#!Q&d1+ok_OooDb_@xus5~lDON1lC$WqQv_g)T2%ws*$kZ^QjJ z3~;Sex`|cd>Asa7lP)mZKSm~h!n!DM;0;G<;qf>i#D|(%_&s+L#gZ{#SJ(gHS1Q}$ieNi)B3wD9WmSjo=W0YNDbb(#;(LvAFrSUAV1Od z=@j%atSyt9 zm#QIXQg$Lp4g!q63OQ1X@sF7cEaR9hQxlCVjQKOCSCfGM`jhC7Z~)9(K6cM%YY7oA z<9P6$)h~^-&eSR!AGYx>&Qh_O3m48d$Yvw-UX}AnCF=dBttXc|t5fy^v6)S0^k1gD z_K}x=J0DiK(;2HC#A)}!Nqs69-)CcYdh6Dog7?l?wf;iwdO5FrM8f}_?D+3*GVafk z@4u*$|M#!;T>kg57^}ms`~RQIq4DKxgWbTA;}K;ji;!p-f+tHph0 zSZL&%+L(j{S?6EeJm3fNz=T;#j2I9cpK}|DAsiSE2ndLE?axi68V)Tt7N#j(AWwQu zyYN$Bk*?0~SL*!GdUx^{GpiUe((3-}|MIAz5r#KzJxC+D=%6>>DU~o`Ly!IuZ7Cj zG;=){fwl$5r9V9120+#ww%It8IHD4jj<(*`(|>LY$nZo$LW1J}n&DkIGZsQ$=b#V$ zvlguQOxT9@W(|b$x^MhTyS?zTjEjiq2`n9KQXhtAH#0L0q`|BV*}Z-(;|KZ_|0*Z` z@9)CE+v;Y#-JC2R=UpR{TL*3ig*3;CDz`Tm%> z+2hX$hhUjzcZSjGiWn|}nj)nCMC0C`ug!_w5j{zd7n(@=ObMTkJ)y1LZu1}3Z(jKK zBa|7KL`bwqLTLMiMFkkV1?fS59Blk>Txl6e>DR_Mxmk+LL5W#(+a92A+KrY7%DBpn zkM<*57icWIY3hQI_J;01u!_&xaD^=LRU7~K^7*o3 z>7DvwA*L2~{hKg%)i``|YhPc9C$Iwqo|06@oCtq$EvI>v2bHY1d}3wJdv%a=KMcqs z+mRX{`rr?%WclPcE?4g5=!Sn2<6Sc5N!w;0lt_%O`N;)FO>4L8pHESXmHtwAX^&0b z2l6IM+d%`>E2`59Q;Jl8(!z29!r?e!J=c0VXsktzQO0d1R?0pSox<+ITO>J;JJ&+> z8Tn{utc3f97F^6#lNN?dIt2J%aW|-y87L<<;f44nDtf z`>YZTh8J7kyXD%btE(=q?cVdr;p*S@|0TF}Qj>kH7>Mas)O7FOz~)bP^c>mf!R`%X z|4JWutL=tYQ~Fx%7>p~m>C!sqbT)q9E#C_3(zX7rcky9OLw+O%X^@Y49&|V~`)epo zq1c*Me+%nRx{TR0K36Rtz*m(s$Yq@0QzVV>v{q5ppLrWue~-s^IFXPvUQHT5?_9_I*JbO!a4Cox*s|Nn2qZ@IE^WPjQpaB# zp*$4G3brJY%{KI|L*^DNRH&=$Z>f`{Cm@EacY}xPr;o#L?B0&?QZx_d5txs|Xj>-> z45Ej4xO)+HPmisGgAnm(Lc;f4j~BR+``UrGmy{i0}`vuoT&0vbD}JvKHj)$38SAJ+p}VB>HgTf?FdvXSO&`%vq?W zJw%zcJQg|dM^q`_LsczJUN3@7pSPc65Qeg;6R3}Ui+7!3u`2WMeW((Fum9P+DoJ&6;4r z%=C?zx|$Q|o5_M)KLZL9yUXQvU-3V{&(>m_A&I0UOr#OfL_bRe%GK6{2@eQE5mhoi z5-kh^b8Kkz-IV!ayqV#oC$rfhY}uO=$sx3aP!&iKw&k>Q#X~N{eP|IhyDRsZ45aQa#M7I5ea>rQHVnz;8U6@Kl|IVu zcJ>>HuW~h#{hGX_WMaX^K!Cj;d5lz`k8@bToEYweAXPXw`|l(IJK$2k`$cbZSSInr zQ?~D89-wAGIQOI}F9M?aIhEzUS#XU^m?pUU`=hBORiPb{sP zG;JDo3uw1#wpOdeEAd_6@puwQjrq@Byo{pFTb9}s%fh4w-7~ab1-WtA+bv6p|Dml@ zPJZPSrmhOnltFc(rMWG zkSxHso)Y(hey*6-Of1~N6 z(jF(j`^$UOrtwV%&s!Pf*r@Kgq(OGe?*8j0?7p=nA;9}WXCN>C(UKHaUJT3Sn9o_* z|JY3oo$8Ot%tx9jYUkqfvc7ZO=ZWI1AifdF*Puet`D_nL?soA#Lb=tj=c{t8+>eFT z6h#&XA*GDltV);U%JL~=UzYu4faNB<;lnFW!*1jZnRI^&;i2-+fu4sB+_XF9`TEX? z_Ks`wrIyTSk=X1zD|f>t^)S_o;or>Hk_Nc*aLMbM0ukHayiCA9a&5~eN+=yaqU-AD zrI^w<3Z29(kyMFo;o{=#x7O&la@l1@PWBd&NA92;N;DuHQxlfl<-peL zup(dDf4Ig2U>=ejOymjSE15iGZZlr97(axIDwBy{a>5rOOR8#aPEPn_X~S>6s8qdrd%B=q$Ugu% z>b!a{rUwVNJi1D}QbsHHrFZF|N^EJ_rp>L4?p%pCUMtRx9SmI|nD>g|xm) zqvv%;qJXB|BK7q-j^6Dwqomj+dtJyRfw&jEi>@pg%}rHxL-uW<#V4nh^5vJC&K9s6 zFnjLY*w0KZ?mQti5{eeBgkX3`{7(=SQE8p1H*e5R9ysC$*%T=0Na)e%x#La0l!7vJ z&;d;m!+BmAtx%srj2q{w{dhoX%4&XCoxXg3^*#lVt%8m`y0K?Rdae8} zfiw2Do*P{?r0{}S!#RS66)-aBZl^>0PL0MDs3Q&TWD;p<3Dnr4nKP%+8@?ckG`+@E zb*;4$h{;$pg%HOp%4m&=t|G?Eun%Qi>XOz4>{gQ^yD_%#92Ks;MxiJ17>x1K5QSbC z1GF`w4BOUvW_RM5nnhxWQ4^WN3ppKKL`g`~`Pj-QB)7!R$s_GrPkNclmrO4N7MY(v zA{@Cv3A3xv=2w%?X{RxZYxkQD*Frh0NSx?*8)QwcM`{+<(to$J9g2#M+)4bqD9eLk zRA+35bzPb%R(xGpu_Dk*CYUV3BtS{e&HbH`qsrG%;hF~I*Q2KJZq@R5mBz{ILrA+{ zr6C4QYiOEjtP|aZ^VEmJQ6#>n9v(ldcgg3?#U15Q*0?GiB+I#S<(tBAmQo_F82x-g(&I?%DDPFRJL-o1eN?MBx2vE%yy{A zRXl(j4oUa1>9dF=nZC>2?bwnXz4q6B47ND8+Fy)Q{9hz>PT-u!SAMhAZr|w53M3U$o>S$ zne$TeA*NatDfNV?^o_7% zAC$cFo<_ciC#Q?11`to$`MrdKv=Bu;(Ld*0+Xg>`Z5w$!-6pF9j$GKjd!`G%ynW;{ zoZLW9Not97efAvi4&J+WZ$Zc0mZzMP&g9iP3@CGLLths7JG7INV5&ZIA1J-o;JNDzuNj?Jw@QRR4m zVj+6&^2#u0pd_Yj?aca32aeHLnys1JZDxLr{cy{Z84hH|)WY%4_sU$4!~u&2pOI-~ z>1_H)2(OwelnGymPTQf9;qwbdL?D;Z%;FN#!eRE&YkCS$P&C;@6ol#qY{Z**#WwC{g!*>@QUdBY=TX^2e@jqrScfuArVf0nW_v4bCMJtS^WtbzY^Mk$C49N5 zz1Gr=vrLTy1?62qPvYi>K{76%$tKlz(d3m6q0m!!*bBU?-nq4?R^OUXQu#RZq%Ac* z@jyYp>xk805!K44PwwD>3j3t#*nCwauFxEnPXYLWC45BdeULD*RC8e6+|NLb`4TU8m}(x#L~VgaJ9Ke_4o>$3*=z~E8<~hsdL8B?`rsE7Jz0#TIz`e2X-MC5f759I2$9R^J&BkOqXJgz1qNzKjvv!2Pp~I{<4KYyf0Qh8 zZ;4xVhtNID7LhYq3_jqjC06{b4<$D08625{DA-*WLSe=x9?nnd;QMWS;KlwNwGwtB zUXouB$@h`#8Pq+PBYdiR>y=18q!Ji_7}{}=irBY%;KAoQzE~bP`>(ZQKVI8iOiRk9 z&+mFhKjG(!H&q(Uum(E_HIsF+PE5P|!2Ix!(Pm+X?1+^tSB>(TQhTb5)O7`c{1HM*}EFYbW7+qI)D{@%C(#lrFIW1KlH$U03Lrl zE{Ze{lX21gZ%2Mo{KR7R2MY&p9SYwYpT)1=+V??d8^^0HKXu;L`LDM5ukN-|ULD&o zX=BzGOhq&1rO1WQRD!9$yWZ*Gg|SCTxQ35D^A-lJobBLHYScXh%*7L)UXMl) zePL~KZc`U>G)+629ShkHX;Sk>!FFcQJ5M-!UJw+pm)v3#IPESe_Lvw4j&Ow*!Zo*fx4(g$@9=~nj=uUZtm{^3=g&5_Rv*mNs- zZA~_tV)L-!m|VhK9NiP|xJC zB3~N;mr{a&!7n}5xLGqF+I8Bkr+vi$`U7S?b{9QQJxEEZglJV7L9I}}t-;_^-)O-; zw%9Cn@pTJ&b-3}mnZ8zKn@!;16pt@#9G;32%l?sYUl6<{g8qbrBCV4=;^47mC<07rW+l!8v9G43eRW$jZq3UpCg=H3)Geses{Yr;8E=vNhU(E} zG_+wMc;KLsqynjAVR}5QwPEG9l6DXcdi-2OM>q>3nu{&*3b$UhSJ?Ri=2e&|yKFa$ z5%Jx)*Cf)ePp3X%uy)L+ncwr<^$d$O?k2&#%yUQg6pxhVb#ipV(Rp;poX50CUF&8O z9(N)a(yTyEf~D_)x<=l7cUuSaFsR>kzp38p`O7zNl2}kOs=_-(ra`DtD)+pc$LaHS z9gH_(y*mt`rt}YWmE>L|!}CK!D5A7J4rcJ#KTvJSkVJ*s?0AZ`p~OT=zO{g0^jrF^ zPn;Io1)DUTMhD`Fv`2g< zDux7q{+MPh>Gb_7ds@A_SCo3U=FJl6Zg{w-68A1q)5^Kj@YiBKo|s)`k&0zbq~sbn zK*bg#HsmuKac@vA%iAAIlyc+T`P^@EYTM_&JwuHeTX*X{_U9t*(||BAw&@s#NXj$} z^@>`&ls&-yR3h?;ZaVh<99py{GXPB84Eh<#VbnTsSMALjKArjINepbo>NjIE0vo)z zQ@w6QsKlg=4cA^j?$k5%?*?SuZ@#l_Sc29PK{<`OqQ0Tc7kz@U zuzzx*cAqm3M^X-=R$`UHx4%F#l#XSws?z&)VEl-aam03m@g~bCUZah(p*4R}pgNcb(M!~)hN7UCXaO^_Kb zFwqn2iIj5}&q6^)2p)OkkkGuBzbt>|?`LRJswf{{Z%KQL30czEYZv;qdh$N<^5t%3 zcM2D2h0TVOB#5!*&_XA;#Bsn$6a%=2qQe2H9E#PXcsZ52T-Kpy%}~7XVNV$2(e3 z1mh>kUR2*dJ0n~7$Xcqn?1Y?Y8@Hx`^aj;fLp zx*T$q{mtX4IA|CzL)7kC^gvMh#Wc}}f!`DNX|R}mQam(A=W zawFA0&(>ZD4_w4NIcgBaPkjVY*wBRIO^G0h+r1}G)&nq0AWPU=Xd)bmDFF?3zKO$E zau&d_2EWM>CFhrm?cmaS)P0KWC-v?=`!TypGuptrEs+FG0Yaaz7bgnDsxZI<~PH z_~lozHLag(=RmIanmN~Ur7H1+o}(D1l0_T!MR+X|IMVR5kLS&qGv$iA@#c>|S9z^p z5e}H1B(qLJD7`eMgIBY-hsb=?~ znT6-1Cwg<@gKs6rp1M8=7#9G43CI<&cKH&;=VBwM4Xy$bw*LTUH z^3V%c7!qipzRkt5#J!H2KJuVth`G|o_(Z$CxAD%>ui(MW>DOA|kgdu49(SkH`{vG> z)BfTg^awLy7Hmg{z^pK5)cxn!%bpx|l4lUpx_0Y!Ii{qs?49x-&K_uQclyT(6Be0J z@Kw27zIgFLpVGHZ|01g))gjw&c5I`Yer>QN=tQU#ub9(KDM_tA(s;zSv8{XxSeHy< zvkBRF`mnZY&zgytwr>jL%SZM1e%g#Scf*go{mJf5G?l9iKrK2~s%&;Q?#C_%iXY}e z?rt*EEbg!=3oOB688u{jWHX`A_9v$pKRd8~_8xEwyMxyyAE)<_N#&f9eiki<1=EE0 zCvHjJOkuvrB>TsW;0yXy>g(+76g*hNchzk)NvQ%~YoUp<#b{o%p7DK-f!mSAg^Dt;T zninvwnw9AHv0u*0Uc)TF?$WzzR1Tf#C2jJukP>OKTx2PIf0pB@hP)-nS~T&R8jAfS z(g`V^P;DIr#WxzOEAVyD-|Ii+{{1z;7?N1hv=(fC@AmV@gXU5S4yXJ;iiM*aebI8pPj0gCX{id{ z=vjzN8r_CPXcyc=-HN|5j>^C)(9!nC;1#+6$k`;j@uh%7)a2<5rxyEA`mCU8*5oST z3BeZ#a1lMn8fJrM##kH@7s3AE=!@`d9f!p)4zBQ>&JtweHALV^uy5al?U`H3w2U@x%0{x6|m4-{LiRg#r^~<=vCBI?1 z2cwa}uN`JU*HHQjZ+v7O>){_!2rNtKtSObU>>{dO2Qwm~BmhyB6fLHH=q8lDgOvYON5okx_qY7T0h5uGYnU(v6yUR^CR4fW2FrFET}^nZ=N zTJ$C2nSp5)EstcaK}IS^L7FsG8m|F`&?A!N2wkdHKgE;(RJmNPaA7+v*eR9nE%d>p zRz$}6z2;_SRp}7%dD{i%&}WAoe+aJ_y;f-Rjur%(OzgDj^Rq<-O^I0v-srXiE zdV9Cg(TAzoWXcC5)f*&kPI7q`2&t8;MT+3tw}V%{+I)2X@-{OUe*LMeYscsp@ejh? zN@-bSA2UPsCRGs_*^fz z@PZj1+ue?LEAsSx#GQDzxQml+jXldBPPw}}X>y;t@osBhPrcQwX;VkCiiCRI9bYoi zGcqk#;(`X#uosMYIOXiY(leese{ind?Ps^XPciP`aJ%(6cQEsqaR;@gy>DsW`D{Vu zy|*$p?mik^xs6r3OVZY2YnlU~sj7mtSY@Dj|7>X~em*JP&c)UY3Z{wLZx z%FE$bCe*cFq01j^`EQUFGOI&u0TmkoTgI9YlTjm>?AaNb9M$@l^-POro*^kP)p-q1~;atyFExglLRDS)xGph+EZ&^ zxITnQpEy%q3}R?p)w5f-R@P$GWD-49$t^vdT`J8x083#+@53>Y z@nQJA0$Z39$V6#}3ZtT68kCl3d|+U47`ET5=}844ZY%-!5qu{v0NRwGC(v%>`o%p* zBbCib*m*qVv#Sc@Lu{fkl=}amLbiVR|D*Cc;&;eVB8J7*%;Pee7E>a{n~Yfj?Jl1J zb{26M6T|BUKw$SRk>fY+Y*boZx+4gNAnv&^Rv@-!zD-NiF%(aZ#sv*rOJJwmBbag7 z7E`@V<4ZZsx(0JstqvI5(U5)=8`rnXy-u2c*<5QN75ylH20Q1xIJH1Jk=coQV3ZWY z-#)*Cdcg&)qFIRzPY-)x}CEQLSTXgU_+X+N^$Y(SyuhS-WZF= z*34bXqka21x*;eOA}{NB&<)kZJV=j$qp-$R>_WCNaR^qy<+clRB{i7zdg$yz1K^Xd z12(w2sp9_KXYer8OalCdEGEu=Vzk);tE7rpKe5#dxzwSbc8TZy7v?~;Zlt|dr82&| zP2vi41(bF_(;R%H90P# ziqCNQnerS)^eXP%LN@hOJqUSga7^I9k%n<0h0l5Z;|2KCGxW1=y|J1S<%M^rNv4A0 zIk#&zYZkuGt=ijG`IeSyIc29fgwQmi0ixAR_3d56+lvEUlqdeGM#+*TRfK|edO)X@ zZWydnE&JG-MK{f2Wtc6peZxlwH1OANiv}BNELy0dUjc{}hbIiyKAe)=5&=K!+RpFn z2K@mbFZ`6Y3QXWg7ExgrgsrR;(O*P<2GbVfwRxRi#y#V;2)NW^htQomyH>oWm+H0i zFyR;-OR~@1#Ut9?UKYmddu)e5UB6jnJ24uTYhWV4LJWN&jINU)w@AH5+TjBHw`KfH zr&){1WVrX$G{_%Ph)%*s{WDWUoe#IBcjy&Ar8|kpLdF-vb1~P>Rq7x~4r{HZWG5iq ziNO8ourYin$;AN?LxP8WWq14ij!@fW?6wzPEZ}?o19>BSv<={3-< znk#f6GzV5w;^N=kFXR92uTX7P_*!B)G1`J)kh@D~?4u=4pjpQZ%Nkr!Po8-BfDfBV zA~cLl5FMQ}x;=HDxtu7Sr|vB3Y*$M>W4jR-97_wpOo&@tmN=+_?b{2VH>~5m-$o!B zprHD(USJfs>waKJy*uUo z9iT(w?w=1^0uc{lxyekw8g|RuuMs}>r+Y>4mElD)&8bi-{O+en=h!_dBB*O<7e!5k1UFmLgaON+wyQzosmHA}Rbkt_8; ztK?Z(69a1XiR(tro``?{DPIjDVeUaB$5-UgAip}_*}9At$Bnu{9;694f?GbTYIhgk{)RA{O$wXJUJk&}FCEmFFWaYzYRsg&%gUqLxK;(U-*aD55 z-f$%XwsH|d_>Msd7(EawWMeH;t6f37^*tgA(+&mMYd!trx;l215b?~ywadD=5*~5-bpk?_7Efb5HYRG*K*>GFDi(oJzva}pfFyH7@20`) zs7WOO!*lB!MO{LBH71x69?wCu91mGr=sc^3Rw$OO$s@Wq{Pat}l7p2IX{?7E)MJ}{ z<;9yIak0VH42*W#5=Fle|L16ScV(XR47qsu`9opYJlxmGb>i;5cP}{SNj2B*hxxQ6 z0@X=a;=Av{fAvMyRsAaB?4BCxOV*fFy`an2_O2lk_|^E% z7P&x)tj1Fa>@3UyGad!r(?JSP={rpZB&y1gL+2`7Q4fyn8b>Ehp1ib8W49fH@+Qva z>{&W*z3f})$=aEz2ZqN5ylQ@S8@{mUl-uCmPGk(xTc4UKbudL-txD-}7xu=W#-eKIG{pn+SX8DWKbRYZd+(s!^+~ zck^&8i-A-mYeMv3lE0Mc`bAurXmK$c;}c2qG-?a?7zkxY%KXF(q~80cV(UXK7gDpR zpFPw<4S4&u-l&_+;nSpHK6C%oMoyg4qpp)7*?+t>zmQFEuOgpUmYkvnSv}?RC81a8IxHXueAZ^KDvN*Gh?L>ey5ivI zrL_t-vCcEU1?Ng1Vvm^2;o;$6^e*R1PwQH|c=7b>pEwmosD}|om|G&I6UF7x$qy?K zD~L5AhfB8^4)-WJao-GE1xy-t5`0U}5L>g5%Y>{xwcif0>&N>OKd;8gJeCeTKJ5kI zp8XYrH<3|E45|B?J^PEv}W_oRdnJmp+bQsd-LNdIzSx z3>=tow!yCf`HO^qD(9DJsu8@1RQe?BXE3sPC0)8fc+va@V%>=g7Zwy+9lW8rPtYCW zgJ#K_QWvb+Tv;k}4~a&_P^(@|zK5y4spf%Q6(hJrZMAvg)ZLYp)Qa zld6z_W^d+SgyyHXp~)Y`6%R)vO;QQmw_(T4N!3=@%;l6nB_(CR)Rn{#On^5K-c4FRo2ONt)9R-Vl*b!fQ$SBvuxaUh}-@m*Py9yds_U?H8+_o;yRLsDka^tuYNB%RD-3>Wl>hPzV=Qk~{T?-$^x1(b#z3~S0 zOSoipmya(Hkxv#vkYZ^WWG34x;LGhhc8L9nAJ?Xk zs^y*XYkJhH`2~4vH)ZomRI0bJPGJ|PNg@)VA5D&XfBk|}?%;6JJk1Mfik(PYJd0OO zohZlL-W?1K-G(v|Mo?#RnPgNskUPzp&UeD3IN>-{Ht)>2c13J$8G#LNkn)#8zd+PW z8+tEw%3XI3xJ-AAE;W7a;a=8N_9 z#14?+iI(r}a;4sscPVtyp>WoDF8xH6;%#04u+mlryCs1eP9PVvZ2Uy0m3lOc&sL2F z$TeOEdmYnE+sZC(xreL#M^s2^6;13Ze9F9*lsSURb_TR0#}qD&pew zD+Rv}6TZMU_RRX;dxk$B=$Tm+Bg=l#>vbPgXzMZ9Ilj;`Pkr+xDlwTYOZkAKI6LIdY;;b)@h)NA}~ z1Ga@RtgB<~3jSi)^UH5gtiu3CXTWEW~UyMJqtKrwV_8kD)viX)AE&R0sk;QEx zUrJ>X=mLrrHomM6)tk0#8=cZtP#nRZW7+!ex0eA4q@uAiYY;nhhk;Mas7hE=BZijw zm9@>|@)qc6HMVdmAD+J`4`%3Bm6<2hhse@}@QZpw49g2@^cIOp} ziU!#SGVSFp%}rlt_gyWQTFo2drz2Zb$bvCeFmu>x{xn#GDZ{LQP+u_SJNMC6-z~fD zfD!Ge<{R>ojBB^)a})x)^t9%@MH1^;o3w!gDWKX0AxJMW&Dv^t{!*chEU~DIOG;y_ zky-EvA^zNLRY%8JPS$Vt8^-vzM0UBYDI!k}7bSRV9L;>mx%tb_Ovd*HhIZi;m9rc) z)Jnm@M>ku|X6(9im&XqsI+W9Ls{fQ_qY7{R$^F%`U!wKOZPpz;lVl|^SpOjKHGJ>v zzteyI4{s-~8+mzCKP3nF5}Y0`BXtf5RIkIgTFpB4$3z3)@MkCs`fn{Q2#R8v3iaB$ z{CFM}(YuQH8f13S0q@^4)WCvY3U{IpK1JKcFL9_0!$+bg+}qc)R~=zv!AkiZ-G(9b z#uyg>NY5$QQI;A98srBBP+26M1*pRMlkMwBR}0{*UJtlCW@{Zl4&B+5+ecif3xIcJ z_xD^%edAI_14$-9HdWRg1>%HVhu>dKjxnH!%f=YW)aI9jXJDX?2$Hm#xHd;Fk%WBr zly_U`SFMycMNsJmTm0O}pb2;)2VW~@7=k)EmSkEnS0);zlFf2VgFZ1f^WD1y=7ABO z6FT*Kc@4l%n@;J92v2;2jI4>nvWSs>PW`wS;jlT$nWMSJSclR!f%Ogb;IQ`A$%Uu- zN}T{>*b~S&OB!T-q-zml zo{x*i-!PJ?lRBKqpL85`nAfkpL@Ny7oI#8wU$JL^fmx1IT&{-POR*0_Uh`{wdpjcK z3!7nJ=Nl_a6LdD<(<*5rh+olf5@y^}JTfQZBB4p#ORhnqG%Oflw!}IbPcE>20z5xW zou?_Wj+mPYTFmBIU}K%;Q{Hxh`*ex1G;-{Qk)v6+@M_Kh zGgm*_hnj-Y1d8+?Q#88360Ju4_Uo@<$Jzq=QE-E|StxhcfVijtD&1rZCE|DC-ESeE z6myib9}Qz77onhbgDM~LU=G#KT4iQ>gqqe~PdZ_J|R z3QYcIfzQZM9Q!Q3O`lhej{qRSw=Wa}zYvsI{Y*sjG53DI8Ia{qYU|i%neAqheYy7iS}x^*l)kdflBK%&RwJa;WID4m0uRAf8V_ktsV?Qr?-}u! z0vz$lVjC(|)0{`3f4@5!H5uMOZ9a~UjOfcO+>Y)~%;^*(0&~$1i79_%abTd0WZ?Qn zdSNN~+MvvyZD^`XnBhfURN3NvJSJ)VYayE+2Z4s#mFxI`ZUhg8$K$ z)JJE~d+(%6%|8)kGw&}QWir!^YqY$^u)0f5^(--$!;6`+ ze|Y=^^0XO5d_4pLmFSgCbW zWc)`l^{}v3z{lQS^CoDvodZ!-hg9^&%zb|W0~YNy0K{dJFe%C&krD5|&<}F$WHr?! z@04m>M#PlJ4TdFtFSmBGo1z~-=CX0Q)E*f?hqpxW_LlH>i~}AaiIGG7`t{F`?YOx1 z0aJoXzu9l-#~kJVgJ=0YLv~*WTw$q6)q6RTJ4lV85uBvl>EY-(3 zQ-V&8*%JUN7ML!ee7w8&0a;jOv+;3ExLFzRMpS5TX%8emD|yG{jFH?2kF99$9n^yy zIag)8q8m99P=HIgfYzyldW_%P0ABhU!aJqnknGH+$- z=l#KN4IIEe^ZjyKUi94^nYi-d!v~A!d-g}QX3b6pOx+0&fLTs51z^>#e4x?vXcspi zzWTPgpAqj>OViO7YQGj037GzZh)fP$q^SeM0wwh<@xYwP6mzsse2dI#5tFb<;0?gPYq~)1^;fuZMjmPu3jg z4^A5iM!|WiF*9CmB$D{yPnR?07HH;A&Qh7P+m3AUmX-W{z`n=>(@2x-cO?y3HshhZ zhH8y}JPrC3U5Y4?YB>)S%S25_bU_enasHKteM?Y7WYQ!<)KQ04W-b4sG+V}Mm=SV* z=$w+d%_rW>HJ)fh3D&(<1=*FonY5yFw{DAf?E{I^WG_kVZ~>|Oc5GaY6@^5XN z#uS`_r7v-{idtk4L#ZUeXMR}uU>DL*HE@L$fM$_Cl=$eD{0#ujk!eVHj`R;^Nhm#i090 zoGY+wf|KCXpv>r&aljElLz!XI8BB$lph9E?1k;%9VsV2y6I;g26h(uOeK=i>Q&>W? zW)*F9t?}b*g9vKPPysQhVZWy}yV;G|#coIlH0Si3|Ns9!@AJOT^A0_IoDK@(&Zr-- zW4+Gx{I=S3CBek96a&0bwAXwF(Fqx1R7PFAjm`8Gln2gzpGNBJ{$qOM_=B;*`thdX zxMT#S9Bn&@Yg85nu;sc`=MvFG-7qXhnny|Wxp2aFmJ`}qzvui(-*S8lRNyb@jZ7n5 zJOtW1gcFsfWwBoSb~ujv&H^A4JLX^>@|rh|EVk*z7;q;h1kS6d})Ct5x(P9CO0+qZ!w}frVS?xMHzb z=f#r`VxHi&z$u|q(iRPQ9xjPkNQFhrtDFNvG! z=hj9SgIz>Ng?0`-2Q%IdZq(7PMXsrHob5#;9=?WjSjDY4}PEQXlJW1w3(?HI~p4P$pV1m@ooDD5927ssb%T+k@Qs2>!CKU|B#DgNkA-ykt&j%JT2fiKUSl zUu8xpZb-!p5?(J3Zqp`5e-Uy_op&TKg`_#6O!XOMb<~zcFHw7l()F45SMEehXL8(c zFx$gQ^ksf}YY&Cn53+n+WR|jEgfiz2Q}C1UFYG~NDXlzBF}D&+#my99Aa0;1Ii;TB)0><5Sbdm@~Nh(jcB}1Xoot!M(DZV>r^}!H3j^t9Rwr zcA%lsv=NL&KI)%F*35_Gd~~K+(hZp?wWLJnOmZ_OgeIaHhNgN;!h(6x4RC4H(eo!wA;PSh30xQpfBxd+|WpI6HWMjTn2t=xfk2WJC3&9yb#Vgec*g zvu{0B%4TRU_7=ocV&V2Sfo4Wru;gvBtXmLZ+1vN;E7O7uZ zW1KM&fRlDW+k5438I_nIK)5a=@Y$%H;#6#9?v=zRnrZ}AU2iV9?QqdA$%9c3n2Kk- zJ57H8wafUF#TrBpVy|nMVA*!0XU*6^T_E;79{uodf27NZ|Ngk@ulRpIR&U!sSNoSk i?tjvq{r^|)9)F_B>*@YvL*G7K+;}VR&5k8&oPPt6Q^PU< literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn/gb300/gdn_fixed_seq_flops.png b/benchmark/linear_attention/results/gdn/gb300/gdn_fixed_seq_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..ce33af58c1f102dfd62367ef82d51976dc51ecaa GIT binary patch literal 107910 zcmd?Rc{JDW-Z%OYMWl?WWJ*agL=usyK}m)XB4bL5P{vFdqCzTDrA%eW7@10xAwo(i zWk|*bnP;K%x~hFY&wkeV^PIKLTF2V=Z{I%~-_LMe@AqqZUzfBs)tDLAFj5r7yno;B zLlniZfud;L8J6HX+h*7e@LxO5?J+#3d*=8#R|_X=>Y&9rhtp@yojz&F?_%wA_T(A+ zt>TiB;*z5LHs{VcoZTTIVfQ~jA%4c`goINI?KJ+%Qipv;XDNz{jrn~| zzgtDmE%s--t09L$>yqK~(VsbTEcS5xwgY@+WPt$gCD%j=G;@P+^WxnM>*`VB1q{bM>8zQoh5`|mFimbDSeyZ`%Zv7bNN|MAyS zl^wJf{r6vEOs7?&=6o_z2&MQuIN z5KvlL8YP%0(Vnm?X!y%>T4|d%RQnUxW*6Dw(c#WrYh@j%iT-A_tGqiO@n|MIw0$q) zIXz7CF8QE;T3T8`g}^!y?WIeX8a_U^i87y?9uYPzDZ-y?`_gY%WcMj|@$=`;r#plA zv>)hgvI`n}?=d^RY4YbM|0DUf30iTAEz5)Oyo8O}R9NWj=vD9l?p^BXIo9^rnd$Z2 zox9GRlRkIuoOa^wr5{^cla9Z-6~1r1NPksjrTNTgFJs+$wx!FKEz`cYk2OMAXOZ)d zrVDrO@Z5Ww!M@+C<%4U7S5Qm-sdTZj^HcT;=O>rFf1vMw<;sEOlZPFRpPrxEduNBU z^aIzg4Ls7PKeA9P+}w41Yw{ewzujQ+_}o(J&12_NvT}0PM&3i8pAJt=sU__VeN%k) zt3vcUp@UXyrEOST23i8I394DvMTx~pmd|y6`xe~aoLx~@#}pP8#@OF*M?MJuzWM#N zee1W#$n5=;Yu)(hr0&O$A5-njj&jFI$;iBTt@Jy?sXg5w+hKmzxx{T`ksp80kB|E` zHDBC65IsCMb9Vju_3@Tfmu=dLBr`HI^`p!l*|7FC+__?ucf7N(!@I7rfHCj<)L`f5 zr{{x1LIT3Vma9jLhvr(>CF&&Y#ixF7aZ~T9);e5Ge^Ojre7QlE$@9DuuUWXb=yi2< zKYsf3{N>A~$B!TX^g#=siMQ@J)_1V7vXY{NG-DYK95@hK5EmZqTNf=+(b2)TY11ZQ z-Q;C>sPvmRr=RYB@!~~!Yb!V2FatAlU}>r1X*;{d#63)ghK4qe=G%@GIS3q+y192K z@I*uG^H;aGTiDuO{{Ak7%f@78lZXi82K5MvQg9!o!;5+6v^!4C+?-$BtTg1!8||2z zH!pqNrAeP<-&Vw$kdQ$67TC2Wx{viOX0jXpWvMQtX+Sjk#l7XrFNMU9y9N zgKuG>YeDb3w{IP$29GLuj59PgHio}Eh7~FM<3r{ZkBJ{D1_tx<=SEL`NIB>1tnh^0=?9ib@WYJ>botm3xs*zp!w4{WElQR&n zBLP>EiU)k*y_UMVUuLEl9)D1ZR$QXz%;=gkSFob@Z6@U+QB|CuUwPF|k4NSlg{~HQ|c4`etu0K>gIBCa#+0QM$|fA-q>hsHS;JlO3e89>hsa!N9ib4 zY&)ltj+=$~XTNHO_P3XM8E0^DbKmYB@IJM6?b^z(>A3YzQA3SML6q{U?e-?65r3Aa ztV2BhUCycJ#AQ*RfzMBkDgJXmKkD?(;>S;(3$^JeeL6fl(XWuNlVK851E0jg=bF5>n$Gb{`#)l-$_VJ>|l(%YnHjCW#Z_> zb{UzK*vT92-M^ofQFij3Py!aS--Qbo4z+)M9e0YO0);664|&u3X!ZN$F_KmRMMbia z6WC50rO$jedGr3g{^}=K6I)KcT`!=hsM!2k%LPS(jw(;wb0t?y9^a<%#hXi8bR~s< z_ujpiU!Oc2H&wcFVPQe_*W`dcuWnRSRQYIc-8TFF%1i8w6VakZhkUP{7&zd`Gv?&= z@@oGs)9kCauy9mw2pz;!)v4_kvrdPGf6W)JAM7$BpV22$UVS zo*@52&bjZ0@*S7)?9da|NepP2&9{Berm%BoUR!i{I7@$f*5c5^d2zvFxs zSSbeDCCh4@xJlg)q)9o0o{*?IFx*+OYR5TYODiiH>cfW*4(HD8{{Hs9z_ooWt5yYH z;gaYYXe~6v-jcCzV_30bMZRm~_>cDoO1d}f+{weI%6Gz2f+n!BU7F52-&Q<|#tM$em4)?3beR-nE#F@JI!{z5a@R!p0w$9PlYBKLOt zFIbDIrcWOi?Aom7F*V3+Y;26apxm0!6QdzikztfaDbE@32nq@YK54R+T8*_%K9cO7 zw&Jr6PEO_i^eeQnzd~E4n^IiAm${EEpIX!rne?c-Vo|VP0Ndx2eH9Z|}sQOs!J#R#+)&=!+d%@#@vV?tzsu zcmdVHtb+Uui?9aXBp>Ad*wT`?@7g*BCZ=85+AQ|%j~C+^DR@q^EMLC7W%T^!qCV_KtW+`isl_vH#b@GsPNIJKbIJ#E#R6QQJ7b)S{0ASw`KeG zYLu^SpBIM8ev6x)KtXQIKV^Vg^}M>8{%gaXiWi~WBeN5yVr3mwzYlhxFg-^zptMWe z93R=WuEe`UW$;Z&*=Snk9jc#xm{jc|ht5Ugh#41r`S{Sv!Nuk9C)4!9nH;%K(0}-4Wmj#~y1QFLW108D%!$hCYO9608FlC1 zzlN0d-9IFK@+}=UziP2lFAE>v3S8^RnCcUk0f!$=n;xBN)=XOnTERzN45{>wA3y%q zlE3-E;mpgp8}ebY=O;y53hb_s&e_(c(9wT&%{Iodudl1`?mWN6>cz6AOk;zAF--yt zoLhJ~{~&}0y#FeB^V6qK2fRzsN*9erwfgz-;VxnQ2N9HVTCC7&`nT`i1>zAcm64IL zG}3pyYHfeO+u$HNe!%$nxyS)qLES|s?Yk;3b@2>qYHBW`EM~{Q8-18rxpjl*%P`)Z zA{2{=K~_dVh>&ZVgjuPH0-r&)>78^LJrw@X9G~Cb0_)fBHaTu*cPFXY8lYzp^}MEL z-^ao;akWx+@7^tc;yR?4b$({_KL53Sbh~!#iu}Y;O6Tb4Xc@(D{``4?EnBp$iz6a9 zo>x@tavT5Fcp_}esrra;R}~c%zuoIl76w}K+1c3HIX`R^5@O_$cj0(#rEjFAc~@bR zo4fnc{rmU(@iUZnRs?M;y)lFT^3xOHq!`e?xmv^JU4F)_>bSM-RKyR7Bail_niS{e zSXD0<*tSjI;^?76*JExfADy3jrm@uQYEI6U1hQ(I`+M(v80skFQczHkwtbBn zOYw`E6lp7H#%vNeF^aO-KOvx$b@i6b`v;LNulGBx;8kGB+r}YbYVz^v`NG;Eo$uej zmydK;2i8VzW}*Us7&k}09K1ELFn>Nl^Ue-@MrGyhef#!li{#<)<*z&NnyBD!@r0C=lyizli=Fn=he$ehRW3@4D`qw@FvzzikeZS9IKj+`s+$xQd1>jF z@AmN6ffev&U?8-s>Pot3dP+*rbAS4zQ_Wf5-lrR66>N-DZ_G5_{>Y|r(ZSm?fn{Y% z$LnJ*m_GGLKywl4pPTK^p8lQ^Mv>mWZ65%y8}REJM? z=hg@TIl0x?MT@AAwQ_8D;YCipj4nSvUg+@oee}C{UiDQztBz7H;8J?Z?fm)8?T)=a zeq6>U+vsI!+>}z6+@q#e@$Mb_na?FEcu*pTGlGGj9^5(MRvpH}z`&r4D%)6KXI%N> z1*zEh4n;L(7_sAi^lM`hzL(ZmDV+Pp!mHrsyM%d7JRqis6^$=QKqqPxL7F#ju0|iB zRZ~;TAEWsyvk+@o=Dm|L$I~pfr=^4r-Usu(~bKqhoeB+h%n)u9T3FkW6YotA5{*o`Ls! zIP-{gtEuMeJ;je6NxQkZJ$wCnh4PEDtyxFqV{YBL{Pmi@KkZ|efm;u>Qh6#4vC zEP48r6_{<^p3BcZf94J4-hNR}L|=rDmGuHXpN2}-idzmKOiB|r_#Qny4pA|&&YwT$ z_Plzx@U2mBWN_e9-iZnzQj4bv2?BkuZU=!eQ53JDC&Q^zrvjziUwS-pA3F}@7!wm? zUU89m3--2Q2iD!PHPQ@tt(AN}a~B-DU#P6R7TkY+;zuAl$mS|$YRQr%hNYfznsJJu zSYn-Dp8KP@d1hJ7$2xhOuDyT%e&>fwWAdV~9?>7P(++KTbovw9%9SfW*>9JY<^ZHt zF*ICN|LRp^k)u^;-h7GAZ}duDw}~H|D1JK08mlF5go22?_?30AY;Ke(F)>lQ)YBck zX%RZ9x;1`t+hbl24-ZQ$g;b+FA+kFbGp*#nwWue{K3t?jUWzG1#C%G_*@n zMFklaEm^)cczW6`=JxGwbiJJXe0`(CB9V#MeKxHHVi%dYqmm4J@EMZ!ZL%S&w=NkO z8DUtqY&W1jXYKg+cWQR?*!C9F!<~i&r&p6@hBq4c`jMoh>uZ zho+}bXOutguD;4d(b3Y5DDX6ex*I^ zFWbPEX=;i#ALVg%b$$30i_7}SycY8s>3tNx9(F{v^R{apH`$!do@P{i;ryz+T-AD~ zm6esn=Ms0@t$IuS`0H16Dph3B4>6BPb1BlD~?jHuOw7&mHRNnN7Tuk+ov6qt{%M-(n^(Gi;mp zQ7=ngqqH{_4G>t-AtqgX+zxN0?f9`{_g=I{{OXN1YbsT$D?323@G5y{d$oMYK-bFo(k5dI9Y8(0krPajOufb-PC);>q`?ELnPfscHCl_j_i}nzxA>@xHfVF%h>2chVCoW=LU0qbq zfPM9xP~6qWs5MwKC~}qPe302L5TO9CY|Sf)wtM^8sb|=H!{9s|s29rJ4QEE(-_n@( z)_3-WTC>eI^)!J)ZexASM+#0~ z+8eqC_2v(a0Bn8;=1tgG*~09p((1b%w#`}0^z`%q1N3FT7v$G$x6cUATL;6?KvD z7YHJt!bA)C(%TzIz!VnXK6UkSw8ex+?V#3{pkWjxFE4-YN7GX12L$M3nLfSDyVD~z z)E#Bqa-}cOY&nFgzP>)tx(di;u( zE9rJ!T-I1*m!|hX=IqhKpkkFFMr2LicGkK>t{)|iikC9>pBwYLe}BURgY2wa!j1`x z#L9OaY}1*xkHqfYX5Y37cqtHc54|#VM3D_!+e`DWW*n#qP==SZlwZ*9z6^n=5MkXG z=HwJTr>V1N&qiWhYz|t=F7ze4Y=IR1i>!isQ#Gy`f4rBHB8Zn*Xy5MV@9$rv04kVy zB;W7q)fEd1zs}qb%SP?6EDu;}Ir|=TPWYZL^U|d%;Bv!1KW)XHT8w(JvU&CD)t%oO zlfX4^ylrBDj0<%}d||qKMg85K+}P~;GqmS$fw=iE-@jji9M$LX8;#JhvwShNT6cR* zC@?A6!@x4Es05&pSNd(6QZEo*-rURuU3U+-#K;n!eVn-iyfErcak$2tVF>dqdv@2oT<0|Mv*zp4n` z5CFU;4-F3r)yn(VK%uBl%*>-ut%I?$I!Szf(zp?k1qPjL)14Z36;_}6ka0mnNLRQy z!)WW=uc0zq-SxDq?@Sy@;{eYo@|dtK)soPDLsUgzQY<)j(5`2Zn+-_Skd>8<_nx1H zeBx2y&oSyGzA#*|qKiWo=!{WGoQ{qz9^4;wqQ2Q{`HB_QfPUAmU-tv)_=1~d9Mbps zJ->a2^N#?Ob*rA5YekOT7i(QqckOZ~(w;eDymkt3F1wsH>yBh?+hJfj&OXywAf-Gi;vn5MfCMV%2VQP%kf2NVJGGiR=X zO!c_=vxgGAyk&pmcsxqa>Mg{FyKW6o_@!Mtc|J4n~&n-iGF)29$0A zn3Q9}LYHA&xEx@q8aQ3jqMR;5%=oa0&+i4VxtXtVlrLmb3-kiUv=7l@#uu?-(nF2! z?)AbFHGJxk|8_Pwn8C3nDR#Q;YdP8m4N)i{`0%t8p2?OT2)}lX)qD2)M$*RziX9+M zXP7=+V_h4eBS+nf@0oCJs{HDGknxC1bC6%DVsdiw<>mlj&j(_z11;MghCE0UcItn3 z3cGk6b_;oao^#Wzcug-*oAORqK(+xLqNT`pVK-EuaEGICSyWsk;=1mGF>D%Ywg0y~ z%TuSWfGt1t^nU+ZLs3{oQ8oaSG*$bpQ}PnjwsjluL@jUCV*RI!6`pDqCkR9RD$0*Y zMBOWCh6B;!oc#)Tp4P@tiC5AI2<|BNo?Swf16S&QP;&U#(}mZwH2X<`cEAahg_j?u*7r5DZdweP>5o=xNLP|j)~&D$|bf9 z0RZxm&ggkrGS(%@zY^_dFQR8&$*#kr)cfEy(iIq=p1zr4L4_Ie3{{c`s|7;znjL?c z8>`^#eRu^?L-G7o>-KjN09biD3ZD5~(BN?ddbmd}-EHV^^+| z-LV6gh62X~X0oRF)lzqn@mK}DgB{_~R=Vt?R{if@OJ-$dLEd=rVsCH65$G~JJI-mJ zH>pdBJ{0{~u~2cUm|v*@)qGHF5^jIKS6r=yT|ejanjNuIJ8yFbR%w z?CLvBf3sVs3XpK?@%m?IQI|oHtlEkk8Q-nsI1HR~xVQw2f|43!y9&iKG?bZAW}%DS z;mk;FaT_+zPaB6e%FM*XL`Z9M)=}MCOW^+z=mA+t87%zz_3Kb{sbE(rB|t(VN_;AX zb=TlB&hEz_oMx!cvr$kjtn%Le8(;eRE}D7IY|v`0Z6MqC)-6UJ9-hDu=D69xCqoW% z(~c&f6}_`zJn|y#IwA&n#}7he5RjIZHnVJ~uc)A<%+1Z?Wv-~8<2FL2jNN&DB@tTj zo+4^F^yaQzyGF%39$y5hdk(?{V;vhC+c0cVBFI9#_kHq29uFvUa@yei-s3Zo8)$rG z@6b_gPtS8f!>edaQscCI0*QBr%g^A{RAG%fJ2-KVdab4pEvtqH+vNfKdyGTkZU9Pz zt(EipH8n(J9W)pbq9noisI02$gi9d{)E65Y>oEL9B~ZEnSW&{Xgn_?Xa|TT&Eu%SM z7d^_ZOx=@jXMfxNi4e=LMFlW4Q*K7d|F*miI&1VN+cGKiS0EY}(^XEKMf!+G~t{ob&(X)d-| zki?%u_rWjjfj`q!a7{Pvmq~cp_L{9UKRe3!P?lJm0ru9StyKbvK7>58j90;M)7DRb z4#Pb!*@;^+;GFrVeFVs4X)WIKu78(`DPM0~AvN(@3!K=kz3#`a_B7p;VHmW+ViF)d z$(nZ_$!TP&l zE~z!8^!LK~K+QbtsXker+#L`PMY?rPyuG*IFz-0Cs;X*^P2;+Zqfcbax`c&6=WXSg zx270Iz23j+uEC!sJ}A~bZZF( zVu}5`Z#`R}i{ilgR~ic*$GyeI>d=y?V{=9K?6qma8ynAqfj*o|VHEM6alH3^g&W*s z+Z5B`iHSX6(!{iYOtiPNde6bunlm?b`TG0&tq?L$_uoM50ZFSD<}kBloqF~~y=$z= zcIH#`tkGHPzTf{&q*HILMN_JdT$Vwh{ZaS@T*@zI#=5GQOo|+6u=1*aO&oh`Ig5&l zf^~~1cX=)d4NLj0sDphR-8c8IXKMl~GNa!>3}cFA<;tku3E54XmimrP7;3i9oH%xj zXh2{WUw_Qiy*d@mg$K49$}C}A%XxN~qwf&|x7i1w2*JG}&mhM)LXU)*uR6!NQHj_i zN^`@sAF|D^Vu>eVOA-g9zP|oTceg+Ic>)0N7AYxJZSC;AablD?=;g-(J8po~wzjq} zKYzxakxYMwECQgpe4P|-TU%aOAiKH`V0olsg%Z#sz0r&CnLKo!t`kf%Gui5g`>|Lr z<2qn~@DQ8_FiFH!=~Ew;!I3=VG4#Iat0I?F+4k}NW-SZlMd2f>9mvCl6$8kh_}~{JSzroFnW)yb)pMihSip#G0Cg`q}{PfQT34C6UfZ0h@pTCk=W8DW95 z^zMK$xngJFnh_2HmF~8TT{$t!?(pf|cG?>4(#I!446D!|$9UTu`|4MZ|NO)*b#oau znhH9ng_RZU)~#Ddf)uuI-%eJu(2cayT4rWS%}R_lcI}Tjm0y_DZ9bf#ZS?K=ZaecnP*cU&Ni6P!Zo%8p%i(#Yt!)irPpb=tD zWN(j>$gzb4Rd(Zo(=R9K{o4=;alGpzdwt*foV+}L2|f zy%v2|GUcm^cktD#YDDM@Ud^97R{Xl`wH>r-qJ1E>u`oYncE9^0!~|ld;`xoh+(1oP ziG5kuyy`>N(JP1Ut6RW>23*T8r>X!+Epgjq9aA>r{*7RFXlIPr|ECf70e^nk?8c;O zdi53BSrVG}J`Ifk0Qn=2Y-qsqp?`3oQc=D{ReJqet20m4)3apycSb{g#s@I%WhYOb zG|Vxl0b+lM(8JHRVs?rs*CE{7m%~Em>=w?z(yh5#sKqH^x)#hGyZvitcekLp)`0^R z04RX)$&pjVb8b9koN~pA9ODy`W0KMDPhr_)Iw-oJluo1l2?3qy z{_L_ZgM(pUBXg*H4z8}vMcybsL{~dE^qGeQXRzZMK?fbZka3|=Ui&Ds0ry9vd{W!S zwEskuyZJs%rD{NPRxk1YtXRZxZb_f?+iZ~a1qkrSrA|mplrOxFW1nauKH*YxPJfaj zq8BnbFJO2wZYh5)arVJ`R6A&LBCR`a4wc*nH9sb`8nUlH>PLFX4HCCNN3Mv|iG4 zST}!4-{gLLB^nw;TpKTKztGT7w~gmvpai)<%R!pVjd;N&nSoCa$!kx@I*u_b#-0u;gzYWo3JHH+y=} z+o;qXSi%gFpb;wzpaEnVjM@h_LO)` zFycGtrF*fXQFAvPuhVud%xc5l&T;BhAEB>Cq5FEiWHG6USo+suVuB$JcF)a>iI^Qw zQc{B6yvfc$BvMuAhPAu+aaZ0j7|f(lgKav3C1M-V!m=e8N1|-b53)vZoS^hX@$10) z*Tq6Z%r`r@;LtHa4qdf+HG%>r)_r_4@fVo5#DiTOevj5kFs@$Xw7JZ*`lsh$^ZK&h z@mcLR&DHgB+_|&$y@-TGOPG<66fC?Mr}|^n&z@1ZnHN=8xH<<0?sU%*r}y{d*qH9B z2<`jooD|Dep)|qQ96}OtW)oX)C=DG-lQzrylUnTE_$M@&sG0^XlBi^+rJk3dCvQH- z!1)M(zYAc83Qyu|K~>Ii8?nT$q#qm{BpxTD$nH#{+=5fFa&ZMgdQCDb^Ip?@P)n;C zt}zV-5iq@$F9p#lgwDo1uNXu^mR|Al<;$GhToS>_6@3Dv7Z4a|fqn1b?93T`%EREH z5Q@^;d&$X`V7qbiVZ9~xT6dXyK=HedW_{rCcly{G;6jhPyhOr1V?2W3d30Dz})rdWoy9d8%C~rj~3T5J@C{-1WovUJ-sis z3~f!#ENP(m6B{*SgVE1|m1M(KNhUtCe09csr%Z2kJ!nMJg4-;h;eBf?He?3eW=1y* zc9dn0&d~FFm)2d@6WIeENt`xnb#tK7!ZaduWM6-6M>8ig-=ZcJUQA7UDv zDu&k$_k2s;Kuul)^{nURjj>+(zBx~xyixSfxrp1qn`V4a1;jL8g`+S5iCLgmqexeS z9!q+Re_Ly}u~3SySuPSRe_{Tl(DoVg_mH_sM2y684$XrA5THpxz=_MoN{Q>oWf#{5 z|1lEe{?j@k3BS0$;Q&#iA6iv+LG{Ra^hk7gp!TcYVyQtts06lQKYXUzUC~V>k(CKF z^O?`rZhwE*fFv~#MVTDxxQd$GW23EVYy05iwG?~Xi#yLx9y_1nXbJ^`hFY?0Rs85+ zS-qn7TrgfiIbt~$+J5LN)G_($$3ndXdw`GuB9OEPM8_bZC)dtMw{tlYBkRL&L}~O~ z#LKYFxx`Jhg?|?#bJQxf=fz$(u^ze*e3*>_; z`JS-@kOjXC4H31*60W1WV(FN%sgzAm-+U?B-kKJs}Yh^%CCW zyp`HnCdH@UO-h&)vH0=F!@(k%9;&CGXK3-nj^mSLjfF@5S!2%5&Lngy$^G1yhE^}C zzcGm(2s98+pex$UTLm?haqb9+rRed)FJGM~#2sGof##`eaTIx|$`F01-4$q?kw5BE zLf@JrUcT|yLt)bJfX3nlSMNiZiNwTQ;byK35f!YXAz zH)5w4{zPK^{QT%Bgl&+O66r_Ug9(eC9qn*^)X`GU;zPytc6SuLj#wL`9$VgI+n~-s z1;7TdDeaaEHjV^1xiW!J9OnlfC2tdoAE~U=Og^Z#AACl8E|u>=CTk#x1ANhtTbkSp^?5Pn7Z|uFs(KpPHWKI%7CrSYITRuuTCJ|}i{@qncwj3j26z;QcU;_da+mJtS!*<`Pf=dDHIgf%@qL32 z5CGG*mMoojJ`)t!JyTsca6AXLmqYhuh2O^YIKIs@Sp zhH=qn8}UjNWE7v;FQKCg&fY{>)W=Aw+x7KLFJ)%tM5A6;TJ7oK{YOE=`dJC{tJ2!r z+uKqB!kKNMg}Hfn@2{g4U}Kz)aoG0v#%D`?ew#w8jyCKAvHVovK0R#la1JdiPs|4~ z>-z(6ZBK1b+6zN5pzutGNaVGb$GRkiBh>=|8m*r^)pGanU<4Rohcvozy^v5}QR&=~ zk?-eoL_qwZK;Bo6Jai;F+A0s?H++0zvDbl!!f6%NL)^skprV)7=*S4`y+dgT!y<+I z*4YC-p=G<4Qa6Ha3`B`(NjB-T>U=l#xZDP<*s=tW&cKia${?E?nD}6&uFV%y(u@){ zN-WSY4XY|C2{XGMYhpI);Uyg0-<<*{VLM&|T8L=fPPbv@Es~O~WgCm1JYfO^5~(Zo z`gNFuZ7F3E{e3(8Gs{TorUs%OB}HbS$A=AlplLtv=rFU(0K{ZfvVt{maNmIgx*~#S zu%OL`$H(;^Nciw!$(PjH_2ievMn*?Q)em6bnaxCAVG#8RH&);jz9uVCdujN7;Vy6Dj8&U4?wWhGAh^?HUOFOuro+1Ux@ zZuiVUVF$t&5hhgwKYxZmKkKT$xfPV5UZ2~jDzgFTm2qxx0BW)&s01)U6>_^dxPV7z zI(SJS@W_!hFfB&NA4Qt~P@0~=ZHWHtTfAn^!P{Yj&8!x*oCooP%Ot>#wTWvanM@#P z9sY-PL((EZEP&6~?PkbLfLf58Vu5`-5gTc1WW@~-S%eCdfFzI2$D9ONy9-dM5z4)h z{xog9A@o4Vni*sEB+9T)enWSeOAUd^emGykrYY z%j-`zPBFbf&|=wYDLP_?;o2?Wnh+D%q}U186rpKVQ$1L@rq+E?qyrIe@Av%td{r4b zWTpc%H{!j>g=T5{p#I6a4z7jouT$(~jcC;uG;5+yKz$`E2GlXFq(I!XWEDCX^b6Bp zh~4H_`@zXaARzP69I6}PkiK&VQsTJXv3*P;iK1`oCGy07v%=>^(gdi1$tc!Ep9-3JUgu6<>>ppaaqPJ2mn^ ze{*2{qQLh6JB|>aiJS~Ueh)Th=a(cc#Ti)tB zT+qJyoBrTs{8CbftYubFsnGaHSPTVa9n#7`M{4+JGHnsD*hnZ z8-b$I+6#h$=n)2~{LSR&3(eA`#ErOI?1dyEALxNM@B$(9OEA|HfuDqrK9(ke4>kx{ zqxEZbFjfQdt=l`U0kT8Z^~m^lT||19Lo3>aAlC@yQ`XA6EQ5bcA|)-~wMUQe)x-!P z`?O3*NJu@7{SvVfIPI=pCYh{SO=V@@Es&d^BTm=ZkXpY2ntCU^RRi~_BYh#XMmrG~ z3j|DML}c(V_Z6X=Y;*$MS+=dW!EF5 ztD1J`zI03J1Nom@hfaWN>wV~dd;bax8(U(5Js!}%z%cA;k?IkyC9u*taRtfYj(ghO zBi`rB7k*9TXYea5EHpoHB1P|WeZ2w&`Ey<+?xbE zGR7f#Gpj!r-PELKft0@=URdqda&VNGZKtB#cK&XH^dRlcqxVz^xbhGff}T++`Vga$ ztKv2YuAzmQgHIi>9v4vrVA%;JW%L*%(-pU^y$#XPXKo}YCUVnQ((X%a@0g8yi9xHi}SJdo#vIF#NM*jkHbtq<97~%EYJY z|IMMn)Lcu37_dDABGvOk1kM^m`NW7&W=k${EHIKl#Y5+FaCB@uXOI_S%;TOMfZ>o4 z`v<58BK1|jU^yd>Y&>u@O8G@eGsyx_?_K?lPa;G}nlt<;^hsZoXOiiK=9gA-n}H%y z05&Pf5bwFNau1LUiE!ch7Zn#HZO#I2bGS%Q{YFvV=#TfCNuU9vm|R?X02@gj@J>HI zah0IV5jGHp?o(cB_QZu59|W+xIDW?y-{fB35czG{2<0G0AOP+I^&$vY@S{09JIAgV zAq;+gehO}*8p@phU*=x6oo4fmKoKJ&J zNZfICZ8iPmfO?cW2NZ6inB8T1$)tNaRam1R(UwXbsMkW8c1mdWLR}iMR(&#%1%9 z+_>;uud>ajfvO!Qe&`cX5I2D569{RS+{5wX$pxQNT%}IzY5!&AW3;M2!epf z1V(Nk-yMt^z{^;Mp{O->#_i`V&>X*Xb|#FjdRGYm4k!`v*z@3nvLXd^I|Ic~cVyHvb-?iX+9?c>a2c_rIWVj>RryTlitm#TFiYe<}t_JlPc} z#0*Q9UVzw%iI~ow9y$yMicaO_Fe0l(y2j*y@we~a6TmykWXbu@C)B)S$vg}ejDe9* zc6Fi&+2g{0_jpb3hSX=;Wj7Ltxh&N@_ zYD5>R4LVM$UK-mil43aJZvqDsG1XywcFFmvhc2^?nM2< z0( zrxO?b-bHvkw42-vR zG~MIO;4tKCJkv7y--LDIq7Q(bnByvc{#>Os0piu4FR#OYnaCz^Zx(S#hv>*(%)EU1Nw$4qyj~$5oueypk04g$oL+|vuAft-k>Vtj_1#me!(<*rvQPP!RwZMw$gxz5C_TEwie2D)InJ6BF zL)|XJq?ipllrq{X$6#N$yi*SYQY#inCxUow%J>CxgB!XKA5SKF$$Tf_LtrjTcJ6#? zQjHw)`G=keiZ|Tdc?C_e0xk;>*e>K=tBP&(>!~oEGsWx z@#)iQ-2F4W3!;}`D9aL)F&HfKGhBd8MTjo|iUryp)?p%^h@7`Q?nrU_Uq?t`zlKI7 zQVb;Of_(7ijIB6YfS6gptGm`+6V7s@&4NiJ0(`Dg^pwR2s#q3eIuSI|06dX+ zKuL9fHZ)jC=O20ue$=Jieh}0Wj=j9jXpIvGn6Q=tizP9#PAD9C?HDLL>4WYQ1f63O zv*jD<<1_IApwpIsk(f(1u=N2N6uG;fG!-nm@@xbh)ax)ea~TK(lUrKWcaSW$gie)e z_%N|{0?(9qRXKL8GLaqc?;z}i@G8!AP)%A@QC|KAAq|Wd6Q)WY2lWL|j?B*D6+DF5 z6}SUpOgRQKC<=r_cTEHf3yUwz3{L9Hj~}-r{yb}BL1-w_a)D3R0mPWVz&u4>s~^8P zq!OH9k#?x#0Le-t&SW$C2#x3D-<_h9bw4E~36@XMx$kTg!5t)IkB`P2h7r6dE3hhJ zAmZ)l6|k;d8zO%6F~67?bBEUy4;e(qsI9bJi(U!$7feVZ!tM_d&u|DD1wa9Xpc`hZ z8Zocn=!NJT87<@QW_sKPz1wlHO%A!Pl^9({=6psv~lCcRd7ea zG_aeiK+afrcrqK~$@F9ba00QXZ^_tQX5Xm!?Co1N_1eovd-w^OLLG-!KNzFIUMr@Z)3)<}^K#$txPSWtb7hz%+UKVP+`qzIX5P z;t_TttFGuZd+b*Ud2{UlY8DyA4F!6szBZh=$C#MeFB*OKZWwGe zE1-HRb!24es&xUQFn0aF~$$fykP^%h~0 z0RJ;b!-#KjMT7Z};Y%dk^=dx@69$oELJ)e1QS{mYRH~;)Z&Vcvpowo|8*zd#KF6En zl#(T8G;RnH$cWkkqMqREkpTJ?N;21PBuq%^2mQc1dpw2UjXhDd14C+%@UdP* zS~U(FV8Lc}z9J^TK0$(6$kYx)Uc(jO3%@>4$ZJHO{X68_tcG8-J%|3l%JHJ^Vs8q>*T- z5Z;|ftc_r_l5t@;R~MqAxo|cBx?i^Ti<%lT8*adT0GXnP7_$m&x{O{z&a?q4C!=EE z+nrc|C}FBNSBGX!BC{*SbxY0t3@OJeTlD$0f99XPw_-NR4sIc~t4{U37Igp`Jv-UsEm5I_kE02}@GE@j2+)X-HZ99-bV zBCWxgRmGwm8SOoQ2DlD~Sx}V={>aOVL@0r>(y91j_{tk+37J%r0~jn3`GZkXLDI)$ zDB4;MatE2DK_l=(Wr=Cj`5Z(J@o^NXuAfJCeF4MQiz#&5^waVwzwR0VDUuUlBKv#h zml2I7E9*zxxxFE)1divfU$;&qQ>^p0j&dLwn<8zQ&m{iY+2gu$gp5El|H)cOZxivi7cTg+apN3x3<1-IzQ zlOzD4kB?5?D4#`An4nn*@5)$#Eq17;$gp7xEUKA)$VXv2-Jk3W{)vG$GMy8sCsGCQ zgbez$XkDT0dqnK`ql@DbfB}eP0^~JhBCkrp_vJV%o(XP0@U#j_EfBku0w>3Ry|5|DbmhpbdX;x0ZFL%;R;V2 zW<3Mk@q;gSv{_(Hlrc4MTq%!{_c z^T&V8u6BAirMi2!-l~6y`0~x0H>X|kx6!HfqHefu@jS9|BLG(*RGDCZHa%-~Y9ZQQ zr*z?6gC8KJDCRen_u5tSm%a=cKd^cd(13fSnKpjx1^~Ud*EOFfgAspZo42Q-X%X}J1c;d_f z6zsB_!5xrcP8{}7jowHs9YAVOZ5mJ^YGJ!D4yB+4w%|zFJN+n5K;l;xQHn-}GGC#}5Tw9%iVRv@n& zBz6-n({{+5%(_7=rl)X_O+qcrQv3om`WpRM*}_y!wM9( zwyK{ige?M(3}-nlhPUwHMqOR$sd9J{KY!Ic_&*mlQ6XAd)=)Xl{f8s1k;6kMm!hyn zeM?KdvkErwnGlI7!>o+g*Ybo)pA^;?nNQ@1dbwf4hE0{pwWyzGIg?@)vf$I+c%UX^Nl;?g%7(t;(1I$2wh#=m zLEI`4!&S1F6KWY9U08(Ktox@!p8Za~jw&LFv0z$?O#C9fpZrGCbn>X~Xpzfc+a;{F zNb4Ri39?~1|7}=IzOhmcXpP8v9=13N6PV>K8z$7?+>6BS;3Af4r3={wn8oq0?Vg$(Y){Y4@YMFh5g+0vMHG}|a!{1@YUgMOA*^fcDS0`KTQ|*7a_)ELeJ-2v?GAE(E`jMBw1@ zKn1TNQ6Y{%q^b6j0L-IIybM7wN2E-E4b+WTATB`e$LKgl77i?;a3qigR^7*3YbI3x zKr9F{z6w7AqFp&;PaUY(sL-44CDt#+)soaKA}BA&nMs*PwvlM^KkN=@lz?~>kcl7+ zA%j-pqv5Oe%wO}jv^*E{jsF@dnxQ}ZPXx_Zq>cMb$-^eV;Ox8)>*@R&51o}PY& z^8iemkw_FbHpLEvh;#iV7>5P^WC;z7jf^~p(@IhQ zG_JFHN-%8+g#@4>0Znq;h<*DSv;>hlWZZb5V1Tr8V#@!v-7gq7T^y_=Q~^6VvX-c2 zYmW^>-bjR6VB0!2F;N8?#LCUhgfNTc8Tu4C6b;UeOSA(ie76l%7kjF2-JI# z4AZ`O!vfzz73ZFip(*&a$68nZtClKcngK_;=x1%kd^{rmfSuV}to=n$2}oYx4|lfG zF3ro^N>B&H75HB?6yZSOCk65l?k2(oSSs`?I9&=r_*I*DwnD1=-xR{!oinay2$JSg z;>B?*T{zYhK|2Q;>I=ko04bbHH?;ys3tZMU?E6}JDtE=r#W+R_iabP*Agnm72s%WX zaf%{2q!Oc8ul1U_phn&k`>Qab)cnuaV$}(U-G8?9|N2LwLRb7hCH#NB`Ty~=Mjy8< z_oYCOF$$IhrN3o)AR$)4!KIR@FGxh;q?(w5T?wK&L;ntCeY7Fli@rWWn*qNe2;S6c zffb6c#_hu3!zimQ%l&hom(u=y0B@s8CV95#stWLz)pcTD9SI}Tx8(0{BTAhM!#$cw zp1*(KT0u|#knhd^M-O2o8gp+5KLgx-T&LF!w378~mkKKW7UNpBU(16Z`Oh!;^HLu; zYL|1Z6A+-m9zmS9*+qDFhcqZ6@r6-O|MO(_(U4!lfC(R%k?`S+RoHWX?GaX;glFHm zODUwskeNB2vPee&lW}v^phiA+XG5IQRdVnNE+0titxNFV zpE83*5uY9P@S!AWpr8XFC@TOu00^2~4FCNly?syv$$$?Lny4=r#5jTh8uIn6zd~i{ z`1>0;L#G3xwt=B5&I2IFPmrd8b%DT`ALIgZkRV{cc55;QN1<9>25bcqXF=*v1-;&( zt8yuWU%ZY9IY%QcZ4;>6C4}~_)Ikp>F+#m;Qyv_QL`-rJJ%Y3VbML!g$^j{o<(O-M z5Sa+ObPJjV+zgWRfx-QttrP?RP-rn66Oyze^QaIE_CEZNToI}sA)GMmgFk?|97~+a zK*l`L-jU)BL)?x~Dx(5tG@KPFNoQJ z9pc3?Y5_PQVHfs1U=%W;JE0IIG?DBvGCyx{6b=A9EBGvtU=XII5YHpWalo|^DJ+9C zOePy~w`2zr^BHjXB!|+UO>-?atALhYL>Tl4YicpVYe7&_NrDmmo6#DtH$`s<$d-u?7$Ql*6sjA;GZT`J?bKvM~Ny;7c)lNW=T0XW_R++`WHO4=s8d=@uKh!kTTv73-cK_@5p5>oad zLkWgO5PL3wjBFNN0j`P=2+lslL07TXCU}smZ~`mg))17*`^JRkA_`!FWTF}8*+x$a zlZ^x4#2nJUy4{P)N}`Qo?3XS2srUREM0@w5ERxB(z)>>(iNlssb2BG}Fkr!r5s^5xOvZc7IG?tF*H@uP1ByP}=M}5fl+J9)+hIg79WHpnjKc0D8jsU^_P~F`{7;_KeJy zizqoO9PhXSFfyrW5Qjz($V)&y6k(M8DhQ;+QzK~>kb17Vy?gdpK#Rtd#<%8dKESnL z2v0NY;#iL;Fx4!0ZQ+|Nk$--UKY?v?KLcl2S&|*tHT#XhD+fiK37ZQppnP_qj6jzVGk& z|BmC|@y^U^>FIgy?|om(d7jsKv%SRV9N}?{eRfB3ji@dc-N@q6Ed)gu>q+oZz`(v@ zypHkTda6wH*n6^OdH3?N{k*O#zQDA1mzm%cAk#?GkVD91QiE>YwgU&eyP6dM!pVFsKpSI#np#d%y^nNd~O!9<0N(9;) znP%_faeY_*nMGJAwC`4PiMh^wqt*@+%nC)3H8TtfZysegvXTsrsGp`|5}sgc^d$9l zt-!)bhkfZ-K)R)mxefoT7z08_&vAUpDI{=xe%IMU=68Ap1?o6$w?zRzjB zPs%9vr}W|n4L7zvvJSIfJ<$zKN#qc;ALiH|X74$6n!GDtH>(Dzus{k0wbNEVKfiNN z@RefJs_65Inc$?dPl3;@d4}e8vQA_mnfQN=rt@4fB~*L;)q^OQB$NqYi%PqRYxCyK zoteoH`@s3b;Qnb!iU?!pTua;bnemsRTlWcC3gS$I6|Zi#%EBZ*%fjQc2FfW0797jdCzqu4>jkL(d9i>+@Xq^cJ`RnffD0*z-(p%g(7iWEU8d()Yb7wyNO{`Gh2ESLzaw8+V+ zIiPS%St_IA-qwYN2t)vjr{ed%xt{WV=_Q5Vgz=b<>(0w(job?j3GQj@Hs179s<=G! zSal1Le-2{GE80_uG-g5=SP{S6s1fSCixl9x-RdbQis6F8-UT0*gEwOk>`lsC&OGqh zW!gnKAYef@H0`s`E>l;CnW>P9gqiz^Xv(8L)S^siprBgZ8#!B42>kLYBSvZFdHP(9-|+=|6nXAO%(neLge6tr){A&rFQC>nZ0&vpTGPg9d?^`FHr$ zej$gDNscs#v=4+8;cIYS)5Z45dwWg_S<0B*MgZAJxH?2M&|UqZ8|9i3ye7kKjbTc~ z0z5u>u~AfW(cR5EI{=T4(N8Y8sfhCh8Ts~UtI>*7jO6^NJI1wanBZjle&o*2o&NfE z>p&Y}d#O-?oXPBblmfS%U)QGeVNs>JNDn0Tuodo4OOv%M z97YgZzQN<(rHF`#bB+|00&7)&de@Yf7Ec-Q0QNe?5d!TqXB3KtN=hY5z6op_8Y&dz zZJ;UVn)5eNoYxaK!Wriritlbsw&`~EMZWvDPbY6w={+4Tt9B^IY}TxH6ux3K3i~LI z9w0$Vlo0h5j6z_Y(h6{Y6T7uVSA*?vL4pP;22YEh3>`y#l*#a?j??2IBKxQOx$|mS ztH3>Gho_0zZfb3Hi-v}&d^Uvwf;SWK1WlG7p_YL{_HFNk?q$QLd_FL;FR37n*-4p6 zz?h~nY1rKEMd9BR2>a~{FGC}{Bzw|C7uIX@Ti(UOtmd6{>zEPz?!3KfzABM3a{pa9 z>iXrva@702{Nrc^;)Mp+_61;wzJ=+LcF{AYe17?qN=A{^A}35jp}=8)UGcx)Zz6Ks z*@eu;C^}RQYx^R3J8~EXg+6@=_OLXsTdXTLZry58H>~eG2WqkPogIs{Bf@1KpbkKn zVw?JXJTis4b;OmAk3I0iBj3M1n3LRTdlcVKmp$X%36=tPB&95n3&Mcb)0UdJ?LU^X z5vcqiu7do9Op@q|)*Hdb(OrDV8h9y{Icor0^Nsaicq&EIeAr(Y`(`B3Zpxdx-R>k# zE-ZTq>9vLNa~n(uB``~G*WcKb9re+v_+Z28--YoM;v9Ws9ncRK-}2$SIdd=}&411x zm~u-7XzmAqI~$6wKKVS>!StnqNMoj!U*v2xG!&fE z8M7Vdr@*CCVMc^vN1L62{y>cMh&R~7Jwq5w`MD(gl74f{GC5$9s`!rbFXcAOtqEK{ zYR$(~hBYhi)e#9(4c|P9=`Nlk@87q;73dW!=r{V_0vIeAAjlexeGC1Rs#1I0Tvc&h zp4mR_8KYo;;OjCjb~|)M5DOVN!c8PSv>k9UCXdKI0ash0qmeBHo`H=+8W8m$>$G1) zp>7vSer&YUdBqiCf%xh$p(l$cvuU?MhWpvfjkQ&lB^KzIG!EyJbZ!}AzoJ|qoJc<= znF}UryoZNARL%yp=lkuAAIG|0kR5R+D zwq+xj+039QS;*gGw~GU(+?paU&Mec)_fP6rT4G6fcqG8S4H;FYhA*xi{`-Ry%0M6Z zG7g98oB!s0{ILRM<{zC$dC9CmSFS z@$h6bOvPZQ4lj3gotfkhT_P-poCzK|whF(C3XW5Xav7;m1$q`Fmrr# z#?bBp)^E%Ti1VR@W|1a>O3Sl=_S;J0UffB>Kg6w+R?3FwAtDEqdbtn}&^C+66*Aq8 zSMv@EN15aj*`jo{l&$)shcigL3D{kROwl-YVyj|)oQmCy_|H3nA>{m|g!=SCs6Lj? z&-tfBiznFe_WkTHYv#d)$aj*B3^_eq&?DSa%qi?xD8x3?oD@@o^g?eyqgfA~n$x=$ zt(FGT97(4l?Ib{z`RK%fb7C9vWX0qi8ALrCk%s(!?-y(%`f^QySjiq9#=v3X^1>Yi zy+vqo&jGE0Cr(ZAaK}S$TS)8JUm3hWQjnUS-Uvae?y9%##gvv>;SkAZHss9D_PZHx zLI&SSE+YDZKLoU!iJ2|)=dGF!zSt@7~w!i}l zw=4$jMFGsh&i)NmS&pheCb?%#Gy#_0bxUytKyIjTet+dA<68T+ogmp1C2uy0 zQrGWUw?!O+?z>IvqyNZl%*U6{^vvWuT7}l3(f{1()usgDn^83^ zZ#-H>!jR{vpX?R8t%*N*J^1*P#)~vsJ#JyVerCw7Us^PKuF{H^j1Z6h8^L}3ea@IV zZ9Q}_YNqn0hFdEt#rI=si=W~G{%@O(46N|&?awA@MQx6{WVNTo_g_cU9~`oM^jLnH zXJh`j)uO<^MqX>SYwX}}o4Yl5U)%0_P6K(@NvZ#e;%B@zbycV@0Kop@wE^6S(oowh z%G>D1&%bCj;y+Sk$?5XdXAGs6F(y4%Vo^hg>##zSK;1o2JfA_%MMj%rQHsg#U|xR{ zllxLvBewh6FQ()t-OStn&sUp+WO{3E+z=C(b2ovbzh3~(l0X7hg)hpBQ9Ysz z)ucldO}6Ye@s45}jLtWB(YB7n{?AMDf7%R!AQR&E<$3mCJq2UChvIMH2rWtn@)kaS z-t07){>9Sx3S~gk!7n^C?ypkF`uxwA)dlfYy|z}1@EKc1{1Vwt8^PALm8z(Z*aX;`il&+!#7wB{po9KTuj@=I z{Ln_2k%`)Oe{Y7g2Np2t7?P>kBEEZhzHr^Q=Gfj_>%nB*j2LYdxr%0H8!$vFoeHAc zoT(FOjm48+E;RtB(O>s{_VcKm8_15QB>X;evmID1&Qaru#c5xyY5zp9Axa3!n(V0_ zNmGHs}cN-HJ+)7T_}Se;K#!bMEN1XjXY z%MiG2MeB!BhcQu#m1R9ei_W^dut)2i zuow=7$8wNkUKC5s%txh}d>+@#T;YuuXd~(KxQm5?6$zL$IoJTTq)Grk?(@M(EWlfN_5~Ud^k9`I zaVXX)65;}v&fnH;shaZ#?BOKspyy6Y8g?0YOtC=S_cv?P3GJrXOzLJkc+RP~9(B5g zdy_5cEL9lu`$8j9P8PHwUQqL$FmZjk;*1zS(KLXvH1p_K2-1KL|6fy|fX;k9@0Eln z)i1ztm^SbDH+-eEvYJqMIohq+%lm|{o#>$vRH}=$D;?>Si)&o{-=xs?W1hb$G&Jij(USTRXdjL6Eo zrC>|ogv?o+_cnp3VJ|(W#R}`Xt1d#l?sPS!fiYfthSIDuFE0cxzsx;%3ddKtQVQ)o zkxpye-R1pp10r`@Tos6lO1KND@EnW}CH@C&C>5zF@I`H|NTpFqOKI?RWvek6-a(KY zQd$eqAmepFvB#gjA~6mEEoj!&pfe(fX?eq^#8?~OJGGz__28}Cl)C8dnN@ee%lxIU zOH11w8z16wn0ootyXdVUs;^6f@IPa7*x(Qm8aYg3{L@VxdP4Z8GA4})967-T|oymss7iNdB2e>!OCyr@CVGY5ZnN;2v>G=;A zUEOg$v&WJsX%Jz-y!i$j>e|^D+T8iPWG0`bAwdfH#j7P3aO~N_M<6?GrxpE^Hd~rR zW@F_x0zQH$VhK0+BaWk=6WCHNL{&dj4&_$LwPcJi23@!H`NhxZsiFPO z*1O^r8=9S~wXzK#{NQCfP?xdKL7>fhHvvNT;nTl7cM^z4E`YgdbUVgoybKnz!7j70 zVo%mmGyF4)tXD8h;rR4?8>uF+4p7)O+x2B-!(BWU@YF_3z4M?ZFD(?E9S`C;u1=Rl z+xC*9$!vT_mNEK|;ah;c=z*r<7^LXXppVRAkdu1foTvPK`LnBRy8o1^Deb8`+Q$8I zjlqi6jAq|(o+qb(hnkDNN|7oX3Nw=D+(-e1W0hD1>vp1ihRQJkxaeA2Ek814cM)aJ zu|KejrUlAP@mz}rsMiU+uk-{J5k`u7QsIqmOK5VLMxsAWPf`9lY%-$xRVJ8m%Rr8p zqhGO59b-~x$Yhn=na5^*v1WALTtv!H)!y9#+wPg@Oby$z%`WGTA5St9(q6dy57(9+ zsh>n&6G?F)zIWc+lcwR5zmd;Omet6w2F-vzymmja-a$5g6RT?f?b}_RWRP&{DcDLg zXB%)vn1kqjc*K@fL=e5N| z6}jzqvHc%RtmP0G7XRI~^0Av(RlI&<|Dsy^N@ z=2<~OLG`@4T1k(;NA4_)m-%oGr^S5kF_2dNIo}GDeLdFy${jZm+8=&N z7Hjix@u#JQ>yvZV|D(sK6Y!<<0b((skeD&)+d{a1+6dc`y)FZ0Lq2+QugZ-5mA0B_ zVf0Q__C~Ry)uRWl(D>>MnQb_;Cv4}`kQyCEwZ~+vtIMHeQ3vtOfld*N3-eMb1_}2? zawl$Va|*cfZ?4N)+ZL|ZpYS5uvr$BimF9KR-dX=Biux=;C1T}|eg=G?0RYT&_Yx!I z<6_)PQO6|By(#%S&=gR$2e6=$q8qh6t#1)|Y+$YQp+YDyEydyT%NjR!tZ0Dgle<66 zzvxwKpg4$rFlpJwx?KGAS$Ypc4dEvrWIfs&E@eyWpUHk+HGpxEr-P!u5hKii5Fb3&}TfHw(N{ znjOO>;!GsB0kbXVJ59&ioxvtPbM}JUb^F&SV&RcZr;X;G|6t+xjj-4!uA5wU#77qQ zkEx*GN6@Nqz|JvykCmCELvmV)^^rZah7BPU@)87vW zXAuWR0zZ5H15buSWex8GOSI-x;$Ctjh3E0Gf(<{PQOdGH^TNxl5PP8M;g@IvVJ(15qG04S2I395idF2MmJbE^M*o zBc5*&A{jvl^kYj1K5sFtLMEq+E6L+^0dFPafyt(G=qufq|13Y|rBhMsqGgXgJmFAA z=a;Q2Bc|uw>7lXv)qwI0%1|+mm?}1tyGwCTn*uIJfuf(KyRPL{N1LjZRW*5@` z;H;9L8bs6S+&LaUd}25^*-n!y+77z{p8%IPQixyA%e|lJXLhe>x^5B{tXWBIFBv&K zjyFV`6T<|Z)7D)Br3i~OX)cf<)+_f`>00%(xK%f=aN{Di1;v@ohIwYMUz^9C{?H*R zT6y-bn)P8{e=j>#v&*jPKyVY4PwK4=04Id#prev&N0yEZ^ANTi!t@Wk((+>dUXkDE$P=`9~6Y=ybFH8=Prl&A}EN7{2N@ z;lueWb_yx7pmAd5e*96|9Sno|1>#@N?1JYXe-c4x7kY>(LvB^ zp!KQeS1#?xex>=nw|bO-{#PdcI}cyC2{+%9k{rvdB8ZJ9S+o&;a&I6TUEwW)Y!0C` zo1N9w;}$Sh^f75r-34qvmk8)hVl`?b)9{=D&6sGq)kHAvN7UH5p6Gxu3v;@%$1$?fIys>v?&3 z(fBEm8;v;!mGif=3yGO&Dpaa{-e`zWxso!g&t(uiJ)I z0@Pm-ow<2{Ovdh}EbM~>hI2J}JpP z@udi5xX~5>rn8<)L%Iv4y1_BvsSO7XsDtZm1!Ox!t(SC4Odx{Qi7&LflNRNPf&|u0 zghC2|!U*bewW-1zYDW_z9Uju%CI4=ukLd!A3^utKwM)x} z8sn=_mdQjxG86V$c2+?^UD8FsKmRNBjepHS7St&+GE(>4m|M(0vtJu3SUJtfu752Q zx*C5mIK$QJRLJbPBuhMBWxTUaoAkMCD_&_Pr4)7s?d;9JG!Z@VsOC{K`XJbt zbm~pG-ipRVfIbfseqJ+Ln_1wQ`Ou@LwyRedbK*CPJHCAOST2!PI zoc`=uPfP-7UD8=WHT_%rFqIa=jx8!(#Zt1UO=(Gqc+e}WsPv@qc+l^DEhLf}5DE=a z?@2#s0}hb8gTU>3sm*gP&s!Yv{Qdh$uq@JpqI+=*D{^^tla;U#r5Z=QL3sZ*p9*8o zF_DTg4I|H3j8n^3M`VWX)OU9wJ!hc3`I}bPS_St+zAR&i#x9KBH*4I0>>fpgLQA=u z>L)*bBnl=4{f~cQ1Ql5&(8~%$czd^*HCtZ!#1P3Jy{cd0t%8DZkRin%T$o1RaD}LG z)ap{r1a2b2*h2*pPb_fRhC?l5==)1Z=(I>XM|-V<1347$tDl3+Vi&BfZq-^IJ0s&l zz#)L#@o*n1j<2sH1HORf^=5>DZx6V8wIkggo>xBHMIxR7yfr1cT;Nb}`hCtd!(s|l zxIHKA=~uqZb2^arU)=Y~ru%bA?jhe%om;t2{!!*t8yDh*sBtt1qBX!El^?Z#^J!cp zTe7BAtH9yLX3ER;g063x_3Ffz!cP|WoNYf|+mv&%lk)!guz(E^xaAg{sC~9k<%F5C zSvI}0Y()aU40TdOVEkatlk9+zsjJxuQ_$RQwBF8>S|HVIN$1v$r#=7Tnl>9Rw=9Vv z)yhccoStgbAt9r^QY=EHVqUy>JkiYQ26Ws+`jTr@Fku1hLII;#<*_tMk=r$nl1&z& z+1=h=X9MA_wA~5`=YeiF`Nc`EPgPZR46UH9+QKTc-ki81>7HIVw>pp=VS)mqwWi16AxaP>fJub zmfd^yb>ae&ZE?_K17Dq2##QtT%Qx&&EpV}q>rk7Y^6|&l)~i1GRyyyuXf^Tuf?wkD z!k*n%ttiRKp@X%2mueZwCz=KqN~M}g4{e4=*aye-%~S?=sr>A3g_*H;)$Y|o5(t7r zg>MhK21Kztz@WOfj{IXBniV@tn;h@(<6+i`?fo=0Rn>d+Nbdgjt^cD=>Z)toFMmRS zIiWVjB(fFbo0x7|Up%<3fVED-ZiYs#b+0Mjki?mf%1CZ{P|(;pKi{ZgaONoE7zf$6 zQ5Ky5mQFwNa&rUaQ=eIA z7}`b?-!wT%y-}g#1cArT4&+li88mJ`Yn8~3IqnkJkUM}vGsl~Vzc#h5j! zp*-n8w=^%cQ=|0tg?FadIfOGzX9JIYph9!bTKXjKQEuAQFY(f~lSO+WX?BAh{- z-cYv~L8~u$aH0d#ppg~}n5+}dnX@XwRL0OoSRU_gvL6vDYhuUgvn9wAH17&HmR>^_ zN?UbXTUo<^Xw3M>FC?JtvCYj_a=fHYKo$HTsS~%|u2!$41mUJS6~)HOV`6QuU!!;m ze%FO?oIy~O%_drywQDQsIp~Nw6nA+;C-aK)B9E0VJ@@Qo)Rg)Zybo4U>B=*Kj|hba z$#nRxshO)2GI8}9_mY48CJ(_BPAYfzk5S0o z-I7cByLkwynHp7uzxz`U4Ut@t6}oQL$+^b-ZL@HFJq#nYyLI!14Fr`?!#baEZ{zqF zzv#f_DPOnnlS0juWFxk6Kl>s(bT34aI5M(Yi!982l3!I+R{9Ma5UY=V>Ot(Lv}>Vi zty|l!xcC(aPU1VgyO=+97;LKue|l{hp3&5WNK=AAPE}jBOhx`XnFQ)oP2;6QzA7d? zNOGoK_!X)g3sfhkUm@?-ClkXl0{-VW*d#3nb$k6Ejh)K56<*Zn2TOL8%7hb8+%_al^xJAS*8ocRoS~ zl3zV%Zojt7aAfhGj&d~|YM^-Sok)qo0ajKZkpbV{%9f#5$+FV&Xg z;u1uSf*^D=K?h{z5O2xx$+ClmyA)IRs!;($+$(I?aY5;^-%d731;FW`W8IKL z6#ssOyX35d-{p?FFXE)$(H{mkf{GP@6zKHVFu7F1V~G6UwW2zwGlfS z`u6Uc72kN8VsdI8A+sLrp|h5ia!(yvf0@dr%Jg2mLWxvEan>gz#@JvYfOX31o;@J& z_7aKasc8eUUEbPt9iM4DBS$Zq7GJ!0u{R^*$vZ+?LPi}SOiI3ydoPA``+P{HrK zpd}sl4%72x7N!a*nA`eorWu`^wGJx~^dTWwTDy)^!3Zlf=kujIM=YA^udT*%gs*}H znmr>+12$WKVaCyC)z>2z@5(A_Ey@bD^nLD z`1I>hGyEvGIr=hnH&mlucyn~mMO3oZ0$EqUbj=AzKtrr-tnkTNE zk2lgTNT;2>r)v{V49U&sK-WJw_7>Rs4(CEna+B7*#-Q$&$_b`($EuvG$eA8}cO2CI zTA1Fie8kv2N$1XyUH4k6sj7};uNJ!+Fy?$>cFYDC(fs7DQmwK5@!Hu-Ihp==Jk!rr zz7Y?Xg6D8Ii3|IUa%5AwbxaDlq%IL8>{}BRm1s7b96!{uJ9|uM3n;~nAle@slkS#$ z{`S4BB$!Z63$UnCPdt7ZWgBdNbIILdi9Jm;BZ9$Z{VN|C%f8TBx}PhG_AN%ci^5UnfKEzNFuWhL_q^|6L< zxWJPanRM=wG~2-CT!pD=6Q)-V0M z))#hCq+>viUK?iXu%~0h^cOe6fYt&&ec4|pCT``P5FFfX z`AOhTG5X>q?J)u;>E&H;J6&ZheqJbRw~;7fxv-A;!|j(B9;8dbqvXM>LnF@nDws*3 z>pn4h{4zhG(EtzIzI!A!DX7hf38p3z<#b1ml#(!X)WCWG7}wDXSndnnIbgwvF8Zrm zo-Wo-BaBnC<(_)Bv-|iKrRnSDW8ZuLP#_lhB{hRx{|GMivY6pP&{Zfjr%O%-@xl9!3R5yz?q)Ayp}JnzRqOJKoV*D9Xb}?i=dRmi4peh#<8tJOWX+ z14gP6Lr52HNRbMGuy~@1@JRC6#j42NA{7;y9JU9E)H@Ge)PTJ`%Uf5XoHS`z$V6)0 zqO!8C@17U}>J_`bM=$gO%Sgs~a?jVVKZY~|8#5*N)4=MmiQ01I$-j@fLEWP!;#m)B z&ka~X7Va;Me^jzkqWo2eiyqUFLhgV^`I1&LO3$$`qIldu(LNk9lYD!gredo0083k{ zV@9kt@Q_T7iev&%lG*0oF$Ttg6jE%7bDrN4T?b}8)Znzy6L2GJlhWy}l@oAj8@%|I zW6(}TuGb#t-5eXFE?chK#?Bc~c`56t!NAx7gUjW#lLss@z|!O7K`?G-kE~$S`oG8( zaO|KPH-1;J+R|dtL01rDW%h`q2G9BILRm`OoWxo?>lfML0uM11XJ+dC`tS6g)Q{&ob_?vX+aVJ_f2+e-AWY#91)=8Sy9(nos;CjCin*$Ht z*Qs@2pikOhZH;<9m2JD^`sZY2`O&%?A6=XK#E`wk$(?-Lxi07xj9*115V+~aPK^d_ z{ThMRIONf}?HlQK%3>riYG?%E#f(B(VRSDL(qW-&N9K~s;Q-wR(&tG@}~WY*( zwYht4U+mF+=<{#H zIqwFvr8v-EgO(TQGbVeD=cAF30<*HSQW}gBASc90gpO^6sEf2pZ zt|A4@NIK1XdgK^bPt=B??tgyy9r}}QrI=d$K`a>l>=H|fxugWu_yctoA=kc{_oZb$ zPW_HLbqm!*IaC0LO-&^4umiT0t*?;7C<9bbzIOwVqYd8)We`LRM8>lJpp92$_HYpS zyLg<^%L>urylT}eI??adrKfQS^%oPQq%kn_>*6OXA@$jUX~n`J9gng!@tG^#+@jI! zGxOnBSDPpDnB=u78Yd7WR9CKZo{9tug<2XfS^nwMD=ytl%gYPlImx(dV+it+9h z;eXxM8a;yOl+>EmMeDMIOp#uUIk&0H1j~y+QX9yjcev6`ax&OkUCYNsth5=iiJZF* za$m@SClDI=ikWPaW?ioWCkhe7RUV=4p^bcDW~Nt6%}H^{#$yyWZ)tNH_?Z(G%7 zGi89u+8@-(%YbYwNX9nvlf=Ehy}t>_War`=JJH1!MFtpH6UtzjWI{^{j9on^FS|Pc zXWG?CWt9X5nRz0@9Au>gqy*RmAj@W1=D=wnWr|I#Wql4UP>%(MMvf|*s+=1gr-O@= zBLwYRLRCVxo_za@@B@@rN9QLEhKTBkwp{eA(m()LnuK(fA4V0in82(&0a0WRC=;bhg@S$VW7Ra0e@Yau8>%(86wxA6po4nu!$O|5)#K=j(z|6}4nN{dq4BaSUwneJ~$QT0aaJGt3Uw{Sa*#W6>c} zDuJQU-rivQMg>X9vqeCis^~yf(=~bfbQA{tGvG#EZqe8h0`d(VK zW}=-*WFTO$TuV$^lokKZQ^GNbquMaXN21?* z1g66eN(?EavsME9+_r0CtEXYoN9Z9B5z%daekc{C4!ZPHR+n0|w0pmhIKhl~%7d*x z)n5h1jRDBa-Wn#CX+}KRv266XxV)R7T?%P&3D6^#?VG5bJa}<>ZQ}CaN5gwZj@R~B z_SE!)%et~{K#+HsMHlD+4>nb&&XRU0$L0J%H}`z3yXz2%Q_6SbwiCgzPKvB6Ju9p4 zja4eORfjGG)zy$LG^frhS4MrF@AlT;%%)M_>6$e>&j@nL|=|SzAaW1x~7<}=DFBjJckzu0c z1wRn?yw#r$jBqZhBf)lUDmU0SXi~^?6(_4W+0AOKwdHUOy;#Cj$6-C+=)QgXwxHB(vniQ|&SV4OD2{TCSPAAe1n(~2BOR0X-7bW!}{bx2qI$qz?KYwlcCNDCT%v5sJ#YscD zW@ZWHD)rD+#FJ=bfxbh`ff+xKM{B{`&mnfP|dA33_5MS(5z3ai@UpeB5Z%$Xw zkF98^UO|#YP1xIl&Q+Q_DqqB{tAiQZH8I+?1x`AbD|CoB>0l$*IjaQ)1u)A)W43D3 zmCcnQ7xqZ#Ll*306=HUgX4u}`C+y3wj2tpo_P$)bV@GpoT34;o9yI8KMYt%~lE=a* z@E3K=T@0z6?(YQJj{*;hF>vy0F`4ms>f6fJA@jmwOO&GkCy){S><5 zNFP*V(*ETCuUraW;XDIgQg*~;q|XXScwnxzk9 zE5n9d228Q+Vw&-r#zcr5iSc%a|btxE7_hbh?Q8VYa_dSLd}a(lKa z>egZl6)V2odQr@5UFV91t)5SvHNxD{M5()RlXmWX)zsvr%U%Dzvza3tNMYQ|m3MMA zKRF5$A*B(*i!0Ec|9%`Df9idf>-o!<#!NcMs3%zL1E-GLGion1 zWD4gQ0SG-%2u*t>{Q$c-4MJ1Z;R{blwJFS!(dvfRpS(3S|6J3*?Jp@^*QZ-gohGLt zEg$j3YN!{QC22jTzG>HQwp4xuk51qU`?Bp}z;|hI>B+yXg37}{R5X?5?-Xgp4>eF4jmKX<3EE*S_PI>=1Z0=*FQix!mr zBC2l?&RA@W$YK5@DJq^Kzkw;wD0WaJO5r3hBt?kow;^>aW@PkY6z&s^X|cJ&r@42P z#xrmQu|p$QrqiFupfI3dqk}ZxTwEX!I$31{2?Obiur;Z1-qrM0G1IH7*!L>4iHj1IxA+ZchD=(wb86?<@coT@DYI>rZW4ubS_^E|WA> z@6St`wt+H8A<7~8;1VW`qYO}S=T)gX3;5a^l!m*F_ku50@a-SS)B=4wgmM@~#2`o)EewfEmk5lQ-kOezv*H@CAjFp_o(V*NJm9#GH?M*?Vz=O1465X_fp zWm1*8dMHxq=ViysM`uPeGWQxAv*7!6AJgpxH=-I+&6P%=hZjPGwsp8jgRs+rq3;N) z!_CzvbWj^?Pn?V^3>%S)@3?b5`;~6F@6RXLjMqK1@bU&y-&Vk)K#g&Yq2bcemYs0I z%Wwh!sUCbi1%`llayzhs;*O5sq!T_5>lgRkILfMyJdj;}p3W$Qws5E33VVk+&;rts z3Rq3uM-s@>@Avlo`yI?0LPvMg*AF9DISd%5~a_$Z`WNzn7p6*20tvae{%&O zk>VhrvqJD_d6RuSQz2P-Px=370T3vo3CkjC zJlqk|ipFaD@x3Lm)I{|@8ircRYx)MMa#l0OclB(TN|ZQsua+Dm6*SMsL3?*}bWdo| zo~$x&=yCk;&}^}`R8TeDo*TY%-|%eP!OwT6tBoI@Y2@i{S3YrYP*XrpvgUS^?$k$w zpQ&Ih#OF*x>Y3U7bTU|$9_CM38OT|%`cH>#tOql)$tiq|%Aj*jMJ?Z5+GT<=)5WVh zTw-caoWzk*KclIp#Bb$Ip(%BRr~*);Gke;DKZ(XLxtaQiV6Gl9>V)aA=`{O6?JbWP z#QMj@RUYrifT5OVw)&%O$@T#rdLu_d=1zir{7Rf8*3CYlLvr6StrnEddPqll!rK$l z(_Z0j6Gcm&Yj994^y=0f({$Z>e|k5n<>_;?o@QosBze+i3+Ba+k%R-2F^Mkz8jKZ( z1&y`4##GlUKTfWH;+UsJZfjR<56=0)2e=3R;U<+I|I`$lZkLBI0i^TOQ@0u zNpsL3XIGQ0*FKgImyDAT|Vw7mJ9dur$`-<9W>Q8>^$ zJnl)%gs8NEamq(rTS)Wd`&LYt8PUk13{`O7ND*)wuhlhgepRzBMa1GfOP?IN9Q0 zH}yvMp6$XPGB=;{^;M@*gK3=SGQV>*>&lPh<>mG8Eo@+Hq_t-M$TMx+vkjmEQht1z z*RO3;ZV`Bavx{JdBI)C{pGW#jT6SoV^C}7plVz13!}(~ogY)!%3Ea7FpOk8h1Bki# zvr5m9LJdpft^*wiF%o?U5OcDq4{i`J_o4QMtBhYNc5Q^AEcOZdz7L(coa9*;p-ugd zq(^o0+Ki81zCM}sv2e0+fzooPX=4!vhS2ZQX)Y$SObxFXFyk+Q3eNO)$UW2uA%;-q z9zUuc%0CZAkilhC_QHj46Ed|Lk6e3Zc*+;wvk!~Y+vWIPUgGhoid-=i-_k=gK=@Dv z5g&6|r#%A}{C+ouLm`=kP9}~jUst`mhE8o2K}u)^4$t0$U))41?zm^qx~aky9i4T$ z3$5KOslAeiYeu+KeQs*{_D`e0+`H-#q3h23v}34G%GlpLtDje|DBJ}yL2d!hBx(E8(4=U#ehy+zIMy|yo#{Ij8Ib!k^GIiVsG zrD$r6m;$Qx18q16gt&@FTD1KF{P>lV$9?h|A`K%{dk}mvGT4ri&}ePgrZm$h>_q`a zsRw|v+eo?rzNRc_!Zr9_C%@U+`0zw1;j{DV0rhf?KLGcO(o@&ZGBo@ZiD? z$32lq!$u0Z1$g89zJbh5qUnGPpSE+D(_A$9H>nw{Rxe|=PZY6~#zl-8kfDV%LboJ3 z^!nsr0`3W~bv1S|vmG<4sJ9&of!In2gVf-92FuTar_L+=8a;n)j>U-6oa=I)@~*zR z{Pur?Lx879_T|*Xg;3)qdn4@cJiB@%c))z$$1^)sR2^*f#?%m};%E3S0PiuW^!8YT{h*5t%FDYHzzi-%u);*dAM?1d@2>Di8zqF6a zz&8zu+|k(G`ID%Nv2$k~WbFuX1NEsVM$=^&G0jf8(_qeoOuI|&hv9cNago}-!CYm zzWFd+8hu48A5Y)ZYyDdn$zUQn%~fRu>W5EJZi#4-!Iq4rU(+^a=$A>UNFJVggLyzx zx-2n41vYwtLx7Zk;;*$6c#_)Nd-1nETQ+Ut62vQX1SdRE240@q1u+}NY90hV#m2T3 zUFoPI0iQiS)hj7bGdA;hoP#^M(;DtFukQ|6Ql_=DhM&p@kNv{=2_%ffF-QA;AC zte;c+{d?)9Vi#A}gilr^B?Y~WTG{9M^a4(8zXm%@bMNtB@H5J<8Zw0V4?iVG0M4i^)tu8g#hMyAf!qj^3VjuMVC25R z&)c=VWp(4(^2TRu3Qb-O2sb+#+aqMSgNCAc=slYU8C{DLoY7 zA^IonYQsjR?ip43_w6M0OW%hC>HYdD^QaiIkBEPBla5x9NDMQ$Rc;#rdOCd-)U=KM zSq4jF*aQ>d9G3J$dAZz4BSIH?L(<6tR{abPo;7+E3EXF)5WYul67&lbXvc66t*k0MB{GfQe!QbVDyH#sOo+>d;S)6Op@+Y5g?Si-$bRNAf z3?H7`P+h{Da)M+~fS=~oKXbd&d*%!b#M_y}-2c|YC}m&YmGA-HOG;X_P}iO4J1zaM z*-84>r)y>Y8l>dE|H0x%K9^JD9Dcu0d#Q8G<@dFxPDfZdKmS_#$JzL$OP-vlmu;~0 zN<|N^w4(vWm3v%1eROABd>q?ds&-HjTRypfefxD)51`m8E-t>x*_y=87Xko$ocYE5 zuyF*0wxLeL!Xvs@Oq_K7)rR!R)#_kA^VT7G+C7@nN;Wo^zn`yYe%c6zuh<9MNIr-x2Q z#vQ*EJkZi#)7^YJh3KB|iJfcnYc_38S-r!i(E*#YUQ1+()zfF#NnSO5S9UZ+mzN+y?aT9{VOwc|3 zVCT;EB8YE*l~QppSAI{7Gl)#$v@bhONEaFTj7|~3A(aiC68kxj6T`UK6_!>}6s-Ro zE739RBe2s92nDJnG*)v^WkA*V>3re^(ovnjVH+LI+NaZM-QU z?|A`V3TH)rsPa93W!%aK-xGG!^GrFk`lxlkZ>7Uq9C&hdbV-X3?q^#lHjOt*J`=Fk ztv0>n$iOyzX5f~VKEjR3e#!rqx^69>hV8`if}ZYQy~_Ary$w}o;(in}yWfM*!dx?I zk`In5*GodO?LjZp%4{g(n5q%)BpXKwPZVue-3mS1l{i*!N@Eu>KP4n}5f8Vh4c zp(4sNCIy;;7s!RNcso>V_QanljIhTO9mk%cG-^Z&1j|Za>ZlNj7*{s5OjpEUGewfiqUpuu6Y**m>}{yxZ;4%<>F?<>=XR?j)VnNZK7R< zM%i9T@ewr8%7C!juJmhuUS8nHdR_!RxQ?u|kO; zM)hBLxVjFx3EUeQ)zEB)kg_bN!05%8nF!fTZyQTv2mrm<=^3Yq=iySu|r@Tn-(`|+s~BswaI?%)kcRBp@cRD5bWv^4=5a5 zjCMdet99ujA{Wt!V?2oeV8P>4NFf>H2tcL#D2*Shv=RTe^$nD{5-TSsU?9Um!-t>k zW#e-+08sS~2zLsjHFCc+Yf7~(qcM!w2BKofSZyiXItPj1vWfR3*JpyUiD`5>urP1K zd0oJvIEGqz4C9!Zu{_2ksC=0{=2MR+nl434BOE@LZ6jv_5n5bY<;gU@TplCW49gfH znMoF-Yu5h-;? zuE&QxD_PqKfZ4xvc;5M?BRdQ}AEZ|qrx(;OZ*cMJ*VooxQc-Q&cA1*h;klF|BeJae z{-y?~v^#6o*F)bF=D`CHfFv4>@F>{PImt!hqOjhKaiHFQ#n>7*n+1uvK*owr+}p;0 zi;z0fT!HtACRsFt(j8*K5>^oT8Wx+%a2!0V+7lUIrnK&8ZoYc_K8uCh&S>4pa$J)X zGmz4dR2)vc2@+jz$*pX}ykftekPUew4h;S0#&@I0!S;W*gsCr4HRw}MboeSv4)wIK z&R{?yrI2CA=om=(ChgkoU43A(hYduM8Q}4u6AQ)B8U}O>(=ak)g3P=Gju!ohzozlF z+4(eF?U94!+?ODHHza`tz)p zg`RbN-;MH7_ALO(ZQSWaWk#Knghd|R_1Q$)OwkC5Qj-k20?Za^O&6O>JeP~E@9|aG zNWQ=={U&{c*uIBl*bnI_;{76517+X8x#pKMSIm+p_AaZkbNlqebLZsIe;bvn+{pcN z=HkJNvu|#&i*sJ-(y$wZxW|uoEmt`ej`LNyF_lW8ZMIjUwcUjsCtu%*Q=6_`GxqN* zyW4+BpE6=qP4HObCXYM!Of;AubRl+0`g|%*!`hl30+}Q6{1cUeH0(0Yx7w1X{{<&e zlv8B>otnl?q)S4(E@L^3H)>dx-L0gb#Rv4#s}^;MdzWG(a%-(m-3r`*lN8SgzEj`a zw2yODzCcH8GN#|lJ~>Ww`bdaT$x#N#Og%W*Gnz5gOoR3URasJX4Jg&fW#X! zK&}0}(Y5b#D2`uQ>uTuebb$T9tN%FxlV&94jOW-}#qJzlogMv8c_y3N$K_S88#Rze z^f>1aDqB&yp@~Y@yZmCNdv2i_q`|`;`ud$INY>=IW)|%Gpj2T1t>eQxfj>I zDVoX?w)Y(N|EHcP+Zgp_l#=Jem-i=ZE3BIJb(H8R&Ml}i-DSr`z&RG9(M2BGarM@% z_x_XnC~~{}s!{CIU;D`UO=)SiL_0y({#D%d9j1}#?{uZRjzQ+W0k{S9%cc5%L|bGo&t{y4PbAvjK3w+RHPfPv|61%-gB4HlHQljO&=@VqlBn{ zqVxCDzTH|lsFy~aEtr;a?t&?GKl{J`a_^7I_~%Fa&%aM^-lh5vCHC)sUL@SkkolmB zTltK@fZ)gXK8~>(a>4i-$C%mTe=F)yqS|5VFmBS(xK;k2*MlbAUH6JF4gi|ziBt+$ z=7b$;l9-x2+rct*k91#BBUOJi1iCmEFu~X^<7eMl_i;=={(Prpo7uK^M3Pr3q5w)J zDLOzgmBh@eeX}jP>Hp8~f{DHZnFs4_NDXjw!G*+xWpthKPWgN2?w-ASXF-VnTXfdb z#yb9AZ+=t{xt{ad$kTu#T9d;;oNodI3P~u8hg7EuCRAQh#R#WI7lWsqHsbr6upp)p z20xk>*C~U4uBFx*(FeD3w`cf^Ne#`It$|TUTO>UqH4J<-u2taKi76gm`Qru=o}W5M zN(F+lp(F*mX~ZZex790AMMXoj1K4y#;sFS#9=%!~YT(0nEC^8jdj3YwPg8cq)0yT{ z5|GMw@!EBNr*<-OL`BebBU*+-DHFA)TVHDan+(xW?Z|QncqpU?qN z+&;|>@|l~^v#32SM`(+ZZ{g3aKVs6mh=18psn4GIBA!Kd7-KlWBIiCDrrY2Ej?ev|5Up;^eQ7LLZ2aY~IEO-1)E>4v_nCGfpg>X zu9P~_?3rP9Y_h|^al-C>`0xh*)xA+q$09M3@hOox@}X#74^o0UZ)?Z?H@h*!t>s0y zK9iAdVrF3Q{lhp0Gbd2OI=o$f-z*brH9qX0lq{42Dr=zlee9j0MPKGBwwk2VSg{j97|t3yPgda;rx)_%#mf zB8CCBiT?K}PrM6Un+)L7Oi#rcP}E1rlBl-@Emfe-YXNorEHC@nwTgk4zTc@%+sp_P zl@=#jR0`BurMEEv>5TLvS>f?h>_%ww6s;;uuq$arGY%x*eOn>BmEV}5~}EuC9FRwr7R z%r89<7UpPxTiq|Bu|^W`-G0&-tsNAc##DMJ-ev9*E{t|(xWh$wTq?*#lx9rwiJOv` zspw`(>`!ivQN~8nw=`>VyA+2Q)FQFwRw6Yp(J~vwDLH84Q0(y3>}U02=AF2Fr_ovZ8=32(NW6~n7by*S zRFAXU$eZ96M_e1g6-qrm(HvXT1UMV)!OTeF8`pa>He`IXmK*WFwLBPiaDM-*hE7gC zSJHrWZHxV0B;uUC1G~b@7ZIAoSVfQ%(8&7IsF#}810;|PPo&L1e@|B~Wz(ly9n7|JIQWwQA&d|Xnk-G^z| z4GxhI%q?%xG(TMQK9vwFvU;&NuybhU&0$+KI0QGi$sUEvd<{9k?FrH#$70~^vto=-9Gmz^Go%YsRwT>X-y6c+q`1$pf_Hl70<$R574hDJ{bP_ea4At z;oD~3C=R*aK+$vYVuv~DeO_MwY-m}d=MrX;Fz=5qw{vx?>J4(e(~P4lHl4j@9&4?@ z`9;IZvufsv68o6hOPJJV?dbu+IKVDk5F=P=+>av>5ncHZ_7oz*Ldw*?NI^lu3{aFW z`8I${1KB$zaJ9~BBI>H07ghgm96&*%61<9j^E@jU-L9mlo}wbpyR z-}il8=XGA^b>0a~ydJ?kfD!t-yxUZ0kd#(=fIMgS=_OX{ZbWRFHBfS<3~x)bhIyN- zPGN6XT`}si>*j838y~c)jGHFMa{emZ9$qN6%V_2(#xmf=bwa-SL5w7%R|HfXzj>P} z7HO3Uv4Q0E$%_~DoSR~25rEi3hBNq|&At}`+N4W_9P$36Zvjl|C2GaA{11dR`38T|#+Kh`@oi|BmNga-Y&ibEZuz0eP(EOd7}$zBN(S zFzs;OcQW~9A2Jyh?AF$HJw)&zQa<+O&Q1Nl^|)^FIYDJS%%81a-kaptdcnFE9S_a; zGjVD6Tsvo*!$+H(e)F=X;fr?mKEc+$<2`nmV$giV^3aOm(d$gRH0*YCnng;0yXlgJ zx$&F)@P&S(u3vb3Uh%Ael0NHTBi=DdVjWgL-be2;fqF*xqzsRn`}&I!LpT!beqTRW zC`j}Jc%E4aewF2x-72SOH!jMD7-93cKYlV%3*QJ8?YDFUlmj1q9^hY2!ZWgp*}5fhD<&cV>8^k+%oatfa&&mL(t_kM2Jxm-A z+{hA4^J!H5uZlDMwCW$f*#w+SEpZK!B(%!Ij%x4i#+&b*OF*cTmNxHeb`8uE-H%Stz3SiQYAri=>dewClika`Ebh>#Nd)j|Tkecvz)*3sN$KO6V+)&~%+nSje*Ct)4NczD z>ofdN|GffMY$~l86zltV0MDL2{bT0YZ5x(PGwpFHJac(THZGsst^;(0g;Ii(%!jl| z6s5q}8Pp*ISsPFKWhUtnx&s<*5~udR3lauq)D5s_0nL z(~I9V2f86vnU`eUac6IAX1g%|_k=Zb7; zXp`K={N%v9q1p%MWnG%mDS{|;H`)9ukuO=Xc@>%z?BVD1j!NI2Vxnt(m zaC7S|;veVchM8ZMr_Ol6D(HL-SuYsgFXSqd1+%JGx?D}8U8%Xhta+szA+5Z|jT_VU zMt*zs^!$kquJh|Zo^$5-xAEfKv8I5iY9Awlk_O1LsNL6FoUh^~!(@QR4A`~}=G#zc zFjsqp-I{u4UMqlP0C-ts8c;XUv-`itV`l?_YPAUL7>IEuhQ?=fGgy4m7D`Ge!BH(e z`h9-oXYF!g@UoL^1cNj)IqPny-n!iPJgaquf$Qmkm80(etH-`|R}f=6iRTh(#WiDO zZUN@|C1GZ1?aOt%Ht*?szfp~jvx5uCev z*zV8Ia>5%}Oq=N9a{JQbm(QN5Z`dw-9qZL&LyZfLE=rh7VRS@UsnkLb)HWr#9ryUx zDpkD9wd>XU>+eafPah7*Jl?IeC!>yVDz2omWDc&9RAZe_LZ-|5ZQ3;L(63?mEpgv6 z7DR(dWI~yHH*`mr&i(ZfPf9=L8n+#XBd6V;M;KS{W4f`&f^5r(1@s-s!ChuQF$4i} zYrb4$Gvar!HF85tjC+LJk;j~mK{t&K&&=zz2%8+5In14lZiphCT}CRO=`#i#HmOe- zys$z9mb>ziBg`S>w$50)33zOKtyRaDbkkWkwjRIz`m2bevHa2*QLvf$K{-4Qn6tEN z1u2`*j|_n>fomBU(fxUg!L!PJdVFT(E8hZ-%?MX!BN?%U1AZP*^j;j?7}dDS$N+Af zw(_Zlp}%BCYyF(u3kh+Xt8TsiZg9z7={QO==S2;>P`h46{YFmcJaqWZm^UxYPQJF8 zQL7g8!O%^O!=38eHT`pKohOgoO3cZv^wp6LYG<6Ot7pE7#Vm)~?3#-p*mK+7-hzbb z{SPanq4TF}T9RP;xE)l`w-onh_OsmD(+@eA24o#muf8Ly<`dZ17xCI9l9Odh2d{G27EC zGO``8jjE)r$$9wvtCNn)seI0<6s8BUbqd_X1Lf{(;}Gu^78W-A5mynqZ`6F+GJ~NU z#7{^xOK)dPnoI}4{#?-cFrFb)Rc*p=0BtWJ_+|SBmSPTyMuzZR zz9}_~Q|2B6G}CAn=_;K%vyk&MT4GMwT|otN<>t)|fK0nnO+FM9Y`8alxtu`h4~`Yj zCV=77Ki@#Ec;UiCsN3+q^Rq@<0Pzr9*C>Y$-VGi5t^R;bD`#Bi#pzX@KHVp6VY5L) z7Ad~C41M%W$L!q0u{{=f_5M~a<;0ou9dx=W?*@#U;l4EG5XPo|0FGK-Ut5o$5P8(P z!5$=~e}Ipvc|-Tv(WfCf=#r#5ZY?^;cnN2khrDT|`8aCD%*Ssx9^2mRxUbhqBlipa zZm~sp-qPg)C1d&xkA6KJ{IK zE>xz>Aj(G9t`=qpBOnC}5Ly?}6V1K{OAR^(t)8YDD0z%S*Ow(-F5TdZ7u&u34G4RM z`ao8fSdK#5F<>MUbd=3C43YDRfK=8U$Ko&wD*qy2g^lI+W)XZpQ+(pK#v_#N0pt?T z2JX%lD2n|3{>M=REAH<*3<$;Q0)WA9Zfs8c|kGC%s>Hv~l01>sv19f50ekQqjZw zLxvZ>beo>y8Mr0DZc}Z$rf1&2nDpXH;L@&+bwrTMWdpwjabRn40}`ZGhAT7NUhIP; zC>^>rLrAwmHjqmg@a;EZ-+=jIECZR#X)@L;o|Tq2$z80QCwc287kX#zRrb`+?=M4?H+awxOV&H$pt;nuM(4MnHJ$4Au^g@Z}Te+N$zZ4 z*7isU7{p1|SlSS1rF2hO$YNQo-}NzvnL_homem@$Eo-SoFIh#j4ZRksE;6pTj(aT! zFz(W2jJx(0mX<6$##WsyW*O}2Nd#d|P^=axo3$1#`q=s%iC0ed6f=G1o}>z205SK3 zCB#EDM=b{FCJ!}`!D$pgYb~0-oiSxq?4jngmsGelp(BrW;nb*#>FMc3MMd6N0s-yK zShO>5&EX2AK8Hn3y8`{vfF`1_Y|g1rYe%@HbQ7C1RnoD~x8On<<5l-S@9<-Wx8t`{ zS;IRGwl4YX(vk^C~;f)}lR^ zFl_$4PdG!R%&f<}#&GDXT8(SVypnh{VJ#!GA|0IGcup%BQmJy~c~2h>&MkVZJFZvv zs&&38WL|PHoq@pGhu^)A8P1*grK-E(#@(r*c{y>D4;OuWTsialmX(crh6^0LwePIq zp3jkJvpT-6(5~X64WNa+G9)6h)t9%;5YJBT8)husYWj;fP5jQs*M#^_VgZUKCsr?F14$@oiX>~jAdR!f3q(~3uc>% z@#4pn)?P9)j};b?c=!Fwszj2&P_#G9Y7IHqxMb#TYvZaVL%wYLcGG*z@b1Sl-M#Ca zf4M&|Dtd|P-PvzPJ#KIHdAsn=!{wZ2whsx(JJ`=>d-ZiUW6--&T7aR!N@W1K04%YP z)mvbm)<1gvsX4hifv)#j6%1~kb+*I5JAM8>KKtJ^N-sy=i8|sDk45V)`fEn6>H#zQ zy5_GU)zWU>b4B~Q-Q^z*)%x9+t?aTSDq~)fS&8P{%{9@@=r0z#f6h7#6)#BQp^6Ti z>I7l@3O(shnOn?ixwY1iyZI%0dIOYRN8pqu)od0wZ@~f)hj{f}e`@VdBL>aiZ<<Lg`apTm~)C3?T#K}H}XVkuEU&LAF{$ z_SXDZ=`XRx0~_ru6(j!m+gRrVrS7GI%F4d}|Jr!!9`uVYzMavuh#xo7-mu2#Ti>PM z@&Yt3W;`kJ;d0}|TH4yPzJ{0er!Q>3U%~r6`YRM^Rch4u``*26C;%sR%br|YRh>7( z@2otq^$5r9$(O1Yll9*iyeb`Ju+g|5p?f zZ6#%3qlD!2G@d0GU1!Ejqfv{}ZcVlb$?pY zA7-2tzOC%8scQN7ysan0$CR^Fhkw+o=UXxS>EQI5o6VZKa*X}!*Q`^`HU4ot@!`ed z47UDh|H#$;^Yb2;H!IQS^ii)X+Ny(H8*Y$+5n>_!i~GucMDyyt^6KCp^iIRutx>99 zesro^`k|-up>wNYltwbCGiy)dmE#i6^o!iye)rf~MkGE4621EFHu7^V6Am6EQg<7Nq(CdiInW;*W&ll^-N64CJ4(en5N>uG6Gq%6H2#(t*qLA zfBnCU`@eGMq*M3LgOT6m2T8=H5Rw*!3|h6XS);~t++byNTEr?S6*zFkCDeo2S}9%_ zzi_y@b1Uij8I`cs;kP61n3M?~zQ1z*2;6=nobPRf{m}>tUEyVEoI4QR>gxJ5Bcp;LJt`u+M-di0^z42`t>+Kv3`>}~VSULs|Np)|(FX#k z2V%)HX~yK^)(6GgQGxqcB>_lWSdZSlw*Uxm)VL1fw$C~JcnYlCzc*d_5{U6HMH!pq zWB{n)hKo;jxdh5LMtv(oMrf8~^x9w!Z96oMwWX6~4pijEObZQTR7ALE$~*UYt=|6z zW&iJwou5K{ga;}Qoy*KWepXi}bYik-&fK}>#pyu&J|O-pU;uUvFN@H&ZBZu;Yq@3k zdDKebsb8j=y=Zdqzdm{BOC+x2nl-}%I!6dKjE59YXv1ZQ5j!de@ybFT$yGJM1*0-L zoeGXPi7^qUWXu|Ihz|!ZIB#9GZU28JLf2NbFE%ijiYNE{`3^Kb1^J1y2)-v?Jb$|! zXC??JxTU8ieNoXcGKWAs=+hH?Kr8#2))qQcjv2~{glR<; zeatYJ64cIchN4cm$~=~6l4*1>hAf91$Q+7@pB9I#y7KQh8yh^Wbi~%YqgSfkS@VH# zV5pV7BEOx7_W)Ul_oTsd8Iz`{k`5%5$+`LK|NSb_OSsKp?e-QG|Dr4$soa?Uz<-r7 zq=-L?^-t0Go)ch6sdtcr7T=gh#l#5Xs#n^qK&Hr+7Z`J87JSxVKVjEGx)#<38dr z!)UE9B(}IX&_NetB^m3a@;l|Uh*+Vh3OBlLovF_E#ktMC^4;^VjniOhil`x0q|1JEW@(=e;`>CmLI0G8g0iPB4q!;8ocTBar3 zfaDFt70*y(!RRt{H3{lIrEs&?F9?}C7dEI>tNfSo2L}wCG>JNNhvRr#-Li0)Ss>0n zh@Pe~%qMdZR~MonSEAV_Pl)boR}=)qNRHmZ>wroA#erI-@MiPJIJ;dwzi*)pAsQ^j zh`oKB+HZPj`}9+C?Y+>deJM>*rL}^w{^@gzsF@-`3}c94^_EKwt_lvC&NltKY*R&>rJvbaix0-su z6FaVhn*FY@KB^U+Z>3eMVcC#FbCsMOxC5&jE;(>))$&VN zLmO_=am8DHz;>I}(jEW2FRM=chY%`vyk*y6$kFM9@|BJ`9KtfbZZl+NrOp+Ib=O#E zn*B0H{~n_m@!2Pj2%!(xuqrI9BaA(@lK7d3RG#Yi;FSg&h9F{)Y49E0U9>?6mmRyA zqgeof3`AFIulgR)m3yg|#0CYsxeM%S9LP6EcIedUcTcFDJB(^8Ff1R=s7Pqd?g;uj zJXi*4RLu~oDwHEf%8uf{|C<9WXz21l4o;1j{ItzWrvWn?bI<@R9#t-rb>WSV1NFhYG!*3tR(S6A&B zGVocgCGEa{gsx~;q*vt^Dsm?JPVGU01o=vyASuP|T9rk*LbND?C5MVUs6PvZ2+yxF z1Po4;6Wg1W1>V@R;$JCla)1rC%s>{S4ZiMlQ7X#;lA@#$PX~mz{OctOKqpKq6|`-i z6XR08L4Xa#N*rj6VT11Vo^ip4dy52CzVb)R!Qy7t&9@vbIR1rbm^cN-q!Du&XeW*E z1QUvx9YBp)ix%Zp^ic|#w1&YevmGKM&0!c>PQYLW7d6fNy!1UJJMsHbh=?2~eL0#O zenZS9mO@d;!Gj1(7~^zF8c$F6czC$Qz)QV{{8O8ijHf{&KP98YtU2@~IKIq%M%S(A z#V(ict1b5IlTQvV)YF`X7zh0oGKCL`(Pu6GN&y@YnK8p9O2nH6?xZI~aScg#AEs97 z=(7HYG&nP6b3gYb0*6KGH5pj`Kn`RS^LH338CMiqwadi96J;SSZw39Yc@+O>LOvlO zLXzkiISX(nq6QAb68XHGpaG(<^CmKpXNuEnZ-fPITP|yv}i>w!W} zWKBl2n6lXQCM^OAMsRiQui@kc12Y%OU--au>HK z2?wl2{Fjm%H98w@^Ya;2s+T9{F1GH}|6}{T_b+=dHtzLU`#X2S|2N}-dP|O#z&VsD zIbfyV%zgCok^Uot$e{Uauz&$H|6x1w7u2v)$|+kWBd*(K)v`zsy~9?a~AXONo*Q@y@VSUDe(HzLmZvCV{byiBq~b75{Z= z!@urQW(=~6UNXzVvK9*!(wwxvGH>l+_R5I(IrBjYk5&dKg`H&s5k)UErX_Z$k>XTM zVRo@mzuKF(?-1`fi45e-g0kI)_?`Q*!vYodAGUsGwq50hpEezroJ~G3`H0*;{oFl>Yd`c zCa#r>dXxW1RgP)5Ie81tfkHy#3vd~RLaWc>PJqWJC=SFGu1?3JQ$7qd&Bu#(HjV zpB+4ep=q;#>w=hIOl;!CL?71=QJ}snC^*F9lsOzw-_zkW#w%~p;70N@tq13gk^2)` zuKNleAf&A##aPD+^kem7HP8liLd7STqy!MgpbegH*f8EGz2Ad#KT#wH3ziNne}Q@~ zjzMa^Wo@ECe%3p}SgOw^EN{h(i1c=^*XYgLBGCkh5}yw|jLwmx#8Lx#M>p`ty#DbE zhvhw^u$6Sqoz7?OERnm~z1!T)MOpX4no^HkM3PP(!!qu6>zN`lA}2DO{Ac71&ItBo zK+HN`G67)fUs{IaGu1)q3W11_gcS^bM62GGxlotU-F?f zkHLY0&t-0G^C~jxDW(!IYo<)$mbJPJW{Sx#1UuTp1?#^`JLB2(YLpPEk3HPOphBh;T1EM_kg+ zmDVV{8uwZ3ec^&3eu@KmtDamP<^?1zGKhna$_mrUW3p zJfV{U5=kaWDeU_qZ9bN|Vj`+unLc6L*u_)JOgMYO82p7zr`Cq38ZA`USQ0#w&V175 zY*883O8p;6uQb?u^^YG*8)= z@p{G|Fz1qk?()tRZ=?!5F3t%C34QyD*py6cMpj4tdnrD?8f`@}fyM3pbAZ_(>h!+LN1z0G;7Rb041*lyI`y&H0~#HT>4s&_3!9pJ~7nV)?(Ys32S$_~8W zHBgQ?JISa5cUNqJxOopE((n%iD9E`}2G40Yi>I(yhaj&2>x&h~C*bL0?(7v6T3G_* z?2wt%F=wrFa)1F)oA%F-8D%yh`EKrUA5&K{7#3SMY)>*Hg;%m;(ow<*QL*I+IUln0 zOEr#&@5t)FEk=i;gXk`!78Iu;Gn?oQhKQhFR)H27H%4H>{* zj}!x{(8K6-6H;KIU;p`z*Uf9oc8k71G?G&ZSmp+*DnwuzQst@tZ95tTmUH68b=g0L zwbW5K1$sj@%4sN#n=~PsZ%k~-!lw!LEGz5NrEhYG0ME_-+Q`916@MTSE9R;u)f`}? zR`8Bjo>3c1VGHrp6`G`V0kHZ238?ec^GEEzL(R;Y-ULOh9Q=i`{PD67E@-LMpjuwr z{1^Jwn;$;fzONj3TF(qdPScSz0I}6t#{i75k{kr})G>#%h2-Q+@u@<>g9*Zw;2*EA zJgytLD>7phDP^WSUACDSi_pC zoy|EMtKuK3@^Z}^*bQ{nFA+aC`uRmPx)!gkoS$Qsrh(NKLTfSZZlHmPp;-HlQZUck>{B1l`5xZG`e~%gTM&{e0uNT4zdJ3@%38Do4lV=iJBL zi|vgiO}>42cx5*~(CWJqn=*^KW=qdb5WAj9WCf;r~dx@}H(#eg{|P_QbrN zxQTRoR}^^BRSI=JErACRk4=@DWCfG zXqi3S9clMBuKIA~YZkr8dMH})gG3nFk35p}*u8Y^X;ciFN0M2$0mFtgLDrd{TlRGc zy+&z@W5$h$Hwh;Xg@9v{3(5Ky)HY&3D!oCJ!O>Yy*BSQ_-(DzbCNknnzJ2U}(&aIt zp4tG;a3|U!y#(>J+P8F)#W(c$^z3hg?i=6{D|9acH;T8IxTMlG>-vVZTDpMptAd^D zLVIDS-S|+hG%JYu!dTHSU^vrw(l-l*!vBTJXn-Y!a&-l!ccQ6-j-1SaYY-7=_MW5E zbNP(DxHcJcD7QGIKPo`pz<5WOLy@o9oifY9j~k_)e9qIPrl=^K3K)aQHJJf*c~Mbw zkBriP|FHq0{UszaD*q4=uODUQ$x~j%`gZDm|$30+-r5a=3%iY zZ(cn*+g(yQXx6~vlpDhs5R7u-gwJWb1m@ClfL>NmZaih6l#qDt_;Es3Y>oh3=5{6v z)x*&r7a1`*IFl(86DFkMV5VnWA2eu?fB}&hp{j8}`{Xj|K9O?wkLuCh9Lr0YrXkfJ z!6qNb*q3N8VH%`XPCE*e5YS%2iQZvO5zX`49KxuEQjpXcM+J%gqpCuRDQ=XU4fWtO z8qlZ2Y^3?z=gPbmOQ&1PR&fT;d}BaENkqxSXW_EmQa3Q)%TWaP1LWLnNFAAfla59F z@RqfqD<*Plgj6HpLttppR!;vANSKaz)%0;$NZZ^)W<*2{OL6<0Zxo@`;*&Gnfd3*T ze%o%2_&rcAiS4jpIQB(ocO6U&XjyUs2uIWJSv`1MUP*~Zr^{_xuRD0~*8=Prrp zqObo^KWXed)Apw_^oF-JjsKhy*lu|H*V}D6>^O6{LGI4tqo>=htv2xJn(}RKye+=G z?|sjC7iwtz^?cdc4o5Sh?ZaNW?aj++{@aZ{FA5&`^9*9+Z`jRoLW8DIbLnn*echq!M;G+}`e6$_=%CnG<1=T^ znyS%AH)o8cZ+C^lX3d^j3i*m%#lJp^cC~;Twi-FoyVt_ktyN2qJQZ*yZS?h%r^isU z+Pl5c-_y_C7P0iU`6U)3@xM;taw5TkpFVw>QtYl++zXO$jJdh``JCy~gHR*1b7}Y` zXMBk{q^c$6WOX~*kHxNK?wmQJ&_F+b@?<~z{uSjSh3p<&TY^{=Sqy1KR_&=!cs`6z zPEO8#$8?@9Jus%WZLd;J1@g4f@-1`9X0HhRbV9=**P0yZJZ9#jC@actgS;k+6AgRo zYCfEI6_!^DZGrZiYBuh%pGxi(O8GYS(WjX-^QXF+4(hJ@>#uiM9tD_B9-*SmFDS5@ zJNE=cL;z6T2}S4J*|YDE$ES2_`25YAnb&$(c-uN5A%UuX8;#OtG}vc8J~s};dxw#L zR;ui@Q(?)Uj~r<|c<^8+)MDz~eSbVcpt`l=c>BP>z|Y*)ZVc#0(tb$G`$eI+f0Y@# zyUZv*I_GXj?=Jyjw_Ul+${zIzeEN?byn5xxJ|6+Vx<~K}fuaqDKYDO_j^AFl+%vDE zYbav74XmR1xDL&rnd^2KYhCH`*PB&LeOe#jhK^$b;W)a204R}QmXUA!M~urIp*Ap4 z?{$6AAbtVz#89S2rnr?T_F{%N6OaDBp*G<|aG}q>b6};cnxWS|jGvWr|G3|?jEwf1 z#*G~tf#G`l@#n_QnKP$h|7Fo_Mu%)4_ojlvbHPk~Y+8n?cLseryu{Q#8 zBZ;Hpg2BuFb~&$KhY*XM87No#)Ag9n)Wc2=89Go=appIEk*^aY9UYJ0emh;C4rEu* zY`aGV#0JxFZNprlG$`BNc%>NNdVX_zaBwhTU7b9$oWip)e(0mPslLjXx0kBfo?jNj zG~yA|izv+(M9s0Ke>gAg6&#?(XwD}z&LW9PhIe{&}%msZhM`Ay8P0OpG=(28P zdCjFLynovgr)2y;&!C-p%a%8f`0K?3QDE$`U-oIrr_Y~F)YOcAJasG~;^N~?>4BWI zeayr+mpzOIbJX)WB*F!pd1w`CYXy)HA!Y%VrPGRI6oym(FwwltlPZ~4x>vw}X8jPq zm^4Mca!Z`Q`61KRg^5$#PVts=55VjO5pKIQ++dQwB!Jo>b->edY?iI< zY|QZ&QRn7`hz!2($A|L>fIej3{e*YBfz=e=yM8ZwyFYHcu~0cdXgUQC)iV;*C^lJ>BH=_QZn2c-QUs^z>|H+o*Bl-MNc- zxjGFS?%X=FoZ-n9KAQV>q4w-f#=rfw``y({_voH%hPj7*eCmW+A0LSL--7mcL858L zxXYI_yk1cHACczNP5ri(NA~w~|LkzDy+WPa^rM5pt~ZbV8a$%U+}{*Ry~^%aE0jKY zg(7nb>DdAjlQa4H$ik*7ummP8fDmuBvwOo0Vnh1*Gs$f}ong21dil-dxqX|GiI#El zxw+3a&r|3ew^v>{k?+!uDvKd^BNjDMcn`T%q*odNxa9mk_}?R> z3~1o-wo2Z$n~pRZ7ToC`Z^VI-@d%Y_MfmA>B~!*9g<=Jez6YcNK2 zlL-3RH{D(>h6sqP0zqJ@$p(CI#lq5sCqI}PL?jF?jK$zdIVlT%3s zgD5t0A`tDRP>JW7@bL^Vaa{s~)wNGY70$LdvQKhHHL-G3}F?`c}=H?oaVWgEp^dB^j*zvW^&|NHUf87&vCbB&2@Nm%lE^!M)E8AQ<# z#BIvP{R{)}8TuaYo(+z{dsh+x?FN6+&$MZ23eVM)CFl4lpy+6u4NZH^Hx7#_+{tlx zejLALNapZ8!wy8eV9NB892)Tf@l7@XDD_dV(u?q{-ke(=dtAKbW~QDPOvNAGu;i8B zrVqDz=W&eY!s&PRFQ-Em3YZzP^>3mUb(7N5ooI4B32O+DMQ^Vbu2< zWEPhoH_Ll3cOr|Fl_ZH5ux@j;cJ0~)pfZX;o9u^Y!j)NI&(G)vlQF6T2xTz_1;W{^ zjC6Tyj}|ZyFN_etUD|Vdh0b|Xb~>g7-N1beC|{qR%dsYRP|pQJqNz0CjILy;eT*&Z z`X0bM6SQG7o~cdQz1-3^oSnWfy=>QcR-bUVqQY?F7Pa~y7m)*-E(c#FKctx1x_$c* zIxIF<8J`M(cz#7OG8BZ-K(Fm&~lPs4ui}D7?T`hAs<$ps?oM z?(!$yxHjH8k4kpfGk*noOgrhsbL-6+_j_?DJW3fjoer=>vE(n3hA>;S=&9ob@i<#r?Z9>p4} zTc?gcd{Q>|b&GOb%n~d1|ERHJcQdS{109JujOgr^9(3jMF4*@tS5D#W$4M=QP|`7Y z5_{)Hhlhuoo16Qn)u5i@lI{!X0jf;G2VFuDb&xZo94YXDjK`cFD?B4GfP=#HLr52U z`*z1i`Wz1AyDy_A6@9Clw?AOf7Ra-mj$0`l`Q-4u?DW@==dPoB4?g7Y?+9dQ*mdf9 zM7BqOcXmxCF2GCg=Jw8mc&q%$FC{LxHIq#;ikiADdajuK>uuHkjGM4r*n5l8U57%; zl0lLQoqTeoif*hA-~n(k?aVD|wr5E(39HoTvN?`RYz(7iZDX8&QZkj?HGT8z{sa*u(ZqVDR zkNq!xWE*8cEAK?xL5F++6Ghz?5H3#j*|>JCIUU)%G~$Qv?$OyUj}NBP`)gi;6_ZON zxt+R?YCbD*nf)F}4S;Yhdp7x&ttZlsAf{~bFAjWG`TVhxFkUZ*eoi9P?DOpGAdpB7 zmDCt2Pwgdd*56eq_NFs-lMI@U&p=D@ia*F5rl}$|h*g;VI4m=^idX@WB)xYV?#+qa z9s6;Dx;wHGTgHpfRyc*(mDA2bSdtCmr64-;asE9Aca_;JkQ#ix_4 z35|C#w_cE!K2CiXl0`0qI>m>P#{gOiZCkdyTFa!zoM)@#fy=_?*n;bYZY+)1S+I;r zEVTv1Mk%ilQ8`EDP;1LUqfYFF2AWNH!gOyHT?XRhkmobc=S-S3DJrMmr_9%4{ z>t%h=EKPvfJxZ-PO;tmoGnq$NS4+#$bvOGU>)VJbHSq^Br4Sj7U(9^YM>`W0Wt;v1 z$A*1KDGxw|#C$DG^WBMLM{3X1`SmYk)NOBT2@duJn=7SktIFwI_^1eCb>^uGaHKon zvLlQW(&U5I*jeiD1Lw2-6vldo2%*fvPZrR35%OLNwax>>4g zX$-kjLw?SW?--hbF`F$5hkERNVvUUcv$W+#k8o}7!(SkiUqQ|1nIs&Qi3YEZJ z$|oz51F_lp$%W^h(wLc*t2fA4g@3XKXuO@<{LH@@acrucox;V`lye zXLL&5fVP9}RT#<+v<5jO+c?T2LXMn>z(h%(s_CY_cK4cm&ijLo*3^~sAYlWQL?GAG zfBqw&wB#~Gf|^8rD>c=|<=m$LNd-u@t;<(x5LiypXNLykO`xG;z0gi^B8hZ=BzR2N z_1m}2=uWeGw_=?ufp!qFcMM<~DkKyWEvpdggt>iICOF>Z{zSqHZ`HFHqNYBPocM={ z=eW1?n(zqb(ZmswW;&K0GTnFZ;6Wc2Dbj(six&r=78phGjrUkjcOdC!RCh`hEyd3W zxU_p~KAY*$-*i#^#}qzrZi9Cc{hW`nkxV~WLf}Zmln3>m52#jR(PT_XSZQUmK|I2t zU3XbLpK!WllIaNZh|^&m#;t!^qn*Rq%4vb1FS{yLs(gMOemepuGTf*g!OjvS=*NbE zMzTSHzL=^i13Btz?!twmp;y}9t6IJK(pQ~Fdxn;NFlX9uj)()JW$}a{Flm= z;}hO}Gd&H$W`^1(YJpjoAD=<=kq^!;3d8fTBc1o5M`QRwWgrZE)~({=;&{hyZ8~=z z1wk{C4pmpn_pK?9vo;PLGNg@;&XdiN3hy!gJM%oOa%@Ka)>9e3^?NybW7n8i1LzQQ zo(#%2T@}LhBoa^i-u-o3www;bP6cz0bP)*NaiYk4aBSjAr4&1;+*g4R{lH)<_2{Ex z1Ibfq&zE_7ywf~5XQX*{mtspoaRAuVC#rDHdsdofOy+};mnfWM`4!|n9^Wzg?WM{0 zTD5FxvZSch>FSkMGGaWRG%Pd`je}bwYbc(?_0UpQUb!+Rt*GSCgR|u_2N8+jq&5SO z+rUdQa^eNf#89)US0_# zTB@=(z0>RFcU^XrQn=6FI0w#TI=QIP=(!I%kCm|KHg{(0iFCxXD&Zq4uLM^&sLM-&dhV;h)=1QmHv zL&hfz)6>(xHmgJjC6FJ%|ZXU!BOJDVslgqLLfHn<5$``@#CVj26f`I zXuiDL&z;hGA}cqF6RD!mDMcm$*1Ijr9V@Ao;@%tXcexaLf*)#V_pwGw5M+J)pi-FI zWZ+Qeu(8_s7V{xYZK>|3eHNP5mc%=7q zlnARH#kh@ONF(c3ec%h4k9$;OWI0{v4NjVfjf=}VaPNlk_fk=Q_|v$()pSvSW_YE* zy9ANl%@%0D=#SU;9{)TzqtTB)@*oNY*Z=c1W5qu-0)Km~N*SYD`W+Wq{=?A7-|YH{ ztl*C`E_Lp`R|#iB#c586vqH$;pt?gd%@1F2hjga+@kX2#Lc=`}p1p8Z8DG44*x?HD z_h{HKd$V$S0iz{dsX#iI1hySKk3-!5wsgwE&8lf$Z>|@R7QwX*JqM4SXOK(<5$50| zqomjb2SRKQa*B+hBn7^(1}_}U{?v0Pv(eqU0vNN zg=9^>3_H9P3V`qbOgay}I=tl;KEQr({OgRipeoN}ZeLXQg=$@4ve=F%jXt zGLVvMbcV3#p!er5UIZZEenp>38YGyS&gbN$ui(5L?xoM(`?7RC02*62ZQ7*aKtA%0 zIn=O%Re|4}6rwM_?CRX}>9c{^ZV}Zr?|5Qlz`!bGL`>AS`1$$4f=tyvMZBb{J3`|r z$nMlpWguM|frUyZgvC2t%iggXZX?EI5)1D4b8wyhDMtTQiGsJNP*mpR><_%5>9fev z6`j9g*q^n)>^{49hrm|FId-$X`R3sn3y}_X4W^NlV_Iw9Ld`|#G(~wAtx0@NFOYNV zNySK99-U$H^}+HuheWy@3_>3u8up(f-Zk0R75rHzlbx3`W&a8=h+*te+6 zDa}K#uB)ThTxDWna{Y#d=At~uKA9NSM5(Kr1^Xr;oTFtzU!2H}T**QTr1)sV!Ie^6 za$g7!lO#ky8hpDGMLQ-46fHd#K0i8hJqe676Jrw_YAzywB_EgCr=@D`h7DGrzA0ZI zje0G9yPp(nh0-T@r>6YnPhHgmpLUPTc%c<>hp8ypKcbxnoq)|2%2j3(WxbP)jWE zoOT&>GrfHZ?+?{;w(rgH#AMEuAGiuVJv|lUSSf6T1qfnd&|sZt69jul-OV4BkT+TgMt9E3|_=g_x+(hUE+R>Khm$_3+ z*k0}1r_YDDR{P*P%vx?y(JPL;!M%ETBJCn=@OIZsdi3t?OzmH_YSl!6$)K=d$(jrF z-seChc;Fo7ggKJhpRnqTm`2(dum%ug?!l3SD%<0e{zQavmpk_3Dt{rv21pH#I8{@n zQi)u@d2_h*H^t{NvJg6=7d0}C4uAgQzW+Yzu)nWe!>V-@Pu=%j<=#D`*$GY?H+Ish zp8X&)Sm{J|+YzLRH#+_2Ed8P$I*&)f>*&vZVkk8R%&0JsEMecNGvS=^Y;6msl6L2h zIK}wiA^9AD++ItIjxi-7mSbBnhs&|U0I*k=1_;sqkk z^Id?5WUG=QuN`k6HFQDqvavH z6Mvl}&}ojCp67q~U`e2BpF@94gEq75UtTPtHtcGp2^xmW)VgsR%AP7_>#kk9h{u^U z3x&L-s(MoYBUyDeVq5LAXf>z`F>UVLuwu+_Dw`i`QW7=`3uTj~s7r2uLy zi^D}i+^Q4`#lB?Z_6S92HfhB7IGp2#rv2?*qTdZUS@Ch3nz68FzgpGm}I~SE2C3e@xr9Ge6t#Ui~EJ6bAgwvOlj}B`1 zjxK|5_x-rr+`Cym9YrlwB6c7N)SuCk5(R9`S}P>Yx%bo`Hi90>gAz)g@2= ztnUK}jt=Dr#l(-tG3UmxcYtQ4ifUp+*OxgtZOhV{<^gz)yv3@^G=2%yI!5nD3ljh6QQzTvz`pb*4pF&F8AUWrf+C*;9S@nAILZM?zpy zZ%d@2%&S_Xh7|$*d1j_hvRUO!Q?fm|gf4l_aa#aRt5#Kn|q=uD%Qdj=q~tO@I3?yoqtJr0?d)U z+`A=;2ryEJI&0)5?ivJZZ`wh_#r&DYDn$mbBK4GNN8Q=;G+{(GyAuJO+`;J94p$bSO_s^6*Pw%><*P@T-OF|Jb+5uXV`O&)w z+PgUmu|}Yjl4$Py`FFuJ5v&^Eh<8T-e(Me$MhIRTGv56WcQ%+4*3|xY?^=SK%ox0~ zpc#>PH!3P>I>kX7+lV8A3K&o)JxqBIZ>P`+1M?iOdOjc^02ujq{-0lPwiw0b%jadM z1#s$jH)kY$fzl7(V%Bf7QZV7`h+pLn8)|!Vhk=J}R0wm_^p1*RBe?~Z@KK@JiavV? zfkA*SlyGWlfIyFu;>W&c7?BtOH{@G#>)SmPPzB84wC&f=o{!<1I%6~u7W4HJ!S1|% zW^>`pnLH$Ln8ZDLG)}Sr!)7~k9|*MzQ$t|dIpsBf*g_7&)0<7_Pr`B-lM_3D#$Dv1 zJfRB2uwu#U&rBLJ-eTfAPOCK9=lT-DSReE0D@(D=e1w^$_;{-lX>N3Rf7kF_G6T@C zKmv=Y+V9P~-S;qoGn>u<8G1KCs+#;O4?!(3>dKWX@YwI)CxL?IgXH(%a=0r`7UP+V zY*uLpvS;)R3`Rq!t&qw9Zr(POkGzXC_&hJqML(Uk3(xfxC?qx2VD9$H9wpsp+#d-5 z+B6PY=q^T+!M1Ym72a+HvDuK&Dsv`#GM_woYow1pR-NndtdikXd^MZ_LQ!rXkCNK! zz>GrxOJ%_hwgQN8kMB}6^RU`Ia_;kMTafa!X*s-XtS&o1;6@O0qr#?r`*ho*jXa+b zdrpL?h7GgiumtH~EDJM=(vDvE_DO+ftBcOkVda4}*|TCXkALB&yWmZ5W4)%87}NrbX=F`Xym# zQzzt-5;LeVLAn<7TME4v^3&)DFLjPK3;Z+W>4dg5&3^B5 zfG9hL_Odh@Sr&C_*WQvF#Zo##9YL++E3E`T#P`}H)gRoD{rLQOtQC(E60#vdgijYS zUuoiN)1`}f$>)3L@3P(8POWQv1zwZ@yG<#5LD^|)wK_4xRHYh)Rt6nLd(~%vc_8a! z+O%NJg~hi%mSK$&5aCOauYSODaELD1uR&6^KYThgt9meonx zP{CCsmN9n543xc90T4WwzM{0bCoMA;!YgWmon5DGl;96C491+n_t78PxS_6)W+RZ; z_9}pqas&y}78Q|#Oy9Vf?`@8x9s{hu=ke`*Q+G}z-q~wG5FaIk56|K8uLvQ+(I52l z(^W~6i){P)VOb?yb;obI#=^c>;_@KWrj%+tCed66>kF^`_*{(C!T7R-EVk7#>nu$W zZ&JmZM5k0|NG5r8(`83Udd{E?rxZ*Koapd;re{y2by(e@G?0#tSC3o) z9b28IUlIb!2%L)dAh-i`w z?3!LGAIoe;8&=Re{O*@K`cm7`rVeBg903le-Y<vuA30d!5Q~39IFMYH*xH8K-(??8()oV&|SK;~dQOO1ePlpIp>dvxpV3 zex(r8upgXC-A1)T|FCH8(CNkc{x!Gc={j9sv$JrP27RO6QcjD`)|A7?8m&wiHDqWj zw^PkjU3Q3wM80%7cD{eM^tYui?cKdh*ID{Ug~8^mj7BWnz0+gj9In4ERRnsDZLY6h z5IDXX{xSyH8cd^bI0dv8=t7b44_LJP9yduiE~Xs+zZI~pZCDSiTZsD;#1An!b>6^`|)+5 zWtX}pJ=_XI1kaMMZ8sbS50P7mlJS=FG~cJQ$nJ!LmwW?+mfZ##SZxdp3@%?ASPBpy zHlsQPk2B7q5|J$G;V-Ar_!EXM57RGAX>(6OcF8wL{%gpAQ~`X0$inUicGbLxca!h; zy!6uUukwr0(Cc$Wf;)U)zj5qdONtA!uMUb_m0E~x*f8r9bb#YT5cYh|z=11RW)2Jb z(uhXN+{bQ7>ANdDUk}vC?iQPn21XgXslw(TETZwI%Iqw${ zWHw^NXha>@0*B((DlPjI7fl0Os-$Y}Cp~9U3_b-R`kHz5jXSOUxf;}|dar=m_#sX6=Q00TFzl}aO^$4KhlnWU+y{_pBPbd*Q)UNd78(g`@Vl?!fo*sQhdiV$cj{v?!UUd#dPyAA@LnZ;J>ZSw# zowA#DubHWoQHA~`pUg1LHij*Uq?h#3NDl6qo^bF_a#?t5&v-^D_HJ>$Cr$zW|rp}>qOx)TH16<9?dYM z22!I;L3Vb*9v2JkB@rz&A1=07b}<4JXsM%~o(i-b-Gi5u_w3_LE)hi^ng7F$uD4%- zXnZfe^=Or4G-Swhd;p@BeT|Zlfr8hmNnJ{qmk?p0(^1^Hjaj@?)A}mRp26Crpcz@f zq1Lj{=?|S;WNPROWiy%-qN{?BvNSSUB~k|%73&mye#}G;&-ww=_`wbwVFySD9e(fw z6|<=tk(sb3qEw?NwD3$*#j!&WIN2P!V@V+>rCRFQGVpnmk7hh<*j>04G;41{vMVd{ zL!<_><)mw#b~C8`4Gp>u0W98yl~Fz_@tJ}5&Rr(*`3TbmYcg4(84!@#9cyftGwBp% zxMuZnhZg4DpNx{R4?xtOzHXx*FMYp3*g2$l*V1Q}vk3qZi{3%TZcd->ArAz>)CpU$ zD96U#dIP-goj5M6=X^HDznf_SyIyL^El8;$WXxOdv?brEQere?&6Zi zsK^s9SiZruT{bKEMFo$TfC*matJpB@*xnpb(qd7NE8Xy}8fYt5;94%Py28)^TRQ*b z4Ejd#Nl6x1jd&|(-amG6dRiVhI1n5w_Y8hBbo%W*y${oTMqT@N;f4xkLYD}}Wm2VQ z{CzrhR*Y?wuM)cer%9xfF=@w)LFJ38X+A8Y1$P%OLW`8Zu&zj5vbYmOXdiqE1ha~2 z0XAYZU@0`E_|qq9I)j@FM2t!~()az1uBpkr-WmbOWUz{lCOq3H=xyi>O8@>gDWYJb z)pUnOy)*)2us0MY5wN(g)y~Q%{y(sR!aH~h@sKJXYV?~$@?ym7tWF`vyq)C_l&=f` z?i3mvp=TD_gKZR5bWF(dAt{Kv5T67zV6rd(igw$c&ym)P7e6HsC+(eY zP7N1<4((x^agzzfQ7lc)h}tX{tlPS^G1Hz|w+P$AVMN><-(SJrOYbdOE_fyg1{tJYa@Jqxz7c z4VvtE{phbhOD6zIe=yUC6``{Z=?nr`w^I{!d+z8y{Rd{9(Em&?pV?nOFOZI@h;s=^ z$nMDUHKaEPO<6N^-Nub6BfmyNlhTGk(wTjxHMMtHjuDI{w=8RTeu zC&j$nIYLnc{eOy;D_uU4P+RxxX(RGe96#D;AeAyFB;fX;CyH&9t|QA@gQYjs)8W{n z3JvNq3)PaQJ#`e)n7v9p;9-|#%iJOj-s)O(Ee>D-9G8Ee#l}2FGK)PMt=wm=ocmA{edGZPiLN!_&Ei z8Zp=R`zt!EW;!~CYywLtB56*tW*uQq*REOPO{0v=7Jx1znAZ8|1thk6{Dmd0P#N_A z)LBx0k$dlA9Lu4H{cq0G=-$4+x*{`iPc33FZPyuSII}5qWIgZ$nJ-@+re=j=8dx*5 zMwKcFfR#uTFw(B2lOkA9DzQVQY*Fd!WiC3?#pf+N+b@<#yD1P~20BCKAK-QFNB^|R zJioLVqpgVy->+kCzyFcG>wile_WyqPf2gMWzjwnfX~xcEk=(z8U|+=OB7LRCu`n@N zEB!aRbs{X?xH0Q|yGdPke1JfA6uidhGv$)B=b(hecJKO$GCfs=vX!QD`xDc~@FDxV zWsnH(fz0*sev?Em?SP$TCMM=|_9HMY4t^*2hl!;J(XQd_?XKfA^|#@&i)Cbx&TZ*Q zAkpA^{A7D_h4Q^3kwMNX*NjMTxWz{(8&M&&8GHJe6iK8NIEcYWgUl^0eMA(YTe_bz z>%T6OL+0-$A~`V$;B*xpP!4!fQJy8j+-}{yyB&a9WsdAb{B~9Hm&w?1s?0%cmZxTo zgXnQ$$eJ}c3QIsJ-@6fvpO)`oAJNeJMXBf?`_+o#&|5z#1b}Y3$e!7aEoBY60BgwhTRwi#fk!UG=hjIDpJJ8`^*g{=H55nKX<(G z#v9`}#?85ju=lTgzu&diTyxI#_;Kdd@W8=SjVAHw{`jL~)EmK#P+pV=2240zfQ~<( zbbAujN6}#+$WrGqQw=cpP7Yu%rGV zRbQlA>gis^slOZ!#nPEGK?i^S7rAOrp1?TrTV6wmN?td{>uQLD4JgJX$k<&?BsAA| zp5(b3&d0(gwh#@Tp9R5>`?49;2DSbn^=e6|Xco5L_cEPyhO~hZHvY z#yaAcJtraS>pY{8}~X%mN3i-7^*t>TK9VW_xD7HktgclE~H!r>gv8a=|Ud+ zIJ;n;&4sV+D1>rTpC5>;d=$vZW6U|aZ2O_3vb@B8B}}6YjjOhM`Le%Hg~)Hf9k&Qa zA=QOpw&TlU<*XbXuhd+<9vYA)7(ZUk=@3n-5SB|xu&|krUQ^7=Nnd$;XL)rf>XLUd z3`WHKQ;UgzcEs4Rn_xk%wG2fYJj7nODVvTSeK#x#JW70_s0=(LBq=(MCgrDNc__U3 zF>n0h<{*#`0Xi;~838?*tBa*hlNu}Ha-o6+Ygt=IHg3zY1q+YS;XmD)TE+Z}x&Lq{ zpRy=j$NA#ZiEt)?7!sddlK!1GI{V?HhnK-K4@cF{XZ~^YU}P_Tii$H-L?LiMN4t}F z^;p_X?ThI2;*y{KDvT^FmfBnAv+2Y!ti?p?GX{jw(XN3pPsxLn|HqVn{5;Zao8QTa zZ-7XiJ$Fu-&X^@5C*M705HCXblc=RzrK-*QbE+ERo_jC_x0C1Ubi_TQUWpP*x2~S4 zAq{|jXORjBpaG`UvUc)J;>8 zKaMM+bNNV$`Dg4?GcVNHy^Zy%2t+1NOD^;=Rn=~P|I$&9L}-Hj23&Tn)FtzyA3MKi z{%5-&l*zQcf>3&_kZZ9Tpu7p0G$LVlloIp;;MrPy;`3)y|5c8eB=0BQ8C zH}MA9>{U6PD>|0l)1FHJkXp=jbpOkNCV3Ye* zJDqU-FTM);dA~TksQPjANih5Aimv=+o!}eV5ZG7}NBT0kYE6sbcAs*?$ZH6aC0YiB zhqpUD(*-jHj==5#&Dd1YWT`+ESaDDx|apSyJ{ds#$dTeIxZ!nGn;f%%WCH*HqtBvRxWYSz4S z)cmp{OMT*Io!Z4V>XIGUYizgu#|M}n`|w7eK7CJ}&%umQ#|EzW%(s9il#;mUWLoam z_`N9qUwsSsfilH1$L0Qg?mrW{f4VI5zx#Io%wU24()=^e1pf6e*OA;`b}jWk`sF2= z61f%NL>bhcJA2E2&mJcuv(e( zik064s0J_in%iZV%Ch)@f0Xd_>FO=0&eze>UvoLg&b`@v1j3i7va#xS$)y73-i-Z> zKt76o55V>^`ac`!4(b8 zf!=`s2WTRbE&LNANQmy)9@yjn1B797_a^<=vw96_0bgbgAE?X7==NRIgaEP1Kw;8F zqyPjkc`;s?#pftJEXSSj{89hF9~@Kz%i3S0hUhO!8pgn&5li2zZOfOZ;A#^4zcWVT zcvtUXexpd`P?KsIVF&d12Npt^Pc7CGgvu1m7xh1f!{;{IjKJ-B17qC4l4}}olSloR zUz<|je~)`yeM>Z+FvTU@81bH%Z3>3EvUR|>XzBrM2A!DZ!mBoG#&RD$|ID(vee~y0 zuZ*WoO35Mm%R+#iG$NHs-edrA_i`TuJqn$AaIgV3y70^LJEllu;T2Jh#@N?izr6YG zv!!9x;(U{hb67vyKxSzu?HBJI=yB`O%coC2+!)ueT(`ZzvfP7dmFJL=v3J1;r>pD+z+M)e_?&s@wTY*_t&EUta-;SwQ54-5o1GAtG9sRRVq061@rjNpFR6M zACAA~*652l@EA?OYFg#xS$Q9mm52Zma83L#g-GP=A%M`YA~S%t)Z{bJ4;4=?Ae{J{ zO!vp%AQCv@>c9UyI5BX_v@^6DW01#i%R7ic(pp!ydKoV!2)S(k%8n}=w$p%CQj0CX z2O_BOKLKiOlJ$oWo}S(c+ZxlX@#mY(!efk?CJWWa6bx84cmDjXM^mXa4i>47%GQPA zN%RqIy65+f#x?F2;#^PZP^aInJ*|+(63)BM@TO)a>>*gAdsBEkD@KO$!hc<~=myCe z#tBA1%Wc#nZnyU*cQ{k7Fkg`_BVrB)H(;kHo@iyqs44 z{_{T*oBh8lkN@+{|KA`P|D#v+Hw>aRTYeH$QBX;v38a*Q%Rlu9+rJj@H2uB3721NG z9!Z%!YZhJ10b?RG1R1#=%JjoufBn^jHx5(6#+E$uV<0J-VwpyWpp0A{z}VbE};rD8}srK~4^i zlnq~Zv0oBR4}$|g1%yJzNeCoIG*e|ElohrbQS!Q|ZeZ=_u?`R%gR)>}?0PZ^VWjNt zuvhi{s!aTh-{Y~9E=gZTCdcA`vO4v>C=Il528DZkk092G^c=ZagQ)i$eLKyfbg8-s zhqopbJyq%!m8#P;`5AK z(!-^+JpMWh|uuBzaubhrTD~jLCUmhr>6)#UG zw>dDAakrC+*pU>JV-zzxr_-$l=LSsTK+t7cej4ljrvBiZz7h@DF&P$scM9N0u;1- zUA^zB)TCAr1k69Luwsf{tnJLTCH^NLJshaPwyW;3SWHXy&vV%cWe4<_j@Nws7nBS%Si ztMyqPf;0Igf3|OKCzhy=ph8w%d)UtpyPuPo|_HZu_`vJ4U^g5a8yNWk!2Eg zLqf=Cs8z6=x<97b$M>%tCq28_=u&E7`yOc`J);l~h#(7cg3Jt(Qp2beU#TEG8Fenv zH1q@O+_`J-q!j1lASQhJt=p~BTKv7#6FR`A>_q`zZ|S;q>+~|nOZw4R*BN;~ zPJbOqky&Dr<-AX7E2;=+wYKqqgWKXo z4{6h|`YBK5M0+&FJ=1bwu+K=RiZf5&|Iw<%`33PQ)7DoD`BWxPiAJ?8;~2HS0vdh$ zZD7Q_l0wwAj6f~-EusXJqEl>t1{en1NN6rg3n(gp1fl03xa>?FVCsUsEMo?T5A(Kv zJZJ7)X3e$le{AwzxNGq8%;~+3gmg@>(GjS&3dkTj`ZK<(qEEtPF)LZxD^{6Gwab z^(j8(H)oT{vuA_I0Ups=t>Tj+Yv*ll`ylzzSlW1Wtlq`{YLhFrzwxnl2Oq;Wyv};4 zZY1pX=8MY!0@I~-=M3@i_Ks7iOIMOVkt8&26NdOByRwejh--{#|RB56g} zOQHlqfV)4qUIEAFTr`F3{vuzii2>2W(R(^fdPQDw==&|SpXZM&F62jtCaF06WoIGw zY6^D?ps*_jB_0#roNTcHqlM+EQ~p$o#(0wOJO09lMw01Jad|C*cS^tYR|pxPOI42E zAKut(I2yOUD3|{Vpi@(X3gVXIu^Aj;%0JYSg)koyPWHBg zrz@?-oUa=9U%uwq{o@0uytRR=8|iCBR~&y#ywM+~do(8n{I+J@%3X9>ljs;gLs6ZA-`OFUH&n|9M0dQZO9OPb2_d4M$gm3mv(&$ zghrErxFd3yW3!xi0%@>3F5pxrpIDXOYPwXz23io^$SuY^Mn)9Ev+Gair$&of=p8sZ zz>%wS3K*l|Imn4ozYaJgyE7RsVu9C7%y`MPsfVHJw*M)$@?;MXKq=+adPm-tEN@8M z@=YTbz6DGDfGl__faUWWwOlU&W-gjp`1^=~xg7Qqd8w0!FLEh&s4G&!6ON+i6tCPO z=M))|+D9_FAkNbtv7HO`rmKWF(d%Y5tX#hQs_m=qCRE9JGhfmkSeD5oS^Z%Cpn?Fa!CiM2^QC8JxC>_8nMj|IN6E)2IC^j@pvAeWePNtcH$dZ2@lonv_Gd9 zRh#Rvps}N>?*o16;P^_DyKuLGhPiX+CS&#J^Au>hW}*zZSnLCokv?BtFim!BMpFNN zM_!jWS0yxR^rhe6o(H$)rnk#c_(&&b*6cN~p--+$9Li>Wr|T&TdJLOghTxyY*`T>@ z7cvL`d=a1jw+YJ~nN8ZVe6cT|vpRELQgr5ZhbV`Z%lmsr!`{7x%bOb>p0b zO3|R!kv8T4V%3>EWvv(yWQ)Sx*uKsU^0yr~0}|LL<3#da`EBK(0ZRv6zxn7Vi#2ay zFhcZ^)e4e=v8P-cI+#FWe(b;ey1#X=-`(P7%L@|SO;7r(xjLhru?&@T0y?>MuZCYO z(sVFBQO%Wy3X-<6v?@H{06%d^Bf zZ$!)_NMU=cam7~S&2BipwmUIWa9$dMP7Be`gcmAGknGQ3v|uH+n^n=(gy-r8JzBVYz2TB7zku1I zcV>%#VK=MV zf=Pkx)V#n*oL)$cxejtv`3-}uN$$?$ zc?zxz`!j|cPy*<4x1_75G-+PxPkPCNBXeBA$ljRNO{M!PY}y#jVQVmQcTOzd!qqJt z#eNn7kETn;yeYU>AGr7dgi(BN0D~A>HMGe0*wKV|vNEbwtBw0BEw-sPd7amlP`5`7 z%#P+3R(IG`el=xqjg)aEHt^DKj0O;QehoFiNuK!d3V{q~HwLY_<}hAFVDg zv{OmF!m`I%+)?LFri6Iyt3>0c55c9cAgf$r_kIpuc~^kg(r6Y8Sh;mPPD%)~ZhPz` z&(|@$y77o2?V2RG6R56}dsa8nRt@mVxqdz8!_mv4(iKiEx44tsNtMWGbcCL}=uW&Z zaXYM8lbp2u2&MHh%5UF$Xzjtx$NxB(xyQ2eZ?o5WI9OM;nV+teKU}3pnv84uo0{Tc zqUjZ59`pEi1`e6wBFhW3V_b&FZuJHo=?l5dnc322&4_Ql0{eE(w5lWi8uVN}EyH$3 z@J`K63{>Ai(rPk4Rd?v3YZ)~~0wI091~&^#nTw>0peJj6JtXqF13aW7G|nZ7BbD6- zATEMBDQ4Y`b7%-3;^l5*gW_Rw2{T&{e*$$1G4D_X)Q)>A0MpNk8TXwCr3AtIQ<5xo zE6y`39>XZ{E%oemTgOgr&*s*9Ztis5THG_g@3bT)pXhKT=+!Wyjn`%q^MUOFInib3 z5F9L|2H`Ba^V=Y+Gw5QRm;f8H7ySh_P!E$-#zO8X!g}$PCIRG2!~x6pVCSQ!I@~)v zu>)pnQeYU8@#nb#&Xe(b(FWVFS6aI61(w*VOu=w|PaET6&Zs%2M+f@kkJkKKJ)ASI zb9SN&XV*Cgq!&yWDP1@3`*`PFJXvFtOw0gvY{+Uuwe0ifJ3+!~TU$Tb72-R7&7u|WL-Y{i`w3xbd(&*7_AIR%NR-ks>lO#N>*<60?mU2 zyPT5FaX&H+4!%m{qaZrcb_}sBzas-7xclzoBVSkWId2?ryX#nYGckTjd44YYGSw26 z{P^#=miKvi?)mH%?p0b6kI{i2Alop5HN=0JO~u@#3c!c6k-7DxoJNB~AY=Nmh-uAb zAgmq@lRk5Fd53ddTdO zgBL#>f|XFGY16(kW6qARR8ZJ^Z{`B`-Q1wj4cadmT9t{Gd)X%}a z>Qa{Nv+uS}pS9L;X|<4v^IGR#Q(7JKVE_d7l#E)fEP*s_bKdvC*8#)Qjp&;+1qI@B zM@>$lyvPvjK?3I-Cjf0+{E3~F?WW#LuJ0T1jv7+Vub_c)3Mx{xXdjTUGLks(8u}>?=YnTl*>Q-ju$OSZyK15ET z3dAYs3PPC}CS-9h7uyh{Cmv7?d26iGI{uHhHpM3h4-YckJ$i`K;m@f%v(M zHt$j`dboqyS99B?MFk}UuzWj{9chyUu!>4UZ{jP zGrtiok!H1wn8r(|n7H3Tlyc>sbDf^UKQ7g*2Y3a^vvW7`?ZrLSu*t4qtc^$Hxk}m~#CDQi}TiiX0vT1~1GQeB2 zh!iA`zdBb@Ar&6<>I{HlWy_c+j+{U4=lzwB^;%Qvgh!W_xeP(QqDUjgukxd+fIgfu zGHCR4YH$sdH%3!^)nST3l+t+_)ET2e)|)sjzr$c5#kOm5brL>-LzzEW3uY;yvH04J z=H&{Gc>Fqy(Nq=sT(TFrCn&%-AAi3t^dY(; z9CSC`NCl^xHKNFxetPBt&3LFq&w-t*? zXs%owe0lKXhXn;4VWP5241ZvaoBL{@H{jVjQS({0{>L?E*T}O7K`0-+E{mA4aN$DH z4cQhK@O$d;ne`eCsGxM`eK)|MGIBn~=UhMeF{445TD?08vA{XQ^}t9XZMwC>g$y^^C% z!D{uio(Y|Vhb?ENJ9k>2X_h$>gA4N%ofUVFEHn${bHB^VQ@==CopeD7HsyPi)02D7UU~calvCu36H^?eq z=docs+!?twjJQNp)Bu?2n1U$flFJdt<``I9LYY;?Y5ba<^1w^io>**#(M^?y+Lz<+!!I?JrF@_4BUoZZ<2Xo?jakeaz);vp$}Y z9+7uk2k*Vnt={|;|CYmT9@rPRSvIWpuisgJ7iqO;=FGz_PxkO@zwh3^Uc~he4e@yq zb|vX**x{i0SEiUPrz}z*^=?nkG~-4e$9tKpBP#pUjoQn7T7%yfEnD|0^^BZa*m#sc zgQ>bY9?Z&az4Ws+xna-I@m&k^o~{z7b7F_NViTe=r}qwaa5x z=l;yfS^xRs|2*o?*2{ku*MGLt|J4J;qiVHk9l!z+ARlU#o9{->S80R-m5#&!OLHdS zJMHJ*?TaG~d9t|0eLpA$l_x*Qhg?$3G{_UXVy>CjtbLmaumM!~%4QCvQe0uXSAUcu z!9JSzGkg3O#^g&oN((_6MRGfnI+UQJ$)asyat)HN*2}^?^+?EC1aBa2N2X=(qu9~- z2$9!Go!xHj+YBLnClGRf%TneG=(?x(@0LpJ_*~gQV-vRn0$m+UNN6zx8f2OZpt`@g z!TjTUE+Vt=V#E=oOu=5D1<{n*F}SO!J3;naaQpie`Bk5NA91a`8BG zwCj1ZXP3i^$&nmP1iG5EQ0pdDZ02T2j@|klNp$L%TsuqG;}^qcU$mLk)}4y?%p~*0 zmc|RGVX=@73H_)RoWTKagnNzx4{LfywuAITcd!)TccdQ7QXPLVqS|-g!3f^L_1||I ze5RqGQKx#3YOi^|Yut*hjf-{IT7T>IzTrEvYXlB0cf{kBf0qF7%biyRhxAJJ$oi>w zn*|1AW5ecwQ9Y?fhey?n4}IS<_VuHdf#p(cR#b@y|Ko=%U!9wJxnZnl{kBUo-nYD5 z{oK?Zi4ha)H;VPF-FAs*trzFs=8>6cVI~ba++O~`A|vRJ?7`=JZ=2KC9FxVGz}jI|e*J{tJ%(4@XIDkYU1X16uXV{QEN zyX|6^=A5bTnNa{y`N?B7!u)eD4e1 zx5~ZasEPFx&;Q_!t1(%7>c>uVZ>)Psq5;e_3)^+qMsrz9~}l zT&p4l&e@p%x9GXIA0o~_wa9j%D{7Dr_odADba?#(0%|bNlSb&%!JTr`tn5?VOXH=~ zp&x@rm@_S_wj2E?RHU$8Zm_hS-0;qaM(pYu$<7rc5C8;?%>A|4sI{$c(m6bs)Fi2R z$X-q^t9N4MTJh&EBjZ@zyZ{7JzE+kPmmfw{X5A<^&8_$;B z$FX#3nVWOJ=PTb_VcNJuU9@O)3y?>Few*Xv@|fca*Ooj0_c^ZdhAFo2VH zn+h2gh}0|V_uoRa!;}UlIQ+EPK)pM z6P|FT&AW?F%bl{DNOi3CCKcqKRz)UUxyuQSM_epeS}5*ZA&?IG3jS%0CZD(6?r0u2XEoq7auQaVjk!BdPVxk}gzT~aV1ocNO>-<+O3zdrj# z>CqbVRC(%?sxiBV3<2wk)95x4;`Bv_7p4xQ>Fr&-Le?|Ie z?1_k>bh_C&YBtFpB|ZHHjde}+2&tBT%V8>2m^qthU~W@B01+i<@}Gr6hUK$TN+Rj*!m?)nCxR&Ahs_X7EAz|&kxFxZGOp-3Hk>~>*_ljwuI>ylqVpN&? zq;6JM-e|Bg#Lnt+f*r*RvW9-qkGoKu6v36AsQh=-_oveW!Rq4EUwp?%0ISD?#taSI zRzHgeLFFI~1u!(6pbH(zH>PYaD~UQ#SzZMU>&r+fF%4xf407wm7zRXHaZi7-57MX# zwy}I<5v4kJv;X-u&ZTaAqp}S~-jx3K+pVm#5Mm+W&+G?3spF~A^M;B(3z5YcD@Is6 z9jZQT)4o=|g1!6qU;SZu7-#QqD}D33^Etwm^ISS{Hk%@jpa$B}E@V(uJFtncRyF)a z(Rr?Jq``F!jG)1Sf9Ef<#h$)e!5HXb@Seli>>Aq0ThZkqA1vbFu`GPF67`l_3^y3O<> zW1FuzGleV$;->^D{$C$D<5;7#gBFe~vtW11Ih#YFByYQ=jKN?<4G+5|n(3Xk?Sg7~ zmUs+Hu+E?Pwrm3xQ!kxMa|1@5>C){%m7v2_XeBs8lh!a)EN}mukjfL;cWbN;UJ=to z4H`1-)!gXl=(=5sChR0pl@qcgCJekeqXR+XL69}~!VvDp;fcd&Z@c)Hwr|&%#w80G z1%12QM<#_-_IcJw_aD1!5S#;Nfw1OA3X?b6_JbfP29fy*n6iJKC)@5r8JBjgMEm3F z)bYW1c_b1gG^tVq@`raZ{7^$s+EYBU!TTGWEYa>~Q+P8+s0V%dymznm?{6y3ALm{l zR-_X?r82EyWb2;VwfBK+6pU7f_ukvm;Q)^0%$*zMD|nea8$@%1NFJ+31)cXefkBxm z`0(MwK5L6Ll-tdRz+Sh?lme2IVrFr*k zy7VseUp2yy=w&?Ar3mNra+uQiUX3?85C2||4>D|1rpdo+vmZ3Ibg7;Xyx?Fe4dAu{ zpJ$^EcldXB{nED7M;a*fA<%hd{TjpT8;=084>~#H-i2-UyXa~;kdCT;xVEXl83@?uK5bGX^PV&*!^qo`y^h+a0cLW|l7eYZ=&rmG^{xzt8T<-F!v9TXU%f zxV(9C(7t*#WW_}5m1hvvvYuaS8qX^XbQXJ)EXB|`)Sz?uyApjQkrf+;ed3Nv-NVBp znnO?46bqLZ@ZJrP!xT(Y_fFA&;Z@KJG+nlRBz3Z{=KaZ=p8nCO_}DLE!^*EZP;$f9 zk&%(+mqj0K{>()q)ioFG8j@_$KKX69+fnQ7--L0TOYJHeDz>L*!qF7i!9vcms_K`e zIuj{_dIp&-{%sOzQX9jhYE?o$nz;vR*LZno8tJGeFg0%6xYCk9yugsndc5(v!=q1! zzjT5A7(g(m=Fx~TKOd8IWIYH7JU7JznSSbDi#z65p6bBs)}YvDNp$cwkanJrii-N(-~RHwajC9QL8M{C(xH;`(5A&Y za0h+Jq+j&vVl|$=cMn;H0g5x_j)6;9FZ1`brpp?8W457w7VV9q%kzg1D>qNxjikTL zqr&whD1K>9@2OI;<%A6yl%bc0j__Duub*v|m*FVUy-k-)^ zzBu-J3DXg9vbXUZ8Ibr>lc*7tNTEe$<$t)&`~EUcG4;@Bzot;t%e0|G)pjN-DV-0- zLo+_o1wVT4*TSTw&X}E-CPb4sEnJ@8X8E1)0%scHds=tq%$dBK3(qF|)*^z6%PN*n z<}hFhB~{O7#zT9;WoTTxRu5i;-5RUPRw@~}kU+%&AqhZKuj#b&+s{hRp%6MW-6C}q zWX;TYcSz?s*C>63Y2a{tbt5)h-0%v1gYZ(iN|o{74}4#kjJFjhtVZTZKg&KwFVG|4 z!%E+Gy&j+Fg4=iIFzN|?)zEUa2Om4^bj|nZpG>|XSl{Yen6NE^w8R&rPbu*^9}fi& z7+`a8tQ7d@ztwjW;y&8$eHqdIcTw8O(+89(=h4vh|a#$c0lm!KB#2PgV^<4QOeC12{GqvsPG*V2%+(MTWN?6dA z*t6G9L!y)*@PB1%e(^C#yaF zZjEg&kGM@<_Lt>bz<&+EiY_}ajz*s*)awvoa!Cl+r88^Z(3`s#zy*L8Ro%$huzY!` zlf|!;c(dELeH5<65@IK4zGXu$>YbJs^%-Wgj|4JPQWO6gte?b!H0Uk4_T{v(Qrnx zmz_7O^M;PbM(EfJ#E1fXT}OE?=X{9IInQ&1&k*eAn?I4#pdVOg_46D3EJ`un4Qp;q*`6El`LiD|xh;sOA*cgr|N$za4er=H{M zZ@Lt=_Je|Lvt(i6F=NK4T$JOR#%7R4XNLx?@TUc zI}96KM~;Z6pjwsC8r7>&yCWH`Pm>>7^>!l#O0c_6SxQ)f(1xnWd8gK$hGZ)aMts=0I3nt5RqavI7bo*8n zrMRZn=y&#ugqi*&dv4>SnMQj4X*u69Gu}KbB+fWhvbn8bJ%?OdW(#$&*M=`|>ewjX zsp)g46e#^AFq_}WlWphH_j6sq2A9vNggVb6tZS6E&FigZ-B|alUWl|Am1n#eN@8Y) ztU~T?UpWC7PYeYSEgM$%_;p7M{ndjYWromPZXjJkgSi841a#IF!2{;bfncCjDk{=h zqZvQPHpc6Ri~Me&><}m0aKD!gnJ+9$wL*2Er7jZm|5y*2qOU zvrkt!IdP z(#^Q8BGtk>dsePvzD49FYG*ieRr#n&pFWC1w;IJn$T0Km`%bea^Z7~z43zOWa9gcbt!_VEnbi%3FMW&6 z6wV zw9VUXwsv-jFlWhbcCeduc0=F~UGgsr`xBk?b{+Uvk-mEAjF1*OXn!_fMg29GwkVk| zQS$L|bk5t^G0NwZ^GVB4vt|nUjnqM0=T=UT*0v*YDyR7)HK%p1|3y~N&~oxM2ER@h zmzPYC1ny+YyEc?f03d9we^(B^=p$y4{AzMOW}OtTj(sBKIwk%X2(6sU%iFwtQf+?3 zSa>UdSYYmDH-BkiHFG0O3j3-QhYiZlt9quP|FUHoB&z|FYW;~1NHBDKL!EwWO+d!268P<;T|JI=-lLG-X94K$TKvtR{l1kAM-7mVT5pR?4+jG^;neQ zgnK0J$51-_=+Qe@q)HMnie$!B{e;vKp`xAab3&Ou|0E+awCSZ`z+c*l--yBkoH&1- z94gHA1iL#kvFOoBDUbJ*NF??=i`%?)s}61om)`thBBg+o0n9i907xJ5_zoot1jjlY zW;%#z!SoqDdi0VZL$E6#va|k4JUreXmO`J6b9z?6%wah2{TOh(dCvQoqu%4^zYE^F zRe}s5y&}}Z8Ix~N;lfo)%4c!P&HV5UMv3froa$~HdxIHUUe@$bw#B5DWl5DWjhEv;V-3i+9CigY zA)>&z$6|9$r6rv^cC=<>5mn{B-6LOiCV#s|Zzy?>cOpF!MJHkxIP!xfSDu665X`iE zRlEZ>CO&(%HspPHFCN#s^?iEx)@%XULJ@#g(y12X8$W-qn(;eK={1Ie$QZdM9YCMShpbX zGAvVwr*hJGbsc=l;x#=CZPd-(OEJU1Va{NLWxnr4yD{bFX|Sl7ugpld8qgkOuC8vI zwmsOdv`|ZJ(;N=of=})#nY2V(kQWJ~QaZAGgjCsltUQfuR%wam3o|i8ZN)0nUjE#^ zLkA#Hzen#-R>eHb7%}7fPMvn}4xirK``htnRqw4XcZ(@eYHN_;ra)Up0~t*nq}G!B zHrRycU%fV`53cjtOn=)lbZ}2T{wvE?eFf0t(~;2gRA6wS%9HJda7eny1XP-rie2X6&9hhBbdX z+;gkls23f5d#-^kuAPV0wYEmV(ug-V@Lm`pY@VT7jT+}3PUhq)X6)3rZ{L83_n@)! z48IWy&k_GT(`~6v8XC%@X*%yQjsQ5%@Cd}^vQs8J?s6cJjnw7H+6bHU9C3Rwxc&ST z=4+00XbcqUedk^3&An%=q3krGZ!-T8I2{+JazAq-@K+~n9v1-+K7L9tb1macCbz+l zLMpIBleS|pZulk%#f=p8=gqU>RLK<`fYty&+jvhj#t4asO8Y-W5Uc1gU2&MJKH zk-g;OE=yL_AJkvXd)Vo;3RgYy0mMkg!D2m?ouGMnm&;L#JK1XKo;r*MnJ93~) zM*v9W?9@7~p0zFy?#_wA&Gf*29P0Qo!uHyA>ljdGpu~)ieTX1EeCl9xKuzoI;PA`z z`&PNkC(W19;U)nM`?`3X!Xv=7@?h%!df>nB?w%&cscxi{je7N(hW{XwYMBrMW%z}l zdv{L{TWZIA_yr5j#~df+-pHd%W|wMm8hXMq%x}=ZX2kJSPnTmp7{&mDP~mGP8Y?qt z!Z_GWreJneB+fK#ydQAU=maqIeab>q$wOnt^TS=t@!eGq`5Mk+xB;l7_6|>b3!#2A z-G_>Or!(v=aq7NGRL}H-DQeNJUpNIA$)~rA@3&Rt?s?G z*E^M%n@Fgl2xFT5OpC!BKdw@|^lTD)%dUXy9bvQH#@lb%vWE?U0l&_!_P99ycLhYu zJ=G4yaYkgdr4A-6_pFVYIV<5fN*Eo~8lJ&v@NE1qZQneYJ+Q4KT!`vMu#zd?wkcu9 ze^SC}u<>s)u4Z2R<;>McgXbp#R;_>^JaYQxN|!&HNpGF~TSh0i{9%3k&xy)P#f2UgL5SxEw=V5YIS zZ{Ttp?}jX+)|V~OX*Inv7{%B~CA$`tVBp<6^i|~hFHWOpg;=wN!b}PTqR%g6`Xj<_ z`BTwzB#iQDKqu9BYi5y6h;!A3Wurgu+VXJ=bs~f7hFA_yq9LbyRTM|h$xdu_Cj~&V zF26{JF`4w@{Pknz+q+v7=w?rjjc2Nr#wJSYV)r5adgjh5)ss0FIidR<%sO)KZab8p z2BZ2DY~QrH%>Kj47-EAVacrs;`H0ACz(g6ePNUA(dN<@6togpC{p1ku+^{^jd&OK^ z8+&saXzq|iU7YcOM~@yQ+cV?GfrcON9dUw{(2JiTUD(Brf67r8OBS^6_$YEu#7@qWAvtdY$LwxFsFGIzD2^N@)}eg@#gHYI zm^URJAmh;z(JfQ+bzF$2`l1miG$Y2F(Gn=<|vgWltikP(K}i zx1%6kTT4|U~z6E zB5`xe@nH>`mUf4*6X%abKc zi(m%Kj}RFkH`-U%;;MSNfw#ssHizo(;z{(5HA8hfojO`*Jy$gzLF}rjm4utim5)$mRzWQ;ufve|AjEP7|k%^pz9wt@n^z z*Uv34xlGPhWp=RldIDx;dQxdiibSm}_N(SLGgo|%tgZRF9YoGs_rm}{-$I;{AK1H} zO3dV{n_8XUf^HE6-FYT3c&QGW>JQJ5-6vw||J<$+Z|D&wb(;P4rcOM^8Zc)9=Z noi%^Of219sJ&^waii_g<$-?N5@_5U1$9-H|G^^ literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb200/gdn2_20260813.csv b/benchmark/linear_attention/results/gdn2/gb200/gdn2_20260813.csv new file mode 100644 index 000000000..6c4b28d45 --- /dev/null +++ b/benchmark/linear_attention/results/gdn2/gb200/gdn2_20260813.csv @@ -0,0 +1,41 @@ +case_tag,backend,variant,batch_size,seqlen,num_q_heads,num_kv_heads,head_dim,fwd_ms,bwd_ms,fwd_tflops,bwd_tflops,max_diff,num_iters,fwd_bw,bwd_bw +gdn2_h64,cudnn,gdn2,1,2048,64,64,128,0.138,0.000,82,0,0.000000,20,1.95,0.00 +gdn2_h64,fla,gdn2,1,2048,64,64,128,0.456,0.000,25,0,0.000000,20,0.59,0.00 +gdn2_h64_bwd,cudnn,gdn2,1,2048,64,64,128,0.138,0.935,82,36,0.000000,20,1.95,1.33 +gdn2_h64_bwd,fla,gdn2,1,2048,64,64,128,0.458,1.561,25,22,0.000000,20,0.59,0.80 +gdn2_h64,cudnn,gdn2,1,4096,64,64,128,0.245,0.000,92,0,0.000000,20,2.19,0.00 +gdn2_h64,fla,gdn2,1,4096,64,64,128,0.870,0.000,26,0,0.000000,20,0.62,0.00 +gdn2_h64_bwd,cudnn,gdn2,1,4096,64,64,128,0.244,1.837,92,37,0.000000,20,2.20,1.35 +gdn2_h64_bwd,fla,gdn2,1,4096,64,64,128,0.870,3.073,26,22,0.000000,20,0.62,0.81 +gdn2_h64,cudnn,gdn2,1,8192,64,64,128,0.474,0.000,95,0,0.000000,20,2.27,0.00 +gdn2_h64,fla,gdn2,1,8192,64,64,128,1.656,0.000,27,0,0.000000,20,0.65,0.00 +gdn2_h64_bwd,cudnn,gdn2,1,8192,64,64,128,0.474,3.090,95,44,0.000000,20,2.27,1.61 +gdn2_h64_bwd,fla,gdn2,1,8192,64,64,128,1.658,6.022,27,22,0.000000,20,0.65,0.82 +gdn2_h64,cudnn,gdn2,1,16384,64,64,128,0.900,0.000,100,0,0.000000,20,2.39,0.00 +gdn2_h64,fla,gdn2,1,16384,64,64,128,3.266,0.000,28,0,0.000000,20,0.66,0.00 +gdn2_h64_bwd,cudnn,gdn2,1,16384,64,64,128,0.900,6.972,100,39,0.000000,20,2.39,1.42 +gdn2_h64_bwd,fla,gdn2,1,16384,64,64,128,3.267,11.986,28,23,0.000000,20,0.66,0.83 +gdn2_h64,cudnn,gdn2,1,32768,64,64,128,1.751,0.000,103,0,0.000000,20,2.45,0.00 +gdn2_h64,fla,gdn2,1,32768,64,64,128,6.434,0.000,28,0,0.000000,20,0.67,0.00 +gdn2_h64_bwd,cudnn,gdn2,1,32768,64,64,128,1.752,12.255,103,44,0.000000,20,2.45,1.62 +gdn2_h64_bwd,fla,gdn2,1,32768,64,64,128,6.450,23.862,28,23,0.000000,20,0.67,0.83 +gdn2_h64,cudnn,gdn2,2,2048,64,64,128,0.136,0.000,166,0,0.000000,20,3.96,0.00 +gdn2_h64,fla,gdn2,2,2048,64,64,128,0.863,0.000,26,0,0.000000,20,0.62,0.00 +gdn2_h64_bwd,cudnn,gdn2,2,2048,64,64,128,0.135,1.031,167,66,0.000000,20,3.97,2.41 +gdn2_h64_bwd,fla,gdn2,2,2048,64,64,128,0.862,3.055,26,22,0.000000,20,0.62,0.81 +gdn2_h64,cudnn,gdn2,2,4096,64,64,128,0.243,0.000,185,0,0.000000,20,4.41,0.00 +gdn2_h64,fla,gdn2,2,4096,64,64,128,1.648,0.000,27,0,0.000000,20,0.65,0.00 +gdn2_h64_bwd,cudnn,gdn2,2,4096,64,64,128,0.244,2.032,185,67,0.000000,20,4.40,2.44 +gdn2_h64_bwd,fla,gdn2,2,4096,64,64,128,1.651,6.010,27,23,0.000000,20,0.65,0.83 +gdn2_h64,cudnn,gdn2,2,8192,64,64,128,0.460,0.000,196,0,0.000000,20,4.67,0.00 +gdn2_h64,fla,gdn2,2,8192,64,64,128,3.254,0.000,28,0,0.000000,20,0.66,0.00 +gdn2_h64_bwd,cudnn,gdn2,2,8192,64,64,128,0.460,4.039,196,67,0.000000,20,4.66,2.46 +gdn2_h64_bwd,fla,gdn2,2,8192,64,64,128,3.261,11.939,28,23,0.000000,20,0.66,0.83 +gdn2_h64,cudnn,gdn2,2,16384,64,64,128,0.894,0.000,202,0,0.000000,20,4.81,0.00 +gdn2_h64,fla,gdn2,2,16384,64,64,128,6.433,0.000,28,0,0.000000,20,0.67,0.00 +gdn2_h64_bwd,cudnn,gdn2,2,16384,64,64,128,0.894,8.072,202,67,0.000000,20,4.80,2.46 +gdn2_h64_bwd,fla,gdn2,2,16384,64,64,128,6.427,23.807,28,23,0.000000,20,0.67,0.83 +gdn2_h64,cudnn,gdn2,2,32768,64,64,128,1.768,0.000,204,0,0.000000,20,4.86,0.00 +gdn2_h64,fla,gdn2,2,32768,64,64,128,12.787,0.000,28,0,0.000000,20,0.67,0.00 +gdn2_h64_bwd,cudnn,gdn2,2,32768,64,64,128,1.768,16.167,204,67,0.000000,20,4.86,2.46 +gdn2_h64_bwd,fla,gdn2,2,32768,64,64,128,12.789,47.523,28,23,0.000000,20,0.67,0.84 diff --git a/benchmark/linear_attention/results/gdn2/gb200/gdn2_b1_bw.png b/benchmark/linear_attention/results/gdn2/gb200/gdn2_b1_bw.png new file mode 100644 index 0000000000000000000000000000000000000000..0d45621433d47ce897226bced3af87a272815489 GIT binary patch literal 105374 zcmeFZhdbBr|2O1l&owGNl2)aj3S{VtE@yqXb_cMiiA{Fga}O}dykAr zR`$9dCw;E_ciqSJ2i(Vf+>Y<}=yT{DuW_Ew=VLujA5D#YEX-?|DT-n_uwO-+qL?;O z6rB?jBfhh1Vuc?5kiM*{fBDFTvzHx=E}fwc8C||;dEv6vFQO7at zccX*;k*@kh0~eUM3u!!VGWiPnGHZlo^hQ`!KDVo|YJ6PxHZ`@X?)lM#m8!jK?OO90 zBoz%>6nG>xb?E7XSw1<n91FnqpaN0>awY9fj&&yM|+*`#m(pxRsbJ5q!)02LBu$WKq z=yMK%Ll1Q-S~Zug6g_q8&6}Nnd#hPXN=j6Dm$|5LbPYu8xtjm&TVh;6v}{|izVqOk z#}^0QdfmKvbN?P6ekLWYMT&UBI$_x^Q!BYd4Dw!^l@4@%;|_iJFu$mX(Vu(wjd$8eN7uYI3oJKb_o09K^eOqZmX?;9>DDK8Ug(Pzuaf(HzU_mL;E|`diY5<+h^T67 zv)84b_wM}mC?Rq9=cg47*-k6&+_`hp&rkVRd2DM>k8hDbx3X|>=c;=WDH@^MY0iAP zU-`VTr!qmzb=ol_Bco-suimL?w#&Fa&B}3nU>$Y3EJ`s>J60`iBxKu(8@T% zdMZ9Syo-5ab?az?uBxM>r0kwOV$S0OO1Ct3x3#sk{QT+p_wQv{*J=F@{h^^DB^4D< zHJLqo_5=xP(H?!Sqd468&FI{@cpL7AljEAH7L{eWZf?QF9hap3G-O`);}UJFOwgll zKO569FyL`^aVga2;pP1vqq@YuD^uj;+ZMbsN$2tHZ|(aYCRlOqe(T6P{5UFbf_;-3 zznIwaMay^;N+b5zHMwb(S5C>NbM&u1RD4D^{X%n*+uY0|Ds<;LcC`?ZlTj(Z?zZvd zockfLboGuWaRovgnR~b_F8%G+GRVBdK(+k-eam%b@|UcxmCK7{6W4%#+>*h&E1ZFR z@umdwFrXb)yol`qr&mr!QOxB!62}Bw#qj_|w@x zx}-77!KSZn*M|=u_=SWP!@eXHdC5YN1kc$8a2MP zQ@V2Ht=iPPt78_HAtS}_-z$d7yXCO)@+OMiXPtT|X0kUe<%GUI6Xj{>Hgh!A63<lkI1g;%_oqUYiYk-acwJNTzj=6ywlj&sE3!x3R7 z+~?1qr*VI2*Q*;|u|=(}sJpvckYdAXSR%Be@!@tO4@!aj;<}-;-yU{2x_+;&=GuE} z)w+!W0-c$xloH-TP+ho`?cVOP=Bcr|>PG$#8Xq6RbKX~wfQjomZqHyi6A;;yB+0reAS33*Z zwZp?@9DGyuxK3x}nf%yPDCPX)=^<5KwQFB`Gd06;`AnPCg4pWEy1zeep{>p@9+mFw z8h0INxmFf?kSn%JzBf>1{qie+yVuEH9o<^*ns%XCq^71Ou21>!;ltg{xw+V{kDU9= zG^$rIfBN((f2iXNgN%%f|J=USt5+|kV_eE+-H;KT7C1dC|16Pqc* zSyZqk8Dwgo?vs!0T}fW!$<`qjQg5l1+m3rq40kDugeb4S6J_65XI}Gi-RIyf-amhu zdU|`ee|;dH;WR9;H*l^0uJcv=f`Uv%MMV}<-14s3j*GO9A79gy_Is0vmWmEQQJc|;soZQ*TCGEw5-ie7;Nn=*cIWBFyqaq?AWPPax3wsR=T;$nv zg#){%V0W|g^|&}e^)N}X>Y1*ikN5g-ztAMBf9x1%orJuujKd!l)SAD4|MG`?NwKPz zm>#XauzmY>!JabT;KeHhTE0En&DT-Rf?lJaWxq}R&>`Efp6XAZ4*atIg>5t_l4DjH z@&3n;75Kow;NbVSfvi|&hko@(Me&D8+R$FPawQI3&*0?A6qhOcKKnm&s(bgwSkP9|lZ_4>GlR623t|fN^THuK)$* zdz#e=gN%zbxVS2(sl$WwO-8WUS9*&F3PoJpwo}>q*}a7Y1ZaECRDZ4)E{>U^S5{VjUs`%N_Ec34R^Mid?FK7fk`3RhM?%rM ziN}?2t)2;b$w6o*vzw-H4zy`Y^=_9n~lC>W1rKOA}|Nc^n*N&y9@=2aHNxJWj7 zMMpgQq(2;`A3w6K*=>!>6+ZfWqd}Jap;!$Ii{}@Aeq8SF^1CKQnc89YVj}cco84E| zlAzmHCtT*bxXjeXFL$>Yga(I%gi!Mpa$K%eeR1YJyy=SgBT8t8UYicyXS&dwb5v+- zXo$W472o#L?sak~>bS(;eSM_<#_q^85IMz1vDx=Gg+Ftt=v=b_mlmg+h*E^*&W7H< zb>qg3afw$^)}v9-apwaq?}a-iX52KmN)y^1(zp*4^SKEItE#9ljgF4eQtulZ-zn6Y z^|hPiTCc!?N>)FokuG5y0NYz z(#$L@e8o(PqE!7>n%BJ%YRYvB?9Z86HrC%b;yJ-&EX9anOP)}xXX2k>4G4(2Qh;7!o!?Cm9M z`ThI%G|D%YDqguFT@ta!51XD}N^148Wy^3Sk9E(V*}Qpk`|sM<8E$iO>?>Ai4^<|Jl#6crt@~8AJ>L$+gN5MN3P?W zr*fQq&?j5*-R)vxiUl`VzT>)O`~C9K483+=Xo|Wz_GgXR!P}oqnU*e9xp;9q_VyBd zWn^U)eEW9DYG(RFaw2ZPvLRzRwcWfdN_K2Ruo_;w8FHqw0>*}P7XTFaNYI9t+R$eo6cdA(`s(6IE%-4jwd;o;%SSFU_I zXdrpHTQf|`ma#tlLh&`aMd$<3fFSpC0cdgki>O_C@YkEq){ER`pL)l7T#;(RFF5Xuaf9 z7E_6`SNwI3dpY78q z)C_XBq{dVxsmhp_@uxbtdwF>+#=Yxim55W(mpVS@KM*&ouogexurF{eo>N>@g?($j z*Kv7y=6eE%*eRyIy3`dQaY?7%2~%uOwBrY+Cv1Z!v2MTFvlzzfJWRN9jFjK*@>svR zN!&yA_!YrFVNF>M$=jWVOcjNLpTD_K`zn@g#fp2+T(mSB!hikx6)zDWZdSsip`l?h zb;-ej%epxy@yB?oUE7DngR()5Zwu-gw(+vFuP?ehKF}&V67Ap6ByZLKiCu z)jY$deW>!V7CLH%^TefSFawL`FTNwi^4VECMT@5aF1)N8-^M3iIDMMmd1j)uF zI2OH?=3@5X#~VIE@ozQEF<_&QE<3CCv9xzwtZZcP-IWoGlw9mr!Qf9Fmk27Q_z4cN zsDJkKDf5jRH&BefJ=~Sqlp|5S3hTh*+O=zx0{XUj(xlt?K=fymx8rUin``$Z11l~6 zlAWDhkrwjFw9WGJ3IX-9b5k$my#C>{iE-^v9|F5K?%Bh;FMx-J0t-5kAS1_Fk$Cdq z2|$lxR{3dnARB-erFLFx;1~UDN3pkdy*KCKXmoqXlW=3pUhf)dpM@Ko2H9Ao404)WT zm7MF=t*gm)5(kivt?2X<%~1e&d1+a@16V!=yCv?(vjF|1jC<{YrAs@Uot?3#_mFi_lX^a+C~F)Q>;|eqeqkX4(4ALKj+}^L&MiU# z(OAa+Wc6PJH8AM+lfXYMdw130$+Kq#4GjRPe-#RSIR^(?mC-5N@qEpzUaTga;o+|H zJsVxECZ?w7J~~?sztVl^&r4DS@2}l+C9k}koq)o@_TulEwr0T+%v_?URB~Ns z2=Ua3b6>_QomW@4W^iy23o{S5Wdl6)LO&%YDncT>tRwT{*q=s8#{vE|JI&qkGq}tf zj&*d|O&b76(^8khh={d#2RE)SVim5e zR>darUf4udVnWw&rE({*koK#q)4LnE9ec3^6wLk6BW~~^4?sZ6UQ}OBQsHqwq@UWj zYjH_f)=>Pm7q=yU(dG!kTZ;zT2WPnE@9#f8{&-&?QA4t?jx9r-%ExaGsVY&XiMPx)F~yP7;t)LI?JDi9$*pow+>tWYUi0Z0y& zMZ8$O`}XY%lXF=E#N*WDhL;B79;N- zc%gAi+350qJUcx2MKresuu;d?sU%El9DJa)ug`xG>)M+xT>?eFH*VhS_r<;9#jzF7 zG{UtXuAit%JYim)#7m0HI^`SYvTJR*bJI$~Wfmz+2|ej=&JD!l^mBXm?AZokVP`OTvd72A`2ni+tX@sFdh?yOuz1)t zsw$M#uB$j0n=U=EC#0*xWfA4$F z>+X2X_s9E!d%El%Ct$3gKe~#RMTq_>8a6 zRnA>rKwi;?%gAd z(jx57v3i&9Z=A1#jE%pgU$iK}Dhot6rNnCq*X1>C8mPU17Pz}tiO)ipNo7SskX(Xp zJ<(H;pTE91)K>i62z1TDw~)(lBak1L3mlHvs{Tmg5EPPV{ic^j%E7TVuim=N<>>F` z=jSh$^ds_+)B{PI7_@u|%TTok62(72MK>2+0yTvw5EaN(nA0(`Sz21Ux_(+yYg_Z} z`bgH03m^M4Wj8-;a`3f%2?1)L`}>AetFCX4HW$sbP7wuyHp=PBRJQQ%vAannUWB|- zO!CtM*UQ@(-Z?9~8t$`lwkRGOA8aq|`i(ZSruUQS4kPT6wjC_T+#c?}AOfA|rxYW~ zY2BN18z{E0@bFz#i^0sNoUmSazXF0HD}sz&=(OX$!W)~|=^a}yYD9hB%xnLG-ePTPBPGwpuY;3=WYLanKn9YO0q2)?cL z_VO>;Li+C6nUkTfWUM&>$NgQVM&q~rsLYg2@6W=oqp0I)mOg+435|bV8Wp@e{hl`U z44}+=u!omsrE8aROQ`>nz0Kz}bE>>BbKv8h4Mco;t`mQ^{l)6H_nWJ=xFgF4-?`3Q zd2u|InR@9sXq=Im8K20-%j*w|q7C2&R9OYso{09Si$qF-A>n@S-U^B?6p*<@&+6s4?OU^%rzLW1$O)bQcGdl~^>pho2`m`9O`FRq&%TjVSMw}4e13iyHP6De!cMY7%IPHnpXd+Vv}RlwHoQE@>1 z17r`KVKrT*3E6cmfw$q|ya9aj5q%1#S`*u~__q{B*+By;Z$I`dNzS%&DJ?B6WxCAs z?9A>B8ngItRy>mGqU!;{Y5qhQ^PgPaVlmT>tu4(}{Ktt4vb*_17RFi4D z8S zf@8Hv&;8w)#Z{ejvH)vvm%t{|J$jv&on7OiM$>KDi=2*63dkp$sHo6G^kw1W;~QM1 zv+swL@Ma?;BWiYc-{n&r6@yK9UdHJannvF{Ga3o5e&RVD@nv{2X9+9oBFg`O5p~+k z%zMX~&-`c|v6j>0g9OPZoy;Nx;SC-zzQ`t6gF4yH<^7Oj^r}#9S3VR1qrctXsU5kY zyTrtV!t&+}=j)qr@fe!JhvCt&X1C!^*T=^;VgeJK?;QFpbK z6kY4hf)|*4%wd*$?7)uQuY*UH+yj2Gcrn&m*W){qVW%k8lH4~L1Ag(L<^A>6jt(y= z+s>%Nk#d=9Zw31JD1yh>^j2+yTeci8YE8f677M;*Rp&SKx;F!iJU_~A5VmiTZ z?N~0pW5%v`!+m=kH-8(l6)7#964~8#+RpGSeg`)Z%ZU)vUc>@2RZw5gtLA z{_C^;+Df#yC(ew4;oCrApl4t(f{7;yKc*_h{HlNSxcXmc8Xrw}?2pwBMxn1z441YG z8Wn%`#Pj#h(JPyFLqDBMg0Q(^&z_TgDU1rvk*<<5`~FYsqNAsWSzKHkT$J#}T8$Ay zHVC7lu0ppTWodD3)pzHWv2OwHCL~8dBP`CYzPsC8Y~A~+s+Cw4(wPall3HKK`61## zYfY>Zyt@N}W{T-2b}H}5fqR7J24Aa%??5|s>BxJSSfJUA6d}@3l1kHN(dd`#6jB@v zlQ>_co*=69;>8Q*t@pFb>>qHbW~Pa+?Dma~-E=D3@fs><_mf=EtQNd`$H~7!F)=Zx zt*jF7xekA>tvxpV&bZKbO>b>&txql#5e+MCrK+y|^`;_TFB}|uv>~4^d181d`7wkx zSPIOxwzlI_QwiZTKd}=_E4lRew4Hy2RzFzDd>4F`mX2&&iKn-lisUgVh8o$fU&R|?!`MGT?C?@uii*RG~fcXHWCJB?S6cO!OqWv$M-v#S z+h00}yAu``R#A22EA!*H*!iB)eWc*Y0ti~YYE=|O;+l6?caQcr2UaH=a`lA7 zb4ysVJ=6L2?PMC~7Tpg}t;kE99{M6mM1?mNT%4Ruy=Fh2KNQ5co0 zE>Pgm?BWeO>|En0N^+a@E7URVtIuj6xY(^Qmyob-VB)uZnFifFe&knU|+vGhoPzCj_cYs}Rk{6%Y_W z=&O1f8j^-S^iU+vBfKKDq)5NE)>bTq;?YNe-}{y5GJf=2@Z--&obJ zBw{!KqR!pj9Lsnl-GQf4T#beX28f=qoP*=Zy;&$7wEW_O&z&cRq<@La-}3kW4!e08 z8(7k5=t*7tZ3e|Wv~$XIZNbC)3R=!$J#B46pPFsre%a}SJ=+=n*x1k=%6#CN;IS#0?`C3Fz-B znmf%-Z0M1+OIx1x24~A9Ls%iU5F8fr_gGQSE2I$Yh znF)rygrcXEdaDw>uvpK%;AUgv588T6o8ngwmp&SA3Ejn59&|;sUJeM8n3{!hOIcVL zO%MZ^@RO&8<9^ zY7tB+T(q_+`j{C!Hn9LjppD3T^@6ca>$ZJ}W}h!hiK-+X9@^qB@pX@NIyGLei? zTeIk?Y__p8h*$`ZAOpkTqq4qA%C1KzslDVAej5V?f5zuFzD#DIiX& z58NYs9)4bJTJ_Tib%7NP4Gp;U#N(-+F!@tQcI?Gz?ut7GjH6tXRQ-yoT5;>=NPN=1kjtTQ8JFT~o`Q@R#2VRJD>~LfS$t zxz!sgnE=HGb`q0P0&S^q%nVB8P=&VytTQ2NGdz2#DpqD@_P}t!js%{22k&oPK6L^e zzsQvYCMY&E#E|51KW$gT;??43@qht7nOa+0)y{yUwq4cwF>$osBnMtH{7^^=OiWCg z@%JJkM0n(!^*aVVSl~(@a2SJxN2dAfpOEa7J zfM<~q^8((&OM}qen%?ZX0=S{F%4Kf$9dTjAlCX({f_vMpY?3cc;AhH5%lR-nH{Q+~ z`wGkBeN)pZihrkNjiy!egjMo@kU=^dRH0wYW(!rYiBv%-u3PnAQh+It*q8NG#9xP_ zm=AFnkuXqrJUX^#0ut~suQRgj9Gw6)B@CD%X2gf$;&uo}On6h(ZQJycc*ripL+=p! z17XP+Y80A}YT8I$`h`Hi02&IN#|xUE#f#p~&Kqb-okD9BoN{+PkZ}+J_!-J(v0BP2 z&7qfM@E&RmNsMhhk+#&T?hUk97Th=*;gW`2H$Eu5*dUt#>F6mUkm(t^c+JgDu`b~f z`2Y|GxAh2-Q*c!;1fMBlflda;<5l@(kUz7PkV z6KS&^9v*Dm+tNS zZLacILmeu)!o*O=ATk@UKKDAyTU>&B8602T3`G#_0vAY8pM!+1 z=k{&>*&MeySDTY(_=ii*?B2aQb`(A91L<85Qy6mRW@SK}u0i_-rz8NjjW-OJQyi!D zxT5Ix-Mh5dKc9+%`GH zS3Vlr{Mw9_I1zw5V7)OYFl`)_=mYOjM~R`qDo&hhoRDBiz+`VN;zX-i~#n zG%;$hz!XSC@3q-s{o}{mP!1&Rf3N!b_3LR>XeMnrGqX#6+)1e*gg33K7VG0PPQV_ z&<<2Lapwkw2RqKZy!=a>7|D1YDnOLPzS2oN&Uv9J`#n19V9R^Dc~wR1T=^4&{>ZfI zRVc>!WQk=tTp+0(@v~p^&~`k~u^;2oV4LlO=M8to6J^4$^j<>3mapHwRgONWp6eQ# z%BEF-376u~e~L%?G81gMLM(wYsMI#PA-K~R(@%Gy7wgDODY@3uMK=8>;aX|TyUc^j z(TTv@juEWfe`{i(7=9+ghWIzXjs%Y$Gh6!cPce3UsJL0cz~snz{}glaBOWLU@^jOV zE7IRSVCQg1v=|n59B46+zfkuly=^mi51Xe)xR4^EQb({+jMI_}f$)Os@RFB+E_&5X zqRMfuMV$nPqA8nEUrgX{dMc`9Vm*Rp`ze0Jp%l-_!v%_Z#i|{zbH*(-elwE?i5(f` zxzn`Gx{W_hP8NM*QK^P~lct?D-6WN0b?f0-HBCvE$v04io=Dg`I5_yfe!@`{j^$K} zHi00;5B+^MT}B%6Xh!UaHN5WIkS@N7BtCmhWV5nNNR3PF+K8%X)R1ALq^9Oul;$gw z2b>dq<61mKAfJjfhl>~4N+RV&BMRi5m!YGH1c@4+YPW47 zxqwpIqh+AJ6)KP5L{FXK@XXE)C_L&KZ~?tBv7UB5ttW8mc|SaxhexWoW#DD9~u|nhCnw{2eB|Z`?2j zP*)2)u#p`YB@d=2u8VkO2tfszEIl^$5Ka}~V`bI7Pp0~>Oc$d)(U>;T)Wf5Ul?IgA zd;IwEfO`5Ni6b6wXG#K$pL!BKK*oNluCDGX8T%L7yoQE`VKNQ?VarzSFuhslip<(e zaQao;+$No0AFN3axpU_l{tePLqiix?sgu_G?2J~A>O$i>hqDbuFa$756$C)S0-P$L2(J?Jiw~xe&Ap9sG+K8ZvRR) zZ*+2UAYK{CDhb<>FhG;n3(3-&y1E#=>e?i!#O}xFk0kk7HR$5F`J8=4qfo?AB_&#o z2w5M5I21)fNZ~aHug0~$?n_4jflv7!Apqu|%i8i|{Hm8zB(4P?euJzm4+08tI`Qmv zX;yx4KF(fsWw_+_RsPCfCgM|q#=rpc1q+Qv(8Qv;SmaiF?8}a^8Ts>7TZvL?ZEX$K z{8smTj9Kub_EBh;cDNpodAmKvWjJI8y zZ1mWdba)*5OieeImzSS5F}aRJdlZUzt=#6qqo+!}9s(aB(GCI-StmW=A}aiI#BOWA zkgc;JDr-|y)A0)j8+m7#LGsWe@*vwnXk~@jhQQKdc)0E`XshZq_dPnl)D9Kmzu+?x zWGK=e@80ccXl$&EnBydSM+EU6r2RILASaeW-q){77h=zuRpRL#sWZKiNNmfED&9XLTS$qu=HcL=libTu1A3qMg z6AQitL?zsFv0J=otgXwxdX<-7^)_6c%JcsQ3=p^=C;-J|-S+Lb4Of}o2@Smok2f0X zK2OcFKn-VSSz@6;Oj~Iu`1e8k)xqJS^SAN~7Oe6S$s@F+Nc@j4PNsJ;%iOUQ{^yt6 zq5mESp2)&i4DI`WxAWh3@T}PW|BwCOPXqt;|J4T)ZCF7=2!x9Wc*YHHR=)g#0y@BR zp`MHXmW`Y+V)RPL{Xi@7g!aW{C#fpDVFMi?h&Xi`q5C+^Xr<-*$X0uxg0xlKbt1>a zkc=KuF3JgSVa#|12Op)`_xD_HP09uWTT!2U=oXRnk1wxp&>7(W3kM#Y?^{v8ekiuy z_wF5OT@*5JIU!gxK0DQ)f=Uy!?AiXHBC7=m~D(ub9dFZ-T)cK?2XJ zyo?J!WH~3hi;K(GNO?o62hd>m2JrYGkV39Nesy5VV*e-y*ue0QnqJA=9#DdlAXDs7~_797_xOXx#BsVK@bG z<;8eA*GW~(%cEdWXbb(9f+b-v*jj35ARr(Pz>Wt;fF<0EXjB&xe*iuh=CK8iD4dLs zkC(LT;rP?^ZVB!|Qc5Zs<*7|N5SL88_^vPCu=n)n?^?&1$z3SdyQ;3DEfRqk6t)#U z_schLR1lSI?d*J7Y5yO~$5OHQ9Zk2P@oG{mZmt7A#}a=FDYw9u4dQ^lIIovuSdZU6i0$NmEcv}*abY}rCG{NOz9h#iP+xyw%C z(;(S00Z=h zbe&ub@O*UY*gIsliPnvX-y7du=vaijB1b{;z$-RuC>%b_O28Ca%fL{_!C=whnVFfw z!Ruf@@f$aRH<%3l`~+bgq%mD%)CK+zg1l@wbF){8oJsf+1eL-eaQB-B7U#~4as%Uf zBU%~URrKY{!Gtc1u*FH9p5GoF$0OxjF1F;+P!IU-!)RNAFd2kM{*4>y38h^t=X?|E zCc$bBRy!HWz;jcEj?~iDMhlbx!=8*mQ0|yAL8^HX0E%%IIJDS0nvXp6A17(#ckSY& zp`oGN(YO5qbDf46K^d+gqE6%rU(PLvBJseLlC)`~C;J&0jDHdNstO@$4fJZ7FxojIW*pNkdLV{#GQhNdaIM$@%*9nDS~hs znVBuhGzh?448ln*C>I1nA?!<#Xlxo3dh;JoSzxr#P#{|h(8Wlq3d#;5E*p@kM-aMN zSf~|L6^Z!`WVj7Kx&RrUhO;OJ1Z$5^PCkD#JUqO@Ef*D^~)lGlA`sc`t#DK!W;V>t10+5tM>48$u}zvK^O0M)^GG zg!LvYvf#Z_?szBMfc_R={!KX2PyjgV1vX-+Xc1ZHD(h)(J1r_CGeZ|!mB_RSG#gPy z*0nFn4S{k$-{0nkX!v7D_`)4{SsAun%dlXB@3M2yG5w3|wPJ1nDD>3UTvQ|YmVmI# z;EsbfkTmfCqp+uMtHr9-s~tG-p7?5rC8$Tr@j~jo5fO1J$plMA5|(OfcQ+F@I|;sT z&4QRiBy7CUcqe-j!X^6~bcwOEG*Xlla|q%WM|zIJT2fqpXMVSH89GOor{R~8YlIRM zuZ_AUPBt_d0z@E*P)(s=RAydOUR)7TiN%aRFxgltt`ij8FQlk&udGaytXUGp_x4_d zG?*udw_k_vqm^V`V?Djc)Zf-4yGw}0|2!xi{8=bx`uEkUD^Q>YMo0G|I2rUA?MpOA z1iPRZ>dR@Ex%W}A)eSmAyf@=nA!6h#DbP;#gpSgOdWPVKT`R7DSlM&gZni!bn8JKG$?+^ zYsZ|oN=qMyJM95R@DkDljr(@fk8H8_kd+{spRaoH>1f!)(LKF#t5&Tddq%I`74R9- za8c1-L@9MnbRiGPVVJN*FX^YS8~{mv(F2kPV4}94{kqB@I>6Zjm?9|vfuu}JmPUO- z48~@5+KKE!2?^aZeFspPLgkekh}-xFo^I8S#>=>KLznJ z?nJtX``n?dP`h6@UYPD!O$-t^JMEx2`wt!BtS;WUV~3Kl@p=UR7D3GmPOB|+?)-2{0G5`LUf_+n4tLG^o|?St11My zicvCLN%A*Dd_G%ysp}1mg9qs-^@lr!N>litQjyRwwv5=7 zzvd_qd;7k6D|AY}N94Grw|7fPcXxk3E5^rQy$-&1uG+K~){0G632(|fkT_BNR_+JUK~5j4NAT_{ieFaN zV3W}M{CuTz!iB7HWY;0m5QZItWF z)6m*OkKpm9@u-+%pbB>+rp2lp%H zB}>RiQ7gu@(Oiw7-Kli|jv$`QSe0xTh}k~N^kFog!M!)(mrxIUW~!>HoP4K9?jqfq z7jG6dB+zNN^UMkP^V%LBZDUyTBveh}>e#G+&!iNg%QJaUa8BTLD#zz$|GBCQ-*!#C zV9{y}DDJBIlynPCF~exSDW2=YZg1;{SORl++n2h%ijBOq$AY(=SP57I8=4U{QwxbW zo0XpUWjE(Uz~zy536-`J#vUT7J(>AsS+i5K=K>xM!4Ng+s6c43E|_qU0CsgFd*0*+HD^umY}fUZBRnfAPEbdVEi;GlmPx{y1w5=)JNLOhA( z&!0bnNc^d*tMlL44~^?OZkci~H|xXNq@f@?1b5Y7t*pEE35_PCFJy|@H^mcLj^9+Y zb=bO{J2`O8fa6iKMc(9j)cTX`1#Ow@Z-Ngp7U1)_LRZN?}`EoZ*4Tc6?9PT_ccyI!P`X=Bp!d4d=GS<0@MFzKhxqL{R z)54@B?;1*EF_55`DANIeQj^TAr50t0jwYbOPUsktJJ6{fyVbJPWcrfI}~LeLDX18}AZq z6ISwCNw6-6=Sv#%5Qfdn2cMZ63vrF`sti1`nmNnV@lGuglY zW|1^zcK}SSBhtyK0r=9X;W#Kst8nSktMMrk1TYnOANA*;6SYR3Zp(e8f5DO=o_fHf z?XVhwCCDttz|hd=v>f9wBv|y$+^46^E1z@1*JD0UcP+!@kiS6H|8`LBEiHN@q>0eSV~0U zd+XijSe6nP0ZCf#iu$_Vx`2mI)zP~)nC>_h(MCVJ`{p2yC?E-zM3lz0n4|pZ#JzBT zDqekfQEmP3mf79(oU5w-N^w)O6J30%ftb`6?MEPJf8Dsdkst^1*b4 zH5R~9<__emsJu+uE)uwm%@Uwj%p)l#z_R&3&`>rn$Xi9SAs_Y9qEeXpe4XJKz*2uM z_6wP-s7TN|9It`-y$^Q6kQ9@4n75!t0qu53X;84*NfMNX0vZKj^wge!*CXaM^wZhu z`aHV18Mht7Vn*O7h~B-t01Tl2G_((wr)2P1K~dlyiJrFAZ7$h9fM45=z`NF( z6Pg*9I(&6suMs)+iXKD}-omO~793ScClk^l)(f>_YY^fRRp9rs2*Mz4bu9NDG6+p@+T&1^l0CNO7rWdavDbb-vNqxWH@VeuD($-|ml zC zU}w@VacyW#N1mIQn9!M-nR$A75uqHI2pYQ9bAUN+LJ`{}3%p;8^%ys&k_wOW_r0-hTmyHW=C)KjQSO63v#_v`;8mEko!X1!68zvnu*7MsA%c~0LmTeHx-cR0 z?Xr#+bMCSep$MXn5sm>t8c8b6x>5N4F@62h=<)EvO#f44IJt4dU=d~dr%(hDCg;JU z*SlhXj4bkmNCkdC@y8~hLoz8P8%-t@rr|>gtLJHkh$W0qOc)j2TCI;s80ZszsDHwh z=PKfs<6Xy%hApEUIzGevADcMOh^CNjzQxj*OpdF*|d za0TSL*PPQ>+{j6(9l(6 zuobY2fvTyme~eWQB}3G zUmz^`V;u$eHXla$BwQD0R#HYLp#mvlF1SHuaatRQ zqiN;lTiS%Y0cwQKLzY&^>|i*I6efF4o>{z~Y8!V?NfA3+5*9z~Qhk>1m9MWa8CxT9 z0Gry+$bu*f+-vQMbYmnF1njV}KBkBgSuy|rAXQ&kC^!Fh^ZXy5_spv~|0v4*A6nNI zB#8g{Jx-?ow|uNjw1<-0jMB6ZBmwY3ry?4PAo-%9a7U+sA3$e0?F8uqco+UcYw%uk z3SCK%=1s&X?tlwd82$SfKCt!R9_UvXmCd!+&@B8I&fOEdGA-(hz4qIL6p&L+B#HKi zuNZYyNmX9jGw)OA7QQ2FJtSkFH&2%qn#;cyqp;A}|L6CZ>E~0{|MOdM|EG_6x@s*_ zc(~YuQe^X$UK+C=LyrKF_|Jt)F!pO_{G0jjTE7=xJFvL$77RFun*{N^dGjV_nRncN zB$hDm%Z|>V%F8@|S6V4|f`UA;`G6$him0dtm@KHug?u3w1or~1kTI&Jvtq;~i) zF~78r99fQO2vm74I}``m7t#z1UXqTfr2YI!S$}O#Yb%KjA$Uze9oz-5 zeDuS2HqLDjwQxR4D+bXWF;BW^<-&s;$W5VxAc(!lX|q#B)$iYx@p&IcO9ofC84z&8 z%S$yP_p2L5ei5*WPicVtWp8iq#3Hs2`SXJkxMa>6`H!!Tiv8no|1N&Hi?!V6 z6GNNW7hf6s;}0CP0S_KJVF3y$;wj#^T3QNV6YcPJZw|t&-l+M(e8?}J?Eg=b|M#Q5 zZ|=h-LCmR1zraUM7}=Si`04JZ6yri>G66+hgeDThd`C_R?eH4%DM0pq(j1BYYxk@{ zWrMUR)T8-t>B~7GF7AoX;DX3!&=GMS5C6J#(Q(QI3n8FdvF}09?G+f#rZ5Je7*R)# zJphC!(Eal2l{94Vb|bsb=RnlO=emi$cQ)*61IJM175MM1-GQkb&nqLS6+;i{oWM+s z!Q{jhY@C1M>$kaFIORD-d=R6{!yF<2iV`4X3a~iuz5{$}hYtNa zgp16yK=t@v3t2_y7IxEmD6+sQ`v6Fsy5#4t;QPL|dk57JyI4!)EI>OlRhwyz?o8zA zXz*>y9g>F$h6yn+A6Mmqv%%tK5UDD)lZ|{1M5e?>KZvoo-AJu&e6E)qfWBpfC#=d# z+A8_tj0UmxX=lVX!f>=lNC+3?$>lg21LItrb~hI`+7qwC!a_#C5@cGwrz+8p2*-L> zTQJ`YTnO=IC5djqksu21xgSrV@SO%WsX@GlMpfg?=O9IJMDmKM@wlq9^Up3O@E$)? zru98!7lMK~Vv9-ryn8q1?}m^scANssX#fnaW7pnT#_Nk7Mc4ALrgDa%0S}71LcvI? zJG;B-c1X(2!|ylW3nFJsfL}0z?GXP2!KX`=IY4P-2$kq zcB;iu97#qGt-M@#B}UbL9G1t8M`1wYrmrvi7fAazXy_R8N=lYNj}ttRwpIFaw>uo` zgug2wph4?O&Y5IJrg@dTt1KWC6ABYzpvFLV7Kbe`+6`fwP$HN^MrY1=k-=wZ1;+~y z&%cwo4NSCnk#x{)N$vz{2;jk93k!u2;b%x5ZLRSjV2~|?A1mbrnYSs zcs!2Z;rh)eX?sLj-4RPdc52zn^mOk#cb0z+5e5`bZjZDu#oeg#+4h(JOJ@+wp#x(3?>;}{h#w5#okMXoLP3n zt4t2F^%UL@nOYV#%%Q(66){kMA@@CE1?UDiw28#Ej+A9-zUbz%qu!E@3qO#L5|=K%sGaiQYy0b|R}oSkLKurzAtUG@6Y1l>W8a);M(?x-8)0`NhSHppcUz3Gr5z zAUlpDFXH(f9@mV=nW;gjz}74VB_*dv5T(9r@nVIuX;RdxHEVjZ`%5bU7yzL@L}E7( zt^tX;kf>CUAYws#CJE^!XEkADmVeVG)oI#XxW@W9&Y9&Q)6@W9H94yhIvLqQYqOtE0}6p+MsD;6p8t zhJ|M$aH|wKW(%eOBaDk8$Vng0D8Z#Fd`l)Zv3`tBpLU1%uo{t4MXWO9WNyG;X@dao zbY*tlt&!7%>4eihxDY|4kmtjs*&>Xsd>R$8;1}1L#B(9%C+w_hz00EVHWk~+rh#~v zB&4>}oQD>9K$8cKg|-OVnP|>9E$Sr$MfMlMdp90GemoxjEFs~1fXVX6 zyMgvk$%&d|+KvP`@WSC^vK%?`!?K=)ys=of@7kpbSn#2kK~Vu`Kh{0D4y1?~Aa;t( zUpM}tk9UNhPqawVIw09QXCpNM1u$5J8+7U8AA`3X?#G=}D=gyrUkfPh}@W+6plu>Qo$Ef;Un!cr` z;W1-6;YIPnPu3rCYbakDK-FBVm}4fvXkZ8UA7Ls z7DWL$kV1`O(2kZ%M%1y{AmT3rnx*p4)QL-z5(t=xqOQ-wv}t{e#~?_C*n4#x*xFO{ z2^%{&Hg5O_AHbDA5J)uM1x_rC8+SQNP!JA!O!XI41kj#@2m_Dy!6x)>KDd% z>9BA-kawO%G8Z)t=ZvZU+9#HHBOkw+SS-P!kCBFl{p=@na7G#fDj(h!{zm{?M8sNh z7!toNb1~~K^D>2VQOb&UvTxg2)gcr?^D*1DT^Z91%7BKpsTUS1xFsyan}LDFsMkBI z9>_ZFzy&=P)52)R%uNgiqj(Tg7aFWd`I7?-9Ae+QOb{hhGg*Xz14)<3ox*xaw@^6~ zzP=K~5tqoAzBymL411gSti{U+LQ-)0-GbvUD0k$^<7UDl!lBX`f4sepj5NWLP*cjs zN+T!W;dl(66jNRqXr*obOg%-*$dHV{MnN=y7^I-6`8hEwoEi&A63oudkux)TBG?s2 z1*caC|F#)VPdO=PzG=^I43!+xmAN zUNJ>>W2%GPnRl%UjWvq86RTxDo@-Emo{95(z2B`{A2SU*MF}A1J_P=wGX#7fSu>7` zc{?bGjAhX?GInN?w{r#OOw!+ekZk~#41vH1_BohDP`SfH%p~nYN(HH~YhauT>+f(J z{pj@vG#1C;C_t{Yot_v_*25fxCe-2BSZ?>P>cj98jG^|Cu{;E5$&t=zy2e0sg-6E3z_4j=Uut69k zlRY-b7GUU4D3}~Iv}X^m!rArCIJ$sDw}G?=HB51!5*nhe)f^11vL91|^42R{IEV>{ z;56Y#SiqD!hGUIEqabVM;qp=VuOW6NwwS~dg=YkdmJJQnUV%nHBwZrWliG4A(;(hp z(5b1UM5Nw^PLT~TA;aLpTtT1tn%y|YksKr%F|~R9`o>4%W~4UyqqF$DmNYy0HUvjf zEjxBWS99xf#NF53H#0T8DIQDn;I$B$=ai6;fSeE+dT}~)TyWn3ODn6R_boKXUYmbk zA8U_zg({+yBqm&LhL^P6qC${>=!G<25!fY}Cik!Hd~y}A6K(^|K<@832wXTuDh~}7 zJ!vmqveDVI0&73`%4~e~93#6p3Ws=qy&ovZRVckAAA=MWH10gWf}kGano@1 zlLY@S_Rc)4=KX#53me<8+1rqLj+8l>lbKecQHByKLuDqEC^Kcq$~+WlP9n;fOesVv z6`7}~gv=t&>sc%I_j_IET<2WZIe(t6@9$?zt@T--&-?v6_jBL(^SndXFrdh5)K!Z3 zwjEn(#KGJv&@!~5`C_J$S-Moo7Q66K$uK$+rb5mCQW_Tz>vI1av_Kai)!?B+9q0y; zK5Urk*_at{1r00GpFKFZ5j=PjR6x1!vtG$?N`;sO(DnX?p?Z?RjD>YX%VQ?}K#>&` zqRQ~|*GE;Sm1hOKHv>2~5V_>V9rSA}$XJ5GlKJRcrYye|$D@~lS|V6tk&CiC%P8TH z6G+I@7*qCyJ1A+AZhG_h>0@Qy70%KjO&a3nY|e~23Q3G?JYI3#or>?saCD$bRfX6$ z=wMt;=iwtTnT#hzTNaRU9tR}}QglyP8M(bj5)Ue?2cFubue;2sulN$GR5=eqddpxr z#W{RL<(hXbd`OK&g>w$?7P7l9u?zKAyn;_o`HiJLLrk3g^Gepu%S#o(X)~TP8WbH_ z!o3b<#?rwgQWtf?P}0*sO`CSyMoj$+2>|_4jaC6bCr}^Lv;;Ptwg9B?1d13~vRyeh()b_QpW&8cJ z-#!{3btJb%5-D9AH_t9vZ0Pa9I3y9+dVuB+8FYr0B@&vHC*o~+eeb+<*WS1nK9b88 z@GkEus1>9J!^==3I|p*HLV)N*5R-%?AUY(iOrVg#nNP0&=^hdmwnpaKDsIM1=tmcb zq-7b5E+R0kELWDLLBJm1I`<00DQF5{~3m~I@XP8av{UFuReaf z$EVl#eJQY>ij(LXJ|v{Nq(s^uY9i2y6_L#>0YzS3agN)fwmd&fY=-0R$(Tzd%W6v! zeSC!Z4h&=UPB}gDDt^zTWfht0ArZ(6KdXRedbfQ(BC0cb+PHY%4Hr;|m`r&@#n7Qo zNe?!9G%=-hxxw-ThnJ_W9g0{xn+X|m9g{Au78wjRYz5P*Mi-e@QUGf2H9o$1FF@@H zFr6V<_0*Ezm#3Q?#-5z}T*~6cuu*Ek7Nu4_f5v%)yBp}b+cs(gvyf70!4wRyQ~Xc* zPZ8A6>dbAYOq{5o{RtdWUcqdtjVMuKa+GQ8mg8xv_5AvLyTm_}e#GRVLK6c6WaLnY zC(HlNE4}OETmk&t`nSFu}qZ;hI%**GP)14Kl3r|M(i47S}H^1;+7QT=L@vqtc9B5E~S#dODc0x)#T@dJeZuu_@Sl5Wge4m$DK0%4=Y0j zNGfmkl2B3^sCAwb7;MZ&d%#Fp~)~%yKyVkyLYWoBe)b9 zoFjB3<~|lfQ-__PU1@{?hYQ&&A_Pse6q>UcGR^di@S;qV6S9m9#1+*~979v4Oc72` z_#S8S+#Cz>Iv~S9h*R9#(z@yK=~We~V``Fp(7ZVA%gGQTSmPJp+;yR& z#j6})6?Q>GFN(JtwR1UT%b*E?pS7tyLg0F({1rSmglE-?#<=!oBSx&U>$v8~kvSHt zL}GoUjs0LqlS?mTw5MV=gy>dgY0I<`Dm^1}vrS)H4mRv4cfqJ1*>Ud4GvOD%v;~0X zKkHcd$|%b6F$$8!>$fMG2L){-qO!e336p%8jLfh~ye`51d%vr1PjA$Wlq(fn!1%#^7`KHjhoL-HuxahfdC5ms+|%voSb2}`{=bW7Rt~!N zc#dT75AI4cvo-tow>bOx*z%@{Q`5HDRXW(9xFJ*3fQ2ceCj`DTm18C@0G8%J^~pf= zGyZun8FUP|1bjTr;`_-%4GPL?=KvwP*gWe zWXJyMl4)>~Pe2{Ng7+KWzbgJweunssS(23%TQIMQcbkQHw&Wh8#hC_QgSr*t2x2kH z$Rpr2Gv)swd~AtXSy`iCTy2nCIbyr-gEapY56JHnG=cbU4)OP~Y*8(L8c% zma~^2I~>aRqJ)%eH=_0gcwuI{+lJkxyCvw%(_K$bTG?ec)@d1JR{Pf#v8P#Vxoxw}kOE;zcCFlv7|Ay9CWFb46L2fOQ8zyVxBS6Cq!Ol4O3*g#jH~mb8`b z6KZQkXGU)yvW42WPGv~)8IR&!t{Ytk9LYOj*YU-RZd50~o5P?-`{%#G?;*2^8Wt{u z!pWZnnTd^)*2bsc5D{W{4%*umQlDFenhr2G{~KO8=1(}(4AaSCrPP3j#n$YV!JdPl>?BPJ-^ENT4W z$&;W@#jxqfAbEZ4H48N8$**IMw8f8(khzY~?8qX!lf8sbN^~Wcb5iRWj$mzZX@m@z0Vym~hJ}Q0%1t6K<*|gA}El=2=PyJr(gBP_+RP z?AimZH5Z^~^m}XQPc^1K@5A&jp<8*?DYg$`C~;{jyoKpOGZWhwO0z^dX-LJlu>xX^ z;$t~>i~|(RtB$lAJh=DNDN`ybWXX!YLVgUQvf_LMi~2ohH-|eD4T7;2rge!aEcab$6stE|1z z$47wO5r%=cX9OiEa3zOfa9o4k+dhK^TPx~R`BU+hIeKkN$I~-GuSGcpV;GQ#%wHB0 zmF27M=sjwa;o6s7b^a_b6$33H;#b6%vt~?ptl|oFA=tR_deS0=%z~n-kg&LME2O8V z%UK6ZDp^i;hpMFSjlH;^8zi-i0Ao_>K)tP6B|mVtFQVVCnf@c+jn7 zklkXAUP6&5W((|pt*VMc zw-7}ACoImb$cKb~floquvH|Bc$)RaOl0NfxYH?$d*Z$M$m`e}v7XfhO^J36eD|jAw zf`l)(;ZcR0K$m*f`|LdkiL!1m8OQgs@O@{h@t{|v`$<$qcd{GwoV_pOA#+=L+4uO! z2~(iT8?zP-9_+4#A}zYr8|Z9WA!c`mS63(`C^DBRsW*}KGB>abQBKykJfW< zXvf|b{yEo42-FY{AVpShT^S-PE#6$TG!8)9IXB6Kb9wyc%_hq83Nq7MhIEAV_WE92 znPUTF_v`hwNBN9fa%fzf$@4=yVB?vn6=JTc<(=B6w#;b29V-L2#_yl*uZBh8Cr%=D z0eV8;eW6LF!w)0+%>{;!`}hK|E)CbPTq2ll>o`8r`T^*f9;BXL172JV>%8DF9%Wp@ zOmLQoM3xDFwrEu?m?j#$*ed1l|I5Ax#@y^3$}AFTOy`;^Dd@6~T;GJsqB=qvM!!QD zPa(~$^J74$BBqd)w2dg?`bfV7R9X8i!G9=()H?h9-Ea|hr2}-!47E&<{$ic7O0z0} z&6Mpi|C|c1^&v#na$So-ZT98u`7`DDJHv;?6H%??Y_ImAl;ux^>z48T0@uOFV-h^# zvWs#Kj-^+z;W2f8l z$tjW>*u6UR+|g(b)C&7>9DJ~Af;fl#!Vmv@eaWlY@Wup7oXnU6EOw8V{S!Vxi^cxb)u z2(^NkEJWjj*?8{Ji~t?J2)-wLOhkG6gtob-?fbQP-fqle_rVWd-)tRZHk>+I4qb}s zE))Ao&Ts@yZ?|{HJSR?ie0!Nbg|4IwD$~mh`9X?_aI=(|{bK$iguHOsEld|2a%YrR zfi1N4GO|k4Npmb7d`x@0tNI9h{YpR7R!oE=Wx4|=K_IQjppe@y_{7p(h?LGbVUHjh ztar_5QP`bH>PKLjf!OZE&3u=`47A=+miI&`A-n}>1qn?_B z?U;S$BQ-Xtw^w`un-kC5#)a3zkAgdKdJ5qPpR|f`&3vj#3Q?h<@FZMu&R-BsVqE0_ zQ3msybm-6_=`76WD)Ia9OFRLhaLdlX$z#*f%hNEf6}IIb_9^ewkmdzBF`Li@82@E9Md+d3zgmz7QU?YPkNZN{4x-(R^t3- zU8huEUA@6_zyYS%7RjH;kqA5RElUr@Hv+7~njPTYh`&avQ|xlg{t#yfjF)Kn!o{;R zL>-q37L=^h@u65zjwEu}nAF&^P=vvNBfS7{g5CIg`+D=~nGEPfxEhLwn&ahUl=ELq z+;2uh*Fk;%Va&=n%sV&8&o3&O z0r5$@Mh(X9RBh6G{y7K-na%C)#L=`$vY~qOYWp$zb_@~Yne$IJ za`>Wj|MHeg9(>%@b$@t;<}HIfEsw8sKz&mmdG`4uVd(;7r#iR_Eyl&Jf;ZCYXt%@h z8}1oaDCe9R~4a#oSv*(ySAwu5)7fjs5t^>*FNH(N*fApQtWK)?Iran@EW2F>w zw<)tF>xwGIQW}-}0!MQYqsSEs1S?UoaR^w9ovUAXyIfg?6Ta_dT{p!C_b!MVXzyE> zLbU{jokcAb^E-6AjpuaWwKN3bJ)S$g8OLWdXHMvu@bRbPGYr<{wr^0}0`!EOHi;x; zvap?JaZ)}0vHOBoxA$8oQZhIoAohAX5PjsI#*GhV9A|OY?k$8>R)6{U!2Vsm zJs>;n!pS3(E+Q*-?r&PPv%dZX!Yf23w7@n;Jm5PFH#Ji1>`~ikeD@Ze94m9@SK?%n zM(L2`Fw7_0re+=KJ+KcQ(KBB~2J(*`}ZjlTwN> z&>z->7Rev~#*AN1L0#|eBZITtUAAWdG|TlbdRLY8m`o+KwW(E)s6pU7VIvrW5{8#a zsVUY!bitaMb>;T5C!rOQ14Dj@MGi8pF^4|U`g%RSCd?%D^^0DW#&2jM@eEZtzgVa( z=2A7bHUBdE)j4_v=u|@<2{Czy%fX?E8k__z$C+KywuseSYu(@?Yv1_z_}j6Qt8v=d zo}x7cEXWE9=-Q-*^A13BoSgZ-1*MsyKSQ2O##&mcOuiR<8rE`i|Blwdh6vw(j9EL5<|SgmXC#T2R7vTcXjFAHnhSdL;P<;Z}oZZ}RzTkjL33aGdA+h#z4!w8yZ zJU&MDPdt&6lh!yW(_n!(w@^-)7iZ)7oecvUp8EMV=F$Ot#`pIst3R>DNU5uW?p6Kk zuOSQ}lDSs#_c-H~c0d{^QsV=`o%;cdREcfZcHSzJs`=8}BsDguCP|tY7}ix>jJgcb zRBwl&``m!z?)e$5IThdlKjHqmtZnF}BYUVj1J_&|_Yt0igpOZj%Zz@cstVrao2e(= z4sL1DXF+GWMdAM@vWpmYQ`xWA=_io35+MP|&)rWGE~Qjz-=V|5pF(PfHiO!DYTX@a zVq&>iX9WyZ`}XY>{hyvIuaGl-@YS9txgt;Ri{FS-0Q9sH&&MN*zm?W{S;|)hrh9t+ zL)*|`vNM8iH4QfSlS7B(+(jgT6~Me-#vGB(0igTQL);(akq^_sw#8r7>l6R&#TWVu z1dYWxR>qjAo`=w+i3P^VxEXLbz9gBJOd2C;Flh3%g9RspxH8axwXo%DSxdjcLM7JJx-+j15Q1}&m?eGH8z>&g?G zrTO`9Dr8D@*_1MK<>bkeHh1^XX9GRQZ{hfo1B-|(t`FQb_pv0{vnru&!K?oCsvV0A zn;B6-L765MJR=5TiyUP{By0z{kfw#uBM+$mWbJSVqC6jrmP?~NOiFChxUsl?&?c=z zW)iLBFI%5$F!PoH>T&-4%uvs&dQ#ry;;dJQh=N4r&vH~aVp-&f2o3_dPvh4I>$dB7n?GLMRk>(jf=-u(0fm6`m4`EA0j{m*>6Cs!m0g>^)nhIz05W^%=dvn8i{=6g9c zC~vdmvyiMN6K>s1UiM~TQU1GxfcsNN>3KHp({?HL1VKa)3=RlpLz-+HDU%KH{>vKc z!kY%w(3v3Nf7jbw%f;Ut51}tK;9=b8J(p{!t8TrmOD&rf`1-Pog_mRKHP-LJ7Hj-x z!G42)We5$|t9dSZPL2EAR8O=d1T*P>NE4qMgmOn#%opsFc+%BBS5loO4x`1Oe4Epi zo+r!F65{69w3>7s8Cb>%FXJrM?C%O&TGkv8H1Oir)Us^&>yQ6wqsRYYKgj>=$HW{f zPcLaNqCh~vr;~ej6|moF1d1@6>EJ*w#;%z9WrT}dgHJ0B3mtr(m@5;JMGS(L>CYO% zhIha)=MCisz@*a;s9#~%dt&vP^s0tB#%coXeofo)=1SDroS#}gC@4rqFA56>7*CzI zgk*rqy0#xj-&A9LL#kZ6a9fk|)b}%JU~=#uK?4_j`!pl^&DsP=G#sA(Z;F~g24MO8 zTA@} zYI#1;2y4j7)vH}b$-%?O%~#&XV*>sg5g7Z-W1y}3Yn?u<2|CR#0pE^AHUl6_;ohs) z$^K6??FSb^nDpfLN$r|(-+Y-C?a|Y-&zxoG1|Z-s1wJFOu(1(2#JYN`RtKiv6$cR+ zNALNA2M@pvL(odzXASiSnMmjsbhu{Cu#F$RNQv;ne!#?50VrTbtE-x#HRZa+pk(IiED)ilLiEycba$ z+HfSPoOH$P!JsUo8t`*`kc*VK$07qqaSl0%QeY3V!~r=Um73|$w<}7#UL@I7obFkd z`80nhWH13AL?SyjfJo~^mXq`&5}Rbe?@Pm`>gAwkN2zQK-)DW$8pG+Cju}0#jy`iC;{==>#Zvficdl^XS)9e3M)azCeKBj7L|fJ z#xpMc8JTgghevP5o$k%KpPegJDZS>PbL!Ls=Z=5gPjN%RR8EkzE*wDKRlm= ze$%m4%d5Vq)-p;+92g@DVr-AsppsexzU?S|BtdEQlWSNDnbdJ@tj%Y7ldXJCG!J3wHo$Yg{>CebD;2rD2L0>PN=?X4d zDo#<#@|sIZFEnZ62~QxmSBL{iS$dhY*~@HODb2*Q6QkLqOYC-RdAnI=`j?fA_Wvvi z-G0rxTPu6~yY+mRRaCHQ#gV(UDF>G%LrFY({dyx<_A#@+EcJR|@q~~(B8CbM?1(CQ z7IJ+J-5ruK#g77c=w8sO$G|~@NckA1K9+xIGqYFk-ZJ!78V;yJULKG$bJjz2vaBup zR4|=c$m3|p9B=d8Cu9`omeDNBwEOqRPk!|X(*XxZ)fWyK{!n^6O`h`}Qh+kUL3);> zX-sPEzh3?Gz`IQXmiccVGx z@gIV{ttVIxwyifmV>r!m80f038!QfUnWNGkRlBpyd_b@c>5kTGY+{m8Fem}7S!|00 z71>ME?#4%FbO#hvh7VVdoa#Ddz0Jb#JnoE8H!z3$DY9LzF;rgCKkgMol6~yaJD=nP zAlB5tHvZ?cvPPY8?f>^5;@&~nC48vcJK04Su_^K9V!ZH9jc5lGEqhV)fRBNJIb!pc zRyP9erDcIKXNia9u$M6&YMBJ5Cnm~0MQ3ZF8B-$)yDIbqb^0OeZBRuxni1n&Mokc` zrk;h#4cm8w7P0gf`BwG&$Z?6xw)GZcJXDUuTrWJ1$kWy6P1%~3$F>k+!yLK2@i)#o zZOtIgDk8x{&tnI)(tE@4Rx+iu+0rlN=$=|OSoQw>PM%mU0v{%s=s91L@IlEbP!Q_S z-!gbQzO*k|AbS7zexF_&8s-)L{Q1*SfAg3X#w}|NzjQWxoNJLo#1dVf>~RCPb{rSI z(6!Te&tnd*(L*Alb8}t7ESCp$Sbn?KO5>I-vyJt?mOge}n(@b^HaoRnA1|Cce&@V^ z82+bimzVn9Z6k-1?+5Yjc#0ILEh)rJge=#zP=1GiTUznkC=NWtz z4_c?!y9=toAy^fG_S(wcOTV9RUy6@u6t?V)6_qXQmwX&WLBA=8LDY)o{=>h%HEjOZ zy;E+(iIZ0Q2d*-Jtlt2WMHB8#gu~P>JTdWP_>b=c>y_~SMxTg)YcwuXz+{3bkHN|~ zmBK%sWL&L(YNMw9fq@fzo%*D#m_1CZVZ5Pno$p63WVokvay*an!%2SfJ(1Lv1C? zR8*ikb?Z+4;lY39t*I|wpLutgsc?(A>8J@NKfk}<_E?V>qJi9z$-If&NzO^bAHSJ%t%wyTNFIqCZ1D> zM~Q1ln?FH!ER8L41be4WF;4IMS#x6v-y?9hdxWEUj%W#NS;X83Sa$2Z0 z1B{gcBYDW$MfXlx$+!{5I%NK+#4XU@3}+Y4W70F<=S>U=X*tBl*kksGt%-?VyHdvz zT{}!TYX|JvP6fC#I0pX>!s#xVKu$q_Ej_(B;%VQsZZ+?pU1G~}r5+p=dlnf!9HBpQ zLgHCD1A|0S2%FK6Q9UrPs!Q0&tq6Bp*})t?Ipe|DIk>qf9s*Gr_Y=7bk=!hJkk9zx zR|(&|eDxT55XHqs@a|QsS2yLDh>?tT-WzS zMLylcMz6Mb&zFN=OF5gVT$?gG$!b~oimJFbjC&#h-ek8;|8ef>#Lkd2N+i~Cs@u?h1;nLn&`4^2(WCT!HGd{7eX(~TurtCsdAKEr!42Y{&l&oyl#k@(BvT~V>D}h#na&%qT$?JE z4LQIyucfb=cTv*LRO* zU*{l$sgjKL{G7y#C2*f`H6LRJQ9z_8CgB#Kadf_sWI`JPzxtg!JtPeH~xhLc!iTG-r%{YqHjERG!2MfAOjl`N(!QI zWmrZwi(C}o8eF-slBy|l_wYh#4_l+G+0^pOo6Qp;y0)fIvu46zY}^X)KtAYgi9=Ja z1T)PoM384-bMgw;u&Y`HFx4np;H)2RZ{75@!TDSXE`BF1WbJXBv$CeqG_P7rMXwY1 z!RyQA?_)8ITB-t0Bs`>XzvE?{rHIYD z;#SacHLz2{2Nf=nU?5en%e@yTFDE4pQQxeLQyi2tDAuvr8z!P4!gjDb(Rr*VX4O&T zP@O!%prsFgmIfjX?9Z`ZeEr^89A|X#p8K9~{8L)X)-`hmkLe>1NWX_qEx(RU1FkJ% zWhH#b7`|ykyD^6+#!T)A0jzk30l<9Ll0E6(rZqZ@URbU3-)hX5+l~tro{Jiv_j(uuS7J2ppCz82XO@z}NX}6JgH0Qv%=a&swi>Lg;ZPwF z4Zpb`;_S}NQBhGu&+P}(xMB|Cx+z<*j*cCw@tq~y%|e2hq`NJwN$cyS;jn}$SFY$P z2Q1&>S#&NVFex+pFYTTGZoeT_jOzhvwo{TwTMCD1rk^Pd&<3#YX!l@(M}BLVYFj>2 zUd<7pqF`wbZfnsp>wO2NW5NcK7)vpphv;0UYsZR!lc^Kvsd{olk$ox@zH=OUFAMy6 ze!lZe3eJ-hSq4*YG-jM##=ReJ&ov zii0@ur%nF(mg@mNv15jNX;rLR#~dcu@msUkA2>bYY1qb!3f(R{O{<=6d}#l?`%cL( zUYR+W_)V!%otEREwrnP+F|*DTuRi-VuhED-JuM_jW_+GqbdEzGc&(c}SxJ)fege7z z@_+W?lS;ORl#HWrbvpgm_Z;$Q91a{1TgR+nDQg`z<{c}!@aImF|S$kdULu#)wv)@i#B{-S^|`R?=Q z?b=JQcuVTUP9FxZTSz*uTD7VrVq6^#Pn>=aFMKH^m#XRX_luY!IOE<43wClA&gUpZ zaKmmr*eqgYCdH2USjvZGb!9JplMqIM>Wog3_sP)x-AjJl3Xw?=n;T9x%9^Tw{^_@A z(dvlVtn(Mic|uMKaAVqCRi-P^0?hGB z`tpMKYL&IV9qi}(lL4bUnM)MIc!CH{nUQdfRcOlV3CCydrC~M{FdAC2_)mrIk*+2~ zzlHUi{Lx|f_;O?x?+y|Wc6D)Fjrtu5VHAf1hT$;xlv|YiB5Cg`=Ys#^}Xvt0p*tMI9K@S$c>C2MplJ)t$VGg-@ zJpQwE`qZI=dd*vms$qb;O2V_8$p#*M_U26|Dp2@>c30|4bj-W4w;i-u6PN)xUjcjJ zd~9r`7(xm#ov1FX{wc5rV;Lr=c7HX{AB=!F?#!Yi$h|>u0Jj={Htgp+tigt%-I&dc zLa-U{JAhILc&FWG_`wN@_B_39#M;QjHL{BIoBGt!Z(z0DDXOMc)6~-M%)^c*`{#3v zAvk-@Gq4Rem(_JvKUvNxMs3AqxfwS_IQO}LNeNk?D`%tfPvq<}U7Yjs&%lPcd8xX5 zGQlt7PRFQdC`%x+fQ#!U#S}gxC zF za`9m}Qx5P6wh6Lkole6J0tdBg!RvcFpM6<7_3K!eHpZupROT9$yle#$Hxum3>$afi zoo8`@dL)jP`XwLIhnX3+t0VX9?JNEFxyzX7DpeR{unRbRE@zeT>2B>GRC!)1n?ncD zY{u=w19)hoS>dwqiDeV6Y+TO~j0x~T7yw#w?xfr?J-ZW!v3v(nNQHfQ?2bdk$}{jM z@wx!_J?Ezlk01Hgkh4zUE8lT^I-*oJrq^jNfm^d?7c+9K_l*0eMj_7K!mM}*AF@uL zmO;Wp+&giEVxbL~>(1S~QN(0<6x0t`QZduVetu68P(roU4asB%gJa>lGo`F!)T>DKGMHGM~gn-8#j*Y!(qWn1$=MxY__o9=XKc5BL(c}II~%GJlAU=5`pIvNZ1W_JGV zRK%78f)%h4*ud;HYxMb8Vm*cZbn+a)>*nE#Ghc0aTc=9#x_5>34)Lt|xw$$x8Gb+1 zuhG(nU3)BjXqC1rvCzyT#^*zFZaGw&Fcw7t(2+eH7@c`!C)lH{k&!*jmX||g*UE~f zYkt~>Hhpt|KAi900z4Wx7kTfkrC;ShdN?`ynv?z=gPNFfwd%H)o+uh2$ASPm&9{(K zkGkPG)6vqFPpk;|%eC^@L1eyTz|B%d&3Syx8a-q0#t{%uDS&+7cl=1HA-wCFZ}D4K zf5XrRx2BeB=J|G6wAJC0u%Nel%fBj8RAc(2e1|PRMr*!p*d!}hu7ty(}8`A82kyFkiqo!NP+*1?<~Me>v6B@uUAlH)bAs2uH?t&(`MAG zXHR1lg-aw$tRUg@1ED(tdM$t(+hRrM`qc@^+c_XS8gxS@xB^fddLTVJqFi@Q zU5VkFN)@VIwQ3E_d8Y^NFPccm#y+$3^%vR=o%&56YeKt3y{OKMI!&pxVgMBiR)|q} z+i+k^o|TqS!``l};HmJ&uIt}0X#^WH`mPLHRR7*@=m85@iujQgeooTg$w3XxaRiz1 z2HQ@b{x&a}{Birt+7ITu8XZsmZBAC;ZFcS?1u=49ANf0~FW~${Kjt3zXmyC>r|*&pdebcHx`;(0u#+c)=4t? z8e>u>*VtMq7D=$gb4V(q9#R=OwIsz0QStoc%M`1}iyj|Yz;`(!-1FVu^DK5C>@{d! zvhDN<8|!IJ9Wcdi(CX^xK)Da+D>s|QM$h{sZQVR8;n<8g-a+__p$Cd1ibT=x-4jA_4Pe$6#79#=@icT z<*j`-ZtQ-XZK6~O8@b7-B<0N0Yne~zBaDF9cV^eHdD20Ux9ITtQ7X4IV7Z(8q3A!Q zzeRb)qn&3yX4{&*Vo=Q)Zj7p6QT~8Q1L<(@aN{reDfL$a-@VbbfBi-N99x@(-e1d~ zbLlubs?b266RrA?qHV*Zu9L3@4=89;li&63*^^7{m6bDe*sykNNKWD~&0a_k*z+s? z$u92YQx>%~QFyo4ZqZ`EDqoMbujZbJwz^X1Uf+B3^Dj)U`s+;}>0;`N5RMNQwD@5` z8G(_7QdHt4zRyXmDSPSu+de@v>hfqM5UGMz zg3$_?Z{0cZ5M_|`LBKZLZ8<{WoOqzr zGt!0&i)yUGcvH7gBl+)X5Lzyop&O9LLP)#qfWAa+f8hV5yNwinQy>|CzuvjpNtZ6Q zWwr#yS{nkPTv3At&aY{OOejKG@miOgnIikkG)O0=5g6a^mcU)7=&;GX9px{OfB`5PhQg&{YPBhb%*UeCl;N#2r(r@CR%mf7gk*9?OSFiB%I?)6qw>T-lDB?+EeZl^yKtj66+L9(JAR|siC|#R z`@+%o?nBVuZBlFWA9}g?-#rDlKH{)d`Cl)f@LShfTN|bDPVWt-axj2|W@kmGIzHIp z`0Br{9(FX+m@bc3W_zO-MeD!D;N?adP)h9svYohf4mZ;KmWi>k`QxCbl#4S9t@SG| ztl4Ayso6U({L#FL#TLyw#s*`fnMTscjh(FK7-#mA9_(%Hor}LvtGRHhE4NO~CT~7{ z`gGD$xze-?PWrlR(bYHH{F?+G?YCKXz^-1M?HC#r^j|N{CYS2TUz>u$6rYAX#;nU; zaCSVMPT&ycJV$M)#w!}u{O^6zNxP4&j^+yew4h2`qx#~&H9@qU%c$zy+rkB~YlQW_ zbN78}W%bW)x_Eku`ak?ye=NG&W5i|IAyPj;61h^(dEgk&bkZP)*z#p}R1t8VB0h-D z0KtisD%@3nM8#X}#%jO^J!Jwe5@;w?E<%59Z#9>v&^KIQqPaxlRaBaP$tmVnw`M|d zwr?1Pn_o9Ru`4K9+0!fCwO#Aa+N{8{#SzGtwrtu)a7sOsCl!dWpCDexX;?PQgo%p zFjP!iN#IKdR%a-K<<`g0rTR|;@kR0y-mDR ze_fVeYm}71%tF%s4lo;4hLi+OhD9-)8o3)Rfa!iPd}ID&H>MKci6 zS&C;&{yVGH2AZHLF8*|IDL#>zas-LNacN&h&XaD;K#1ED`VPOxdpFp7KXVX&s=ti;00J^l45H!rT`@C=azQdK zuH#638pDJPbgbZc+xX!{P5*t*jO&n!&~p0F?mA{|@;U9(_+1GrM`;gVYD*ghmFw)> z){;-7Ce3Q6d9&slite8tjSgXu_~V(s*|WU7PnTRFfuu~mk5WUUs5Q@Na_9P%_iNt1 zN+yth*-H>meZV7;0s}KE;Ug``dp=Oj>NRiPpO(Z^+Ro>^%jsDLuceHO0Rhq^u+)-a zVDNr)U8|h#4N$xIy(W z08%bc2wO_=R6D1mh!h`RrVoNYlmPSN$G2HuOGUS15cI4^MHMo_BdJqqX zVYjm^BVqEs4!Q%Tioj4-lKAsmKX(rgqqt^=Lx2dv{%$Io1!1VlVJ-;#eS$C^4rd_b z8BV#6B6m!e;k-|9y$iqc-rA?MLur7Vst5g^h0z&_zbz{H9M5_7HZ_czCcXxMPNL{YMkIN3ho&%+3G^+HR z7dI_@wna_wovTl;+1+vc3DZdgEANVn)|91or~bH08+LJ6sqHLImNAF-$;25t zP<7=ccK!z(?fNN9zqN7K%Bv~S)oFTSxnG}D(daJnYy7N-he-ItRWg8!8CK`+HEYr& zniK?<*5TFsZ#QUtAmMzrvs3wX@gw;N&4;Jl8d61P&d&I@kJ;*CC;=y0pIJ1Asl0`Jr{wkN zH}m0%g?Hf4fvbW2ydp1G2LN-A+!!d56=5}ud4ro-B>2}4a@WGKX30A-jVj~IYt4oE zWt}*E+LhL$O+_)|;v$8T3yww?Giz_KXq@H}{@5AY_WU#hJxu)|hT+*gsDXtDWNM@k zBCOEukzM4}^7|*|3?lro8BRMHG;FQJ2jLI#EDL}~EyL+;66$H*_O?Bz1d*VkGgIoo zv81|AmG8ebkhhxHIe3g;b0r*@74az!$!R=|yIGF3_Kj10xHC1dwY$}1y326bN*C9- zqf<68*{H*c^Aqos0oCRgjsF)=Qr;y(rR!TOAP)qf3P$t^)nkIb@W6fv2?-D!eg_i) z{6dBI+SsbG{DMLeTLZ{&_}b%!y>mt^@VxP3aOfY-dFN)c4>W5%*koHf6B&Vp0^5f0 z5Q)1a;#3=L?ZGOMPY8f*3VSr%Ag+js6ZbK5FJRmj4xC(7=Pp!z6ih&{UKxH2?j+E1 z>Y7xY;m6$6n^C_3;Bw*!;H^|Nl!tLJmd`1EM&(c+4Oa>FPrl4lG^GkFm9u-0=91O_ z|BNCwbz~Jtf>CscNmwJ@MQ|KP#Mw`eZRQaVH*HwAY14M?J9qn)+Afepbhp{@73K94 zAs$EU?Va^q&b?%RWY>=2wBb;RB+%^P(7703e7}A!1YoeB$Eg-Unk;#HzH+vi2Cu5$ zJ;Qk0ZQT~!gqSn$eCXg&Hd(uLJByjC^Ct1K`&t%V#s`|Wif zoTty$71LiN>t2Q#T->GJWvINDQh;vS2?3z<`dL}jls9c)5&BSOigqX?GbOPb=nBvj3C1bmPCixPorzf#fyVr-c;fh zdG>NO9FZ}8m@K4~EzcHd>L>NJrKIRjR*M62NlHt4U4S?^Mz$T$VyIvo{$!B^Mg1WK zYAd1t0N}!aKBHZ6&TDUNc54CAG?$JxYIXx!0s)&Q;4`x>XPMFz3B`fHvGvBze;>oS z_t&-zsX~H81kAE}`r})<7{VNzqcy9l&Qk&|-`B$rS%BFVhlVvx2G+sG|3p38P%nq7 zTG|BTWx63U&xZsVH}7F`+C| zA#%()&hi<9J*jU+i6gdWQ8C1U2A9B{3uMyHr=Ih8f4#@3u#ruOdZJUxi`XwN>elGt ze=rKc_Mz<=yAZ?;)`P-CzBzKT0|k^y8>6O>_S`jhU`?3M2*J63^L;rV~#qUHaAi+=s2 zFUF7dY5!|7;n&Y86k4!povQygN$~q$QwF}8zW4Xv_>Yubzpmr3$>zc&B> zpXEOMKfW!mW|TdI{^QuAxBC73F4IDVs1jO^k~Yw_Jz1adsYjQn(0VYZA2Wt`(uD`K zxuIS^-UpYJJt~U(+_>S4bjUYB9#KVx>w_?Z+NHY6S~Hzg!n_2;#BN#^xfv zDQ1g16WDFe3P&Xtoc~~@WM`rOyhKfr;l=_QUNpUi(SpyH4c@FXE%M1Ho5<*JUlGLZ*6=t;nKo%;2|5uwM< zu-9BipI^+=VQ;KgYGVay^ymzI^*=jx{#EIYnTUN+G%M()1K1L7|DvwyY%GJ(i$;R5 z3o^qtff)^u3#=!6KJjD~A0d)9gKU$U>KEKCRJMFQ>V*RIl2Ie0NDE>z={6(xRQw_KjloOL`w%UG|v0r_P4B1c43Y28vzG zfRPloVQNoW|J6f~CgiKX-d>zwAP{;VCD$-UxZm#|8d~;4=YC2jI|Y9@wM!&=i|Q<{ z)ZyYbKn9R06X-Cv+e-1sv#j}>tC`P`s8V{b@^Y{tZr2uZ8|HG0wiMkU3;kr7lt1o> z!zzq|{=sb@qbj;EbqZ`4!3!dRzNBO~={^;njx}`T*eOl;)^GCfJK(Udv#@J@%D=CL zQ2si2)b?QE9ftI@MO%EMoQG@HWhO0zWsmOdpTB=u{Mz8>zFk6Dyh{5Vredg6P41hY zSyUXhv9(g%uG=(9?byb@*L);E=LzRGlY1Xm*9joCT^Lm6Q$O2n~UHPmn6)Dm9dl;_u20Hhlk8BSrLN!^u!96x_ z9EMkM4v5_c?#MrhLo&umeF!+t0_ruWUa66jM*lsS^dyKGdVfkT6~_xpMf=J$16e<~ z1h)D6`_H_0Vn6R*2n)zv-71(J6)}nixe^8^J})S6an|I%<|}$LL|Ys@cv(lGq^N?K z{u0hk;+1T${9wiop|QGR#frKd4h+`Yiit%eR@jrPz=+EWHTx`J4JYJ^S6(zq45b1y zg*%X#16by^+Fg0V`$x)=Q_ksCv2J5&(B`@WhBJmO1=wB=>0fMVO?@hN@H7YLKHXG z;rF}f*~TA);0uF<@0;@xV~3R(RU)i(N*SDKb3?QIf-C#_)M~G5Z0sx4Eg|Oea}C(=5?Ve1jicS~Yleo=22^lu9B0Vq2aC;Uy}gMYDhmS=@@H;m zO_ZDXmrW;=9bKnx)OKc^cwW5ND>WsHId`sZy?QzbE3RgmDqsHucolb~8IyAwGDRu4 z@@Z{2=?EqzQsbC2mso6d!P7MB%4sAoO$*ZwZE~kc9;VZHXU2U}+iqh~v{Fi=KwOyS z)-|{X{1*yENHoqmPz6(u0kDUcygHi3A+jCVnx?g|tjL#BMP8UuuKr@!R;pye275k| znn4vzVZY<9`u)%OK2OAtJRisUb1_7>v)OVCaO(4^nXlS}GCb<3HU}Uu^Rg?Qn5WM<2kPBrO}9SeL;T&7$(0SN+!~YQiUy`JdX( zPcE&KU?oI3&aU0xID@o}(gm;R?_0G8-vXl~n&+R-^+xo7`Uz#6-3~{M+dX!~6@#Af1(Dm|} zcARsz5}%lE&6=CEPwS{ZhrcVAypu9 zM7Gs4!CB*SiqN@ zk1a>_NJOc)L^-7yhx42==H7&67ExegFEQgo8ZM_BW> z!r$heoEL_OfWtXLJdKkzDYW?tfzWVgAD?fZz#x8O2-8d`fKiBrrKHG$+_)$%^CKS8 zJH1D#-{tL|jALydTR@ZuK`3EP9tl7QIdT`wR%Sbm&gr*IP?%tQ7J*3IUkOMUc_h(B zYu30T?aBi!|K7cNMey_s9weLw+|Tl@qkfzIm26-OCEIv_;l`?s^or)SkLHf7LV5<5 zk6e)b6vTl=ud`$|7R4>G>EQsJm&h6HahB?rQ8c&>il|HUJD4JaAk9&UKEh0e6B?(RAF>f?BS2~<^!~;!TVFLw=wXV8UHl+1zaUw%LQP@9d zShiuSccfGo)fZ}AjGusA7e?8rg5?Q_$Nbd%2)SY@XGj+eAp>@YxeG>Es8AWNyuEReP4J8!k zoHo+O|Me%UH2$6~EgKBT7R)5BRNx zvVe%C`ogodQgJF+m*KxN6Cn+$ETnwJstk^`kVJtSLe_(j@6dLm;7wACvE!ZDcCIYJ z>xXrtFNBwqWO(Z8@U5vCx*T~p;@Ya-Tu>df(Tq&>&+IK zoaS1_{>JV@i0Jn@cNyJ^-4|&}gYFK+r@uiDjcuX%e13JY-Rn>E*#vRp7Wgn|D7jvFfui+Rdn?DL(TrxD2UcSmq0Bs)Gm^X&G|0>-hHvi`Bo^ z{OOT#7Uj)4s{i7h3>>OjmiW(j|6dYQ7anAk6KK0Z-57z@&YJUepz;S3V&dKO)bDt% zQhPlUW}Ay2uKmcCgkL!5_(hr>={JY5LP!CznupFa3VPZ|lxEopRNS723eEQP+~4ED zqCs#L$F<8IYdHvGQ|aZT3!Gv;@M{T6$EH|VT#&Xk&NhUI z9SKjHK^uk}!`q?%g6jver)B!Q9F=Cj^y_3AhwMRf+5AcerG`{=>M!68zL&zO6Qm}b z{zwrb=}Kv~_&PooOnxyG&DZYUwFRB#V8?u3Sh#!XI?b;~9eMd>HEP&3j-rQa$s|$F ze{`;#(h@1qnko7WFd@heLI14I1w5w!Dhe~i@H ztz*^{--4wOYMAnAd$J!MrrARc?6}Oh$=9^Ra3BGfobCoXUbSY8j_=2}Z^a)Z?oYYB zpFX7}&X<#jFu2)LKUxD=P9_jg)5dS%pr(C<{X*bnUU$4AADMT9pgMQ{e9zaO>g|)_ zBgjOowsefCqci?wamN9~PBCd(x-z(1h&$J&8H{r^QGYnUbLzVq}P0oY#dbFnspo3*t;; zWr|mvH6p&xOx!fl+tdoHV(hXkpIWNwF0*2Y&{0%8&Rsn9iM72kxAANQzhIp2_rAWg z_)eSl((xNOU%C{l#cBdwO%Y~Um!_G$#KK3Cjs*VTJyX$(OL0!2X30&H0R&R(Bd+;r zOEQI`8W-!d4vZwkzUzHW1lei7t4{0SHKbx^+O-9&3alkLDl=FLxKq8SHk}2`qw0F( z_>J5d$zJO_>y-M=HQ!5@(0La}>6wOxK=8Rq!k~>dW5v_v($y$7N2V2qYDhZoVNSV0 zD}{N)@ut$oWyen!P!4NtDl90_>(S%5QxVAI_#%xd&99df{(vEoMrcg;|G2Q=mQn`q zf-+|KX{>GyMu&DVVGvq6$Q5Vw0hBZsZ1f@&Cq1n}uMQ-ca;dnfR+sOx z=SKw%u2M~3jS8DP9y#h(zCk)mxehEVH|1_`4OI6lhpRxT+B^R0a9x~}-`yy_Pd!sV z(*9HYoEMrHx09|tjL@kuR2NawcupU`>iopRlcuPz@SF|BTq`BvLV4upq|?x=gp9!6 z4=`L>t&i1_c(+YShtPPk!s7>LGapJHOuh3b6RZZYcbY@xdl{uBl4A$G$ zCv!Hb-O19%mA^}qY8#B-RkVd;8zXUa(!TZX$W&b#`g=x)dQFj9Cp+)etAoVJb{#v8 z6jGLrAzq-cjY(ST)t}t(KBdrn0=M{X5pu^S;<=X|u|vJ286Nx5D55*fTixa@rqh5n zh8ZvwqK``d-^n*@KP0ei+CpXL@9Ly}%?5iZ#c5?X;1p7s)6m90f8CJ=5p5qUdEqdz zZj^$+KD=aAeIU9_4U#TTG_@~ZxMDFW9FUjy|Ia@m~yE{gSUxFrtOgKcRPx< z9X;mAKpIw*mRp)Dg*D?c+K6G4ki?ssJ2?f5F$FST3`Yc%i}PNn3H|;yP9KA0xXd*A zarNt0r$uZp^m}{UKCWi|ht#-n=MPOaoARd=aX=I+BD^*B^JjQZ`ka2IW2)88CN;=i zj(foZ+Ij3Z+@uaa_~CS=PpyDpV6ym1KH!pZg&Y%7=VT0lUyL45O+$LAvzT8Nbc0Ng zQw7uf=L}t(-Sk0AO>h#;B7~c}`!0vcijkUIrgORtk5PujYKM4i)&!a0_80?D1~;cq zyM=vARWx+s2*Iz=AA$9)2w2)NyK#0AaQbzR;pBmoMK9~WN3GN20P^~NoJsg1vc+U$CaqxgR?^k%bSv~|Z8pU1oTtgtYON5(xI zpZ5v4?!IVIIO4V_(EMwtq==q#2D1Kh*h&)I*?YnHebpJoFRQOz4~`$6z6%o2lF&>0 z)tRZM^4OXgppB20c3=*YPMo9K-PG&KzCMf!$H!UGC4vPZZq3LiL^yam%h*riVqsCL zf+=pVJbd^w!jW#Ye0nC0R2iQ4)i14Zs?KXOp6bAq z)E)c1y%j9_tGO3-aIIpxPF^b1mxxma%yK?2qIAIZaqcgyi^Jy&yX`2!H zgocnpixv^vQVqsjl+MI+OA?DU3JOy7vsbUC7ixff|Aw(17L1&$Ej?pm{Sc*`4i=Xp zU2P=^rgO7Qzi|R3(1_#ed;?J2O7!{}7SK)O`fq;}GYEQ&H9;tw+a=GmOd?TatR-bJ zjX-(iJ_40=?!naxl$2o!^^em9IQ$oTew(LreOUV+QUzV_qh*6Fi3KuJiyo+o(iHqF z7&)f|1+9dCGCcB=J70pl*1k;8cd{g{n~m?oY_6LyCRVR}*RMDAJ*K|7{d>oTx19tt zoPD(n{B;3q9i3Kmj+qiZWz7V(hr`>-ysv+MhPtzimwUVX=+RH!6)IJ~>0Pcu`;F?= zqZpe;Br%vsj|b|yR+9Riyi?D^XF(9qSw6RG@m23MwH=P83Si6b6(7V zyvMaAjA!pCV{K>C6&hpo{p}#7B{1}4hmg1xxKnBL@BbgXr>@EEGw$;ubDIjfkp^{F z(_eKY!#A7?{~#iGy~-WH6rT^58_f#$`uQV&vu0r#53MwO;DUL!{8AA$`UckEY{ zcFm)TdJ%ZfpGqZy%4#{iCW(}f_nZ!2{)hrBirWJ?BaRlKBk0oE_Up#>gab`J)TA*F zf(Z{6XsMFe#4jj%WLcxIQ1(<;CNIRij0O-l)_O~4JR(jrUZe$%hKZ9xL)yy`gbh9; zenh|o_MZL78=7$9m0R*V3YcPcppR>1kt_1j0;TV?kY;3a9?-#g#7=uH10ZVQpuExR6iOfe000bhXeLU_*)LuT2%2^816| zK>SB~wM3H%o}^4aoR?$I4SmN|h;TMg|;V$R*O+00Od5w?#XrR6OpsOY_jG z9z1JxVUW~)ER_rwO~vT94w7yfs*6l!C+4*G$P`{G5fkS%nUlbeu z+7uZb#@f0VdT3C7$~XZr(!TDIAF|y5cd+)1h~k%S>R)R&?$7Iv*wJ|a4bu$4 zi2YPY^Rxakq=A2iz>a}bgq2PyGb(r+IB^4X1(bu;2@4-__UTg1y5t^TB`1#pd6_eh~p)c;}{ zi?-4by;_I)=-PEV+1rVyiq&_Mk=>NONyf{*KhOo7&G>ig?mrgW~4@arZMD8xc0 zZG4CyJ9BC<&Nha1=8A1ru>S82a(H-Q6|R7uxQ+};rI0sYNxSS^}(rnkjYA4+aZOY=( zsI#75WoDb$WMCMWYPKWp7wt0!O~LKctFcn;$c==mI0j%77mG?PO>&PY17vYS%uKzz zQj@}lpgK2K%77Va8BSX1q0C^K$1(ULc_LC=q7t=I{?{TBGbAzsW(6_b;s9X5;*^md zXHoi56rBsLd3nDiBxo7&Ahrie#ejxh9MvO)4?s@3-m6y{r1~LR9d`hXf^E}vT*zOB z+of{kTkl}2z~_MtrLTc$CZN=-7caIN`xE-FE}m;9ubEqKOpYyK8;^9DI`xO+&g(x2v78S#G%ETHvFW$`q}H(ha{;>%MOo?u6cSVv2>!4 z=`!DfJv$tjkK6PBpUvO?cm>0BDlqVW3+abpt!|Hrc~kr0!US5xWO9W_u=uOOi3!4h zY3JjGFL;=kC@rPbPB=%c*s7{4P9`yT2eeVO8C9NwU+9wu9)%qUHW6e=5BegHEiLuH ziEhQgNurQsI=HBpqF)sn)#Wy;l{7m$Yq%v@8X}nDAmPZzNGh@1-Xr*Fr%61}3>lYU zX7qJ1AjM+#?8A)kj6^K#o%H2@u=n0kRi9h;FB%hrCTg&E)QzH8i3K|`Ds0p!sE7?c zV#kJ9P*L!x$A$`9jlmifE7*I-L@`#BX0JpL5rqhf4St`wH|Cu8-ZAd)_x^FmxMSQm zW1O5=*!x?a=UHpbHRoJ}QT6ro-TF-$Sao5ES86-`U_{aGAy_IoAfuHpAjnBJt2!MY zbK`{*g+0pI*lZGDD*qI=n3^aIJn{gk^;wi9^*$rC!5g-V)+7%YDl?|0)H5rHF+C~+ z5n5+~T7z(oBWPUJZ9Ber_}s*0;~6)(R6h#+K;8p$i2C)}tXjEL$`=psvjnzA|AImm z2~a2E$uXLRf#UC@;SemyL%iMa2k!r5V;wt0178>qrfOBXx1%s<(06~w{0|TFP3K)Y zcXpee+4q<1N8j%SCwLCC!en611dDpLh{h6sqtwAIbjcb!iyPvaWGVz)o@$~6oW1TuRsd^Y^Q>p&^K z!l6$?-|m)W>$s7R0v+lsISx>SY&}`SZ>h!RHbuQ8;=RoLkIbn9M);y+@Cr_@xFxJ06EEpJxD~1HJwiBVMh{R3E zNT64c%?lX?>TG;Maj#w&zyFNYBBVC%ujawMTJk~4Deoe~G`)Ku)`!q()=a z90~~GyP3l=MJEUFnvh{OJq8PH&*$o>L06-PEAAbK9jl|a00yD)WO5Q*_QCE@&qgOb z=!T3|;@@2hjsQR`3mc2vFc*<2PjcG{wyV79U;Bc5#%xLf=T6ckoz^Pn4Mn zt_N_DI)hq#b;{eqZLTKIddO_ao*~4{6Vu|j>Q9+qUrsv7l0n=%GY%h{4!0hLH@r8iKeN`1HlpdB;n#VD$y?b)Xs8zu^v^B8-E%uSy{

6FR|&Q<7vtKcoJ>c z9B?u>6Tw#9SLB7Y&eyp>qdk(noW5n1x8o66)6%V2VNfjw1IqEOlw15!94B)8<8i@t zBAM^<&RS4p7n2}%QkpeBN^!_llwcz^lS&A$o{FHv>()^LN3estOwWA#_(q)@{`ioA z8jYgJNDJuWd`X^BJlk|iHRTu)5ocb>sgk9Jd*7GZ_4+_zDfpb3k1pDu`L`avB?pY; zgAlw9X!Wq7;}`#jg>3LVmg{D6z)M3N#d1jb^3#Qd5Q>1~8_+sJ9%-0n;|F}gQd?tUGZrYU4eKJNV(Z;piQJhUlB&aymWHP)vYG)#75?qC%GKf zGKQlJ`scxu-`e(8(#hJ|j%@TjM?+b-$FhQ=h)VfI4~LV2`f3&DZuZ4G8=m20(HoMW z>QT=1UUcSH8Kh=Vx}1R3Ygzjqs;qD1V{@2B&-X}TgTll0w+LKPJBa!v3L`2RGo8sJ zAmAwH`ew=NW_@};8e?i40=Gy!$P*EjTBe+>rIuFx2(&HH-$C5CKXJ8^Q#Tl2y5b6I zi~RfRlQ-gLvun_q2LqZ2Y1gVJHD`SUOM+$^E|nPXzR^X$Sq zSxaP0+f)zJgpx&*qN*OQe~eXTfJ|H=ZsGrhHc!+PQabao(_|KJM$sRTx*kBG45wDN zopG3>qJi=U3=j~5cXP19Y_4!%IWM&t|q5Bjyds^B9V@mBIHelvj(Ag@g&slDpsH<_5g7h zJ9+UR(X_0gCH}PkFbEyMX+M_s3X$T#i&lS7r;iXNa1680y+aFMM-4ZA ztPdw&4n6_)6ecOB@$lP$jhKwr0Q)~=cP($Vek2_*F}0=jHU5>8lPKa9)pD6+d6X`x z*0vv^*klmZl|!|0dAmS1iK$xDzMsM!?W(@Ca;*wut`mkeVC}@u<=Ky}WX+$MFS8eE zQ`us@*?)$%0n-nYVGCr!%)#uH;ZBRI7nWIN@`J2yxv{A+hl#lG=EH{%b*6z8Pv)t; z+Lwj*;7-|WM?Nj;wH!8X z8o2VdcwJ@>Zt@#`H?&_xNZ!@ff|3962s1XJ@&xTsg87Qs7%0U(MI@%`J%npY^iDZHhj*TT}Y6ZEL<|?dHvs0^1!kV1d~|%y_cb*MWt)oc#FLW%w*N<*+&y zkkOq-^Yq1w`e|pgYL$%rraiP?!~~?(z5N4uJWBjPI*lH#t_xJFtA>${BsD;*2j4*k zX$GO!^`gXNY5V&7w;2fZ37dWKA!l<|nBfLm;KW))c- zQ}jG1&mej}=(+60$yuL{bCPW%8{J8Psx{Rqxlr2U$Mg5#G|~7-+mG)Fsx(1Jwj~S&*0jfz?45WV^3jsaX<|Vkm_A!+3w11O86EHzra~6zbFMC*mF8&fctW9 z`TqvBbvNkg4w52SRLf13b3jLK(C)0@Y#W$F#+0$o@Gttu!yui?ytjDhx>x3XO;tN; zZQa3Db^W?v4q|&cNjgkr!|;-M5W6W~?b_N8VIa{*>tR@8-&6J*!9kQrP^S)W*C@lC3S_0>bx;TM zE012h8MeIGgrpOCm8ANH#eENt2{$B_aGV36E@Mo=D;B*60stDP6$z`$WC#kv z!2ISQQv_6nx_slXODmdOSk<7vr`VDuQ>|UC0?NJVJZFDa>cXJhQmxY;yaK0;h$OZ2 zOPAhkZg;YU(SPQ#3K;z|2QKCzlZ5 z3-_;f_w`Yne@4^0snZB=svX4tMNe}Pz7PfzYR+QRSXS(NlXO-%!HZx;)V;yt7}m6s zj9>ddsOMPdT9xiquT?2(Wt%Mg4H^bnp8in$6VhjijFd0%Q@)TRjMvs%K&D3JC@pNkaK~_R%Z5{@z8#vy`Xv?9YaZGz@3qp~H*e(a zs8BniQpvsZ#M42;SN9FUz12nWtkOwCXT9($=$kf+Oqn0G?E4En`ClQcKb+#C3Y2Zp zKTFR+NPAAhK&wPb4G3OxwxJ$eNx7KANxE(NNiYzB0HO5j6i12IjFng;L zLt&VCY_W}aUy#?pT}Kh+Qm_RO#QRl=D)z8GNM@V%?qp52Dg>5lGLf!|VJ4TY)0EHY zP7a!W?60d)d~PIPIhsACvVyZ2G^D(L5Krdm;617lE~~>R$USU1lCe(YO=uCho=V; z-CDfmNt)dh66gY?UTb*(3WSNd29aexP^zu`^AI&IdFfy0I?SoFq-exBPKa!`qlN2~ zS=sjPq4qcKf2e(`>6=Yv2iEA~F*M9&tZa}UHf~Wz-ej7u+O~9Y%p-ZJ)H#Q!vy}gL zO94j-F$gEA`vxY6w`mfuqSvxxWz+tUP=rw#+DQ~vt1P@D2I zHve$j-YsC0(`j+`_7(!TaF{V8}3dFszolVx)u2JLh z4j11ivthaS-)6qYzCQy`QOcF6RH@$P8zVC|TCSmXYD!9NW7^?ynMZ5Rc3HNVxtv`S zyW(?BoN>${TkysfbY(ia7nU&|M1bSF|Q zd${Uk$ov0(&JH+gke+Tw;R0LfB8HjWQlM-ti^vbUFgG8Q&!4k@b*f1<-M#87Le2E# zB`wO(8l@uK$?i%y4ef>@UH!#Gk*6Ipy0()~nsv8%N2``ukGRHzZ0yUQ?CZ@q>>IT6 z#M^GuN4@A&z{Bfn!dPqlRR}1;DMdh$(B;s4OIt7e9+7=TgZ|CGt=7hiC}UjSQW>F< zkx6d*+S}XP8}G$`1Y0vC6hS4^6I$ydh6QSFP3~YEIGDkevhN(r#tF` z@&>%YKKjEUk*^M=?7I4g zNpJv+@A+SS4eD?iH7-~w?Q++{&G(Vo7(IBx9?iTVT^Q<8t<-&U_ilTyA;*9g{GS%E zz=qMebJ_6qQUBDXvQ+ZzZoWJ#M`S_^489t^N;k`Y(LQAwZM%aXvD?1TqyU|+wDKFx zd3-iL6PjZZE7Hy=0?5Ssbex3PiB?5+*6heC*wLng^mIM}#qES_d8`r2c-@sj=zr`w zbuAFD2`eUUj>xECS%T%g}S~hIZ-s zX#3>;5&6^{q?|nm=X}?C#mYBaJ9O3D`K&Va z-jzVnapJ{>(jn0N8)7}tr$O6s<`OBm>%4+b58WcaI!wZMEXp8{!7=vTxnFcRcpWs~ zpOvZ0@=bHuDvaXjp@RvRI}MvDlZ>--c)55pDj)pKOY%6OHj=0!k3E0>yxrny3}BJ8 zxbjfTriePvk;!Io=iVnnI`=JUq3i4q#+?8o$j+GB1r1qy;>3(r!z|A-Zqd3!MJb>g z>92D-_60>sd4pyQLzTV&`3ck-%G~}=nG1o{&>oJQ`s#sv3=Hyoxhb%e#6*6JVtNyc z$E>YuzPY_CSU%>?omnSlHs@!-3I^c(m1C5Oy1&JS_sF+(ZbADQ&G{``GPC#@Ru62s zp5!hRK7$#ObGLb4^XCt4!ZE!ThI!l^|LWC;IffgI0Cg)2$j>f(=1hF^^6~K>$T2k} zM3Tz<{NuB;2WLDY<>_~H!p3eBEsr9}hxM@ehP_d-(4hG$rU;gW(S(V#U0bL2mY(uS zul0w=*;+i8t(-m8WH+O`AQT2;r$uGL3|c>?GUdwEHUiP@{acqgl#d~u}%j!tQeGrchmKN~xv z5Pi6|w7~2s6#zu9c-H;gpoXVHE2Ae7kNU>2gBK&NT~o3Q5sry$aTE|5JMj04nLYL4 z%s#J9zFzv`MYT2#H;{TmT@=wD38pB8v?jNakKJ;zw7CbXyY7k%JnkRFjh&=?EgNjL ze0zAv*}<-i0zx$~Ipa>^XZ6g%Avwtw%7$e>Ss5J4*_}IoUT`eEN4Gef(x(a51z(YO zXAaNC9xSQf-^ystH~CT1TUmZEALj+cSx1BofvU~BU(S5){m~)HV?|@|b40-AVHku# z(R%3yEY80S^#B z*eshl5z}dc9U2>FzN6@R2pnz^k<^_9|3k)rgl07PoB(lA0TlVs-%HZXwy8xANf$!Z zmGnn-^G6JwK-j%WIFQD=XwjmI2^)ZWRnlW z!p%g@G(0WA{3Et=S`&|8wD$VdxsVK>5Rse3dkiGukll{^fJ(@gfSJGCwD~M%etiJN z9+9rR&VseCdW_SKp!o8hvKN{WyADi-KzHia|fh|Hme~QCC zJ_SaCNt{>zyPF5Kc5*TRP}~{n*!S;eLIr1MH_#yiu|LU_Hi{&Rq{q|&? z4K2xW|5j~LpQd`Z-h1T8ov)Ue|59}4`m?5P;DB_9up}G%Ytnl+JG|2co$VKBSq7gk z;s3(m`oVmKdCZmk{^X+3K_7BY^{lb6bgQ}E5mUb&xY})b?Nd|hx*e#tI<8a4QbkG> z85LKgqjQ+|mM#m6tb8z|xLxS+_2t`ypJ{0{guWPKb9zJilU74_HS?X7Tye^q^)q8F zj8(q+#Xoyk4g$+%>@D8Gf<{_?8Qt(}3meGtORE#MD+>P0{D1$uA0F_e)ekt0dI#L) zX`Y|_*8D?{><(-G*~sgU<{jRJM=w2)%CTk5Pbhe9nbaN~oSmHm6D&{6^ecnR%AcN+ z$*K<6ZE52e4m!=ZFy_el5h`o6^2h$8U!XCTI&;c9rgB&Fds{v@xvayazNjkVlUqmyrw_j30r= z)L|D*oj9B{)S&9X+U6_TyF6f8;2eX9CL3nH30D)JtU>}h=>bTUR@C_ME0V4s^mD#@ zPL*r)0eoYPgivhdj-$t4|MhI~So{lz!_yx?rM)31U)`xenA0=l5T4} zC~P{+@^ZjDQGfBI>9`mQ4fu`@6` zJ=N~ogHsk)vyNpO4sd$##vh}38SVn=UoToc@K+-B`BA62c4@RED5DClN!~P*s4yKo zGw0FSFEo%;?bwfH%-LwP!xcC=fb=)UpZTi`Op$WZ@w*lutm*OI7k|4qE4WMKFGKi z9ho_lquYCxKf`8^8RWPD9D4Z9*yd|^jSY?Hg}yBBVt&gDiqe+#phQ|}*QK1?dey52 z76!yIxU915mnUIe0=s4BX8A~Z4|`Z;m|+KVuLU$$nmJ zO#m=Ocs^n$$yk>A=KD(R;^OUU-M@ElQ+Df?E$c#OYQ+!@#vtTh>Cpioe$!a_)&G@p zwQ}lo^Qv>L&ol@*?XlW4a?%qX(*EIBcZv+97Xo-wWnfLqHrUKHKn-hTR9d8+D>ROs z>({V;4QBWDFy$S3*=vV~(ZuNBP}^qa&krs00+}BHPC`Ig5OKL01-mHwSx57yhE6(v zIl1zf=4%8fBC{C@Ouz3z!#05=9#kraQ?W!jKLdx8!l7*2t1OZj{ zpz&WVxW$Dt{HMjcs|*fVn7%u?+y0>!Wl&9l<-*~0oi0n|9YKIy z6g*On^sQ@tSiz}}aGNyjw(yIC%O|h6-ufh-EOHG3Jg<|z`w)aTopNxo7slO$E?|t~ zTtx9Z{?G=*7P2=m==`Y!cJ$orvnfW#g6hC}(7xTAR`K_=`w`H)PGBmY$*~ioa za>zga!7*p>S=b6l@UrMe|VE(4RB_7uXqPgwh63oGzVHe0Wq*G z)?D36=H`H~imYACx2i2;<5{yXq|D)OFdx$4P8N_-BDI#z3)bo73Y*#K4%DfByDfsl zHk{ziKq0$xaL80IaXl%vuumweC87#0Vp~9QGpCimqnT%Tb2Q?ivd_gY5h2B3WF;=$ zbDwqj?pW$&Xg1tS&J{m+vYxP*2SqhDzxYy}nBpN=?LYQ4;!BNJ7DZ3vrijl!*;>|} zp*E46S$v&wvJ){eV%wl+Z`0ykc$t0Z#wQQ3HGinIPvt% z8C!l@*N?d8KPJrlm!ro$sp1y!>eO3#tVKG&pLkcSsUFE$!r>Ey!>@JzN!wiJUmxhw(dvPr-Fcx zsT^*h@BX2RDq^CacbTaxyVYhZ#U<+D-05a_li-G}Q!a*-%#b}p?; zHJeGZwh#_m9q(%X@WFMm-sJwPe&7AiG}X3%S1jd?JnVswb~PC^GiN2V9h*TXEa%%o zS5A#)O!H8?pDlGtEF^ZtXbh9gZc(E_yr$=7Zk$tK43lN|po8^y9J88SA<2xMowP?x zr|>Xos+9biXk;f)geA|oX}&Jo?zXF9=FPt6YVh5Vju;7liaQZ62r4`YU<)r45h;i~B zWUt3uZU?hN!tsyr`li`z&2@rTM4{%|2>KSgIxy(lR{@ z4LkL+9IjSg5N?E+shHJ@qT4l@Jjp5c1bsdcRKSduBzQwfr(xU`&hD(;@8rnK;CA8E zP>;giYg`nix;@+9Z?`#!2S?OA%ahgKf`fy?*O0y2ynLN&hPLz{uB;9G+kb6JOH^b9 z<84;bA*QGV>pY^n6Zah9?R6l~4ZhdZdZP7eiU8jgsIvd|7?GgV?4*pgaWg;WN7Zeb zt{5D0L+1@0V|f^`)4V&pw5Y^R{s~8uG_v+61d`%)aDZh0=C1*uv~E1vrLs(D*|P1- zMNj1q17b50wQavMvrat-NM7P~^$5Kj36LS6OItxpr}~b3h^p}8ravyo92PyX(26zT`r^KF&c`GI) z_QW*P`gY~c5(hMUgZ(k|z-^-DyYde-97tHO_=`ZY2MaS38-AKL7?DGysUcO}Dzu&w zW!S-%an~Ocam!l{-r$IOoNf@?^x{#I`9{1EPYmj?2M|LdEHZVZP3A(Jb;i9cOaJ@| zC8w4Mgil?P(2k5+w!w8hB9N9}1r$SjPYi7aeHE_N`+0AV@moS4se&$HBd{V*okNx( z&u>sviU`I_tG91xjQVI!UZedN_}l-3;O=R=kzgd50_G)~$c1Gmte3$M&`9WeY;Jc( zU(Fvdz$SZ#rAmsPjp^C19M_lNtbwET#Li3J?`oEbEtqq@i;0%u zc21{$86Z(N(JM3q8Y)}Yqa!E8liypK|Ms3=AOt8!Fhzj-?PP+qDT4M4U5IWnV=$1U z$+Uarnl=4TzIY47X^Oz`$>r__Ox%s#`veDm%-v_I(>-s`YxTiQa;;W-jW5J^VteAN zl$5RcA5+`oPU98z+<~Uw!3@oZ1WNKLu=-gzJO04@k0%yvDGs-z^o~;`KS>YqDeqs{ zax|a(C%E+vmIDXg@5N}l;2I}lpV2McaSK$Kvg2pZz4bLf^u{bz06-C%d7zmrfcGA> zvL&g2(y9jh8+&tT5ANu2&A|)+nSkGG9ZJ0t9{+v@IZ5(yOQIk8_4!Hno!_7FEeZmO z#%;vOmn~pWm?EgiOBEo_n1UaFOa5vnH@pWsx=$E0{p7-)0idC za{o*IYw)D}J6oZaaiZVCwc~SBV<{N??iZx7b8C85$bFdpRsNU*7beeH;9~oc8tld! zmxCAHxLZ?l9LdYW#;sFYKJ(YuXEJq7cek|1ZL56-E*h<2SqOX+BbE2LDSJ=f1D#D! z?n_6$Q+KIMT@TmyXDT<%OWQz6mC?z~TqoHY7rjodT(eBQUeAtw7;?P7CEObtz)W5? zai(qK#*Jgz8RZ^FGbq2qhs(b-*(~ZX2dgBy$`m=O?TlnRiHwsQWVXkuIra!fn6fkD z9jSkvxW0a0N0q6fq{tm#^8$)hui(>^+Pb0-2E*01bl;FSZ?xr->MlCc<2NGZrw-;M zz{)ml2E4?On?;8_zsy!$`=%wkSbbtm-Vtj%k@k(GAv6v}L6J0;_Hx+azBB{~Q1zgC zyKXFE9;;@a3yWcz%0akw4awL@EYqZwh(HBZ|EFf=co&**uH}lo93{Ei&A+xX{6LyK z>#=W9JdhT}pQf3-Lz@O=rA2$cArTv>9~G@*n++n&Df`z3f|Cnt<;Z)FtzCEMDB}!T z-1Dk={~4N*14+l!?z#EbR>mGw_f@86CVZ1oGN-&kk{+M7=`%ex%S60mhb39x?(AagI9%7h7yG3K3;9l;6i8 zDe9M)(x}je=(CjaHT{{5CCIkXYlUNyz%fp!af*>AwI=iRraZ~L)4+Veu7+~;y?&L2 zKm5BThaZA>(d(5B2SB&Y(N^ZaG`M7zjoU3;ICbe9Vo-L~t`{haVds|MU@HywESND2 zk+qqK1i;A4mVM4<216^>)-^~~!uEFOR9Z>Na35zhHaeijHeodS7;e4~`C^rR1qI3p zN^4p)O3!iGmVfPgn!^A>8|{+?Ozj@1i4+}CxoUa=yCeDd4D(fInm!P~b4YXCH0oH> z4(pPfxX0{bqmr`4kz1=LMMcwkV;2iC1|F;xw4PHeN%*SzJphP?ZFnysnL3| zzu7|C6hSc#4G1Rcc_!D2fok6V^HXSa zEcMj5%-DScTALrfZ59_-!^tT(t>eCvch7~2!Ygrd*c@Wmh;5omh{`)Uc z%KB%RKO?wL4{jF@>cVA~)$D9Tiy#fSH&b7`j&>npUy18Ax`c$YpPA=FE4j|4TmrEQt|>b#N2rytw1atx zIos?dl~=)`lM81VVX|^>Z?aU(bH#8fEs0O3Q$U!zU>+a`px@=wVikNrfSPYCVd>vn z8H*HrYR$KQUMwxi)oLfdO;D%oj7t0k1_R%?oahGLf(Un-(2nM7y2Vr=ZW~Rc`S2zxHi)=u)gyLvPbEnrJs+UNg==mq`B2_VHh8MP+3#a6P*dWFmH_`7O z``TKrpiTANY?{nP3co7gsP2&ReF{&mPQ?U9DI@bsBuk02MD%ASRUG~&uj}!3S>E4g zwm}Ed6iDFH+cWvSIgIU$0PH;gw9SHj;UI^n@s7=kCUAx@gO(6Md!epBrUF2{>)7ww zSN7Qyf+I+8WJM&nn~?|(GRbmUg{s5G;fiKAqU{;pSL*{=tRUO{46;L3*_7G~p?d1YaVHoDl#?Z`ytHC_L|ddLseK zizRK4&=$Ay+(;IbNdjDRvO4UABK#k2MOxTRy4492GQa;&$VDwMYM71HXXTz94cjW5n;pGtdSmDk4oy8VNg4J>=nrQQ zmp99kq{`oaIV!NkDab_S;e}n2+po1O(ZSi{caaPN&esRKV14cTd3SXTJkbl}2yc3!00a5r9(g4`@XMB&$5%Aox=mGn}K(WM*bwFFz=7BQD%jFF%SlCCrSN z-p@xXK$;8Q+eBQk`9WFh%RCcFg&JLuEy|}oyZ|L*8#!D$hs30F|FGPlt@WeIq|M$n zu(;r--l|@YyvhJanoaR6qs<{DVWr~MP=l;hOPjyrNzx2l?&RAQE+i0-o!EGClS1{H zjoixcN0wFoR?Jtu< zOp8b1wxQiA(u6H!?8+6x-se|r=L@S>W2QyOCZlH)EvHm3HR>97JDNu}*OI7PW&S2$ zD!7j;&Q=lmC+4*IC!ChVx7}elImSFgNY={l$y~TShj1rI-0uFPmJJPm^Y+#3rtGHY zuH!>su7Jp!UIg*%U6YWy!veDKtAlr5$V#rf!3 zTU^6B2wnpM-3D=|&4KCx6^RpSMlLu!9#xq89eI+msn0uAUpCgVVVno}ouH4hClm*Q z%fk<(frNUlY0{aJ$wSuSA>ApbA<@g(oncTG3py-KR06oioH%xD6!O~lf}ufqB13e{{20z=!tLC2BH!f9-_`d$GpCu zoV#iQceQFSpE-I@hi$R6VZk+Kg}@T{7zFZp1JSq+!yQmE^N+9an~R7+TutpOPB#b0 zO_8s7CpvXV`FmrzKkVx2#Pv_V*w>t?FRBDt5FRjcgaHECTy5o|F^H0+AUj`U7)9$` zP7@G6cir@!(G9rB6Mc-CQO_1!sd9sH#=cr<>M_ZXl?A`FRY%)Jzx?(^lT*#xJM{YT z*;lc(h{-MehbBIaY_Yy=LBMt$LbwbT!APtDDZ41)1SI1J)RF$e4oVjJYON# zdU2KiA=J!KZTo!WQ0yhN4={jt-r!QMrc@w6Iu0LKUJlsxZKu=X8ANhZYe_5}qH67D zMeW_Y*L7I{=S|tEOD|NLy3~R|Sh-#?zeUMtXcmXv5q*@ia^wZHm+73U+Ci+x&5$!C zmJujfcFclOKwwmwl1CP?$?RHxMD-cmn zRHjt8MI0sREPVNP z`|3X*EXZiB9zAs-=Q0TG;UjxN2C*}WQbK35m)SoQd5+9G;CGy@bcV3$`+*8o-i#on z%MEmxU0HQM8&FMqvj(jLn!3L5y4G%rSDXckC#79gePfZ2;=qZ{q24jh2s>$ai;jC| zpHME2&{~Itq!L;W%xZ^J08YhL08@!krSieO@Wu3RoX1EuM%%cj^s=CmH1~FbC)E@C zYZUjW%~sYemlh>l*_LV)llR1NeyptB&Fp_~N;vB0BcU+b-$Y=qs87tdmFCdG0NBvFjWa8PY6 za+4lVG8C+g9tEZWG&-8c%>pU0NikoqmvC-kzPlDLNE8_!W-sdktS`o{ZP`=O0)yH_ zK)b`%rkbl)+qyMY1XmEIjGvfx_0uO$Hj!+b1)R9N0JoOOXikRWfnoHfjLKLxD-X`k zdwa$dLH|j%+FX4fFsqCyRh-8FH+3UtG#;F(01e_&^AHK3H(y^G3NNY1vgj+V$UBt+UDx+ z2tJn`m&or8xmTNh3u)s<#acp7=jW|e&Xsi-{%I}z73Ga^j}FFKf{yp70&a_tOgeUH z3|+jAGmSl%j1D*+$IQpaeguFb9V62WKk6PZJ1cM7?r`z{MAE}O4sk{2u0 z1zx~n+YtQ^-kHIC4fSDJN&TP*ghUp~TZ79;Iy@i+K}ZZ0Hw?32sS#SEYzr|Lb~Hfe z#p~oS06&dzX`w|;hbe(*jtOsX^_Y1^iu5b#xochtRdni7n>IbEbtk6$vGa1D@lMCh z=;p$6tX(P4E~2Zrl$@y8DTXjBR35+^TCbQu`XI)p^mu?PWj$2 zE*X9kj=F^$efG>z4{ZKj+_gfQNN-1$RDDan^wvC^LqwU>)i*0_X3sj%Mnm5*AaJEc ztzmi!s+0MbTH>=9MXQ+>rfkYmVXOL0LmGTQEzCm^d8BZ)c17*Q>#%?>7HUw}mugyB zgy2PXdtbJ(JJe^J+&@CSxXeE!$anQ@v1 z%odR4ah=Q%Q)aQjTZ)`h)DrDm0futTmyB9hqSM?cxrT~xKvPxF6n#=M&-mu7GkPt6X&^`$pane*r1Lxc;oCr6l+;mTC&A= zYLbrV-(_zFxkHd&c=!{iF@b6P+7vVN`Dj=N`$A1ruBd{i5iV`1${<=nyYjSJX(E$H z{%lZ>NPS$+B!=B@g(>Ix;K75WT$Dhgk)$b*`Q~@yd!MW6`S!!0 z{4rA}F1p{rBxVURKzj3Qj+VFd2^}Do=B?y60a>b!9RwVn!0F)%(lM+jPr_$yBNEin zG>uOtS$-gY(bs2P+L$tfIvpg_u1oD7yLLS2{(FEQF)&3_JJv2JD1YQ#pMF1>FD3Zg zq8=S;vYE;ySFN7g{@)pEXf&^;j`2Nm&dnL^K_(0`|J%2EJX~>}f4C|@qDiOc#bvTG z6V-+M$e8GweTUBO$TTfn(4Dz9HapnrPp^daUbW}Jgh_SHzkhNPu%|8he_mS}>jrJZ zh=@G4iP5C375PrRV)F8loWV_&zN6D)Fror3pw33;yV%zVK?pS?=ha7%YJT|X48=BUa2uFb-weez7&z54POI5U%Mcb+XN(Pc zvDPvsSqKk?cHCLZ@*~y^u(tacSt{>??kn11l+Xfn_p|cejZlg$KSI!wEo5wUp-0JQ zoA*&`>;bYk`H3(*BCLb8%5lH;a3V#+qgxY25qviz!2GCOLkjrA!G^R$H@)Mb;cOxv zY4CttJQ8n$wOn1>GJbZV`F&=N{>(2nMUkJ&Cr+ z=|e6I-LlZFD4ovtOf-M3p=trl>ugFPQc{xbqyZOOh4X$Oy@0pJQeGuS-dy~%d9OH| zQvW}TA2lXEfrs9{srhR@BEtJJfQY zZ^P89UCLEVe_{+sj}RP6Nsw$c#ngh1@vw9~m6*Izmr;x;A9OO|KR+b0=8b>Zh{C21 zA=lJy?#*j3#!&}xCAE};B5>*f`ntjQfiMdEzou+2DA;r%!F?{jZcr=JCY(gk%!c*E z_g35?5a$6V!N2npEDaz1Z7VCAQTCdj&Ldi3Q2pi>MA%Zl2L>Ri@3Uj6XYlWX)UcZa zGv=75<)_U5_U^sVam54*Xn0Su@)hSdZUandWlS*@RHVi>drSC3Ygb#jFJ(+GN)J4_ zn^yFB5?LUTw|tg$!=()$b)wF4F^Pp);R38)BTILGOiU8>mL8~2hz4&_Lhk^kRknau z9zx=qD=irqW;6B!#UR1|LF&%WnLnQ#e6F7R?iExs^^F|xOZ>Wa9Mq#nkEoO@EDkZd z2sQF2Rn9I8&>BDfOU-bV9h9O^ye*t(%RTQ|(D9guARI$o^|zaRHvWazYT;+DBY(k>?Iq6iANkr&qDj%Hc`FUjfE z9$ZS`$;`53c9tJG_tU0t?1o-l-MgaWP;MX;3oo(*_&ORyNOWDv5$g2n73qN~0@UOD zw1$?SGymJWEHBE|NVqy|P-3_st_7`dqX%>c*{vS}u<5V@Vr*L_f@|l}t-Yxu}R6*mdlvx6xiE6|13xm7a4vL=Z-H(~Smy{ey zgXm$tTpe8!UjRS~`X7~bN=fR-VJIQ`Oo|H*s52TB4s)c=iy!`@T@|{(?oN*mT0f9d z#O8vemjr;ms~s?kEQ?^UEK3{!p-ECIPQzvdTKi9=UYaS=8-tdBNK3_SzKh_9hHN&2 z2%>Zh=2D`ItXR>f*TIqQq8)*rEOLAR`Ut6B3iq0a2()RC1O!R39k+tDJg0uXWU^aK zJ`+Whw5K4lp6vA>Pu^g?uoX|9v^1V0V@R{ZoIa>6rXAGU(Jt1sz1;eL*TCApKQG^G zVJ{bajwFP8O&IoRzcaUDr@B_@bEOq?>2 z0P@mTSFLA07;Up^sSn3LrYOSLDCqcXn0I0X8hO)Epjz&&??iUb&j`UT?_%qrzqwA)VjJ1cZQxMx9c-gpi6gq@KbHso;o zO>Nt>89=+C(a8-!5PH*>LTk(+$I1KcwdLk5Ej|9*b>iB!>mSVjj;v#B5EWF;)Da_6 z%ip;`Pgdg-sg-+E%k>rpdTQ61GeHM4n(X@3{29(e&bwKNceWRS43`+vNLp{ad8km1 zt+0%0Qr8Dj9*NUskSZQx*vG|Xi|MDGZ3NEgA45y*JVML7UQ~o*ak-&veXq=8sbn~Z z9t6Jb#=sC&gq*YY7MK6O{UB)b3QXdl(O@ue(f$(xj8FqL_tRr3wCHvN7Vi|tl1o6w zFf?#QLEYTx?{6mX$wz-!h&zZhgKBC;ju+O}V*Gl;4q$<>!-TihWE|V5D>{e!v+m6D zH+=8u6^I%Brmdw%YO*3Sns3Kya7oEq>G5lNiHvdiS$xlq97N9t0vZXV!gC0W7>i5v z4(%#_A!>T!X{L{;i={O(jvRp_nHObnw3x#>ZgqY|fS)U0QF zrc!Ce;!bnUD|T|~wW{f_SU=1B?#Z{v0z|E?U!Ml}vWZl_dc;v-ND@(*$cmW*aS zvb(ZeGpJ?aDRb8!I|Co+LYMoNcUJcA(xDG}ReEyo!;`5kHc{Gj92JW~nfKwdEYAFw zUE|hP8EAPnmVXztT zMCTYI;krD>kkDj#T(Oi|RniHxV)Y7qz(iI?<>b{E!3Wk&F{8QkbrLqt75^7q2Q|xr zgm!f8MSK?q1&Pb*JtLtY2|J>d-dk`nT$d`2mYQNh6jfW_ z{11%I=$j$%XWx+XYElXMq3W6Z$A5mdIY4QuO0iO|rmMr_VNuu_$zE18iK+405)Lo4 zB*&8>J&o@4IVIaINLU5NC3S3$ime_F?=TbW4_4*PFEX*1ANE-yd7e}j*)WApmA z{qbA}TZ*U947>Ns&cCZRW9_SbZyndaTALm_|8?6pP7h1BxR#Up`@8i={Z3!-4qTrO z&HrZxAGGO}Gw~SpxS_5(=BvQ9MjOCAu^ zZtbgul~ZTDI2Dq5{M@^|Hck(Hmc4G9l^nS2)T*>QKTtNek#ZH@)};y;I%c;G$h1A1 zx_IH$;fs$HZwr5d8(X^SnCUo1CQz~Z_Ki!KJgM%&4nH4Uy4$wiwYRsD1B2Q%y8G_h zvAjU1hZ{c6V9(Dt6B6Pwyt2t}~AROdWP? zN~ryh*63m{mw;d9*Oi8<;r#PCfq}89uU_R}Y2xbi-PBc?`Nh4zT>C0u<%HA8z0V$7 z+Q&Mz^^ZbL^&SUah@AVntsmz2g&Nfy zduLnNQquGDPAMd7rU=jrL#}0oM(4MYx*@;c}%>YrtW5kTk#lb9whC!z|a3IgvKumIu>Ps~SCEu}mU4iX6{HNQa4V zj*_X$s1u^w%Z8Ft==0BcPd0;7li}CU`AJmIiEKw=e%23JrH1ZFl1A0k`Ho5mjX=+l z3q!c&p`1J^@&&|6U!WH9r&auK8;`$li&`h~Fco7lVys|xbzivIC94YVGx2svlb9Qw zPRbv12Rav>v9L}IMo_vkPe!0$PnJh)1r(J;=!H&vbb z__r?s@8x^#fH<=EimV$ z4U4+=U^{F^(jZ$5dXUjCp^p|`44ZA{9n0DV%I!3{7&d$M`FBh^&koki+rLnQGH?~+!IYQ|n8W_q< zclyKXO13^8(Pz=s3O-c^7J1|fmMZmWSsR-nhorp2P{W<_zp7WoQu*)SzS;x4)x{)+ z1-PQLF(q~vTM$Hxw^GZA5n}p5!!%0vuX!tkzm`W^Md!hu)MJ`Acdn3B_DgSeNY#Pm z2A3y__M{(|4NEKX>hJ?<;>mgB3x_X4doj*MYn+nHJK_!0tLG#XLe}Qg`{Kz%h_Uq4 zx~Y$inx!Xu2&F|G5an|>wiL>GNma#^feq4>F4$wlo_aH9`c7KY2*3B9QIk#`H?cOo z<<5<+w(5YgrSjQF31@N22(qQRzX_fsx`)SZ6NB ze)-1G3EZVOe@m@rtkmEYEG%0()Sw)Tc5EC<0y$dCtwRQKAx+*R;i@vQTCgFy!e;JP z=?+tZZX&PDVcNhBntd9w!y9?@%w~2#4t`rB0X<|RCT%V(rBbfW!Yb@Y%trDE9b+^M zkuKhKD3l5Ks=Qo(-k}Do#1sKks_72Y>a`teV|lIRnD(!+V)bjYr zK3_T5JzA;h=7Kx$CKWSFzW!-xU(QFO%X95n|K>cy7F??3)0O>n)gWw-eqdkhUBe!r z;vMC-rui-z2Ox+8*wOhbiY}P`Z~pFCZc0WTFxm=e8C+=z&*{&#?Pm59q{_Nzh7?R4 zNzXx$b>!VdFXHm;+qY-;F6{cE=1beIaDwFNM^W3wq_71ZCempFZX$m^sy92=@6-Fq z5=L|Bnry`}p8m{ab>eGlYj@EGSrgJtF)h%{D8)C0m!(v#7d7^#d+x;z&7`pbl~&gk z0~5^+i~1$lFoI^S80&Hw#xR6*VGE+D_{^83<^#*+#wIaArl8kI3K5DS)d zm0opAwLlQL2K1pez}hbFQ~h{$E3Xch{Vb@wkK7NT%1b}KxyzR?u0bVb_a*YO5}_S< zc6BzrCite)2#~czg!4ZQn#n}wr%4ekk8Ob8t+u;aErhq3YykD?w!g4-T>R5h18IN5 z*OOoeI#f!`SBB~IljIpklcKa;=mbiMnM~Ew#+P9LfC!&s`%jBUYcn5bdKl zPA-8UE-T-;+3Hb)8(It&5?-dJ$|&?Xa3gPh4UmO_!ESQ&g{m{zd5iPCP9Gg7Pn~+? z=ci^La8RbMY?mY&Lo$2UI?x2E@(H2#GoeA6)}Ou|Mb~ge9+W6I({ArmtACwbXz8Qm zr=uQ^&Y#PkmB#}&=L#6k5ca$A6}Ipd&m6n5dVJBmlSEm83GynQ)Q9>ScEM#RJ6x2nuL71rW0-cHiR>iAH=0_9}vcd%|Y#g)L2L$V91K?Vnp`(I-%+1ZUmpe3AY=r&Uxjd#F z1YU_?4o=DFSF}&@HfNqah}{7Fg_ErhI;?S^(y|QU^+WkyDR(MJ0)BploTKD?!69#| zc_vBiEXLCRsxDv;NMI6*Ls63V^K&c>{asHv58zceK_Q*|$rk{Sl5AzQHUYn~EaWMn zLxE&d=@j9$_5Pucj?p_cQt^T*`bU(VkXP3~<;s8-(GI_}D`lfTVZwxiXXnzIRD@(d zwrlau_J$vHTn<;ePP5wQBbfJcaC=Dh0{-5Ue1L~oV=Sz0>p?FfqpkLAr=5Jm)zQ=y z7R?pv-mx4?OAKUm75#|`zAp$%OPID1HWYPDdi2iOd{Sp=&ob}JjV0;s0&+LLa< z)wmlleR?J1aC#6WqAEELc})ph*%*WQDCRW@$X2etugcnVc}-H``vR0V#E|^*QFQ@>twbp#$?)^rM4LTA%Ols48SOHOq@6utN^3!JfDy+S^b{5K{w z=@Pxj>H`12IBY1#&xk`9KUAn*eS_=G5;D~&>S=xQD9?>+VNf&To1$NSX*BVwY90zAlVBN^of=!y=!fA4JyH7m4_q^VuZ5iFyk=fs4FLbS4NC@$ zLmasv+h6e%7NC%8n&daxTJq(RkO0P)yC@O+^y;|;_veHsPtM?M z7NxX@qK$(c6CE9aUOc9Bc~3hHZcYnfWkk6}Q`Ry?RlS){fw=8U!3zy0snbz0SfHg^d$h=` zsjy9%{Rb^=0HOs6Z`G3H8)fy*pV8*~Qp^kw+Zu&mr4~A8;0?ihPJ%>QVM{Ed!UT9> zPbY#Y=cv3*MS-<>UK|Dj4I_iMZ{|e#qI0Sd7jo=pN}Q&=8i$QT;~#qS`+YboH-((g z$Yd-DJIxqaD4;K+;k>Y6*!!S(ObkSinx|r1ir7fgMTVArZ1;~=OW}I9TvDZ_2Z*8s zMVme?Jx3HEJ6k*{V=(N)q|G~pRdCR6C(a?P@3QyK>{KZN6|z~R*V&6>4`G)!3Pzjc zN~~TDb<-GcDu+Q4!TYka{eL)jru2i@3++2tyT+*(s;Ld+E*eC4@XVww+Xnpvm&2iH z)25dehnmUYIR9v%E?mqpshl<6v+p&kfI5Kz5y!Nn>E4(J1!ZTidc_f^!Gr7Id5{9$ zAXVIa<$L`1*KY@xRMIWk;gRWU+!R@De){3>HqKoz;tL*6NRvuf zp-^KCdHa6$Q6)_B)kTjbi!BL)fK;#yt3E7>Wg!B!h)7fm($rVm`-cZBm0qF;+{tKbdJnM@Nr*7)wpjpSmH;z&c6T-Ruk3K-t|0 zv*1-PX#R_B+fQwv4n>tMwQ0t#cTW~K((pWq71;1oqACP55um5(ylfNA+u=;{1SW0D zQy^$}F>y?9h4K6Y++?XC2%4ob4$YAp2(UA+Htc2NBHrZ~>*Gn4&UMx!?Dv3ICpKHu zxXuA=n#uQVYldc06q)sG#ZE+Deh#0+Q^FN+BFRYD>8C~8&6sMSZ=U34&6@Ne#~v1P zqs7C(&4h=`#9h85i4au!edFK}d^y#)FmExP!G)glC+;!GDMy^%fChEHqB}L%4v^J1 z4L>C&?=9+LW&MZ&iiBLgD#Xwe>dR}?kRva=fA_AUu{Hsp75=^F2a>#Q8~{rUuc_G# zz0#un({TRH3ETrT5YET-jrF<)T*OLP*6a;TDlzPna!F;vVOo~1Qe}){QF2Pka6>pH zWy{P3!OvJKYbX=?r;g{*cIAIj8@jBlH|(diPTwWJp}7CvbcO8vilhOl^WjZ5FSM+1 zfFHW!1K!&P;X)ZtP27_dzShMAy4;s?DSav!hm%ZDo;B@kqH5n_{q47hVnV96P8x7# zR@)jSzgVym-Dv;VL)9FIx<C=yDXDtF%^6eiTcjmP4M}i3~FLRjY3USLvaKx3mZyE z?8@n}1f}KV(?>=pUTOm-93vL@2-MQ!e*69RZfI(k7?PQJLG8FU*rm|Pqnrl)QfJ`9 zaTP_|c!ZQA4dJAzZ7$^!-YXl2!~IJ7un~u~nCF0^jiF22{=WV4LQ6BxmmgHYSex2< zI7nk`mFj+{SJ}J&hL^Bc1_GzXklH(&-jUU0^!>|WaSz6vk=|D{4u5&?Skk=QU;b%L zVz`yqowT?QiCtH!oTx@q5ZjKOmG)#yQ&7M3njG-Oqux-;^cyt5uKW6XGD!1$NK)HP zy1*9OU<@<{%Q!fz8iT27mH0AXD`#J2qa5yrlb?EipwSZb`X)`^7Z1<0zmvhC=R0$6 z&CuX;DdhOIDRerU(lCx^*i4g{3d*#W1Jf+0HW+z?ru>?mSII$NSVv!u;_TvYa-c>X zLHjp5=K*eZ51R1v`x@3$dc;w1ZfV&sT%*92%#VLM-3 zIWYxWE9(JmOSOh<$u0t0lsbIZ8>oR^spC0bYtB2#!l+R9XeflkX`tqYfVf=DP?o5jK=VYdF+^ z5%)7o`}HjH0-}oPAlNvG;mw;jd7$~>XXkR3uHjL-y#MkA=`K{`!2@%PdNXySM$)<< z4&9}3_+RyL@X}7km8H^-e4I@k0h^kEq$=5EYH2&qQ5~Yc+l`Ad6!RstFD5pu?9&yI z%C6UKNn27_j3JI&QQs%6`=N+U=&h58>XHVI0j&#rkKTQxmUc*!+Kn4upF4kk|Cvzq zny~ixo7?fiyg9ww86LxmJT~$gJ|i3y<~OvQ=SH6)Tu%P({e~d8ub2rshX8VoHiXkH zxlr$yy#OxDp)k7c`Cf#RNsp@sgm;W8u^RCHH5ggPG$^hca%SATd$)DmB5V9!n^SZ& zVJ|e{h>wYnkLOShEsJ_ia219vrA6U&ruf$e z+kFdjXZhr2J`TI{J3E#0^~#J_r$&S{aX(!h258CIMGURJtFe|n(K)Sni4x7pB9>PK zuM7v6Iu`hDf8taI4J?Or==v&^V)oke8&f)y=6Nfl9oEmMSZ#b$lI|U!e7VBw?0Mqq zZDmyshi02X(Oe?3YQWF1xBE?X`1q=PFLnbvcMS-)*VJ#PPn#AFWZvpm58IbOwL=Wy zfKJFHm~Moct6NXR-nA~4X*l2(-iDnjy@piNcYIT8?x!vY>%lI6XbxM1YuInScJ{j1 zZ&L!>F{DD1--^|E!N6_j!Zs_(8~ZOClDiEyhfBYVx=&)s4Sm-o16&OU(%j1T3`dTg zLffH)Q(>!x#&gL_YtFh5Q>LpIhUhuRG~+adq`WEHz6s^pYI`l6^C{psVGvvYZK6(D@}+&|P< zx=xvsp45JNmQMA2Tw&fTZN=F6dW+RsE) z*9@|gK!p?{Ytol<8Jkxx&Izz4`RUV6+>!^K5hAlXx?{-)nv+c)^t3`Qx0VUvCv#>; z(m*XiTH^mOV!?RRq-xCYGeFyxazHya1-s>E(;SF*0=@%Y50rv`mh z8Yhi0nkd$m8yv&IIm{3a_RuM&SBr{Pdjd&q-mzRW{wU0iWp0jIS>oW*z#@&Dqp~x>$8O($1*E zTzk#DkSZH4*r-t0PgzHh50QH1f>-58CX;@Mm;; z8X7q*?Y}JFQ7d+SRtEuEd!ty&IOQ1EsR5sVnWetI&ye znWJiCttw6vqAU1BU6|FWyi@X>LN<})LS7E(NIz0j@JRWdb=29Em6f&6v&YxPiy!E) zij}kYjo;1a%P)mv%*a__dyNYFABOO$52jt1-myE0{vp#(pvF~+`b!8ICn-_xyE>xi z3rWb+ifiOa9g(uCq0O!Zt*aTr*>S6B5dMNZ)XNyXpv7vsQI06NJDYCf>o6zsEAOu7 zs+QzH-wGJJl@!ABFJ{YaO7eZD+UL`W1p#eJhTbB}UV|Fy`tcbJNaku21C%n|XxM!a z8oF8+1GyjTLG?4MaC<|EnBQ&rL{s(1`uJIJDPIFhR)AFiI2BH@TkZY<8VYmmD>~a~ z5E>D*!$4!9f>{Vx5=>54dJY){ywz|f0I98Sb5hMBt$-<0;YfJ!l5j@sxI?G zBPzF8(nKSnw18Y4)tKvXKKcONr(bx~buQKFw&dNJx9JKw*n29bsIz>@XC@9cQ9Y>iE^uEj;<43P#i&wl+1xUjszsQ#8DaY`q=T~cTNno zTA08eN_FXThVl_9u4bk8A4^Ri&k-g{Cs#l!l}1eHx?tjLCrmL+Bb8jBi(2h3?>T3k zwSIs6)^Dw!Wf|-FJkN9A_cgrV@9Vlm4(O1~9E^ZFG_(^$o#+f}iHNuBdw&MNbr%Kc zfM>TvQ7rxg_EgYzaiyGzNw+qfSX!+esu4R!pgOpRCKiE>!V`=J{dPHhcnPvf(7>!Q zMQn+tD)z82AA)`Xj0?&UWMnRN$T3!Js*_JQ{9_~XTVf(NkO@)vAiBAT4m^EFZMSj( z);uNfY*CmfxMAkph;9iJ7l97GLSGc0MyFt_*Mza3AqL=k2bKQ9y9vTnrrMTzg+`y@sO`fa7c#yPw)B)}hRRK@W?k6SOJlQKIz{Uo2UX>Rr0s4g+kfL2~_n_kpfh>v68n9WL)8-t~C8(G{AiJ4-MGPR<%T>5)pdWh} z)NaI|(;D&Gx<1aJFU-UB9`WZP3lZzBd^epG_diM(u*U|ijTs&CT@^mq@!->Y9W=JC$x&JWCfrc?K6`3jQSe(}ygBN`u+gKuxM^od`xY9mhn_^DId zz~F{W9DPH32AgjVYIM(Nnl`21JjR}QlRXaGG5Y*a=|eo#qJ1FWmqzVMM`Si#pgBIG z+h`Jx;<4*2Jz)QWVwwJ#I{xcX?fRIc{6S`lCELo^*snZz&#$ilMUrA;W8E)n)|!4K zxVUq|x4MMoEDcz>avPp?8-U^I4+F65dvQj?;&sfu><5fjzI}`3&bk!<_4)M46E8y@ zGqXc;I(Z5$k->$du(7TUm>%CLYwH=>qp>eguL>jQOmGlwn80?kpx27Q1-C;}2i~nk z!=3YMp7;6KvpVd2oxIY#G&q`2z+qTK!3ioNq$t#U3c zG-EY1sf_b}h=QQ&J&Z~c)kg8 zLLK;g!(?8O^exq#D+X!DWs+B2U|emoPRSUnS(x}U*?XEe|CdvNk5XCoQ&cWyBxP38NG ztD8@3yhTQaZw!LP0ek_oiVGet6;I{mWql1bYv+8Nv|BWgA?+qY1mtPo7$(287xC$v zlXt}6V0K?q*bmdi%j3Sf-V<$ct494HvT0gLTjF2yPVasPnx_z>$Wym>yp0u6Y0<(0 zx7a(L!ocCF5B|a9^%I!M!>hNDnpEb73alq&qhcc+6C+03do>72X63Taug8sLw5`Df zrlno%Fevagd>IY6Mv;(_*;RC-F?XCd`@L=YdmNZZb^tRliRGNHJy8=+F3u#)TK9S6 zSHME2@FO8FmlA1-Ap{}N0>hy}R(n@1wXO{O8uj{VJ_3}+XJ)r<2+Y%ve# zn{^tk0Jss!M#Wz%ku4+u2X|doj7IhP{nJ4(APOM6-#Py@pxY(>r^#lFnS$xMnqWYO;VDFG!ExR=pSC{Z^s8qcfzeIRbz_xPF3rT${mz+s*UK^$+gAY386FaSQ@Q=@(xPEzss&9#Lra z_}E`Bova}traMf@U!2;)(7qnbnQRX@UHh0wHbfX+`{BdkE6a$!>Nvn%?7!7hOBVZz ztn`*+Z_!ROZ!nMulN|3koqR|WDzg5ecWI0GDAZIlJSV*0226T%UZd!JRlS#p`URm^ z+;;*zKViTGr~=0xCo@toOT%weX&oumwwQBm_sdqs?895M#o2~w-$r7rW{xfU{y=ms&jHp6 zHQ|}K7mx0-!^^h*@ocInckDaswizXjm&{@tz>B(?7s3=2$g$%E0Pl?nB8mft%>OxRua}x3XNKr$6rMKv)TsuGPIGnJwDAUNpT+qWg zhWgmaDhL}+wuumISHvtwy&^hEaJQnii?+@~72qo=wv?D*rxJ6N(2t9=`)8)HX&&e6^ zY+`Vc7?Yr2q8RH23@loY#iyZZD6Vh#$fbmsQ0UIt#QW&ruHtSa8!R++K=MMt27DRI z^wFbF!3p<4=XIrV7tc42=-l9Q-9%NhQ-F_x!GJTk#Y;3CL8d4g#2uyS$NwUMYh2QC zhUfq^`{zzcV*^N)paQKK2}~D_dAXfvMa?OO%ar1VVgjEk-#Vga(Zm7xUev^i`Zrec*OHFgE)N4=swAlw zWm=9QVmeL+f<5Zs_dZOpx740(76N>{` zh1WewI1*~|M|eL#h4IQSckR@4uBRD?Nd3CFB?nIdANQ#)42iodx_NP2MVBNY8N-Ss z8Vqm6X1So6`V(!8SA~UPU!j}GzRG~pQ0rm|`NB#P~Gqh;`~@z0tqk-5o-t z?gsQRg%LD>AtMU+$tDQzzz5CcMaQRcK;{Cdna6ytt&KArH0N>ptSHgebP501HAu*{ z*n2|#gMVmzQ(}2#nPFQ&asrEerUeh(teww>3IP^JRFJPoj6AQ#hXgG8`p*<092Lzv z-QC@zo+7VF=d{g*hoiUBg0#~hl_SNNt?9QsL%j(Dei%ri!Qz8p4$yvvG{1p4TJ*w@tnf$K~f(OB<@Q`lugJp^dstY&CXzT z<_LKqOrwQBig~FMpuYNe6^CngQc%l}5=v#DGIyA&`dj{F!rx!**(kP(WB0C(lWlA( z+h%(daRlDow)-$y*{C{wG2x%U8`uv*NldB!cE_J$Dzp&9g0P4tEyOjGj@E5wqDlOw zo>Q!aU_#a;x{VYIn6OwbcMZlD z(?KVVY-oyavnQ{SZGUGYiw^MOfIzlo5n$~)F@-c*G`noptl7Xi`3{mFBke>BGq^-C zoA8a!QxuRVsJIdAZ4E9&w1j~jY4dJ?SeSoU7_!XGqMsL#i-U?W$<$m(K{8z-Q~b<| zn&|22T^s0R-w_ne zV@M8RH#}PCAT_=5_*0#l0cKN99>tN4-sCSpbfhxitK4JMp+kp=UygmJz{rQ>Nkq5( zK@#mc%FDXbkZLddL)D3YNCU)wHZcb7S4C*$*f>k1&1L&S>T`%|5`by;G~Rl+qpWQ4 z=haeUx%ZLt`VsEy(iF2dJ7xH$sgIPkOz)g`dt_MXK3^B7dq*>#4>4Kl+nDKk)ODZN za?{Rv;eE0d6EhU=Y&l#0?dRjf25pN7zl69?b*l-Lp{Ep~MIXJB2yOL;M- zTYu1?)9wv|0Ho2{K1q*VG4^EV2&W4Ga$$*yiC4Y_0v+aFnf6gAToZf2+xq1vO7wf)7$h(*Iom*dbo0?8)59E8K6zZ& zrY7mpUQ%S}#lRSbLH?rZBdh8i#oL`kF}Luch@R+V`yzuf~K+&oAPuJSH zuCDP_Nui;N73c+P-Y0Mzv>E2|;nSy0cc=E8rMF^7adENZqRFaZsXP|>%LeTHh+XsO z&t%R#lbT24Teogqi~ZDjJWi%*YGX4gpQd2)!2$F^DItIhli7*kiszp%gzea1_ilLB z_zk$09KDc>T-n}*3_)X>lBkPtpERk3^n{3~M@guC`fMUGR~LU^qZ0gx13oHkTWpI( z7JqWTz54ajGB!3YIsI+@#kLzXUgi3~DA%beLbU5P_a1Rz3)5b=@!n$I{rEnK>i}-0Iw=f zsQIO?ZqL7x1*OY(XlpoTxrU~uk>+!6Z|_%rx8_(r!Aa>`SXev?g6AB2``*145|7$8 ztmm`NCL>Y4C5%@@cTjORkKs^qwx5vik7*F%Ah+{F?LVdFZ`1Mf^DD;O<>ujb^d8A8 zBsv+1UM{05&jcv-n9wQp)((%Xw&Exub!ZI>xpJnb-HH__m0PuHJ8RY~k6p`@l$3U~ zUll%8>rr4`^wIb-V&0>9LXTK<`SQEOhB5nZR<`srb=>_?Xfi5CDSt=CU~gz>_^csl zy;Hgd4_?`FC#mc6$>s%{1%@rb0kz1+G-b`B{6jt%{?#+(OXAmlt3i{RJ36IDb6W0- zT2e_5oAGzfsC3r|qzVy?*L`L)mA{a4L|`%_sR{50Vi;O#WYXtR(AOPw3X)+$x*hKI zxWw!TKZ@Zl3sC5iTNl=s57?m+C>!%)sG=pma>J9A3v_wrwy@>Hio(AJ+<2R^|STz z(l;Kx-M#PG2&V{{ycC<5Xp}o}`A0>9@IsDRdLEqjW2{XHys5g5j>_=i!*AcY^O%lM zg_NE3VzK~Y>&?qor50JO{9;OY&;md}w40xO%Gr1*n^#>+YeL4)iSF*z1h+lkCE2@O zVaBm`O-+rqxtAhHR8~#_Iy7la_5F|?J9LH2CnMaxIg0EU&h(Hy6@`x;b&`4k!XvV1 zE6LinT}Qqm7qBX8rmwFm8(;W?9u_Vx&-yL<01GPGbc}xll*y%S z3Beyfe?GJRnti(Cqx(Yw0|$`lg_94AoTsCslhkj1M}`mugPwJ5qngigR#sNFb#hWg z+8M>`OG^W_W|N&zjY`{L9p;U-R6lMlVv?fQ=VZ{n-eFK9E+QbR0TAR4c^5NC1e;^>k zdfoEC{yl#8E*h1C>gLUxS2TPG+!^k)4=kANUnHcD%TBRo(uesi$2pP&o0p+y_jW|RlCAcgiNAYx$sQLoV1){iYcg zH`Cj@u(DEvC0TK7Wk7)5z=67G5^wmTaWWzOPk8^Vi_3B2E4u^-f36zJkPE3B!VIga-9T&ESSrKgCiZrojp z<9_Ji@Z5u4-JwGV!9fKw|IxL9WYnBQUkc95>;_XB4G^H+-Pk3&q_ngM^|y!lV+S<+ z6({%fWkq`S>>2Q*-dyx2gi-n1bdQEkF~oJ+w-AVp-JIq*917^!7|f1uP-8YZs90C# zhL$lYFiJBy!(G*Smy*=Id-wWp-#oDr9`CR-4j8E17jc<^o|!5IeR0wsdpJrvS7)Y= znKRh`87US)QZeAdp|P4H-FAvhsK>NFB*C(&YDm)wWbqK)-psK#n&smYLC8EENP+o} zroc#RaC$>C-Msi0>n`!j6H2}t>xW-7Ae0iSA!(=0ve`YyJVqJac0tx)=A6jos3@yDS$C8p++|hzPU?uNr_i55+I; z20*qR8F=sBy*=1E$!eLc=5St8dRZNNhq9IyQn1iGzg*`GT_qI~eO$COaPbMa*;0i$ z71l57NYX=3ow9){TE25^Om^3hi_r%Tm}U*P5uXJlR>ZeVpEqyH^dSlrZ>~Iepox zk#18n-1f;IQ)=E^gu2(}Si;vo&AIIx;@v;jEo#S zMr~Q74_a8k=%Qi7FG4;5pG%u&1MM@|-wWL?~U)g@Lrm?>=l<0b*jugr2fQw+*HzNug0N>b!?!}grTfToP!pPR=JXvxLJ%KM;l&i-!PVQNT^%H5tfEAyV$TKLN4F+=T*zASVp+R#je9Vr4M z^QY>GNo*xFmg{gJS!+VW!Z`3Hxtqg9M)B&8j@k|hHun2HXgSB zD%!=UO)(jZvD2!_&c@Fco)7{x5h??tsbh}YNTJcu(N|vXdbQ;9uErn3Pi_V~TJgM3 zBl(Rzra)2Z46Hy&ZZVswgx>4=ZB)Y~J@(F&VKotvLdE|(i^aQnVW5kEek2({Y_|n( zS)XQ-xo44LEqI&LqQ3xwO!j>8cI@4`{zFTXPUV*`Gi6(sYMrF+hGW7kiCExp$9Z(_ ztfyCaJ1$&p7RlO&y1MJjn89ur}psYt0L|qH4BYqEnc#l5bADW$=jHDu(32 zNF6S``WGf)x~FFd9>+gSQNMRMA49{nwuq}*5q6x@)W|3zeUwCe^e>K>KndM_vjnOHWUKOnK)y zAcLdh+a+G`uBWj`F9sP5s@yu&hS2aF(Yxo28EeVHQi=Jbor6z4i|+SO1n+5O?_Vt< zz@TKY4Qe<$XJ@qujv3S4H`zX4gB7()iCr~&i2OMF{q(6 zlUa}9FJHV^NAn0#Aw6vLFgGjFikG)H2G+2{;8h*KWeM2*QdC<@Bp{k}df?%-rq7zS zj;J#QUFytDJ2U1$E`*RkOouZODN9uzshY@1)Ya9)V`7E@Fjb%h$*Nl>9nQ`kGHaHx zbXYGWTuonJU!FXr@;yOhq);abM?hHwx47>^s4S_^+_tC{)|Qt$15?mDPXQf{Jw_^8 zcVkm38b*1>3^ra8pg%6CmL$+l(^#@4CT56IGzKGu15gj(T0uLd-cEnI1rT#FPNgI0 zcQ8ifDO7uG9{a?~+By~Lh`_c3IC==F^B%;G9t|Q{5gh_*mbEo>GZi-bF?lTLQRgR5 zpH7Um)!ZtR@10zJ+dTYYluRC1B{D4Nfthcu>>VoUG->bP;K}ZNbtk8b>)5db$y3sb z6jMJ|6Gzca?<~sAkmJX#!08?DQP!>6uU|hAa#Ma#h^$VMSkfZS+vB%yO+U}WG=r%Y zuVF7H@#a&&NiknDS>8F@ZFaA2&ql7V4*(M+d(g3I<&u)VcduQhn_bEny}o_h;VC^mAodZ5p%h7K@WOLbKbG$$#QUp@b3%S#b;hotl zMf^)76y)3;+Ya{%F-c`#EPhkfl2>ulHa%1^$E#$(%fpEa8AhuByl&2d=QB zEl|(TgZ-ZeIGd8HXms4u$%I}}LIgc78 zH(=W4Y@+4kH2$IgL?oN$TuDhYDKbs*Mp5A?w#G^|r8hOjd}+KSvHCEr8Y_MjhI7%Tp5R4A|dn zudzCD?QJA+H5Ui3*|XO}Htpsn9G#to0~38Eq^m77kUmLhXRs>T{kyqTJkvCkg4uqM z0BaCaK0z_$N|R$OBh}tYg>_{%O%JOc*OzDz*|F5BRVzx#l^?Ed4Dy~mTR|ejdyF*z z@oL}S$fY9RJyw!H?}t(5)QdUx=)R|z5NHo>OV^2$U{~QAHgtcJ;jL)uq{r&$o6P*K z+8SDzjxE;J1YSVC4x>lMWY5q@JP$7l8>zy*yz#rVZ46L^=^JK)g$J7rCi{n}+TrR`x`Z)yv%6+^e_#IpkC( z!ptCvZx2w_cWDO`g~{E-z79WcOU!W8Se?XB^8u-wOeT}0`j3~)IQmUD|49{@ViL3G zKH0R;$Vk0MG7Hwz;>6?_QMzT*DpCF4EvDnhbHUK=9=~6hjfccZ44|4MA~H;D5{jnb=MUYkJ1{&VYzEjZZ@80ZRm;x=7A;lcto zE76gv$Re^-W8-FWjbd(QTLC7kf2LY%$ukYm$OZCkg>hPC+#h zK+}(GZ~-bD3va8}K)#uVa391DP4|&;e~h*;VJ;4v5)#&8FdmcsdAtKzJ4Y=_b7#VdcRtH+)7PZzsE0Z6BjV!7 z@}*JI*=l}Jq93A)?a@<7CU{_%gKH+!JX4afS;AiL+Pc+5 z6i>Xb-LZYUKyQzh!l#XuWH5Yo#tAi1=KcEJTK*os-Vm`t>Q`-oO_@}T-nUz*OutK) zzYD*<2RtdcBA!J&P8sviC7IyWkX&@Xp$+xMc(#uPFLI6@(*o}Fm;v&WKBM5r^+VGJ zd(NC`tKhFuava;G_H3@;JqzAkeW|wkp<&@QwXJ(mmE>kgc-#V6^ zJGZC!gz87CWK<=6J-yvye$pA;Oq3S9ueDk7+7J?`N*muNu7BCAW?S6_@a1+npabN> zpHHt}-k_qYs$r*`4TT$6mv7a@b$bVH{`-R8mME$3ymqMhA_q7AXiW@bPaZ(UvfB^h z2O_x}HZPFB2%!BjR*a2h&>6vFA}!%mAV0?s{6!=olQ;E_=14? z51qwuAVR-qr;gaVXm**j6D&Xzsg4MPaHirLn;^?zp-iG(UflP8?&h{2NCztr@o?&% zJ#ir6prT&j%>lG7;`BNhP4LmSn571?+j82pX{e61A$aY8u>{{CaCGWvfJzTM%m&hf z=N#teXL=+jM_pI9B}`O?n+e#=pZniq>T9VNIDX=Uu=`?#%JBf-A%(UD@?ZRy{6yL7-bm6SO`aho=!Ss2sUWo~-t06951 z`Cv9`>_b0?%l|p?@5vFKkdMn=ym;ZUNC^+;h15X!me<>M?C60(4coJ44+F`Ue;hwH zf5qp0GQ03io7#B7OoBwg1x1c-B(JO4TIb*}rCW;CozVJ6PAa0YOHCttF=b3wa`W^X zo0F5f0XT)BK0tzeYvpz=&3+SxL>_T1LY=__gHjdTONfsTo!nCk$5H8fEH_tEDkj=p zTxv?gWl1I^nK!sSV<{Za@%1NwOd%`q=x+4PADpmR&?vVX!VG+S{5xKF^6FJMuf22s z{yH#4#e7UJszvC4A@W72`W|%s`J;Nd!*^V%B#}>`1!4P#Ks`NM*q@)!ifU42j{2^F zE7#$uZF_DJpoewYYL@64dtKv9%oFdGE*xXe`STB5|3$hkCkddn`dghYoW2T>red>Z zIs+z*&;`=5Q)VDgM`*^%+Pat?2^VCoTPx&xWPSa%iSq3V(Lu_*yu3V()xWZHV2kQ+ zQ`Eq3uIpG(@1{`@Q}m)KXxC4z{CdUa{dD3v^1Kpm{V@9r%`tcP6n&f+W^~k7@yEt@V7%5ip zNh|>ju}gOqh=E6EnVlQJ!Ni7c0& z97}#yeq6r%^)_4Vp1ODOd9gEv{ z?-rGomR6r|GI^gmNjYW7k1ZQ_JgWU%U`D(n-v3H@36xmU>ycT1`wxpF&5!d7 zh)HN@90QT!$Mk5p4uLX0dGh2i3U%q*wDm8hY&hNZy|pEfob9h$F$ACxl15jiZTB-X zd3%2dtX*)?)n%J3CyXcvFc>yuk<#~=c|JZW5NW)cN39DFFH&;2z{%UxDGdHuUH#`h zASk_Fy<{-DR!Ci?!n=1p_H9}Xdq=D^?ZmEp^FKNb3Ux}aoRsw8<047UF`?zRDLj0a zHv>nz9MN1)tOtvW`=Q|wyZGJsEsf~(!jR<3#{T)e%# z+p7N7)&dV=>{#rJRv!-zrG_>=QDK_VeCJiSV!(|yJ~T^gvDF3#f*hvuWGmkDlk<}z zTpk6kA2c)Z&A75ZTC_L^&e3vdhby`=^@=w!-36kfBghds+kKY(%*}^Dn+V?JfO~f8 zmADXC=Lpow#p`EkrJja-DXo7Jh@RZl74xg;@0OB}&65KCk?nobbcfufvB8Yc9kU_*g;GkI8g9zPfP`5!`{%DN>wNrk|JCOvMA_`SIfDFKIKYrW9n zThWAvPYWYTNZY$t-^4@}dasnwLHaTOA9i{>*tak=5pc#E!NRM_I$1;q5S!H0RGEf` zV1-APW#{A!rG8AD?8WaN>ca6kczCq?^{ZjN$TNwoR@6PetQ$ycaUr}xLABuh!+{XE zUQBbe{<$F`%L-;*rmd~qSrMdv9RSf&LS90%vjl&_JFB(td!T&L^S^m$aolkW8wHd` z7*q!Ah0s&hsnbUETb}quA?n4JVwUcg-5?C2_er$S!Dal}*q2|4fIC z;ZwS~G$&AyS7BG0ETJ$&T~htwzhsG~aQf}gaXnH=b{=1S90WP@46-&*oa^^QZ#Hy0 zs?rl0Jb0AV+&gyYp*a%>NP#=b2OlRJj5zX>a8^E8P!^v~i+H82R7^eE!JEI>ahd^9 z?BmT{Ck*L1HNXC1<6lxS?-uDH%b?2e9{Gk9RKx4;fnp&bfc1K)Hh%p0qN=L#`+GoJ zVL@shG!h~|7_7QDTRy5F!q@XGI!BS={reAHF*WVXGRC+umUIzsp9x?;1#GXFJ z=H@QM6vN5ff*M{I`~p5~1Ezi5Ml@6^`rlIoQC{;kNX(0n^swGXAD&{#wZ2>DTm8-t z+P*&XfNlppNLC=;@ZC9ur`eU)t=DB`WkJ1GuKYpU8uW!ciN`xI>RUF0^%E`>(-^Ip zRErwiH(In<9ld(^@G_0(c+krAzb#cE>g;ULc=QoUXwV!ugveJT1-Vb2iQ4ZRfL_|{ z&M)XO(~js@T_y|W44o@4SNrnii?C(lj8QlZvM}+r*_KP<^aldXpk|iCjqe5v_bOK2 zZsjk}39H&Bh7Tc=>rFd=mSe|`h4&EK(~8O3RTiB`#%CQDvvxT_yLk{q!`57)`t92& zl;PF9TFmFL3+WrPvU>b#c+}`cTp&1jGCFy2t^WPbUccTNO1}gack+{`LZSpSEl3?S zBq9X|F&B~+5{Kp=&ZPIL6LcC)bXtQfjS+&=+5d?xX7I&yLY|H+^40wl7If0v2<&wB z{{0R}bA}Kf)N)p5utzP$xr76B077*o-RR>7ELssWljQC%&Yvs z*h>T+MZ7<94pF50aZ1%Aju7I~vlN^NWK3Wph@k`&O6K)BrW;=8^fJzS3E6seO^pii zEbekQj0R6qwY=PO;`57t^5qNg8{+)WxpTxwKny$tC;S{41v{dQ>Hw#}?bIPu^Lj#o zEHDR%E25Dvd4e^GwUi#5SPSpuiOqtg+WD!g5~p=&tV-w5?4kIzI8R}Cgo|GNYk+9g zq`O{iIvoPw+~nx15sEk--sd;4m1t2_RmFM*ewx-Sy<(1tCNeU*lKm)wH^I&Za9;h_Y$*aYe&G{q*cJ z6FWaVGSUuC^4f_-vuBhU>Zh*mR`~(Q+f3Ux4kG6z z9WMc-T9OA%DP&O4B^GwQdG>=%km;wpIeB@*0s^cC)mJN6fugZ`$O4AYsaMRK1_IQ! z(f|F}>N+kI42oEk8v9nnMZ{SGDHF;J9N!ZXn0`y`_4U7NCH&{D{`-whzqyp0Yl7td veblDk{=Wfk|9yP%+yCD%<^QK!t6#mTz%Hw5Z@VlpC}xDiX#0b9)7ShDHAEEa literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb200/gdn2_b1_flops.png b/benchmark/linear_attention/results/gdn2/gb200/gdn2_b1_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..6666ca617d30c5f1c61886078567ad60ef195d43 GIT binary patch literal 85416 zcmeFZc{Gr1`!#&Y6ipH(k)bG(p;D4e5lMvOPW ztuAgCm5>mX5aB&{>5{dbw3wLXf4)N0%GONGwwZPkKZVI!^@JTotzjkq(3nNnX;Cy3 zbwEi$+wsBR7YALo#_63S&S3^@N^II}7j&P-WZb;b%bC8>kPMb|bda!1iakwVP-oYq&$rbmsn+GF~ev8;zKKR+J=JsMwUXL69 z{joXdfIT<0^1nY^Bf>-EmHzwVkLLlV|Ne|ph}i!HUpD^F>$^tI%+A{Me-a82J;m8w z71U@iHBPUad3Kqs%cRG}FZr=uP-*i9=ZCwFP6nZ4cRXFyuH5L;JQ*GLtQ zdr4W@qN9%v*V{{lcvsw9vq37d#1H5E!n1h>|bSEb#d3Wq!)-QHkVXXC|gE$bewK|;k+i@uz3HaOFKliZq)@>B#la z(9q*U4XJuQ2Z9AlU%zJhJ=|V9I#;}UkHe3bKYMFwsC_=ml@ruLx1|=__NcwAsEE}` zH%P*prw#Po$&omotVJ(oQ0U>~!?-_y^O(oVZvo%GpVLk~$)22?yqK9;%Sp`gOFlav zAD{40yq3`0ckeWlwFGi=b4!|=Pj<`h_%*s)MMY&P=MGiV0)5e{Ym6&81_o}vdZkLe z`e3n1TAFi>m;swfvVPvSlUe6tM`arG?93iW+cQ@M@u-A}o_dy@UM46ORn_`x_;+Dm z-tMmMZkx#wqw((_Hq=fxCKO(oH4E#KeAwq2@nLrQx6Me00#))U`KXb(c|>|fk3hw4 zmzEq$4q2C6>@3UYy=%_fj(t5qM@L6dk9E=mX=oX~I;AuVKdv>o*w&60%C+fc@bvU# zS-*a{0?TPLGsZP*)`W-|8a$b;KC?~##TEA7Kfdh4TKoI^pSH4ELH<_gsGv?2!x`6r z*qVnbLDoON9a2(O#&XacKYl#vcq%97u8T{L9z7Z`G}b??Zy=LKv8-L|Yu8)T`D>t2 zeAh0OFZuSJU0sWOmF`##6~_;(#`%ep*}#GjzRkPX(|V{`G8UWRsV&S-CCw%ueH8fA z{G%YBq~x)L*{igSJ94TgyE{5~-`?MK#PRo!$Vy$=;ejwix7Rx+?g$)Yro03W-c1U5 z7qML>Nh`sx((z50nAqUw%uUlP!td!u?{!YJFUYoNWTxmgZF65n0 z1NX^JACZ=(rl!S|7x$jaUk~g(RkyS9P3V>-6y1syL5uuEtPVxVoS+tdwWZU~Z|^c? zioXvGEWs+Q#x~itGE%X4PqiUY{aoHGt9FCU28Fq5Uj_>=w-vjK`LEr+|9xYlPG1$y z0@(zkB_a=($dOmGc*cWgMIn>+qv!STXl+;OV&loaH`Q8d)^z;CHeEu zv{vL^=ZPB`^z`&S&KH~x*eq|)vAmzQ@3q73y=xw?;^GnszkmPcP=Uj6l>1eS*rdHd zc~0(f&J*h2Tk|sOx}!%`_-`-%QsA&gao@h}LuQR>rxYW$nSPq!3sv)0xW>SQaD7#8 zbJji}_EvcK=9xde(Tab@dL-upwr$(S%CKU+NNbU+o+VG-?4B^4APZyTIB}aUrR?aT z^PiIVaKygqO^fyAdm{&IPPhKUfPLksmaGwdS?KhWH>{fPwyH+KWy>8GnphBgipIwL z28E7S@%J4+z7%lmxh&FO3^j}jC897esbUoR^!r^OdY*?seG zaCNEq@IkZB0Waiu<%*(kXx#DClSz8dMK2FECwwWLbtgG&pdoei>+4mj30k>%d9lcV z;yZWlx3(6qiQMCZcYb22SM27BgS0*>Dyk{Ra$TkW+Tc{zHT#`LWwc6P-E5S*t$4A< zyB{Zu!N$ghd^66G?EX2^H}9tRNJ(*QIub=kh2OguckIdWB!d^ytgNiovs3o1`Swdx zg4S2OyT7ZmxA)P6wA=K=5Ql^@9aZx3CEcll%ihpTdbM*@#W_H?67;d zjMKB`Iol}*QL*(p3M}I1-jmbKL3Q-^`(gWdg@u`YmUEXOlHTXrTcCQxV4a-W<_`X? zFK;AInJJSD(bwEnakHYpK>yS!Z|qL?FJoh4FYhI5m@6wQ7g2oD(kGMk@2{mrG!}CDpnm&g?v@n0hkXEA3PP^RA1n8bV8! zFJIl?kh-PqS4cg#O8?hOiF9=a5BCSqQDwDq^CCD;$PCEmXPduSJP?ZAJpHTDke(`w zQMk6F;dTGTzO1YrM(<-3x~4z0x68G-p6ePO7JDG;dh&*^?@Bra7KFny`$vu^Pu_bU ze_$1jheoU-edo7#_l5261P6Ps+I0Bbpb4_=qFb^ihcxe!OkkbpHCR0~AopDAXj#|> zu^R8Ss659g-b?+T4!A^;zkU5~db%~w&NCq);pwG5Exw&Qk0-d`SaqG6z=Ixd@rjI# zq~xDIf4;_`*ez9CFs!;_fR-vNa-Fulm-<+vWL^CxCmoxbo6Ej@;hvb7h{$oC zpWB;eSRDBNy+)z?{QA|aS6g4cygyj*P|Tx8k8-WP)k9-W>{?H@x87LW-NfrdtTO0SaJvq@>@|zBhQBpRFL#L4Qu9MPIDSKu7 z`&C3{H&S`-m0#!6*8eOM*;8wB`ZVu)IXC~OPsLyL*oJJ?!!~Ehdo#0Z_ka7gmxq(n zuR2U@10I}#fdOY!CG+gN(WLO)IY%>vz>|I`dmU?N(&tmtKHW>#1?`LYF_R zcui|tTgB?TTO|!>O+y|$SZZZuMG_Sg6Vua-jFpXHyS8utIF)NJgpgsz`b>^i@fe*w zdmV3adZeTD&&p>f^PJW%x!+n?`~;u~$jd@zO(2KFqBWxWB}fO>09<%rR9x3xRJrg$c9_(9=Qxq`ZRfLpga*E5JG2IteB*{vm z!gbdr96a~=nQ2rGV^P#XXz{1^NpJB)Y^jD#{T>#hSo{M5ya6RrTfDu!nGPH{pw{wi zk4Z_i`e$|7YTX;0|U=%*RH)enTB&7QL`Gg+Z%7Y?`c++cEReQ_p$rx+h`^x zM@?c;R?as%G4N`q=xXL#cX~Fvp7o#p`Z`z<4<6bqs{2%api0H4|5NfJD)!L*ou0gl z%L3NzqN6-f&sbKkPSkR5&N7ck>wNO)QCRAgwe*aPOCt9;9M@}aXgJ>eGd9<5s^ltd z*$TNC-tT>V$yrvPpCz!cvxh!3(bQ;-?Cb1|-5Ch{uo#bVtVO50GQfNK_i(TIO!L_I zI9u($h}1nP@4nPsy?WJH9a~(h$^E+I=zYtEMDvP zUO^p^o1Zz#LKjf*RNlD#>U3t zKY<5wey6{a+^?#7<20g+V%hJ{aXWP`TFyP~*;cW+8dHJHDoH6RU6Gc2`zOs4+alB- z1Y8{Z`r4^k_MuqkKj4h*1>4yL)O`o35QT@LuKemxsOOGm?r#u%m!|n}ztPlqpX#AQ z-binids=~tR*M<KXct?rGfDu49spRbe&#pZf;I_;I#IhD0b>Z7XIFpv0?nz zFA0=*l!y)J0)aJ@k#hhbmu}j$DLYdI^_N-i`9*r_Q<5g_j~_pPlclgKUH}>0?e6o+ zfIy-Kg-JVpDrji16c!eCbaYT=W@hM1*6-iHA7NhF_wmt6q{l$+JzYq<=RPIfZqDUF zrit3?ta#+e3Uo#)J#RvdPM^Ltbb=M2=&4=rVXEZS?F}>(y541|@;6cNF0>bM7tc=I z*SZh1QQF$dg>Te{N+s_@zLn`;?2^?D#4`BYM1O*2OQJQ&Ur9xm;XbZTjj2{JD0sVlm!L zWSr}M8?pU+L+U!MqenFh9c?ZRw{c&-eA#hsW;fpHIugJ`U@D|i;qwhC5t~LWs#{vt z$xqqc56+Hb|7x4(Xcq15tsI-TK}cx3L(CVvYwu>Bu!hitSdqG9pAp)?HLrH z2;Sh@dxZ^={f9GC)cmJR)NVTlAkgbKZ!Qr&o^s=vX*E9*9o{bY@(}0c;kE!ImIrd~ z>sfv2pBCNuRoS~YUn#GG3dlVPDqt(3B=ZTs}puuEMnBhyB%EG5k zKhIKq*DQ9Qy31dzw10mb76To_Qg9?2F1r626X>sh;uFLpXFSxLjc|XVRRCb1qpqr( zXDi?~GpdrHFox4n4rCH0W>|^RBjGf5CL;v_jP+CoPCCZWU}9oI!jG^D+t0o&0bZ!D zoc;E0opzf3+TPk|pKI6l1qRJGWtiOX^HcO?6?iEoPIVLv7nLJ-XV?!MfBRtfQK1^d zL@b(trY!SSz|$X|=!h*{yG<#<%5s3F?^+kAp_2Q%tPq+WnP#;PvBMy6%IACS)J7?6NJSqFP@)!vh(&^MBl0WOQrZ?o0&QZmQ>m;Cg7d-!>#wg?f{Ic~J`9Ba=4ZI3+Y7_LWLxot?APazP{k!B-`O~y zdNb(yBxbTsFS0eN+HNeL)&s|fv~ae0Tqu$iNTHf>N=z9uGk<=5Z_eIw=gyr^l|Z)c z`&j6}GIik7;@h@q2_I2aef7a!Q=$0oSik<{+^n?{nj|4XTHJCIyHM z%|?cY-@rD~P|eS+(sxGKq9?i$7+47as+nhNN_fEK%a>11j&8v6`ZKOrAEO?ztqBP& zhc$^kE3Xo#ivHHzXQhv0I=gSLY24C(=2tG@JDdEJ+5NX^s-8eoEET zNl=$ib}x3D-SNZyL!2@?8T8!);q-j2(sncR$&clt6QxaB@7;BL6$o}9DRS&rxz7sE zjEoFQeq?M+tEHi`a{rTLpX_7@aG(N)#w z98yP~G*RS{#?GERBEfBY zR*tYdGcz-`^(wFL2xR*2-t$qhkUF}KZr%>8*3kob8ZgF~fdv}6nq~@&p80ahBGJ#Jj1(+rk_=&5j za>iTV;Naj&A!r}!?=wtPE;MH`f$Y`VGi#M@@ZwXBB)A)}Vk{gSNn*r0ybR=c$obQ8N&xSvI8Yle8&zY91zXy^T$ZXE-NcLGCHb&AVhs@5NV(9KW7&Z zJIQtDb8jr-_PonQl>p9!%V&cMC-?l`ZMnYjQF1x(u=7 zR0kaW56SubuY6>f>VM5_Eu){@E|i;%8#mr4-qO%rvjkO2%U57KxQ9yLRhz7T|2Q?+ z_F|99(a*Gfv>}!)IRxc`oDTZ<=!ls#OY)ZJP0VNFIkFBPK7716%BH`cFrzs0R2`$@ z_PP6a?+P|7S+?w0b0fONtE7MeDMeJS0a<$4`EWzyY-^5X3pzE;bc4O0p6DD&m?8BX zaM=3$d%hIC=ZA-EnVghI!~6UCI#Ej)W2PLLJv{&hXjSLS)FL0qL^~<049l<5Q~-R& z5_y9#)19~bv(jBr(K9zNlmCFw77#_8j3ZzWe65OSS7X~${xU3+7k=MvWV<#YKc!uD zbM;oO6P%`~c2d?I)WxT0hX)86*a{>ht>ZUcJzdamz23M|M{Lj0miZYW}` z|8Obr%^S1;G&Aj4A9v$C{l%TKt@?6-5c<%RR_^Y}4Cc`}dE3m~CT9+mOp)MmpX z8AwG}q0odrgWh8V%#rK#Q;;ky7No4cp8e3h9SX>$nKNA)!Q@0u=ud+3_vhZTT7GIx zCb$IpRg?2!ExnrS4Bt#U9x*|ju?qpRcs7r)9K}#SwKnGkY`XHhUN58oX9*sX8{=_9|Q>CD1r|rK_uJ zf|qA&PVMGhja!KrX~-!kfOPahXG`}QK_r4Y-Ql0v_3Q&Sz4R3x#W@2v5tgg)@a4$S z63!DB#s?b3TCR<1YHFVQk}rv*#kgnB9!Do9^TO^!jj}u`jr%KUiV?u)>i6H~T0|{b zEyBWhsycLw4`{~nnwsTj&z|ji%Z5*T@GkE95*t13@?v0+m5v6I5$!Fv#(ViRr=S9z zKJ9_yk8Gwg6UQ&NdHqMUN^}eiXHUp=cSmR*j;bw;u^bDEE9C9oB`N708_P%3i+d6R zTlQUFS_O*Ewzgl*X{YvZAhCVAt-A61r{q=SP=HS77ZN(qw?xL< z-|+HT{c8Pnd3nyewi9;#r`U&W&g+0{GNTKMCc-SXqVO@Upr$zzy*OzcD|le#>}fhL84cUJQ8 zr1AEGfx0zWtx-$dQEUs{W;xs4rniELII3ZE({@_MZ7X4|WB2(eiiVlA=Gy2!zZhAW zk8E9rm!hG7zID`8sk-7pgQuryWcKdu93Bn=1>H5?_wi&^+>qq;uI-w-x*TV=tsMu} z0sC>35xG)Rx+dig>$c!81EGU5X@140NRM{@{Nv`|oYbiO)Sp4FJj-a!i)ngbcy1K! zP*($nic3F^5X1A*!P%y*!_eDBJMiX>9pc(ucEZbi= zZxY-XvPEY(x;-ab9e1O+@KF-fAlK?12k;ARKo$r6Nl&RqO0%Qk)m|HZ=+L2e&&77J zqv^g86T?fDkoF7xv-$ui7XiayR<$X(e}{PCCi>i(=gc=%j_x?h}r%i1Mle%aZQ z5I#mb%Y9_}(xD=ajE}zrt2Tmu16tbG1I4?Ky|`jOI*<02b08Gb;>{KUBJq(3UUNuyaJ zK&sUZNyhuU-*3+l^qdzB9Q^DaCwhr_zpeRzlTt=19kdm!8k3uw8=#&d@-4!M@yL-Q zt%Xk8^?HZ@W`pmYoix0&v&o_-G;eQpEkHoa$vCw(2iHXjk)p3d>enT3RePJ(5~Ex(5Gb#oXFD=6 zQImPKzf4KuyIAmRaoew|+DRG;133=EZS0M=Pvu(2Ht(rG(G1yj^{g={Rn!{`z}b`x zgOevY3F|}D9Ap|bho|PwVb6B{f{eTzcmOto{F26}47To7_BbjKhc+#rd$J+;HK}ye z=8@mZ`0UE{>%p17m+2Gh&g%DT%oIfdUV@Yh-d#yoH@xzSUw4@5M$X~)wN0`GOvmya zhrV4;>>Wi{OL@SrV6J~>W9HA0ox_hfCC`(p-c`gLyxpwkG_dFHi*hzyFVlAl@%@c^ zw%(4nXU!HbTUOuyV+vp|7WmJYzwh=2mG|j}ax%^nOZQwJeAO^0PCt9{%@aGL?nuI|+gj+>dALv@reez_R@ugm=G@200l`|kF(_*a*kZz%XhGn6Kg zkDeajaRczc{hgM=Pvhw<({yada@~Hu+toGi>f-WAkR9JVIW_h8Q|--(wLRY!Shw#7 z%j4~oNFQ-R z4k9ebGRN`nw{y>whlSsw;*W?~Km|dKGD7^H6zimB+MA!U`YLs2^}3_mj(VAVKwW+5 zavs^G)Fjm051@S12MhtT5E4!K_FDiR1up8SKZ>lsH`M%T0NrMeReK;1NjG|SF!GJx zG{*dMs^Siy`N^fX%W+YtZ*XQYo^K3>$|MaS!~kn3B&d?8PTM6m&CC)!wH z`<#nk9@~qo{q_9=E$jwW_-7Qfz=iCs-y5|q)_R|KH}MPv2f!@p4>-h6Qyztfo>kZO zYQ`oN6+gFZxeoHv_u!Hg!(xLcbWcGB(5Sk%VN>51xjJY6BCMpg+#W#Hg>~_`X=Hh}^#vh?X{vPV8ms%v@H<3#y53m(I=F|Q~iGM{i zkp*XxVIJrOcnv7UC&yF0fJDqwoQWAiH^X=lWrX^qX3Dsni)GqrVi24#w2m@7z)RVs z%a+lLiHU_?Sv&0)74r=hV^EO$jt#A%7i$mqkHagUP6gBOm>{IPqt zV*m1UcCGy>rwWc!yi&V&Cx^^J1OQWyR-Lm2G_)?o3&}Gnrq`C)Q4K_<>mWssctlmM3-;6=SK>`oD^?(c=%$dUMCt1^K29F%TGLu zzxCjO$VIE>EU?>z7*8C^PTmy!nhpRSZSa%hTRWk&oF9Oo$pqO%IGuL;_U$wtM3D)Y zh9H6;628abVxTb_okEg+-ZOOZBO@Bk>v?#@P;G9EswgRK4wZ8pkr%OgA9Gc$I4?kM z3sCMFYB88&wc5_!d%LVeDHdsx;c5U0UN#aNgxx2foruksvx6Y7tEX_9yPSJ_quq}? z9YLDY9<-XB`o%fj*Y*P4EfTJAT32I2t87s5%-Bu5lxCdQNLR&@glFhgZ>B3B>eaDq z{|T9am0G-W=gu55jc^ zvM`|Fn#QiJ#~!l*?KAA!^_;6>FC=3e>eZc{oe>QdEje2uyDg%u?d;02-8&lWA>4w_ zwq;up`w<80bKyWny9@uA#E$L&=JV-{NfkS$VhG?y6@<@V4x-Pg>?Pl5fP>* zit&Rc`yNG)fU>jUw4ly?Ok4(Kk-8Om9D+ALN(?%57Q>)>U_!J_`+tFnE`w~07@?&?Hft{0 zay;c#CWnc!amd-O8#iyRnx986>N_Az+vgyPUSvzQ6$I$BH4hf>2%sC#pc2X?baEen zPG1LW?9*y|%Zp|X)W43Qp=H$f_G0%ouJGkO<4#@8xX@Cgv)g=uKXefB;WxH zu05BRqT@}{i1o}Zn(57Qt;GxTq@|@f0`5SKxel_$@2EB-BV&1E z0S^fkb%TdcgrkBkSzEYm?ax0!+1c6R_WjzOKncP_+iN&(eGYo@ z;)OUm)K0YfoF*U6Cc^OI1@c@k*~)&qz~&z(>%W-+OMP~0&Dwjh;s_ca-o=0R9QE|{ z@$rG}Fs#om!{v`{!cOe5V!`F1PYqVky@>FJ*5wE_=9cIMrqy@4InyktVF+k&BgI1a z>`PqBICp65X4t!TZ)kUYVifOUk`TomM^2AU%FTbkQD|*#^+Mci$*vxX{McNXWS&Up zjU`$_MB{bNfK8G$ceGd>+-mPh9;A4ea)`^Da92e7oS20ri)jUq?AxrM!I}^jHn#X* z&N7(=Ci9csE3$hx0+}0iz4TSo(z;Wb>1yn3WpNSar_ zr0{SKz+e828|e^;5GxduwGs#z0^~9;te!`jYASXwuG{IWv}o0;RcvAgT)-S8r3q^% z-7;OecKV)#2?O9`eM9i;GsHOom6RsFoo1k`D^A-CfKK(mf!DUt@-=7GG#pwZG~b4F zpa+L(G!~)8A?H@875U$mpZbZPXzU=-u7OW1T*Bmf3YI#rs*g&;`>GP``&XPz4O-HE~Z~xTVs{iCN z5!Zr(R-o?}xBI4!g{GnC6g<+?#V)oN1z=TPo!OS1x#XPQkxsxs;x$rKRJ?WP&Z2ee z))99MYR~b4)mdGVQm0EO?R0}vmeFW4RnWNN6o0B&A9u*#q9LUeCfU92-mP#e|#z4^jzp-?79!bIo7yLd3c7 z_;@OT0p+0N8u?b5r{b+fe*P@aFsUNC$0?tEh7v`qi-|Q~_aAG1#X{SOG_J@C^EuL< z^7-?^uza63HBA_}#T)>_t{eT<41a(hSB1X_*5q1oadBFCY&wVGO~YLVCTt=G3$#RX zU86lkSW6rv0tL|8ctuC^6pO#&1*H_gDXEA?XZO=1%7@W&5-|nAsN5L)UMDcUB7I%D z1oQIr1#;_C_kz;%_b2`RcZgVg{__8Gw4+FUfds?4{pTajrSzY-`Ol~L->munzwCcj z1AqFzbwOg&2dD*d^fGKWS8?(PZURfxC(ZTu{o@o>ke*9D=_wDCnDWw6h37PVO^Do4 z0M637I#%QlOt;|3Mcjeh{9T%$8)1DiAR}qfaPHW1!(z82xKL$P#!c@_+Y1vU3ihQN z2tR+h5Bb$#X06xlyikH{dn{+5uaEdW2r`7~aoXJc#7L?6t-E&_fDI^*urLBWF0H#2 zIMkZAiZIN`2`jm{Fg=4Gv#nymCaYS3TrLjnHnqRu@v)o#nNU)Gg+>TY#cS#V9GslS zgoG|~*^T_RWb-yW^AxLu&{gtN>cI%A2QLr5_0P1E_Ky0$u$k5U$bkb2j*e2`r->yS zKy3rCJxGOmqYx~B_NRY;0?08C*D8%hM7+m`=K3e-#h@2FdbU?mk{$B~FVSaZShdTF z=;yK^eQceid|~jtS+Wp`Z|ud4pB&0;-`XbTPU4`Eh_sCx^Cd-nNN*sCsR#$Z_mm(c=trIEL)CSI-|mNLb^QqIQrf_ z;k(XGPRZlF0|Q%tB7CFMgYVq&f>XH^rb>$U!CvR%3Gbe_yKR}+4YUF ze0!Z?CUYIe#>K=0e|}4-T17qxEf!F>adpKQEQ0lU6Kv&&C}L5xaPi0SU|v$Qu%=FD z(Rb@UJHtekl$O$9ovuejtR>!Ww6BEYAQo@f`HU`HNXZF;cZ&}ONBm+X%n6)6f8GyR zD;A%9nv=61wxGfnFFq7FSY<%8w;I6&*XVEH(l~6I+Uyl4CmITXI3de*xa|e3LCP3_ zAOlzy78bCD5x+VCUc@v79Bu?5kTXc{=vxYtT7g(~K14I*XksDc;^xMxU)`V*cp4N6 zr`avyzQISGHwF@BV11@2Jv}{w787cqM3e9%0zBx_%|{cLlI(}4>q5kXNh6uf21cgP z985&Wqj6_hw{A(YHnW3+13=k6!Qf6rHZktu2W18RM${Nz%tpP3 zVGf(K8Y-<4AVFxeP$jd+c!pQ4Ip@Jw?=rSPQdkc1DCtzePEahETw_8b!0Yv7AroDc7X%IQklGA`^`VF?Z_j*k;iO+ z?{=JfFE5k^^twer`=6A`9)jRAZYy-^8YTlx{{CK%9&LhVrupp5fx+hNgQlNpsqkG^ z>(SQIZP~IV^cgIrap*osk4`*q;AS#JcJE&6IyFwb`vi!SyomQw8#=0`Wj#0Jlp1IR zfsUwC=m);lMyEfUQ2>klGTw_=Vz0x1Ppq~$XyxcJM*2Q(V%u_DJ>{X05Z|M!D9v-{ zHoMKwIv`CSRo}0nn|+=gv!oab!%{8Bda8=(fem9=%0Ygu`RI4N7CUQx)MHYDn4|rx zcb%EoZrQve;~i2u*lRK}cDPpR{HM({9;c0r_%;d%__k~p^#*Auch27exdS9Kn#l9h z0JQ~3(r_{JZ`e>VhG6r{Fby+St47t?Y}qLI4S9Z{|6g!nSw?zzP#HX;?f(=zdWF#w zyE1t|9Crf4?ry7kYE-UWxg0I?POCN%(OM1`mKb~Y{<~J;a_&XBXUYHqQgb4sqoZ{< zb>F#zvkiV0IfCuV?;l%5lVe8EA!5X#iX$twuW9@GeW>X>a?-&!Atmm%jc*RQR4qYd%tiJ)n|&k0TZjFgPq3rI9@IbI-bM%uAQPMEgEEF{Fnk zU6F;3>N_g9s00UOM@~K4PfV}z2EUF;PY;44=>roUF~cG+D41nsWh|i5-J;xn^|Mp;wI9LyETUjL z-26^3oEzpJG?(SzqEBDA;15Se900}nft}~xQ>YbHh*jcpwz*sGZ^DGP>w4>tEj=!r z4qh4t$+53Z+v|4RCE|mZmls9BxgMJnn2;cVv)~Q6_WWSAsFo*bzCGJ(>>i>ReO!f=*R{O zsD+dughtC7Z~xSM1cQ6nKL`@-uh@#yZK259VJ|S z@=VBywE$yA<$qCDHkw)Ad5k6JR$c)eFD@aW;NnuKbsuB7{@cFjAk?3l)vlKiQjk!M z30GDaO*fv#HiNa=nDg!x_*-u2OY(qEZ{NNpJTlVQL&woBadB~qBBaQXC>i+L8aaX! z!`Z9CKbyoI+Kny1ho-`P$jf9L5{>kNY3u+KlE^i9!I92rQNnjvwSlz#(HCtJ}}3NC=d>|TzW{SR0)>$fwIJQD451tizpe#5egK)0)!Rfb|w9!>6;J* z5bT&*S%EAFQIA%Go74-lka}HDC4Pkr;ix75&G4Wd?zGwW8k>uL=8ID zd`6`*9?N%t2}JikprKI^h;WdoiKOdh@B@E`PMri*54*~H$hw5Yg;z%Vb31f~ZIKrx zw)>fU3h{`&_1R?q-?lV<3)saHph6q?kr97>)snzctI2bN(Xz&PBq3=Mh7f#-mj!7m z!ulfK@!twXweNt}U;^Lt-w9#KHstD0IAk%dV=l0RL*~Gdm4RG9ZHs ztNo=qoD@epUBR!Ys;Oz|RbhTseu^ysJjbEHiKB2QlZg*%1ZzpgNWk)JK9!%Pt8&}p z)-f$l{= zXumk^qJ^L}LVw5Ec=*$GV~x|u)l|sv1&xq{8(E3Wj@dIc%WYb%kr{5YS3;Hi!24i| zWLuTQB0@mD&onzil@KWng+@43W&eI-u#%SI`%ynl3rC);5L(~ReSRk&d?fMAOq7h5@ySkKizW?wJk z7?YD`tbZC?Mx+X)-T15>iikEN_z;l)yJ1P9w5!(HIuAB;%DKsSjK_PCVje0WC}@m> zgi+Yn^<+bs?-Zrc5{r}AK8#V05Ksj(A@C%l3&eKV2B4L7!UqET3Ihd0;|emgR$m0X zIEuOoC#u2B-iMevD*t=I10y9REgc6Zg77S{!IPc}tSgW)aUkfPJlTa#q^m;(gy#r; zC|A`K#N7pmJw37Y-cCzDgf%e)0GqueMjPB<(e(MV@9R~6^MDe44;(OnXi<{|yo)iB z?wy{VE_L&2YU);KF!7%*LN;!CX1WC7qw#S6GBS#TK(0Urw#Kx8%hWg%yRf!ARM9uG z0sj8h9VIkmHVA${MWjF2XD#4kUxQ(S($Z1@P-f7_xH2K4M}kbSgMh`B(h`mru)h;$ z6Sz4?Yi~u<)x-=s-n}-&@^~XC$OlF7$&)8)k*!<>Ic}4kjKp^V5MKgN*3sE{^Udag2s1_&lci&-mvL9>)gH2$1(d>jCnWKS5 z-8B{hT^cBI@51!k+V!MsWM1eXjVyJ^AK(123flx-7 z)@G1kLl^&Yb}c98*}NcTcHz_TVymjEqJoAtD|o`MRE9K4y-zr}CcyO_)NwSHo_G)v zuaN0tv5$oF03Csf*NNpJc3P=#kU+a&bR?PrTxhfr>eun3V_-8T`7C0)8R)5qp{jfT zE_Y=RPY}w-;|s7~q4)I0PQe%vi0?T!_|ScRj%)#$)Pnkn`xqYL7h5|x_<-0Z{C%EX z@A*MI7AN6-0MR)f5Qp%6j5^S%c*nlf~fHTp|Jec#2kl=+(b$s-v&T{2Qp5qc*yHeR51N}+QMQ9 z+E6s?WXc#OkhErrl|-G~|37!n{C&NJe}AvuQ2#Go{ZABQy1*3J<8N;K&%b+?EO>DL z^WXS`X3^>YuY8#+ioOT(3_!3ZW>Zi35TO7c8=0Ayt^x?4akA%rqjR;J_V4GlIeIR; zcx)G7TQoX4it)C3dyc<8Z6yy+`ha*5Cl}9Ojv%gV^Yd<6+-1O`(MmJgQA*3L z^3Ts7cSmSk;x5zi{y%^6KTYkw>-GO?t;pMb-mrKPox*!Or!-hA2<(@*85Vx{?HU_@ zkD{~xJgcyDc!-7)?DgcX2)tYVa&OgM;8|FLm%;4t@4{%upaBMi3<6;06A(~t>}2U< zt72R@5`GM#?HY8@;Gob*@Ll~re={gZWx=T%k+zRqV+33xjx-Egx+4buSFzqG8Kohg zjf?Zdiqa6|t1fsAp$7oX2$uJhk5c@0%;`GLPD$bphfy#qFj4eNZzSKuAGc@}77uj4 z#C;1RkoflP*R1g9p>?S3&SCd{y9zg`HsMR(~Y+l}lp1d~> z?nFiHlj4jk`}eFU(+0CL=lSy_q265LVAAZoJ^G?VfpOtmrZ~^c43-P5yHRK3vY^W( zo$Gj{yYN^4^RNFWp&?ELgMxI(m2GWp`OB3pL`6jj?peOP!G7~UhtbdrAbB5Z4AL6; z5}1>LS4)w>?77Lr{>qiE6@X3NWhnN`eU;+IOkjdWfi}VwDA=19Y2|;{m_N?tYj^j< z%Pu9aUf~xkCwvb9DFfnvyWh?0$WJ*5YXm;)fgBtQ*4c4-A~kK_KkpH@=q_k$uq~a) z?8=y=A+R1B`}gA{@&&u_DnWG^=7Go2022-cq5x@H;Jfa2pP!CS+xMS``S$he#=sSe zJv|`y5+4xMw0)>kiASQ-23Y<+lIlTqwHU>N{I_vm6emcmDtM4+C}^CGHrflnbUa}d z9S!FByMoY@>F&zhrfjGb;!N(Gu(z+kLoV|mA*r2(aaLXd0fDL>{D;)&Oxj*S0gc;Q z7)eTK2Tw}+K{DKjIU6sWSo8D5M;1Y1RkGNHv3^AeuG{Ee_ z2K8POTEPsiNa6e~hYKx;O9d5~_$CF9MEL_@rFLyv*n?ldHVg+39t2D)!~HV=&e3X- zD-wi0?h1YzYJr^ba2Kajd@egWb09kWJ@ z7A+z+gc%4ch(F2$T5+<-cH$W$*bv`U0dDX;_~}qtGE!?*^oN14MB_J-X|VfmoSBy91OS;set(0P%a3?P6P;Yi2~Yn*bld4 zM}UTb>OdG{;%%&rmLqmAvTWpbCAi`g6D8X6>=>cKmqG9(#I4B5ELC;YCQQ!F{Vtx5 z#oCiUBjRAd^}ltCMZ~a(ixLM*3oB*PcmdJ{G@H_@Dn@8AFz0gsSJF{<4&S=0r^FmY z>_-SuG9y6RB%)~GRwQ!EAIy$#CQObj9ERr4u$9AD=J}Tfr8BAYR{%dsnzlt0xmN+Z zRtBaHav))=!GMu;-TlhfFAxI@H@az2Hb~%teMNb@ia(m5M5z5r&=Ml|E}*d^_7czq z@Of>7HEJ2SxETza3-8)Q{ElcWt``-_CLWITLpMPL1oTu*1rB29R71@D;TcrHsmQQw z*-6^OO@|+_tYRhCkdbL0boLmBc?o6%x1Zer@HB?LM=XAG@JQmUJQ*K=%LuJP2QD^1 z4cSVh6G%TqQzX;u;C5c-6R!v~P~K2A36sjDgn`6zzQiLF^oFn#N5hUI;|oD}83OqI zwh7uFHMsRtZr~h}6^GbJLf_Gh=rv@s zaSdI0b@dVuS*u%cwHYlaxmP;^iPsM*G5~%Bp)?@NB-mD53V;ab0lQxA3tTxv#sXG? z6h;f!fkFf|$3v}^=soTve$6F?h7R-Rn*MU}uSIP)9_*_a1;n)E)X4fp=g zsx2IW-Ip-|;(}=iwU++I^dM*-!j=cIVK}fPa0%6iFr{BB_+gC@RIH++BAh2BwB|2| zFeg=t0}|5T+|+a$%S_TgS^{e_6J9$Cb1@@D7!pFd=6->T2A4CzJz88Tg)Hk2r#882 zfV8Yoj2>whp^rs45jIFIQVxxqHIRmrWNa8}-^&dZ$L(3w@bDI`iFVTs+ z8j-o+c|F2aOU5s#(?oqo80Z#5{FcIvO%NEb*zqtR=bvI6 zwf%w6RpwBpRf6&?bfe_fM?#Fk;YQ|cao8SbeJ*yFBPSZxBpe|5!kfmJ^P@Zvm>FsC zHo$NNt|OF{H7f(JG2UN~%gL5wxHli`sISijx{fdSwFW3M?EF69OI(0?ODzFuzKbfw zc=xfX;iim-;kRON5GxdJBqeR8e2D8Y-)8>>Zn{?F3H_ztnKjf|BQ#IloX zG>Rq<9zJ{>br0S*3Dj;P?BRfjeAn>x6T!_U38PmrAKzN+&VzU)S~~7?^n~V-u#<*{ z20iy;+$pjc`P9ZG21`zbR58lB&D+e%J#sx^@gm}S|q0Q40;Gi{%m=KXN`M<_h-R*nYzVU9(qNN>UKQwCCoC;SQ89PS2K!X@X%g_y$a20So@^WTM+GYIEL4r==~ z2=5Di$5`DZt0~;!MMGh@>T_TP3ItFeQrOXZ^a`E8Yk+|iL>KZuTxy65)rOeS*$NLG z3>8keTnva~98JAU;Mvx;axCuw7taEiHGc%UUHITLWfc{et`%p#0Ii~ukbBF}%Sv2m z5(gQb@3BE($`%t7CE&D(aD*0{m>qBz<&7TrMD_#BWRM{oM#e1x^qZ5`EwlqyuPWpi z=h+Xe#!Yw{n0ChfiW-*O%3U7$qgVY+= z3n|4$P$9jAGNKP2ptmYx6OGE_aNvGme)A0La9;Nb2J z3ny=guzQ%eP-_8E2|^=9s}TtmH&!Ua{!OM37x~@#UY!{JX9TdjD~DW5h8~TWI3Vti zV3ZdP9RN3w;p1_%NbrLPyy=}zfx<(B1tWSVxv&PADe04@|4Qh@dq0lSSXm8U7}ERb zMhyBn4+x&lx0#=FB_#y!jwJCCNEdfbQoyBj^z^1D{`cd=isP}8lask){>M_sv9C;M zzL5VVu8g0Pl9IadaXTPWQi=c*g~{WqFz+L4R(xf!WQWZaF}5F586^9VD{(NxW1I(4 zg&3J2mvmtO|8>QOvuR4ctV!C*tmMiwOmByd5zz=L;GZ0KvSM-R3UD1e>e$1_hi!aW z5RBP%ZxjL(NL0iQPp(bCCM{jJb16lp{fU8zq*zo;85tQetywH~X z_u77fBnLj}8uBUHcY1&$Ya1IH3T^cwim-MF@eb@gAp`#*is6t{Nhrp4b>j*%Wpu&N zTnL3pZi<6P-*IMA0>r5|8GP}tf?1T@Gl{_&8(bzw948OPR>HnZOA$tloGmnBDoC)1 zCS1mF18QR|{4$_?-_>_-!w!$OVvY+F!o~lJ1Sythma7G|q7I$^+|=b;s*(KxYF*uoTbw0XAbi;x?V2YWRdG z3bUXuaVsNqMK4sqQshRgL={f8Z9M_P%P`Ca5A<7``}(45$XzLz8N;QrxcpFGY983j>V-gFgxj)x`ts9W~%V&+RbU^TycO{is-)RDB*4)L%;MMS4c+asVwri zc{JgyHLz(!25KSsUZ^PSt z6ioS2IMQ%mVp)*f+zn*Ivune^w4mjEdge7Nc=;6oV_n(-c(`j|&q?182OI|{XWW_B zcZf#--MUB`cW_Hp52T22NsHBJ)adr@+eZxRa08HUA_@UH0OUi{C=RW(R)ShAx<*oI z$wa2MEWj{HuFz6_aH%ywM#qmIWYE-1`e6r-2f6GE+q)J`GqH~2#szXe6S>)dT@Q>4 zi5XPqSe%kB+dBZ-m1uG#AhQ$0KIjW_tu$I6@;wivY!>0Frq`n;aU_+I%R`B!IU_>? zm#7fi{N$fMWHvbO2iyku*!+AW?j_7$-eu8?YlA7rk&dfs5pWkE=Ta1^-m1oZ8>jA# zE^N!V{hj~C-kXQjyua<=%eJwxvDt=go-#z{L<8C8l{rL2hEirqk;ZM_+NM=Wl0wOl zqD+x8r&3Xf3>i{1phBh8^SV}fe*1eIzi0U8xqr{|9QQho`~KS0TA%g#yx-S#UgvpU z*E`Mv8{DOaN5+D-?xm(C*ctjb&kc%-EvRug?^W4`ZQ<8*$K=0#_@y)2`ODHdj+-r= zh8dwJyQ;J9cMxPL1{7n#|cc<<4ra(Gsg_%{5%IF>Tk#X`erphwiv;>ykZu zsVusGz5Z{H#{V&XzHu`b=u7;Z+e#ei-=k(GgGGFO?T@tT5waYTAZM4OO$)w!nnYnpehGW?Ak8EG#wy_gLKt));f zCrs2iGJ5KXe_ikoGx}jOkoXqbaZ+?>HWPD!)Nj?VZ~dE#{59)40(kplz8P&{=zDH0sSUEy2=<_+&6$@%=`q8kkMn2<@PXG6p-JUmI2A7Q% zpRuLoW_YZ56cRLS>cCVCSiY9hT=+FW(=GZr`m!-Zv<2!h;4MBvXWC~!c10D78Tp|n zRx+ly-FyG}rp#BPm=#vb@0OSg%zIN$^S|Af*KgEljI=oPfTZgYB8_EuOz98oKj~YU zv}sT~7`*Qp3`=mci>LZY9=qD~iBA()3dtjIch~R%Q{15wk2tzq-ara8L^?VgN5W?b zbZeux<`_>NQnz_5lBlq2wrx}lhjGBU&sg-sv&V}HrJXNF#RbGz$0vXqo=S#;$|MKuYZJ= zGDV72sVH@j6EYwXNiRi~ ziA~tCU5!YMwPBK{&&Cnep`hsO;TNF$j$VtLV-m>#6117me&fcC&QBK(Ciab?yGU*1 zkf(8C4o*W^^j`Tac}x>9-2-keXH4h-;ci(pRz1%j2!}ahe9e8BBU)&h-x;drl;7n} zxA15p)g*}I5Hugo9yzxmV!V;85h=cKmVpv}C%FEswqpxXM7}-kTIoW9G@P-#o(gHh zxioxT+X9nrWiz_ZII2B&^OkJumojuwM!)(ln+)e2v>vO&eLct(P?*eH2d>^)4jCs| zqLsGw-T45H?LoXI$194h8_jHJ8zR zBHNRhgf&p?wq*0XP4ezLbk&!f>3L5tG$JG!D|H|*P8eN5PhR5XDTfLO4N5J+?H8-- z*`4o+QiIpL`wcGc)^h({jmlvWMdOsJhXwujQ^T0~f6mAJ2`y}#ZV!S4@Hd@le4R&f zh5?h*$)Q)jDrG;9G!N_K>j~vjwCv%e@#Alz>naqkOqkur{41=`wm^iV2~z?3F{gqNKGxGIgCdK9{%NEb z1}zg1=09kJ3B|8)ywQ~vj$-=dQ6-re%V>hhXBLct?B4!#Xr+@YJ`%F|A*|0V?WFKz z7V2VGMd?j=dP(QF>OQ4TGFYLwULNk+LpS#5$U5DTWT7a&{^CRD;kHD-2Iw-mi@s zH4=j!jPWKo8LqYK#6h?O$E{ZWO$>bneWXyJHM0Xp=M@2o1ak=;DX@m1A#(`Qon^AK zko5QvWs)MU=82YlM`Gdv#)=z+TC1ds98I%05uCrilw@rO+>6gf}$>c*W%TR6sT71k6xy0|Zi2pe(Z-MDqD7OCYbg!UF$ zn+bz{Z9lXPbvW&e&3|u4T=CjF7?JoIoyF2eD;;<29Auj?v+E$W zLthWizi0xJd)m3oULX={^NHS2bzVE1SF$7}fJW+uir0Q-{)D_%%vq&-fDj(r=0eBk z<5=Lb7O9V*qhw~%MF`8M%HqU!9A$@@tnqC=??)X02t^k zU5~P5sTNw|@tgLVDSMg-=L~4_KUK{(bn3B$=Yk2c9z2$i_88VHE6T!5mP@+{8_3ep zNufYS`3vB3g#0Kbe3?vupH$(-IOG(Y--MgsL1G-~yt9$ikXnYwV;q7}gI;CrB@cc< z%)N~hRIHZFObBwG%Y+nWWBo}ZryP#L^c-Pn+U&Y>j99Hu^yw3}tbG+xG{xR&Annqs z+AvlR_qJ}?@*1&{Dpu-UYVd&J{;eC+Dme<2YCuw)-YW9lq>yxX#jbSC|h=%9&Tqif6q^Gi<--dsgsHzm;^NDt5I zF!HIybTZ`0HkaWdW!$+4APla2{T238`p2yqtN`iTV?_phKaP{mFWyVTqxSOjro>ux z#q`Z2>JjAr?tIJ1Jhx+?_WT<=NA8~}GRbWUJ+DPwiCQ-hBg3Ch;ueXvf@e&j5WpVe z>MFG}^T!wN&2MI&nZ4#N{TTxWulv*i0r2Mz&Q8PpARrcU2~hPR@g>7Zs*=pvwQo>; zUf=zN?C>XaLtltY>4q>WGhiAR6OtMzB(%-9vf9f1cty6KlJgW?LA68?@aN8*yVR&= z>}>N1#v62+=4RL3usdw|*y-E_DEUS~%i#xga}w|@lE+4X@DzcDW5^Uku!PCrPb5g~ z)UULJG9P5*&aVvu8~xh7NfWovUp(HtejWZWmaDEY%yPE{Kt-x5Y4i5fWDmN( z>7|%wZN2Y^3nutq=_`gh9`{U#V-U(=g6P~x>_5d2K~Z62FnTfsYCj=f+usQlU<$dp zk8?Uwj2l2nxcU0FuDT|1=yydvL=j-t=YyS%6z>=QTkZ;Mi(hE6?BT6JZ}JJ z1LvUVkhE^MhmOyEPlwROwKs?T&A@>8=L+2dJoa>kA4Y^X`hM&7?bpO)TEPubZ4WMP z4>h~Qd&oT#0-?<_laBOW2@lESgcN%$5i(4i(I|^AU%IR#h-llkU%c``4+@*1#{@!e z-ukZ^DaI<<)2wz#`|r(dA`eJg-DZft@1D_a$#tC(uLSn1Ys71 zL~F0)uZ$Ff$~F{F-S}ABrvARo{EtNR47*(i2ClQ1%kTmly}XlDf{lUvkOdnoueX=O zx@t-viPR&ES{D_CDO)|g{3#a{_XuHT=Q=)zsJ<3cS>cYDam>%|8m6}mJ|x8(-Fk1qFyE)oc|e?$}x1PwLNzj z*4^MnYsdpqLTrSQ0moI~$@#HBoxHz4z3Sc_)kCH>s}0aF34dFn-@pFm*wQd0E3m=) zB$5w#cQXO=Sg*BqcFnBD-5C#?O+~_aROiZnnQB3o5;JLOTsY>rgR}vVy1{r5joyuA zhCb;T`Oq-+Pmd5Y#;=2##;|y7p8fFQ&TmH77WTu%ZQ8REq_qS~=AOldtn}f$Q;{<{ z2Uj3_OZ@a=I)|Tu``21w&XEwn=uPciN;<}tQEv871TddL_d}I)a%}5S_D44fL65BY z`s4QFpJjj-h@_?>x-_db^2n5sh*htK3(v@sQTt7C#EK;qee@QR%zcc`(r>cZP z`iuKNpD ztbHzFb`@1J<$m!wVCGfwL&rVGdYezfS4)E0L5U!fbjJJYGytr~_Zj%8wXThZaNcNA zpqwn_vmB{FBule)rhq~r0v15!u&qS9Co^m%cL<&k5?Agn@M7N0ZH;wbWW$uweb%+N zd6kywX}*1#@o)^Y;xrBaj?=nHB?gs@o?W`YzrB8U|HvO`x~yq2ul{W*>-W%ztq(#6 z?0c?LOUbEEWmrH4hH)3%}F%44zz)iHH&9@t6i?&T^SmHb z%zX#y%KXOFzL%0S>p_H=d|TE?f!IBa`zSmm-!Eu&6Ie#sD>B@H;+R=w8;A$T|5<9C zo{iyu7j#h1S0@Y#@(0nCnWRV&~@VuC2yvwO#oi19<^)}Z}U6d5ho-pQW?#+ zu-GJ{q#iz;zTDndT>~SL|85XAnnfEu{ z?{=2)IlAsr$pB>V z)(t|2U0b8jRk-c?-+)3aC~88Ri$ToNu)aGQHGMM47_?Es!n1ww2n`jyoLH<=Yt*Dk zAKlBYn*YH_k7m?6ml}3XzIprhf{OA|CdZ%rVQM_=bhY5G@&fYzjr;c-(-JIW*g|i_ z{Z4CB{F*2Asz83!G&qE9g}$J2I%VwIjy`w>>o5m2{P1Ja^#%)5c+2@9>-|FL2(^Xu zaM&-|+Eqf{4-yPd-v4W+G;rjl+qRTBtTQ3O?d_AoKR^UgCH=+n$g(z<6A(ebB&G(` z*xE}UX^r`A@s@I4^fw%^;UTS|Cjm!t)(s<=?s%^QXl;Nhk!CqsI7w{;g6)Or{&f;Rgkcd=lJ_xYcJImtkQrOZL zvH>W(WO_zw>hIE40?aS!(oTJId(0efRPqyHQ24Ri=s~!!f>Zx!D*81jMe%+$KRHbDRF2hW z3}R=OJgi6Fmlj?Ap{SSUAxfcrRij2VSLoBWkFX{AKj;z?oh-~(ASKAw>~RJoN0#Pu z2dvXV+X8EZQlRErC^OYAuIi|G33wL26T@3OpGR-3GSGLQw2ESw;PhMb=Cu_BYH5fj z&Y`_EKoPe{KT1Eo9P?sSDrR<`ANMos#E!0V^X?z11Mmr&ZhRflLylp9C9*Uvwq^7d zS1k>JI1TD8T=>RmoDsdCL#bR>y}8|BwJ*kUd={J~H$BI3{rUwkZZbPM(0Jl!b@Z1Z z35T|!Tb({Rx%r+~EA3isUH&BHm-==4G_Bc)a5n}4mY%*E9ftfVPBCeS5vLmq>;^{% z1i`ng1Vb&Pn%4BfwJYR@__`L|-M8nY&VHF&4{iCENzI0l=hlB&0%(+`Zq{jObWHLW zdPro(*WTW9P8jP@x*$2VhOcTFdO*eG9l&CaVg!Y_mEuT^A1@A$b#-#eHG-T8p8grGKmC)EX(VN(xOkxVRSF zpFW=+t4pZvE7kWg;qA6xUES-p3+njreztSuuQLt5^~$tyIP}LFg&X#)mvA#iPw^~% zdcHP8dgx)^B)@yblj2r$Y@s9>{yTLENTn!}=r8o0&`ndy^47!i(ia`zv1wCCwT{H$ zDMJ9?K{9V6JRk5aEc53tseR5!M90DXhURbjFgdgY2Mb7VGH1y$H+iCWM>5U*78~Bw z+)vUP8#T0Mis4qVipUH>vCXoAdp`6w3gSy&hv1{_xb~^9dpvYS4)$le={C3Y?Aq55C5oEd34g$rRL*o>@juLd8kqjoY|*q+m(cLE@wcuU}eoOe!khAI(GbXd{Y@cdNB<4$=wdqxL8M}s_aqLwNsCJa)=Ka#rBxk zTesveIy#DfR>+(WqNg-Eid{emTFImJyjd90!_2Rj45K?5-d(Mipy?S|f^gV`M~!>| zK8pwbj`aWJ$$Wr^SPT#wHo?OL+29G40q*VEwKGby_lZ;bCYP3!$UI2#uie&zUy_rn zjPB-)#BhTzjyHYXzj;0K{z&zV+St!5HVOQ0pJR;+e7+Np<#+%yvjawzkq*6OmLtX_ ze%ZZD&!I5T*be~EnMTXp)$OYSaTK<4y<+^D9G6#vD18l1LK3WW#gt$IR!D*Mj$>zm zF~KSJfM5nn20bbZKXc|9u2w-JQo494gN((837vBl((znQAD-V@xKi;uQ5a0stpCd| z!>AJO!8l;({Dn!Hg~$q90bm+ZaA(a!o@>di_b;GFS& z;Y$K8E(e~<0Ct&Uf@%Kjt^-3`$dQEns6r0xd>u`{(y?jDFpG@gd;Rj4##5#g^2V=_ zh(wxWgo%h~GJRwGrjt{S)Rx9zSmW35EMkm9XmNx4d_Pz%h&{(ccALY3Ctq5#SSCJD z^lSk6%jqSLN63*%sx6fCnrC-%a zRMb7wHQl#wm!qh}VhWupBD1vjr@KJ-_2ncaf%?!>0;S1N!do*=rKSQVV+V!__eyLI zm)E7b?b@}g#7E$~$O!WnEh9k$SRZ%cCl`Bz*hGG9{8jiLsH`w2Mj!lM(bA!hkc2N)wa z()@}jY$z8pzX#cgDEIy($T=&aqUpk)az5O=P$gf9`&{TA4r?;~>g}P^vb3yBQe*zb z>%?A>Xyv0o4RsWEIg1yIq{(sp#VHzM=w)!|CMYPk&q1wVooHYCTb$01Ia5|Ws^Z{~ z*6@sS?&)J{)i~j;jQy$DKP z{hK~b#s*7jlyN>Ftp&q>Qebjkq&H{&3Tm(#WBql9`GL*fu%%kXXeXc48Mgkl92AU) z{kSo;_E2WEETCYJBkUAn`zP9QAe5)^bJmh5z{1Pi-1M_T=A=lA9XVievq_MYBC$Y! z$?P>bHJix`Oo%zMtnG{y-P^gd*nWE2jZ<;pe*Z+tJA+q4sO@9)v5W5sT2%3y_uVN{ z!19}uqn#Y)Y!Z{(G;1 zKhd_moP#44U(KmUx53?sPD!(SkB1FNN#xCb9P$iJsF1WN&mC9Y>t%I)pmSx;lB`=6 z>{ZTggPBCkeKIuk83l$$b<mVo(8|JdD+LuC%q7`_8RbUi$Y8k z>r>zEDqDAO*7-?+NI6aOG717J1G-?qS=>=M3sf7fvr=jVcGDG$$A%6VYZdQfkFVGf zn=Ew|z1gE6VFm627`N3Nz^wD;R(6WNE~udppqgTj8A>L{B7wg?Ps$w8+gqmgSvqxKi;FRhyDouzhw4}o(%I|iq<4o(e)HS& zdffLs@wzDjc+*z7GOSCE4M37UJGXxSZp$IlDHyPNpEAY<^(z@v&gAfP9y(8j!bV^q zk{*9BuwOs=V;zsKmY!FZ@-4^`IYW1K8Bbn@Q5Bp?&dna~N7Y_a0SoKZ4RO2~A-0>e zB@kijC0j2v#bi)IVH8@yQDrV!{_%NgqI*~2DfN$-H16gDNKKF{+qq5OSulv8i*DF)3M zVszJ|g1T~lM1<|pge+c>l+*p(yzW?DnaOu$pS(sytVrHC} z#zyCu0}=jmoqIlGSORNOj-@x2^CV{+_kEf@ywe_wSqJeL!X4p69C4Cs*pmK&$vrY)k1H2`j6M=_qT$yXF;nLq>+wsy?j4Lq zj%xh(^T%h>9$#*<^z_IhF3p=P9eU)<&Cz*>Z?|()-n9N_-(!T2?m9m=of3FKGsdu7 z*Su@ZnJ<$p_hp;9pF0py9G!jTM!(fb&Q|`V=gTT*b9Tu@4&99Un!Z#w`+#lMr7o{s z_43cAroiMeyXc+1|C_BLPu8SPC`uXE=b>Ze<^1!rjf{+J9!{U}PI5D> z+=&2CVU}Ewk=(a(WkT=E%a$xzQW)3G|3N!n0|%QGQ3bd4ikjcfxW$|JfishB<;hN> z`rNqUNnM3v*y>XkUaU8@dpO3SbCRMqzqRrkTHyQ1^(;lp#W*R`vcEmvxYDuw`kdg#bn{N%2Vdu51SzIiyh@@g2AV&HI>mXOkZ9p zev!cc`8FC_(!qP~`o%5mT#&VVDidn`?G)(TGQR5y?X%v-U47Om?jp zU|1SAY00jz=x;ftU&37dY?Fr!GIuH%rIT)bM19di9PR!5epiLX?7HyWes)8>cL5w7 z+vD)c5jE~h8QlugG+eL2dfmPS*`sgg5wIcHE-Ztys z9s1I4RDpA4?~2)cUt78mEKi+#WF$5(%B~Qe-nlol+pRkPVtp=l+Y0?F?XEtXHhucT z$oUCTUVHaGD(y9x!|uEH>Xk@+>bLy-ldi(T?HSX~2aKP6E^ThY!d*j5%KrSW?B?f5 zUK_&yIWshC_oShv8gxoI**{%=!Jy#g@3zkQTidPg4s`JuxYPvF!Gl&a1AfYapYEH{ zHnbv@8@-R#%fv6MYt>(x*ib)jH-Iw>fSj~{X{f1V6ykj3t1Yd=S*RAgYa)jM`X&!8 z)527;&YrJk@b&S+bG9(>wjZw)FJf&av|1Xn=SD+)Ezor$q;28d0m+#s!=mGX_d}9@ zT^bs>==?LAg!1Id@=0AM9@qyY%AVx0n#?#~sV^A|?%GK0ONN$)w!ntN^;; z7Oc4R{;22Py+?6ly~n(QvCoqqo*$YV{A|VL{490?KDDeM@44{+tK1!M{R_ES#7}MT)%@tcX*$+?VZERqZd+LI$ z`LU{yvnfWAA1>vl&=r>M(0cjVDOWN^3X3<_G5_4J?F&P;`cfln? zkri~}M302ExvELAJNwvsjy^Qi+p;h&c40H@js5QL``9Riz2blY&o-ggg-rRDvsYg6 zI`?Q+Pg!C42d0~tcIgxKX?WnSQvxHN!y7^HrxTRpac1xGY$)9$E;N-t{y%W zm{m3{N^=|KQ#U2RZOY1!OPBOMMios|g}Kfw|JYu;g;%4wP621;wOjY{uRJc_fqj#; zern+H!7TCq3vO;RU^NywpU=GT-LZ!Da=?rdPR#bRT|(f4%3g^x?56iG`BVTaXMd%2 ztaif;vyZVaE{>0n&#pAfIvRQ+yz7keyI)@Hi+mAtd9%ZnY zXRnCkPS4F7Jj#}Zx`;QBs}8^0KjE(J!LpiH%S%4SeoRUpH*Dc+*Nm9Uquy7z`88=3 z86F<)R9Nu%@X9I6bF*tjTuELJ!Rzq$UzDTr&m@V3WuVsZ4ls!&_^K>qS$q`*yIeDm`mR{wcGfn# z;_B-|VDrdK)wupnC7-&^F$Y%IkS+Tn7~076B^Q`3og=Jdg`1>J91KFQW;&Xe!ovL=%=)a*9^{V5mPb>GIozF zcH*hi3iQuq#0$zs}Y#Cdb*TGP4aU9j=@?vQw`IG2fx!R*M1iuW#C5QA?7s zk!B6I4UYVsZRVaai-P^^+Kj zai?`Z-y2bs<-2lGM50at=6(+i^^Lpo<JZWJxrLr+kBz9^OxzQl#9#GJv7lRdATvht^x<2N~2k` zW<>{5DLysg7lCf(1Z^5+hT=hqevs^H31eppa4??x zA#BIdlPpgfnUyNk`DYWhj-FRuQeb!f;&OTl_nI#9F(9GozI3aZ)3k}o0`r@)@KnsH z+6sl?1Azjd#q6AuVHFdLVl>;_C1f46<$M%Vs}JR1?**Z;#KIxT14$Z-p>)K-MdPrA zkL6zFs8OTh_l#K8!k!#6_ zJt;paPG59n^T2<5Or11jAiKh@;u-r-sWLBr-9ocZpJV}P8m8>nR0OsM4<2Ol^+SvI zON^5%w5Yh;R!hzXunh7tF=FfF9UkkEU7=8DwW5)Hzn^`&reofjF)R2MdB7EtqFo{H z&{au|U_<#}0gz7ab>W%H`RkKzPnSg`I<4(kv$>(mhUFR(k($jg>o;)VK%_=Z`!?NM zCI7w5NpH=&-Cz(Mh?Bc5%$@d7tH?Tz-$kBF*Ohxu`t;^AhOT|KdW_`ln>PuSwWd#> zUex9!r``;*64=LH_XfIkZM`yTwO?u4oV9aKoH$YVrhVnb0R1i+=9MPv(8E(WORe}9 z>y&5h!DYe+*u4I`n{4@$-`DRL_~;G}iy~sW-r5%qR_%z965x$!r5M1t+0Xe;2hA{B zJ!+!f`@q**>a_aDczEYGb$SP!X{l>tA@>H5+C|y>(!0YXtroiP0ye5NsG}UHOHQ)Y zx?y@O7?0t(Ue4N>vN=guS0i_}mFP{5mK0z~($ehpJ*Gj|UF_Ru>)z~$%s>A$1wQ~! zE@mveqYdBcZCXr)ohlP7uyWgl$q7S#QYf}B2e*Xw($J7y!eL$q@F!|0=Q_Ogk|RH$ZC>ul?TEc`yf2W*UzK)453 z0Cu;F0kL@)rZ(5bvgCddKgc539f5@X7pDZu7F4Qa`5tuqpd_y5yDp58-Gi;fy=4pQ zc}8^=O{YRzczHC6bCo9)KIuNTm*F3q)r%=SiEP!<-G85Qf1X1PaboRj!4ow z`PJX`TdIH7#Wy<>oQt1(gHH8UMiq<}EE+|vDElS_mbkn$*|qoFnE}^GNZO5A(&@n& zQx-3N@3e9x8D`!7j3aHyjNiDQ5n2&<``g(!;9$a~FWAxPVN2O@w0U$>atxN&^ciyo zGL>c7r;uR<-FdGXyq9V8Y;Pw$d#5u;wCnj@((;%zwB5;l1bBs_Z~MyfY(sT)F=$0P$1$^Islw`*#546;<3I>cR=v@B)GcJ2VLfcr857Y<-RaseT@tCTG@ zyZ+RCU^`EmLe;OPoBWzo{VAaF$rC4x?Ck8$EYz+C&}jF(euMFvl6!fzI!y!~W!{RKIE+`td1>rkj4e zkz(?1|IhvK+glCQpFJ$KFLF#d6HF%3k0z)n{S8!EB0a#liRSCy2Sr?~-CEmXBnYCv z8p#o^mT=E?3-5H6J;*f20kVwwD~~$uotU6{Hjch6n=cP%q?s0dPMu`6YslBhtN`mw z_Km!{i>=$T`1#u8_LU{E8gUe&5w}|{&5Z9EcK3Mj!Tn_0!4k6b%VFk&Q66-mgWQ|7 zRJd&pBtxp-V`h{cqYcQUOY_tCL?viPP+??J3`AE1;nng+gF(vzHYpTMw^y$K>)&L7 z4{#|jF(`g!TS~aJeY0cGBj2f}ran^LVWI3Xd^lfIr}}HQx(Qw4Kj_7}&N%bzr3R+) z?b3?_@+~oyC;e5Y-ONpVg4VKj8S0A~w(%k%qmXybCcxa?RQGqAyfd+nd~LZGtj(`+ zO~z?Wsrn83sx1)^AZ!9*_o2(2xpNObG@0FtC>1sUT=^n_&FMq-J9DV>Pm0MMs#ikO z2hb>c4XGidEEE<`Vuz4rg6Q>%LLrYN#)D4n_oztzrwJqSnmic}Yo!b#VbgY|y+J@m zA#~GGs#m+L2~2#JJ7CO()%&w*U$|{3i<{)4@-NTqR^&Ip;?C~cMl}_~Mrii3SHC0w zZ(e~#@XzueZe0&knN#oF+Ihdb@DK3mkNdH)hE&~5x2^&Ihn2gcdG*%$zltF*;lQ zOIFxXSw+oT2UTx@30U*rgl8XgJcQdwFSGOG$iA3>CZ|C z36JlgR8a}MJ)1f)Au{{m?92!9Ep}AJuBKDy*;Z-HUUp_w*YW#ygH$^ST_vek<30O5 z{~4s96=fN5*T~e=bV?zy)2V&+J{t8Y(_>N=qtU{?ucv3JrOvXMj7IX&5C)i>dr&16_pBdRt=jF``m;9Q&5Mbw@Zs z0f%y=y0F$@fZ#iE@<8>gmTdP8UNh$E$|Dr_F7EC?5$_RZ{d5I54W){R`toSD8aLBs z>A5$(yeU!6peo7*Y?H0V=(Z$Vh`_A(c#ug=3dPKIRsU|CLdG$kGiOfTA3KJ_lT*rU zZ_9JE9$EeG;rx5J(}mupYw}7Kj9J`N&(5@~zmtJUKQ6H(>3{#BlP8PbKQ7SycFEE> z$M$RKiBKbklnrGg%5u26_Nb>~RDYYyJI`>#wuBmyi>gtxb|6vp+#B`sg!#o0*G1xm z5Xh8pQQQ9JGv*acm+j)+RR4CTQ1ym!8-0xM8=$!!Iz^a184~ZiEL=I1@GM9evxSlB z0BxqaZ~hq|JWC+rUnE6V1h>gCx;oe0JH~U`{*qSJ<=F$E6M?rgDq6J zusyo)PfN1VS`3jl)ifSBuW3&mi8g$*;swD!ci({UlL`Nj3~X0vNbTco?&2c_g)*F| znC4?hGns}Gxwo}W_v&B#>Gaf}nqC7fd9TfI9t6j`9#fj1MEPz9*DbJr;gG%FmN4M= z`^3Q=rL1A!Oxbj{>X#LU|M)ct_+>5&$&3S}pHN^3L8Rs-SS&ug%aJ5upDmMz6bmgw zr;TyofBte3Xlhx^_pgzkuGja_*xJTstROkFRo7|>wpVF@Xp#kH<>cfDLn%i~ySHpv z_4zLyBjn;qBYOlTQ8g)5EE6YgnJhyO4M|JF2ZGTI$j|kvp~kQygz4@Z1|%B{kf0+G zMSZ)$yB->X&Q%sdk9~b&rRKYKk7#&&&e{}ZH_QBUX}w2VRDHPH#|b>Rd0x{^B)=EF zcJGtLt)D~*6`rJ*8yFIzrL+-}CbsR`kPFf<)1WN_zvcC^&Qv3}PU3?TG8&3iS~C_wq2*zBCXZq+|3oLU&w)i(V<1+D}REBPVTyzG@W&d>X6e$g#+nm(h; z#zyV^b8?nc=X1qNeLgC#XRmSb@#D6)UD$ZW)V%7C8P<8)tRH_?Tz##qiX{AF_ibHE zOSa%Iv*Lgd;gC|`Nf7%w16|7=*-3nU0hp&Hx0BQXd*`lyUwbS?h5EH)sz2r)OA{4SfC9WvDU|DH#>G4=zLJ z)8*#h%)|NzG2ZoK=qVieNU=;~8aR0Km`V;2uc^GJKbde3@p#da_2k-0_`p+Tfc(7=G z78@Jr+<_01-+pKcMVoEG9l>@r2~pK6hE-b%rRNiTaD5M`rBY$UVk*9b-CCz#{Q;-P zF6|$*iHJuZ0`%ndy4?cLey)7+O|-MlI|p}C_6MY;zmbvV$cFnh1lAi>*|{c z*Ed1M=!3wQrRsb|9dKqb?^jq@C^fYbH81QzYgb5P4-JWG`DX@YI1LcefRyJeUUrw(kQYD7 z2d!`Wr)j<-OD1z!si2HH@2LZ2Z*$Dn5WHLKg)WxASCa-PWEx5~l8T+W>|mHFAmzd; zoY9+I$vW!Sy=~e)#@G20QuJmX*)=qNREs&D5(g=lP%#_73fR~%9%OsKW1^mBnwjkc zRUn=-r3z!r-7f>#!u{byG>-S`e>a(`E^EEhT>a{(s~EP>gbmi3mnWwG~IeGj-R+#JerK&*oL@gtj2lZ>m z3=hpNApl9GO4#{IMnUH8$Y58(Mk)Ec2x$4PrI)Urrm`18Uno|;Qh^L|Yc!sK%IdXl zSN&$K4#DACcNh7~1gb^OArNI+(3Z$ja%BhC=NuPNO{tP3%*+1y6Dxl|GEXN_pk%93 z8xNEZXkYPZYZToOrt>#9vmqcf%jHN5_3LV`P zQBh1~S&ZGzQwUBX$@OQPzCccgmw(k+5-aH+e5e7(*nJI6RqB`Zn*HN7J1R})*gX_L zBT2XjrmeFO*A5O2B)~mnrrvlbD$LTp=UEvhKnm<pz?2NF})}EsQ&bs<(|!d1P160sXwY-^@jhC zDE!Nh5UvR5Pz8PX8Mh5mR{r-7SKWdC58aJ;cQt<#aA*R7%C$7XTx=hi=-Lwz4b`Gv zu;KT`k}w@Y#a{9O`kd~41C9U?Wj!1z--7VI^5Q4H`n_#cEXssy%3M<7ZP8W*rzWJCb}I>Du9jaIq3PCsiGTgump&xg(lw2+E`MUD64V> z0t{gyr3D${cT6FQ+7LW66D7t9(|vBKaJ+5TKIe8TB>Q`C!gzo&4JRtHv(mU=I{%7ZA1-xjkPJ`OBs%l831W>m7P-S@PME!^D{4vM8pxD$<0xWV*@$DZ{ zb9XuELF(X^RxqF{z4ER}@52j8^?6|d!vHnW< zUOsdpP3C@(81|uI4`Bf1yVAxvA4}b!jH0g}o;!xdGPTOf%VgI3Ix+2e@0m@jxQq!Y zI6-Y;z8~u9P&o+e%ao1dLv((i{p`|N?Bo%+VG zK4cFNcIh}!x{4=d$gtMLOSz%G8k#+Oj#F{@$wlXonLRY#zRhqsX-Nm9?U12iZK?7U z0sW>{J;p#$A*8m=PwVFIWaCnNIpeM8)QNHT$F%pu=%c9&e_%YWi&AJF5IxMuBga#H zqDu}ZLHjFSF8$)VU2w4V;hnl!`#in8-cm-yu}D=K(tIXBpF3xfd1ud;M<{|>8I=`l zRTlivihEJ~+Ua9eS5ZMQGh#UaGiGR=BX@DpeG?NEQxKXo4H#fu2u`P-54~^eJx=_?Ps43 zRoVb6(lpiBN$GG#jUcoGMP8CEC&jwxRn)!c4&%iH!p#iEgeNk0PKcz3DQQSdaQVlS9OTk6BEzROfRr~c!5ott(eM#`$0 zH=k7B3E2@y8oHth7J@ru^(@8^|7hmI9m4GtnWX`uo>Bfv?N$Uk5`^$6)hUScncI8P zbHM8A-5tp2aAA`&9Os}FiG}JX4K}IUtzEH$_)zuhzn=50+TMKl|61z#zb|8?NAE6o z4-V0nc*kVTO24dRxH(}pZ|`jDi~Rpna5XA-yM0Qzw@C#mw3RUCQX)ZmSU~ZX)zJl;=lAW%B45AH?uZ9TDyOnLDG z?`g&Fe=xCbd*{znpy0zKIl$97NOdefEAqLO!OVrGLllrJxo|EsavwYx&JCQJ0WX@D8?Y*yW-Mf$=jU* zg`;;lR;zaxiC$_OE>&zBs%NRXH3oKN}0;_`)xpn{RE%)UnYnpI;x)wPodO&GcLQB94`o zN{$jiU%1WIdOi3?pKA76r8%D=v*3_8a<7@_1cnD`cX}&8QfhsvGIeu~9g$kDi*loP z6Ilxy4y0Ns_QjuFiWpPK1!ha>4TJAT`6_bwxux_Zdu3MEYfyQL8dSY&q2aS;4^TT~ zT;=i9a(3i0?!7Z0MJiQ1m%yZgJ~8F}#m^+hOL$H_p_at3YKEw%Y*lekK;x$yy(Vj7 zx2h}EDN`RjVGi{cW*F3)54<{M>m*XJO57{;ZE`*?XK{vo&*rQ+tHgn6Ap`88`T`Lo z#3?wmqU4Q69J;W#hR|eL-}d`E6)J%wDSSH&!_n&eIIo~6F*&1W$m!vJZ^V&#;^fH) zYRgBF8J`k|YzEWzfuU%y_jj(W#JD1_5^s%N@r_Gae(bW30k&ojMOrypQZ>i^ zNzmz6MCW<^J|{YF#$SCpsQ|K2S1J6uUaU|m#1?R&b8u{Td0w|J3=7&MS~qds37?b? zJ1F973CPP!9=FJ*d`eD;;4W3AqxSF-xXWFKD{VFjssp8sR0C*Dslc4g0P~lWk|@^wigl7LqqDQ{4=uBp$6{? zV#W0G(5Dh!L(ICs*b{4GBmvFT_UiMlzSxLS+E(m@pJw@X;x?arKK-Ro&Vwpt^BY%Q zXxnNy(}-jN236uQHBt#PSbu#^`^;^fcixdlD)NO;ZQ{2e_gINT5r^L7k6Xs5 z%D-NSNyLWgBUb=~Cnny}%MI}W{pyH!tMLm-AlfdXl1YRx!&E6$oekclF3L+BNKdl7 z{Gxn8##DYz5gkMPX7E4{=IX2wo3|`#@U1j&h@sbW?;~funiT^m8mhVl}xjY8TL1C7qIve(LbEe%MvXHeDB2$8-ihE{MLqTa}t-+zu0hS z$;?)y&BM5rIVsIvG2${@!Su#J$fFC@)Z$Bn*fdCSpP4O*7 z07v+mv}u`asl=7sgauLzY<_2=JPS2F$P382xej{+f-28&}xSG_~M5y{c zxB^z6d9A+YYrOL(A?3;Y@>Q01eAQZnew8FE)7|Qo<9>YQhnCV5uM(xMG^o;Dpkm`X{~dq?)}sXOy%Oa#m&T|2fKPAsSQ6HuzFU2@o-YRN(#-2 z`ra9Q>bsfzB@cA6QC)#tDb!gX2R;He2q9Vj&1e2~Sm*Po@=hPmkT(o4T6Q1bfxn%1 z$TZ-8R8)5pxErUz92~u%YiOPS>|hIF%eC8LzA(j&=yET;uyAvDX=na&^1S-xzeoA< zCwBi76BCoXU-;v&zZ{%?@ZpnxtMw#HysRyUtY0I}Q?bX5G5j_~^3k@ba{Jx)kTtR> zG}Zfut3HaVBQ@ihh4Rma_l7^!oKNtH1jwfHb+kZSrj*0dyoAYasOgg9LE*TfgcqMX zQ;qqXca}CAM7)Tz9A+V+lFsursbc96cd0lapRwU_RrR(X&qus;oYh^;gTL)&SgPDP zwtjfoEi4Mi&5YV;;! z=isC)WLwiW?A{g&&aG$F%1@AMWre8Pf~%v*+*mL0OUnulr$u2ALGvzM3ey75o}Op+ zQF{~7L+l5HxrJzCLEx9JF{Ozs`!g0&N(E_alEEL+$RPB9N~49@sy{*FQMDQh&ee8_ zDY?~0%ymJS5;0g?&m~$4V#UdF2g0NfcE6`cnc`5Eh9Plm@W{8fZOuhLZfSL34-B!A>WMY77sb=$yZpQ!kMk>2M8-nb;!bSZ-zth~rfWFx8)7D=4Aqcb zo%4?&ek!rvi!@d{;g|aExqe$tgHy0}TS1yF=43GOt9(40g@9zFD(7~a?Vc*zY{xUA zlCOp?0!2)!Q!&h%wYAvPWl>?_KmB6mW;*forI;0h2h~plI<&X3(agbSADhPiG$eXH z`%|exp6QL&n6Y>Q!=gMi*zQj@9FWjvof#b&cKY@9o-VO3Ny30P*6D2yL{`g4BARad5ttv}6(i5m_o3o5Y; zheS1F2k|@e$JZ*p5;mE8#M0;zx3=Ofp{1nbP_Vgp@5Sh81MbJmQ<3#1!*gjFC7ZCt zWPl5e7BBYXtEKbR4nbInzek)Vd_b1|DRCF6G+=kle|%28o*}z4D?c=IlD;8{dg6u> zL-bWtdu>ZJ-z1^D09FQro~;^jrI9r%WO@_{^PG7Mfl}Hn5N`dq?5tP+?w5%pW)j>X z;g`1VD`}Jt0VY%|xlr~PB;7*o@EO+m0^MS)C1^ur(d^mTl^-iBRyL7m*FD7id(I3s zeTD$6YxwQ{F0ppXZla_$S6TSkJ&NlW+r~qzuL)WayQp-2`lq5FY0(S~XI4d(yV zE@QWbaH;LEUx@cgdK?HVoI|NvUnE>>9^HiE-xIsKq_z6%m++_q8tX%06v6iC2;2V^ zo3TVnN#r?+zR=C$zIuisN|PQ%IuE9at3$h4Oxg2~g;1oIUKGGZ>nOt?&06}_$Le5Q zpwgp1RUs`$^F#f2{JHSu`rwXQRC$7|V2u-`#YGUa1PoCFP=eP=p@vf^6Y!fz{`C>s zl|r1zbJ@y}%sA}#G?Y)dtS9gyws@MYC>o#e;03fDrTfrF&u)njrq75X@*O?YfeFUn$PfgF+pX=I;n8ASOnMNKjM$xQU$R9}wH;%<*D+DW#uQBF-Y3 z*&P5jQo~6!W0>)s=ChaWk!%AK(A$0*0dp+1;H97x6rTG^gyoPt>>Uerz7-B z$&IJQPCHqbU7R51Y{8;zD)W8S(N=t3g3{@P>zCmm(VE1i&bw^N2A&NSZd**K`grr~ zeA~-GQFn`Sv#oCD<%Bpqdbc2UUX6<{V)WJDcwF6#Tf$9AWEL$3BD599D2NZ-&R7Rx|%1DM6knxx!R1SZCd@Fra$5DT)zd~`}9-}XrGdl)+WRM)|TAw zPa|Kn4z5;i<`oq1$gL<@lchpI_30~iX_8lgU+kfNtFNSbUVKr*t^J8>YmK<4c(ewx zLZzEkRAQ9|qDw|ZX_l`Qn>=2cYEHemx9{SM#YScy0XadQ@;g_DN3b9sm0m&!7bHln zTH$y5UvD0-)^QZw_fb8l;~smzc;ml*;$yrTKq#8t6d$)3{9S>+(oyqqNW&d`$D=pD zG{ntU<#C;N=rUJSM9epk;#eF&|M6qy_0LYd4tEm{hHwoVIKV22+#n0pUB(J#LtK2D zyOcCNZC_ulX_A0#Znw0u;9g4@jV>On-8FRW{w8Vqiccd1#nLZKwJ@W(Nps#JpsoWB zrb0-Bruwz(KN*;2=CN?6ZdY-m6d};Og=jHS(ax!dnZ4}7&Jhb=uMgH-eWfO?Nk4ft z5#d{Y)X;i4XtjBD_+zgYTLS5TmyofMyHih}7GMKLnoaBE9$GmcwEM_Lq<;I)+kcLX zD8wANseVtXpzm!GU+Ar!!)mmljYapv#WyaW{{3~l{;iA83h-+pv9BDePdryb+w}>Q zN@UnnF#|0kvx)C6g~zZU)-;dS?4V1REGxwDg2u zp-Vq|L;c=k8R?Vj*+j}GN@hLLH24ZFv;`=i{zD2yxrv#99urSYHF^i?i$;ctVs%T& z(fD_rxop+t@e+auYm*Jp(;#Y%g(Y%=;dEfnvL=4$EFhtwi z2Higq8k)3k^_!xvkU-`l8;|~Aa@0V?_)VzB5 zNQWvv!1l1H5RF79>w)IBPOi;chZe8QUT^Ni7xc?*t7zW#hrgE()H?V-w+$S2njdvX zRv)Eknu`#*k?krc$)+QGNA{iAh~L#-z_5!+ps(kL8S|84wyQE&Eg?ZOZOoK!<8pDc zu@MsIB5APs=v3~D@tI@cXw7nLJ4pqt+_Z;a*weppHx8j~i6ZxHJ7Qy2X?QLHH1 zHj*E?P2V>@N@wf`!EbA=9$m1T>+#X%Gfn0Fx>|Tl(9yJ)dNSauQ45QLA7UIx`P%h( zxe)EfKZXfdu^evglx@`X4og{#CBPpof!8fe#IO?Z_eCC`y2*$w^RFM}-_)?o!d!g=qfqODMj@SJIrIa>QO+BSk*5ek#8rMsifcjHMIccbea>LaCu}a2bvh zsJd&Y)Ia%@pA5N_bHaXz{HM!q>Zg050=ZA`EBrJ{{*Kpw_(o=S-?dWy(t4R}DXCtB zdrg594uNBrqAc3(`Nh?53eh5ht2Mq|bQNFU3S;9onx<6wWn;D2D}CDpsae_UV2`Cm zI17T!85zcW<#WK3_-~}!j)E<8NVH6e5QQJuHu>E<-sIjp30feC1nK@yYyUvrIV&4O zaw0c5m5Q?VkdI_aZwR&^@St=crZS{QuDNBu1tSG~NEehKLzy_Bmn)tI1ISou>;tjx zr4Y0LV@QP;0Zd-R7)J4mw*GvDPr7<|#a~8s;ouVHZ*6=2bU1&dm~cl3EYG5Y^bLzq zJpw@VP?xDv#p<*I_r8}Zt}`idRsLh)<#QP=IHL(sURcUh_PbEE`FzC5wX72DCuA%u%FOY&oBM$I-07-0v z#lMUiFDqx$UFD%6IxbW)CT*1k-OML1EIN7Wyb(U60nA~Rn^Y@V%K9CDtL6V;YR5Xj zP*#r%%~+4xRnnm{z@acxW}6B@e#MRkNpYE|G;lX9J1sK%y+1r#s4YrY55FWoC*u|H zCw1r{%p4z50c)oApn0dHL{_@Q^62K!yowU_FaJW>X1VrJ2h<*ax2_+>^U3e|(G}TW zI)(kt5C=7UtU7iqU4LPd2CdavW)j=2H|435w)SEH#DclS7b*@d*?4+yguYf`-?ls$ z){iftwmQdraiX_$Uv7azegJQw*x^(Cserm3@cA>x)DofWfpRXvjv$ z`Zfy*e6Gr3b2F3f78DW+>ow}|>DJW>6R+ADNkJ($JOS>*pCxwJZUxIfg-Svrj;8Y| zE6X_qDTxUX(pN0D*IJ1Z=yp`~2iyx3%Q-aXoDVUT74rNeu~Fv?34axnPgSf`3Ek_{ zZ{cXaGXs(acUFdjSdt>7O+qnY`VYH-CqQc%Jui~(rOXg+Ryewx#Mw{-MIQd;fnRfF z*abjL{D|Sm0dJ}6CNUyi%p-5H+>Di4Cqd|ttCiKt{XaNUvV`td$?}l~ifp}=m{g8W zHR4c*=^#45H{r}!mcNm3xyU1i95~ZrJuxn+0^__Ctd{p_-Q>t=;j5f0# z$MJXy(b*F1=F%1_WV6_`g!|$-N8D;YTTo_$1AkQieyd#!XSgV%dDy*b6RjJ2U45{f>V zeu5>yUp%%-wK&TbTX{^q__>mM*4j^uuZz0~O~;HFSmalnSi&43b$2x!$a^+jZ-{N# z4h-Bl|6HmsdClv+-$L`1Oprl=-i;}1Fo+<}Ao~g?Q>n!y6i6%im)pDaa*k=l`O_}9 zLA*lpa=U#Z4j6$iOw*q>K1%5~A$eK7l5DB|<t-P@;q-1s`3S;C=Y!<$u3?BXW<&2eUHl_hcpTz`lVJkZXE-U?Xa<#y{&e7*wH zse9KUhxnRT6neuJ5ek;3B_(jR9vTc~`}QcUb-x8S&2}X(ty6vN3hTSLV_#v~O_Z(& zqGMMj#A$A#l*lX_f{;vXP*VqDj>iZc%hThV1R_L}S$mqTtGWn+xsiL@fJ0KQ@yqd) zRoK!p;G>zHmEwwg@L}60|AUOS(1;Vm13y3Tp1xoNUUULf z`GBXInCPS#W|U;*rlf?$^b&bh%)G*m5}1w`@0YSkr6EQ6jVK9r^0FQ&-wXUtPxM=} zN-c)gsFQFnT$WF0cwCa^hs#~5q6<+M1HG+0d(>nNyu8cAdxcOe3|;q+Ki;spB?kJ_ z$Mj)=2T%*kSR~wIBXa)n^7OP|qN_9vufAS=?N>X6#N~PRHOw$bKZK6(Vy2LyC^BZ? z8)21&rxTWl-L;%t5cBrlh(^7JFQ~e7#mvu7G&_ohF*u`uZ6G2!_PK?vy8c%E16sAK z@}~Ufe^Ec}f18^AYx5VrJtZ`^iK5_fwvNL|lC$Yi6& zH!%E%nwCw!WuCYpODY9p9{?Vxy{eK3bY*thyF*)LYJ*_P^N$y}3*D(yNgFjppbXd( zXIF~>dQK~=uBz1r!NujS0>%8o5cu~a3QENz=~1we;nfK$m^E>E(EYjc-LcbG;gWc1 z|12f=K6idTn^v$S@`@kPsf$u9Ig;JP>yknUmp(KxbEYtGv{UfrT5r}%X%;Y9@X5;< zOs$KkEGF?CrSm5Jjf}@2t^RN9y?Ip5d;9jEX`8lf@62L#x!VANR%-{MG+ckVw)nNK~f4KN|Gj1^}No@zMt>+_pJ3>&)>h_`daH=_kB~> z^%>sp^E}SuIL`AerYk&14uVMecrvSV!l)@HUjO)IAAZ?;sRi^MBYsbDZ6c@Ny;oZ@ zx~uq|IQT3k_n`2Q#WaE3IdFcOgnmT*MPkF1as}KR+-U@}U@nr2Uc&&Hs6>Sl##)#F zk+=Jq&BMK=zSow_H_+%mTgG~c$&ATa1L4fYq$cH;kT^<}m}1a6E|&?bF2)8i`I^<; zl>)sP0l3vsO3;Lq%xHH6ig*9adE1_b>-Fj-_87qM$*%LSkA3mu?z)Phy=<^Fs#Bc` zp)Z3}q%D#s^#JR$uR@w{0>99lCC#(aelhRHmIi7l0!@>Qn0S~nEwb|SgY}gcdtbPr zIgFgM0^UrzdSrM>nb3^r8{(weexUvOwh%)6X(o-|2f~3K!IeiunS$z<&@F@b=;J6% z4unRRVUGcBx7s!9)S~tFW<#E6!4o<$o!hFwgM6rZMelBk7|(?xkr6VK*uH2EdQz(W zmU?}Y@M~o8U*#Q^ycW@1mNI&iLV8pW+S_jyj|)IchrrRl+Sg9lMi4CsTCoV2CNM~bz^K0hFe6rCF&o}&+N|mG%`K+fk*+Cey$;*5`Bl*@O;O5_bi+5MU5FMS zQ5808;KCgGd^c|Tafj!ladv;MBXhwzu6#Hqh7w<&MJsa}MLCdRo!aWgTW8+Juos(K z2&=B6oHFcn%airnpOorU7#0~jr*>b``==&41>LG-H%LnF8~uFKya@{iO#kmZ_}QQ* zZmk{l7g{a4e{n(^Xw$@snrAV&lDFIUI9cDj%X1tN!PNyax{|H5PJpg_AkcFdEcivK zvUcjHoC#b_iP*Axuiode*)NbTIxv!D!DnF%7?7#0lrGw{7jCr|^<&3xdv2Pqu~r`Z z;^Q9w=5tN&Aw2&SSY#H;P^zik_2qSm3i8c1Uk$JM8fo}f7k{W+(o>Woyv3`akJV_W zUU|b8;m=6H(vzX#D60XuG)}k9oI0b3M==P3B8g{o{zh;uCO;yBD8%TnqurqiGC*+ttz8eY3*1e4uNU%rr!Gtm zOF6h`S6{VJRzrh<7QM&G%olM}d|O2vlW{r!t|X;8>EIe1Z_=pg12NJ_jLiL|vGkau z2#b@+ZuPTOlJf*UpCxz1P-&-0l3~)rbbCwHLX*a2gKt+!w!wkZj;|Y3Yt38*QU>+K z{$Oc6#G&lp%pj+~&{s0xL;d`btAC%kr9mb0MAsJu$YeE9@?ZS(<5?zlGtu3+$piTu zjlbX(8}ZzzNJ^+h*eguY%t(ZBGU!O%E$f%?9l30M=)aB7bKFksO&n{$yhQa3Hu0}X zA4O-We`K(MLZGi0cx7~t5CsuTGHIpPcb6S66|dxuPV@H|P~6O$_~TlRHa0VvbF4kn z5M>vBFSJUyO9g3=7bAkXN(PuZc{}v}o5hhDx5RH}I|F9h{P<1pm}&f`y5k#e<<0e` z{gT-MHcP+D|AGq?uge1DVG(~#N$<5iI%a*+%WkSi4uh%|ZXuVCwr zd;RtW(S^G1ekp^~JY@`0Vb${+E8S942QoC>8(Do~F-Lr)a55QYr(m=yJIJoJ{| zkS49AfitMhmG{>#G~y4t-L@(T0`)3IKzQCw%s5h>kV{U4&aO_78U?k~ntnMy!^ouHK*$O5q`Ht8gIw0)-^ z<4-2v6|$Kd39BqZA+4~@lu}OWT!FFb@mHT|{!)oLE(n2Su%E4U^w&K+j-!tv)RJFf z)aQG%Yg7vH0~GD*gu)>HC9JLiGzJ)~9K`p{4EjN?CEv!8q>n_0{o4VNFU%mGYn?s9BSRnIIAT)7Z}*8#Z0Sy`eVkeJj27(TD`yf;0>4NlPZ#kDw5I~;%@e@`Qu?*vDCtU8e4#ro>v@SFuGXv6 zUu@)?hNDTia#^C-Fb=6r!^IbXRhj+?II*Rz!$c+HECjgHn(Ze%k^o5Avr-r|t1*=O zoqqZUvpo#Howt0b;0SBcbBPSbekDzLVs2vO@uF?o`_-;lkm?Ze8vm0Zgu~+cg|(Z7 zQ9gg*5VrN^Hc_m|g;K`ck!(Mw^heJa|MUN;{QTcLhG|Yc+I3f#(T2@Oel!wEqu`I` zhId$OGP_auW|_<@o%(CSh)EM=ERska()CH(OQ6hDe8)iy_2T8R$)Cj|g=i5vTslO7 znVz?gc)Z*uLhb`;zl1Xj&wVO{)EByZW*0n>;l5agUkO#mDxLlCKVRFD5%WKI=RIYD(zPAdYjviZoT=wt7Hfr&_UQD~2%< zc_7pGp61K!VfAowe6?Tx=~CkZdw*g~?vE5KJ7)62)0>6!(o?LPf3uZUvOE!m{~$-R z;!;rJYkDcC;F*_zHGB!oniM@fl$`;+*EdU>B!9dooQ3pyBQ7z6DS@ ztgxQHYj6ZadAe;C+`mF8)Gm(VkdI@2BoYiljstfuJ#a**PcekbfH^i?T;*rduWwMw zh1yH_k)(eFV)*WjZ7tGx%U=$k=ChOT3TCmbiQ$~LX0<35x?d^O z<(l|gSXF$bN&oZXGEP%v3TnaSKO~6ApFs;JyPw`Hle7hF0>)$1RR=|W+_1cnf4`6> z%AzGQT7%mVv*CUlr{J--{^c{hC!D6AOoTP5Pee|XA#O4&YDHiaXhT~m6*S=_qXUPW zd8lMkP@bc@0aLzwvB^O@-MJ5sj**%p6-TR>&c9@rwtW zBD3OvxNnd_dV8#wYvIpgXW6dDWU{t;9)*mTwI=(#8D1h@acN(D8ZI5?Z|LVtmPTY8 zLZhX%7C_4=~lD$DNR1dCOhoY)f z;evcczwZUp|Gb{$-=UhNI+Ql9gjM#8sx0bcD{gXhsy8C+N*f&PsrZF{m1$1$XU>I3 zJQyfw6AX-P-VP=%OJiK0pm9I`oyiFQil^$IF8$v_3IAW{&5yNDua*t%P*hu+|3Lrh z0EhR#N1WEwoU6L4c{kqjaqscJ_3GwgGF~yRk#&Uj-NP+EZR?%of9dF*_VNjBR5U^ z&*PB4`q*f!#O(k4N&KH**7%u=sz1!T(=_!RV!Je5QY={5wbwV9ock zsSO%52Zl8QTt)*zel6KHx)Mv2QJcAcU2CnZ>49MoPAQAAbtA& zW)j_m-t*}UEp6GMLy_~LfeW@X8|@5Ar6q~>NY(Zqc-wIvnrq1Hgm_L=e-r6N|6uzO z?q`t^@=xAXf1PncwAVkaIEvH+c(G~T{P`6?+Cm^qIw;WM)TDN_*%))A+>DhgFGbx$ zdxHIdqm21gBsVubQ_{eq1Bb7sq8VI6IiLDQ!3bljB_#N_x~T*+e=A zL%9?IQ8OS8W9@X+zwoZ9p0?dB`YJBZyV)d$7EU0Rs|#AYC9pwTiR19Hf!YbEq2Z_~ z%}D^dPSNW62<5kr(uOl1Lv_`8`D2&OntkVkQMd2joyk|6!%U>(7(V^PR)r&XGw|Z2 zzEW*!XasHKeDYr1x{XztgH060G6B0u%brIM9B5no@d`&d>C4E-n`6UPzP!TRC)-+G;@Bifc&?Jiwk(YB;rm;CC& z0lE5yx4vb}k6pBGXI1Fx?zg|ufcd(lB>c{uI~~kt-MYlF@2&dU&FtE>t8PtS2&Czh zSG8%_GS8t_%&zoEZXVPxHr{Sj?$rPOLAN~~d{%$a;nD%Q;r5;fyL`MFHZdUye}hsL z;gG~xRN>*_oBsF%PT(Y8rMFen?B(*9S0yD!3WwLOJqsu=*xIpZVH77tHu&jluioWf zd#rVBF!xw)n3k^kdS`dIwf5F6kAxvRt9I@xUGKvEbjYuMnscV~16NN`mUrNf1#_$p zc8R`J>{+*Ca&wP{;r%kVR;B&-pYETlKjhZ8%X4EFIq$4m;%on=VIyt3tw-__5)Ma4 zN4NaOakeI@7L(h{G0Fy~`se(*&Fa+0l1;XC@;gOo-WuVmy>oSrN!yZ-I&T`fy2i#o zx4iCg``qTzzik_Vf-(s0IBdBbCj{a?qPnyJJWgH0 z7_)Px|8Dx8r8yh$WGkQ2uxrbklCQAZ(`|g}72%9DG*SnTZS41=px`vsnX$^=<3$5$ zrgOBtvND(+FKwk$UVNNY;lCf&oex9wvWfG&e9?>gkajh{0K>k1_w*`$1S9(|Yp!~i ze_&kWL-yt+4%KVb$Hts+W$xe*G{iNhr_w1a;v@v;o7cV6zv0~@BYgJkilWTS?_bI) z;Mm?#0Z!wn*GuljHaBnFcuPx$89O*`mX7*z-~D|rIXQcL&}s}t(qc|?Ioipbs zPLMTTxc{0oYV>j5D)rB5_2C@eA}Z&{jO^tp>M57U*m+Vo4`yQG>jy{j?zK|yd7qkJ zO`?B-=ikk-w85OTxUcUl8(JYM+?78*XBTCW2`2z<-?1YN^Ru$;k4wMze?a@|x3m8( zCDp&?eW=B;A3;+6e>I-}w)sDXBl!#OiNC5l6@L69k#tN?^~e7E|2^ve-+y`N%lv%t z*e&&|qw(=P<9_tIcDp1Nd3+|1Ul)0x>CEuX=YhnNSe@B$STFW1;Td_zs_{@&Kd;aMGd;JJMp z8qc7W)8eGbzI2X|-PAbx-OXFK-~gtF5G6TuFbDz7x1W_p&|N6mawyp|^KNdJe}f_F zu+*=*EeVVc_V<4rHY=2-ptpFpRsL(PDzxt3xN+kQ@=l1& z<&eL-jQ$`OV++yS-oW+j^4E8R=>_~si$?6M)IK*C#NMl$(~2xb$ulKJMgj_fVeU_FcOy|mFt=GU%W%kA4N`o;R1FN;Q@ zX@7+c3vg-G%lsihrB&9&TyFNt&dH#RkBnoeu; zc+QZeEv)_M^|4T=I`8t98I&B0_U{PFV8)EOv-4R<>|=DYg)mD~u1JhcaU=*Hy0;TB`Y zj48|r^~3LQhKnj>&QAM*4u3_N#x7t#*5u=0&g3AaS4BmkCVSNsamYc0pr%w9;^ER_!!`P5ZXoCDyQIUZKl0BGT zJh0C0rp=rC9XWD@mWIjw>lmCv@O4DC>ow!lPH~ek94my~)qa%cB}J)!=R}KP%U}J) z^F8tTvqu~Xq78AGLr0F%(-=yK!e~9HYR;|63t&tjPExTTo`RZ-P<3>*;j+zmmy$j} ze1(!E*e=pjCU?R?JdcROv9olZyS;evf@PTjuh&@T`KiI&iL7EiK7*4-2QPl8H43c@ zkaXeu_wNU~82`Rm)4NvAOIHLl$BUi6=i(?WEhB#30>3rnP1kewaW=4H6_);q&D;Qp zE>F&nQKiSMSQHfrKHAl3gry)q{D^}%zW3(Mn`=%?dVZxiFJU&@aKA$S9lC5E?@J3~1`bc*duvmj!V*|>3= zABWElfAVDJ(tS;|0z2q0+&Sfl;W>_}Z`OI_6PLj;B&c2+YQB>&9`Yh9cs8CF%%0z3 zGjvhtW@W3^AtdnBPH@%J7p~G&CCh;9_v@s@$+wdOvMrl;F|_04*VRv1dOD*=kKU&{V*;7%5i4@l zo8)#&_j#IR`O>NP$@2$>-_kTySXf+@{fOB_YY; z%(O9c=gvJiVs%)XUJZP5wc`(SesNwOpJl8C;RsWf=MmDTZPU~?=QQx*XIC?AU)Rm- zQ@XA;_1m_IgaHPlgv>boH_d`8ya+I!gj|aV2PJ_t$@XO3;7XFX6QG<+-FtEStt~rWv{4A?s@QcxcPi#y}T6tFV>(FEbf^ub>G|V7FYqe*IT`v=xLw z&y$wVJzu8;+H#7piLvoR@`0jyZSyz0x*}S(roFhnr2<>(8FaS0D(xMT&iE0}sKr{X z5Nl3jU>%AUt$vmCPO!Cg_Wd9kYEK5>gt4^JY??-GjzsHJF44Yd|e+JC> zr8(E?E=W?1r@QzOLoq@T_ascp3t|Tq-`u`&i3}4`*5-yOdiLz898p#;I)mlTe@XW z-148j51~>XQ209VWFmRHhbStPGAK8B%Hfgk%Bu;z0r9qx&ZYA*j0#85AFq={Ptnqa z3zx=!Ii-wfKiKIB*1N#<`1&q;du#YQbQt3N(K5y;BtSaV84NV<8!KY6inG79nuXSl zEuh=sJS!{O1smUC^4W+ZE*xOLkRiuqW?PI3iBHzVwMnYiclP{4_36kt$Og(c`T6sa zO8nCd3=MM$5%W}M%}$!wpKWjCdC4t*LP5tf6t0iKR`*y8%80RBnphK&(y9_S*4y8G z;xE1q^xc-p9Z*o;Q&^Y2yxFI3-xOBNLe*KLjdd&8#6zpQb?<(TLQp5k@ax;|{VabD zN+CwiaNeo81FXp`ri3oU0n=-PKl%&-=^mKm%uL@pFf|&L-KDETHh=j zE4fY*!TT&yQY9dJ*!-(Zz-UcID(}R>&B}^!^jI!zC^Z5Ed z|6JxN^5}Ns*-a*?h!fDe(ati+S0-Y%ZPQ zISyX^!ETYEk?Payd#9O+VWL@9?@lyAs$#>24Rl`3U=yVRBDIx_+Ivhj@UFaR82};*lA#ZqopQ@@d(R)I8Z@OAqiSM#`#;njH7BEMu+S>T7PwT$d z+g9Zt%q#Y!2z;iK-+~M@`OoHE9<#{?t$aHTMU>WyRm;C^8d5rR+r4z&WuIof8k#L7 zhd#ZyXsYU4SBeN58L|xJ~EIVQ=3qUZnYj z0-nFwPygAmx(%A#$;+Efv@rvx%#3XL&T{2SuVI^#>0@f795=@N5|*}eP-4MU04en& z%1b__N$5L%;>0Q#6S9eMe0=;AjTdIysvg9naHoc>)OeQ+x><=8w1{w}e9Cv7;^un} z8DdGIOCuOo;A{&fH7dXA#2@ZIno1SVC?Uo#d+q7+62EC`_`zX)wl)dt{4y?WEIvLr8@4|XDW7G_;MJ+ZL?tn0{P1EEO|*+NVCVBGD6-lUcj-srFZBMYSDc?r;i1)q+%5aEC$%NAGFwq zSQkuw59MtMHUs1K?$V_yJc-db#otH2(68P#v1gsAAp1B^M~)Z}WGp5a=l)~9R^1I| zMH{Q?)~n}x38%#LhQD;2W^DWy-}*X9wbA_0AV(D))xXFyWQka^4<&I8-Pqpm-btEJ zOtv&>B^eZkZuGFK+41R&T?+(K)2sSw{qK36t zVdUvfx|ajWogNVfN5v~*?(p^BJtE5LQN#;MQP@xQSuY?}n2Ez9*OJA)e)?-S*vJokGu!AWzlq%V$`={24cCymX z^)1W$vhjw|bymH3q_Z`rWdDkb!9EGGlkAx0bL-`X{Utw)#(Re1feY12yD zrWSIev$8c!vKW>70JzganA^#}7LmG))o14XrF zy+(fDQkKHv{>w@PlYZ7*`e3Vy`p}_{hkR3?{{0`a2qvp=;3Pxm`WAy9^A0NX%!2+B zn?#o3{98XaZPH|*atM=qF7@gZt%F=0v0_6|xAEI&^16~UGc#j)yn=OoRbGvgAmCdc z(+JL*nQ3PiK+XS@ZzuhC7 zcO3dq;eYRS1n^{S#4Fp@$Li;(AkiJ6=O_c*ZqB(51yKy2Hc+`=&#Lbdh&X`zwP6E_ za$+vFIb3Ji0zByhm92mMbq>os=3+G!k-OpMpMLu3Dd06Al7hjS+1l8aS02TU4|iP*!|X8z^lxP+&^NqtFHsZbR;06xm7c*IQY?JeTVd%dH#8;KN{1{ zyi-3Z@67b>Yni{8oR^oU$f9R&4vA`V{vr*#ENGYy0c}>jOT9U7-n``G_iX)?06)cA z90mbiFJHcV4|G|*DVImTAH)9Xr5+y9+|DiAsf~@NJKg$r(lX4>QFWG-S6|boYaAqo zP7-;bSRO^>12(oI3q*X#;mE)D;{N&L+=9WiPPdB*n7vtBDO+3!BJOK9v>m?k%%UCf zTYfW1?BmGR*GYmzo5#Jz&+Gq!dTvFSr}6jFs;Vd{Et_W4vufR?ONm{KkKOjmq;f%* zs^kX`9`H*qAIIZA4xM#1a?9q;2LQwS^sG{NqkAl0*UNn6dHbrrH648cirzf;K@i1p z$_U=qQ(nijr*s(TuZXdWTwP+nW7n=#I$ zmUc&H#vZu8xZdEj3qx#DzKwY~6?E#tJvdq$dau-hW5h2#1lND&dmcCU({%d@)ErfySSzSAK>g2+~(c0Wc z0leI@zJ{N}o3`ou1cOj9sO~b{TlB2-^Atkt@I%=JU*w@M12Qe*yX`NsH;9ORh7Ns# z)uxmF#g>0GX1dnCvN!9%H?q}b3DL+o2aYl->TbiZo~2GlXu0c?b#yYg=^Vx9;~-@V z0$IcY(^3;Ht+=bd6I`Dv3WE(;#fG6D!nfC;ay5z+_d)MN(-{x|Ruc(jr^j*n~1)%wuj(8t#S__FNQQUGW6INNKR8X&?srL<}xRdt}D*VB9Z>IKs@ zs_*N-#FCYfSjDclE5W81`R4wi5mAfo95N`e$a_;K@!UI7Vx*7HuRC_^(7zaTZ8M39 zBL@F^M!Ssr$LC}ESv9r0dgEa(g&9vUKKw)0#oIA~^DoT1Ikex{t*helM-3VBnDTF| zR-JaKpuRxcPxzkFvucPlBck%f4SW_IrFHAp3st?DuX?ugpFbCo;|D7(j^O49EBezQN^+`iOhv*xmUg=wdEZ`r!_ir3rNK?i)ICuJgM#Lm?A z8D8+38#>VHdwP#b0PEu9VPi7z1-Z-$sfZX}>=Qnoi{`(b=(&NH1HMWG5dUAOE zSH}W(1wl2EQyToU654!%Cb_S$1_;F24f7ekEy`(ij9tp2FTjQ2N1nA=dj$WYM_htoYHO6IwB-_S+HfhyMakb@8pWhT=m0go4V?N~6GgCo~q;a~& zl7$Ng3>ZMm^+I0IJr;+&0V`DUl=uVq@7BXxJB7ClL^^LNN7;M{##0d z>Cm^}Rzev%@usfON}*7~jrRh3s?WJ}v>&mo zyFAsucJ5541giilN4wCZ5)Zv9hiA#an_iY@4+#ChmQoDc3Tv)N%FHzAWAc;P0wI8z z(s5~hwP&+V!_&|8S^XhLvyPJj_NOhDI*f-oWR>tmtK9*fCQti~^fp zASC*A%tyy+f8WKPk!q<+(HvrL9z-g~{=5kOQ(UbO4$C;cxk4^&oEEQnp(IAK-J zN;ctfV2CX+iGw(NaRGO|A22Mfc$c*bcK9UdeW>p*6s2kSr4F8Q0Nqf#nB|VV`MXC(Z%{oSGilOJ6qlY# zQkCUF!-mhDw13iMtLZ5Z_Mf{c!Hc_Ga9Q#V^CX9V&&aF$uB=ilB?i~wO`PDMSJjuVyOa={h#{UeZBG3FQf!N~woH0PTrJ`u1 zes$EadTo*)2c_~fX2oe?*0B7Veud(dv9ib=iN!;=y|7C0^EuqdI^tEtTzy{d!p4mo zw+GT)zkXc^hEOIu^_m?YyP`%>-Cxdv>mjPxbDoh|guV#`4_ba3p<^|1oI!Gy z{=H%oAJn&8`QmpRxi}C!MQP(NOM_dU(VI&LdEL zM`3B#N#c&*qX0TqKvqOz94YAhfvWiuNaUcR!|;`hRq@lOPq#cVp;3vw&=bTRK}I-f{pidizJ*x)sPqT{q8Mb>0KoG78qXIR#r!#)?nu+ zK2=kLOf!e)&d>z-K)@3vodJ!VB}*Wn`h$R3ME7BWW_WeHHDcg#R^5^(=Z}@O@6xvI zVxlC4mS4Xk?+qs(95g`5RM=mmieX(Ytg!W^t{K(`FqZ`HahAQWlfNvYjM^T0nbfdVd?we6qy<(H4Bd?x=a+Gq-zpFAHsMM{TKW?sB!*$GJKFm&0` z1$g(@bXCi@UO%cXy)2`;K8H^(jP0?AMUXPeU#4G7$f%e31~O;mA#+og{u@jG-uadf zNvWPnW#EebFi8I*pjqybl2R+Hn)yR7b84BGm-{$j!-lK)zb2_DtFs||Qa{N`PT05^ z(^WRLZeTz2nzf^iFK+1ZTJw)3s@`Apq`H zgnf$edyWD%B_^FfRs0^n`69X5cLN`dphdy$N|LN0!$4bi6xL`Isb@T&KLumo9v1Gm zw{T9~{7&sMAV2d1il!bMl*ER0p^4h^s8JnX2gK4T5i&_P;K5J;|7AKlZ6VYGYA134 z!nQVt8!uZ^+f3n{9eKa6UGyC2o`GGu9C5gPg&ZHGZJ+F`ZQ2@*^HiB<2`d1_$?^N7 zWx8Z@N3@0>hd%4;XUDCLlH);H=&zFUW9M`lx-79t>)ys-u_;o_$-;}$YkHM~23jwu zm-${8@-|}ETCTG#T>C(x$HWQy57h04&Y{xN*Ru$5pmd(>Z?}cFk#7H5+b2nq^Zg#;V@aY;xhRzy7+^vy0JiIr+$6fwp3=isVyB zT9f2vo=z3yX3}Zo$FKDq?9I8#nq)a{#41$$zEABD5V}<{`{$;jj?e$ z1J;1Op5?}FwREqh5tv`Jo(GV5vY?=)&k(@)DUQwp+lbXHqPWM@!+Fj6#!{v{woW*ZJV)7CgpUF)w!D!rflo>sAhH$N5v3`_5x zJ@3$|?!aJ{)3(`2|56n{zOrgb5p1bo&Pw`|vvtDJEpBt-&9ZLy{VW_Qz` zvhwig4Y?RhCa?%hoNli@j`V4b@Ys9m;gOY0q+51qM3Aj)G-d1d?VZ??Q8*9nSE#q* zS!hL_B(ke2NH|SPERP*VdYmUFJ>*Cd7k1K$A|%m{OW-#xg3WGXPnb*VoUlfr?$pRF z>or~|vXMT@uDq^5AYsI9G0-4g2Sta}P3=tKIf#flDRn`3%?Qu(Gx{&GAVde9{U!Q> z>kW_5AnG2b^z-v-i$! zRN$~x0nS?2-sdT|-FRRq0@>It8UqiCK|zrPww&?4MVBs1k(>0D+?l__(%;YPS_g<%Z*Qv)!&eQAsCqeQI9Qaq#wL%Gt znNDS<&f7~KPa+YCy$pHu?UwBu(@EKn=vP0@ineDUo2|+nR4qiRYLNP3e=(Y{*CT$A z+skq2OxoG3qKUv(OVZ%wRYjISp%Hy}7YC?JkSTicPIqTEvD+=KA2cw}7hBo{xFA|4 z?C(1(<94_AxD}=6w9Ctg(I^w(3_3#r7jDUy<3WqCHN4fjgN8MUX5AnIGm)= z&7|GQSL<%<@E2}H88)}aA$#AZ#`paCWfIr}Ew$@*rc<*i280J+zQI-xTK3%N%a<>X zq$f;O0sPkq)F~%+mdMl$o$PYX`r@q}7WFdwDiq@&!Y5ydqX-no1ihatVXX(9{n=?S*lH6IUa;yU)!mBX#WKy(hs@S%4PyVZ7i9)?9GXT7|pk$h>Si1dqcP zFTp)yLs{A0a!b+po^=ewKrP~v`M;i@YHXgmmCbP7l$YEv^s)E%ht@gDBJ1HeKQ@1% z=;{=gP7Har+P8eN?g3$pfWlLmrLY$RNy)p2jUBpyg++ZL3Y_E8gPw$wE=5QnAexfW z?w;#|IycNwh_A(QMmADK9~|NmB4(F%`nR%QYZw<06oZhvl`7iXMu|*xGd`eS*AsB) zTN%pdr^s$s5^c>2F=a-=4!}+L6N8g;uCrqhp)8mZvIO|KxV?F^oVGCP8SJJ|K+S-g zhGJ2GF-r#?h~9!K=2{HA*?;4^hU*lu(cEy_-mclP`VKu2#4Sk96||s*U^_afzvyq< zZ+M-`fuBV{JOSB5mgWY$r|s<9Ql%LzwXPRGo7OiDJLF^nhCwMm_>%U zCV#;n!-#*D&|xa)JPv4!bbZ!~-S2*{h1RHTSg_H%EfUsyZTa)hA_j705^=(v@@{Ci zhoQaS$3){TLZ909$H7V=#0VYP?p->ZcbcV8b{OQD|&b~Fr| zK{+#g^@p>ViNrjJO(o{NiJRzVOnlhmu$HAKc3Wj_O`b+-DElpx$RMUlVB0Cc>#U6R z;&5BM_>2^)T;ctHEM9IMp!Ej>gpYjr`t_h9>Ig2%v~zhx-C~H0{>s)JJ38a?mva5I z7xCv!LBS%JeTvP)kEhm)?nT;J(V+kpF#?`m``vBs>?jW@WoB~(9%7fT15L1gvtm6; zL;f!1Q63_O{GApyYXo%6$7Qb*D_FaiRNAJ^#RnBj($73< z7N!|J0Ndm9b*?exJxJHt>gU*r#eB4&?}>GJ!r2P6>J}OCA;AE=FRoR z^c_&i3wVT?A+QpK@1F2Us`tQ(id!@4gqd(Nfo|()P|kw4P?gVRQ}$o+W)c-cf90E! zlBF2-bduDNg6hzs(|m#fsk`v|?4GufX9GvBN&RPEJ%b6T$N_6|Y4x<7G-*=E-;BY4 zK!MbB#|_cJPJ3B3dcaw>gVON9x36EHQk8`jwiRyh&Bi?gel;=LDm+%f_+SFCjJJ5o za5qO)^6TYU5Qfjb;!(0;odkMq=j=}m(&ssp;5rFcN{j4G&fioNiYl@o7ltdX=w(w#i=O zig8(FkYcJ{_xPM=BtUuY>v@x_;-@7Aj))QypbYRA$H-(AY2sbT-?VGOEJ-fYgX>`9n_3GnF{BD&HJh`e{q6)HtE7 zDI=T-a#la<92k(hr}TYx)PtD1^H4NgT{sV=zfyO~rCET41?;CuKtwatZ>&J}u|u;e zaY)-$j(g8@jKUt0r?948e~5Z7c8-AIjsuT44x2lrWmd0{+UykKAG1Ds}@SgR!T0)Xs3=chJ`Cv=LwD*)=Qdyfb zv3T>B&!6SP!+NPvzpsza^Rrr+YSX0b8bub**yCot+!1j~@U0n0q2F4!EKv4)ICtHY zwXz*}VXy7(_}fiuVceYad*`WoQ-`RlZdrEzL8112j$xG3Ny4d7vdzrxkKas;t^i-h z7i*N{i9Vq139POfd}8ZQzx&iIoD6# z8!#|;&mP5k4KIeQsV<45N%lCO5y2V}Xzs!$SANBIIAV68S|0g4Y=W6eyImVbFf-Q5 z?J1`(8QNZ$x5f4d$KwvO+Rmd-mBGrFdUuI-+SWCninGVJtXEV*LzI<_jIzrp&D~Jz z18T*}YadnX~nNSminTs6bd5y=pKmzZRCr=c&m9d}p8`T|nTvOZR zfZ`io^JkH@5KESr>=v~OIDfvsNJq9u@UcJPOf4vXE^ytL9V^9<(ppn<9^2wgO5mZO zYV2LH@3-8kt&n@EFd6>QaBO4aOct<%x}sEGm`ZZ62)7?qyuEJf9T+=j)q9ZqKz-98 z2l`{`ksysP@bIbQ1OVBMdUL4IMz<=NJ|0lK zEo$(WlHE|L@?zgrvDl?1cZNjjeQn3ebC?LIfTkWf?EDShbF!Jr;*rL4Gb;HZl(9{l zHJj|#>KL^e1jW+!RzG>zW6~Bn0yc69p0WOOVf>_I5DhOD=U=g;L0<|FmYjbKSp)dWN$bWZ^Hq*K3zfq$0} z`s~A6N4xXRgc^q2K83kMgb5pVA|*%=`eZNcvImE{9S01a5OWYOD4peaman_gNnPF1 z!U$IpS3b!*{VMNWyE^ZCA}SgN34P&UNvKaR%>NU>`FfqzKw1K-G1RJpa2&!}&k&;x zA01a#-#hf7)FxQd`8(4{9E^uL)P@*m!eNGrXK1#ab`M}U_*NOYlr{(2ArJR%{_za` zIQ; z)cJ+}6ego{F9VEPX6NVUYwIR0^3?l;Jn>hXHG^uK>c|S>xlcDY--{RQCqFKKxP9-aweD!Z78|~o z{<*v$7xkiL>(=|7*P4{oQ(c<-&riD_|MURM;mUpk+U-+e z>!W^*aj}hG7XyYq%eeDjCmC&R-MRCJA@8)R5A7etmM^Pfuuq}*-Db64_KZ#OA5%J8 zwmr-#?F?E37fPa8Kt0!L%GG|#;j}#%+-)%OI9sL3&p)SA!-rAB&SHf>Vou%((sJ&( z*|boKHl|GF8)B#TS~jb1mA~Qe;Wnxq!?Z*G(kB5zhg?WZdtK*_7UV`tO-n;PGFh&^-Fr&r2qfN@Hsh);ais+xFONzctr$nEgDPG&J9=N=M4D#W%- zEhLqOLMhvvmGkPcf!q4$@P*rV?xe1*vzrsJVmP*_FL-;p6d+iD0xW*7En?=3FNN8_ zQ%eH7_|#PYBHO$$J1V?~=^uZRhPOg54vW&>;n#0=7ywC^TY=$8(g}g#896e?`iiXz zYU`Bv!29}{asb9^*Vogy1$Oo=GC^<|=1oV=(-b!%Bcd82VVMx&rl!iAnQV)^NyW&l9=W`tr@8>6!)!VjlGbmfPXHVme8#mg+YZ;;ssb1Md zpU2QM3KNI4sR&{0uJl zKJmBSUW|k&(Fj|cglUTF2|OVBA*QP0M}PnLhvH2`pLkT8agchCr9F~dH7ldQw>G7@ zaqUl=Ri@^En((PNfLU>Ta*L(^x@#}kzm9_SgtJYv)g;=u)HrC=Rqr3}a ziqRx+zMD=fv*quC{yDjEGuCcl%vissJ5Fmn*IDM+^F^xBSki^GLw*?W_s@OU`n`DO z+-iPn&AqDl zX$3tXCi{9hQmjjZ7;N$hW+ypOdboOl@e`f1GcE8M)wyZk-z81~_cdr9s&TB_^@ z9O&=-q(8xNDtksca>;jtk+P+u4#n;<>9&=o=5f0J$~1qQ_&4POLKucpCL`i$L zph_XFByi5@w4)CKBG9ig9p>4Je-lTJM~unyLJoqqH{IX+5wcQP+ewR=!b{;P#_c41E=*tn1;CHjN>sl?~jp`)PXE2YVL@jM?(Tm@TLm*oY%2VWzMw1W^ zvWRbTz8bRMt2-}->|TWz^j^TQiyW!=yWg`~!PiPlCK+H$(@BCQeEu$C&jh=Le%l=C zyvqR@KHDEOe@p$Gov?F4O0n5et-IEV_J=>5BHacT$IYj+I-O^9dGVv`_x9DXDxfU8 zQkF7eRn7M`(gY{8894DZ-GyNDm)qwL>)LgUymt0Kdw!T88EJPTPoEm(`BP$sWLAXV zFsxcaQQn0CNE3UlMVOs6TWmU20(|ueLI%&^UU6HFH#C=bN8|aGmNfap)a+4M z8YfN;1v3x#Ebm!1J6gM=;kub}#jX3=_0*ridrpsAySkz{+lu%~A7wP#sb2GuM~R|N z((2W;>5Lz4kJD5{W3X+;zFhJJ$+3d#eaHPss5$Ies0-88Zx>F|3URN)BlN^L&5KWi z^een1xsYzgX+l9FDoG0;h-WW>^^N;ZZPWc{Sc@rn)?Ht`d09|g*SknWD)ASvQ!pzY zvShtVPP;Y4wH*RwgS~1fa?8V0u%#-2#y)xIB^VpVe)wDu@4Jth5P%E|tRdm2(Nrd_d24 z-}A7pg_HhTwP^9X)(f2*CXMfWlq|sqIN{a`@S%x3{OYn7o1|%!Myj4fkX_pL z#zqs))BY**Hv}$cFDjKj8G?AcLZ**MH;43Q5T))SFi^6)UOPmAGn))9_wOg|fSmvG zh(+C4Au~OWp~QsDyl_1Gi^#lTYi`!piW|X0AjzBrUx(t7OsOO(gu*vqQ=hT1Zi8}O zy&kPZl-Dvx1oLU}*~2}R)L(&M&2;)vmX2)Z@vFM|U2B(J-ve_N(xkF`MaEE{>2|Rm zk6gaw5KjtmH14N}1>_$W!oUTUAcM4Ge$u2Bv#h7)_Eb+{i2Y!4P2h;O_0!JAB{d^P zpI>;ty+CGU2AQ=Y_E<)*C{^*bHESiEf?4Dx62IdthNS+rJ4!v`*tHdAni=Q}tq==H zDT^6ItEy?Hpw)E@OFolG9S6neg8ZHbYzsy&I|!|Bt2IU?_BCZHrR)BM>JbS>p5jUr ztV^iE0=f5!dvVB(C4Yg&&xoZ54yt_lzQoI4O1>M><4`VKJ%rdnTDMym`vk#+^D5-@wNwuQ&Z3J4;3eEYoWz-Vwd5lvfV7rPsfI ze`Pjhr>;``IW~UkxdrcJxha%zfDg$+3MH>T;V{zx{r!d3y8p?$2-55L1&H3)F5Zg85SFF&5DRC+Xg*C~29&w{$j@)LDKj}_j zE%WCf9QpJzcWdv7H*Vh!1!tqV9F#(i|EWt<&F!mxeW_cJn~P9Pe8*|kF-X7|{1#N_ zhc8T;G==KR*6!Nuv&U8Wg6Ed=7a~k_-&j9KQ$QY1+=;E!1T7JuAj_T$uQdSqN}IAK zKbAv_Mg^jr2o!z_oTi(MuJ_se@6%BC1O*J;V^r7G38g)qNwI2A9Bi?akD?utBG+71 z592UyBjqIU@J_NCQoskm|A(?Q#d@@9)24F1B9sc1kpYf>{re3vzJ+ehE%FT{YjKfD zMvy>F>@x#B#m$lvA2otHz_d;0Rmx{0ADreXITD?vuTyMQM836Gdj}2h^Y@S1A-~`% z=e1bzVR5s5dOrQpqesJWeG%@8ip}JowPwv*uMxE!ZqgJMxYWJfN6r6Zm|e8jeWb~+ov>guAU+VXlDLo%aSpHrw$7*;NMNs~@YIZPiHRm;%lVb=(x`PIp%S%Gc z0YP*lV?N5(emPnUe&VaNi+-4WlA+0SWpaqtu2FR5vo?cU_p_JMtQP{XwEck)k7Kn7 z1x0;@>Y>Bm)l6nUPQ^$cPZ?6|BQ9QeerXA~g)0loJG< zrj3h`X2`qp^{#bfj*0Q_B8VdW^yZ(?`lMpEfUl4}18gYdn=L_juk!z~{V{09Ae32^ zA!wI~+0!8Qq)QaK+6>X27LSJ@9&_kU9>SWd=H!;5H-G>Qjp%*7Z7vzB(vSV#Q2Jc6 z4htx_v*&k-O`Mjly>#D?5vr+)ph&p~4l6eW812D9m1ao`xzu4eem>3H!t*xx#cyt1 z4WF&VgC0t;fQGNhlQ9wO1hFtmT7^DBcwPcKtouQqMlEYpYhWfaDzctobrpg3Bf=$9;IZ{7rH&7v}FT z?Sc~4iC{osKN<1`y>a^Uh_|pKQXr#K0qZ8?LwhL0G*BLXU%$v8m3Nf^sYe!`3Us~8 z5f!p>MHyG-{nutqm&qg?iJ?0?ex0yMwCx7du+vM(mGuyb(s0QNdS+&QYD^1tvZA@rcFUFw4q?V_6K8(xWYh zS@^v-gd>^y=`+RMmG}A1dGXa(HZ0QJMk`MxdTs2yT5J)V*&*|N=+S3OzUroqo z&ax%lNOzxDnBNkL44Wwe*RUv zFhvDUo+k=xgx47(cW=NX@vKrw^8gf>PIeW$={x}HfWnsPaQ^Lh~Fo#b+viBwTc$Er{w zcw2wxC(>k8-iF%sUL4K#72}3{Y*Y<#W_oQ+wYF0H%l^YgV_TTKN`~2McrSq>9MAcD zGvu>{>_sYi$0VgRup-3pR)+^|XyC|*cAGD>3?nSWNGy}!B&z^-Br4&1J_dRp{w=`J z%Z4OszHVJC?@*1`sCE#`rc={%l5kLmPwUKP*ow8+i}#McaRBFPD2qqPJw7cB!T2QL z=xT)v`$-;~Sb@;qhkJZHD)(BEp;W8Ywm3 z@Qs&w_qpdbP$5rNiT^AsU$(Bon$jcC1zf(&G-|dJb+ed3QY*su`W$?!X=T!=QKR!* z>Zl~cZI?GNA3r7XhMo@N0S>*-|8C_}cqL$owQg62OOPQKQh=X_z}3!{zGj^yF*rG! z+8uk27d--G9aV#{EuYTb7D-rNpWf0)Bo^rk<63&G*-2Li_LO~^$;r&2U#fD(1(x*K z{oId{Yi39^xl&c>k%v@YHUeF<2-i6&_rxjX>p;|>TP3iU^JHL)p zjd@vJBb|@1;dH22@$}GW_R#A`X3FJk|iCVC3psG%4|rb?^6@Xwfa zrVGuOHoDYqHFMpdepjRzw0N*;1>&~eAx)!(-Vn98QG(1t@8V7S0ZdgweKN=Xq!wk zaGY*ep15}#fhlM(8PE)unIIVV`Q?s*`r@;B=j*Y&4J9}pD0)C)ey&+t_Im`H2^`&H z{oH`j8)_W7R3XPFbgejjDsmj(ZBluFl0IcX3&Kj_!NT~%SHdz+_Q+2fRdX_|pbGTS zPN`el;m^&R<-umw)DOGx1NT(iBTPsaNgSubea)+_IW5|!s+R|FiadKEAYHY20BGuG@ch;jk*wLh#trg7+^=QcK9ynQM&!{0_T zGD77-o*9hpI^*n!VU8A=uRCAGO6lk2tUm*|VpcsA!|S1-y|ugO0wWb=nRn?TQT+KR z63q<4q}VBCO2pTMvimd~&91a7Be$furFyP^ZsTOSD?O#T$Ca>GpTB`X?a!@zS=|F12Xa=foky`ZK9TH-%cX=)Dg6Nc~oDI=a$tbiN7F|xm zbg6=fk3Af-6V!&;gAQbxENvxOaB8Z1VY*DU8<|v@p1!|5l>Gkkl2+ZUx}PgFnZi_q zy&xMJVQ1BT%c~t(IaHeB;M^~HeRFqtc4e^g8_RXi`}K5mt6uGCtfqYuce#E4>?SE( zC4pU8yE<$HgT}C*%+7GrENBSxxq)53C=6ABa zL>~%&GWuNMm7M%TYM8p)aLUVsG600?61MK4*Y=wZ;29X3{-}!Z-9O}Lt&=h?k*pwW z5_*bET$SV@Gs6^0?4>dSPVr7ytp8Qq`Nl+D$8r1?Zc&a$!J3#*>kvz)YiLCe4oWR9 z?I4JBh={_b9B`);e`dk3C;BMzz-~%T8~;--Jq3cB!V{)8=ePinuE(oM8Tb* zz21kl9=4w3LkHa5?|#3}=kxyad4HiH)Q^x(cWDUV=ecL@(uYbxYr^JRhL1D4Ia=wl z$B4P)4n(j%>9R*K zncS|Smpcx|12Ub)h>16~qE$6kLlMI`NhzYo z*UoTtHK7F^ettEgEQVQ~L=C>ModJe{le>TZ)|)VhzeBlJLzMo&9xzZLNX{HyMaGtI zel&1h#3S)_&8Lg7_yN%O1nhhxpHs7kCjFjj``?hl9ER%ni05RZ13Q|1a*(9+=IP%F#vwJ5eK1Df{w3P;GlzBlXqPL> zP~UV)hk};uf@BpA6veKGk0wfwl#fN;>JWH9_Y;QV$=)$+uHEQEA-=#(=%#V8v90W4 zjnjYMrBI)(ncBgTE=A^qQsy9qK=O$WaUB(Bh2E2cp5*9eE4D7oxs&ib+f?ybRr9^2qt6_(BApk0~kah#pNtF*U`dXC#4coqh}h%orY z+xQlRQ+a|fzSnsikhB$%bGkCk0zB+tQBOKFY?iYAcqex_H)cYUFQxowpu#AnmV2ZF z#7+UaK2WL_W61Yl|F%+6LYG?h>(3&i$|b-Eo>n47Z+f zE-IaQ%&_~!1ba}DEnon~Ibr2?RqCyJ&hYvbuSG7yKo8-2c4ve)&Df7T*C|1PyFQ-$ z%D$m}k8_)>mA;jg^%;-c`g`>BxVq7my6|0oJU)_wAxW|@cU6+UqJ>_Of5tBE=K|OK zDbO`aks(~RoguUn{zpv*q|${(uS~Vj@Rw2h78vUNk$~bK&Wj~iTO8@Va6VGK4d3+7 zF`)AC)7|Cr?R_xC|Bq^@F+3Y}pPOqn(GNIh3ixhw9+`A>i6lAp%_pN9N(vclgvPY% z0i~vxF2zwHwP+KW&v+4S*I?VeG9|_S^gpAKZKkW~(fYuT{{5S%1mD>`y-o-Jx8FhjgoDO~a}F2J*qx=0o^i0Tyx?GIZp7(y*3RDi zg0-lih=`!b4o))%2OInSLPA#m`3b=bcBVphjfaLn$zxDS^hxO^jD7D7B5E5E&u&h zYP^3jJGJJ&zudyU@Ljq2-(Mq(4{$U7_m8RNpN^;d@5gN^W$q7_IrQIO(^n(b{`bRf z@)|zZp#ATst?69hr2Kz-d(p0MRhRwu!wv-t{hxhvvZ;?2k(Z4g)Hr>bQ#E`ajevl_ zN)D;KNADhAc=O_H+0d`=2ORpptjRcE$KP#Z_r9!b1q}^NclF~%=NnQ^RkymXX=`s! zyEJUHUQnMUMlnFJ+lF_==+6qd^XK_5PxLR_zCHfxYVZ6lBNmssDkEC^`}M0^kFOqf zoUTCKfBhkxVM&lY}`oUHcp+|e&zzL>DSi+A*6+r3==_J+DpkJiHL zN-8Rc@OE0;+M@RyGc)70ZqE1n@IeV7^GGRhOM+U=5;ie2>eQ+4JB&Z^)Hl1&t}60j z`uy~p50ikfU5AopqWa>CgFh;U=L?vdGOgYPZ9hg)ZnG2eId+}9DhkYMw!AMXiLq!% zNicYQIc1b-o0ahn!+Z%@S=s0Azb~(36YCaL2;FOBWYp5#eKS2>Ou}VCrSo-1PtUEP zA)64p{rmR|i;BwOVJ&TKv`3B{@tdvIEQ#mWOz_0>5-sn4G|9d=xLNSjD=(c?0~#6s z%81ZVmZ_;JUL#u@8<`-UqhbbKjTsi+>$a=5bl1d~HRro!PK?KXyxkzOcVt%b!nt!d zyu9c(NnB!#IB>CYbY_LJt}a{3soWc;RgtY&Wk%UI#l=em1qGjHWK=Y`l~u7G9W3+x z_2czzd`KEfAz-uL7I~iz3rpw2?yH*9&DYIL4jN+DH++4);k@14l&${j%S=>DM~7FW zq{~m|XN@zuh9%!z^E->i5Auh!jdYe}+jp(T=Ra*`Mt|VIf$Z$;H;s+^bxM|)xCK0} zytb6V=J$-_A$fT&5s@`mc{%)1D{yxU4GlHhabG9O-r4cH+juWiWw`h?Y+LAq2agr} z*lcWVuN>O)F>9px)1LrRi(Es_$u=*>!scefbgL%q?(S}S>O4nXk`A*){j)XZ=H~pJ zlfS+vrRf)YzkdBXRNOx9((FWX@d>G9i?;W-9u!-jIm1QWESR4;!fiD())VbIJ;K(W zmXX19;Nq|H`E_wT^(6ru7N@MIM>=Ue6Vzjs{U?9Fo*HVsCM+VNCQ)Tv?ECSB%B{8B zbX1ySU%kXfojd7%HE-V@q!{}eQWsPLJChMxM=3^8(@M>-AKA9;J5q%%U%qTK)be%< z7Z+EFBgHt-n4!VnH88MKH{(2ac}0cR$hod>xBUG#$lX}E20zO`&cep#Gyi)wY3D^O z8jXzRvlD7h;^NM~E)Y^P$n(z$Y#wyh(9ocf!F~%H_AY$eEbYDn(XqX}yu3DFd+n`5 z4!yO9B|PROnvL#6iALoM)qTyekKdAsVCd9SnLhYui!}~<-yO8Oyi;?#n3!hITz_q% z#{A-x>!|~d%cUeS?yc!3cLin}TBC0F?*ESloj;taZ zN&PH;a1B{omM7@omcU0?j|9CeK_6xz|0hqjd3kxgX=>V-b)mUfs9^4Dh=9%(VPQrE z1%>l{8^!IO7}2RC_53LC2xw_JO`;QT0^7}_ouYrMB5ZGrRa554;w;OryL>l^uZ=x& zzj7jNpt-Tm@rp;OsfN{$-PGJ?2lgU`{Yuh*#U&xJ zDM=^w#;se2?`izTvFG=@r+VawhHsn8{G51ws$l@~!6j#B^HYCL|D&;sBppYfJh8KD zChIcoK41H0VxTGGK=y*ru3c*@9`1hk&YJ-na4qg=SoFDyhr+vd$s)axbAl}Vbf;dD z_RjS4b<5*bBMR{&!X_W?SPsk{31&{++F4LteSDL&`zGqNkrD01ix>Bai<9H0lcN9D zZ{uE@si8Ca1@7!f^Wrn%af3&!Q?B=}WZgwKJUmQZ#m>%7N@3orlmvr5$DRv6d}$T| z9{BqC)jXD4`lYF<9VvPrf^4(IB~SJPPMrJpv5;rIn*2PmdFzfHD^KM*72$O#>iTjf z+4f?;R-9E)QPI51K-ep=gAQv%$Px-UTicw$hBNznBgprZlU6R)oc~ zjJNgn@?X7rwf5(2KR+h>pXF;1IkcWPZ{1Q;Za7!$iENpMQm2-1oQp<=g@uLQ(9kf% zm9>xGATOKZT(XMo$uP^VFuup<>z+o-d9QB6@!~f}h(#kU`KTQeXYG0L`N3Kxm+{`Z zb<}&WW$O+ho)%MS7n<``%3K3Pxm_#fr`j24WX5{yju=NiQ#cYIAAcLkp6^G%l`B`i zrpN`oc_U*x(s6jr#*G4AjBTA|_rE^mB?Dp40xlO8Ycn$?_k$*ooJz6w7IjHqXQc{vB#tgw2AtYiG=XF8R zK!1hk$k^C=>JXxqscV_3@`uR3iTiDFu>!^DW2HB{XnVeOl09MmHPe`L8S0Y1@#y2p<*#;Haur_4Ubt@7U_PAe%n zo~3ks1#>YnP))Ml-gN8l`xR$JMMT~{Kl85o$>DWk=WFS0ZEdHfyP}4>Yqt8$M(lUw z$39#?8n)-mw`^M*JG;_)slA9$>{8OvmoHzwyRmBH@ht08Po6w!nHp|q+3#SM86dEo zcF`i+w)e|UoIH63DP7rIN=k|ZaR0zSO@W8Uli1kSZ#nj)5Z}Ff_wgsDNBMx{g25d$Kr{O+w&&%6;X@7tJuAR*6?0y0|DJ@7KH~sw;5CT9xzK9+f9UUF3 z2glk~j~*?;N3&U&cOy{5{BxX*qhsBSIIUC)!)8R?twUE^-+6D9lG=={QC#e|arhmB zbV7%;+l-i?VZQIw)FtZ8l|@U}i=L%88b%|e++NjB*E&!1vkK~Gd)^l^EPV5ZQdL#8 zxp=W*pZ$kC=g~V4ANrs~7k>P>Ovo_*J>p}it!M?y#*H!7-(Q&_x%mgkTYr6V_R{1p z_We%%3AUMZiJCs1p47Kjjw=Akysuwh+}n`KU}|cbF>SxDAX8mKBh9v5fsz5_u-6Yr zNfG2x4Zn#{q^Mub1p%)d`+jEU``YPMVaeBTl?XtjI%z(>a)kza5uXMibKu>#Z<|OK zM}*sthkJZ{F}i=r;>BD@7r=yfKAP}qJzf7$%>3%Ev!w|g6%{V?SWU7m26>lg+}zy0 zeEs@ZEk=P?E&9;ClAwmc`&AYAaBttfWeae8<4H3#F;R?+X*<+%@Po-N-~rd4y!Je~ z4R$&G;TNNP0s=}t25lc6>(L-vLs zKkvnNUK#Alzm9{YWU=_Wz*YQ50_S*ljT}WvWE%jH?)}?0Z!&)T_))p(#R20t1q<^@ z9}29R_V_u*AGt4hzOU3Wj?U^sgIJ`L+vUu%pAK; zN>NHJ)AD5Nz(BH76l=Sb`&>4*IavMc*O#Uizvn*qZI;xf7*VI0W$A4P8h6vkboci9 z+`LI2GKWf0=CH~TP~Eb#j2GA>-J(I&YW^lz_b$DQC&Fqz9Xxo@C)9tl z4D!LtbMAT{BZl$j|Uukw^29OY*pxZ8i^Q4ZO4)clm_#~Oz7WBm7eeE{uG|q z6(#Ayv)`fn(VhZi#i`*E4z1FT$=S{$=|)A%i_$uszF%}4xg6&eZct`#_v383%cQm1 zvlAjAFFH#@)-ekilskC+{`EcgdTeZLqgxoxDkA&r^f%XKBfYx3UIe^vpx%?$^& zit{4jcCFt%|19^*zH2Hx>(B_%dY*WG`by&nA0HnMKMDo%Px-_=5;i>p1M|$x46P@N z@M)RMUXPKETcw$KQ{iRG=c+8s%o4U#eEzIq<(#PbG}P}`Rnl{je*`Oj`t-@B|BDDEgCd>5Hq3XQ*+PMVO^hDz>Hzt--daAtt=}9R9E30YkZ|a( zrdi6c`909t@dEeX`h$kQu&GMNjxD8-Wjjy4c5X#}Bbk0|Y^*v-M+m36m7H(aDPe-$ z04d3#JZ~DnB4)-YAt^~i0WMxeeg!Zd#^Kx~d3n8FrsY9k9#>b_;B!)!ko;WT+|FFM zKyuWV#>T?xYDQF@ea<7>koa!wv1)87yuKnqCpEG$``Wc@HL*vS5nIv7-ZnNip85Ii znWl%8kd`y8nkY(oVNR5K7RP#a1f=Q zsb?h6r158v%iNR*Nm(e+a)yQ+R*mTgvB81C!E{1GLK3c1jMDQ{Tk+Ib+x5+;KWjE@ zxWTXW^wG(e=c+${euu@f>G{MRe&FJ2^7^Yeq{0V`Yi``X&y0*ry2sG4uo#Va)#qtx zZ_$X_bbMH&pJTTeSc>FOHFS{@w+!{%-Q6jv{7mta0=lZG-3k*-g8EapSp%8Zths`0 zMN7SM=%J;6$$3a6$*R@j*(v5*D@* z`D8JQ8EH4s4DPe*xLMr(A$79sJ}+P_g|6@>Hig+n>_J&YhQ#)+h;NVOyoU-Felq}0 zqd!%|0X=PQehXb#OUK6`v!={sW!4;|r^k3AqwFK}=Vo+cR0Y)s{s_0+dkS$tT(Kq$Fw10*(P^miAQ2Lp{Yc_li z1(Y(i7bb`0Uoq0YG_6uVAF-TS@R`$%XV1*+18foefC4lW3fjHo8TL(^Ua)Hzd`mT% z$TP@wIy&sZ%*>q8>FD54oo#EpX}^OKpNBzkRkus_9~}B=!yg8z=Ar;EFa5G*%cwg8 z9JfcWK6tf%4<>j5WvP%9kkNQ>n?BqHQDc3hx-ZZ;TYL(nOOPMZR zy3_(Jcp}G+n*eJ0he1K>2&;gHKNWjc;IXjz_2uzGyri)Cr)}UD68vTX)NnA~ATi@i zMYXp}+jY=KM@Jt|H&ZKOV4;L;3l`|oj`XQy&E07}azX(!jt3p3+P zAo|yN%nf_1mw%2s`VIxt7NIWTK6gUxqLWd%3R*}b);pyk0_~volr^?BqoN|al!WcQ zPz}81H(1hNoSHn^Rkf(veR&TK_!;z*J_VmYeHxk>w+NSX*>L8}8R5NqOMv4@7EbQ_ z@ZtS?IwZ2(`eD;fw0J%XO+XiD6_GujsBcC9FZ>?#IPv=OZls^x_1uGZ90z|$=DE#o zK#W$s+$WudjEn;H?%qzlTBmKJqoZ;te68*6q#27gF7c;cOuFAyjwOX*4HMJ#wcN@r z)sN-OsvliZj~p9w2x$G_P>?)3 zI!rQbQ}wb|%hH2VCrdPeuRXK_iA>6f&orQOc7iRIv@?bg{9-8d!K)9ZvPZ} zgpzXKTa!MgrP@*S+h_kt$yA^m+M(gxqvs}Dk*st|qjWk&XYT#_(*xKg%XFC&8j@c| z(SMVgN<~j+XRJswTIu(`tfHEmNtG_WA9Qx*E2jZ|>ZVbVkNGnJ9v%gHuJ?ErQq9U2WMXdAu z*)w3@YaeyQ*>PuoUDADu(|zIh#n^%A*RFEbhKg3M%LWJY^z#ok^`P`@P~goGE35=w9e>BC7+sk3#LVS&5fYWDq{e0FyPZY z$$%mhX{-3HJf?=YK&!2!SazA*EPmxN)w&e8U@@Y*udQ|a{Uz^Wr}XHrOb#};Bc0OI z)60No7Y3QBqNwSitEk{CSt=$aWiY(3aj*61La${6Vt(qLUc6$%=}%7%XI~!YN5Z?Y zY#mQRa#>-a?C=7b?GE$sC_T`Tz+QOZ8>HdDJ9nrDDoa}09fjKqzB;1QrevTeR6jfU za#LNP;oyDb;TW*-;y%khG>y98gPq8C6{k-B$aA)Fa;kj!s_tXIo501%U#I%P)LH2o zoGN?i^26E1WfAr5+qdKBCo*Q)_t|iQ28-R33`j(JKVUvmqGJ1GtX+TE^9iL~FY#x8 z2d?CJihj-x(~1ukId1G3!`!b6?2k0{y=%(4P?KYCN;YoEk|o`>iR;iStU+D`qtw<` zblb+k!7D!A^xmL{V|q@>9S??&{j=2rOk=EK9(&-d~-lIoa3Gly#$z^f+3(tss(zvs@*AL)y*BeQBLbWDE%;txO+D` zU9!El?#GWGLz9zBhtjyj+6#JmdVE}~qNH^i8}KniF8b`(u_LBMHeaIOxx1V zpxxWiFEC1P zuuF^oicAtsjvc$5V!dlALM9Z&Pw=U-H9*91cKk`_C z;JuTRcj~5_(SWi6^+ti#uNd)`J1RAq6WnCc>dnA0R3=nFO-;RfmXHIsGPJ$MR%z;B`$S9Q^w$E!HVw%658#=tFGAOvU1wg;#{*gGP*yK z9`AhRvqi!B1X(^A80Gi4VA6m5nZIk|8c=w#s~6|&j8e1yTQ>7||0j&3L%ptFH!q*; zY3v;t&mO&+_tvm!Tplf;ulw(rx1T?+1CWwK&-?N_3kUEA_MjCE3>t|msi~>OYc4Dv z_*{J1%EWSLwlB3Hoqy}#CRz#*SX=DXJ;76K=rC^h`K?kpdX!02qdZb_qbz+(+r`<=nknD~VO(#i#gV9F~X?!dL^uXPob-y1=UVI@9v&T6C}c z&fC^5Ys|E|4S>>)$W&5Q)t;d_CfFVj!P>B^LBU#K@Z32RI_LSVy5~}j_4PM_AVz<+ z1lWz_eDopD)6+FwHv?z40k5*9s+|msx}DS0uX%iMvg!<=O?Ok45Y7R9`UZ-?dIu%l zkLXGEcAaucdE4t=o}!)121fom@@OlLCR!PK!cw-hi}Np>5zX!U@U^^cyqve>G;etn;-AG*E@MZQDi)#{<>zD2}$aw)N7>O-#jlk%FHa zm#p@>al;60EX(fm!S!d+1toRgIsVGQd}?Oq=f{M3phxH54f>2g<#(34y1Mzlv-!FJd4cq4DAK{KU=}oa=z(fB-N<)vK75b$w-ZSm zvd^A98`*g;yxQr(dgoV}q8mDxpnK+B8m1lm@wzN%rnffn8VWQQgo0GV0tHBFBTs&W zLv&D|YX&iGETyNXSF|AQI`t@1L$P&C`hcVok<5`I#l27?5-+;5V>^0U0<5y<1`8GSWnD|+(p&SqQ20OtL`3M+D@qqsF-v^ zX`;$R%l&D8@uDJxAC&Nrfl$;Lvv1iVklXyh_kzz}L{iX%yPa&)Y)HRUDoA*W0=J3H zE3Bzevyz&ACOk6dtR)rm?OPUcE4W$wH4EhH;Q&DzmUJ5Y;kRh%>J~IhQR5}Gt2XZ8 zjnk~CB^8#(fdob2Cu9nZN8d4&PT~UId6Kp23Gs>)h z-|5dz!7uhqhlhtVGBQ#E0!dZ*H+J8dUDL37^=d$d<6`qkTSJs>XD6(vss6zgOP4Ms z;ei@f6*RL;HAzr9YK3~;Q>`O%bW0$qH*=6=r z{Ty1V`^W3tV(!Dca#hVo-JfbEt_AJ!>`TLaVr2jlt+1UY)ZUb08F1oh5XWGip$qT= z#E)xL&gb=2~&*JLO zSa3c33Iq29bXJ3OGR`bL7Ae8HagXI2Y(bh?P4te@0tjwE-H^_%7=65PAx(~U$oBF{ zi1a8WXOInag<7e(=EmpHjoZ_^;Blx3+PC|bpI_12YfEXUbv!CE_~@b)g98J8&HYE7m&J^Z9(d>;~Bu z^aUWhym#-svSr~+I$(P9+)cTosWEf=V|&<`n3zI(f*Tw2qDbLD5o<%u12&xRAG}pg zkpJhz7ccLA=Gd{5UY7M4)N|0ep30FR7}|hCDNh`)pQJ5{|JxSrSVo$o!EY!+bQFzDc5d!% zfI74CzQIAEZnzrHLkk{N?<)W81>pSNI}4HS|exwblFy?JbsU( zrGtyWD=jUJ;+&u96AE#~2Vf+1lJZJUPQC3V9!pn1LB>Iil#3=f>JO$}{cBI^u6)aJL|C~fmZDLTd zM2xTgks}0dYP=V{`0JY`M?mHz3&JKE z90;vvbwdO5iVZs$vBKv^i;IeGi5M62wq2f|%Td1!#M?3PDi>0e{BLw!+R7^s6_BB^ zji5FfU)k#!{L=f&rj`nX|MBSMyMQKK3iUzQqJ9U^N;FUs;YUG;tN^M2DV^v3n{9r6 z-X{@>%TFy{mEF%#P%m>OO7Krrxym&Ur+pkIetM??vV0(nN{?f#}L20-p&JgFu@Z7V&O&986b zr0Vb4v*)I-ujWI$JwhMvTNBL$K>THSGk(VxXtSoK26PzBBj?edh>jAR5=K_mTR7qb zio%(9hWggp72u5-^;s63(*>4nAlTXr^x^*dr1L z(#+u8x(c)qgFZCdWl|6&yBe8~zyXw#w@?ZK@7}!%v`#d!vX|Kzok-wNCNWt$09q*9A>_T{6z|IvH(;s zAwtjG@84>VWfv2L_TIgF#0W;f{~%G*k3h ziRJ{8+Avrj{cIZ=a864o%_Qwrs1ubF!{#mlYS}ht!lm5U{;}Z*+qNB=18yMJC;Z+S zYis}a?+;11&8){8(~>v_=aFYlPEO-~`x){3%njdRRXGEsP6#xBJa4#=Vo*ALPM#vN z9nSa}_&?5@8h)^HZfVN>$AUvUu$EYG#ug7xPBMd4XF^!|1d2pS?bx}q2)0G}6DLBI zr+r12m`pr>`I01$br*i*)v(;&Bgzeq7hZzkUhTaaS6yqL?g>Ccdv@0O^F(%M03$Pk6F_7%Kg z<8)NbQ!Rnm_5_W1|56eB(9=z+d#2&KLHtI8A@*rs7<(Kazy0AJD}IpR(3KC_*}W)!F|wuwky@tRZ}<={9@r#v2^46jyssEFJS zy-bz%@!baor%@bgUO8H14EOZ)-RAI^Q;RLn+M`j3YWZ@U_2THyqapjzmqsf|d(69L z8h=Q7u(ktkzN!l6?Ou!^tdHw~9x^~^9JF1vFvhm)Od%c7y!@(i!ou_-t7t=-?d*5f z9twM~*3Lz6dEjWn@R5+Ps;sk57Zn$`0q_D9D5|KeM5$HFyJWlXz=0UlK{mzd<^m56bk1sxJb_B5 zEiG5t+uH~8Yb{4Y#<5npCt~u<0_-HbYU*Lf_(P25=cbj899cAEOZV*d?c3^+NR(W= zckeE9qZJS1JNVt!R6$BZme}q7<9^oX3kV2csEm?kaB*=N8*DbTl7z41ONIrrudgpr z$oyuH96LrjHRN%|b?X-49S{{2O#wk1#dfHmMNlRP@rg22o1}A0WWP0feSjUX45^=H z@*CerJiy5%PR2JSfsfCfJEt3R0Fn);VS3VP1FVBlqbM#qBGW z*@>ORdIZxT44?$75VH*YIjK%N&8Ee_yfi%qQ(t1TJvIRisyt3qUFGKJqrOnDNx7~D zp9uJRWWfE2Iv>)MEyCt~QPJZQX+{Z#1yY~_6Zd!+6h(z}RYkpf`<4do(@WDM+(Zq% zTLQPh0I$ikW1K_JC$lfl&%{}%mh~kZe@u(w8|&yA<-e`M_tk^7yNJgI#f*4t(4_|R zX)HklP3$^x?CDfcB2h zeFArWPtJXgKV~~W=S+Np#Q3cdw~}qI^)n~8{;Fa3>2T|B*;i0*%_0oH-~7&}5%&*! zj2yK$U3Z;iOn=@5q^kgiDtcW<;wL9HjWL9 zTy)ky@@^4-7sTqs#6)65hlLE#zDOl1NJvXH_4>mPRqM8I-wvd&VHNo;^0pm{Pr;#_ zY1gvbeb@9aU*1nv7o99fS7Jzjd+#7ReInkDOV!r=+$0Ygt=d2I+Qf$B7UM_I{=CO9@eAB#GlHw#ylCvlu-1L{vbwr6Q91Z1apC5UddG6r9VXV z6Q6&F7xF0)5fS|UA^69ModTiGJ1t37g~c%ill3dXkXHZ{EE5_fZ%A$D@@0 z88yHOe}w+?OQzLET3XtvYX_GA48oSN8C>hz8*VebSSlmfYlxTqpto$Zc;xNoy+4dy zrT!W^)#0K1-$pU2`MN1+K>zcryXpPE>w(hq_v`=k^u+(CM?3!iH}}7XiNN15$>;fJ zmZJr*fPoy(P#P;KB=c5ZFjpA{y7!RcX0O4 zHuiY3|2v$Ne+=~|bid=%whLTzP~NVA@M#6#0uEIU8Ox@tlKq6P?uOG(@UmmcXW}vY z6n)_XKCc?2ysN9P{awJD`NF6bp%|}dOa~8Z{NWDQC5&&7_W(rPKDtB^zuqkNXVz(^ z5{AQfB_&JY#=soPj~^hTHa~y<{CL;`1i^g{-Hb3>h&c71K=lA!+k$TrGc8{3Gz?h8 z>5U}igpG(VvC1LUF^UYa>9BPZX%Cv@rO}_He}Y@; zEEz`m@PW4kyZh~Apfor4V$;fSkXsCo;^UVpDk_Ru)T^YqZ~Z%|Rg-0cwg?NvPZkGB z+oF6DBTA$}12d@z!2w`a4c=nl6pM@~BB{b00jtkJI7$%%i{MJre5%RUB!Y5b45uw7 zlhB8A6PKt~k~SAV|8mecirC@c5%lvk#QRP?0^sScPYzR#__uJwwFZ)7YePx?e+BAJ za3aKR0&wVxUq(QB0%dJkK;m!d?7WV~hsQzb6uL8oJMR3sFyc&>-kIC_@jYIaeNf1r$_B zS22K5QWvbZ-A9W49(^`6ulFJyE$N`);jX>su`n-* zp@r+{ksqO>#dOhff_Y;UR*6}EW1)y?>09na2A~tQ;1%G<9hC6EoT{lJNh~_)s3{C_ z6iuLCX@yYy{KX4d5c#K#jR`D;cdre`Hxl(2Yg&Z9HY0}pg%2x5*m%$n#PR^}NfM8x zl@)0sNr#O1X?gV!M-^XV6@<|QYfM97$F&X@KmtiIEbu4BMZ3;Yke1g~c;u9n=wV8; zfn^u0S}Ts~Ut2HQ0$9&T=T99^RObc@NnGDJhS`pN8=!h&e;-5Bg25MjEF1V_IUH|9 zZ0whKaIe^-#T* zj2Sfss6_C5^X}hP9=7+GNRSB*TusWU1DGpoL1#e%03=UP=P2a(F>d33vKb8Br_Y{+ zhX8a(f>zQ7io}=C>Sch>@6go4El9a;Vif@&NkmZMZ}nLC?Ttf`Sj)`85dcio3hgKX z)V=a$*j0&g0xun}qw<}-E5*z|(J>3^dy)`_=}mry^atVLgnc1kFYS9_AuS-W*^fMN zIG95wLc+s+35pYER!vPU+i8Fe3(gg&B#Z-}ZuwO`))L);E#4z&gOl6?kS0;_ zc9PjY{3pp!$~}bSmyx0HhSD$;vvRYVLHAj5HcN%mHar7Z2o!}cFJYz#Fs@oU% z5D4_{LHa1dXxH6NdpYcm{5leZS_dwVB~W)djX^K?>_U*Nr4A9vtiB0@AT%;ax&l_1 zPYjB#%&=%UzeC#SJw1&Kn3t30xoG+*&dNwh0a%R>0lIr>@@qXg1}&3fguefwe)LXw z_zskW#T4403d<5#G#T=n5qBBRnbzN*eh>v{w=5!s3tyUyIPZaH!>)i6!5a?xG&$OT zC+jHe%Y0TIzh^Sgx-e4jpBxT)p=AonijEqFpuU{TEBn%LG_P)Y0>ugY?H0_D)ZTlE z=|bu~QY}-{k9;?QZEJ6#zt{k70y8-nX$mRTO4h}DL{OJ&hd=a}Y!~T9$B(0u(FrSy z07VZ<5gq2EJyi_SbB``1rx?6`m^_8;$fx_mj&^U{W%)(ePn7$E>0!(_y#m6d&rz8^uBAzUZuLHKvH`NB??gL9xLG}C^wziMKZ z5{Vs>reDd?zl%D?{i&|52x%%Llbf5HR2U+PFImariEhOh&&80ehF%FQ7dFFf@)FJ? zbTGWipt+)W@43^jSb>)%8I#6!E+yjO!{ro<Q-O^FwyD8 zp%!D8Lb#S%AiIQ4*Hl$8z!;gEE08~Gs^8X%5)40MWVEZnCKm;a#j=@Ry_2QrF!+NV zE$8{sm9WMj>~+Ev5R$w941oVH5Tr5+)=YDFYkuBV&;28+D=EgcYokB$y@fH#+~@r< z1k6EzikfAl6GJ#`fitY;#n~fZoeIByXGb0%8Xi7~vWGqOBo6-GjWkXWrs1w4tnq`* z?RaHac6R(kk*8!(<#yL60q{WdYK1}I!LO*l5Ul%13^b~AS2+db2slh>ke92oE*Ngw zXCuQ$Dh6hS-viho`GohlgSAU#5ni)t)5?~1$`iiS4&Gk`wc>r;QH}+W<00{GKJOc` zwxeTVa%e~fiu`ear*9x((5e&fd}^wY7b6;bI+TPbnue+Yl3StK`kn0 zF@ia0{$dVOQa=~=?%mrEDrBr%yvwfl^1La$jv~M{i=c*kAwy!eJUMjn+jsBeh6f>F zVXv+c+XZj|n4D`cO~WM+quC6957^NQ^55Z-c%GHzjekI@g26)jnT&&j7$#8~LwDUi z>F)bLa(m;Gf5C(mKDe-Buck_<@Shn4h+h~%Oi)^YXXE+BK?6z#h9yibxCQSxxwg2t z_>R&xH0OoL>@+ex2c{8)6a^it($*5v_jPzQ{QXtqq<@utY}y29zP{u!G9DZ%*iAJD zWvZ6p&WD~f)8LaXhh$}GsevXlazYjUICsV(_%svlv@MJj(>U&mJG&!@9Sn(&Dt27u zINJk?f>-tG#-ECWu^1y5!k30l%e%af zk63GG-`}`v_ilN7Gj3pwzsRug<_2%nW)(smrh&Sqx&4YM>ythL)onAe0J*E`39DJzx z!4(V0c!=*+u~v4h4VJZHOl^1oSBSFx`t3s=e(PZOBTkku@W+zSN~(sX{+{%k`I;!3 zT5|<(p!zJ76$Dw#NUf&O3u)vM>oI)ZUW~Ly?(O)fr3&Q(SFL7w?@b3jD?}HNP4{7H5*Uj!M45;X zgn;Bb210p#-?4cZAo=}h0DwYwomKX4r=#GyBeDfW-Y@1l1Y(;}V#t_%-G4s)Ap{~Z zmUsq8f7xsP=};g~)BqT)?RRdu(hCSWHAbR=ORqnDPXlYw}7pW)BR& zj4UiS(U}%Cv(}{=a-hQaI3OlcCeEttfDLbDVjSZc2?JEE+=fo}$hpCC`=F83UG zfnnERBqN9*5z|aQuKfq(qbn{+*2^LwmR%pmtS%5G6-T9b42@GOY-#cUav_<0y}fN8 z0=JSL3>Mo(aHkPt>n98c_zbs~#Gny8-?w|uo*SsMWX2W%w+(Scd^_rq;M~c)cAxPB zwBktK!frDTd5@D&WSG>yHP~YBAtdTacO2O z(P&}`h0bLU2Kt|ceh);hw%E3&_op(#B|5bjtv&GxQg8ktxROWEQb6HVIB|l7dQ(}s z8hGIv8D5;rvi@F(^Ku$GVw!Quxsx77Mq4l$N>BZ(5Zvv;3<}vR>Jb`W6q1G$&B)s% z!QyTcqG&?MA)FZLIxszn3IE6N{6L~p^*2^MeE2QV6fglua8pgq8uG%4iF}yay8$Kk zOkIs7f)f1}Axg|wKtA|gCOFVY zt^bn)feU~r9yIR95m<`B4_JV3A;sJJ`VDx4m{d^znEPKGOmgE)M{^DL+GRIZvPO$- zEQInvM)W8dY_irpAG$|oiqNDA(Ho4+eaR5N;cI}Y14!IJHOxMpJ2&DU8jM_{v8AQ( zgsT`GK6?^Fdo}mH{w3l6N&8?Q4ZBSu$dK><{NkPcKYKUO|N8{;b%TRThqwh!zSy>Q z?b_pUDi5x%{^vvS>R~YC%=z=gi^Al(06`Ro7DElkxQob7s%9=;TK6cjrE{-c8!5>=q1=i>C5dN#XlMjfR= zY1`d@3DoYSl8MDLKhpgE+(6acG*aiJOuu}2Ax5FZ;`Un9_rG*d1}-N&C>Td{5TkA$ zhYWYYQvhms27);oP?W?j{Lnv} z`}(z8{-7Rz2$;7`un? z=d!D@fvMgD+#plQOvvIFgD@_EkdxEVVdhp2_Qcr)d|JM3L-bXcAZ`Fdk;@m*XYrnV zVRF#>pS7Qjej<$`Ce+ihU`I~+qV}{!Mtj;fZu{GW;d7L z13HGEcnCx+)_v?(PfEY6JJFw+ybn6aDNp@L!?93mwAHUvRw%zMb%lTZ6wq~JZ&l0lA z(HBZ|n)%LvHu#d~{PNQtjDV7Ajd}R#OhYcOwgvqcSGr?-EhJw2^VXCOur{&ma!=hd z6ZxM#9@e_3j9Y{b!fQ)7G8lhQGEo6YyMv+m5}%~|!i;V~fwZig z9FeKv07l)80e1MB(g!O}w%crGu>%7G1N=fidGyJL#m612*JF2J~b}$x4KYCYFN-pamrR$$-st$UC+`-pa zo%2WsNtsu{7t8U7@W=LG)C5%fiB}GMpo7v=M)4A;_PNCln1FCno+{1&RY;};Fe`QZ zkNBV{fV(b;JqxWRv;pnv*1xCRd<@6ugD>!eXFv=O??sGwc>#VUSndTh#y|rg7*X?s zL9tNLTmSipN6ARJn;Wqlkg*RK&|q3C0e~eu2eI~$@rFES`^>&y4XGTM;?T-`DJS~_ zoP!upu^l8#(HvOpHJjE)aU^xpmqp_Q?~@ZQJOu zsb5~Z?1iSk)YP==>T1wDhL0ZLI) zs;?ED-&Its!dyo?D3^V>W`sa35Yng7X)k8q@;7MQ$n7T8>E;?CtCvxvXF}4ErN4p+ zRN9;V>9F#xl%Kr{Oa~&erKe{Zs7=BW-~ukbZpOdS%FEyuBFg7+LwPYO9X@;o)Nmt5 zN?j)|*~qDsGhDK0(IV=@#ExZ@O<(=aZ@9b!7a|ajIycwFzn5#Er@Q+R>=N+KQpCp( z^!l9S+Wh6>InpYssy8AdH;;|gE{MHzQJuNl*)bRV(S!hFAVLQR&62~T;qgMJR4dg` zs9`>WkL&5q+1HVihMg+Q;ISv;y~CJ7B$q3}Nm~fxl1*#jVx)ND0mJ-9JDXRO8t%gh znZKtTOKv6re7g>?QTXKxi@eYJE8tN9WbT=tB9>^#kI)hqgG>t+GTaQmn9h!}Kqf9a zo@0?K;Sz(fyaf`QPdd8T7;yjOL*i}XX*XaeYlVS~e;l%z3@*@uii1X>2pAJ4sYQ6; zzJ2>nW8=@8_A$zy#tj9;Gl|{;Wv^W(6=3oZ22+c2(IuDu=KF_WJIi!C)SKG)ENBn!pgv1=O^Vh z(*kaWf1KQ|f~ia}MQ`v^H5nE@dMEMaNO`BAk&HC>Fmeg5>rj*a;ygX8DB?TSNx<0meu8W`sWLh-+NMEj@ty*C0-A zg(L$yfp{m0{}da9D>F7Y+-%Mx$|d|LEC-x+P>+s9Nh$Y`m&L^kf0{+o%0b3>gQ7%k zeT;Q@pmKuVyge;hB+*?V-8;ty{Oq6qF5;4aS5P;X@M#KXn=eKFc1<&`+yQ z!P})8CFKWSk1_flH2yxy5%~P@6mea>c8#eCe2)*z-<&YtVE$CIRGRFn6+Uti76{k5 z_~G=NWt0As4mvBb1f)eF8tZ4*usv3Vh)r84j*tf4p=E`zDhq>M4deo%kizCkE{=ee zWeHTW9s1df=nr`HGW$*Uvz7XTVI?=7p(7W6I9~vU3%3K|&X8--ugTaPxkZ6scSP`c zErKJ(ith*cuWjDXMTZL#h~5hEUItgf5szH(2njWh`RVvinHVZ;N0Yqo@)$41AH4A> zfmlVfH*ll@0cuZ1ymon;P~BT?$@&VSe@)-vW*n zxhf6JE`_=IgMcAgy?K!5c#b~AEqQcBnK<~5ks5#)$yTgyYKKY;lcU2*0tql94Fe9E zpGfmL%%o1DZS{s4VT}9)72^%)qMEp)8_DPf457w35GGncqG;qC11on8bORwUa8VwV z1oA87{uTJYqnf?to`m4@pKf_`1x9HB;7Ae7;UbLXJEi(GU~+$hPosugBw`1!n6U5uQbgw;(2YXEup32*!0VCp_qI0eY9BgD%){Ig;sAy<&x$m9gD!SP^mB8hPl z7wNYgdwJaCL#}4S0yCnE^_d|T8EsFky8qL~*^t1Yn)`g*-0~pVTSn-n@uoNIIKhef&kL*&lCS4_bl0rvZ1S>ItC--aWkQ{6LlA!z^Z*`$ zi0~{OSH#^S0SPvNo@xUfs9VSF@k81T+1q{6YGsUqUun7@xvPsP(fplY19%mDS-7Cd z^P<{tLkxQle&Id-5d_p4P*=n^ny3Q)1M?nMRy+4te&Iwe;761A9+~kyI3Yd|Aql|+ zNba11+y-VOya3&4VWp&t&DBBpd=dUkqnvmq@&NjO`JG&li_2( z|BjL1BZ>VacYKnS1)W(4u?QW6RvfJ;;VEV8NL*HZ&iCiR0 zF(MNmf0AMTS%FBi^uyzklQ~w#Q~dWYJQ6AaoVJBpc1TfCzzW(c-|9{KJOM8F<5+gU zE9yQ!ZAxt4jIv~g%4bfN9^AJy<%ujx=DZUyCr7F)*m^7mGqOt!3mL2=1{dN*fE}O& zG)_pp@idMe&QuyM!oQFws0fFV{uSmK~B-C2rnn{pauvV=`Um}lWA#%Q! z*8x{V!Dc~}i9A4u?BC$IJ32RBZiR zr&VBg2(*Elf!s?(x*e*etIHd0-6I${e9XUgRfOHIQj`%V5*OMtoM|HniWup`5~D*zT`43gg*dqA#* zL1Blplw{Pza;2kVvBRJT;a=DsS7x#cCTb4i2LeT?JOhf07&jnXl8=aCsA6~kMnyY> zRv+NLqm3&aIK7Tbz|cfo$vC5n!MI-c_V!keA(z}$zjnC@wf!yZy1;MvAF9dHAzL@? z9LOW&Y8m2$2JpU@`~|`%41PYKJjrMf8diR1;sc`08dBJ?K2T!_7mch%YBe@|36;^u zS%+Jq(DU&(p|q@SX%D_*hpN4lA}(-l4+w)TZEbgxjsG@_1LXQG*r|)LFBzrH08wO$ z_OFXg1`<9QMj*atitGxR?*V93ghOiL*LUKvNK$D~UEGIEOs;%FYO%q!+M67eKKQsk zJZn|Yz&${W|A~HF`}ecqv&lnm6T#JLzHGad`3h8c-l10403{$&4jwyptfrWgL;p^i z-@$w*3pB^I?on0cIHBYMOCX@Fpy;8+9#>|7?1haWa?VX|p|v>aKGsaK)`)AEfhMqC*7e6W3l4oVlKf`KB(B{c{MtB9_`sw$zVXr9sta_I^B8ITyaiY16KjE~hql@K@8Tk*UO72C5r72*2YUgh-f;9`i46kx>9>a- zt!Xg?Y);&Hh>*>AQY=$|qB?&CMU4>ZrG^7U&{`|lCp8Q4>?! zH+#;U1IU$V*B5!_s37Q~GBOO@B?MIUE>*arrznTwq+QC-mxS)RCm`9Lh@gBqGH2jy z-m`C%1W)RKT?8rtRK>sks?|Z4DN(&u~cjd zO2ww2b?rijX8nyljL)c^kPI484YBMS+!sU0QyR1p*nIT3dJ>5dPMY8yl-fY;6$nR2 zbdu}fITP%fC9n3v>!U-4lVn9QK=*b2)lD%=nSd(}T){X@3$1VEYQBzGp}q#gps2SNv; znE(aY2jSl1Fn#2n(fPQ^)4x$ec--f(9Gu`E{y0r!Ti8m|y5bybz^AH#)848^8Hp*Amcb`5M zKSae(kYDHkQ2c>RIwhD3;(4HyMuSWMiJ6?QTmh|$swlP7?w0^j@*?TW)Z8E`KXA&I z_&!HSSByhQQmc<40th!#1ZiL*zc8MFxJq?4f;{n<3t^bZu4id%t@z%85DRZxw|)B( zB+>*I0$fh~O`x&*LloUZHrWatl+Q_e~L^6 zE^|rDFyl7dGO7zaB-bEfYG#OK;~*pv$Un3-5AwPJSyMQ%aETYzmim2Zng!JXKyPId zzyMmux6sAH9>)NQ5>?<6l!H%|0-QyNHqNQRi7tVz=cgcLh2k^GG@~m=aNrJ6j}cie zX-t?lkK-@YBMDqXv4%7};q+etRE)NLe{T|EmSXHtf?bM+gd%yZ5tjMzWO3y-zmUiufq6yioKJM+*u=}ME=6b=vn&Q@|>V#@8dfrGhv&h+h1>)u_oV%AkDRx?zpnUTZP)p(y z16Lv;M40DdB~x9%40Jvb7LbT2GG(7kTqc!=Kuy_)C}|;B5U}@@yJJnr&p<|UriaIb zzQkrgV$&F)@G>I@*8fh#ig-V2i zAp+GHJ&wqKIv6E52c|@4KXi9F`llR3UM(sAZi2*F%Z16AGM> zijZ99C|1s={20tvF)sZJB=8XA9znH>kRUPihAi;jzXIi9!T1MswBS?X&}y47RT+o= zGRzcH!V7!_GLwjX)&QUcpSnT6h6byOCh2_ z$T#D&=?Kwc7!3H}7dY4pVCeu9AxCUNG)c0}rE*D~73Jkm(aSl9L9qwng)q?6bq8#< z(7s^oTJrGUzt2DJ3DAcT21KxAk5XdO_wUo~?d>7lAR*g!fXxoMH(>Lk+xsrC$2_?V zABewXBn^Z|4)=0p;dm1E4P~dv)2B}-8Xk05q(K~zeAT_cstE{-41z#NUnDy=K*n+o zgZeSnty|~Y;(O%^qW0v$57^{X0)c%{LyCI$@cqft=W*->#`k;d@L!o(u;^IS+dvl- zzUPfTx&s8WG%p|=lC;<+CiiwQ$gm&*dj=av;|H*%s5JnlCbH#B{?x{@@CCsefY(=^ zS;t07C#oVyhL#&cW?}Gqq{kLOP?YOlRYBf>;T%tqZfLd>U<%=$i-=DM&J>Ip5+0FS zw-fEvK$brvWZDNMq!W+{BAz35K@kG~t;q66kw9Bp2r~jYHj5M1M=OtbeiXF?DxtAk zA+wW3;P8-(bkW^*5R-)hF?9Zzhn0cgfNCR1B}*B%<%8Cf^eu^0p$upOD+k2mDNZ8s z&hUJjfFPhp!s*(!Een;`i60EnEJz_^hm9Q?{xcu@%q& zL|Y67b(rG`QI7y7Qz!_e0X`=uOrm~q5df59Kw5(f*#X;mC$IrHnUbl9h{$soq^)Kp z3m`Bs8$u3Vg$h!qL}J*$c>l|I1LF{ zUW>Rr|7+omO?q+11R~)1VsO?T`uXw~n4Q`aU@FzEsMH6kg~QplVBH|(U$|g_C*T1S^o2mh<a3gqADq2`5=K(+6b`2&*bF0?K90sNEL>+ck!+a|LmD-2}c%{Aq-AGbJV` zs11n8rZb4ajBP{QIrRSoDvSeGxD9}f8s0qpJ5HltU;+B9sc=sNt)Rss7&3F-JQ;L4 z_e>bKCi4Zo#7(Ul&25+^W;^gQP-)g(j!|@q-F>oZ&R=paVD!Rd$zZ=hkA}uwA=-!K zH`CdH=mgL%dKh3ngG?T|?pV8KDp)H-cyhfQ$xYV<2WFK`tMcQWf;!@K>(bo36m8?mZA*|5f69j@S3R z>@y7>if0;ZI$nu()v;#swXSMH2G)C;nAW&(lHg(x>O(#VktgFmznFMm^24F1^Jfsi zL9_66f&T}g@e7a&VjofOq-HU)WD2Nh1rdRuOQ`j`IJMWNLHc`^y1KftwRO@~74W?f z{d&SQG$9a;+$uKAB^Re2QnyoX8ydqVM^;=XqonuZ58Jd6?yFQ_%nlGUT*{s13>3kE zzP@mVGLRSQ7vImiVg(ul0UA(44ZIaUdJ*cyv5gTUp%N+t8!dU#t2YI9=bnJZtTe;N z0>(qPq76CVzYk3Q+pD8V{q0LL;m6M1(+W~^&d;-tN#_ZgykUC5Ii>YYZhZI# zHA!Z#F_xpuE2bvhKc+C_RfAg-a8YBmGS4nVyQpx}GN z(sHn_DZ$>~Z`1AM;(42N7PCe^mhbfcL#vcr?5F#R%TIS5R$Lk@4q$OAYQm_8OU;-B zh-dM6fUN-iKY#vA(nT1dBcJFBHE@uPg0nruG6F~KHx01HJ{qn1XE|Q)vR2mP$bx4x z^2X_aN(Yhg01^27AXXg6M+~!Ep$oa0aDN~cEJQ75nK$|~qET$d;@e7cvf<`&`{VyL zW-?lz(pu=@C2lo9=n2@h=DH!aa)=##P-)P6YykYsTKV0Dg-MaY(b&!He3j7*~MjP+3sB(CU=2A28v_gszylh1fn`Kq!Mlw;< zTAY(}I-)8xs2E=09i&FUjuzJLoAH!4nECzuUB`zmxuP))<<40m)6hupc&FQPfAxex1t|_tSO}o(04qJ?hdN#lS+Vf0ix=C z!~)9BLwE%xn{&|OrUSwt7UcNYfDs|RH#Ri}gd)`RK2mt;2v}I6sTe6TzdE_r6Qd6c zOPEXD>QZF1r{`~A(zomc4&=LP+&&SUoh|jgXPKzA2p;Fff`Wqjx6^P$6o!EH#XBXC zDwYYF6oB#(2MVR0EWb9`Wi*%+B74Ski%XY4M9)x)ib6`h75$h5nv`-9X|>?o znXsQ20{7dZEttVT2v&%!;=cK-rPMqVX&woi5`(uA4ok68{H;aQQHmvsR*jMZy?!lH zlS|0aqmZ1DGywUk-(AqJ#M1@riWM(i)+n`#I=HEqChHb;>xca2<>9fc7T7oA9N6EK zZ6MYqwM!J7(smf>>Wba%?Dd>eaNj=S(G-~%)0Y3T;R6&J6?ksDSo~T3kTJJjq2TQM zFYa{j56=FeB@AA-Lq?cR_t(SuKR55Hy*q2eh`$wgMIdnqGTT2#zrWV#e)HEh_Z3^X zO!+3}j5wI58=Ye`c*X6X@sEKI=6f%&CK4wE%YI?xWa(a&9OL{aaf)n$Dzm+n*1zhO zn}#3^x-i?yOq|i+O@t3bs+=UjiT=lXV78~cfBh!2^(J5Woda)nrv#>T@byK;gGd+|CqexezJ3Pzs0=NfB^dlB0XW z#-ln~5)l@5$>78MXg&XVK7MoJ6haRqq}m1hD6N>_6WVw=(N?>->h%HMNLP=*IU{fF zxi*G9d3=10?S%cF__tg)fBc@TpUWgN|MmOYK~TL@z$e&S;K{{St$Otaa$Uj?*U4OY zf*giA+0fPu5*4s9%JGomUjH$MqV)oBf2ehiwD?fKFvjL(c%sTy!?FYozu*}$v?h?aeU(E{@1s%mox4;RT_?pS*~^G6j|s!5UH88?Q_NU z*Eh14$9St0J=Re?-tmIdcf_^t_uQouS+h)Kl?vOi?mKLNFUT zdZsgI@DfvV0Ob*4HNsv{(g76g3W7P1NR`#Fe;-jJ64-~gVZ1PDg%a;~AQORgD)6HR z90;@1ry6t3XXrv5r1ItR}|52k)Uo-s&BHS0do++H}}Xg~y@ zMs1@3q3{h57TC*c4@{EUKaAXmc=tRbo4s}}RZ}s?vMBvB+(IF60rm_%^P-80?QliW z&b|tD^<5+H=kIcEW#2T&X8b%ro?o+M&eCf|zcvX@!;DV)i)T`MvUeU{nzPrm*57Wq zRKe`^;T-IJ$`huOoE^EYJb%tpmC|)3$3FVq{T7wOzXxbytU5+cy~E8@2Ce6NRs#337q}o=y)$`t%sQ0LIj|M zDKoi!xh=&QbCbuxiArW7Da#R#fSf?LMMYj7f__tpg?0F=1yXYIap-2d6MPz87PG8g zxrQ0jgP6?7Qrzf#=1jBYin~?ETHj}7spk%~^}K)V z&KLi7<>fU6?zYR?KBcWv(AE1A$(H=(>mhEZcF_&LyBeL?i{JkAOuR9Z@#N`!A@`qS ztKLS4`7cfR6x^Pjku!;b!#8=l><9-ZOYB9>5b`>~0<5A<)5n^1iKNKMdG( zCU_lMU7>~WCf)V*qr>c*0*BHKZ&=mdzZkuLGGqCQRpyWU)=htmMAkEZiVMFT_pr?# zX=j-qyK^+5Kf=%+@ZYwnp7?X5PbSvRU&9Ldu>V>Gy5e4l07gBlB zs8i@es1Jr4Z6mB{Y34j4YwxwTw)QiD{H3d3sX6&eX8*Nj1rgAL_(E!JGg&dNC`;`+i5?K)z&bDfd@tHi$xQ5 zc6K4{VBCq3UL3B=y{3s8W`=b`HW`Si)Gu$ounGmw;C6x}P}3#x>P%FnE5$&w>;B@V z?`L{%v&~JXcp)!u*=M<$GncOrIDLGhdgc}%9d_w6hhr;FJyPknGQJHd%-6rZWwn34 z6das65$Ub$fou!%5x)lCcu8KyvE9G+>YZCsnBZ7JMOLJ&p>6YjqYOV4o1q)W_1ElXJ*b8KRYT^ycL7Ezdg7co*%)bu3k0&F{UwGI(II=V#gFW zw-DDwGgk-A4Eb??>wSjj*F)DH((}R(e-3${%j6(|<66re0E&485j7Z{Tii#xWqqBP zFb$R4;ca*uP1qP{_{id%jv#ds13WQ>jMZ=;BrfCA8y=A66mjBjh+LPtkoQ2PirwQ? z?b^4y7C%GOifL5G!U0jq{-xmrMtG3a6(Oqcq>zfn%U4`bx`2)_-VyrsNaYsL8?c^!PzE;1g<+n1Bx<7X0 z*J7UaJW&qSKB5g>V~_k%B36w#I-VWLGVWIU6K+pITPU#SFI`wu0e*ya!7~cW396Je2%eh5609wGbn;-?IA-(F}Q_qrcJ-ea=U0V6~`_NgY zfXmht2nes&0ld~j-and$30u~|!?ShRMXVNz^g+r&DYfGsxdYh&Zw z^pC9(*gErM{6?85siUoR2`BvqT*j;yBx$DnWwKQIr2h0jb^um%NXQ_me(|{8$I~Vz z(e(>w5f)6@mhaqY42lSd91Xkyy#(4fWr3Ol04>60|3VdH@4$rrj*|JHk4jPa0OB_3 z4r7xNBMldG0^{e2GJ&lv1%eG>Jz%;WfuV&%+V3~;5PJX-icYsH^Kuq0H1EyJ+zQg5 zvfZi+85!FM8$|sHD{W|32iTS}5|k6xk)S@TRMt>Z^&Kz4YQe81lQw4_`+hZ{+rrGu z)~jJG_efv5@&mUXE~Ph0<>|^Aj>A!NOLt!@O7itE{d!&Jc*=)`OKx!-@A|o)iBf0t zDrE4P$vRf;)<_BZ(&Q^dX~^WZe;YY#X|KdJ!51CV@bp>w^?GrS%{SRj_DJ~@*p&u9^Kt4?dt^8_ zVQ*TgM6k$==X;WqCXqE9!g6|okt(Kh-G|2CuhZY^ZIcW^=P#qXBhz` z$3;eOSG#-}7=MY7bX={=>gv#Y(|fl-aQlHCst9uq_3&`W9^IYT?j~I(`90ZRrogUn z*2eEYZ#{C(K4mj^d4-0=NPbw9=oOim=!3ka#l^+(T^)6%CA-EyjhGGEzYRavIKo%h z<@WYT*I3Y(0ZEI`_%5?JiD?HF?yYrk8L=HYDv`WO!Z~vs6ZTL!R0PbDy{Xt1&2!Fu z)!Ka@PnOl?vNK8?(Q}%6Mp9u)yi2K{hq>e-hd_}ym0&o$A^SNDhQo^f zVYaBL|Mjc!60b%7^{c1W0)_wjmGS@m&0NG)0$!q#`{L*O`}tt+J$`(*!lhImnzb+md?)65*x}cP)ifjngBA?+|m*UpL^5kP2tCU zH$Cvls&;ohhTUE%acXp#k!uuE`VI&O_;ZdJflq1O@c1F}`r9BzB!XdLT^?PuxGadP z>v4}C!1lnkgZH4V@$kWeJJeV7VgfTp!*-pBt~)oIE_d&Xd&L12_)?|D`OuSsaZ^D; zE@+hhxU;ije9qgjJv&P*McUgek`gWNC#d%tTz;6!`Nbnq*SAzqE&V~Fgk$xz6}~T3 z9N!djeDpY+cB(b4#l~H}R$|`IJ^FU_S?`uDmQ<)6oPJB}>>&fg_RV~`QF8F}6nS-5*TE zNTD$DjxW+RP>}!EOTI{cH0G&D^aaDxuHU&chkXpmf;MoEK2Y~*^FXQsc(b!n6E$V_MT-KH>`~HAg6tts^D0mnl+xzX3Oj7; zU#@P`n4bqGKr|UH#kuo-=_HobiTJhHj>((b(Jvo_VnBgE}zxz|X`%cB~6W#R>n&yk{ zGW*%z2=KQGA&)Qj9>cEukz+oR@XB*hbF$ALiXZO(akJfmj^ApFwZX=`|79-T%5+r+ z1;q-uMPuiN^X{0#^AhS0fRB~SPa!mz%dtxpypLgk#C;Wto8wwfL$X0rGdO?rw<5-j ziy3s7HHS@Lx!%D~R)KQg0{^-Hr1;-pA)^lY5QGQ7JBGjAw4cFC>$7*g zss9=~d~#y{a*VV>>!h<4(v$jh^UFwCu17bD`e)Rne2PxFxwh9f#<@+GZ(Z$Z?U((s zSx%Mn-i)?9nqx2#>n9VTTIUcaED;dE1x`c-Li6q50=xB=2SrXM+p~H^enfot;VZZD z%({&4y_^urQltLk6vXSu<5ba`;ggTeZ-Ymjm|BPKXvCJGsY9q=VkG(Ce$t9W@Su?b zsl_PZG96H%t3gGoV!MS*>QEXuq^*E$+tD`^HYO9JMgv|zB67lg6u(`@$(aJG+{H_m z+98!F@J%PzF%jgo4HuqLwbTj`d%Wyk+s1qYEOj+rkp)T|sB;_NSZogK$v_)mIk+57 zn4VCni?&GRt?}RoLk#!U!3go8_bzO-hKCIgzBdI2C>CG31QL)ShlSOHc%6xKNTjjQ z+sn%mFkf3(BdU^!4rvDhgTIwDaq*k4X1eAHH_7%@D|}x#zPWMLX1t@;$X$DU@=(z@?@EWMZGroyZ2yMq5*3g1Uf?Mgv=uF4LK(}GD;vX#k-_XI|w>)>Y}9#ye((8J<9Xh_#b34B!Zy?KxaPZC^HqX#7$5W#*rP1mbglMA&hP!?#obpB zpJw^pPr~Z$$>FL3>)Ir5U5N>w{E1WXk}Gx`bV}Y?`OGL(J}`QdLCv$FSB@LMU5ef$ zZS!T+Tz8kpjp&$EIru(ZE9;wEe=N(Z$d6oEJ-aGuimZ|i$DUIPIOuQQm3>7;H4I~U zc&u>I@e-H7kceOY9h$Ic6YGFjf&UQXtT1S8W_rVlV&7-Mi1-tsYrzeP~sJOTT zPb|CC8HsHu5Agd9)d?%mw-^Jkx)m0<8j?|(VBZwK4ZUi^#g5&MvT^*wBgD}xdj7l> zM;|&%!3eZSHwKyFtMc-#Meji+Bil^_prD4gv8`cM4;<=CXw1T!*5cT02b!-OAud{7 z9>ncFc*Lg`=NLZz5RQFcXGU2PO8hF0Mpid)Ys4!ByE|5QA|KyfieYhtLLsSoZ3r$P zgAuTs6m^kuJw_GkE>!gI0Ei5X#iUrfy7VGJ8;BSgp*4eWT~yN_vb$|QWtq65(!FM| zj4#>z=CbC_&ffP!!VYiTa!euradokEMQ@#^Tj(a4WPAS{t=gPx@oyq-i=Pd5YI>oZ zx6>?m(W1o~SMr>;9ZJjYNXDij1ufRO-y0Qs3pr2kaa?Raz!S(O%bIib>$622n4MDw7Z;8(?g)QL`en4 zBIS;+n<$s%7VsD!b;c2?lz`EE*ad?ri1z2CeoS4E#?U!1jp)Jgz*)3rlu1^Csib}d zS_}nfUx}VCIrxNKE+7_uH3?YfFnn>PQ?DsP1yDER$o?X0SB0*D6MBA=?HXc>5adFQ z;PtPjE%Znj4=X{Aa>60R46P!35)Tr(Ec?a=K^sYH|EKuM!mIXeoi}>Iwl&o{zQNY| zdD$<2ZA>};fYi^#;icobN+^I=@I}tkiV%Mr7>mMCqDAAT+>;~6lF)1$UVHsSq0+gj z^3ExzEkZUJ>1M>2DQZiA`=$B3UqZL%yNsQFfpWu=()Pp6GYximv~@4jD;XPG+P@r^ z=K1kl(4v5Nm>k920ihSK9lpQKvsexJ@c25Td&c|7b>nDNqsmnU5>fG680nvv{ry9O zG}=`p(e9!b2r5P%`wyf9gn;-MKLdQv^J(`dC}A9si7987qUoM+)zr=o&|fms55`Uh zOv|eo85yuRYeDxyK~OFbo-Op}$o;7x4iyy@nYV}0%6WF$fw={+aT`{7|62S3MfU!=ULOh&0H{Iii(lH<$}el7ij2mO3<9d!a7 zyQPACmUsw$8Ze({@!e(8oPX9!b#jH{z|TC9{F^sFtShdotBV_wp8(5-&#(u9abRzW z=?L^3ViAGZ0KCBLPr2I1bdd;%qoG->`D37Q%cBPmV%twTIVmqxAfi2w6>LrX?cxiD zuH8klHOIk4&%z0S{uQ<@^s9^g%gCZaux_Qn`dfN)J}pu}50YPP;BXsApd2;fA@!`UW2bsw zQ86#SO2gyC`di@~Ev?rbKKy)p>yg9pd$smG6LQ=0|B_v!bysuzs*{k;1F6b{x7EQb znyTXMPHK$U_gri5`ZQgs(M54WsKUXc%S&P!u4eV=)xn-3nCPq+NXk{V)8Tu2k&K`n<8144Jaf0+$0twU7;_JR-$^5W5v@8 zu)pit8YB)uCPBIpob8suCw$yL_vAQ0SjPWxn)giLY`{RAF&FLY;zi)CD{Zg&7+K`8 z=UlfD_+J&++gh;Mj+%a*U(p|n@{Cg7qfwqFe-2TT`s;$KxA*!q9VWK`;F0;x+EkSQ~*1>+iyTq&vFkS|bMwOCaf}A6MGY{$5n|h_R|?Wbi@ z1vDSrp0!Ni&bEYct=qGBp`Q!JeBz6+kqmzQ2ltMBlp9Ng9B^bWkiB8andk4?!@uBg2Li!buQ zsGI=a0V%D}v(Gd4Ri`^KGX-)^^T8ysLp0q629~48QxoJkG>^o!M((f+L-JplcL=uw z?E5wXPb*No1SEF_T+7)rBk)$wm9-B-%WO_cG`{lO|nMS6-CMF zS2pOr)?k~({ti!AJu_jz;>*vUo2F0qF8;Z-`OQsjWD5Sb=*;cyFP5JvrvY(nVwX_p zsDaSG3Q^k<2w`KtQ~`v`1d~F(0_|wch(MqMOltuDp@j*35gy}bZ%Mj|RX>KNmo9?x zSeN3R5uzt$fc;g$g7((v*;{qKzwe*&#(0Y*?%NueLT(q*542kE193kaCG=Dzr8nz) zT3#g}XxN9A12wb<;N5!oWL*>mwD<1yMBPK#6eli*0y`YGtC)u5+}j!jjI<3-Nw>f~ z4~3~r#0cIspCMF$2z3KLzZ{0>9D-28b~IjOp+5t0yTA*{wz97UhG)L8A>-18M4~3p z*%pw8ml>XUT2+9=0c96m6c>~R1{Ubl;^Fq8;UyX2Hhqas6L!P~`F3%iIq z2%R(0N_`g;Zep%jqjw*{6LxnDq9PkisdqSj{2i=n-_J)~rHGe6uqcNwq8Q9OV*E7J zQ7XBJ2TKv_MpzU~0l=FxkOI~g*Br{co-jPssq*yV_=FL6VFzX7N8fa_TvI4vyXe!R z^do<+O|nZ_XkT54z2ncu%Sq#oOMZOJm%VtIJ@~R>=}4+>Uf>Jm(F5G`#+1D0b7};| zmz7q((MmWNQ69OWrmaPLa`o;zANJd{e$2XcRAoZz)9$&d0}-1$bYIWiIBxeacfeK0 z$Zczp$lIq+_o0^~H-D^OPXg(7+pvy!NkaprqG7vFr^2rDfpQcAgcr9Z?28?t23}5c zb93;|t~JgY&DJYCA9Ad9o5Ig!z``&0oK+g)UfMYx(|y zvHqK=;A&L=Lckfu75eKoB4I-Uau9UIgLZh`O<5O3hj@@*Y(TaakeV4tlT%^nZSY;g zfGt`e!wpe@HU!GB-ErzyQZR~*&<1?DG@S{!c&&J)6>~YoM5N`YlgzohXqkq72NatT zhPOjERvXL#Pzn-|XuX1HFQ^(SC%ndkMujLlf~icfbU_f;^7CgZ=k8NbdQcXG4!b}w z)tLaV$blF$zAkc-PiGTqW16T-YUaBRIbK{8;V6!EWW0Ly)@2UxZ|jc_?i1w(8|NUZ zNTH=tR z8BZ(B<7~s-iv`wI3S$pE$EO%o-jCr6xgKz5QJhag>uk%q!N1ew6?+aTYhq6>-8`dw zI(Na1=&s7m-pOB1_GO;xld)^KXW$!Ye;uzEKzUi>w*|xiZ{u#ZRv`o95Cz?IAj4%){K{dhXnUC7lV)wVbcRUKAJ!DZ55vRLD<6(#8LZ;C4QE%rC$Qtkc^h* z`aG8$+T7?m16xkxsDFWn`ZGk2>ME1P!5{h;D(9N$toyi0p!|-!L<~DO_r&{{zpifY z-aTP9c&lv1V4jEYwb49}Di>Wd$^PZGe9(abI&sZn#qo=M;VZ0hyQMdGrWF+vczsX2f4Re@g9i2OX}3@kEpD`4`L3j<;{ms{wI!nJ6lcRcYV=mduPs2=S1%iaAJBLl0gsNJ zk!17)U#|yKP{g5G4!GVHgQl`@(HI0S{Oi}tAkV2n(<^32Cjs)%9XQJT9Qs#4%R&W) zr~X}}b+qApvXQUdL5p)6*B+2Jfwz|T<^sEh`1xJJe0_UNtrLJi@XK!JQd5^)`32{Rfi^AmJBJKR61-vA@8^Oa`Nt>IgM044w0+(m_oDg^+Ye5%N$nv#dNizMw!^aq%WtgI8+n_9GuYh&?S!}QaiU53t#_AJGRiz*@p!|@XBUnV59X3aj7 z%Vvdrd+L(nD5oG|Z9QQLWY7S0<{gRWSlNlinqcF7kB>~odl`&xCLT}IVa(Z=Elc1r zeQee|GQ3M(CI)P9GYbno^cm@e(s5ip{qrp*)=BTd&%Ig98)N-~yg`tKCjaw$@@lYY z|NXgp9w+GEpZ|UH|1KDI!~gpxDd|H?&k`=KRN#mk1O(ddH`#xD5Cz{14Rl%5POSVy zPyY<3vIS5s-CX^Dw^+#z2TlGjpz$5T;Rxna;o7MG;9X72>Q$iWg#*jJGtl6of@DAq z-rT9VAm@Qkrz<(&TPBj=&eb7KIfEe-If&N#L{=;-IZtM)^ zrC)^?Ev>853B=J&pj;RogYqXk!9C}dhn8CakLa#lBs@+zJOqiB!0%Gp6zSG~W0O(- zUHE!|O`8tlKW~Cv{^L6so_Ak7tzkR-4B!`ePGG3g0A{nDL`#lRWD90R2bwp-puZ2= zQ?W=frGHQ!%f*Nw1k3B?#ZG>^f^Ty|^EUuuetZKk`x+!;DU;b|wIZ__Mpcpg#_Pw- ziWKW1mMltl#1f=+UpbcVxd_=mlF#c&=eQS4U9U$cZCXwem~G%wixX$rPBQ zMnHZZU1*!y8mu_cgmR6i1&J#nj%!bi-ysvY0|I$6%&Nob(`9${{(X#|LTmK)=cm>7wK0^A*}Nc-i@`2k4g(FL^z z=%&0nN()FZc;t~5t3y}_q6gQl2LAggN@q(79D(cs65Am3HOHgUYJMgHv2?kiN)V1afG-%*9t3iFDhl5s&lT?Ahc&{X6 zuN{NkO<~X+MQPl8mLLsjK~bT0XQS&n`B`XcMyf|{3(Y4A=>D%qNhQ{WPk;oWB3*y~ zjU;&1QlUGDf0MwXI%?;`{Oa*C&`_&D=>LK70$Pzzja7kf^`bT-2yU>7NEdK{cswq0 z(*pHlV`Etm8HDm($=zJ$+e=c??f8#?^OByCQ3>h#*AX2@WIOhBPGxJJgO|O-bRLo( zrn+QAL_`=g5iw*;*%Z`wV=>-+flxw8!s6RN4>}1lN+=^H14h3&jEOCp(C(o08&;{k zGQl78+R(%SrpFYmgM@4r1m*sK1DYr~wjkf5qGdtHzCU*ZD~7m+2ruO5>V1KjZD&9H>lv=KVtXSBx=kQXb31yAy_c=^7j~91BGn0%m)eM5+@^!$%uIB zcffZLbXxu3glWh|y&VR%ANQz5A?&m2*9LT3w2S#Yzs%BG8sG006GP={>l;agSe4fi zMw%Vfxpx}@qk%Yg+@wkg^Ww()d;X;l&19O!k`Tg^@uUR6FP=ZPdcQ7M^#bw);D(K1 z*iv9D9@wXcCuMa=<*KFI?B`2lP`CCeCAB_?^yL4fv2cGtU}XSjKhA+-@T9dF3<2y z#TOTG^;uFDh6LxNEu0H^Wq{elLBZUhJ>49zNP#~k#@B2FBoxS_bw_ngp%e370&dpT zVW>F$c=9a)#Rv*s$P-ZNKgu7lS{_hkI_7b}9H8)2r#S{-JBSCS>`6EidSjGi{Rvt< zZOw7hu&lQT2t=vUk!>3F;pL6|d^BtBT;*~@H0RR2;cz?myxAFY{kmm4BpXA?$;h|Y zXC<54GcU+fe{HcuK5Idm^`R85h7M#ik(Dh#r0__rfF~xt$Qy1?#x~rVXPEE$5Lo|e zlz-T!PK~ABox@~#aEi2GZF|vy)&V4l>@cM{sI-|6@=$#NV8re};rHqfyAT1vR`OVZ z%>=m1Jgf;0BxG&yFMgUY->bq1WXpYGzQ6g<9~Rl(Y&CQZ zz%=~$JI&@(ZWg@dSjkq57$JyAMv>+ItnOuCuGzmoM27kIHwNQ>^M(ISu>Sv=RC3(i z-VIO44%!lq;LA^+43I?-`zGre>+tC8g>}TVuQU)E$kr2l3xGfiwQ4=+5YUh96O$+} zz|EFlo?XPq6!e_k1QjVx(=@`lT>xf54zTYb#BCuv?F|(-GApPI+h#;;s6a+$9(}VC zv57JS5fWculA#*#%J%p7W2o+d8qzAwSGbSMU#4fk$-jc&a*cbsX`@0&`0C{&1j{xu{j{xF1 z;z5Gc9gC=prq1NL4yB`*vjh_Y(Kefm3NRVtCG6}_m*88~$O|iIl%c5=3FblY`;S+)Jyb4Bp<`5(=b1A+O+!5o0vNo`s&6xy!fgEfeoi3Ip%qcW=-h_mYpYw>7|sACSE(e&U7=C`a~jB$oKP}~M{yhzwFFD51iT+7>( zRDp%FG<6>HRc5?LFD`mLljpE0lLyK$c0($aRU!@q4*`982`dR-Z6n;#{V%eJcg{EM zRy_~q7V=pX@S@m%K=?})-Rp1*8-R+}RJ>xkP{yi9&~qe0EeK0D>#^>njg1Z5Uaf6Z zjX=@%@EYcIl-Mqv&kLz6DROg&RgJAbAQ?g`6$3g7f%)VHv8(Lc;#jf9Tu20{(~A+Z zE88J$t;G{dK|F1RaNZe#)skh)ED8mgFW!4J{S5yRd{|)Y&W1CERXY%$Y2hePNs-PC z=G20yrsHhIycFYgR2122dWnOBqbfm#TYli6{B;qlKtty?UOO-g(b7?Y9joC*a2-;w zw@15GpJ5$%S&;dj^NKFEGpz};WCCWlv3J;KX^ANEr&=Gx*D2d9?T`3a4gJiOyiuD8?E=~NL zE>{7tfGRjRSc)bp%=deKEcqJch5e%#h#{S;gyK%rAQ) z12`m($i=7w=sr3{1AmjSl@d{XkKfwK3QGb;4vPnq2B~}n@QfrJR%?7jdt*UCq6bcJ zU@X4;-=}~mgQ}~sE?q+?d?lff+?rKuxXt&-Ket!>v|=$@cC3MW4?_fxR~{wWT2%L~ z+atj%Bc?qR92GaS_N{Fz3VKAV0HK(2=!NTKt{;>}cSa48J1s~_u_y4cmM#3v*C=q0 zntXTmzF1`+TbBbumJMdM(2uPU2BVWr=tXTJuDu7a0{Zl9RhKaT_8xBrccW^~F$1^$ zIt#Hgzi#ukv*MzjV*Dl5(I~40G8nRO7h={S z9^a^jajy6&InTk#iOE7@eNcKvN}`Q!+djbmg21+G{u~IQ=bfrj;3a{z+K20I1x~bb zkNab?a}e;vl;kL<1o5UO z?$2o!-W~iI$u6Fg6;_?FAoM3f$#4tOZ8;oqZA_NYM0ibkD3S6|5ff6|$y%KI$oo2o zqmB9v`9mCRLcCCXDMvaRMT~n??N@Fw_M~z-Di(U7EO#P&1B-+T*R<*sJ#BIZP#=he zdK#76aGjOd;(hJOX{BJ-5hWd2rz7gVd4#>c6 zK$kYa0Sfy`3K|$)U9K7Pf&*p>tyCCHgUez9BS7}ZO=CZ|LEC{kZpl|coR4ylFlIu> z@KibqtPb`3cGP7wL5rXzB{nURo?$$%Dri=nJ#(fNh*!2#BJ*3Gl-^yba7Gm_A7eK6 z5PAX$SfYFyx(}HH=Ht)!)4;KVrpdSse(|n{=mjQ*C&Og5)mcOmli$u@C2Ng^r|Svf z&FEo9{U;`Tgglbsd}72*47(!pA_u>wh9lPtK0W}x zw^dz8WD{9!C7yGizj)COW<#RXG`J{r^lSB^U8XHA)UX1er+RM&>lE#MYjIVT)Sak` zhO~upR%`I~fdFTr^-U|y&;X!94VEg?xz9SUuSO5878g-m+IU}tbt_j1tpNBSc=+CJ z?6nxci%AqKN4*zOy*tO=2=rcMUHrGnsI2)JdW<oI0KHfVflbn-#k_M%?Du$;1L!tv{P^cE`%?EK?m94*8nk!pI7EIN^px#LxIreg zLdL#w_o=%LXTNg-w@ifoo5I0^H8@zd`V!2`FA>5LpgIfGbeh-An>Qg`FXG&5Nxml# zfLjiGlS^;w8hNqfpFJP`;{s@82bpc0r;ZErB?B;1hFWQe%7WcabW#Y>nys|ovRuh! z>OUskl*-!W{EA#tMinw9d+!|1lLTH2cpRs~_||#m|K5Aeg}{gL%=hxPx*6Z(HHI6iy& zaVCYfe~U5SzITUu!a+`|8ZjI9{sj0(5XmB;09dgx<1(}+HKC37CV~|%A8tfHiaBTn z|6Gtz23I5I^Jx)N2r<~(cYUNVntQq{6;3itgOzxd9Ds1EDms!xAPi@Kc1QQloa!V0<^9#QGjmu9cs%2RY)4497PG49uw}oyu5Lul29iH@3#_=Vlp{8 ztPMr=P&l#vH(ZBJoHS|DO7taCEgFdB*zZ@HK~$>Rp9bCPY%+F~b>IjQt;G>F1z?Bf zhF_SMu)6X_-{2)+!cs7AxV&Is<_Tz)mk)#{Eyn!a29<$u#@rK4m*|iq=n#Ni2}Ct2 z8iU7LgH$BdFgQe5zYro!y`bB-V!Jv(lbmpU08uB8v2aCkDRi@ELJo_PO0e-kCix50 zy-m!d1tcj=N)`bS11J2G;VE;QK-2g-#9Cvh8V8<6_G4DOWNeN{#*+y410ZPvy5or6 zNtpzkE&5oxG);*N2)}#>fjznCS?v4)Uz|_~OGG8X8W12rV1Z$*W*n8i5J0{9HOgF$ z&*elEgKjq%3h6sy1wrl9a733I(;MK}GLgGcV++u;LpXNST4aZLNgfkp zIh3T*j2Rflr8=x&BsK*h@-PT#z3|+${shB;dP0aG%x*FonFV#yV^dzSQh*)CA^yA= zPDdYcXztXh5MctB5dnpxPe1SiDhpwUzJr_Hj>hWbYSvAtJ2*LMVJ++o{%qs5owf#h ziHj$wwR&IfKxZjbMjAK z+t*fET*I_CL23~zdRct|^+U+@%tfV1 zIB&=DITXQ3PK=%u-4)x_k0X2S=m<(VTA+1Yu40)>AyYVAF}LAN+WZu-g5w8D%XXIr zl&mx;Uaf|)!D%K zxEu|@BfRpnmkQmJIUVG`R(@@b1s<2a`T6_T;;0cnRnrYrW2nPj<*5c)dw&nhm0sD65Q5+yl(f*? z8sCm#0S{1Svs0}rMAhWb=EuLzWAs`Yj?V^haeed$zN7em@!}XL1SHr*cuUOs&8lfU z=2c`p3cY)0_)$XK2bw@Fj8I5;_-M)f6?1v1e;4t-29i+pekP+5L)|TS#W3P{C&aGu z6##s;iR@wC)ERVs+(LQ$79L}btY6Q-K$&`!7a33-Rn(5<%JZS2{D%yzn{m_^7UF2k zEavkq$w%r9m8>g>&;-z02wRMM)<8zO)Bu9HI#>;Lm?$W_X!I?^P9mZJ;3$i777qTT zFz`G^V&LR#Vr6bsII zvor6YqzzYrc?zjKR^a)sL_=F#LMXHP4OOCY$j@dTDV%mo{H!{#JwdRiK=Pwgi1K)? zpbhKkRMF%GAWg3t8VmuST>}jwMKOH_dZ^_qu>N-Xtg})VRJ4C4mMe${_N%BuBLUT1 zEv*#Y>UP%N@sxd3z^(ewp@m~H+J=y3D+Z!#z`BWbI7=ebBcm+WQtL)v68h4-&wG z5gVDOe&`Z{2o7Qx;#~7KR?@e3sR*W*kS+eRVbKw365@o8(nJtZW34BU87LcJOi{uN zUU4M^CD_9o$ew~Up{P$06enIkl`^@_G)h zkm2LOt7Bkc)BwYKt-~@^W?PZ6`hQwXUv?VnmR?{M#-qlfrO_~TjMJTE-;oz|S z&+E`Hw-4IKM2^-mWqsRQ6FllBWW;iT@bKGQtXRvue_&uLq6FtK9_H^QGQtwUqyi&H zJy3yl|0GcV0p5z&aw}sXf^=>@Id1Ke(uW2_RDkheM>p^ANllK}cT;8Dx@TYQQ0w_TedSIwH2qzh068_XU z0F7+mP7&)YE`oD%auPR#?F2YS9Ss1$$X@HP z5SsULIBJz27c-|PaB$ouA_pdwtDT$7rsM|Wqz+*VRCTwG;RQEzSjXhvn%#;jgHNY( z^NWe;Ho6TM_<&4dUulAlB_AIj>c+nnOLWm9Bv4ogYYi$Aby|}I03NV; z%s-HL1|V_t@CL?W@#5<|3fvHUTr>Q0l_lUmI{`$a5&{}Ol}^t2^JJSPfzy9{d=_{# zUXPCUoW=(y3jc#5pZ)0mw0L{>WAifg&j^Hq~{7^j$wGTT3*1DGP%)1ux%XeqPJ7A^TVZ*7UfRsjQ zdpwRM0l@!R*CVu6Z0@ozQ;T(Xk4yT?-O3*MU#qamKp8KEY^}L=xuKT|B z>vavNwZ6mWbAHbAIF9o;oeus$%~uK`MkO_q9MA36-t~kN=^v#BWj~vebLh-18%~dbt{qQ&c3L=addUi;I3hOo|q|4)b5#~FBwDdY{J>nAc^Hh$< zwN=u5(MjQE{|{Bn_)VH;a#)Mo58g=I`3%B=Yz^@^Y2112NYp~S(yJU-JNehKK1%h4 zI5{K7nYVCZwiJ9g(M-XQW<2Fvx#JxK*Q%4F+(yDErszMY*Law+YB$VaK=D0<+ed_a*H4>s@ zH7T;zmSo{5I&tO$7e#SAGDuztsu=kT(;sTml|6`IrR%THIFj;SeQC+nK{!QhNR}#O zt!Vq%_rrWE3*X6;lQI@RiL=?y)Db*jnA7yv)i?s}6FnTs-9|_{K#YnxRdE7VW?4(M zbu%o0(<0Vd6Zf~IcLFy9pX8~IJLQuA7eQNiKU6%Xrl!#ViIJ%qG9X|(Eti7<2kjYf z#-6>kiJ+WB~q#dpcsfK$t*K6# zZY)%(dimBlg|_NE&a(Z%&v`9xOt_Gd@ljKY1^eE~$%(qx@XA1x{tM~&y#*%Xndrq}9=pldPRh>QG|4AzVYP~dI_=%a z0aS!XkYsXUN)_7LTiEecY&0nKK$iXO zGy!8E$0St2V2bl2ACG9lL!JF}gkVwe4^+m_K>a6++frA9D%|e?K$mEwNrqke6GVYc z(#M@7;QuS@VK+9?(FuE8i2fpmzWlKV+o?Z3*q&f2v!Wf>v|KwlK28&{x6HwW$MtnB zyyD!!D>fQx?$5=M4<~eNaxDeuz;LM@6bE2+G@$)gI6wh`Fb~*hdV|NQ;MeJdD-X>qb%g7(NUU}g3*L`x9r8U;9|MA9#A>G)!qi1n|hCqu8 zAj4f;rlaUqQxp=i<=eFzK=^w8BSDS;@S6Fimdu$FXb>&-UZ!8}wOm#?@-!t|G)Ja* zaEnAey|V^cz7tF>@a@s^`ob-;w~PjXbtTO-o_Rw7=r@y~_cK?pVs|1!e+rXH?Y*a_ z!pfF_C7vZBhbR0W9`$b#T^zrZwmiI>ug>J2rUv{;DL(~nI6n9BAMEHDK#fS#mFAjv zH?Iw6lUmG4eLrq%SxF9^4pV+e7uK51hQ8?cgXTA<%HNCAl|lrCGe3btXCjcitofv; zbpfuWshmx^hjE!bQt#&)I#tkj_s_FR$%3s_;)RMb7#l!40k24wq#I2HDXLr0ia)RT z0vxHdLtLLRnjPGe>2M+`@bvT~bqbgMHxN^anVhqUMw;j9qFsaR6GnLkla8b>VDP$D z(FCT*7B@*vgrWQST(5zy?4+nKAYZoSr|pW-#k$e(2*eTjOGN>f==5%bHd~TvOw5_8 zRj+>e$`xzT%uwrs#!4SJJlrLxZexE4G6u5mZ*Va7K(`^dErtAgI4!UMu@%BsY%-Fz13F(~X`;)x7Eq~*$3#sSx>T8vt2w#UTSzOw<>H3_ z9T0XVVWVa{^_oY|#A{}Pp^FW^K4VzcWT0ej;oeRY%_h9d%Vac`xSaL${p5j=*!?V0H6uWV3iVRIl$JIes)-DnK)j}4pem1-M?)Btywa3*Egy}TC-~yY{~e+f z9p1{3fe`7kRJRN9jv4ABR7c3VtZg)P?%|FAO+xjq zU%&piPw~E=s)rREyrgWS$SR}}zJSL4jbRmK;3&}g5X4G|$qxYkZUz`t^f)q#BJoa_PnuO}J?Zs=5=YaTK|l8;ZGm2d69t8ABYi(ai(k zt_S1YTaMP;l*u+Ogi#y4{5S??5u?z_FNRry&6A$&czy@-vgrVHg5 z;U!~+oAfcH;!%Bg_fA<7W{L7BwgNQ2)Jb;rnV#IxiJ|e zHT7s`wgQ>^*R<*yP$eK;&@1+Q7Kb7T!&o0T^%n*ISw_|(CpR{VI;L^J16wLs!Gk^5 zSe$+=3I^glDN!_Z;FfzB>>r?ho+Ppy6gxz>nLh&BFQQ>bJ4ZQrlda!LNxgj&`Cm^a z!;|>Uq!k#4u+X&?{Q_Gq61F%=tX1-wz}z?ATSFf{nT5z3fX259jrKEUJ*9g^L{!^% z>?lTclOG)wrcw~3@G~4Q1UsD-vQ~HwcN52I?rkOypE(UjBYZPgEg4B)w;~!Dwpu#y zFMUx-Nfm^c9iecMmMF`U+UP#WX~=9P60djzGURt3POBw}tX{~xn zQ)~Xi`R&!q-!kLv(K4;oKZ3s=!5`w>P$}$#LPRa92pYz$RCJ(qRO0<}_(%CSSSPWK zOv03Q*dIl6ONxkVyes7m(sJ9A-{An|-2%c7RdKBT>lA53ZG)-$EuN%wW0CR=6IdBm zK*UM;9M015@{?f~H5nz?Dn&m1YMT@SWQuPBq8uC?Y-Xa4L-+Od=?0sB6}_dtm3gY6 z3aN`#qIwV^5kgc7nLm5XxTZTkRFh4ZY+1gi!^d~;Mj`bid@0g+Gf+Y|z)Tn55HDAT zKhky9+rQ*{BKZ)(=d|gIin=!dg_6)D9XV`A*bX-}jN8=6BX6kY5xXAp?$}?|s+H}C z()8|`Gcc#h9;=DsF27b^-Zm@DQaYE=*R-_PY&egrPlD$B+n&-2?%Yg5vmA~KKz|y& zmUoz=f}@DRFE37d4k8Qi0%CW1mD|JaMLoABxv4E|RC z(9;1;YUwjrR80#~jfvF*@Pi@D9;}fA$$3xCEDEdsB za?Z(JNj zE7eUIys3y`P$xz3u3mCF-o_=U_@4U#=#m-aF`{kMFQ_=sOi+ak-|?G0CLLn-M-j>i z+wtm_?n8KGSWkT^=JN`HA8pxd3T1`-$|X;epIEvY_YLVrilwyWkxG&-dK>g5*#s;x zPm~IV?Jt^LL7%?MYl*Mc<23!f+!mk6(M7W+kTvY z^4+_3JD#4GMZzYn^b#z?&f<%MYX;yvatM#>Rv*l$2}|35b@ml^yoI& zq(Mrq{|<1AH$RX_L{R#B?pN`CTZ}+P$K%AAP3onlHLo9vBF+D+`QQIl33Bf7yMZ@S zow}Kgs!+9jhb7$n+_!vHh3w`2pBuFQ?E?brH~ZNR0Kw5O(ERP)zrw>OuN0`e^TN3m z0q{xs!)kLEElRS~BqF+fJ?GA?$Un8rn*8Su`+x5){BKwD|LvC_JH4q2M$`QCCyK@A zKcs(7>>M#eTp3pk0*q4OPeV<*=eVBt2_-rHr@)7~6|UYkk8mn&5~nGWUF?xiiGki; z6laoPvL!n$KkKx5W}AGJ?ZjS_4hno2(5b^VteL1mu zbFllOKN$Hdh^X{43-m?hoG%Ix=xkx9@cJA}K-jo9OZ-D{Ljdc&No`S#g-$0*H9@TT zIcYhjQ9y{SgqO$FpZ-V(`0^JfqKVptNSK9y8h4eEeqTQ_pyZ;UnbdAqv!`j^L(BL} zc`X&uAQ}?yrHbK-4g2&N%Zs}StuIPl;8ud?BuV)0y0fd~X;Us3+#}-buV7{YOS8!L zM6kR0lqQk%?gJqI8`_xmD#BDx1b=QhgycR^)o8WQRR&<+moy+%3 zIahV~=5GB_&bBN3);oA^IM|CcTwLAS(;8t+6UPcIO&Ho7Y~G>~Zx;|*ud@t#P?O2o zER7x%=v@wH-~cQ+D?;6<8Rai`2br`QN}AN>Bc{KdaEHJa(8eN`9C05;`px6iMO(oA z^jE39&4zMLi*`p)e8SmQ>pgQB@u;-rGP=(iu6|!{{iRcw3~N8{K;?3%zJb*ZDoQW< zRLwu6cNdb1%-J=22`RU9`64K#Iw^*5cmMeEdMCJkX`7`s8+T->`UB@Rguc~Pau|z8 zWY6gtt(ed{Niej!bw}gc7JFzyZ5EzbXo>SDwf~;Nt+FwGG)X^HX|mhk-BOGTKgS3R z&W2EAt;BU3R(1TH7DV)0q!+A@wn{+=IC4zPa#yuk+L3ojYOg*$VTbtDjcw*r|U0ma{i)-n{Dr;&Z;55YaXI%kWQ{s zw`O8Rpw0*jW0P+9?J&*sJ=vArC{P7zQDEdO#z@D%&}4S`N%S<)&D;J#UbycY={Udq zcjG;|Wv5n?TSP%+nG>-9iW5KY?x=ny2r0COc5AT~)H~tAXd4$AqYECMIw8p#c*mE| zpIv-31?y8+406zX!C6BfIqz``qv+TX;k#@35yYumPLIemXkgY(eTD{vpjcH)MAvmd z%+o{OC)zL(lCZ94Z_qFtm)2qoWOA|tQF(o))z}rp0YyG%$;?l=a~3UJxRIgMuA%C) zz;fM`Pe#dQy^Cxv&*vw zn;6hVI{u~oJod`^1#^m$VG%`MMhf=y_3It~lrP68pm!H z(g2`TJ@ha46Zd{qBgTetifv2r!2h0uTEw8XE3_42WhgpOxGx?}tn7Pn>Z6_w#SjU3 z-6ixRF5Y>#3gJ6$~N`=6hkV#VPDzVAu^1#-Jp(-M}dJgZqZ^NXJjGFhgfMSl*rwMupk?7 zUSDC9uoj)~9a@~_aKZwSu_*)5cyJEbp+u0XNjbDwso7n@rv%>znUS!H zWkp4T#9eeVYpNmKS1X*NSyt^ZM7zUvuq+) z$`T#Ui7r?PcOTdrR|PPv?}nKLMaiDDH$*;6RCj zb(x88k0c(_jUy#Fc#s@Ujhi%a{YgLp7Ci1xJz89O)%bR*XBv{Tl&^2-r!7Lm=c7!YdPD zWk7(SSG+n3nY%~b)b|wZ$aa!}e=J<#K<$qBe|ffv(RN|rC&7}ODOGdKIqxf zf7Jcg@KAY%IV)U2BzofF*EwVA)oWt9Q?sq_TodSpZxefY@lN8_&^_#j=@gjb zI&n9FmR!k`Mal`E=P}VleT$b|T%)Qy&kXOX^eKIvB#FJ9$do@4Xa(00DWH51uW;*$ zipGb^vvW_!%9dB*LaQy^KPs_fL&Y_d14$JzX42w8(A^tRZCrKJ+*_ja^fy)2ei)Ww)N0cZQeSytbqHIU;T%g#B!Xi|Ms@ap~1dg@S~d|@z#6w!%MD?^2zp8 z(HVm3z*0rD9166go9yYQ{%5E0P3q_))>91hQ~z1FR16OX=9XB*8~~2aHqul{F271Q z*ZK2ZMro3XOM`*)$*F=V2GjGU$Q>v+0)Is17SH)y4Y!zFGdS9wzOC9@GYdh#QUF+h}BQnKET6?vsiY*(z4<) zqCudXu$LY0e%t6SuIVkrmtSq%LB%3DH(2;MBK0lw9QMdOZyu$!LhL`uL+zPj;rdYf zaPu`k97eRX4xd1LB(p_?2?I-T>DElg^gEb)wwQ!d2@}6B;W9G_a;a2AVj?RRg1e`6 z4HFOeHZimma+L$bd|nJ?EUAkZZNt)^z|ymR_d-EtjQi^iQbT+6;@m{q<*X_;8C3a* zvx#bB2Ow}U%(f95G;*|!3Xz$M>ocuydUNI!rJrc-V%_8y z{bp+iOQ9t8NaE*+D6(WC6Y>8IIu<*_4|!hzFf7m1XAex zJl`^1R)?JUh+Qy--6p-OvmbnK<6F{NXHapH|F?%9EARUA;hK*R^>j~l+}UW)Ukj>6 zSjR7^G-_mnkyZO2+}UGK&A$Tv=r}y&p>I*o5Uph^j+mdZ_&iPLwnAHXoBrFS6UW58 z*k(FCHS_KER~cc)UJM(Ldd!f$Ao;2EtP>x~PFHH9d+hhYgZBY#idEE*ejWtHbSqy4bG-L?eP{VUv9 z9EESte?M#16_x%Pg!i!=xG61%GKON;^Nbz)!8ba=_eo2Xw2wFLGVWHvO%bi;-&|GX zV-=#PBTaDj!XNR>8@{qbS>(r2g22hcwrsj4KUu?zwVrdTcq0cs0p4945C2Y7S9eKMCKZCIi;oAVT^w0hzY*o6 z_8qCY1wlgOAob(gLO#+E9Vpd&lF>*JI%dDQ*SPRWW?W>BQu}Q4`AU`}$ zs_jXr)jM)vt0Y-q-qTf|uz5V_)*FjPY-qohFdl_O!lh3OT~G8&B8?mG+!INSc!~2D zt>$!W3wGGt#O0vTP}p}v3gPUd?|pMJLknqSsRf`kM6;l9?b)0cq+1q7n&{(74E*jAb{HU-l@Ml&(mvYIsz==n(qr~p4b)UJG0)Emm(CVvz!P37hHei6$(H!O( zzg2y-OzJw0)kSwUvZcjA`{8%Oy8xzQqYfoJ6R1w^`TWsdDKEtf9S#S@&JZ!1BKRyS z@YJd=11zAJx13KzB3yuX#Jf*zZ~y=ej$m8t!w>l@s$u&xOZ`?o5<5MC287t5Y8;7_ zxAn$jH72CF#z%vF-lH#G;+^saA1;s^$&$$kf9F7{yV-Z+`|qGImqy3ygf=-XID||y zUJMaw9qRJaMQtYqwVLdRK^M$xWM{|2FWF~7y>WDrs)%zqj@!>Cz{uFPHpM?y`AaDm zU0fmvpHUoIDJ>CQb81NJJbc$fZKi`>KRxuxl)JAN3ZaO0#AM|Z+GA>)9p_(xrhoJJ@c#YbH!I4LSQXPU!zL13u2YlaYWCZ+j2+`snyp&O zPQD3FKqFHxDjCDGWtHOI?#%0lyb?)AHtO#qQ!mz*1ByR7kRD~FHdA}5a70SJaK3iC zijUG?xFdWIi63<$>mw^hfn-FFz^=l zzGGqwRk9peT*Y2i_T}xS@l?&9!<&Xme?I3!f$U_Wxs(mO^}B9!5Kv2hCJ$58@oAnl zldBNfthXnp69Ga2i@i&2fbik`ZgV6hIGwF5&yd(LlBn1ED;sQ*5H~>0LcTKr;MUAk zEwjBfT9-&#Bj^An)C`cud-2nQXb;GrYxD?z778DkYjT5wA%S;oWc&Eg`<;2G*{8Bo zB$7(q30x=jbgLAW@fIoZlG;}+h(J({$4 zDasA7shICDHP<vVP`m}tO-*3$I z5lo+RtMkL}l;t;t+-8{EV1X+*;sjY(lX7|aQUwuL} zWtNPIpb=dE!HJH=gq`tu-&rGq9+5K|=07}|$=o3h(+J;6D{!6C@AHO-7|T)9{mr|v zaK9_rGp?~N4d~9`Q^)3g=Ya{rm}S8pz#`+&(vsq0dtOY#RwW$GXkDD|=N+0Fe^*!< ziA^*!h+gG;dO}JttjVey|Do0259jrFxH*+!TVP0?H~uL zkK`VTef}&S{RgFoYpWl=ZRDMzVxrilZR4N3S?t7<|14!b*dI&KS_SzP$%`YDI)ZsP zZT}7q75rd!Qdsk_PWGKRd#@hfQq8KC#3(tc>8=?mt&yk`1*?!wiv>k@uSgEdsTpeQ zJXm}UWplER_9D@d*DiK4AgFpu3CS|gESeHuoUZs4piGl8)S7W-!;<^6d`W;6r)V%> z%uBSKd1S_7{Hh*OxR0+9G5cYcw0}s4_f{OYY{f{~Gazr-1xj&rLa8xYR_d{1`2rTP zcLXIKqtpJ;PQVb4C!X*&RAwE|_j z?x-^7()A70WvA{lpKD(N{69?_@#x(mgvHYLPJW!7$Qf851MNR`Me7ZX8A)q~Y-+*Q zSvb2vn>&n2OV*3O>mh!SBJ2=^ih;9Y0esl?DiW%a5C zIyrzu`lu9@rCOjt?nCI_BzZkWpxTeJ!f1c1%W|^ z)9@VhmB2R~fZLuj?gURA-u7ro>tv*lQZzi`TGN^~dg(O;6anqB&QKf5Kp-jD_#*Kj5V0u&1}9?drWXB;9AVRfIn6cPPO!Y-y^bpAD z)k);Q3nWOKPZXyaFvePM4gkFEBShG4tf@E#mXhz^=!6Bt{^~)>-}{d)FP(gH7PSBB z?r*My8bu|tsNAn7V# z3GV|okt9l5d4}1`(ll7}VD{}VYtH`Gt=%TG`?KPVUL0zwGjmMz zirYt@(7;9p7YKRU<{cU|Y4ZH3TkvS=n!U&xgblj9UT+mDx-oHM5B-z%g4{={2JkTf z_MfG1xp{nkHhN4}jRR-;1Y3)+VU)dxb6&?8c(TD7Xev!|6CuL?n0HmRBHHxiRcqJ5zD zis>gkf}{`%C1!4tQA)@wL%c1gqofwd1suzAap)m&c<}WUc1=QDAa_zNsj;%rnmub9 z22EQVUN`Vdjq^wSO}f2lQ#Vh?a_D<5h!5-V^MY#Ctm!o3QLRn={Cs4h=Gjj?7SnH& zNMxiH@$7fR7k;*(E2ElOx6EyGrGqnnKe+7QUc1!t0oKQkY2+dm7xmWV zKOJMY|L25>x&LK6g~5Ni#YFw*c#8k|Wv%~tU|Pxlvq-f5+iVQ82@@@CkAj(qPzM)U z074nYH{VNH5g_FMm=)W;8d^)3Hr7fL;be78)>)TBwy2iasfQST=+p0MLkkO<@ z{oIbJKh`YN)}v!vs&Mh2k!*@ky`K|aG)ns`DB?6Haq9rY_-FU-*jeInfh=jS)X?}A zt$uhLYyb}aWYruFmJt=wpH(4K2Z|SkJ(&V0wEDZOTxEnEK&zqJ$^wDvP7pRXI)w$n z6{JHd(%G}?f4X%UT2C7MJrg&9c_<6iAMj|Ve5m@P9_5RATPdTwd$^`IbN2pTbqw{b$R5U?%n52^5eF^YVZKHPsoPe5TkPKZ&%6<_`sO4+MYYi-iEWi4 zU+PvNAfbywx=$Y=K94*I-=`V@h4mcoHTMqdfq zwkFAZ;R)DJDBgw%nV%~&MHM|a1g%m$4bbah+x$F!=WlEixQ$=8Q+oJ-C3 z-AO)&C{E?#uf9^VS+=k8T9RoBU|d)tbvBb|YZRxp@wYm@sN{FhpozAc=Vj$X8-WTU zN6ag6KiE$Tf4$f6WFZPxXcy&oUyz7k1#pzP@{<~TOupIvVq8cACi?(wjMID25Rihsbh+7XDpeXyZ@G=fGapG%TV=tuUCYlGO4kzLfefywms5hL z5Rese({odNfdPcY1LLtLBNILC{Ceu%gMbbb{+?)XmbNyY@Au4pG94mP!%kbylYw!7 z+KbAlSB3f)FVu;reC&ZSLW~E)&iQN)ZP**%MV{tJbtiG+VMh3k?9_8gZn@4K6W;1i(_x-~EQq{<~0vTGO zR7FKai3AusM}?2t!j&y3+C$mHEXJgJho@)80gQ~9bMoC_RU37Esb#aKQOlv~uhFtO zAl>Rbb2XlXYd`LSvYV(7)+G6ZE=zw8!1?_tzRdCIfqp`C3SuuzCJa#^OmSla+uvu; zHu#jLQIwlC`MG-cgK%R!_SCObT6sEl?09w|v=(kf5+8nNfsGXN+uVrqx2#oKP{2&iJIF1!;ehllupP=Iwm z`<;SI?KKDj1EyDqpM@|@3MGKXP0X?LPDVmK$~cVpdYTogWn(JtixDGPcXApvM&t%; zWKzpiH&N-*(~KNysYsN-9ruA+wD02}MUJQpfxGNL zQ#g}5(|#>ZKSE{+_PD=@%^>v(j9-Ui^=chGz)`l3N=d{%#VDE8vr>sW;?}|MUMSs4 zm~*FO_dyXVDm*dBrUh79k8T~AX@^wfZh;-DZM)G&$SwB3)nwBB9!z5&@lvy=s+F#52I zQf;iq%QjM84l_T&mY-c5{+ri6G>=)>7C$xB)%{fv>pjX1gGGn}N-45&xl%0Kf_hti zkai&02+@QQ4-`syez706a?%=dc@E`5U*Ys9#OeE18b=(hu%WZ#a6Hk;qDkAHix5)` z6vSDmlp|OI9qw3wvq;Mg#2?SqRKf(mf9PHz2@Wq^+T61<(I9Nd8yWDsDKbQ{i~)-j zlL`^wn7F3zC7Vjgv4CE~x=-zNhxW@(G*B#Smk0Vw&n$f8iLk7?xYJ~2X5Mo+v2Yec zE{z|4x8B%US_80=Z);BTGkW1YGDl_d4bJxJ+pdZI!dzD*Q4)}&jFJxVw;kC$_h>GNs+hN>XngXtRMmE^=#b7q zA_$>$)bXC|&@L~41FKrv(zyp!5wMk#opVxLS6C8C6;V;DCCTiCAK*u`4o(?(u9kPq zidC!BTu}}gbC>*wJ64o0EX?jAbY|a0YT+}<*JuzaQTqV?sUn2ypz|&ZjO>VWYCWN^ zxVc4N4{5CqHTPXO@#x&MQ*vZE!LU>@W1BO<-!Gc1j_ zmVo7~d;S-QGBM8;-)coPklNpP&e7?r|6qKj>8v6l0a|_POZFpBo%kQ5?5?C8#U~3X z#2x7=MM@!50FdX<<4qG|kOWCDbFl>|&XwirzjVMU%s zav)L%8sribB7XTwT@A3@MQMxGa1TbOiEWm+i}GNnCFK;SbqMeCszmz2!Fs(%*VnO@ znuZM{0}DlCCf&%G5u!QOSC$N@fq*fou`ZsrG8ugjbdB`3$xLWrDUu%gT)8)0L+`bI zVccF5sRH7?j~>@j?4X2@V@|p$%*@`Do%w{`(t$-H-Z;}(^t&*>3*=ev7IS@sH@7&Q zLYWJSedQok2=o=CM|wdxm?^(xjV{RM^zzDYcEb(Jf+3tzO(~SpgakD>9rIHrHVa|E zNg%2Vu!5Mrwe=nM`(utXju7E#G>pyLGsCTJfT1YTAP>ZY`PQz1HH5nmKTeU`7dNMi zbl%fTBq_BG7#Jqwwjg+-(d|fc8E5!yiCM~$Sv7Q;0_odRc#AfJTI#_RuV5O_-T35s zi3cBxc;0e{!k%bLH&{6uXgEM{ZW5u%Vxb?CrCv?qQsUeh9J`VQ&ZegU}(gxJ1#}pF0|&t$jS( zWll(OegWs0#dFq^pwD|H=-Y>xEspsFy(Xixvx}gnvXB(foiADdG57L$7CJ`zK|M30 z8;;*ro#_54v|mA4uk%cWdjiA&4v6b6#I>%RkAH z#uvs~&WpKoW`lnXZD9f}xzA7zFFQ>`ik8+?^;2Xcsoo4fo%(lK?x9)bjaszmq!gV6 zD@+g94L<^l&+CNS!T_&QW{TK{)^py@+@p8unAMXh^1rm{H$mjrh{*bVdpl5gVh3Ng zU~8Rc-H$5d#rxGoqxhhZAmecFKwtG<42WvU8DjNn&}~UpfHBl& z{z{dJ%cWqaa(G*@8NG38+ImhvjExpXqX1C7sn$XSakw0pu~P5b|Czduwl&$P6uuX7 zPGdp6D%afHoQvwVDs{s*JBkzpM|p|^4Yk5Xv(YZ^=)Mh3VWF3STo;%%y*co6rY>2fcp-;EX@&6zMU!@0{(GQ+H0%D3fH z?rW~a>;WdO{Ygi&&iMO_+=@IzrI4W+XdEIzYt((`cC1R7o&h|SPxCsmV7~~Wz9HNZ z4S@2tBFNrfQfi4b4Q(myEt5rAq*Aa(R=o(RQvT-}uI%F74pkOqKAHLa@$3y9n$9S< zYgFFPrV*4{w>O#-DQxx%?XYg5Lk2h!4{+jcEjClay*ixg)H{x6N^q*iRvtS4fM_pp z*-%?Qpz0Jg2Bk>;L+930rA?V$Mmv+%zuKA@WjvXA;X!=;l_xaPOV)$;;a`V(pS$DH zON_78e5oJL-ZN1Y`><<&#hX)pa4wC}9`jdJt)|o4)NMLFyz_nYZGG;WYh>I^fZHI% zQdHg(k0*0IH7!rF(rj7`sFfyE@4^!gvZh7UB+SKCcwng);@iKNj=TXu)U%8&!(Vi* zexkzTQ2meUOvhzv5<2Ip2kQl1(yza9#+d_etxgR6#iFo4qk7l;U{Juw0VW^jCZ%l{ zdCqUpyE&R37WKawgl*Fl>YAVAlK#thg8$(X{`+tV|HBRbx1oLh&v;dD=XzBNDiFIJ z)Q(bOI0s60O{Fd1e_Tb=R-fyD-aVup{$G3(gXr7)`#vgwV+8~Of*scmveh^mSA zBOHcw!b(e>DCor~Lclm&3%HbXSsEvX07W7&HMvCjh6l69P$~EDMRj`uePQQ1O@0Y*9dj$ErhMY?TaenCMjp7=qBdcN$ zUFeSGB_L}_iAm%=p!L^~`3en}F!N@LeyQdj%R4|P9Hih=-k{ei#_|sZ8V)3uqg8^K zH+P=!lc~VT{Wb}gh{}O&V5Kev_ZL>`qyLBMX{RjI-}j6M}2r8!X`;&pQo;+9zpoNCF{hjZ+Q^$CCFa1{Su1CM})O3{Sad};Qf{z~I=7pMCcnRn>$YzuIvK;x3KMZ*fpj8or!e(d-1?AtHf-Wxw}Q`h->TJll< zZh@s8s{HGtKlHh39C1l)3Z}25co=ce#$)`987~KxFY@R$UN!s?$2<&d59O!g9^#^P z^3;RSo)edj*`%84GW6Y6(}Gder7FSGvwP{5`iY!FH`LHgP=L z4p%dD^h&lh{0kXk4*iG=bN!b`Je)jWv-8~64_E253O{4e()s&Yj(Kx!!M=7oeoI3RNBRVqWddT+BH)Rd$mLFTX}VxhAHM@gHhjCD0;LU-zZFrxIP2KxRs7`IW+eujIPbli<==R)X2Ar?y0iVL zZ#F3QAWt(Ls)3Qy9xX*q_MsAkeBSkYfBkO7%3Npb(s36($I=u7`bTrrvp%ocU`N7g zIpX2?w4zzGkMdQ?H}tqEIj4c`RxSOHrAU$}1+(%7h02p7T^US39||o_wupvQC(JZ! zRv{c5*{ZY)k?LM=)L$RZ*SAoleOLdU)que?TXywJnyT(ARmPUr5INp)@;$>KE0d+J-vRdFYQ(Z`1y-Sl8jHOqGL)53I-`i z--l702GZ_xp29eb(w3U$`aKajiog;C4o9;ItqxO|NfRm!Q|1&?-`1CCS|qeP3|gf< zRWKV|4T6mai)9aAG;yM)$U7m7x4Wvfs23EvodKn!eO0`Kgu?bR z4X~DZKbONT&LAdzR$v?8G9UUybmB-G)VfeM#PQ<_Et`WN-7+9ujE==$mCr_W~S{;!{AeI)+uYM?qI@rNvln@+{q4 zf)O_zQ>~8;wXZl10o!$g`&UWxj<1MIGD zHXaBxahv~%odq8)W-YAfSLZHkna#vbQ3i;(Q}ou$=|1gp1huhXK}P%P&JAN2umBeb zdu4MAAuIxDG>xPQbGNcnq)|=Q?$oZUBZhBt04{WZRhu}Wb?}nYpP$qz#?VF;LdRC@ zq*LR7<`#I`2YkSVgqGE%5E!$MplxN_?W}3}Tp1Q%c;MteXedM; zErT~{-FRK_UMp%W0RdYnUAzQjQVP3QFF2sXI7mNCulCUfGg8~qnf#1^)#Am2xkjAPuXFzvCI;B)GU9E#687<;w5^gG0&UJ0wGRJhTHq)W zF1qleq%(o0f#6=w81v z7*?!M$IXHNcpR8loOJJBQn5?vBjzdprAq{13O#;hI{p-)3OM0 zY?3`pR2DybZlb6_r4_|Hw*~u80E9;zFhSN6hy|$BJ{BH|p+R_!P2b*k#auuQ0MlJo5%S3`=fUhO;p|6BbNM|JHkF*2`B#cx?X6!vU?U$3ar*c|*x<0v7AiNE? zkn5^1KR&ac9kZH%VqZ3(GYj(!-pDZKmfj%8mZ?!q?Rmt;*dQuzFfk@pW^0(vA%&6a z7Rm^+^Jt~YHi5*TPGliG%{2|Jj?yP7?t#L1h`|EbYTiG*hVBbrjg<~&VFl2KL8%+i z(9(of=mRndmH2kz=uq7+jlhrvwnmmD?N*SYvPj2r1gpet^M-Ifyzz=LSNSY;;aTw~ ze6!Cayu}J0we-uXXwYl8>b0QHpzDojSOI=5lvXFwU17;-S3bJaRyY5`(q=yR-d}N7 ziV#ZtTg1+Vd^k%SndvV8fODzJGRxj&z3vk^ZTHzCvNgj5QX5X;9-c4um!j%J8?}?p4d`*SaQ=VYRvEH!7`|S-_ow zT1pgz0g_ftLQ5zpr?i~%7>#EJH0lD-r+O3n_gtDnlqE(f2=m9w&?zP;$gpG~ZIPM@ z{Xmp_bnfqEtD54K$;WXnc08MQfKE6D5H0kbx3T)s(fI11DU<#_<=+j5HG@)s8{LnE z$qe!_qqrvC-tTr42}lphI~qcTHYpyHp}b*kTX-+g5VZeA%j_VB!zJYUsyyEenTE_Q zTB~Ri6DzrW9N2Oqz~6jC;dUZol5X^c4C7b#60U6RDx~r)&D1KQrkhAvZqYHxMw9`% z*uBi1mXvNAT3e zix(RR>)R@`#2;YReNX}Yv(X$@ti4c}g1yqV`elvZxMsT5zTh1ZsL)F@qs{p_1BXo@ z_y?;D$O%mO=B!0@kEGxZZDm4MG=EA~@@_-=_}9Dme0(`2(Abq;+rg^EkN<3%PW2nR zl&|SYqsU;@U8vLjm#gWdu^g{bv}${_p$UAwi_9+xw67HBMH6)%lY(Tat$NK4IFhke zu?6;UD~ANJ!|XWeW^*zGy)@3RyFJ{q)0uYCLK#VPeNI4e$c6cA>F@8amzEx8p7W=D z6{~1Oo$iX}&~!HmXCIiK>=+)+RrTh+6TVVy-?fr^dk&sz6w|`HhqOyR{{{#xAL7Ww_SBrY~jcIP8k1%wXjx^$LK>Q z$-O?eFML8wNI352WVF~j#JTKiW#vQlpG|hvPASq~Qe((9XGYoKbftE$;X@$4ttlre)N zaxnAV2Nh|Ektf<4X{(d*>7B??J-?>(>(G^enPJr2+a;x$sD2T~tov|jJrw3%iXsUS zl0!-&8*P_5Zt2VLsp0##^RF(^Exj96s41vM!e8Fz)Ec5{Wpr{WudU$zv3OQg6d6PN z#XADJi!wyiK{P!7L(bO3+j?Q=v@6 zBo2INRieOBEO-zRAZ}WB^Es&>yCRw1I zu37ZGlJ|Y|5n{iVZ8~)-H&4{YQto!xai<#`?^v)AeM7qE(R#il$1&!O_+F#npi- zaaotWh>ZYx!_^!1SJ%0UB~C1@YhrRD^}Do{^2Ma;<5h3`ys=*!3G#J~3L=5@kU2*t z8A~H=!sBqV>_j+FIKCUZh84oI5K-X|`7F9DF=62Z`2ND(d0?$WnOf_9Q8aTFe2}?` zxAKdtD5p_Lf))ge9V$t`4EX|q3t_Too2WS)fvm6y7OgukK^A~)Fk&#e;eq0o%dwNP zu64_zzwMIqo?!hcY^~^_WMW9I>YmtKctNw{VsX*;^chrWQa-69myF7;tNTm@5$Lv> zVvN@7q0=7=ib^+Lb&Yqh{Hw?ug^jV~&`e7mjp0oPhq`84<;f=pR2sUiSFEUZBv47O zJ6&m9hQffC_m$K|(mhI6gC0G4JTzS?G_6UBsGI98(ykn|jkh933etFrSU?1hxRCT3 z)w_$VpRJSs`bDvXU4s&7O}t6hw?`2ZW(xVv5kE|XfnvXc0xZRMt$xRjm&O@9MuFl$ zPU{kQoesn}R+$V67St+(km`9P_Th0q#uQxO`c!xdL)*H6-lQ%rmWI8 zPuBV5)s65GM!s{a1_y{r!^(s-Vz3JLrsD{TMx6A+Rn=Lsg@);E2rtLxAoEzXfzn4W zt!|4R9ghvIp))i{$2{i60)!IDjDtSwvvD4LXu9m|)3i<)lW8P;t&c#1!gVuKxjv9c zwj{#pr48fg`*TRuZ%O{GE zfZ*M!c~E8V1ymJD50ln3GMLIaFdcR2G=&TvxR~#cQjN=N)#n;Tb+(lWs-qs%DGuSp zgDw14hZj?{>bp>AXU~sWRk!(*ZcFp#Q>y5tF*au#pvm|g@iN*#^SNG{NEb4UT3Xho zkka4qo0nVR_IGEV*A(A9#TEgL(+38&?wbvU>KuPJoeiYg?eS>Wij^y6W@vL|ifjB9 z{&Fke-?-cax;AIBsho-@(;57(xVWb(kYmM>c3I<;%_tKKeGwRJx{IE4zldfj-?mbV z@+s%fpOWN?iXSn37;^F9 zG$IBnJ}_1ALOzn%VR~tSI5+%eBNBqxPuqIW*~-ygemeKA))=W|*Sk^wx{`}$ukamkw(5xovstXJr}C*_ zFCzyze|%>t9=Y#!txmlMfEnAh94*b^!~>4;DaS_=V%^0AX8&uXInQAd4vsLf+cR{U ze1BO#@S`!UY6s6iLES?&Y-Qc%>DZD?nH7$ueb04Os#Lj3Pyucb4c*>9t`2Khg1I6ROk*G&*E9{f*v-VPl|cX$!0=Ke}1v) zxROkl{L+5Ah5y44%;z3gxpnpGRk6^_u4?hN525k!o42LZ4^NIDlfGN~j`!ll zTR67*9FaLjk>o$vaprn^dv7LJY2WFWnn!O?7_Oxo;?P|W&y*v^DYV)=Xi-8o)1ChE zz?)Hs`m%Re$g4wYka~Q4m-g*bcEy4ssTX!LXX=$e7=?TE`M49;YjOWr&>ni2$KdH^ zQGRS&9U%PM;6fuNI=wDodc zFFZkWZ#UEwZ1z64$8hv1bt@S-cr=URQ1HBm9`h;*y$2xWl=e1Z9%NN>5;(B4*x?sB z=5OELlohq1lH_-yV(W5#>&R|Xs0p6~Jfv*)ozo4IoNi!KCKMpOfxPEBzP6mo`n-C) z&g0ys5!4cxn?^dF{UiGUYj@dmNJOK(jjVnH7wD;yLB)Bq=3w9z&2)8HhU;Bq<}(Rv ztFLCf-0WI<%c@3?wQ6^Ze02Sr}nYV<`mfIVp2c8d5}{V3zb)>7OM@>($)Xl@7t(X(`OGjyMpX= zeC`zzo~fK4(|JTQMOk09D6X1L2g>m7DscvWGOH`O0`s$urGXW$caia9>8N=Rpb#6Z zYL0j2_C%_+hRl{D;hY+4E83SnWmDTs8NdV2LJt*NG_w5UTem=C4i|17ASTAWGb#F_ zqx+Z8C6_kQBv>H;tX>)g{Iw~iNOy)aTqtivJwOH5;$vVDTL78!(Y-aNd>lBop?V+f zfeqcvVLSdzdEd>F5lXPMBay(1^ZB`<%T|?;a5Y!9m)!dvIn5NrgHveD_W2T;X`*JoTKTb z%PK;|w7@e0fiEqQ#_)2pDk+S z!i;x({0TvzMBRV(S)%ip%kp@L5F3ll-GNf-m0|BPi`YJ#7mLN&X@(bl_M%wIn&SuE9$ok2u%Ys2R!YJw2*K>#aU}j8n1M{WYRbVIkJ(UD zhq?o5d3Po$O+$gbg#&F|&iCfS{wiyZfuq#D!>JFwt`~<}R}o5#V<>fs?JLr#UaDc_ zKsBG}4(hw$Ww4j_(IsEzxtDUz#WcKdXL|8z=GN3QI(u!{K?cXBVc!>D_X0P~E~p8@WSnyUL~|>Z zyeI$+)dhM5W-7LjCv7257?;BwBj?z42fYd%oC6Q()MzqUYprSZhy{?G2@X%oFjini z`2_-8t#<*UiD}%3?!cICa%$)KP3RSD+sb0;{oghP{qmFva=BI}2Ts2KyZBNVtepF_ z9c1QPCVU*}q23d9`~9iOZ%|`c=ID@Oz)uA?4=H?jw>%Sf!ud^;UTL-m8#=y>D{J7t%ydz_SxDGC{}t3b_3Sgn^Twc;m)Dm zS-pEDzs<~AysX1T{(iXlMW~x}B+a#hN?g^_Vy*FWAV|Lcrj?0I++5SCx^`CzUK#Y} zsk6*2y?jUJCc`lqxHh(+XM&)S-zTWbX_};qOFRKCo_1$yJ6`Ff?=@(od0U<8`l4Qmck+%Q$mIT1uVA~+u+_}0_gfX1m5ix!nU?bOnHM!4Un zEf1G`7%qH@Xy*rx=R!r{Eni0v{5vpE^zbsCzG}&f{QUe^s!9=aNrI{3l?Tl!en|Di z@+KjcgM1vbTAk;Y6+e1xe7TWT=fBHdVb&tL1-IZ$R7^Mu?`_jqRbiBIifZ|@mI zN!*_glIw_3JAkR_$9S|qX5Hu4jU!h}6Vop-vj?5gCvlOcP#*c3`Ma_ayO3-p4X6cb zVn<30r?UH3y}>U@Dn^r!z_$EG?Q~hP&3gb9QPd=zNP}={x2GBxfTg)Jt)m|i{;RJ7 z=(&1*ofw%tL*lBQ2kiIzph5XO4OIkt@^}ILVYM&>gWBEbB<;&!mIpXp3px3uyK1+z z#hxkuE++r6OTla7M>K;X4QaKfiqeguM1({{$~}lwDV>M2a2LMZAQTdd5;2qv3vO&# z8bw9sP6Doq03*`3ew}>&?_U&FC8s~y(8jdu0rv0W(@7(aU=-5(3o8*6AI`BNM(7Yp zqdBZL>{+OB$stV-g_@)35Fm1L*OPDmV5(2+s5vB3&wzce8k3x;=ggtUW;yKRB)IGv zuO0w^A?&<16eFxK^~7wrCBceBElwnfNgmEa78F9JmC1Z5X`5qvlBX=j-YMx5F}ykS zz*H#E6HmtmT#$v%8NR_uWWd2*KbALn!TEG!6QE%zICDvq7LN?p=c37lk9G-%Mkp=|_KL{OUrC zCa)O%BuQ2icz732qBxcam*&^e@}G!T*yDeo7GE$MN9U1zQedA zET{zLvY9fH+&;~_YVz`ySe5n67{ax%?T4dO&g{2*0Zox*i-e4J1J8JCHrqD#lb= z2$P?Aw{9_s4>#xn8?K#jfO|ai>7M(DIN*Cg=WbMLCYT3y13Oy!vW%F%m3PrX>567W zIks)E*~)sY-A=vm_3>!|E1;JK0hUsF=_c?*b7dqCs9w&Pn6Xo?+ApDfAkDtt+ho2a zo$K&WY=P6++;-qfZl2wMdnR*SihHYQ8kvtF=zifsv-8AoaqF-l0iPQTwLJWp9@4c} zUAxOOrx=$yUQ&No`NVRTKWlO;}= z+r#q7x9-7jsd1Y*2hPk%j3o<@mfOkk>>I-cJi=<#wt%=ez9@$#??${$Tii^`_Al7} z7RvV2T&^sE_LvK_qNP)iA)mqXcuB1WJmSmZA5PZXIH)59GLI3WdW_Lrrg9IcO zaYe}(m90c(EOmttZOT@mlxVSxoiRhC<*EqTM%h|SmMBXGMU*ItK_x8+p-GhL_c||g zKhJ$Ue?G@^9FOC;?>VThuIuyte9q5#zTfZjeSV)!y!HXs^;C41Rt)>iYp>qslT+QZ z-c_YlzqaloPfX4$Cm5a#hRf{Xmu+KPCkLPf%8_z*zcl5IRej*F8x0F8BWe=d^+&)_ z7DP)ac#6E)gRt}!kah#n13BzP__PZGB_Ef@yTqX@*}I1o>fO_@@C6OkTG&(1;e|;) z$UswWBl&v(2t2Dk!@)}9_1>F%hI>z+I^h~A(F96Z^6S&S)2mJd@<1wh$P<(JNGGtg z_8ybC0ja}X{McA6kO{xPE$g55lIwuZ$c5aNPqO{jS$(6c>-*@dVI+3}q)Tl(_U89t zc->8iTPX{vuOd1`wNl2jdSvKqrI-|gX0!xPu#IRW*U=pDj2&wpAfEY!1}&MTIvymg z^SKeNEIP7-J9mJwklh)~j;DUPQ0<;9`vn6VIr!0zD}wI@8;@NZA?>!J8YC-{NnjZd zD!C&>Rlu7)K#k^1cH(a0sHqUoQ6n09aRRYp?Mq(@c!byf4DmRkt#T0D)5dpfPD3_& z*5v-LuZb-(6u?tl?F-Ie(!5V*_o0RHqG-cX`sJ#+%V(@?22&_OWk@=iACNnI-U7Nh zw^8C@q_=M9F+(g2q&1H^s<(UG$UMYnzlq38Iw{3_B<3B_KAuWmk};9BtZ&!C!b^lw z>~y7y&A+FlvJ%y{w0R=Dk=)1*Uy_Jqo1R`T@1L*|DJ#G%a+H2UhMZp3z@)CRTw(U$ znSiqgNrKEz)=Y12m_cqTLrMW5B_oYgJc2Kj{Hy6>%|sD$6C@niw#@;SnOaONx?KNI z!)_-q;&W0q@tNhqa$iu4>!mnRu;N#KnLt{~kiYD*eqF2u^(qu=cj-muVQ$VZVnYx- zC&(8dy(`zZdt*>)`RivYClDTe<;+#J_Ul)1m&bwCUMF`Je_s-qhxAN8OPs zYs}8MvHQ3Oqlu|%-5_>ZRKDBQgL;{tl4bvWSef)7h+j2yC+eBm>l@)3ju4AnWRh># z(3dLhl zc~~-gLX^53@%fxEv4SP)4#cmxo5L3t=O7NnIpIwkMnw^ARk55j0}~SyvvH$3OJKW= z&jIDkGaxY0A%)FK|9;G7u@6;UvC*`C!&sM|%6ddaYA3D+=w>Sz`P+c=I49?o_3dQs zUsP1&pjEl|Rk!i)(ar%xyesS5Wl>+W45&eiXFuTGsI9W`&!GIVlM)hpso~s#4g+2i z*SfZk))S$^iuDU-%$OlME`=BUrdjr#@fGrkt_DQ7#pmt(0xQ}+qOK#4s?V+VxcKDN z-pXj{_@g>j%_>HXe1kE09OOJybgKB#J+lDZ#*soTy|*M?Zbh|c8P89DY3kZ3tp86q zNd(@*&SC&krJq>x0xo*W5+hVIW4H$Mnw#n@m3iDeu9N$RDVn@P_H*9E>RF^pY|!5B z_SWqXg7uSniIpCw5?Kv7zusRL#5?~=w?46-XKKkrZQ${se57)7@Q<*kau=JU=78hD zexZ?)pi93xwQBU?Ce6>fh{#IXzU9DDsGE*!qjJUlNJfq##xU&NyZ2wKMfxh1W@O@r zDLy(bAnT4&P?(#aa<~`;_}3putPw%uxx!Y~oyF-?_A&$32l+l6`d8gIJf5olMi;#G z^gn0FC9kBl(CHNU9|lCRGSrFlWq?qjXca+Uw6Q74BH2}kuk%TayOr|QFW_{A(z$vnH4yWd8YcW7eu3)MO+q^o4W zViCiZ>{B%Bk^~u!qY?JdEXf@%gwjXM5-=P?#FNfXCv(c$o3#uE(ccF$_H)`_iW&-RpQpdwi;FwKZtal;+b@ zs$S<8dUZNKB5H5ml3IdDK)^CJw_%TaX1DQ}I;YOP86!#+*^ya+tw-i*QKJ#f4~p*& z6@4LfNS7B?s3I`e;WQ1gb$OvZiYk1!gHhO2aCBPn&9SDk7TFQY-K92-V!AJc^557T zTwdBbR-=t|OXi*!^H#^IE>A?esa^lu#*MqEi(pIv+Jbe5|4r#N?Ii|g12AeHt8!MC z@NHg?q>6bypor|ln~kP>h+Cy_Je<^4MFdhIQ}pqYw!Y)Rh)_!8P$Vs?n2j11VMt6w zI5o|^i0lohnSxD}j77PqgSDHMP%u|rIl3;EoVTJXk_ zpDqX=*?AYh_?x=A(0~j0Mg(lE*}35(3M9%0#;OEu+GPDZ+~ZVgnS=6BiWUmx2XIq) zs}4iJ#SRGO9fuDeF0)|912ui78hekPxZAw8Rc@DuC~ZZ5S5U>3k;C1x+tn=W zq?-H|8c+VD0Kk)IvCkZz)j29mo|)k>Hdj5KrtN= zKgRC+$hB7AH0aM%afm~@Q^}%yJ+XK{`iwap8%egtvxO51`H{=;34aYlI;-A8y?b)3!QC+fPli!Y;rgEC(`&Zh4~S^vY5otu)nUAVk3!x zOZ5W+xGB_;33=4-i(g3x!RVKL-YuNRC25TR2w+&#loTK5s#Qn1;wo6eA-v>?l*VD%8WthskM21^K!P8kV7SYA(*RbWb&SkX zxXr}eX>dz5P!DHI4Ax$eLLCYZevD?cY%HnZ z8_KNUjXg-f3cAJ7aA1BCn6uWhRU6BXuYb+Laeq`hV6z?EwdM(XG%RkS$M%G0al>SA zQL(yi1adn0YV+1c;Z}Q%4K7*VRaeH;n5u%2+}%Iv6*1}o{Y_hAN9(7i!aSG`@2&y46bKP0;ygdQ{OBh#8l7dQ(ZSZ}BU~+mu_{+oWml((L2ker30r z+FMSQ*kJCUG%X$yy@(<@pScxO8|4FDnQ}8!w$Ur`tAP@6FzUtu_0?=J>)Eo` z$VkcJs-afuJ`D@Ig4=)tNe2~I8HG5vd8tvthBQy zK+!5VXbyYPweZgLy)V7T^`J5Lb<5X7mw{_-RpisKF?0dX(F2Q@u1KLLHvruZA%hse zTT{|yX3;{JV00V7iW3mmLtoOCHbj-pbib&Q5yZJ{0#-I|3m#kzbNSZNY}Gmi?W{Ug zm!?7{^DsAB=?M-ZZ0bsGqPZNZH*MyvyE@^8dyl5`r4DcanjJa}6tM_W0#hi0!Bk(y zWa&XCLiq?^S&!UYi1r!93qp>L>T+AnDgP0M7RZ$TmW_4IfeS0AoE#VIW4fEqQToKiBY6Zq-{9n`$i`<&CIu zfvBWmz;Sy=i!kb!aVnE}H(&g@is&dq2!f4?gb}A!fdWaplrZfJWIzzt_7PkP53gx% z%9;WN6l|=DrPy+Mz`l`LL5Z%Zd4GR{8eY6!IIrSL1Zb2*eL4Si$zPO~JBwB~h`G}b z_p93KULh2nf>M?C5u$2PzVC_qqHNP)4&da5nm2C*ZBeC-*O<6TvrBK8ObL1$K}(eG zq4Ol0s(xn=pP89iQ z4GArVwkyKF9|Nb-RI{>KJV}oM+xPK{TGV@pW4E=DV%;aLO>Mth*%Q;IWew5;=Ih(R z)FSm?VQ`h34=*4yqoh!f*~tKFGCB1r4M<{bwI%*D-6DEkjFdYu7L-z28Xgf9B~Lr; z`-6M1Om(1Za!i!`GNBwOIb^isyhtzKnd>t_mo85o@NQ@K$JI&26>6!sf(Al>~>#9vMgFo zX`^Rf=cDZsib^abflSnMD@cZEd=cf=WGK(et4YDeaaBxvp?w+1E@6S#^pQ)(Q5z80 zn(Sbeg!bdsli`i^By$NDWHdX+NAx%{ZZwM78?Vt8(#Dd{hw^vzd7WW-=RA0OWe}5^=)aWBot4jF-6Lvf)u_2AY{Wwk5WjJ{c><` zSNrc@UxUn*5wm<|V_O%{Cz&(KwdpK0X`Iws=#YVr?fkd38!CW`jM@W)i>JKCBEf;`BWo$N1^x` zD5X)0C18yV9iucYj)S(ulYtj()@&wW8@O?p=cu25o{CukvWw|L)FF8&FYEX9qND5> z=bF!nHB*7+#nwlp&(!VyXI8)Ea(f0`EX^=P$daSfVQ*(vHpm?DV*qzvT!bCDGK-UV z!sgkQy~_1`*ITIwgsod?=AGZ-q^lb;QB%76h>-`GcBW^W_O*2opqTcy#O<;4Lfoh2 zN=9bm2N_n@{|jpq8FYBscegO!NN_v>n?!dkD@>_&#Y``Fch``G>swf7fpkmfo=w{6 z4NrM>Y0jhf=Cb%k;cpPehM|3jdin>)$o3W{*jn^@dDZ~`Y2v@v8lKl;bBpKGX^ z0@j->X)2*Z3N_1AV6=e%dGoflQEVX@Ji>L?F`xsvzsymuU4bHy%iQ3V=_nfqQc;OS zBu$f$bO`<;Xc2W%(23ZkTuzZ7$jWTvlB_8N+A6={G_l;F@Yh!o7W`s*m{UyrpiIMx z>v+NEinxYHXRvbCtz6lRv#A{mPDOw7!^C#vceaZGur^u^d%C+BY(0wE%-`e$H>VO6MaNpNN=` zNWGn_>i}=ZlXN?1`J?pQI+?77f z-fN_)zH)0P;pK4*lfZ+|++@N)if;KW#>RAGD%(zKTW_X?$^8f#aEwb-Y$yEc2{%sh zuT8G5jeNRV&7Dpqt-y;;-^inTvG!&1$^J?|-%T32R(!%KoKE|5%l!D%21R$5Hvd5L zF@xM{8iiL!nM`j@BpHsAq26!3A~g=xS1QUaTWJFO-r-|3*CZMBjQhM@JC%YUnxz*u zfEL>GH8+Qe#6dPElvi@#kUd)125tv-61hK}l(*gqhtDsTu|kwW)2HgaUL+|pvjv@C z)+hWKN~gUHYU2EGpuN)&=}?eYh82vUw@RQTr5*b6Wo!&@)J~4vIs>^hCBjflPXcrY z+&R;5;oaWKhUN|mM4i*Q7~5Bl&`@QkkJ99ykQe_?l`(WfQVmwY3)ST zQ?$J52MLXD-@S9-OiZ8NH_QXa-o8q4VZHp0OG@#iwL{i+&DOAu#6oWhpvVd4w~7Yglk)H3`4xqC<60=i_w^tu&9N#WzrZ<` zcv>B#vgS!_*oTD&e1lemq>7CiWTR41QA>J;qFD_0@2 z7t|sLJa7UYj<6~WQ4>bQ!@MPC;wYoKt10KX)B>64c&$x{8H*OP_hp^KJ=DC{4n)K| z2@ym8k+!-mM_a`_p@2uXF3^W`m2_cfJW{{8+(`)RB6Ok^{aiy7R4EMxXLuA&C!BfR0tYdHs5Tl*}K{!;J?kdPV~4x~aN z-MO(p5qX$|UD{MjUc6nm(H}LVXi|XJ>KT8av*&6PS%Y-F;hFn~`fG1b&>4)1O7>9-)cP znvb1d(WyGMsd;GA`xicKch%f;3wdmGsJyD*jr*I*CXFpM@@TMmdp2xRqQS-=4L63_ z1sYyS>b$wE%EIgN_j5aDYUud4$9v2r!v&>Ygt^=40N702P;-uy0TtQUms_aLQ9Xho8qya__ z3Ae!3kbf=43SbJFufNmRH0uoNZEa^`vzgT65Yn=1OsnO+OaT?%VmN%#P`PoW9D;TA zBv(jHVkbBzCnrmu2%cyu7?rqj3~JD_>`-!otQqzqfvP1JLge z0*)8J%0NZQh1tQk+5KW)-uH(eCfM5Aip#W&DaCmUMliug`N29X|K`w%xlE=?Fyp zIUQyoU7Q2Nh?9~ctG2tC55dX$@Fq8`=rlp|&t1q};R>d~k||i)AGO>)lAW}5S{W?K z)Y4Zdu5ks!i1`O<@0%pl9~drmt}Si{lUP9+BC#*Cb&0PkENmY|(9?Bob#DkM{wp5;iV_X+W82iM>A zJN8bmPjiPD(}eaMja^E}?dY{D9zmqaNKo(+r>kNNDFy_ee3lMDsUF}t$~Qaw{PTM9 zi$h#PRY96upZU}}3AFg%;&8u`a4dHhjw&$+4q(AjRz<4ph-x%x&52w57UyZxc9Mnt z0a#RNUhYLWhQT}l5hlqi7xWFcWKXDjvD=m*7qrn%tE~R_Vg(xNwYtN-?h|B?%g7N; z3C53!7&4%o*Vm5jY;cZR?AqPC_IcG;e`wRTtqd2nBWB@R634T%R;eN)2pGfld-tA? zghjHXUhd+ub#Z!S?zcqp&T@|0_HNy-nstgG38-z5>V)TOGj5!OVZ;I^2-L=@WC8{$ z!HujeC4Xeo9&kILWoBm1t7*JmD%w+mk+2{^K|!$5iKS1T=rpFouiU@?0E#thU$N8tBWtJj=YyMXPR1w?q5SBwWXle&%s_2@tU z1S(~As*Dnl=I;A>}RM>o%6tE;V>I@8I13k*u;dsr0u4{pb9+&oO}RQa$AKjnk# z5umZ%{P{7CU(`8MYv0if zpf{d#b`*fUb>s@A+haG?plPL@_yLanEcla}1pb*NR zJDl_*Q7R(s4PAHfeLxW zGgVS+fNO;lAYXOL)S9rvWQ;hOFIkcz$IVLjmn*lECk1<|$AL7B_V+&yFc*jzjhCHy zkIHz@^5=#mdE<^8IU=e^;#5ZO?())I)$Qa$^9&jkMZB#EHkPzj44?{Ute8NEkU6Y? zVUqwEMq zk9K6WDxLd%lX3N`*pSGL#ltuV@i^4{8|{m3!Nwu#Ss5Q6&6zi^3bKi+B9gP6QA)f& z3i=rnGjx{PYOl%0(9rPhn>YKpvbKU?9|q_M32D*x55s@0=jB&x-1{v3?SGp?y;!f_ zit4ZNf3vy`R6ooA{Qc~{|Kq3CkLW*Nz1aWL(*Eb~OMaU2->>~=`<STOz~bC_{q#~QMa0!8tH~V;I5(ZYO;KSMLnFIotI-xAS1|O zG{+(_Twb!S?Hw{RiMzCKhJ)!XiAg{v$MD^;tmR~i3nn{E z5V~6Cqj1k?>wLB5HAOb>!IRe8kdT%Hu$+wBEb{xaB!FoWU&$rQzm@SGP>%=T*Tw7H zBy*^N-mAszPcbAsNizgS8U?rg)JyVPAlE2=cK}t%5m4PvG_xXx7g8;H?U_m$9ZfF( z!o7ut_WB6c_1abbDqb9*R5Dk>#@hN8_fAw~WGpF06@a!q(N{;Oi~1LQ%)!jXPI|{F zjzu?q<;v(e9v*_#iIX>zO8W!K5JWUI{%mi{f$903dU(=#vBw4%N_;|-nUrkcYf^4m zWvD}=#?p1c!5zfUKqW{ZmPszhKF#g1#*25+RPm5I`y`fEee3t~??FmCf>t}%2IT%U zW6U^;A-l}Zpvy$;q`-?#nl%&r^FeI?p)EwDFJuQ|D#zr5F(4NczS%N{T&ds#xI|=z z+igI0pVc$C2j?U^b?w&8G1iBeyNxe!okm4RA4U!#NTv*3B?n`E^^29EjUY$6ZWb4eL5Su5m=yd<&Ah*1^jhu@ETb$_NLg!Spo4VD70;#Mxj zGO3p&oaB2IJF0m(Iz_s>3m$wJ$nNdy*OO2aB1&3oaqQ!@H-L$K(VMYqBDLU7jpxk@ z&&3LD-n1#~pN7@zWL0h^HngYg{AI-HeP#swUTuhAVhx1?k54FK;iay1SVnxS9Go5# z6O#=P6Ofa8{kjRcZL*p8!--~jtcpA!xq!U)vb6~na>|bVYW1C2;L}Or#|hAM#nQrI zLQ!6Tn}gfcugp!!K^sT9Bql8hnYXi9qWUGo4$u23cqiqhY|fB$o(`doHRCs(E2 zoPL3a&gqG1nZ_qG#|dM{{!SB+`-LiUr{mXBeKA!kLc8qbbWbNIG$_az zjx~2_BgD;)?CfmULEu{ATa(slj`uXID8Vo3O8{3IFL7w@+=ECj z;|ZzP83Ds<+t}C~FX%Vzq%|-Oj;7nm@DA~?o}*r@$L}v{=gytgSuH=akM=Q>0XSmr z0*005U%r_fM-mOCs;Opgv$)R}qzbx4#>$?M0V5finSBQhvX?xZlIs9(E6XkUqkd{r zyWHV*G4Mc#fX-Ke=M%hDdHMO$8zU!J6meV?3kq)T8Zwa*Tj~P7{B@5Wl7AYRnofZf zy*6kmr9s-t566Y4A@qy3SE8{@HR8OL@2q?g`NvtC9F{I4BA`g8sM zeP_WPxZP#qhK$a-MII2jbLadM6}ZXCygu>C6$*l;=}A0S!OW-@49v`&1u96|36}g5 zM_DEy)bb;J`2xvY0kKqO8&n+$8zW0c2hm{)a~TRw8oLv<$x%K#$%QundTK=ozGfOy-x7y1zaWI&Q z`WGbZH?$;HE3cnQUj(2k>$Y#Cc44O$8W)R;Sv=vX-+ui4Y=)aLA?rr|^kOdkU46Qi zErb42)3YtBoB&w1Drm?l=YqujPjvr zlYo-oE__0`2YTlmimjSEpvYzBhI&@eK;a%(2+lWk`6)mh0kr3l_f;fc)itGOFZG=k z3|=-5Cx_xIApoh6bf}|}zgDxW*392_x`lt)CYrR%$~>ODc;UlY&jwOby_e%xBjQeNKoY5ksLJwYxVWp}7NkA^*gGDxA|HUdf{!!qFyUr z8V6k7opd`Ac7s5o{TQNkrO<3_mYkAO@h*b?#KoE9YMXlXf>ef%wJ ziIGclc>k=OW=2NiR0OZ1Y?bTCQMu=ry)1v?ON6B_wr-G>BTcSPp;FaC$iGu zkKZc`4s>odNPjbI@gc5y{)AMLco1g28K*i8)h{+8B5thWK9d3Lu^;z32JCTO7*%tQ zBKLd4=6U0ZEM~)obu(Kt`}m{hwsl${4*HKiT(F)T@b%rq^gsR14sYY)O=r{f8#jz{ z-+xyfp|OE$GswdA#j})w$7vvF0>S zz3@o)GEXIneNgqw`2qLa^{Tuw)}`V5l&+3RmM&{}hTk&M<59tErpqlFQYu+@yJu*N zBNg6lQ#nww*T`p{LZhEOkh{pwPllsO=;c^erd|Nlr&Q^eyNt9&UkT-VJ>VrFVe^(P zHwHOG^U>JbBvghx+iJ*zy*GbMa{Z8H)+GF_%$0dmpxLowN|Jlk;me^T>aWe+9UmXB zFznUqvhV#*&$I68@ZSNENm;NnoCYGj^q=mKNs+uYL3BXv4RdR;L-;%4ET5FMcpLn6 zZg(%2>vP6&Yu}}JKknec=fvB)*mwjY(wH-MZeK$~ zCDBnFL2F?GzVX_PWalW?lVm>TOJ61M&W`ebAuq1b}i;kQ^nHC@NjvDKC<{F zuT(pB9N`D&)3{krb+fqiiYE1+fxiA29)w8!DP|0wty$3l&8lo8PW_!z-j3qaFMg3E z90D|f&$58Rj~6T!5f8fgM~oFCanI*!5W-o+D`G%}CTBs#jr;D)gu`SYC5+DmNv8-{ zptgrCYT8F64mfx2+(1NdG%Z7*%G}O3Z{A1^4PJL@)?Y1!ZpheUWJz;EEGSZRvuk}j&W;-Tk3*Gh<6feP8>fzo?OM}W<0GZUW_RV z_dr7GpO_Cqy=#z=WJLUds%)sl&5;XumUIvzBwJ8)fZMr;vVymKZO{lB2OlKKarwSH zZ8vM!_^f!BqK1}@9`3;ana0#W=`SCmBdnMP-=_P?5i+O4zd3*5J#wwB#FCuc?*Oa9 zJxDhrQ(q2HrMCX0jGZ7 zm@X7VnOL$b3GowQ%Y!gEf-ZSWi`>mX-%ZK+&{6#X*nRK`>T zB_!e7ca8Fir+d50M2r1DUU>iWcdOsePBMofh13xl(Q{l2iBNqU9UaB9nG>BazKLwu zvZF`H#3c5j6UStJCZ9`stx&GrT^U+i_x5cb9O0exPo|Efr_^;JWZ(x7*xgSsB)S!y zC4dWMFrPC)vRnQyR)Bgmx9ZC+MvklHaj-t;@sbdX9ht6t4;7LTST0~hK5=c!xtq*W zON1+Z7H*{*PVL9H-9_3cJ_hq|?OPXnZbb(a5_kDtq1>zDNx}^Je}hwhAcNz=(tRJL zc~KqXxG)*gPDiKhk^yMqBzDP9!6Iwxs1L`YP>_mkTeh@6_W3z0vKjG)moa?8$591Y zS+&#{sk$N+twu%(yA7OtmrTKhv*=46CrUN4k3q~%0X`w4x_H}I4X$1+ekg(+&W#|_ z06jKJj1{Rfd=W` q_n&*8{r?7k|MOn{ALGu=o!nfPm74fDkOFFGD96~2PWpM)`u_o?^e&?S literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb200/gdn2_b2_flops.png b/benchmark/linear_attention/results/gdn2/gb200/gdn2_b2_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..985165d97cf11d94c7c1963615e6082638083660 GIT binary patch literal 90061 zcmdqJc{tW@+b#YfB@IXvGB+4fQYurXkTl3tNK_J$nG6|AB^is5sYu9_iYQSDB_(Aj zWXzm-$ZW5xzTaoR?|U4-zkhr0`*?bup3;5apX)lW^IYdz>%6WXIlOP>GR|cbMXlVg zrmRI#OE*%~BG;u$@FzQ`Rv*LvNIR>F1}Sn~| zudJx!67%bavz~U{0{xKl8=hqXcU!363afsvG5gVA@cW^`@06o5=eaWs_Vv|@3}n_T>Q;=;<=+@hC$AV3|DN7q#doCoLqNf{tk<}%-H5ucb2xcwxK#> zuh-?}%WiEtR6b}cv~INX<6d?3Wz%Clm&?O9cR5Syc)!bP{qfxCa&+`@&ue?7+eRmQ z+ROriFUKDzpI&`yDZjEOy(0EuO!)fs>#vgzL@;x5`a2Icy>4veVq|1A|9qDwCMHHA z_l4Jj+{D0-w5NvzLSNzUjvhT~Yiq0U@y<@o$7kN3H<^5tb$zv9f?B8mXP=m19y0?2 zgZYo=JMT!GOUONJYrAcBZf>}(WVy16%Io@i&Yc$33YV9xJbChDh}FURx6-HH(0%*% z&Ai6Cqdc5l-c#1l(9rhxz=_!33rXJ!J!Il!U1=$H33KN1+gqYd=Sps@UB7iJ;~uv$ z@3b__%)SmmD-%)ULfJSuk7Z0uOe=WhX4k8x@M>ymrX0CPtEH{Iik0>Lh(Udx-FhV@ zC7t^HNm_aa1_|fCKQ{kzpW(cf)y{#v+Pu6E^|A$~$NQ2WpV)M(wdv)>Jol;X2wO@a z*+Ec5WLejzSnao3nTAplR`ojVPMO#=$qT>O8F}Rb4#!G&m>b7)ddLRnht_4e9yoB| zZCP3SE1yN(wHeV(GnHbgsi}E(?Mp6Qy2Q-G!^lskep~eVrbBn`>ZEG9KDU{YO6+lp z`u^jEbGA*h9QEq{{#z%ltYVY1dv48g7M}aMX$8mj4MoLMLiTQ3s(*G@CFXg~NmF<2 zJIcGiCI>f})Ergx;Bgq8e-PZa*Ue2j(=h*%w>L%B`t4h~!-o%($K>MTvUfVKFcB|a z?K;##W8GA6eYZ^$A3s0Mu3fuQbW&M*>#~>}92^4s$HwK7v-+%+mFW)0%LT3#GkW#q z3k%sXgc)C`+DQ6b*I51)gpZmH9XXl|{Jwc^;}5$(-@6>5oIataFLA{`s>3-=?@-C59XDxeg4CrZ8II!+SZmZT3Ez7*j%)>v$NB`%ys1F`|r8J zN7ExIzFez@2N7;l`y1Flgl`t4mNGE~2eU?b9*W$iNHIS-`%&w76Sm-dme|#+S5Lq7 zTO}YTXVfJ}`2}&yh~&szoJcos$juNk%zLE%BPK@1rn!)9{k|Yp{IJMeWMrft|K6)9 zR?efHhiDXZ9_sABfB(Mu%OZ_Xxwx{%cYSpQSFT*4_!zk)Lj>o%&f_bu;g-3;di(Zm zi}=mgl?9$(=oeAiyLVJ{$&}P+YoAu^i5C}yPcL7#to%vj$fZAf{9u-GV`Y!q79BxJ zNxh!gU-dbbO^Y+HT$+&W?3_DfTz3s`xRjorKF32XNvkT$Sblim#|uMlvkUR!KUr3< zF1fk!fO4u<9ByZG=1aPNpKsHqB|bhrBswo$zASE8D@uMYdbf@0HAe2o zfi_bk9prSYhY5wYn=}-86}2&5T)TFy#JMkY$R{q-W5Pi^d;C9tJ~Gzdur|x+<(s$H zS35KoW%kyjU5nab_NJ+6KC%DFT`bL778Vx1aCQD$?;m;0ToBcJM5(H(PI1=@n!fR6 za6#(uF?`OyVFR6IZMsi>zH6@b+|iUHiapjn2+5KY?)UE98yXr?P*YoaI7afG)1R}S z@TJJZ!^3Xm=YA}(PTJ9x2>DkCj;Att3%`-No?6r_W8#(b)e%uap4EJE_E6-VI|7oD z?8zGOrLD!+YH~Al)7I4ITHUxMsB!x1gG206_Gf}7=7hhLjjFkd;Rp8x^DL$!qoWg5 zZmdZ$%-@b@uUZPD<TNc zY#XW6wr*-3v-9{?;JV`~Blj|D+m0Q3k(OG2{R)VTWH-#SrD5Wcef23;W)(a8QeIx( zSh<;VGaT(xqg}`u^wvL~FQ$ZaQkXe+TL&y-6CuYM+fVWmEiLUMlQPC%O)uAZEzIrS zbTIO6uKbDSuCd{~W&EEJY+RDI6!ke>e}{zmXUe9j;57nlxaIXCvM3Q-PbiJ|){HmJ zJIppJp_YVMPQ|j3q7v5J`7t^!dLcMGBT0s;fCL_|cWzpU{1{{1@xb$W8Bl{1(>EsEu#ZrUbk=K)dc#=Lu4 zYq+>XEvi%Q<@N24klEuDE9~?1SeDTq8im=RVx~9chGeE(3V?TS`K%` zd--EK!bbwv?G_nG=x;9aihKJiibalhI&aO{T)TEv8U@Gh&x&?&whw$`-hZ5lmdj+? zc;GfG<$LSaEtlc8D_T(pH)*@eQG64>z9)Vaw;1JEFD-4Lldh*pF(b}THx;-FY~6a; z>IaV2GG}LJ8if@+vU*9f*vouu2CP~ptL3ohP>t?oE%zrKloYiKa3h!j^hMIaYnS-!Rf!HitXw{s_3 zS7m(J%ZtB5A|scYnVCJctUH{R^B9@TZhqEP)@^L*@bIvZQGujE?s*15!DP{fmmV`~ z>M~F4H8&S(D)d;sdGqGH{QP^0mzQkYw(UqmXK|jkmX_^sn@VGm7Xb(6O$AbSc3M6d zH=#R|&NR^1xBR;DC2^BdK41&vjn{a@&ri%W!l#oD$G!>Qe2jVRT6$`7uxNpGhbhI; zH#I-eU}I3HEhr?k6h-pwhX}D#=g$WKc#-qXX~r0}+lEI=OY3v82Cq)0A%~c8;aeop zRjB_vcI+Svf~EX2rT6o;m~jvg*S!M~TRF`RsA*_i3l3%)Xw1KiAY$EOz&hSncP%

xcf7|!W`p1zx+f$P>Gwt}I`EC{Rrc$mvbF;I|?CgGz9&LMfX1k=c zbi%o>4}?xUf2#hYv7mGB-o19MZx;JryH@<}-BN-ks#8{>9N+FA?w%PR$=1cQA?b4l zI{^WZ5^>?zH=jlHj0&4>DT;)9{uFY$_0=-p@bIS>OeG~Httb{Q6TdhrDk@sq+Gr8H zf$=`u4;(&x_1UxCJhCp{zP@ykBGq)` zNE!L2AGMAv(>R2jXn)be@A`E`Jw0~nBx1?d(NV`jEL3fqL+8QS@j7F*{reRa6ezdf z1F?zhIdcKxrsb?8eps&4Va@ho@pjgd++FbUB{{rpNaj-h^zY_94)fOiz+Tc&cXnC% zW8;ld(*fA@a?f)-cj~)pS(n*5J2h(S>|AGgvE{?mpWJspXZDjnsix%`&F-Ut6yL!^ zhg6i6eQW#Qym_OnrFHX5Zp0*Ul+-}tzTjp1 z_U%i`b?B_9njQ{~mN<8l;xl+=y~UTgtu$~Qtv5~#^Qlv(M8=a3M&0b-EL^f=iPXag zeOuc{-QThk>!$(1>n7db4aUs>ZuTnq`To`wgn;@*Z0B(2$91tmDu+FtA6&b3tzPm| z+@8%MIw^N^vw*E{yl@`u-RWr+E9QLga9qZ0a!M?o=Ir*U`><4qcxlZCj|!6ECh}t-IV7>_bCC z)ean3L?OxcWf*(7Ob+qaKeJ)Nksj--W5QWpx^iX7y}j3%SFgT|ie8K$>;JW?_!?6g zz8wvv`%p*m!|g4a$QZINL)0}U-qPBPY#3W^8ia&l_nWP?m z>H2jBv{<&YlXi5A7vJ-knaFb)765!p-Zk**m4f476E`AhvNPJ|2pTD@PlwBG->@*& z^Ua0a)VA~AwowZG^*Ne}sw*c^mjJUSJ0gt3>fOg{jy-z%^le+oja9t7!2rI+H8r}L zTZ?WTY;dy&+E&GS4ENS-#ygS^O){s7o**u|l- zv9~#vb)WO>Obqkw7we`StvN7TKQb|~BxJa;_wmuEF{s^*-_1--$KN-Gu3fX{Dq^0N zdhR^PNacCV>>|zCw{Loql9G3^XWu^=(NZfpC0=0@S~@zylf}CPt>Vg73m$I45hHzD zd6@7boYl8)-@dA>WCFw~A^2|9DxV!@pA-Z_sdrO-J+Cj*o7bju02VB|u1s=j^+Vm| z$Zm>?ih1@O%VhWL*~zod=%taIikMLWMWLN}RbDOjI%vG6OVFpp$Fjf zNi(x+2%tB$wQFc-Xe2#nC8Zs^?|xAoz#5_1Wb$Q3c2#c3wbD%26Ib514{=UMs$b~( zeUGqU1#aKQl;Z2&?72A6((?Lpigq$%X)temvYpxO=&s=d;Wgum0-+~|T3+itIlB_A z#${CJdw@Lt_WX366;YQF3wrwcsj>W0d_3`m4xhG<+e?d-eWS9a_sIjQLygd5xMG8S)qP6sB z&H)QraYBa1$3H%{Ms@%E^qjU~fy*i&xMM4}^%mprXwIz2?0GJ#Vszcl4=lrG^a(WR zzDg#{D&y{*{dmV^VSdlibUh#Jzi>{&bEh-`{!<`0Cg*1c!zJ{}m8`<)Wec;TOl|G$Rf+qU zR``7Hs%<>Ewcu3U*g>CaSd(r!?{xf zI24gG&RadFdjyjrQAv)?4wvzgSZ^;2X7lLUP@0Y=P~9Ggxs3O+eTtE~3_O=;+xq5PuGPBp=g&VK zSib{&!@=lXhx#U;ILu8ul`P0eN#P(cBJA?CxPitBJ!VdWVj^96Y2i->{oeyWioqY8 zGBZnlCMloWC>spO`5HN?X46gAix(5|kdn>=T!fR^dg3|t?t7o`o;}>;{Nrp~K3(G9 ziXZ`}<)an*^r=L6m-v$cx0nBLYf=`szCS(p^`y0Rro#-=#!Z_}mj!WC-dLZ`o-Zr) zUs`wkly^(J)nJ*wwTqiVJei zVwK5sW6c&N9UWF=u0Sw7B)i*QxKIkjLf}~A3uh+qbKd|^gIH2Jo=yeWO^pDSHXoAM z;+3@A!`(fP6`+TuIA zK;TK#9)EPwCsNAcL3uNhU3hQL;{Zj}9y*E?&g`s|6d`_J=3|+Ln&)NQ#!|FAPDI!& zUAmO{?Af!l-t2PjEekA^zcX@wtaP$}-MfzZPQ-ezRWKxcTKpL13rFRBJ~^Qm$3@>5`{+ z-1MT|_62dRQYd}AHbTt!=y5@Qe#Jix&S{`Rb<>fa#^X5__-U*)X(mg1;&S$^IrL%o z`NllW;ey6|hkaHJVA(eJ8a{X2pEYb&_WCu*p&M((K0S_em;RYPFQWT!gS)%Cea*hA z1eGU&Qi5NPXX!st)qIA+PU~&!@t>_%YLSF%>*}-0r7b#CJH~ zKFiwE?ppB)%*A4Axc^!6!?1IsZYa9(0=JQ|f0{SD&bfv1MrV3u85eyTmV2_@bJm4M zfgP+X_?o^P8!4+#98O-mTbn}~EL|^1#5#{yJ_20<%p6spi+9j@JJ;%3^-y;+jd@&? z(J|f`!!LC?kB~}ey(4#7-T5+6U{uKetTJ(z+?Bd#auGJ4C#FN<;GS`;yTjFHmr7_~Jjsx;B9?JN8urTYMsO;GH}20{Z!Yw{Mj}sWBY5En1bIRiJfGX))-RLsn5YZY-x- zIzB|GH}J~2`=LEwM5zXGrDhMt?YY492~6|F1*`DF{$Ia7KQ=wk&!}}7olcCj)3R3!x2;2NRmMEBtlN72+anru zt(7%}kJlS_z6BC$XMKUNHg7K6Ge6bE8zb*!)IU}P()mb3ZFMy(mhMV?e0=$6D$@wZ z77%4Oz-p&td`&wh1}eJ+QAT~`|$l{c_NAk%l%h(8?Qc`q<87<3eAc>at zc90C5CsF?Nn>6Q6@kaH`Z2g*-Ct0%43}#4vc4*NWQN6p4zh^HkUOvImq*v%J z?xazS*{^ijdIx{O9bN_#DM=9o@Dzuh%f z0HRcsdiCQ6w`qBp=8I;VqPaCs%qou_Kbm`9KW5LxH5Y#mlss$pOmpOZ7f8}vyMYVv z>#xTA)mX5P7iw2{eCQud3DoErk3C**#A{UK8Hk>>70u|jUAt5f7n<1q0yOeqmEJ&( zID-F+(<|OkS0oKY5aPx3zIP_1sHmv4>jH=&3$uyOWiyS}p1Kl>lZGlSE!CeewXj&m zXVMvu6pwS&o8)me{%hA7^kfN*e-$Q=?Cnf(raVIPDEwbTkdr5$x$m*|@Cyvoy}ofX z7N8h#re$z2I9A5FS0a2*h`7S{2Zz?t(bFfQ|3h_hxxFZKs-sxhr{d@3TJRd`4LdYW zcuf4#H+<>FrckP2-nsA~DM`5Z3fSZRzO178H@sf6KHc5jqU|@XU)OwbB4XT09-qAl z{dRS_{_35UwK~EJ;1jYDiNBhQcsC!*toZ6w(Gi^L<3lq)+0H~g3{GaI)rga&2^qc| zF8y3Kewr`%z>jtzGf)7CyjLGS9MD|Y>$=KIN$JvQZZp@LS{F!)wWGX1gRx86@|~@W zTY2%~MbI^y$~?NG-9PiZi|lm)&meLj5k+_4I{K6S4^x(UJA z{dDXQn(dGf<^ax}%LEU{94S~*Q{@;~e)!}`KJSG?G8s2eNS|5O4cyYRFh@h}-(~vi zTQ+BnNrxzE0TLt||0d;7L>sqzxG^PLGQEUdAxH@V97=On=IV zG2}%jeFckm7ag9Y>&X4n`zgNmC%rLJ4usrJ^jerl4Xuqlaz7{+y!l{;z6YXNPNQg_@4Fe`0(*>5LK?c#e(Ln`$qT_ z{ViY`%kRSz!}+K`evv5vbu4Eg6P|ufk042`1$xKbh5sn zI7?-kMjEbN9UZthONI0wZx9s~W!<5T??|t- zwB*M<-f7#HeK9ObC(-m!+XoD^ci-LY>m3@p3M~(bpAsfESH=0r>8YvLm6c)*(^0&Y z6^^cQQJ=}XkizT&_J#y$fk#0Horpt!dasahU@b4FYexqcBOhI*j#^4Fq&Y}iNwi!F#^5; z>LwWESUk$f(JkPeoStq2`NNr|`8_oLM*jW_x;VB2gimmm&|0%52E^ z*=Z0LJ-St}?xd6aS@b;If*Nse!-TbMXC|y%+}z%Ps8lGG5?MI4Bf0O3*LlPRBsexS zH?N?%Re_udX3JHwjxQ5aIh@HvXhaZ;6ormYOI?lH?)fWjhZ!S!X9iP1YI|?*GH6*z?G;;v`8P|=72Ld4raRxnwr<@G=m4R8r`k=38UJ$P1py~QK$#d<9+4IVxV&Hl~bb0)0NAyI{v(i?d^B>f{Dp? z=sJvyVuCo2l(b8zwL%TgP*W4qOrR?$C>XOpwvenW*M1r2|7=C%J>7M#TB`2uvh%Z} z8h-@BM=rys!+pM2S69o<*lFV!J1kwcjFuv84EnK($MabH5eZ=g!Y=DZy)<5>U;X`~ zqH9i=zc}b$VA#O1e5?dLUTbSB)RI`E=`?g|Sdr^ktoE9;V?vPBjHYXJ(Q>gZ7(YI< zKX4qPT+-yi@4@D@L+bBl+0R8+Y>b#<)bOwX6B##_*5f)hH1uk;tCFw`=OKN}iq-HR&B105Xnml8zfqnU=+fydgQGDto5^!nu>{H#t1 z3NXcyrQ!76e?++sq=U+oH_aj)&+giAa7e#?{_c}`X7A=fT5<1yGhq)7MO&oTC8#VX z;R)m}6|_IPk{!%_DWo4TzSlqk-9v?vY6{-^6anzHEC+1_MMWLF3KmBPqLS5~sq}3l z)6*95`qqEJk%#rajpzFH>m7Tm87@xzdJSFW+Pin}f~QS-QNP)=A9nuS+5k@BbYq^K zUV)2v=cm{kr;4whs~N!Zo7X&K1Iuv@N#IpO!*QEMHSsJ{cfFwPVpp$%xw8CG+uUrN z9rO#VWah--Y&7uQf}ceRh*2m#X@$;ZXDu4t;sV!ETXf3cdpG}h{hs?{M%AR{$P3xdQ4}7#xC>E3O0l1 zrrmjVzOSrvw(ha_cvm1Bv&-Iq_w&FGRW$wEK)T&eN@|Vwnx8V?&rt7fRMlaZapFa8 z+k)7M=l6G|g1{!skjNiS;eQ@)bNRWk)$?<|jW=xEcnXZXq|4CV(w&OqJ=iw?7Z>n= zH&FJa_x0o5&C-pFcmVfH!CyVe;01R>xB);_|BdV9%6#egOx*MZ1qEfj=DDCdRJk$G zw|H3=t8ZWl{2B));i#CM1q(1OICY95pl@v8hs=j*5D`}(6yS9G0Vj#%en|fM$@=33 zt6rypMh;R-huh1BXQv#{ADwd$a!c&bHvfXQ0cCknU6%2&{Y@b%3E1$|Z9Fz|vChD~r$wuj&ay zL%A6l#+`eDNBY7ZSXSfHAIE4NkFL{(uKDOx@e(>Zx)!87-npHj;Xs|HQo0u|IL^Jc zCn~9&d*12ZA=FH^6fA}FpR>Q^c~K%6pgA!*g$^Xw99v(KQ{-%Z#pAjM+P zH4fDie{sRt-3=c=0s(>QfT!v1P;}gBhN~tn#PZf9wuEEX8k7o)J`r3Pc{aUem_iiLv zdi?#H&-eD~_HxO&vtv`2P-ERydr50_^Y7_5JwQZtk5(ilxDY599M%Z%@hO=601@L4 z1_qQcI6ZY!;)_U#M^}?(6(179cQnf=qq`W)po^<(YW0W4V>&uv^@W|4@nY0coa9qy z&u*gpz|y2uFK1#}8!n>DLJ__HE%Kj)JiGMg&b%7^?9(tIz&oQ;?m5-5mPR4(#S6>x zUM;U^XcP#m7G{Mum}1sSg_4-91n8OL;p^+`2P2E{JU0(drsVBr&zVC>h2ZU>|NBcr z*#2bK_+3BWK8fPPsaM8Z=Z+&xL@7T!mjBP6XG$iJkYL~daju_R_rS=_?YXUk!)i)7 z$D&60L<)f_9twtrD!w-CK_h zcO^H<1tXBwQp%wMDsS<-@N#k{B-iddVVvXf@fbQ$iW!~JYQPZP)LVD%h=5&6f4m1W z!xaB$ju-SM&Sq^1*V)NzLYv^%*`OxD4*QYkKXNQ1+NPM6Ez=VD@G(brch`yY=hK0O z*eK=Vt1ETo>VQgB;UN&Ix2b=+KXn$+1pq?HDjj@_miq&PLU-3S!4=7h#&t?gKfCrL z;&;}*|Ck@y(3O4nUC+#M>rAG>rkAT>GNJV*zL|2%&9Gw-#Od#XmLZsnHy9u{Au&+} zUw<)m>dcu9Xe9XnNNQ>W0|N<@ar7-4C+9<*%;3t+`7Z~on+s1=Er2ac(pkg8A^_+OV4ANaE13DJU9)J8uj=RNK(TSC&+wZOoY6j65}wG%x8GBBIvaQ*l1r;rRF$>txn(}_<+*tAW;0MILT#Y4M(%bb)?J=+U9k(aSGh$Pg8TA_g{4u8)w|2)IT~P(o|L=lcrB z%&xC?OPHuW3ozx8JN8H8e;(NV)TWsSBGLESLFnQS4#)aGvuPH2kiEnkIjJ3uwO+pc zMxz23A9T@gpp-HUR+bpz z?go;4_xQj;t6-JYAV<$P6>z`>_!=FAX1=|tV_)scjp|{quyeMcQZ@(*7Q_8_JokLn z@y2=!;BNo5VnKd>OVNlQwUe_L>8@HIo>~X7lnIOgX-z58aETf`RkIOS-M4S^$G5B2 zu1%g?Kq}J6&fFei*Yy~kzt#)@bQ^@wQsldA^D2QDSyvX|ud9HfpW@}S@-9C)6uoP5 z;Kg8dX-HTYosEqR^=fysdt&^e5Br|W1rzz`vFQh9HnuB-Zh%WbNcSNtR)z*@7bJs3 z=mh@b9rE}4Ej2^gmkn=V2g1CwkkR2h)!8Mj}kHx3qOcJVZo-qfJzmO3}VG`ea2%0b1kQ5mD@F+`;A3shC5C{bgEi^DnYJ@ zf7n=1)G;yrXC+~DglY3N_Txy7g)RvY&{bs+e2C3J)^nDfI`Y!Z>EIm+;j*&iezO&@ zBoN*SBoYC%+dQTnD`I8Vf+1FfHHrY|4I4K6`3er50c~~Gpcy3n(o-0c^N6qX9v$K{4Yv>q-YnJN2e;q0jLtAGXleSW3Dm<%Rh< zwz7rRcrc$NC*)oHovHPru}Lb!D&Wpp^$;1X54@e1>c2q^$g;Z~7Hrq`8_WhBRRy<2 zpwmxhp0mH}3S1rR@J6CD?|Ke0V5<#^N}oKJnE)=+x_D^&_U$a~rGcyW@88eaP(DZ@ z5x51NUa0PFca^pO=_jOh-%ouVI3;nI2(n?w4Xmq+Dd>!s^AI7Hm|b#!_QQy?w5L;z zXVt;qzpJ{~2q|B88ls6#m~oyhf0=l<|KMaMRsDViL)0dqZk4Z(I#g56L79C$2y8wJ@PG|CMkaLC!uz{^>-6Et}M&mMz*=ND+O zZ-DAEh3_PHu509|8Y1)u<>A#ODgglf-C)WBt7}BtHvO*Qy5(9$54xB`3E-rc#Db?HwR<5-x)T9Ww|q zU>@FIi1)VMu`xwvApM=0I)u2AY?CsyB*F0C2-QeI!`W(>N5S<&q$MkP1jo0R7k5Js z7zP1q(P(U7z=^`q+SPRpS%P(|Av?4XVq!&oe~|qHMZm73oDCXHF?w9k^4g8uyUjjv z0rG%rP&?;gY}NLu90IEyC_9S7`Kp0|fkn*#ia?%Y56g-bD~3iZ<%?gvq9_ytoM=1H z4n*tRv*$$rFWUOrTNmf1cOn%AW07D4y{ngfny8|4)8jRni?_(r&-^uDr}aC4OI*sx zsEYro`NWfOC;$bJq=(N*2i5|`9_aV{^7X47PG)1F$2zbBkFv8B@h-$%m;*XfV*Nr}_gPX|mAhzin*%|h7tT)w=BbJux4Kz!4wNcV@1FV|QI$ZLok%M6h7m`=&K z)!?9cxUOqhJ1qM5ZVyA=pRn-RVtLX3!a>KmY11aynk3-nSq3dPy`6>+xP+bfaHG+O8&2dJ`)jRLH^jzY9vR+bAGAO)G3Slh=dte)pC82+n?{NV+W zZRKd7vb;eJBo3tTo}eN}Km}FeYyg|x`10b~KW0^y&{{kT@o9q6N`SmK#m(sP+GVR} z{v=0xd)ptfQ9$<~(OK}Cro(Y03a%OqMMsV>LqQ#pxv!y$&WzaT=0j8$2f3@>2C#Tj zSGT*U_?{ShWr^52c@uZTzpl*t<=cuD{e3TiyYT4~V?B6)$;*nHV5Cbqmc;|8RqW#- zVhhMP{6VbfE@ZAL?qwdW_)6c&S8DzHmB#-9Kw!ZCygC@pdD0WD{lC8A1^?fi-S~fC z{N(?aNBDQ251CarI- zAgRu)h2ncub@}aQz%z#>5sv7?{Y9HL5Y7NEwu3-M$ai~K7Y@U<2>OkXvPih1$8%TU z?@!`xi5F3h#HwUKZTs`D<){pQ7h01buor=|aU1dS2UkZ)`(H`vrP zo!d8W5@`~oH%X%;8DnUNbV7i|u_inqf)WZL*n+DkJU_U(*WbT$Sk*tBx?kJ1;qSxI zQovnAt-=~DreNeXw$i%9Obd?~;oASb_m z{h9{DTuS(M4zu6zLmiv{%w72qgiO(TPr^v)P<;VKmzdY9jOGPH)!us3EK)vrFhD&T z;Uv<#n2yd5M?fei<5*T)oOd+Ogz^RLs z#Z9pXaY+eG{~pY$J}MF}(pz}(_hE49`;)vXfa^n-S^UM9M+#8 z!noJeu;u3F65|yd!BueBs6>POX(fcM7tAY+=*Exr)*(Sqp8=?n`T^*!=I2YHFSrIm zpIGN`yz*>YX_0)c#(PdPfCGyZH@RBcUdy_J0uezB;(!U>_wMmOckHL(}7yc`sfRyxe zH~|P3t=<5W*j<=Z;4<*R$O<7sz?(p6kpP)lY?k0wO?_7MLe6NY@L3;9TB>m?*uNv^^hpR%%A3Ev@~$y5Nb z6x7wNr9Pwgs!lt`WM^k5={~s)PQP1;W!L=tgo?!f$YYP6Iu9BF2_3W22jwzUbIh3L zk8c$$-D-*$Brr~}|C~WFIw}*m1@nL{p;)m)zw3o zxX6G4_(>fOH5WL>Y0-D4Fm6D*vX)!g@yPz0aBC5%<$*?g5WJVCa8@{zgDONiA${U} z(;|;adQ407R;MPig7?b^V-dkFk(b;WFE1~WQiliOEWrtBf#n&j4RHyfabXNMWui{v zAqk2?pzD+qLl{g!EntdCAdoQwA(-4rq9$HjC^%%;g7~5=;&0x(1S-_rX$5-OG9XLX zb%H=cq=|Zw@KVuII!hd`Ft9$o7?SQQ8uWOb6*xUt*qR}_o}ZNc*MBu}TS zsfHYX?kJ4@2)%x`Rl_cLeKj<5p%;$ze5Hq_&_Grij}Y%S8~2b%m2!KcB5dCE+JKez z-v!tU_7|a@==%K3#yDw&Y4H5J3b=L&`Um?8QiUXI@{i(>x$rAR#|R5VrZSM1qaI}YLXGEx@0Z%< z*nK3&DLo?t`GARd>`WF&V6ptcL7KNQa1jVr#Z+GqP9&s6${XjvBK;Wb-59Y$GY-h1 zY^6mkuM)XgT1m_Pr7w|V=VpHI!yFGrdi|ln5!;PXe` z3%)r!xWu-yi-+W9<+d0bWn)xIx@^ydUwU*)mXLX7AQ)xf8Hw7R+qNlyqqZCC)+Bb_ z&@jkTE8#tc(6hNzV!POINR8g5%(6KB3nmPd&h4CcqG^~PPSE&k7ZIvEZDIl~9m^az z9uN?4^6c5Gprw;`MPXj^On%myEe0n+@%!JWo8t3vU$c11tD^yX7z6;L(olg$?Q`Y{ z8ITRH-n^kD))*`r*)nzx4jPJBM8T0OZMtsjoj7ms0sbAF$6(w>MSuP26XV*gh6>1Jy7luTAH*!oSV9&e<7z}{zhi!C5Aj`L z4&feZ1#uOVDVFR)*=#<%4@Mw0ISx?X*n1P~+R(&=DhQZi6lS#zEU3+!$HVa#SKUPw zm->K)3EUxO|M!{y>5~`71C=1$|I7sBmb$@x>HmTbJapFA!k>$l04&1IL85#E%k?j@ z%cIN=gfP*jL|YF7WKiD1+7D6=sZk;Qj~VqtHA*HC`#_lI(!094l1JtKZoA)zKeN(s zGtuZSO#LP!0%>!aJfCMho`XD5_vj`t1$A>TVrdnI6!0Ksu6uo=qlq<}&(|CFq|@nq z4qBI9#jfpQeIYe2G-a?rrSP5la}z_)mDvw7kZU0Z+|LcVc8%UU zK$sH?p7GTabmKM1_B0e)fopJ5<>lpp#W&L27v~*>P+KH8!M}fZ*f0<_uxIDMIW@%htf<@&~~-PvX-w~ zw=M==L@0-(gMx6BDbB$+8bELMDdsg`@F|Ra!1p22dxqygNag*zAM})tmSqN#JOfUP z@`k#Rnz0&+JAu?)9zc~I=l{|$FUk1w1{N~ovd#Xd8jhn-IOdcNgobUycx0fZpv7J} zp+~1nmQm%-yWV}+$ije;aBnyzaLnHie!?t3xVivrqr4z#A24d7u5h7&1rmw;B(?93R@EN&6V)S1BXC3WH4*(((Q%1oe3m-2sx z0X4^YHW~BjUCqWz$=!nftekwt7aLdo1z95&O zN-pv4b=hkjeBwC74VsY9FyfC*dJLN`#!T6v+vV5O196v~iEO69;#75!8OumH4{2gf zI^U4Hl1S`$dkm2IAPinNG_0eJfYKx_QGWd~B#&&TzN27gH_yjLy?nU`vNE~tfl|N` z(o9fh2=~OyNHB&#Td*ZW#F4)6Ybhpc;d$2(pCms~;4-Y#r57Qpe`)#Jt$bkeiNP4r zteL8{5l6lSe8FB6ZQQbusuuST=QXWIX4f&3{;&5I^^NTbJFA!x7q~ZFi`^J&Ck6l{ zI%UY~pV4Wfg(TBL1i>PClPO^6_q!aLk0umBoCg2cs?*8?gjh%(b?9sDlWw>I2nU&R-48WBm zQsFhahRj%8Q6YM4=DlnmdI8;ouP0A&yW(i;)0@bprC^}#rpL~y z9BLlw60Ly%N#}t^ zMfknQd!y5RRa&})ux2?qPC>ku|1LNbuO;aBL>iE2FpY?tSoTX;H?qSHRQs(AitJ_B zAc*IZ)Dqkx0DylPoEZ28s5XYLAb0GXAlw0#r4={`+Q-c{nfzO~Ruk9yi8;%k&dm7je|?Qjh5jFNRnWf- z=AYi>0ZM8+WC{IR@on2)p@r1V-KVU468#A2Rv|nbfnrY*CcJO&(`-TFk=HVb|NfM` z2f#lLqcIch)5$Pk!+?`^{RJ>aj{Ws)m{C`6z+memw{Z&ukl|o9xvv2)HphQe0Zd?^ z#2zCa_T2Nx00gJZ&(GM9MlE{|LM=B5*Q9>8n*R>bkSZS$Ks{6l!3tqm<7KbkqHaN^#hDxfz%EkE`^$T5(BhC z(R(iLg2UkokD4d_+iRA(|14RD;M%njvy82Y1}`pbp-y6SK)nG|yKK$h zk7Gjm3#Kydi8MrpSD|pmM%cDYQh&dXhrhjy+^hTKx&-o?ZaY!r$qm9$DQdgvXXs-| zkHMx0z&DA}4kcuVnq0BtxuSR@W)_y4U?(5Ej)9RX0Uz+3`=7V067>JD8sKl1ap`coN30Fa;vupZs@?VzNpg7NksrauqN#&u9#hz~cP3}H_E&n^ooPklm91I7)! zYvtgPKOd3eJvuz>1E54QZnnw3!12Z(KkSBD6p8k{QMeU%aHy%Nc_W7)sohE*egLT% z&qinwb#?W)fg9c0Wy3UNzY;F_eAvxOp~+sle7T7dZPmh!;NX3FdXc5)wEv#Tr#Tn; z4X9Tk{Cja9)!pnk_cd1t=VoMNlhHUoq5KOv&2 z_&SPZnKzgj2*lik)`o=}9o~q{h6HyQ#&DdB$aoTIx-cRNe>?eJ0wQ;zvt@$@Q`>O! z{OV=EIDTlBPeH6O2fqLk0G7Y2I6MOAi%=JeLA=H%4`U*74XP$V7hvE|o;pR78umN- zbHw%bMSr$?G5sxk7uX0X4okf|k&!iqL&rt`9<=}b_y5=zVN3dBnc2*lV7F=UEc;O{p+_nzp- zUzBnG8(K$PZm|;PgyCW)`4+tG6gT0#(I_eUGGjqFKbUUkxc?=6bPj|+rtFvQ-Rrqn z@Jo1jI2$OStKjj7ryfardWHKW&*oe2LvG$o(9#eIfAj6zq{&s)`_8h>JqO;5#ujdn zZO$o+Vo6cRe)2?n+F47B1r+lYl!ZiGXhGHv;u1MJghBm9C^_ao+U053LEU@8R^r&BgIOc0g`G=TwXCn5X3R02Bl;}Q{c3|O#4 zOlLfL@`PN*g!jA(7n4Qf0;b=tqD3jj@xtW?xOr+d$lXNU**{x&-v~aot4zF%$dI_V z?~XJl+~8|(H#X3uCBoYsQ@Y#6sHk?DOT?tNgki7r8L3}%31RreZ^R$jcS(s}yk zkbANiL!~n}L%B0CdE=Mv@!Ob49cV5RBfsE)k+(m?TZxF=O8J4V6v=&mW;?k?gfu^z zPt7%H6k_B&3{rmpa??>cUM$y@(Wfbz8pAM?0P2df!3j3tR8LsF{!R;jTf@V%6%(18 zDL?pI^g%SLd`I@o%C^BClQ&J8-yuJ&*}%}>HLrt z87KiEA)#aq6`!G`g01A(2@$*RUZPDDc+MGsbGeS0X)^ikhr4<(7>$`{SU=U!10RBu z&ks?u3>Gp1kCQy1MyLSmU4g?A2*%BmOFh$c+#`G}xP6flrgk8}ZQZ(6y#a1bv8_f0 zLLjZyQ+^yf&I*O2Bph#dwxdqN?Pu|OH!fQWw8BhSSZ{ez5eCk+v$t&AsIr1vT8CnW z$3)vu5Cl4O={IQDO%{mDw|4=v02uUzY)F39?}DdPQ|cZE z*k4#Zy60K@q8MQxIbo>l6oVRo!&U*u znrLOnJjI_sGsD}ch%3RgqOrMT0t1)WU>-@&htSTruMkaS88$2sB8nMDJA{KHxc3Xn zvS-s_Y52(cj7va`j(o?hYgoTLGzqXMfWxj*oWGg<*#+!#Ze zZ_))^XW59$P_)9tjQ@=N;Mv)nka1WSxObgrCnyRffebwo=XQ_yCK5pcOl~$!wsp+ zk#Zo`tiVM$B{)4a3Z_xve?(+LOp+ZW3yoF|RQk}z-OVi!c{$scu2@0*i{ugsOki;m zqdrWR3GP@ZHFfnj_%h_ytgtP|*FZQzA>4}ula279ULj(2r+1+RhM-Ky$8X=i--B$z zjz?X>$hDMG|L~s7KT(7sB?k)>0+B9>wn^NHSg$YB*&E9(5my8_<8G!)ApOa;w)j59 zFe=v68l&V9PG$Z2#9%IA&4J;+C56|YPlL3a>2qu z6j?ym*T_vU4FP+5BdaM3yvFTcG!%?A+C7NX31S&q(eoA-ACrQxMo=5hr9nI*MJ^&^=Xz!K-JDJVZoP+U= zEoA-y;hl2)S!TWT$Xgnx_C--;OYL{aQD@{;Uk%ethrKf!AlZwwcF4HEVQhFWZG&7^ zid;iMuEG*Rf%NoM53~F)Ala$xKAi|NkR35@;ev)2&NLL6henZvB6=N&n;-WS;#OvV zu_>4=o~X;K__NYbFvoOAUL`dO3^2J86(B0~&Yg!W>v88n6n^8luk4x^Z>C4o>Jgt$ z5c30_9Hr#2ZVBmFeR5wY0$+tpQ)2yaXZ7LKPsnzJH6=HLBE(e9&CP@AS9_>uq8@-Y zFu|m9Tg|UbnQhjfgY(bU)b` zKeY|?a<&!jwmxmuZhGqq3S(VeGk>6)ZG)GcZrE~eCXvn9cK26hEFGkXhM!>B*H z7t8q3=KqfXrsWqAQ5R3yQnaH5ScX~YF4lrw{4^b0W^7+~Jh*?KjGTzfkt2@`Uq7h8sRX6R-uRi3!~}FyR6G&cYKx+t2y?*<%=kbMp2sTYk*qW|!>$ z`-j8#!NHl|b9$!0vIz^@3uviaO9W++TQGHd?Sb%RFOH$OS8k_?PnhVp9Ik{!V1$)j zx)=Zk8ss*nV*)7995qSj5#^}E@BuLl1g8i1W5D@5hDja~j~FD^BUvYN6!^>mw@pXX$rS~%HE6@|I1@i(1a935D))b1M(XFZ5ph`xChY0 za6{oplghvxy`s@d&=$~C_y^ecBkd*VH$p$wUff4&J_FDC87>P#L~ykD3F}-(v%;wx zd7v}!lX>8eXzm?W<=GfrMeTt+-dGSb#0i8ZW9+T?)mY3 zZaZ+8`z&`#J15qyvU$qO%M`M`fr2DlGwm`##je8UIIi5(q551+dB7HoO6J@DWqy zF;YGeb}I@x(2zhNEKoKzQ=GK;& zv4>Z=EARXI=e>Hhc|tcH8y~}|mM;<3z6NvD3FLt5NyvhsFkvhoJmkZRiQu7jttCY} zAMOj``U&wlSU>aA!P!4pSe;kE^pcc`4;)=I^*a#4CrtaK#@6E}PSDNybk{Qb8MYCD zuT!25Dr3!s(C;493Ow?y7Y9jqwydTVVUM?+yiKhNesU+ zXjp#Y>UHn^ZzOO5HjJS-xrmtd0vHm$S=EEAZY(L#1IKZvuV*3&`~?detiNn?$93jm z5fL#@FWy$hi49K=E(|CiR4JA5-_pz2R zqoo;c^lRi+D-YhZ8|Y$_M)eK|r@$UC_Yy&|ZvA?& zg=ERv_xw-0mc0_M20sLT8a00Qx=+doS79dEU~{F-k@H7s-@c#+ZXA}-Ck~JFKH@n| zRi=Gtz3~K^4BtWXAhSKJ$bc&^PzN^PcWawgHr) zL8tJEG3L(Y8(#H#qi4+a1&f(@9FzsKN`~ZgJKUl`g5du>I_CAyoXS%k>)r$ zAVek~fU{>n0^~V@%N))2VU^;XcW@BGQxFXh%z3ft_b?^n?1T`gB0d4wy3n?II;*zP zjBhM4GUMPw#i&RSk^aY#KNZ(_FTfppj|4>4Aae&~JQ^AVmqMhE!?|-OP;3#uQT+{L ziqKk({*?Ccd7Pg;Ca<8dKfl!T+AOc4ZpAs5K6?(#;+k!A0_Bvm5>5gk4+6?eFoeIs zK@J-IedbTM;FqIn_^l)uquhHAgE=uO*47umm0;@rdAMG<$%TOoPea(l?GPdMAyuO) zqPeE`0cL*z*s0(RY4i=st1F?Q$K$IxdX4kk|FU{rjY`;ptJKBTEf5X3K~MZgequ_> zC7|!(7S5g98zGAKnFEBm#|^_0yMfxDQyrH5Oo(MUGvTbUoj*-t~FMn{<;x zo00TW^>VStvRNX|?ISz-BBt+j{_Nygv(@+_=f}?UR2BJ9sf|Li384C8mc0nL9-o4i z*>k|x>ht!BMkz;UJje@_gH=z=sVRA$#o{VEF;xm0476!j0^Xw zU6_6=K(|?Dxvx=Zzi1>026+s5ZH&1 z&!Z_%>LG}wF89I^lC88GVADk_$ojmHi8+o6CYImVb8z4)qM4x%*v|)8CCnMHga}} zbB45g7+C}X9?5q^?C*QbQt{o@TQwiX^s<3lK0cSAI)rCtrsNl(F|EwP-pVRJEp7wi z+iCumYdj+BZNKf|mD~BT$>G%F!_&A?$X!ZJO|3S#SR{XfddX0vSMJ(;95e7RspAEp z|5h;OFn_6c+@=x$-QGl*G01V!yZ!?mlw@FV2?pTJ<|3LYk#eqNYscJ`QZGdi{`8cf zqx2nNFJLr+h2D!42F}49aKqWhdBGpf#W?iH3!MuLSyZIA{2o4;-vddFZa($P(Fr)z zEl8^kS3(e2PHqkjB%v-LL?U<=)X`~yGPd#S*E_sdDZorGYuh4C7JV$2nWzdBS>kV= zq&$35kDZ<>Cr9%sXy6Et%>%IQI3p{yV(V~2hF8b{oZ)K+In~ZI42Zl>|N=@AAzQhJ-P96yg~lO zyEU%gA`i;*M2?uwG7t?Yw%>zcF86Vv=iJ-z0m#|3MC8~FoDXpvd@$5G&e%~R$<0e@#WJ6@y2`&HCT|Qs6ce&0}6V};}<$wHW%A2b?k*6x| z-x`>CbAM6M+jrSE0%i?K+zlQp0u#2maa=I(+Y)45n|9i?&czuc|6pO0)fgTU2I$Rc zC<);xp$;XL0(8{PVgSJ%iLY+v*)whSjox;VoEf{5K7aZ|Q^V0;_Xk;SFe#(L97Vn- za_FXqm&+<>6aWTR5D$+1L<6`%nbS!6wW#0GmvtU$3WElMES0ckGRU$^8ZVvi2E@3z z-=V8o>|luWDj0^J6VZi~>@n(uzBO{LQwYz2Dg~7LV8H_*96T@*KN0?0zYIakY=eSJ zI3?vRiWoE!{&urR;~^?Fy7?Fucm!Q1bgG)nUwpvrCBNm(^jU=sN*o16N4RBiD3EN| zprI`_+5dQFoMKTUm?tk?G-9~6VlsJEe*WfjcN0zg$!GEhK|@|& zr8RK;>!pC9ru3YiE#LpCURsl4>iMos~lIcNzo456)o_&_+G z_xbf7I(U#~!c5~Z6o7d9G%&l|Pf&w%sGMSawc><4Vm@Lh5to>?K)g6DbsAlu;z;vt zmL0Wj+Y3&EbTIt+M`smI%DF;w0$?!5eM|(=L74)RsTcJ!m!o%*X7wP-lfQoP_*;w@ zqACB7RjB_IJpMhzEaE{6q%mynl4bVr42e|FNc#cdz9{TBYj0)^?BM)ruPC5JIdz9Rb)UO7DJa+x1{VRFN zqz?FVc+s1d73g1kk9pxl@`FIdur_)?Dqo@1P-C{*m?Px~45&xOptMXm3Cs`;)Q7W! z3KcwoLs8s1)r!SZ=(aL+xQNhBqt_(9nF6w2^YT(F+DlC}Bx)g%=0O@wHg|>l4>P%* zzkk0qZ*+zmRNvf&243(C({A2kLa>cjj)1s@qgr-j6!mP3aPrk3?CE(1FD*?bp@ZzM zcP-{u&c$|ohmpZ&zSU|ZI+G{VrfoiGp(zGTNDG1}{J3mKnobTd5da?MED=R?vSJ5BQ7x`y5x- z@POHi9nqrNS^ zF~r0DUD6@6Q7N5ZSD#z8EB$o?S4D32@r0r563 zSim5s4H{D=zUrfnVcZivK=L7N7A!JlIk_)5cxK(&DU9XYPr0Fp{)A=6@63ZDP4;zN z21>Vdj3-Hiyo=81MW}EgPwj#JOI$jJrSFq+8B>q)Ks5hsLn8;?`$r<2-;|r&kr9FA zd3c<(P%NiC)ZC(!e*F@OWFV(qG?nfr?99&1ap#*8H#Ky(F;@lreRJTx$2)20K) zSxy5Ni#j_yX;!;$7jPNl3~w3e{jNH6F(UDhJ)g`0urN)hC;MdoJlLio#l1u@Bw-U4 z20dgiXS^Iy$#_ zSMj5p8ejT&7;Z@|fyxC1e)g|@qval zPnTwa-1&oYbnXHTE42{um7`e^sVuG$vpu{a=*YlbeP~xT6ACNAj+(Gbu*1`+8I&XK z#dFOO3O41LOGCA}6vpiTuxkU88Oaogl-~k}he>w|K$(!+4g9M6wf|5&@?41HZHQ3C z-KCLE1b*o#LKp@9B!miMEkuk)M4K&J^OtTHR*PRr+yhKVRN@j1wc&7Jq&iLfx_0f1 z4O$-HQ(BHvUUk?)H}S+iIt`1gAs`jE4;(iN+4@1REbCOziARr8-bLx0yfPWGGNKgJe-n^FNCMSZgpzUpb9y?e7c6SKGZZQz_v5hHF1CB$e;?D46^%P?-&u z(Zomx#3Y$ZP9u`S>Y}1A15V@_xHQgR#Hb3DeD-bV4;8k`#hF21Q}!bllWq7T(vv5| zxO>~wu5jp4AZ7rZ8Sg-lM@)r zKI}VuXZviR361%84B8;j)3iK)-nHvW7h>+GugnAPhq-7yRlIxi<}w&;@%4koX8;R+ z*eMN__&oY_sM;%Z2@I6K?{jo^ZRR(Bp5y$RzbVcYd9LiO(@|l6F-A)zRwrY6=vt9Q zH9KPK&TsSzK0XvVv8%i9q2+5UF{^|G8{VI>qPp?RpwT)Z9oyyEB~}eIOm{aIr#lQS zXhzRyu$bKdZWn$prLE9Au%*(zuQ-L)83`?=Gwe?|Q{f_a>Pa%u%-nxBr);61M z&X?jUZhms!5<9t<5|^B>@RY0*`?i{X+m15-teMV*tql`yX>ZRxI2rWy?4bA@h1JuK z7kKHv%hKHuH0UtYxOuyo_RL~Kx%O_^pR2!2PwB5IXp%Yd{Ml+1SCywV6Z-t0^b(T( zevk$xmkUoX?kmobi53!23Y&%Quqogx1)E6tBwh-s9ZcS-*d3_tiBU(9wGW(XS%pTL zSwN}RPDH|2==4*hvnw)xmbE-|;AIMgBiH@VV(7>|c$LDP#VqSrqFj9e8wgRKWWa%e z>iXtq$u(`zD4+nwrF7B#g^*dvzwv&FCN&oM+3nqX*1+Ye(war~Df`tA>|D4IA8XFz zlH*Q#80QKzkycuWd~^SHWP(>@^2xqvB#vN=rnu*|wUUL-{VU2A#`s&lytOCdyRB}& z{N@4P+J%Q|J6>AQmOPC9y6au&||p5-+(v{zkyOk~RQ zXNgPauWYz}l{PA4cDB~2Jd_t5zdf>7q`qMadOeO8RILvlg!YXN2V=+ z#y0y+=g4$N@*e3$Xm#w}9jN`j)%MT5y76c`p)?YERVpl`Qp>wOzx2Gy`gMr8_mi4` zTRgY7&w1=`$E8b8KHdAqUs&pp-m5La-P4zWNN?|RD)~Cf-_Dyiv-c_n*kJwXb(=-n zwO$T)uQ)MTzy5pXxyED->yE4cB>(R?aZ5L?FmY} zsV(n<*Z)SfFm`I}8`{u+dE1sPn@;Nvaa};2Oviu>uewr?lfm>*mf&b`jtBk6kpuH` zxx=eYSnyM8=B1`-4*F$}pKa1Ao}Jj;r1SHoZ=n7n6$phREJzBZR6 z8=OB#{iQLPP`+pq3*`A;!PW2};Oo9ZUGl~ftDYxbsa2Z(djKnd%;G5RaTOcs&V z4Mlz2S0m~MCfOU^Aw*meJBeW;D+eEVki3o({B5%|Agg_VFrIdup2p zr5%@Do}+c&u5NjT$paSR%oXlCHwJ<44tpOGy4^^u>-T*%AZ7cgT0`&~zBgq7}hnjip8_bLqo!&X7ru4LSY+eK~B z($WR?>+w0sW=dm&TyXH2cIx|Pa`0j-#=A^gQyp$CQ{0|C5GL`PF(oLtZFE!-wZm2n za2*?N3*nqyzJkP8&|Jp8MZa3F(1dc&1h9>0NVr-_#z(xy@1kx?j5U>@Dc!!jFDv7! zufLe7@VvBiDo;G`mAD`erD30*oKTZgxsPe~w(-@cl1yu+JaPKt{2%%=T0Xx?wRu0v z@5w8D(UseB@A#Ee5X`)Hw(8G-A6WUdvfB+SHeiQ7jnx-5c(dQ@Cp+@P9k&Z-6Ykl2 zmihYJ$31rx8U_v9bEE&DGt5F>-kZjtr;Ca}x!MlS58~uNO5-GE9+PsViRb1fCf5PX zj*~9~t{c_S_u+4^;z(0kd_ca(A_gRE0r;qAP_U!;^UQ+GMiaEsg{fBwA~_wh3!9VLovCn2Kct%6 z1=IZL6ietJRRdA-ce%}>)T4cZCeUaF-?=jfCyKxvLt+A1x|4Dc5}{QJPszQoy|0=GF|@!*U@kFc=sEq}oa5 zOT;j=-(heE&Hy0g8kB5oiY4s|a76XspJ~PrirF=YIams4tqQu|2`366b6CH8tLeyb zl&EwLSK2t*=oiokD|F3}gNJ7KM%6-ca-P}$a^TS9=bG*r_skqN&Xa11e|y!**`lf4 zTdyc^^y2GX+~1`BYL3|vsTd`=QhrfMQqGqzgO7Q?{|~!KI8H-x!t~d7yDnI_1)3!JCDc|xa<~H?2Bzwid!Hk=xb^5=B*8GMAW2yclL`| z{_fN%GTZj}-e0rs!m3wO`RzJaP4QTs^?HH++O3H%n*%>Ckm7&cy!X`-({*Vd1qS?1 zjV;>yJD2$N$Td^%9|#I9a&kQzXdpUG>EyP#HVu`pY~Q*RYUZRGIUF9iW0Yf=J=R+Y zrXam{SQgPS8LLmlQ{wZr_0}JcUv2T&QLi4>Ce2{v_7*-gE6YFwMFa%KD!q&ItiNTn zwdPE8h~KQlX3TucooxM+nVJv-k}=-%SXo7=omIr*STK>O>8%3FWjI! zA*q+^=G2?Nk;R_5`s}@=uCFl(Sl7-;-!eN|ebtUdS$p!0>wxxH52yP9_LIKi-M($- zdS3Cw+^BJA3)+UN4HUA$C(D%#21CFnDJdz?>h&^XSbbaul0(AL57myCTD-IAM#0vN z!DmmzKj|$nvBB#3`fB8FXYea7!-cvJV`BwfM{63Q`$L91j+Q=gNY=LNQE_)F;tS~O zKIUMoc>VhIr1G@8Y8|5{1(gzKCPSKYkL9)85wOVLdNt^dz~tFI9(g!D(Za>Pm^bO| zL&p|&gHEt%!kD~W%|!0l$PX?hRq>6i-Y*vwTI%ejW5abMQxI`!;>OHzB-sZqw2dw2H}8+O^y z+^Gykq^{GvfL&t144ug((m2ffSiQ?=7mzw1OBDA;a{R}K=@Leuajf5pRW@V29KiR{ zKf3VZb};tB*+ZXKLT+rx;(x|29=1m#SCm!9Rm71Nw`ubafX?V^_EvZZ9zQ_J(GDjZ zhcp9{Mv2wNdGc3^_yIn0J#U&a*+jLSN^J9ze!p4Tg? zj6$5Ff28W99&)jJJ$tx`uhQU6wMIg#$tO(uMFBVuzM{4 zbV0Z9ux)&H&tS$#@7=zL23`LciznL!62=m`*11?0ydTcrykAYC)lKVL{`9u7ty-2@ z`>InrpQu-=c;B_i+|29zE-IG*^VMOEA~-Q@oWY|wr&Iq#JVFcy%i58 zZZ4DAV9{rf{mj!G*dWC_gt=v=oib6~UP*uFmD{vmE?1kTeiPd&D1TvufvF9pf_qBn0cb;Q30Hn~`z{r05m zd_nnwB{WUMVX%w4_p{Ap(Kdzb+jcRjZ{dLHs}`^n9IzGgP*Kwchh&uGt6jwAm8*=`nvW(G7Vhx*znq z;J$tP$_EbBo^A3F*fXbrZi~0m3hT32%14hJxpn=zZFO0jtKbHeV=(CJgdd)DV}PGd!wkTIv};4F73z@ydWwvGSa(knm}BNlE?Vl zJ39&!N+iT`c6D8l?D=R@)$QhNvNAozCVQLFx7rjFm2r(%yDeRBGOJF+Sg2I5fWB#Q z*OI;C81J#Ff!25SO2>``|Lw7&W!UIge6Q2JGj^dPk*86}$o}GS z+Z-SN5(R&c9S7W(2!uy&qOHfHdr-5vBWfNT0R7M|8LEuGW|u`lIa{=NvC#J^vb|l> zUMzl@y^~qp6OlPPrH2}8xcgREJpME_Pur>IRR$KD4gCLYh(*W3nuPNMu|^-FE9AAM zp^vXG8Dxxsn^UtV=PJY2Ab^`nKN8~seBv;GtaQ1e2dk;|0+;X`+e#h(qY6@~NzPzl zhTCK&HHPae2fFWX0C&P-$EH=Aq*_bj68&6Z#bL40gCcxJ+sdj()LHuUn?GdU2<@o# zl{sd8=-fcP^DWOTrVTsHyhf$BrTJ^jE*E=q?1Pw!^inyMnv*3H1I{Z&Po4eSW#_}{ zq2QuAo$Z-+(ok0WBXK;m79?pln&@wE7`X{>oV`^)T4s~FeVr{ztU$EENt?!J zoHQH-xf-K0h}{)}+@$T$;;Ui#w11*cv7sYma?A*SXZ`t~jjG1HTTkrutxO%xQoq?< z?~V|hqUxEKv;!`wT4Yb(1lOS#yeA)ch)0W_gmJ+EKhuj!!6+`Zq5x?`Eko`#DvUch z<6V|2uk8W-@>V;VeR7}4V0?W!V};Y`TVi=+9J^90r{={Wrx{y2G& zcTDfR{&jbEo8s=`H&*+{B<<@48rQ9|XSFi(XXsRvXua5PfF106+>NPW8t2JkE97~W zh}YOAh_-nrHbp-<<`{IC#eU%>z2+*f^SgB+qsV)3Od9#OK6)@eQt8E2-W$>4cE+we zSVvr4B0@rM(c5hO?$4P*r9DU$EoxReJhCtBe!SlwfsNb_=`r=pH~^-%Qx-qajky6G^j9nkxepP!#V(@9L-$oO*KJ=*;I_mE(_ifoBnGF4mfj?6*Gk&jPY(KbhS|MEGDFkeTd2Z)Sq}u3K7gCVZFWJjWf~IXjS5 z;?nEZt*}UOnXqnZP*PQOesux)Li^ObE>+XJ%weaooY%Am{5ra0m#kB-Zg@2)#fziW zP7#)n8u+Mx%MY8KG{0iJvpA!3E1h|1)aRm5-^b> z9qoL(YU2p>GjE6QDk{nPuimcFvz0pbEsvW#3*nfz!8#1brJ^0+;A+Rt#Llel?2Kl% z{1WvSUM@c#Xxmj8N_@|q)>o)IA&RYCghQBv7b-|5R;w;@AI%z4aO{3+zxv#lP_v8{ zKMnKL@}$?xO$|`)CUo-dJgtW+J^-;*xU@%oA~v~rm+zDS(}bsKnv+pg7b_$i#FPmX|ELK2-ff*&6u8ctP$UxU!u?qF|=x!@P%X|>d zUEjg#?a-avMcB9$g=paE=d+jXu+YXBD4&bm(mSaf@W^vP?!@{H1oo!*R3bfUcI@n@ z%k*L_zR7jv_Vt%@4u(j1MO$TEsJ@fjCY=>#Sh&PtBEUb{INdc)yb`L_ShosKYnXmX zQR3B*JJYzo`R04)6v#|YXpVH|s7>^Ij`7uZM}zkiPV|LnApRypt!}%v-ZOHgO|4a4INIzSzCWAUhfH`92P_rR zdIII9rnMv+Zq#7}?rkW17U5tcFz6ck9%9z@4sC{bWsH?;0Uk!>HlCNn;;qv6lCpk2dt);arbDU? zul1X?RmZQcvPS$m31rD)Lndx(y=!?GypTX%SIURO<=Ou{sy2x+q^O6Qs~2;rIhES@ zKn+E=%43dka*cXcE<7_=o|a8Dmq4<=qK$OqfEw(mrsf?Keet3`=oo)PK-~))cre?t zOc(i80rl!p?Xj*CQZNk4!84y!Dkq@gi8|@91yDL5g&|;LG3o^!wU((2*{4b&A>7n` z2+VuT0=EK%9h1;$TCH^Gb^>YP<>d;h^lguZ<0AydO^##&1s8RO0$Iv+DIQ4vX2| zojvZ7X`apU&E^X@>g^Y&*?e>TK*`0y`9;P?LCS`P7wQJ9H!D0jZT~I1FZ9We&!$yp zKg_dkGWT!0H8c3&5v%gciaS1@ekwUxaM7Xjlac6;{`B5&HN0UTG)_; zuo$0L!78cGim-a%`qUCRYuV{Rv5bzinY|Tv*O$g>d|Q z^R(F~HhJz=bzEYFYCR12$ziy~zYnX&DBM>ydM=-jzkeHIlGfp!pDt~;tRHi1*v(+* z4XY|A?)<}Y4mq7!Y!vm%9ReaEC2)G)W^z#cjF)8Z0nc4uBXQnobZ_W^J&3Ct4I!Bk?&Zl$Zyp@Cf7VO#+OjQrvh$^W zyemi}fRiUPVTA$ML2>EzyAqxbXj1kYYY}l}VI^Q_$nDxQ9}cnb5I%jn%3J(?KW`8s zY+}6d#d9Z7P8!2Oe|Xe%)MvZqkaX(2x-o~v8Lb<=MG?VFxILqIYrEZx$B#{BH1Oz{ z*LQP`K9TF{bGm1NJ75G=**M4I!DW{vI!m%|Z`qt>?kXGq&x?nR+>0k&VkiIS?+F@xI$@I|lG)Ne?B=`R zYX+Fh8>2t^_@gE=6=6?GSiMuwa{jI__X5FR&~vdVJF>srR>JlK3a4RgGHcwe%Hch0 z=9uE61?;ps0k6y*aAYUp2@2S9{I7=x1K5~YVKClbnUG}r0ukRI3RW9V0xUV^*rjsS zcN+0JedibNxhhuOmd-Q23+r^kdYPQMYW=y6IBnZ<6|Q-p;u!r`6`#+!7Z&z%=EUM$ zU{0M%zOpEI|C>QuCUl6Z&nT82?Ig(T(0G0(e+?xME|Di&cGRBUliGtq)VKokWS#(w z{)LL}(eJ_Bu}eLNdqNvzPEALA9L=%S0n?!99&0=!R}^ysJ28JGMNSJK0%h8ru#r+3 zP&;p708ajNWrc9^a>Pu0MXpknW871H8?<|ezpjfzO9?G1c*cCeZ=Z~bODkxP2*Kdw z5d-YgDlCj`PD9@%G1_r>v|%!s>42nSjWW^*DfKNVcN=6dBb?lz0BX0RH;MBb80u4T z!DZ1vyJ+_iWA#!l<4~Ay~;rJPOh=YD$h^-`PPrlTw1`0RKawmd_#Yow0B#PUhP~E>qQ>rfP<%eDWo=o z^V-}bRqe{@k4|GpqlS`W7v0^F;2Cqi<%RvE(AcOeFJ2^&^PY?kEU^<~RhGqui(5471Ec<$gjr1;muQ2yXt5}Vbqep zF(kHke+8nmkeasDKCHzU{w5>Lr=r3K)LvxLolnFVYFT9PQHAqohdF3pkRwM`Rs8@d zV(+z?9LOHsGg|hK?c~*gk zZo)C~nI>@21psqzCTx?c`23IkD{lcnO5#yeS>>(z0Jh}`xEqJo0HY)%Orm8EpuC%- z+9xLv=Yy1Mg!z8~7zk*-+VnjPGa&;V;>Fgz_5&@d`}QF+)tWu+udvm*-Zl-}%xe~3 z*+MY>g}4S3cXL92TfTjCuW-<3c39Z0LCa5L4eGeDbtnLBhCW?wL-|1|Gzf1igLHb# za8hP+xP($*Kp#MFb59UwNn{A7DGdGn}*532Y)jCEbc``7IL8{FFX_R$#&d5{bWf+WiQ*Lj-?lB4UX66jPwd9{Ro0U|dY1&dCkWe&b=E|EGVc&e;U0CyENR8C` z$W^qm1q2Gmu1l~k5851WRZ|WFF-w0dEF9lq4a|D(c?4<$#=Sp@Ht=Q=p_?lE{KAtT zrSdE`YJ)IMUBrhEuoKX^sc0>ZdiodEvEKAGKKL=l_NAy)9GX0KusK+d-Hf?lMT8?T z`o<8FTX`#>J&$i;xn;`xzspC6Siz%w+^deC&CU6kO@Zyy77Ts3i~GBQRiUo$K`_w) zcT`{$6B=)AE0NsBbL{N#uueb-*T<5al=v1KYwr@$i-p&-tXz4#t?0c2bgCYpSi3kFNeEO8&3=n0HmwIQre zTPq*7F!o0-Vj$f$GbG6b`?x`uTbP ztOAbpW&$h6CAyZYs2yT+?kh#ax8gN|tlrP=BC0DUffi0;&0!M=5S5JgkNuOFTXQRF)=;{xssU=fT&sJ@@DN-CQCT0GYR=XCBWv>Wu6Tz3i`Nl zJkh9u7?J+V%WdhdF$zmb|BxAPR~QD0xrNrm=8yVbhw#ie`viNj1dlWM)e!u3C-jW@$EmVJJi7ycQIH1FLjo{b-R1j5UIW;$=;rc|!kOkr4-8N7m1q_vo=6d7 zDC~(DM%Z_SAl-e5O`05QlOa(Ar+i9dX*v=mwVdro-!8iJzkl)e=#Hr-FR(Pdxuu8JEMC009nKz@wKU9&AQ^~alv=k` z1>!~^B&D$4{*56{Y{*A~r^l9?#@@0;hRL5pkpW+Bi`Adb<8uP*lYtvT1z0O2trCoz zU^5AnVmll$A45-0(u?`Kow}`X45Bvz$^G7b-1z{C_UD^Qpcnn5St7Q4Y^h}K*KOYxx&@y3Xrl5bim3p@BS1UutcR~jE;}g0dSf2 ziC=%B1%}&|`}J2A5VhAZF_2Co3a;oDO#Cf~M%s--<*HgjXqAXg>_%_D90=r6%)lUK zUd%HCan9yT=+z3ju3y*n?)S19#plRDYNmOac)|q8z#?pgu`fEc>DObC-J@!`XLf2k zHiaCUI1;XvL|S7i$4Ma7cBFTB6LVqn@#*sK|z5vkU5p)VxGhAyyQa&G=Glc`0>-RG@?$MA@3?Y@ zj<#os;)7|%aQ*yk4An*R#C^UaUY*1t!0j;dfXMRV+w~ffE+mWPisB4 znY&~fI5!HJ)+m?--=hzf^K~zj|Nl5*?R}6b%vgQ_*iSIM&Q7bPV-L5~c2urbsa6;c zYZVOG2XUae@gjF|G{T4a_^;ntxKMyES5y%n0RjpVTPvVW0b;V2c@Y9q2!@(TLWLvV zC#W;E^4Kt#7oH4CkNDDa&=3DjeFouK+AJ(KxJtDSSKKU{*UOBWyIm(8iGF0}^XNLk z`^Vn*^;h&$Up<>I*BaE)lqw*vq}~F7w(A1%Z$N2$R zZFp1L=0a$xuc?}XxTA4tdBlpiR2ir1KV)03brPWOVNIRh8A-2C}H7Uw}#xqE-!MR8P6SlE$y<6J0U z9UmW`t{p)=MMS=fgfo`07hQr3j zS_d`{*biLh4++t>?+rsa!^J^Dc(MDzP3LDMJZ5e-R#vVw6l~-Fn)&N3?ydemU;FZ^G6W*C4G6bz zIC(Df^WZ6d75jyk1H?>Kr6I*nA4q@?KrF2R`A`CxW*Y|E6{L}ShqMi7R^IX=PgBz5 zk(S-gl?=?od&piq10>gi!YqCu;i1X`Fha>=%^PL%0gS96U6Jpn1iLf{4jU0H@8R)o zO!5wK$($cgnb6M?Y!1Gbhhd(CA+#c$utYfIN-y+Q#%#z=Vq7va45_l)R z&#c*rAIWungp-o+-@&C&F4)aQ+e%_928pXd=|H?0;Z;MerFe;8+b{Ilj`Uv~pY`ny z@J1MhR!B76*d~h1Y|=zCt&!sRSa$e0CgW&WC6@}3BTb}OpL&CshOg}hILcWiB@!s2 zZQ53^7E6Hd=C9d^6oZ^gMhas97>_(n=$G5zStsJ4!Lk0op^~lf^k#mp{pZ3jfo5-` zI)EFZsTr^xxvWG%T21%thO{XDQSajP33TW+Ol_0ydAU&H$Gbhrh@?g&6cCe-Zn#Gb zH3CYDltBBS6WN#QU%Ks>3WWzbXmE9v-5Yuw6yo98h+Qdz6Hb^cAa$OzMJT&SfDY97 z_!;5xUT{>p0H5}`_HYS&(CAcuKR@Bq#h|&i0Zm>WsBDHRQ~*G$5pIqyos9kptv)0u z3{IAIR~N%IC{_-@Yrhh*fl77S#}n==k+LPeMn146!wJ>rC}aYd#z^aI1H@l0$&(ry z8tgC>1r3f9)6IJx{aY8t;4(%>(rR%FtvpzPAt*Wm!oEh&0$hAcX%lTrBnel+ zHxgY$8?a@G7obXelE4!BmFMYKJ|D1(6(Z9cV8JbLoGmB_V@lOPsSqQ;>ID{D{DfY9 z?>k{5WPqVahY{uYZS3H29P9PZDWb~ETR~a^a|3dopP5k`Pop-nD|NV(m>8YD=OMaInAQYSVH97!;HnM?K(W12p+_3osE`=y{ zG~cmO9f;J9&uG09oAoUdYf3$GM!)fJsk}(sfSVq~ zhhlJYA&{xX(KA|%fgIK-U-?)h4lu#vew*qg0nLNS0>7R;IELFDndtndz$UyABQ*gz zp&e#fwgKAf19086_@P}Ig!Un(^%5zOGxSQN7g5Cssf4H_{Jr14D*Ujbh*+fWf+asZ zF5{>kW#K~V*)168PSYPk*Pi{*Snju_{KsA zX|a-^6{x0}gJ^^IBUKc07sEg3*GV&p01B=Lda?`QcP`cVOJK{NVEr!9MV6?9~HuvWhG0LVP zIlJ+RA&{>`fbva{0oBdShiK{t6?g;7Q+ni!HvC*HHYr^6HrT2Dc$--GUFZoQrXF*+ zLB855q`&CEHj2E&=3wnhp>93^y+IUl9Opp`v8;iwjogLOHhoF`RbR?FD`Stt!?X-m z+tdM6$|@6*SgQpgh;?_!f&eDp2or6I?_jcsV$!jz;^V9`?17l-9-@Xw#=!q9h}7xjk``mHD-pr%(7sQd-d83NJgR>+;gUsD3)yF){1v0U^M+i+~`CI}ju z_ig_Dq9{hYtR$^ZP?H0%Gp`2XFF|Lbex z+TXT1xutiI2FTDt0x5i&Z&(|He8fQghBiUoE_Z?jlu|2>_qa{!R!4l6g+~*Z=yqZy z&gXR?=>hAdruhH6m<0l{W6r{Ahr-?;CGXk*6p4n87)5U~mybO98v{@Obqu?hqMRHB zG>SoFJvD~I)HfTTe3u%+{?rbzl-vHZb?(go8}c2zjv@L%D1s-!tk+C98a&UEoi5nD zBv6%-w*^r`1GyAV_dfDVJAgg221jWK;NgJJV;bm+%xn1J(2lr@dL${$tPQ_5qWUf3 z8<7KH(I4UHNyvpHJYK%CfbIZY8<90Qn@v015>>dwr%MZaUoKQI0R2a7VCpbj2xrMM zpqWElK7FYlSzKv5MLrrhpX4+MZ%GH5K&OYm?UJs!W-ZHQtclkMq~c*V^xP!X4iZc7 z<#r{yKHf*MCYwtDMAUMz0aujc$jGLm;9xNycPo(-M4=dJto!*Bj4}bZW=N!}*ZX88`Q>q^iF4w!Ss|j1MtuwebGIUq2S9w%lpcag1xDe=dbx362z@&BF27 z5B8yfiG*Vu7a&a807Fvm?5IDx9!F(4Vo;_(Y=F|hunms?o$ZWD_%{&xZJf8uXA1&8 zrnPJXAk-j-;K_&}yJ+!NJf^~M|6_Mx=!%jXUGF8y#{Wb`vWHHr1|K~3{i=+%s^Kp( zNA~GEy*;z(4F+J8gK>kOJn=Sonw2{(Ah$;+j#Hlo8Y65#ezrk;KNhC=920a+%yi$) zA&r8*6hp+`C_V>`rDtM@u z>>PlJtr$Z+L{W~KVPVnJG_9RaXyKGw=p{;s#7Cm-&<5r!q4G2?vih(MAacdT_2N}- zhOtkcNRr8vEM`o4mb52=B_;OrI-d!KLyy1fu2~Z6+ta~%v$U#{g<;3=Q;`bGm2XU5 z<~jBT&)S#LG^EY(dfDet$^>{1j72~#5XD2$XNLhjrydi)&pa|C6VtNorxXFcf~I@Y z9;Awh90=DUV5CAchR(vI52yJf0ha>cnj%6f18F*=Vs4d=p@wFJ$HmrPs1) z1F({yUKx;CflW0INIP9{(9{cIgsmwyV?Nb>P?(r3D#Fp)FrQZ92Q)LpLaG0E717;d zC1Gg^m75T&BwIHOEraO7mfQNCEG#Nw0+#310rWcnI46^fi?+V&c)vFjw0R+Hhs5<# z6HEr=X@5hKY#-IeREA!f91fX`s+jQ484Kr-yQ)_f&}xMbftlY|BBN^_^{SgHWZ%$B zWC?_XCQrKtR;rmQuYJLG($WOlHEcLQ40GOlar=}fYO$rdSA~R;#t0ly zfHsmq@Fh(2iJqfVwW)@O^RWtWEE(QN!C}shvi*v}HJ|vr-QOOsN7bR#_!zG&C||<8 zv(e3D9AMoPV8t8Et0=W!h*ueSUnnOq*njBhd_JlT-6gZJncc-E8shN&Ty<)+ej{}O z_hP8Gs0!S;4G0s#Go9Gn!6@1u;)9d@I=UYdm5kt~E`XyKmG1!#Xc^jpxQusTJPLE^ zkI^+m<(dKPn{D4oYFBfO5U)Ea0OUDj=0i%#9z2T~4RMLi@LE7f=o>5swd*#rmji}Q zDN-hyyL8JMa#2*E4`yF*`aM~gPfcdCP`YKOv>?so&e`|#)mzAo1@4}sonXsnAFv@j zZ}2H*jZ^xazYx1}2G9Q23PaX;!!M|M*EB-9UxL$0E1hbWe^w5}ZUUvO^Jh zdqHE94}v^qeLp@Zw1s&ejuv%+0vdpdk1m_E_~#kYaDIzogUSR5{+)t zOgeWX-uHr2YreT>e!hrHk2}9}M4+`?Jdih8i_oA{g6MY_m(WJc8;Be-AB=so&(T`q zPY=DnK$MFQwrnXO&6ERZPe|IuM@wNUPRdb5k_^PItJIiBb`R9I0VJwWNECkhqqHEK zlt3&tdva&(ilIuT(&8pxlgIc9vqyi{<8{=LHC)&2 z;*WZN2WcN=d>~bKI$WoXbqf6~^Ni9V3Iq86Ihi88q z8&u+lOG)9n6cs#HOYI^l5k$ystd>YQ>Lw(QF+k4_7L_2VuO$ddaQfLGMO1Wfax-V= z9fYmG8eX-T(ub+xilLMVKlg%WA;X6Qi0F5LEfS=+~%wwj?E`(HP(&Zt_ ziNQ@VR2sY-7ZaJcHnWyncDBre}SoHe!mG^wd0Tx3{X+88nW0IMa$dbIejqgCW90SEd)PS={ejR}6{UhwF*jOUQ{Vp4K|JXvfw9@UI- zX)fgT0%1mtJnjpyyxhW5ltV9}5H&(fIS93esG@>_oXPQS;&9Mim4z9#VFiKcs@llH zgz%JqFPZG8EKz7+1!?$b8$_Ob|4TF7A_)3!biokOdEh7Lrlz(LF`nBS^}cj<@Hf1}c?corUCaI%z9V13jduCJ1*fG$+loLo!#B(P+N|%UuQ(m{06kqaw z&|ac!7cRx&e@jq7s1TJzge2J~Nz5mn8t3S#tf)aYB3+oG;MaRf_-%SDLEBAhSroH~ zi36nS{VOlktV)#hH}gQsk?MbvFb)_qQE!E=-+a+XTA7iDUNy-4LhKB^o^rXLK)NF~ znFEt3T@q{qe=Lpp-3cBq2wn+eikdw}Xql#YkpUgNTyd zg%l-;fR4vOfQZ?pN^w40Tqnke-(s;z-9{h&K{iEd7&-H)@#E_1bDLy_o^ktSb8hwJ z-gkSr`GyC$xT(>{aFl{%2+*hlydTyOOe$3z@W!%2>e0R&*oII?ttP(bFLkf9Ejz+u z6EACgm0dj;m81iJc8|`V;ZPl*7vODI|d0ToRK22l-`T0=JRVQy0jc1!VpJkHD+19jFxR z@ra;nhki_t1>wHTmgAQ@U&XQa(O+-6c0tgFN-|Qz z$u3KP5kmGBq-lYdQFw*h;x~wbyC6E%cL9K&H(bNw zKyw@+ftnR4<5B}XVe59F2UB=VYXBYcCJ=8) zFcXKX7~pxMN>H=LI^DpUKt5~+>#g(kIfe6)ZurOEfbOO>NFF82Cp~{7==UL+o}?ed z`}W;!7{hsC+w-vz@4^lxiN$8CBTk?9n%^#;@91igz~LQ*CNXh-=!|K`XD6GiZdC(n za&qC(uBZBTBTG2*%Y7$&etAbu+F89q+)SM&8KC8+4Y%qo6>F3zAjxx zz}g!El(=^Zo&sd8SW=ECP67avMQH(0^lYfU!tSmV6tf#~tEo=*A1K2sWu*oxb0)-a zG$V1xm_58w&dz^W)%8b}5!R`29Q!=%=o zqZ9}GYhePr!z7LOf3WxFVKuMq-+zX-9UFU_$FQwX2<^;8+cvbykTfDwnHo?TGNeq~ zM#z#mnTn(VDNSU|4q>N38A8fdDpQ4|=XI{^`@X-w-7rm4#+e)62TYOezYLRJ05U;gM}Iq-uSDsfq@^?(@e>Y|QY? zGv?Ssx6ZBSFc=eP3-b#MG=9uu{vqnMI= zFN*8VsCL(rYgfLTUQ<3@se*0Oml(+!ckaNMqt{+1Q^Za75G_7q>`+s4Ysc{c2J}F& zRtnhJHt6{u41Pq4J8D##KsJEXAQ8RGqkJy-soiY5t&W8lUlJ9RZMUanTLKUtftIEF zMmnye@N?;ApWfZ-^By!(C|M^{;XaG^OUXi?=t>DShKoc*6orS!`$CFdB#Aw?ZHOLA z+d-lV)qze@dBsF~&vn!;nqXqwlBt9FfYYJOdjD04w`U;+c;H_x6vy(Y)(%$raj^13 zB|8xPEq73F-feNWBiK;Fh?E13-EXLi8QpDbK4ZA?Bx5*wBH$3nd_g zg9dWQ7CZ?bPUHa*#g;NFu3Pmyvc60xIpnWtGLSGX1Xecwm~&tVWRlRd`|1gwA(nYj zBgnIEj<|6FW<_*S35NJsz1mvKYV37#a$}udo8>Bmm*JGM{W28xhqU>}&>NE03BIKx z(|881D4_#Ce!8*+EF*uB*QL$M|3h)%kJ0A8EC-FqENJC2e*XMd>EFQ5Ye*?1!=hPf zG8zphNzkfn-}y-08y(2vo#I0_p+PJA-eo27(|@o zN8SSsccYv4W^Rv@{syvfu#V#KmpU&IPWSX8TxsqCc`*i@iZqu&Zz!vl=Wz#H$VMT} ztaFEhheSLTLS;`_+=h6do!V9{?q}IiMsL07(|t2QP^eLv%JR)pqd+?nDXSH`0c2IO zO`+ULUK#GYoAU9_N7jS&^@NERxHu$Co$VU`oJx&O zae@HO8M-YM0jGO?A$p~W_e3%_tS04^+oIT9#dfNYQp~Zu8~KmrG@2 z$bOj^@>+EYVysm*(P^)^?nI2^nb<|tP!BypWyIV!@Bhz}{AjaIOGj#flq#_nvae-+ zJ`X3TNR?V=W`R!7zdv@OL+jJwtALV9Df-BGkjKypBfw53wJ-Nc19kRC_x&$K)X{0O z{m3h$5ooOq^9u_0z|CyL%l@%7iFhnJL-$Q^3tYWp)p^yg6#caxbR5P9BfzO}AEo*! zZJ7(d!6|w*D@MNMGBX~0EfYpdsdi(e=6C|q`H`Nq zr;Rr}G0jz~tW?r+sDA~$s)h;vA=wjVsCI;U?H4#sT4#SW-bTp>`F@_svBWTjO+1Fsl)MLm{ShiamCFsh7dLrR;?X`ZE1{oPHj%?~C+dff%_h zlQcK%#0)7QK{Fm8ZF|H@pyy2nG=WnTcPgm1y=0jtzuGE|jtqyyM*?mRFO~$sx2`?N zUuwUk22f;`)gY$~n{ivDU`8~6g5t6YRR=^Ei2yGihIFu<;udqWpH#gy8T`UuXioW3 zsSXfwiY`!xq-dU788W;o=WbZw+}g*LSH`;k69LJC;31{TRDW>+78>R;mLnWiV7HW- zKbx=ZS&pg~P;RGOtJwag%)SF17AP0Z@ZN&!*h`*2caOu;R8htu3kxtZM>d;fmoD3! zye?THE}((aLX2r3dd{mQ(2|*I^5;dE{iKTvwz_{xa<4nR1P+zGyfzdGHT%YGj^V$7`14f z{m?;|;Xiyil;1*q2tZI@c%~2|OMkG*0=K~0twsvjt_S_{Ei=)`ly8bT$?gi8!1n!V zR+|0%kX_FCwXh}|;_Z46b=6kBX1vii*M@zOiiT@pvWhP?YddslxV0onGc;$vR|7@< z{=XrQ%cc}h+X%7l%0$jpf8M;#0#%8h(yr@2!Z33-6%o_wpZ$o+mr2<=e3OReoGMA# z<@N1beZNx^)`S&7va`w7&3K~G(EM}7xqS$Tb{nZ^jprd0&DStAqC5%`ka;aUH&ou4 zAEM+Fi?)a4s&lNv7uDuMy1!b_?@8AVQvV=PI}l`SrF=&&_vYT7k#RpKf6j~wJJ*;d z3q*r7Sz^NK16(wt)OAEStmTs%n@Peh)uQ;EWM?3)_(jASJWca+&d&b5X{R6|u(j!XRr5I(hwNlVX*{iw&J{|r z-s_c2FfO-Jy*M$vdVb7cVU~GGO^^s}-z;E12k<)@hVw}Mq=vYZoJW=~fJ`}_{Nv_V zGJpd5HH?Fhlz8m?-j-)j+W`#6p1^^uX9HXHj-Z=I~$O?8kJ{E`b4xiaH#tATycxFog27A zT`Zv@j=8S6p(k|?nvub8%37Vfes!&l1)t*k`L^0lWU?-cQYsP5#O?nMA)%WRBf4!4 z3TKrfmHFfl?|j9l-SE|`)BU>^K-=tz+q@FjVOQnQL{x6Wx3^ z6r&WkMW~0V=GSlH4b5W`r=rg-XYGjVRg9AyRC${eI7qmErN_0O#stTGoW6bBczj4w zRzRG*BO3tCBjz&o#G>bMS+li5d53-v330N~*+oiy2!GMo9fIDM8Q1vgGU75f%)9vm`gx?Es0}TxK)q zOZ%2B#><3BAWl#B4db6@Tz670H~#uh@R7g%wMaH^8ZJ85ac4j(>Io5+tLgODijg6yS$+bnB-CKRbd#5 z=4Kp=oLbkmzZZx^(kQ7Y66c+B2Z17!TS|zPSF7LL0l7x%LMx{=rxdk6rBEX3a(KY$ z&YVkqTd)XvL!s~lPvamZh6V-^q)+K3_dW??#z>380xkaqjNfufBcI=|?X2j;tql8z7}fl*B~4 zqSH|+2m>xEpPpNTj*BhSR!H%o!;-1nDYeE(SsAqOKLNCSVCdb=#R2<#;wS z&269Vnp{~XI8|L{Cm1oe&p&{xD4s&LkCZLFCmlHf2`vKM?!Vihx|^#&1&IU9)fq?8 z4md~IXB)ki1c=Z~#o*d2?#cK4?nkXxEB_Nz5h0PHj8 z%$MskF@7UO>vY*MrT8eCTAC7Ga+lAn`Es|$NjNlgwX@vQT&x}RMeB%#Ih&vl zW8*4MGHkJ2(u~3GR6Z3TPK#SurQn`rT1p9&LMfDqv?dYPn92!aSzXNx3zcP`b~^Y` znOuns%dHUhWC+u^5%Du1pL>k|JYzh>nDbSnR8J zsKb5{#8v?bV`~(iE{h3(x+yiI&u22YtJ6r<&XJLS%_^1xIQ@B>nG=} zRR`tHe|x_A#JzSezS*J8i^Z}e9LpB{gdWoGi}Bi{kdk2{PKc;UQkyGrP#owYGfS#$ z^ySS}*uh@I{xza^&d#XaIem5BFqP-*vbITWX**Sl*sZ><#Aryn6Oqoy7r%VGG5n5r zn|-j<$-)uge@JA9R)_v|g>&s|%XKtjoGE3MRq!6}L%p?2PKTB=s&wP$=MYJE`1yrP zU%@{G*oN4v41 zE!S0f{}q>K`$K8vsQlGYhia{nq5Y+E-mdJt4NVa*lT52m$H^xw$tv3Wwt2VACAt=i z{{3)U9;_`-*L*MhvUJuGNUc+}v5X3%Al_eaVN3S-W2&PG`g7w4tQ@~QHttPyg8sS? zu3&L#<>$NA7yo`#R(^;JiO(zUUh7(Rc3>_Go|A5IH9P_7_Gia+wA^?=6qSNT;d>1QscmHzjz$aUw-$oH&?cfIyhQq)Dnj~9&QzZ?_AxV z%~e-RUTSEr#?Lb}Ged)WY~9eTm2+ez3G?B@Y*!K(?ZC)0`>t}=VEKKoSO8sY!OA=-Em?p=1jcZvg_emy(3B+#898B zMLe*>Uuzxi?E+{&(2Lshu@qLtT$E1PLY|2@=YFM)w$n@iN~j|6tvrEz~2 z)3s{z-kU(NJzT?aDGMR++raXO^;5|3Cq6aRvXquXxTO;-Zw`js6Fy($Y0{>iVzU&! zMc6JWL6k29|EMduI!XdManRRgp!rMug z1Ntiu{~BkOCl&eswht1PrDLK}^^Ka-waSrk=u9F6&Gwkp2@OtRe5jYBQy;MILEYngYx^Wr@cQ$hZ62fwRzejIuFX&7o0c4-_T?htj?GMdoXr(1l|}kv#EM@|v7|Vr=o;2XHmaoVXd- zD5H&88$^ETTPWRMtaGKgaiJ)i4ExmLe-~V3A09z#68u0zmX%J8@MapH%o+e)C^D0D zmljbAwBb*m^^<7%Y7x$f9Ut$hp$uUsr6gu91$=mt>mb zq=yZm@r9IHvQ|B=x3qQQFU!$`d)`=$HlS2dX_(;^4iy+egcV^~xVEy!I{&2})ZA;- zw;MdY%Gu)KKlRrr>OT8)z5%!9rujUI|s9RadSBIXLc2>7;ir^>prLvY1mX!s`ZqM8hU(Y+Z zhT{+q!tcfSUY2c{q12NhmdGQ{XE8C#@f=dpE8ZozJliqq!gcwMTTxIjcG8d-y;n}? z5NEGtM2!$Q;*X*KOxQwoNirGDK(0jn7^OX)^=FQawsZT3J-FvPLJJe%_|blCq)I_J zBjgWPZcoaHorHbqMy*8fn{nrdub`I{K zv*MkuNW#llzzyr0{A+^E`d9WQ(Di@Ox#~+zMXAx9l+QIaAK%|z;5YlVW8NO8>D_gw zgub5g^U$AW>;4=PF!JZ%QGfnuf41zWcbj_Uw6)s(=bagQH`wPT5C5E2zGQ#fTl92G zF)BU(@X><870FS>Nq4;N?(@8G_x+0LK=UsDj-ql08*BF+`K!D-tgX#PYX3`P(f9xA zt|dSD?_d7!0sG&WB##2sfR*X~OEqqZ&3=AX+d2#m=UGj5!{orq%CmMhnHPQN?Hs|t12D1=&6Y}E?dMN z`{NT zjf|#Z?7IikNk-hIyVduKPANmvoAak7UoSuG|vEeXMi56N~682mdaOpELmc0 zV&Y3ybX>gmlp5@aii(PL&GE?FC3O>A`gZlDKe|r@3n%)XTJ`B&!$ytnBV+e>O!5f(KOD;L6}e6}MtG;_oWQBZZx*#FGo8SnyUtnRC&{&jWelU1 zBSvu&4hw4dRLlh|d`MJDH0?NoGi|hZB0YAiL5|7(F#?nyO-fbil!G52EzMPioB>S% zh70E+A9Nic>dXmF0GkKnstRVq4js9vS-ZY7eA%-LZ>O)>nBp~N#TBmx+IM)PjuPX; z=4^21%JPzjworc(kExgKkt)GJTH;6 zfb_;}>!54j?aiM&gSnO7K%N$@S{)A!?ZFAxJ-T%py?XU(o0WL>3Z6gz*>SAdo`yyS zwNK>D)5z(`k(@1m>-zOk92EFRc4=PHa=mM+O^$VJx$TyBb1s56q*#^{6v{G8c}7aFj+Ve_p-%o62z3 zweKMo%`l8kIx=Fz2U_2RfiT0ST}aq@N>jUWe#ci@`5oiVA3oe!Y00w>&)ZS+?)`i5 zz6d<2(vEV?Ijgl``0{tb97VYMgLD#QVr(E}n>2frw0siBk=bykI!d10aqb`#_lEiB zV^!3`naceJLy_%mb{eOLZHMUX_q}j8+ z%69SMb0nu>WHv?L7C>ee2jcbThzMDh;t^+1^55P12Svx)G4tyCw)stq|B8tDeD=eG8}af7D>g6aL~)2GzzY+E5m zk~ZyTK84M?zWL@sPBwdacj352b>}}uL6%YST2o&+IbrchZZ3oC>2CF%Q340B$GAPT zk#2|-IUMDk4`ZfZSYiF?T|xB4y8*g8&(2A6?(O+Qu#Vrd;O>6Qo?e!(n-i@&XpkM1 zYG$YJVj@&10iokk)71lLTFwGbox_9vjDleuF}eS!kt0KC?d(i9(rwztdP?!F!Wf2A zF(ReOo40Mt!Uyu=McA4#l`x_6SQfcl-`W!5p*72C7Y^takw2JfQ|hA{u<%JZ?4b)7 zGphm={O64uW3c^aVE>O|!f|y_|EtprZtbDiv}bpl#)@zA(Q6T7t2goJdNhnakSq&V zpmUYv_L-IxN`iYcGt(N(AG|ZHmV93_E2O^S07z{r?H2sEY}tZ&;4nRt{s!fqF1;|A zaKS&A(l5<&(W28v#>V|ue3;kWwz1KE+?!*f=ibeNz6h(U+3=c64Vyy@=Q5PPx_?6} zvrg(U(;YuM=bQ~;bsg(HCDv}bd!_kTQlXd`m+L;}pi&NDq;ZUK2=gaw-d{(@M|SY? z`ZoB_=+ie=Wr!Wu%Z<~O;}x%{rJA_Am%ZgVef8j54y8fd)$qwNX4YX=XLd*Dp){i} z9BZZ{=i0j%q|8H_0&t%XoT~fgjbA^BX-G2N-5w|OzihJhLz$iy>6D>1E zxU+fUVW%O;u@<4q7(yIToFo;3tHM*j2(t1o(22Y3 zws>SU6i-PePuo;%p{b=;arMfRmLVV5eC*sHPP3Wtnas7`cBf4mwd!^HVI{2DeVD74 zAH`cwRC4@Yr?Z?6v1|pZYs~8E6>Qd*bAzZd{8Uvnr81k7D`TwMcS9v9pHcDMtmEhXsSxV#6I|oU-jxw^qGX0A^L041`QU zw?}akIY;}f3u@;65x@Ob61!pj`ad5&9I9hpUv-`By!7D-!24&m(@BRht@P{HPtU4z z)Sc-`&tF?=Y;;WOYCJQk-W^XA3Pp)6qdpGRXrH(Z@YlTEwct0clr}QNZ4I2_<|Y%U z^4{O%9NoKB+FdWS`DKnDg6b(LXQR;$BtW1d_3qsXk&%%y&tx9r@EVW9JBzr#_{E<; z@z}Fdchxf~YTds5srRj;{{B03t%qmK;KV?ylBG`JdVWU5uMOUO-=?hTwzokQ{hoB{ zkl#_RM5D8t*}x;~|NdKjzXbVA)aJ#7x7)Iv>0!tZn{qx9V{jNXiPs*MG2{MXc$yoR z;uyzpt!Q8qw6vE)f&^eoGRTtHvPbcgm<_hnp~TF(-mqtq#aSmM=UmtWZkAeJ_v&!^ z(c5)18g24CQSUYO1r&ZWtC&n@L_GrDsOnzxt;-h2z?p`$^>& z(&p?;OiWz6Zk^O~@mNR5%#`r=^ee&0W7)Fp=MP#p`+67fQzm_3MpI5Wv7yzv555K4 z)I%=!>YLH~@%m);?un+m`Z`M)#Ua+lGhg%*0@u+HCYRfeqVGE{q&mpNqP8}}FJS7-Zvq1wF z;;r+exHHZ5#t!Q>w`PXjov&{%d)%nJ;c-K*Mg3az`{L$iaeL&XEZk7J@t!9*ExB2V zJVUk<$jy)$OPX#hc>S+C(9k$ZBW@-8MLNB4Up#x(#mA?neidx86?G35l&!4t_GX7N z#*k)3*fxLWT(-5~HCLVUudCMZb^OW9-w=T?~1cKUbpFXasT`q5Fde*I7^r=N+% z`7y2atW4po{=y!#^@gYk zBQDrPPSqWF{^Rg%v_a4DKGfy6?~LhzlzV#SI_&*z-SRFvV21sR&AqJ33-qU4v~pH$ zJF0J==)XJU)~;qm=|PJQL+bvPDZR+DB6e=ye)qSISJ|;j9-iN$=xDqmonC)a_k|JQ3qABPRi7UW%wU_TYeE4x#x zUUr?EHr+n5s=XtayP0dE=gVz)$9pUFre93@t>`~~y>oAK*5ZMI3&XIwiM0tHAq?-F zE;+=rXU~>avvqdnsBNX33BsAn!*Q99W7b(Qhg-C37lyVzA9icVs*k5B{0e209h9#& zO?|sQ|FDn)7FW=D04f8N=|J!Ra*z5|;{pdwgeeYC@bI~4a|#m7PA>$go?G;6L_ykt z>@NNPJ$p15%L5i`S&aZ+kG$3Hrxd1^1vhuakQ2!e@yqFCBeVS`f@rOnijoZ6JTyGH zt29#h=hZn)!Va7X@aN-^%2QM0&+6Gm27b}}kE#As80M@DEmBzaAQ8?pBCii^Ib&RaB^ zwCK>mRkIoUR3wR+Gv!MBi6>LdI5wtfNnUbZPVG+NUv%2?inm{f!Imj)gteI7bD__E z4UOr&l_P((@-E*Ma{*`(W_41vt<&&G0;3Bpr^bhi@qieV4yr%NMTbE$2=p*AGEyi> zAJ0mzPAd2c|DRaZNix*GaIz8YEUt2Fxuiiv>#c42U5V@sVc*Wk>$GA%Jj|UQwR&5y zf1GCxJi+8O>j%+YV~s*O$%Y^QUOwiM&yQwt8;SVBhCt{~3^H2&JZ9j$dnT%LJ~gY` zI$9>w>ID9Ef9~3~Wt%oVlnlePqzaXed6>zXx|{#_;~tsgq;K!C?0?|d{SPC7eC9&K z(fC5_@m?h$;&*S}Sh%`|3yTGQc>LZAC*rt8RX72=fHl~eFiQ`t@m>viY6zt1tI^a(`G#)jG$d z6W)OSJ%RahZYDs`mSf%xtoIB!ILv$UsLd^QY~McN=5a(V(wE$K?ZVculDKPjqZt0w9gcFdY80!gTxBy^R_--lbrubT2QpwY6Qoa%E3L zL#r-bx>PpSQKZa?PJR5?ICaU_N;^4M<7|gw_hs*1D3`BDxOunoukI_;>kXei*Y?NS ztM1e!-`I`M?vS!FH^s|mkYDAwt#7|BQn@6trS_CQB%Y@YU$1^&Z`pZcGv9Te>b!hf zUhwQqrQ!eNml%rn`PI*YcYmrgLaXv=VMW>Y?n8zQ`D+-9MfJgC=XZjIg0A@YdLB9U zS&inaI<@48kV^9B&b2R>^0#KU7CPcT{#2ezy`G~PJm=!Yn$cs{u3g*AwP~Y9lZ55b z1s3YsNWW+Dxxw!5>s`Kl8AeEX`Yx%6Ldo(Tck*oQzc&9~=h1JZR|@5tHERq~uDJ|{ z22cFa8!rG_N15f_WG;?V&KPfFSbOQzZMk|HP1Xe9$6K&q)xvI!X{y=JhSD zBVtK1i!cW4RxBxd@>sWAjWT=#ep*&qgKcO&yh&CC&~u zJTGM;%?ZT^NIp(-)I*^d2{W@xvE=oGiRDF^o*tsov($D-SR$pK4@6o)>8j8;v;Xw} z6{5_FAvN2hA3>Lgp^0PFZ@27tpkKh4W;yA>VmA1C_ z`0!eRDtJA#;|0{p{ewLb%#JFQ0}hN6get)gGI((Oh}sY32&AUZU^)JxYO>(ftCQ~v zZqWkk_8ooogU|>*-G1PL9;90E?0WWR6*FrCVCkCwhqMD%rvAKd|D!$O|6{l{W7DQh ze-SzY=|_L*GH5pV!2`Rw(xlMW$`zA$f_GJH^ev*O1p~?Z3SS#&1lL>N%y;F37?(Y) z_+ZG*Pzd=Ui=O&Mo7e3iJtg$)P9GMk``OivLlZk(IwD}5cv_5fCo%1yLlgg_yzLl| zEevZ#{pN2Tz%8WX%7~!hTPRPmUB6lNJ4I@ks^F%IsL!`V@Xa{XikwRHEh_>W9eYu zs}c6mqiNM(3GX1gjEEt3^mQH1wEcRst`^_B9>LbFa`9G~lhOEIa>jTJmadw2n3IO`opmJ^kWTtRXh0rhh!H;Fv%m ztH*~mT;#{ShOvLr#!P+f$v!?Y^Y4!)GryzX*mNoO_XBXpA#k?#4`oAzQ@q82_M3Lq z(LKhhY54v3{t7-~JgJWj%1_|bmi}MZc`_LTAvPAQWP@7#^irLII|m;scZKx1eMLiy z^LRrg*Hl_=*ETj>S>YY7aeg<_!LbJ%-pspbO^Hs*}OA2r*MvSB=}-_i~QT@%Ai<8af@U zgXZqe?$z7=>_}#X1rbp=8Xm@k?h}F*FWqEvmU?Fjei4l_(C%NqaMrjX?F0J5^uS5Z zWBX&Kipxb3!}x>*XMO@0ShRGX?^Q^=TWM+IxW{gLeQ(%s$k`J1K**3^MVxthd0qb* zmo&VwT&S+CT%pnQ1wp9jQG7In#|I2B^O4Dm{yMg?+k~KS7)!lDgZ8fMn}5c@{rSiZ zzyDrD)!u!KaQ4b3+arkKuLLMUc5qCjpr;~vECe{9d=U+x`NuCx~%z$W0&n4 zBQ`Wl@OpbRc41y*Pus{rr^a@+uMfq81zQzNo6!d`F!R!3xai)Ib?7~ZL5zAshUB)L zU3dFyOpooDtLjemE3nEJl$Po%MIGZvqbTG{&F{V&5rP^^JCa z{qlYSr1tHGZC~Erci(x-l+EX+#(OmU;fD-3E+A|{xMiJS4UKPwUa)G~81rtOEpD|-?R8^U)fCT4&pKG)UE z{L=W)7G8@v0N%gfXnlQs*r@|C)og242)P-F%NB~efx$zneE^Dnwmd zo<}CL)2jqHvRmO0x2Q{qc|FUikAkm~3uL0<)?cTE3Y9?(OIlGE)1A+s+#OD_J#loe zgY=y*;BjLIg^lnK4eraAFXxyr-K`pSj@h!(SWb6y==|*Y^(~+2DjHF6k*OC6a6bv4 z9O7B%7kyJx+y2p~ua7I6H`UUbNQuD*Oy@-u?c8V0lv5@b1_0ab=NDMaAr)_sCz^u65X7(CxQ?FRGHVO1lt-`A`ztQ5AM`6zSJ*d}zg6q=1jx<9eRNFgDNaAz4LCcr|$E1HA!G!thEw#C>}L zS$J@W&C`hh6LZFS^VY4I6w4{9S2iuAO0if*e8sLh8^{mzD1w*kS6pw@jf(l6@UT0m%=@Kro4%J)(H#1e zR)0aRTKxILe5ECFD4S|9k4z6PE#9Lboj75AVZ{b%Ft$ln)+B=?X%Fw++orp1Bcs5$%pVn1cfY{y9j4CKRE5lX40iG)8II_1;E7Q1#L$o@ zH-hTSn9J~uR_$;?taFt=Qym<(Dx@p?K91v<;_ZWQbgGQe547v=zZl;37u8r+%hrA= z9FTRz)Sz)Y!@g|YXFqo|^1A2c<<&+BfoOb3ueGU)0200E*_DwGNzmLB`&fq|Gpv2z zC%w7S=s5Dk(uX$Tb~B;8BPs^~d7&8_(-Fo6>P~`Ozv0 zy!z#>u8PPUh^|7ZeXgkiE{w&qGBqrZQ7|(3r3ui)0FLk%7=HuR*k@NZjGHmzaAqaD zqI}Y56`e3u%};%&TELtctz{MngXmo-?>P0Zd`R%R1oT6XN{ z=C+bz2kWCFwTi3EcIoNTevIVZtH+KVJ8;09ySNMlz(UMCJKJ1{df-8pTL)I7@dcb} zHq$XbH106SLiGW>b7J0O4a+^Dp`l(6VNt`Og-_C4Mxu*g1w?G}QwMwv=L_p868ze#)X+ zf#I}8n>K&{neN--Wmy^Zay=5Kyb~x%-V=SXkHukMP)eEKJKv41+O>18tY+ix@AT)7 zvv2R6NZX)Y*jSj4)+feyL=kgfpR&mVT|00Y2{7aI`Upr58D4={=wOf%N~PioJlZAQ z^7A?EaQYs7hjaiay6s58WL$4-mG6MkLGHIQX2U;d>NQZ65|xG^TO?I=LEceP5a&0` z(J>QG3DYYKl|$r?Bu7I0DAnsLgH4O@14xKc!pgs9`0ght*D19`T+5^upiz6eFF1kl`=K;kedK7FPA2?`c<@rgUD}M{+ zZDuO!E4)jdT$pn4;ze}41#55k@9kt#v`6lD#QS={ze!B07{?n;8fzOp=(42lw$r38 zq8w2|9VW#(PmSyeMgW#bocE@mlkbkl^U`(U!i8cEkPQOW9+Oi*`h+@OGpIC~M8a={ z)Om2QXG$&)G!hT~E9S6dYQktSRU}{aa6ePC*J#k72cM0f8GFC$8sNKFwwTZKB$BS2 zOYliuo`HnEYf^(I9gxFUtVnTb`u+F!*a8dk^HI^;4Lws!1{$yzH92y>lpD8n;=G|in)l`uP$Ffvu-?@K$9s}_x__wJB1N8lW z)Ygt96SHsbY2v)}?ekH_n2?3=!*$=&n;E!*W~*|jxR*D(zCMeu~HreJ+Ng@8aq)g|L!FRFB;S77Aad%76e3Sk>@S zy_NoNgZyjmG`*(TjA?X)fHWUJU6ZJeURIDvD`*O81#KSjS8m$8*+4u7DODKz0zdlsUNgS!RTIVVwO7s|b(zDwPI>t7p{W6j~a9UE_VDLreMG~&k zwwtW*d}L*WS>G}>z-M+k@y3!_5Ds$``>SW>*(-P7|IR@xk%NLB@u?%pdK8nss*XBOyM4|!wJs?Ri6 zNagV4mH5mIYBy~hB)1tK$MVKan+_yvkGT`oZh-4N7?e4F{R&~E`)M|8(N{H=8$VdS z*mvcMy>GuxXLIkTx{Z-!7Rb*9bjsy0Np!rXX$B8~ILdN~gTLf7pM#Vw?CF0V88FwY zikO}E;_jh5siJ2yuBeDT&~y-wU$V+c@8)YXbwz*9D)t_cUzJRei)YtVDNN{QWb`O^ z7{;KksVwDaI{u8=9k}2pw(k$Lp=fs?Z4Tv;K6x{)pxTbZ(Xmyrlm2V>ai2^AI~HFH zr*Z9gukS8ZGsPKUwwOXdcyw$HUZt!97AUCEgqc$%@Z*%x-!mKg6r!r=UyApt@S7J2722jte+A+~-N|6S9!5x|c1+Pn|MiP5sXOr?3=v zIB|~HO#SH>znD~o(*&zs8EdjHqI_Rqt%1sK`}XZQD7x!Tedj-!jBj%5_JIArFJ2bn zYVyu6WO{}aOsy5bqZ-Szx=mhc`LV4=Wu-W5UXH@L^UF^^{gl$ik&w19ZMo6_i~sJJ z3C!T-cxoaJpJ(_}cXZvDnVpxHhxhNk{RAyjE6bFs$FU0@J1h2qE}NN}?Eio-tc@}y zJ3IS}j79}9wY{B&#H-BNCZm1=Kme9fB}cmM;~HLc8gm93jMcXWvNx7ZxRMxH{$AvIc?!_6!$ z!Clgicrswvuwf~gnVD86Ew9r6dU(jq2HH1|ZCmE(M``ljUF&uHeb3FoAURqkPiQb! z7>9?f5*NIss6l=#-?QCbFmFK_-JosiH}fy#B-zKN2FNa3vzL|^S<~G62WgkRpL;z3 z4`=5_jl}7%+Sk0rJ^K1_;y|5S4g7CjOG(+!0^5u2?8X<`7*saywtoHky(B0PF6=fn z5XX_0mX`aKUo=Xi`0T8tY#(t-nx6eSni94vk<~QK6R&PJXl*2ceOQF{V?@cR-l}Q2 zb4=T^ls50Fwa-aw`lG4I20fKIDF z9_g2&XgBtw5%UY6`s}Jf|cO%d_;eds2*2q5K>?Ppz?LMmTVbwfTphHT0i_?NcCqBC-WwM;u*bOJ47Qrir?AK zaSZJo#~!`Bo}j-E<8T|tuhs1nasT`Rk3_|P*@MS7>zHnRZZqg34=X?BdPFxNlI(qk z6C~&79NSijykzIbe8{@C!I@P4d5kxG8)OnnVj~+0fdn>k0ro*hzfYC#x36El&^O~X zEf)Hz#u86UpnEPpZlP;=ZNchGH*VbMH8t*DktYV(_fReS$Rzh+X?ghgV#jSb3tnYG z3c5&c=UtOGW5lEC>+KEaUD`Ow)bn|%&mNYySN=eS;+lL~3;n4NWPv7JNrLFI4O!JWa_T$VRc?XarF2I0<`J%jO_NLu41sjHdfZM;Kci z?OCxnJMHo>$fiT~kyX+bH1z(``Q;k^!_;@p<&PXQroH3y2F+y}@f60gy{Bzx)H-l# z?>BFrk5~!R`Xjql<+5^=hpdtg@OXEY|CAm+rehiV|8Y_ZMzR}DuABs&GPv6C5*VU0 z3vTSxF>iGr_B;RO%ejhuKeRPCy`n>UH~{h{DjcLxY&D%3vU+!fN}5YqxDHOBwp~-> z5)$@vuGi2r&p&LnsGA-EU)>ffaN@v&%NjI)zWbFZ8qH7DR~3S*y74RQ?0sjhjj0Bd z7SfV{%?V`X*DnREe`7_xo7EF`w9?7;N?uyhuK7aKop@uNSwwz{_Y{xr1%vLRfavCs zoP?@rUND`8w9jBKV@oJbI=W^IEsE=RO?`pQh2-YOqd51gbho-37ZlQ7X z#u;&A*EY4RXIk<+QZK2hUU=dgXf$6~*ye5U)_3LT(NoEtfZcai+sq2$eq+elP zqvbq)Mik;=aJ)z7`Og<*E{M(vuYAY0a{5W)jmHZBLN^mGOkl4FwD(9D%@nkGWI2_2 z8~yMJo<^4|M@SpA^fj9VYr3EEHZ>qwK3`f#Hx)wfDoO-9$yoB_RJOhyPI$|o;~Ea0 znbh^dlryt7ye%(}S^K?AHFb)fUyj>qLPRCm(o%?=)%j=h_F;m`6)4m>G1H{wQCQ`hq7bzf0BQt zp{c!oUF}H(L~<&#bC2`#rmN1u$Gbd)))&(o+B%H)-yExBOGya+gpxNa){NP4&&vnP zuc7OMzPy2KKq>xY=+<(|4x9n+qAn>?vG(KK#GW4K-u1dm164ib)=_8z!k=+K(2WHv z;Sv^;WPo2LwC*#_3eUqX+7Oe^_kGgu(W6HSB~DM>i3a`(+@F6DSWZz9?85O_OWlMd zW0EP9Lf+CntF_V+s=c{!5F?qtS<}BrJf%=3j-oVSA$;L&Ulc z)|wnWXKVk~i_Q}{Z_SCGPb*nD?Sj1=Du3VKdzUU*GCOIg*6<}pOUX2k!Pto_9gFNe zw$2EmRc`gn-NDRW@pESdUzMjd?_Osfh8T+{GjlPJOZqMUdB;odlK106}JMZUuKC!b2t1G4uwo`X*e7LJ~X3_l<+jlro9(r)hkS6&Z z=l_KQGZr=?Jo_V|+pb2Nt|V)#B=pJF!xm}(5ariRcBHPUA3uE(9`;1WBn|Bch#wIv zC{IUlc8fGxjE`9c?DNz>qdndKC{l=l;HqKrZU8HGPKf&@+Aw{{VX`DcpbqX&r{G zdT^ONLa3iM$aItNPBz!p4tkN__sNb{te?GTCw5V9gMxx;ku=-uxJ8E$2uV~qTw>`% zfTU+}Y&YcADP{ppjRL^Bdr3drD!VaH3y}>5sK&ygH-9~x09L&07&Qi2C953Swy2E` zzpH^;E4#(2R&ni*)HfO+Frm5&L_yRzcBJ@4&(^lZSEbI9Mo61%mqUQS^arl7_3LSw zIPVUCJ^+u~RXPQ6##}?|rbdC9jaoez7pBvH?lPSTP^SI_5Eb|=VB%2TA-Hs4n z;~<5bJ&Gbd$f}Ma)H-v)vvEbZDcZs_MUzwntZH|u-lv6~yFeHL*0VpG6eL%!Gc3$g z#>YL~uMqTkKXhW(ofHyuO+_Z%q&1*(N6}*h0;|Si_)6pHj3xf=qZOiL;!E^n5QAL~ z5${|UJWZAq&itm=gg+71#zE`C%)@O2Ta;hAI@tU3xh2#3=%!w-Pr*i*(vQ>QlTP@M8*<*$<;T{n`vym9*h=cbjp4LBeiM#JIF9|4TC+ru(yb^A(s!$dEn zLA;@AEFTfkIvmR50IK9qq;xw2-mX5+E#^AdKS=he3Fy(@Yo3ti!wyT|f^g;YjauE9 zLc6brT!I7cRCbe|jxPHtWKqOTpEobG{QUD$A+e3$?D8-%Gwb31<4X=%F;Lwmsx^P? z0}D_HLWzlNrg93Ri&*nEZT2=RsAoEx#n!QFs@MspP2pqaQ&yQy}Cn2povQ7&h((GDphwz`q zK2wtWJo3lkK3`X@twMYs{^X1Ao;`b{{JWu56QjWOb+z1S2MHBH%;}a;1Hm&J`E^F? zSv{vEeKdaA!r$O~ou_AZFeAIBN=~O0%S3xDsivs`BKg@Ti@x$xIc-PRPWR2p5TiU~%mgX_q zrILzZ`6}b&P8uPdUQV>Q%?Y_PG5;+nhx*$l`pEoyhu6p-`~iNC8(c7PehZhXXlL3t zMOqNS-oR2<8fOt)ZsQF~BNBGB*7r4hitfepH!(3;E8e=}XgOJKJDztNLYm@6!NN!e z+syLD%0RIDessxw;MoIM82leT>XIqj)D9=A1IAfc_~%d;_*;YKlzrY}gkIkL98LS@e;lM+zTC=UL=nMNBz)Y&^=CZ_&LOkHs4N8PDj_&ck~l( zOTrjYcL3X1!pk!fU12YqJ6PAz-7Yg3U2}B0anx5Ks{22a7TzajEHQa!w(YHUixw~b zY_`VfErdjSa>!_!;kZn(Tr0C-6qb+ysFQQxhu`K(Q-r6+>%3R33a697e8E&FiGpjn zgs%q!{#aS&iAjVK+s+Zop=`G*q|wUADv;k^ZUg?11*7%k=kH7#sT`B?l9ummI(J7A z%zk=Ye0)yt3kL&c)Me|K$N&kEa*}*2DVE(*XVCB&(K)v%a4lW$yk-pfr!O(jUD8~| zKK8SaMYFxEfRoKZ*n6bEZ12B@F`#1iQ7A=CL#*BNb4;Do-h)%1(Q-Q4cJ?1Dd@oiX zK!o+#SWIXRI5Pax=G-R|8*$1G2s_K zzosQ6hewVaDSB|`7>1dH7D#Ep23#+P&dt2~-E0h@(*TBv3|w!e982ENwn69Kn}<#p zm&oG9i`RM+h{QZo@aK--q+AYOanjk5opTg7yf}b78Z6pjTF*B|mB?3APAJm_!iwb%agGzx5ecgDWV1ROsoq`pt}xBT+lK+$+Ht52 zf}xBmoN6SXE|em%{q(yL{Qz`6hBEL)vobGTegF3DM50i0x3b5H=Se3Gi9!zX9y*#m z>-o1rqxh8>{TVQqKki9h8fByPzWc#nA-aREwYGn_lvTs3w&`QBD0#*eRz7vn4+8XC znl6Rfp9De-FFSVZn0R6Y855Nte0LUQNA(##e0bwd!={Q24CgGH%8P3xW~pRaN8QAO zgtit1bM6eSt!vj(yQS>aQp8K6z~8>PfhY*uF|z72UX9PoW_2=+V7Q4i*fyTW2-3r` z`4@3z3A{lAv-obU#y&_|9xYFpWZMP#ben#A(^do+sc;n+9a~NJ+$gWN&qZ<&5yf`e z^6QalSFc_jG9M+^eclBVvs-uoxPxOzsi+T`IRzS~1ybmVW;R*%0s6@W1NEkgf>m1c zufr;q!*JYqc*pBTei4}SGcmZOw)X10i<>l?vsg~R{S-;98UHrvuyg0-cxYgD%cZbZ zHLnvE6G=xUaiC$KIX$=aO3ggcQy0iby?1cUaV(J}w>Mpsuy2iT0v|s#NVC62(o`6_n%hxvhf$j6{AbP{Z zHf+^P9G_AbyaIEo6um!LmR$2)b5Xa5F?v+$K$TjenoT;q;^|+3!+oYnkr|MuKNhX> z*E#QlF3X9Er^pB1#vU?WX~U`uNShO#SfJ&zyPVw3h6Tl2*8;V(XNn zZz1lUwbIcL*8;BZ5%Q@B68)*fq{WPV2~^}gY%RSV{n%!YLK_adoWJPe`g2b)B3KZ} zGmtxwk;rStvvxwn83`5(n|SPZ@cdyg>Lc}i(7sVqileAQ6&IQ4zc*SpH9E?3rRsJJ zaAbc!$KNSPTy=NXbLl96;@0PhFo6*ci8krU$?Z*?W6)4VdL>|3k*N@(VjJYjsa+B= zc!WpEf(FN#DcaWy{t1D~kX(1P>Txz%?d#7J&K)2-XsZ;-Wh^s^A+b|a#VlGu(>#+C z>I8?LmL)06Nr!JE0p*uFCgr>LrKZg#eC6`xZJ8onQv>uJaz~CHJ?vh;i&dpV9br%M z7Z+!x!SfD@^LY9fqWWIM8@2@m#5n{s<&g2P>GL0)KC;6}X3A~rx~ch0)~u@CiOIaP z;!@cp+*xqxeD~0}$r>kf2!-PSviTQ+D96ext#)L|N>`|e3YOzl=wrJQ#@>KUq`&Gm zt&R%(PSiE>XZG0V4s}e>;AcbnI$TaXqpGoiUL}})IE;uRL!iusa7G@8JuB68ETJ8w~ zdZL5fc0zmMDfFIHnBhk%&etyLLp=Wq4Z-$n}>pyjczOL_iU-;}fxU+@@wa z5Nts321D*pa=cYBDc)mt1KS@u$4rj*2w8G@glH#7Ca$t)UT7W2o|oh;LU{3`KM_6X z!EsssVDdysVe08BGWn4>Bpv(7z-?Gr`$CD z++(*|G4lY#e6Z2-cld~iA%J3-w4m;@e7?Nhc#=xI9ZnE0PMjk(aLlfTM$%ES7CVXR zm{sBL|DzC`U0(MgR&!Vnn3{-6K=CJioyrw02{bgz(j?fQnu#wHd5j2N#qe4vE z10-{=N85|Lq@!N zz?IXz#$p5JD73F@>Yg@n7K=ym!r69qc1SCUFB>pb%q2N<=(1WEs#t<{8zO+{%^Y6j z!jal7MQ?(oYFReO-S!C@;$*8ZE8VWCbOgM-pz4g{joUa3=0&WP$V2b6_K6}#^|L?g zgo?tQfAVd0GK=g~6i}GpIhX?8=1SkimDB$CGM)AFzFt&ty_c9ktlW4zX%?=i<|X;K zuz*8{JVw>geF1gF3IIzu`}NcMt~tHM=CSvx z)?h!D4TAkJq*={(vP@g+Pd%Md0dx#6aX;VxQ`sIKpgnA1W*Ix>0HBndT;rrQE!dg+ zhx5iS-pqA6F?mOeh1P}|aS!1u#Y8|99=EvV5uQgoTia|}X)H>@(~@6O;wHkJBj&Qb zruu9GiJ)aC6z9ccKW;Foe*81On&+Yh<7$^QanbW&^ z*{7Qtz==l3s)d&7J3h%WpUNdRinRWTGbDs_3bZhOGXs$fWkC)KzFa|GPud8I)ScSa za_tsht8kP@f^L8uz&^6diGwcm^?#Ucw3oc=P zVKg)q${8ehP;?;yPReu;kN9$=E}Xz?31=vU+|rx@)+`(0GD0X{dgg#jvw-*|+W@$Y zrHc_o2zR{NnW*p!=|V@Vg4wIhRnXDL3As1<4blGeX&fTJn-wY^5LQhjJhD>ttznUn z%9!z&W!={(Ev8qeEx6(;F+mQ{iG@}CkL#WYnqND@8r?1JLr;^4V;6%J52it4;Cmk1&m96}+4J6eEL3N3gP zH`Cijdau7?KJQeBUmvzf-9l*gt-`?W8b-dtFC${}GpDsBx0@P0`8a)4*zWEVw8d@o z`KewPU^$fmThCWhK!DA3snc@b-|$BXAf?mG#+csvd!H5;p8~R*tGbQd>;Jc6xIed% zH5REs+UejplS+OWQ$Y6af2!vppRBkR?6ZBv5~vahq`Pesg|Lg}DymK*h>>C#N@*iz zD1qLRF42``9$UP)ub{?`v%ZGJJ3oB}JvfoP9SMT^Le=vNCkCcS!;K~pjZJc&6&4l9g(63Ey;ZM6Btg@r$FHB0XN;hxW-;Dz89u-hhR3KtI~{*RQ9pQg;=kg{ z#$&Y-3!A`j6rZWN1OV}`6H`wznX(3fuVW=Go@(rGM0V0uDQTbBn-DPN+yg0{ki-4x zi!&OpPA1(5kHMmwStd1F=9>Y*D}MWcLx-lXE!mXT)HEF=&|z}>4)7V47vh|(W?i{@ zwE%tZyc?{lnAqW-O#0TDzaQ}aIC$NMg_A3Q`dO8jTdA8cdzJ1x23r zAw_R+U|DRRQ8(St$H&JZd9{OZSTXp?yv4v)UQL*jek*eW8W^1=!xw?8o#8xx7`I65 zQRb?PAD=gBiI^R^@X?n`Ot6;CnyJ!5>NcThYv?Y0hsiiu#l$8^B%-INI8__?uICratI5{p;Qco z4&p{FBT6}Rl0u4gP>EVy_w#+({rKIF`=8(6_a2Y!vB#FH@Adh9KJUZp{d&DV`A0(! z42bAxe=nB#WZ&^?&JS|fxDiH4^hZF%=jJsCrf_!Aj2}Zof6vY~zWk+Lgd4I0PlrGU z?i2d=xQG35JvGXf?o74`glvT)T&-s4L5pEqiYW^&F=JT8;d3&%j(e@D!?B?AB3kXM z{U412$Lg@lA@uNtv1WF>Z)1wyBa=|AT3bu#zhUEJBLCfj~!E+HWzqL zF|)3^va>et6e9f$D#HG=EU}PUTh+ayv$ojx08lc7I^fa2ej&1q!8xhlfAb&n8eUaV z>Gf}rujLNavpdaT+MXi+f*dg(MX7>8(C_5jl0Vn6>kgl5>g2qSlc(>kXaiQwgpzumSIN}nCrGwy8zy~8Yga$lTU z^(cr6^QgY}^0rrg&%4#sTH65GHR0FlAMPK!$baVtARwoDDg@_TMSPZ74huOIpzTR7 z3xl4-P0;lz=@GK!NBZRUWBQhXo3>N@2SKAr9@9CM)OYEtUww?*2D#W!*~$SG@r;BD$i9!sE8?+{y1e`lb~!?^6~h-6{wQtjLei?l1ZXa6ZUw79wm<(x z#Agd`^dZ#Vc+`9aQ+z|mt)ADfUAeMR6kxDD-I;@Y8@>50?E9%N_24#Ey zLS&Szgek}g%ypkrYqv6TE=UdV)%R{HOJS5sh2uxf9B5E!Gh8d`Y z#9c|WJwOa}N%_1Cq0`dLHm(YHse*4HRx2dXL10j#oC6bzmZ>M&TL-dy`?!|O)hoZk zEXiOJ#Vpb1T}An?PoRKvIEuRIR6OmHo{=H4yR0t^J|^izDgvtWC$TJra6OWdj1?ZA zZp-$OUxVqnnX$XJmE5}LODJpw5U%@p%ECKGw~1n*^fW)TLcXI&%_w=C1OiknBI`i1 zw%$_E%j!6wgZvLl5)~nme(yL@tp*>jJYrEZnA5{$>a+=sgCgqtSXlQ3chdtC7(TZw zX!UA}B_7L&0{Ov_PB}*zIeev(;*_nDJFNTn$01VoVO&0FNZCF1Lhkb~^1~>oKqY7) z?JNK+o&u|!uM0sqI}(;em}N&J{1y0XIU%BmE737QcWCA?>MWe($WhiEb&OCG^JHNB$oX zV9PJ;P|P^tX11&{>wTZy?^;E$E&D2kB$Gu)XlH>uE~C!(H3jQU+_yej`X_kEUM4I@ zWS+HmwEtmvT|T##Scc0)dMA^SMp}PXO31Nx79<2qW|4Ll@WcFA4xp(b{Ih~y^ zP#@nnQ)c(j4(dl_K0&q=%sJVnbN%RI)V9Ka_1&Ds`4ZzLk-_aW?P73)iMaHe!^Yy?860rkt!oCfIUcu#6`u{Eu-2 zi~D=?6(w*qk26(R%@H%ydJ~~z;8tp4Vj_Z7Px@=gY5K)>maOj~3SH4WPXF$^p)IzP zq8o|bLohjHF5p6O3=%L{sUDViAAm@i>hsxU>TCu~=|59s_m2B0s5kK|QX%5dVG)Mn zPrMUXKJH;W37N%Sm7bp7L??M1J&{``OKA5=gjRVTY2E|+o>9cQ>>^fpe%LqLwW@89 z<(~cFiG4SmJmfo51_s3V8=Adl)GYK+sR7uIQTvKB2jmE1&K^rvlv?fqTGT@#J$Oo1=gc?};5(Kg!?PHQT%t^xTT6cfV%u=Djk$?Ho}g1r%k8gjKNs^KB!D7bMIafz(O_y&_}b(GN)g1|?=kCA&?HjExCt z6^8_whm837oqkL^kk?1p)ap(7EK7FpUDrzMSs&`_FFLoT7C5{a0w>6iBMjDQ)DVd& zkZ*kPe3FnF(%$Uu6REic%qfte12Ve`GrwN3E7S)D@XkUd0fOU-5sOTvc+mohb>{p*Kb{|*CL({sj6IBI$lxM+lRrk8yR&Jj;UN}FS}S0}y2qcc4{dzd z_6AhXh0}f)y!4OG+A3+#98}cOyS~+!D_Ae)(oatg>r~n@iS^C=ZCcy6g10BtH|3O5 zl1|WQ9`!gQNa;I8-NcSe;nM)GZAI4r&e=-fn33u-dObcf@{IXAHH;F`4Kq0~MaU{a zgFQtm((BjnMM3Mj$*wfI?c{@(O-*6ER1+=C{%O9z!G zsMw4-eSV>O{NjvN`3^K)zJinG_7c^Q*qCBR7ZVes8*ghrNOPr2>?sI=0RO6{y3IvW zpio;IyS<|b-(=K}1_aKYiBfI(<{f$B>X|l|yDDWOLk;mX;qO?{p?VZ#k_v;b<&o9d#u`>q|1 zFa9!gea7TNSTf44I`yyjjSYyGfr2PhI3V%Bd}kbNdVK%ZVas}zci41bS8au%*SkG> zIK#JY=2L}y#W2|@LOxJSJ(pz#3^dwINLorKDwpdvqbgO(yw1jOVE?xlC&ZotlyK#Z z@_Rkf$YM022gQO?-VAh)vtWXp{%fROK9$2HlaXh6wpo2T6R0^ZGAok(1j;yn@2NRRR zqmCc*Qk+QVVgew>AXKMbTyeKok|W*Fod@$P%%uVpjv>O0zrIU;vN$r(b!uRMnDL23 zLk3RIvB!{kIA$FqX>5FUZ_Wm~DdwpQPz}m^x9ct@_z@eh{)?wvmQRI`AWbsHpxAuJ z@{MX31&i(FlZt=FyuC!xz#Z7}HuYl?m8#?z+Q2)^6@|7ZRLi|1CWGK|8{6(P4y<+1 zyd}AK&#r3%tsA9^rF&lp8Tsxj90^2JEhiQ&!sCROnpL#!ai4)Pc2q8~Gl>^dC`#Qt zDT-Vn$z~wDc1rjgLEz2z`bJt>Lrla1*@aUYTGm2iC0#j7u0RQclr{#3rq=Ytuasn9O zFn4`OOA|?9E-)V3Y6#Do7c-Ihvg_G}(*eK4(3GmI0CaI)m~vOl%pc6Np2@V5rk)5W z{9AFsttrb$1G4$>q?!pP??kLFsJ5m$MvvTqlvIYyJd-aFy7qV{jxL4BF_O^lc#(@R z4{oHMzJ46Tj^A$np|&_KIZDHyJ(=+;8cIQP)a{JV1{~Em>Rj5mb>9FJB+ID$CrYR% zSh%vJPiXXb3-J&XqM@NfI?ySpB8|98tRAy$fjCOYXr8O|BFbl5Q@!xPaM%u%|GI1V z++my@ez<*y@9!XoZ1{O$c?8HKd|D0<34rKAO+tFT@a5WfWaLcaD18}af4uBfi}_o? znRY($*|lG4UuIS!;T2~s_|XNhFWSkq4XquN@xZNO8k96ABVIxURmX1l{mi9(i_32x??pLy=@pw=l0!6#QVcA?zd9{3_Ou8Y>X zu3Wkm@h#r8zKXq!M56&3C)9^p|J*yMmj~sb>v>p{ ziP&XOP7}8fU+!D6{T$=A$?<=%8EK8Eq(yzLTO#)`H5jaA4$!h@KWj1J<*lLSaP97f zh}YI115iy<`$s+VMo^!_Wai3}7=UD^Q&g+S1V@yOr+G>6`1PMlV!(2v=_dxi?gIz! zu*}T-0|Js;aU(U!i$z}3} z_rYz`waOIHZu*6FOu^_TKXv`|a$g#LfOrnkxF7t84weu!g`1^aJA;j`iv;@IEecQJ z?db;ZN2g)+gI5bqJ!WZ*&)W2H)~E5*9-ne^!B+P=O2#mSSj&8QtCCTlV_Gnx0!O%5 zIxjo{xkYHf;H5OR5b{2mU#ER#I+wmHlXx>%HFqKoIga*CfPePJVQ^5fwa#Ni9LF|$ z28U3)wvpke#9F{c!PAx_hZM>#tsVBw4A^8^H46%gtYm0HaCg(; zq0VMRKJ&CyOl-m5eF$tk`7S{|Sav~HqbWC7uVx(W1wwaw$jdyzqKHEfU*{~MV~M@`&noM;Y@m)lLFp$< zZfw{!TqrC}vieZf7bVq2476R2{fypFk7=fE3Ac&59I!Dml4zxY6PNq64)G{qW)4S- zp=y}>9pQ zHH7gV@4qjtag^oKITV(fEA5{bbDF#^u9->kdXM%o0J3x`O=qUMBTakVU{1P_WO$vOUdG;Phjr8N3K@MeIIhdBoe1{K{QL6~z;Q|z$_y48zS08v;WMq4l@Tt?@tfz4^zCcLTd6I5L z)~*)tNgt#)CuYLh^i3=2QEyQDzG!X1$lcC@3e%}Rh}ZO)>IS~j34W+1PdS`>_y?N(}9l+Z+t9d5)(YT&1kZbr#n5tb7ocvW2Y+&iuDG^wwT@+s}a z)cPPQmJu{PGU>gW0uZCGJj#x0$ZeoFO3V%^h|*Cx?=x2?MH=SZ*gu94=8u(bS!vPd zBsxOOkQrHX)WYMikgB^mj^FRgdcRhgVLhjtEg>h$#)o@Lo3SqF_MY`Lx4+IkiK%6G9PQO zgkYx1rqvHuckOV;>89}D#EtV-4r80=3OEfr^;+>9Y7V6{DLa|C%Ro#lVjcyIliEmP z15xBUuugjRxd-3PB9a~se+Z3$6qV)&@+|-h-cbpre}iQzyPFd#_))|5JM4$as|`}{%S%?Y+xIH7iS3&q+lM#BLY^#hak;XFt) zVmx}nZ9XxmDu^!{zNYrINpfkThP_ql8*-4mBNF~A&sVwlhs&s?Yn3sl)_BOHH|zNV zP&#Heq!>3X+uXL6_`hsJyKb~)(6nANc(%>bEv?hl8$M??N}@d8;ZeEhVw+_d#zz7V zv|c?>at%fpLp0hy{CSdh6>il93TXeLwo4_n$HA}9!xx^qt97Td=YE#TC};{5-Sa*E zj2auJ1x&E_3a_c%*|5n#y(IkQGC?t7ud2gg&r-BJ4PD#0*UY!UE>8l5AVg)sEnW;u z)OT1Q6J;yzMMCCKssk_TBWDY|xHtM~cY~uTqsegEKW7?M{b}u7v&to~du#tOOGV(1 zAE8)da{9Duciva4H+~&LxzJBjJlmtF)3~9<0$_`KrYb@AI;pm(w6ZR9u2`rAB%BXM z+#FNxea&%U$xU3johythSmABw%!VWn{_*6=rH2EI6-R1P?|JWL9P@LyUsxVjd#N60 z!*ks#-_6$6mRFOg7{Il!z(H0rQigHT8$)#X~=atUR)IUr5RO~xAvGp8M(v6w^OS{}V@HIT^Dg zo9ZV|1}nuAMWLi2U0GfU{5DAir}jmVAC<)&CV-WGWYT?#YKonPBHqm*aFFccpCgM@ z;zg?J$gp`lG1kRPdk7sq4SX!Q^0}CmUNNYG@*f!*8Y(y!YnA1zCK~IO+``U9)z9LQ zrX#g808&qdRjn8TXsha|n;i&@P3OAyVCB`Dz1ASkYjCSW-s4zbVvcj5>{nqVmxq=> zLEEfZ+er-Y0Gk46p4(ncjZ3XTF46#PaVwOM_zkw{}Zdr*Dz6v*@xV@Aj1x^5d;ZOhAXoT z@fbisNNjcv>ozCk&HQUwSu)a>krMY2NDdB0HzSu?=#~t}3r!Wv&(}~EG#DMxRG8aA zjI6}4MGlkShOwC-gCpLSSjcf| z4pS3lKaY`3NEiB-e$0qLaN?YdS*)Wo=}}ghJ4oQ98nRSVN-n$+TPs0PLBfSg0mLo8 z6MmUE>#SJTSDBZwI#8J}gLFV!XByyX{>gK-k6xu{;M zVb=LJFTIykI<5`xqtaS)F1&!`$);C$NIM*+ScX>9x0tQx$a~ST>JY+{3{hx4zYg3_ z5A-n?0tB_Y|3_D0Y-m`1v1$*IaseIWiZqQ;L%pBEkYStR4&tcENoE8T>2NN`FJL$|b84O1i`7Wz0OOxhUlntIf7aWcg4EJ=fEsVDEtbbbn=G@v@?WvQUI;8q zGsjDL&|l`U4Xv5~)lw{E=r#*3l`qi^clCWJhTxK5WXfHVB#QKwmwvM9AozwpV!1cge7uoSzn8JKY>J;(d!G+gSbxcgyhB6_V=Rp zpdqOdg)^t!1x6=4BDwNKaAt6aBSM>KcskNRP&oX+Ml!qhbuy+I6ZA+{ixyb9#d5L$ zfx^BLne5h{13v1c)h~O&d%mwaYekg3N8!k*;x>Sc;0r#xyL3n3@(=lV1WOBf5+Jy1SZ>-)1LCd_81LT#JIUXUEc9Zjx;33u!+;YCa|?4(sbQsZTue+oXu zVu}gvlxyAI-m-V8@(-E)^VB-una(^N8_+<~&9#<$?+o8Z8678+2h`cmnQel}+}53H zbqGZA8O>tAij|s2FzgnF=tQVXfak&@gRmbbS69Ki25CD?WEz|HrIz*B1tYTh_^^Ke w_m{zh5^IfR`(OX*e+*9k_ZRv99{bM;?{E4?oVuoQiTnxWB&RQvY`)&`FRBjciU0rr literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb200/gdn2_fixed_batch_bw.png b/benchmark/linear_attention/results/gdn2/gb200/gdn2_fixed_batch_bw.png new file mode 100644 index 0000000000000000000000000000000000000000..9361f470193dc225ef91a5ffeea401f81dddc47b GIT binary patch literal 100289 zcmdqJc{G-9`#yS8Nfaujl&Mn4kg1SaC6XzrM3f{`=1OHqM41{45y?zQG876C3L(mv zp(G?jC}jE_mpxQIw#S28yCq zsO{gQOHuT^6t&2Oo(?~eoLYSZ|B-Rpd(`QW?HQ*FCmqbFgD0KrtZkjFElzQtH*;{b zustU(EGjB2D#(4-$;r-9MnuHs-|rB%bubrkXk9dkPg!QS|Cl31v8^Ki(3nTp>ryln zrM5>&?_%VS_6vHfO`UWj=Eoj!uVvYzq`gN+u=xOMS;ujIcFEA@Ri=KtTi+jDZ5ohi zrTs&0U)l8~uNh9g>R{Q+y5V3jQ~)ssp6jhenoFRug<;czh1FdUTH4-?|1D97HQ_c`aeH!$!=bjmH+k9DKtr1 z{9hloX;UTJkKq6Lty7*+tNzEQ@fklkMEhSKwsK_;a^0~XYqoaeKo?fxHCQ2tsGs=^xH;#pk z&1bLQ+L)u6r`GK{XQozI?-<+w+~TrKVcp4brd2 zCVzerFZX%XyOu1?yVJ__I(159o3Ib|;q`|j;Jb?Q{t*RNh5KIq9_8s9(o@@s$p zjVDibA1P5 zbL`BSGrmDV6&)`xzfV8@Og3ZQ)S<7A$MD6)l~mV{AO7;Lf4)|^1vmNUPMGc6W!v$( zXnu~7!c*LcwRs>S+OxDr?#k?+U+W~!YAgASDV?4;k^Iglf3365(OT1UbA<_siC^jy z4qP1XTlqllG9AxiYa5&QxRd6Vmi}yaZ{@3Fy|vxt0ULa8+_-xC_R1fFgA2cHZvOhc z*&{7lR3gu@mr2;T(ErVw{adzdd3gHmon*s2F?NZw9-H>wJS!A-#JzX)i}Ml>Hjxuu zlRvxh;C#Zv*IPEG-Pn3CLUHTep5s$m{dwMqU%py;5=pC+=|2w-i^w?jagUFWFQx(% zBrQL%q#nswo05`Z*>hpIBR$8koNMXQrJ|KCL;N0W^CidA3uJ){yo^1HScNRD=8&)wAZ03RA@Ej<>z-GaAG#^;@GzUgsAVh z%=~0`(6P|W+}yR)>u>Mwk8Lqls+ZiSs;Ww(c=6)JD-Yj{Hr^98T}+Yh4d^$%@^f$K z1e>d?tEhGJj+IEXTn;d-vY>T~7Uet=+N*sx(kfO8*0xzKg`W?x^QA&0qs z?K<@vH*TEz`K7F+rbct1lW+6p&F?de6;`sc#%Bh8nv1GRZITof)j=j#?=~+GIQ+=h zk4@y;f1YB7-4q>pj#QR;ZMpw?@mmibFokPe|Nb#G?!?PWe1{*gu3EjCY%eY@F2f=> zj!xGJ4*j&FSBr}k*KOEfm1aBM_r9J!J~){5b$R)hq4t937k_S2R#73Vo^~{ocEe6< zMIR=Pr~bD6^$F|bE=g2|3f&(xZObwbk|^mYD_e3fLP9C?R2j>vRh}w7OrPK0-8T7W zAmis~&jEjq-8l4?t`fbl_Rh5-#5pB5EK%wKgCLJ$acI?=} z*HKnFWu&a9k>8u%zEv~JcCxg5Byn>X-!U>WqImS^x)(2B zIzR4q4G}o3J0-Vc2a9SDr}u*goR1EtUPq!jHPD!z&+%!V5$Ov-Xg}N`hvkoP8T*!e zICZUmK)_0N_M6=mLEXhy7qM40J~rBbFe<}S&UBl-w46&`_W1PNx}GwZcu(;wlXkDm z%3d`$uTxV~Tf)J#W>a1sn`McW=mm$d2_XpyR;&>%#mcX{cxY(I?)>?ExImX=k6+x$?7&&CbEW6TAAvi;GuLXg(ufH0L{SKmVio zYSG*uMl9j`qsE%^H9chm0$z%pg|3+whEGr)_wR3D-)ZfO%<-coYw3ldPfBD>P&o85 zPRP|(x!CHvBhIUrs(;XF z=+iO7d?z&_teD=1v}1?Ze+7T^-F(nN<`8ZFW4GBqv>x{)Ebj2>r`@Bt=Vm85HL9)z ziu1CbDViDeqqzV49!!F`y*%gsfP$Gqe`&%s>8)q z5l|M@m#0R}-lrWCr%_zKQA%4#?%6^DKP)d!vG}bO(p6(*WL*9G_itK{vvm)Z3WIBp zs-AQ|$Ab-Dxjk|LP03p|JvJ)+kaCEI8f<&!6Bfoc{!3C})0*x2%MT@K?yY?w_x`a_ z!N$Ri{kJzw&P@-$&$c)uaqiPP+s?wWXPs_-(-Xs1?Zfzc(epo!rWqG;uHLHQk(K3` z`#bMsDII>UY8vtKr5tvL-=$yQd8DP+Bh*WLS8jglYAC((zaWsF+SspOW&I6FpS!v| zl&;b7NJ_2&<|#q)yLIm#J!(j{Rg-2?&ePSaSKEL8!2hYpo$Ke%pL|C75(e34X)a%$ z)~MRMckkyCZ^nzGUzf$k#uBjdJU{;*#KB(mg6o31IzvsQ^fhFW&8Y0}?(g!sfB$|> z#>$a%t|M63H>bt?H%hOvx3@onVhG_Ja|7A zS;NxBaqNLRR+D5H^V$cmii;`4)7ekYMbpyLvt%Ea5#*Jl$hK6;B*z{V(^}O4*mDbiWlVpN~hYw8n z)pPLRL1+0P&tSF*lr)Mt+Kg-xo2_AhyQFVxWe=da+yzgpim=1&r zDm^>bZuhxlNu-Q3!@CH{s)o_x-my>0$bj)#o#xQS_#!rYO*iQB`uT zfAqHS@}^i<4f)fc4u_7~wmjWyWu&8{6N}A3>(STovY^{E|Tcu<`(!u+ogHNTS z_)oSxHKX;gv$xkda9_;qCdG~7v=w=hf8)lD9~vEmP>>U|mc%C|`Jpek@ukeq%FypT z!h>Daqy6ROBhL#8tPFeCd+-?-z8LxW^ZoN$e|s(&r~SeFhxR4>NIRD0X6EVuqZXAo*DE;#C z#JQi{K?)3chL80Nk7b?NBqk=NS`iz+F{=Ob=NM5YK!?v?zs6@irdz@8dw-YhH;b8k znM>ozPF$7N?I(@)9WX99fAD5Ms;KlM6}FZr{uqNC%kaRn9}XimEulm$KO`G|JalXH zi>&a87e|A)AJOd2Ni%exnSGxi?NX}%h))WGsoSDOi++AQVw@84c6j1|5{mbIihHfd z2|+IwZ-+#$$c`#Uhjz1U$6igHr>E6&o~{wlV?ji}bMl(4iQ4U*kdRRBdOsrKCbs&S zsn#pYN57xo+w;oMPx{_~(+3C=bo~p0|zxJxCrp6~=u>qOssZ&57>{Z*gZDUkX zQK3-;=GoLX-PE&wv!I~Hx^?UDhLNsUEsy43U0Z6|Hu`1QP+#n_5#E|0p{1o2(6ffY zTVg{yfa+-Wm3`4Aj*NiFIw$zA7~Q*p8ale)(oz8NPEYl{kMq&oEFU6~ZWLhzpj4)% zFiRSojdGXt2IzOm$;qK4kU%a+?_rO)lsH?*-86j!VYQ~dzMi7=_4Qr!ZBIPEFwEKd z%$6D8799^8wlYbTzlS@6nwyV~4>U2O0b@50S|{VgCo3mMLmhskr-VXca_(Fp@I@D% zQFFG%Y6QXOfq_)9u~%`bL3X2G)ug1Pia&l_bN>AK9b#h2Kvsbpr9FYBVqB+xOF4d9 z{=F$9Lv#rHHx@t{&0k-8!LE%Q94AdoZVet=1w{SCtZFS)jDo^QB_B#+A{_?`6FS8e z_Ym>w;X7?wi>s@bGqMTqxpGA||H5z}yQJm*{>s~@^S>QpU}D;bOiVx{mi!^UGua?V z0KoP0;9vm0@N;TLVF|E)cHo!$QudoT72KAyv9V=ax2*c}=g)Jy&$M{{KB#t|+iZ)< z=4Pj}Tm|z-e*fMVCTw&oB7(uu(a~wJbxm$=?!}QVninr#kbO^pl$86N>?8g3h;2Uy zDv`#D3mos?&4~@EZ)ixk?TWT^`s}{1*z>ID^cGW2!yOzH3oEM^>CsSXpJZk#jeUEU z75Q>8MS#`p^hEd9w|A%59hK^PWG`L5yjfOuBSFVgqdhu_8tVzZ296q;n5f9LX&XU8 ziIlRZ6$@Xk_`6dJ8T2Xu+NRB$dG5X?+4Td^D1Ju@6ROVVo}PqXQlkU2y0^{`eY$}K zib0zTIPOF!hP=G6-mm$G5Efj8bAPOmN8{((AjuoZMtgf4Hk$N! zf5TE`dL$1zz+La09LWCvu&2IGPQHBcf@S^s8>@KL_Zyo~#kY7=cATkRck$At zFHOeohi_yZyub5oL*fekj1vKqlOP7(FG3(4Nj=_lrMO_P z&dM+|Qp$WZ>&%LM{_CzHmL5s6EGouD|E%D0?+P<^)44H)jaWKdG|wa=BC>et(gV)s z+sdoAY~89}<)N2uu%;r2>qz*fDU^(}A0MqjpXd#=X5p%EW|x~$l)Nju$Izkuod9Rz zXR2?xX&pRRRuv{f?nu^cW*xvs^9vVo^-#eSM~?Mt(D5kDj^C~MY2JgnRX;xn$|6Q( zDM91BXn{*{lL3cQ^38AEn{P2D!_|f z0|S1GmNGp=Z$mwuaMR^uo&27ffkj+qWo7lF?HTim4LkMT-VF^+JkKd!nK9w_?Ab2h zt*3{##znjRS+@5k>)Z2r$;o_#jzWGwt5}!sujN+x!!FMiF`FIx|-w z@P~xu2YR4dWjv+$@zvWjNn^HOUA=zYgu?vHBA&$&5*D8LaWQ%)UO65f9@W~tTz?EI zQ~&M{xuAT$#j;K7!H1)bt*r)0DL&3`F96Weyg2^*3ve3y7)UJo%a<>s?i%Ll@*b3q zx{Rh&GW4K^1}mV@N167l#}!x+I@H?!7ltn{9X4&@*nNJ#OU(nh`$<2|%dnd%78I3g ziD(CzuhnjZ-yzNOVIObry`iG!=9dN#eQGYKMM9J>N{%lsUPmKRN(bm&OFiRA#)H;+ zsrBte?#7h^!^6>Lkz8@;2#>AbeLjuizP@~YY-nQDbU%_;$SG<2FMD!-M|BRf^d6K4BnsCK$)*Vo@C9;~rxiW=oi%g7Mnl)sXk`2%@3 z;dy6~utBahKQ-9U*89PyFccu=sCLt-bG3284Buy?bWR9&?zl95wqKHwfuTM^D@NOw z_dw|WDv8rG7WMHr=ouNS?3d@unr5GUuS~Jj)YKS^83KHD_r@ToQstO?Cpz=ePBeib zVeD$xwGG#CTU*!ERNFAR-j(5RfA8WgP8j$1<^$-R>gV58S7(2_ui!3^ZtFv?*1a7x zHm%uj4==~+ANJdOe_sH54A!o>$X&s0s)r4aR`_^BQ_(PTQs1XMag=DCJO?wu6ECDH zoO{@SqDY%>8sX>1NOhrZ3A7Ej7x;h`ql!m9m*zCi_JkRqLU;1qeO~^7g8O5!)r+Vt z+qQLscYY3LlN2H_QTzwD-sO>#<9PMz6|h90K|81fWq=r`-TQ?e$h+P&tq7!(`4>q$@y#kybOHVg7gDdll1T!18*@ z+B5*BhjvH1C`rfdeR=7ZcGj7L0WOz*|G067_rN9@8FsWMlVAKqK8$`nGIRU(ZR+9J zekI%-;h6uR%pt4|pWD&k1%iPmQ7igEzwh^nfj4{^s^R5jWoq+94m}JYT)l8rDgF*tswu@^-#@-bsoMDQw()QnI-Qt{Kfg%dT=u5rmn$}ubzANVz~MIziJmjTEz|n_ zAXJ+{(UoVv$PY80$OlDALrGXQF2{3gdfv&Sa^2)xbvSziSa}NQ9{K>I%Jt*8PLM`qWF*U) zHQrz$zn~){3{_H+dFZhFL0|xFH>S5g#_E0zH03-XX~#0MWwXd!%}cE?!fmdkOwOLY zXJJ%cUQTtqn?YWub9J4S<&S=!G``S}1t0$_{zRwy2bizlaIZ9_NJ< z7pH%x8wiGzRvk#&cc3Z56zq<4L?!3NCdHO~XDf&pAXk}G12?|S=6U+?MW+t;X0teF z2ZyJXzx@v#nVy_1?(F0$F1TLjKcAS8pf>eD)+P3726n*biB9(le4u!M$j+Vm-M=H# zGNuc08=DcFIhG%l(+zSy)cY)7?}eC(hHrgOKoj!`{j~vWoKKC z)r3+Q`=zmOD+;HFc2mQJ%f&@Sr%a~ZqJ@XskZ{iq2H$GC?4p@baO)NeRoBYDoC43U z9lK}Q>({T769Q0aK@=Awx=xy#`=AJxzI(RId&SY%OLpm`f4gesIByYW693pJ{%J?lGK+Z~sGTa&hiiHXTzH>*YL zfcW)MTdOnCK4>bhBIg|_GH;|&=NEZ!M6CXo^Wnl>V(0l9_vC%>1sYTL@%$88%eU)q z;)`Zq5CUUs^~*&x&_bk1c1~NRZg1+YVXUNcH)>+T(-fd%+DMODGxNL6whwzVcSAQJ z;2o8s3*~VOh6ExfpZ|1c2s3*MjlAd)>4P z6A7CK?E17i`sc4|zHPnYB7Ln>s^N1y-6Bx-makZG`_7%%27mrz{hdV$xz;V7=-^Jx z%}mKij`lwDk2}$pZGk>|QI0IS2Vj2kOza^6658U*)uiB`RW<%r;RBm?#a$2=2W2cf zze*@;Z1VTu@#lGYpMQ2&+QgFEJTF(d!Y0fZ9eX>s*j@|l64kIT7J-$ zu1S;OJ-xM{c6*D9QHXm%N1>}9I_`jBNvhuY&)koVc|y@T=N=p&BO6!5kW6wpVK)rpCT-`c|Nt0PE>7D#pBgVDTb zBu+J@9}mljS{0np&t`ljrh$zIegD+j1DbvL8aKs^n$|Op-b_f~1p|2<3X*I}2HHTW zOXGZqXa6t}`k1y=IKej81SzLm6tyCslA;90w}>VySExW5G|bi!f4%|zXsIrbCh2M zVA$a7SFgu-{H{jZ4?kK=?bp&$g+PH)CGqW04MSXY^pPI%%6AK#&HJS5|8$yWCMF)Z z*pqV<|2oWsYIPq>kVG?DMjh*0p3 zj_y>QN>Dq;zhQO_!;-?;-^U5<)%z48Kv?7LD7nkI<+o3v>3@4)Q9Scpy9_nVZngXT zk7P+>0|R!H*=t1Df$}b}x!H*$A?<)Xp6KGmi;F8OA1O8-svY#Tf>4Sd><^faz!DK< zkCLC9E_wfcCF)S=-ECT<@rC{kkG5y@dZOc#gxbW7QVV|gV}zE=j99qF0R60gPgYyZ zBksrTpX$zYiWokpDCE4bMqV|s0*}W=$JYE(U1AB6Q#vpSC6{gw|9{;I=xHbVVn3qw1%X0j!#H9 z2{go&cF0q>cDuBT(ZGk4&9n1B9qhZ;kQ9Q(!g6XX_R79Imj$Kd&xgOR5Z6WmVSjYI z+yD)45*a?zwo^Xc%JW+SE=oP_{C2%$kBq`U0>^vYXMY_5{k|3i#L?$=Cqt)Mml#0fC(oW$pzqGK{3s9=W=G9E+nJ`ZwLxZZ3Q1&ii>3|1vtU-L6%|XdnMTIP3C$v8b3`*K>SM%U{?`GZ>7CZi9&nO$Aq(S^n|icI{n7m# zL`lhX9?%DR)JY$^UJrh7<(f6TJ@dCdefnhgt$O`pI=XLau2B3Jk38-^44HN;zUI@8 zojdn|=KYBX{?eRj%ATaWR=Eopoa%V#lOuFBcIMQFva+7LT=vO>vI4`0T8DIibHg=L z(yOLZKu;2d;@qd_G|<_Z=N`P#C^$bT7A3HId&_KzSnB$npca3AeY;xkvEh2xKjSnb zpLg%<7uYp1`pNKukU@4a005GSPhFihIFRI?pX1{nq(lAlod>RC7jnZ@`aaO4DSIRJcb|~@MOboT_%AqX*VZ3FF zZ^-r|>dw%MZeqJV>Zv+BJb!LxOp8YGUU>Mjg{FFDsyEtl3JH5F=}js6-Me@H z?0U6mP-GudpTnmJ!Q;6a6bndOON&E^8Y`(isBzh$odp*%C~iWwg?{Vr=cnkw4`*7p zh@mZboEZqNj`&ylC%}HIRfyTNO61HSjVo`xtL75OrSM?zdDc_dac~j$?)jn}5*)+k z)}~n2u3ftc3TSIbvd(52MIz5PjOKqm#K&6l@#A5N1r=>I{2m|gi3kZDoSd3UF+ruDiDK z#_Da0sgtKq$2%}}PL#13gNQ))W-w;m@#0v%JQ(T7I2os(klid4NM;aoH!*oQ-VqKF zxb?Eb(5F0ISMVNn23r3S63UEkIb7xhryxlAf>iSbAPva|+tY9G;k^6DUvkFCb~K7d zj~{2mv&)f%i{%R*x@Ftg^z3M+&VyQrp_cDfZ`>$|woRC{Zs|pLhlf?dCI&%1CXCG^ zFha8bLYbGY)B5ljF;KCD@2i#ErWwhCE=6|=EkJ5KI@=>JF8)+DO+)z$dyo<;Y!vZ! z`cHTt@fs$^q^=jO8=%ybcvQ9{-0;$RPd>9`087 zg_ck{kn;nadvjmZCyo5=pN>c^xZ+%Yll&K-n0N|d4j{XQKxj}R50ROkWM_MUtwR+f zZlHq)txuiu#D29*cSJtnJN9%7#SJkRx?nl-8OyqLeCcJ!AL;QEy%;n?t##A*|Aj?Q4+Qt8}VElBbp#Cz|K~sBkS?q-bx%H3(c?RB3J%8*^4`%3i*Pi zIf+e)bjw4uV0dM4fFi9Pt)uG!OBGgo+2&#JdAQW)Wo+ z{<3>;GU=TNyMq5k2hB`!?AW)1gf)c`0;Ka^7*a+cRLEZbz5MFctG+jHmLMaKIXNYg zT#a9%AksPY*Drx+;FXmnPO=^r<_q*A!lK-{dshddwl(zE+AobE;QZCYgaeSgiz_Nr z^Eq?!+~*5hKMa5-R8mx=W@cw;C|>p8V$UUvqk`o?52@9!SF*8fo1I6<+-*Ee*AZZN z-r4yuzcHTr42(B0RVh3%c|{9di4_fvCva%Wp``egkr3g_q1sE7-wF<1te~JkDFV8U zeJf_r)~xM@OC$62+f``h9zt*eZeNP*%EHM>_+ywV*n28N?ly+GM?W5(f~V;fnmZBW z!aZ=R5$l4*&F{$ck-IN2iHeHW(_g9|!o6~p)iV)Y;#a$3xx4Yg`Q^n_U*v}ht)5+N=^+G4C|d)%%>B( z_*-ps5I%)G2_i2+E(t=0C3KH@O{9{Y-42S}Jh5cv25Dg{o+`Wf8Ksk~#$&@gPoR6O z?VRG>tp~MnY4`5m-^{~9LoN+TF1{%R!MJ1BuBM~r#Cr1^9y}0(r?3miQ{Sp_&jYl6 z9Q3gIG-2old7W#!f06%nwX!cF*B8bewTk(BvFc6 z0lu&h9PaAkhQDNUdueBpk4A)qTClK@!NBDNC|fS_jCyWW?_Pq zoQXtn8!Cpr>N!Nr$AO4jx3~*Y2m+!9Bcr0q&?$0v%dyFY+ra;?ivGp1n8btoZ&LI*nwsMu&b${pC znXy`c#KwE*n@vz@_a8jCWaQjpYlOGnDAA(fQ#m%Y`b~?*vWURl;+5g6XcsLaume86 zlc;9`E>~P#?f;Bh0OFazo5Vhl2jdzT1sYJSGx%_WiLlVn>#*d2pk9H@fX(3(5U>I$ z$RC7(h0ukcU-l9L5%>YsXAfM03GeNJew1NgpS_+3F;tbs;m6rW0r>K0dy< zwUynhG8hrKLgMj~^cQr~*GuAgSK>R_(yJo;itO0SiLMFnI`3ZOk ze=MDz7N?t_4!tm==I2X=I}+j%XVl;D!3+(QUJ4sIpk1ItPvxujb`Ery;f-x5T+k~` z(dtZzc}}Xl^&6w7h-nKc3}#DOSP;-sGQmH&2L^2MmqoNK?zIY6CRviSqRWZi8NYue zc>`os@A6-n3a_VoOk-uEAsO^FBn6er0&Tv6;@kE8`%Sc3ec4;{6J7g83tzg7ZH4pB zm@0m5-!0K`Hv}f2r4AR`%>?Hmn!@^53eP9}GS{$VPRvbC%qx|-`S|DofmA^U!3bbt zYI=>R2Dq2-!mV7Xu*l|2u{6c%qRdqAAxk5%(4wqFO(avSN`M6IQh?HL2MAYy(cJn8qxAbHu>GPMtHp6 zKcp-I%m3+x%W670x+NT|Zr1=Fd4aXK=;9LN*joeRr)&8|A-BRw|DPv+F4(B|bNp+7 z!FBO}+1BaNFAMhFl4l2f`WjVfYA^d5nacIY5c}R>{XCg`dydMnXkk0cux$s0U7_wKr@fD`E}`41&Ngymrg2ZQT3Tv7)v z>GR0nOA@d7zYa?%XZ-uKLEbOCVzpNP>)Mf1{=WVXAHMiMdNSny4;QEWPY4QV5Na82 zkYZpxwDf#^c7!IyOI{5;RWZmPogz0^r@p$Fho$`J*DV&*4ho4(HC)V$K?)HT`vVh1 zdz6+EE1mQOxb>h?9ZY%whQli)Ydg>>X+Lek-k#ne zDyoFfLw#8U2;n-}C5E|-T|YXDxF8%X3KcOvQTLfOIX-?1T3w1FjRf%aT08PNEv4aP zK8PU?4p&o@5t*LBmaJ|6j(FF7Gcz+zmb;MmrGbe6>B^W{)?gR7YA%c^^{pO-eq-L+ z@4@k}F(Y{`h&I@BA#QBST~G&Lvs)!>luyvKgiR|e1apss1T&g_>iG75)P_G^#Epv1 zdVAlFh2;ad+Z>mX5wu?X%vp`GkrASc5_2I&9*A{g>d7ly!uxFj3uL6cBbphc5M;qjbI55m;wa4iFOFSijDZCDr)9GmQzeW!~n)(Or{jw69LiVU}w_<-RGv3 zf*QPb^(sxGMtE$f+mB{ZQhtd+44uH?dZ>=X7~tjY{TcTLP`($JO~y}<^n0bgfoN8n zx_kGo{mi627`CNwRzT%b204}RYXkAR;U&)tzZdZ$|NQ=OJF@n1t3ovDT_6F8tcmK$ z2uXq1W* zcVO;nsj;!KmzP%;{+Kw}0J9Qiu#%o|kd%~`lFCAC0fSPw7lzFXa?u|R_zFJ*X_TSx z--7gV=gir&zVPCl0!plFI^t^h{L(LhGc}PnNU4VhX`PfEH{jIf1m<-+C?dcBz?`(P zNpIvQ-f4hrl~boSQ4is*vkYKHO(Ckm;l>yKfq~`VDTvF4NHrWmAt7t#{tSpf`_d9R z7hoCVN6U21ePEysHOAjzlqzG!Hb1@Xn)HSb_2IR{o4!+uN*Ul`Z;h4 z#yW%DZ1~WHyMV4E;g*O-ULHoWWuVBSP6gb!p#-pFhfx63RifI|H8hk$v3P>@Ls>N( zth#~_Pl<}bms&=xBGm^jik1ES{dlkHgN@VBZ=Sf%yAfLw(Gp<6@+X=}KUA1oGQ8FI zcQJ!EQib5!YSrD}pVBY@3($DxDN>AMYcYQC@(Gzx7?xlB43<9hdywCspC4pjD|plc zDX0srZo^S|L<uBvc3`G}J}-t`MS2;0sJKR8;i%z(aF!;2Z;QSfdgVvB*Xz+ zl|*_#oMT=E@RFA(@$h-?MP}2h+`4sZEVO0}2~rendrROv2UJ#t2lFxZIPkM9T*pb8 z%ZRrd?Tj6qLs&A7j*gCc`Qz}X0bdk1H62O+gg{D|C$=rz58v_JwCq9bM0_!dqzcZUkKg_L{8Ru|iMDwAidv5uaoaMtkzlpD#XKZnx@d zqZNXX+{s)GJoLA6F>zK5IqEO5airyluI?%#sv!zY>f_Z&r;hu1$}t8$<2EJ^0L$@F z#0Ib1iY*T#IU`~0N>dMz0)ZH`M?NaWmBx2^v-o0CEa5;H$Sryp&yC$@*T4y*v%L+u z`x8_h@I{9YTt*thsGJ9a@Dw)r(R`;A^*^w4u&}UDd4_=?gik?q*WKWy3m#K_DUHp! z&cXmFSzn-3+c`RxBD{27T(svml79XfGz7_tAX?%j4i2vDyfPMG00sWt;V)J1GR>=d zzkQCcdsU_S><)KUoXIi&{cbyi5h@&g}@BIXGWnq5ZIx0oWiCkE*Q);@k# z2Jc{b=$|x-*-)pBXQ=4v-a$!NL+#Pl*3LCW`yR+3u7^b(tXpqEM>4+ zoD=2kDlxs6kD@BFQCg)zb@ZjktrX1TRCqK_Rv(#LGO1*%%C(ty;B;#0w0@gy}>+ zzE)JEK-fwc;9*g6y172=+?)L|+Fj5*K#OcedAc1LDVToQ;`C`R*gZcLTp+d3(oEd1 z1pi3Zqk@!my}FhOKRnU2 zz$>#4V7hVztZ{xMYvRG|KX70XWzp}0zGyRw54tO=!?jUUO^t@~->^%#A`|{XE=0N{ z7W-%Wh0c@kxw4z@U)+(i&AK&v8Tg#~>L52aHx}h$C?bEVqwR0_`1;agI~)**W~+Fh zgSg4ASsY^>1mTeFCM$oeXe|t`Pt0pM11xae-ZTqv%T;+_;Kmzhe9R(x(FxUuO;!d{ z5q!HvDaHN!eGQTp1JQ)=uj*_<)h44LQQYXnuA!0J1TQf*;Z;LnpMrwj=o4XD_ruci z$jfu$K~oaDXo>R)S5bm@i^-M4m=(|_?kBxv%VLTce$#)k!%q%KfvLt*iPI!Sz5f(+1Z7=RX}$5iI8Xq1p$}+rE!of;kwS9LvL`ZG}zA&dG_me1P9zwSJ_R602*h z86>wExhdT&cyJ~5;)jTBSrRCPB_4lKpE%n5kU86P=N?NMmuLH%1xx&zsZA!czQ?k* z5;p}_LB{#}{w!`{1N?IVNQ8hyG82Ou4uT&Otjh^emtqT(OT2Uau&{WqE10=vY0_bstFqp(mOmFA-$z9|aKYG`*e7W<(#-@~M&u>P=V^~#+- zzWAzxaksy(?@^9pvxw%$IVAi-r>Y*B0 ziKnHA!|%a6Eb6y}++If2J-0Vqw|@O+KrDp%-FjHK8C+dIRs@q;MAcz#k{ALY*;q~= zhXjZ_c3@2%T_@>W{Dv>9$^UPFNOJ-99({Ux8Sx2uEAt3dJygqc=+TZ*ULp(X1sjIoZbTKCfP=KK=wj1} zOh|F4LJNiA^6TlSRSu3n^2GqJF-Mg+xYF8>8&6(yESA)y@wbn(V`#zId0gO%%z1k+ zv1zD$t3J$Jv=lOaVx~$7Dirwp z2XRtY=Q+CJC(8b&1|K)(OB`M>FKmKMu|XjYDXJ8}P4JvMDl#=3YEn0zAWNl3hlCv9 zAzp7}>uP&z_xagfgusR{e`R{`8MNzaiK3@+B}o4^p^GWfK%wfADH1OqpKg>Y>OAApZzIzz@UDmwZ4V$ntlO2Sxp=vz zp(Es*Edl_jpAU(Co8!v4)B}x>YWHslXC=Uo%Jehn^73xlV(9Rmtcw^%KZyRE(yqMP ztiUrmn(NAyD^&7%bs~WgxePSaDa>cpQu>gYgcl1fu`of9p(u0$RX^QeyTP##98@trk4oN2dwRwd z1TFFkc*J!m;pumhG{VK||J=n>*0)-Yp_XzCZa8e*M!~K{`aH@5hRRK733zAW`J99! z#WCmK518aPQS+LOF!VjK#n}hJiIIb(eK4q0R0Hb^$fUcWWEz{9_uUR68iBP{5U^oU z2dV$T;x}mlh*_6K+2pnFN{my6lwYaTTn4ya0v`m;S=*qx#uE!GAbILF{XhKYI{7Pv z1=OpbpX%kBftxF@btwpIrG{D=G8*M-32N5^n407jk~_%+Af5$E`zsVc)Y~Ow&_=Go zncbPdHFz746;Aw-i^J^MLyWNMUcw3@p|slSf&nIOG1S6LoH&7JyaYsrM`ord9y&c5 z7AQ2p^7QC+YV6ZMHSEP-66O@BoXqmT9t9i1E5s{ejSR2=8g8nX{)d1Qpa-Xc3%p*L z3jj*$EdV#h5U)Zd#j`R&6W3#Z;erpG&S*QlApw>)G^|F$L&gdK^D%Po2?{jDdFV(4 z?7xIN#~vfb7P6hti<|@{Pnu&$W6wcZkeOC|?irVj=yrC0b&w|7Ca7LzJSQ011YB^Y z2RVTPBUmI&g2u)CkmaLuM$}$IH@vYD>*^vjXL-Q+vO788x zAPjWG3D?Il-uAjW7BaB|%}uzc=`<|D7w2ZAcb@y?d;HnC&oF@KMxclgN6XbkOTRrj zqzrzas0Yi@7315>5UdFjsQaYM!n^SA<6XQ1m19WDoRfG2Hf=;b;ou#blUJ!UU#eh!GKcNwU!jk~NGNr!Y-RW@7-)ufboC{51u=E;M0h`A|4%m5q(L z&}5XNRUp#K&W1Pu3*v=U922C6zzegyD(1$6=UwyU&Ku1_oF~07E^-9yx5aO|h2b{b z75BTL{g}L1_tq;h?+(^6xsK@6K>BplI%$V3=r5lhmKPISg-(`q#8h!d2Pdd{FEXS! zgD5Y9I7mbpSbfZoxr63_ILS;U>!qwBL++uy&=diO1u3GeVECubpcAQ^q*x-+f*U46 zG&(r=9Z7lFvu|H4c&TJ8PoSx$re-<#3A^4JjyF9HV9jD+NCS2+!#2Yx<02#u3+oU6 zg!!L)VA@{Q)@qqXK!Oz%6x2@D7y6KXoDB{!D>-!d1XzF$!eY^3`UZdd3KlaEF$6(S zCYTOE!D~3G>f|JWdDa`yf#JX^h1(ja7RMmyR~7s>2P4s#G~+Lf`7r?OkQSDIv;ayG z5BWpr&Bfd~o5419BjjqvOq+VviCsclPeu`m)1~S-7M_K{o0jYlid1MwZ5pA@B&}%t z-yALEeWLN$-xB|ye?JXcfLQ|KQM|{|D4?ADa`!m4!oNHe=hPHf-{aqnHrcR<)V!4|*PO_+G40Y=7LLwG)zZ>Z$%inocm>&nN5u{p7zN_wRoPi!eY) zMXz3g&LulBj8BIa5NgKb@8$gG-#Yuea}SlEUhTjUF6sT)4@5ebH~IGm{qsM;Ix&O6 z4^P|LGJ~JW8I0upAE!noo2>bxxeVc7g4UfuW;x}-LaXfS$#ktR4x@=cqxjjv9SD3S zMp=+-JUF3)=!`izdqG(06keIKZpuO91kKRmAV9JZ60+m!bGc7kukPPvZMv{I1U53b zCV6=#YV(o72Jl(vMiL&D@(8eM96Wdu_65)+tWJCj*R1!3oMnJ3QU>+Z$~J*+5xw|p zkZmT`)@hDWOPMwjQ(R8s{qS(tTbj%80^NW~*eiv@etm~t?OZs+B{ryY+k5`cs7U0O zDY^*@q;#W-L5w!4_HuM1T&LalMBhZ?2s)2!M2EZ=xFT||vC3QUmbtk(a*wd5nqc){ z11AhGwDuCT&>%1GSZrHlga5tGutp%+=7OVRm0z8Bf;QjCct2cVH?MTEy-)J8S-x;f zTt&*A{3lVC?nyf)ju(KA#PKrM(8#D5E4VX@2naCXdVtrS`)H@LV zO({OvV4KPZ|2AMB42EDXpeUp;5GsP^OBKh%Ma9HCaaWLUJ@Ig#zVcYuR!_@N9sv#5 z8{8&8FRhM{WCY$(2Dr=J&C9yQY{NDD3Cy`@*7+V>Ele_&Uph z=eNSa{K4SMjlLcH;n|j_yl{7o$!JI_)58HvL+#$eVz#<%-NHr7yJ2H_Cib&pZf?iP zlEoM`KK1)Y3(g+FvtI>#l3-tuyPnnK!U{Td!>3pIE{oNel+Z0&crKOOiIWOv$d@8ne8^g*U_(7 zA@uL$)VGM3zR5HRoREa6ZV@&H#==nsrHETNV2Jq&pZx^%5^kLwk|Xo+ZUgV8e@O|4 zqfaM!A%v(mO*(%glJIq8fEy|(8I>i>BE*Zg`cpXHzy!iW{pc-q5TwdD83yyxOMn0a zvUaZen|^e@G4$jwQl`h@Y*rcv_U|u#;WCC(``9HWZ=HOvO8gI)dq7QdQSyn+8ZMk> z!VU(BBKWWSVTeLQMd*F3cQL@;dqk>rlMaey4ioy&)FUEf8lSR*s1o%ZL~&?2SIr&m zNzBWUPeMnGPjbTa@N!U#psqcc$Prw=zWa6gcJpEYE)F1{j1oi83Pj>CA>bJEOvEdJ z;l^UjvVqPL>C;wFgCXad#@0jAqVV^Etr0~XMd$hCVd=k^=twY7Fs+Szd$$2c-Rw~1 zSoQP^zL!EusUFK;276Ov`A#~RvLLn)BOu17Y?OS|7Q$~&o;F_?K|I$$cazCFY?{=B zFoDAi;1RCfy?cme&%~6vrY4bWl$DjAAcK`TqErgqy>sGsp-o#N(#QtIE=cJY!;bFd!I6It4g! z$%GmFM}FWOUZH8krpDm%VI`lbw^og5>oH!V1popS;yU(RH;69~Whi?X90kYe`>iLS5Q#@c%KnR2Y$StW5M>o=mqFo}jNovUW`{MZqvYE;9f8pg~T!czchhx9!dFo&cf`@YH<=;Pas+w>fU`Wd} zIo$#9k4)2=ahJkJhUmMj_egIUfSFHvx)6K^U?F|c8mK$>G-)i4*TgtIOeDlxiQzCG zWYmRfFE4XUPQ~&L{C_w-qHiG8UGnbTKfdzuk%n7>Yp*EJE6lA3NSO>R` zi5XBVDIv3w8B=&Lv}>3!jj20j1X2_8Hza5grORP;#<@rvfuP}9IFRT(h|P&i=*&PfKMzMwh~1V^a~EET7`D!UbN31nTtgMJaF>h5>>HLPNMX}jE36Bj{e)HsHmv% z5^C=VDBU=C`1mj^3#!x;jh`vHYXO@j;Wq*Th_nObLXc6&c@Nz1Lx^3(m5WD$BiH-@ z3cA6WS^mP&Qvm17;nb}_U_pOlJ_=C;83I&z2^tWIAtH_9>h3sw!t7TCj-vb z0RK4>w<1o%gZXk9G60dG0KDr{e&F~5;#Qx*$w>9N$W}y(zKZ98B_%m-Bbxf6l5Lf@ zA3o$J2M~cq_sPqX&Ufnf#NPR?cOIe#lzJVP$1Z3BF;ve4114OzBw-Tf)y-Qgcr&WX z9vl||3i3MfP`S^~VFSH+8p%Qnv*$I^pGNK8X5)?o|7X1a8s0)!OvI2P`X4vW64j(S z*jl)v$?Pox2KjI)MU-vcS==FP2-g-Gbwb(`Hz$aE%xlEoc>q@x8qi112I%#bkhUQD zv#_&ASQr9=O@6y?JAyHc7`WbX2a9mGG!&Xqa!eLDf%qA?)}+q=SVs~I-o}A$Lnn0% zYBC1oZcA9wU7S#q!FVEAFH?A%1*UvfY)E`6v}qG9+;=`fLCiQbCE5b(j#qpE=9Eq$ zH~?7G6Q>~K1VK>x4E&2u&KvSG630z2$LeIY)7 zM%_F5#Dr$6W+Y!3R8W%~CP6Z61Yr9U%k!$L>Oi+c+>ytIuW%%!KPt3E8xj;b&5#Un zjlJ7d{~L-1HdM++6^vmKJ1(qM05fLn3OJ{=1{?Gm?)M~kpB>;H@K{NnBRnMX?t6nF zD1FV(?Kguq)yw6=i6&&^4hTXUf?C>%mt?Q0!o-UT5Z39V5`P!LZ~EpI4>_y?yKl$o zH!NU%j0VHMyCPw!PFU5Tn(6 zP_N1m#tV~6r^Uj76IO%D38}s?salT-oQmh@FoLKQSH9D?banE4wG za@@DrpaQ0c9MJf&@?EroF*y`7Qtq-o(R6XR$xYyXKePZ0;Tl^(`0NFbqaKdE%RrUj zmsWgW30drg0!!~+@y-e1KDG|?%0(}C~`~=j~5Ga$4W5l92Eg4+$`!0CWpFu_Wm`0OA-fQv9Z&pDC12YP$`={h54fE;^zrPw)Q1|Hc>Odk}%}(|&bMp4$4%W;qTa9Q_2^jVMoKP8(w=^}j*x#i57_ zxK!aFI@_YzwQt|PCA1h$%2^K2EvBE>6Vj6E_{IIbB`ZnqF`_>i!UwVaN(Leh$pOk_ zvI42){WDwRka1KT6)>S#yR&V%>DlcVhzCygf|&^uV?LNkAmexfW`3;vv>0-SjpaZa zB8CxmA3R1LoD7QWuyo|yPC9B5UwrtC5!vMA;2u=r&AhyOrx}T9YtJ5;aB$Kvea8RJ zQ@M+SQ%n$WI8^TL8FgROXneh+9@omAASC#1L@w#du!XW+!K7QdL`6oDX?4N`AgPpq zMJ4`Wsu-!p4u=S%9-=ul?iss+C#DS*M5CLeGg2+lmf|51(nL?q7!xpn2efEkf(4K1 zwj$PObs0OD93TM~GF6NX z&^0t9{E_IpX}ab|kKP-fM{Dw#*tN+)gFcImhyo1an5cIEg6{#sb7a9eLX7y) zypoESRO_5S($Kn}h^iLF+K+%Xapn$0Pkx&M9PPS5(^^hp$k+>$o^a#r+U=vd5KqH- z=x)f7T}Wu(gNu-5)=65j5*J^%1|9erk^v&N?kf1TQfCNOWR8UsmxAyJ0bt0`sI=oZ zZWy0%?O~9n8Zv=Dty`Wh!5|PQ*X+uA|Kft3bY@A5doH1CIsb9eMP8gJOhEm_o=Nx! zR7ia_cW}_er{2}BxKuI6Mrq1j(Q$(cmg*z>W*jX^&Tzrya)Z=?Yxy}qS zu}gEJWr~u_^N^_$ng<$Wj*3b$Wt->eJI=LgKhOJqzu))Q_vh>P?EUPtEcd$a>$=YK zJdWc$&TU(Qu3!Ix!U*)k7zXtnW7R&x;jwC1y&I0xZECq{R?gp~3*6I5n5I<)8=!YI~E4tDsLZb^@xRX9V$s2HHe+74f z6XUzdD=~@O(^EWDM1hb-mt)SV<9~mH8bwS^P?)nX`1x&!xP+VK<>WX`h9`yx6d1Sh z#Drge|IX9LG`Pdgj~Z8(S>?<@%tY`wJ#8$=CUaW}@Ylk$SioCQyBz^{$b#TFkR?*K zN#l3|Bmi{eAj(YoV!7c9)>LE{WJETA_0b~%l-JYz{CM%!syTk+_<(4ZDKQya=U%@Z z=d~c>Q-HM!Ffn+EKnH1A{qi>)z10>>uQ z7ms1iLrI+1arl~o_r3rH%wuTenpf?C@CMyvo~UQxXBLiJK;)UGSQIrnA-nHGGC*dt zUdK_NlBW^YIzIsD;!ac8IBUPmM=$Uq=sYPA$C*r>WJoH%oDrF{?I^wU;c(A9G<$@( z{P6zr#hQHC(3Tx|M0Ha2`oS2oX(1CqaC5lwaO`F{n%?PBtm7UG_2AFo>&MR^vdH7u zqB-Qg!)T38$VQ9Gw{PD7EQ)(R!jms;l0u=*5XZswQI=XUsbnT119_<3NduJqA`Ie8 z;uf8b4kh9a){q%ujxXD>0k)S>f$2B5?o~13pZ!~B@Ox0uZEvZ^U+UiiuqU$ar~oo* z|GhUC4wxeESOgRrA|iF@eMtpI4J*`8hHt_6Ycr?)&jM66ke19RPj_VQqSas~uXL zhu&-y3&CREi4Vrr+63f`t6qO*$wN;zgm5I?h1$3=+Z%x&or-Pnii$5IRu42Nu#Y)V z{D^%wy>LVUsX9eOs4eVu*6jXieH7Xp>K*|pisJfS>BW11jfe^_Yf@OV<|R380!VCJ z^{yTQn&u?oFDS$i)QlgFuX=3CJ)oFlcDZeK6jCk(it`^pK!kNk`Fg zbquuM$oHu1Cn3_0ALkGs;=bFVk&_X-dGxxN-)W%1xcWv{SdYzU=3L)lXNe5o8 zEEwXWv3MN*r>M3JnFI2&O<*66#Cu;*xq?6FL`3>K3gEX*V01N={fQlU zo>$GoU2+=iND$v+%R14I&2U}wq!nmM<)ttjThtVcbK?&D3`lppCvpe;c!(OO*%C;8 zUKe%np)Jh(HRc#Bga;BVr_|t?*Vfl=gMw9CrUn$_y(8m_yi3Q3{o8X zJ6I|D2BWOUJ5T)bnd6A--+-wY7;)i&^7}DTQqcc>ZIevYKt^_KMovN~CpAZw{6w`d z@zI0oA!w1Hrg{Nlh)5+*&++tVi7LWNOPhT2ch6BhrR;I6dEp2SBeFyzWgc46Nseck zeI#E#8cB-{;A&D6k}V*I3d_z#5x;P*7IT(Lm;_H8rXxDX-C- z%wRz6`xpnv^jWjc0qY?_Ent=}T`9C?4O>$gz|SI3sVzsf zO4ov|;rEa_rI73)O@Ts6_BP5shPX^vD*QO;%tWc;ioi*jC`8&+v?b*sg@{0_H&L2M z$-iQX0=Q+a=FV4;LZ1qu&v;wBR1|L9e?p+ih|StT;5>ZsALCp*PiQX^ z9rSCOX6(CG)gT>(vkQrgWz^fTeKl#5Vejwsrts0t|G^O(k&z-BN6C(DAfDd^LIsD(q44yzvgX*oHy+lKa{QYptw<0CCLV z2kcKe?k&6p2S=>Lxc#Hp8w%x?$n=Wh3?#>>5&lr`y!8Y7U8@+F(5Vs&7pkMCxB%7I zMRI=^m@B7Ip9Z>lCLkQ&Q{a1($jw#z7B>7P9Hw<aw3}o@kU?A>lG6iq(Js(Z`9BrN6CGk3xRMLb=-CQXv}rZ^vS{t zS)uLo6zrJ@YGjuzp=ru7Muc1xPoy3CFNxl{uunh zoUd|n2@I6u?wkT8$93AcfZP3}OlqJ3A2HPXG5Q72LgkNTs;_eyo)W+;iwAduC)owS zt5^?*wkPzOkYe52(oLkyw#WI7^Tyd~-&Us9BY@u#;KbJ+`oaJPm%>p&Jf$KD@qH8g z3kVh?-U_;3!jWDmb$4}Cx zqwj~>r(hXSV0<4*AoT@2XwCD@s;sP}u@}Xy;Gl7yYVO;&&+04$l#x$>>LV&WLP0`@ z6l|Z3v{5%b#V(aSHRQThNr~8l)vH(2Od^^I;o=tpKP{9Gq<@U1#a^!i=u3cf7uK zhHsxm4PCgQW!LKg3cB|EoC^sw$$7qly?{b!V%tBjKOqX;#>@0aP*$MZWhTIGv2WA( zC+E5Izr$mumXORAszb53DDBz_l$Lz;K%YbewBkHLYWR1?GL~`o^gU+_Z&v!66y7vU zb^UBRHDyq5T*~0je$%QsAK1af*fq?cgNh6~Kt_yZAZaD?1gDR2(3r1m029On0Kke# z%Y~ATLNF1bs06I>OKGMAL{c5yYa2>PonVnHV5RJ4^ge8+aMWwluXZ0+tqzzjq>mhl zh-zfeNtqkeNfGhn32|Uv&FB+~Nnqb!jygt6`~l7hy00mu zQbraNSkQv50BX#{OO2Y`&=~-hmDB-@RwYGp1P8;`zjBhsS#vZG;!~UzQ%=17@@VL? zsHGl4Ppx3Un)_&;z=b7)sGbj_dz1J*2qVlp$q^lt8!YaqrAxkQDT;EeWZ`wO$yP`oAL()bLc5N(PBlI-mzo7Csk(+9BIfOb3bojpY;hb6H}&i zA~1fM2mvyT>TPK|$Rr{o5hiyu9csfZLY0WvgkOh;CEtD)?`F9v9?Jjt(BXw#;|S>T zQ9OW>b9h^UAeb_^V~BR*e$>2fGLVe5DCD>1@TncQJ%uSwC(9B&1VU*V=wdc zh7IhSQVYa@7696ty-xs!P$;q#r3cAnA5lrT1or!50HjIP;M8`pIyi_LH5-6tw^Pgcy1bw0 z^}-493<3+{r5c($9a$QFfgrVvPC=@*7FBX&&QrJ zItt053aI%D@RDTK3o(bmTPJXPB33m00p zeJnf~j!1gYWa8Jt9YXs=_BbZC_2s9FCGP^S!d2d5y%qqt13@@RoqlZlLSJ{0M@XM4rb9l!2g#Bw)t$IGVY?=11epTmf z=|>eP$|z6yx$e>rnR)I|kup{~{j|O^M}GREmjdTDoOKf09ZFF^LnCNsjOR&cAFj65 zryO!Ss@U*)&ji5@!Roi04(>!laACU^aSwngB@1_5^74BAaCC&5w*stu{wpgFd7Xli z0DQ+;$|E14*p|t|ZT^CLs`TN##Y1VkVbwgv!mq?IvL&Hrrlzq&katI#&5-*T za1B;RdCxR)L6P|}#V@wPQbO6$2AZq!VAecoRvoIvm*A|5(rxuZ3Zf$;cfAbAZK!;Z zE^ia2JJ;bzW6>6gnv@yt=cD%UWwr#*lvOB_+HuPHVA;$c&oAwIea>ChrMwSLD%>OS zLWp3;jlAs>&y2NS**C`#Ck@gtVp}|?(P@hsnlWRB@3Pl<$2%v|2(zxn%Ft7XP%u#f z4avzk6s^=mf={V|bBx4BAg^lChpdJSj*N~8BS&ejZ*$StcLvp?WwXIg;G86ily*6s z{Vzgo`PX9KxR!K4p{5Ux%8}8K4^Tc`a5||vA;i&y$i>7be*)6+y zW{?U{0@zKM|Q->$^Xw*h`9~%6z{>>w&mR%l;xB!B z^h?UtjMyOUE9cHl+*chg+tZ@>Rp$J25b{g`df!>ky{W0oE;hWHgE1E0@E5Kk1If9M)BGmn9dfHU zw=GFiD5K@e)hy9Z+w%L}#|cFb%T3LFFd4pN3!4-UZG4uXb;Pc7@`zQcreo+x{qmik zUd63kVLa16Wa+NAmvkFG+Kl&{W`DRm*5syS4;Y5{dvjguL|wsYS<$00`Mthh_#7(f@xI-nH@ z14KttYzqQcq#XT89bRbTeV=T(4h-6k;JGwi0ld4+tmWG;0K(l^@DS+WO%=82COe=Vqz#Jv!?l`7tgJDmis1n$%;t$FA35cMlTKSEcCvLrVxr7WeP(me zJcY{Kr+-$TS{G%;&k^)RH5u(>M&FD%PULsNkXP%P}I#cP2;g-N;~E-BQ=?_)vE3;VyaeyAIL% zcXQ9kFusiBPFt&__`P8cPnq1ozfNLnG}2;Q5g@wOCVWSP>ETY1mU#@NqjpJY`%mAwcsQ%MZJ=`n8b;H#UsgG?+P2SG2Rg(@Jz zW)bgh4QvOhGYbb9NySQwv7GSCs+O&j@Jq{jbmq$@#jB6sypeyo$6uhU``XRK<6ocy z6U3=ym^D{;PHdfTzDL)Q?#c|SmpKy|eraiMTy9d_$Mn|cTll*Ef&m2in+k8zAc-e@ z$f(P)9R{3sNfC%o3=gUio#SMPS-AiSkoGU6wn>0BS|4rk9|SWY zTYuB80`&}y%!5mV@=KTB-N6OqafMPo$x#5?@9$}2@stWPZY_Q*JeHBXJ^NO@>|VpF z7yMu{zUqZ0M0JaG2?+>v$vQWwM@Hva99ST`V0yZ{g=hH2x9?-z?~1hcc06}D6lXkF zLPq=hy(!!W9iHyTg;^UI|m2L2P+D>QTSfKabJ(l8Bb^s^WY4Z+Mr zG8Cav00;`CTkw;wa=Or5uTIa2NI~@OJSERwPP5Mkn04!Xit6RO^*JpsXW6 z){$ous3Aioew;ddpn~Wdl#~KlCwE6!msHq=k_rlKABI_$NDBdPKSdppDW*(^EvnW4 zm9wkQfz1v>?}k!cNyg2E42YGUV8$2vb%M%?+QjHJ@`#1f#JrQeE?n$XNZ+LJg%LY< zn(5oznxJF*sSZa!|2$+ZPU6MU(wXUA zOln>LTSpBWwm*OpJceW#38LKFdKhe=@O}(i^s0cvdwmj%xv_sq?}ryYy%1)kKCw}m zKVDV3VQa_c{XZ=>kJ<<-9XSx>!UeEEKzW7;#=e(=6k$uc^vqch4gf-5j;701&d{Tw z;g(ujKt65@=trJ(8Mm@-EwDj+HP{%2g|C-3%s`7~#}$`G{f6RPbm#E;HzOC?{y7G~ zP?*r{+2@IIA<=oW-`>4o+A(-`9mxog_wA4QWeLW;*+umi$?1l@mq-TiiQ%}}B*_ox z2h4I6${pO0ufQ2%jgy2aq9QG9_S?d{fxLf*t{_lx;0mC$TBPnEor>>Y!C;TBQM}zZ zV3gO<-wt(b#IMK8{k7O1dG}6^!E{3ZrBlhnvTH6JvMy<#{~HC*esRBZri~5{`G6vvFhWSaNc)d4k5y@5!thg zD9UXd{{1eDS0Db6V5m*(c`@%`ESh;QR-GDbSqK0z#?$h475!s9s1&d%6B0keE$_YtT;#DtRjlZ>%ay1|M=FIP{5J6jDZtIP(CTH zTc_%B6_QUx!5fk8SPLDPhm7g4_T@Jv@4NLTW2{Z=RF%Fj_c!pNm1`^d+l(NZst>kp zRCt3qJSh;h+GF9sW#u&9vJ3*N5EZsL5^rBf`XIXv$9c2_yQrbZ@87?l4=;C+upHnY zY0l-t_gR_|+xF~`iu-ijNuue&bQgmMM}Kb1FMjT~re<_K=Rix7uAK0UTMqAhMO0ij zAH4fBs;>EmN`(WvP270ih7tXuT_YVb*GKRFFn5~lj7G36D6jk@A|jAIM}vTlJ=x!Z z$|DF~Z7E2frJ!Li2v$fS1{vbOX2602T_t*s`YY&RyVc{tEZr4fPc@?j=!x^B2tp3vM6$k8Lj*=)a@eKMK{2Eei z;VdG({az}1SHch_K8AlkB7@0pUriR49?K5=xqsAT^;?|_fs0+dx1!-NxbL6{K7+N) zO<86coG*@_IDsKG+MGy`=gnZ1v%#mM%!q+t3|g*d3r^o^onn0QWKL6QLbBPzk?XOo z&)Qctjl2{05>?&aqLk&j|J|g*_4O>3d9qe6u#S$Cdw#1qv-&=J)90GGk$ReWDYu4| z)>qKSNJcVd4wvH!Fv)da+T1sBfIb?dbd`rtL!*{je|oFk^wyu~(&}gx#6_wNs4sb} zUHoLl?2>OQex~it4@Y{%v|k73G@P+9pgS9}{E?+633jhjRFr~d0aDdW)IuTQY^&Yx z+%enI9=h@-v^JqRx=Y<5@))Em0gtV>A_?-%m@TEx%#YDS^G_tAKO4D_r{EiF+ZC1T z=IpoBRI<)+8ADeQ|K-Yp;~>zronAnByBA}0jd0P$pdgGQwXSIlzYDsVxnOrfmLs20 zX0VV!M&ac*{V6t^w%C6Ru;jnB?(5kf4+Wr;PlP3pBtH}G6L8=(XaWkzX-CdngboLJ z2-JpK-A8ra>eFvDof9^GX?n|w*)pAfJ^)Z_g5Gx@Tr~Eo|9ZEh1~oXP>aalY7Lvi~ z=pdyD=Ppg9$A1i9%M|0=-I(?o59o*f313e;AGtppL!g>IKe!8#cIO*~9k@S9m>3G( zJ657&+AwNt)ZEvhTQP*PvhH2T$I`o)>iZduHYg1hM19ch8jlsqnLlz8?dcNhbN0$% zpPZ=kHxXPf*>2{bZ1obv_DmcR*vKjsF_S=lCqdUK4Y3qXtd*aocghQA!do7ljV2S% z{Drm}9r;#VZAV9kT$#|EIr|2Vl}GE&>uxOBgtEgwG<3%$2f7~&<%=z8SVY+ zv|nQTtNW{FAL)5B;q!y9w~-PTynQRPF=YidCL)9h^z(Qq-33ciu0NTC8;{1^8_T!f zyG^YF{G3m?iB2ujreUAgyRJ(3~mQ)7w6wG~;uj#t0H&1QE#+vn$Jnl!ZoLOsQ&okizuOX0F)R#Y5bqL=K@MmK zH9^r-0zsV-dLIy>9H3Ne!nk*wXT7C*$=flt7FkjSBAO9;IiMPXFil5z{fP`%8Zje^ zVCuHF5gAx?idb(=wzl7ro%|9o__V}L1U}SMojd3}`iZ-eR)nG+ibGvjh0&gffJ<3O z3Mw$zF$RY?oqzW5u%{+J&=bpwK0)8ygr<)&K*-nM6tIMrnQn!az5*IwY&R+V@qKCi z_lDR7PX`)vt)6BaDY92^C~F96&V6-rv?}{}V8H}G?ISuK%+L2llWv8mzOVi~$*{+t zbLsNsiu>ua%i@|Xnp9&SgeTXOrae!PU=8+)mcN~Mm&sDFH(EV`KVaGRD6AG8PIyFX zW7}h!g5bBhC40RLF6wVI$avB394Mb!D5-SnyPvkdRJ6nNHQk`n?E&~Tl{PkdyM4%X z2!zq*ZK&?BCp1xmP{4zhj?$;IXV2O}@sfRNI2%m(j1XDq#??3(ct#sHZiIp(f2Ow2 z{9CdBLe+pS8^YQvbg+I37M-}GxSxlleNA32WgnPN8H?JPGe6Jl<|^B(Mm-6jqnVA+ ztZot1hVP9L)L-4OrrZZSQhHnXvsD#t*A%_~oIKeGnpcj<9L#XphK#fX>_=H_(Ws~f z{X*yq3_wxI10tbQ7aC<&GRH4cPoYSP{6Vko5IK%yj|{5D6I9-e~a)T!PYwtT2Gd?B-` zX7^fRyRXe3@K#xp2EbaK54S^2X>ae(1+S=6F9s6CMm*omT`x-V^Q9{5hs@N0A23yy2hdNMN7wupD_lK@0R>5D6@SxJY2CS+x6?E7gvT& zZ&{5Vn^KIPJWPJq!02|@Z=XPrV&B;f^|eQDTEDx%k42F7KQ#|i0JN;HXzokdR4;c^ zmeyk9yfqTyW`I-uy4uXH#MR-6a^|b7TB!)9@(M_k<)IqH9c6)5IPoKRgNzspyZ(U^-*FE@e>;8GaPj&LG@ai1*u0o*Xg|8*N*Lz5q;2 zKmN6!F5V;XC8AFqO6KeZvI@%0SHiTr`QtmyC+mu+ZaA2EFd!Uh$yH{_AX{?NOD0?Q z#>3=|e;PMr3MZ9Dnz41=7w)x9;Baf6lIi(n5=r|oaR)JLI)>`OKa-EJI zEnk|F!--w*J~D(_NH*lJoF!4Kn_@{3gGB^q>}W0LGlW;LHU3vUB|d z>=8Lf{>o^n8GG~dE4Q&H9Vv4YQy#o_;j;UVxJH9hMUiuD$|5V9qkC$8$P}r%O^s@} zf=>5Ar$ea~2Rj3Ut9Jb(><`8P2c-@&_>9d4Pnp)CGAU&5ZQo)0PWhzhuV?VFq{dAA z>4}!iitn|x5$Hf(oiB78scLiz^7aH|K}%Iss!OvF{b#N}{pBz^<iH@3BaM*gH zG<;{m`0@L2ho!WK9yZ)Wi3Jc?BHw`zFbsBYqZ{&1mw!H)VM}G7=pMaQ2EU#ypR@hD z45oQcF^2=?K6@e!z%7STTN2>iINHD?hHD0WY!Eo)$Yh-_b99|d&tho{)# zBY3hZ6}1y;YirAJFcUyPX5u2g$9F%eoyab=INEb`VB|)eRCVpfABKq?@gtRPOIU8M zAIdap9@rK}N6QFnagsM$1v2-%{jnI9Ye@J)cIU;IP-&*Ot>WPID*SB)XJf()FSjb; zq>l>zkFDRAUQFk7&GphmGPb_m1rz_wIh>nzZ?4O3z`0araWJ?0*?lQ8r$+ZWnO^Ko zqCNd6>z>q@eCT$4!6nC1MF7mo1STYYxP1_;S|F<~L1rSs!i0f4=(|sbbS54rz7hs+ zQ#T+onFeIAta3Pem!ZI5uFuJ9#}WW0?GK@DGMWx|>|l47t#Vj3M_>R3ABM=bWx*LK#b5swpNb3x~{%n z5`l%j1CHzj`0g@vMu(`nTLHa-z2_~I3(??g7sr2PIZy<2A8J2O=sEl?kZZG!cYH6x zIIj%w+c-JQkPU3+8Iq9NCqPIV*=Or_6SpTp?eDSi z=9%$k&9oR8R#T9-P*NWYG_e`?0w}@7E1D7b7%^if@I%dV*a?o26i(sM0hT!q_z(6{_^tmnf1-tMf5-7=nA*rYS1AiE-DdiCW`Y(6E@eh> zO$p{hgZy>9-P=%_7ThSbkkFZJ@Nvpp1&4u@6EV6wa!cxDK3~X=+W1hWVx47m@!cI; zE&{?7JYe`NK5>6t(f+TK5zqXYB@$7mH|g0zii^+Y+5t?;%l;hll;y z^XGBU$J3+&bg|r~Jq*Z6YGkg2E#!yxm1$AXuD&s!gj?s~I9)q6&C`{oT`Ad_l-_%p zGJZ%sl97KnK=b3|djR~PmH$y^nYe~Dlrsm|T{2@RPiw9eqk*k^CI}~u zs4SYTE$HI0buw^(9d~t&#T5g546=IzN|MfBwCmZUM5aeUn*zadU4DcF&M4jI7;cK+ zo|c~O3n3pJkQl(MO|}2(S8e8J)cltRcF8hw2XYe(BnzdNuigR>R_qo zeW+1OPR<06*gGR%qr!#YG_zyK{BfT7p*y+Wb_7FA0E;+Iw9#%&EA*ft3@jy}ube!P z#-ggHwsQ>O_Rf6h;RP7JdZ?QJq2SJ8_0WZAtXpn*uVcH3=(kZc&uvB$UWRLbyFq{GU~A{s zQRh67KIhSs_{Zr>;c(~I!r|9{tM(R3%7n-sGC1uI;b${|c5N-KR69|G+QxrU#bu;0 zVlQu~L#O4_ec!F>{Pdxbz}6a!qxz2FQU7%A)7N3TgL|ve-5!oqcY8OLr`R<+6=y&g zik*u6@vc0IUYW{J=Q9RCRd}O2kQ#;@E!DGSG4sMAkp7mjLgL+=VpR&d>vsBq^q5Vc z7fzuM*#h*}v!1Z`(*OVZdL#`F=fA!Zcre%R?Z}x1J7S^|b@1@(btEt9`PjYp>iuWK8VV1sgRr zn^1jkzw<}rqX<35aW}W6a97I097p=)o+#&~P|IXH)m_3KwI0~0w2Vd)b#%BGD)oy# z(4Fh0|LneL?m$yQ*7CZb-RzXT@##isOt{k6HS@yVf$sSH@&A%4&F1)`tj+T zD@;vc#|kxnsat%^Pf=IZfo1K7HFX<>;z08kpc}iqPFaX|y9d;kJU;EZhzp@G-aPL= zy$3yKQi~UBw!fIfxfa@-{%oEXYC}j|LqLY@w!KZ+>{W7B0-O;bhp(5 zogp?1eYNOxlfk2rn46CNK|+Hm_dz>dlG$pyPTp6f88}52ic_j@kHEwkwU+LbDXdX@ zbP>iO^k2GkNgu+|F4STTxQ|uvdxEJMaNYx-Ad-1|1T(xSRuRgHWJ>_o=PPP5J;ePosw|L7^j)mQ_+7GbpD=NvHix7UXq={Olb@fT!=xG!<7Smkbj5;- zAj}eIR7?RcAK|{WN5;9WW94#ymBgd!N8|jE!Ak_e{MWBvcje+9VOA(Ql^j`953(SU z&nr*1Ou$AYuobN&TM<7J&HuwxG9ry|jbVmL5a0bkEx5qm3r)U7h&qYeRQ~gcBkhi6EoY?q0}Mz&Kqs)52$k*5n2LTI z7;G%WkA$2?p7%cH6A3~E9k4C~Eq80=;bAaYRLKyG_N)}K|GZ(bk~-h3RI))sQi!$( zHNdiEK=7F?_?VO+s|sD)3Wx_TF*NT!29-oPU+AtZ=Vx3v=ZBBtHw$Z$5$PG%#X)q6a2U1#ECkafnR=pKNP<&32I z*47lHe`qh*aPftX{@-IKBn~_M*So(xoa!>hBME}A&ES4lvBd-h!vTxz1I6TY$hFVJ z22mm&=@5J~;sIbuLCA+xNFRWhJkCTGRgB137U2AWJt{tei$12h6EKBb1_OdmA=wC( zvBCtb-(MeQ(q?#lV@(QL3#HaQ7=54?iNve}Zc0o*G-!c0=0v)W^uLt<(QYw~-W$n@ z3UQE+wj6B@+{8G>XRoCx8?&R4m_4R>~s)~p)AZb!v z*}s1|c764S^==rc_3ziC&f4hlk!Dyk6i!^`HlCxs3p(9$;=iw9T%V?`7ECNqPJgb@ zDTycLQIYfR=b{1WiO^|oc;UUYSU{am%NN^Z<|zqvc8pv8)tWo12I@KTac-XY?tg~@ ztyO?vbkoxjX$7*DFA`l@mHnSz$IyE6u6)9O{n;aM1`oFV^*w7LoCg2-@-V_J_^&VH zzn1*h4Puy1_`e_i-#2XShirz1`>GHMabYA73 zy_|?n2P7%K;t}sf9s(^~CefbKAd?HQV_+Sd1dWS%pAh%g z_}t-siatQbE)Y_nuTU4r_gxo7?%zJr22ntx%O&h!-55{yz3F z$HAHFVE^E7?K!Hvfnu|ywm}*kp+)1gPb4uESV4>6Se`{89KM9a3`_?y_P4y z8n4AyLI`{ zGkhQYWO493c=h<}^G+e$kj*`zXezTv<-N{Y7ac_%7*WLhi~;T&ory3222b@3BJ#xn zoE*RdR2jxu2r*UT|8A zH#*}}h_m7$z5Mx;Gv62Oq#HR$+P?htT8O*2)FyJ4i`DqRyA^N*EgXOmtKD*e#!7y) zf=OfE<~WnH{y3EcM$Ii=8>NnV}1_Rwv3 zNW~&iz;e&iPAcZ!95v}d1hMXiqr-Z5+cG@q?_Yd6YOf-3V{zbI!D)HuKAP9q{Bj(> z-V?_w+@E0vEZ@Kf9{0pI|MQSn!P+SqII9FgGM-VSlMDAn;+ZGWqszsCAR+)c({cZx z9DHCnOoH<&M*s%ta1g~t-fUXK{Q-|iT<}iBLSeW-q_F`Qy+@esLK&3+6e@1}T<)v1 zCL@f1ntg!5pK%pdZn~i1wqc=#x`kkuusFm7Aqkx;a86^``+&nqdESCn7F%aAoBIZg zHf3T+;38m87!VK~SbvYG|2>|C+$jfkR5Rz#Peytu*|bX7D_Dh{&5T3);NFVK^wTH~ ziq8sS2nwasfRQS*XL8@vdMEC&F~~&HDdcvSTbSp#-oaxDRcsA8R=o}9R#&BMsD%Ap zWXka4V6sTIiJcIK7D8m@@yLUS%SWA97Ns*gA!k1KU9FBbcC=5{n;{~yznK?{Jp9pZ zY6|Nq4pRnaBD(eSuigM1fMmyZU<61eR^v9 z9$wneW$ro=ALrV5;8$o{u_B^q-@w4O_u}{6xhKkAT=`6-D=>00#HQp#g7Znhrl?BY zc?Pi0!S)L7d#Gu)7k}o83jmT?p^eKB(vVQaci{GlddYGF=d#44^WVMIac%Nw@A8_z z-W-KcT6(QBZ;zUe{S`lb4xsXUU0pqLzg|&E36+8DMjJjR?D~f#CPVut4Sh~|74K+`9E0S7Km+tS&;T*S z0D?*jZmoZigdv#pwc7g5)+flSx{xnWidU7k=Q2#mf_HC*Qq>UQLE*qBQ|O2SK-@%S zag!p(Y9|N~h7p^Pk%bVy(9qDpv>~A=7$G7(NILr1I`uCuqQnr-3=-AGq=izb$>Sh8 zYJvuc!-ObQXvtPJMW^pu0d~4{{?G+qPE_sy!b&$?%_EO7P4L=L8R@gZ=L)vq9WYvZ z$Xy5E2c73!RvQPGAv~Ozyio?2d}tD zU7W7YN9w4&fu^M-iX%sD024+qs3j-`5?cuR0DuhOYN!nl6qz2NV-v8Al)57J1R3yB zUM&d_!@|n0sdSnG2|!jGhsa0-1u$coN#`S9PDa6uQ$G-99!X?34SK#5!ztxqz@y+; zRXQ<+2ZcpWT<5&ly(3>T-ts{fY%k*otA-vl9)hSeL_*U20>)H;3lRKh)G>CRTywkf zB}1Up8o2UG|7-*mFx@0G~YH7>t43&X~MFt;1&7+@v1_@KZ_Zp)amXb`JgWObt zv407zA?NqVzl3*IxG*WY^FqpNN6 zZRD*Gu5;GT;J&%|BHXvHMKNL+6no&rt=)M0@87;HWZ~oIFbS9ps3rkCgYiLYWT>As zPH=;ihHQ~42GpZQ@JKG1^NyE)zj9qRUmKk(Da^+7LKTv&qkkj{($=5>G!`>iDY-qt zQFnR!V>KqfBqC8_o#?xziqmia>}WUS+QBalE$040wHLM4jT4Z`5o%`#CYlpT-3qj* z;sC<-6Q&9fQ6^V``|{Ku$WqM_VtMUr!HZ9t+Z;_c#+cT{!QMNV~0$RfP!eI=TeYwAA5)V1!GDfQ!vWlOrIpV0^eTIQl19 z<(fl7Ac^ykir2t)M6lTMriIKB_49|5(q5z$O}7KO7m{vg7gi1 zxDwJP$is+=AaoFT1AVpL{);wE(G69x1pUQ<{SPSH2$(%$V_5{m&g1^@2UElkNTAk1 zCZdMBZhqe>3NEjJ5iM8z2c!w-9u5rS_dJ~5c(VOu7Y=GcY z3Bzw-sFZ-SzX=PY0#ud^JP^>pLV%_b@Dho;zoH;f0@w&Qq^_|s4qbYc*@oO7vEE}mSweX^ z7C5=dQ`7;~LjwXJLI?nW$uXDUz4g2$kW{Pmf1g2019XX&B5GnCD4;XteR&jjbuyMZ zr#_EH1TsSLP?qIr0Xx@6vwr{lD_}FE-iN8@;a@vwqV=5{f&kAU2D#5H^n~e3B8&-A za84xP5(m&tCxjEU=@_iC40#CRqlr+hx?p!P7n}j38;j0Wf{N)}1a~OeRKJ;fwH|?? z8`b%MNC0^a1j0|k1RT;t{h;2)0T$H-(I?~sy5{M7y$DXCCx^`$@ZtAqvVHqa(h8wi ze~n?iASFxi*dbTOYEg=YlP9ecuoK+UQQcNu(~#fu91{*_^#(AXqEc$*E#2w(>Q zgcJ}38;Aox(fdylu7RY;!m6ioNNI_Zj+A@Qcc*IbI-4G8qL(2_${>0P+@)Zol#L1= zYbc7ib|`A8MTJreoHfSO?#^VlKc7uaB3?^*i!Sc@&#h1JfFxL83d-@%5m@0+iUxyD z{>hWe@KJQ^6B-6l;6@;oj1TAhSd~oHQ&9>uj@)m>7 z;p?K>_khtbu{jS+i&kS-qfsly4)yV|{->O~pdM*LB*v6b8Rt4QIC^W6 za&=F`GTxfc8NyHVwg~Uv%=7?F`SGqE-pB-eh~qB^R_p1TJgJ@7tpE8{L?GaZa-GTh-LBH< z2Y*2eyo=MC^8XE!oBn+4jf3puKZ?jV(@J}|KlNxLj``5R&6opb01=RJJnut9O3eCG z0@jEF_v`0)Y3exgg3W%KjaA5AjIms?Kx@rn#JTU`6K%T8?~x=AYJIaf&K{?L+AL=b|&_0Vm*KZ1UAp*@l+O?kb0rMwlfbRvl{+;@v;} z5Hs(CWClw@Fw(1Z{&;HuoS!KLB#<%?r|+*8z`d1_amwNVu+9_}m0=a!R7|Ic4cFwQ z*x!Y@y~M!cOwH%gq=X3)W(_S8)rL?5E=IDHe>0G`*hZd$mWO=i0ac=CaeGYoR3ydE;8{ z-HL!-7Pes$INFx8y>Bsy2@cAMB=RFaN_=4lL8!ivRCpk_i@}j9U<}o4_rtu0a*?7w zVnGvbHvg0<`uh-#^?r)2+AHvDt6TXXqHQ9+5~A3BsJDrNMNv0%`t&d&|381WM9T#<%eN8h zWYTJ2EWpX2@)2rAN_i+9ZK+IuxzLTfS3D$_8pit2H`!onK`)gftO;RG9tzT>h&2Te zcp(+Dy)#VxI5L*mhVWvh8X1k!HrHORU5)Y%&xKVyLTyyJtGPE|uK#m&zKgS54sC@E zuVJLtbcbc5o1xn;eyX>V`vQ+rUkROAC?5*q-S9Zo=$_+&`6)OU1Me_p`jPD}NT1h3 zV|iD3(MX*yP72Md;D9bcH-rfhz?qYY!9_M;fE;(fB@g9IrFy(A3cx^KB!KT zUUQEvaZ8=vF!&)%LW2~k65 zO(X8d@yUxC+^P;eC%V1XVC7n(vkFu-NvujU9ZZmdAPMUaasUIfwz5a^^KI^XGOU&( z-$p;^P3o1zv$ltB9-7^FLR-MVZG%!myGZHN3+^6M+qFXXO^x+R0*@%0B%3c5xPU~j z1m5v$XNesDNPowkv**s)V^bh)^T%ZM6v#2QbL5YA>_&rN#S0_ui+xn025$&*3^4l( z-D78Ym(wOs>%SPoKobG0*r8j6ns)&yL_^|ioBaGYz0p(r$*=7mnv}FSX!dce<%KY{ z6UH!-W$7b3_gc!mE8=YJz%>$3eQto%6ytP0qXv?wD-7L%E=9_5(X7&qHZT?k4U+q6 zwYfi|b^>^dB1<0zDElpVXG^1BnL3~Mt>>;SRSJ;hNv?|q!M7qK3lAUl1dqM3L-~a? zv1GP6PYscC4^OKRj2(7rz!gm!&M6dDfOzDM9H=0q7D{|34ey6eDOjZ?Nz# zpX#rh&C*A7)lcB1@K(~O7vZmlh;M~$9)QqsOQ~n$20lt!5jseULHQ-*4o0ZDi6DX2 z5t_(2;4mn)S7h)6AFih$fHFowixHApF<1W-5RhN$cL=>QNoMIUy4a92F&k0q1foBP z+30U##qAvz?kv$$wZlm-EW1l1eCrdOZxLu(j764D$Oy?2QDLDYQwNe^4n7CY9a?m= zleuJ$$CWqV!aOO@T)Wm3Hyshk2)+g1Aq#4BB%pvY+gUq!#81w3T(lk16^aBXrxro< z0A-QHT$9^sW72yn0VyFq+=^sFdLEsk+W*~GBg|8h^rF=91#-*|`L848fEP_1H5Eqk+r`F9hXT+fr$16K0(E*AtLSN3T~RBrn#$q>l5s82^Mrfce{Ai zv$)rZBlBrINy0=yW*iU@U_URM-!`HX@i`x;VZ)f+R^nb?+r7wROM)qQ zaiGIH*vKov)P$cO=WP|OtH5pmMFujV^(!gn;kX|11*im}y(|XUM7{#QN$TQ1E}s`9 zZ+<95>>y+$qBts$S4`L$!bl35S)yz~ZdR1naesq{{G-P=^RW)Kvp$1ih5wHU+8d}% z6R^bMbq7$m{@DLJrNclwS|Dj`a8SRaGnxcmm6a*vUq=xQ57KSr%XyGbgVZ}Guz&K~P1zXxg6TVZ%w;LAh)RpQnss4kWn@h)3S z5@NMCN;6)eSelT)djq>5WNAPNE#iNra$9!44fbEf`%9mNM+2-<5e#8PexuTc`ytf^ z4kOYdaOzT9fEn+bk6u5=qQw%LFkwQ_dR_t^DFz-Df-j=9-wN{*H>>5?n94ek%`x+< zt5q_8BTHQkn;$)XoEXF69&{c@m}MLIs7sSc-hYE_({X(sbVdW|kJSs^%X)a>NiR0yXyN z?EjN{oA$?I3SD&XD4v%wB(;WE1hBSQRkZfZV{vp;QAk55%331Cef5V6D9*76<=E?? zuwy!Mzxkz(;&aL1C|186a*U4N-e1K$<2s^q+dcF*RTt~sdI`v~a@vLjJ(L9N7ZY z0aouE($wJC0(w(ZdJH860o!=4sNfZBe*?u}BUn9~t?@pz`bZy*Y-%aaq3Aqp*aV<4 zB$GxbNI-BM_vSI?rKNYBrW%{Dx1CQsHU1vo_%-Y~-~zaYgh3_u;a^PlBJv(Y{5s(F z$oCdg3npMWB%W&U2sJ|EL&i3L-tEf-0*zyRB1<1@)3?RFbiSbAu75FS(ivlaNJD*$ zH1gKZ{{hHi(n=Py5o`}va)D_0F9h3>*y-_d@ZXJ)>bfKhB9{M$y*Cf5d2jo_Giw{S zv5a9wW*H;VHl&pdl{sZ7W0^LR48>-fidrH=Y9o@$)Ls%}D zd;fmN@45eaj^nwX=e~|(@9Vm#wZ6mWbAHbAeZJrCQx(Ls*MB&0R&q0CotBzlgtCKt z3e%8%B%VfkL05Ko*k$L3qg-ob9@(pMkl4ZQ_Rz0bpGsGnM9xxYZ=>qR}J- zfnQMdZMyM6;M8+jSZ;6jEmM6q?$>WF${z7IsC85mJ))-;?5&5+ZSC5%k&JpEH5fBs zzyN9N0Q%iyIG6x2U`7~3ocR1RXiz%s=UmBwYb_`3tLM*Kfyv`&6H2lt(szua=jL=* zuWi!~r@oHR-qsUXh}9dN{!#NLYYFEW`L^hxZ2dG#h8Xa)oE@P4AuYY%|Ah*So1UK4 z6ttn85~ZUub!WY8DVm$ez5;?%c&S=Tal)aHoO~eK*L9^PZFW7oyP?GdR9Krc)gKyn zqYa!^IY)&5@W1?=5!vCdpF0RqM9s?Q=OPcqXMHOF{aaz9*IvBn{_E$i;M12>Qub2w z`C9Kgoqxu|>3m+pMljE!WK40;uu@Fxu|#O+3>I|^6`#!Tfgd%&O%vd}6L5R8p$mCi zb{K$vHrpk-iw1|{z%seesxNx=%yUaa$;ppOi3;~eIIH$Z)zXennZn=hd{R$+p$~SU zNr77uh0Pftre*4t9&3T8IAz9C;>??@ul_0Xd0Y`b&em)gH4y|}nDmweCZ@5WuS69! z4r$0(o}jb_Qy*#`c~iz>_|wUaT$X2lcXu4&rx(?Tfa4@G!jO?GI{)qUUishQ!!^fO zNZgD4m6h;IZCH|NSPGNIcI?^n-V(jvn>0~3H~<7GI?^9#-rWNaa$879N5@ZDHCE(i zz_La zBMn|c;;K&4+1qX81yBt{F8Ud*=p3T*8ayxK)KH4l9g2pXCyn7*+Q{>yuR9}2bv4zt zJX{*1{3nJ60bz7RxxrHu3nUK11Ari*EJfnL>7|;MncfcKU_K;k3l&H9KLq=uoM$_q zXg(^)kRZs>DqXO9fFHG(^F!qv-9@v=9_-q8s4=66VqwL9<}*5ArX-Xu>gh-ve0e3d zZkm>xcUAlxCA*YL9K-D~C#L=(@UC=*;84%|S(yILwmpHjLn*E4!*7NivpqeMVJY_j z*j|13aE!|19dAi2IL4Ae4_{E8t;rNd;ZD;VJqX4$n@}eu39vRc{R${&}CUe!grJQ3z|H0)z%g!0!m1*EQXLR0EUtxATpsG7c`TeI~}Z0UII1t zed6(mSrq}Pr5`T|FM%nuyi?)*Tc;mi|90$iGNMPRW~Fzpy<$a)VpheK=9;tG^GXLz zF-HvPV|O2a&E+R=s@rGKe=9N?mi5aaHgq8&We5U!Zw&&TmO@r4ayea{k=K%a)DDd14G3 z^ZbwE9Egxnc5Boh8`qWwJ?`I~sk@#JR>hM`w$6Z8k*E@|n=Z^}1u zW*R_(u=UeCWan$7o`e8{DHm4mv~Q%j*l8%gj{X3^13kK>5$KQ)$MZTWYIc`n|M~Oh zN2}E_=%oH2&mfbTmj0T@+uKP~{h6;~$^-_+4+<`SqBj*n^LWCE*6Ppq9!H>t?7nH@ z-lkq=5wF!x-{amgTHz7Nm7y8P&cD%6vtv^`L6-oBvy3hU#F1tv5`!`4-Il((wO7Qd zVxLsVSV&o}oM*3{HC+?u_=gWmHqEL?dLwE?ydpx#q;CK%Nq?H7SzdtfV9B{gzS^vL z;o-X)S}=TYt}2?tRGbI|J*1_p@9yqy^OO=t_C8oc2O^pO&+bM8%%3uPtl@Jy7#Nr@ zxbW>TI7K+e)9zwve~}+&^A5z@aPH3Xw|gx)wZ`htcZO}|_0V{&SEXLih7FBCd&d9@ z2^8{+5ujkL2!8~_73ht-4ZQV~#GQN4MWRR)$%*fpY2Xcydo!s6u~btiS#~3$%dA$Q zJo}aT{E1l(>R^$_LBm;b%HdZMbdq*BF#BcU`jz2c_i3|1 z5Mj+~P-yrK&DsvR_M(q||I{;3Oy)m&FzJS7BlHx}Dg6QQ zy!q8xsnlt;jV9=;Z&k~~@#@}s$jHKQMwey){Enop*=eDEQNP$yLSw=2gp869X_=Y_ zYD?Ol`H!X@?43#IV30Po$9aS(ok&jlFYWXbp>6TMLN^Lcx@KB+8q17)ra`l2ozS0% zBAiEt^+oj0pyLIMwokg!!VCp}5#IJbyVQ2W8X1g^tD2_XXKhs5FVi2pkW}r6&FR?Z?^$6FLX8#N_)~P@uYk|0>N~Xjsf2x}`S)G=5GeR*!@h-|t0_UOfI5@}m;>C;7h!6r7!TYCZ zAn13pCQscSWrmtO6#XRDk+@Gb z>}o%mk`+rC{2mzOL@>x0ZS+))DY3UEZ@ZtV27g|8lAWmrq0J%CAbd z3$FcFXH}$*i~3Vjvg?IQAw)U3(_WKL1})-mUuibImSWm})$H0b@WLNL4DTsUwZJ#r z;cZwy+nX;V=s5j^Y+jPiOM^gOEnNf}(Y=#)acsRKF^T?Tsb&L9)wi(dgB`pITO#*P zdrdbtXc}+^VFLeDlsczfO}P z%~d^_&(4c{bT6RYs21`efh#|567~eFx?8`t|2F_l(SnQh7_Bmn9Q7)+XLgGCu}BjY zdQdh7Wzn6>58EazN6^{M+0Q?QS>v*R~-UF4wXWY$| zDj0~rp1PM*LUhlpux%0;lo_^1;G7AaL3{$xa^Rn&{6+Ip@X%4{E7cgjGL>y7Eq}05 zhL}&@({Jff%2+%x=l^=CCjIK~Vr?nr@I9&^Xb9IIKC~6^h$h^%+Nj__CnVkMx_HZ^>ev|ap*#?Z&<^owT28=ZoYX2ARREml`O0F+I7exJ^s zr~=B7M#)jVI+oWZV0o31@tMcI#|-EI-8Nf?j+*ylrc?88(o5Qz;~>t=8}qyRk5BYs z18D+>`u}U0o|kG!&NQz(#sj zB{VO@yha>v*)T5}4ORb8cRfHiC~sZh27UYXEr}*!=Sh*Hf$y0ZDm6zo75+Vp^XsNVe{LIEr-xtWGjhL8+R)_%JU^3V(1}Ar8o2Du4$W!HIdTG6MX>E zWGh}A`Jb@C1a=`!OD7lFLDNmdp<>v9yuY)u_{Em4OI}|4z3KS!7V#E3j^uAtQGMoz zk)*!`9ugrIuu4M;Qf{RK4Fe&^a%znT+$h4>vp=ynP^1IkkB28bR@~+CiRBh75)#s(mz$O4%SX@F{d1?!nZ5lVIYSF?J^ULdaS&N$qXu7@< zY{|0c1!zF3AR0KkfALGOHGIc8Buh)UqkWBxu2Jc-4DR!5I|jP1#th?b&nM{CldJzq z=DHdCE~ICqw5DlkDf2LTt;(fSKYJzOFMUxZcUx9ov$WZkT!~`IfG(g-$-y3LAfcH5 zOxAvfpvG`tbsWjK_vPz-;^L-+fyDz_wYu~lq@Jr+l@=?j=6@|JGpXYmn|o)XCOvWe z#&2sDvgY6a8^fb^N()u@s((E{o2h=crp2eh-yENo3=CrqnIkK=}{KfBkRljsMb2;Q#KIp?)x?tf+Co3UF++?_WHNqzXef=76zMmZ+cqkCpXSKG?bF^JZily>6mPH;V`v{QezV`Un;U01eU zh+w_OcujVmDw|N{*i8ou9y>9fm0BC1FaFdh6t-Pqa*+Qi=GLiE<1U&Z;c|(T7+%>p zwN;RohiCHDt3wGW)^w}Jo^niTa0}SJ=ZdG$&gKx8CdIgfQ>%7?gbqsFkz%rI>}&G0 zUVy6m`Ytx3Dz&0lF0mMsfxUKJ)mN%3)HcAI*>s9z>tHJ^yc9BJv6m6?rT$#Y%d>l4 zrz^#(7hnPVpI`=NzJ-b3gt@^2j?of5iU=ALEbp;%)t{@^hAW7@Q1)X59hGd-%sY%8 zKEPod3Ya4%KM<9}qKn|I9HCxot6X7ed9%iGLfa_v)y`HADQ|iqw3a7LBVgt8DD^_U zcJ2L5yJuo|Na4N|8%Tj$$h(^`+xNEy2LA{t6aJo1J%@z1g<~LGg|?#t!YBbls0T%r zEl`IPE}T15Km7wW30KEOOP8k6D_rs{t^(aP8_|LR`K7_smLrM$K8IMP>cXyZ5NUs^?)Q{0y|QTOEBisQo3JNuLJK(Fx*^&U=B!XDQGpt zVhGz0nAM7gsyM{({&RS`J2OY(Q0MpI#S4*FsQJ}bzi1XyQYgIy&WgjF+dN>oH`1nY z5ci}p$+wOamHxv#pv^lxRB1vB9~Yo&W_S_PC5NkszQLj(6T<<~$Z_a^Dd&g`gTL== z)laiLw!*V}x^%!js^=NVrXhP7hBfdMlKCl=%(~+tCB*Wuv7S z`OwPp9& zCTIxHoO0iCM~^j)3*aZ1s|1LF7^o1dk@70+dAx=h6P~mOlBbkE8Q*hF|$QtL5(|x!gHS8q*LNM zlKtGF{r0S!92{#A@eCStxqi!b->SDp)USd7VO4qN z&OSUB-%jc#g~`c;VY!fN1Ud~3bvl}+#-`$K#KAh{Vk8_1`&x*bv@3+-Dj|$Ob9#tZ zN~BxiT$Y{%A?6MUZ#DfV$j#<$%$1(!7EOuZh)zA`CTNqV$HNG+(X$ANpzY`oJNirHP}M0UgmbN#w? zM=**AiPoGkx@iC?f)#LLA7OBX+%KMD8qDp>WZ|MIVZ2}vtxgvCK36 z3Y_qda5*5I-3Mn4e6bwvHfd`PMHf!~fqM>Vez+Ch%BAa5DuX3_)ZO%^JB%qI^{*=aYz3pq zG8g7XDo8*896PevHYx=?70Iw4snoC3L)Tj0w{GLc!2~G-_N**;jP@tyz-7Ta^va-e%*E*} zjs!AgUO0IK1eKW8N2D|F>K-vauwTNB8`N-;LyLSve~!C?btjH`5)XLT9f5J-XZ8*q zss7rjuSQ?y#WY}d6QuSI*#C?c09!PaP@Dcpllwoo%`@)lkWf>iH(CDn&-t&{fDIjG zea{Tfg+=G}If;w~Bv&hv>T)5n_a7|#T@?7#Ec>Y< z@K~=YvtEy~cW8q3h%r?^GVd>wF)WuSOiFNw(^F~95*I<%()SOngkHs|>TIh}|L}>v zq-;`nDzc@iBG;&nDigkkXnG2IwS=i5?ptznkuZtTImeN6&MEZ=m^N{k6|x?@;b?yT zLfP+gm71$axEddC(7*p=nn&DKM7ec^o|mxfQ8)IQlR5;k;~3z_9bg?&k;#VFf2TS9 z=LpwV(jzhD1UwOVmP{|0M*jp(h*L|KtG`9hi(=%qw3X>d3iqw?l$EW>Gj9c$*b*?t z8N>&dDgLQtbkAXVpQ zz=|=h4b6{tYGmOW&yfekY>)A>V@Mu*E_yKp$=gWPXZqLDZ#fFvON!iIFb1em4kV6r zV0cbQd*7+TBx31VJ&ROoyj8Se`r|nGUaS z`}?n?!ZKNcntdN_pF*gA+3a*toCd(WTH3la|t z&k3G`JR;drgv{XdboKg+GZR-3#Fk=>$U)YAk936-oRKg>X^`ZGIrbLI!6q%XFc`1=}t>A@OTCrAlg#4fmdOBP_}d;O)LU z^Cx5=uDsH8;C*Lm93igNmCpc^Vw!9ls~GwMG}xXxzLANY0{DRyi|?wqcf)v*nL3IP z-Z;6OHAn}PyyGy_m8vC__c<4If^0)vY}P6u-clEhlFXFy!Fq9daX(np!?+le;T$Kw z&CuAMT&v3oN3IC@ecuribU4Hw3zuY098tWE*xh}oEbb?6Ih`fZ;kEBjNNoWQE0vhI zhVf(K=z___ICm5Cz)pYKti3b)9<7f3pr=1|H8LuI37fkT!bm6xlXr-N3MDb~RDAiH z`5U&^L7~7flH@c|c5JNcB1vcA$f$xi+$G|oJ;wIf50xA@H*c~UsdlVMdr4Yy(gtQd z4p)gf02?6DuPPJhCCX$HIk9B;rxNY*VWIMJY=*#=>0lcjCch}nPBt9(%u&fg)oZvKc;9wU;nn+AZTS3w+cOT$o=J4%Q z=A&gr(gvl)JqXAbp0hP4+{ZU5U6rCVWX%g(RU{3Bgt2UovFDb|vAtj!G;QX<3!)1? zY&wX)6>1gBX}DOVpi>pG7lfCVL`Lfys0uO`Kh6H+^no<9!Q!wzZ~M4B$N+mko!~StR*Fl!j$02(!F75~w!FT4nMJ zz{6Np@xFzBn~q^*uM~=MlyCwMWA6zMBn=okFO&~UPZ1%t7+0Z#m!v?dK$Su{y>Qro z3(whN^zC*OejT9KHU;_dlDFQ95DpcWTn0c1^-3WEXFZlH|A`y^5=$&8e7TfxTnY3? z`cJ??>kdh&Z_hc_bd7eK4yxz^bSA@v=m>kJjSC|dra`?CbYyLn8rN^-mhB9XBvho& zUYxIgKh}9RbRkmq=8SM7yB2bNfDwX(Pr{Y74jviK|GY6XLa74bl6pDjB!_Hz@1(db zQ+Wo`GscL?K-r|b<3@2jm~;1VhC>F(}*NN6&IWOJo8 z-b2D+6x4~fS~?Gtts}#C1f*6)VoCkl#NM18Yc269au&&hh0ZOoK*ESu!33Nys7I&x z-{0tWcd`4s7^&tbrqtc^IZrtWy33nV&50COLvgrJE4wVcG6!}$rggZ6*H|=5u84rh zwmZm+6+H`C)mYE6K=QvSu-?y$=#08AX=a3aJ{QM^I;z@`JMu0Z&P(;c=%PWoD9S`i zEMs+q-a+5^7^%X9Dcwu=8k+G^Zi6gI>=N$^3{vl-oN8MhJ{QCm{LKcN`FPVVg|glt zZP)#zPJ=1xPB;mONS2_R`}JDP^*YV_1>xy9o2*#0ToT`u!aN_le^8ZL&5TC#ifo9f z(tAPqlJWda#rSmC&Q}aNOgqn>)UE@)l!ze;BVhPzS1|@ zQzUC$&QOsG(1`;PJ`eM!^lLwT#X=whMrJH@(bXOV6F6dcTo1PoqgMGSkC(0X_+FH9 zXWlYRj$ZHBwV@GjFO1&m>|mnzTrd_%1cAbi0)XPqC=OfR;HFcb>(*SxQXVNq7^Yb<%|v?;&%a%f~vCllP+vh|D6lKf=xy z^LNnzB8e6`Jl>$@ZjpDdaX^Z#h#Wm&8aCj1JB$`hhbENp{7t2JGs{mu$X4%cT|g-! z(=SO8JI+ZBVw>M3yoW9PK*13;<3=MfnSyI@oBSF_O9vtUaV)re{iZjq2D1dxZ|W{7 z_}FWfUqzz^R!h|JLhHdd-Z5R8eoMR+`Xly?k;O!~K3Ge?a^=eADo!ay_MLIZIR9>= z$;wR-D^xOVHt6`sln&MW=kN?98Dyb%&c4$-fj%b~@s*{#X7}wH`;T|^hRv5e$(Zp>H+~`|LrTp}A$hQE z78TytEWmaTSogjur92A5FX}26j!BqnNS&0|5M4!Y9`!0#!E88Nj9ffjR)SE;5J7>7 zVbtCSeT?J+V(jWkH;wxi%X6U{gC_+M9WzRsd-Py4Uv_(B9>ftWNt2h$N}dMB+*VNw zp4yX;-bbq>gQx_iPd(^8h(#+DYRQ>o5%Ewd?F=>q)~IU>*L(BFBY#7>8M-`Za1ig} zDA1i7eiMXU%5rwB6el5kwd9|YR1`Jg)CK!9dR^ILR$bjAn;bpw(0B&XIzKxRbj4OC zClPODOH)~-0r5zvXxL@)1dfxa6}3LK730_%mjQqCXl{~MSXL(7IAks&Uupf`H%|fF(FL`C%hvOyw4IB z<&FNRfXOP~)6!7u?$Tk8 zWako-q^cA;1J!q@>^lz2S}(ig)8bYM8x4JpttyDts1rTb^jOOB^WwsZV8d2LRX4H} zxz)`TT3`L%JZXRF>C=H--f8x>wmAn$!157sNbI(T{xNy@_O@Qf>>}Kc z%Z{ftwmMc4f3P6~fNFxc_(Kb*j2@K`%<51NPN3~g{qn(ku~q=u!mB-JTyG%IkTky_ zObhWkXHkDrL(xn&G%ulyUkaQd^KfJx5OPKKOOs8x&84x-wV~plZ&L9H*CD=}{)^X| z_LKroOr*(7$Dmx~V-HHlBd>_j+fqmOT1{(3QcvJmJ@9Qg4!|+f53UmaDmpUJ;&ER2 zFgR9FN&Yc<^s6rWm^R{>zKA4gH;xbvkGc;{Bslpnc?Eg` z*-WXyqmFZk;EHiy^dRKba+}Iju!kL;CU%dxD3;^b$>aL_*Q#p_#(9tOU}~Q!WhQx; zFGRtwVhM}{z!B?n@ojMmL+T2%=rCF9@Xh`5fIx-dLj~F)Af={jtl*DakE|tCyHQ;l z`;|SMdKUSelnaG3=E-x$K}Odhtnd7Yj0K16gm%mNH~dnm>sp!oBk(k&G=6t}pHX6l zimLibabHT^N$1_7=+cH4l%?@zNdqQI6Y_omK_n1bat6-a4H^^llSqp(Z@}YBd;_8Cppw-WYKBKU0fFso$ zrgZy4ee-o0KmLAVvJ=xSUYlse>x~?&ouDHS3v`ToL^W}W6X#cH6|wdugeRp)*6Y+pf(MFEpWYBF!e zHSa%C59XeOl(wJ6;zXWcp;By;6-@XTu6nY%WFGm+T<%|JNm957{Lfk)*|(2gKMH%X zc99ON;4r$~9le2t3MVhp3_^QYTrqUIV}fUqmez(=${~MHV`ef@RH`@X0Rz$;;l4>9 zNGTgpkRHLMI37hcA;M%CWwOaRi4GbUG5ku(We zfOF0(+MUwcW#AiF)Yy=1E^w8Ucg6Qg`K=juu^Y31X7d6e`p5*8m}v5bP{ewMEsm2I z3S#G{69-@ba|{*u||jn1CU4EVlw{N&N*>o(W!m-oJVzW=VGQ5)phE&niD zGG^foQ$C9w6lbyw20Y+tOk9_SB%L4e2azxjV>htqxaOip{)v-@kOS2dPz7M3+8pb< z2r8O-Dgbg$;YGns+LUKKrJwV>W-FZMeg8Q1Nn-8AKVH3fJ$JkDl;i6t9*y% z>oor!lGyoJZAgC4dbVk@zukYnDA&IK7Sq-Kw=um@gq z`JHEV|Ltl&_}^}+|MSaQ|MS4K{%4VBdHnx!W1Q(576V(u5;Ga5(M!gjtLJPmjr>Cd zX>>;4XVU!74%)1aNTw_Mg0D&Qj0AY%Jn8`(;IGfw!GYIjMqDkPK(9jvd^@~v30jK1 zU)khBbs)??YH?!!Iu8#maS8%9pq$-_>q8LelTnOD4e@L$Smij*HvzA3l{gBQLy|pk zy2Iib1K&ZM`vZ#4h?ka7O_0_Hm~^?WK181F@SQqXTl$=|`r7w7JuEms&$x~Hn&WQd zAV-vm0>XfkdeeehQZa{QQs{<~HKJb>IqImTkHou&9HRCS9r<04@qSw(YPgP5suU;7 z)pgI(aLlMArirSaaOXOD>nWuNN((>)^;9{=p+nSPo8MKtcR5wo-h`}dEYt=;Q^|kL zvKKJ+O<{7kO$)3@MeeDdV8b1L+Nn8(d_7t|t?LH|SZ7`d3Goip_pZk@r)4}e`Zg+= zd_3e~%ZwI%Kg>bFHVp7FJw*(xq>~DRV&LFI>Z>qqU49Mqycz2=Tm|0Ue4e}G3NSq2 zZdnQwS{jo93zbH>XX}h?+CQAC;QC=4LKNjNV(mttmJ8#$N*&6k<9WACex31Sd^uK zw7&d0W=);B8(OjWInFK_wXfd{uZRril+q$;cBz(-f4>=^(RLhfJxFuP>s_Mi4^oKN z3|X;f>O~SDr8QO4iW zSb!e-m3K3ht)y0_J>D$$DPVyib@gx+W`~W(2dh6auH)5P8CCj+{_~GsOqDgfW^)zk zy>n;N1{(NwnVE*M5%-4_L$nbjH54QO%rczfY&M{j__xT)VnaPRKa;^mlK>ZFcmzRU zVlEv8^*}+4aOkkh2(fc`FbFpnz^px#BoS!CO~@)zs1q7jrQ*TpL6i@O51Jk-l|G5s z@3vJ`vDG>(1_!e9emL_Fd!OnR0>eV5h%s`i4#$zLe0xE4$x@}b27Gg6-{UzWya;u@bQ<$^z822WG z`6ryZqI<`U`~bUpE?otYDmwSxD~<>>^hD_Fx249Yi)wY&vx)i-Mz7Vt4zJ&LS3h=- z?Y^CD$DyG0N_!x|1^(Fb$W=&GrRTwtZM>nJn51m|PQ6mqzon(s?BC<*7FI7`Lz=&i zYxSQ88c}Wjn~whf-d}!Ho}}uzfgXtWDGpifh26}$un}j}=EjieaK`R#u>yFOxg*1M z9mJ?YY$;$5N|RTNRgqvC75CIUjQro)5PD22E`>KO-FG|@j^baA8}h;C%Yb|&5L;#T zaJ=>ph`)sjnof|Kbj3h>;mvPItJQxS2OIQ@Rebc+n zEuJ~^JeVNsQ?K2~W?8$qCDv>kix@CmVsGia#7h3J=**%O`b(O8Qb`iYLO6k74n8$- zs8q3{h+|@+1?Ogmjx%@;#3nwjLX)Q!{@o(3Mm96O)MzYKowR-VxikvZR(0xZGjQOz zp3^7K%dog0v1KpuxeJ?U!|^b#jP|G)5zt0^{yopVY&$AUdO{ud-a1TdQF|0Kn}mCB zG-D;jx5&+M-UtRJ=PjKQedS+OYgu#fhW!v+N!k=Nk!b^d_*!36?ZkE5H2U)J*T4q6 ztsOtUe;TD$@CkVfIHbe%2P6Psgg1}nLLfVp;VdYhl=JUgJt9C8%w)6_lq`J*Pr}ah_DqTIZ_KeUCxNskXzTEi_Db3g?e7 z1B$=VRo>HW+FH_1c>a{*wI^7q z42vE2@M0xxm3ZMPH`lwV4oX^DFI1!!rjOiU5sG3;BAdW>WC?(E^>Qab$7A%))eobD zg~*0;>i9CaZ;NN0I6fV{vOqhuLnrQCMR0a&U*&hmlSZi|@)t81y_ zZc~58D7g8?nZjSeytlC31sBpkH!5&UbAF#!M)X#bB{8?=iZ^d})5Lm@w{{5-Yv9BJCkidf~tUo&!_ulW_&K zToc|Ex}#1*sJP&9-;!AQS^iW+1v$wRmD0RII$jcgziPE=PAFH>SD1K@mbFDoJj!WH zZtO+QPEM)h*kNn88(6glth{|J7Xb65TjV`j^|&;21P)PHJf&qJe)0lM(yrMU@vr+q zz%B3WFr z;pjaq`jem2aR_TZuQlzA4y{|i*mT@3a<&g?H`{yhupQNm;y>Li5pQT1Nn(q6@Oq_Y ztw|K&9PYAv2s8}7ek}3A2Z&GpH7L?nfXRY$zP^YE%Lc7%POIh4s??UCORWol{wg2+zGq@sbrWwA=%ffid!#M^d zGmnYpJxVWg)XH3WUQlz%>H8k-AJ-~i>62l!f=ET)r5@Vy_(N;cLBcUDg()Ir2!&WU zN@o*ny|q%Pq!7FJgBNC;pAW7tel#fK-e(@4a#0vcGPFbznMaQwXT8y+)LQu#ZG7|V zDZxM$877mSt6NLJ*!{mm2~iS%1`hv5vVMkxyC3L3QW5=WiEubbZt0KIQ~wQ>4W8o5 z!8otXEs<_5TRi6t9mQ6)gc9Xy&Q)dOPI!q(lTt!_H!r$ZN8}BxPT`rVJ ze$TA0J--o2-YX3_`2aXP>Mwp0+xJPbCPmdc-lET}#vW(3BQ4}Tx%+ZIMTu6=ohkaufueUXu1WyZ!xqx-v)WZiX_tW z&zm%LL=e*tosHZs54F9lTRK#+{`K<}($ymVh(U-L&&iMhI;fv9FST&{{K^>?HM7Lq z9~!YY-R-fgE&iGP^a?G=V^E$&6^c)suxzBV5XQ)p4(U1C=Di1IQHwMcO4NQ z2?wa7WhW&KIRcaqDN2J^6L@fECA=OgeR{E@CQtD=V`lJN|Hbp?`*|DhXe6Ul->p!J zf}E_K6_ztb>M%w=7pK1Y){TcJw9gQ=827f{yA> zCK=38XFE?=wcyl(jFThex7L9a!y0dkw1YmHwP;rfR^dND@Y}QQU5+aQ`kxh?Kes@f zFl5}`lzU{9NvDC{1yP|BH@Bt+(2Joqx^j$`hoeH-@fYVpj9LyFz zc(a7mZVqi`!xni2x~(@p=5N-fx6Y=IxM5xw{^LG7O%|f%zmY5?fwK?}-hb zsb9``U)sL4T5NSP&7&ao@elIy@>(96$74@f31Xf<_uniN&Ekm7myEt{;+Yk_znod| z<<+IwZOvbQzag_P{!grn@=`!qT|d4}qa25ARZ9G-eH!p(S5en$9|Eq1Zd|@?{Eoe$ zBR;IF7hvF9W~TY^<2?sy7p%ML7Q1;@k!!<6MepxE|B&HVNq(=~8AHpy*SdNyw6_$F z+Q27P`t&2aTi%o|k4%~xKQbx#bJ~-Ex{Eb#U0QLWe~%v2-{Ir{mkB4_&(ffx$)fod z01ZDcvW}nsPyV}WyZ>*LF#k;}um8F1|29SP|GcXIHm?diBkmLDyh@TR8tO?p2LbCs z=pc)mbcDurNUOGZv*OY{N{GPBEfV=U&C(>y$<`Zbf~onl{p-kDj!*&#QPZx3JX`Q7 znFTQNsK(B~C{;8GGVlpy^|*)d8w)uxXH%z<3i;V=(_p79lqa?=IKw4L%b%i6EMoU? zT#7D^KYB}=z|fQ3Fos{fFSqP$W{aVkSmz7N2T|UnGYj{NBa0}jfb-o-a!&*IzmC+P zY93|Vr!KS5F`T|mv@E1MV&(VeqUFBFXCrzL5fVV<2m$EfkL@eB(AhX#?45oTF{(B) z6EewBDS|nJCGn;Ko$dRdKYPL{$4v_1K9AG;Lj~G);p>YUN}(;z@LXiEN`0v4eb|SI zB{}^LtoW$dobLO=?0#K(ciNM7D{p=azT)rRw_9G|$jQkiU;+$8FsXgi z?W8}RX)0g7V0KBHhbe&ryE@#gUuK+8STMoCwIF}M=OtfD_vRbE2^8W%T&#P)o4*e6 z+BZvrKeY}wR9qWT#RHx@JQ;ZY%$_1B?WFlz_x^R> zG$68V-HaAAE8^nf+WqLWF(>Qlx?WY^JYV%TyP%}cG>7DYCa+gsPK=IB`NJaoyj4TUdt;$Zy%-wcGsP2K#S{UA(SrL4Jp6 zPqN0w$0mmU-FMsg-#@ijl&f^A=#x{y`h9xk7sakO3mezT8|<3a?M*?4>E%DUae3gv zvqmu`PmlFva@w_+q7)HO2fjO+VzjW z>#><+ps^(2ur9n(-RX+u?Dj^i;Fb4Lv*!L9vO4)G?7PTP%9x0zjh`l#w+`w#8nj}D ziwr(c%z^7_Rg5AnY~#l2;u8wQvE|n%2xbf#Ard$M{4 zBS6$Jk!-9Fle~i{2y$cqodWp(PWvcLW}Rrhr)P+|4rXI+C+nY4DC8NsNZW7CwBZ7C z?fby|7w1+ehxY#EYdwDJUTH@m0ZPO6Oe9FcmZ3P=ackyXs$uE=LAwuuQkYzBupWOB zpSDTREn@`oXJ3TC6_UQ%(z}<%V}o{7=J}aa=E9Eyq-#i#tiG3%4m}cdW;HGd6G2!D z6SuciH6n9TBq1;moe5lFq3yLBud#1x_MF%#&Rx*J!h6HN74skvL)DforLVoA9o+|@ z>WN5)g6Zwq@~Aky-5(q3wU*b$GgBFwi5eT?^FBL`E_7ox*z2j(4=?E!aHXKK#+i!* z@}kld{VN#tn_r*x=Fq>6XphEqR@5mk3DkG=|McNc((!- z``gv7+ZCuR9QbrL%wY*^5jQIMzW@RlM1 zg0_o}UeEwIiz3ua?xMX=O!DMCQ9O^Qa+DdX#xV*#dqK=7o4_4!EMY6n@~dPt6d-;q zP!|;Nl#D}K1v5EHxEqG+w%!mvbVJg0jn|4%i%^*<8x5#m8w}M?K=l-El zflrh>Dr~KfL>{zy4ib@%`T3%Q2hA~--ay8o3bhyG1XTF;&fpr2I*oogzO@ipc{JkF z({6mV_(1AN8K|QQLO(Kx)>Q}Vpp8H81IijYhOGCQSlu6njx0#X$fNJmu8h<+Unu$s z_LKpbwNQ}R%=ci}uivmC2Sj_3%W{vwGC5pefdfM}MYFpntri1T9>Ec14rre2 zRZl~y(IakD%CBdba0EG*JR>?6VPe$D&vc-32P`6XV*O%<#I+I-7D2HAY=g`Tt732m zRq9pfD(IW&0TBxM8JfYV^rf{RVFSo!@##+X?K#nQC`6sg#IZ5(;~p6DOX9f+o0)ek z8h8W>%K4`@#@ciPB^GBTz=<|Wpz(7wMS{o#?Lgp{ z{kg;Rk%GXe1hp1p4Zx>;Ov&A}qvE1B#2)G9u&eH2V-s_M2Qb3l$VzsuJ>7b=X2mbJkCG`a;%i!z_jH*{h-qxWr%yr-mCiV~ z+-3Gqh|ntJG5@v4jB+X&n^Fcm-U4GL>U1x=yT#FhunkpMZ%>EV$$Hgak&e zTNu!sC5tOnwr@hiU^J0{0Hmw$y%20#q>J3W_{WH~XS=c5@%vc0tb04j`#B(kIYcbn zmYn`d4lJ(NWSjlYTIl_F2}z%0CLK~tA>Fy8l$339t#E>ODdx~omyA~k3g{yxD-ah= zNEY&@X6^WPUC-B}@(d1bbzkXWR31@ENZo+BzJ?BFI~X7**cxZyl9Dg;+Ku@we&4kz ze?$9?j=rx7k^_K22W}V#_CiP^G-(od8{X=~ZiCl;0cPcb@8Au5)u5E~R;94Ju(Gdx zPDOt%aui=`L(9nX(mJf_WY*ilKM%!y+LY~9#_Et09eh=N4Y2U(n~&y`ap&p9HR$b+ zp%b#S#})Udi-nv4!b+VmL&AngcqO30G?;$HO0mSpsV2=^mt1VFt!WI(87};2X^63x z1-|Wf&{A(Cyq{q*IE7#zMZ`KZa$n}Lfsip|Rv3HzWzcUG-iTd2-0{{eTL$XMo zMP>@sg1-|V8{V_X&%}rclza}i1C_tkma68rk@gh*xA> zd?QY#$h1`o0{c5)cx$NN;$z7G*|!{?ODxWtPTn-xK}fw(_Fp*okMfxkH;62sVLXJA zE6`Mx8C-yxSvOT{X1O5RmEw^S*v0t8cUwy02~*-aCbDw4Piu2?^TxfTI_VO0f$W9JMEhCs!`&kPngqA!#l;8X*H@_L4W6E1-d%Om z>G=7X5=ys2VsDe=+i7CzMZU%G5wX_c5NAsgn_F^2r6Y;k;mgS=dbxc6T41J9`Wu}TpPu@m zRRdCJHiRwx3o%?12ha9Qzw&Y92%g>N3oXnZDoe5qOQ7lcN2>%pWr@!~ApN^T8B&co z-C2(MnJ1aXhP61y#zT#=Ykww$$X)b$WCCluD6SR&c2U?UP?=a)`~YC&@|VcGJy z9yO|0KT&cP62qi(OG3oRv99G%5XoKpcQ5zVU?vJQBBL7ca!zUCtL@ zWWc2etHk7+Z9|mOAF;3BSc#OCa%^xud-KNHA)nXcFd+VUvqF+Xj135$z@C} z)36`cNK>fkE1x!We?%DCkSIY{S^TBwyqt(`S$z81n-0N9ne5+H4H!nzCi;B%5(&k< z7|QLfi0NuCT{Wncb5W{Gm)HNkwoxi)aL;L%E3+Y$Q&twN2c4;#YlKk*4KLe)d5A*f zYB_PEuBGEogW$+mK0lFjRfIkw&>@_;tl|Mg%NKfk*A=!Nlp-4;#Z}byIrF4izJiGl zLM^2o)%Z{fUbg~C5oIM%LjI7c9?UDU^7Vcy#JLtNx@;Z zp}$+qiuvvvh)~G=Hh2HPN2(NiZ#Bv-u9Sa38}=E#0AWt?4_xz~t$Dm+wWHoNYD>xK z0_L8t==!@9IbszBf2jOfCR12_*#?jvt zjguF~=`=^Tt(pe_@=-U=XH8`XPk2zm1~Q6Wt#>R8S}{iqEQrTq2_nmt3L%Xg>7ZvJ zi5sPn z@)#J@?Bhk01z{vpN)?g%?d?4k6q7+uMc6A70%rfEoq90Ew)fs!nH5@{EAnSDuGCGE z&X5!}Wu>@s|xl9p&K_xD7u%jJ7Nl4Jb3fEsVy$t61>eZ`;GiKR88(IX#2n&Lh zBE%Q%D6+@V)aIUFj2p3GoPyU;DvF_L_XNqxZqt7ibQ#_4+!yhi3RX2S4byeF#|T*I z)HCA-^6~m5Y;l7u=iU$NA1_K|Iv->yTH)i6el1**eY|b3iBZLkm(cb)D|87%ZIZ?u zo1D=fTj|V`WPuzxJmXu0@31dSAHNcWg5SKmXvvPJu9@|boZB`<9Jl=Bs4DFA3Lr?A z{NL$4Tf~xZ8D0Qh)eV|hTlfjzu~BJDd^J<)lIYh@sqPxfUbdm?b!tKC`=9~;6rUxy zy@o1mctH%3%~WR@=C!-`ygh}@Fy#9CYSu)plsw9ls3)f>3m3@vh|P^U&sFTEni{I> zj?TDE5)U*{F)HAZd3RoS5BN0cyZF`JG|Lm-%+VS(gL5T)aTsmx1%2EPE{3$xCdn0p zT3*PcT(Kuo?(H4!;|Lb={_Uv>imBv3y@Z@~OtP`?3KrRp3gwS=_^;kQMIH#m0VyB(q|vX z*x1;cBqvz~ojl$YU2QyN!Y3K_{u%}Y#El?B)lFdQgPtgtcltPTa8I{uC+;!h$_R$H zNXNd#r#O4`ppxb7a<9Dz>Z@OTy|#P`()z_9HKD7Z(j=s%rI~jpr(5cnB%Ni~=bdb$ zZ<=?Pq4SL%IsDdk$dDns!1^XG=*OxT`2FOC+pB0Nd^@zMaVD&busUUgx#&zLGA~$V z^(s4i4wR|+g$(&frxqo} z#d|p7hn5sm1Wsz{TV0{IU)aRL;PG)3J|2C0o@{?uSTE6ue*OA&NXa?SkV$1De>64M zs<#$gOh+kvlas4&N%mw{9|yd6;>_Jioe<>9Ji%lFlNZk4(I_%kW_PiKi56 zDl2Bg6=F@0fbIrQNm~t}Gwa>Exv0Ct4UYHa94lP#^5UAxdacfoS}$Gvi5Nb;a1U{R zFY`Kdm2EjE4x<`OuQkZ0UX)yeJ_qpWR{WVx?jn)Acx+)bC(@#gm+0(ekbap|H( zhBD$WDy~lKMjmLm@pnRey+ap~+m4HqjXV>+o=YjfL?2GF?oXaPNw{_EmYf>n=9tFq z=1`q0cYAB*mhQn`hB9hBJ$95&>-Bw6EczP4TRHvVdn<2`b?^SFGvLIBaS$a==deiw z$aXdwChS5^SO53lrxhZKcuG=Q&)jsa6uLPkKValq6%*W@Wgvi=r_MAYjYAYpQI97% z|0oKivs2$(D}M;+S~t~ABBnz_%Lfp|l)JkhdR#Jsm7Lq?>zc+^FXtZscAv>9YgLR| zWkzX>WVYGkt=dTSX5t8Q7=Ac>kH-=mCVUlp)rzEmuzUG_<-fB|AO5DerTF&oqCs zVzcZT+S-Ql7)cup!I%2bDTpj)q|AJdoLS4x=VfOneYjhIF^9dm(A1-+d9~jqYEr=w z9c~`%vkT769u9-OQ1f@w((4=Uq8Cl>G41jC{HI-mJPp->fMe|BhuWVD3wyBvmkl~( z`e-+n_tY}J?d#8T=WrdtYqX?Qp{Z!>Lk~ z_etj3dr0_eHtBiow^q)Xk(;<_akQVfZTi>K1gp&4iVeTL!Eq|$wHp=GH$VqVFu9xL zD$_`rbJz9I@~%5{zz>6u%jR2NS||2Bfj5p#xj3KDH3J}LCU)K>I(f&c*ele9nv_rn zbGw4q%mH(jo#x{x1iX}n6&K}!>F{Qd4ITJQ)+1-xU9w3hzjI_nLvcP~`9Jy3j) z3~S)(B4>+_Bi;us59P5&+rs>6%ULhP=Cv>NV-_3K!GT4`C7XZ$4PL846>P>i3+u{sx z;_SX5rTqdBU&1OmMw$y5ZI0D_y5_y;xksfpqkTD%;obE+R^4AY)*D!O7D(ew&b07o z_iFW9m(|qm9=~a9#guj&%f+dMPi-r)%5PBgU72s?OkPCotx{*$zDi4!^y(R}pKf~dmt3E;L=q68mRnb)mdilVYS@OqnpI%ybEhZQ! zJuB$9HE&K6_H(ubySrzLY{_ZaI>6;5olkFP>M0KshCnbj*5}F&vXp|94#HTDg4_Be zv-va*PZ&5;O6aFS5&}2nAZHb2xt;rmR>Nfpm5C{YuKP*YhemY;!8?4u= z;u8?ih+}=Y%y&WWJkHoqVRqbg8&>pPoJzBvKWE>;pq#XA=;oT;#EGfINU68K`ftCT zihhf7?JRw`aCu0ww^f!r@bHs975;Y5ORf>yN&2n!a07GBwS^@q2PEZQf0sV1n>gI0 zUF*f-@o|({NbSbfjJK!YWhW?ohyBy!ll%Qo*kJUeY-&r(_%NufRf}pXrP=&65h8F_ z=#uXnwrI7hUbE&Hz%|cOnD;*PLMYp&N72GDoyk32H<8) zbjpAdOEm~6c3J>hGUi#RuBFr)*S%XY6^C&PpvgXp#}`%^udY$;#RE_mu~L=h5uji? zY+zh1|AA97n^Pa_D3wB#JYBkM>oRs%k@+>LnrY#@^5_4(`9ZIw*C@;ZKM&8#n6LJqiODyvZ(aAPiDeA1q@Nh)O4h&V2St%!f6VT%-w z<2(A5%%71417fytxQH5u+$0LipU`mfIh+gj(P-)f1r!VoKwx!&RhPQ$xT;`U?=jKx zhkP2gIdVr)^hKn)CYq#?jE6fV;R&O3`&is8|G=mGe8F4}gA->Ht)+t{^SY?7WM)J_ z|BNP0`{qrYVmfH1=N*Xtas@F(dCah>%MM9{eTzWm+Wg6+_)*C>oBZ{uu7zo=%t920JPu&E zvxA|pq?0)xTxpeq^9uW8qI$N*5lR*eqV9XQ8vL1yY0tqg1bK?^*Pb>Nrq2WYk$M)& zY6m(GV~Zpa(Q|D>xE=$MLtsear zkOh;3)i|Pv-M5%b;fQoV<~&HgE@Wx&n@L$y9m^ZI#I#w zdNHVy$#HZoe6yT^hv??-WW(`sI$fhGnNJ|9^u!5F5D)ZtqFDyz!j$5ifx~cLgZ&(# z6P;>`d!*Pc!hR>c1L9K(@{M19`Z2qvPlGGQXR?nANn-@_5R4rpL>#(1F5KRK`JZCy zCof34nY5N?lMe{t2?|(vmhgB|4|YAt+I;W3xWe=tUK<&L4H{ z?vyUaiO1pm*wHT|5+fXwE~jsH-H*rbZj=;FfS-HN>3!309Lb=k!iGkJ(b3u)2O>G? z3BB7)#VnhXEiO4kWZVRLJLrj7b03u@C2!cQC%T{BescwF>aDUQC45m((<2z=)zgWe zR^pN;!Bf%;tLPfN5_Sn*A&F{}i|q2y1+-ezv%*;V^#g;o^wzKmd$7G-JOiW1<{gM9 z@9k}(ohQmMkkexiHM+Q6rj=pMvImeWp-4Oo5mfOr8MEKS-kRA16r>mR+R1+7xJTQl zxvvCOTTBF#A@tj!utd;wo;6@3GdbHM9016#)uh3Oh7}94khq?B@`+I9MS1Vzh|TiB zRWGx_H#`-0>DR}6fl_yCQmCNzFG5#73!%5WhvcTIvMnL{Fibf!mH0$QX3=+=U)oej z8>_F|=Obwdy^L}SLU@jz!97fuRM*$D|4tPu7==7@%paweUks$!gP(M|bcf_!It6aG zAY_Dr2Gmk$Xq@==581#K#dA(>X|%=F)n*MT%)jLK`+q z;YTy6OXVkgSx)IIbSaHSpA}DGf%hbT?4k1H z>?IeU&_AF;{%4>BN4t&#oVv-tiNo|sNGbydoIgwF<5A<^j3l0fEsd7KjF7gN*6ycF zr{lWHnIPm=_KPXub6#)cMA}X3XfMHR!jzlQcwB1Fsu-W_UP+sC&`{tr{@YANCOs3KSL~+oH7whhrIW;f5g$j%`=tdi<0hOcOS-~gQ}!U|TeQ-*R{sc)nNeF+1R(&y!wm^F<-h3QIxvMVV(F0vE?=I+jJI+`bWL3sGZP1Gs9AqL@DcOZ1VV``_q-*Uir19)spT%3S1}~ zmR(rhm&*BbH*MV54dOyi)lHz>gP#|$f`iD!e=8ii`kPEFmsrd_dP}WG^~AIP>nbKO!!yT(xKFWqSGQ zf)#5#Dnicp2aoGkQ&jeG!e{H&cbdvBtEj7bt8Vhd53b4;$#C-{W#=Q3DW~E`#=1@$ z+Sq)f`0chy5`JmI{lLD96A+tTBD5YjJ8xS08{NTT<6cj~;r0KkxN{H7F>U{SHZvGAjD5&%gf^7I zC<()$r?N{Wgdy7^yLPfE^JWZEdX!X%M3gciN|70(w3A9@7bTUX@T$-#>vKNd_qW#H z>sZIJ=6K&ZCZ6Z+zVGY0&dvAyp5L^vHdRaHKrJP2{j}q2k1nJ~$Yza9J4`w) zn;yLG*{}&cP1rHCU*6C#ar1Re==FP!xq6$FzM`gwDUHuj0YYF~tY>?X$R4E(8nQybh#0>Ho!I);4`Sx69`lr&YQ!0TCrC zRVZb21grK!;x-;x6vBFFTRo3YH%NQI3KqToZr};dxvv!cs0Zt^qiz3z<0pc}GWasn ziD;jMX63C_v=R3nz4`!n(N^gTMfZPgU<<&HOjIQcJ^&2&!snZ}<0Ddk{py(=an=xp zD1R)|V8UZpQC*?vydcFRUc@PuRJ7{UC2UK$-lB0bt4Ap;BY~*n!f&l)iTfov8JOLd@&te=}wJYTi~fd2*L6%|;YrV{#y+h|b*N zt+iul9~M$j0V~;mt43JPw1KTUJY+@WW*>dkl`|;4`!De^!WrAqR6Ps;aK229Ag&dD zC@q8O0(&@9{6zQpeG9eC=+g@-AGu?Vw4gu^QiM`ow!9tY7E-(D^5+%kx-(??|P0`LE?KoR)v4bHwmQ1H;(H4CW2jGiRFugewA)?ne+DV`ucn9 zMAQlQe9-Cg`)0Q09E|X*hsU~GiaMdl^YgL~=M_w_b7TOHmKTQY1GfJ+F=o zagmZDD<(=p!J6WEdRMgG_-v!ox7@A`r^*F*p6xlYf6Qci=t&*{gqBjJRid7`Y8|b- zY4Idb%D>wU(`=unF5E&aj9cJ!AV7IOQ3)pQ9gu97#>RZdyOgRbQKMhshr_>i5ULgN zja1p$!8rj#+H9;Nh>;ts$fFu+%{1>geG+kHB(coyqHl~;1JOh! zG;FN@dr!5E5nywkvXzOH0WS2@-uEQ(tkEMr8Z;;oCsO=E!*Gf{BO5Qx{5id>iKKbG_6(}E5S3GXhRm`!G5A3!+?!|(!E^4iZe$r;=6Up#$WRu>Q`=Bw{k_pH zd#h%5!ZacF$&&Jak+zSvI+f}0`re-Ur@u^Q!9~Thw?qAdRpBBK{C30FlOEf_^prW0 z!d#Z|Jw)&h>0XTwRxux7dSK)2ewzS+hlb@;9$2t5aTDRQB;}%^DLw8Y45{kv&mw{5 zSHrq49j5n3t4X4NCAyZX9NSNG-~eL^}^CB*L^)c*23%(%a8! z&BI+Y*dt2oPaj^E6Ir8-=wN#`;Dc*`Q0Dk`>)CT<^VVOUluvc16a&KX6ALaC?I7dL zEAa}odx{U|ebr5xk+93OJS|+Zy}@Su(N^D&z5Y*~@SkT3myZza6fP&tsEALHQp=e1 z!0Wfi{DuQJI%hjDjym#GWY-UX?ht95hX?TYOIJ3&YWN_|HaWMwncBDCw;MY1$ZAsR zwVDr_t2wkKi4aWZqFw@xDLFX|SSE}LbBq`1F_X@+%nU)K_P;JrJC;f0TU)smbCk_} zMEE8{ztRSI2v`nZXc^nd5xn4B3;2p;s}e3@2z!z^jt_#`F662Co1bvS#Kc7Eh@c4Z z6J`=#)wb07Y8dD5v}v<(J&{Ek?agL}GHt*ZJ3p#l6~;e}oXgA*fO7w%A9fo3hyg9H zt?gIgnF>_yRy@YwBm0X2ATi$>=jZOqOf`C%z?`zcTPb%)uVkM^6=TK1*>As8^s zo~6YTA%Fqze)w>h(m?uCg<9?}Al!3`?nV65FBzk7kml;a88A|_J^>xrE&*4n#zF=x z2SZp&7FPbQyLXgqv4bY=&Q5oR%?PbOTik(IX1%zH8du7D!p-j#r}6tjPQ)0c#MP1m z0}yXHUH%C?SQ1V8t~O)ojpDlgZo-K;4f%*ao|;rSVAM*>bAATol7lmtu5#ng!XNYy zo!Pwa(*gYqf#jj)K1PZwuPK6BzKvTsWH9#dx^u-8s3dg0Bi~|K`zlToH{LPsU8;?R z%tZINIMO1xF8)s+(wqf5H53xipXL@T$$hWv_z078WXmiyk8jveBbAoClpG?*8RRLF zG%L^miGQ#*?@sdzQLhM008x0t(8|in#{0kgg?F04X!CQ%i;!Uc0qU=hLL`nga$5&Dju_R(~a6JM^Op<3w|8PUgp)Q91 zvpFAM?84nC6BUJt=J{!<^PWRQHFe7d(H-h9(%*5=}`Mih>|XYq5ssbUnv z&(;29f@ zYg4uFkKPXJPh{pO(j3XxiPZmAAKAWBryr%us%QD9O=A zY+E+6jA-RF4KV`pySOh*^^0$LjTgG1RVm4{= z7?!_g<*pa#&>kjhO#8510W(kT&-Gucu^96m(mJk#JF3b@M*@iQczJDXAVht~bA`z+1|3mRKTo=po<*`B3N9Gw-h2@po*QRxETEC> z;Q|PKT7Q1!(0|?ezFX#vppb1H@aSbxv`to=Df-plGWtU(6IYWja?whpu8PDYg@G|G z=$Hm$@s7jIjyLv|;&3^RobZTxV(7yFqG1#k(_Y$9em{*{sboKHTr`w1RXDx<{{H>} z%bBsHamtH(AgRN3JB&{wD98~o@_I{?(vPsUZkL8iFCorP{M)*9ycfb5_fLq9H7MRsNZ`JWoR@-hbXKx;-@ndzogm$j3RwNXQ`Ptv5*be14ptRQ9%LE(W2-_g!S~nsW!7J z2t1XM02B`s^l1YkIiod>Oey{$r1CSO@=w!|?LzWX{OxdNSttmr_lEzq(`cn{fF>E& z4M&IybNItc3~!DZ0X3c&Q^r(RFrSE*t3#nd>K77@N$NpHx%Yk3&ZzkR3}oO$9K>U* zpL)9TqBfOp)Q0TN`BkBjB6T{(JTC4kR7SR!aM;KIw9LIw7S*hdcls7QQV7BQYy@(foZq3xgErc21CT6R&uA{!QVY9Y|ER~X#j}s0bn>~x3--iY~C?bOe-|C;I^XE2{t0u!ANt9%%CA5^KhV0%1@Z{Q5zXmb= z!i>V!xp829N;c!rYV5gG_oP`|IbA0LaINFfr;ViQQEv@a)&-Y z2rEDBvv;tgbE2^Ej>&83OcW2>((=!G8DdJaql^N;PoE@=wsRmSieudZco#mRqJ`;| ztS6$@-PM)m7)9a}uRRYemv`^z@iiMk%AX9azSInh@-A;j`oxkO31nqkbNB zt__mzc!FFjAMQ-O(O4z=G>w`6rK zz;X?}Y(RdcvspIbvkO>+!C#G4m+dVW3J!68$8VIshgcnRY*d3`QCkvqD%oTp6VlTxD8?MU_Je!v zI=S!{wXTK`_P3LtDf_P*1krH8B6^G&y>opL7o@{)IDZZMsG9KLj#e)ZXFRgO*IgrD zoPY1n>Eku9vJ2v$xOGv}G@x^!4BSz?ij!INUrqyY?7@(|GENX-oIi2-&$*(sxAH31U+L#X!=Q%5Mq~Y%kHZ- zKKAyi5ug9rzJjAJGgry$m?KwyF5laz;dS2u>ZC&zb=|OhzEoo?>(d-;RxHhzN!pm2I2UIV;d27JGpYHVGp{4J^ z|60vAW?HO^rb1kGN6l>Jvr4-utg$He=UD>2wI%z5)5v4H;}9{dAg_<(){}&Zsv2fM z#s9KtKRyha)_gX90m-j}Mma_SL|>#$uafX3Ji$WI7dWPPg4(leuPGfqchZD=^)qm>`^qgG zFE$I_Q81I_Nq-UXH1LK*J=L6a0(0@|mDNroAu}rS38r2>BySc1gitde&`E*#h;+nA zVVy5ebW4B1lUL(e<`7aabH`Ybk7KG$lhy*N)`6P+yZH86K!zv%WkBq!>8y|;9}L!u zAzLQ$7RBLAryNBBPUW#n~RH%XimNR3@iD2u+EA|S0TSqFC zU;3jsbN3sTkh;NmHaa`*hJjShG(+e!2@(D*V09QaT3sx%A022NND_3s^4H{7HulLAZTe;{Vaz&J zGW-r77=rT;A)+oZGb94@eC5P5QuhH23k7*u;N${ovM1_g+3Fu1Jk;8045Pd+E)Y+0 zt5XM$voa@_@Z^e{87-!#E?TtKaQ@PpxioOq7CNv*leLN3&r$2%Z|Z#wM5z}Mi6BOs zL2_D1(j|CbAaqXkCA~N~0NzfNOfW3R7A@g^q}N{nhP;Y4xgw4(;R97+39Rm$`Zvd3 zd6LgwBB0!#ey5D6njz0qXf}9tK<&l}eviyj3z7*hZm)jX-8SVChDYQ?kz>qzIB4gH znzuSHfL}nxlC^4y`HBJh?hk3zAye9WL0||{v_O@`le4JjNGzwKTy1c@4&VH;B>9R^ zM6ZlG3)2cmO(|m9TeGz(fAFU#@V7DvQ8h9UUZ=vp|57FB92JEkPmNhF3;;fd$2|kN z8GK=YiDrsV`yn@$GQ*0o#4DQ_4y=C><1KF)z0Hn0+oo8-(vRyr;lquGi{qfF_c8Un zL~-He^GxrkQLt7dc;-Od!+)F%c7!8Bo;NKW$nSjm-&y+5b0 zl}5=Lr6HmO0}4<;s$5U{UV|A%!y;|&SKYjQo>Re-I7$i=__+*heA04H`OlYQtncjdah!T$N~XWosTWpS zars)x^7qyYhc5HEFk<4B2d}1YYiw%COUR9H47uO*;pg{{Tx(zc+cWi8c(KF!&-FXD z>{$Hz*)o|!2xA8FotwKx>E4Rx&!yvv7)dk&&=Uv@4fVXWzr&4$=K{v#i+l9#8$&FZ znwef;=Q)W6ZgURW6zMa)%0E@C}rP5^)_>JbCyA36mL|B`sN4^SHg_1A|}(I*}< zWQewbL0-SqukC$Lw?*sacfrL8g9i_8Aj*>Q z<+$Mlzz_1y=b|4ano=;gc zRWx)GK^<>hY=f>`kibdEe_|$(b}&=Y5fO(UtqGXkf8%EJgbpYZw?cx z@m+^}{qu9yae!aBB?J9#NI3b|jMb_^e0qP*clZlPw0n;pr%2j#h7FretX}8W;gg^l96Q6)P0W{L6BEipR>M1eZOVXB;RsPN_>0x+EL)F7>3OKG(%Xg)D&hY`XM= zeAN2STjw`v$?SHS?FB~XZ25KK6QbsZzdGu_px_`R!N>jD&Fe4>=};~`$jB%$yHf4D z4j(Q`@BXuA&omW)E}F98)zoHBWtH+f>Zg=`7$qO^z!#2Tun2CEqV~;_jT?I8sYZC% z|0dc#Y>o?$YDa69&T;^Im#O13!zaWZKCCVy7)k=JYV0+7 z*+YBFwXIe2MZUaY|KHl$zxOo|A|?LmJVM8|SO{YLLu5cTb*mFI$CvbzFMT-K@tbd~ zmM@Rg%sb#caTk4QYw#;q;Y^I7wUyJbz0Ob0j#}|9GJ2U>;YelqyAFwUoVxmYsUOb; zoh|@)bbYk$N|H&>$2Sbq)ZKT>cUoSUo0|uQg~hNrIoff12`gnH~BlrMi~U@>Q#x$gmwtt|mZQr=U6peu*6rqXhJ> z4BB&_{`dJ5pcw9f)W*%dyP8_unqzfXiW>uPcqgP#4QvwBdm_y-0&#)UuBIqvn3kwL zD9)Yl6akwhkcOeVQCPB3^=pF|H}exe+w|_;yCRL=^)r!?nnK`_{`9S_pIsH%YG2=P z8{^f~oc~m?MAp9m8bcI4NO&&iNU$f22j(!Jc&j0MRPGsUFSbN={gzTtol{%@V0$$H zpO{6AzJy*vjfRqkewC}cB_9qhjbxMR`~jk>$lsi68a3>g$6p5h`}B!MU7J{NCXmaf zd0#Z?HhhPr%O_~nRL}`Ho2_5I0Eb9qDH8eaw@_HyU_29Bcf2)0$U#kRw0^HsDtd}? z>WFIsAWS1$6HAiih!vVy^%=__I?grPIjN#2MD1vTKF=2?b_)+*^t%qN&>l0U(7=l< zfcKBv+tjorFE5Y(h6Hfd>N2g#w=WDQgj*0264D2@rE|*LX|y_@0g8h#zj9Z43n{cB zK0cng+y{a1rI@?5E@?X4h?Yfl4_=>5OtiyGU%)g*QM#3{f)9C;Kr_BIW79j*6y2-~ z^%cQO96`ZF|D9bTE&Zub;U3avS>TH7SR`j##KMS5LnRadoYKMi99kjrD9RlV{@{i| zs)IDAPMw-@VR(A3fUVmbgPyAv%ifef*3s4u(4+_S`Cdm0b915QmoNrXw3&EiykH^l zNnie&9I{{;DNQ+R9`ZF;D-JCbDX2w>8-Bu7658mEdivn{&0Fjf*PGvswKiTjdGfEr zvt&4eOk@((FDN_0gF>2)0m%=)dHp9i2A3Nv2Zu|Fvc$9U@&>B3pY`6qdhfz1Xojq0 z3q~qPvA4I{)M?X1T%QR&laoGjIa(zS7)(2#^6$Np zyfpv}=^mGd9_8;`!#!c2q7< z9XJ>kw)KH)?)Z1U0nh8xS6&PO%L@uwf925(Ac3QI)LqkrD=UpL#JXdZ*+;TKo#GvP*&%dU5=F#0}0kLNB6ulHQOJBXZ0PiDZMv zOdK{W{juAlbv@z+;12!RWv5Ti3ZF1dxyRps?4cDwF`Of9_t)=f{seIG1?$n$ z)>feH5Q<(qACNOotLE0stgM(f%XjM9gseR^;X`Q~xe_^UW=@Wct?i*>+y1Rk$>5?FRwVFFOC^$F@$(x^G;xLEi=<@kRua!0L-;jl~?-uRdvqvRV z!S-M@)g+;U;F=>zXK3i0Gt;RmJLIH~Putq3sY_@UFrP8Q1_NMYZ+{M?F*t;5XTwp% zHOSwJ2n$6dLd-BhPb7sw`;~Fbrtme`yKmn?O%Vu5Q%|2okATimmGI<-1|jniS9wzF zl?fa&4;uh3MQ}87ho~{LOdA3v0mX@ku@TKSR#h4?vA`~sau~Aijc>f~AX1&s z3Xcoth`$&I{vtImfjc(pEVr>4y6`N)xy;mh$UF+NxG0b?-JwIL5!f>n*gPbOM=k_3 zjpjr;0YdpzJ>e5hkbjYuDtmn>CYr}+@b~&OMFylh_?w=lsZn?0#3ckBQrUC9)?mBt z{vSU<%GR5=6#xv5oE1M5aSO+J8c85Xt=OP&wL={0cznR$<4>eJCPn97DmnyZ*m_#YEM8y&%cRi! zR~Z=@e(}&Vp}|wCoLT%aA@ljwKwV?)GK45bA6kLn!^6W$W!I?9nDeEy|5?tQ>F02+ z-se}2cu*JwSAe+3khlHfq1>Xjz)2N?SA`?g|I8zSvvu_J?wA>ZFS~@+nr~K}|HrHF ziIs$)KZ3QcXjI2Q=g`iYLkiw+Gm)e(rPU4{JLGA-zer+p*)iusVPr<7=51d^&zE~|y<0o4@==azTDV{&NN+BdG z_9;!&CA?{y7E_&rwaeTeZ`=)Lp1J=D;G0rxh9WiP%0_mFXe4dhmXfvErjR4G!gr3j z`CdSHjhf)Nb@LC=?mZ&8q^|zjLH4=+*dj1@l#ER3_Gk)eQMu!z+hh`!%tDZmNb}V6 zib~a*CFVYDX&f87_}T*5_n_I3(V`L)jmTnz8!f47`8^*ho(nJ+1X+RM;Y*ULR^ZnH z12wC?a4~h3}7`G ztzlpNw9I@HftV_kFEs>56##*jCb6>;tdLca&B{C*Wh6)TC2ZhBnh$E-_#fw!Nc8WG z!xx{;-vV~Py^FDzf`v;hbWEM9^Ol$Q8UM4}Md6q2mb^%bB4!kf6p?5G70}DIu{qVb zAnmYW&1cWry+!VBONM7DX;vveW@hE=9H(jQXZ|T+Z<>Dbf8Vc+`<5&Va9)ZHU`5_o zq^LISItgU1)Y-9k#P*ff?5f{6asVUuh8s+oGNq!dY)-|epbWS8tP+RYckUc=`S{Nd zHaRvny}Xu|Tk$H28sEJ;n40R9^mfV9ZW#$%zn!PVf$5f2*RiPf`y_(K7D*>6-8u=H z+B9e9wen(MG??e7%BsMkBaG?0b?es5zTI{-@0h6Yo8hy!e*_h=YL}zB8MeK7Quw#l zJP=0AYxB{VpqL*Bf|QZ>@7qcBPH*8b-iIgPQeYmu^KWXkxcRuXzQk8 ztY$Q?Hn8t`+B!6~M~sMloFHbO2x>)|4!?~pL?k>cxg-vnO2gQyC9(EBv`t$!iaVlw zkyV7ba5-20>dZ}y#iKPxSz67rXU^PS@j!K4>fYCw{&xGenfQ5QIs>Ce z(+Q6ANTQAw#iz5ehe;B8W)j^EGJ2fLo8zPlD4W>Ko_$>WoPog{Foi%(KphiDN!mZW zroT7(-aTtklcLYn+`^&&sv3#+Fo~(>ygTpJY@r;DT@)%l;fZLGiCh^A8uh?@fJ^X* zg{P?#ED~P}QY&Q~PEpiT=QEah5v_G7*pSuB)qSA*d^zS}t@%*seKBQBE}EB<2CA!3IQt z?T7*u$yh!X`e+d?dFDz9kJ1f}s5Y3B1#uQA#cA@FYIr!sQp1jj_(4EU)FVx@BT=OmP?L>NTzN+ZCOI9 zyJ~pW)#W9i`2F71x+PbGlNM}2am^64k@ikbO^7Qhnx&Uc(|IC%P8#bhEMlxKL*?qL zsLY!_z}3@V)L%41;0m0{{y+erpOi-66%$xvMZ&S;-Rl|3Eugr*25HZqKR-;OzGRa# zt}5$X9wHorGhRJR$XfF?dR3@3Td!OT_$iHH)m5F5jW9FBck9i)htA4nbzUEIWl2bWmQ#JnDf<)H926} z!eA48Ht6E5^mOi@&FbPAE{Qj++=_~zBzHo*)K^sv5(>>*yrtqoOP#bsIQSai#RxCQ z1s^e4`j?NPQ`@h;5|OQ_n%CSvYFthA+M})`jT-fj79j`x;iqPX{-%v~A)&x>fnfuB z_Ov}E8J*r2XaI?wqQ2`>PEb{e;eMi#JHk)9l0%#b$$W}y|e~9ioB+m40wAzD8SJUq)y#^J03giNoh9zr07ewx8=n$G!m#i=ben7%$FaUieSWW(Zs&tGqfCOFh=etkRJ zinQS_)p)6)$IyFeb4O)96Ps{N3B;B{!(lPzQE(7MN>GU%%}rMc80y(|)l~j;=2r2X zJdYCdr2uyl%ku=Dl|)xe@9zGdW|Tgp?;H)O)zJf~=Xr0d9OkxUmLbEcFSY@}RgWi1(mDCmz>)f8I83FEutR(3m{= zm0V0~;J`@&BHaJu1evR7d?Q^Pmsv9&&|NrW&a|j;es%d63_qE791SFHu9KQR5&}7& zojJC|bOt-PvM* za;`dD8ON9M!<%t}3`;c<`v<;JEfIl%0dDkFht zOLf|}Z+{nq?gDZSBYZNS5GI{)#*|Xjk-4d&agAdjgSMa-qL7+X zfEsrY?_r&_hZ%|B~ L`E%UlUw-{x<7ApE literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb200/gdn2_fixed_batch_flops.png b/benchmark/linear_attention/results/gdn2/gb200/gdn2_fixed_batch_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..9a654b708ec8e1ca04328b845d150aa3b9c340bd GIT binary patch literal 89913 zcmeFZc{rDC+ctU`DnlX}N@l63P)afr6;j61L`Y>yBqCEqLKzCpnUbLaWh!GLMbRKa znMLM#9=`qLzTfwGzi(TAt$)_q*0Q}l_x-wmab4$moX4>r`@SEizwW->EKEF16h*OU z?orpLCEvSOXi4dqIi0n&ce1rI7j(9CJZEKp zMp|O?W{J%k1y4IUojoTfDQWj#pOCP3JSFK^OE-?6!gzMip>q_)%})N%oQnLWPtj16 zrn;)(g$LcgU5pK?=a%%l{M3%{zPwR`x3M5smm|H`?s@Eaqq;Yp*2nmQ4)mlvuRDD7 z`Nbc4W3-%{4z$Z22(G`J;N2ZQH1*DOP;~93l%*Y0BMJ(oiY=aJ9M5U)QySy-XH)aO z_CLQ8eEy!6r-c9Km)rKuVwe8s=dNijy2bv#eJjCG>i_*8>#$mQeK+MrOO~{JiQbv= z>eaiSsTS#c%W~-=B#s5#mNbnyRq^Qb&y?tlcVah1b_WTrU(cZEHmnl6>qZ*izPlke zZ(eTjoMPeP;<~fd>So5fXxfd&Db%4uhgQm-6}0}56xVY$fj9KnyNhGJERu)_znvOI z?Wx_(7q4et={rIe)$kUBkWa=HL2QG#_RX6&*;cN+*3oe~$9;l*?b@{|nVGwX{(Np~ zZjQ6?wLvrgrZxp|!a1D%pR@ExQ?0&xu7= zR#qKfqUmySbB(I}uU}U`eR};P1=q_yKJ>0!}HB1I_ZBo#DrO8f}Z{6!keKXA(V@Y%ZXE`M1+Oa{-&p$`7Nie zp&^$3{p;6DckixBGJ3k$-roM_&z}ju{oW|m8^-QpS<0_qY?W;&Ui|s~c8A_yJBr@F zr&S3pe~9OPAa`DiKk)wj)rT|BJRg;-etu*-ubdOZ+{~E38<#$l1ds4-q5}$CyRGv7 zeE#$)Vz+?A%kk zh{7JSZ``y=TlC2{!R)KaruqEu{8+1pC_rJtO8x$1OKRSB#)~#E+rYb&vE}j@_U(UsK70WWv`&Lw5?e9*pr{{ovwPeYVY2?l^Gin^UsM6WYMbF+HMRIP!3wR z|53rGPfJI-s{JxEx36K>OicH>U7yK@Umw0wRUKby3_=n|vjpEb_lP?=Inh($%Git* zUb7SNAIi%QSm&Q>-Ur0~Qsi_|R`+ev}u6c0)t+#R3*=YTm zu?8>6q=U&XWcb&uTbFvitEy1>xVU+)OW&jSJrSFZtfd(C@2?QMPp)Phk8!oFc#L=n^+3S$;p}bH@S;C&vXHH$Jja`se7Y;WL_d)O2OWE$aciy4aT}Y!;bA9*3;h%p zoSfyn@n+^{#;LS#-@cXFvPHk9!H020tBaterKOOV7-LgYQ&zf z*s4^&IL3V4{_nRqa>A1g6ZJbR>T)k;PA|KN4Y_df;=95^T2_9Ag11+eEa&69p?YPp zkc7mt73=nGbehxnk!EG^>5<}UXJ==7yU4a5RnL!vMno*;Cr*B)E#;80uEj>Ik zlFoPd?jW|DEKQr^->V4;>z^G?XF|qWv~Q%VxA(=_OebgOjA@BHrOZ1Dfa$)O^08bW>?3> z!n4&oG;}%M(_wntVH)u(UMFMwlkK70`NFTUyHh=9+}uZNPWjfl95Q6D5V5K_73py` zER192s#QWFB0KAvba&`Kz~(D(2gb{KHISc!JVAN;`!iyT{0^sE3vJrO!Yyf1^yBHl zRQm>{qpuwosj8}qH+0%;)=ybY^0(k|I_fedUzp@FQ3<-ywD{SJkR0#ay(=Um!-ZP2%h-6OjD7ueySj`f zyhzDyQ_sYg`7sC`{}LmHOV|{Pibsl+w~SqFl>Q=r?rs*`w$13-(bxKh3A(Bj+v3HG3mY0t6HV&#Z0+n8QJbwR*Qb|+H0)(O-&wgF zHINNYd_X+#`gIN*qT3B4hF|!0CvlvALjUdP_a{1wsDY7cQ)Yb8sC3t+Z)lL|YuU@# zR)fIC&$`fmJpDY=nSPfdw@L_f!4dD zHvQ@JMC)G0;)C_QDUtyUIcr_rcN+8u2?^m8#gQ^==iej=Zzs&PP zh>-ZyStlnB$|o=|@WNoLzuH|x@$h_QN-)E|!Q`<;Wvk*YEMJ0N>{Ut-(O!Ic5wp6M zRzT?+JlS$3B_&$#l`^(>^j#h1%EOCV#4lD*D$}EN_Dq&~dU~h?$X%cBZQBq&>N>T) zZvg^mF`d#v{f`+tyk`$X-puF`O69i1u@tX4&yK%|o`EOK%r4pI&v4C5jqPqPe;Ami zykf#L*}0}t#%l!ZQca6MFHcXRLWZ0oV9p3DaG<|2RNd{4zE`yK`$$T+Pd$=L^F zF^qVfm317y{`9xUj9a9oNvONePPF?fy@+90w(NcMPQT-)PWh!;l!%An05P2^e{kvP z(+yWy1%goU9w>S6Y`3phcXN|FdhFQO*XK^%s4CsJe}Bo}@9_>}y~mKuV;tK``$tB6 z0|T}5dQ5UITnglt6qb|Yku=Hi_Vrzqlauo$$MsxCXJ?Y_(8*uO-?A4+H=g zoMyY)w6EivX-$$@`FVMl`m0~YqX>(NE_!<4c_EJD*JM-WxXF?rzIUjvp*L^RQsX!$ zZPJ!ha`n*6>{Q>j7x7x^>h$H06a(Y;M-~D$99a9)C-F!qU>LvEWjI z(NnHRN*)0%CBY(V*Dikl{ykZc!x^@RS}j5ky>SvV$#eIAq0mjvYI;H&Ti3NQNy9_3O=fW}SOm z#T>m{IWr?suRiqkE1YfgF3o0QS-SKKuuHnfaPRNcfByVoU|}JMA#mi*ZA)KQ{+()S zZ2bH|CBXta#ls8VzNNuNUw!rJmHzMMoIkO#u?~Mettu)iQZY3ZkdTl#UGq{@+$hPg zaUg8COQVjwmzcqmg33yc4F_Kog$V6Th5^Y{NtoXh%(SStHo;B><^|`B&SagqMn!UTqz`#&|!|5aM(#s*v zAJv$DuGr%!O=GpG*CGlqlE>i5UcL3_932a9ifVt!cCkO2b7495{AzIUe*cUm^^s$n zNHOL4?tFkA)iqScWBBWCT5nWJ_SDo=p3IBWS!qGIF>V0i;6_omqB%@+f zGoxN<`(H~~`8k#?TgGJ9{`r3VK#Pd9)weY?Do!0=R2hVdP(&9|(Kvvwi0B}mp;$Y=d+X;F@m!zN)v63IPvnanmeHgOqPj}@0L zU7Dyj%lDXaHnXsZb!>R@`+g~Zyd@JPHX3g z2M-<`KYjWd@E5O=`%0XGj&E_yBsbz|ldj7vDDZkr4qtllLPBY>LnQ099{_r6IbAC1^Kp-7f20 z^F8g+r`(HUU*4RzDJm|;PZT)z#_5uu-;!_NzdN*kSpMtvxg`Rg<3bsBbx%em@{;o= znS%KhFUQBPtIf1$Ra8_Y=NMJ)9g50k$JRaFb(zUuN@D!cdq<%aM>liz_3Ir7#YMY! z?1L%GQB!2SW)*JUx@Bf>AB<<>lD)`CZMCoGof_@AlArID`2LyBJ%wPv)3>+Q z?QiVqxq+6Iq-0G^O$J8Bx4z8WDVAki=ur`MM_OcPuEB`$vtjiaU~^1ifrYvY-b~Dk#lH{(Z{l>G04iwMj6EKOE64iXJ4@b z!S4I&R7G>aB|77r3l3I_wxTg>l6`CCxC4h2@{+*;&{9%cxBB?{`l{VUVoou8mty@x zoM7~we# zW{&B{K8IZQiI>xJ=yApCx;i?djGi9&neCFSHgofIWc5`4JR03&uy+CE9017CH!z@y z`eMp;n_gO4`hu%#)3-Qn0Fd&`Gh@BKzan-XC|>0H9>4!Kt5k#!N+)$eG3!93rI?mB zI^xbBPyG-M1!%+aW}4plo&FL#Dh@(*u={4(5 z&P7ihw}97ls(!=0DmoL^09I_4gM$N3XE6$6V{fm7B;g9O99mM`Myv@C>!?h;^^Mw0 zDYy+Y47FF304;Nx&-P~b^|u5oo8tkp(23{t6&BrzL^C+CMG)aG>p7iO$ng<1m9RZP zEmyH7Pdya5WIS~q?l`yD`;Wn%hTMzX=l*;OoqP1~;f<^;*$1)?wB*>~D9N1p#efWa zW9ma$Q4u}P{dd{f%iu{gXT7+&xt)4yIsI4#4kU<#?vXnEMUVsu0E_k1P0JupWHc)TBL<6+f@8OYZWI)xerLOsBpSr`qm)GB=u%8$Zl!o#KLO8cI2p0} zaasrx$$Q)FU+98t*!|b_)6+B)NgOp*^i0vUt=sHsx8AvPhvUs}?4OibA$`Z+zi%rl z_V;X2iWAGPetx}M+ukVP;wH&b*CkY9Vj|%R1jDty#%M6}DqLVYaNxk;AX?Wbv}UBz zuNpJ;npq@74?fJHxrh~5V$k)qiQf|Qni)B`X3d)8bs2VN&z;+ewQKC`3^R;R~{v3U6~N=@JTJ_Q919NveC%>?}qI+j{Ih4^(8Pr+#oN{eL^2CGb2TMLd!a^ zJyu&sQr3>32@~|LE;D%`@3PPGA*d`cmBQ4u@C^sW!#+MahccO4nt`>9QS)W8j=xwn zY+SjiAq6bp#E|36#eIF2aeX*g!3oWj`nXkW=`hFz&aJrTW!H2Zo2?vuO-BECTIo|m+SsfHfYa8IU-eI0~$=NP_rClZ< zvw9lEP>ihQzt4$=}}8!6IyuTJVr50nUK^gd0?%Fd46_fSr}3}i43g>KyFrza50uWT1qs=ww` zWMWUYX};(9pZoS_f#4N9CJoKjE~Y+w`cwqe=`b~Fw`T2H;SC$KP1c6)!O3*!t5@7= zUAZUG{>YIdGF7aBNdf$7PYKSb;p7#=p1 zFMg4ly7`UM-^)lZD**Ek?3h7pc^GAE+p{BXK{lEL8tWv^LYYHOasm*2nnt}J1Vn3M2P0c|5;l5;I8N|4lk z6wmUppw8`6AD%w$+oGeX89FzIqBB0(l{_$1p}gUj;vD{Afig%aHX;-x=g%ItH09iI zAZ%`~=G7@tp7y!}>=hPjqMbu$M-M(3)){Z!7QIT@WAXnRJ$KeNd){M0ioU?)`RihsfPJr&~sj9X^n3 zas%~}hT__2OhK3kA8BrB`HD7*U?@;OBu0My{yorK%uK&{v9OdBTW@c#apsw0my*4V zfCYr6cyBheM}PUEn_kfaF2u0BTaVqVz5emXmV1vLEu*BQq&{Ryiin6%$){#Ne*Ace zL~YuCr@N#7f`R(CuX5;$?eEFh@qGLCtu?=_szrhke?cp?_0-$TiwN3Q+qp9p&nm2A z$(q9-)%`P71T0-sYpV~4qIP+<#7*{VnOa3xryAhRtR-5YU-z@FXIAl7e(^Z}9XDF_5Wibl?7hT*SSxhj} z#UNBt(PV^|iH331SR_4t8i;DX+vmF!-mZfOxd}y| zpP!#IdtXar7U^(8uKeYixXX}3cJ@%Sz}@{H*F_!&fBBEp#Qb}=<}H?5y5x8{XaRZO zzO6?Eepmib-l46O)AnbIs-3=Cy^W=6a>e%WssWILIeWAlINDvW2Lu#yo4Q{D4*Fnv zB`G0$7v~z_`GUD84yLYu_D9Ze99L}@D9eNWtse-Pt zB+%z*3a7n=c3v&`@x#bYEmZtoUP;J|-RHg`T`=`h7e{R;`X)qGKB8)d=WO*jYMFc^ zhp{;6Lj=#)@81pftY3d-DkorQ9#VE3d-s?geVLnm+e~3A=vC{k+}9azY!!QN3a;+2 zcJk0;Wo2EuT4rfi)$?~Q4IX|pbc;FMq-K$v2#Z`;SlHOn!QkcPmF|*bRd@f&$h{Bp zjhSbg)Na&%xVe)$UZ3MSs9Ftfg#cP`f<6El?k594MWQj!mmk)#f8lb>SdHNsgrb73 z>?%W+4kn?fd9yFb&cRs``Y1^`!T#H!AIix_)R&8jp`p_|q^zlNO=0#5*Fokn0zf@#KS z_pj}{{m`{~OSQT~Z~f;A)HLmB?X>AfC9SO|(yYEOCMXY`2a2k0{60PiZ6D_NL9({= zH@&k?)>{<%FtAwD`CC<2?qiRU9|ysAGAd29t{~h)T@#14!;7}!9eqsaSPqO>I)hhr zz)tRcD8~sM;T_O$Dkw@Sxk9go`@Zk7+p~5Xdee35NR}xXD@HR#KNG8S}6W`pwZ19>hiE4xy8x}R>DfYLLEjU*&I*1Dlrb3Atr+{aVH zcvLr1?+YEc!oor#3-e6HKDCHX)rEumMBg2ryp0+h8vb0fBcM(xxjDyPh=fJ;f<|3` zgq+I6L@{(74wJ)w-Qy=ES&(7|B^NQB+ujMNhV;UvzH65`67C&YhnV+vEiEl=jEo@F zE};kX|MkN(;=@VzFF$@1XEt~~)mwc?e;2eS&&j{*sDj6uw^B}j6^^@9gPvZ0%F*Bt zng;+miXuG>aPQ~LS+tqj+r(5y?_^m2(2A8*Bi;X|h?p-M#mD~sIZ+iSSuDr{o^&P!E8wZDB z>nQTghDwe;kVBM-^4cW!q7xr|As!75g55FxTUPusO165i`GLzzo1`~Bl6R+Hd(;^ z#fha1`zWaspe@q@q9_m_DXghE)RGY$9X)vdgcnr1aF>AJc^=s(Xg`-=%V~1>pY>Ed zzhCPH0fX>C5~g|AQ89hxe{RjyK6J~CaMQZFY>0-mmyg^W`m&tSTIlMAxc8fe|N3>* z=jv73f+JqXz|07NrT+T$D`KPG>W5|fE14gO21`I&?*aP`68lHhExvqpVZDofp7o=? zItkz2zdpj-`KSba6D_rP>B=Z)&70FN41jz=Al*P?SvF=A!MnqeUOQr=nBx?o^!{p> zUCyJ=KGzdP$iVF(3<&_SP6Q<)!MnoU(UTF)1T7pX)8i zo{%2e(b2IVpUq2uOs`X)%+cmxSD6qxJqJcm{1y{-7!@(zFn>*LyU~LMVz-ZHttj;G zuNUq9e7gJd^|^Ihwrr6ie9`ahgFW}6ME?RW4Q4w(HF|m#^(IBP=hWwWzF-mrm6Z6A z48>DC(ImNhco392ag)Vzfce%<-DDHRtgNg?R1)KK>+tv5veev>)dz$_6CRGbgWwWn zkC*fJ_y76o6sV)F#VAgS=MDJo7unrdonm-g$4E;RiJa;(sXUs!n+$s(ZrpI(WpqAs61Hi~N?= zV&|(9P=MRz<1*GJ$3uoT$H8S_W_|~HxM9*pSQ%7txl2;_WsXtivdbrb0^RXJ<0kl@ zuu27W`)$7&E`$KqDb;o8Bn|+f8ehD4p>-YwDBfYC>j(r4;DjPDaIk({QA_uzYu9%G z*`&m+J>(kZ3Rs|d2vQUnOi|Ore8ynxLeZwtrl#Nr|kc}$BFXk+D?)@d! z*AB&BybgSTKYG{FDd1N&5UE-tgQMXgAyF@rLG&?Eg`Ym{h1knXnKu>sYK@qio2x@i zTSNhjuIX*aPo90do>nL5NI1|cI%U9>#P0j{dCMuE;NaleQS37Q6|9?n&71RG^i*_g ztO)T`Jl0PvRf>s;DS}MUnqN{>^dY!ysxSXwQeS)fx;5J- z=^vj#E5}I1z`P+WEp3pOi~pC_8|BM9uV-80Z7|tT+vq7hRQ=`r{4ZXb?AsRvO+{CZ zb*~Q8+0;`N{OMEu0|U#cBE(xDM=YBf5)B?342#)C2tTMhHx*{lYl(+71p2OR3g5E0G;-z2 zmA2%Eq)Ga_ZQ}cW<&RTGQq6s9J^x!#*x+d=-a91gG3i930)EXOpmcp$?u7w0Pl=@L zJnPAO$p-O3lpwI!8u(nI4A;raA0hj$Ri`~QqF?plQ3U`(;`4LpFgF82;!n5Q%tQBc z0{3#x;Mv5zW??|$C=EEbNwVDa_Jao-ZEbDoDMA&+jcr@E?*ZGcUAqQH5mE=&H@rF( zeC@^!qgBu>T-y2)(cj&x5HXMmt4!Ao|E9zpIHeH5?K?9SF_zl|v)GTkcEdK*Z4w(4 z8|vagH#^Hyrvm%(ruOkykRsaA@^$!s(~LpdZv+}Ks?yoFj|=TeLtj(L3qhU}zdp5;=lL>lr0%&1E*rfjp z{8`aU%QChVD{k562Jy)^BN0>Hz;?vyW8LNE>KYAul}5l4U4=8}^X6tRc64-bxC-1` z31JY~%Lhl@tk8#%lvN}VLb9V1CP)Xp6+rQ&;9ypC^|Tc6xq;qHMQS6&Gy094zzRY> z1w>mO=`@m+lWRuaVlpU?SX1+jg=h+iiB^|J!2Sml4FLjk$bJhJicij|07o?iZJUDY zApJJ$O7FZpC0L^RXT}=J>TLA)@7}!x^w6&RGInQM-m_e|a3Ko1U;p6XWjH>Xo15{b zSKvR%xiCPB7!v!MExir-8m=so$%ngZR%}1>O9e4cBu4j6xA3TtR#My?t4s(QZPh+B9MlJgm@$8#DFtFJ6DK^Bklt9!34qNhZbn zXDS`^ch#dud7*?eLe-q$_yKK9)}@yhJz=ZMms@UlTgesL@*M4lO&$!P-(Cub**c}^ z{+&m&&eCA9RlspVyPMZ3P?#X#L1<7%r>#%8M<(!8@KOrFU>1avjKFLJRZZk*@J>yQ zjif6Ko3t=KD(bLlV^H&hz+w;nvZUr|4|&gNS*59NX$B$K6P6%ACt9ACqWz7Q67fdI z>N(NIeVtF93s2_5l|@^CLYE3d9i%B% z!g%Ay5Ld&*GYO~&#Mm=EJ#AQ?r~7NPAbadzC(iv9#EE0*jD=DKscwXG@}cj#>rl!2 z_q+Dq+o}n$zbDb7eihx|6fpnt29L3a7e>49)*5fUD~4`!cCt&-H_zyEm|(Vz(XB}E zpPZ8p--oj-l_5#0UHkS>O!YkyH;&1lsiV__Ws*#GP=bn=2)IYR^u$Rj23+$&;ij}a zpZGjIm|QxRe!lC#jXH4bhSYo&mJLE)*uZxMANn?6t-qf^$lSuB81{kIlDR*@u;87p z%Xki{;;LG9bN!CzR@XaEYK81@X!ap$@~VqalPk+_4$AaS&mwCtW+ycG%t+Qv(>ehhhIk=Q(>mCI#?;lDMK;Pa) z$3K9Y&Ae`1ynApQO0oI7t1QBxo)Z7H{%zW%gGP84tbb@}{DJoVOc`@z&SVcRM9AYk zKI`L-4%j+2b7@Ed&QvQ(K!al=y_)vN>Qc6du1q8Gm(JpaBHu zJ1COyq`)5~A$2v!Tu68MKyLfR!y`vlC9+|8W1jcp2H>f7Z4xg9ShJ&Ss0NwKHw4t%B&;!EKO7 zgpLFOgg{3is@-`5XTK_;_k+Rsn$T#Ip|eWMYA2 z9a|Y@q3`IwzA|nn=HI@Y!UH&r_nQ%qCukvT<}Rqvc%+AYBb@7}ZDvWb1bu>Y3Fj^C zsZ*!YKqs_K{;RFIai{g2sO8sCuD|kQDt^zay6osOq z@2V3a)}79~zTWkA<_xl(*@+YMaDhSxDS&yxljY!fmgQuMgt* z!NZ3{-Nlcw$zdfta&#%${=}gY_|2{zf4kRQjB{uGejfXCSiQk~(GMxwD&jYG}9%jp5Sq^mI(rTCa z@3Gi)I86oZH}p1iXKttEdEJ5sj=FmdD88>NUT88)7uW;j?ZpuP3LqFD+v0?Jr>Cc9 z>=c(qNxcd#e;{zHxXTIYms?MNq2R`?UFKJBp}^f>?;bqhJTPzW4c($>w}cQ8_#zbx zU7iob22Zn)-5`>1J^~OZj>jWQAL8hf-Rr2&&Cb{vJ;qy6Jhr;QB1h4EUhR3zGw&2c zTk~IEG5XQ5Jrv0s|MAuFyJ(&n{jX2qU&J}*IrsnkR*e4J|NmqEpI8vL+GS+?r^CYq z9vrY0Z{NM6qg%WbhLeS$RJRPm#6^zl5Je#s_7Jfxv7CP%@ zoJp;^^XJc-ojiFJ*ee#cCj5B!ske!tw{FparK7y_^Ob8e?4JJiPO}9$Io^6}e`9m= zSq`$?|zhPm6do(ePLsWS^>wzdsiDkcz6q!Z#NugtA4iqj?fVJDzBhfHG`cyed_#dFx!Eh+&+->}%s(sDE0imU^ zIeYKRu%I=kC+1ddOa04iv}kCG(I*eIeq2F$gU}#9bjgw>H%2*^FTaF30#TR)zmGW5 zFuEds;F)np7P^odplawSSTXd#xxsYKPDw4eJ=RyUGBFi`TcvXH#9BVr2m;^?fEiR1 z4m}R0kEzUgia01vu33$%JFs89B3{lb;$Yo&2kl1@w(MKJOL0c4Wm6VTaojNSobmY zQ_cvd$V7YB_3KMR!^1`AsUL^#pPS=kBHai5b1H~$DIBEx0g4E`qK@6J!A7WUxbHsjr};EL9>h78(8+ACH35LTdA7RTz#5{R+OK5%L^}Di!39 zRy%a$X9+{w2Y;g)&J)Mv$k^Btir8q)rf?#Uo10$-CGqa-*X1%YGUS*7{P*`a?fmp$ z#|qHy<7h^H@MpfcLnDa5C-@7rFVNhVSEu$vl%<2Gp%~2HW;kxsrT375DtuY(+_!HT zvF}nUU|zqbnC*;{Te6H-#ob-O-Q69p$8_=HMKYiV$NyU-wZ`^#KWvg2nqMtD7*emJ zK^KI96Pf}s!^4S51P?qEfwK5PUu5f%HY@84(w(VP;|%7)L1q7 z8e^J%w~b9Yo$uKL&UJ4=I#YFW+6wCcGaMrrW1A`p1%UE%Luce$V!OkE ze}_o;0@2#_(8-f)C&3dve!NCjUfu#UQ-W!Jp4t@fEPQZku@=U!PO^|`D)a*8kmj|@ z1W0^63Ja-5j_n^Gu&8`-Y&9}(mixpOV067`wMDLc=QSBM6i0Kwm=6$d9{jz7C+P3u zREwZ3!p{H9au6Y2BFQ7*z7{Xzy@oBf>F+>@g)gstltjEq;-Nk87FBTN=tLYDdP;ug6lYGlX^CLVsNKvF|htKAg zWTM~-etwAt)B4SebrR%9b4rN$H@)ZK!-t>&AjyP7N@YZ;x>>fJZMoiS%;?}L)|(O+ zu!4euVMMxZjsC1x4c`*DD>h#T^IA2BwkY-FXPz~fBLI!Y)MJSSOpxJ1%u^~CF34iW zfgC|OttsZ_{n6`{B68VBSs!n#PeZe~K8c)9ro~ zyH5^VVY}7#?`I{6uv89C_+8*k*?4%8GQ1WlUd%V$e-S}88!g^}ScP94eSLW`3(s*- z$9J|^2GlBHlHwW&L*iqCJz%inJDLgFsA<@h&K5E5SEizz8ioUYjD@r232t32`nh?c7zR6a5P?4Vh6q)S^QBw{amth}1d? z?5agZ??FJKh3N+|MCVCRYV1Km^DP4*LvE0S)qPtVK{`!wxs z-91`b`bVBf{2}fUij48X$eZ2+0=?qjj>4@AU?@JteWwY;{}K!*N+He(;bO)}Ly?}6 z9Jz_xW1Tq@u7@*n%)uZ9@QxujVfJO&h7B7?vZ=4Hx9)=G69bqid!b(tNKbUX7OI1H zP|z}9r40h#KV%o}^t6~ii%dM)R8-nC4U>s*WbyEA3oy-Gd*8l@xdsTTc8Y*7&~HRA z=oYPG0$4K9`xJunPn;PAF;$s8g7-C4N0V)*AY$X33fbR3i2hK{rT02g#hrWiV!#x` z{dom!2Zjk~DA;J;K}3VI?Z^Aa5+`dm{~5b;T7TaH7#z;HA)w4p@+i^9h;3|*Xi-T? zEPlo{7>nV$C&?aY>s@Uv588do8|bS5W&7{YFbFz|!c4hFP!FmnMPY3xHU>-#jEsb@ z6Qv4}_?B~X`+?iQus0KYt_0%1RSnA@4az-aI!c#cdwqFoyU`b8;{Jg5uQH?E7??AB z-9L*`HYIdOCM!w{l@8=2bwCQmTsP}7&uNqOdE&w5HpIGQQMnQ1qr!$#5BT~4a3wB zGdJ82rBtx&m`px^*Ng(`0_D$7T^K?_u$pP{4riSY@7aCWK&9)+4+)nb(ZdRga$`fM3rs!~SZH(^XQ0-N$fZgT$1AqU}$qaMMz{ zPUZdJuZ~0q$E2jDzJ(*2xXehtKpOZwc9j--CI-Nfp@3m0j8(A@Miz_+cu;3I7T&4# zSd~!z0BsNyZ#~z-`2!}t{3UB2+8hvaS4-vQ-5dMA{)7`xxToavXBOzFXryK*nc*7f zA0GBWGR_IwNE03PnEud%&L;*6Al*^A0y)-E0AWN;09YafXQ>=Ojq2&sr$fi)2g|W; zhv8dT+Boui3JGVhXOYJv06{V;Xc%!G?r;hh7Od~fUk|6H1C($w$*_6zWoaLAiCS zTj&nZ)A8EG3Yx8)~NIr`#g~7QP=sfcoERdVwHjN#fM&Fq5&p&Qnr0% zTtD`f8$)W;7uLY}*cHTksC6X`1_$^t_(DSwjEJ|@TOSPEh;$v8{$Kl;Vq3EXoG2aT zHrD%GckPryE+P)})B3RD;^JsDFig@LTDP6}z=nx>HHJ%Q;GX8RD8uR+D>?f7*|W1y z4#=D!M$JxFzu*J)nfUUB?a|{f=v+Y041|`Hvh^~EMKbY&Q9lr$II{NZdsA(!VQ{2O z)`=1WS@cFlMLrP8;6BY$N3kEnGJhBDtZ>^-ym+V-CBXbXC|<;(2?xRz*qn%y)!Un* zRA6W%vKP0FXljR`?z8u z2ddj^qbWt(CNg)7;S%r{v56}MssdGS_L_C~bvAV$SVXjb1V{`_*<>h(mzTG{Pi4D zB0`eO9iTB>g^}tXI8DkC4im4opm2<%?|h{-H0P2y%7E2`{JI3W!#L0Vb z|M(J#4+C8()AtQLt5>&RMM_YaQcirlDcpBu0m&t-hl>%(@ySd4UNWmdqTPL!#wsKg zGLnzazehFc$KNe0gNoS?+X~!LbVOe&&07r`0ki#s*L*OU?ivn;;W@HL;n%@8~U zivaGRX~o?vO9&pfV%zy1?zcQ;a5=PNP2JtsF=n$0mz+SxKCojCG5;Wq(ov*Y%}7QV zo1?*#bsJufN7MymCm+#0VS~`EXXX_k zHX(wa6S7+99*9LRg>0n(5+&o^!haCf4}rG=z6CqKf%?XAYT?I(YZUVdH11`z^zL4E z4Tacx2m3!DU=9-tmCg)(oI$J$i>Pc(+E)_$me}2D}z%BMtf0)sU#)2NDcA0O&FR5+Yh%|?kkoM2Q==md) z0;@`fz*CDN3%1oz;;j4ptxrLGd*;Hfz_dTcXB* zPc8gSCp8Ax17T!19CS6oy=h_hL!CE^MPDzQQq{ys4Gd)qS9Ud(=5?A6mzcqgeW!RN zYE%qm|B0+`5}pB~3$gL<|NdVJFXETs7+Q(cf_z-NOv3^V5P?3|uU9&+|0i_%$Ur2# z6UTAU2>A%^pV|axI4(Vj>d~FA3LT1&b;MtJV$=aa5hq|^U;rq+MEc~X0&H&@-`a%; zULGC;f;PN`Zh@aICVbD(A00bC&4u4&4MnY#%yj+2pdWgEX~H^#=dzOjhY}WcD}#_c zU#-#Koc@)y8IKf;eVzZ>vuLfT?p;WZvJ66kKHeCx0DlHpzZkt8i0EdNq-y7D|Gbo8 zDMpOiTIkdmU`lBQ?@5L_iKP>)u$|wYg-?l-2jYb24Y#yA#`%dzOf>?=91aSY|CRMd zYS-M$;`FUjYgg!fJ!EW5yglP6c@#E0AxEske$Cv~Z`nTW^11sjB&WMDV`OGAj2Q&33I9D-_1oDE+mm<_` zIJDi|OtO zbe`M>gGVQ<2H0FQ%uzWTVVp*>=R*c0%mca|GZ0|*Q&QLJy+a%8MVmFvY$LjMs5n z+ojl8!rp<@<|)~4$n)~mtIg;B>;-AC3eVPoc`i&TZIf6h{?G2sZzFf8;PavWe!ysc z@iGvt#pn%+@JId3ncgEod$Y-PN;E3hS8v~!rz0L7fG#o+Pv(z7U?!@0e4qvM!nl@z zMx`xSU3fVnOCJEq3 z-^n24-2GDs0s3Q98K}*7wvQ+(*I-=?AQueSeh0@MX90KBo6AKF+T5*u7en}}^EaI?kx^_5=IVoktIg#eit4h}$5C0>ac=U0^AuV254&|tRG z-Gbqy8P_s}mV||b2-=IRuA~G>R|_0h447R6PP7TWWApUvKFA@u6dT$oV!alY zkvV`X5&ZM#W{(J^Uit?n`L0;KdINP0d6`MbhfK4c{{1Fh?I9?}YuHy4I{_R#a|jD@ zJg2OV>lfA)577KLp~=|kiQBt`F}x=T=R8L(?$5afyCu2&>>3z3)QhEnHybi+YlMOQ zFwcra?<)g#j+-L4sgb)1;RIiSmBE7P!`P!k5O(zJeND!SNM&y1I9Q1>x2m?6#27e+ zLPWyG!)bfqO;I8DoS_W(L&9SQHji5^hzFX#E{Pm>;#xzG8)uyD{2Y0Yo5EdX2|1r3 z>fjm^2!tm;-4WtfxFATzRSBJk300B4U`@v146L;Q>J@P-pOp@N(x%?VSmELp+3y}d z_HoKRkK4uFyKKAe>+={jCEg!gPQX+Lii$;9S(%ttaLrFkvBVOjaaobDBGJ!qCj^<-1i~P<5>W)8-{9G%pYR>B z6GUHe+p-TI3c#R(7xO9r-3LTqzv*Pmm?nr_C|`b%Wqfp!#w54h@AUzKUEyhdaDrpcMoC(EoL}0MVPQTgYu?AYsqqS_z`aBF+0k zVSvsx*ajV<0I#%}Fu3^n2nbC57i0V(1u2UvKvfT$tipKICAuYF^1Zx%*47ec65UJD zx=43gib?wusd$tWR8k@lV$O=pBY$~uOdf*_OX0=;GM*|bt_jdX77&YLk4wmC(8a@2 zB}FEg^>%>xBFSJOE97>dY+J&S-XdsH(Mcmj4}&V9C2*C58H~-uW2S$fPu@8SF9CAY z4<2AZBzWl+UJHtgYng0^l#c85kRV_Th28K-p8F0A)-wan5jBjA3a{UIYxa96#Afb-xTT5sxiH8_uJOV&xd>24^k-~%KVl^o_gf&$BZ66% z!lmHdJYInOL>$k+3(c_LZ!)w+B_y@JA6HtC`x2ln>LpDCLc}-|z!QtldzTn9kd(ixMCF1x)RcYb|lh#IHOL@~-2} zfs$Lr|F@#ecZ)sYS87_CI+Vi!P)BcJ)*!=K;CNcwaJSRj-2xsy2p~adey9vu9w$zm z_%IiV`$Q?U+#8)XB37Vekjt+S#9^hT;J$_EDM4JVXjYawT)g-Fe$wq9w45!Vc?MlXiRVX@;3+tv;0Tb3JH<;8*E@;5%XI?m;};AT*w*K)!rx@FpxKhN z(0zD#%%~MRLQ*BhYnaK1q5shC2-zrWcfr1ust)g$cwa7IkX2Ht6}}(Wg7c1nQ!&Da z{Wxga1SSQqv?$^~E2Ba~V2~!G;h95+4?{U*q{#kK1;pJAI|J?&YywC_pHELFs|5xz z0Tovn93IX|HDaK|H;7Cu5xp38Whp=$3<7K@X2{`B`-B@qvtnC5N5aU$;WoGB<=cuo zcUGpK4{*foPU`Ag)EF>i#f_T}Re&T15KB#n3#Rj+>BuD=ggC?#yeBG_B{9qUE#qrd zM!oQ;^HVWg59d4ijuhxW_ldUKOqRI7Q3Znsg&7dhaD2r^VlhAqKOWJn6#D$NLT&%N z!<`lr8NiJq*j+Lo4FO~Tlg&iHW4HvpTx>*WwE$EVgGqPO5U9vzC@Ms(g<8)sySVhj zdj@c#clyR*k5`EveSSt6iCynd`2fwg2;c(bM;3L&7&=W~bz!lT2MG#I6Z`sBJn9=( zYML-{+N%}ixB=k+_K%u%IaNR;A0KiUaamtfXhm`s5y{3BPCLiek_~7Y3Ra?Yh4566#?z@E`OK8VdK2 znt>sQzhE&b6gZG}{LA$6py3e!H;vm2#>ZQNNpJ%$OdMk&vjPBt#Ebx((gc_THh~@s zw3s4S9}s{`EH*fhkQ>O&2bf|@{`^kZE#4QCisKFA=<$fpoLp5(s5t;~;)Zu}GGX=Y z!B}1*EAkxa7r;RP9oR*WN`~$sv$N#NWMo4?ZcZSBDBXBKG|EJB(V=_9wMmJW5G(o) z8)hpVFl|IUAr%<+?!E;ANyWG}$#Crcj|&`Q2LGeAyZ9A1S(iQp>x?05a*Z>wSm8jy z$wNbd7g|gaX#y8BYdAV?AzJ~AM1r?E+4Nv1>|4tKP8or&zQDe|(KL@2w|F0$)5#w>xX)%t;r2E5VWbnh~7E}*_G#`Mk zpb1nlu1=0vr;Xb&2y*!e;{G3^rW??~8) zo2lQT+O`?2aJz6}{x&Bp)0g=D0={ofo;RMBEb!4+K7z?3MoMfcL2dc#xn*5j0Rr_9O57=gQLD2mwscX*M|{} z9@~+H`=I6pAN4-Tny{)ZFRc~6@jZJ&AHsilp-`90s=XqY&J#4K9?;tux zUcimvPGx0iV^vj3?iNDY5SKBnbqUG?n#)1@H8v=TT4d5BvnM9KDj7T0#fmtfKY+T~ z`5MvSNXkQPbztJ1_J}13MQP*rrzbf8WE_kA4%j z|KpE8$O4KR)q=WCF_}TG)M1RpBBcto+5^mz8vgvyi;Uz5Fnf)I0V$~$iuM~mI|QEu zlNqMz37}rTfB?z8#1A?lw$e3pi(xp@AAr?;Q6$BfLRY382b6Ki-F+Gvjew&+;=*47 zTDAVm{mMa)T>Z56fMYM>HAvBfZ1Ib^@7_Jpo*=ED0Ll_IlrNFCVL}p{#(;mQ@3)yt z>sCf1hlXRD@({aJ6}1p!Wj&$x3f(C~*9Hx01;&6ob{4ow%*go!d^b`mh;Vyo#TIJ@ zNqnGfaYIxw=ml<<-=QQddsTzMsI-%j#mG12BwYZF<{y+2P~mq1pI{=?A21P6hC%Dx zb3l1uT7JXHef%;q%i(P43%+p5av1syQ4nNfCxoAtwg-Xf??sC)!H#c!<_}8#yLR#I zUUSXSaWi)SZl+u9(kUx8{3Ny=^B5u27oIx*-^e3Q6X3mlvLKIBdp8~hH665C*r#J2 z1;T*VU2b!jVbD?j8uh}WQH<%vS6+_7VG8)x$Mq+EtDpB!4>3mQcKD&bu`ESCua+6z zy#M?LW8qAE6BxNLT5mPDTCjF_gBc7`>`>Zf)rBk>9zFOnC*mHas7Hj!Zfn|<-~Vj9 z1zj3J-EF*O_9Z?(z0<^-e!B6OvG(nwR1U%IDq@!clBz-b_wSx)5frYF}sawa}w*p53_>+%McWKiD8HEsYCU3f+AcQpikjlOS6i6fW$C=zwYx zl-qAmfy8zzbx)s^6xyMO@{!v*+y2+noYTmVL<~8u(@aS=T zv4%MYofnHyW>H~B^JI}y5gLTy6{22GbD?ABYtdT&^7XbCTjXUvTQ>?CR_Gmh_1P3i z@i6b1)epuI`ged01XXeo0z7IN64c=PaX3b!UNF&?IDI@Q2z?K+pSvgCPpWSmcZ`}uf1A zZE!C_X~0qA{-xm$Z>o5%Mrs!tA5u7WYz<15w~+_e(&S~# zxr}2zfi&^J?%tbx<8i~35nWH@f0=1(j zvmc~j?8<|o7rzP+MlLYLICV-KSQuXR-xIIabtYxy_Jh=~#>adE2oi$5#c+doSClRQY7FZM9L8N4`}jkyhR#2@8Akvm zetspUZFAmCDp){iFx&uEA0RFO%+6-4My$gVfJ88x0bU{mtQe42&EaD8Y*b=-Z|kOE zsg1w=n2*09hS$5)W34Shq^9mf==5kS*a4yn)F>ANZJifj?Et)nCuQ!hx&Ywsu;)F_ z65VIpX7lLMg|M(NxC^xs#GC9eEfC-?1hB8qvOf#G3BNNqD5;);N&&?Z)MtH7k{dRV zDzBo^S@y2f>kM{b} zr5H70N)C?=yJ4rmXoX1a#0P~o=+Aawi{NvhOF9gc^6`CZ%vcC+Z#(J{@`#QM0v%!E zcVnY^S_}w5>NiJyN)~aL2zHx$^J1C!G{7?!V#kYRqwaD5sF;TDV&8+1?w{9a`k_1!)ijLnd1K=W zEHX7g?~)+$JNTUZF)%>>c!YPRfBCjA(=GA~h1F#l z?49nQkZHI$5*r0y5>Qa0Z7+IDLFk$MrN92VFvs#97SOGIcmMb)7T$9-WiNou>0dx2 zO#I^_yF6&jMTN#{DT?8TQl0Va80YR=s#a~D`6}|-)1`}WA(2i>Xb=VO3JU#C21x6d z=vBQ29J`7^sRiA2$j7{lBiI5kY2dz{5Hq3TXz1o&)8XPkkbi3@*mfa1dldp>*-IJ) zwq{MmJDJ;etUD|CXs$B80og{Wok%?&%`%K047Brc4hs**;SVzyM=%t)bu-?0V3M?N zup(_!Xgk1r|*ON1r3S4@f>xAIzN+f zt1K|&RN4i2a~KRD%OfTxnys4$z7yg`;?!DGDTO-Z7@7y(fmWQeOLHSp++a(rrzV}_ zUTxns5MNMQ`WmGZm8n#Hpqjy8n)NU$G8kwQh9I+*=G?*IPgHN_Sf{!?&_$?W1IobLBx5Cy zKIHcqZcgXh9;Cg!jIHG4clX?K?4;Hr{Y5o2>kh4vKE|j>)>1%7(84Ssl#qKLx(s2K zC9vBA5I-~ubS}3V%JkQsE3e~U?7)lxeAZnltDrq65#z1{%7RkI4Z@=$4-m+@>C||; z9$XIb{%E{(4E-cB7s@I&O$$of*LaDdvnzuA3?z62duG#C033~gjdK{tf7E)Rv}FVC z(Rp5s?fK2#An^SwPLWnv(T>v8;ve@`#mTbN!FJRuV+a`k`>+D9!2QAnc)bU4U9%G!)V7<8_eZ ziS|IL`x}F5R}}O!dqzE9eM)g?AgcRQtDAsIu$kB_M&vQ8D|_Fr-@WYIJB;Z?n$h52 zj*%5YRWSz7p1x+yLD{zx(E~z;Je{zDv{1X{5ikr|!lyw8`@NUCtt7Kg|XYDu%>w%%>|R zmKe@0A2v}KK3`zzVm%DIL~}v0&?rHYMvVo2Vujl0AG%w{o929bm%b-EPOa8%?%A~Q z#pBei0WQ3!Lmaw*)p;%YJ!>d7jGiti*(lett7eJL$88rFL>x zT-Q(B^udsjAdHs5&Vx%Cxj5AcM8B2}&Vf0K$?Dg~hbVM`P=AR==w_)X!VwW3T?AutR3IyDyVX7)@A8qL08^ zG$Xdr_!MN~%OGVw;^CG@#{D%NsYe0)>rwO3^Xr}iLkeY6MT$GS>{VPugw}(6EqKpT z6;Prk+O#Pm=)9GQ6K&+vHdxegY7bVcu*?V4K@4IO2L=>P-UuUVBj&s$E&$u~D5^#e zMa6edTxJLAMTh7VO_h|^B`16m)pC!J_~i7F537{x>orX)yU?%$nZ7e?gigNZfBm`Q zq3q^~XI#NvypImrkS7uu^Po>r*rhDBiP^Jf zehOXvz^zMU>_p-4odf419V!HziB)(7%~z-``GZgOA++`J@eyqXHEjk1<7nFENHMJ8 zQ71Ox+~|N1rYyXLW{sE`*=UNAl===ofn!6}@B%C<2CzT;>miH`pr;?(bsfg1(8PSz z{C&IBSN%djiRdPW(#A@sy)x&z5D$@fYh#ZgO%KDspDfS z20Qn@OEl(<*PnASJbb3yipGVTEHt#PB^q_4y~{W&A8#;kcgrIKkF9c88Ho0If_|(y zG}jADcvJMyBScN=K25i3dW|T36kc51)I6$OL&-KQ(u7q7{k}#s!2hl1GR$wSleI4k%d4}6@=epi!R=wLWapscz7Z?%KzL9@>fc3tY|mFO+0 z?N2;fqG|adbs7Jo*L}|C&(-8sD}`D|efd4%^4Z?*(iy|16PRt0xO9{GQ=ymd9-o$F zPaaua?)DCH&z5!&oCd=)DV)Y>6;UTM?_rMf*R@l~K8Wcu%rj|Q*VbBk?DlMH& z{%GigwAbNRHg@mA*-0~zUy# zoN;2)fxSHC)h+rLMi&0;RNq$s$gs$SA3nubsFJzftHAO^C&c9>zCyzcuk*m);07KZ zJe2oLX8tsWxW@gt&#V}~Z#lclHGYgcdnJs(8!9B#YrfV$oZ&L%!2M0QL0Ih1+rIZa z|24|qasPa3;>m8qKL5Y#UCU?5$`3EenMhgCG|`ij=o%{?d*3sA_pi;e=D_LaTZ9)9Q3&iHGL9otrZ0sj0c`a` zD2?dfjQfUlvx1s64j(@3-i3+l^F(4U)VirhAX)TX`i+6g8;4(kZlA#j-@!&E#Qt4^ zLxc{#`w*ig;SnW(-}Ri!E&LIcu%fdTY8}R3yu5~=pKwst)vJoSP?i?Go{em;8Rmlf zauJ4o>Qd-CiBSFu=@kblW=wC$ak~9G1I&v=#blRo-J69A7`boWMV31U$~sS*Be8J( zv4Pbd;mf{A_3XN?c{9;|=%~jivf6HM0b@ z+HdEFP!le49OPoaM?hT!s5a&LeCPB2yK;$yI(UPIl! z=*$fRA4n$PS@iZcC9mL27@)j}McduUuv=5<40y=OYeA z*woB~#g5GFupfd^XxUNWNK3;xDy#B^Pg+$JbKx64K0nfQ)${zCrfj<( zCXG2`e?81za;VkWV2+Ek-UqpoYrk&RZT>B4c`|DRv>cXY+n;X2l4y#zp}8VAv14H}7XlGZMyh3hFf0|RHMzedD;jWAenQccr^d&yzzR_?Fz=l{Of_BZA*#3L?6V*_V!0fr5J)25;K-_b~(<3BBuWV%236# zH*ef{jF?%KU}*T0xCM2$7RV8P^tof^@J0~h2iVXI^8p>h0~7;;dE{F;Ke%>a&+#}( zHx)gUG@ULrUs+ub`$GY*7#~8?dI$nZ@dwq?SWQndGhZ||W|enw20eUs{ov8-4u13j zRY<|9{V@@q&~;#>dNi}AxW0#(Zkkq`i&yfO8{yH}&MoP4BX!z$i3dQMY z9gv32ciS~jclGG3?;jh&9zIn27~;Ma<2EN%4Xe(NprLUd><;JRUP7A*MP*DUOHyxD z>e;>x@H|RipF!I_N(`;GY4!f|VPvIxKlOBbdYM!UdInyhT<(R%&Qbj~oHs@Qkj|3W>{Qh(o0vOnrXZ)6GqqL0;|{5AEkesp zEES%2L7aD1ZyDa+Iecth+B&-CGJy-;#$S};-!D{OaqH(M@z6*a8Myjk z27*F&)%!Xu`6W1S8dfuOxjgHjBfg;j75G_VzUvOE0KrL#?cKxq-LT>}?|zpZvvV;E29w7db7iRH$yIaq~LRxYE*^52ioI^wQ+NZC;Jfm?NT($62lYMtY7~(T1xBZ z-TZlTf8F9Gbm;i!xI#>|1jRzJo?~qy%RSCm)GxaI-&@n42U(o?7CwTQG=a=eG{uXq zO5Aviw8gSog<~sU;+)QP=*Xp0d~mP?naRn&a?tUA{aV8EoQ@_wVK^tyF2P`I#8I+z z_!qpM^uc6KN2cZ&-8(vFGss(mO6bw)|LwaJ0>z>DAF1g?69Hkf*1?~T@8|!k%YFXe zoxgti(Qgkn5QRjFwV$6~+RnGv&?W<2gw9k1H?RZh{4@r-0ce;rjKe6A=0M0=A@8yR z;r${84EsJ5H$?pY$44=B<(`a?hYx393;)*~)E#x(KAW_uWokmK*oyR!0 z(hq0C(Huai@~P_!>#JZF#OhNMRARB7Bi7M{+S?BuOFbxnbLx<A7(TDY8C#nZZVL%qt6+#W7NNO}k7Iq2h-KKtVoWAJAr{sHr??a#9 z|4yk@Gl{q0-12)Lhut2a`ftIFWT01LI*5Hz4p2#=p(C+y^FOtx)(o1h5Y|ZZ(0cJy zu?%Qi(B#%}(CJ9MO~yx?=i`}A_woaZ}gSBD+iXi*ob+NJo^>(eM15HO@J z1(hafn9p^|0_Z%X3Ke+wIrawVEVS=aWJA-jyXQQCOPsDC;TV9)QXe0QWknS{=KzxT zVgnK4_;~h0E>VU!zh>ZT?RTOtNf*H z`3chjUM?CvH#X3D5xJ`c#JcdA2KKjyTgU+n6AqkQbYx<-GrDsTF=x(OwJlP03Pi|W z5a49txyhFM9k>;m_vON(3pUB2QG{&E1a;xZiubt6Nn!gI?pGA-D9q!zi4pHvh_psfZRa-7Y+_)iMbM*g z%%41(CnHr)p+5%O=@`^^sxS782`L6h$)Zo62y8GQ*QzG#Jb^<6Y8`AHv8`}W3nRCt zW-$ixLHK%V4I25dCD0M6_olxsMs1r>Jg<}k>%1wn(sfMV<|aFhl|LWjc<6LL@j84TE|>0ePN z@$)4IMsGPlq=&Qa=n>rao6DXz>LpKG0J8_QD13z%po8cHMHF`onyW<3v!MRag6Ok= zMb16ckRyZejM5wfB<0Vw3B2SXr-p^2)<2QVXXioi$E42UiiwUzV1W`TA|ZngVOX`w zBX57|<4iL{du%z%*Xd*SXU@FD3jWyAa&gW=ezJn83PO8-5fVD_*la@CX>=q%QuF=p z11hg9`X7A^u{dWfzaY88H7;q}`d;qu?;YHVi$&G#?ROiGDfT5vPImP68LzQPvb7#4 z7h)vjPpfzBInRIf@vVld4tK4S9>_QCNSb#NBVr6ogzErU&-#4yx+QegTdGV^aUr$%dY3hf>^eTj-F{LaN!0JF;m6fH?(GjuHzch{$k zOpedr`Z;8Ub?!8WmYY(GzfS(eU_2flQCg*O3U$A9`(2~HJT8gj7ysP6536tk82&)} z4-V3OZ`MlU34hD%&d%t4z1;R&V!m&{Sd8I>Fk*(o_w2FnuMWo`Xi9&3{KBSML#qu3 zikGzqOpT4)zz~KE*vgYDm1WxMEfPx5YM%9`MXVrjYP0pnT+1iTm(5V&WemcV*l}y; zyn=-)+io9_dcSe^?%g8@(>9+&ue7t3#0y|atAOo@S>V|HOlxq=B;cgxV!Q~Zb zd`Q>6^zQn#-C6z0uoHXZDrIy}rdA(YX#|go^q0toeHBU);ShfP*BkADhWb#?`uR-h zwmWKz&z}N$tb|IS4~LwDA-gS=wmFVC>-{x8Q+(`urWFK{JK&W~@kai#j&^mUQ=Rf( z>y7>N3$srq<2kh5vxq1DE9uo@-SNBWD;Ny7;1K#QUasRt@tNPBo&n&FiA_?3Px8fh zpI=Tfx0c>*Cf(_z_jV0!o-*GBrG^Sc5{kW5K|?<|wN;eTOsEpWD%Wi!n{Y3PI;dGj&7uTUvL zfYn}d-%mw;sdqy5vf$*>34PBkqajS4*7dk00sUp20n^fDZ^0q?{bi?s|^RNK* z9J^Av?bZ*gpf~L#W^mkx8-81WWfBW#n%_g}Zyl&qDa+^x3V^ex(V^qM!`HZUHWRDL zG(*CL?7_Ggp-lP5_2w}?9t+a&2eUx3Y};D6{~7WJ73Qc%!Q6g4f2DmfNaa2_2~F;w ztzq>|T&afAo3z^ERQ5=F#+T@rzSqmG?r`xj3vf&C-nXld68-BftdqM}S+Uy3BTz4^ zkAA+g{M3_wrp9un0&rnrAlz7N#E|Nz>900}#wcfeg`S4<#va2_Xz_c|fUZ(N=dy-XaYxeIKk*aqM*mG{e%q2*<%R8Q+p-&R*}i<s!(p^P?LL@HbkViI{-faLCN|5IY>nUy+PJGdlH@^uG*5RXNP=ML|_h>Bx zbc;Kxzkbk<D*QH=z4S_xszW7}#!|)@V}%z-Q%di!&s?!)U%4P7xuQD*!JMm=zhN zoBmy)L)?@1Lo%3RvFg)U(vjyON$Q+>_M;~7Oa;r>{Sq?@oF3NflMAsUFDBdp=C$fw zzx7@*@v%E;)12WaF}U9EcPgL5q*>uT#%Q~QGXjJ`54o=;E z$#3uH_cuyPv_<6yE5-Qen&4kLw6*h%Wk+{t9ya5{AVi!*t?le$R(4ayY8J|B-O%nUT|${>I@!pS z#+hAdMVlKey+LwH4E9b9rsvRB(>YtMQ3(_RM{9`a4QOxglHSi++=}PG>T^E#DKUSQ z63~dEQjY7M6X<^E!$Fqf*56^%_72Mvp9Mv0KPu0nQNODB4GQ9)AK`Kt)%mal1c62r z&X6w$%b_^-RVp&LvUHn}>u{H0U&7v?g71!9+MQ3r@AqGnw(USryEY{3lIAcJn>H#e z{rzf_ahbDSsjsfpJ+ZV=9gkaidD|vla*b4)@q~pPcd5~_kHZm}>vr#M>Q){8@uc^9 z;8aaOcR0Ej&YIVjDoH%MgYWYQ0KQ{tZ+RKK=t+Hr z*InhNz?hx?j`f|Za#wY_22q?s!or6?{^`~_*0GIWXv}EXb+p-QRQ@#%<-N<)L6qUy z(w@~{S|mHv=$%|9Cu16T`t`Au_N?|43`D|C^l=d}syrZ7f_&+xdn$tu%iRfp$`4^U z5V{;C(?t`W|>QbIFvTE#Tk=%`vbSEXZe14G?RiiNvDeJ-Lra83;BJW zaG#i8FDa!jwVEf`Rl~0SZ04)2-?2*AzG49#0du(R&stW5oenzJ9MpJr?BddaycoLE z$Lmu%J2P5pI)}XaE3;nZ@vfCN`_ww#u4Q_>LqA<9$3kKsv~xDIGNR z@|Vn+8d|zxIPMQeM|D&Kn%}cuesJv8@8l6@ySn;^BuQKuFCTo7GsV7#Gk7vqZkdLp zy*Tu@=&HjOaXwfuG_k_UwPJl2v^uf^z?n5@V!729^ zP5g&In?j+CR<=@7!>LTMnnb_j$~YRLM#uC%DWE=s3CX2fjlC|B58mc`Mn^k>Rtb{M z{bz&q7NPkrxYu-#jXb3v>5aif&J&Zy?1Yg|QAx=bqPb}a^2#>{Rt5Df-MIq;znqZ0 zQ3x&sJE0I(jj4y8LrpH{%6td6;J&LrORYI9!lLMSe5@R8d-{B>gjCEA(L&R)BAVX;;{G25-dGu791LmQRV_%=Mc{XGa2RK@aei&jyaP4o_ zdW5^;I$9ar2gb3(%5_W^M!Bcuy{179Ch#r$<2hI))@rk*@tjUx{$uH(=5f@Jy5G+t z1KNTO6=0!l_-oRjkLF<^%wo{@Xh}1x>^99Y&_B2%hPst;(&LqGX zX+R0zegl*d8)x}O-hJ7wPM_$7INJP9>YqzDFe$Nk9qxuPn^uAMGpxN(!^8Wq_L?^6 zp3KyVe=;yy9F%&_>vw#sfT^s#CZ%_MyyOIV7Z01{h2e=Me8cRw)a0Q_LT!6SO!9JO zEGyC7TvioTDY7EHx=_rUY3MmI^z0HAe&d<0mD~!yb8X$@6@0d+{=Q7R@A6sdPeoq2 zHoX{Dp&CLHz*-MSZ2JQQ>5PF?=E)qVTZIy%5A#DYKx%8oj~ZQXW=Ei{vgfwGH!B>!CD!59lF0fXP%Pb9 zqe$azV5^ncU=(H1@N#<9xsUEtk9We&rwlN^y9O~cIna5qPWnfc8qb)M@9|P>fj}ey zxv3%7DZjY1`smnaNXtt%o=_cZv@IiOPM|d&+YmzAdfF^?PgdH!`)9t~@DVft2O<4P z6L?3yN?vjPb10yJ${<3?_;>r9BGlNn_?3>kMxIeU=l-3!@*yUkEnRnHJ0YGM%Qgv* zeM?q3oO0r}irfg(Yg^wwyAKW>f#_TD3DP1C8e^;iT}C_oHyA70y*qk^f=(wKQ&`Ty z^!=|-tkP(w$F3~XbqAP6{1<$nf(nZu8HIuA!<(B6!KwvoXzK96Q)Cx}aD!7+MOik$ zr){M&-NmOeyL_Q_cfgDEhSt@@V1oqwJFx@r=!y`+C85v}Z~?_~CI~lg;QM@WNnTGC^12x_C(z>UTb;QRla`ul5W^S~4 zD>`+))!#DteY?Y_#6ClYPWgIVnyvFxw)CeE!w7UPNOy*LC%s-~aU5kTozl9R`kde*c zWL3tAuP}VaG}L6+Ia+aHUm#x z4GBie&^c8v#ArwUzU-#nBiF^!-mJ85ybU2ufQ8n(J18aAFF%>`R-vqms@F8sr`RR3 z4yg7kPePK=mWCJo)?6uO^_gGny|i?|ZpW<+%5O%>{&J0!}@R%1iHZ z9ZU{3#{D-A_E^60#4QTRv_q-hi+%cK$crh6!ZDyS&e8AA9pBcRvvo(_kAL?PIQ}k^ zH|*=>uRdRO+0s5{jbR4|-Y|uFDS0X>~W&(+&>kpS7C0UYZw3ls+4DPb8=f?u?bgk7`YTYp5&=J=hg`OWz z>B(}-hW-wqjjl5W36*QSZZj?7o|(+bKI1y8abOo3eQ&3KI4=TUM_Gc)kC@$Zulsm% zhIeyr%K5KxOSf1b0YVNy!7E8sW`^SmlpCdlR+^h^V&c+-(~I)vD=I45blkUp1Od{l zJmmSBCL;zT#C(y@SM~QL1+SyC(Jen+1>JYCx-e*aZpxpbkrFP_Z@DTw+*ut-pN2My-{yU~r3dqQ!PaZH*s#)p0&yQm^}Z*$;z>j1N3bNm(Cw z1|81sR>fIju9N-2wD|h*g^aO5bgZ8JuzB;(l~yo2%3xkZiXA{zRluzF$I7i6QNk5H z{Td%Y*OI@&wpc+vSajad?C3oZP%!G|Ud>8#i61%56{F^j!X ze1ycrN`~L|CxM-sV#*mcoz7oLA)^n`F#@=OYemOoZq7(k<*SY^<=T)n8PP*M+ z5S;D9X*FO9}P^z zMPZi>-={C_h%(tBy^KL@woO#?K2iw|Lg)Q6y(VN6o#xYS5V}jzzS|G{AK^MR=^^gu z-pH%HK1DzL?BsZQC`7b@`!2TNP#5A61m3>(18d7s+0;cCd9l^2++%!JODH3!-nL9t?|C!sjf$NYn&F z_2IUWR&QEK6kD#WIxi^mt*6^6%pRk9aM`;(q_qqjjES@a0$N9fv4wo|rG*gr(9*08@bDOG z3E~%H8J8B|54UAT7fMte4Z2;W z&v`fm^Nf5}*@NU1ld?OC1MCiUmVsvVZLh++eH35`HVed~=0mJtvB808oZh7RY`mwb z*mcq#U$x2-f#pEoM8_qJ)Y_UeTAe^wMAZezs1hum0I{@pYlu#&3&a>neinj#dTz^R z$hIsxQs6?>L~yWKd+B}MTbqvTZkxnss>ju9=rg#q@C;Cfn+$%dTPkB`si}Xqx+=aa z_|L^eJJG*khe7l|wH3d?VpAPB;qr=Gt2NNJwBX?0l|IBzVPw=uk>Mm81-zj&;5k8z z2*~ut?iaymO7uLTdakeAIxp`JUb(D_L2%Kc*2=)Ul;!}9Op%+3@4_|vVIPUv$S#gY zOky26eCXcr=O?Ow(Ws_^G-gwsyS=)~ zpmHh>itlX_0Tbgi33}W22bm5CprL?zM&%X7(0N-vy2WPo6|&giPfWp9`plZ|GW^gK zA~%<&BN!e^kzhJx&`9I(ye7)bWq9XUx}jFEe2P{oxIpzx`R|$XxHlSm1w)tN&;gek zZKIns_5x2qx$cVbk>@V&jKgrXZP{~B+*VlvhILYQJJ)j4GOf;pdlVlPK)DUoSphn| z*-TVUMQCyeBA$tsP}+y{Uu|sB)y&55QNe;ddl~J+&`S0L{%!z(fa9~;Xq8_mzS`D#zp7YL6K zuTNAcp&;Q}DX{+_31nDoz>x4*XFq+r0$SqOMA-_0&3Iigh@(o(p1tt>&Ml{@lilQN z0q6GpZ_%}oF-!1@z=a9A(y8WSD>Bygclfl|FXTUtavu_(vW8!>b|Wt~QMAAv)so@W z8nVu~V}wqN47x$t=vc(178+1!aX>L7Qrv*rAbF%cK!9Ze>ImSZ9lhGcmzSTqi&g)i z#{=Ik*3A=rBM|SU5CTrpDqeB6>Yy1!G)kqmrP2a~{ZZ8W0=gtxmO@SlBz6PL;Tx*n z)s&_AU^7GrZuBM)$6~9g$ezszy5jg-(HgFDpp^a2s; zqs4&;Bl(@qyR{vmO(c8%48|zjm!ItQasciq`k@lIB{9|&PFaWUXG49NO%w6o{KIHH zAqcR!#GkAtI0vBP2MHr_ZYg$q$f&NdkA@8n2@_yV4n)X@WBq^=JcX{Vd!P?t)~YJN zT=hVTktMX6#It(xm<1HM31-sbfCP%uT4D)*N)MFpjQMni6J|=3=R95!3(AJ7A9m}b zfK$=suDR1@x4O-kC+Ii<__mhFYS0S_%!1&l@eZnQ%C`0I2fN*e0wWyHwiC50R&GF> z4eb035jI6>i`c|HbPnke{Uz((LDAj+X06xne0Z^FDDJ^F zLXKYdlCgG7@mg`!-S{t$KmD!f7JL_Jgpa_2y-_SGETB;PLIah18-)-L3ox%CCIsD5 z^yBslqJjK+kmUMy!2T(4)y8(IxQ?oUrpW|s76DW(*g0SJ4uYM=QJU7cBw7Y=O)dt6MCVwP2 z^}ga6c_ra;8$sIVW+ou94Pf^pTxnZ7eOI7;i~ouXJP%zYEpOzhBN;(Nb>b)2>z_M} zq8%VfY1MLarZ*G&x)haA(QDFKa7*-U8c*kI+|4Q7`n45YRH?rGdmoTPt>JzK!aOEF z-w4@wfudVm2rdMRt4SfBsF_eBDiA<3I@T?o_49SA+g~B?{Q7x(EJmdf=`E1%GS*=1nQ;W}3iN63 zo+Rz9t^Ga00Shx~imA~H8%{;9(%_b!j2|(P^Q?vtZbQtV~ym8p%i`hKzs(oLJf=cCiF-wSzwaCd~o%_EUnfyo-9%mPH~%wo@9cbBnAKlM*O zQ4ttZS!mI^OW=T1i;qtbuSpLnC1^w{ElB4*I2`198wXR-yQ@o95BC09RQRV<<1KvC z6{lbRO)|*z{AVakjdQ#}HBg)c!t?`~CIQ`}xSq{wOFM{QBE<*)A= zx!8O_dCq*<0oB)Dyuo7WjT<098iX5x9mxVf^s)Ggz)SdnDrl0gDj_oyF>_tdYPhG_ zCr#qS?EjWiWGTHDp06{z^+}7T?#H`R^O)lya8Y$af@im)?lx#o{cj zGWlQ>2ki*X+7FE2wUS_MaGe}C)>-!J8e{)5w-m&lKeZ40cb;_XEtuJQ_LfPYsKKcK z!OYIajQ%8XpeD{w^s{A46)|;4nIg~eX0KD{a2ISqA+;gEmY`wWn@Pb1<@{}|QAv>Y zieLb3@d7cgr2m%U;VAv;nW9n99VKxiZ0o*)KibTKVwo9c*VV1^l}HqXg~La`BQM;d zsD^J?1~`zFyNQx0+MSO>x?{0X)0>c5Atc0NqdXC*)w8{MpP)7Uge<7u%z1!G@3|%` zArch21aCfhINX-A>bRQt{$Y`J1nNe>zVY;4G-3wA@gl$ieDpy}QF}_Ia#5MfU^1WV ze~&;rRHH#Pk3YbY4X20bnf>^7!Y5rJsUE_(RU}f0GcI5vZAfFyoiTqu8z)Cd0(e5` zZeCFR2j`*~j7==8Zwz;rfp&2CK?d}|iUlRQs&9VX97bg1lW4AAuVrZN|1at=w(sC% zEY5$KiFo_#XVE22K}7$g`q%$?@&B*xMrzDcdhTv*W&n-;n2?;UKuvKQE`51ex zN#KsO&aZuV*%6Op5mfPp>~Z+?uRv0vaExe=DY04~8{wpp1}t~MNUvK^ugT?8M79Yk zz49gLBfh*)rU1SDW);*-9PSgQK$VT~O4oLOQ?LwjEH*p{;V7S$qTHieshw;%r0QZj zai>szk398_vudmM0nC*G)o3)EqCcJc^*l%ou2W-8RLaN1KNSYLF99{#2cqT?CNj{~ z5{}h+u7h{URSDoM_}n$bKoa#r=&xzKU5cs#_Hi%SLBR&uW)wg1g$uB!s4S+f6`4q! z6qtOfNx4=I!5Jbj;$rVHWd{<0Q~*NO8(=iLP(@#~+UFn`>WUV}pNtS;^dHKB3_5_A zu>lNL7Ni~rhULIs)-a3CK;&;>oHn^JUC8RZvGU5`{iJ}NJ_-Za&zYJ z(aXnO<-c84%^b{;^0UN=9GKj3mkEALOSpKcyo(drSWqWH!ah5)h`5*PmveiEFOy=2 z7h@G1z$2`HBCMDXrBn-bp4H-WgssE|mj4OWxD<*eRr9e}_M~lpsmGddYa_G=wc1ss z1y(8+8*}~Q^v0l2`onS9M#d=UAL{Yl_}hLYNAd^Z;XGKk8^z-GO&TZd6{(emD8z3l zbj|9LjH$XJ3I{BI>2x}nQ((E}|0h!f0v%n8@AayIjnO%w13MM&P2bWvKVMgE? z0MPJ*N+?2l2wBB$ELpz>_4)~dYQ&AP*iDmz>22+#p+|uX1a^0AgF?X3*->+CK_{|H z-NYrT8;d6NaGO3x#MghnV~x7J{Q{TxJ~rilk}~0rU2O547y@7e%@c+w6@^o99{gIF zZ;y8p-^uKRzkx_a_z2nw^x&<7$dw^P#U(>Kh_p(*bIl$ZJ$YOr1E`Tjcnr=R#YLvu zbXmk_n4E&whV&dHd0V2*Z6+du&YP2-aiNB@H4Y*T0v`U8Gxoka2@XAD{OSSNK>MLP ziPba$42Y67fXCt+puj!I&nDo#O2E(h-rQU-h_p($lxm7_)LNv35R(uty%JY_8o^eVgvg1I!cUe#^1I-n%>@*D$oe720Mb7jb0gt!=9u#Pg zu@z!Vz?k+h3()7!r73`4p&xavO&ji5Q;7!9PHv1!h>w=I#}1J3L~b|&M|v~iHaiMN zOCvM~&O#QO@GADcu;tsI@4-+lQE$kX61|Ca_z0CP?Mi;-z`H0IO%t9C7JBd)+=VcZ zY}tq-o!LxWS=7K57>h|a-2mB+cCuMRT9-m?7h3GYyAUK4u?2m5{)4_BBck?n-FXaP z*&cdjRUuK9BRw;0To{ga0Rl%UYJImWKtQ z9~y^YSro?lP%O1ZMFbVzxBuV|pudh)fe^_oKrj-PHNY+bZ$6Pc%mN7hnw^0X*~4p4BdS1wti;&HJf8n5lIp9fc~ep# z+?gzV`@(ucjm)KRV__P9!(3a8hPuP7J96>QLweuhXmjAlZA#r}k}U_95=;~7^-|!S z8}aB_Y}k+Qq^4uq6}VYh_1GiBOO}jvw#5F@VpfoQW}To&8bNJ^>=L3pDBv|`TrpA|5Jzjf4$H;NI~N;mEj-*+0cx_v2|vz zu-*<8KRLG-n6Chyh5M8eNUZ_|;YZ33jErERz_J9UH@P^v=Qo^qcryuhTCpy(Uppap zZbbd%1qR6aM3PHn^(YW}sL3wi4^UsGh?5``;2dviL#iyIa{C8JUI~FfYXv&P6__(! zuZb81k@zRa+|QD0?SGW3s{fiVS43~(kyS9CfHV}qc0P+B^^DRC*9v*0Lria8O~?i2 zdtV^a2NG6hyupm=%6L*D^byPdY!N2mM3Xd+&hOI{;Mvde`{J3!|B8+ipH!#-s z4DmNqF{~QUkRF#8ppoi6V8L4Qalo~8xW)h#J#0hmu9F?Eq<^=A+)*M^-Q0dJ80g7O zFZwd-=Uc-m9fg`*`w;G`Gtu6tkB%q4PDgNtGI=6ny8;_zPaF6Ik%pPgxCMsoslTpj z`vG;JNbMSz-$h75@);zKih>>~w-ig^Om?^@WtF``8Cb+g*4Marfo{!D9FUF2oug#7y;p0Xs1FI6G*{;nV$E-;7}$Z1|9FQ4=v<= zTK@Pts$EsXs&tODEu?Y+H^gXT%;vB!Jr=Uo(@nrUU@^$)n)DWW;JfjMN?-ZT`C>JP z$&!)c3{YW1TA1b?P?w3ze`f~O$BS(tHTiyUZ*5scI&qJ{*rZM^3N z)sE32=kZoYbk0#7iLXPMgKMQd@KZYsU}adT6?m6eP?4m!!L*e24Uftn^BZiU;R0kt zdo5e8IE;#d?5A0yUt-89MGIYH>nT)&1*i`6;^R;S=C5+J^aO5a767HDX?f<1PyCYyj6Rd?)0>lc4|h!N0@4qe8h2p~N_3CP-n?quhqfWEJWQ|JHn+GJi8 z$z-8$@zZd?OLJ}d;Qj1C0a5C28sv{!-!;?-dejJ$g@zX4fiCpHz%H_b76^^~ni<7S z9}u&QTM@Id_mLclqn5RLEDpyY3$_-upqKRiH0(?gT1kj)9|fOkyW~hcWdNI)M&9i^ZRvR7-9vh>&CeB4_l!rE3wJzH&R#T;tL<2FyVgk z-O?riC@mqM0N7@Tb8Ba>JfnJ4h3=*HBMN%thr^j%HZZbemM3BogJR!=^1nFj-bExF z!HlCAOW1Tl%vP_L!ySez(Cl_Uwrt4OD1a4;(6mcLQZT$O{o4>eDO>bT4|GRZlg(Ic zfuM71RJQf*3eSIrY2if*b4dKD!B<9mMUeyKP>~z&VWP6dSCZ(tOaY{)Qu5f#tAzc? z*wEh}Jmqw|1@83#CfCHcApktH0HvYt0Z6)x#}^{gTn|`bCAvVAmpD@l(OZ9T43%=m zn_gDZ+da#DCO#lWLXh7^_W`~Vg7|lQeW{=z4uMp1zIceV4DCs(oTylXL;$(I>{tm{ z(WC6+D2%yQ^uOTw2ReSro0dO0;{MF85d03KTIXkp7+R4pA-WIlG?%L3sTAyPQ`EBt z4>ItQT$Oy7DfF`c`DT+m9PCrr?mlvE3v8jj395}#5R~ZQ7>$4U94+)n@b{V9qSNr3DCt@ zY>XUC%v&!H#|O?M-HN5C4DrOTl49xhAF2rM_zPn=5l!U5C7T;!E3!e@w-L{CW!}mP z#9}=1s8}l~90f36+ZOQo9Y^`4c)B+yr{nb7-8_6O0;kk?`IuAwRp0o@OBs!$d7%U* z;|9gCc_`W`J(97eFQ94skzNx?)8z6<$m&xh=??T*4hz(i4eme(LK!(HsQPLepl=o4 z8rqR(EW+?l{zD&enykP57*N5-f}_NTgC@|9g>~;|M*u2h+44>1>XbdH{Dhmx_4=_l zxBjU+nf|^HP=}9&S4yG-1^v;Iuyh-1+#wY})IEG2A9eq*NF#r&H-o+i-)TJ zA)KIy(ut^@b4aYgy5M+4x`=}!zX;N<6q?;^h>)X9PDzf(fea%>!zS|coFim1h6li>5~4ToMhd!R)1vv5UZVlyd2 zWDH_rsVT%MQlimmpS4Yqvvw*Dd2rAs0c#L4g`k=AYbF6iOo2x0aM5ZAR5rF4Od7zX z#w0%z&f>ZE74OdugJ;k9Bx#LwtFewv@Ok1=li==4$xDOhI(KL}zJ*)xqk1JrH*#O2 zaxWsEHZ~?1s(zX_ps+=+o1`#zsJDUm-|Zz87-o+Ey^`GUARGcIpHZhIwNlc9hhaoW z>H|j$i*)hbddK?pQq$w$?;XIm+QLELm)ebOBK^^AhVTv7*4um8Ogzr93UQ(l)QvD@ zrbC%HNAma&$Cp*Xp zdeyYt8AZ8=pGEMB5HpKSZFs5X)rFLj z$zz0Ss-r@SGEA)MDyN?3%qG|7X7qTAT2RZ3Ek3;i8o_=XH4_uPjox{wcJ%!Pr0J9I z1&4tMazWt+(aj>;+6gmXyoRBvD&Hu%1u(TJbqE0jagee-Y@Zp+Ob|R8 z!UIUj@r{Jz-KJ8o??G3o(A~R9cRV^Gd@dgV3qK3qB;i$0hR1fo&@Ea*fgpEop);XR z?cQpG#j~*tqk2j_5D&y3OR&f)2-;8ddK)K9St1_5kv-}4#R4%Sw1^gmuei=;}r&x`y96Q!5d+`&>1&!ZRKhi&nSL=H>h>J|1PKNFH{SN}0QC zMH~&7`1h)|WH5lQD9sq_G%hlW1=UG>PSrl0S{U@Y3T zBTX9T$T^YjLR~Dlxo=RZ-bca6b4&;?J2G)S;rPAP+38xzzcz z8FfvHd9^Tnh5Jyo{q*e6^~zUln8Qa^0lZEcE@KT%?I@L1*d)5)G6?At@l3v7dP^IG zAJOZ4^!2%O(IG(LixR5?LJ`raq_9)3Eg3?A&dK9VvLm`q?;$M3M=Oz)$mJ5UD_Ju^ zJM1OD6j?l|D5O#hJP9R9#KK}CMGPnV{8;Rqsflh%`F>BW@%tg8GU((dWsM4aG~*Od zZ2?t^5MBZLF8i^*sR>;LWkf&z>>ytN(usy2g+puR=z3qE+*7%C0vP0)XLxk}TTi#$x(9wiQjOb3$y0AH(qtTb%$%51EEm;bJq+6G64o?X~YT`@#n z0Oo{4G@cntwhnDg-}4cds4iR;FkHGYapBk~ad0GQ&Wqzn)+A}edLnu&viGke@7o}#DrvLfakRX7O#YiA*6 z7Tly<9E&$JA;2|L#LhBW-2WGQZyuL({{R1G8xu2QWXqOjTvAqBtlvXEG3If$V@LztCjZ; z-8w1!rG;EqJJ+3SwN^5=DMbow*|W`!XOIS@8oEh2AavX%cY#Xw?{cHl#ed^iXpQK8 zXa+|ab`lI$sv5R<*TXOQE+4&wF~crmxlb+r?wrt8JvLfVcCq0PdQz~ z^V>#}vVae7dmssk`sDO%zFlKojAigzDXzuUIw|4PEXmKT^<8wI{bfc9aYxXUNVxHa zd-{sqi*8rg9-50|B%vq5-4GPEzhDN6#?&*?k%V&s9WJ{V61-)o7&|0#_o9e(FBZ4@ z&}a97*DAB-ubf!2TU&@*QDqLqc_2*QwoG{;`8x7q6L*Iow5Qh<dlQ($DzH#%G5h%-$WCx))(pXmPcx+*h()YS(R!U4R8~pVyx%41={%3T0!JSFHvG zMEFgQI&0%Ixso9w9DRE*FTe#vbe+C>8T~=(h+%#LLae=PE1t%bU7@SuMTNpmx=d1i zi{*bhosvQBD)=5fb;BOIOWn$-%k-N4xI|hWpO@3v-C{rzP$|-_^ARf0XKb%afb2WP zHwtLYy`8I4$V`W5a+M+zem>Pw?kghrC(`V6ImNCM*|&I>g=ZB>>!t8R=M4cLKd~da zE*}J?qsU}x?Oq6Hg;F&0L71$jJIk69r&J)h59W= z3}B6jazf@R>HLWqPNT~ad3)}u03?#t3U$ml%iQ7%PVD1Dt56ADTq&?-(kbzE%I6T& z{v>UO*m+bTJ~m!J#+eCX|czz^Y)h<*5Nb?!f;|L@R# z)9_B`X*_Z|o$@WaCNfM8=6~L|UP62z?9F!vpGL-}x9V*0&Owl)4`Gu#1h1umbm6)_ zP4Ez>2n(y*x1I&$;LCN74r6xs(J?gJFBQd9TRVOPofVT%9*z=(_EDUTy3}|IJX?P7 z8C7nx{aWhplotmnxEd`V>9iJC7VjX>ermnKN9uBrKmP|+96P3_9WlQz{Rd&~0zt`w zk0Lm#EzDGle)H*`#A0FiRfmIV{GIy|H8s)ktJQ89Fi_e1wCX{Edjsz|L?30gme5Et z171)|6VbN6~x;ot#|E4R}rWw<|HPJ!LhIBNH5>ng|7h z5?A09ycKgupi(ws97?pmDs|Y(m+V;%fBsXv_Lsut1MdG$=9h)L+!v8Jfpd;Hpyk-M zNNLk_^pJWO9zLZS+xYlu{**QReU{WcmR8;dVt5_YjG*omw2f zzm=95ucPsydI~ZpMJBRr`{>V9wPvbUlmX42zM_Z{YV{N(gkW^*y32|Q_Yd`2A@|v2 zaRo5L6Xt%8!&@usk1C-9C{UVE3Tl?L^?P8{LSTi_5V>*GsFud+8%sMggAPf=T{*z} zh0A1Zz7FW{RaVMsu>-BJiY}(mvXU#8+BCr%%tNUn%!HgcQ8u=tH9}5z=N5;syKhb0 z!l(q^!_N02nP>mUpVgntErhopBwUUNbU;tUpmAAD1#)9ykh(W)C4;b(&9zN&vv>Q2 z(Pg`>KZtW+f2+n{UjnedE0FQhJ4J}fpW!!B4_U*YCxfO;RcZ;3A&a%1PLX_2m2k%X zMQY0M-8cT@&YJe(veYm#6|h|1fgJ*48Sgox|1rwWKqouy;!-B@Bi zFNdm8_J`GzvET~#Azd!fn0^`_SNe68fOpNFo+N2kLIlb<9pOMPC5T|=Ls*lV&k@K- zTZ5RRNbF=Ni_l)ZiM5fRu@ji8>=#`h!64BYNyKcA8ttu=WDU)G@)DMg|0PeBx*LAx z?Hn|Fw$7W4WZTQ&2aH#yLUSrI+f{$aaQ7J3|3!$nn$`9vnR`JAC7mSY&E6l{826xN z@7A!R+>+|A551GfmgO-+0a-ndK<=dW#z1#;&dp~d3m$%ltt=3GAtBkIXg1um+O@$} zeVD+CG+C^k_oi{H6S;L9J=&gsRD(PpF~iC25?&wJBK*cO@gh*h`mv9?2=67}0}Wjp z%;mOB{@eD}mg_UA(5+dO#zG+SaaWd~Bt}0)=GWb11m6(j^dAxa|8epDkA~tug6)6% z1ODH;CCq=9`FrphcF%WEc?i8H9;e^j9I@)MW#hN^miCF!LW}`rR{mp;oga{S~scW%P}}Kz;K)ovi|uM%KCbs1x&v-1;T&2oqAKU?5#XO*+q!0p13p@=Y6KYjKXqs zTCHcnivrlXBEb*!=4gRcA~{9K*$*b*=%O+7dM%8FF6L{0|JLM8G32TNGP5pGfeLZ* zdJs!w#z=JD<0CmRwm$pT5CR4(#j-eQp%iBFbtzZsMd6>|XBWz6i+d+z$kMbb2aL)o z@-PKGA{g@94*f{xhNZs>jM0^;R>G}_sS^Fs``Q@}UN1uXeJ;P*i_ao@%dbp^{kyIr zY2eG}r3{0)Os^6kC>=F}XVNLb0Hw`RPuvs*yFkX$B{S&a=qC5DKipRQW4e;d<3eVx zEb>1i1cO&LXuW@Rt`fkh+U6n+C$viCS3aj;%KV1KjNn=^<@QeZ<8B{4h7cq5TXFtz3?WJTf$)+lSvaoW6JGIZz-aex@qMOFgj0xu@~HS zMQp|c6fX55|7d2lIluEGvgP==hYniJ4^a%ud{&i0#QJevrDoormkd056QG#GXoqx9 zEw>QlhkcS9Sq)}n%GAt%M$F0v71;>Z8!H>5t}S`bjYWHQovZjMW={6)JzC-!M-u;w zSn+GsZxb^-;|TU^l?#IrFU1K#>*K~z2TNom=?$t*MJA%=T_WF2L)I@^c0sVm52~<~ zT(8(k|02Dh8lj-Bh=_wf_^*#^HI0`Qj^d@^$};fD$=P}_b;Z}W-a+f9LFL!f1^w0T z#0Z)5@T4k3D02znKZ}K3B7vUmhl8#2!3)I^Aa=w)d9x0R_(E_l(W=2;^;HXDuqVM1 ztys2IU;^_faY)EJ$|`U#7DbcLjquHeti|z?bM?c2-q!Y?s~8u&H89ZS)rmLDs!tkO zq)eJ%wf${CACGa$_KqhHd&nXUa4o;_7k)%Mki3Ao1*} z=U>VQt_B-rdYy(!d>`O)mtgxGAKm?35D-~`;!l(&lwEq0bRXWi>9$(`T+@{@OD^Q) z_j+A#*_Vz_kSiE6FnQXv6efd`_{)I&p8)Ay5Pb}!p6FvBq(vViK5ZA#vgqhZWo+4{ zg~KBC05@qMZK0Wky`&#ylbX}c6mZ`(^oL9l2i@Q=T9AQ$0+~>t&eCYPy)W z6u?{R$YtaK=B;gU5nBhPmKdE}*ss!AN5r_<7=GQS8;jaby%X+V^hs)OGmOdNiY| z*n;$DpWo(~7Tb+b&+)n3Pw&N&l4n*9-;F*~aq5G8Y4*M%?>9;L5kq)CkJLIt7S~jF zT|i+rS!9g;v~E|?XWMBf@>^t8oO;Yv>?-m;|H#X+w_J?%|IB%=o;4#+%V*6$b&E!W zn*KokxzqRmvZ?=nG_$aanI>w90R)WNGC4((Q&MDQHU>AZO#Pt()9G#^<)l_>) z0CXRFXh@&YrCr_7xV@mcd~d#zv>fsJGVk;7@F?{{IWi&o#N$os>CJTZN$Su&?H+6iL)dy@>lmw@MM2t*WtJRtx>2 zP|}oM$`jQRlM)q~A}gJCMbbR2whIt7ZTh*g&Q1Bba830J^HGT0Tqd?lYOAl~In}lb z*kLCq;b9mg!GOA4?m{90YTK<~8u}CW2r*=Y|1ML8_%gBTg3}uhqR6ZYiRf8SgjP8< z)*MI!45XGXKGBSKtQ>qhrqe%Cr?s9@JUva0x|kk-I+S*A?oWXJ#4Q zI`QexpMUVc368y+>1KY!+<{3O{C*@>(17+^1{#VgMvU@8oK3IwdplZpMP}O4F-I;_ z;7y+Q>g$N2Z(aP(uO3ZTVGIC&zl~F?MBBYn)G69122+IQ7FLBvH|0gF6c6e#tiR2M zGWB^mvSx8C6k&_tE0wSgTLq34d)>OWS}V_fBO8zOGrZ$6c3#R8Z76uqAfVftba%p{3yO< zKAT9gW&Bgz!102Qh1XNBm;LEGFer(HSdmG@526dI4qJ4O?{_c=G-uPQiy^9XwFv;E zZMwIJ5-5fD5Ez=qFOjGA*Vj>72f6mjzuSF@xv-F`E2;*k1;7+NoZ-J_$%y+C3v|;1 z_VUy3{`J#a2S-m^n9a&1lq{EtiqmbR1Bg8FRX56?(a|W>IYIvX|9fXX_zl)_U}?-1 zXKPnMZxxDVwc)FMU_K%wku}SlK)X$UPVXa>gxWY({4=y>*37N> zof7OR<-=7frae;e$=MG`Su7rgtCJ3WfLGXrE%QIba&aE)qz+1<@zf_%sGS^?H8LD& zA{8XiP_X_bQTFlWvoA!f0@6Z>4;#J*ZL=sq=tIU3pN^q_v3foPpjV}UN9k(wZI+6; zL-qypp}gcV88a$_d12CpZF+*2F1$yHS_mWORdhCp{xE4V<&4r0p$D_cB4WghEMF!@ z6XDE`G$4qm$9l09L@OlR(lOFPq38e!KMsB7k)o6;Mx$GM1tCfcCG?6BQ0(js>Y5&0 zv;?yL#yb?Uq13#iJ-^-7*EcPBICAp23^NslmW;!aP_hE=pA6Tn8KNr)TAa9yn8_hs zZN{hI`7}xO)B#woG<=`9FdIGky*eI8btss#G{{oh$(T=0!J4cJp*d5kpzbZbPJ+G2 zAPwb>6AKGQL=V9sVRpAg{O{k0Z_AlRZim;dyF+~+u%f!tx^^PzR;p-%t^2D-&&psR zr?uHT%!dNSzQvh=`EoF0zd8l|o1zr_h4!MfUU`TMqx1)r-| zeOeYb!uNLYyYTQRQ>&~Sd2W`)wwqdY3r&vCvpD!6(8BIkq{ZZ|2kcV%^mL0Dd(L$2 z1<&d)&%5X5x@!IE?rx~_{%*J9yX{NgFW!^8ap|}#6@#x9dAqV$Chgv`WlM7d`%WA) zKWdwFe_y{$HU-WYstM&0X<&b1KTB%DfF-Z4A+&Sxoj_BIdbV4F>&=lA30oFFK*Sn> z^k!F2=RY`;sKfa2<57@!yyX9taX{y}t6w*wytGf?FIj7~6rD%?)#fFJoi*>GCLH_& zwY2|6wC5Y4}borV3xo74EwJN zTmJsvKdkZJH|&41NZx2QZ+^(n&u@KV{qMf(Fl~fSL63a@Z9Vd97Ux4#Sy-oswH>n3 zo(5uKw0%+_TJ)`5Z7n#!G(K;0Pnp$XIP*$m?RpK4nEL;k7dQRSw++7At5ML_RkJ{* zAn>JJEd{&h5rlXSIHScWwW#Xd+=N-zhf?BS2XoU?LJH-g_DGm{RptR%zJPu3OY@x!h0Xaz6>9}QOkYd z!s~K2Hy;u=!}-jub+&AI^H==ZH$RzjB%R%f1;P0kvMOHO90fgkxBYv~h7E6WK3N6n zauyulu6`be=H;K{kW2ObFb)1*PxpEotUTCxFtH5^T~S#k+U=aRhZO7n_`?m7NZ*ys!Lhuif7>#_$=J*~mH z8+M#ecbzwWQdk%Y=q51SMyooSe|wIy^I$lJRhY#H&M}jR3(+v_&YV8V8_Ylp{AJj8 zDPKsBxA;rY1pU|NzFo7#P|z0{plCtG!Ow?|D)FFT$mZ}`B;UsyH0`t>F|{dW(>i_2 zBl|i15~t&sj^p=^S@W{yUfbRaa?r5Yf?42>_Yjv0Kg;yA1`Qhq96o#)9Y;rL4wLg< zGO=eLBM2b1`h>x{Ug^}*w1?bo_ri-WxZQ^7&#BohL4FoLJY8!L0D4(Pg_XIvpF{Qj z{koMOKlTr^dze|XE*C4*3r)|__t@-j1Yh zZv(dB8#nW+!TkC23$N~Tw2gCF=#+3eVA{#{-O!SsM7ea3#~%*Hy2;n~7W=O7*|UyH z_>%3tm%p1ccBkP8LYFL?Nk^y5J#P2CpGD2{Ui~3u!J{>2&z^l!TpY#Ns&jdV(ZZ-g z*w1>?UDeM~vHWs}UnX?J7zx;NgPtLqlL%81xgo9;1uBKJFKA{cMU5FI^Q9xhPB(Er z**>b1P0W4-&tI;IaNSaF_mN*uFZ^~()~#J@szQ^hP5HFbP^sb|?RLvfa^!P*@@MpK z#E%8F_0<)GZso6mKEsib53DG;Kbg}Wk;o)C#aE-?E-NduAe1gDd!poQ3oCga%H=)g z-CnzBm&z^8n|ADuLWw$V;k%9VJQhxypoSGSbkxBBheon@RU&HR`Nuz(rd3b+TH%06 zkA_X#wPMAJTfxKXdm~~W7v9{{(08fvFc+66!#bXQbM*D=1v9t%3_tC;rCs{+m+IdJ z1ao6Q=LTx=|85NYDC}&wmF`Wu%@Qi?c6fBu3{I|NkX74{ zRihnw9CW&tg;x3im7R-ANZGTjsD|64`=>NL{pjlryZG?gv%?_tMesIoPL{rtyAEdz zJ!4N*A;h`<2ZyU4m8#oNP1D}j+oca~(C*r=MpI9=4};@-aW*uf|G6;AC@s;Y|*7qW!RaRATdgNleQ!A1{q^IV7A#OZpF&x%vL-o&<9ewk6+0n2%1%p;Sf^jMPOG#I(eqs@|Db6^ zWbSe@W!c*q${S^cml%w`Old*>EhjbEe7?)^T3E?MGp>(0Lc$Q@<`-~w*;~6<2X?VG zNJUYhtgg8?rcb}u3o-0k-D*|W>Rn|$^9#1Tk<$0}?t#nVSpTmuY$cB6r2-Vqx=*=h z5cu%%;{%14whJ+2cWlOH3AYi&l>6C4)U5=skj`q{-3M%8sQ&q~qg&FsA@vJBUsta^ zpH#FaT{v;|$`VJ%2ikWfJcD^!NjnYd*|c%vU{dU*2M-+kv@T6j2dE7{e!azhPo>E| zQ?-o>gHv+HoO6?#p<$Y8Pcrz|m+77x_slexmgZ-(Di(h%ca@?6`9luA*!|Ed>3P-g zFS$&kE*LmvU4v$p$7YNjwN=QquU{scwo>nNw(SJ^-2@IcC4J{5Bqb$<5r$e3R``0A z$Dt#K5W=2Yc8>=AKPqzF=B^zA7hpj7OrKr$?%gEz&Q<_;bbjOT`QG52E8vG;0=H6S zOpC1QOSB}t{~7z4<*;bci*Fag?;)nVmLJayJB%q_*-a-?Q;YNH?2f>{N)LX6HnM_U zpK&`4oSDQu!s0(a%d^L?YOBmx9k=MKv{%zN%eG?CTB`>Hhq-6g98TwS|%kYr#~+!EYw@I zY89u@j``)6U-FL4+#)p&12#v(kX=V@_PD}^hwr<%3YD+bi3N{3da2jj*Gp3Lb;b8b zkdca@|H8*>!F57FFwZ}K3sWbCZcZ;hmXM`=kWUj%MfedCHr~C^%m`b7rdt00^_6rh zM;{Ab%NffNG~Nxq|Na`NIZDZ+HdRD}jN3Dg-4Td*?)_vda(@oYq1I9Jvy@3r_jPg` zH2C@=@>__DD_9TK#BA^kI^7)nAh)Os>b5;hu26-L^At{t|NR3$xf%p=`e!Y%%*(ho zfk4=X9X{AF7&<^PJyWF@d+{r#B0r~_y4N%)l1pV=z0yC4>a^!fWfd{ z-MZ)U-Vv~70rsdus8Bk3Mg9BT7P-y{UF+kcA(s~%92{)Q#&O6>!tm)LUx*;Wd${D)kQHQV5|ZTh(%u}^ZBl8JtKdhXR3$HFM+b{S6J#DZu$dEmcZsk8YmO}(b%dm+Nx zS6Lk~Kl5?vsTGy+v;=Jjudw;gew_W>OyBZ?znZfA`YL`0LU)=E-KeGKlVLNA_>Xo;{(kzG8oK7&kF2i|P<{Jrn>A|g;oDyqeE;7+ z{NFcBMqX}8k zZgD`BD?xbNh)qows)rx&Eu+_JJ)TCC1dKb%8QxKInWmFNJ5m8SE{)EWE;XQ_Vk6aC z1W` z-Mja#JJV}w;QGnS-X3aZdc-nJMSl6Dq9&n4M5kORO(8?q`(*z8_b^pCt!o5@oj76# ze=eU$Pk1hKtNKFxPR3ZA9agtqJy#|!#od|nku!LU^H5M6`u0>h)Z}o}+BBxtCMG_LFxr`(d4D$Ra!4{w*REqY!)e^hGIeY< z&6)GZ_?rd2J)VwPTyoDAH~y_n&AK?nj~Y4hA|H;vCVA!OWN?$@@addUYv)B&HqEY-80;?I*l2tBh?WN@=KlJm=-I`qFB|*%&$Iv7cfa%Kkwdzs zFaM-2hxgW0#sB`Td(RB1KMJXE9zJ|HJltS9IwIc&&AWDL*KVKjs^+x6*T9K`@jciI ztUYW?#wU}qvqhsny?Nie(D`IWyMjUg=ieONcePuv{J^O3g+qr9RVX};$2 z{Cs-s2@4jyM!2&u_RbvlrAx;z*i$_y2Pa`brwp^=aeD`L9zWi?vT|^x%Xqto&x&3( zaB`A&`TV9w{=`1%RRz;d9-ia-KI3oC@prUKw{(n6=YFJ!F2=b2=rcM-Ltan48&KDH zN4o_-O>xv|v}*kS#Sg82vp241e!KDRJKO07zW#aMyy`EWRPv8sHPO*AP`=sK+j!A) z^T$p97~a@>dV}ijrVYPB5`t9mYe9#n^)32Nar6U06cq8>uZbPVx_&{GAHTG2tNFtZ zz6qZx+Xc)~U~lZ4`PDwC?C7=fAuSheyl=u)^gnt^3X5^!aWzY#F)M8No5$z%l(lNr zGE6@=Ysi9!K8hE}DxCU_R)32{7^S~H5urh&M!D3?&hbM|4DwQ2@L(hUG(S5`#Dy2( ztDw3M4XOUrYh)F=MTL@*%IjN|tTJcC(^xvie@2ilz=r^RL)SIXTkNWS7sc$gS~_RJ z9dWGriH=K-tc%NoFgh$Xtm;ece7Ic`;hmgvCXZW>=vdlC~~lOV$hzGw+%g zvz~XwKsTuh@;}}sQ%ee+bq!lgD!jP0S(`SI$c5CMVg%)FzoeQL{-G)6s!43zgcX(U zsYtZEQ0fL6IP{@re{alI_hCLEKeuWXF8TrRSZhzu6N=0k$6mkz3HYA&*R3GgkeD;CY@?|ZUFH;!lasT^WZf+j8OjAFGX>B8s?uU0fEi7>bL5@=*Y24 zwc5O-A(BEE0b~T|-GG!fx=Kz9o@5nnIe{Md`wqI<4OyRPCJusU9N{M={#GuiXR;<} zz|wJ)zkktTMe_egm(u3?(}=~xOS&dhZi1oEJyQA} z06x0O61Ga5WBaVxcgCkd&C6LFW~wRP6;`Q?yAt?#PM_TYUFv8(K>B$YO;DFkoi60% zBzer_=u@kC^X83<{K2#p%Qpv1T%jcQWP^!5c|3Gd%do5bGnR+;#rIQk=Pd^&OjuHE zmp;hHb25|quyyW#_;6}UF__>4)U835%^qsgmRZyLHr3IIZqQ8s2;L7ttZ9w+IJEM6 z4hDCC`&>lo&Pn7Jx^n$xZ{K=2O*gsm{6^@X$fRW{e_L*~gKY6`hlD_Vn`)R8j zxs};OiC2lwE^ZxN^{#j#8st5ews~i}{_yjke5N9F$^i-i)fjpF`xHhX-A`R^&8dR^ z0TFeya>=mjJ;U|~tGo&59tpbH*wiGvH#-|n+yPzzqwB$nKRj@s%9Gt=-`1};kXj)7 zCn+UmHV(ST9X^&X=!&G3lV!(HYBRL9IoKq(wjcFV6z3`epGO=UUDt1Dq)qHlfC2Lb z3u3_kLm3+}j>WgNeEx~uX~CvrrOcpl)E_v|9wH@|p_JJ`ceOQ5w?cCsVI{oe7LiP> zQM(tud>JmVJU-E-OO~0aU^sricmKX@?6w0v%^8~FbiQs#f3m9-GrebCX|rtEGI72L z(|pge>MFILsYA_`RN2up*+VU03KruKX$y2+dH;s;sE zt~~B522B{uU*$d_ zwo@q4S^z_3MHcg^WrdLW4)UX}K>x{J72(G&*3~-Clc8cYP+Bk^p*Rn0t~a2G5YOl+ zt^?ULYuxyeui@&BOMnA^)EQ`_lK(ICRY^&SDujY640b2k zd3|pY^8Ec<5fV^}&dKdF?QGp;o}Nc2PbC`hOEF}~5EzA9npCpoGJ%pFs((XnQg?ZK^rP-h3H{)-@b3(zQp+tqJJH+>0GNZJZPTJhXX_PJd`jC_DgHz`@L3#+Yw9%&C)NN0?Uo1S?As)KtjJia8++?zgJ?(2 z#d{sMEdiA04^F!t%iWnk`2UD0F*|kn(!;kJ3=9MMxp4mc2x5uotU^|mp0Q6^GFR50 zKJ*CVL+UVr!j$^=nuM&xxs#tY|50%o#4eCSBf7M#a!hhBR4JmhM(r|>wCOoLYnK)B ziR~fUXV!uwg;B4y!s|u5zNhu zxOUBa=C&UdHe?IE%m%Hsoo~fWA00(#<`b9$Axn%Cb})HGWt^b#Y@hzIP7mL(t;XF* zJKp6@S=pxRS3YUnXhr;(y;3)E>QLfLIU;k3U{t(|+!Ib0Y@G7rZ*kR^OM&uk;a*ZG zVR>&dr0iCtZLbl-hd&pZkw9A#=rwUrozqtclyem=l~2jjeLyGU7A{_Fq`FL4b6K-y zP2?#_A=yg-ZO28nGChIo;EJgsYSfNY+U%yrYAOR z_?{fzvVEpY)~;-(>*X}=!D9n+n zOip;OWG6?hJagvEA)diVk!<-k>GGYH973`%5ynkh`Ba3?$!jz^%2AyQmoDuKd61BZ zu;Cu0TgI+-uiw4vQDB_`QJ9_kJ_Hh=jzHMBH zqNLX!fe}ceR3WrDZseA{)aA!lczY*IJyU!71-~;dt*NMIYo_pI282%14W{m4QcZ;M z-GkeYY#DcDTVq!#FH%cE%V86 zXD?9yJouPTx5=Rn6G%l(X)z!!Cm&xZ^LES8a@%-2*xTPqTeV7g1IlJ=$l^MtQ-DdF z=X9a;gg=dIwaK?L0X++T_rtAiuON;Z}xJL$n-?~fPm z-*=E*1hSz@F1OrbUi${Lt=IdKC@Dtf84kAN<1DT$n@+Dap04U9`S%u=pVDa7!7WWv z$2iCwmHqCzkBG$)s2b0mXlFs$SivL}eU8m0?oTo`-GHJixGMmQaXR-?FVi5Xj$+ip z&F@ceQ%2oLYv(dTo)@kpLtY`5X;f7+BIl*d#X1oHT3J5qx@)a@WC-a_YvFmzLcWj+p-;>;QtC z3)$J(4Ri-~`uOFJ49*F^QA^Q_IT2QsWrcj;LhZTavm0pHxYN;u$mgU#Va+W`|JvRM z9mn$|-IwBI-($S+Y{~q?<)!P*(I;$Ept~HEm*jDnXQ6M8l?b?$pVYEo{Xr$n{IlYGClLUBH8r z%Wi%g^k|Ux64_=*I(YW(XOGp=DWP$dh7Nc!l#W#&?R)=KUsm|8ydIFr z+6?G9l8tg~HW59SjnP>tWm!SP`hKsvI-auUr7eNq2p-L5_xSDO`-&%Qu?B!Li&yIE z>Y8wkThA3~s*bX(L{rWQUeMv?? zrn7SQctct}i^27D^B?!hcYgbza`khQv)G6&qwV|QYkKe~q0V?|Sb$3oiPlD{W_%_S zd6p>KQ?$KPPoA`-5v9Yu-$22j9b49hU98hOCX|jZ%nTS+zmZOSevw9|63GB#l!Niu zJtx*kWTok7dC+l@#-nNbDmS<3XF-|(D&9f5DZG|^Q!(>?-`Wcf(J-|_wOR1u1-T89 zch~FJuWhQnP{xvcERXmlfu$yRql!qsbm@|*AtB#1{Ko2xGMM|x??)k6@(TmQCEX^J zXjT&2?YzhD=|BDcd|Q`;~{p9RO z$FzLU<;zKg?8pv863u<}{CWT+8NyZ@s>VQoUq;tEjjGvA(UNOz(zIz)=Y7nP*^L5h zH_aG7`Ze=Tsv;Smn5j^Sc6cIIj`rfgPA}2PUC0SEvHmW2A{}N=rh#NG=emmJ9&a9x z7`5$3-JMdB`@2w-yC`-N*mv;k&*#0TB1q55$~uvfvYWog_AXPuXQJ1ds@Mrhsb}@# zw6kseEdMvJU&nkYv5KZ5dAlrPL#?SaFi7dU-S)fnnNr$I++t9DaM6d4iG?=$e=B0B#2 zyY7Cb(s6oq^Z0znx5)2y0-@YAHDqlpd0PuuyswO6zj(OTxw~lUqH?Qlr@vn%=WvZw z$vDQb1jYn!W{8ed0j~;b|5w+3Stx9J3woYD*{R9{Ot;=C|v;;=_XWnZB)zHks762q7idG@0JQz&`;oMy|?Mi?Rk{9Ty9H(s|^ z=Nr_ka}+JnR3Dpusrl?GE@HFdEfsP4?T*Ra5>0CR1v#8ramOobOX-PJ-U)L1&&AUw z6RL{83})8mmDmx7r>)iU>rkAZO5sIy(i0YbrAv=T0`FKgy6(tz$TC?lwBqzZ(Ue&m zoZIFEmUNs0t$V6`ckkZKXWi^jMTsP1GUoud_(nuTgb^gg7HP4y(ZFpT+m@zl_d-e7 zMRPP4okhF2SrI_D;XyNo$Wt3{OR@A7#w6CEIB8tDw2#n9_n zDCb;3-y}FOmgL0!2R6F>mZ{#%^X(~B{{2rHtLug+`p(afH%PQ4P&(Wtjm}l<nWN-5PlUD7ta;(sHX|?PcPlGU5}|g72gH25q3Q{m)TnLjt@e5 zO1E=UigL@NQ|_NQaYCEuhu_(wX!t8)d|fR+ImIQ{)R2sE?CxN!Wd#KV&V9qok=b~> ze)wC&c(R)uUnUcMv#5`X*Ywx85l!2=i?XDZ^O&GBYbio@6D+iyzkF_;0_=VhGEE#{ zckW-TW%}0vYvT`Gj8puGqaDhbb!M|_h;5v9=Q{y^y~jFaJC{qiF?_w-r%#`p;=OT% zrgPFW<23GhdU>_loi&3V-$m38zLoFaZn?d3zR#0a65&*1NN>f7^9lYl)K0O?upVkOW;8nu@_ZZ-FnQ1Q)!++OCJ_A=sd3c zb19vLmh-3Y>et^*x4w&NZwEMst@VzjFK(PyX4f9yp=Wv(Vx%#YPCWAwjLX>KtC$#O zp*Q=}$Ppv%jEzT2`GC<*my(?h*u*+LrT82d-NtkW*|3#z219vr5;>tZuT?28(uaa9 zZr)WSO3|JVp4KwSE5||Fx?Q_=O&T|z=DV@w7#m3l1S5f6l04rW2#BbsWf9UO*sYA;R9f-&3?(Sad^4yx7#hBHE#0SZXFG*W#YDi_&n+Snu zwCKl9?}v;q3$!Vq!z>ze@|O|)SA2*e#rTYt!D zuO2&2Z?ZFeF~#eJUi(Zr;U(v#JMhUx|pby7LX1#j%ghu8W#l2<4>SPxt={p=g7N7?8eWRre=Mf#2ru0xFTM|6kJ3rXI~Wby$E!DRhOw#_ShTw_3&L9vKaKW*QrydCjA`D zT2uGuV|-7ETD@X$ebXy|D$Y^|!rZk?DduWoqrE<`U?Zn{Qu4Mc&=x4(--V_yAbgHS z=WC#NzFc_2K7?r5;MaXOp52PfC}!}rA2l@>TAV+9P(mbAD^8af4YZ?W z+N_vJdr8ICwtc3+Aj3zF%&Qo%{9U9`{=vT+nA#cCwg>}|>_l;JWeF)$d4pmqYA)b~ zQ1&jz?mR(2&Eg#moGS{2zl-Ya9X_YvGuVXK6!?aCMB1N;`~pMCN8lTkym5d+Kt%K4 zp{fu8NtYFyIH7QjBA4Xw4i&V{x92jqE>_q8plleB(IPh$y)_f;GmQTrLn<<}vyD2N zeW&#V^}mFcB_;R5z`yS#2u3DUkm3b_>KcMz1LaHT)1V9w8V~ze4HctEN?x#(7bc?6 zv(+;TjM?4p_Q_?(Dv}@L81VD^8Nv3b?cE)hc&45X*f95`HHHB}B1KX_UBXC?rLuwI zW@Lq<5|6-EH|YUhayzYOqfL{bj=O*{i2gy^sC4bwRiu_ET1|cW5yoy>P#o$hMcB?m zA271~lTX%+fIl0yYw_zc(N54!$t3UD1`mIuQ0IGYvhs{5vrzR?f_A&ov-su9i=Ioh zR=zY{EPNfM?uCquQDpR+A{RwRcl-0SZ6Vy12<~uB$ls(H;c5AKZwB0PP~c7|&ICFg z`Rf%={nPCSxt-ji-E%0nC7aqO5-p^Ryceaz;v0HO=!aw$q$2a`)vKMa*70oW>`PdE zcWp3RVFu^4<}kE2jLI$(fSsb#n#_+wWk$uD@^VYc-#~nj^4vMDabx3w-(yK(CW=BS zO!JLr8t1GR=|19GRUvG%BA)&dZvg`E#g3qEYenQyod2NFf%b|Uqf$V@_0rU@?{^R0 zW*B6RALt8UlGTCXGGNn;)^X$Y<8((kpH7nkDi|ox3DF<0We!5eKx(J9{oIGku(+2L z)|U>ZS1}SdoCPhC&sOAbyx@}o2*k+~J7Ct52WC zn46nh45AN~2zUQ)b@dK1M*!?Q&))YG>Q}*Fb4rJiREkTsv`U?i68196hky?|H5MeY zSAb==*j(KX1L^{PKQDI`g}3x76e2B2z(s@OufMH@#G?;QL1Ae}RcJ=r(uH8BJwPwz z``AVOPq^t$BL&Rn8po2!$IO~FYt0tCJ!Z1#q`^#0Ury=LqD2d$8VFsEcTK6rIMBSg z)i|6Z@9tu?>UF+YU*{5`U<|q1R}sJ}eGBD5Pe*2seb%c+omBh2#vTD(l^6cefnr z8T4FEX2@WpoIyL&m;j*&*YO-EQRRM!-O%4P*-OJ&y|E>$XLh)~U+sp> zD|?zXTd|X}BK`JE@_EkkAePoHBJys4G266J9Jf{-^m>fX`l|W5k2sX~DQ;+KR>^XD zIkEFDr?m(f;dnU!dlo}l9f~UQSqRA36>oZ-`JJ4e{rdg;EkQo2t_9s|}Da(S^`o6E)Ek=d^PQG+IS1D_Y!0<>L&^4hVcyBMzSi zHfURL@iKMq4l$l3I|7JS9DjV~sF-T1%=+neKHuDQ!#m_6t(7x~x(iD%LVHb2^;%q{ z|BLbvxLXI1Csvo=<&z9*;6?!-`6iN9I01Kb`^*!`$-A&4w08cIx<=pYaR67RlB<($ zk$JV(K`L4+pMt~mWkV?^^nZQ(K!yh+)O?Q~Ep@Kng#41pskSU`CjVT2g#aPC+s6iP z08`H;ONHuuxsm6xWji^WMDMpkIGApJKhMRxGE(*8LE5UIDgd0zB=2;CbCW_${nuaV ztn2!-nMt@mC+W4%3>xg+Kj$4*vWdY%>$T5pihaWC?`8NA#GS2(Z0hN**|zk$z*f?v zm~HFo3R;+r62#G(00D8?#8KGHjmN~ zm~I9mWS)W_{iWZ+qygN1o%Wf8^7c9Jkj6;CRFr6Xy~!R4V?_Cmem+h0f(r)aeQhPi z63DU`jCFQyiQgP@RHmsRWnBlvCa>~St76r=(gjUdPt|2YrE|Q<+kuk}XD}7y5O-Mp zAqaL$yaBkJ;Mtt$*k=GE%pVtI>~9%b=gkr>&$y0ey0!;LU>7`nIzu27teu~)IU~f# zt&7w>DZI8yC%A4%tCcHPj=VAK=$90F2ct)K2!o2qpG@sH_ROzSr%qejU;R1>@>THo zu@h0mdG2Ua^&OstyM&5B+tKYdU{|1kdI z2owDYaHh%|JlKL)5^uAB$yMdzE8(FkEn55C)^a{M6M}4Ez1kaPzJsVF1!bK*87Q%w z86@bpTnWTzlb*L%$9!?Z$ji_x9KSmliK!63st^nil3~S8LA?{k-bl4`N;|f<01Nf&630vUTSAKX@pByz7 z@;{fLpo+ups7Uv`itYhDM;qLX3ZX-|)pCS)H9&Xg^S&?- zNU7qi_}N&(@b-BQ1#EJY^C1wwipLkKrNzuNAn`C78W9h&2?L@{bQ_EHSd_NuiKFT@ zk!#?c`h}U*Z6hGuMUHu^_zXtWnPm)pJ?3E_fQHz<3p4#W~)j4Ln+c5;id?| z5WL;s;`Z3knMr|pEQ?JF!aS8rdfGf$nC*IHjX1bwjDTdpd4R~_>>pEt9bu@ zdwJ=(!G88FoqIc-`@Y_H-=)vO^Jpa2OFE5&RR~5y?F$-o07?3d0@DI@O9GOSQQLo1 zY+~1tQEZ?%a^YMazp@T7Z7k+WN$M=EnYm9NJ({*B9zJ0}{1n|VM3AFJ`2>$j40-&@ zT|?m^MLV!>d*w$4N`fGqG_G<#wKmWRtX{0TPIG3SY@Z;j_{+zl?s&Qe$IZ7tvlekj z1mLw7m7E-6s5sKfv<=LuHN;>A&~(O~4D?(Ar2WyTT{I8$BFtHID*U&wnJ;f09^XzG zd1%KC;{K+%Rn;pg3nvO&DMUZwi`xkda5fVr2il#sup7W+-<$$+gPkf+Z*=qZxj!h@ z_8iH2&fzAqbHlrg<{n#a zq|KJUE43`vh^>lbB*L1=Gj-DkSOf@NCHh3n&LUj|>31XCZMj#Pc4jTHP42xs6OoWn z@Af<8`A`#qWf)gd_WphDSJ~LWMWH9av7R&HaziXQpjq0dvOhyst?KLZ#Ep;*X_p#q z8Fqk9O8D|Oc}Lk)5-->;LSGu6FjKUu6Y?|P+KUo!r0RYyLr`bsmeSTdW#(cpmGAN6 z$00&DZS?hx&$EuQn@%JGN9YBcG)CRs@WEcCFGCP@84}p3T-NpT2We?WIq*?}F@q{# zDexdedA2QJ-Sq9VJFEXV>N`PZe(zF$uRR<~XGkc27a>fDD%A-?j$B3J@@nm7&87=; z8u3}OJ{pHfRp=|nAR{DKKqEpe=uQ!_(dGg&?M|M>x#v{(ZZSv+3fNGJmUQuW?xw4u zw4G_yUj|IcxQ*Q4O+`h*$`(A7nXKD3_Wk^f$r4ku-m)(4Knv(SHp!3pE3YR6Iyk0m~BCR?b1^;bb1>ztd&#~Mq2A+WnnD|PbZz{LvT33#>@Cwq=dJwd=(3#a^DHnr%!_ut8>K zo|5Y3y~Q6%pwIP zC4k^$+nsn-QW`}8nKwRKfbRE7Q+>+~Qg_pVW;IYC#OZP-gElb;vPv2`3C&LHOdfCW z_3bUacI{aLR|c4 z@Nl?`=wfq-amPFz0j%H!#=@wGJ(*`hKC!ubl!NH5HHk$&xQhq}mxi!M<{|v0-^}w@ z@&XhmTk$ewWn{6)c8IkjEl$=pA|m%fCf^rQPhF&TZE0e-2!3Yvy0r`*yd9;5YN|hN zYLuN(jyIDemQJJ**S2;xa=mfx-pOUX!uHno(?z$4am-(1&C+KU5&3!g?)A8sdn~FC-;{*CCWLm57md34G zxe>f(P(iwKE>mj{+JyR(eo*YihN(@t*rZjfkJqN2{ryr;L;9-WX|=pLe#H<%}SMinx2{tJ5%v$iEokF0{mK!J8J$#c2*bw{OCE!ul zZfR8yj?!?QH@-q2ki`(;5iIm805egs-dG%FFE5`v_DdC7-x=~~D+U113&_5czHs#G zUu~&BH~G78&sxl`f~WBf(T+G!>-6tLSXa=P%}0;U4~qDn)Owj7x5L8eJ_SgNTZ1qN zd0KOE#1S>R&YI57%&xtql!Z&2;g%F%DF*VLcE6-3&5@$Qh~{@mMiQ-%>)pg|1Vgc# z!xJ!BB62QgKE}*Td;8!hNTu!PlKj-!APLh4fBE3-fhe-0WutRf!JoM)Xu!_ptwcr* z#!+&Az{W1aEEY8O^Cggs=R@Z*$Gq{(eQ*zRFI6n}Mo$TSe;wrYo0^>J zKO59;y&!wy2AsS7bPXVBa5kwj$6(GkFAbLWW%^ zM9*WFwlbCZI3in{Iq1iS{v2F!kk@{}e6lSGgFe3Zd0(!f)4}o73b(xYFj1G|Ur5he zk5FwAc$&zOjfOmAS_r{7oWM?6-zMTv!Cv8X1SnQBUr0EfFwBKX0AKMsTD={YY z4@$-(Y$Ue}^sL%@l$S|#H@J6ivw*vmc` ztrEY97%}+#fjvip_ub;8D{iPkA;*`%08N1DTFj#4J+5TZG94zz=g|)a6?3_&rUipi zQDVP)wQ$y44U3-$o1Zz%LPo0vc|YFO&Un?AlLn5bUR@~DSTB38nVH|0?mNtr+^5rA zj%J6rA{1PKddFd|1kuX#x^JN74#=DS9r&b~32Z*%#6ArAEnOI<1o zQk3;yJcxhAmWeE7-hA-s)2BlWe)M$evwQ3};W}Wa=$&}cttcP3)g8Nc@1F7ne*dt-RkdwD$NX(@I|; z8_>%vk$WyLj8K14q|Kw}Wf~!P1Lu{!d9(4@pv216U;8W4PxCW%LuN56szHMWa(rmy zTppE7mWnLx%NBrE=znnF!;Q46qai(R5Fey3-Q2R*NL4vUp??AghLzH zadDW>FvuML1tJSUb3G zh^K(VljA(4bzt5eLjJ;nf+n*MG%)Q9kr)-4q9@>mP#E+|^nLU0Uks)MkUh;N);h#0 z-etV-fN(2h$g#-tEZ9s=sb{e%CMITmVvBBemhDIV6{wI;QU9kl=CIsO zNUEa)++*RE@K-* z=}zsr7C(gma6LbcxZ3DLW###0quc2wC9SGH<=b;%;&M;VS+sg(<>j|%RE(xyY+Cu@ zLnrvXo*%!UbCb@uERldLpn-Iz;^p#89a35P$)|k2ZZN^kFufCJf&3KF;xDE~ciz== zB%oo!l>ItCC>G(_MSwEsz4oY0&GAWa%zBE+Sz6npC@D=~FfF;2mh?SeblY2f|HBWr zhk(^4u5WTcJeD}Lq7{S6w+%LNm*IM(L=k$^1X#j~65#ylBn1B`dr1Lr{<1s><0pAZus&ek zB0lg(SU1MUc(imDBa7X%wz~ajp!T}uMn{R|ior8me&_(yJcSJv0Lcp6U`3FkJ{co^R_J72p?#G)ow9B;NbN_|l`;ZG;wgB-O_q~Ra`o?*dTE(zyjIBe zeDvry-CAMW2r|-mwJ_bVw~;&_uI>Nm?%e-!zW+bo*!b9iDW*0OE>e!6EvNX-~DYo{#6_`FK8_4tKrX$^F|KwLX*!Q=lHgc_}+4?f|+4IiA0 z(>LNct`%V50kwomaBP*5F0Z@@nKZd%_0$*Jdj4u9&xc7z>P`raxv>zktT(7_g*QWc zSD5II$haKPWjIruBgn34JN@GpCt6xw;H#{u?y%3!LkZlVs}X8t;QL+QVyF+VUfM)x zgw3MWxW#2mWIM2-Q@!|G@;OvYlSGkWel#UZc!iB|5tIz*+R46BQJza^Z~q)Ty;CuM z7;f9eU^Blx=&FFNVTfx9Ir@@AFE4oO>Gdy!9~FHDYz~92IhmJE!ddg>-?BD=o@QW&GV7br&l>5qZj2W$4c4+9-UKmXL5@U*Bw- zYsQo@Ns}Su#svpEkr)KBVQUCT_Ql@4Mc1*@%iy)g`uetbH1y5oDn`j{4P+~wvB0;n zW?uMl?jY~{p`SLPnJ1d5(p5+W3`X=G_K*|?8FbIZcuMUa0PzAOx@vv0;!;N5#+Lqe zDEnf%7}bBVnC}zkWv+{`a{Oz3-%p8m`=zW6?C(Ou3j7slj#vNjmnK;1Fo z-UJ*lC0J;<1XmsdY%y1z2xfI!>Lxa9Z#Camf_|2-fIhVB!HYFRzzFP?Gb3uk$roTIx@If4osFlTCKrM|$GvM%V}P2^%<9zHzilV@$l z+#uenJ+%+uy#Wp$P676>&-jWrF=JkRCjCpflii;2{!3hP`s9J~1VbaTcMneUf#5&k zZtBB3A-2@*uX+P_r#rwdvf9h2P1KTZPAIv4*TaAMBQc7~eQNJha>BPs{m8fH?=;xCeX*ypIooo! z*$fQh6#t;+;S4u%m-j=d_|_!c`biA#&8;rN3lSC zEYSA@jFv{WY7!*~N6<#c4L*N>yw6`v`R_TG0ZpAyc*<*R-8m6BIJD&*!Et^@)j=1Fk3{4-ov=Aat)H| z7)z}_6;1)+Mc;2RY$q-%1R0E652`avkIy5?NepnjcZ_AhajujaAatZybc%G63N^Xd zZu=9HIrHUDk9hb9ruk^+;uW>qwgCZRaK1w9jaGp0#h|F?%rW472A3=wlMP;>(E5W0Y`R0@H{6^pB@1)Jwzk<@8$UkpFKSvX_P)rCA4eEA4IT^b!4GvcC?^k4=9%7q`q&{|6_h*T_Jr} z0~LW$i9t{N@@AVMvc{9Pas)q^>teIWcaa>`M&ILp+zHX9b|ci2H}L!<_LNbAs}t*y#|`q_SrOInr(ikrMU`TIhF7GT1H)@isZ*mL9lHc2$br?T`P z@~Ji{_A<8io{FH_2a^ez8iD7=)3qaNQMJRBjspWZu!ZcFit=8ohsTftsaj0^D075N zN*490pa)Pt`;^kxapc4s2UwYdF+em=|0U2Zh%jz!=aP>c#X9q7G-4mah#nNLjyjlrNwV&Mfz zFR5v&cQB$0r7R_1aJGFjhVaP8exCl7zPL4UMRi?y`LQFCO)d>z3a}U$yHK+hp$F-< z@`EIKRe*{_(M!_^VeaERI`KS;rRkYP^2!Q~;V8I#eP%1|>p9VB6p+dNq@*0MuhfI~FrQ(P~h&^5*8tu6Pi7p7)FqpdOp%!Ex*=KBbpKi0z-DU--wnZRkI5 zx7uW3sZQV{+fBOZTtRaTpHdk@mpL#bl;GK-CFFbg!)cVSfy;D^? zn8>(4_J`RQ|GTU~$({h?F#z98Go_2o6khl9bNsvIc;xLlCDXUUKY}bGkdB$7Jh5 zfqa;j8;D&B%L34Ta4bs#3m&{zP8h>9*eh!>EwwH*(hbxBrU#)tVOMyhKyF@{Vu~Um zlm_Af8F#<;h>^q|cE_%@{pZbu{pn1nkCtH})R27)gWo1RQP8SVcY$WKZwU;ipSqM2 zikZUQGUPB_T0qj_Xo3^yi3Tff0Xmk!!A4-o&NoHv()CWs#1(!Z3bS&r?0-H1o#c2t zyk78(&ET{bIflL9Pms8)NFiOop%N|(tBSit*(E@DtB@p7@K>PlFZF>d_wEJXw6`YR zKjWx(FJ_T{p0)J$^=^~)-HVIpPhT=^HS>ReZXCR-|D$ly()e5^#oQd)_7)7lnA+Ye zM&N6d^^Tmj9=}CxmUHHQHuo%6C`E9dS3M;cG>;@=uS`u1jgR3G0^oJ8da?+;`F5+S z<#`{hLgYzr>BmK$uS5Kp#rj9p^wuiQo4Md~k2gZPLM{?S) zdP1tX@l5d#-b2(<=DfRX3glA6JjXf_GE_yn73S_Kn&DQ;otCwF8kRx!w-(gWfFHGY z3`uS{2~t|jgFjD)B0UT-tOu$PukKLQNgQNitd#Q+tV&;Q<%i#tQxj<&#QZ;110P~O zy+EV8sLufldKJddO16WX`vj2{CgBnRU4!m)o08Lx2XhKfigCUYN}9K!=aj{)oAd^J z_&Bj`g}Z}x^1>r{{c-m{w)L6#12{{LTrlL{lSc7!E-SpE5CqzNl?1iVHGl zzCOug@rSs(z;nLIp&5_znO<6R=Q$mhiV35fSIA(NRPOUqK)Y-%AQHE-4PD8qxV_@5 zmoJ7_-_DgS4Y!gO5?mn|#9Z$Jjl zk{rp#gv><|MTskfjokWh3rWm6#{$WfgibhRz5*e1#B1iStU(~wN3OlKCgDo5@GUjI z0wAqSUzhaEjg`;E1HSjWkrBA8*EDY2SjD^RrKJ&&)wAN=I${2(t}vw4nUJIQCO3?aI-H!j zg=|c1e5z#xo90m+ zkus8B3W@r<-<%z*rvY#6uea{ezWveYa_1mDo5O>0O_@GJp!8N6GST$-7c*D^{)bxe#Ny-$o$pmwyAz`0GS1N%5@uMh zUaexb@4E&GijUnsj2bF(OZ@G1w()N7(vU~1EzVI>cXey`-&aT+dQhVjdIW}44sRsl z4@vbREY0^p7;hVwHRfixe?4;rI35;k4|j4qKi$KEuHJ{$6E zkG2-h+d7&`if_{YFOsEKulxfI+VT5fIR#XTb+29b;D-1(`Bw9iCr7yn>WXBgBoxUn zu-U87UEU@ixxf-_bu{}8kST>$#6XRNXFS_$d=u5`fl90 zv&=53zo#*(87=w?;-i{<-ImE#*&N1O_YQ3It-6j%$-VHOIHY_dmd7mIL}GFZMk>A{ zqi#~NC8__1w_lF5T)z3zA7ug@PTd7+XD1^@-^K}e6@wg^N617x#~$c$xzsM^{Z{yM zqQgT7sVN_Uem$gV0lFz^GBm5MHudT6TT~YVsMvtWUQ1(?G<;F_?IfINr zV@i*#sl4oH`S+OnAMEG$PT6NM#b9p6!sf;{Fa=mQ^0kv{DUy}>Z?Tq`I`%Ftrbz~) zgzHjs7%lUDbU@ZL@5l9tHyb;q?g@j5$=GD`?mG^XEM#xW|`S+S)8i(HPW4({4b%nCl0ATXp}w z*`To;W`VVSzvYVhHHhx~JWs}7&|BSe9s+5-%}pH#*l(_&#Gndkd_|M(-P(Z!TYd*|KH)BnWVYB^k zMz`#XIg~@Z&NUlJX@FZVW|}2sN7)q9toS_q)mzti{El)jS}8qB0w% z&A@11g7X9dR{7#aqy%dWE1%RYE)bz*dc#dnhCFRmwbwze49b{!;0zR9ZSRnDs+6sI zJrvCy5W3q#qjyx3JP1ume2D|ph}X+Y3-EjgI$87sn&ho8e$ESmGH4Qgn=(~LBjGL& zlq}~kIS2|v^C1$=0=X;V9G&TKie@w(%eVzKaRiByVtx0?2e*mP;<09?J(dL4Y`tLk zVs4$t9r;%d@ht`{??3b)6d5OvBSC2*~a&Kxfi|k!g3!!-r5p5|MTQo)rA$8vcnx9zn>=rEw zu8^}D*{7s6y8<#h4YPb!@Wsm#?UE15j7`2odNQ(VATIP^ch-IkWRCa!1Ao}Y%&eCK z7F$hDdjcNVKw=vjMt(!Oy7X$|upzvlH_cTvXIo8LyE+6m$vg@dahoXpUd!PgWSn==6XKEuJS+i`7~#C5g0z9^3*Rwq&}Jwk2cw!9<`%Ol!*Qp${&74RgJYx8Ox$OUU>tSey&Mnce*SX8D03QM0JMsz@LC(aMx7)gzd{JUn8TS zIWmIx7&`9_jXV6Vm)w{|7&Yk{6l=RNogVHrOeHBWs}F(V`(~iahU>C8goep+bRiorK|rY_Ruz7uk`(B4<(h> ztyhB*O&m6D16L{@q3Vw0JV~N3o<1)ODP3TdWbALW84a1el~1K~<$ap2qw zV0>SCX8p@hKYb;k2THa{GGD~{kb|p}<7#ab11sAK^l8`QR+lb2PKZqbWp+!d3a_*( zj^5tobtu2U?}9&(^By7%SN4FcHA!+9an(^HSD2A?+HTNjaPkj$Z76x7^oc|YDd-Yi zFyVkmNSp6xo^IX^bSKG>E^I8Nop)O!%)P%Q{HAQRJ=RiBmS9G^FOr*zoYqRdHcs`I7*=dU?WQU!7p@;MNaYy80Jocc9KVF_A}dR4-eH*k2mMrDXRBnfTJSt0 zeB%2q1}g@T)FJRFy79-++&67(ax)x$-f#%o&K;q50g0-6Z2iYpHto-!o?()6e;KcW zP1XLA7-Y@Eog5?%Qz-^>AkD=%+r4qq*?-@g0%98tX+rh8(eDjy7L8Mdj32`!ctHFR zx`_RO-N`4{m+h!{KXVG%X#f2%VEkMQqZGc5OM?7$Gh9?>7-?x)2;TA=#BX%n;0887 zg$Tf#q35_-q+PDgYmHe2`(#4ee#?0 zzvtI(IUdhb)LuZnU3@70UeEHOSqR%6t!&c zo}GFW#ki58Xx$hY@Ew_{m51;@@-DlKT=dTzcR6>=$(qtW=Hg&?#>LLYg72)gle5hk z`>kS0lT+aW2j0iuV$W;VQEn3mRLylNM)kjt+BMX73G_JY~VL%W}B* zXp-PBhQL62c2)V+_bPrAc}@P9F*?T)wdrKs8@Y=w3am@EDr|jrv9G?EO_i07b;W;w z^+?%xE>_w3-(MY@&T8_l|F5q^#pgOq|NUD#BQ}2!x^(5gzkbpwZ2aHeLpSbd3f+Ib z*!uOgE5Aql@9*m76U+ABFLp(6|3m#n|9!zLdYJy7y|}k^!xVYj{<nI($*eR;9Z}n7F%LdaxV{CKF_zVTkY%X zd-PR|q@Zr%PS1d`*B7S8g?|0`9CS;@wku$r=gVOSaz%VA=u}@0^O1j2d~$MJ+t0MW zz@@RSTuFOV%ip3hFhTV)L#l2P!v?miRZclMIYtFe$+c2fcP>&_SC6@QGr=gwBGvTS zHsM2A%xTiymlBn-(+_2vr5;R=YM#0H>-%%O*nv!Anem~vLzZzdF`S=DJb2Eu7pd;t zNvEZyMSk@7WW%COn>KM(CD~@|xcF=5(8oMi;d6cUYot%TW28n#yEWQRzVb~;*{rx@ z2Mu+)E#I%W*u#J7_fXqnFE1}?`%j#)3g_w_OJ?rb2GcHPdGX`(GfDSJ2fNnXo~65E zF5cRHCe*ei>v>xn&+FUU5A9X(np4a&eRkh)_xkX=2NKjC=H=~by05KU;_j9?VVmCV z?6+i9`5Wc=49Owf*{5fRxXp^r)2OoMx_;lVd2>Zm)5nqCs?AyOm8K-uCn?)+)<6^wPMvO`kPt|JUl!fK7LGe?5+|tNIl>g zkh0rt#>&P0+K;ZTE{Ca~)`}N@F*`asO329U>Zy%9cIHf2x!=+g#!7f5=Z4!WLwL5c z^6)IBVq_iHi-?H07<=Igii(Q93=fAzM6gGQ9A?9PyPTKjdf!^vbLPsCC-(dsHqaeA zc1%J-LM2%<@@=m5K6X)KgOrIAEnSsC2?qpSrbS$woh$IlM^Bv!9B4}EvoR?Ul9%VM z3|vJ^DZ2fnd;j2IUEh_G;FfIjin_X`crihfrw`97S><(qyRUtlb@gfn%E85DXV7ZV zpX)zqdAuIB)X5SraP)XNFd?}WOKB%=Y{SE2>YoZ-w=~Vz*w{S1_*-o4jtjn62!iZO zj1C`8(212#J@Q0seEhNUnUtdA9yLc(ujsbu%$YMg zH8d(38`(sSAD5l$@Q6OqbL0B;18I62i#-0U-XE{DYoIaN)rBgGO9NMn3J0%IFg*YB>(yo>tl)#` zhFn#1v%fo*QodMD>B&Rw#YaxQy?co-Qd6eN^7!#hB4$NKd$hG*cdgYAbM32-m5+Yr z@h9KtM|X8ttMPBIy%qk;Dret6wrSED!{alh_=0)1iyivev)R;=SK_EQrNXJ?KFJX% za&D;gkp{)*^8JIZ3H zr0MM%oZBuT!TK@(^cHMDg;Mo{35uRd%1VN^jg5`IQ)7Kgu>-j~=GFX`Fgf?WT7mCM zM|BjrJ$jpQcHJRV!E+Oo2{$6`%$r%l)#M|I;#Bo#(h$+6>J(e(0Ct6+<8?oXtVgvUAwOS zkxm*8>R5t(oZ#{27k8M@(xpp}Vg1XuxPE&l6#27se(vJh-EleDqaLqnCN^%*S}s?t z^i3_{0HgLT=@k^Cy}dnGZ^)G^toQfFi=iCZ%)ZKh_>lF!*3I=rZezPG*VNh??iSy= zb?dRIpFP}lea)`hrKJt}-=*ECl9zYszbiBMNwC!ZvpkLW+idev<`$Hto#PuFrWw#n zY7c3It*yMquTDx3rP7*f-B9G_{Ncj~mG$9YCI(xcx(u+j7r9-o4C1iFI(ImCu4UTb z%8eU0WG%c+#Hs_9uVZCr_r1OS3|?!|U`uv&bLNrJ!R%6tkB@C`R`vDt_-#E=Z#B60 z?74I2J{4c&ppM$wE?vBMG2Y<|DwQni)OrB{Nv}C~vKLWHYn*0V59V6!6)}29QL?V; zo7O6x_f|OfLq#|ecjU|T#2^+h(}P3VaUBV(1W75yzCTdvIXlC$v5tKSeebKAv{YB= zN9?I|#pJ95f~;IzS8;vi)zypp-oytK7w;gixPDh)zd?1Ml9#5kpde!a%bG=0#Fj%W zuW!jzph%2%RxFOqUp_hh&N|zy*kO9yhEhT0X-M3)H0Jhg0esZBxH#O4>b56m-p9QV zF?lM1-W0YgX!Wtqaz6)G*Y|G4^sj#nwMp82kYMzoP#W=>4IUUU9Y21Yartr$2L}oJ z_97hzjFdLiQ zJ18i~c&aI7fA_~cTM21tQhtuv+lQhDeJh&l@cg4)RF87K+O*I`6&K?;+`i+Eve(+% z+fJ9GX*poU<)wI`IWeKoP~MS|t9aL}(apCzymR;N@`?&teBSa7ulbNazlXbV8E;!! z-u$pnoWFwZlAxcWb0s9iFDFOxOJ%BuABk*kiT?Cd>FfA(YLI1hXlGRj52_5!!pe+)@?^W`%&3NMKmaY3 zk(o*EaDvCqsY*>?J4YVeQizy;T?vEEVg z;6~4$oa&2Tv2IB{=SLT9*11zw z;N176zn^@n=bt_)w$z1jN-kQ;A@-uZ_@bn9uhzRXeT_lS&18c(Pkr9vgZn=BOw8md zpVD(Z^w7@lt&h0JlJDJPR5<&cRWm}6pE^*NQg?;MslWtR>3aabBbj|arL?i-V1|(- z%E7aOPiwr7J+czAeK z1A7Sr8grN6ie=o=tY)R2nUdz3nt@+_{18*TFy6mZBStbb;+pgG=g)n1u1872W?0F# za^=Csl`B{Jg|1Nu?8~{YyI&_>N%Z{1i%a+J-Rr~0Sl*DN;p%z4_0608ymD#i)fQ_s zaqC&Tk32daVmt*f!MJLRA^X|y9}>2e>*?uTmzBDd|Fhr{KBEFwBK?*vTWGwGo;i~} zx_1!mh3UtSAM)?}zJC{?SXYaiiLBksE5vW>LWjsO(xv$9K0H(v-G96#gS=#(ZOdwk5nVw|P0dI1g1yPP z$kL7D6BF<2Q|h9{g9I$K?#N2+uT9CWXl*@gyB5nKQ5@F?T&`PkB`mDkn}(Jzx{jbK zH2)Kp?S1>$<3uA0G>0zmWAMD<_v=QsBg0 z_v)6+(yhl|m0e!E{Naf=YngdvCH4*SZnx(>)snUBrq=x^6byz#hYsCQILES1$-S(z zBA_98U-Wl7<0wCWe+zUf%DeT^iS*#8k?*Zru`Cx~xNspx;hY%t1wH+#+t{Y|Qm^H< zwzjhJ@`+kErEX?vh(&j!NI&(OSHh3N5_bh9+BDqOSOHF82I0v^_5+VXd+!0jy;m9qTBE!Di9C%~^M1V@I($V}C}99M%)P514fR{CPf< zVW$FKw8hIwNlDFa{^!1bxKviALV$Bsk1&9iRZx{M=FSg41dvlnJ2yTiE0Y%UhEQo;tQ z-rzPBK-wt&)hHB?*6jBj{q<|L#@EB34b5(m6B82+K#p32b>9k7Bch@hQ05igCt1r< zJle8M6=H7P3hdJbc5i>oSyE&^SBZk>VNYl7q|3D3V3@@vR>W{Hl|Uj=ML)V z^!NZEbLEZA`yF~}R}pkrUcM;v$dkDTy*6Is{o#}2zdjW>AG5N$cirB~%9noW>ay3b zmr-B(`T}yT>mGcqsZZUjDJCX%K#*NRQt~+p@UfF8Q?Iw@*&f1zV?}p;f!~v`|0KI- z1DoXMni?jwmqS536!WrWIZi#AcXnJ{{U&Zl1s0#A=gg#C_4G#Ti4O*v`}S3!hp*t| zyaMplKR8HM#mV;%B6XaL0{r}*;~F|UJ0-Sn4~UJ8txuXQy7-#~oA~_H&y8Q5u&dJaQw}_JIF_f_+uJK|x_pPC;+L*2A8wgbN731h9-q`T z^UcuJX|ML#gtl_WY@Jj0bj;0^{qdxCO;1nvw`9jxDf;joE%T>=BXQ0BDhqu?Ch{xp}f3fW2$G zvgV&(Lo4I8bTl;yrmb{49kzDIgQ$5>cJ6MS`9k3S`l&`hkCj!SYqsE7r}xNg+qN1s z@PQ)_FE209@2i_z{j_gyv$+%f1nZz26)iVURFjs0nU!L!2w2|2Sq*gUXP9YhlaT`6 zT8<~}B;#pX?D5CY-D#>mR>62|;Acc3sx_EDJym{3&N=sBKvu4lUip<30(6ubfZn^^ zb~S;3C*T^=xM^*!UsF(Au2jD&zW6)WbpP2myHr_yv7Q+ZJ}}VTa_HWk70250Pj|V2 zWeeuz%E@`6o%+eUemki+j{=+N=!JcG;MLejr+UJ^Xt7h)8`-XY`TqS*(o|*X1{46v zv*P07A#ROc%HVYN_g#Q!pg@jm^~R@3=D1{PjHiuHj(+V!5? z!k0tgY`M;MOUhFQjd9OqqP7^MUVC`-`Jzk#6DA)%KwFh=xldSAoq!u%^U7Z3W&rz3 zSBtR5Yt7AApFS;W*Ip#oLWJ97~v*N%LH=siK7SZTBnYsM8J-6~~nl?Gc>gwvsXX^_k z%|4hBrBOLQGnrHM6Fi?|41Htsp}Le4cxQ_B?9j(Zzf6GK^|hwF3%bfr6ThM1|K#*!13Y9syxtDxt z`jkn(*VvdF8^RCO!sW*&E_C1BE-un|@&VweBF2vy0rcKa-ovHemN@>36Tlgb%Tebe zKpwBcS>{7oCaP$71kpwa>Z*xEM@0BSRl1AMeDZUljNR$ex~LDnetw;e$@|Wa^)5r9 zTa5OLi?5Byf1Prj>>i(3yf4_>7I9f)HUTyZAoj!&){{SKn z(35a|{@HLz1$NfMGaX8b9@FgTJ9-ztWm?r-hrDr3XDWh?jqPort1XJ}V(KlR>xb4> z)lD}#@di&XObGYZ#|Aw))6tb(I@kDA6o9%K0Q>;B&GP!03khFLM8xwF{2=<_WloVT3R1}`x zRh_Y=+ie=@D2Mqu5BJFt8i*1#O;4R~JH@$BM#UFoOiWE5&Fpm6sjocoChj?I;c!Q3 zKzR6y!gIr+C^Up2%AWns>A#F8s{740ZRi|+X2loyH*Tay`6d{sHbUrfYN{ZwvL`Qa znM%50rbBntGWXw~&v6fupZoauynS$R3lX@m#0lGsQ}PHzS4jdqijlE3%&LxSLs9m_ zTXbQiOb$KiIC|nlij=Xclao}G$l*m4$b{aJ5yEy(mrW@cuhVd1*y0fbt(#~vI^f1yjAdY4Lb@#00o3l$Z4#m!56!D%2` zL63Q9@=3qo4H{ETO%0{Ag9&W(OK0b$#mm?7iHa_R@TP&{v~|rCJ%!$O+1r~6=GwZc zuG&`T>sODN?Are6k&RhlV44(x1etCTrOUOov?R`t9E}}}(o8niD5=|HXO|&sVVJZ# z9-tDYm7A_qWZSpq?y`Q_glYgyh=Bl%2qArff+b&n_`q2mwoW5OC-&X_-H@Ov zA-?3`vAl{^;86ExZa7bCEEX+*2DEnVS|E;t`A_DFSaE)Iz4w0o?SjkJDe)4*i)Xq5 z`W<9R$sw|rp*D7QcV7lvAR6T<{k2rQeedA){Wd^YPo4W%f`fw{x?V6nwr$bb=z(U> zDd)5S4a*N@+Scy7$Hcc)gsgu5kfYu+E9)`+_WWoFseCSUeIF{PjS^z;>#|yc3MwHafvXV0 zRfQuhAn|dv$4nK4zR|uTtDC~3TFAI`=}Mlu!RE{$6befS6+m;sQxLlSz2;{R4YJmh zrBq!=Ggwk!61wG3mV3^?ai@K;@~&_1Yx4kJAFU2u^VEGx3evkjwo)Q?X)6A^AIt9K zBo@lGb@!b`)t^6`_VlV9J<12|<}$!g-?wcVJ9iSKOsM?j%d`;d@BKXd#G-PE%eQw- zEm@`&Bb^nFAPgZBr0Sxq3CHV0xpQ!GD!;k)#KTkXgVg+kB8eOBbr^R|Mb~20Z~!W3dP!Cw_9JwKM?Vc2oyjeX(AkSG(?t~9PK7Z zRx3t=CO<#lqA)vKd~ULPIaLN|@$gs$L*&5ax#>Z1t#~C<+fTK}-fh;3@%EaV(h%Jt zC8c-gm*AF?@dAfGzfM!$d3iFR&p{A((zO+=?Ch{-j~{*RLvQ@Xk7v967l1+4n(G^7 z&Qdj7L|SgQdV2i&&gH#7c4Vs_^k3YNpeg5c_Y1W;Ottm=#6_A~AFpu=7E|K< zUDPqhm>Te%Q?H)@tsKKOoyrDFc<{BZUYY?|kfRRzyOyOXO8F|f&L-Aj)aXUjF(@p; z6ISP@{jtfHMhYKfqZpw})-;c_xXF3YQqLYXuAxW7qL*&lwrvpw?P;lT_uNE_xoejGZE{&_9+UuEaIx%z z5O^-uzqw_jIcSKi^e9Y*yxzf&dFfD$70*Acb?O;kjGM3$KvX8`pbmec&CBzqPj3vRyQ@kIZ2>Ylv?}Ato@jGlEWxF0S9el;(Ce8Xg4I4$ zf=o-}jYa%G%TnZug@wfn%jvObJj1sj5uMog+#MBn?gS0IxeN$;QsmsP?}sJ#+iQ17 zocm!iS`5ARZA^7Xhq>Z>_iWE4}Sh@!V*5DPiJ-tit}Cp1WCI=IhjXaM!r zkJRMYFBqELx_{N*`7 zre9kogo;1iD&gPK67Z==={^TL`-bN3p`lC}^8__NFag|~yk0rf$EuD1ui1%cV>8pN z!$WOPIMD5Li?j>BAL@B$y++>UEDlmhpAP(`P78XAE=W#dAQSk)q0Pg6PR9#vC`4@nKM8ToF`50xLdzlxc0pozi zGi~{z@Js@s6A?gns)>)$hX~Qc!~|L2t*6IEr%F+d$~%Azp-^xS4nq)V-Zs7!>=S(* zK8!4nlDjMv2092hGe7IZ<%|yF_0-+5mvoT$ zhzNQxYU73tWpCar=MXbdfiV8iYu*z8Ut@4IJ1unF$u@$H>T+SD9De^WMWbUupFMPS z2vTqVha3U$4N2E;2E0n{Mz$Wni0A+O62-Q57OfJmf#!fFav7DAg;&-aJ?6{Y>>pSo z>Lw<<1o)wNKW}ee3rhgLlHBILE))jJ{eoV5P0e1*0`!iu=H_h`DgBSWK{TP1v_B`D zmYoNNN<9Ah*2w7SrRZo5+~<@ijkXi2)BF?AIAEOSiBxX4c)pX9gA z&diW|hBe72DXBY?1<7E!ipOK7o`MghUS5$N@$Kk_c{)P}6V&KID_Nna?E(XO+#G(; zeA?uI=Qcr&MY88d_&g^&7fX6T=;D+(O0ld}WaQxBVCI(IbK2?=G*O~gqklGDFs)nY zAK~j-Qi>7>!SS-gYKM<_|C$wuuB#k0+g-Z+4SWI6`83fdd3lw%r!rAfGq;lAc)*Os zsv)XBpiu>+iAOFXgSqC=YH9eIZ2g=J7_H|%3CG9b271qj{G5={8Y8kr$J&KJK%#j4 zt5;ex3bWde-b@U4m?t;vlPmSGhXR*Wb^h^9X&biYjKi0K+C&cL(m`c=-F%^`vGFbL z0eMH*aGCm$uLNyl`LI ztozfSkTonC@9l*&WbHrdF*%~z?F6QK_WH{#Y~A&c2C=q@-e^2sc8PWr;IkW|9q^3P z{q4t{%CXkVQf}X@O-Pg(dIMvghVr?5nV)eH?0n4~11UEr@BEzAnm6l>!Wy9_U7YtG zNm)dZAOed1=H7V|v&wA6P?zFMhf)vlXX=YjPEOv1{6dJ$?hx)|CuiLjd8Z@O31Q2a znIF219nY(0!+z(KImOD2BaHg*QV)ROa8lIgU!h`q2K07+xUMy(* zI5lPtDk~5=Gc3Mc`0)rj=IACJdqUf*g1IgOpMU9ZNJ=%%6E1Wa@Ik-s2GAvzoQSCC zF(4bT^4OBwR2kM*&gr&o0BlbS3lp*A3RXYbhKxcbaPHPuV|bU$ST?5%c4p7cF0-r( zMuA)|70f?-j$W1Z#qjFdG4mV2jrVn81JMPUu<^=(4d15eZ~CET^XVNmTMEtA z{hXeTnKOC!D|X+e8wm(J?Af#D<;d@2(6$cBFFXE@MsshQ%sy7S(e3SF%5zuHNKeQX z`(Drbb8sql%f^ixE2bk_3g46_^*naJFrcgtVf@WPN+&}6Z!)Ju7GQvrlamk3R#jEi zc+A67-H7apJsoAHz#m#;;24&~`#XEK3i7a|b!~)J-|ra{C>{*VoZf&>Fqm0UKlN|l zD7yQy-sI!J8G|1m^Y*1k4JZUWEhtF9Ln8X@)~)Iw$)s%)>$@nt@7uRPJO=~`LMFI4 zIj#QO_ilt-Oyhpb)(yLxd(7y_5dmzFtgI}zk3y;i`#S(m(I2gEnI|0JOzgr1O84rw8|?8)QsX$1Z&x$!uX+~RN6%zc_|EmJ zC~dk0PS)4}i$Fli(FPm_8W%*&BDwGz&XVbUs`k)kHBOxprmhI6SoQ9U_cH$ov4*5b zXlZF@{tTC%s?LFQn}4Pw6t3JCButE+ofTazVstHCGQ;%Q>W!KaW$+nBAv8s@ghDnV z#hu_X>g|Ak1BgftxCg{+=#2DOjrH%(kE!W|X`7pyL;YPZDY*jm=yx>OCLLNVsCgK!7KmUgU=gnt-d}5-Ac7O{GfQA|9 z$Zjy8%o9=F)|Ziy0c|3HTrm;$e&P?w_Cq9O>Ze>Z&v3WbF2k(lZXO<5UazdjW9zD_ z7-(o{xaO3;e=682f9{9KoN^M&ft%rbHZ1UK^lg~3h$pQg4aVO(H`hFfU1$+yk#hPp zUVkxlDpt+eSgCk?je73%j!^J;!6$!XULKB8H<=M0cn*!h5upJ0tDQhals7C}s}J5c z+QE26TFkvVL8ORRgv3<(*fio0$OkA#vM>>7rqfSvIGtdq5yDMxYdfe_chBpmJ+ePo z0S*ohWsuZll=iy*{JNKJ@!~$6GHw!0!Rn@^irlBzAOqG)%%cp9B4YIQaQ73Z_$WBi z*aYv>v{VbZ65W??eKwG-^=6R2S~=zWwTQ!(*;x?+&#=f*Q8srdY>W^$H*>#Uj8KR1 zSfrS#@ywAn!3|~ezix=FDQK*#k@&z>bMkhsIvyMJERFz%QZ(rgLj0Ti5`R8zKHFq2 zv|YYhaFwtDBkluSC6DRx2AflU2oLmqoOTVru|gX0A_>Q?7b1}b@WH6NP$i91j!I$k zU1nJJrKg8}-MV=|ao z#zm69n)WOATqttmPpR&O>KFLp#m;vR4q`o+yqz?I%gefaosz+kYy)9GwS<==Ca=CL z@=99JfsgwGI(8G;&~sK^QBjeVmp9nlbBZ1$oTF5u|J%O!r31eU#Nw9?D0BR1FAjo8 z@Ej5eU^2JHobFj}2p_PDe~XvSBuc5NpY3V>6LV91HtuVF#=pvVQGItu0SSpgWFl52 z9NGe>S5{T1NqMKzKQf&UL%@KNw}=8POdjoZK?Q{Z z6=yqoD%Z9}jHJ^(e&hybtCbjrPK@Rlir0J&f7T>xP`|S9tA4vI5sV<)mQ1d!lQWHX zk`9O96SzO`RH7YoHkMRqbM(R5NvHIrAg2$GeBEvyrR=y|5FHXt;A%J7TRQtOE-ol6 zj6-AT%Wr?iWtLqwy|2YJdZy>KG~WpAXoV^&l)HzA%nvWB>vu&knlUQw^ZI(0 zvXsGcV&5S{LuA(9KR(y#ZH^cB5W6=#l2~^_eHu)!&&0xFJzVoUofjgK>Z!7?H8-4l z7@X}(!nmdyhxgjo)-MPYAe3G2n7KUqobYg=H4|41@|DHcm*Izv_n%Ex&dRH)_dIyk z5Vu@3PSGvVv9<(|h7ZCg1)1owiD-P>PZMaM?)kf4EI0_u{>D?M$V*J3i>yEz2gRcs z@(K9O%drXJzMb?+J1T*Z-ibsGzfb4*{{6edSg$sW@hwd$jrw|eBi&)j*nle~q@+ls zptJ@`1d+wTIXXecK0I}_jFR7Y@BaP3{mnMFZyf{rxv6AxM;{%L@`j{u<3253jE#46 z;@f*fa}EwffCW(0&H9|IeI&!A|LUzxYZS+(CF~^_9tUa%av!mApsHU1p%Q^*6*^@qzlqw z=&*+@cR-K!S4;3V5I)G&`PFs>FE6u)$E=oy!(OkI!m}l{H3g|f1vAJU@$R@_(5?L# z@t>>vU-m`887PN$4l+?m9-6y*U;droyWz3JOHtk^YVyw?$oYERIh+u-;@#s&v z5vywO*t8BaROeck4|7P2sk72S^(0uDx|Y7bsPFxsnD%JT{Z8sS3tjLXVmYu|DjOJ4 zm{TOparbo}9oI%8sx#W$6N009oOuvLoyzco(jMpPYCMM!X_K{hpMSdTA)tSrVy#U# zMEMactG-iH;}Lz%s9~;wv7{01X9S_yIrSBwmPk|7LevD@J6mQLpT-w5=n~2oKrQ_q z{o=(-DTo%}0y0t2bv@-*S8n!&w{gV{ep6bMxChb8-Wx>+1O&vSAPBsOB4OoIQ`c|Y z@DnpFWTfaIAatT{aeWWTXtI7LQ}CaRq_EBA{c-FRBeGD&buh%Q_8{Mxc)ho~TL1jqj?x- zQm==HhD3XRkBzO2-hWeS7sZEaU~Od9;$B~0k4uCn#uX<(qA;+(vvOZS-l6!Qkm%bM z0Da@LgC|%iRMF5fFjBto_0yB_qB;~GY>cH(>_6-64Z3=D1-2q=*YNeb5M_-q2~pk( z_lF}}b-Vqix3|tfB!+C%*r;c_7XBZ>-1vX1s&)@?HYc-~3cZPh!ft)*H>?t$$jC@I zlPy`XL@>PWIoh?$aUEg2wmV?3Jj0I=$q4O!vz!Q%uU@_Kg;=-*ULE%s2@@@diO<1E z&(F^lYhHjA1^63OUDem}8ex#p9rS!8qM2F$omUyzU7wpbZ%%SeOid-TuUjWh_7W6| zmC-wZ>w-6nh-eMUBh#>TrbuLun0bkaJv58hckLY{3#6l?V-bh7aoo5MRi<(V#^@XA z(v*~xj~D1Wn=U3R?j{LZ61Wr%Q%hhCQ-v66ho;pu;Za{vp>Fi>=p|zD@=Hoe*7ZcT z1in$eECaH9ol=3M?Q3kT6Z#rf0u$wn#hwmXTJ=2q*8`-l?3kL_IsicDV<5bWVuYh< z@JweOx~WC$o6_zL8#XW!N##h2-n3=s*qcJ1lZdle6v0F?*t>7<-W9}=ketQpJ^bK^ z+Z_VmjmN%zHF{O^XHur7c4A{nOF+?6B~N?oSF8*46|UZkBVR5-4_iN1{}CQG5==6^ z9%H>BZ%aHbE~ujEQE{y@`(R0v=-87%AjO0G-VC@NI$V0`(j_=p3D?hpsE<0kAi?0K zXIAs^qcO$z{bSyix;8S45OG9Sj|?01@Mvgg@X9(+0DimBy@)0c(4KVo(eZ~lIhPU= z_z8-|;xQ@pT;s!s#HrO}#1DcF++=#3|R zUh9D9pvmzEa)^h%eECxGs_E3d@XSonTX^0!a{w;So10f7WBDA{No)ryK6DTyr4t2* z;2fk`Hs~Bd&>$Fz;p0Lb0Dhft#cj2?WPtEm|V8 zB#&D`06qEXsU#d?5?*#0Xr%vN9AYG@Mr$64&$=AL-h~*u7*Ps_6ZLmi1aviT;^$XA zO&0=WO6n}iBfyfMfp8^OWkO2yMdduWbeG^AkIvWHup>^D%(fpluSvI8lv6;dYQCkiHpLk!{5BX2Y#EX zD}eapa=hA=;NTaChEx{Z+sMX5R3&h+)QC4V1y5ngprDZy8oIVRvM7WRgKKm{umHDb z&ECw_;KJ;g1uq_y+aIP3ftko2biwH%?2IUbpgV{SW0Zgyc>SqkHv=Mz#1CA#^2Ya} zPuVJP_$1+`!q1~c>Dh(GMYceM@WCbEdQTlXeI{>03GfOLctY+Kar)+GXJ;!*5w=_x zY5M{1bj;d1S!E`RZYj+mLK)u?x5#qi2G?d;COj`ms@i*cXMfRz~8 zYzg~fIr{v;BR>WPuA)oQye#eJZTJ1k z(7ATNu37_QCm<+jDI8~_sX(Lnf;~n=%jR4->is}esCW(c_O2wMU=-6mnwsUf=bZxs zOcaSs*1lXs{~mYM5oC_2EyRyQB$!Mzyr`}wYoC#snT`SuP^jSVe6(eAwFQY^kyP)0 z;#cduW>#Q9RsbqOf=YIj=ATtL{{G&j%a;dSy?XWI@0|<5P2tjuyU0WpJp;oMf!)EZFnbN+ z>@?@4ur#NC4`+`KJQ^2Zuo{=zzMW%t*jiCV&lw|Ik;32zKF0XCz^FYNEX=e8HVVf7 zZ0lM2sCa1+bpv~73#(!p_Q2m^BuSp{Lqjsn`bcwk0xWXZ^-L`M z1WgX69ykOC`x3t@#oCm2u5w|YKYfzQ5Ks`Unv&n&sR==(}*N0BB{f$o_P^9}@3K?;`YPtRYpq#~yDN z5o@W>N_Pxqb62X?fI0CHQQo6FPh3aGMjU5L|G+?6{;Nd<;U*+eat%XZDq# z_xdw)t$$Hfb^KUZZ}ueoxbgmmu2k9qa}+8PiI2^^;L0TgYHxA2%HV2~+ zWl?Jkd3{D;+Lamy8^#+kMg^d`$>gbnDuy&HC6l5N<{mZeL4_%Ob9ML^zt z4q>!dRyMZloh2`A+WsH3zLe&~+{_Tb(y z;tcMMB3oOzgjCzk+9n}Zjr6$tw-bm@3pf;qtg?%WSpnxs-S6C z{Yw3x$qxi#^C-~nivbPNpTb#?o?Z=A&pNsD~# zRnq*C9l}eTkPUd68wv!`iTr21UT}O^#NCkl;cSij_U*e2Pm5%c(ND3<7XAA5D-}1% ze?km$BI$-`DG=84MJmnSLi^!@wy zaq&{9FuKU+X7(s~U6&L1sI(3Q4HBcZqSqYnI%Q8^bYk)=CGJzqg9;ef0HI82D$Mg( zEiWruhJ;DCv_r5^A!ARLEHH2et)Eqz&{Lt10u}HpR zNF){UU^1ry_|AZWvGAgpB$+6B6Z})w@$Yn>bZ^Z-PHM#G-#L7>f5)&iTQP^#g)lp` zF16wd6E30Kk=r`=~WIbRV(_VJH~iij+vh`c(CAP3NXOaI>Rz>J5o1=F7b}0j*dUnb2`;B z)Yh%v71bl6&x_1GkvaX59o(%9!I_2}%jy}HzS~cLvoI96n#ghh<7D0zspYV`NnKU^ z0~c||1b#vTASQtZ?d|PkNH-TH86k8_cw67Esf@3BhO8rEbHKB&ZkI)5lefw=E9Pdq zN(WF-H+-PP4RYfo;1BVX2)rn*;KytGnEhv9I-&!$-4`r@A7(|Vy3hfTz=sZuY?6t z^LPGN)`OOS5QGXmyT_LiVE|v_nSTQYnz<;R*X>8m`Uvwn=XYRU37;`w}=l{!> zo7$}on*aR5-&bAH@Z{?MD-+lMpB@MQ|Leu|_nf21dN98M7}SYU_VCFQlH>flcNX?c zBl;=<>nKQMj1POJgv?4Hl}tl{c?urRwZcd|h_VctyYCDO886En##2flx6(cpj^68}w2`lkzauD^t7Jd9E>At8!D^Hkm9=u7B0 zU%*X>=!YQi?T*f`=}7WmAd;4{?ti}8a^1#i}%ZSlcVzZztkFVoE6-L$p6H7RgflP3c(Rs}7R?R3WDG>)Br9=XW z7Gf~NF+{A2TnWa(mtX?Y05h40ZyBUpl3KECnO{kX62iDiQJ!-%ZYJ=v39XJ- z@-S#b7|n6$qfC+KEH?&o$Q%u3vjb4#%U-?|Y?J)m-=O{~{?46X^mUTqATx(BLOUTN z*LUpscY8#+5GW)@q=LiJ)559(lgj52M(@?l)FQ81{HSANKpScVb zx*YR8B;YrikHFIMxw$z4+TcD01EoAgl#-8+g5ypY8zADQ-mu_c-`lr&$Q&lv`CTB4 zQ*EfY<1!ZHFp9+aRJfc(Bt{B#JGhHMBP3Y~%s|ilYCH3`Nq@HR3lg^=w?q;_qWA@w zHHNJr=`r%1)qucw3jd){|;Vc;4lnN zLt&+5M0sL{11V4rnN#n`5eES%jX;EaSb8LJg(NnCd{71>Fd>+1hnL4te-MBoMIqT9 zO5$0sUNPfA`y-zYk;)J0*;x6FU{iO&i_eeuuKU<&V^?WML|u{dHURx)r?i zQP>ShfZk-nBg?Ef5Dt=cp*e2w5~+VxF~hucE&4)B4Aj5vn6X+%J+!Q5$x$!IeSCPb zVLfsQRD4`83LHNyxnMk(F3?Ldg#YdvCaf+azfY3j49mDvlSxPsOv?w_0`f1`6-2Xg z#1J4hswE1V#WZAlvUI^e;05ZK*xT4J;d3M9ptk8oTE-TnhTzatk%=11D(3O;tt<7? zr0XQfxHF`YR3v9Vef~^pi{xwAR=8Qa&YyqkU%c<{Q!{voQc6bJyXNUtNkaX{r-I9% zOsea{kAfXyn5NkHFxW52DRp@Kjx{#MU^^Z#v$C?X$dP>J`I)gW&1rNAGW3DSBe^rs zx7pkHB8arusH$=s@xvtCX_Du|Xg+3}S>Sg_noc=XK#M0nIF?3`&8?3DxE=SR+A(bt zIq8%;G%(iM*47QhL{RVE70r)efiHe1d(F>%$Un`40`y`Wxj#bSZ~oxHW?G~T2{|$Q zBkD3eb+g38#N>q(h%+Lzs7UXO&)}xgQ))=$AB;lmQ#->M6yCTFVPi9j4=deiVmnz(dgP`R!&X*tV%QN;ExJas7HV9A|=|xzd@R`=Y*KwiXr% zn)w|BP>~B2B?p{Pe2`I?@C3Q*NIU{#%^fACCaYJi+K9zPT$m5f&Sm4Tau|&1fVWLu9NcK1}O8?sb`71yF0j0(zcX)$o{~|$|Lts6v$%>iJhI3Cwj7f|WWSyGZe#c@uji>QOT@M*no|%~;AwjtOL1Sjz+}!IC-6SWT zkYS11BF@)Rp6l@0=sDw8K{yy_k-7SdSLVS zj?#6|sC}UyRLP@8b|PO(jB7}LNas*|I$3rsR|a&s@596zFfoEIhG_VcSg;zT(3Y(gilFRW+V0jGly*cc zCxxP@lBrM5BdKM*8gKjl!K!^*Cl@@y=*o!2{!l4#<&FL6TfCWK3d9JK@R5E;}-fcijyII1& z!wbg`K`%gEhA1f!$}5+6Jq|DjE}{i?3Do3ThaJRwUKfLTw#6d@G~VduuxJ2$i4=i- zvQX#{-8wcm9?KRH;q~Nn+d_bg6n<=f0yv;5VYp#2ovO-Jwv7jhFStfd=RO-6z~KqH zwp`X1ccRHW?yV6-sJ`E}YymdTOG0g6pWn81Ytt=p594Vm&081dQcge>LzI9dlgWKR znCFG=!htYzf7;HJKL7lA4X*hz2DIR=LhuMAQ0X0sl{0fnsARMuzKq3nIW0}dN{Wj* z3Zo5Z@se7?TTcDml3SigCV3?8lmEMi8(Qw8N0Ic@@mDdINDW6K6#_|ajMB-se7uV1 zS*ZibnyW}g19=t1&7h+qHeLmOPpR~5cf)`&b$3tr`qofU=vv4qp)Ob4{-asDEdX27 zQ9(LSgJ=doE5KPQW3+6opMQ-@oeD#6Zt>lVJc!6Qv|RZM5iFN*h>6T)+VQ^KKyJig z44?Dl=$E7H8pLT)-o%bV&WETE#>Tf|+M&Vs52Sf&-`sv?Ep->f_e1f;?GVG?HRHi- zd-39h)fU5^ZKu9@hnwyG6a05M^xHTqPa8s9-;ILeZ;2M}xUoX~QKA*N&MeB5-Wcxhf#{0Cx%=%Hpk!7GI9NR0+EV zGi0GK{{n|r#GliAS(86l3nK+t2CjUKhoYcJC1-DKHeKLXhVT#g!uKE}+L@s}Eo~aN z^X<;osWOb@!+a&DF9Aot5Mm1s2_e1*QA$wLny0PQ3=G&8(bCdTkVRTP@bhE$!=$S* zyi+!x-_X)>$Tkkie4InGp=}U#k9-NIM(FQYq+(%_IQbg_CQ_F^s!iH%l(E;AEh%b@ zOO~jiPUc%4!Kfr!ECci)h*Hnrzn7_)sr|(WVv?uJM%_i}y{4HwGBQ$zgJ{S(4b-=x z6>_a-O$Qp+hU;>ZN9q0VQId%Rlwlew-~RIhnRw7H>MmL(5X!6OLgRcpCTw)zbVN=U zVa=eRu2$GZ&Hs6hV*qqPB=%e1)SH&$$y?_46guzAo}YG=Of%VzWT13peEeNZAL7%f zn03G8|323%OCA7N2YE%~(1V#HQ@2gph8-z5qmQb3SXfvjq@{iN7%>Ox8rpRv$AbQN z{T(KJ#1#U&uMDleCD^tKhm|?PrJ)|gvcaDshn(PckZnUzM~)nc6x!l_=k&9z2*)9m zUnV9d?lG%oCLCpxde`zVceo2XnudZC9}I$!rnn55Z6G3M0QrT8(xc+FK%del-%`}W zX)+AZ4T*!$8oH7qDrCB$=)#28 z2o&c?7l0BB@o&`lpND$|%GoID0?njLMF|GxEp9nICWCK5`dorFJx-{Z>UQdrCOHu}6+%L* zP+}k&$uXY>ls9ef5$xal!zF@*IGm z8$?|QsE>+qt>i0l=2}#gKRn9Y{OB&0)XW`T8L0{lXNq`y8qWyeKj;nGVw^B{@<8!O3ZN^o2z(TjNkb!Tm?34 ze>!GAa=;)m2O#~Hfu6qw!-(ub$d#nmz!OGrk^%Mrj^5DLf}pe+Cx(Th{@Y}V0;lwA|i#b8~1kmU_9%AB2ge*ANVu9Z{ou+o{Irx?bG+F3DbfZ6-lG6Oh6;wa2?$x<*n04oM;e63h{nWpKLY-}X$MX6vk@h#l= zJVgLEGGk301>r>xxzjw2DB_~!%a@Y_Mo>ryT0C~*#A0j#u%q=jScIH~1!DhbjWdSE z$pjWYzX}Eg;H@Lg>*B$-+z1t-U17g5GZ1tiH4G&%#zHD@|X`~YqMcj-|*!Do_C(NhgGZyC3f z=qm0n$y%So1Un zfz^Y}uHfbt);ZIPR)@MMVp_;f{U=&tj6(u-Rm=bR2w2hoz9Kf?yLp0W@c-*qt^5Li z{a?TJ|MA006QVn4*sYqIo5?i6O)2Z23haNcK2gMNc3K8UZ<5&===mS2{t1-k*?00K z9q#!uBibM!BO_B<@vqU>=HgM|n|w@uSC}g@y8l>OjD?$d!#miw< z!8Q4kf6ah~7U7iB%zw|z_}5z({^$SBIj(BrrL>b#I0i$N^``E8 z>KulC2+Aw`l>YSR|6ao3V8Oz8O=;*cV!`|w@S--aCHS7+&+ z<{Z|vN04a6v9Nt7|30ms(R_>upU_fg_wd!O|Myf^xqpwiraj>G)uEOlF(bUjs;k3J zV&TdX=L^S6gBs_})+fHo30XM2H}y;RXP3Y0?|=U9|JCul{kL-g9!R^yv9hJ=4Xp^^ z@Q3fg>3nJbxyv|l&KyVp--X*7-I36r0lq;<1mN57fxj2M6Ds&&49-g8mGYu-E(bJv zk}8E=-Fq(P?<(18P^KNHc-WGM&>Ec95`+rbiD2K0}UHMq7ySs#4txZkEFLf1OB~J>kA#G2wMVyQI3xx_(hEz zd|mLL%b>q(AiN9oa0EjZ9qED2WYd6zIgO(AdfOQ&5)h zijl`m$s`;KEn1KxSwrZ2kVpjAD!HfXivY$!)50Al^KtlqFCJqI!J^EYhQ!IBDIEp3 zL~54g+)hmpGXYX8nQtJcsG(!t#bf~h^G;j_InENvmQyD4V0#1=qihpN6%&lgFeqID z|B>we((hy22%Luov<%)ZIerr-b1{jl~ zg(KEZ6i-|O0F9^{12B4f4+*k!N?BboPVs}@Ku1SchD;2Zu9wL2^`%1w zm=Wr=D*Tm)9)Hf_F42$$NY4I2CzG7V{da&3L&i#xOQ=MO@!dD~`MDh=9SK>&5y#&0 z;RqNX9Pn}!(Qn}?yjWVU2ucj%LnJ5h7I_)wZgPwxwkc4-QV_pGhYj(87jm+7=;4sp zc))@EP=Lr0o=_RBo++)bMIM!$NCwq|44whz;9;a(KY<5CjO>F^yLRsU0xTm7vWPX| z3(GC_9)!>e9Fc>mSK&Gg3zOs5$dw_G0!| z+zdV#PD0xq_rjdj@g806HqbECl~HSZdzQtE7o&K06H-sYcs~aR2A;!4LNufbR{#Ne zLwlLH@nbIR8*2u zY*8{qg-s*LJeG_pLsaMSyzlJK_xn4)bAD@`wa!_;KThkje74m4eh;trbGWYSdA&BZ zZzg>TOHj~#9%yriO;k8ETU5_~G1K^nkusOE8F-B<#K%NQ=sN{mKuJ~Aght{~j>xTw#h*!czLD%Nu$ zbZ1S({_XR0qakHKjsFOcXFAW0M!wsOvjgii?XX3FCSDu{iIdGL;l^enhpyJzAT)hX zpD%Vb4c66cu4`_&3{!fB2^%mVMQ<__#2H!T$Q3Jw5_~qus1#YT_2t6XuN%YUiqMtS zW$*KYf??Ojm^9mogbL_R{9s`N#WPB53kMBqEi}dA#c@-T;GowK=!6bod7SkuYry1V z9R&rUjm}$yzK@EFg;sU|q80xFC#PKW0Ttevjjt6)4{+62Sx{yj5Hfge+`kc8}1sbSD z%XpHRg4>^Ey!{fRhkn3^aK4Y}ovDJ`cw1Ps7A8+|e94P(VghY{npXAz3F5J@+gKTQ zolMznh0zI_>~ zjtrKIS1&J?7*G;fYkW3nQ%e-BQ)lO_lQAMW6QU~gW&B%9b#BYr2(LaELk8>n+(nuG zA%3s>d?pOqPw%&WC#*XD^lU4&hggP7gHCBs!jRjMXVGY+BqkA49uXdqZvQ;wS1}Lc zU0F|~A>$Y&%adu!j?huAUnDOC3g*!ywuKBvFlue0<3$buDgF_7g_Yl_Hv?5eg(f|I zyzk$C|NRhJ*X587KUr7Wa|mli3P>6J_`u~Kd!i0pPbR*ATLUXkJp)CP5ke9B0Pd>%&n%4piCKcT#fV>59s^-> zN!q8_5ZF>ij%I_Z*M3*Damx)`1{Zlp)3Q#g>zeIzSm+~2En@EWjhNde*mT0+6`%Vi zW4A5m3Xmca36f*aZAW+JmT%}bmjUAgS(jlm)<8ZTgu)TKv7FHk$Oo7}A0kf4R1UzK z8|OEw+oYZ1z@5trF)>%j;8_34pc`qdlYZD02~jUoG{t^GD}ZAl_1zE~y1;=%{kT@c zy?0t+pw@e-sii)?^#B(&6abg@`CLi}13-H1n*|8lHKD2cLS|1d7q&LDMfB05ZhFi4 zJfYiUOk}bVnqUd;OP4M^{mYA6Yvt|ZNW)2Wv9#MVsleNsLYg%=Vf1$!XItAfSC+qqhU!4;p)@Rf`@I%pGPx`6dj0fRn6( zodj|x=?*Cn{mRIZk^F-AiztjuP3w_Yx3bp609MaJw_6fQ36v#k7(b&15k?a@NQxRU zY};Af<>1}fvu4#Gvxv7JnIILpj8sli!YkB89_4TAyzNuxHv$D$6Vy)oA9t!)Q-4|h zWxvT^nd@4EUA)=cM(Q^2>uQ$I7HA2>{oe_MCZMI8pA#ub_CdomEgIBJm zn`bTb|L3YbxltjJAtSq0@Hr}hfM;c{Q+!Xpraej@U!lH7rA6U0xDq6HCNdHa3L_9T zX3_`xoda+Yts8-_^kr^C$}SVg#N))fdATMdmmLDGgx{=3)hr0dL+e8ifktKm456uc z%lI}@sGuDB=eQRtnJpcM8C@-| zT48wSj=CPda3OB`F!f4{DEaVV1m#LBb4-Dw-B&T>Da8#rVB=MGa1euE;!n!$+XMSt z>~#~~YH=XtHc>_KEoR~>=wXrj0=`Mw15OkfWN4O!&qIQJ&dcruF_^}8B>SF!HsZZz z7Bf|-Vc0-wnJ#FKbu4gpP$-b~Uw^4PN6AGfR_t9>vIIfFkB7Hw`mVdy7;5AH-+AcJ zX%3#SHM4MWs?@Dvbnlya)ukm9UUoTnR>p?NSBlSeZ3Plk9N@*0HE6c0cdv0jwWbo` z^^VNV#uHc}F9$y#5Y`GkZyi7+@uEW(Lfv1>Z|`2ca7;mJkej8~F(L!e*I@hl7{xzTL|vMYowLkYTvtR+Lsd#H5mwQBpOM#Kh%R|= zDRu-+X2*#3JZTxscy;%5b3B`_u*{ak+XXDSEP@_EImJct_U%dHSpmn_#^(=hx%7ia z^P!wQo-|VkSc<+$NJE7@T-o;F?5ta7{#+mg#kbcb47sAA8Tab!{>fe=15OF2Orf^1 z@lw*>2wWy=81Mi6krda=)Bxv-SG4%c1FDT+pNlD8EUi?qR9`r21J}Gpq#km|>(G{#z0({&hgb zuh-PrfQpJ_Av4$gmX$chCd*6=HN&zqood(XdLeY)hmsO0&G#D7d&SbWr6_3WM7$+w z6K&UoYh8DkJz`pnV@Bml=S6!ylawWVh7$- z1ZDH?#Mc*N7RD{37HQW~>>jpiWq5I#DM}*o9>&$hp6M&(bvt)6{?$*$$&dzQWNKKV z%3}H-iosfVoCp>p2sNt^C5}c|MEwWN2X^C2w2&fLtaZc}R;XjwJo9-rX-8?Pri(i) zGDCy#qT=FW7w%ugSOQni&AFr@(~0IrsNBYC_5m*=zz|gAl9BaK{kyF6a8o z&>t`FrYhgfn`W_PT45Hn2?;_<*uYjJ85pyKJs9kb%S1 zJk+)%NYONbQdo+i%7`#Fvs?(Fcd zaysKzINq>~fRP?fjw^jia*y|(+^$p1h=wB%)llbau>JD`)*tQBZGI!Q999mrY&1H$ zp$5&KOxrXfWBIE+R%w5^cj>}TNF_<Z&fd9dvU3b*bH&m|l zdH*v|vqGKAkWFo!-zZUU;QU5eKtGz_{rAu0vHyN#_4)JJ|4%m_+e+sp6ImjKMQ1a3 zhtk~q65P!@=2GT`HP1XX3M5>e|21bux>})q}cq>KwE?w~|MbcB&PB78aajXEdA_sMIDol!MlR<*%Xe zc0j%-`fbVsz{7s&JPAuS7w-p8x!h|ZmD?UuK_NcAOi>ERWqEoFXwcML&{ zUE%hQE5kLbA=&A7vTa|?mwTp`q8Jlx?W)QzT@??hFDGHTKW;XHmyOi(kv(y8xq=|* zQDI@XlY7>(YX)c0>eo4qh^0HiKF7)Qx9mxPF1sP_7(?wk@^V$25@w$aAp0<{m!mZ#J>(M2I*=`e#?#_s#R_N5dd(mb$Gle%Bc|4-F6lEHD{dk|aahlax zK6pjL`He)-Olmb)IsoXL{)E{kL-cuQ(FvLO?vjUjSQHwf{ti3Vaa_{!KF*u6HoOy zBa&ETt>Qe;W5Q9*u7rg>y4I`7i%}Pq5JQ2rxygD($ismJTbOKe$&U;BJTt-yJuN)@ zG4-n&`lWr!sc{y%f2}l1Dv1i|+(iH}^nZd|^S(Mz5z-LmM9^jRWGkMn1d`dyl6Y*g zDVN}{i~>AlrCp%dS114_Q$&oWKoiOB>q9M~AZZd)T1weV8)5TudF8i3Li(Umkh%_I z$8J9izxnq+moAN56o@>vA(k)(mv19NxQm#ZX8&r*M@?qRJmovTQKYa*sE4F3!;LIG z%dIxIaioon&CUXGTDW;?ft`3iP*7r@QA1`_xvxSIat1<~D^gHpwR=x}*x0`9#TQ+M zcrIHu3*!+{A|giGh#<{*`dpAqQ7v#)_UNh$R0Rp#e zJ(0y8)eoPRO1f~@D+akeJ*~-*A!mBVe8LD+fM6boID)h4#Gq8MkO_OaT!of@`c1U* z!a#9rp#JL0b_4pC9wSs2iN2uu^pNY=4@T|@2xX`K{cUFlGB7ZL1R&9zh4X|^A&LvC zs~meO{O*bkTU%D=;?T)-`xq2pth8NaZyWaJXW1W~Do z@tbUP36udjfdAc4>=gPKZ4uU$M50h&ZPn*wziuY0e((lIJ;>QfjY>ri!_Gwyk*xyf z2QuE=%E0NzYuB%fk&l3gROdF!O>2J6xy+!%dN9yA&R?a5FxsV3ttM1EuxJ4A6F!%} zdsIttKt_lQPDbxO2lj@7?HAsA=*=3$hR@~Y6lS9jS0~;KOLlC!b8`)yx#-h&7AJzK z$o!;Kiz$q*aMNZ1va?bahCf3p+DL)e-G1k-#>U@j#OyB^pIxspLmaZwrBJ|JB4Iw0 zE|++60@R3+RQHiZ6`$W<=_oY@%xnYkIX1hO@tP+& z{^Uttxkjn9vJGEg z6SD)2Xw)`$?3OW66z;aW5F?_W(lcToUujU;bpos`*!c^km%^Bl))NVpq|}Y=n<4YL zJ@Bgg{(*~XqWasdKYP<{UNdQx`S*y%TPK}c=_PfNY>kO32J_$XQ3VK;U~5@@8H=Y? zuj1N)vK?gG5#2hPi?cuy(9X6yo!jKfZm}Mf-%ejH z$ND4nb#+x!zLT9%pPRK^Q}U<|%XvpR+>>Y`_T!R*JSqKn6Pz#4Vn$fd@9BiLBS?g@ zpHtjjaD4U^vv0dL>&5&=bj)H9iX-?Gc4}j4UP3K3R1he3W;7RX09MjZ z*(WaEVzR>~e0O!jaIoqNu?3^$4FEN+?cHYqKeL&FV8?l$QWbzDfF=&LSa7$v$jiku zJMXe=vn4}MV54#z>&-Xwi&qap>V-_4jbmnPHSLk!uiT>#1d`TMvF5NiC-uv2}sLp0pr9XZ`%qCysJ*##MSP#zXG|xf2lEnB2>{i4aZLUt4H0c1wW3q)0 z6dT(m9qeZZq&WW6AJbnk?Fi>{y3Hz20*%-=%NbJ$9>P_n@P5(|o|F%BV`3`U3R_OJ zG;4{Y7f|?V1eTVq4LkKD2zA zS+5#zKn61no_fOJ6-6eEO)bzI@?pYwN$s20kK8P(*wucEx?IupeuD`J6N*jj&xe)-rj zb{z^@HR!;nfhnahc&RDe7morKlSu~1+gZ3$erf+tOSO8gJm&F&=%eY^A4~~Y<#BiJ z*x@b?4|!uub^X1phvhZg#BI39hIq^~u$ABSw+k1W(=0Nuh70lRl|I$7!rCngS}hI+ z4wQD4-1;Eo>0X&8MVo+;)5PyTr_D%6`Ac9g*ZosVTy>}>xg+12Xc*lMg~JZBB$!1t zTj{f%jqK)q*GbiGa^+X?l=&}T|2qjhMPGE}0M@D(c_-N$T&}(?ntf3+5cLS6T@+f( zGm8E|801~OxAMk5A*RZhD}N2V@l$h6i1>LZm8i%xlE1z=!$PnV@d{cn`v z0znn;#%hdSLnv*Ke_cCnP7g93o4JHi79(38{KAf^Y2#bZ=QR?8F7dRu8rV#xE>Y${ zl4y^4w)o7p{gg5iL}7k1$#Af)jp$!SqR1m7if;zd=m3FTpmiZwC^%%7voFp1sINHl z)F#?3hF3eikx?*EUMU7;s>&vE*-XpXdw6{qz@DJ<8cL_SewDo;pC8#WfZQe+I{$DY zKAPDC*#twJzga}HBj){)x80i70Ym*-00YAA#m~!chIKr%mtBQg7p|L~BE)tG^@#}1 zk*U}-F8=-8p0TY!T9^MInaP)DO^+M+`wX97HjG{aqa_jwpb$d^+baA-bQq1Q3+09A zTcX@sKvftaoRSf72_?Kdwn|a9i&+q@`!gOLxIm-^I7-De$MI}~4{A3L8#jQY!|FRgv^=5XrfvnR8B00)|K z{*wFGf?(#$d7#Dik(3h#9uPaFn<|r zKu(ZR1jE8C@^u8F6V~GINGk3UOq#NKNMyZ97V3*oP;;X=R>^k?5e_=cL*O@az znBR`d_KdTBz64_S5?nORaQeI*RlkUqfldO;e~|@MIlN6ibSU!UC*NVe7S*m-uhUxZ z5VN0Fm)@^G_J>FIg;}FUKJ7dy$>CnryNezM(X-v{}tdYn!<=Q|b7C82IP?OomU6(CLTI zjZpA4Z;R#Xz}A7d@#EfwVc8S$Z-;E&W z9@90pO^fky%jA8s!IG)$RlYUvylLEMNP8R9pI+}%TY7%D{@V_;FIWEgV&G6YAI--m z?C0pKtj>05-#!||NDoYoOdJp#99(vX73|JHOa-!)yvEJApZ(|Ew#-TjwxA~nU4F&f z=+oULo<(81RA;-sU$kWCq$>os+4_50v}jSvHXZ}rUj|4lV03}cyzd#gHq~CvE}qQX zx=|fACL5amH$CMStka4ktq*T2J9h$!ciHQLi9D|HY~o`zy`kKYhnaB~I$T5v>c$+x zm(QP%fOO^csev!XSZx2tvo2JI7h~9fnsH_u5?RL~t7KA>r6g0yvg;@yanMswZ~}~~ zdN+ox=8{$C-+c~#k-g>1DP)gwRHneX&!C-+mkp%3Sovj+=C7T4>5>zBSgSQpLvt&nH`dlU;AMhkRVfnAsF`S4N*7)~Z1L7P0q(^W~vV6s;wt zrRE>Kd*2%}McOu7>a3@1vIN~X74SKWy!Y(cvoJpVCBu^_v~MA%&{mRtPSk0DvTvrB zkK>9JXScD9D=frII7(|&C^|*4OBj$g;qciqC=DKc`t)gMP^Fjm&YZ&V^j+G5W1^ON zNza0$@)+??PfyQ`m%Y63kgoyLos)b1a-GRI0X)jyHZA{iI_PfP`evkz?weB&Qho7p zGc3NbC8pdkLwZ6RZdg2eq}sJ>D^&!avecG#g}D>$a>0sAxk)b2aunXSs#tm%Yf!HK z&RL;0(YA03N!-M}!&W8BlZ88;5<}Vf(%-^pJlE16B-*|3=A+mThSztmntE)JGN_b6W`Lp$JC)^4UEq9a!%yPa2wvDn$!*&O+o-FAMcD3s&K^8? z)7Gs!+!pkiiw{Q0`}dj1xqe0)5;=&48BKLsOnIPZmwXoAhe(2c(Y^)AnXo05Y0JNI z%W_HYq@=NQIJqRWUZeqVva#gK90HyJz_;xBhB$9r+!ffad)n??%#;!Jvyu#DnC>MR z6$12j^QyZw z`Tp^;+CWhqm)vdI?rzJrZKJ@-qvY5;*A7Fj(5d9IS=_UK0;e)!cRVO%AQhYY+xycA z__1#qRuquT4*|(V0_5I-o*~MHqaI88c*j*Bd`h(ge9MJ~O4jCL?x15I5y<(TJ@qw2 zsBM4Cnq0cy#?~Dl)V^_$4`#2BXKnNyO;vR*CQiK7;=zZx4SDlb+N_=asl_7uV@jbv zySdriAeFq?%-}XRY}e5WLZM1~)v7A(*Pz-N^5|cjTmI=7Yb2U#(V2j%XL-OIb&pj- zrzM3qJM|*{%TQb$ZYE2{lbcq?eY~DA)dnBZ{(f3Ye$E<3qbz@Ja9DW< zD|>@BJAR!z4&18y*bnMw7q=enNyw{mR6@~z{h(x3*n-Zyj(dLSW2;7W));+#^=ly5 zd+y;rL$B}tT8V}3cU&$x2s%AM!<{6NJI&1?fGa*Gfe&pwmU)6Yzh_LSEt z8nE)pw!nT4>&ucmtk7{}4%h}UrWc8|Y5O7b?-*b9&RpaNEmGw=>1EGa3e`{~v=aX-n6}P`OK&2i5hvEumafk6|)GJ?a4+@@h z&5QP6M^uz++(%TLvI*^WB=o@Fl@}+^?y(&nrFeGw?swp+ zv-^XUqoZS-VI3@Z+QW#12)EP&$|R{YYmD*^AoCF(_0&&VL3tI;yWe(@^V9$Bckb)7 zY5TR)3ktq&uh=r$Vbg=Ob5$u7+TZvlr z%}o&>W>brmvUPzA=?Um%Cv!x&{ZAdsuADSFXSoTG&vu3!;jk(#M zdr$iuf&jUY7_EAfLa-M#A@hDaG-5tc;&8+y; zOAtbeQBS9H0F+ZY{@Bs2ywv&hnKQ#O_p;&w&d;=@*y?ac)(%DG1PR$7PU^GPj=(KD zuX+8->#%A#Am4h(97I9psdurXqMfZ!FBdE)T^Vasl^Fwv??ZxM` z$j7ydD4Ul(z0pI5bfF5Bf|~M6c`R_~GAp!0&JDDsct~WM0@}y2oF#I96>-~0-7ds6 zYF_jABZn#TZq@m=+*jPkExx7tz$Qv8B2SR4vPOKB@=sV<@B}Muh9n>XrPUM&8=jPh-8vAED<&X*-=C68l`h^>_;Eo^eKX4kfs`6|? zFA8071oOO4vzD+)_9WiXa_+GIw^whM=iE4lG*{{r((JCMM(|XjOcLv9Yd;NfuzpgR z-k4xDCVQwn0`H?CNYV7$Y;9xc=@#eu=~Jtc7dK*F1T7zj*41>e7p1X9spoM2^iZiJ zxOcg8gS~W-LzsN5qV9Y8^l2#lW8Yj?f;V;3VGqwI=Yk(Bd6p4Igd3xB_dS2*iV225 zP47AU4R@vLYeJNxrrs6$;LvlprzlbYt!VJ8y{Y!G$qhQ|kkogCIxcTRX`M^1QmxYo zvLPsg!`o097+kN#NvFgP>|Ue$;y?eq#rnx1)`=es&rw|=utJt)4B4}Y>SRl_nn`4 zlPeAv-F^AWpgF)a^QG90o)r5#GeNN@t0)NC3|o0t8rhaJJcbkF_hQN@`5uya%2#R7 zI(n6EPsX%~;>h&={rfJrkB6XaIK(td;q~7hsA3UlfQ!yB)hlr+VjX*j`Dk>)ym$H&vnWS92bZH)MK6Q{5Eny^R zmgGQrfp7Y^>N7pjk%tP$0)G@Pbt-FmuaQ?GtGV>{9fW)RnYd|DtNF=x*~g|2sh}n+ zM~G#MEr;t@UN(2)GxhvP;>9MquHPQ4IK+;fM*!(npmJR)noV#Am_RUi_T}(4JCbfz7OH3A2WdxiHZQJrJI7u z2$!eZ{XWA&36gN6HIts3cR~tOsT*Emt;J&qRNpx^d!a~9!@EUO(k$%xd;O(z1}Lpw zChu7goDe*N(%GF++v{wif*q0@~+t?0q zrx`e!oI?Twq{}=!3Yo&SMTSM$Gw1b0l#>YzH0OVo_viD2iI5Jg=TIaWMkUij)B&jL zu05Yx-Cyxxtkfe?Ga+HvG33ubu%cA*h~hrB9=Py;1SKg}sW)BNn_x=3QwZ;S*X0S0 zu&@9e(nPi1I(pgiwwCiDn$q8ce6`TvdfdOwz|Gsuv%4H;Z^ra!KbuBaZ?#jX;vn0L2gh zOBuJ+l@8UDmoCBMz5OT?6IbdY<&21T#|MQjU*4Xj#Z18myv+v;0CT!`-no1CZVvuTJo%paXUQKo>07DYj$Wx; zq3#Li)fwY*BW_h4Ns(41pWZ1|`SbGVqP3NM&c6FS&MsxIxo+(ir4yf~cKh10yxz*n zhPD0^e>Oz8eB09AGuj~sc#lfqvP^d-<}M8x#yQ6U&A_hblAe5=7AACrGP11 ztbbN+Iav$rY=z^nkF6An8+t(rL0ftSCp;YZZAq1*epZKCU3G46#5lb3g+3Yc1`LUB zaws!x&hR#Pr2kIiXA8O?verhnk)=*8O*57bNpV{?=`AC;y#y0YWM>4lM*s`czkK;( z_>KemcnPs!&>Nn1`;DH(*z4&k|7o!Qrkz_+b;ZI0~S z_(!wjU5G^;@=ka|J`GA-Ywkco+}2y}x$!T5GHWVfhKACgqR4;R5~gifNK)Gk%{*ni zvhvH^>w!Z)h1|m2+QDaiGi^S&=Aq0Zv_yjvx0rXb2pT#i&T-?Rt8yJPyR2;{|F19( z7t0sGpaq_!eH&cn4Iw=jFItpIN=*PRK{BO8!ZF^-sj&Htdw>h_@9B52fSbKJOHNJU zq5G4W{6D*yjYfb`#?p=8`WG)9ml!R3U;{mvOLf{ccty#jn8hfKd?=q?FS@9I+dDH; zE}yT^36w*yBD;Ii>mfnbf)n61`kX5Eyq zFaeyYL5pq&h)Wr_Zr#e$Q9rPE=A8oG1(5g4-jXcy#x_;>?%KV(ThaYh>L2mRI%w``TOKL;w z6^6CFLvK5zwGyC+*E>5gy9=&^yqvsn;?RV&E4gH^4m0!Mo9g--mt738roJcP8kTeEV`!<%AH=A!= zRUCcFfT0kYfT;yNf7uKL13u;HEM3Rf*QZ{WZ|=t+2K9s-n}{7(FxYs)r4DNx=qDqf zGK6IhTK;BGdv>W2`R?+!NuO@Vf{w--eR5|)5C_7=`R7Tix(|Yba+$wn>VF_|Gdw#)KansQjxL1j!=ZkiMiZMENgxP zHYvezK24m|bWB}`10xIG8&RHmn@-Sb-m2U68+_urqAqr%V0+JB8t$u;;_z;piQM9oanuG?&`hEbE9bHVC?7P*N z(1R$RVCXQ7%b$qwK(%#pywi3yTDr8Mp~85=aopD$vyTQ4fF-%p z`qc+YPbctOb`o2)=~hFc01)2PVGHPvFtw(kQ$ZcR9zWn6*JdA&h#bPig#2d1)>Kyg z-%ym}YPSd6^9V@ulmc={0zV|ghbOcN=&NH4v2J0jYK%}Tn;ob&6N$Ufs(`z>a6w2i z$Hc8*JV&yI8#dljBHD}wAsFv!##*KtBE|=W>bB7MS*PK59QV>OA%d;Z~sidgNfv4Zvs^9OU4`r6WTnFx^ z$GbOp$#X2^Ttaydipb45?1!szTmf45lp%MDZ&Our`O6-GMqX1$t=qLFbj&$18OF-& z*4LUH!e~j9Mnv>UDmRczpKSo`TZ}47t>s<;=FV2R5@^Ph3?iM=NarbZiO97>)QL>t z=X@4uOenb^kGrlhN#VCMb@|&G`p6ZPX|} zD9BG0FfNH)J)Q_L_GPVC^AX!ge}v23pRF6MZ}D{n1!G2#;nyw}-}>$`m(2q?t1$B2 zRr}a(q(iX8k9zHS%DbFL)X&V!q(#gb5%VH#`v^c>^DCRaf3ooe)IojaB^4S`_{LYw zDYqqPA4tW9NGC2kkNcC$1gug;x-2UPj+e%AN3W@}1?|}2eKizX?H%t}$_BXX zDHr22W5LY>=ezj-UHwUZKkqTu-E)18$*%3rt+tLH#8+Y?$C|%v9z|q6x$^7l-{#*- zv;|aA9^?4xoerS`b(C;AiKMfi+qdtflv-0rE2@GqDq-^IQHwp`R_TJeq2}}YqaIhR z;)*{#!#N|r>fIPC0QGj3gn5PEdkj_6C`lOm_I=&9Wy`L}{r6p`rt&To!q)Pp-S(D; z+D$hLfAr#Q)P{GPe`x-0k`&a;ks2_nkDWx;9mEt%e(q*z%w%^Clv4y^1u)hE&zcv! z&~HzqxkpCTO_{}*NHh>*6zzomzUxzxSDY3MkFvP*Y%hi4Rqt${W+)4sD>^Qe+oKe6 zO6E)O4Wxd^@%h;H*Ew6{4S{w|d+@+c_#}e!mz)n|kCcAb=A7uw1Riae?l*GBh*Zg)h``79J4U`G7@0i)A5`vCtgjRwY{ zdp`njC5Q_^WUz~o+(*9t?vouH(3fgeK!&^kEc8|`RGm?foe~6l0oM7C?zAF3+UxtI!AwxJ3+Ba9!`+s)u-QHui1Z^E#mxt-|9p;C?sZE7*BLWBlp zwpCT%UJO%X9tkdBY>C89r=0G z>iR73?JRISRk8@hxg=bWZly}zeqNO+q}spmfjkPrCA?^pV=XwCsU3@p*LjVAUTjIp zes==EOOd^p$>^i2IH?vGhl49xV&l`GNs|eHZUKIN@BTfReb+agZ54KEW$BLX4`PSy z%-8lR=wUYJ#2zxF)^uKbxdo~O&;VEAb$L(e@uR4b59YxO|NZ-dYXXs zcoz9sTbiov;C-|!En2tkP9cFAs~-J#o`0$g2cyh4P5kt7J>{P(gU%6q)hvO2&HuK- z6pEmJzN=0R*UbT=Qf7hWx1T19*RK+Q|1g(Z96WKxvQgkIC+pM`WEL{uB^ZDz6p@fC z?gD~QB1%5OzjZ^PNMANJQoR5;h__F=Dr7D3%uGd}(oTKghaM{WWlH70@}BS6TG-Mf!Sp(+TY zfB>*yu)G5k*_-;`ub|0$^aExZw0Wi>JzTlT>BnA>O9l9&c5H7cRqDHiy8cQ~@yywd zZ6vMl)v&_!wie-ybeK`s12vaEA6QvH#njYPJSA2LJuOB(+~2{8<{U?>K2OwpbtOyZ zS)rAF<6*1pMy<8e-d%kZgS1KpX z@h_bp(~aCwU3n^ub$9Rz@T10aX1l5O?{B=VcbDeJ2PRe@SK;jF<|aq_{-x#GS*;oQ z{`Xg_s|!2LH;y;_eqiGAi6#;L=h6b?q=@c1DXbwrwCeFFm5} zocVF@qiO`hc<0g=|FoC#9I)mPZhD}ZhI^iI8|C*~NXet(31sf(g(#$=)u*o?DvOsk00U&`@Vd-`+2hNaOE`*lL z%>#eYmj016XTZKPfD;!$oDBc2zC3v$?QWl0@i1a*2g509%8Ozb8OCCF_pw4;qU@yxnp2ikJcoK_otnXAs(54e7mc$)Lgm`Al=qC;ZZ@zviYRKK8D2bhV z{re|6PT0@;cH@8qXA~+3FgEB~1bw@?k_G{6N$yF1+&Emn($wfEfh+{UZrJDS%%nVu zGZ}7dGfmOq3b%KNyQ1k_gEoV9e19r_&Ob7mJ9kUEqt&k7(+y@__yOHS(RMGuJPX8O zHbQlRQA9w43T-38HMh5am)EUT|I>>bJA3WB_tGTK{(oL7en0y?HEBaUF5~PogA^@6 zvg0y==YaF6J6)KlW8#mg!%$M*O9Ac?PaLh7b0a94Qd6GR;w4LjvwrQ>Hm&pjb)EP9 zJbBw<4jhT>{LdV+qKB< z_{`x-E85}c`4De6d0IcS4+_F@r4=P-F1@8Pt8T{?2Tna$ITf&SobOaM=pr9uTxX8N zZ12)+R0bmWLkkUTx1C;K%jiSYW=BoVuln}>%AHsTA8pBaLbg(JMK6*4R*>abQJPaa zcjv!#(%UdDE-H)ido8}-DTRJdB7^r_Q*av93NZgTe)}-Q|M^pmY08IrZ`nY0 z8G2EsA}IKEovA%54+8;A9rh>wJ18}?=y-IApI~HWuy{MQQR6v-j88c+XKU{nXR&Q{ zaV8hD2hG&P`x?ww4y>VN+U#k(H?!L`=VME*~!&F??gx&`YxP3wQXA{BnaLBhQo zT`trfG^lO6>7#XLF8VLY*rU;*Lux_m_XAIEUfs8AzA>|9H72k5tM_!?*#Gjiba*v> zrup9O?@hnvumArqEB^l*2FUuq`yw$8pQXP?grow<(ZWg(Arflf45j(p!7GQz`XpYdby{ZNX`G3Hp&B71tU7-_!^y2Hj3o_jJt_sLiF!t`vJn~ z>hjQl|1tjzZ6DLhhp4=q37XX2vC0bd5r?{>@=*VmV;ewpDJB@(8A3;FOc_bW+9^1V zFrL&6FSEinQ&?FWzMz*Xqb%q_Gbcy~)IgNDx*-(+`w)f){N}aCM9qPx|I4|_B%n|Q z3zENt%YIw!1}#owXJ--V5>}-{;V3Tw#v=CKaMVYKtW<@;?Bw8je zda|DDpL3%?1L9x(Uhc=O1+)@ror)^OFR~w<<#^OECC{%%5nOdo&*|GrAUS0gt+Pw^ z6!o=xO!yvhCbLUT0MIKK*8oCvn9DVCrH)dx_UC7sE={7)QI@NJ$Ghv3SB2HNY~CZ$ z*1@CTOWzG(D%8cC!)T{JcrYxrn2vat=tpg7uqgU`_s8Jy)%|2Jz$Y?RrR`YFP12DP zNWS!wK;jBO~CldPIrQ{p7G$KWHl7xWUyOke0PPvRg61V$BKZHwj-K zWtnT+H&^%l7NZT+(K)pkT-Hz9kg*3}FL!}Sz!$cC2g=>#;WND>LOW62N{J0YXY_t3 zg!guBDEl!Wes$J#N-sNp{J6Ge*`@d^E0(QSA4$FAquHDta&l` zsh&zJ>8pqRvRNnnZ#w=oBpIlM#;cY>6C5cWqG&i+X2qTTD6W|gl>*N<7y3h9-gC9pVy*x=$n zUk7?NGZS*r2r8l12 z)qZ-D|8j`WRr+O0+1N+4vmh*o&^QcAZ1-P&;+^8m{(yy`5^Qy3$0FIZapQJhUthY# zLu@h6Z(Fl=TS^0H=5XlI(ljLotYac6;4oH?Dr|THL z9idr}kt2ZUJ$X}a{!8vOW!2ydnNhXk+#m$ahUN4_eQgQ43rXjyDFqh(qL7BP>ex$KcoM-+myiY6JV&(_q! zxq*(f6&N*UBIIBY>aESz_W;H~YdLK0B8>o2&>ZH(gPiV>rOZ?)Dywc9$z0#c&yVWT zmj_gb!~vp;rY&~Sq}uJ7PUg9P(~1lKE-6$nN`|wYi0{>zG3`QD3o{I)=S5zu*Zuv+ zk9qQ}31DA0UKWYoKmT;!#jwW|eD9ASE`a|&4tn@4r%dyXKIJb*Wx{Fj7C#jiIKw*J zTmZ*;lUl1UF3pLfn~u@y)awJ>+E}@i_fxp87C-$I!iyP24}7wl8fH}teKXD0<2 ztpnh8!@#-j1P!*CYC2q>{=A)Bx9rImDhXn<@Vq|5@Q&{C3NMWM($s>fWC zpkpffcB83%8moW2-BVs-GgDK!W3+18%E1dCdW%;%;#ScQ3L&QbQNwk5?Bvkm9-@TK zP-zQ)O4f=`9@hytRuzi(>wk{=;A@Snq<6#Vd_`k?EgweQTnX2&W6WK;OwIo&8e?7qy(Om^(6B1Pz zXBA~~SD|A)MQ1^|#s(p;=Or1aZ_Gk9N^!&R(|%ktB6wk)UAM-HP|?&_T`LTwZ*yK&5a77FeM%b6RL{DlN+fdV=g!2`iiI_civ!-Y^uJisv!#+|~3}$##CIUnt zE;&!MwSXle)7U$3(LoKprwLWHxM^BSY8D$}c9|fU)byf3q0_wRFcs{>MyF*P{TG3$rQz{CP{=($&A3@(;1n!w;#Y%dKP4)CN{K zMM&zDHlg+pKRB_Gj3=<~j;4`jdg~0qiMpp_9o+0?QAuj%|dUIw@A{r_0}wt&|3SFk{=OQryQU8 z7jv!$Pzg*a09h3XIGs}m!{cHDdFBhEW>-IRQ~*NsqnX2Q<;>C7vU4&OC5L9lp{3ji zJh?V)-n?Yy_vjh+CG>|A`A4N{G%(1y!~QKtcwSHw=1kY+eZ*N)DekvgvP05s4`*nhZ5>)=Vvl2O?*T?+1gK;*BS*(ty4_m9A`!~bR z&bhHR1lyIK2aas__@P?eL6b#n)+2Puy{zSYkKltM-9#xm(0g%&Or00)#G_l z8wsUBN7}2u;kc457fefDRUz64yIuLPzxuoHULBv@O{_#Lay0#i{yB6mVV_@B9bc;X zfqBX$-5!C?93Q1Qj#l&crKKxg8veyozw@`|W%3&@i2)&`X((;YVt*8HBX zee5Jh{XMr^YL4rBsaV~{X#V!U#01x!h}MObxY^wI3zfnAgzDGFo1Vz(#NjNS+chQ zpu=98_`T0HpH}pO*7^<#sJ$;vFZ&QKGQf|k)&FF?en+_F@1y}6EC^*HpDBCKgR%UX zC0jr9O<^W_%~t=gw_6J|HN1W@cmppNXs(;xeCxS|k5*%w5)A|>fJ35Lu*fJ5lKFC3)UoRpli~oJPm(=5)=emp=67i~2mY+J_BlEOVPQ z1`)uF0=^(n(i3_2U;-D3B^*~WG(v1jI!!u6D&tb>%P348Pae_eF$S74(I}JkMXRd5 z21`pwi#3*$%EeBH(Z9x`zyx9``SfWbke`hh3H@4w#e9vvay<=r5Y zRV!-FJ8SS7$MJ%q(#HZ+&6*EzXUkA78OpJ5t9Fw#L1Urz8Qt3Z0|U$MF1eAvq`vy{ z{A$o#q+PstF;lZ*6m@?coT&bn>OcJYe;+o`eB=LB!26$%yA9}>$fYtB$PUpx{nG>N z&mrQx?XXP!`46!R>c=H+r>YGFSV*&v)h@P1Jqnb2twydUHI~l#VEH?}x2QsE)on8L zWS?r@Cyso&v{0}C+KG<&F-YpF8)1*pbT@t3#76*V%ETNDV5)60@;zItKicmvvVsWX z=?6r=(tl=(f?25yC-chZX`(|p@Vn*|>kVkCgE^>~%7A6dfK*mqwz`P%ZdAxD2I|NF z1PJl7F4o~BZe;4zC=!)t-nl9Wwee829fr(d7ERsD~rqwDVl$4F~;BVAmKK z*P}LzWZ!v1O;>QntkQj*fxu7N61(%Pt3U8%mm0DR$zC+aoi?REQG=vr2G5uH>bk%_(L z<{KKSuRG0~;(2!8R&GoKnCz-K@KS|;Gtir|+sOi<;XNyLEW^`E!I!1bRc6r-BrN+d zCve!R8SD%%C&L2@CvZ4J{w_VEjw;9N?h6jMiP^9+3u1G=p(bUf9S}^O*T0y?in8i& znVX0w{PX~Kr?!^zR;DGXORrKLs^S?><;^nVWAgfO8$?3a-d0~ud}x1Kf0n08-J_Wq508jga6SLkUtu0lKYs-L&JBnK2ZDGL|U73SxH&GcKitnW|6*G~v6u zrn8G&NjK((e|jd_)p&1cQCVZ!(C#Ov_WVUgIfPsA@wqa>H<}%NO|O10s&Kvyp=b;D z=tZSNNEuI=&yZD@K7C>wy#TSoF}KLSKf7jYDg0PCVU%!@>J}P7p%=7u8;}Sl%38E- zyU&q5dHzJ4^?bu(c{8-eg2(&~$m{U+(&fwHk(%I?){f9_U3Ro1syuc|b$q>|pzV#s zufv7Um|DO%heSw(B$R*VqMo7JLZzdv#2RZN#((Xs`p8`j9kqAHgPF0DvP!FSfsv?f zvR9@6v$gkXXq+gW1ah}1fDENf2Z)t~K+`JdV)1klqZ1SsXnk*S()bGYRA%vl2#gMh zJn&Tq`ILvwVMU^vQGZz>+Ld7?yb3I(Stl_eaqyDfh-N5{h3xjFWVX_Zq6Rykp^R~k z7l2am4VoNR1~XKl(rg3mzRW*ySRe!$cOlTfd~ohumjyq`rvz;RYUq>v`JlEueCL;% zc0}KohAn~sE*KD_HL=^PX!+EX%XDyTlDJ`Px%^e(eeDSgR%DxECK7IBJja$> z?=$1qHd`gP#VrLQ1+(}P@&sAuG1~3?gX+p^;cmB8UL(}@45_I)N#F4cw^bG?)G6n8 zJe=(CKj-Tme=6!=CXV{rqG2lf4pjVznuI4}3Zhels;K*c1Lsf&4@wL$zj))p#Gf9u ztgkDld%XU>wX$!$Rqih>TOMXzDzlqhmEjkj+#!bA;H4+6we&o_cGi(kRy3hJJ$j=`*T4JZ0x#B+P$|$m zx$J}{KUy@S$ISnejBoOmvA@?RzQ$GFo7q>}Rr81^TQC;Fk&tFgU{ZwbBznOV+FJe9eJ!0`9#U(u=v9JFL~NJxl9KZ7nrkS)S-F{t%k zym0c4P47X24f}c{Z;WQS)&;$vfTpYe%5NUxR0Ku=!nqSRN>VBT>~}_i{P`;uip7QM zbf7RE9J{Pr2xGH*G3ze>^frc9+X;e5d5kkGX68UV2j+i&_YXiF6q14kO&6-(3y4m6 z47sNry|{{*x;1*5HC}Z6dB_h+p7$vRu*1o~t_138YD{1<78i5eDIKnAR&`xeqFG|e zhXPyZ=Pa4sqHAKrH-en&f)tf1qy2PsWKo!!EUm9&YGrJ!h;dRs&xftWe$V!Z<@q|}u<`xnZw-YzwitSYd&g{T5Q8Fj;v>|xh?V*^_L!63OY zn#O&UTd?d8^_d3Q(qP?P)$}OCo6H_5C#8%DVhUTGHS`ZLWHAc8p$1ER6Tb>sT@P!&hC-Ww{#%gt>XE)C zk!lXe``T4Lf|kG;VzeAjED^PusTduU+f=)5-6#}%8OU#=cWDAs6B{&$yfWrOlF|Se z;0&llXFmI=-4y&tL`mI#n)70iz3wOfkM`a>s_L`b_Q#SKVvAj@s2dd(3-$)6QQ2S% zc2OisY=9+}*ufraaN8@EZ5Ml2RE)g~HWaagh&5tYxS#pm;QO9)$8Y@py628N#+z}@ zn*#=hfaV=TV}0T6Aa{iU*kg>W}4Cb)lOQb%;GJ8K%k1YU{iB4=R=-Zlm`>xLEwG z3(ryLGKE>)mVw4ig|7Ppi7=b2m7K3xGnhJ5G5_=KU90z9bVC3aox3{?x{I8UNyL+lrhU%28XjA&v)V>As{+FKHl8e9u8L< z0=3#DYUK-b(fA^)Jg~)WY`F^28Y!{<8t%p2FjJ$;-JMTfzT8R;UA|78PSQ`)stelF zO6q&iwt#Y#!IGMRlw4s6692l^vh@o+v(dHk0c9?Lq|FQ(F;@^b>ueL-=AT!uw#9Z~ z%SO|yw(NE<+>ZInt@C;fU&v{yp3E)mW%#*yj~3%}rQ=54LeHBTyZ6fB1b>B&QSVcYE_`f?4mh-ub6cyQwOp(F zCQ`9a&vsra5xWjp)$E3ugIV>aB(ThY@4 zGH&5$gi<}=PCn)hY&TTil1M|5+1#=Fn0Yr`jAQxLjBUJ-!|LjjB>I2`P^Yo1D$AVj z?wI$>v`P3TT3iD^y!utQcl{FiwwkDPMQ1;)`(Nx6YTPhl*3q5E{Hs1$v%oLfyGo>t zVxDqbqU4?~y{i;|G+fortu-M2*%R9l?NM)=FBd}RsPmEHuR#-?sj1h#xqjMZ`c)z8HL*XoJglrxN{Cj2#c24c@j(#&!TW+nbB=-1S z)Cycu+Wu9g-TE-dOTYd-&G{&zM)$xskgV)7XGrO0=Dmh|4THii1NY<@;@{o&)sHT%j{ zWyl0>i*0fI^Bh+10sE>wX|kvYWH;XHylg9CwL@^j`=ate)?bV`F zu?R(~j%);oYe{B!f>GMk~>bhPK^yk5p;c1)(en&LO&ae;0y zB4>a7uYJEJ>ib=TJUJIMF=vSeo#;6Wd5BnG3uz92p6Kxl2j@_Lz{jgXz(X zlaB_>f*7_gOM1yV%b!!0TF+!OzEM*B3)GgCHr>fHxQk<&*oKQJmVS7gwzGtED>c(- zGt)8J_V%jniYFX6Qc|;VGuuk~qh~2AfXoq2#~K!?aV!kC!i7wruCSPo6v( zVk;Bv>MZ6L3Qj-qFWij#PKQ5217^&sY&^yE{|U%oD%$=%wA;-uZ*!&zRb*&#{;l}P z-yD_E8Ny?`gFN7d(ZS{rufL+S@k?Rx&wlr5Jbd^*vEV>vW#GWk;hbt)BAw5XJc(-n z6%Bud`X|1ILbE`RY&^@*AwAyA$qQ;mLg^%SE|2Z{JG$-_1OcT$L^>TC`(la>LJJ+T zYvkL0B0|7nKqnS0^gSub8P^CFZ)ZlA;drwUM}bG*a+~cR%|i61GGHhL8_V()DxAGF zmQqG$D4b4njvP4xFS%z1&Xh$iJbZZgn0Jdrx-g56nx9buw5(2Jd9T(Mjn?8_osC=f ztirC^Oxk0qfnheBRHyy$ibu#iSc-9Z<7IsE1hA824&Sr0%S<6R4}Uh2ghTz`nT#R_ z18tR-*>+?!;h3j|@SWL*9+)ADv~YTu%=rFT1c2Ne+1L2)dyenlXPK7Yxir2e4m02H zDfZ%JHqUP;7ukL(%4W0Ms1YN~76FKcBd~Q;2I$cF?}zWQeB&M#jBPNt4O}O))aE8L z?k96hviml4_3#ECpZymI$SkBz-%p4-P*T4@ZZGE-h$)tv=}=zh0J?)@6;5XOoZkP8 z!0S*_5-J%-Gezrxf3f^%GDGQ1Vi$?lgMGnD@~iSBM;PQ@msSs~?O(thSlWFqsUr!3 zYA4q|i`14UM3-%eC2-{#y6eIC3~DsMaQoxuLJ`-XoJr!t{^Xv&-0OcSBx^ ztkLKiQ`%HPeX^^vt?DWE2u!J8v~?Ibdru$u}0e{(39Y z%eNo;x;g&}P42o!V~ThTsjTQNMBG-AMrWf2YU<}Vw>Lo$(g6fw2wQdx{z!c*+n}~O z7kT-+N4@WluzfviNO>@ztxmTj^3aYv{oTw#qZihadQMKjl(}$$%Qjz-%XmZvF=`<_ zVC$%&c`$eiGet7gNXlsWVbWKoDc9ZtV%T1ulVqUco4I{?_4l{?cBFj>n|+QcNxVTK zeSReSTuNW&j8Y?u`>5S%mXm_I0)SW2aIxG52c%OU42Bq5VK0la|3F{h0lwXIr~aPT zjQ|H`oA4`jjmQ9?>wI54kJ%<3Y^U4coM{6x^^A6U5pvkw z_TU2Cps)Buf3-_WdzUpevg9lO+h4aEkJK^Qd+>izbUHd++_PA=15*;YJp!LJmUHged>%NydNoD2pqP^e5$t%yADHTfcoes=Jtv z=Z66;_&>3A$riTnxBW^%B==5c1cK_GVrV1{I@0899c{fp*StNul+%nR@r%$!JcV|2 z1C(GZ2&USO&0=P$gI8-D}@up#3 zXY@A)uHb$*=X03JB#R9?;%PWN!3U|W7(}8>;pQ$JfCgXxO^uN2B-hYXj^!JV zkjhK#=B|T=(>MCf9^JlUlbamQ5}9(kd0Un)d;U+P4IdASiXGkDk&{7?KO_!7rApis z5PDxOQI^}mk;PtJMf0!bCic{wZVR20a}utLBOjy!$KPNlM94d*CINp8R&4JgjlRZd zM>00`Rx$l}%FkG=VBi~h?XvtY)-|1Qm6gaLkP(%cfC8vXlFLz#Tw@S4r1{$&&e_$P zgMfwny`tGyYL>7t|2e5dYadNI-aBBxlJQ!2j#;Rx^t@O&yD7CGtfA~`jo08WEPGk| zHD6%6>0;$sps>m`2<7_2!WA*ACI}m3JrIO0>#{{(fF~i_W^$7AWxp~IVKbmC6zZ7y zM#}3kc_3PqmJ6#kl7}n^jMTC84>oXfI2Jl=wSnw*mY|=!p1_k}Z?cM3@Fku8+5-(C z+xQ*K+25$hu}ZtZBZ;Sm;Nig8te49j_4Zs}JX9>jt?7?&7TrjFpeAtha7i(~vZ@ zXP)+WBTLzF2((S28Q90>w=&}wxBp#5qh)0mP#CiBp;yPU?^&N^;GYcks>$3ZNwENw zXFvM{3O<)uB-(uYuTDuU$;CM2z}N`5VH_DpXQL+I$m=h)wXhEH9P~Z^P-G{?UDDiKz`5lKLh*<%p5IdzPce(5tm^ z<8}?^xAeRFs)qCf>S^^G;Q`xkBxYwOGg;F0cTg;n?;a#LX8!I+>wzOC+*1Ql5&=c+ z^)*hdutnVt4gUr9(>Fr>1gQ5Wv|{v+BnX7}y%E#8l&&Q_OM@(rycP9#gAdatUrkwH zRRejy(K0;U50GdjnOq>RiiV1m;2G7hXSO7pCX*$qkqb?mHS@BtU!!}b*8?69Rob%+ zRXoZL)S|XpwvIN0EPDE##`XkqQTx`mTDB3QPB%+HjUw$4N6}pIOZ#{!`{p>t2C$LdEpf^a;vrLVJ09J6Q%X zz?8q$C=zjl1W0lf-zrlpk+1xJ(OL%b?N)Kj_-%0%p&txweNSjM)Un5J({3Gn`DFdZ z)aQS{w>HW=F>7Y$A$flKDNiQn(ve=-OGnygF5NUw7SFbA+GNdRHNWh(_=nuZlbTMO zSh>`nDa-AWeoOc|q4t}>EkciT0pWoWo14CiPh9$PYV#VO<97bBC#cpI8)R+Wli%Jc zD=r@7kV5k@dvi9+XjocUb^fV8ET+u%o;45|->&^~x?90 z8c}yzz|Nfuo)yorr*fI%Rby>v>Z0?kUmYgWeNeIzOgj>*bfD6#Z)Dj;c6_^_?MMo$ z9J_KN(2dz5^I^8XNJnQX+xDn6Shza9X0;dtiQMWNLUZb$yu;WW9z}XCh|708Vscf& zv1EFjg5JY^G5*Xkc<{jCpXyFwAdaZc%AWyl{U2_${brXr&)Udfye42NP(mMa0#$%Q zn-l<%V0%w3VjTP)McOh1lYG~Q6(ze}s=0ME8*q5(qQi~_6x7XzH+W{T6!&5#^%dYQ zp~PZ}@k<(yaKX2(l6$R-rpZf-WH<)4OU8$8+a53_ z?*m3Ev0c5TqTxKLi84f>7T4hV7d_J5*SB`xx_X^+s)YJ~#s{2zD;$^+8sZLhm^kt}@?Fg4MIgzL`|Alg+8=f-JH- zMbfEcwWL5AgDa2GgGQIb!jVfMEvOjZ0g-Yg>_fYDPi0&2ZOyxBV&2pZ#B|YE$mG_5 z7|2#ZjuF`JMvSoq3;#0j-WeNL84wS!C4A`!v?FWTc4DaPNw4nsJRS>BkR+}gm*Mn+ z9NpcITus+^Y%7IHvc_eo-6ul{7JS@wCtEJp&eCt?#E{>I43Wk&Xi)k$H=&6oF4#~9 z+X|Xz?N|OjeFIrTif*7eiZ#>qlR-}a5mj7rFm51Z-MMbd$U@s$O0S%VOJ^WCmSknL z_I{H{j)UFxee-6`o{C&UuURkO>;vi7=Rvgbbw?*-|GSzvzn<3g$j9?KUR~8RJL>Yg zaUo1N6Qz=Ea*ePn+zu~axQ}mt6aOFYCx!*{wLRdUbn2~GSlT)-|X}4Wv>zVoYRPim} z3!<;EEseLVSGJDK@-+6R*PYVucRpK}P3R;tzuJ=~bMUQQzi^IT zqdJowU+rt^g-7?op4prMLTflO6a_ogxyzOkQ zPJolF^|vjn0Jk%(ew-+mInoZ7TWR}EJ&9OzM%h+s*yIL*ZXTbG)Ey`GT710QiLDJm z{g_-5^CdFMKkQ7vr%Q?10Zc$+4iq-P>@sKqbs@CNc;%~O=ODpX@u>@sUDqHE^|nxV_E=`e6Ga<}|j@x*?i z4}==fI9-F3=6GVy0OLC8Tga~4EbMLkSDMgsO;9UqUF&C@n~r77@4W0c4>Nw$BcUx- zMUHCa2EL4MXtqF69RGFDb9$s=#!dM9%{m`7%>75Pq!6<$o?+|Id4~H0eV%-^AR`KS zaB5R)b91TX#Q0@$9BLDpBu99sA1Pgb{&}M5%$YOJQK>}#E;FE5kJ@PCPyct{D`QYpNf8?&dr-Msv5KqUpF}EA(Q<(^`2$8)#l?UVs|lOmqxn zjJjrWa(Hf_Xa*=;do-OIlUSEI2)ZPCJIcAI{rP7bO$0;bA&EDKq-FK$)e+}rlevJ3q(>=Pg8jpW5^~(+U7vk)f3no zulMv>k8Dv$=^>u^y(hPv&BkfX&o7CJiCCb&Uw8=yI`PCWl@qk@J8QatSV;}vLg$4 zKN-auv=~nSD;6N(<*F8J4os~d_7f*^0}Wy*Sh=gbik*^+7@v9XBUjo)^`)g9gI!g@ z5R4?+ltB*dpFl7@mC#hREyW0uclg_Se;+%RADu6Yqyo_<;T`Wq#;R)2NvHugXyNB? zyG(NxpVl-YZ&)5v{ka;eEvW) zn9yIGt&nKLhoU)Am3)HFw_N1f6t?Bm+CUm^LYh%ZtK+tTf2COT-La);zI;8m`>YoI zw&93>epaU;f0fbFgkY5Iqv?;ZsQdP>T{wpfm<^)Fi#xvH4sw-xarekp!LC$=BJa@z zTD*C&EhcQXL^?bmM&~b9Y%?#Z4=iPS=3<^fx6BId$WfZ|>V6Zx!*&0HR+m*aqaum? z^3S=csrj3be&_qbjG$d!HUudkD`#6+%8=a?CuX)Eq>aF7?F@}VHi8`Jb2lgVXFLMI zK4B|r1u#{{84kV;c93t(VJXLTUIBt%!vlz1^a(2|224RS*eJp|CtNzGzktq8_2+Ku zc`BE=wfo0=_g;OOR0utWresZj=wpy)<;sMbOnE7P;`1eaoooQY^Rlpt(~fPg5}9Iw zIZzp4r`wAo#p0G!*aY4z(32 zyL}a75iaVyWAA`hP?t;^bUd|4$=WY?;AbLtO#&>FhKNAdC356$HNc+qgig4`e-ux! z?M9mb)O9Y!p-y{9WNoJAb;>*Ryv{zv$4;YKVJ7{3LU571f#4%lJTIz$e*XNqa+Adxiww5)S&rN1%-Np40{5G*rW>6;U^~&<`h=uBnaxL>zgwtM zr#8ceo$*^y(xKCi@L6n4&yfdm^yq>AbQ4Po73d?sE|wuc8)EkSRnq-B^|hB{dfpj! z^`o8h<3wA;u2PWhIOvJwgL*oeFV8Y?AN&Qjm9)~dgbc=%IBtZRn7AWMp%R0Hrz6P- zYxM7{`*K=)@&u6ecmb_8S#|Q%%;{67nqRDuzKRM**pqlx(2Ey5OTWuP+_>BJH<{9k z6)j#oc-~^;X}s1nYT4sZFlb+Jj&35HV4mSMa5OBJN1#C@w&Jhx{nu!sRHP6$KfS$o zD=}Kgj5Q(HvAFlUcR2@H4lL;TyP|=l26RbwfkMh-dc(*u#&^A{u{7*v-EkCW$pwwr z@@H>yVqM}0???iyAPh~cyNapR^zAZjXLbj0Wmy`$=4i~_YcNMkzC%#>&2K-=PF>Tf zd=y4yYg)Z+Ohwy{HA=ri2Be5O9is*TXI>*Bw~es1m|d5^JheVuYFvxsYj8mEln4Y1 z)mRY5Ek2N6`KT*be#w<9SAxrDYufGcFG=~C(Hw5=RK8SAI>NXl)4;_0i({~CofmXq z%@=kUA8GsC3xCd-G51Oz&F^)&HJBda<;$0cKP+CX*i}4Ad0Z_t=NBm#Y&}!+=FK5d ztmzPY+MlfY&+OT(WtBnQ-urUWLz?eCD{VX3HJre3E1;F}1!2R1g3d0kxv7NP=ss6c zxxe09M%HFx8jZ#-@EyXSghS4U#&Gsv$1lB;eqS@OQs*72x%Q-cD;OUWy$d+L4;>yE zotV_~R7J;vEN>PQj>Z~YFH^&uuA|9uHSX8R%%rk5FvCEQep^!1l zW5SxSBf%!h@sx_jm5R>rk+VI8M(R0rl0IZA-^0WvMr?y`g-K|u^bRwAT#vD4aq+G^-mzIK)x~3CU z4|&OcAh!to^5M1ptgZAA*abaiy3TGCn~sB|Ka#L&+hr`B@z+fchUKy1BF=dhlJ82l z%&BEMcxC*l;~X6tIw0f~D}P(yk9Tpe<;p}E#ws~3kdL$LqSaz z?_|W5^p51zYVe`r3$J?}ydKO=K?Z8N=wJ}B<9nPZz>BgcD;|c|2~(Q)($1kN7x-g= zS9r+e>pt}Z?ENJXX|wOU_gG^KTU@H^N@Bb4U0Hp>72JgZh{!77+imIWKAd@%0#XEF zIP&XRCEC5}k%$=!sg#i714mA{n_N1eTwdcCEKi|G=2-ddfS){%;&moVJXBWQ`HZX7 zyIbY3$GgEvdS>lZsYQ6-@^e{FUqmcimQh7=VlUAg;j`1XTMH#KkkUguVW@78l9M-q zG~B+IYxZ&^`*-#Y?U;DDa`|kdCPbVwzd5#gMuqzSERHyLG{oauvoIy)#8INsV<^)U zXB$XM$#oN}pPDjhQeVNR%;kYUtr32xep*{n9-YYt75HqAL*!sG{M{6vB`m)tEFyy2 zD7!L?)=yt*H{@bt4<&XqtdO|tRw4zOB=2Q{g_z@f>pBX4>D*O=0H>PFz{ss4i)PMy zKi#SkT%o=>T4NrG(_$(!W+?E`YaEO<_H7eSz{?_G7U}d^=6mp9uIf3%uGUb?@BfZj zaF)WvE3#61c0!C-lDf)Xscr}2?b zpvyDeX9|$7xb(F5H&VS>tVH0aOi^t|VA85E)P7jFph4X|;5*M{Pnv+1QqU=z)&8Q8(xy8BJ4l8yroy`DUxD_%v9@86%@?|mlI z365=1Gp{7DKj}G@t#g=OljOMZV?PJuz!H1=b}}P?q~@w@rUF>5X|&97#9DrkWZ@)i z&ah=PPNc)(qK<=>zm0PrnNmLLHN!&XH9(iuc|w#U-Zd_rcl-?b-?H58y_Bz1?n`dI+3*PL@<-CQ?nv@4Q%G{hy2v+)y}SqtKpUzTX9o zyPw^!oV?U=b7fUP|76p(++PZ*8^#^F+#*@AVxVfOGZun`VFeD9+i>)UHX2U>rIL64xjiaFV6-pI1(9`NsuSs%lh~;{4Sg>4Vq!FTji-%P? zGr{5-DcNI)z_NM*4iwFnX7A>)RXlpgVhSzu-ahqNxy*0co%PgKv?Dxo8Ec`W);dyE zb>z!G(?5;|m@Ihe=Iz_(`qb!>_Ya|X>Y81^D=1GVX!xGYEA>)bq$al&oe&0$42 zy1VK*F%Ei+|M-U?xEYfE^rbpT*aI$|48S9N`r*t=kkS-(*i>tS?t8e-T(YU89TL{PDG?o0o=5xh(#6&hl6i!Yb53! zxCgiAi7Ho$3e+gZ{xJI`PZ_wMvz=`e&cgb?x? z@MQ>5UeVRVlfiFcM+|H59!vOizYK*F;I6C9OOCO!wKeqAP^SaJu@by2oQJD~*b;Z~ z=Dd!YVLg90LODmq(8p&B2p1$}o^O)(3B4u3!jGRkS+)=clDB6JZbo8!RThC_@4PoaafhKM(lCbOwZ# zu+@=Di6{HwKA^7(A-Mo>H1=az(378Ql6NEtlYAixpzhXIT=|#iYtho(b@A`K8b31O3kr(^nloYcE!HX;1+R^I*q}98lawHS^hGV?D zR{Q3Gr8TNFIJ<%$4=I&gE0UcR8&UUj<11fgxaME} z`S|_)*}|4*aH2vdp~6{@)0nN$EB48rGv~b3t9u^u7;iXoqwZd=2+I!Y zLq1<`#6MQ%%`ER?>up4r&!_}KHRJFp(18#1<)zD(86vQlidzp^?tQ=pq%-CQm(kFl z4+fu5=N8J%l6D)6c0gFIZ}mncX5_Wj>5VSWWQHYuokpsty>AvG)YomzHDzdV2@XzX zVFe%n&;u=>P8dmGbpCIb^d0F35Udwf=?#{+&|Tq8teC2Q!9t$FPH-}_0xxUNvxVSP z`%VleoD02#PFNPq zd-_=Rx6F1eI2%d$DSk4hYHcWnG=OsYYoe+Tt(2xqNIr)SFlsLU!5LvI%#q%;AUykb zkgw|b`Euwn!m)}|o#hb*N#1dAhPpfKB4`Tj@>>UH3@L@??7EWv5!2qFq?`M(v98Jk#$24bd!-p`&tpEG1RT&rXbnM=b=ZdGKYGX5Fu5#~>IlZ^|NRdnxnxO9Z4XG&># z(#_DqwmiFYu%pa%!J$OqQyET^V{!G~vm|-~3X{n$i~)1KmtYMWfn81ohQRe`yK#sS zN^Ha6_(#WX4xoE{jN7LtWm<;2-HD5b=A+UB%cE#S0eG3bF)iDR)d<|xUO#p;sDB;_ zy?Q5SwmjrCQr6AgnF#QBC<+cxvN%>U0B34Lo|@@B zjpLegMOhufdGNdHza*9i03ZKc-~tpHOQ3NkIYxT#T{)3nqwnUqRVm(!cCf)MK7u(- z^aE=~rS5wrn2=Mav=U=ZCkjOq>fa_w%xy0KE3FJn7Uty}5n0 zsV~)K+M2GmUydAd(5JCik0Z^Ee~q33C}Oq%wTJtsQI(?RVWiaT z%83fLd$+nMA&1UFD?kHS;3Y1GA$EHmfTP($r5!XV4m=R}po(hsRbN@=DiB!BO-E6{ zf+|^Ag5DMs(?OFC8Z8{hhdn>r*q&iqG<8EgTAF67a8O!L`l*6U7&7xf$*)@thbN=s{fM5& zw}`_kwruLB47Pm`7qn9v4=B%A7>zI!l_` zFV40=913Q{m&)K>no10%?o;+ttn`j7-Z8yG)XN8P%)i<83!au`5!pX%g*GbiEcMp5 zUniK7AfngWOIdXKWV$1?u`0qwthqf;JCgD&|5mv$(G;>+>AqQqbvAy&_CNR1w6|Ep z70Z`@vcG4sV$si^8@F!z_CI)JPMVN@ zDGjk49-pm%X+=MQ96KaUHD|u8K$` zyKR8h5w~sWJqw!49+YOqUM5rFE-+Ci^`V1|8Qm87NwWF@cieaP|ITx+TKW9HoVPim zYC=iUhtW#(8u-O(o`YtiN2;M}r`OEyd0E8M6Oay^C1$cf9)KI{y6G(1-k0N?U$y_D zSHfPB2d+3>RBb}JnZWK6wOz*!h2}@OynU1X+h@_q_`&mc-6{5nK{7vZGJ|Sg=HldDD`QD@WWf4|ynEJ} z|61kRi|oU7W^bb1kv0BK#LO|}Ufpv(S|j!6Q0*-FwMiV#59 zlxwCe1K6$8YR5xSp(rnSN|s+>vU5!^^v-}n$GM|N6CzDmt$xJr`=ICaWWeN?HudE* zfE<$wm4L6(4c1pHj^}AN3SrC%^dScuF1gl}q3QP==+9eCipNSx5Jk5mW4mR`Rjds( zj3IyqSx*(o@jdj_YYzg!u(3!=3#nJqKRXwEl?rh;O?{D{Wl$MEBC$Bl&w5d>HQ4d( z+nOQd6iHaX$xMGl3uq~f_zq01;SzSe7)%`Ki4cR(0IS$n*mSY#aso+jE!yh z_3qt}@AF#M`8Jsdp&W3{*|$zu4*cK?-Y;Sn=?SdspJP2a@iqbY$;-9ibLfJak^zTI zlht8%NDl0E<$!JZhK8t%mehr!USt&!V1U==8N)0vzwsYGYUZesBc-iLcv;eI^m@%~ zp%syqG&Qz)}lSJ4fSYLRy-|547wrO}Vm`$QIwY>#?>Fbqd59e$$LUIxrIeDFps)j` z@8K<%Cr{%>G);n3m_YX6t-j{vp*OPtkyf+m)^&>X@PbCM6fiXC$}lXd-ptPhc=tLr zGE8#Qr|j*7(BB%9S;NXGBix_B(IWO(##`Ztb?kN!=N#IjE`DA@8TfUwjqBO}N_+o% z{F#O&!-^CuR*d=KeXkT`h&a)Ac=(@R_mWD8u`Ys1P4BEeUBPsQg0xTHLwJxC(WQpk zw47{B#y48JVud3F$d2lF5_{&+kINt7;JHNyKKn*YD~T-Vjy01Fl%#8BP4QgIZdZC< z@RTARtP1rUE8TKu0ZF&X1mPQ`DH=N?Z{F8?hNo{TpS+yMN%2SA9ZJ0aI^FN$3sPP( z(bx0eq;5^j;&V>IF0gcD&DY~J?*Nd-?R_1D4j@JnmNvNL`3GcyHRLaWM0kspZm$fv zp2qFpXgjZpZe0Os@q z;uA|)lN`8}@;p4t^2rjx_ImxtC!i(V`btP+_xj>{wMtpOYR1sES;eaDMvS~8nJw==)M*Ze-+`@#lu5#USO8yEl~Eb*DduUX6xQg0fBy zUCN`d4Um75%$ggV1hb~^qVL-uEW#xVf;G@<`i^k{XV6)Q9S5oMEJ+o|v1knx1r^ND zIIlnf58giQPk>Tl7>umD!y7Hd5UCXzlMkLdO;_|rK@3pW{ApbNupd}gY{OG|pKaot zKJ>gOYnnWM>Am44OO~W9Q-4i5r_*lv^fsA8hiJH$!VTZwf9zB>+j$7y{OUYY%cXUG zQ9BYC`B%3WRDI$vBCVVI=uM_Q2_X4$m|6Xoo1$6@n4iWne1iLVp{H`X-NCo7KiG%K z_Iet~tc&?pHDA<_1;Qo2@1TY>v~u0*HAMUdCkWcL9DTW_-?rarO!f+9AJRGN1(Gu< zt*;a`Zk1Je=p7BQ&g(Lk<7w*;P5~P6DA1E1CM7PHZAziEo`>m&R%RkCspQEeGyeMX z&&yo2*hU@wMl_}AcD+Jz&mCMpbbG^FQXLOMi-yIBSTN!|U}OsPW#5Sg@sm(|@^+KD z)e^HrkEhm?b&h&mnnDB5qtocTqglfnJ8fTeq8eiQj=1*aZP1-6q5T@K0&m$iU|@5AHQ+21g7{oe~4;zC6A(w`WD-D_qgEyV&VH+)Yp8(V|4s z?~z{fCbgJWXxG%g&&AFt^;?II!wK~ z{&x?%pzoOen&tF~`K36j22tKUC4Z9(A5rVK5;0UUZ)8}h{vq9swHI@iYI5&!(1X{% z*XnuiaV6I>ewWLZsM5Q7L8Hfz(7pWPa9chtq_Oiw#D3*gtm8iLD{dEwIHo6N*B6PTu#kHPvuu-$4=Q;;8JG8WA5x>j5 zDjixerkbDH5BlqL5YOKcr%a2LCBFFfgVbUU4h{oGgJWwFW@EBtOJrmut=G-sb5mv; zO)D|?rG}*%o5$K2d^ywbJbo9;=G5)*_gq!_WYjmOZ$1m^X6FOnVgy~Vlcie4g+Kh1 zx5!-*YpOI<5SYEt8%LeEeYRNdMaUN8Fx6CJD|a!D!o6BOf3{;gBf(? z0W~J(bq%8=El;~y+Rpu?2f^~dGP*cop-}2qG3#Uy#1eQT_2ryc0DC%Anxjoy*2mxYm^OX0)gW z7&~anCh(W)4fQz1=!V27g}6+*ps&i&hBUqn=_)xuttD|~^i8(E9KSlw4U0-)m+E{b z`9WWT#RGk*3ttssiP7ip9wC<&mzc2}8`%irC}65JC(2*$KypH&4DTkjBTu^q(lY9u z3fEr#TWJ}V%AyX{W+5#ocax{FZ+I{*5K*StYgE;8iD-T0Q^sZS_eKAEbz0HMs(cJ) ziSL5hm2pqHec&58i%x}W9xqQP-du%tN1M}$*ge$w-#*UzV`KXb$NSWHN)NH#`=O0! zK?HScwYX4;-`d5_z7f>T25%U@;OOv!lhsw*!`|son;P!__P9LXYgzr#_dmHYO8-4EFWlkv)Iv88z?|o= zuxX&59BQj&A3DO#_;0QUhd@aNGC-B8MCI|tWxjB`7*lwK%r64J3Ey<@kIj#$IX^)o z2hS_Z`K;+>IEY;#Wsg2?gLjX%&9$epV_nkfwd9=%AovmngW2oqA^oZkMok8X2}G}C0J`Mw|T zH5b@;X1QbhnWs#plg9Ul&L~x(f){23vrUrZKz5m9k}ML6LOTu#nbC8y#6GfJ?NOft zIYipm#FaTF7;sL0X+-`iW?4XkmbKk!a~AV&%9(Re#5*9M^(cg1f8vC$VAg@#@}f;r zQ)$h)HjPn_VJ**tU%S9>q_f|Vy?gg!o{q+SD3)DC+u4L~o9_b*Sghhw-%LAMbU>#W z-p}9IG6iqm%e^D}0I*FAwM<^rli#LJae?q~Z&ISHP{%*lQ0~>MA6EZ7=a{(fMOq%( zkai>{>+-)pq=oo-Pi}ZGao!JUX{Xk3@-E)%YR*3V#hl5BZheQpsQ&%(&({0UcGxa-_&sImrf*gzYiRqdbPP%_+ND= z@0xOCie++I>e`S7Z_IyJ53lwru3XMbV@@6&9A2ZrdxtFWPpoU|P9EamCBD5iF@89J zg~^QaFzsVKhJc?aI}jLy-rBsR9 z$CWF7Y0RgS)x(E0c#~9;&t7wN@Jq-^^yk)`KNB5R=z{(63)wtdCuT*6zIB1K7|h)(%`-AQ|1|0IFQktCX{vVgIg%Ub9y>uFi6W>=N!&uU(6}< zWoM&a#|Hm({?Eh`i*jCS;96$+DC;iU7k(KW)NlD;y&FEi|FTp-tp;yS&lvt1v7#aG zoI<+a8c7UB?*peTHHaa0rd9`XI%(`7<)Fk(FsUL$izFJfgW#R6C&P;I^mW6^#4a>{ zt>Lfy&Cz(luA*Q~f+s7HI0)=o1YWX|XF+eTvzB2>k|xrZnVa56vYYzQKRI*iCUVO5 z3}XBx+-;2Xrl6~1UEfn2G1}=PslZ;(|w1_BlvR4C8kYBgUCszMjI-nKMt=6h< zo3!)6{d+L39Ygb+83ahCY5`zCV=crB*YIl^9@~cYwSD3=$d+q?Pc*0${P!1%eN6+1 zr04SbHibZ;(wSM5>`P>JE(x1 zEsodZ`yp%|1HGVbKl&5TmtN7!0nCXt2D}H@sq&zO+ArBPwA~*BL|M^e^R{!SbLfR2 zbOLlBqCt)MbxUZQ%t10%A%EUB)WKsT)0F2(V8AXHNniDTgm>o6Pid%e_RNP*SAz*( zz6-Kgr|G1Ld>=oD_z2Lm5n6xWFn7a**A>jjd*FwFR)7n}SS;-6*btk*ZT3{Q-*x~)AqdVg zFx=Va4}{{ZXjxbmZU22IITpG2oy-N#RgTHbj@g_{xR(__{-aohK)0hVfy<8!6#{Xe zbN__4tImkfF?9B2hCuav?MO%n25UEIaywk+7~}Q0Xo!pqQ0Wc?cgyQP6bz9*W9LJy z?LVPbgeV~5eDIH1Hg5(2zqNnxI_tW@{UCUDpr#4#TEM@dy7R1eM5j(Dc&IunQrMzh z5)cG)w|{cpOq5%| z7Q0Q-k^tUF;hXM%Qr6!%^`?fj0+ym z2GyY;nj=X#h(b+mZ*Wn>@n))Bd$KFNpU z`CSijSo^YHWB4VdN@ARgyP^6QE;can*gci=Y+KAabq3Ye3&u^VQ?*PEx3IF6Uq8As zO?@Ljny*8jFfPAQ8gILYPW9Nh5QYW^1|z|$6vxH`gt79ti9;rbblEnQ9+XrI>fca* z+n050TNwR;ei)9kwvy(`$z=e6uBNL9!{bkTCc9?Zv>sCYvuDpVzMj}LkCnTNCzH%- zRx^hmUPGDhX%TS|bnTz)`CMmmgD~VaQu50i@lYxd0)!#l#tr`}=_MPd?ZHZ>Od-BeWr+cx z_Zt{BBdLh=AU~9KY?3`k4vCAZWXqOK)W)rcKW*H9>Egv!CC8-9HYKqh_EgBjE{vS-g%-@8=9>UeW;9GS#zMP*d$y{00A!s_GC&*=@Z^*TpGaQn*s z?dfS+M40Zhi$%~44s=^|k&Jg+O%JKh`GwU)G%ladqh296$i{^6viNiHc=NRIn7kL= zrwGOzmN(e&k6yg!|KbgLS)xDzG^9A7#BTFCI+B-F$*hr05O`9~`)7_SX8pr}VU{cv$rISC>5=%}YXw^|}@v>ZU#+ke>|I<#Tnto!H)g@EI#Ap2UiiID5?$(}V|Sby2v0gM~QO@j}D4Poh_HzoL-&%DMdsMfInn|QaQ9?gDw z|HlZDr~O8+5r-A|1yF;STW6`S=uMyr@+lXZaskB?U04m})I24IjLv2i9pan3M;@r~ z3jox^x7AOD&bWxZGi}8fNswTd`&R{3%3k4@RTDLCH2L>&Y0r3{7At9wGY5LYJ52Tk z<+7RukN3jbfz=L4n-~S+{1My?tcSCnoy?x;E z>3O_7HlBm%t*uou9)4cehflb(1=lWz+ds z35Qk&PG^vSPn)m31!y9@Vm8~QO)6KL|Q&T?= z&;hM~28x!G6vxE1`M*8v{=vQN&7Wz?U1Or|^gYM3=9BPh+qP|2D;4m}!{LTNechv{ zPZ#@##7!@+rrsC;zn%SpJ^D@c^{duD+GAvq7KOnRO%mcsO~mE8xc!>TT3dcL^u1&Z zPyk7|+Fc|s73$ZouWq-BK6SjV`wr;KsUb@j`cfAvPPKD5G3sSxIkRZi;@%LAxjb)~ zPl<$3JAQZ*TJ55A8@9wEC%AZjAJ&RiO$SQp?R~skRJ~6QVZPuxE`Q9{?(T@!5*B^E zw74e{1q?UWZSGPutP7D~Gmmmf$Bomomu2ir1u9eetZiB7>X&l_7VVh*OKl;Y#8gKv zn&8liPPGp&It3S?3a}0pNz{bY`OkljHtJusFi?xl+?36yLtL@H@mwiD7SGyo<||~$ z&W@0P8e0#g^c$H}N?96vvee8{b3*UP?WCJ(z0xhq7k+1I*)$y7P$Asg#T zDw2Usu*4Qu{dIgz?<_tiV7!;cmTG#N#&F|Kq!n|cJmjq1(P*g0*$tdpDDxB?v?DdQ z0xFhl#%O;W?-2x->3pfB`+jp6H)9n?dP49fp$?&;c4p4WIS;AG=MPIGR{xMDHH`v;XkJ zwuOUnKez#ZCFqwDVLeot=$%TzM9<OygU}qX;bxAw%yaqD<|~m+4C*p2AzHsJkAyR zmz08h@NMa2NhHqtS8XtO(szyxs};^6!P|)OjxH`@xtfs z=;O}4J{f%XaJT5RU~nF1rF6guFgHNFjx2I=RX`@%-Q8Jve$qCTWPVg zAey|+Y%h@_2fvkvckcG>B@28`w^?5E)Ig}O)Wx2b6xw9#Mt1Y~y@8F?H1htmE85k7 zdL}dDt*?QFiOe+k2xX~Azv$}PLZcC-uzhl~Xstl$GS!<;cxi8PEHmA=%at={rh1Sq zQrwhhg-O`k)Nv)28kXgi&4sYQ_&&TmGsbyzN3OpMIRX63)&c>{JE5%b15ni*z zSx2a^y_;su4(Kn|(y_*h-_9Y|vl(=;z zc!^}l(ijzaoaMBY)4F&34n30t>ekZXs&%Q_Po5slWLb`${Pv+wUAsDFI(=I62GLSc z-&FIvD?dLDh_hJHyv?2^b+xSfNHx^!10u4nt7tw_zU$yG9|paB7TXqK@3ua~>>Kp+ zRU>zb==G)c#Jpi;)w3i?GKGb<=*eR;=)!qCJtqGFUq`=4jp@jrm7%WJL4w?pG|-u+ zC@v9*QzR|8pl&6?Ozfp=9LF^W-t&;_fQUdarSleO5oYDk3#1&B*AKDb8b%uhT)Nbl zw=r9ogXBWk=9c0s$jeKJ>~7N6*M`Fe|MwT7QW^eC<90O^7$CVkmZQkXG&DhH!qTNn z_4R#7fh#a#gJO3LSCT|XNC@`Ko3BOK^6#%Iq2nNvIU^~4nT2zjk?JjGBJ(Cbb`v7` zta-zn#lB~OBc9c>hzf8$<6qSPNi;_VRjt;5v}WKHrdUx9>9voM4M%RM9s1h?BGkT$wOo{OJ8(}td1$|y)8@{ z4CjNQvys^EB(hTi(*6tUr;Hl%mE0b$uK_-45JPL4LNbN~Bhg$Iam-#n7`Lhj@3Ja?8W7)2FU~AS{&J07=sOjVFff9ng zTXgZKJO+Mf^5uw1Ig<9ZHKbGtBh;H#Y99^3O0e2rb|uq;wzOf>>mJ- zBW@9cw13_7&#RIdQVWlhKNV^*oP6Va)IbM4?COat~XB`$H8ObyEy_mJSPBp04n+u4gs%)gbX7Ova~%ruFBB0949R z?%)dsRE`;4CpB1p2$P2(o!I2x6XvbxTqW*5PijaxR%1_Gw;1Pv1| z$>PtY4^6iSaDEE!M}6&xbG9E$ilc1B-Qd?`*RcPN8{Yiy_9p)?Dbzx*>A1(}6Y|8i z&N*?A#M$vO>0nU02*YzZrxN?|2^Dg!O;Iqy_+en?U7cY*Sj!1R^(;pTQyL-v%X9dJ zkpXoi)fvL5bl(Mw`f*IHqI8m%IpsooaI*IJq{@R6F!>7aBE#3UuMY`%zNI|@-3*1h zXn&0E-BY>xwXTee(MhM|m%3eDGzkR=P`e@MzZq;w4ei_?O+Mzbs zw}$A%$H!~pnLuS+4@=oH0;}FU(85@`(YvtZS&~?Y1X;~15hgkDuV4=ubRcKX zet&{8!ex@qn<}+0_^V1@+HG&;e9Dom4CkM7BDO<~CAidgp248-I1tdgcR88Rf+N>s6al|5|2_#EHYjraCB%u@ z1ttI27Zo{9WMP2c>g@tQ*m-;(<;9Z;s0t{y{ev4-t`L7sd?I-?96~vGRj5J0KUF^! z9OgPnk;zP2x$~y)@BFP`PZ}s2NWRMRs$~(yr;BIm{zvwyocS9O0dROPz!!jYX+39A z*-xn)06YjMGnH(MltKU68RvtM8hH-PeE~ak>Z%6~fIa{lJ96K1$p@J&w1M6c)-y5= z%iGP?u<3h;R9`9saY><}NubU7DZGn(S3K2%J-1R>utq7Dy(}!Bp07U-gDN>92yjO; zaD|c5B{&OA0>^9y*o24wLbNY}huavJy*}W{FMZNQQ4$s4*q{_v% zIXbQulZy*otTsCB>-*PR+<|hjob^LM9mefT;Dtsai7h*IOE!etMo9EbM9VCv)IdWT z4~wu%$^cRX==|IIOy}s}Q7U1QrzG%hJIfXaU^r-($Ft|lnl)>AM@J9N-Bzz+@5?rckOH}s!JW+f)F%t0n@dD{9%IG}IB84gZ1c5wKw zvjE=$Sy^4M(38+;Z~Vw}{oWdD-9IN3I+@BU7eaA{r=^aNf<1+)e#Nx7U@!of=1R*Z zsSi6`__Jlps5c2M!9B{wXDcRU6?%Xq%0fd}@Q~@jKW+UQlJ+@V3AzLvB=c1cy`xX1 ze3qt}#CIAjF{j-qpRa>w-6yZXvvC%cPw#386kDP*DnBRNbN8#7)h;yUPn|N%+YG)H zFnb4yVMhj^K0@+A!SQrLZntfPv6!_Y5(-2)C$DUE%E>O3T&rcl-&O;w-)@k~*=qA2 zBa~Kq>C)GA(RuHJi2SC(a!J-TRL0+X_442_M{^W#>}XSM$#m)$n~iBn~qv zSb4e`q0W;9K1A|Qq>FMU@_()=KqMZksBWCG8EHPN*cG^=Yw^61!QRvZA*e2O9T(@W+6&98+}wg9wa|6BF0*#+RG|`0Y$2)tFP69ZEca^AmZs zm~5g%;s8|j%8hJxjxDQavI`yZE6UEY=1X4?j7?XIZtcB$_kIo^jQ>$tvj%{7*(D2Q z{+YDuc>S`d?vefXvIN`$yIr+KU8*^zwg90m=r9x3~+mYh%(+)BjZ8mn#K^&FFfZ1*V0K?g5OUI z!7{II>Nib`@IjsXBU9CxdebZJ@7)4=9^Nw&(azOwMi;KOe&*5`vMlw(#!QuFiA z(Ed-=PawRNH7hv9kZ0!`x=fqabf!k1%PWSDVFlDQyQ&3pWhlhi7|KB7U_C=a>_Or^ zZdx2isGrArjOKi5X=wq3e`@fhQOF9~Qr)$bgsBg+f@_;yQ4`dTdI%uXf<;mr5bgpK z;g(rPWwLX0G3?r+7ZDqLO)p9kiNVqp>be{lvVm`)wh0(?o>%!+HK>~ZyqJ$|e`1+g zaYsmz4d?g@5zRQNSU-+Xg z;k$G95Tzx_J$TeTLM5J1ZRu>?pcR)N*@js1v<+eiV|m9c=h?~+k?nbfmn$rBdTL0k zJ}`~s?!ti~d*UBV&gX1g+prFgU~`e&U8OQGnGvC0B|kX-l}FV9CmJ1cIKaC{-F+Bb z_f*)@kKY-(CxIRCZJ&^D;wCahTzy^#QMlVww@YCCn%xok_PQ@7cZY!~nuf#`0|D8w zu;-uIr+mpbIEP*63_AFBb`BHq=^3&06uZWIc?1MK4*ff zaG7VTCaVBBp!hlHsOqjv<8U?&uMB=`whuY5646jPu&U8bfpK0t5L|Y!%tb6yXjaW4 zGv)I&Htp*lth_gi!Y?8)-z5Wd0UyL;=Pou^-o9sO&=L$>lF||iKS(Hs+IcfaY(?ZY z2`;6PHSlI%WWDtEP~FJ{CvS@yE~LK|*9<5VOGi5YPd_<21D}D8x8Xm_qhBU@$P?^m zP0wDBS5mfRJX3a@XX-U3bp@PnwY1Jkz9E@_vxD4*-*Rf$t>)4Act1LyS_ts)@&i^8 zbA_nW8)!Q|F$JldNp5f`X7+TdcbVNIXD;iy&$V@~hopX7M^4JX)b-pBE3`w0$VB`v zpJ2cl!z)MP=_JgA>`jypF>&ZudRmBYAC3*?!1g9liQz?3gI(!8YcOjIO{vMOQRNy^ zrS3h)hoC-<0pvelY_Q!}CLSZ-U;r(5O2nYEO_J17f>}~R27jjtK&mWsjBh>=y~Q!N zUtayiKB$4QJu3bnOZY?LED{ad1ADTPud^pqbFj`4?x|F_&U?{8{S~#-RQZu)pyIwE zUv6GZU&=S}5-^voKs4mHU+zy7GeKqz;7x+KXg7~dxG9TOLS9DW4(4ML`JT`X_R4O& z4K=w+jLf#5=4Z0oW^bsa>@2-Fq+h=){be19O!T3dO=xDM&`^^AR0t7w)vH7zDH?gM zq+hQu{=$wHgOV+dpHHp5={|XyiKrZs*d-QCk!n3Ao_LLVpw)|Dw~nM>{O)RILEK-| z6Z?aecwskq-+-@<;U1hC9!y}Z=vFLNrQqxn+?)23s;%W;PZSM@BXm68mPOSP)~Adj z>aZ|zj=>!}M}3Mt$s6CmBG9-cVp4Fm>);&#<}rN8gJtVbND1ZRTh=irYSHZp)u4xtS_VxV7=`4eC2@Ff}d?N#c&>UE*WrntbR8c#_a)Os1=J&nh+p3__%V2xX1*Pd)J=$}gFZl$lV70Iw;IAd@qYpbER zTp@Z132M-ieu|rF;{ZqRkhF)YvIF1p=;g}++v|nWuAzMrK~^XC05~z%UhD;C*D(lYxdqf^ z1GBXFWLYB13Y+prdY*)N`NGR>7@UI({Y1?#W)^XvB^`5JMS9JxY9}GL4Tx$&2q-SPs+Un;mjs3^}FSKtzZ2$lO literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb200/gdn2_fixed_seq_flops.png b/benchmark/linear_attention/results/gdn2/gb200/gdn2_fixed_seq_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..dbd40c6bcabda530a766f96ccd764d834bc4a6e1 GIT binary patch literal 79705 zcmdqJc{rEtyFU6DDwL^78A2kIF_j^dNU2PT$WSCxk||V%ifBMG7nzfU2xW*$Q4~>8 z$e1Zf=2`o^)%yO{I`*;u+WU`vEbDmR@4M=Q=eh6ezOM5+&-1#kYHO-9GjTFe6ve!6 zuc|IZF$hu=od?5W{K?j-6$bbpc^9=KE{7aWy0{&8vZM|kcR6#)!R3^VIp0}JCubXn z(_6%)q{O8*^I5yNoN<Vh4tGGH{iiV>0 zsqWHqkNNq{O>f25rX{}}RI=75USck?Q@4D5>DsgY#+k;VOjrBM zrF@B(Oz2;;Tyj~uH}uxbeEZwtR~y)F>rBkZZB&mgn5fn=V7v zm1D<`DxMqN`Tq9SyolC3JC1cq7Xq?MrkOlv#y=nF?y8D>dUk-*wAlUP{rkd4jvQ%A z&nmf;p1x`G=FK@b)EkZzxeW#x<=OV>4Ni~sPfqt|(NGSpC0(q6Uw;n|Uy6+6pyKfx zts8S~BDyAie!UzY&-W?IjF(5=<&CqPXWiEFi*!rOd#YpCC|!`(cH`}xIaR*>6zsL%inwq+g_Ojo)bxS+{l;P8-PoLM;>R3$ide5?tbiXfS zRi5>U^O_b>Z<-GKR_MC)$?0z@2M#R5S6f?~&n?0IiC%NllH0d8xVhVg^=Rq$Ea?;H z=cj3DX(8*reEITYHa1r^!bKBoKHhtjlT(>Lk0m+NSw2Qvbz16IYiTgH zPNRn1le;EPxx#NL2OB#(&8AJ8`o?a4eD&G2zaAk(OMQ57c+=Cu!quuzzfQ-*RTLYW z4R=o8Z0NT*J3Bl3;nDHkvI+{FKR%~_jnLB4s=#irtyvQgDPg9Pp#8kMnkCP!;Yw)e zie=1stq!ThCq3VM_`v4Jv`Qsh^zet|eJi4(qb=9&_A+rBY%OKw<>h@+8+2fa}G?#QfC{DAo@&J?S?dY|OLMefGqmwcEDok6?9m zb={Edmq*9*yNb>Zh*AMVofSR%7b28~ZZQL8KKZ+yS0rZ^FE38BGuN&`vvV3d-scjC ze+M$Kuv8!7QoFWNCv9T1?@;YyE3Njnws`$*8+$ZuZ66#dcF#s^=-MHGudWs|B8&Op zGY%&QzJ>kXy?g67Y*6hP@O%)&qaY+|kojfn-D_0$-9LynMdJI8uH&A(v-J#uNlwQDaC`Pnufg=S|*YNEE;8|4p~JUv6RQS0uz z&#^z(ty}lhv2!sMf8y;e>#sS2+}zy1J6hT#ubmww5x1B@`$>#-J#y2DDHN~(I@6O7i=oepFgL>j^?Yr zzP1Y67x?DQUf(NMUgDXsC=3h?K93(iF4=wW!nWft=^vUDt){de9+7hC{XjE5K2DbP zZRke!Rjd5syyw^I=;+`lUN)M;Bee~8RUu|~A)z0)u(*6ERkN>n&dh5{t*}~FUOo~3 zKTgie_jk5CcE4Nu@aPkk7cXAOBHkLmm14hs4-L^!U%q}#IGnD3#>GX|#YJX^q9Qvt z_l;GX^xKeLW!-*p-?(wZ>3meLatD5ub!&-YM`!2plP9livu})d>Z+uCP?KU5&#&&O ziK_?`Ih13+32ED+{KmzrR~dB=9paZ2whQX&n!d%`;9qRY>*_f9`#5rYqUp19)=l{t z^-fAkO1+=cIo1mZ&`@|+lIZ&CQjVWGb>q<^Np}yAb{vc&`KQ*Ryagnw2fuv({-DKq zypX}#Z-OEsjK_~3mv!u*$3sb3lrP&LBxL^L(jw8Z;m)Te01sI3S93EH$E~dcG-Krx zP-xs|C#8bcDKfdax#c3CB9&YT3_MuUz$;}=zk2f#0vRm2-$mayDaxTFg!RuCaRfdOF z+7`}iaz2Xsy^2x=W>DR;=Sun6$DAU^kqi^@I32yUNhk{UA6~zHeZ{l$6A}5YKfg!? z4p<335fHrjt7PVvLLbZIy@Pj{DD&yjKEwJciJ-9Gatco011LUGz5DTbH^kl2BuonR z&MD51HJBCH?hd_qlP%l(j~Dy8b>d>qE-vqCN2C!@G&@mNt^$+TZ0_vq6T#Z2IDQM= zsO5V=Xa({or~08u?hP9@>`hYrly0E4!+YM?qTtgfJzzI}S~@yqCtt(Py^J0loSdR3 zq@yW5qbH}uZiR*(uHU$E%%gZt@0tUa#&*nmc1T?(L1j_4ebcr)_6S{F-Q*C)+|MB&>}%JG z{QUWoo~p$$4RkX?zA|!mcklhwa$hHjp31haUEjB(eJ`V2Xk`C>(=660N6$}c03&%` zn!3Xi6OnuD9zA}%0(GcvQgO3@z;14Zvj_CS;%NFF~oJU$+=r+o5P(I8W@SNAp`$dtTgfAeL?a*{74makYL zb?iBH;>3w~;{vCr=f*_wz%mhArKMlS@4EaU?a)S-{(3QkrqHUc-sXZWw`FWDU18>0 zFD%Tk#j08rAc=sfK-ZHscLR}=*CD%o0eD=@!X1@lAH=J?4yFFm#fujk<&n@&wG^-Y z_3Kx__3LG&f99_vaJo)5IBnjsgS)>WYYFu(dW*VcuCG5UuX#hJiKL9otL6e{8fyFL zW}mn?-j-qyy7TAHlTf#;410T4VeQ(r zb`J;e<;RaJ`1$#9c6NTvv8)BW${1PZ(3H=t6{q9}und6ghs521H+7#L6`|tKjs39x z{2;tBU~S>A>(b?ym$1HUZRJJ!SBn%kktwI8uyju^Oi7!Yn;)~Z^uzZjIChpB_0Uz@ z&&p+=e9uWe$1XX2TTC6eEyIY&*{z~N`|Dd#WNNTsY79HWaswY2Wur;a{3&AfmA{yx|5KYzvy4lYq)z-BE0mQy=*YV*U444o`f zMPTfm;*sg$!9hXgD5f;j$5buoj#r9Sm!Y^K1_}KB|7lFZEzIvr<>50BU z>P*|tQ>`UDrSsF9%7b}XqYnsW4Ay6G-P4N8T)mw_({l{T7};(mX>7dz%yFRcdgJ<# z_mF1I!IaTam&vKA_P&p)46LmFAtCIQkrKhcl6^82=T%kd0a-|yL$Q4R`n6h5t(R%3 z7c(m>>yGo|%MgabI*GnAw*v3gT!>3Fe0-80?}Y^{2XZ`aX_=y5US7T@wLc*>b>p^E zUwp6-BrP9#dgk~p--kM;cPq3Gnr2FPJX88!F(#$Jx^9OLK8aHGA$#B@CR0nVz*E+>~CpBRygU*6>3?X>dxt-wHLymoMHlDdGD6gyDAI$(J6n1^EK z(9jUUI@Gb@?{96hQiTrQK2{kfGKn;sefpdH>||%~qx}5pC(|0OUNd6z)BO_U6dfJdbLzrgl`hYDwZMp1kR4jg!e{H+S4@~K;V ztBlObk)E0wCoc6MZgzBV4YCt9z@@JqTh)B}u>naM!EF~?m5z%i%6P!$`<^5g)t6$ABk~+t`Ru0lZ$5{08>Dq8kMS zZ>*70q1XUP^tiGGH?Ed2-Apl9S@jq{Nzj@Z9|*0E2H5H@jBPGi?Z?)EMmiC=#CD8n zmFR~2Q(xCV3j%o0(BCf66O1oX*vlKt5Cbynf^jM}EPu5;+XUVrS>FKdg-+6*E*r0* z;J)#JCf%PL;>I722#xey=GuPxMAJ#lUnfp~d6Yo4w1#KCZ+Uh;WgLYqY+zhj2FHkI zrye@$3Rl;VRsq59-OVdI=3Fzd;GF%@ckW!D8tJj)Y1d%j2cpdy+S-gx&vB&tVBa9? z;3EU!mE_Q3@p+Yg4h+1;8$4FrHocmQYtz7PQ}xe#4;)yzN>qCs-N_o&vfx$mQ**2F$Z z3;XPN*ejCaLp-Er&jU^+OQTL!HcUr@uMn?0(R7c;kg|L)uL#G&llH^r))r5xg~GEIuufE~Bm)gP-%-sjOc?*Fo^ zjHWoQXzb9fyLW?-8Te7PKwj*`^0RSqEd>Yz2sTGse_uD{TDd|jAg6z}MTH6j4T_eH zW+NaVx6*|*Xg*Xs%LB=Z5qjqVsZ_}NiCA9b!QF3f3F;s*;MldzPK}VBi{!Eb;~+pn z>%by_+khK4RPSuF*G|>siMo9|Q8z_HPH8csW-+yFjpWY4q&)P0RBE~B*CIEGOP4R}B&srkb6A4p4G!ZaUh_AaU6)_YYjMc@qoWOj z#??3g&(Vgmlgb6miiS23Ek^(lvVYHVf;3*Cp-qaQ-**GbpU z;Ql-ZZX*5W&6^aTgPTJ4_O1(`S}%++pO9XASJ^uV$b(N_o)bOuE^MC5ZL_ENU&;q1 ziCWVobI|%M@lZpZF6Xe=~TEqGHi4KL=7pyfD6w*eOig`ObtbFXm-k( zYz-QZs*&?|gero00els2-oDL*=9{Eh;Ei3Nf`Yi^XuZ9?W$fygfq1#EpK-f<8;+YV zkSh5z9DH?i^Yx^~#6lmNn;sKBn!g#qD0VQr=gIA@b}zqvT?_J}0*Hc^gM%ZymI)`% zpYTtVc9!Y0Q|e=H)A09n6u^B--hpoR9SFlKz-(RCToR_m)bBpedmlDP^mK#l2tgE* zz6GqKuu-mXVp0-e$p~~p2=3jt??qPW{Db^&a?ahyhB{tTK7^P8Y6fq4<;j!n;-=k4f3TzH=yvB{9fpsQyQNCmwNqr(<+*H&$M+0wEu zBqRiVb77@hdU1WtyVtK5p+a0lrChvhP5cbYa^So!0U<%~T(#5W{vu{rwlPYhf0vcJj6w48CA#&*6F#y;) z`7K+vaLYN-gNiV?!^tlu#!PrdY|OO+=N|LlZ7(>zeX0Q!FI?DxUKRD^vGL#^R6oaG z&2rRW>S=9Gr3v?vxYdd_Skyf98}`aut>_>YuXq>~2lRRzNE zxUKCqWW+?^NYQ@4=YS9ab#v@=@3*2S<<%{1ZN~rumjc`#@=(%@k>#Ijb8mZ2W5Yye zGNHzSLQHs193pxoweU%=KXY!NxmZZ#z6lYi&9U4csC?B?z zAPXW$wl;0>D=uD*l)vfV#m@HjgyGJYSc=+W4`+0MJIp5JKye^b5VqAQ|I}5wB`o)g z8^jEA$ktQHRoGB-?44Ae9F|zC+zr*TVfw@cP#<{3tuL9VQJU$s%Il2p}dLu!-TTgdM{y>br zGV`8G$ybt>H@ZD8yisY~j$kl~LRlh--M%qb=g!$T@7^tI_5PzsYWi4STYvv`;OM1- z8et!zC|&L0=^Z~hER;7c5-UQLHsk$RSC?$}k`$7>X4Hse>D%`XyvbnAGJN~?EyL2K zWjI=F931`#A<|jMo!(y5IsLnRH#Hdg`>}4exf=KbjjoY&ualL@^p;KxSo09mUXSF( zsMgc0*Mfo~u0$I@I(D(%iIBHA5C-Q@HRkS1F+P1yifAS&@~pf{pT5+m9ZL0O8}xU7 zlAX;?z4|jhp&WkJNx`!W)sS#gizpo+0%K-xlbav6Q*2z@PJKA?=lMlCTA!O$UF-Sz zcUi_dF?`W&n z_Jh02`wJx+?>Rhm8@xMO40250;v34%(%rjv)9gfI*6iB`0~wcmw*PbbMk;{t;(!MM=sd0h;xM4|p%@et6hMB( zSe{3Xru6~qWSB4Ibq(r}NbE;mrOjeuYN?vBuMoB}O8y=mVkfG21eu)w7spwRP4>CMg*#=XSv?JcW|J^GtE-wE}NJbG9&dtsN z5kVoLYiL01PPu{AvmLZUJpBGqX>WAo2>J(xK+DCf>o(^hk0{4>?mc*L6$FLVl<(!s zmji->8wyTq3j3?nyxskAl-WaWyKBOZCrY#2TWmiG)9eh}WT4p@W~{Y{i~HHX!tpy$4XB)Tu9zcD%mnk$d^wkhLyw-@6#O71;Q4 zNOe+Pv+m(GOPoYeHo!fUqsah4co9Xo$9>IEaKibCU!0L>w|T(ozDi0;A|e>&!$yk& zOiwN2l-b*_JuPOGyAyC=XnM@fxX@*vJtsXsJ%|q$)NY6|>jecDsW4DCgFXU)5-BKf zU}B(YyOqK;4L?2F4O)t`<}tvU1rYF;Vvj8Vz2T|$mOUUZ_U_+*8O%gw05k88+N31_ zcUOR#t&LkVO%(DJG{PSkWU+uq0F3><|6R0)yE{dp&&v%m2Ti>w_te+r=pkj1nZXF& z)4vzr5q$M34Hc^{B)Mfv*^RYZ^dbZW1^H27arV@}nLTnFH2v9_x4(ZXD=qNduw58E zRp+vtBH~)^>2I6>0ySV+14(3^#JVAQlpH<F`k))+R`%j1&<{9h2%Y7FWt}mR4ISuqXo^b@l>l3E_UG9* ztw!2?H!MCK1F1vSXX22XHCJ^h-~_m!#e`_VGQ@+|vbD^8#&Keut}@EDndjKypbjpu zvdFuUN%9&6p0R{0LAR`O=;5`3!ph6m zO4B2c?gk!#4C=3Ve*8J8D7f zUzi$xdVWHjbP|i1Ij!vlQc*m1O|`^xGamahKS#9aTrZF+grFE7Yz=1S+IH+i{4QI8 zJ98ivo_haLLIJ%uR;a&FKvZ}eQZFU;k0tPB1y|M_^%?IB4GG~lnaG*;a>tv@BC*?6 z?T^@e6=~h};Yj<|{R!H9LPCpC6=k9OmQ4TH1Y%-~dQXmc#a21FwN%bCePMM!?TDJJ z>F8A=y7bTg1P_nN9T?e-urpit4+>h@+l$R9j|89O{n1)ls*xJ$C&a&AGMTed`1O!b$f@8eDabto6wA0)9% z*B^eQr~UkS721MwF!YDLj6dZC&PI6jTHx!US*fX*VIo?}HHznC^s`I@%luibD>QJdq8=pj#co-_qL;j`8__qr?td0 zMa0;r_}S{X9Uio8p0o$M`OD!eIdLB!DiIQ-U%Z%>f->^BcwK68GNoO*Zj;UVx#;NV z7q4E?gDI-|+%K`c#>5m#RRUluWX4^}!K1JmZ1{_W-TvAs8mmaZrmwI6)MG>lK8?%k)vi4Uwn<1xkoVk1uJuJ6yOV}^?d?XP%ZJaMMxk75Q4!=M{bhUKp_R(M8jtcVAxc^yu z1_CrTNF6{w|Ma&9x5DCMlwQDs0XlEi+L3N*_0>DD-S@O_SASKhC(8f%Y? zIK%FVFf_W7A{SALM*q5VMr*$|HopAwWexNm21dqOqhDpbzTqQHJ^t@_nj!DJ&o7RS zUQ5Iyut&$y!A412YUGcc_Y%gb0RIK_wB(q%`6Wb3`ulzJX>Z;eH%5EkZ_c*x*L0QM z=pO@50_oGy$%6tW3>iu{3$~+<)dQ$(4`LZIaQXO+z(D7@B%eCuf2ugnj&z}%RbQH# z1VGs1bK)P8>AR#n$9}AupD6ufdDIk{00P}G+Q#o54zRH#)jexq(WkY=cZ@A)MuJp2T29Pxe_UjXE z(V?aI1(*#GGP4%~kAHq}SlF~!mb6&(OO~W>kkwDqU37l@=PrOVYG*3%4dD17pqgDm zp~vxbljBVeBU@W8-E43k!TE!7unOW8kucHZ#^Ygy_3pDFQ&AKMh*y9Lhm|%#N7V4P z8Z)Z0{RXD^CTIh=LDp~HybQUa0xv~F5vqD}sFPGpksz(Z-4lzurpEaJAknY4fi=QW zx|fRvl?MdL?sMk~OJ_{MWZTma#G14Jv+2G(KUu8Sf$f(Cicwk;xj{QWdl2 z*2fx0WN_6fzXe0tXzkiHYoKu~MwS6}@x;{pDX1cf2)2f2+fIN{mzAy&p@UN9vk`S!D0m2Is_?c21$l54*l^Z+XChH zeVor$cf=hFuhKHgef;Oz)#AocmQyGQo#@*|A5}YaNx8bXyuwntJQxv~Ta$Q!Klh)T()=Pt)utvQ>SuKjU|=g+LYL!=Zv6Jv6nQkGOCwTTY|gAnc*FBQyc0^Gs=A_-y_JX< z8>8&K4&2*iV{vu(y+}*{>eZ{gNtP$vp8SIH{piA!Q=fk}GHUnv;zj&fJ>6)3ZhZfK z;?qOJ6{H`UG)CIza2sqn+>~i7?yr*YW;ieETmKm}p=-9CT0>|nKoOFjevgi-;?x5> z`XSrxjT3Mg%QjmY9b6zj$Q<3uak}~GMeM~(q-b~nIMwHMgqUL#+%yX>pRa#tC@LWH zUEfPF`0Lb{ntfhXeShOkdHad6=yb?nfDMe0;;w~-gCggF5D6|a7O!d0Sui>}{Y{I>958%c1}}DZhsP!0Z?8}6bQe55 z2~MQm)UsMGUc7h`F-F`xB#_Z4KhqqmtE>C)#KDYvhr71MUf}@V-q8$rL14#Hg#<@R z3YMtfcssTX3=BoRyQ_G#Gx%d)Umut#os!q-Z;^_ALvuvhvl#8IHHOHCuF;b}5WxXCW?#*AepH;uEHHEiuhrL8<*ie^2 z;&rDlj(q(dp31Ay#ePaVv;-`r;r!EE9)I$_DMm&`2?wse`LUE5D)F9F6OSwdtB2%EwU}{u&5aqr;KT}S{{mInIqdrg ziJKLsE*=hVW=rBP#6f{7hubZuImqeLS}JOXhf~Sy^O(3A3w@<==UZ`3&Yr@({YmI9 zWw!3tk1HTt4w3=M)+F8?4)@(Vs3^Bj>=+P$H)=Vc&y|y`%hv=l= z>&&fFJO{~e5gaN}is#SfyQQSwyO#*;D0^;H7-Y9&?}r2lX?_}N88b6+IQ?ub<%K}? zUPo8O$mq6_JPK#6e;Rusv!Q^b>dsr(H0Fws3{5%` zpn|&j?s~|o$$zZo9goJH8~@1&B9)$=o=;ZxNasqH%Jz@U%*^fIzb9wM;yHIEXcK|= z#+l&)Bu~GP5S@I1rzhswMMttLtL${9I-T&C;3SfV)MY>>_OVRO))hEgK}0`}HjC`p z-I~GK2qZD55sw7)8w(yBIGpTa)`7~`)VO&l;?EJd)qXcQl+sY=CVpi&sxIfJCrT%@ z8T@Vw6s5SsDwlLks}EY{+{~|GLRa@GDvT=j7E|&VrU!$xzzhX89rj{*!zdsifcA9> zLRID5Ie8rSTS>{j!E;9U%}SM!oC806(1yoi8Ms2-O)H>UnXqozC>17pShVT*42PHz z`e&33MyMpzn+AoJr9V90{RA&I#V^s|f2l-?lan(MQD0|G59svqNG4W@^o6vvw32r9 zo6+~?JkNR!up7EZzjpMim_$V$cB?Mn@X`EJFVJ#k>hF=WkIBxKl7}*iVtA?~3KK=^^QFYI zsC*hM7V+jJVrg~1qR>AGuWBZ|nM{5c+%w@KM!EMXzQVHu5A>m&~0Yi9xCbvDgc=AD&+V; z`(oH!kSGl-qd`nie0UvUWJ|sJsmVzO@b5N7%JY*JL7mmr;Hp!`CCo}<>^(piq+Qs# z(okx<*blGCwHrZF z5K_jrNw4yP_=QQ+2@yzf;VTCrW*x3ixp(gx8k&9mS!Sholk&rR-r`r~MZB#lh>GkP za=sP1V2@hl=sEiVdJUcggv1HM>ef~>IAHg-J_e}V1jRrm!gHv@&l*NticceAb6UPq zkzIsJ0voIzH^=*X;=J{b#x?Y3=6It2ZSU(#%c*W^I(m*bR`R&Y#ZLw~Po8j4uMp%; z#jJl3r~uKELRyID2=aUxP~Y$GZ&&2nd|V9g*KQ~!u!d7O4+wZpV#tJ?!RPC9K&Wb1 z{2NF$jS<_ECr%zcnF8m!lh(0zZmIejT1}H&KO4E_gtd z!{MYQzj0&)2+v^c!Q&mTu0|<&%0uh9h~|Tw7GfI*Ts$-W(-78}_#@fokY>eiehGKu z4Bu?HoS9SRP5uSEpmxrQeQ@Q{;)lNkD@jwVoE0f{bg{Csa;|kPEtG{k8T%JVa@vWi zSGmKFLT*L2J2N$GNft>{QlL~D{06e{mBZS_7n}=rz5n~%jjxyz=KIx_6{sJz_YQKQa4`{ADQd>(-Ih{s??8bksO4>F^|Xk zUmzDoZCan)oHmG1 zNm#*W=Vu0?8_-^@kmKYajIXY@gXe*wvIS^6B83Zq-ycvcgMpNFMRmH%6ISIvqR|u z8ULcHYALEM)e`KL66o*01a7&MQa?7t^$TG4A;@1gMKChyrbY5Ld;`_ej?S7itz>s` zDw4n<6N#FDw%lA9y)<35eivbn4A$Z}Au)haa38jDE3N_wD2aMnsUa<~D|$}r5kC=Z zAP<_qhE0J75JE}L8v6D4oxtJ~V~-$*4}}3`6so)=-DSKg;M|8Wwjj zS$Ft>-rFI^kFKTR#E-|s&vZ7KUJ$-EqN`?Pe`##I1}lB~Cp5DiMS~@WpDJig?vHhMC-`whAjR}_Yx-c{FgmYpBiAr=PeZL%kOK_po z`SDZ46*m?In&c>a7e`V|P1QL+Y%PDdM}tiz05tYh&_cGSDlnZx{H_+yF0B*U#lR0U zDRddM{fvGp3E4M>#a&HID^j)Ms&fLHZ-i+VyF13_XZ5knyqhsZuN^o>PzQ9myn1xZ zY2Z&^!Uhc0qg!@f3opg03Z3ro9^=|^`N|)SaZL@0Y$^mQ=snQemr)!Z9u9u+@u+;I zZ8E*dIuJ&O($2#M+xN@qAS+ytwU`(PpSUameTUx{V#o@#?kgf9A~=Kn@HN@)&zsyo zqBnj+%gjo6z+oI zN+kf9^)P3+$Vd|073OHw4NEqPtYngc8wU>X`}(E8S{)DrIOEC?+qD=Q$jBnY9YkS+ z*yMahb!-#7Z_rZqaJur1lmQ4mJd(ZMVBm+eu8L{R>t?0kxSapY`y`2@6a!{K3AZ}B zO;l9lz@2ReIS;FSEJL0mb8X13`k2dbmP+?W{vkBfRzDR zCDX$Fid#@jj~qQ(My88wKa)u!zR0S_H~5{8LbE5LZeGNdt&`Sx?v1)pJAU($m zOmSW8$**tU?`3QpKg!j$ml2eHory9QS10=V1Qr{fG_PpR@iz1z|KRBra_s1^yG;;lA&IHM)Hm&CMG7tKpZ%826hlAlf?MBM)pj6x%Ai9 z#5q8wK0y4#330u?Go`p{2VAEJLGbOXIk%r)hE6B2Yh=^>LolugYDVO~*I@o+T5Q}f zKar6XL&`pa3qAhn2jw9AYPnScKQnlS|`~g`@TaCb8LUP2;4n+Kuj+XaZD#o zH)V4ZMq&5~PYj*(7zlN_YjcN^UqBW5FfM6JN7E9cZ8x+g+_YBtr6n-Y)=`@Z## z3KwGC$I9jb8p9mSj3)VrkH1!?ed1c8bs1)J=NZVD61$YS_`Vp~GP#;`ZU zY6V7-6XRyU*%96L4WrzcUwGNjK#X(5gnaQL4Il^^8~{UPRS*~V0M;x5a}YB=a40{} zOQS{mkdZa>Q@AD+BLFWxe?DvxU3~XC1jcpBY90k8Fcj|+dD5V6LhNmzxOOABT!M(g z1rc8T*fBofD`eA)sMtx^ATqW2c!ySaPlrFIlQ7sxwue|TdGh#`gEvvY`3ssUnkk*+ zz3lj#L~N~mUR={Opcg5Ju%S63P0Gs2^%X1P$S2O0FDT$R{8~+qtytk!@`rBH0rMak zj2(s!M;!R*Yv3cigl4g1_q4fXc-YP`1F|RZH2bm|h+37^);z#O80`V3JE3JfkFn zS5Svj>7Hh6r7wMA%@?%BH{rxYRvl75R=4Ty^*{Fbi2nhhG6DxY{fA%gho=s`%rflZ zEb`wpi$7{H!?KzYBzL0#V4NaB(-Jnq7ylB@`ox+yX!_XKtckK*i*}#L z#HD}c6(Bt->z=^3jS8AC93IOw(6NR>Jd*tXPFU}V-s6)8T?SiBrWHVi0ZNbt2n?70 z!y^}wg8|&tgLPmyt?da~I5c4?p!$>0NKUg{{&&!Z4dDxabnFFfUKdhhFXm84zXKmn zM+L99YKlgF3`>$FLtIeL0Yiv4UAG1A2{^I%_x!yZtm>B$60guo;6ejQG!fx8o1GpHGWpSh z%%!D>M+OAOT9cDL4Bsh z!i$r)?SX!VXt**gW{uPEe}IvM4PUfs$+9(k5aK>}EaF4={2FV6-sR2VcN$<$IY8m) z+qJ^nx!J757~@;}C^q!CZ)&rJNajZF4n+K7k^!uyQ5AH&UDNaDhC0;p#=xE?k(DhU z{0L>r{QH}b^&r>mX3)_S%P(jDov5e)fZp^lt^NB$>(8G%_aM)$uTGE{2|Y_S_w8dx za|W!ANffK<7@hPj)*txN-oq+>Kch9*CKMByhaZGV%-r?hB|R7MSL+&xNA9;Ydv;Ec zjF;4x7>jeD$zBEC8qEvJcD=;`1an&3!g&mHK%`zq7|>Cb5n|WDr{QaOB>1WXVHI1|u0V`g$Uj%)F=XG5S>A-r)Rj-wRZGVAB+ zS8t)fLnYx{V35K$A%VZprSB5h6@t`=O$OXjUJxG{lEqgtE530O$bMm-7f3nNC>lht z09;9sO?RUe#&Ew6(nmX5F)~$1=DT3}C0%XCFSHT_z{An93xXF;AV_+ZJ$n}8T7zpu z9mRQ+ufROE&geiB2b{BO6wYo2j9XM-%tDb~1NId<^y0wBMk#l5vMBe~$lv_-o{Sz} z-BJ)^>oF~36dOi{bFQMKOh(lW6V42?YmFw=w0-_`> zgx;Q`x+p%g1+PYERe80V>}G}uMp2EeQUE!itag|zkSbvR96|xm%yr9 z>qo}M!YfbQ)pbIsVN_1?_$6{aWl}EbYHQ!jefG|CS>cEL!RUzuP-I5^knm$*GrGn1$b`4 z&%(dJz)uzPO0!d#oQTiM%#V@))rc{@HJqI0u#$0B5rP;;@b|<-L!;7Pc&xMQKF&^| z+o1TInD2A2o*;89apMS`>yJ+_z%Wa~Hj`n$!*hBi@_-tm1f!ZMgC9Y^Q=dw;V1`>x zwf*?GGEP4tTkX0?V)KXk^R2QEXNwcXtR^XD=QN5875=tLt{C+r_Rvm7y>9I_S@ zlJ|2;z|El2yO;btf3k6UIgEcda6thbb%llJK{n6~GoFpez;^Z!jd~BgT);n-y6B(& zddOh_TEC!X6Hc!SA5}iw()w!>1$^7c68k8e~JeS#!U?H{jEU+ zU~D?QJ_mJ~djs9qsva<8>HN zLn_+mx&`5`28D)9gDqXURK~uM3^4lSNFd4nIJGEhF? zox0@n5#j&MOF#4&XvoT=^*%BCNt>u^`vN0pJtX1 z=L|+$HbnA;zQyo(N}kdTM*e%9|3%2-cV@T3W0&40dJ%lU8Je0WG&QLBh%h4g5SoN! zy}f$_a!cUVIOF1?XG88D0y=$-!H+=FfI}{|Z7PD~q*_66<8sNc9Ql)fUtV>( z191(Vs2NTnRJS^pK$vkO8VLQQUEC!{ElO7VHk@bx-9 zV2=w7T%P>ZUuY!WYou9VFbif7x&!C^(J=GJG0Z`R1Jx z--qKS9N;Weqopk=3TU$S2(jRz(L9#<1Q={ITWW#57=Hcg>I%UA-0iKf{$(N1z;mZ}~F;?t4eY_L{z2wRnV`F1BASS$)fTX0hg`s8Xb1VmuLvj0= znWe2S3<GF^m2Fe$+AeT{h>f1 z*rugrtUpkghJ8hb3Q@w#Dk_%1;e!FSMPzcZt7{QO1WNq&$W;40)J-tu7$Zm=bvud7 zPHu-IhI8t^ae*|E%j>JL&Qd4duwhz+5nA>hOwppY=EjT zClv^xnjk?!aH4n+d62*>U~t{xg{$e1gey8g3WEvT>%EkpOoajZ?n_z&PP3Ef@oC#n zzFULoE`k^D8|5vG+$|T>AUAj5mzM!P>D)W8mfYiFe@942=u4@$GBIspmCJ%)I!2aD?UaYGxRwf=S;ZvtBCX5dTp*KuIpCQ^6=D7rw)Mw)AQpA9o zx%qm~otTq&pf7~8Y*M%q&;oM3+QJDTu^PsOqwpH= zc55d5_PB~jY<_;m!G==}akCM#xO4*QL8`Exlle-USXj!5-3u3-w0ZxTi7U^CpM}ss z@KchailX-#XCh?`i%ryq>KWKjnBmdCna-PwL@3$mr1+OoF)@*0}KE~%W zcSNPN3`5M2gF#e0Eh@4AszRFBfcHr06S0(D3x5kU2S?wE>h5 zfMC~s;y{&zxem#{?f>v0A5f+9G?%5I9PDe_vTa-SSuU`nFEB@Z1><}83AR5s5br6> z1{fuhcdcuz2bz)sk&5RaPY4$}2{7z46IcjbB~b>uIPp2Sz5_NP19dpEi{Sp@f{ZJC z=-x809?Td*CZ6_f+qTIm)&ENj1?1j{1@a1W9Z*fwA&7jyMJmJsW5)%xkxVW){`e$9 z=BUBx3mar|6MTwt{sMQ(APO5A_uy9&o&qiB%Zdv1Y0Y1~y#YA%`&;|MzdIF-Di*^m zW;*KligVkk`}KQ(P?GI&BDS|qC>D9X zc?S%zQ8VT_UaAda;UJfA-WFO=7Z=~sQe`scnXLr?-Frx2g$je`l79{Fzc(cFGX*7d z|C+Cbx<*Qy#EMXIXV8EDy1(=97XJ4?W1|26%g5FrP152X3JhQdNLyC+Dy;nXE^o03 zD8Gcw1GgXde4gg-Urev;%JcRS5WGMO6dw{1ac@xa-<#zgtq3 z)I0`wL3+edT|8{)vd#rzNy%65pTk+ECawIlssFdn{%48*{>T5DJ5pb2x6)`=Gw}2I zesEi(8n*)!K=gtCJTlX=8HRwIihr**XQBJj3N#U8c6K&y;rQextQyA*Kns=yH)$HT zCj71RUDweL6N4Kzk(-q0SHP2BcXebo4%ecMp zZW90Wj(?UWGEaxU``Wkg461R8mVd~P(Yde9uc*Fz)Z^Hng_~_d=A~liS8Vhp_=f@w zEbJea6%B6+E?DJe{`(-m82OV$xFCt)PilMq^V?r1`)V0?OK70#oV(%MI}$pzwM=u# zLM%lC+1)_gZ;sl6}wF|rN<}V zzl63m#v`=wYV7Q*_~5+^oTEd=CqM=kf*UCKp8o&nUL%}vB?UN@7?v%2g><_Z@DACp zUtz<)!!-Q{xryB90bAJ%TyX)@&NK^gs^Cio{I=&Xxx<8?-j_{VcqQ?}fUVPp0TFQ9 z0w8soCFJjC1DK8gkfh9G`V{`i%Rr1|l9JqI0X}$?XZ_#DwRw)0dWfY#6c92^i$p*mhMFup0 zqWV@U|9zp_o*!joQBXt)Hq%SjXU09ZxU*>wXccmW69a#4e%G&GLAdT^G46;UmncHJ zCc_EFMQ0E8xRZL1>5+JNX`!7&M6^Q3#kUgv@;ICiaO9%D=f*XZ26mk&xCj>XCA;DH zI$0ef@)_5NEhE?^>S1D)ctyZ6ZS@;HcYDCZxl?2R{xT42 z*_KtS$?Y2;GyOn1P-O6Sjf^!L6sL{hS_cmj_gissu_PGOz>j)D8#fYa8(qtT+|8hH z$W;);)c^!|KP+ym-D<)@^bO)(L*hOJjie4=XU>hk`zKQbeje_Ehxx@gcXh*H)6+7P z&;DEkrh{omiPl=ola^oT*MlDf87P@abREfNlgkoYi%22phvJrfeq%#x_*B-J3~OTO$Z=-i{@SL^b3QEX;77-oz`2xC zD&x%RPFsT+T^7()Lc3DNFQ9aK+^?1<0ad%>FC0^rXn`+K2Bm_p$F`AD0)Ki$d$;y^ z!y0^Qvy6Q%j!%}{CFyE62l;Pcazn&iVwki|$XG4%!dZgDLYMx`bv^Bt-GqS|$ff(J z=EhzSM>B=Pgsbj3*PAf}OF?v$HcebmF|m#&4lnUvK7aM9t3LBZdnQ&_EfcpkcE_s{ zv9A9*2nuQ`O*N2P&k2O`5Rk%tP{@qhPi|Cq5xvA^OHPoYcY>C+CS=rB z+#vTdjB0BYGu1lD#ZerVziSq$Q zm|`gvhR$gZmGCO#&@b3XL`@Zf#JQ4tB>on-fv62%XOVn}AD9AvBxPS=IdmvqG*%RZfuOH zPEHCM;21vyN;tGq!jJ1awx*-iFl%*c?j3(UWJQHc?iAKunu;4Vu%cqLq?gX)lv~Zx zwb}8D5Ck_A_@e76=&Rk5@)b6^6W%EdK6Z0-_4OM=SdA%j-?r8xl0c(VSw}EZ{rcjX zkeSQaqke~1jYuQ)37V5mXofhCt?ti5tQA$#6vDqVGcF`vnlE3nY1(<_d{WJ=Wfw?B z(eG4Cfoe{FbV@GypV&a=s(tIRg><=Z4BLr6vi67S$~u}c7#goeMv5u47CtDGhrKl` z^Uhqz5xHBSo5Qio5fs_O^<-2lxv$^gFS8jv`zecxU;xf!kqNONwI?84p>iI>yC^P# zc}`G?>OBI6ZLO+ikcgr`<*rBj0P+qdc`(cYqlSOq0Z?{YrBKjBI3vrJy5V7LQ^RRi z02-ZnI&0S5Tt4*r8Vt_bUI#V6e(QXKy~N|6I-32;Dhm#djED$lOAWKKx(q@(!H|@o z+sy)-bTnH_jS+Ny-@2}T%>51LK0n}m1fhaiAk*l2_|@38h2j|z6Mwc{aP~Fh&rSId zX4i@@fFXaO+xoAQCwnATeVwz>IdEN_x>**A8QDX(?5b+2xj|!F;?qOSc*zRVNgW;5 zx2|F>!)fVqx4y8{3!!gv+p>zJ^_{hExujyDebeHn`t=nGDOUKvYw)eyJ2quAlu0KF zE1U?H_1P6ek95d2GmK9j|FCAl00;*O{Z9ox5ZCUtMLCdOsQ0f*KYV=tIK(78VVtg+ znZukmGp9+uX_h+zKC18H_ZuWN0uOcGast7Dvd-txsc}?1$JgOH54xxo?fA$7xz`%* zdS#(9DSA%Ai*SbVLJ0hkrp>>=J+i_m*DUKZmVC&O##BGd^LhBCykG-^d9iZ-!&a0g zeU^Mw(3p~Dj6ikHvVb-W+COj3qcmb<&!a9iHPV=;^Bx?pC0s4_!}wsv*fe2n4cuey zbDSZ~82+ZB|9ovw^b6|3*SQynurS1ms_L{Aw4(Q~FYWWwF)C1C#Pl?qE-S=At&s5z zY*UDyVabip#hyNW6;ApVWBak#)FbZ=9y?ZjDiHycXxSE9^G6JLIaA$E+EP7y^9>5_ z$h^0wkNe;Im6{SZw?8e1qGuziF??;%*R*hDNb2pQlTD@D1Gr;}0xcfiAR`&I$}*0e z|J&F4=*zV#{Hj-2hRZk(nc5^SKY8Dnn3&%}k_sD|E-yVku^moTY3YL=U^M)}_VcFD zna>NB=7vmJ@a6qvrp?>yk2!>oQ1Nth-3eP@0P88*mUn_2jW9h^1Ctlis3hGd8#?yu z`zV$aIbg<&oS|#q+{H%owE*OQ2^M$qxrGCmKt{!hde~5*5iMwpZXzB9fKf9oEjdzQ zVZnoDL^j(w#{dNa3vui|$2cT|IS6|=ONU@YmXrowoxSlILl3K~$_-cl)aGvya4?J9 z*|HfQau0L?iS-|Axdzu|QD{*JE07}|Q6O1F}`h)Yu$>_mG{+|^!@$cPG!o;WIO zJu97Fpy#Cd2h6909^T~+w1tC2=597{pTw&fWLqZT~c1651zfn^r-~q?u~v|0}bzYU}O;8i~l~m zw9+Pc3hXoJ?oPI_z|AP{GJ2AB zO{`59&t&@sa5JQl=FY%hIt>bbbUlp-E{Fkz#))2IowqW^1upCvIo$i99G%eYw%;YJ zDAMdMB{l2fP7$|q|I~k0!_Zg~f+9?wlClEmfQnk)WHj}-l!1h8-%q9M636qIsTe2T zZ`z@63;4$1s@+D;4aEV}rpHVt+HrK8$(u)d={M4Nv{+dGf*(mLMtYM@k8r-=??3po zJCik=GpJ0ACt@ifBJXFxiYy_PaJ=5nA>{ikc(&r|SAa?CqIMHzgR~^d3F_x^diYkA zxy%j&zhO2z6C!~-vodiOz&-)XVHihA`|vn79$c+f)qxy5_=fec3f)Z%sIN)>sq zJF_v7QGtycur7!jN#`vgVl88l8w*MvHmpAONAcxJ`wq{%?ITV~kfUuR3sCL-PBs_r z7`|6p)rizC?Lx2+GvV_uiROa#yu7Z^u}Vm*EAfRVQb#85kWB=OLs1LyEOh~&S+%aE zw}}A`txF;j5Dq&rwL{8oz@*q`g3`v%fdev>lFGjP*G|7wc44Z-N0whjrWlc zyRSNy6d|^A*Hn7D8gdOk)UErRJQhWJ3zI?*uBmU)&uHuYM<=_b2X>j4_uMQHn1bJf zu}YzxQ{tPh>$+@tWlg3SX`5}=t@?!DiYTWKpJi&)pupZWgsx`!4Ei*ILGEM(*wsDa z$^umStog>22NpiC*n56-gx`wqzUfcNsm$^`WTgFbn~p~ct<4*iXs_>7;9c6~_Qx9f zaZuot2Nqj;?6Im=d?(0XZHu(-%I|Od*lzuRwCt>t2d*osI<{$yOYqA#9uW`yzg4xK z_*hle8DUk!$nQ&E{aEqmpMkGUH;fLwwOCPewS(%eudH*fbm7-}-_&kRw)ysdi6g%M zv1Z9=Ze{KNSY`k7J)NKbw_n@W{t=2Bg9E>ouda(n1i z&9vtKy7;cw5JXbyHz4z^rYC|&Usr)0LHcy1u@b1dA1|V8EELO6?M<5b{O_N2pV_4I zi}Xo)wb5qdye#XBH%mGoAQL|Q+@aa)M&F;uw|DJ2T1L4E!sJ6}Ty8&-&qQnGot($k zzU}uDe{qV^x{rq19JKopOkf9^kpch@c>`y}|AID30Ff$kX~1`1v1c)MMCQ^orNk?{ z^Hj#=(&lA6t2&P((-wvAXFxS8S}&KS6A&;dhf zJSZ|?gVVr$tUDlz4V82FJ+@paY&O32{rb8KoZ=WA7aC@SBF=bE`WDhtNGR=zfT8v3XD0A7`FIGBSI)W!#gtWDo?UJ3 z5y+h!VO4s=nc*iV7B)y&yY}9k_SzlRcT)B=v+K}QmK=tMt20)z%Ay0K?N97WQ#HVF zd6{P+^^%woK!km-ouUavXayS06><#CbFmC`>j6b0%n4(LttU9rG80$AR+6D+M8ioF zF+Q8hJQ;d`w6wd`Qov^Ti-8i*h;_4SXx2A|zGZ@8xSt%4Bk)G-j=^)~)V4Se;J~eJ z-KGM>$&CtNrEtD9t6Sx7m%%#X%)|+FC$?#MJyKLt+$GX0PL{b-)YQ%1`YWK7*0+7J z1%L#ATfxxbR=@V1-=0!?6$sMj)I2e{hs;BW=Wr40;VIBfy#Z`;uoi}Mfe;V)4*s*F zk;B#vV=&5x9pk#TSkkQ7#$^`4e}xZ7GpZ~ezI1i}xaxi~Bk1#!pGjYRvuE!OK&E)k zpUXTb$>Id~sSvGwGh13Z;p4$c(FNp-SI_y`Kc$RyYQ)A3d(o?B&l~qEQ;(pN%KR9R zpz-&OoA}!l!m5IiYwcFAF1za-1ksBhLI&$dKQkQsb_RV=BFZ0f2;HGm1^*O&!S=b5 z3DQvgKbo4JpZjPog9F~1TKA``SBPY(vS{ME-z{Ej{atn8-u@w1I&ow`EI6=4`oIO~ z5K8tBTGt%(l9bkaZ}ZzBf2kTwsXwqzn+=~bUVrHJke+5HKgdi#7&c&4|)SuhlRjB6>(SKh}*5^Y%5e-|c+# zpTl%^s5YE#<+5&6k7ISid97m$;p4&^1a2*O*Nu5?@{oF_xCcn>KR8UDyfrXz6|eqMx}+Z+ zq_&W(M$TYk)e{dEo&7in54UpR>5CdoZpoWfT@x$Un)k-N)n8gGAsv1KMvHIw^t`)= z-eZRPNafuAOH;)~?vNM&Wb7ZMowRvsx?q9+I(G-iLlyMsm~b}3;?_B@IkPMICE=xU zKM0H7)i&l=xeb`u@d@tS}OA7wl*=P>PT5<92-5%G!js1Yx*iD}Ks9`ErTkVhpPB;)E2P1Mu$rGABkms*~EsihwIBm8! z-7+X`*Qq15m&3cxQr(~;4`?G2 z)S5V4QIuPOFBfryfqPYjw@~>Cr#W+^vTgV$!Ss-vC~~EB~M|G`R&cUBaN{0&7QGJcwO$ZocvbByx^dD=q@eb;5EZ|>+=~d zi&X{Ic4)*rcv2?JUebr)kCxXb+8tQ+!!w`i{-eUAVU-ej zakfvSks|Zni7(Hsz_$mhetFRv>Kr~ zF#T_zb<^rE>zMZM5BoN+mHR@A$GG$De(R0=xPZ(j^5<_8qwTr{vhi z+>DKPJH9%M^D+KtG(uGmvX=Bb>!gkxG2$7H8_?Auz55IA4?&V;rG%fZfJIYyWFOL0 z1Du$)MMh2))mc6tVpDusM4hV)BOHIPO}lo}S$bK2Xm?HOoqu}sZ%JS4X@(NIzLmyY zQmq^z1>drwq)N^r*!V7K*_TZkC%_a1boTWRvo>T~|Kw4Xwp)Afj~~8Vm&usSto0c* z!fLN!3c*=sR&Km&&I}G&o4k=&<_HLg=virg2i64z*=%~doC>8r28p)!nV5V?TX)u* z{uJYF*SIc41zPc?{F{dVNxLsq8J+GjhBB_O1f+bGb+eA@a4%Kk#7EY?ZnnOg(+*X< z(NgR^v00{Vaz=~xU!A*DLLY)`cGg__&+)aKoVunaKJ(itg-5kYu677Sl}tk=Ld|LK zbqSmX4{(+6Y0YT*z`hclxMf+PJ6JPf$S5sour7HilASHdCVe$nmo1 zym?E{#x&Q^rzwA#DsYu7uZkEHFnLo{yC~S=FNdLqL>HgyhLeVW zl+W1?v_g%WeQH&VjH||2!QiQGYyZ^72_~a+2vC{#8WXD7H8NQTT0tQUFQT6j62!@Z zp2=whLh;U1A8Y^Dhd9hg3emrhRMvs`7f(B%YKyZezdV#8sbPYjLUa7b3BSECAPW{P zx+?4e=Em7&RJRMNt9QCV`fap{P<4;6Voq$`O5X0)yQnna2mn=sC*mTZz0Usai0ZFj_`CzJBfzuVA#jvV$US&eysLa2Ph5a1V4u31|{ zXeyaWr9=RKeU(UVe+)<+hSl`^&ywL zej`Bnhi=}EiiQBWbmz|4NDo{cZSS|lP7i6cIxK8*u)7Ge2s-_l=ZLE?E_fHjmb8nZ z+0UxaP{c8&uXoqqz+H*k3f+G}_}K zMaEPfn``!^9_>g(%vsnvZ=XzJ`am9^MyJuQMl;r4HKHCJB9#%d#Psie3UhcJGR1dH z+5DyoVefHpUwDFrz*ngDnFzdWg3 zSDtD}&}O+!=>qoaMk`*F=RU^RbJ10q|{}EAv%M882FFj%c|W_STDixh>iCach#yZ zoBh@BYlXwlrKT4SKcHon?e(SG=cdcsWFM${iO7<8*05sF-$dA#m|MR#>6{G}Kb0Yu zuO7#9o~c|9S5*6A)n9*qgb|a031~$FVM0E27?lWWj#8`2U2Z6IL1mT~1( zS?)1vYZf0`8WIY7i|+#kN)C?|<^`N96rUUxn!i@#F6*jG z>0X<}+&|pi zt=-}gs8_F^Aa{20zYM7RK}v1=6PAV_HG1dme5s+uzaQ*O6apPbx<4U?$tgLs-@KHg z)B^$<%`lo!YtYUf1zOf#m0!JZR@PGBG2?d2v<;bxhI~BXUN;UCMw2;qx(don20o)1 zZ;$NzP+o`%WdPB~Gc*v}ZyjI;K{hYS(f7<&JE)XP@+3IbTh0x`|H4_kmhenlDV0eP zQY4HWdl}R%wzccI?SBrI-1ur79>;VCG)1eRZh~4aT%9^)->;W4A%g?B57HF#@4tg4n$x`=<9MMhHJ*$V{Pc5oNm9*T zGmLnT$$!I!{d}_RW+?nNZ&+;^UaIWdL9rK`OCMy&lvrT9oV_bmAhy6UG=c?vrp}TU zUS<#pn;&+1+pfM^^sI@Rjof(EA+x&q4?q665Bz}PAtjgEc)V^lVO{7dQKtz07Gnc7 zck)w&h>76Eo=3$L#@8*T#|7o0aU)D#0zS&|o6^}0!ewVlmV_M<4XK6x#81-;{%%{+ zbk-MM_42I`)#opY{fa(Gj+vVZjqI9j$^lQiT1-9`rP5n zN0UZB{q)0!#3}jzwBKob<-jXgsKzwtiiQwaf<*BdM94D@N-vRhF^5Kr)N zlAqSMK}bR(DJa#LfTzdU{{6i!Ut*0>@-^bn#kr4|es|2B35c?{qAv3KhO;Lm1c42n+WTE?o@aS46}Rp{>J3q{gm{$Iq%J`UiGuwNOnoX)&?8c z5SpeydS>boPRx8Oe&v(6u2W*RVlw%TQx(+@&KEAYkYDciEZuWdiTc9Norfqb1Apt< zsaIULIJ;9Bu0gHlWzOoB+4a!Kx`!qln(%|M>+osYhW%`$jGeLBv#9(Z!~92gN0{WN zC967aew&gR`lWQa#{Mu1c(|M2t2{ zK2vKwgia(FWp+l?ZKEkKliq@;hLw$JIETwIvXJSF#RLgP2=&dH7NDPUzmGcS#I=sP z+2PNVyX?`M(%MQ50I`dTW;tLL7^!yKysXcsTLX)+1RC7^O54@(7iG%LuDvOWLYKT) zSH@8XVH7_H0mN+p;|B3L>h{YDP9UR>Xkzx^;^XDTHyi0U{0wRQX#V+m@8k1#JD;&7p6)&|;mcKur&l*o};}Uuz<_jiNmOSEswncJ51;_tslDcpsP4qcuUv5ebFihO8f9A0f zhn!rOmrXS|ujjJcJFoekcNe}qO>H)`{Z^z+qe*j545r`>#@(B_qkvuoQ z@<|BD+qlY*?CPY-#gmu4k9R3MGjGv@scR4jdb=fjylNDd+JIZxldQXta~ihJJDv3h zS99bRtZb4pnX&CtIu0AAmPAO$e44@e0yPLbf3)A%yH;_-`;RoDg*TMfjeown-5u%| zC@cdlpHE|&nB;76@VsfPQvWjAtUKh%jme9j*i3ytrK%{T;_Z~`6b=jsdF^=R$`zCH zf|H&fF4fCNOm=+r>PmJ+Uhnvqn@o#Rr&JkLmtM07e!VlbM}AV{_I>^wJa}-x^3BU$ zM}*|@xP7Q5)U2|6)@d5cVmA*~A7b%(ou+16xR!%>Pb&I-`ny9vwOgI0>E`8g)icO5 zZF{>s(@+@iO zZZi`e!t#!ID(1#z+U|hul8UJ~&D5~<PV4h> z!(5ID%;I^3EZxY7Mg4Fyr%K4h{P~Bh*}A#UL$C0<(~B-QGRz6GbL?g@s`%ip z`6=BZL)8!g@{-rN&tbz2FZ4{R`h3ijE3$hqI2;-+;7BxHQi*epe9S9Pwdg8!yfR#? zw!dfLg)e#kG_|;4V>j7m>YRUMCw};+sp0j0RR6n`JHT_lm&-PpiNz(L;Bw*}bv%)3X&(8zQ`k zO+`BJWJvX;2!r zu}u#=u(cG5VTnF&AGyLvO7ODhP5sVVXY+RBxPYQtrGB~)5>0>SETXyo8Rrj|;o|qi z!J3~jZnS<=r4kEhvSp{OhhtBD{_A8~UtMyAt?$QxDSE}mAuegPk&h=AeC@A3g_LSp z&a&9$#q-&5Y|6xmJLw=1G;PGE3w9ualYo5vC&&ph@R?)&xbp5S`T)dyJS zCZ99LdH&P5_Y5FyK&YvlBUg02vwSzTPJ`s?cfRCaJA>jLdZW05F;wCgt!|8JFh07j zOWpUU?vBze*o{+ggu0>2FI8t)gJvuw%BI)b?v65@x8rEMnWx%qvB5Akt5`4NjMK6d zy!tt7J8v2W<8YGDX4|LGeYQbj=?e#pC=-a1Ct0SER1moV`bZr*+4G#-9r=+mdv%wN zKA&#iV5e|^#j+u@;7p6}O>)y;^d1^jJtKVbPju#L9$$i=xBi1x9_|4h6$(Z66+%X^ zY){_R{*}*t>?LaDoR||s2ve<0l4w|OdDTEgsn{yZe0k{yjBp>%2VO{7LICN(**Zb4 zKGL*H3LWlI@$p)|WUNkW+Q(|JL0v>U$dsLW8HxWiF`W_SbuYPAz4lwQsy?iFjYPQU zsdqkEVE&F{m8^pNYZg`RG-I56eQ?)7Rs<>1idF&QZ(K&%u?r;=Q&LjK0v`PD)uUZz z2k>$xcJVBNTI-$X&wl=^?mWM$;ypclD@qY&U3cus_scvo>Hb$fW6Zy+nn&5fR?6!! z*7*2??pLm4&wq8|+0FNUlRku3E&h0WsFt>>*W(5MT-l`gD6*B((xyn?|4Kk-fdyj-G_%T-MESoS9>Bd-h1{(^;TE>+?UKz9>5-yQ<{OYaE1SWm{A% znaw#@T2FDvDJ=FweeXh|}KPUK-}Cy!lOO0Lvh z=<{s*^i9cosw?tC5ks=js6$?#OYPC6tfX|!vBa;g;Uzqo8P~UTiZ9FCRm`@^l?MRI z+UZl0)rGL7oBeUBJC5GBlHE+6t*GOXAGR2GlTu}b8ifIK$&UN=)Hx9$jw$uDmLIpq z@K~61-Oxc<{l)q3!TuJlPU)BCCnfwbV!QR!dZQ$r{Z2Z2=(Mu2ksQjzY6%!|w~Gp3 zrLM?cbE^@$bZ7Q$<)|;aEc|@Q@rk_0*~^PoR?i_Nr0sCoy@Nz-w2WtCGW^a!&z$|@9PntsRIlTger%^y znOA)H2N8D8iI;e0Pu?OnT}eFVsIFV9Li<7zp)ReZox@h3wl3NFrP1WY-g(O8KX%Pk z^Pimo@41)$s!K=p&M6fyT9;ih^xZ03mbjKWW@v{XNfTsMFSTAtY9qG`pKWD>5Iv4w z_R8jV`Cm17-?q^ved$N{0{r{Nt5?$IU+`Rlc!HvVsKhyi~WfUU9!*mo&-af=t5mtFzzzvWsJD z_HQ=ttI}ESN%`3-z~R?9^FFevN*>P9J%!p)sX_Cm1;?+XO>QO2ovmf}>fUYw^x@f;>Rs!?r~bYCKEN}B zW^OW|eSz&4V&?E7Kdq-lPv)My*(+?Yzj7YE{DoP_nMH2zyCk=B zOW3-Y0A~86kld((2R?Gf6i$+3^WCvR_TvZDRWF)HwVXC#>zo|GT5N zn`2ozeFZ6k%q2h;vmUrrt#ju!N?h?Fsne<}li_$evhqC17ALPyZyvnxhd~tWvEanq ztL@xw@RQU>G~?~4Rp-Z?i*=fgfM;%P>Nor8(zi!wlj&^=R`lSAb46D@Q*G-b_rNxf z-LG{pDS0scvIb+)9l9_ePDhvJT$xY{=U!{tVvyqkV;v0i#8aKua= zGLgA@#+nX`y1Ete=Z;IB`&T@xXE6-24(m>i0|;;e|MR>xpkC1CM{L64&3+Y~rCgzd zCy2{)-Zk%ZGWU5$s$r5DVa6}byu4t<9&-)6Vq2fR^|5)t`Kx{3#%v>Qm|kn|IbwBF zt=)B+^!W`MF_jQeM2d0Q+^^(1@#@_5{5J@x21yk~p*%ynsjL<~@wzV|sqF5aszCnC zY3rnYzO6T#6%AXI?eisK{y3?9l%(90>6`n>EUjGD;KZEm=qSqzmu06Cnbh1Aqb{Kf zEd8rcZ*t)pt@!;|c5%$wSUVGfa(rQ8dF-{0%MMWNn&bwKyF(Q!#d-DD`$;JlQ5IAd zQVDH`!dXi*bgzkOb4&cJ|?`S*#SoP0dT4*nY&4x^@L>jb9KGPpp~yUN?{!YvXw`Su0`ELME8y8(jM=iIT3R-c z3LgG5P`0831MZT?)iy2yu;R5pn1b~u*OXm`4<}Z3UJMuAM z-uu(-p?A5)PleeQ^*Szb@^_mqh&-U;kE@!@{BHOfYJb_$tfCKN{gSw^!#$vZq*9Xm zi6i{<19y4+#VNYSa-?o<()ZO~8oYa|m?pkcswR~Oz~48rLuJG8KaMq>d+zl?A$;y| z-B1MWyvfb6S6WTVVb$tu-39W?MaP5aivIm&^o-~ur&CzlyQKOHcYB1Q(GOIn<7Y?% zP$jCs@AWz?w4M*ZiiywL*(ehu>lZMOe#TQ|Zij`p2e;`t%R2GPg+iO9Y5QbV%Tq=> z3u$C}uO_%zo4WzW`&H`Yxmrv=91%VW>cg1DKH+x2bC$x-+g zE@VvClT@%AZbIT{52znWlu2JRk4ZGOwSV~ll0cImKBapQ2BVIrUak$!U<#q%CZ>#{ zzxy3ZnyygnUE1PiwLvqthSM&sIU3M+Tx?8?9^$6E+AVu~%hq;g?)!OGbG&BlIkEp* z%fwsSvmPaSCAK$N8s4MngkCf%U6+J1+PrMW)_~u~Llkz&1?{90G!q^a8*GcHF0Z5L ze-l^AW4XS_?0bE+<62c$*HT&K-%O9|vLO{KnogYd(%76Y(b0d?J{5E3>C!T)!Xnay zzI(fB!3Na7H_>fQvL-2EH>rS%#U!8{Ae5C<9k2kA;JDPun?KjWHxtJ4|5K`9xJ`#A zxiOz)Er%|BbEqt5+vq!#{8u)eG$$*kl}9++1r>%wb#yFDo$owR>-ad!#&$F&84lg# zIOBJ}!8!&8r=M9jsMmf$|54^;ISC{hVI~fr4-I-q-Pi-Bvx4G&!nEJ0EuCfy{URwf z3&0smxux$u7vZ>w1Cw3Y6{$j#L7SaTI0o7%!=&nXsK2H21TA}GYDRNGQ|xGE;PIcH z(#kFT)bO8_Y?#c^U8lCR9y|6$<7%B}hJIzky+W)z_vCC$)`p)~2=5$I8z0fGP`x}Z?iC^j^x!UBqjty29;ZkdxUH@dU;Z5NYlE*kx8 z#rr92hZ78EY@+Hc>=_ATX#$1d6VQR51aGaAA-}=bioa`qS^mADWmm33^VPL}_^KcW ziXLh?wG#9do#(oD=7bN0H~gxX{h&}-|5Ev7*-nZ4)3vRO?V%D@ys?Qs*T^ZEKN!pT zHjH#Nm7KS8&e0tc<0_`Fj1cG|l$esey=ibgR_qX63N@a44}>oX_v}>x7EFE^`y}F$ z)lI3u##`)}e_w%5G#xUx2kh`qTILDoQvYfjK{>&=WL>;12Syz}A7;3V0L)EmI1a_LSXO@LGpYOj9Th|7~sAcnl*c zVgSQCINQs*wqBk5LA4c%=zTy1nyz}m-oy=2q+aB6O^1ll5DXMnd4l28?O|L6ju7_) zDhr?J?Pi}yT_ms~E~Am6e`YhQsHq8Cc|k?7 zDG9Ds0=7_bx@X67eG}ciT5HzE#l!EUg275N%A)d90eezNE6<|jF8S+qkJ`^Fz~L@s z>-autwG=CMrjBAw^3?(fC(ekGoR8TSS-|9!N4{UqXXhpqr$zr+B8dr1capqrH}nwC zqPr;?24B|J?dBdIpcV?Co7*I4HhGjVFTrPmpkIO~%}<8>oBD_+C>*bi-@Cd8JfYZDqt6%`1VT z0@`bk9Mu4ia(b&Qi)sI^t)S{m1`#=4A6rXduQJB7e7+!d`02+Uss6PasA5Ue&d48! zSEuZColLqZdp+83rxZEtvSeD(J*Zd>JS`M4Rt15)r(UM0EXXRR?DF9fg^@;b4o4rg zLbOS}7V4R1qT$`ywaZalH5Ful!MUy~BaUuqzu;mm@8X-iHTjw%20CK02bugJ>niip zc-HP7QURIRW^o&1OLzAZZbwdvBQi_~cyEpYIZ1fcRtX%V@>sm%Yb__6&NWwsZL#Rq zz^2aB=~n8;0DIX#MIv}{KQ+9l03*p9Si~hbY6>q-Haf|}&prKUekyQ&7xGRd3tWUM zh_b+>>dHc4PY;*9CO$T$I-U?cuzW*@en-y@Kh*5uX}2&^MynCeOqa2m1rf>lVX}$G zebb&yUT)|Sd`L@SYiuczc5DZ)j4>^J$>3q*fH}aeNKnFmz&b3TN?qtvxGal+-~q;T zKL74?D$6Q1t?%0dT$f7Ckra`*m)W@~>wQ}&R%F^TQ0(fajv;sF&kvin=cM773A68f z$#L(dtNV-9Gt+kcmM#>cMLs*~*-d6)dkS78Pu~DJUC8iN*k{(4os%t$esJ7d%ZfMHBTXIEPr8=_jjlp#7 z-bv>g?fo&D+JpXwAW*#tQwm0?M}+q2Zc(0 z+S;DZsCx0@MaxNlDq3u>jwK4G(k5O;$EzHhWh$H;hWO{7jizfQ|Mp{t#s{ zk0%5;qANA0j=gI?WDOCJWffnz<{e zFvgnCCFM=$Asg&ZH08>zotT)B%D&t#iV~S@G<22`C9Q}psTstCwwjnQJQE{zD3w|~ znyny}pU|jOu>gfg++XPyApR^Bkoa7oiO7ybKkX| zvfN!tYth80GSetHc1&`+V2r9eMs8kDjyqI&yja%-g1nbExV9MnJ(ASrP2`P*Kd`;u znUCLMJmBIU0#5GxdMj-Ft+|-v<3o5jW3M3+_swJI}dDt?ZlTeZFjL&$d_oD384Vp_Y2tW7p&apv{R z_rTkO1W%K9r|xRr8Z4PENresHkA*z{k5Qr92R7~3lScNVe-=_Hr}N==@uK4__Qke=Y6~P{tc=RW3;OFB~>WaWFzeOQxnBgaaGk>2dJCO*YCpePYQc_qK6}fMd z?7ueLDxs$+8oMlh;$_!y$+N#&R(6$ZR#^XRY!rB$iaAfRfqb5UO-z&oh;#(Pod6qy z^nPC)@TKh`Ps!fpq7mput1I3wPD=G}?YJEY=>XheJemA1OJzh@Ut_#pJ`wo}g)*5= z6wiN~4Gq-Njf0GuDUuBN;)^o7_qRE`HGCBh zA|fhk)>a?6aq{GqEIqfBN4#AaoR<)Z0#qhW9R;J31}&^ih5>XqWTH?sl8^$2s8=1& z4j*@LL^9mW7NrRovzQt@_n2Sh9FegdyfSTea9(xU$0jxR<8J2R_q7i{B=E`a@%CMX z7xXUY`GqOP(?TD4!mWf3iym}R74i5Yd3oG4smApbPJhY$iVXFz5@lJWH9?w`zCA!a z%4F!BFq9f|(6=MYd>b^ej@0Zai)}S;y<+G5phnsbD13rEqy(c!B|O0g##uEs{9Tf? z;|Ph1x2^|pR%EC6JiY%#D|0`T{-Yv8Z)gXv@mR`&Bu@(+D(Jj`#UQy#og`#B!aT~e7lfn~GU^Vig>N4Cx?Y2&SLmEd zDG2-^fLE+5eL3M65;EBEy&?0^pRi&IkN08Ij8aogxy9$}CBw@mT`#%vZZAdfW7N7L zD*nLgN~-MZvO)0aMRG+#;>ME%;nOd+53ZT1`_HkyY#(a&BYpZ2-7I@$ZcQtd{$X>EnQ`2WvO#xGnQVWA1 z#W%~S^p$8lYBvjO`{M&I7u~33|HSLntxn=CQM)D8HK#CYSgBz>*&`JtXOeD;(MQs| zxNG=l5O8irQBHCjvn30vut5Z;Lxd);U;cyrOyRzGSZDt2Zt?Z@=A!*tmF#we6_eKo zwt2uOwcKZ>x!R~j#(e`90GV@P>Zc2}Y{vY{*_nqkr(Iiw^^0lc*QJ;O{O9t&Be(o` zUR_i|rd!?Qf<-D2NJ2tBFeALmoI`)9u+xtDeg^5yKmRV6)dIQsJ{c-^n^IW2GGomf zE283EXLJo`lgqvqvQYd~W%rY+;znHSQjyceB3WLf;!~u>NEp?V+%^{`6p0eckm)@D zPoY|ZcGtQ1VxyLYxfrd=(n**j*#Jd=@t`%$8N#@mno~SPS+3DXJXkW3RF#=FM#k3E z%dG6f=Qp$sKYMn`)>^;U+>M3=?GK`%vLZ&Rm$Prhs@BTneea~Qk)!%NUsbdx9dW`8 z;zA4qvB=>EgzPZ!oGI_76xBp9rp?)_JWr>OrzwBT@UWHfbaoSlg%M7sX;st&rAne% zm)%Q#R~g}v6wXV<5V>Mhp#w+4sS!*j(waOdfb^xyLq4NxQui7uYkIxNn?<~B+_m#u0dS`uK;VByBx z9o)FdaIqpNpJcf^OA3L0y9-$>JecYZ;iTcv-lFeZ$I}!hyjjaGBcQ4Bcwzuw^H!V#GF#ubz(%a@U3WgG;rt9v&wFVY{oxvj>*Zh zTdaWzikKI&qPTSvNXEKe8(AZ~{K03Qil|p&7^4?toAAoeMDZ`y>k7OV(nlr!HWwn- zNF2G_lp=qavj>E8&z#Up^KlfZ;X_t8SJ->s3^nN?uB5VTzw$@fdBm(#(X(Zh$>me5 zhsqQaf|eYTCOEw&23_k-vgynz0cjXsdmt=}SRncDdn{C>4(M8Ig^mqGg)roe;@o&% zDm4brJREuHgw8!mX1@+!zrM;Xpc1TN%4-i59}@Dpd?McOX~RRpCjxs9YX>U&2kCIg z6#_>{pMxaZUGvYqZhnkT^!qwTf2#SdmLt2?axxx6sm$YiW3YcnR|5kMyHaWBPkzPv zWdm(R%#Y&buw7Fk0NpEOZ0A$+vinZi)LwtP=0z0ND!_B}U;kGN;@j_S{ojzg<~3rB z)<>VOc{K>%Ol8dy;v4@1*eys}{xr$}Mc1b4<7U4c3|BV^^(Wb4IbyG5@y1>MD0_o| zyj%Ew;a0fO9zamTMG^;N-fPIJ6`6MgsEK272LRf?M67=DpgoX-Y-iCTw!rtpO05|* z<+Xfhc<$9c5uH-8$&CH5lO?$Kpr%Yy*qREt1IQr_5&}Ii(j0)C7MI(5idZ4DTVAEW z)W_lHCj1MSJdYGO3SL{oOE3~r@Raa4qReD|?87x4ztg;iNY)jhABS*)GMUe_OZVv6 zeU{a3TqeT?Pg_m`iu53y(ATalZW}?EN=?*%6}~2qOnw9YDpcP2ikUEG89!giWK@Ne z0t?YjopDCWI`d`TgJH%Lf~i2smJi=2RaYrB5xJf8k?Y3col_ z&jg>T)Z$*obbmv}u{ zR@7;NL5B&!W5@dbJ8;5;tZBzg>B?5Bj!Yar#nqcis&!W4*$yqZsJ(wkElW|{j$3er?0{2xOS6bT5;24kG zcZ9c_g1nSMq>STt9=wnTu=(j zIXry|T~<-xVv$O*^VUBP{ilwG^%6qU8znOJ5t#UzMv;cjw;LqL-+F%yvk)Cd@s$fT zf4y#Nhb4abDA{9S1B@6eJLTCx6Htg=O(zJmaQ zo&>C#4bA8Ql%qL;Cl z|33g+6j#02YFbW~>=Bdk9UoyH3ZYW&OIpK zghN$}RTmY7S&nMXusvk9{6i8rX7W@?kRS9l`)x(F+ofiWL8-cKXzUY&809`Toa$ef zG!)EmtUl|W2&?wY4}i{Bg`e=J>KUkWhi}2tpYNgqcb@jo6Iay^)5RK!55&|mk@+Ty zr~%&|W{UXHgp>i^=(!ep$ZLv1|v0?Pr5HAmXql&MxCrVIzVU|*f^v>NWAtHw@eBJ8KAKO|8zX-ngAhtgxLiJZ|WzzyjjN|OeQ7ccxnMqo#`$Pww9 zEh|#7_zFdybn`9VCG8}D(aaA$dGX^OC&$UMIX*Fq%Mq2ki%JL~HsTBvWBVCDKlcm6 zbt9J}!m1=-YBz(0@%)kX{+gQ}eT4E|de5@G9U<)+S))iPVwSeg<{z4`|3USlQ8sFR zdC-wJ276=WD$rfJq1ic#XI3aGiAp&S(3s6(-W?0#PCu9yAh0c&467%Yqr-@>x>J|7wNFDC#gaTuUK7&^` z7vff9uVZ)lmuOM@zosh;zw)`|qLOL8qK~(2XSIZdq6wLlgh5dQ%WC$zyYf9QEUrWo4>1{bB@im2G?jX0Nawj@<} zbV}7gRDwa28Wrz}#x5?d%ri~Vgp<#l|6MOd?BcX=F;>h#s#coC z$DZBy?e_0oJ>i{Y1G(#<_ppt<9(E;6rgDFbXlW4&DOnT?K=s+>1)3HR?3a^r;=~s& z+7WvvwS^T8+^N4mVxYioT!;qwzkGY`iz&0?8fv#+`YOB(V|*A5Oo3-^(uhJR7%EOJ z>6r1xTYHj#ZnxcT{P zsC)4Hnu{7>yUY4AvsyYngKov7i%+Y3J%Qtcxoq6j*GjNanbmy3{ICUICw zo6uZ}$$$%Kq9voGxo{MuXlM)t+XXQvw(g(5KadqyqE76II)?h(a?xOWTd_^>L6eFT zc;2~~a;npczi(f0tmr|ay9)&{-!Bh6`9Y(`io_Dk1$#)wJbq~ zSb1Z2iE+R?JN%s+UFTlG=oZIEe7{O@bha^e9bc9%O}(NaQ;GJ5HWcHRv@IvcpOjl9-Hbw^{Q0TT0B%4ZPNgD!}wV!iSL=54W5dB8*& zo{D}U9RV6TlcqHB(Q{j69G2*q*iXjxt1B*m>LYoPkG`hcx&9t55j9MclkKL>= ze)D2c^!TpuQ+g|ZY;1+~P%JZcuY@!+f(R3|DV&X3DX~WUno4ng>v+!LPxr1~Mgy^0 zDQxFsMI|2#4)phR#&VCNR5~rKI3#<7v^c|D>PlSuDlN9kWWb_h zxDP>fm|E--MrR~mx}N0~|4`kd>TCF}(Y+SZqOJLtM7pQ3!JYX~SD<2$?IG*S$Rpdt zemg13jhd&%OxE%tr<&|giUQ<3Uvp0u1WdBCf4tLD&eY^yDDMetv-e;WpC46NrN(tS z6R4E?4i+samsn#A)~sTt%rzNF@jO1AB(P!c@Sq;3TGnTdiVo&mBqOIjyI<>5O6QLuC&*yKIifE;Z#UXL@R$l_eid?(b)|ONz;*F9zTfE1_j98e! znE=x%vkJtM`P{#?bgl}e_U&C3NJf<2BF!WNf}V7@&Xl(FoR2?gT43ZAKSg)*tViRt@O#nNM!sV_ga>5ru8dAUIsJ+H?&{ZG_B%y$)s$GY1%&V_&% zIbGDrqFBCrQRgV3QGhO@M)Y|NN8Hdx)s>5Mye@H+C?$IQgy|cNH9cR*eMi@)%Epe4 zlHEd}9!Z6Fmw1Qs=Qlq29NGI%BIG@7_TIVHI{~#?42nh!3Mi7E^aysoh{`e(fxfyk zv1jCWZM(~C5}9lurtXA!JVJ86OObdSM9J=jLEbfnIUS0q7^}j-1zaH&poz%;yBV(y z8QW9jf5D=H4 zfV0V)uAsvwMFiU0aJ*~eDxSNn&Dn4{-oH)#iz-B0Wlk4l%41oj_Jj59dpv+k(K+>n z30MRdu>nX?)uHBg9vB8wYf4_BxS4`wzF98>8&1xC^=~=g=?PFk`vhpUB_kH zQXd?;)e9hfr9?Yfdi|gaj%^XQKC4|je3iJDNV$&|zj=#}fH~vB;j{htsrAYZ6ieCh zZVX_M=`-P0^2UdUaXDfhApgZXc=mqawDNl}l8}SAX?Wk^yprGSf7AMqwI!m8*rRQ2 z<7HRf1ZrSg5-8C~d zL0%$YTyEjcE(kPI4@(^>;xx})6fvoEL%-DLxAgAe+9?Li=83|~#uUm(hEiBCN>`~EITnU@2WphzBo7rW8oSIWHO-N2#nkM93?ZAp?h&acT=(fT(w2o; zmU+xKNOCXCT%*rtTr+h9l=s74D++#@ zloA+jB=#L-$?gCm%%}Nl?ujcJsWRxPAKa5zTIIn*Yx$jcDRy1CQr3Mutcf(~fA0)r zy>FG}D3;vlQ@was`nYSFM2S2bFR4fRyFv=Bh$t% zLZleti6oT6Q5|@NsOJQmIyS3{RBl3c_y=fkKOt`V}+wI!0 z?uFI517##e!$~h5z+lYaiOAF{5ZGyJKgcXmo|R(kvwr=Mmvpij8*Q|WQGe3IZ6sKO zLu2Acc$PqO$do0qf_(=+0u+nhO}rHQfIOa(4}XwUQQtq;9dac{nlHs5Cl*Wh0v@|s zxnF(t#diUTj=>G}Uo@}A8o<07oLNqENHs;wd+oV?Mf7JoNf?YwlGP@ysB zTE*z<+3VWx#AOo!!5|(hwAyozzf@v*){KFXjVL)hV$&Yhx>~Q-mMsS=9;{>S>o1-x zcTVSU#;693HqEg=zV^6Ys}1w8aeV(+V%?zWLnv|QhJO&nx(I10{iepQr+5#bOeoJa zZ#3x3Xrr(X>KTk;3g{s1RLThl&pm|^8hqMtemxCJO(uz1Vllr%K;vvNb5zGPqnpx$ zvF$)b%wW0apHDf>b7DQD_N-!2kAz~8xfU&28)mDo%CH{UXqs*kK5Q=gL8n@nHKhtr z7rh*fppEIGxs1@7iK!z^dS@`J4|e<_y+Sfg#W(&9L3V|W4dI>&NK}DsO6fuccLevT zqE{e!3hm6w>cveY*dJf|P{U~1Ik*qT&Y$$}36k0kDRvKd%xQ?HccwZ=EK<^J-_Wlk z{&!_E4ca?JMJFmuRUd%Z$YPsqQwp=v>YR*7?ZF(9z`(!2a2R74RA8i^3>C?=Ak(H)nFy+)L( zh+;(`CKjX^4XCkU0og$f3b7yv_xF5|UCuq@o^$RP_x{WHuQB$PP*(ZA_nq&Y&wQR| zf{bClocV?PkSbNP47XG7y@Lm5@W44+b|r$Ca6cvzwHDW={%7Wv?Q+LTNPgFG)COOv#6QC*X zOx2Mi!4alQaYt8!Io^ThA%T%&$}Igwf{P+ZDyUm`nkKkIoGxUIIrSNLJ^dqT^(f-R z7YmFjyVS{B%?1fM%OoNNR*Ly?!I^$7qTxa1*M_d_I1{h0>hvRcqG!6;WOa2B7^i=RnnrHq>hg4=Oc7*9XsMNe=0HfFI)>7?7u@9IZ3bdVzY zCk{Odgu{+OIzaRM{*lV7rtJz7Iyw;{~dk8`-!J~{rH3h zljlJh3UuD@RL2=WijDndw}>W6|0IEn3p5h?*I_+m5{ISTAOQ~lr{=}`(KFmI z@(oc89Tnp4=JtCrUP(AzRr&aHuD|DWp$>i940ntKvzclDuwEUprcU@(pCi0p$ULom zJ%;?-N?W^m#5a*e)%CQ!OCg;FP}&u19Jgl5XA=(TK#}b{wDQ;=dqb0h(lq1XUuWnU zEU&F}NgK<07G0y9<9x)V8p^@5RF^6M)G>*+t*>7ks1PH^9hL8{b6He+4TV{{ZVB5< z_?_^$--U*J?Ejqg@THGu=oZqql{H89?)_x>-I_!a2$CSr(78O>=Fc5FbQoRup1t`WErIPeN$dzlHRo8SvIk5io8 zytrGM?u;JxU%K6m{E>bR%>U4Xw_FYxw=75q9=Lb5drNtz*!UfU zznOH%U0>TKc$|0#u%j)Qi+OeGz4CfR?(%e^UoLedoRpgi++ z^Ag&rDI{cq=3nBk9(*gjUM%M%Ul@-ihD6Z#@?Sb$DRDR2N*+MuD&KHR<^#m9iJq60 zqr`ltqz&o+;~qKc&`6_P&r>&V_)5oyc1m{?Nz#5=581yKP$Ut3G;^b23x%thjq$NP zW8GXEvfQ$%i7GeTQrho1y%sXMR3{TGed|O=z11OIhnwxka`@-K#*V+CkC725I^5pd zx~yP)|2+y~UR-ve1o1!%(|0nIyKlL09A(rvl|6fX=oZpUolP0R#?neGh&p;RstnArH6<2lJ z#c79u&p?$KQgO4-)F+htg5!qY<462qpZw7oWrwKlgVD8y?dYydw(y(xy7*;;m+hkm ze@m1zWtVC+aVYIX+Ee@3{D;y=EFz3@-$=O8{Z8bF@hw@e&Ifi2{Sn8@6>9>S<;#dV zg^-&vF#%nUFmdnMkh&)dZ~3ZQH?RQ1WeY8?kuVRCv(E~XVL0>cG8aeI;{f!jWwN6Z zRah-uoe2N(ERT1JVyrYxAgcu55lz*L*kMTK!8j{xyuu_E1-0k1|H!RNl|LFyMKAPdipso2B zShIRfH2+I-tEY>}w5;3Er1`6fZ(`Qk@8MrTki{JU_*KS>iX*FpP&g2RxR~&TU-YRO zi;&9q>LV)o3GC9ud&OiC3CE3Niswr^oIhd5B34d2v*f*$c_6Nf^a4H1z za}O40Br{ZM!5~a|B;1s$ND>ttDXr(-dsmg2*gD+xOxDAxVy9nTs_PntuRIgDXV_6# zfvZSA)hDI%UEumuiYu#;Z|1$~)m(tzpq}@C5YYcKFQ60MP9F^3e%1+?*ds1DY@t<^ z?)N<+msoKRK^!TNF(pxGkAoiVFfS1@H z?q$}@g8o*C=EN44X-YKiCn|MABkOKl(&fOZ_SdOb2LfzXY!W@hED2EAJ~VeHAaS5% z5zAi`$5&BEW6wKOloGmECbI}bikk>2-1m@66^cn;ni zLe(Q+NI&tCavBE9a4P7vg@W4~7S;-r^l73V%Vd`BpqD5(feCDEzx>Xe>FW0DTFByv z+a~I#`-XQYD`d*5FwD7fmZbk^3uR-@!)f8G{Y0K$J#(WxNoX&|S9yKznzPVOE;6z- zxy-33;Tlx*7ge6qc(-i2flycDc8pXq&qw8Tf z7mI;R$>PC|Gn_YYFFgyrSE;@FPcru#I9La8`%YWx$NYhSHD78*sTcIFl&}Nc2m>)o z5ZX~Tt?IlW%P5u@k4DqAk#bG&^y41q($0>OmJ@MACZBMPFcTS&2(K1n8#~Wtx2gb+ zPs309hL&ysSHFUgxrPMw9hD_gNeXSi2C8-3FJJM@enph3}J@kpKPf3I-_H$i)5Ttq{(C_**+WCn!Q+cBdXzL zY(H>eW?#MNGo{uoEwObNzoqZdnum>)q$PAy+rT;e<9bZ>?1Yf0 z46ijkRr=u^v7w1qA}vfg6d%fpd4Y<yGApnf64vH( z6Q2ufbiCs~YZ$ruKi_!WP=&`Ym#SVyQdSv!ITSqDOaqn+I|vAWISbY)_*YuP)_I33 z1wr3iP}>!+PKWvr%xfZzkw$8T3H(n)AFBy@UY=`+NQ{&!Zz~s7%e$>eKjAAk8YNtk z53Q7Y<0>gsld($C@itHq{!(MWO;e`aEy?gxQjgFbZ87R&+*Qy6NF3E@xHE4eZ$MqI zKfGBiVoJHYh-@K`HNA1lFhiQ9ZoM_K!JnHymj43#?``cOb+ILl6`~J>yGeUMP6t6W zu1jVqLGp^NqqD7)5x6!HIoRGIW0Vn*h8L@i^5a!b%bp}0f{D6PT*!&EC``*-5mTg& z=XMFO4Qy$2`JvNps;PPd(Zy*cWaLnW=`sOX+PDB@DQ;}U7TtGobjoVXz(oiE!t8XG zi-IHxqV0SgaWOB~@fBrB^Ap|E(w@pYq>u^WE;irnsE_fW>F+Wy_CTEZpS_b5*7W!< z6ng*L@SQF1u{Ltliy*mBciG4Q;GULMtM|qZOuRRdjk+`K)7a&Ko^=X3V%sc1d`cQ+ zV?UV=fRN*DfH)%R2;}Fe!%4G)koOAFGy)S=ukHW)<@Q5w6TQmv_~778y`_HX0{*5h zThU!!arAd9cRNo1J0u{3aCy6d3Li)n{exORTh5PidsGwH+0uXD-^E}_YhMA-lmt^& zkAl&y+c@skDg{7GG*qkL+%nOVY%_mU2j{PTM^G{Xr;_s+Tu;!YB96=UL7fPoVi=v;JZ$%0UH0lrBAKUc|xp%5F@-PsI*RI8jvc(YPji-LR|9x@lg zh1H}>`(EQ-k=#7AThSbDS1W2YZ~A@9dvGW18~?>gZrrvppL?$S{loNi2HFtzjMv85 zbufuTS7ikM-J>j~ojiLc-hKG8^8~bk_5R_QF=U{aIDy#*=6!7bBG1tw?$=;TVAtZv zfbu12F~4?syzK9nH_kiJx;3x4o7dKZ^ZPh^{3p_t7&QO_0Cx!$SKC(N>bjAXB!Few z{y>+QIliR{gHo?P6Zd)T`_Bkwg8j#0SIP+i;L(k)uH460Jz|t>Z}s$LZYiHmbwH(H z_w+xORF!(wd&8_F&knC;W79`{lZ}H%Qo67*DOzvg$q^z-q@WcuXXz$cw`Pkz$lH;f zbe6Ur9jdgo?Y}6^y}cp-tMfhj;)PR3RLgr`lLaC+!MeF-%&fquSpF+$*S>XKz(gW> zRcjr$PaY7{ZwggWQrIZnxST zTz9b@*_ytmRMo;d>7@|x<4mEUWU4Sl26RNw2v)vr&f-OmH|qYh?Ds#tP7fjnxWqWf zfv@kmK9#!AS|~P^C;?ROw|4Y-`pe>uFUBo8-y{CTT{FYuhW<{>%NwrjU3LjlMP;Fh z^W72zH=x;kUotM83dZP><#w(s@6VYSdgHpa*hV4n?mN}Uo)ods*2Y-uNkf-sB@`_f70x`V2R(DmGz5x4oxf4A_! zm@-$C%R*OvKGwq69UD)HO)a@GSw^NuB{+dsN-aP(BtS-lwtbk8CR=jdQFUXYAAVnk zM*Ls*aNlP4=8SpUw>16_yB-DHeDHk2o!X0IZ%{$iB@u09)YGK5{@n&Iu3Z;?`L!rg z-*=UV9sJwYC9>t+!P+l4*gC2o@#2G}FRyCc@t1FW?BLDix_xwOee%@#XIkWs{=>>^ z57O?WUY*jLmoEOtOUDGOT>*<|(|A~+pv1*9fBm8~&SD-KSove%SGS%#P3V$(bye=| zu>9)}(mwGm%{aC5K&|%IFJCTdw~+1U$kdk%yr{<`YOJJ^B!lK}<9UlC$Q^xPxj5rB zyN*;ozaqKkg)hQ7xO=|ax8GE+H=8tVwD`G^3Gd#y7dY;(AGDn@^y}j>yQeq}{mQFt zyR_Z!FI{}4?~mb&$0fB{?04)hU)tNA9XqzT^84160R{c)2fXB!Ti&hPvO6JtPG>u> z=&|TX@!Cpx?TCib&b;qpV0;3kn}Y82Df`~tYH%|%nsjBSY56Hj<1mX~wu#=iir(}P z(z5W2{g`jgzjoWzVB%^#dJW)q^9<~K6|wc`dv|;-tHNgz$bT%^W9ZibzTEoQ*ZFB) zpql~FreqUIlB?%TnRxl}Y4=vjOPj6tP$LwY#a{r+{oEw#i`{<~>} z@f!NckfycogfIIJwf_%)x!)BJ!O}Kf7pW&#!2>A5it8l7R%p>wP?zpVEQJvS9hpY0 zZx~sL3Bha^k;vAQZ5DeY^Nt#)GR13t?W?qp1bgnPx@yain9lo%9BNCQgd;QD{|c&4d7JJ!8 z%heKgsArG)2(iY~b<+6Z^8M*gJE)(Ze?d$)Qebtbgp- z6r0wZC-pj)?+~hI509R#Y;P!&eZ-2C7nyZD7*zi8*8ovWnJNh4TwM*Mf&9qbO8xet zl-;|>v6C`_&Bk)PR`vzOEfdfHy`O47kjC&i618xR`<=N1m?uhIIOEw!+YJuhkG-)Q~H! z(Yp0}d7H7!67Eq*M)A3Y+A14L=FSX%#ofvpT5$V8`3Xt2FK^~AlGfK<;z>%1@fO-d z8rXqaXtr7+fj2`-=7l`UB4Y^(`E4{n|Ios}xn+5wm)2lk2u3Hc1_`5Yo z*o$T5q45NAPjt=|bImF<7B*snqeInxb^XlAC2{f1ojd98g959Y3+hj*$N1f;niV11 zotmx9z3T3YkQNw0az8r%(k6-_6?Z0by@5l!>MoRhVB7_->!~!BMXnpB!G;iZ6=#|#^FzoK@1A67l9(5~CN4Mzs289_Oftt9bskb2H473_FIZFQc zWfXbEI-XXT$dt0{1sQ$%_Q{8$31mFmUY&)|@8)uy*Ba))m5&Akgicw19(GCK;Wt#q z$w%366ogirBR?3mZDBRY9QAF;WkE-t^2uxxPS#!B4~SoSK9gX0ba6*A%DKyy#K<8Q zOO$*THGN2%$?ySfsWwjt)*aEny8UwWzOr67VwPV7gB!oUOdp|Ow(5fsH@&F24dXB% zlN^55h2=}~-wtGAa_6nw;=YNSYuDAA(jx%nmo(^goIl;4aaIdp$gmByL5if^rgM+|0=({bz1DRP6K6oApKNOs~TYmY8QMwl_XJ~ zf$CsJbIz*((|jESaLr0O)K%Ew{r0(rZLKEzQP;mwuZizQFA_{C$^i(<@Vm31y_U1e z)VLqHwR>HcWAocyTihC!w?ZfTGyIAvuYwMSE%lcXBHkKSZmr?eDW!_Tukbf#!9ciAh760u_OC+`|ss(bBc*Ih%u*?W(!Y@!ie@`fQf zG%Gkt@gYi@no5Fw#pxxstuPS>2U-&jmBclgmq2|K5WUiJ3Sg@%tdmkFJI%Xm)U#T{ z`VdYP)by)+k&G^rA{_06;}bu{;~jDLG00G|{By<1Q-!IcIRAK~h@<;^PaxB_Zc9j* zn%Us*m-m9Lr`x1U`|j5JYbiOIsAy$0f7(a`=;<)F`uT?JQlD#~B2senTTO(-irnJX zoyYx9@2?9kJ#Slg!IFB$lL%!k7Qe_`#%o(zT>=7KVG@z5-Y)e7Y$II7!R`O|z2~*u z-ma<-p1ST<4aO?DAaKmn;%$eBDzRlvkSY}Gc#*_fzVQ21J(zqnvPoS|5{(A;;(gaG z-v}9zHL=x17!Jz!;weK|a3-^eHxQP-4Kz@F)#`?K*pgPiua>ADq3*5xCX1peqKL8z zdQr1w#34+D@q8Z`Azu=&WT*op4*#kSgR)w&9Xa%KXaoENTx zt{}xDJm8}3FgGtUNy8|GcAYewDZ6=M*=}SrpJwRILjn;N)KtrXTSFNwcVoUWDQ(c` zHS&*Uy%75t52ql@Y7f_}s1D_@or-~Sv*`_NRFf&fwT;>YsIXVFx#!(Z5ywEi%AT}H z1?!1ot>jIevQ;o* zY}nG*unI*bD2E#;o!RP21IBtT?Sqq#I)X;0P`oUV4EGkwLBorpR10L-PU74*cw~r( zl$@x2AIzv2Wr46>|Jv(q8NJXD8ggSBe^;Awg_pjCKuCH3k-4acTg=L3&(K(!-46eN zV`u-|z4xruGyOPBbyb8?AB?<;o`IOWsD(5S8OF);GR5s9PIR zr`7VMt2cbq^CtEa(kGB#QU`ez8?X_OPOSfT9{1t2 z7xU^lG8j;}mc&3+>HOkQ!b27l2C@X!NKzvCGTO}`2NO{-yl9wF`$T=DvqMU4P`e)H zxaOrA?vapu%2D$uW#}YGkYg!FFuo*R8trm!R7g&f9bX*tPusIqv#!K49>Ca2&0Nvk zGIsJxm*!2csL8eoPd$gGw)xAbKF&8f@+(Q^X$l6B11zmx;g4Tqu8Oc#A(l(5Vnatw z#8%<)M%g+bRW8(irrw$ z9x0PkvvOQ0hn|ZM-1}^GX~HB?5v7RYZ06af7&DguYonJv*Y6z;PJ@;Y6-ew1DOsN8 zPEk|33bmQ^FcQ9Mh~vu$)^q#RRB=t-K$pAH$O-(OAu~(+LYg9(Iw~}!(v?%{B33}$ zefhqL;|(RKD`Yc{<=%P^ZmVCKzZ#`OF}Pl~`t^d)Vat2AzRV^1LcO;5+(eF05pre2 z9=*6xpD$U1TR9<0`mAES8#+377FrpRedU|YoLWhABQ5M@7MM(obhwjn3#EliEV{K) zRS~lG(84Un2=+99+Xj=!hD%zhwG!21T7k{IUb2?X|Mi{2>HvWZbcaPU%-44#3#Y5x zB?M8K%q%i#w2Ns8$36V>C@){px5QmwYe&*p?5W(OWqeFbf7WLFh60il-o?bpCl=Wq zWuc$JZpvUdpt9&flA#d+qN%82Fc}4kRy!BV-H+d z4`-JNvB-K0yO65BB15YbR16EfJ-_^X33HVrseH4j6J}BlO3y9LfVB{|^572`35Ee$tZDZ3FWxNhUM~N}3I^nsCEnCIi!k?a z(-*%>o&sN$U9IYq=S|KCfjh7gAowpdEqH7Y88@YiqYQldS+W9gGG*ko2CI1s zvi_Bx8Z2d(%Y4{a#jqVt61n#PS_Oc^NmS+5@T@4csTymJ5I`BriJ9#t3AJS~v+ zMCG5gken+EPSqw9CTfc3D)W)a8~4y|(DO@9&uifKu5uNVCPVMvxQ77qh8f;#+VC!{ zg^V&z$7^$OneY*UDUF)!+*9U$&DH;IHr?uSSzvQUi61XVF}XS}`s#Y6%m@fWo8&?# zR*U2~#dKeGK8f;!f~BEMvX+@0{r&~q3g13xe*fL!)G+e>D+ZqJG(GU~{Op--WB)yT zmQ-&F-AHT1>F*({zF|%&Xq1+-|H_T)mug*Iiq?44%XsuADYyVUO)M3#0pmLT;$tzp zRSQdPZ&lk&=c)vVQEfX`{YCmM+6QSrHul|*i-$^g_s-3leo>Ko_H9nNg!HQyUw)^$ z+~dP%nvdgd47SeW|7Cx>bV1D<1($EMFO@1*PgxzE{HdTM?19#TWR z*DUP z(C(J{4Qhncu_Ci5-PUrFhNUd%aTj`Frr#nRolK zGG0-XDqvVIANbCdw>V43#HEdXu3**OBQ2KRJu;<1dDxlNr70hRVdjzg#@LoNw%exw zNEK_y7+OXz60bzsVk$5b>J9n|K?qcVfu%IH^?uF#%14LKm;L|pc!vt4&zX}D(vn)Q7qnSM! zzv!!HqFV2J zKjx_;AESh{HU$Nm;u`01be=S$#QhZ}V9 z4cG5mSAS!0)sl%da&$N?W6DFJ9ro}hGHQzU2Q0oJr8(@%VRuK`i7_EPgsvW!WVB8j zHHtZ(GQpKBhmxH3Pc3C*%p1dQn`TP^y6il>mgBoQlx1q50X+iUAx*x6>Tk}eQ;|RL zO4lyg+a>+LDIaE}rrd$?FvQ|ru2N+)B)2oIvM^(Uu<$nKqxFfdpVs2i+)gf+<}!mT zlnSW2@gYrYb+;GZo;QvAIaH1*xm+=IrQz%Nr|hO%m3N_K2#rck((1Ec_wO)%ZfWU+ zxk3C554SCYVxJu{mz&#OIIl2SW(@5s@m1Q8DYh(Wu- ztm|zku-w}y(C86;>*4QT_8khxu{qo?U&r?C-COyu83FCEL|RNeQr3F7cIf)Av8?td ze$VZQ9FjHMVwpfAqC7Vs(PMvVLf-VK$z6+wPsm!*$01Z21Rmi&?had`;zs&L&2ekk z$n*MBtx6vDaW1vUqy8gr(0|%g_-}O}{?9M}zi!_ApG)^&x^zyTb4aVDFX%h=TP{%&^O_>058zqt5+eUw|bq)!;hOloFjgiDn0~=IjyW4-*=5#H`}Nb0zbtWx2hB?|;lxORb>VFfQ#Y+J!808P01yRoVaf!RT}eH# z{h|)qjGDm?W7`N_gXXu5-jh^&OYZ~Ybu^cW&N2BzH6o(6(m64isp7{_yFfA16(GOu z!RUy~XO{`b%z7(&QN%3jQNxm?s9tHqgSCu&vQWU3xv>+BCt9Wk5Pm5A=^M%qCI6){ zlcQH^QoX6B#z$b?y`^h6?Hv}t@HoThyW_{H93v{UV4tsm=Mr{%b~&UxpxEB@c_3+) zjy%!gLkcD9Cu!Gzid^~h+86Qpjx_~yLr+(XnX@bH`le)2bp9bJtG8{kVLV9E6B>y%kL*Nb{bjrH2H5; zHNTXdo;T)`Tb2L8KUrs=gnJxy%`SNAH#X>F>RuU1z|p@wx6fd2@1L8DEEH@_=5^@( z*W&k8s++g@-Pz{?&mGDe8r5iubNkRw*Vew@*Lc4Z(mMU?KO60w3uJ0f!)8AU&yVXNs`D|59aH zEC;qzmgqyKOihF9Cd=aqG^E8N=X6Ya>s!{Pjk_7>bQrxW*}{W)HR?IB`r>z7;}qlP z8C|q0H7(AR2?)FrAmH3f@)@@Vwd`J0ROId~Zhg`x3G@nuJK!nDAk(GyMHp65r+gA) z#}0QT*Gw8f^82OmBc3YM(a}*#`apj&2vNMPq9{4_Q(W+Kka3IEq!A`dcSC1s1C}d@ zf*S15C!g`YGrA0NxZXpQC$m+4^oCzPox6d8InWV7)Z;n(N1s>J$h zuk^zwGF}Q(N#ZdN*k}ODaA8B_qLiH6#Tc8$Wr=iY$FDnuoC5#flGi4xfjsAA5)IdNRT^|oE?)a9QRaV2Hx7hpgX{5Mpf>!s zPsgOoYgu-3*um&J(#6)Pna~BzrRD?d${Xqz3vA${DTl&9)S{2Q$-V#jAXuelDSDg; zj|Pb9BA8PCEfPWIP4|m+Y-%ePIoTfsKr@S8_I+Cqx6S7;F`JWyyNdoyS-(00;ujao zIv=FWy;Z}-@F;W)eHJ~Fz<|C>?}>kV64(M z&3_@(i`3}xVZEqflJCa8p5Z}AEiDvfN0udOih)OV3N_iUmF(Q`ALKp_?7YyYRGbZ2 z$#&Rxp{*9Ssx30xRW8k)n#pfUpsyHFT%I7Bd0xlX7rpPjWr%&GOZ}!j&lX50iEoUrEQYj`qt0ON+K()kYIr znoXXp0YaRPW*FOU>;G->L~0Dsq|tKt(_APsn0_)FgGO9J-Wjj@Gu$>?dN8zHqkzSv z{29=y3~+9hOOvL@1gV0^7jeyn4a#UR+2%Ay8=) zeJdqMGV;(2qrBytCuuKe@}-=U&6-5L?$$8R zhl4y&jQ|Bag5{QR^-?Z!7OfG-LR~CW`)5I+^cLl7BDCigpZ^pH>By4RN$+#_e7r&3 zWk(|x{o;msZW?%4jrx6}Ilrm27!o)b0&6g(Z**#E z>hbQZ{2^)en+K9NX$`G2sde7$#h83>gh)(byNTu~QnOgcf*aJ?(@-w66^ttB2Zynl zxApm{P~j!@XQNHl6k9Or6(#_eP4Z&ajh)OJF(!bSH@NgJ`*y}{HnTNo#v-2|bN{B& zp82V(W;iupp1vg}MS2zbqre5}C9S{ZV}2m{hk#M?vCEX$^6iN`P9CfgM0RGfG0&}w zhV&FWnftSz`&Bt|Nka~-G;?n7ZEy**RY6WjMBLJ)G}!x~ljC#lFVw(lVMidBtJ=0} zDnf_VM8%xA@98vfWPqFXYH>1~N13Ho6D4&6J2zXWteu=p5`*(m2W!xsXx*Mfw77(X zY-;={K7r;yi5;ABU(ExW;=6nIP0Ry9(IFy4uer~F)kNWj!;U^WuVoEXEH4KsQv&^E zR>TL5BFT*A7Ix>B%eXA6oqf|xpvyA4E_UXJY`;K%HRm;u>-Q91!cmgZJxx1%|1r9L zpo~lc5NiHB>D#4}^Z~iLai?yJ!Xg&AIASzfcXm#LYu~P7yWJmC1|pKs=x(`YHF9dF z?w*33+Rad~B3(KFj)dbp?Q&@J{Hq}K7A z0Qt*WX|5$OifmMo+q}104HSafqjwt(c@O4~RDPpa=sIRGq=;YLH}N-gY=!dc131<^ zu=6iT7Lr4Oz)ee8ts*CU(6 ztmLIPsEUzigM5WeLkDa==YnOTn8k&78;rR&i}QcQg|sCXV*#)gWg*!yD0}FDNr-Y3 z2xN<@2DjV#=FJOaKWwVF*}7(RoA*A~;3CnK%XoiX%)xRnQFjWVz5*i$y3WJikfnlb<>_DWCOgF^B)dsXO!8kO?2(HmBO=eIrO*1MNfld`l41B;GXoR3Qopbu6nSL*kwpPMKBh- zkmK21btjdKlZY=Gt)kc}<6IFPbs=G6%EACZYb5o~IQj{mG$y)y$Oo78?ZvQNSSJ7;QJ5iGhzc%7Lv+ z7ISaLN1_|qA!&S)6uInN(I?}VZO~w72&l^3Myw3=n|+>WAv96ry4Al`H&|7GTDDbe z`kgLdbh8y$_3;XselTU}NS~oC19vBU+v8OnPh~tRb9W z0g{L+rPFD8@t}@9`c&4cnoBOOAHBKY%wvry;z0a{NF{^iT^0zE)TEs>3d~x_+3Hi$ zePPk*qcaA))Mb2sKKCw(jM7FQ3l122``OHQrqr*Q8RiEpVSu(cgwWpH@#Qa@4~VQn zTh21_{9VeFlqnhe_DPV=rczWI~f!E=x%rqp-#;iI_!|ESYq!LCiqR zuhww>cULN_ujQb{IxyZP2i;UusAelly5y!qy=|iFTlzP0Y&UB9v1W1~vzpXP-~@DA z-z4hmibo@!lwRorzA@Ul0JJ}!A(D2{S~t;)_#N4z9z+5ysmCaiMbG`gL%R9T&gZZb zqHWmk@ZltOmbb>z(Hn@>`R`o4RdeIHtk1x4I(?E#7AvLP3R61n;9be#rh?2k=gtLK zkwuDpf=Kc@x>+_q^d8drI+&7SpU;`6y#SG*edqwMaWbj7Td*7eDRynCq7*DL_tnqO zUoC|+F<|GRyCcXm&%z^ZY8#=7vF^Q|XK@%dC6XfJ#1UaSs!&hEY}JNsm@k13u<{%y8e7ivDq!D~&o>EpzV2<&mqaA7+Z@%plEZDBDu1WwyL zqayv-B7yVb6?1Nvt7z@LE!t91>Tz^IizeAK13%M1)6925D$^S^eCNt1!#eUF<5K)0 zx&|(hnH{k+acHuJ29=|paWHF0;i|SRo%TsIDb={#Fe5hNBe-9pOLN-6ey0_f=F$4} zyeWx0yHVr?%_=zs8sV9J!RhnlPj~$(`P0K(jgU;cRi^vTbV&J0GIdI-4l4S&d(E1} zgi32HxIlD4QL*A&k1i-^Jc}rK`~o3Aw; zc%lc#xLhS+R<)lnb#qkN(&?7^0tVdLCq}9%%22qkDh4Hfg8tDn?2l8PF1h$CO|5}s z5TbCj^j8u{YH_o4W>h0B0p;{@4;J?8&`#k)xGBfP(7Uq)BBNdGrA&MA%j=HOAho@X zR$8Qz6^|P@rd*O0JhP0ql}s$(aB&i;uf!Q4#1ltTk}GSWK`=R15es{uRZ{@OU(%$l zsBSMityLW^U$#B)S|*a%%B#8zTQR5L&W5!Ut6FBBsN|hUKSRU^$cw%Cd_=~LS2ztZ zVJLpqC-4ZdwTo_R{^?siQqBn7?o(6VheZ&Uuwb|AOAH181Ga>^r%KQ%6&Eb=5jVys zO;T&r*eIgV5?;swYfcjBcb`1Ah_qk5MbX1Jk2NP-lQ5L*N1bi&H`Ku{>F8QubFHse z`IOZQ-4Y+FbFIOV{rW|?)=TjLnq@!4`e%4ZSp$LVO>6@P0RXzUYS28Z>Z1>WxZ$GD z>c&xGAMiR>KiuvY%?)M&T;rpq0HKhEy$wS&m1kW0;2F#N8uzC7Dn{cae9SyqHz!2x;IR+e4y(M3+n>XZlNIi1>N7jzyg9XUS@ zp{FuYEeN=HI;TqDdf4<`K6-vdk=ZI|a1Ec~VAP~;$Ok(BRHPdI>c!-k%QD*v56E3m zaB6HCi$%(WeG_4`hc`P}HL`p5c?ty0dKLKPwtDd~L*JCqWwOSTQj&G?KmMl(e%r=t z;YZwA;Rj#_!$RFe*s`hKrJG3`)mH6{q)&l7ernnzD#Sn%orY2Jt)veUz&o$~g?ge& z{d9O{P{g95T7_0tB*5eZT?$P_2yb>-UUx8-`?VR{eT2onscrQ0!kBB$OOis1A<>^Hc3vy+_)YP&%XDtSS&Z?!?%YxBgWe)zUkH~BvxpV z0<1;}n>i?jDI1g}aBgL;@G0Z+)H~X2FY1wh*k+PjVT1w;H0mUJ*@59t6(A&g)iFtm zKA7+*?^I25)i@oa1^$9f)GGm?%|lB5&8F{UQC06=+CS!RhS7pXc5|1XZZ4h!AH-l} zQx(bocm9w5b_I7LKZc4UCZA56`S)&$bVkGI)V1$EOMkGQY!dkPQG-5WV>$_*Ukl%0 z&X!_TJqLd~>%QQ}auLTBW&mKL`d8F|C28WR;pPBvRnmBVs!5)EI~n~Lu~DWqL}au%nD7u=QR8 zv9?(Jo_;Mx>j2uD*ICyxMzgF^ddu)jf(V^D)yr)u_EnH(NU;XEBtDFx z;0Jw^k*vx#=#0Mbf0L?WMwZZsNZvs){Ig^}ylulImVedza|sO$+H@aDteaJ^R`;7< zca2-NpUL4Gtpx}Y0@|?3UOUk#Os6%6^UW%now;(`5&S2SfKU*R58JP;xc#h-V2z;; z^fz2NA*IMTDf-Jl*kmPMtjvTu{6hj-PG{SOy76ilM+wJ!b*m}egu;FEaet22np0Y@@Uu1A(s(N9 zoIt7Lcl(O5*z~8CQxb;WEe2ufqx3@f0Aq4?V5(zCDOo=~G)gxm1U15OL7wI^i20Vv z(X)xSarY0X*z*wOucthJV`7CPLg)56FLB(GP3ae zqp))gZsl%R$F5fqlD`alsy@}Z&Ywa9I&l1UZ{mLPnGarxCO~_0M-4kFFhK9Zo;FDd zyXi6w`CjlD;Fc6LgRC;M%0DjR`5j)!dFNbu4m6#16^d$#7G>{+({ni+iuqQ;jHx6l z(jY3Ar$lLa9y6|*O!S4rKB_#)M-jnSz_Z?dzd64mKa_j4N@i+Cuay2Cf6|Dalua7@ z+wxIil}v>J^=`3}Pl}E$uZMny>wY5%evD@)XsqK7DIDyYKIMB!HDo|E;|^ctPzSE# z5Oe|{7>L}Z`R&=QNdU#WmBuoEw`5PC4)Ru3PgVtP{!VcL#ZQ;BK*~GB*3KIum<0e-_$1s};=j_<61%V3m}@?%?Gz^lB@||^xZ4H}u0JO68B_R(??&bO7F6{S?N2&> zB`7{)Zvxw#mP%I1p|oUNMcpx>>`IK`*R2BejXhVe2Vq%w!1n+ZUManQ{CHw~r9j+V6=-5Z0K@C1%laHPXL)6mI(d2~!#aT^H%joJz;Tc&k^4H94O+wq$MM?A zGTr*H&y%47I<)gNWC1GiL9fyx^b1&VpcbOTxdB0B{unKmcYlvrz$h)cSF#*zv8L)o zHI#6%66L4Gcm?`MyJl3w2)8xcl37-{wg%2P7@?h>#OZl%Rwq7fu`4O~*us;`8>K!3 z8RaF>wM|?;MzPOnC%uT!EmP{lj%!bs%5Qn=Qp(ScT;}~r@JB}U=uN5VYLzp0^I2Ck zH_+93l*Z(kn1qC?++yjrJdbnypL?kf7md$}EyUBd6Q!EDE>{_mw4JYhLDR1}`1i=W zRWNdSI!fF0^4)V8BqsO7>ktKY&Zr{m=gkT8Xq5UZom3~2gUJ0$s$6XRyfysrFTAz+ zDaWh9uH=!d)9B?l{rb*ev@kP#qAxm&!7if8Hb7d@pY!+IKP0l4Fnt?4RHZG@v&PWN zsN~qGG&Fkw*z6}x=b(LOuoYU*^yRImNU9@-s#YEa*+J^p5DI;DPV{P8M>JbIS<l#t@=HG3}X5;I}`mt-fh8OtF%T1%t76PGuKrSIw7n-QwQ^`zfaJw!0de;hVr z`nd+y@c^}ED-m|)?w#$A3#Vpow@6azmno@jG5QlcNMKToOgJ%*GGAbykx(Jh_3)oJ zfAY9@Z$De}bfZsehF3|V@03>+)Kp>7^8^@kD&3DlW(H-UEtl)C+~J^|z-Ku-uFr>n z9x?%z*<{xV*Gp!q8xskA58kDt@J=2F%9D0> z>bK=q!A`&GcN$s8vk$$+^8y1G^5c1_Yr9NV#-WT8+KSfa7QD9=ripk0%CUv;HBA51 z&YxVvY!$(2c`lW3mfXZUD}1T}@Z3gm5NK7j0Yw|c=Z*5R@F~5{%9+{Z3>iveEKZ3? zK@q4~hp6~rs!LilrDyI@Ztn0a+pn>7<96RZn(SBoq;K-1DfQKRe7@IjKN`?9Lfo_m zsdRcen}Z1tomMcgcRnph%&kd8ij=UZH;M;jJ2MH$aA(P3)b>KaEpFPSB;NMCVt@6D^wNacbyBu9B0mT@J+Lh2wq^~M0b@&1RzZ7MQwEGl z^J>)5PknmryvWK+{MkQp>6pe{lIOu-kC3px7&EcdhMBJTWy zg8jZ1=m4cZhI|$}$Z@+Ry`%6UVZwK{C#Vl1a3% zHdT7qZ%%8vF%=t1zt{69m5AR-7m7oca^z6S1;WUY{-wX5Zi?Tjo}(>&j8m?s(>d{4 z{1Qr~`2*avzJKNb^M+ueIRr&A!zIZb^HG%o7qpU99F=dY+9qBlQw0!%7bfDJLi;79Oo}t`oqa>3PRfmsI zKwPqm_@)sigp+lG#CD~Yxk*YCcOPMb26$^o*Evq8w*vF*$&RvAx^Q7iN49>X-VBX> z^~bk~dh%FMJel(AfHhf~ynz&BqTZ{lsrZ{+F^3&|HM10RMw#zQ_gbeT+sF|%fy6kA z*=4JlO}3rA-zZGPghG-X--P0;q}O6E?4TOcr4Z<#R-0;eV@{5|Dds;!NczpH1~zs> wEnb&BL->Gtg!l6!YpQ+x?_|nxqvqL%FVB6kUC6+5`f%ocgWlii>i6mY1!D~Pga7~l literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb300/gdn2_20260813.csv b/benchmark/linear_attention/results/gdn2/gb300/gdn2_20260813.csv new file mode 100644 index 000000000..ac5bb7075 --- /dev/null +++ b/benchmark/linear_attention/results/gdn2/gb300/gdn2_20260813.csv @@ -0,0 +1,41 @@ +case_tag,backend,variant,batch_size,seqlen,num_q_heads,num_kv_heads,head_dim,fwd_ms,bwd_ms,fwd_tflops,bwd_tflops,max_diff,num_iters,fwd_bw,bwd_bw +gdn2_h64,cudnn,gdn2,1,2048,64,64,128,0.140,0.000,81,0,0.000000,20,1.92,0.00 +gdn2_h64,fla,gdn2,1,2048,64,64,128,0.435,0.000,26,0,0.000000,20,0.62,0.00 +gdn2_h64_bwd,cudnn,gdn2,1,2048,64,64,128,0.140,0.934,81,36,0.000000,20,1.92,1.33 +gdn2_h64_bwd,fla,gdn2,1,2048,64,64,128,0.434,1.511,26,22,0.000000,20,0.62,0.82 +gdn2_h64,cudnn,gdn2,1,4096,64,64,128,0.250,0.000,90,0,0.000000,20,2.15,0.00 +gdn2_h64,fla,gdn2,1,4096,64,64,128,0.830,0.000,27,0,0.000000,20,0.65,0.00 +gdn2_h64_bwd,cudnn,gdn2,1,4096,64,64,128,0.249,1.835,91,37,0.000000,20,2.16,1.35 +gdn2_h64_bwd,fla,gdn2,1,4096,64,64,128,0.831,2.967,27,23,0.000000,20,0.65,0.84 +gdn2_h64,cudnn,gdn2,1,8192,64,64,128,0.475,0.000,95,0,0.000000,20,2.26,0.00 +gdn2_h64,fla,gdn2,1,8192,64,64,128,1.582,0.000,29,0,0.000000,20,0.68,0.00 +gdn2_h64_bwd,cudnn,gdn2,1,8192,64,64,128,0.475,3.084,95,44,0.000000,20,2.26,1.61 +gdn2_h64_bwd,fla,gdn2,1,8192,64,64,128,1.584,5.832,28,23,0.000000,20,0.68,0.85 +gdn2_h64,cudnn,gdn2,1,16384,64,64,128,0.914,0.000,99,0,0.000000,20,2.35,0.00 +gdn2_h64,fla,gdn2,1,16384,64,64,128,3.115,0.000,29,0,0.000000,20,0.69,0.00 +gdn2_h64_bwd,cudnn,gdn2,1,16384,64,64,128,0.913,7.021,99,39,0.000000,20,2.35,1.41 +gdn2_h64_bwd,fla,gdn2,1,16384,64,64,128,3.119,11.608,29,23,0.000000,20,0.69,0.86 +gdn2_h64,cudnn,gdn2,1,32768,64,64,128,1.788,0.000,101,0,0.000000,20,2.40,0.00 +gdn2_h64,fla,gdn2,1,32768,64,64,128,6.128,0.000,29,0,0.000000,20,0.70,0.00 +gdn2_h64_bwd,cudnn,gdn2,1,32768,64,64,128,1.788,12.276,101,44,0.000000,20,2.40,1.62 +gdn2_h64_bwd,fla,gdn2,1,32768,64,64,128,6.142,23.122,29,23,0.000000,20,0.70,0.86 +gdn2_h64,cudnn,gdn2,2,2048,64,64,128,0.134,0.000,169,0,0.000000,20,4.01,0.00 +gdn2_h64,fla,gdn2,2,2048,64,64,128,0.825,0.000,27,0,0.000000,20,0.65,0.00 +gdn2_h64_bwd,cudnn,gdn2,2,2048,64,64,128,0.134,1.027,168,66,0.000000,20,4.00,2.42 +gdn2_h64_bwd,fla,gdn2,2,2048,64,64,128,0.826,2.963,27,23,0.000000,20,0.65,0.84 +gdn2_h64,cudnn,gdn2,2,4096,64,64,128,0.246,0.000,183,0,0.000000,20,4.36,0.00 +gdn2_h64,fla,gdn2,2,4096,64,64,128,1.579,0.000,29,0,0.000000,20,0.68,0.00 +gdn2_h64_bwd,cudnn,gdn2,2,4096,64,64,128,0.246,2.027,184,67,0.000000,20,4.37,2.45 +gdn2_h64_bwd,fla,gdn2,2,4096,64,64,128,1.581,5.827,29,23,0.000000,20,0.68,0.85 +gdn2_h64,cudnn,gdn2,2,8192,64,64,128,0.469,0.000,192,0,0.000000,20,4.58,0.00 +gdn2_h64,fla,gdn2,2,8192,64,64,128,3.113,0.000,29,0,0.000000,20,0.69,0.00 +gdn2_h64_bwd,cudnn,gdn2,2,8192,64,64,128,0.469,4.030,192,67,0.000000,20,4.58,2.46 +gdn2_h64_bwd,fla,gdn2,2,8192,64,64,128,3.119,11.590,29,23,0.000000,20,0.69,0.86 +gdn2_h64,cudnn,gdn2,2,16384,64,64,128,0.915,0.000,197,0,0.000000,20,4.69,0.00 +gdn2_h64,fla,gdn2,2,16384,64,64,128,6.140,0.000,29,0,0.000000,20,0.70,0.00 +gdn2_h64_bwd,cudnn,gdn2,2,16384,64,64,128,0.916,8.034,197,67,0.000000,20,4.69,2.47 +gdn2_h64_bwd,fla,gdn2,2,16384,64,64,128,6.131,23.109,29,23,0.000000,20,0.70,0.86 +gdn2_h64,cudnn,gdn2,2,32768,64,64,128,1.809,0.000,199,0,0.000000,20,4.75,0.00 +gdn2_h64,fla,gdn2,2,32768,64,64,128,12.200,0.000,30,0,0.000000,20,0.70,0.00 +gdn2_h64_bwd,cudnn,gdn2,2,32768,64,64,128,1.808,16.042,200,67,0.000000,20,4.75,2.48 +gdn2_h64_bwd,fla,gdn2,2,32768,64,64,128,12.194,46.135,30,23,0.000000,20,0.70,0.86 diff --git a/benchmark/linear_attention/results/gdn2/gb300/gdn2_b1_bw.png b/benchmark/linear_attention/results/gdn2/gb300/gdn2_b1_bw.png new file mode 100644 index 0000000000000000000000000000000000000000..1fe00f1eaad52532cf256a494689de533631bb1e GIT binary patch literal 105914 zcmeFZcU;eHA2MToac2v*Zti8KduF{hJj1NVBe3|8AKfm;_SW2qocbdgwognZ60`etYz3dV=x++jIsL?Cg9QDu z%Zqq;m?}AV?~1RuIWpIAe`NK#3mG~^QN5kIRa>jPZbmu`e^abp%}K99asKz04$Wt= ze(LgnfBpDe8b`nCzrVzJ_G=3N_aDWmzWV?Fw?!ZRO2dI?@~P)-bv1j!!3TZ;0SxQ* zM`(ZT__*Gp<%P@T;9#|Z)`CxuR4#V1t=CdeP-yS%4K^z{A8lE3sZ8+TmFp1^m$%#2 z7uVME=p`K7)bAk5^e$Xdsi>=~>)*_GA7L~~GsS<B}^XCnX z{d2R^G7jy&HPTnq{dD6sU*X3Z8X6{=7b)7=+dE(FC@Flr|E`}?cjc+nBByT6HS5>M zT33XetV<4Sl8cUx&h?m(ShadJr;t$S(_`;qa33e@Q@nz1{+VPrUcD?Cl1^id8T=~&fr1Y!) z$^C)zWqX_XQVdcqUBAweo0n(#K0<02pXIB|p`n4Xv8%Ui*)sV1%hoXGw=FG#9UUFw zeP_>})qZjIlzx&f!xNKizq@zumV2H2)umRKVpQ=lTFoHmUAGfG14GBJx)isuU%b?} zpV{0(LhM>vT2gnC4N_uUdh7Yydd`pjxPS-WF*(%r<4e|wv~hzPp3;#Bp#ZO{aET-Q zd-v{@@toW|HQsl)eRf%b-}%{|JDuBOf9Agz-@KV)er{TlviMb-Bzic^jyVcQ@ZJX;lu(GqO%hJMv^6>B=|1aLxA315^ z9^1t(D;3;)4?jOGvThyY+qZ8;%?lN5Y7;L!K5#zH>yz1pvn{G*L0K$p zWiEOZ4o^={|NfSji{#|wXsK)^JU(9OGp!qs9zB})Ov!EVQTe$;J^}&){=0&gvV8jV ziQoE9MmnD8i4`6Jti1HpYssRS7~LF;SG3V4**5%LnORxf}o5T3*hI_04Js3eRcxHU9XVmM4dw(^IO~__zCyj-GuoVZ_Dz&yrZNV#OlL zza)@-ndW?1^dqx*_jtDto^EEzS*`i61#Q0lP&&Z_gI1UgO4Az zqtE`Vv6@!&n&L5QH<@gFSTPgXhMQhWE^Bo1#O#edYgD5Yz5KDC?WVsJcs#c_b}T-J zT~JWa^2f($RZ~;b!jP>Awu=L9-{$FZ0vxdD8!;tIg zvLN10+r2GX_}w|bb>H5-oa+uoCXQzv`EYN00M;Ro$+Xtbg}RaW$l(tosmav7aEY>* zUBSZP??i6w(M~uhxb5^;zVF|^@7cRoWzQaF3DaEPm#za=UvnKBa~)UD%*v?(bk`V=KS5Vh0C5e-IXcWmoyN#!PFb7p?;jtv!3|pOnPyy=7r- zeXG}6{_KzIZh-|T4Oj*YdU|^P{{HQ#9Jur1>mq6#yu1NodXK-on9sMViDOwRVamK{ zQShqfo(db((LDzbUJ40e`{pEjuCFP;+-H{e{Q2``f+knCxDOvKEi3c7y>EwqV4(Hq z#|K0WlJ!IMX0ckuTQ)toaYLlFY#Apf&5j-98%jz_)Or;a^$X6s#JA1P%s701zx>CK zACK^=DPGQ_4M+pGpiVA%%qFe z2xX-BmM@PsN;h+Ga45dNF3{QA=;&iH-dNzyO!C{7ppi@(Tg8t`}OlS@8BOlPQJgpRU^JYn3+%RtSFn1;$q6d z#bsC4@edr_+!xgX**acbWWwU5OEb;$L($=Jo7NgySb3stL9$|gc;ShSo6HMM^Lxwc z>I|5fnY9l~iioHfzc`zO1tv;ypzeiLX5W<(78Bb$IZ~|@u;o}KGez&^<<-$y7A!vY zbo|)q(-IU%`PJpILwrx~i$&x8sp10t@m!K`2VA(IX&7_=~%S1 zlx2PDA-|-gBuaS_dnwC)M7HYopw#`XCqgJ?+)QG*oGjMN-MeCo1?BV=gs?!IS_0R3 zdwajeQwi|*M>VAH=zbTT_d-u$>sCGVz`K-kqJGj%hu->4VRENVoqBzRd;PZY%oEis zY21{oKUAa&W6QViXUPk1Yw+=QeC)5Kp_0Gscz1t4t{=NZqVS-Mo@)QMo*6~g{z&Z{ zyF-@P!LM;2`jss$EoCo%2QFFhIENh_g2vb4%o$1b-MhuZ-gY|;O;NrXKJ#-5g0a#K zd@L+1s<>rT(T5KoW_bQ_5|NV9v2Q_-V6ks{#z`@+)(FuTle}Ey46MS1_8p%y+uM*H zrdTBU@MS7b!(z%GU_>-5ATV&z`1m+26@BXSW1EQ^ZR3ZgCmYib|NBxr(A%gOZM~;kz?|Jj2=&=3|vSc5Q`%TaUl@y>&}3Z9LDqf^C~!qqhCsf`Xknr(1$h zSXOV^v}C>3!&28aukhZk%0)^_N|ACdtV@?Jjo57Ir+YUcVZFqW7xYX_Ojr50vo2r0 zd~kexx3)HiuC8u-dpo%jn{2~#nwqTn1qDMtKCND}c5Sp(S2*_5w5{pv z{XIQQ08=d7>x8AGIPXi_zs|959xPjCo>;5oF}@K$`BG`AI#uj5KfAu_Hch^Hwd*TC z29pN89yG^X&p%tt3O!k{q-A7fNd@3P!FH(7Q?};wXDPQq3;ZEjXQbAc<-5_WUcH)> z=1nHg-i&05rXGII=IH47qvhrE@-JLRw=Bor?-&>e1*UkU8o+w8>6sY%Ms0ny)=ZNo z?f}f+0mTlWGJQfjtG|zn{9W5<~fWXzB>L@jd@RFt`e)P@Vc-{KHzV}3i zqMoDH5M=Y?=g(KItt)ZazVY$LLMrq=_Uv;MO&m##d)w!cr+V7VRY-IH{!;9%q0ygu zC~vR6d|8h5`pIp{Z37b%{`~y>!S+{l0HY-}HC#D4Il!a5SFc^mR_yQ8=NJfJl{7Cb z9sT*)($4Ox();c8AV6b*#|~1j@=V## zU^8dXzTDa1T;;gzuu3%Z!3rUtM`zj!bG>Kfutq-t#L?Qbdnz_;0t!<%+0dWL zj8+}fx$WdfVN?J#;Gxq3E)SJ_B;p=F7Po)bca^4Se(t5QPea-f{^_7=*LVR-S5q8l zSP41lzmIEvykvEti(%2yZ!hf4PG}kzc^6R14(HApirtrYO{0`getvw-VW@N0t5>g7 zj%_m1)B9|9V$t;scPuTowfDp#+fnYJ!+0Uqv#z0s68yYN&4|#*j4@!5fKW4!G4|n`w!SI zHqUiXaX5Qczu5?#KA^a`nD(0a;S*JnoNa+m54#%Qe;JRpSsH=<{4hcB*^OrTGTVl< z2YnA>ntpvw2&I%GWE`yur^F623@OabO%JGI(bg8;;q49eacekjR`mx!j-}xInC@Y7 z(1pq2cajE8JmY_KxjcVwUHfgUKtVG7W$O4qo8)atbF+B+FmVI_4Mu54JpyOJuNby& z-I_fuEiJtiRO)7zwH_Bwtw%H{T3ewP5bQwFyacnK_ECILkNXTC&*+~5%M!-SqTF1W z_$_&+61T+-G%4Yksor#(=7jiomZ_;JG%Re;yw)OAcu*fw+tH_Xo6dSY8^$K%-8izq^|SnGrf5paUl z`Q2llHQl^u_wnQFsN&bJV~VFXF#{;6KR?~_C?zF0S}pK3_B^<;%=uB#)_nJiAiUVu zhx&eVT9;qF@baZR_2~GAd!z#I-@l*iZviE5f%%ycc2zaCj-Nl1wvD_(jUsDOMnHhY1i_zW&c9b>%GS#|dgXn~W@ZLjQ zD5aUU4H8r_y3P{nWMhU17-68VFZCOM%DJb?z~*zF^9e$L0E>3(>T+Sr1io~0Pdg44?gq#{awJ- zivYKBXMa#fjvO&9@)pE%3wZu~N8#L*t-#Ln+Zrljmawz4kM_5$K%d$yC8Yx3Z((a2 zjQ-XR@JK3vNsiqWtk`J7CngMBML8&dRKCyYk*9{w{7fa54gB7^efuUU*Ka$0<^(QW zxIp=O&rZ=${a^Dg;xBcyymZZY`c$n5+^Jp~3nluk>J~%ms9srS&C-!mPsuD4er0 zPT!dMx7+vw!SULBJ=BfeW4deiZdz*b;>8v=Hp%+yZtU6Y^lgt_(=!&*hKbOSX|*fU zl!Jcbp{MjHHNld@K6eBrXecjBV0`;j{_ z8RVIxWdL|FrKYBq_Gv{a?wXw%8-nnVYF@Mw%uU~BqFQPGG7Afq=|bCJe2e`TGTztR zRTICDFf;7*jdx@L8L+@lw&v$m7Eht-Za(`%n_wlSxyjWEJ9pNV3xZ;y|8&m$83BiQ zGcqJ3e{Qjql$5AZTI3`HBjq{rVMm05`wy;0IDmqF6 zQ%0ZI?`)raG>O;9$;-=n7wV`)f8h*jSHbtM4Sp?F^HF9!%~wH%xpU_Z zS*8Ry>aPbg=yrdfoKdw5+kJ9s>U;75L#3HPI?XWgu%7WUv6mx;Dns$l<^Fg{R=?|I zqi){3x%u30<2T(-zz71U2I_fE-KE1LJ=LW2?m2M4V8sL|FPiDcWP=TnN6OyX=a%|hdiPXN#lO~0)T3U{Hn$674*5$N3KNDr#gR;xO?{g44 z9aoOsn)M_hf=%{i6zUsuVPWB)U$Fr2vgMKYe?Hk^-?XOQ{^O$yV3>!Wr){=x+GdFj zqzXh(HVkHzvhmujI@I^l7mA9CvgL;sw5nT>P?#zIGlhS4$M3&C^0Uonp4VE|XU;Qi z{77R@aA)K26t|$@gI+K1#N0b)-t<)mzT5F_qCjE&Ju}y!2@5o?4wPz}W=KEUSEoxF zb_KEt(kqXQj=LMwrpNo8mlOa?27B%Z6uO#aX(j+*IXkzOrSQ~$xiL4}G(AGYttngc_ z%aQLNz-bNTXV0J)*5z#9woM(llY@if(Xi)I`m(D_p}y?~X05GXzS)ApmI(!-G|hFO zK@rwZ(lZ{aC@rOwb6i=EcmMct8SpHqXpx^UxMDl-&fwS>5pFHj)41X=`$0DRC@RKFtMro<%irPF!mMoPtOU+3|A(7-NVyH*yq!N{Szl2=Af?sZGc&WaIpLHjf*o+!1a(+X!N`j1?z?nA|? zt@^FQ;nTBm<3_?4DP>br0aaC1hyJg!c3*Qj0fRWhrQV9KUCTf{N>5MEG;g16TirhU zUI}IeO7y1dyH;t(^MXOASbeI8xQcYjMvoJ9J(82-6iT5Qh6@cLy6LC&N_I+ ztK&7pi#u_(7`Sqdv`|!U(J&M>9OZL%<3CvKLS*Ugul}% z^7g7tH#d);xqqhc6-uh(kB=gDjTtI^PAm+&h9WfMbvGW)S?v3(ON^QJp@KUtEE2KH7K#iPJb7{SE!V~)FL{gD+mw&)g+(X}x})5AwHSGuZbFP^l0S$Go08G+2XFJD^E&HUk$aj-~Z>aw=7 zf`7(D)M02Xxo$%-B@PrQ>XI$5{-h5#+K@6JX}rX%)tb|0S}+t=9C z^t!RpIP$L5+nZ}S=vDINN7aPVTpt|v6Slwur&`;xAD>n>;-H5bB!V4*ddFLoD7rYP zVw&TRsoU%1404P=xfTh1eu-NIXE&xh|2atjc)ZJllNLr^&B4z#2xZ( zeK#`l;@t<2j2Dtwy|qafx1Ol7fb^W}+`|SAbO{1KjoUoGfoEn?pWW87wM7E~_zrat zt-`5a8vt}N6JIawo&an4-Ta&#I-IYmL$IgCxD@r;pS=Ic)S(##vMI%|Y)p zBbaaw?Zq^W&CRRj^j_{q^2r*YCg7Stem9e(VP%jeWvM5zZ%JUKnXV z3vrd#M%S-X%KppKctdl4V`F0;la21teOttWoa|3*6cs&w{J7gtCj;zl*0>@Vc4R$( zb*q5^z$9E0*Ou?(-d5h}kpju>lP7Zd6o1g75P*Dh#{nJ=IfG2DLvF-ZXqkKC~3rL^-8eu;R zOhvgxU9EwPer*gOmbw;b}y6N8t)Q^Yb;QBldr%)AgD zZggRZe)yUo6`#BNFQpgF@dLBeyy(NVmbn}SEleIin90Pxl4t*;FO50&D?o>{cb*^p zXe_yPD>r&{1vaK5x^29C+|uYr8PX`MF%f(;-s0cfl;lj$L#ocL6l zM{lZw3Ax1!K;;v?KKJxsN}AiwWeIp>C2!s^1L#5P%_~12)D^yG4gbK$>L$hKECyvm zO&{0~RCnPfX(&1-w%Xh^d(RE%HAQN{4=F|EF7*mq1EMq86{>fe%} zm(O%!hrAjZs-bH`L!u7dcAHub?!BP`vVIrnm^z2LDjGqc)B1BH{&KYMqq`>9F zh5p6c*!IlNbLI9w5!C}ebAOnCPI08Ar*OF@`FONefS@7I`=#~5h7^xrb zZe_wFSGTbdUy!6^oz%PTJz@~syjeTn_Bk*W(L0^%<`Sa3U*lzS3JPAsEABC#$FD@K zL+C(Q;>)1ex;4~5LU=aVG@M3h`=+=tF1F(N`b%MWtO+~Y&=M|`ChY|fUa$ruw%aZx zBD(!7)KcceC}UIG{nI z*`NNJ8+|iB+F^3g62^VdSny3WWmJvDN=ikvKL=1bsn+#o2C2rG%2V&z)zs8Re|(}N z8I7ukJ5$byA64OS_*E;0y=Qq~-y_jNK9_Un=DrQc)Q&2MwOFI{>K^&FoM?u^( zwPymTOA6|%*EV64AfoDn(ql+ zRrvTlx9-sY&XOn}4`detUx%wGehf>7mQvBs&>-#w#KpRC&Xp@AplxZ+%woTHW^Odj zP>wGI$wG#uR1O-@JD0DrUotE&Qp%gRY~kYK=5|gkIzMJZ_AmqjSl9vgV+%@QY5J5(EBEtbP0D z4F^drJT1tm4!mj|HP{R5th6lR8R^s};S4T;fXW4yuYAO^8xbt70 zHld-cPzh~%3q2+dMo8P=<1<3x)phrtuiEK8te$&*EV;a581Svr<1Ru^94l9*>7IZm zW;JhHpQx=q$6$Ac)Wxv;V|AOTmC#nRHYO)yWECcvmg^m{t zJ0Qlft9wywYIB!Xja!@b`qFVw1#3R8&;%()7GM z*_<Q@Yza`UO0HK|C>L8uE13*&`mNE}uLD4~GRge2oq87b)huYGCg z2EjfE9G8I+ZSw3SjS5G5U*9zpjFS5LrFc4;dR0r;l+`Q&*SMIO=@>G`1lJy>&njtY z!^m=eO77fmW+5RVP(B)bccNK=-00}&BiKRk5ij>;CnextQJY^OYlTIf2-(&Y$`S4Y zaTXDAaaKvQd|#kfD{E^9ZcIpw9zHzpV!G*Gj z>(?J`1vSCuL3y`-1wt0b9J1q7#!C!c^P@U zk%7YLgZ7KT9R`PoVJ+})OSE81 zY`7Z2=db7V%%JVJ|N0dKEJP9yBzcjL$hKrj89FGUlaj3}^J4D?dgoF60=7KxKHc3@ zprDlg5-PqmR36fPJG#0qLKck0gTb0BfeKDK7f^XgaWMt6M(320^Y{G<9^;Fb@NZMj zwy8t5iVl`EDf|HY2yR^f@B<_;{1IyD&0FeW>HT;AWyV!j|M7x8aaB-$sh=>6J6Z(2 zI=fP;avi&FAE$>0AYhngdIe0p9hXO5DHhZvo4RBX2?;h-rgHQS8Y1Dmg1hVN<=JFr z9dYko=kG6B20g51ZB z4RH}rl7HJtYQu&N5qIw*vd4moa}l1|maSV6Ny|9(c^%k!(r*vMiDcXcg+o^E3_>Gn zM?Q*VZ)8&50S;+D-oG5V(WD1;tBU$AXbt=?QKQnqp&?F^v_a16vTM2Lq&3Li{s(df zpbmPcqFl;-e|w`c1mGeULENtA3zpuB*pu+gScAEySIT`c6~~5TX(hO<@5j0x_c?Cx zhJQq_BKq|%7Y`~w{o%vSc2U7ljl{F}?%Bg3=h~l|;|+JhpA=ZQ*c7YxV}ESD;OVB9 zuIcrCFfGv?U!46Jb5;OJLtRiNk{H9uNdKKhV!#3MBZXTLcnjt|(REQ1qjEv&h z?*<-xZYoba^_V^Yw!k7tRW{aVnw}+}8kbE&n?X8r?efRv_R%%7ch}_;_ zbTIxJHa5|=EKwu)+svJvs{;M~nE{3jRz$&vCdNndvp+m1G~ZxBV_RGOLFmJ&VBYVZ zn2JdHK=&W_78XBYEDsHP4>&{}P_Q~ZCIdq|K*0Rb(ri56H$OY2*Pf-!n=_NY%qHf; zewII8n^;v_qbhESEA>1_H?Oy+VRr5Rs`%M473%6f;5QQ-`cYVK*R6K6&ARfq!0p5ck`*$7kbYOY4Gv$CqIEIo?%tS ziar4ah{+C{VCuRG&-!$)M`4u8BA9R-sF-x-NL4Pd}OVMmOm^2o^ zjyBUQ2wb9BAPI6=$Y(~UYYuy*1ELs=ZIq}Iq;*KJ8=5xg-;>N1AWDb6rtP5OXyl6z z+}p-E5OVnGQ5HlgcVFd;epZ++?RcUGZ-*$Vz7wvOAUT+%oP!Nqfi*20rb(;D~bbc~%IiBy$CK9>TZfsZ%U?PuuL9^!rRc27%?iy4dmU zTU<^xUJ4Np!14{`JuXX2kEL z9E5A6jY9w9*RM`AT;k{^b@lhBjY}LYn?@vP$-NzC9`@b)9Hf|RlpX?=+eWZ0(DmkV zjg|+Uht{rJM+%kO)TkJ4TihVoA1LotP0f~O@0aMBFrn<=nrin8ONb*FLR$>c)d4Oc z0;Okvxb-kBr_Eg6JvpBI&#q@n1-fW_lu8e|9?c* ztgWqKz_CMUtU3QZCqHLi|DPBN>w+{1zM)BEq0UgHxnlHpgU9l7Ut98=nV_8bBlb^A zM~4`P(N_`9Q%_Lch_sSvUGWh}nao94dv5YBbi1bW%xxj89v8Hi1=|&0Yn44c6=323 zwOk-U5zjxvWrDg_kpE%_T?jId`=p^EwR#^~54MaEhBZ!HO)slnkuDXq^l!*f|L0Oy z>hEXKqYkn`;qgmPmn_=+W)->Bzv;l_te{U4Ycl1XbZ{2k}1f<)sg~AQ1B5wo^6q*v^vu4nYQhG@zA;h|8bb zhz=Aii;r0KVGWGbGC_L$CEidlG|TFTJ5A~yY__C@Z^bGbqjebk$Po#6tYaeqnSbWA z#BC611PIpsB5`An#nGdr@1mab0CVgTGL^s>}zPa2{|KQiko07p}%wcETg9(F$AI!BF*mR?jDT<4M}UhfB!y6 zN|!voh`-?+D&PDeq4Z^fjO5wjiN0M*!&v=PMxFGCGo)U3#R2`Qe* z28+meQ*^ZDL+k$L=jp~uh(bf!D1mx}pxq+KArM?uOnYuDJe|uwtOxs^w-N2#ZFXuq zN_jf*`sfKVhwPN^F~J7?{N&eMDKw>LP1OL-WP$)aP8gqy7UqZGvkgFXZv?k+UO2@O#BPxO5zBd5fNnTsDE5e7N z^c5pDNn9TcoTQl*1fhMC1h}~QcMo^>D_u;0UPgNXNB0yzJM|eF*e4?4$&Qn60~?w| zn3Z$s(o2Mcp(a?l6BjYgpA5=C<54#;xmRgT;8I-|hP(8+{%%7j`}X1sB-!aZA#?}h zITVN_zFQc!9N_6|u<}9Z|4H`q-#@It7!w0X=`OJG_TRr313!@Nj)(z0rM_#Ih^Qzt zu#1&E%zr5nOg=DQAEWa|f*W!PMZsX>eqK>l)=_$eJ0l}Q866a{$3>XM!Z1T8$T(TB z@c7v9soAc(zVU3OzUI4g;}+DAU4o8yjl|Gl$D$m*0t(oSD0EuU-oH;@Xo;(y zNs^DmyZ!`dCLpv7QK_4M(EvR~CShQfN9Ux%&O7<>5e?OjeaXqq1>GzZGN{_B8-!Yr ziLMM7%n&U3Af@cUo=B=O#{?3IhG5iTA&}CI=Y#u!4VR#+>t{tR6i`@kxSyffiMHTw zQjA~}fW`zsdI# z2*KgYJb;)ih6tgBZXa)kX-DJR-q{)R3=5{uDhfH2PmRPyf@+KHq}hBp+h#4rfyyHZ zX2XgNzwGFiyT*!VS`}z~KLQ_{?qtKG_QPxnr3@}`FwMH~?9ZFV6`w!r**}1I807N3 z>cRb#$-h0|GHVU6w6nk>2rZnDAJazj)KZ0HF8oG zVB!2HG_~K~y9*P)Zj=w4|q}e}q^xHMgVBsy=?j)0=OPl4Laq1?RG5frx53dhJJ{0U_={ znAO42QRmeCBk$q%P!uW2UnelKz1e%lZOyuMHVUT+j>0C4Sreuyg_jaMK7YGUy#i_Vop9O_|u(@bG2yD#~CXrPafS!rd;1gkZYq zdSjt*$WGK~lT(o_s!~;(9om&qJ2dSdAP&U{uBv-@Hx_5j-!e~8XyM8zZdp%8FlIBf z&vVx*U%8+)j1)5qi_TunMD1w58vP(48!%7k&ZTQE1J{xJ*rTPT0l*;=ruixDh&*hB zV!U1$r5eb}L#3b+g*G;3onVKV8Gv}eQID%cpnSRWWrP}{H54FOU1R#ghbsXpuHfE| zzV!L`VaBASq&)LfGsMgZ86#IvP}po$$~=SQux2-sOA$Lgq)B0^t*y120N`6OwEHsf zE;YbCy=l{?5uOPs)fw(1$KM>?0wBkXmi0czyFQMdp==&H8~eU}`v`a<1psIRl_&_- zlPU*2fItkcRdcp`tT8LuBs;lWI>_-7eDPDMw~j1_^0W7^Q(6O^ZlHAjeniB{)bQ+$ zwksH1T$zs=V8ETUJm^t)HS77M?L@2mJ{Ed;47FjUmf*>dxG*G>>!6-cbYpd5cWcff z{mq1w5BKyr7ngvG7wJekJV~FGYRs^tgF23^RbBb*BkzN(-}g&>dK~$87bnJH9P~Ar zkh87dNXj^3(wAamV~ywPjLbWrs8GH{BPQ~Qapo$dV6ipH=q2Dct#8eVJ}g68YX5-) zBLdjB+b|!7xB7I}$y`P14k?F2GPT#XlAsJ^JYeldbLD~gD?JRqT-%C0LG#i3Hx3s_ zjI&-q;isM;McFyedy)Cq(M|7~*6je^Tfc5CN&o$1W28=giebRIcK9=V?CA01LES`8 zNJ|@DTLqtphC+(C0_LP8W|p27p?RK$c4)ZsS*~c(4R$5wWSFv2C9AQ!+?y*H1 zNoz{df%l3ogb1^R?al1{_vM$?>Z}nWXdX3~MA5)8O4oAXseep8w3WbWRJD(8Olo-4 zInvI{mqVMo=7Rr%NOMtP;Nm~8IZ^D61HCVH|Ij=nNTxw= z7DgBIO1h7e%Z)R6RAH_|WEmD{#v_-qE?>^s1nUk}$A!!DX$bfMPsZ-eg z3Kc9A5_gGu32c-F4y#~hXEzdOU%a>!zN2uI>I8P-s$f(+iaF-0%^5az;tO_z^&(cXnrsY92*?3b{dJ0tiqr*OJo{vTD8uz!fKVU_b!y^=Fv=zp4+@IZ8GKIF0DeSDg~ z$|prjW^~Z_GEaV5MQV$n+*x1sRj=T%wFC50;BV$Ry|OkWt&BXJny&86Ne{yV_oa6w z>Bg51<3f}*G-P{pk@7<8CXq~MCLN^9{bbJBlW51Hr5lU=7#3xCOq@o;e1toMFYp>amSl=?+nunP z+*=pk%3qzMy9wK7Gn{8K)e9?EvzPBzZ?82*@+V=j3>QQ}y;um*wN|@n$L?Md?*t6@ z%Iv&=bTc>S*c zP@R7(!Jq{7rl<7N%>+@C43eDZX52}!rwC0Wc<}cCY&GZvt8gfR4ghj)Ztf#=-s_t2 z3q2f>*{l;<+wjfRQkdH8AT|TwAyyGo=ub$~!n?HwnIf1F#G;PP``_Zofbin_HDqVvRLo0!mmvWIYE2V7;O+sLTV3TZovdLJpv4F|%oSp&&) z3A6mmi1ndCfLFf4z$C(Q;w}^ar4kx7M|EaB$3{nE4Lh!u>oqM+Cif{aL|(~tohluo zL2haWF*gS9-%SE)hM0OpsC*ly&qX8-e?hAwXbYilii|2@+P9|>r)uapS}wo@Qtki! z;$?jhnexv~{`+T~-7EiHXa5itKn(5wh=Kq4GggNGFTO3{v?J4XBRq1vrxIXGvf`j9 z5b+BVV+BwLUVNVEnCtMjy{RVIn%3@sbeVvXGnWqhgJJ&pZS>2e=q)mlj|PPUP2zQP za6AlI{7h^D%AhXU{Uc}zu*>R=(^Brw2PQ*=!ss9wLmqCy^CVw7DrhKynVC}X%MHFp zbI6<{!!Gz*tL0t6rdSz=Lnt_q%|pMA#h^=~Y2FsX>>+zu0OR9PDi?XrlG#-_h~JV2 zAX8eQFo`!i%a8cUp+I83_l>L4-;Y#hSi5>P9sUsd=&VwR;NRD#v-eh8=Ecx0VlYhs z?DH|aA73+0W>K)@$uTLwB@W--a3V?|ZR8~{Bl8G&f`PAZUbNo{qJsGCxhsF)sMEVI zU-EL*9&P-e_izCBaF$gxWYvVFs%m{7Ci-Y74EenfRr}jF|MP3iM#ajzTuxjVFr*S8 zF{Cx)dGKP7l6lnkA3mTxELtq2C;-`zL2%vQNB8fqpVs%?+NVioR1pym#J-7vs*-ow z2YWldZQ}Qri{zvlOl2cgaM94~KzF`=3?4@GpDzabXy~lPyM=q&LuY zi(%wDBCb8!*Sy214_8A3v9Lbqq=?@MLJuKX3=5={+(UGvE~
iMo;RDS8L(kTHB-fsAI=q-bmS?<#N5DJ)UtZ)KRq!Jiw~3$2<69AX*qr4;?_W96WDS$HCIh!;oXaK;u4@CRQc1#%>R2UHwR)Lijmq% zvL+ZtMh+(c5r?If-cU~1DI9Sod3%l*7mg!#Si!GA0a16=G*FEZGm;SjOp}DmUO}oD zCs|<42E34rxmkr!4;~C&BC(uhGX8FK|M_cz*tRIlT7s(AmKz2Il0t5*L9ZQOK9LqMxuZa>rD}b>-3de>SrI zuotccR2>IJ;24^W&He!aNl%QB`gt<-xuKx~yCMT(6|$$`e1;pdva`#e{Sp?lC%$He zDgA;lbLV{CQMykahpJ`SHZGK%PdyU7jVhKQI=GPk`Z0jQr9kk`sV8tW)~E+i2@(b5 zXbN%r20#Qfaw$wdk@N51ZxPWFcnv4zm|u_P{XepCh#aLWnF}bgtT`Cd8#cw{d*Jl}1P7PED1bn*B;RX#DHN6s{T%<4>_k2? zlPRzRV2~W1hf#1H%uy4yfCSI~{kA=e_G>=kBDvEawMpFdX-5`OLUm177w&b`k&9lW z!5&x+n=v`FD1UUXy6opqpDx75uiYKG@*2kYY1$iU7G4SOL);9`F^YN(v}||skp(%v|P!^sp9gR zoF)W5MZ{~YVs>&3NI{aYTxe)$xi=V5CPvywz7S7>urhLR8*y+?Ef$t1=Q#kyAUj9Q zKyj}^rL#Xs){GWsll3;C-LnT43dfTC|+m6E|8t^R2fju3_Q)Ox1nMF92z9R&|4P2U?z1`NX^ zrw(H)BjkS-^sEbg+!)z4U7V9r0RV4sb1BC6IAH7ijo=?&VdUhrg^XJf5s`1(Ne(at z0%$2;V)BLxvESXl4ky!ulF3_yrVkM$OO62fxfKCUlBhxCJGT!PfkVi|aL^?g0*Idi zy?auVg9Fcgx4c{ig;*Jc=IOP+ulx2c4UI0e(-N2gI+&ayN3mfF*AEK|^Xv_F&6qtv z0KB#sxyi5`I9JOt8sj!IDTN~k;obRoPPDkfK{B^o$@=L8)+)rPVw@`ibngoX5VCNL z{s|D{$v-_&VL7{Y?drg7%XrVokN^=NgHe(KxBx;`>dMN#HRH(BqWC~?;*vl2`;BxC zYJChmW;e_(LnRIXs11@rdX^tF@wx6SK%E74%_Ovc2RI!tb#&sZvF5rk;HwVCpZlS$ zO^6I9;*A=ad#ydKwST`qG}{^}j5}CCR)b}>3xq0XXra8^K;+7dhQhg}sRpyy-W|}6 z$vGR~^nnm($!!4!ltW?|gqRC1PpnR)BP*b^Tbwu%Nc4g4mbdvL#V!Pq15;2VC-tzh z2yH`;A)`0stg!egP)Oo0lan>UcThKi%-h2e(!pzZSra}(EgWWtf20e-%Aa8Q6qOHd%sXHX@T5ah-&;t_mrwgp+{tG-CPmkn|>Fph%&S`AD38 znxyr3U1Sg)k3;?8 z$^`*DraBB~#gYsMM%Yv^YXzaa3 z67FQ#(9lqTyQq3pKH6e%7bA{3gsr-8z$z9ZIjaxhVD02H00Qu^8$WO2ObTKiMMrK2N&v09yjLcU^H6*a}n1Q)EXJvt04V! z6vX=g5UN<^}V#4s9mknfLy>B(gwqoY-uWfBr<97kN{y)#Vh0 zRKTs8)J2dsOVIgAAtxiR-pUKjSoAR76Sk^4Ud;A-cYsH7uIcZ#BJ*Q?6%dv%#D{{t^FPJQ1(j-qwh83`P$Z@rB#{n&9C~#smVs0s- ziZY>II||>%L4y^+3aHSel?H`T}qO&^VGYCx80(*#{idNBH%ST~)=y z$p3Ig5OyJ-v>l)Cgr1Bymj7-|IziT;Zvm;cpIebfD4v$zv2g4tPVyBus(?L*ugb?- zQ6VHp7^P%x{Q#6e>%=8bN1>a=bIC_D9A7-x$ijG%!EoQYAb zutQ6`aUd${6r>y=5n9y8&c;{J6DiS9K~vCWim=h_sLxqMCL{+WBDxG!S8wkgVoG8| z;~0PXuyrxExDBwW?mB2$v852c~(l{=LILIt4tma;0VVV#R^fPeA z87*eO^wUh2V<_NL@&{yH=n(YAW}`t=tn>E^_dXm27-R&23ewroV22vDIFTIxe+&#}1?B z1f<#<6M!9PF>qbJ+x~vO02UR#3UJ)YwnzncE^Ijc7s0YAKnESL@rV}*$W>-L1EdNG zOo~RuvG3v7JfU#>N}qZjBVkI;Iz?rqrH~5Riul&1J7~2grs><|F zAvqX5j~>1H6>PQF1doPAWnPRdUxIQ5zTzaqziL$_J7jF&03yCNB+udA34$cX0J%3f z!3*QaWD^u&CJ#fw`KB0s099-!2Xlaa-Y)#?i84V3H&}VaEkK8We#>~BZote76GeIs zAe5FO!;M&uf+U&_IvT%^6HY~G>HlEw&%<(D+wgz*HqWd@EMv7OROXpd8A1bTqB4aH z$rNQyS>_C-GDRXKDN2P~R&w9A<;I1@X=&G#ltV|8b-E8qx?3m);C6a@hDa6q`{Cpq7 zpFgaB%bzU?Q`)1Wy%fC_; zH0sGecWzTDHVdF$pCe)sEsV$#h?zRKy%kc8ieYa(g#h)khBcDhKZ^Sdt$0#V4uQmX z|9(xZQ>XGQtP7-xR zk{*StH1EU;q^T(FmkNWA4mGJwOBAod{h)#YY@jU_Kdf`7`tgo0frFw+YvyX_g4s4WE{kNIpzPhw10<{|{yMGxQoMZoxaKH$+uf z8^IIe4BC^nJ8lEiv+e68akMx>%{BXXZ@ZVr_j5FGqMGft^vlZ?MRc4s z%)9G-2Wp6Fwl+1+aDevi!-o$Y>QIC>m4;ZR1SH7^5~hGP*V@mWIkPtAqrZMp z@L#5Jr$`*aC%eavlE|v?rayZ@F)7WE37PjXuEUN9NviR67+($;;GDMa>HbYNIy7Z$z&#HET(s+^+CbmT1RBUN5W+ zYeC|TjCN5Tlj|}|ZvF(Rq!OzJ6l9Fs$-ml7F8+AcijJ}th(Kr2Hd3lmo+V;eEO-;C z_upyiU#3#13n?hR8|*?rbE)SyAFuQ9V?r(#VJT+&} znflIL^QSL)=7L&sEBSFuTyKo|-v=aw62>+0e$>B(QVWcp$=}9wQi}fLs zL72`Y1BV=ly1Cq+DlVOPv~Aa}+KS`x@oRZrG?V{+k%}m90n=^G#%>_(uN~DEf7?fc zR&;^oucDwB*PTWOLxTYbRAStw&8tm+e7+z#tK=v7@p|5u=M7&RWUAOkbFssWVu$TM z>d6f^ELi`##-B}3qlzIEO{SU0@kg7bD6KX{jpBQ5VJ;m*RfR;J2Z%8tv_*6)R-dAs zr81c8>>N1NYv|B_V^=@ws57;e#rF0}{4}&NHR0t`{=FwAESd1=zzQ{8$8k@We&1BJ z#rW^7IYi(Y#s+}jf~u}sHRxig1&^N^8JAj|@WW^C)B%PKwPdyL5r_59iz$CactNDX z88BPMP-A~miLHTU6&nwv$q62B*!++MM`JH)))sXk!V^(^)4{@cjo;fg2ke(5U^lBLymS5L-S&_6 zU2QlQ09E2s@!V$joC%CPglWMoAc*a$Ex((}F8bz*1R_+!{o04d6^ z9j{j+?Dk;+KBudj5(2i<+-Ri%c7yF!M*95)1979xzQ3fj3;ByKJVt9voB0*)hK`)q27cw|xVjOxnK%9my1esKT2IdV%-|z3{EEAuPjW?AN#(2*K*{09fM& zxbH|?%R7JB{;7g2?f>Gzsy=3S9#p9ylxtJhuQ_}6?DzAwx=9qsz1nNb^d+5C6R*?k zGm+?N0M|IixB4bn5_=V%^`Sebv^ORfrAB#Vf z)(YKrPYL?ulC}@eK|v)!m&nN;g6m+Tl!)-?0Nw#qFJf`5@D^b|f*P@6g}`B^)m{zC z#CAXOQYv&gy#k!Sih``S4#&boW&p4p?D(oS0Lh3p$NC@J;(y{*KQ`9yDLMe(zVkw9 zD(%nA|63~jsh-R)p;ur&fKg8Z#iNU!tz?KLavW~Xl+JB2L}7NghPwWrR+E?GFQwQ| z0+MMWYV{hV^H{Fz73i~!m>#q9LODO{G_%nA0iAwq-KoU>(MvuZzgqUkP%#7-14=O} zLk0-8thD|8sL`e1>5CW4>R06&6jDT0e^&Om4*`;B{ZDXk8_r>>_fZq{!-`;_5Xy~y zaR?)$t?{I!7Z)@A^C}T%Q-2wAlIQQgR;8u>)AJRm3RwR?G>1=VMnLBL>=%j8 z_s5+c*Ov|d#G;L5qMN>2J~P=!!Y9qALcu_RjCu+wMG*&}F$G<---v#5bpQ0+3-MV+ zX=p=YKaQS3(nDgYgPORqp{$)!QkYMUTeh{VMvWSgkB_<3tWj}gzD+AOpW#B3TW}0_ zPu;=y>@(fP=VB6<;IGumYnZvCc|@4kJ9 zlQ(*uN{QUdtQ00 zxg!=sH|oK!N{K*+mei~oZ`-sJ-+^WRHSK2!;9{N-97%n2dV(pja*7GJ>Y&$=mUz|URI(U2&e7a++`z7>BV$)a@JpLvg$ z1C9YUUcm_j^P9`W389K+Wz40p!WHA-jc;6CJ0ckcbHbT9SSw1YD#Lj1zArG>-F~HF zCX6fYH?jvc=-2}dYesj*8$*!(?nY)Q= z*arI&WR!ZpGWv_!w0+`!Cd0^FKCIkR_ylyKofS{9qtwYP4)OJTFVCfAz)dmcK-cY4 z#0E$br^GhGQ5A)l*z>_AM4h@+_N|qS0iDvO&S1IA zXYbeS3#KX-A%<!smxKd=Vh8P6+PZc2cvC zPA9i)uW_aAmp+L^5EF6S8QBL_!q_8=A@+_2G?wcWK_>lcbFAP&xK3`(C(AHrIhZZ3 zNJI%ht7dluU1&=w%ca)M0ePPm@5v=k%saLVY?(o__!0h%72-Z90qNI|*BYV;Hi@(3 zvUFP$OkyTpxTdTC7QgSWssL~mn}JA6NEvG6oj1N=&t(0xoT5=&xQ~`llqaDhSS9cy z=}_X9^-`(%@xW=c+{MUlu6rv!R^KpN~n*9KKjNmA*i+n6?$j? zFy@?Cq@OuQ#Yvhbvc$Icaz+&MpOzM8-$$@xhhAKFS0V#mEf35uftdCi`IB=mxxE;M#BRj+b`Fg4Xp?-tJ>wu7$K<1P z6Qd3@4=Q6`GG?_~X0i)6NgDyUpJdW!rw`B-r&k7hq= zsOAPsx&g>5+Y!9e;6H=|7to5fSe;5u_-ZA4w|x1g)eQl7Axa}3XuHnePFmCgJ>6(Gm$dN@0Jyo)Sp&IAujIU&6 z^&LA03yLF!OGgHiv0u6TNDRFaTa5#s<{ox9(VpU4s1?)?Bj1H7n@ds|q3BMA;ngcw zx`6`0w#kVRun|Mh0k15{??B5lm7e~&^$zs4-}56v6bi|?_&X~0W987ihx3L95%7*; z<_Xy=Sl*_t7e@!p`TLzA?RzYvCTrHOKgd#)xI{Y;IOTp;)^K_gIp?XO;<7rBCP!i_ zn?L~VzJ!*~j~C}YBk5AY?;kUVp0w(2iybc}^nBZTfO~B@s}Vmhlza_l%$UIwR48a) z8&U!)KmtvqVWZ&&K$AH!oQh-8Y&XM}$Pp93z^S;G(SSPPt|W&rfz<|nTZ+wxfOU{m zGdV#G{&|xO^03%+yK`V4Ud(way%<|*IJW-+te@HYoQ&*-FA&V`tc)l4+=Etu9^A@U z`QpWkSZNmY7dNCnTSb;;A&T-4_(k?BIFUku(n@Gr1saBGZtsn|N3~N}z;#RMLd8#= zx0a?%4wJIyY?vE{2^TJWVurGl7qk0UguRdq>Chm3=Np9L1J_L{u>@=u|Cd^oo09 zA8iMs3$+DZVIDP^p(r!?6( zkQjOwzuTXaoh|3@iL@!OWK8R^qeqY4w-tgzz7F?0JZ`YerXj$eeXO*EK#zIFdLXr` z3e@T1otKCce11L8kz}<-Owqv$>SmSJ^oBf6%*~y&H(c5z^1PXc?wIHepnl9zwwQ^* z&yCm+NC_H@WY}~OP$?3HlAAhpH90dB+=&sKwQ1dRNg3#YYbwN5i#rbAyyk{~fJ1Ua zz`Jb`Q&j?@8+B%w?QhG$eI2I%)LY4DYGA}gxH*xPf1X=aAqYr#L_)= z;=1hd+Znr>9e#xrDJ4nTH&HX>Ie>MIuowZt*j+JNf000Uir=y#UAzZOIQq&``-M?g zI4;RGlnJ^;9h65M9;f(hoi&6_TuVWee61t)19_1X#(+Ur6D|0J@MwyEghwaS`goie zKmd6mriZ1RH}QwBDjQKllHF`Hr`Gb>(~L>q{r7WuN`GPt26S?era$}JftVOylv)PK zqeoAzMM#2Yw9R3OF9jLe$f$<~I2)QA9DBqwFkc^A!e~`Ax3#BT)CgxO4{5vYaPX{D zbUSiF4rA~83f8~HKB=mLMl`AX^)Q@`7d(D71l57UTZY%;@HX21J)?~;64CKp#%AwB zgup{FXw#|^ewqtRHIvOI=R;Z_Vx1DDU+oH%GgA<;Xx{CvcK^Yub8gk1mM!?>M+<#$ z@EVE=X|UA+h88>nSu=7tNBrVZjJ(wJ`uVw*q%f%~rfSeysVFA;q_xfT6d)u=%07Lh z+*SzlcrE}V;%ZnWOj$Y+0$Smn*0_l>fsw*iizz)2u#_Oz9R3nHkI=9%BrQu_?jQcM zp}N5a)^^jke-)oQ_i_E3UR&BoYYVu7iQJ#3Yms-E&di-!i*5t>;Lbt~)hqW;mB_r{ z9aRh#IHHfr*fEa7ap{N25;sqUut&v6j(^nI+-k7-wC#D3%% z$1m=|;*sMb6#Lzrsz`Uc4yB}cEH!M{aMnBS`xHh#IP_Psvb3Nty348?Cj^TTAvr_5 z62K%yjYZTLj0gof{a-K;D-0>b?1QxeLwuDN70Vrij|-(hCJB9CQn89&KU>&B{|u+( zbD(Lkn7xHXdrrErk5-iby5@>yo);VZ`O|2$(F=Nn+U*HCXv zv>1m7dM({X`ib!DCJV&Fl<^432A9?AE3YWE)~>)$ z+ZY*HMIk=x_SoCIx|;PF)9hYq%ERidsFk3W_KqovyVKHdJK2>@gHK;wy1p%*qWd8p ztkrZjYTiTD4N8p#a7$VZWzNECA{z*c0p9BG?&ipoona1>tmR(t~8KFArfGFc|ZTTJlDEU6UD z?Bu1#LSR>51`@HIphFZ7T~L5cZ~aI%Z<>75+1!TSp%(fC#|)d(_p2xfC^T7B!~*SG zEWNnF%Ad`lUtk(D?dM^8A$1FVrJdnNJ?WguzRy3hN+`v;$^?&f7OrN}aX8F%qr4U9 z4_E(xxISn&^G@%8)?rxP%1S(BECLTIp4XUjcW%uMd3#Mw@K==UD~dA4w)72F_4rgM)i8fP~7Y~4EQz@3hxTXTSss88rM z51tvK0|;KY3YZS5)7VD`UT*3GB5Wn@9ZdwFpeGb7CXk0=j3J5)>2>5vAQNe?$u_{) zB8Xyb6m%%+yP@vHVZ&-dj&)CY47gad*e$L6dXF0Tj`U_wu6O%CtWuJ)&VQ@>C4S%I z8FsqMTI+XQ3W~yX?*^DQC+btq>Ru)1;nE1olorL!*)vyDKtcc5*8ak&`|3u+LG;&r z{FtL>I=NTrt6^`j2HKQ zU;g7u-e<x+22Er$%y2uwg4MatR+%1_3cx*`yy> zHGm+GZs$2=YsU|xmCxW!PeimO@3(gzH>RLvpoSh|8cbK3E^bbsT@ziWra~A3Sh6v5 z2LIEisJ!%}kwA}#IC{$H0V7~?ymiP>>(*?q)CjQ;N7%m1ZM~Hfh*9M{hsnCHyJh>i z$cO|_oJbpW zW1%MwJz}EF7L&7)g|(t&LYH--+tm)LIzs-`?V|#0Zd`-4xyQ*!^;`9+5Bhd%Y#I9z zr_(V%C$!y$%%dh1aSippm2b@af+_b#UZMe&RxO+XErNR0;ZOrB9nQ1X6z!!)88=G? z4j7uKqLB7RW*Wewl@Ois2Fo8l+ZY`Z#M#!`la>{k#hpScvn)FEiVUuS$a$ahvpMY(SEU<76>i<|QQ?@6Wg z8Z?jzzH^HTf?ltTdr?rpn2hqs%cznLqxKr3MSxry+wt=ssjvTENtk-=+shPyXLY82 zohDCSMcmJ`FQ!k0a5CYy8BnIWk@HWemDb?*2L~1yu`&+0ZT`>iZlLy8Q&YPqmmrN4 zH$o9lJ3HGx{j3}di(sIjdbMh+XFo*F zakWj3TYvz-0Nzqw=FRM%eb)YbIaWG4#pAAKV5WVU-oc% zaXJQ90P|b9!W!tdZmmBWVyOG)&)|F=Z4T8ZYb$LK-@I{tcnC(Hio!}4liRE0aVflU zPk$Y?r9B0fINXYHdPet#>jOvPcq1v0~61xssGAypD*6pme)_TB2NVhi5Xi-_FYVL(R^niygYyA6Gni@ycTnaJ5DRoFEc2YnWGss~5oN3G4LK#8-cCfh;=}jl!G<^Kq3vpfo%4U*@Zg#z zKKcr|iB-9Y8JE1gg!-g)QFsG)i}%L}&YoqKMusKCB5ORf5yA7#((e%)eh(to}}qxj*Lhg%j-h$;3+cryR(tFucIVqBNR8NwND zY0+~SV@RhkW!v+7adm}4p}G7($G`VnuWPw_dQwEEuP;*$vR<_v#^J$0l^e!`e**+; zO9Tlh(&vbDl*=J2zHAhx(7lST@X)pSCT% zJ^ciw=wXWeNU#`P`{PfCByaNZN#mWj<7`9xb==po+n(FX*JpFzbn5m7s4GAV9l|K! zLL27xcrmTV56{IJuc~^49O~1vW%}3iGR^?c6i&jjZ?+OZ_&$)1xdor9w^O1bc;NCe z+Gc@E-r%*vS6Y`2(QWqfJqpt?_>kISQeo~^&Qgp=Vx19b(;;cbS2l^6Aq( zK}L4%>S(%P{`}FL=4Z!HtMl-QVXqexrWzat#P)Y5lHL4?%PSI!ps?kQLN9!}QZER) zN4buS`|K=vw$!-hcTA*lTImYYH-bt#6}gLi4C&|^rxR(jfBn7`ZOJ&@f zBeOZgf#ad=bV?#Dc89_;I4?B15itJiJiyN#=tA?Z7F|N?VtUuGG#cGaG@a04E)4wE z@yLru7d^3{D6-zx)}-*m`H!=JO?9n8*P#6h3=GsdksVFQehW~x<0HbT!@@%OpPY8H znfXIHgbV=Qrk(6ZbHd}0xTR@d&x^FeomzRqg(ayyvb?++kwyZuXJ-fsTkh}r%auOo2yyg?{$D^VcHCzfd0^l^Y(|LZ7|fL{Joa)j z(P2uPBUT638J4_h*FfFCo`_w-ht@JhaOQYs{s9_=&beL3pSkzt4^H6v>EyC!fX5LY z?jg{ycfi4?Zf)C_E2T^)BaxRmyej!vCdalzbY)ScYJ3im-RieJ)~0HYk@V15UoD!$ z^~i0|dYrI)EQR77zaLzio@owY*{5eKj+y9%hmj{eMBUo8M>7qt-v5a_JZVb!oAm^H z2F(IsLXMG|BU$A=_T1(t?*XyBeEQU{(4-cdzopEu4h5T8?C(N(+C(`2<<=gh6bPKet(P-WNcogA#WvG)OALixp%t5@44%_uH!W(U7B9N6K$ygMj;#E*UI zVZ=e&y=_FkB957M1qg_|_SozAR6d_Bd*j^e8hkvbR__D7E!?Q;%HStj_FoVdt~uyj zvDzRokRHyXkSLp4)2<#VD=*D0fBEXwA;{_QrMC#_7x9^F+Htg|iPXOFKyF)*L8E63 z&}vb?rst>2wWtmq;~o%x-k|nMM7r+Q zkK^&xR$tzaeWawjbZ%IBFBa9pTtc&RpFK7c^x;4b(CtL>bR4(FUEk9;Mrwch^oR`! zM&)JyR5yiI^dpBIL_<7ay?;X!cq++>wZ~7Pz~ge7|B>3-$O#_s+OYO&|o9JYC%=?dRyE zfbs1fkL&AKm>9pBlyC$lB_+o-ma$4<^;Vu<`Dh=@*oz#|)iY1)cO`EFhjbt>;mXmw{FPNy_2{&nwHH_(bm!+fvu z)jwYJTlytTYoPn`OJZS2m0^jguQteKCgSS>|JLRC%5C~%dFpzl9KkN+2g)0{;bJo z2*6h@PvWw*_C-&ZRTW4$c5t~j%w^{M?bF$ck8Ge6S z&4cYQ@`YhaZ)$pfxj*+a8xVtST9<~dlzR7wbz%6i0kXaV)+nqa|xui-BHJ)Rc?0;zFPRS zs~5K1%i_|dfs-Ank}PVj9t51`>9m=tFwl}{6fv_nKGUUTvFWjGVTg&wEgJB>FBpnx z@rTcsEDfM@+K(I0oZfaVDyf$TiijU-kb&J@YTwvl(9ibS8cK4f>-OyUPdwN)~+46=ub~xN0>Fqdq956oW`?O zl`(~fM{;)zy0+5a71`QL&T}W_guVIt`Ex1@+XD*AIjIg?#>_1jHRCxZ>qN4UM^YcT zu>2vq1)aWqPdnGBV|^ojb>Xir>hW_mQ@)&VDv*11{fYYcWex4E5Te*ZUwVGao}qK< z5#?Ciib>Rfb8;_EV6=9}DHe;-x7e!uSQ^Eu)DkG|e;kO#&2cnYq1W_VyKEQNJf7K` z6S~hKJNm>)*AUj8I%^BBAD5+z=I5yF7UZn>RO8eV$7T=lmSO0={=U7?Af-3gYvwWb8b1PP zmX83!G}924Gy7o&uUBs7* zi?^|3S*At{3N#soI${%Xr(Lo#O?yb1Ho@QI7>O>GvZ342i3yGf)lQr|f9~8{KA#mOLm^1k5xhPI zc>TIS9KFw(VjMF*Zpy;E8{sSw2eyJ8H>NPhH6{|~=iK~f|A@k(zrOxEGRrcjhCJA1 ztPbh=f{nZ;B5cf*mB6G1f|l+M?_d1t@+%5iS5QB<&_knwk2KacyPlL}!YVvnNS<>< zH9eM#O8?nYnetzbr>n~#gGiw%FUSRKr+}cFvAwXw%@ZL;-09Ov>J$MupV^(q9`SK%GKK6K6OYk^v(F3WouO?|a;fs{Su z(Ly-ye5z8i3_9lcLaxDa3T4khCX~flfMsFBjVUq3*_y5mLTUq|WQm%R>NtVZ=L1!-x}eKnc{DRU6q?u_m@ z=-lZyBKVeflA&P8&jW-uvlK6%fc9T+=tsY#mw?#KTH zzqH^cWD+Oi=`I+nGi9A+KwH_gZjlW=fKthY&MNN2i2$;^kD5-XuVKju`qWWHzOSXT zD_Zyq?T81ue5%GU`wmO|U9Hnu^i8{rx0k1l1anvPJSt)-UlHmYB}=BjKU08Y0bJU& z{xg+e*&9`kprQ2d%=p{UU%=>$=hK|{mw(bqdcBl9L;9zMy`3h75L=*(k!2tHVlFoC z!-u&WN51=b<;jzYa#3!x^Voa6;I5?!1fZKmu+5u@mCjK=zdulXz1e2eS8kBQp38eP z6JBqd3y&2T_@%y~X^-6&!cuER!dU418oh^(yKT~irf&UvylCRN;h!fH&$&PQGIh?z z4dn~@%mjG{>Gp5{KAIl8Tlh$XO|hIS*!i!&cGkK!@a?5~3d_?BT=pL@VAiJ5Pe#RhW7G&Y~5)H964c+4y6}yE`u4W zAs;14T;FJ~fBhF%-iwpu$3;xGVS|=nOfV<7raU*j10AaDp1>qwI_C{564IfjY#Wxi$)weKRjelgM7m#6g)3OfTWbt33ly z?r=Avw{`jb0<$-SR#zl8Ige+RjDi0>WL-xZ5@0e?Mm*v@*%vbGA_&N(m7W;DCg0$U zzKSo+6K*SpH2eH<;e#fP8!s8$+^^Ha;#T$7IhSwIbj8~@Q|b-^`dbpE=W{o-Ew=lO z#h=2Gr{49#r+H8CMn9lq6 z`&PUNMoSDBq+Qwuv{BE%;8Dv+`7Hg{U3T*0Au`z8v{|!5+_9Ta_Jy}GNT{f3YZ;!M z{x_vEZJW)C6&En5oB7V&yVK>Fy>2{w_%PGu=_I~&7(gTn?U_I@J|-a~h_a7f<$_wJ z7rq?Be8T-hBU?#>MOQPHrX@3GQVjjUjQgGC<7Mx_=?E}+KmEkob?b5lw9z!Ls1kGb z{Y)KW9}MVYwxd%d?cvw3YWtRP2gMg}O!O}H-n4h_pSqF&G6?bR96cu?Pe@IIx(5Sj zrfKxn0iVp_a)=^2f7rnIfL{BsfF9&>-mK41?=z#BkZPRaX_w`X1V`@7FsBd$(U z?Nt>P%aH>A-vy|G>B zb>ilFN2t5j;ycFQjI9PYyH-n2;b29TuAcMp*|W*nO$VJ@eC+k5Nk$teA%FtXcedtd zKdDWAEU%*QYco2K2W){2NZy<327B2ags-FN%72tUuE%yXQ+T)5($O(B@mZ?(d_mYz zo0MjCmo3jvTYFQj{qLVsZd%P+-RV}(w2oZ0@Pc!4ZnnmaK&{(vn&)F}s5o{}~Xn85`)_3n@DuZIh3w!U~$OioH%kS@g7k~e>B5{ade6)4` zVYJ;*_{X|j?9;aGU;^L{`cu1W)%+6V^HqL5Js_?nJX)!T031i-1Qv9Rh`aZQZDc4N zlcUccX+71w->-kss`7WpfpcGj-j(-S^lYbC(l(H{Kl|8aq1iqX5J z+`Orm!DU-El0a7ZWp76TE!8Ke*~0-%6@7;NcTW_GC{r`DG}oAfL+~;71l&`jYI$$5 z`k!l0oJAk=H{&W96gAX)wB~*U0-#!?L34uDy)CBCd-p%L%6Xp5wi69r5-rgxy#r+W zQe3=Ki*7-e>K)Dqfb3j&o5b{0!T)oa2DY}gfsD)xrkRwKBtaJx^myUvSs7;1;%d3T z+t;mACw9ech9r@O%73eA8O$0}fq*ng- zISG!O>^Kb*c7a(6Hi$ z8gD&GPNSF@fnxSPH?(tsbSTVir43xk%pva?hPKnx8ux&@%{uM+;Kriyj-Cm)6GV)w zR+@EmTyf$>Lee|HBj4WY-g{S9tTVrMxAA8{m$OAHk`p{>7~>i@ZXDg`7x42@2vE8; z3Nwc}a_tJmg2unE$QC~j=KM_Q-)*7KUQ+A9BREWZ4|Xcedr<#{hsYsj%BMbCmkrfE zASKhqxgL!pP|K-N)s`HoT&xSi`aW6GZpgJV!0}$B@!kNg(x5Qs(bHL7?pkM5*|uT3 zPPgqcJZHf6*$4KYCcB-nm|WFwBO6xvwWL0>LMvue@O;u-eVK}aZ+{Bx`Tu!~{`b2h z4l?Zv#x{f)Xu}Zq>|nvVoF>l*-26@Lw&C7vcB!RyF$`F9X94l9HWMS#nPT%!qw zhjP?83Xey3$Q2J$zTzrxmLl!xL0`i(M0QP^H}43gr=olm#;1%c_C#VkIqz3v#iv!+ zF7B5Gb@>IsFmitFZ=rQ;?+ z%3pKtuU1iM7+hi9G=%{Hu*qN5ouYWBmXxe)jNRK?Du39arHb-WD9UpYBOd+qX+9vY z8B|By$h28L_u<%wiE|+{=|5AB{{jJ$kkeg0RgZ-#OO33?7 z85cLzb%+at1)H<+2j|Gdqt7QPjRL+sebNiRs78A`K&5fWau)Xx@NbY_uClXGWVfog zEY0CLCuaCc=NX!SRBp#duBuOI8LQK_}heg}G-Pwq|G zI(JDRsMT9A+yx63EDJTLq8agn>w}X}zxfXR%8i-53=4ypxO1VIqe7pB_BU?AY-yOt zsM?$6;gkos@A~*(KD6K8zu(ZBI#aTzQD$#+U&o$!7C%lt_X>n*dGr#+^?V`20FWgf zP~K(X4W;fF*bM^=M(Axobh79rKMZbe_+j;5IrMw`N^_wp?;4|9S zseD3&P?yK&;$a?-(nh(k5`8zvFzbj`ahA^bT}ECP1?}bi$Fbl1!$|3GfqQP=yC9d{ zz4EdZiU+s6d?zzRAFtN#(P9kADby!W+k8RIcwq3_9z9KX*RX2;z3U!y7MZZJhoJTl z+Ic{59SiSv0J6zH4BM(Qp!c`9?h5Qh8JDuI1(kv3(KrvQ!4?*qQ8yg|FzZO6&wF63 zwGX7A_}{75l9OH8?*kYgB7%y$HQl2gP|lcJlD_3@{v=8*xJI=WEg~t3I2`0oS{5Z# zyi5AEFS5c|5D5@}%d;QOc&9y=@E0-o)X%^6q&Qon))77na;C3|7mq2PnMXLI;(s&`cwn^8DsLb|j!m8%2&JHn+!5v_Hm0Hw#kl)huv zv9g_~N7UtAlLO~$YAnT^V9X#L!nC2W^8o)mI{vJ+P(Jjk?ikF1=R~;N!e{hHX{lbP zP8I-$Z*$_KXhZ}W2sZR|l;Z^n8nn^dE-#jvN51-J$xdf$dSD~x?@n*U72Zkv2zt@! zR|Tw+SHxUG9w$?R@;Q&X!f%b(Iol!7saMN*j=mDDL6XsEr0B%=q-F@_FhcM)!H$Pk2KZmOo{ z>~J0!`Ls{7&PSlE-+&dAP$CvF3>}3Q)IaR>lV#?@m;uN(CD_tR&jfy9C&y(Eq~g*n zct1NhF`!%X`a(1LKye5an71OfwEWG;P0~I2_X2@5AM}rwplPcc8Tx5;#*T6zvupD8 zR44E4TV48pW@+;UR&jSP9P;Az?L84(tph8}jz$DgQ;clqvl_?4siitnfMMg1DqSAj zi7tNy`1B70Jv+ks+VNdr6YhnXg@rA)zC|=YE*xIHqwOB{Z9cNNF%+Ya_K`xoqKQ78 zlE4;y`gz*T;Kpgf4!QVj+PE2FUB zZO0S<8$=w+mpNYmFd(5!`*CnKszJ%xA_~wQf96v=9*J<(EYoj^zbN*#m(SrXSUORJ zy2xt}Ih+`n@-05XO;1p28Bb5yp|5Rl3%v0pS_DK1<*Uj!l*O>4NBGzMt4lPpm#{aB z-Q5H6A`DA@=&4K#3dNGfWDv7OYFCMIvM8yQA|f^9dgFXr$J^kS5zTBusDMfgKHLZ~ z@(Mb}$)!8AbTE8uv^iyG*GX{?)(bCsq(GVi_FB*aRehb9UO)v^f}5KM^_$=^BBE!% zr9WyE*8X?R(du{2!5SHNv%5(hM02w^6p8V?7{<5{vn_oGuFIai(kZvh>u~w`s>&vl zoYs-93JlGP##~ybqxP|b8UGChIZD6Z=;v4-osZhrG5_X`7#HC`N3HN%T6n#kIu?Ai z5$juju9JskM%F(ludKU-;wQ^}J<7`3G=}T_mAH#yZiGL;Jd7!`2i9}E?lmNY$XB8l z-K>tLrm=r7&W;)Q^;XXk#u)GOQ@x6+l~3!jTg|P%b+y+^Qnp@-HS*Un4NC1`-9A8B zntQt+KjBd|zp1)<7rcg;S`{R6>3jBqq+gUSz_m@AGzkbT2X)7wWv^Bw3n_nO3)MT1 zm=1}tth}tabIjuR54J^@lQmJz_#ui5Dp|EYwx`*!<<%zneh;0`mPs5Sft{LM4!H$F zpQU2A{%x}51ywBN!p+%zT!|1mB?t^-fKPs*{^aW$+s*x7zJ49S>bB2`K;c8hqSvt_a>!97x#hni2vlKbE0!}sc0S^E!9bzW zkBmQy`;YS5U%qo^l%Ogua(k|~kDF%o&-wG`4FN@mU&BUFJNq5+C12PN;+xHuVjyP-JT*17 zFr1ZG0a0*mlu@f#dv*&{Y0Nm&|2Nv1mKo_>uSPOm?d0T}K*xYHQjMy&oY!Nw&^hDA~bp3tXXUf|(u_onwl(zYfnY4_{ z^0tpv)6j_FJT)t_yjio;biMI2QwNi|V<#>z=-p~sy~;ONC@j|nH}0p9u?k^%16fb&=H~G6mxMRg+19T$%hHO>}Nk^YX;?l@BT)AIT)q*Do zivwJr^Zr-myO7k~KsfA8c@B9*DrR!>QHcal;d6p)CsV2zYYGFj5*>lZAYt^b{nnnqL4dLIOoCpZOaeBp^2gff@FRm;Pc{krpz&Uziomj0D?j}a6 z+d1NdM9XZwr5wlEB8sYUy#>mv__{v({r_Mu3Pq3V`r1z;Io(bRSpQwGxEcU4AOYhd z{<~?Kwf1+c(tqOrkpVmZZw9P*X}`H@+M3D#moB{W8Wi5{2Rl}=Y#=~c-vbhipl@A`xO7hqCc|KtDv&)k^*AK#X5mCs1C9nfzX@I(bH zhNz$&JXMF55dAvrB~rKzg`d!T)SoxdoZoOz7U6z9|3ihOvn4n3y8+l%{|3F(U889^ zTw#cXr)i2IO!Qq230z&FKs0qx<0P0jsmr93DZG&43 zJfF}c024>4C2wB7JP0375g@7k@yC8pHN*;+Z>R#g@we$SD%WI>!Iam9d_ahU@eK2v z<^Y_KCDE`2@W~E>qcOfhciLVUOp$1fo0p(%S_1MRrK)RqBU!b^TN6jA@L$d0()?vy z6INUZ*U*80Awk`zu9Im{&S(|bYhPgVJmc`pmn+OXRf>ae$D!TxA&=uVi3M6i%4`Mzh5)7;x#q5h~R)irilf%b&!qQDX>}amMgt$t`+IXuw z*3<&6T*eck8Xcv0ZM_ zz>Idy)L+p(Cw?MQ|MS(q$$wZ6-_{&bdHJfJ=~4avyMKN# z<9;H_TOrQ@`_N$A z(Ep3(2lOwJH;iniHNsjb7`i2JE4_aG>_wIbrrw)kq%ces8shfLnn!|>3JNL}RT)8C zgkn-T%>tUC^^GL*3xPnd%DIkal->Ez&~M*AYc@Am)lbfC zYSe3X@`>Qy1g=UB2%d9wFW{no?qPf2W zXW7tnLJXgDa!5-Penz3#*QIb;wNNSy z$L0HoeQc>}k$7>}A#6uPF=vNz3E z@`6#$blZXYi|3lAVLGBMJyzyE6>s9~MnDMQ2nk8E^qhK-ebU5HS#;{I(km$Z7TPCt z6vs^eFc=HLwqd#$Wa?@R z*7!0WHlp}*1F~#dEsYiXsl*)GDIpPs-)fO~3zE&?)Ilha3hqnM!{OYf#xzQWm&(g5 zx$)NdY2m%WDg;o5DQx*ewbf_eV(YT5k(-opsYkhE??l!E$b8gBwbyrRJ~pWXT9U(n zwOl&%fG6@msg^DzBxGROBRm%h!T4dvlozYHClaQFa~)1%9Pg2zW+FTQi9{ARBH=;s zh^aC{)Mw9}U|r>JTvz0GF=H@qG#N#-Uea-b8D=mHh#>4(|Ans(BN3F`MT!=F;qir~ z=`EKkug`lngU=&D*y9Ae1)mbl66|3rv`-P!G$OOmbv$`kjsQoo#(83wGj&v-OpmN| zk3uzAaeV>GQKVkV|QtA!B3WT>~Ab@*EsA8GJUT`eohhS^Q z0P5!eyCSpv8**AmCwSXHNHR2z)6&UN0NtZ4f{&?90IqN%a4f^0X%I$E3AA{&Wx3OeE zv!=hZRsKq1lN5{chVC#7$#HhS<;wT>o5)73UAL~k`@G2<7HWbkhp3>z zvb0PY{j+FRj;h+!AC@%eNoDk3%;_CKJyFOnmhma4&6^j4Vp_71&&G`m@+X17v@^eO z+39$+_VJ3;@NMUZ&^6FKX8vd_rn=~jI~AQ*O7GWUQ&-lSmT=AgeW>EB20(ti?mU-E zmlk~@*=$q_rqEOPyZ@yP(rW6D6shx^@?TGw^LF@ImeoE$CqWh7ygPO2-rdUjOZ-1x zmwtUbclpx84>A3fpVj-l9)VA_I8}c}R<;YOzdF}09lkzn%Vnn}(W=Y-RQLdf^oVQq zMzd5gqJOmt35)6%yz6iK5G>@x+${#m?-(~krggQ$M1Ac3a%%c2zn3^Htt{V=UUSR{ zRg`);l+0SO_&=$clLx3pY*Eu6-KK);u205pB8;s+i+g%%l&T_qV0><|87A=&y}zj5 zy#Ht_Z2UG-tm-SjVGy2{plT~s|E@5eRoO49zR}{Jiatm6SHHib7wvxm+^6ae&II?<@b*oKe}& zs=n!5yHZx7{JZhGn7VHP?haDD$Abv-kshTPV(-DMY-w!rL&^;ly%ez0%CvM$F)ANWTgU24ED8;=>`=rxf6 zN!v{d_3)t^8!H(-IC92tBbgnr47}=51K(lUZ&DO*`i86d;a`Q zMjZfKY=LJ4QEt^r&ve0}MPsPS3(*lrjo8*U|56h@Q)b$kMR6~BYbU)mOHZc0j;cBk z&4-_vFO$ZZ9#{m*fK#c6F0^HmP~@U;rJx&KCR|!MLHKRTs=UeT;o2ntV-&NCxD?XQ zmy9G#BFzhtM?y9C1W$EQG*6{^y){0 zdnn&0doX`dPCP?Zj0P2zK7v9KJ)&p=8ME<25iu+N$I4$nW(cSty&0XmFw*pSMxU1e zB`jl1#{)msraIO`nI2MVd{MpJ-raLJicy)2&G@2RdbC!KR)gaPG5jEIrO2b&t-Je& zl#@hFQFWV-V=q8SL<1|lLkoHzghEt#t)?rZjqx474wmYmYdS+-;|V1SCF(-mzJ2>H zU9u!Bs^c?;WHRW2AfD>U#qM~aej42mWyxC_y|LxL$^?~ieXI(K*%Q|x@*f`Axq0&~ z!xFybxYDj=Q9FuGB0RF=GxUm)r;KLSQY8&1Pnjjju@^9dU`4aWP}^+u9PS_cZ4f9% z4mHH1yHbB!J_WosZnel<{N17TTFp?s>+H!)1ykURCTuYbv?ufuJJml7quN`eRKQrL zEum|AG3Xhf#Y{m?KNUviRF50&A0~`%j+fWse;Ev&;iw`doNHNTd~U$vNJYh+m!Gdg zrDlFU#W@Y!?&XUY0ndX3N)GB}D#GWhx6D*}4d=h1+WfT^FD|w%72^?8Eo7!WymMQbA^9H!| zYS}rrgSPfGv&v7IeX8R#*2!C72A^+a0aqr2{KKTU&hIq*U}{IKXJtpSqs>w$QMr{t zH__e9ENP;Od)ar&+O^>wch4#>Tg1=LnDKh}R+I*@whuK@P7>1!#Z)t-i zgmVmdQKUdk-@QLReY&4bTkzxNqsa)H(-}BaoA2@gV4?TUzVf6gs);0 zCmsYo-*CmVzo>!HsE8HFK^w)#RHgv{*aQpf?1?Zki`CzWvF&4fmA&6}7ci(#yo9;G z=*@}K_RE(jAGF^V`n8G3KqFB3!=&F;JO8J>4HbRit3C4!d&$`hDHAyx&6YNy(XA+P zPsxr8XI02pugvPvqDyjeo)WX8!~ z{7Y^J>leo!%g0m@_Pg}nE{GQwOuh3>RnaA4*o}lg)0NPcbFY?12li54?)6vpsx|dA zUyvuE!Tpg@I2RmHeoo^)V+p!KYY>HSmpkP#K>2H$t3hohldr)8_sS`=3{w8S1Gh!q zjWyQWteB9wxfkasRng!zHRCMrCW%NaK7ly`$xl)h5mRrZr}xY_LrrjK#xm7ST#JIu z0ua|cK8DgvfN#-pQeZJus%>H+y*X!1ZXTtIkqGRJ`HuSNKu%0iRg#I#C`L!0 zo;$%cPkBQW3!0O4l;iiP%x+{mEKg(;%45vms``QccWHFRXF>p>d>_>}vsVKR3P#R| zY`DH^tG_gWrI5iO&OP08thwrg4+OG%W(7!~#Z*!E6BTqsc4Aq4wdJMm!2?vqcbvCH zS2MAs)e?V--#?{LPN_gsw;r?fB{sN5Bj=EZ$%mJ;Oo2vEU8_KHH|e7*1!vJQl}CCq z{M4fPL<&@pV;fh#LiI5^W-Oi|K{+2>F}yV_ec$sxu;9&`H)`*ylMub0=Bz>iK^bi) z(_NKPjqAqzcE1HLH{_MFKqF?JzYCxSsA!9%?P&MT%1?T|CguG!hYlE6idHb3Wgz2n zDbJDYkcwwXwXm70F3>VY1X7uGm_G{V+x)xr?-uB| z88DV`g_-IqF!ALdz#vmp{eXAGh;u%T0UQF3r96b`J@!tu+A}5M6+qK&jOhx!NvxWY zv{`up#?z@R#bqMB_u1^(qq2&W-_?94Qm1p76}?-ZciJjYv&RHp9)#(r?L$nog+O8$ z;|N`aE}c5wg=BqWF>WDkSpjOL_w22@C(i*7HO?%OLm?A5!g?9^S$6wd9y((W(d{>p&4S zulp1tw&;YYppFi#{2(HVx#Z;~W-szo>o(Q`{~RMqJBDBdg*n(sdCR@y1*1my?jI(Z zCCBTMCso6Ls5CaJwg8N>j6icDT3)Th+Sjw8;HS{iQG6UJD}G3Vcli%pKkTvEtGE() z`!<8nM)hFs&xtwyv&!L0wIA!>P;E_v2@IR*Q1(k?f;$@3`_-?Ysulad*$WQ6C)5RDPiv?yR@ zX(Y%EoJT}tRIR%K+2fWc0)M|j@EnAWp2Hzxz~3S^EG0?Gkm+weW0f2rCKH|I$Q~-G#9XGr| z>wcpt5l>dM^P1HhJS%hq%Jr&H{2$|73co>V9xI0M(6voraz>@@XjXnv#k9QaBpQb& zOJBFa#={O)YiZo6li!OyR9kPk*;pm;n08+kPWKr~g-*$jXV$LJ&YgdGR%il=v-{}LvyuNYM>Oj1$CkN(usD9bDl+sR z%8`u8(t~NPsps4pOTyT^!mdGZ*XEVJq3{Abeu=JwGYSHsj(DXE zpnTBMcB?;D&ECe2;#)-s@1+x$xfdp$<0Dj0Et|<|Klu&NJegXe9yq1ckyu8uX^x0X z7F?<@#%v9&=a-gH?^tiSv>?xC;>sV>NWK=_C@XC9M(yo)04ai_oO5#!z{`co^qgnu zDZzO8>$J2sRT?bVedVlsyx4SUyLTk6pg5{E9yeQ8th$SWnn?V zg58J(L1R}05iE%f6_?mzi-Lt_EMV`kH$(*$dqqJ-BcOud{XJ`8l6~&J!fs z#&-O7b});zzV&_YJKs5<`8?13qilWe#7>~(yT>l$zW9`pFPM&T+73E}Akl3BJz7(6+U_?%rsE|u5AMt~$v$L}+t$@1Oy8^qbJ0gNVHVR_k zi`P%FsCUJ;+3h%-ymM?;PLIU~P%r1W7H?NmsMs2XLdiCPZ0s}*An_8D&+o4^q(;xX z_V?M>O00qOqX|rBp-EGTaz^bI+D%Q~o>F8-O$!JcxVZz3H3?tnyLf{)doid-_f>5t z=m==Ax~l~a#y^7UdRLc{h+mROT}zZLyXwh1EiyKjeG-hr4ZPeoJL_%c-N5K_O~$pw zh0m-xA4P-d3PpA_# z#|@n4yl%b;b5J_2Uf>W66rKXtxDOoPEH zJ2@#n(;KnWI5btC)f$ND>*=7r`6|%UN&IWpTYtVo52|*UEN;i~o#ZGuq;_W&is6Ll zzP)&Qu9UKd+ch4|W3n}hd)Ar3P0)R<0EUHD=avW zgHVc;aC(gzbr9y4_Es_x8t7R_B`2g@wO49_!uR|uzaKM~RBBh~=!*_%X3O@lRiv{ck||vEzR6O1{QP-MzwD4|zo(J7O3M!j6NaKg5+3z&I@UQj zq_m6aPQXWh0<%4(beWb@_v{*ZtIyjwAS_f3vwFBl54x})n<90;87Ro%ws zNZqxhTDflze>~$^KZ0^ajxp2FD139M*pLqP9~XIjPIE|M1dujHh&MI`!*=#&?I{il zq~$0YcKj1}f`MN9m>Cj98tz4#VxQ0J4PkyRDKT&?q-Mq4bL47U*2p>p0QT@O@OM?N zhmnHR7^+5@6y9UUsKd7E^X%i@226e3js?gb?noerM{{Trk3vQ`G=)!}znQN>1R)jU zh8|#t$fOKLF=JHq0{}UI>MkR7ha|I}La}%58uE^55RI?wDAt-qYnpFOo#7c<;lTBuTjl^wwW9*IS6(c&VWsG8xutXzwLK;Ri~=B zr=0ez@4WBrG_*P~M{ie(+?n+;U*A|STkP+|i&{4=D_m_q^oSB7Ks^_dB#p%TiuKGV zkdLqQs-&_@XV0FkcC9#5q%(<3#FKp{Cs5~BIq6H3Mv~y&Y}((-sZ!<254K3puy~*#fm59WyL>zpkjQwZN#-W&2ZVoeg*)FoozLPq?`$L2ibeA6lZs7z0FZZm4K{oNDP1iGGWko8Aq_@V z_nd)(7|>D4{V@?|tBrtJPW#?=xT}U0Wfu=Efru9k&L05^v8+|nXS*(AN-CQb{<9=B zLejVyqSu(haGgLJHIV1K%pf7N;0I{&l7E7skH9By+1qUKC%G}H;^lGQgwCrzN{J7h zec@%%-piHH9k!hfw%QU_;jTdzVC;KNH>Co|k7WLIkU3DxHeW{cF`t`CgpphI4OCm4 zmLOM4iD4$MsiX=Dt^~w$2suF9WrL~i(U7qm_HNYRXoa={H?OTc`vNq%vsCrHZ|2l` z=D`GzgQ)Nw1o(F+?y5>6cqO8u8Ej^)JkA!CONA48f!p&?%@QrAqr!?Rl;&m@-c2-sCC z{kDp7AN3+*U#DM|lf`aIryr!SY|_{DXPfuN^FD zEOw=|r9}gg6B9wCCkp-y(4|O>e0FsyYUL_(8I}hqoHuqxZm`a9;!G>JcGWFhZzO0S zO<#hw8K+SOR>V)qe9k-|i?Ry{2w*4K)cZNVPZ)n$1F{rcm>svLvQamt9oBGitx7p! zu~Im1BtJ*v*q(mFka>p-;Y!s*uCjEZ?VTN$bq#YWN{W**J%`>-=gfWFGJxB2`5(4u z-dtW*y=%2P@*7>CEqsYhNy*S_z=I9msdo{v58H&UHA$a&E#E042_taw+muSL@Y zx4bIv2HsB3N30KGqjVQ^8>*dXWwESqyx5DA0{4>YV4UD--Qms@FI8%bzyWr+RCTCa zI2$91e?8c0Pz}+FB`k(5caBt#xK^gwq$nvX7$VaBkG&I~$=_3!5JXyK+r~ZkUK5PD;m0I*+JinVxP&{k(z847n-a}>uvKi0ocouE_-d-=(1BYvZ|Ceap_Yng zTx|~fb8d|wFhH!^5mo*?xEmJF-2l$S&Ai^Uht(yRck)%H9oHv`u8s|5DdBIE?>C*H zLujL8FzN`8hWR2UT26zfczqxY3Y=H+G;dq*Mjt=FNP z_wU20Xv;thRI`>_JOU5Y6d*`&zi67K18L3$L-~<6l{kP*2UBZFrw%BY7ed;G2RaTZ z$u|i9ewWQ&?o;2*RyFSBtZjJU?#|8M*gLLDY(Lq*9^<-JA^v4L1`uI&Hf1TPqMsYFxB<(|Z? z8$D%9QJ-K2GnA;4620x!)yZM`##!GL~nB9FIpTJIO;8^rkn?)Q&Il%k5v4{Uo zCcjapH{pP(x%PmF_ ztd!MvfNG4FX`Gp*(+~(nwVQwlTpC9;2Z!d=2hpK#ShwP!8Y(X!k%v<`)lJCd^tScu zPdJ~w*8UEYh$cS29zU=2gAs#}eSTjd{Na*$XYP1zo7l8>(whBMdz24AnP*ep=kVt8 zX>XsONoiMzh<=6Ecg^9UI!rO{7pScyh-~ECpnI*VT4Gsy!jkObltp&zOOpk8X$O5z z(Dehf)JHD2u?pKjDSH;{SQg1o?*pz0nk$RwK+Ckw3_ z(YOG!R+R))p2g#;LEW9W*0Rfnw1FZVQ60v;RzS##Oj zsaWL6%SfS~55uebM14%H@fUMtU1y>F6Dtv7fQ}4$7>kB}ph_c!?4vF>A!_8rRi~>q z9^ZUa|AsYMP(K>3nPbGu0L8OdSzBvHG?dY-4?H4#WnXYa2suraBK}*Q}<}c?zrG9lNQe$ z^MnT4>4j0bfAssx0^}Z<}oYrE@5X))(i#Ag{Y!4?q# zO`=r*?e-v(!({k9MBFW0xwsBQo|HQW_bX}SZVb(g&S#G$vZ5Hpf=@&a;nbM9k3$JA z;-ueBZiUQ?B4p*ua*N-Led3ceR#(@%!~b_MdC({hk4qvZu-J?y?b*(&8+W5@kr$TR zai+v=;gKx6Ms!77lw`B zx77jz_P$1WeK~g0J-G2~@ZNPQx%XhpK-|~_tKU6I^)#~}ewj5ZtWU@*4i}|{s0ua< z1p-7;ZW;daz)Eb*$WaDRVzq96=ZkT32y>ixeK#+DOj>`vsX8sqe*t8BmDqRm#-s|= zQ1FLN$H#XQCkxYJ?OLq7Wq=~t1hW0rf5Xru06Dn-GLB9xAZd!gyJVF43@=UzdCHY9 zFOUk7dXpvfPCncNm}2NX$+)ME#kj>K<2Q(NvgyylL?fip=mBI45kN5Z4*;*!SjeO1 zCc0E;;w3t*z2H_$eamA{Y&mK4^F^3V!pEF4SsnuOv z;)=f}ZXi@rgY4wO@->W2VH8m|ZbdpO{|eK~7s0Qn=&Pe<;CKbA@IHC>n;i|{9d3K& zRDNCePMIUG<#wI?^0)32KeRI|Bm*}G{r?KiFaDkcR?f8~L!KjU#(FQu14|?dBX!8; z801aH%N$M=%){(k28*6x$7UD4WWXPL00^sC2=YLIR5biS`AYkDHY)Sz$h9_y?UK0h zfwd&3xGMC_M4F$|Jv(hTH}J+MFuLYT9zV2s(&dW9XCg4o`uNFhv#F+(udSR1|B~2`?q)@i4 zJ^U-6Z#DF5<`9*ZP+>pL;1oz_XL(Up%fAoRu~@0X0#QWsKPkSU7n?4$v8YLh45aB` z2OtDxG=h@*Da)&K{9#?->18c2`+W*8CHy)QGP-Bg> zt4o!>PEJnSCVpjpe-FCQM+g1D@Wm-Ii)#Skuzce&Qc`s#%*$3Mf{ar^Cw8-?A$(&N z#~8=vMu1FD9E>z21>8a2PhxSgT7Uv}Bs`gqClu=HiBHa#R@;PSsOPrM=Kw0uR3eoe zl2Km@_5w_Efg*P|_^-L(FX2AjM*s2)j$tKqg-6^ja`?VQllkidOH(1Z`adm_^OvRo zM{R%zUY%3Bw;wtRN&Mx~GbIY>W_;%jLHDB2T3K8-4G>brebrp2tJxw{#(m%@+ zVsu@cY!Df$;|7e-0Vr3t)JeNX|3M-hPK{rBvfCoE!C)Mk7KI#%GqY9ecH#qP{qoBQ zQkQ+c84t9eB};T1^+W)i zf(>j*7ct7n3!{WA!A)6It_E~=ZvS!vh*>B2cH7fgeS+^LGXVSx*_cOb-PVAFjwug} znw=)(p;Of>Z9MFq&G+Npvtk@joPqLDEwbRvn>T(B%(QC3a#YS&keiIko1zfT;(=%O z8^pBLn>NhiB;f&{xpPP1%{eg{Ul}%p-bcvjTDyVf4;WXS4`=?F5pvaX+(KDx>dgT` z_Vz1denoI+6@$9y)oa!yFq(%EJEt#Rtlh5P>jx*+`79($vBO7(b*0?OEqyoS7OHn` z^DFfSkez)Rl+btANLjbn^e2cAph65R;iq%QoAXv1Ged?Ftd3FoJe77Y?pwNz4JV-t zlL8J;JlR;`!@S^|mDQ*XOv;YBXV;)kG&Y62^xLU^+&2y-ecU zjf^XS2b*5a{FEIFi$fv{)CXSoKX>1cR9JPRY<@JE8SxY{F0rjdAA4E8O=(zE zq?rZ^CZh#u|W5xaR3erX3c^We=5x5E^$|G3?au+X)4%nb3XvS|tRr2&+- zkDsUI_Ffp8x$o%Ac?EL)`^}ufvSTqRlV=u@*k?i<(~zpJGrbMVWM zk13GB_`5{^2C%1INBN0~|EX>k7#C{8Csj@{NeF#%k)$6j-O1-?>D_AiaC}@LvQESx z&AUk6Cfdv0lUf`=>k3<>-jKo|0Ok|f^+l1wg}Z$J)(&)RKSwnv(m723O-zpn14VQ; zkyd1cSu{SuRp`jh5^d!_q2XKy5%1&oZyJuAFq}i|R1YxId6ev}K+s%|_e7-(i`Zww z6u4!za7v*nsLr1gNykCO3`W)ASA0NnO@@L6XLrwj@gl9(_I@1pbN@2OVw=r6W)H9X zau3Zv6?-o%ghR)M@C%lXd|`uS8w4_^2ALF*F3nx=hLb8nLkU>0mtiVMIA?C7(f>9W zX_IuEbeqnP49bAR29k{Jj-gXDz{KfVRRBH3$@X-lsQyf%Fon^uYccksdCfd(%aJS9 z`agYIg`WeHn3cF}>s|>!RA$Rz9{1~L5lgu!^C1Yx0}0Yy+3^lhGlY|XcbZecKA)agoOEp9Y9yN) z+*j-UZ0?f=j{GI>&4WW>3~T}P8!_(%Kp`9K^`#Z6tNO&4zm1ja+29Sc=#mS!C&|7- zepnoP2CrK{DfI^DtvIuWnj`8m?f!5^0LgZJPB=G@_&#LjbMrfyIV1P!V|hW=BR%*2 zL{=+22DC#D17%*31*p6RRUMi~$eBlW`~fRD@!d%&%=)1iqEK%U9cfSLLP@p%mZW~0 z4xg2`5r81mZ=);6!$e2}YQ?C*2ulN+%;<*}Et;dR_;A#0rZ5TR$mn**lmw-qF26>d z+5pBd`sQ1o(w&+wiFZkd{P)9jN=FlwYL@+jY^a+{sCVPT1n81sU)*)Z48#Xc5CN0c zL6~21%UnDLO@@r8@oh{ZBmd0~*^{06#n7`M-H77a8##CPO`kCO#XSxVt=!JzEH@+3 zg9QN(R?PloW}&z#orBhFs8$p3ak2R>-+T7gmVc>+tU1*}?7Fo!P2lh_x^ght8lfI9 zHZW?_$;8_Dhx=0@=I1e#;p{VG9vv%q2`r!WgkDn^hnvAK2SYzQCgn+FZU|=|{zLdd z`L2d8&0BqtkRJ9^0`Xb*Nxm0!VT1ox+)k%$D@lj(GI*D*kpQo8J@T&@zIIA`jRgVn zrqMt8QySZFOu&mlWb39dv%ZaiWWrL&Y|hY7CxGqhG0*(%k2pV%ZZU`$fL0l2R~xfE zj$li^gi2GPnEbMQcK^Qn)A905Q3<`~_QBz{#K5Yg!=xwn7%UMyWu(*{_wlmB&B?3P z<#VKlCU~AxuMxR4#Gl+&$LYT1_`!tnVRoyvrVmIX_zimbI2hgW8P}TeHc8}RQjl@)Ep$*&J z&ipCkzKbkw%iZjt0t+WevRn$K4s>Z!cxN@ZHo2_ZN>Vn0MOvomge_1KmabUgx!@DC zY&m#=24c`~&9kv*;Ry354=RC>?m9@Rz#{S) zI)-R?Qpl|ml?q520I9(D;etj=g%2XZ8{9(aPx9R^YN5=r{P75>1S9k7d*V z#0Ou!7+h(~4!4}}mnE-brj+Srt(uH07XeGa7AiGEIM4RXgMsG7PD+2JnT2ZiF5vm? z$v(wjLjoyUNszD&w%DH zjV3Uugv_|Z`}gi`SGHotcyu?dCySG1Wo6onEvYg7j}jl=BnJ#^^lo_Hs|_sZSMWI7 z#v$g{Vh|KRCo*5}9= z<;zH(xRGUi^o_K!Crfci*cXkkRHWv2yLV&BV7?rg823PaoHbyS^6i!b5^e_l%MDKs znltATdnbfv$gj(c;OyEK`=wE1Hd@=Lf4Q^;UafPqhHaS3y4>*H4BAU{8@(S2tSsFV z<1k*OyX~Ly-ViQIQ@!DIao}V%T-8~&3`CfTukQU=tM?AWk8jME_bTcJ+ziYfUv<9; zOTIU8!}*aVMmxGXkFir8Ft^*OV$8$p25nw~FvjL|;F1q<3}jt>Lo?DD6n8NyV&X(3 zdBo{qJ7r$%o0D-(ZDDzwtD-y}YQ7UT6B!*Ckm6mCHQw=&m-+7;T|17!q+n#jRU}W+ zgvgj~$BmJ59bN=P^xeHx@Q3Fci`3?Dz(>!cGif4tW8<9lG<1>1Smmv}=ATP41KoiF zQuA{j#4dn`v?kKhYR`NUa;Xp#c(+hwk03TE1c@Q97=^-zCU8z%t23OpvuH6Jj8NVD zgL0{woqS^+4FMH#U1rWB9p`bMns8!vk&gp#IxXvoVg;yn-oHOPU_hf+uV0Uy@ZR!u zr?Xjc!rLg)FBRShX^?r+Jc*@ft^{tV>+SCDKKQCvBj~tSIb@eI_YqvPRr>(&i;!A< z8>&T13NgX)7xSBjb;8~wLADHlp(9k+p;!l$OQ2uNao$<*AQfe~{#DH{-?pT{Z0il2 zsvSfFQG(6Ks!~%YIN`vQ@sCe0ZX^2w;$AWD$#YZ4)=Xg}{U@7LC61cHe*gXVQx|(h zxo>y>xVrh2x?{hxB-Eed`ZGfu9$<}*(s)ZuNgtgA!&&wR&`ac>+jL(i^JN6jv91*| zsPofDVVvDw=;H+Pv|m&qnue5&ZKNq0oL0MA^QrH;UJhs#d6|^Kl+KMZgj;gqtMX)o zux4)Q!;n%oEUJzYcj~Vu`z++T;5q%CH<`kRt641VoFK~}hs;~I-QJ|9hZq3lk2iTj zv^7=e-+0Pu6k_f-=a;CFSvn2pI_Xktg* zVduASglg~^QjvZ!xnjN_n4*C zU0V}DY>ninUCCnP!2|oF_R(KjEOcm3TD*N4T!Li=uHqTAOH=VGQ-2tJr@72{<$et5 z{+j0-*-&%qo!0 zGRP_9RHN%78>+@tj|*7l;X$Y~Z`b;+dz*H*7xlp482)7-^~iFATB+1J3$9Hi7ZZbB zu5$z#&2H$q?!vGu(n8W@Y&r3u`OY*?RqL8&&T+q~6eyB2H0x=(6VGdPYhCDMsoRGG zS~+Z**N7e5$?xm<5yQQRYD;i6<_~F9H_6HtRqGKC5my#k#S) zK3u(4xL-f(#{<4HU-v!7Lb(%AE@y2{x>#z-rT*p@1`fG+;>0g`^#105t~UA3mc0i_ z=2rB4wi#P3X=Hf7EDURj&!{wveB6uNeJ|c9(wYiJX*bjZ67-+Rds2C>;bbl zFU_51KnAg4jBtU+4t`+?7-2R=ix!nf2BfX^L~qSJ7W0BI(Bpv5SgmnTE<>>8QmZey z*Yy|+PHrS(k(xppc=x0|v4!pfC*8mqUt5-y{WTSQ2$^LH0#DXt2xru;4iW0+YOu5= z@?BWVk~>}RJflq^o%e=kOM4BF+5vnD1(X8uvzgRj+=)e6L(F%vApvm+OE21B+5Qd`fhkX3Up z;33uaa_6GAl|WD|7TU#fS)U=gPf=8+uvfaIwTX9P8Cu>TIe0*IW>ubzA8OfL&5SXp z8nGce=l|lHOSHViR#owo#g!5R>v*3R0}-FBs)@Dp7K$r4k3T_52Z-`P>QiqnUBpMj zGb#d&x1Nx8r$xbDmYrtZalNf{X^5exrSQdBvDC5(15dy@QNvp}k@guS zDA`h{SLzt`A_YPyf5GzwpeWGEd`Q-B+P)!`dy27CJn@#%0%fBAAkomgIEt;1d;2UO zS<6&2L}mxDGsZR}dn#Zuk#RIY4wsDzO8w zq?bL>rG7Gfws2~KjwUhg$%oEx=CD~sP@=tQ{A2>@bh+ychxV|LFJUDh|GHG97cOUS z=f%*MWG6aKl^v8m6nD6wnr(du{4n|F>(1 zE_FCCbfZXO<|hWw`h!CL>C_2pKiAs%hm%hkv3LoRlBn{sFjM(2aiI5k{Uw|uFZ2gy zeZ1jhUhMvA3`YhW$911f2r`tgx3u`7*1Ru2GO>|{caw5HT_yzTq%OLx0wmc}`HEhQr$VG!elu2y?* zHMJBzsRPXUlNq7bC{TKmSAni)y|na#!mMZ?xxq+G=N@m@o~$T%QY*d9X%T@=@ixYZx*@ zG>GXL6h*ZWb>C8~ubo(^OTn4|!zK+E=IfmFN&`ezb6@$o=pB>-@5h)Aog?+Ca@=7G z7HSwl)RpO);TeYank`MhsAk4)_*%9MK5ly>4Q$I`M82>+bNFe*W}zi)5a%;H>h-!e z{jnf1b&qm)&_>rVV;*5b+LI^d5n-T#3w!w^HJ%@vBgC)k?~VtaSoWUn7~c=UAHR>C z@ROsXV+^>^{=Nb0C_JWHj<2;pVo%kJ_X61*%1RQp2#&Ydz%imuUdI^6)>GTIZ{ITE z9hs#>51Nypo2{k{>rwn~tx!s!8=?KopXnM^;E#|oU@k^iHA`cY&n!8W#%o0p2uPg>NmW+cJpRm(*&?~_w3N5{{8z`-0wUiJ)lNyID+l@wwE|l zLNkg0Xj9T%g9ADE)}ErP%;mFJV!V#_FAMZ5=kgedHb+~N(+-K`XHC**#gtMi#>*4(B zVc8AE?Oh+u&3>@2ujPuuYSV3775uMp4mvk=RqhG&@{iDODhl-$V~)AwZUU_JisKaI9x5*jVFiFjQ{_WfnL zRIxBqq|?P^M)P}OMiH(z(H;>kz7M#A80hN0v77YUyr$}^3!BITg~*!1H0k=s2Ij~d zoMdPmvJE)<_!diA9M+n%WD01;ojZ35B~2zH*AOg8>-d_N0j#4AiQNr>!lnK|Y9t5- zO<^mvw`k;IvUIM(+k3X4(H*C*7tG%(tRW#qo5sPCfLetIW60FzkMe9?7m3=ao<;Cf zlCp960GTX$l)uXWQ2BuH^%le)hTn&9dJZs#l+YR|13z8?Z$_Kqmd|KEH!}|@3w$K~ z7iar|YuA+@vnM__ISCF)obAcMQ7qI_RlCNv{g#y~DF}9`y>;fZ^QC4IqYy|%7QlmF z=B2Ga6sgc$eOZ}>D)ZyVuByn9$wI#Gi(X^Qgt_KRu_>yI7%<4pi4ORf+xS#Sjb2>{ zUOgaXSbOibyy}+~Or?^!5*ytyoyFCvs{VWJSB^Ge7fx8<)coAa`?uZy2@+emL zhXpi&q>x+Fkwj^-r{$^H+$YDxGemqf`8kqfh&BU-$|-0pPh02eCy}qlVei~IzV&1a zC~exSwMx%|>KeY>~)vmKf-@HDCWJ&2+pPI}`Y<)ufR+W)Yz2Gf2j_ zPaa`@bmr{4F;qi?kiFMsDmM6;l%mk>yc+BYZ@&??XbloD3Z+HY8uqim!!{XCQ~`od z!l)cicx&pB;Tf6ZKfk)6<;RlH>$`M$z5kn|dH#E@ivYW@g0Epj2e|OsyDekMYK%|w za+kVq!>)as+hkc@ff_cIy*BIDUt9iWw{~)~#?6{_G*xIhb;aMcZ0(dc0aO1d>-2H* z3qnD`Nfsc=%CLsdFn_tce++(TaxVtdWZ1sRQWS)rCy#jG|AI(){bGXoRe@n9OZcks z_lr6?IeqyFy8Z$<{r@Sl%IclfnQlYRfbsNkkWo&Wvw{~X-?KZTY5 z*M50VW6Sq&e6SZb6Kq+`mr5tEfcQcvv2DMfLAmt#GmpQqoJv+>f7)nH44gh^fo%j) zuK{=FY;dKSL?woP1iPQAz)%1pd{E{o*oiDt-v|pjYZbW1(h>;l{yU*lyIsps6O8Dg z(LU`KXyx}!AS8dquhz8$lXu9Wx5UiJ7zE9moIA@dp^C#KBmbKtTF{C$JvBE zLv&!E6+yL1p@m<}xd|))J9wYxC=Sw|cLVkU4U?B|WX82^wyFjnqQ2C0v5;qei_vS`YBRbN?@oc!-^$uuSS6 zy-NgFY5G_`B|N4=$5T-buqB-GG6}c%q)Th{?P*E}$Er?aT|}2%6b`@1+|r}my)mWh z4!MLKL})DipFdNw>yU*mAzqP6EVKKxOh2}z`U@O>p88vaLCJ{0TCYh_avQut+`r4ARt4^<^H1Yp1$51!vBITc*y`jlwD@X; zCMzaQ0(%0dY9Dkr|Lc{Ov&w2tkqUR=UDkUd3-&frxHZ~SYQOOSUP|Sp0F&9VmcNI5 z)kT0r0Mhd&1w4ja#YeSU{O+QqKC(Jqu!b$;lD;W~tptG{%HFq;7cgMPsUeeEcyU#;GcKS_ZG_iK-&pP&<<0|*A-i0GR|S+8Lo^U7D}l7xC>Moj zPxXZ$R+JyLil`gqV0ybDbgMXeND z4aVGalsJgB>c)4|AX`!P5TB6^_Mi%S40VHVDSyXtlP9}G{Q2479$c9i_rO`ZUcMXv zYl=Hv1!KI_;lPIK3^K~E8*pkY;E`mJmqR-=DCH4#>8SL!T!E7Cx+kTA+SVw;STb0?hR5wug&R9V(38GcGaXFI$Y~p#O&LwG9y2tRH8fN#ZSZQ@ zfe-;Uf^+oELEyfWA=O)HTSU-HQQ1XiYkKYzJPx1?y3pnLp$0#k<`!CDN-L|mLFOC3 zfc#hv=c7P)s`Z@W^F>>( z(PiuNePfL*uVRz2f%0d;W^2=QmR{Kr!+ow7sjH#wGf}0y>~oCuea+^ zy1-SQ>>kgD=;Jq1J14#QF3_hm)-N)cB^sZlUk=<0yKp50Vp|PW6J2h^f{GkqoY@di ze^$p-Tz*a$;qYemO$Je1LUzUutx+OEXY2IcyLT&{Drj}4Gado8`|XRU^5?j7CvH4> zI?faue938PnQ<56hvuEDMU|}?frIw5h&NL@MsQo?nycfB^S(*qYZ8mad_h2HjZQSW z5>sPQks2@aEYMYM;wB^|Jz*d0{5aj}^*w4c&ZJbH&Axz-hwvM%UUKuz7||6$u||CIX=MlPwb ze%8n0Cl6bW1;>asoNeizd$Z{OixzIGP%C2%NW6{}j@*R~(;l5(jBmmvva_K(TkP}6 zjT^qOqnw5b#=}28n7)~qH{Jc_o5$x$dixQ`sR6gTXG;RWerp@~ZFt3VH}c}ak)6|7 zmrhCWtBCI-a#j;^nY!l=(|1mTkZprceANK|Kfw?@+z{C=q+W9&!0kL)4m6GPM^=U6 zE0dz1y?widabZFGl9LynsyKPk=8-jrR}83`#}=}(v8@8d#l(;d#<`S@zCis0d1(8> z!(=DI&RE0V;CfkV2CZ-Xx!QxeU81B@c&TYGFhBi@3nhP5EE+4@7cOXDeDcDFy^oK} zTy$z&W?p*kT9+)+Mlcpl#6^jdXbf+p4o`edkQ2Cz<$;Y+olT`runs)Ye4I7e&ga-= z1UD#*fE?;leO%44iyDtjez)ZQ%dwx5e^V>HeYg%>gismdIrFtX316q0ir02+d|Zo? z(Y3&o_YokY9*%!}*5dnBxku2|pk}8Y&fb42Z^v0V<*qJ=`ovIBc)79+_10uAS=lJ2 zhG7Iaq;j(chd8syzn9h?aKO%}mL#cCIM8ozlK@|-B}~%!vd*jX!2qXiP30|igSzZ{ zo>3uzA5~{dmI>I<1MYqUpI3v}cvUXGE>f}ORkof|U$J8O?}^J}*t2TwY8)}T|1H>V zh+SHD$m!tjv-2O<%)m>eyc$C!uCZ)`Nb{<3Y#BjemaB*5J(az#wuhDz>C0slGa9Uy z3aY-?fQ(=ZjBe+3Ee~!Ta4>?hI|cAO81ht`g?ZT#S3agd>q2v`=(OWZQsYe?Ny5Ow zxf|w;`2PK(9#g^1VYNi!daA5W(ULDFcw}hY+WxekwKXUtYRUuyr+j$D3FIL|cWy7x zsGNkAY8|L+hOsEsF-zxmN*Ysrk-Ux0c@FMDt?!NgbszpvY`K!P#=Jt1qFUlIbakON zTz=JOuRkpjE2}g4))zQFg7&d2q=9Ox7$XK8+y|}}0a6x0PU&zG;H0nQxV*P{%&PQj z5J?TjkJ)!WZuq4%aegCe!Ka*|s-$v?d^u+}1(Q7-5{D#6$7Q^?9bpT>Y}$TN`Y{$@ zHQxn)UxK4EW#U+EYZlN?InNogpq!E{Z~L#-YTY3>6i-9|$GAtQ*Hb=-YHS)DwRH@X zbWDA#bLLHIQ8=tHi7S?Q6DE{CGl0b&+VSttu&O!B`n%2WSxQX~BqyqFXJ?oA8u6~V z1;VC~GF8N3g$IH79IW#iW)h>cl;k`DT$=QvRB+S(>^S06UKRM#)}{+2TO#hXm|bM+ zH7EhMFPzM@DGqwb(~XU;6recKHzdia*T?QYk4HC>mRvME#_N$&!zx3J);YKEn{%~F z4qp{|&Y0M`b(J1or<-(G{%IeQVN_;^$PS`>3*Y*EZgN6}6MT33Hn53sR_jnQ@KGcgpwvqxI=Rz(=FE%kdW?fu zDp4~ZS&|iqI4rO>rZA596ctLaVL^Nmd@0#hM~s$Vfq%gyHcLG=7C0`gk#E|;x3pVI z>l9V{7jqpM=ovLoObqhU0OGWtc4+X3*c6n+vf9$XoK(bmOAVS9p~Q`L#Ado)y|Y)M z!8yCE)~fV^7I%O11K(Dob)k1I5;L$*x6qappsY+Jp%*<|wKi+l{y4dey2UhWRE#jn zs}?zMsZj{#Y>Fcxz*RJIg-`vq%?7qYChA-`)@5Pj`;1QZJ}mpsW+!Tlro#MHwZtrr^H_#{$*EtC;?5M`v9 z@-FcwE^5{m0O{VqFKxrd)V|noueI|#m!Cdl>kA&KH-grO`u$R~Q`DyrB{cy@J`Ms) zSwji9e`IzaKO4_~BBAu882(%g&2L`CbMoi$ErYA;5wK^~Aj9w1EA}w(MRg{-?fUt) zka_Cg_Abreg5^qM9W_Up^ zCZvDIs;$kYDI=dM%GMT|KQPy|j`?YYm-uusDVlC{vXtt50O)z~1EG0%-&*p+Ni&Or zbe^PRrCqG~1A~A2L^usjk>jxis!U2tPA)mB-&AfuXJVg*+=YN`%ql@uru!DCw3nSJ{pJ*d&|@v0tK7m5dXQ zF-J&JOQ5A1M$i%v-^PH@Wm5dEY)(QUIwGi1UF}SwRTDvb_xQd)EK<>a0?X8yfk}|` zY>WBLou;Us<4<6B&8@pcG?cFM8R?Tm)WRWTl0^9=YYu4q)}vO*KZ-Js4#k4+oqa%> z9^L}S_gC7p`;Ko%`Z2LLEhliW^(Uw{{4)3G5Aid(R&9L_rJ)(>3sB`x%wrKXFZ=h! zMB0$Zgmc(o3N!UoB9rj!Q8|E8$0mUfwPBJ(7Bs8n=Ov8H2E@Y@)+E% z4ahU7Gulwx`8>bQ=WgM1UFv^insIi_M)P(JPWj)7FWa{ad4CY!{qSzU;aKeuT4z9MA zvDSA0hFpVMaT<1c-sWGIVAG@H_8PWRu>vC{5phf254~T&UuCe>Zb>Z`RA|wMwd%KF z3J^l160=Er|MdpV$GbjvE(9^#WRN|WLzcvq#y_uem*%{_DNsf{MR? z_bW(&(L}aYY1M`AQ&%7+HT&an^uqdh?-S<__aHS*l`wl=-$n2d>Fc~}<*2y+)!Fy{ z*KM`2?YOvxZGnOXJDNZs9+Z2?Yz3ndrUOqVj2$bQc?`D_gQvD2v{2#}LuLIP^j~w` zkk)kZ`B&O%)x3)9Bj-Xyoy7)>sZKU|0#hOWWXp>p;(++I^{xREnbT>L`mpR&Ih3hs zeSI%!jmM}(-k*Q|uP^NyT70kT4CcFpee?@zGm_$^)TrkV8#g(T>vEs0n&yCUUan^z zXlVYuS9e{9*i}+uv83t(Q3r@#C!gXbumK#mCiVc-5{-{$&-d-r*2l+ZMzSUJn}79Q z@5Pnd6p}`e{Le6gt8&O1ettzIqnH*Y?40J%ukUE0+P7|QacHt!XxzNwrJaL^v~c|i zuw4v)a&S>rIQ9mt^Uqvqu*Z!csUgRCHvl6hM_%!KLBT}BeL)Pxaet9Ah6KgqGa+Is z$$UqXY<`kM|8;3L8Dqvy$mf#EQcZyHXf~H(p__+k?Yx0Jx2Bl|!hQXI)GCHEsK*Pc zH7D`spm?%n`HPxmC7ehZ_m1ZG?gXDqrf}I6EDdPOHwkx|*WaGe1X%o&720zXWdsEN z4yYJf=g_;>e22Mjo9jrS2_HtbSA)i)+&@6FA7Pll=X*gE)-x@?$Z{N*-#PGVwG&%I zY>b#}Xwppj(SjA9QmmT{b?W@l_!K$u05Z*GI8Ou}qNtE*s40w6On3)O7TD35_W#yf zbg$iRwsr2a!^ght#)5_0ve@&X3GXE{nO-S%U}G}&O-Fcd(nSUS@K`&mVG)# zEuZy&Oq1`wFME;{M3$%aT$)EkI!;|%827^b0f3Ox`90Vxb2pJuARi0I8~5aVIE6qY zB8y2|%(A+`kVT&ovsfq`VLGO2);#Z37T>seCF^VjJ+^~1|N6IN>NUac3?nKVnnwE4 z18864gNL3Dt`u(>AZOFy@1#H@9n|Ikucw9M%ekdAN4`|Ei&RXMT_7!{b7!8tcG-OZ zQwsG*$+mT2q;QMC?qDi5lpGnE`6Ui-U5Mf6VoNqL?$ZJYzYN2-rbm#>gfuJU1!Tvr z>L6fEDBTbcX2A1)m;zj;vm{v1^989<50XJOdo+0y0J4hfKuB^D!MHuzjvPH9aCWLFG4xtibU4;}Yj1yFS&2`Zlvt3aEh4J@~(-!|9(P$a0xi^}Qt0%(8< z)6-INo@37`Edr-T14_}&Yt4sJ;MI9~gjvUkhmX>qwhY2)(LK9L<;qcqar$eO(2{Q% z3WMk@7?L!iaFQM#95rarpr{vPqklJFzN7OA^^f>tlVAkwGGLf%muRu*Q8WJK%iQB{ za1-iHvKVX_b$#29G9vGse0|Mw`y3I0~R$=-`}sc){mq>>qrw;iNGl* zDCx5PZ@;*iJ0i1nMj22pRtOCM9y5h;th9f}5FNEHy83O{9SaDle8JR6CID%_F z@<;Q%uo_fEDVus40){j@f;BmT4IzwMQg(1YNcI&+tzxJ{XV8Z3xE*S4bEHU)@{u#x zlZcn8JH#m}n;J7xCcFFdFi-n$Kb4==)n3zvQ3As!x<^(-sVW9M9Gtmt`*F%{%RNLI z-1wCIFY=b?yMligL24nB+GzWbh&f9M1VT&~5SAE|R3&eL5EYa-0Gfb#rcHJv{?8k( zT8e3}Plr#KFRQ8>6>x=STLJbWNd1B`Vc%$GQl_2x#ev-~oQeN;_7r$^ZeAd=+_G(7 z=UtWM$0N1#o{;57tNouMjpgU(GyeOR|NDmh_ad?SU%N3>5O6*4d4+AGnDI=^yb7-eh9xvaapMn3S1Jr3q^FlGye6j?uFoHqrPxxpS`PZ^ zbxF7d=*z$MVILDYFiz<}%N zSlAm2wkU=ILWHrIt>$hiQ8s1aLAzD8j$Ya5wfN)5yD9q@&3w1pH)zW*j$otvS#s8r`{tiQ7L3n1Gv?6dk*m{QFJFUVqM z`~n|Q`^l5hLw7wAzcK8~8+9A+o3Wzt-G__bb;Ke;s>hbHCMT zE>*W*oCtJ0Wy%!Wi$l(we%La#Lu`lk?Qx2HxBcT6b6X7TXPugq?~fKPzg}Bc?a#EN z|M0te*Q8n}XD_)oHJ~mRT$X<|*1lF4c;M)X$K7s#g^pcVi1CwQKK{Wj&);)KS8Vo$ zv0BPY>)e2&I|JX8kF&lwxp|Go*Iw7WKefYP-&@0Tw#2=6ebBF&&*k@x(eT3@p9+1tI1P~6j|h6 z@v&!`<*$8uhMn%(T7Jwl;^;0uWAi&?u_{NkNZk%n{?8KI5|>7+p+i&-FzLL8wex5> z7VJW9GFY}6Mv$*NWGzRq%x;v)4psC&?;!U|x?u7`n@E4HR1SO9WEN~zvQU;@ML8{A zU%;pg${>M!;_WX@^7Hk_iq#ZGP1NotfuVtvDD>ch^B8r3kL*eB#7Z$dgEf>$#us&n z)W#D75lVPu0Yr?}e=FG@ z&#jxcxtp$XIN5mL95tAW@1PZDZk^Ch5C<-qGgk$V-~xLkkhL5))O&8WK9 z2&JaO@&eaSP%tt*V6?0nK)IyCOnz#z+K(k*3@zL+!P6Xq3u_m2as_nLTwy&UI)m45 zL-Emx7*64$0wx`XTa$fZ!{=Qn(F^Z2q0ZL(uJBvhK0vtv05LxMZsPLuiu5EL65Tu(`~d&Re`~9 zDcAq|K_$gzrbQ45-@+aBP-ix_5`gmVr30^4u357a+anQ{YG`aL7#Fe|hPVCX_p^b= z5$=dY>B(IYZpuyD1XwGd;f1xob($c?9dTVtxt)J!6UqmF=HkgpZDwTt=c9oK=_Umm zv}!pLyK{re_bUIQqM2feGPJ>I8 z|Ea_{>O4Y{4Lz6NIsq1q?vVzD@vgXAR+?TQ%vo(6%+Olqq`M&`ItPgX7XuB>qeH!< zRLLdNui7!AL2ru{fUsD;%(lUwxylB03vDgmSNEn%yGT2X+Z7)+@UgPJkp#1 zOgK&V2cT7tV6WfzakC~oyKFS60s8yzHgtDpmM9inT_8G(Bm^HXO*-U*BPs77{_JAT z=*y>ntt}1nX1JLIt7krYwl}r1d?Uj;lq|;~A(rvBekEO(if@D&+a)+Xib2ixl{@mY zo+G%GJvRfn%07l3RQ3qc;4!&8@Nk=#ED<-0yfY>C+JGfEAuwUq-63DN5+hl?;>=+l zlSE%;NQT-`az};32o0~iyKP`d?UcgDfU{Hh$ZlqT2ncX)AQ%~#IctQ)gfUAUAD{}H zCngH!0pxTY>FkW3U<~Y!dulki0!FLrgo(+pN|+J|R>Jx=1qC$00dRP^<#X3nd_$2u z)iQ*eyOq>-C(m*XvR)B!p~7lVwUjO$O$bdQ`92ZcX-UUsMow)AaI?0zh`dB4@w{sU z+qCGD(!gy~CkR0yTL&K28>YhbE*w?dX%BeTQ|@9g9(g1**3osa3FW0~f{5~c0m==_ zvv5Rlrxiq?{-9naEN#SwAyY;5ryU8v(F^V;1YV(aP4;y^@{(nt1tafszE;$V!0y9g zV+dU*Hib$v$r>i9yuuC@fi#CTR5-%g(YX;5RE(Etl1DqvNcWsFnq)}lxj3FU%Iw*m z&;E`=5ZxFWRbbLf-VAV?iv>{;o-?| zJ1l+Q$wupx)|IV>uP#iSMu>!zW`y_K1989RUD3(Q4iuEd2Ish^cQ3M-sf zxeRLKGoF~-S!dr{`G#z_zT`L9?Ov`#!jfHGSUtGD7MDo1`OlikuO?2F7Db1pIQ1#GECeX_a3QugXh??aWrbJwKa zr|{Xvv|Z^qBDcT!l7yd zFJbJMjYpRF5t(8KzdU?5t`0ygHUjay|EGZv0Ism<4$7J#(94ZJyU(z82k(EN=N-L^ zGc1yk3gk-H=vFJ4Qe**b>T_u7UR-_8p7=b>i5z2bx@V7BgRj^YtCG#p=6{J`-f zRR@pn5GUf}dtTG4mAsk#Ub`2PIVfqOE(|ufMiW5KQPD|IJ$2%9tJhB+KCA-o!5BZ# z>?ukE$Y`PuoUn7aiw89xGGUDs7?3?D>Sxz;Oi2y40j43fD*PgG@uhEfuf3kL{&fGjbJ)@4!qQH)z65>>A z+33B^iLv9yFX7AzdzzNaS{W5>=MM{9IU?BqvUJa1Vt}3v{@dd|%ilb7+PnJ&tDJGF z3xwN1mG0QdlN*^j5d$s|q@uTzs5OE*+TPR~0%rcmFV+s2Yi%23xB{=oBglcHr-7*x z<2KGyJw}b?Kxv3rr@g5OQ;S2-e!liBpsEamA>*?h78u53DQ<}RYD^HPyrwh10kv)G zUF6Fzx82EatV_vRm(qq!@8heYdhEDybxoQe-I-9ZfLUUThrm4cq8b~qAcrPE5O%_g zhaAd&*{opCZ*4Zdr#%;BV1CD7){GaajaN=#ENBl-VO=(u#L?5da#MJs;S5M<9e1&?l${FPe zNt3H^s@5|FAwJ_!56U0eWeseaIP93Wl^R6=O2l}WFX(YBMg zr6u_$eiK)$jWf!hMnUt+`QHDfWDLSl&lj96%Q+x2-Z0UtGINi@Y8UYt z%xYbbIUZuSybv{zi2P02eFzR@00%w!%2#JDtWtmDzw#j``dDS?b!iK@Rx&7K@x|lK@(NfS?xIzuBwIObz6U zUUvmj4w|Z_8=Bvno0FY%6j7$=QM9jq)cL^|hbfnS?CUzx-o_s)I|UJ}#1)Vh-D@nW z{Q5g*$B0H#-;I&H1Fn@rmi$p91rwndn%0Dux0=Yi1yV%cpzYn_5<1fVIN0%vwZA#E z*Ad8r8r+y(gnHz--zv6b zN7tA-gQzz9ax0Ad?NtuPVh6h%sUX$>8IXB$9CN$1Fr{1j9jQGTn#p7guWiAz6V?5?KGy!0FDhW5*_ZcsJH8yu3@^ zDJw04mY7t~`MXjK3=U#7{bs=Eecm;FmWZFB`=3rC2)_3*`m)8ci;5AHYtaT-HEH#vUMS+1hMrA z1J5XzG+}OV&c+%ovc`$bL+o-GK)QkCK`Jg5Tmn3WX1%^`zJ^tAPv4ZPZXJB8ND3Jh z+WVM{71rVKR2p&-N@dAkREy`b?@PTo51@cK#qvdqiWG7;bksQl*^uT7GKNa>m7{NT z36?&7G+FCzw#U_H?YHGhIQ?+a6C$TlmD{NWt-YWtNmzsdu13X-95f^S>6;M7Q1*13^CET_woj^XSe+Wb+N|d{ z&DV%$PjC?>9-4~Wrk3JitO^md*I-Z3m9e%u!Hj&BrZCaFp#08*97N!ZY4*+ZN0nKD z3R>{pnOswokSSW`>wTr~L)b{1ICG2RF){ua>r$%5^Uu!ka2|Fi8v>ej$6NZUOc2>e zr?Se=MHv(8s`@~{4Il>+Ezgg+IUv`)=(i$KFkPLUuQ^RYSqQ4TrWD5K^#cvs3mS{} z#rCvho}Ju#MI@IM@4=1%nT?+xE$V3f4kqpb!^0OZMj2K@xm(G8KGignetLV;nH3J9 zOXO9onta2`^>>i76%;_d)4u(fy7MDjjEe#$Znyu?4=_YMcHRiO zFsz=8Md$69UTqxifzbCpuX2X7l}e}d!{p@Q{>yyf$6bK$6#W`p{|Ai3+M6gO7m$E& zbU3+!Iv-Ni`~nqy832G?s-Vf z-k0%#`3<(^Ij6ehrcbCGjmJW**4`n`M!h*m-uQ$(`Ug#TUOWUJkTM=0*waCVdYme$ zIhSS?Di)PXI9x>!AbtC~Ggci+VLY^W2;|V=9CCkMf88zFkHq~vD>yn6x%A1#&6<@p z4&(I6toeQ=ETC|Wfm&dwM+%YJdTPI;vj9llm{*Z{B9fC4QbkH)$^E2oWe!hRT+<(p zXkxn=bUpgKpa9%)Xb&;=2RZR1OlBAi6vh^*7e-Jvbf#8}T1T<>VosZU732=VGNg8R zuhcw~n!ax`9JW7hknly_+&$`MV%DH`Hu>x?@XkTxn2U#CvQ(Zy9m9ego*;MK1eSFY zUTESATHrtBynUVK_*w93+9bD~>Jo()y={ms`?fpTa3+pj-P-@=zAXg)&Tk-g5d~~E zYH;Ng9AM59(JPkiw1@cgIwK?YvwIv)^)Oe9opy~;J5A?C7JMM!lS_pj!bL7hkou@^ zG63q7&|eO`eSV`@tv$g_#t(a#lHz43&FI>l4CU!i?yFbpKz5?6E;3?u(Cw+;bKQLZ z!E55E?dy>%FPKQZH*N)>iM5mCJaOEroMDvwO5KMS(shrWG^ruk&%EsyWXt!lVQzUR zE7MMnPxnT)bugG7a1$={l}G1SUU8~Rvi8I11FwIb_@~R&&PBHs(dhvsIM}cf$Y%wI zO^b#rz{fv#Lwx0VIFLiA69q~1*B>3nS#x2x-!}rpDV4*Cy<^hl{kZ^tEu2RW698O}QbD`JiUv3 zcWHdVSGSEW{9Dz|W!?a<9DhoU_$t9t^eWIB;IYknIl}E zI_(KqM5B`{O5GKO{X3p$u@cAZ8I%%q2=lWyP*;EBQ&tikR+2=y`>TGDQ&;A+JAeK; zGa>W^s#Vv6q|<%R#hgUILJ71NlURo*@n0u!glmlDc@oj{*m#xe+3Zzg@~iDHd%k;H zvir$WZy*YmlA3wzj~qnfxmN|de`UCZci3pdN;dMTB(#nG^f$kKLliD$+yqs`LO6xX zd(Ai0$nEcpQq7}(>zCZYs^Ae%@aWmG%q}A%qXDSc_SRpL0hR?*6h5%M!iiTdls54_ z{pMx8L6Xor8II_l-nhy=V`^S$^rkls2OQ47(8|%-I%7}hmb=@>eqKzqQ|kC@Al~N* zXCufG&T}X}IB@fOr^1Iw*eO^@q7rk#6IRkTS&7)5D<#qx`&A0|x#Rz~KL=FmwL0ww z@>#DlGaFJ&?27&BHm6B>kA7D}J8HSj$KoN#&MHRxbJ%V|Wa!YkAl1RPlK>Ov@YA`2hi-!C&+)BSnzvI1EmvR1z$9+sNWFLn+xh#@G_diiO zrg$Iik>BZzf+gA4u$&KZgIeoIWoboqup`bgScV(Me=FtB@^Wan>cRmOEl*fSyICdVcKzOpBbFLz)q6~ zorr5lJjp_&b;*Wp17{Y0CJD!#=38-x_G(4LBI9J;l+uS}M7Fx)y9XnoCwKkj$dzn7GNS5i$bJ7fm9+Aw=!GP?pYhOE z`59J%(K2`3=0^#N3@e$!aB(%`KD_RtWsYrnjPJR?;iT{LUL&X1+gjdcqjK-q9WP1f zE^wKno407eC8VW|j{g0-qD6oDr8Xq?>|JLGS;Ff*^*YYP%25b zX(Q7}8&R30LK`U(MInizO)-`BMU-hJOH@j$HffPct5y={`PTgI-#O=T{yOKMGmm+6 z&ppHU`}w?=>v~vdfh)p2vqim4^*r~l)4EdAZm1RsK2Z!x89Bkr!JqShDtPr%(G z$~Ya(sbgNSfW?XV=ULvpd-v#XUcemt`_AX6t$!HOyBQMYU~`}JF4tQ9y%4fv4<_l( zBR`=2AgrzhV(FMg!3}WDT28Yl&~uvJ>w_Fq5d8$boUxpLb0_HjwQW}3(}Jrtg*zg> zs_8xVR>m?~qo-&XwQ`JtJoE;$d&`~jqY(4#5pqqtSM6`5c9>{mV!vGiw{RFxn)bI( z4CZ1VD%Kk%uf>viFCT4~@DTFT{rANLd6g6aRi3FU?ODWYIGPKI-GG6n7d~A3WAVzQ zcbu;m$jh@U&uf^Bzi=uySDU1w@1S|u8@Rz(V)+d|RJAf*B(6jsNKWm=iU*c`BB`XrFWOa7#tUCxken>*wq++z<_f_JR7{{YBW# z9ve9cO*w@6A3;K+S;re9h40d$=POtbc7m|0j@g*r7S2y$T8v(W15LM3myM+6q)|YHA7s}386({ zt4O=to|Sd)Ge+no1j~J_Jy<3kWVerHHidMKwE`~~I*Cr&e+>Lkz^d&<-#Z8QNtKS< ziE~hwVm78IXQticAzY*&H^3L`f`&ofFWcNsFG~DJfWy-_--v}Mh!sP9vIVtB=#T`9 z105ssjbRo4{l+ zg3#gu;Y9}v(OQ@P8C#?!rsx2HvWo1tzn?P5#+k|kas$*~Rr(&eE86C8PsKzjQSR|L z^iv!(gqRCm0vs)}o-)KHup{MCbh7istqKKi>^Do%(}6`1m7=1wpQRDG{^u`Vbnu@- z+_~s%ieH;}bc{XjKr?IzsI&G9z4jKg#(I6TBz(l;fboL0u?+2#Atbu59v>FEHSu?h^l%n_&ScXMg#2Bujyck_g8k; z)O1YfGF9h0>~zMLWx>IJv5a+fbvH&ExbuGy0~+qrwJB90iOD|o z#P`xiZ4;y|$8#?{XOY%qf;RG_lkO^u1R|Bjr57)bghISU8zlo8^YLqg-4-9i;K(RfC=!VjLhF44@Jk1h`EJ8Z5ej-1C2{Iwj^xFXuB@zakIGiKn<(F;) zJHO)I27IM2rvtRuP1sg(MFlq^#OR{c=JJo77M%;)-mb@0kebkeQNAZ+tEJ38p^&tb z|7fU-FErHfBfd@4sLfslY!rJ!K|0W2bmA>#>lo+;5}V|HeoiN-JYgnsgzz;+L;orJ zfLC`vq8kcjB`UcH`;rLue{@`*qI1Tz)dx44_4@bDs(U|Nu=4;l1(KmAE$>&yUo z11Vye1>d8%MWn8QJKT8vKS;|9RMx_;@yd#{>!Ype&qr7FFVJa;+J+57{cv1f8 zGx52Pp@4!$PgJFkgBXZc+t`KlhA^K9{E0?7gHWk^HxK<7f+se(VJra7YdVax9d90y5+@q z+*H21AmlLfYi^#LKb?rP!;#$d9iy6xI?u1jom2H{+u%i|&&#$y;3${So#Vs48pZbQ zg`V9kQGj%{W40H>cmbcYMW_GyrxT&RLCN=qhN1?An~;~FTA)e!eMlC+sjE`@dD*%q z|5h*=1NT?uncNLNOgz6rd*wL83D{xxV$_K@8s-~N5%5CrhYbP+4OJGUe@+YKz7GNz z1>kRHaH~ZQ!@xeaA1gVK54RwtbfS-ALqo$B@QTNSUW^)R(EHjFBnw@_ox-Cmd@5y7 zL10D;_~qKCpvQU3AT`Ey@{m3pyUii%w{?0$1E@|{S!wcLcB`Lh%tydeueac970T3 z9iAc6^k8@^Me5K77HQo)JavdcNo?mqeeU>k=VwRx8)}nB6AVo4&z}rxn1jLHSl3T7 z41x|LiG)SF3iQ6h|GUcF7F5x}+1&+x^SE>xGU-lj>G@WQHi2KMqV4Qt%v53#;ff&W z1$pd##Jy|3o~}z9aVJcq78Ml~`X$EFBEyPPr`APAB}}wVt5hH6-|q?9!#$LJ5hqf( zPp7|GPdh4ApCe0w(vK`w8Ka$DpC`48^fDF~+wRHAH;9 zI_UAj3mc^7w~tpw;^k9V)Q$9TGAPj@A#vv|-y_1=*=N&^aQ9C5DT2GlJT=H04oq9C zo?P(`>=1j7a5;7Jv=EYNgrKtsWs0=CF}U8eRuNHM3-@3sqRzsXKR{rk8N3~9xY58t zne&vY<&EX?WyJKnpEpF!{>JpVRWalhiU{Ylr~$l5+Pb^4vEZWkgh|a;p^3JF8Mp&_ z0ko1xf&At)lfy8k57X<@1Ya9`XjrUWOSSS_3sHw2;4p;vu=#M)MS>y^xa!GuQca&#|#hg00kc+9N`wd12_LZXWY?picC! z?eF2R24bClUt9OFE6p|vd(dhGq&WX^7r!vr5ax+)N?r)Zzn_SI&57op@57uXnI?F; z%e)wdARJpFW3EPQ6Cv6|L4ctSz;`1H@n#dODifm&yr&gf{qP(=RZoh?9g{aO4m5v3~&%2{cHl2H1}n>El% z%U4)YG>M~0z=@Qu_mpvBTzN3!v(Y%m(P$x_Hm;Cg@ON}{v}G%Xm8ffI_)sMod;0Vb z@2-A7Nje45`!a}>(BX|n>I1!FRTCokJbM`QyEy7EObsX;ES^aa?nLWiB8?y0<21!6 z!P1=8X^_9KG2=uuZU?`5sln&!9vV&xfh*#?HgM(MJFpZa1Q)ez^914j zj=Ji3-Jx|M(auY(@C_0)9BqXaUN~HYu;Z*X_-lc zE-ogXCNn%d-0+~9qmIyy2T`-10ucwL3 z%_IlQ`}GZk>-#b50zR=0{aN*2`{7G>Fs5IOK4FuOm`OA|6XAg9ij-pJy^vv-BC9TX zmf~i_>{X-#*13$;_(LT42PEh@u{v&8zRy_Zx+Yzli0|}j!o-;u`NEb5#ETkmV&I-m zA{!?<{JI&*#_S>Y69UE-J(RflMON>4yop$TBPYZjvE?m0;PPG_Zgg%oitxXLmXgYB z6xO17IeE#K!)vgWaG3g{_k*a<>fmTeUWZWN2+Mr{5t*0}fiCKAoVMt*P26{z#w#@t zHr)sLTvB1!Y*8o2gukI9eivP|(yw2?j@CtyR-rw{02RX zyXF3NC+yf{D7plSl=8M`B&E2Hrmq0p9=U-tU6b1wc7iDC8To14`nJZQhM5UqmlQ?l>bao#C|f| zpYAK`ji5$kpE${EwirwMEOZR$^Z2{`r&$__S;F+usAAoKr2Bxt2EV0M}kgQ zr&b4(FS>OZmMt+111Vhx;4Q*(7RSf+Oc4Q`=rY2~$9yuoU=t{m0Yb|`kbN~Qek@v(|1ot)>_-3vAfC$ygH@uP`{rF+9CV1+S5ptX4>q5((JYQN-2Ek!iD{%odjJ)@uQP^+A6f`Vy#8YwvudCUVbEQRt$59Ubk)}rZIqS z?W=w~t=X5XY-%8gmp-)Kj{@axr?nI=n@ zEqlh#wElL{uZQUyH&aaxcpTu+6R|-^l;XsR6D!UwA3_0e98qxI-Mbwby&CePt z<}M#Tjg3ln-klHJLbz^4G^RmstuC0T%8)*PW%kgDg>TZ-m#tb6(L<&?KIIFWSn)w1F4WHFmo8$j7uo%}zsI48vARPhN-6|7nn*28 zFKxC-GC+-_fz{tUy!PwYukS;Drfu7Xuyeol~z9?L;_E&hm;s0sX#571$vbB z?d81F)+oqa3g-x`ast#KUpJjUg|j#^HZ0qtHxy|1^`!SSY3VF}NtGC0HvHw|^XLDX z*3Lzl8iTfQW2{tGZ`#>ACbSKZqTvjqF`aG6q1AVNk0I1-Y^ztg5&@2)lwV%%G8cg6 z+rLu)(lD{ahEA=Ae_DNTN^YXOqT6%dY!4SOw2Tc`+|prsX(!y1t_wRQV*?F7`0kHx z=6mo2CFl2I9s9tH^stZB>Xx>3{{0g@9DbBE4gnpvHt24go^F5O_F5pHl9d1O;S@4n zA}z?z#FkN*&gzWnQ;rFm!>F(v$Bgq*1}-tl?*w!i;2Jo}b|YrUe;-LQN2*}3bv zG>Q}!%Gd}>t~kotPW}}jzHY6a=s5HM?#)z+#%)&nPRAT2w+)Vtp0+vIV%6ERXDw-6 zRh9#zHFJ{ZPHr^eOSMzO-S<1IUypRF;PARID$NXm&M`p!$qvSb#>Q(2ti1mEYu~5f zB@B}~{bWt%(eJeMeV%N+V|du%thY2wR|7$Z)eh_5DK4~C=eBB@nfQSCQ1yszgjyPA zW@eLCJl-|&z$j*HhOU|o;IsO3bd=!wQ&xiC`3n#5vk>rFt=l=D^LS*SC9`(R9xxXYQo>dE=mWR5{t-T=07ED|L2($S|{d zPt!BsjSXxor4#&wJUHQT$b@`oCLAblh_xnA>Er6^S~tX4RnrSai6*6V8d3qj@+GG$ z^AjW$da+*%>}s0q-1xZpXBxRl4cN(41OyfN1(VrU5=hS*-`NSW$%CBw-iLJ<~%RhQF+}o6Nkja#OJ~NMn>fk zH>pYm(5-*3PGyjZ)Ckim2@Dkmm^hu0!LBlPS@Z{<~NmKc!28ng!>DnU+GR8;H) z;qQrOD({BCD;9F8Red|=Az?CMA-^hyb^3@BJq8D368{O$`l>R zlc>}Z{$GGPJhi@=nJS=%yJ(E~xNj^?)w{fZ|9(9u=a}9~r2%XW0c{nU#g%}kykXX9 z-P6y|k+wS-n~yX^)qeQyk0ZV`4jipV0||vCCNz?;R`U}wLjV}chi$dINkD-wYW#U=j54>hM^oA536(x`OVI5(5 zs;RNdVd93iq6x!V#Uqn3xn;W9rpZd|Cn2=_iu&q~VhEjHhMIOEW}v!$CBgiKM~`%g zzU}$V0O-^0?}x)sN0XV7MFxreOWLQeUhTZN=9A><=hqgOfz@@{`xEJfFICoM*5;bM zdkxrSd$F=|*yRYhsIrKN}aUOx|nHy^m1`|N@hJPbf`p9(xH!| z6b@`1x!YB9VgGdNtYgRWpa+g(?_%NA2&L>1HSTmXJ;uK|t6p-YWX4E!Rl89eQd6~| z*Z2}6`-}PcA3ndDz|Xtx)svAZs%;C-^6~M}GczmK2#5(hWEZ10se_@k`dUp!dU{oL zbrjV4i>1N#`VC7i&y%$XnpSo7@c{bgh(jTwv(6TsHn><>S?!_>ad1!@5I_OH@qGCC zkS|Jf46hpM>!xx0;3ReA+zj1TfKZGhskDo{;Y;S>R;oRQq05|jsFPHtJN}~!)~EiX zHKzL%Y6{5{0}wLECuHeTNdkOK+o>H>ck#hIN5|iPzd0oBakgJa!X`8FK|5+Rya3wz z21Z7sYRZ+xcd=t1iFH7KGbdlSajQThpg(x9$GHT6S_cS21>Rc}YOP3Gg_t~XC{QMQPD{^ z&}{dv+*PEA)uqq#7J|p=WxAUAF+0JLXjoR|{juB|R+4tcov(0sT z^xvakxIQPF4ji4pk7YSv1iEL*a_zOOmmZB#Q#-pAxVsB8a*Ag8o0e-R$&-7C$BkC; zqmE8ad7?$sghxMLmK(Iw9jqbs1LO=FzFuiJE@c-9Ihn;Ww||qx}$!=AEH-S znVhUi*~@6%WD~-MWvwM4z#V(qJ|hptZNrWo%9KxXXCG>OjIO*#uowlpnR)tYmg~L( zc-(l*>JuVjDI7=i`ZGmf{cyl-rO;b_HJv-fwFc%!oIE**JF^<_E}Sw?8RF-azqRJm zk&cFB7RNTdnV@PEy;i}&!QlxNLcClQiatpX#S7@^pM4ps)on?F{fgm449^97HFteX zjH<_>J9e+4>o${glJvxsgBdes*ezUmLa3}IJt>;myCt_&V*-)pTEH0CN-7Z5DlRTA z_r8vEbX1|sMBVI1RI)ab)t>?T{41;%7L1MPvGY%xq#!1<+^K&<I=NN4@c8xcYlT_8zX1o@Movejy zIK5<6OM0;I1|lVKvjzso*x4cJJUeV;=%fZ)0myR%wFFR8R1Rw^V^i_qLGALU7^MTv zjWxsTvXeG%zB~Cy{gLVC`;gl(loXgIX)zPiO8SJ z*x-cLfiR+ReF{X^gv69^JUGVKh>lVLh`a#rQ*Ct;*v~JS8#^fs3thFgbxKVpi;qtJ(@~*ayZ(y|^)v2KnjtF5cgPq!sga=z7a1hGlIXy(3M;wD-K?#3 zFoWawk@4Rp3)o$Y7-Y89*aNp`KuiO*x+bKgWM&-#oK@ubcgCf~>88Ehj;Na2f-d6^ z+=gn(DGRy>YcipcB*Ri*Y);xE@;G_-?|0JH)}GRBWpu1Q-@?f#6o3y084x9Bx)>V+ z!x^2Dq7@t^*JtXu!h1xi#~`rIF6OieTQkLI<=RND1YHkwot&H&ELdO@8fwjb_y9z> zSX5+uIRL)wJ9^1uum{_d?TwuB3ESgISAN`Z$L;08$tRVe`y6kB4TOvy{6nq zuxOCs-G{FzP5^uecZxiyfaFT9t*zO`bU9PC{+-&xv{Vnz0u5z6f9^x8_$QFH-Xmmx z_QV~wfw$bWIpYB3olGiT55Ks@Qro2rA*VJtcmx$OVrqf-C7iC6>1ZkZ`}d`A2A%N^ z5yT;a1?1LsXxV(?fH#j$cNAqG{Pdz6#9O*Zv>?|;`uG?M7|F?bwe5VXaNYxFArmgI zXy>CrtOKpHq4^33G@iWFN-U@+7N?iv&KctrR-$P4tl4F z4`$Z=Y{-CyuW#4kTk?yG+t9w}nv0jLWar@Ejj1`cw}Ww?2?;AJ9~HGc{Er@TtoY7!o%JIP1T|Iyi#wk!)~f$1vlxx0v*fD=~G-*^p$h za{bALjKcIK?ATGp+TF2SV$(cuPIf1r3SY4RGJ26ty3V@w>({vMi_Fcn;?~RKxk&Jz z3-Xe6aXCs@23sgc*u;(4T9$()QzTHa#SaEB-X(WvzkVv1rcE{4zF7V@atxz&8kwvfDL=Sa4*b;X#jVrI?r)!J|{g z-sjtJBI~crutD%Y;Y>So{SHRRXH`|JGfrw66Rzz7y%E6=@^VgJ!jQn8?(Xi=_t}$* zAj6-6eIe+J@n8Uz){-zb6ZR_4@b>2LDS!X*W2^Z<@lmQSyFTUQN0RYQr|OO7_MhK@ zJQRQb{rCE75G?jkm#fznI0=Nq|Z>lRr$*%7@+@AE(Xw9uk(fPMz z#4AUyRmYN$B^UBPySa0FQ6nF~9B8Ro!;o@WHN;_#&d%lmuIWoQPD+p@7WgGc@!}$${sYwwdIJV%N-Bh2&`jP0hsI-_ zKF^!h7sSk42&5$tm9YFB4f&uB+0e?swCJ^KO7?Y=b zQ1trs4jvv`_IaySmk_K9NI}~FmH*{!UNUz={52|KSAHaH#V&vg?_x^?3bYZw zwFNt~_)W!1vbHvfhTHwAd|ul4-3+YTIQhfJ_AS3{U)PYKW1f5Z^fSUH6KXvaH>zr1 zLvAg@ODCDQ$WnY+r54-rvv&8E{&Z1Jr9t^hB9V*4agim_v9p;flLD$8MqtjcraQL+ zGNlE(<-{B58w4y|cq`S;bA+ts5^?7|J@q6{)Q#eRioZfop8Y1ODw`}I@=iTCKC>;P zS*s}9l^F2U!OEH{{~j@ICteF5wyWINJ;A21l8kCORgKnRO*&LHr&tE0?=xsQKdPD` z#Hw0D7Wa}m7>@sF9C-nle4VHZ;u*l*N|dq0kCKlZB`%(wo&8SNeb-UZPdUG0JFL_} zR6VUxmqgI3f;jv7`LM7?e9JaFK%#iub$G6S*Leo-G}%^R?&EX1#9ZIWVB^u_A&+KI zP`@YMUjEX<|2l#OZQ8WyKes|f5|8fT?5sp!u?iMl@B}anoB#9ZiijXaaJk1rckuMb zjGS76cTV_D;AM(ckTJjI^V;3*f?>DFev1^uGo?RuJaAjTTQ?b8_?~!MA2-l~1H{#? z@e9m`i678UU!vNz>u<}JEfXI}etxLiA&=X)%r4nsjDGFwfBv-3d*J@-kpJh;OD6xP zA4~tWzb&8N@`t(7zdQA77sQ_%T(J26`J7+>5F7FD!vE(h@nip<|Ns6l{R~eR+5i5a zmVeTgU#IrJec1o|qrIFi#(pFiD$*CsVv)wbPhc7eOXdZpeXV6@=hm*}EiO3%2+*_2 zFhB-**Tow*)C7p2F-knvknSrNZ>hd&&lxA*v4C=^|N5|oj+nadFSgDf56J*T%)mxw zPCGkTC@@eO>JTcKEwT%_2wTz3NexYk_iGc@X4LB3eXTIAnht;c^_SqE2$1&NY$EEg z4*9HrF5&`QI%mMe*7O~}w2J#EE-{d|@#;?#@IVWv^kAzlDF}233=zN$Y*^lCu#V1|rBEZZYlh@201YhnFQJkR;T54!H4Zf0}vO^EEUu={%uOT*uNs z|9xg?ZApR+Sy6Kg)-sB#OHKE0xQwLUlRK~mMHD2q{~A_bXGA(Fh5$UDw<9LFFgtDe zYIIRFz*6Q20Js91F*_p?4Ta>bstHvyTS6y;Eqmwk7;6ljv@B+|!)2LKmk#%(pPwJiGSsP&v9Y&r{pQV^s+TV_Z$7>rnqPbH z&>@)qSJt!JDl4z%>j^Z*rVWb&YUh=gtKv+3VO5oNEEB4i9VO5r!`BI=-O-#Jdzf|= z4UI^6$?D6?Hl+#ha^zft65F^MijXh4e$;CuQ#&Q%hmfDTE%&iaH@Ov5T5|H+0Qm$B z!O_q7c4_@+MB?^H1H9(XUk@Xl|KLGq1W2L^hz&%`Ak%2^8)^MBX?JVdL#lrH`s1(_ zNBFZapruPh>QkaQ!CO0;8^jk2&97{DURUBZAq~4Zo!lO2lFo0YGxbjZF&P_?8t-DU z6KNdIM=Urn;~~g>MW%KKw(`{&mk=t zj7WNY#5W+~S_itH85~Ns<0D14+s%NxYLXpQ?d;7((W66KS+oX1z$9-3Gi8?a!32*~Ri6L(pq z-MZzr-3*41*g3yySxRmiUCVd_gjCneuhjEJ$3ZTJF#iX6p z_=+`$4)sPr7J(_WUiP&e#FGW?{rHqgRa2{tC-1>X!aV$b2RT4u7?lPCyd5k=F` z$^cv_=s;3*AFLh*Tl4my=L#$=NE@7hk>F`>r_6{AJR7W$U2d;N2nM z2Xl4BxN|S=h*-viq$HmOLh4D~zYINZ+F@xf&`1=(A5W@Xu-CB8+=c7fq6JVD`5O9q zDN#(+HrY9fl!7P7tO9N5nfu<;#G(-_XK;O?qs+0u3&{2p?T)~zh@9SXT*i8|2I({t zNI|R-I&>L7z}P7{(U$P!l;GVH50q1cY`wfegZ5VPf?y}eE+{TG=+&zoDb;RRe+a|< zg9aKI$x}0DNym6Cia zeTq?+U{Oo_g^EQgKv4R*L1ava&6@V7^QAF4SCZ9|oOi?3M^&a=d4sJDd_XmIFDY77 z23VJE4cyps$ou22`DlU#btFCoV$3s*sL)2xW$*&Iy@R46L<5sAOa=Q#(qlehx0As6 zF_Mb1va)`kK4a&LQt&co#Z>N^ycF>KlIp5^l>`3EvY!VbM$^Q=jidkSimu2;lc`=y z603Cigb5Sic;ko|%1>tvoA6@JT5>8XwhFrlZAp!hgX>5?#=Es$)ztK-2c3I2RlL-| zZcQg$Z83?T8<=%t3oq5SbLY;vy{)V!2^kQw6ya05)GW%qn!J6grs2wTdLJS*~7{(&oz}a(fch}-0){=OT{k~WKCtg94 zh$eVeB9nGpg7GFA{H-;7g|&3)n*9}WtmLc6>Rg9wJNpbJcCnZ|w`01)yyIsNLv9Jy z+^?CARg9zFF|^f(YoB!a+V(K@a`N)pL;sw%-8xv!RsrKLR5EoRE!OvTQsdyZi?jFs zP@hCD=BD2P@BqHF`kS0Ynb&VsX{WW|P%;MX32SZ&4$-x`ijfn4)xRe-nNKvV(zWY) zgkN6h1h~~Goi4DCk+5;fWN^66Acl* za~O}&luqyvL}89&#uO54NfMO7QqYz}rd0fW;I{37+ohLlW|qF$Oze(m4=vrr($dnZ zV^gP2{j_a3Pd)qPx`k5UtT_8tl89l!83HDOwRzg);PT#)*R7hi7MH#LzPhrbQ=+Gr zSGV)srGx|`_PWHi^Go*+-5a=NRf`Rbjx7e>5t0e`f-)fz4IY{qRVWsk1!#sJJTlVa z$f)U?gLOL*73tObIqY^-w6MssuDLcQ=Wg}FcQ(6SWiHi+I!p_yzFj+LhR>iSo-Zqa zGDowsZ~KLP8Y&YzWXZp)F=HP2XFI#*fxr>*b-;u*)SyZd_~p5CJ#E-8l$E_Z^rJ-Q z)0=x1c;Z!2it~rHQ3m>=ny04kEbwua;z*O(!|155V zHfq0O-?m2&N6eU^FGL%Qmd-m~ZU4eKcJ+%P?RsQRnLnfnu+)#1kQusbg4z)3R0Ood z?dl>uB3(*;!6g!j69haW!~5K~>x&mJ&=Ge5MY6JgpZp2RfW)}r*Y}V88d4SbC*%Px zCkmAF`UE~=9s(&Lj3m*(HC^G?98sE_d}XHoZ#tIY1SDE-En#Jn)_eOp74Q#QIC8&Y3%GYx7dSog zNbf=S0u4S{;)h33_!0l3^N>F$n%5}P>CpdH?WZfbtww$tN!EP^7AKm#iHe$91l?a+OI5YCJ=be9cc<|&6uJ%(1O6G*C`${G2+Y${ z{A>|k@3|Qvg}g*cqBSy)Vw5%LcH0W7fbD9XHFKH@ooqB&rPYW#=_YuL0W>&I?K5rl zdHQD{NI7Zr4r^_y)P&7SAY0AWGgYK5WzyGkS7q!B)F!c`Ne2y)kr%2J)?P4=oH2x3 zTJ(3OZ>9Qm)G^k&#jx8H?b!32gfs!%eRN&V@b4*u7nx#~=PcOfx=&6LBWI=^{x`H! zYLo5J9$|tIp8&3lkO4rJ#^bBlhMhDLw_H#qg*RW-CBx}O5D1&~{rSdm{lc0;1yA|v z)hnUawX>50QeP)ztG7gy@$$~0&gSOkLLef1xFGIP^!K9#8*rs{WwX_i4^+24<-dzW z8%`MjuojITA?t)>aB#?r$FKhtN1!&(3- zS`M+(l8}J(x;1A&dPYV7*r%P2r3&j(b~6A?lrjK#8)lu#S{kemhTj&71hKAERaKor z1wk$b4A^^|wlIH20%ScFZPuhqUyCV>J%e|g{g9$?JwvC`#ZyRbJ^W9+fU4dYet;|w zhyS(CUcgf8Vl;H)KwqOJ6=Hyr2{{xfjgnME!|}XtzJN>F@b9d_!uxax-#T|a(t-ol z)um;et;&Yx_Td$$0y0!=quB;9W2(QXBYdTkf&ZY@3-Z&vcEIiYyu1;6-;%{bi&ajYfd~M9(H;tfA ztA?;|2xt|%MS{yj4a>Eh_t=g_@dO`wsF&M!X(p$F@buQcjbnfCuWdvkAcX=VkHcQw>7)T zw+U$sQX}CEHf`F3x3`7q>o=o7K^y>9c+-!b8cHJFfL(SVi-rtQ5h@&9_5^TYGU@o= zd7(%TE>JM+LiI50_5}3l>CH#39Bg9)(2#*az7 ztO6a_$jIP1i$+pv zp6p>U0oe7CCiJ^R;sep?4vQ8(Ue%@PSWZq6GD-1aLMFhzjf6J%pNI)@%8m+f9Kc#s zWX=8!9p_s7R|bP4X?kgc5UQLJoT9QsRJv=^`Z^fpV{2+Ev|(((I8V^G(`UKQ<(WVI z{rv-|022g-2=9*{7wI474A>HW-Ka`==yUn)vXf!WD?~m4rfSWqRjmYbfZRgp9*4d- zioe^9mpK*k#Z)9^k*7;=`+_GRmsC|#vxax^#-gz(Amk6=>kF4Jx8<-SFd7M6vfxA2 z2yWJI;Y|6$#EZ~>pqrsnOZ%ArX3d`CJP7Yq(3F9~1p=g-^}^S;6xnT) zKM(6b3wys#5nnK~>1!XJ+s?%Ww|-RJii6Av7t2&>oBf86G8 zh4L3#DA+G`Ls+t>&MiE5)Q;`jbp<*BTxxLCSMy_Ba*+>(?b8*#o)+2ta=H5EyU0rl z-Kvm=iLLI{i%KVXvKp-jD2lqLh0b^#EgsI8>%4E|uM_u+V&YGij~9O}$}ce1^5_3M oPy0W8SoHsA6!QPgqkRw4QLwr`*kz%Q7@^=aYOLL1ThEC90c&<4@Bjb+ literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb300/gdn2_b1_flops.png b/benchmark/linear_attention/results/gdn2/gb300/gdn2_b1_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..2e3e3331f94ac300f4b82978453bb63816fa7b84 GIT binary patch literal 85617 zcmeEuc{G>(`tD<>3=I;I$WRH1XfltDD)W#MNu^|_P^LsA848WaOq3*~WXupUMKVN4 zl*}cwbA9yg{oB8D*7@(Ob=ErTSbM*_?bX-we1`kJ?(4el`?{w3zGX~YOcX^eJFtJZ zHbpUPq$oNUhQ;{Kj>%O9_#YX^Jx3gMY|c13ncAPG4x2jKTG}{To;%5R@wC0eIUDP3 zViFQ!5~6%(9UW~QWW>d-{{0hTHumP?_APW1_!LIl{e}({#lcGcOJg2YuT9ZV)Pdc~ zy3P-Oes(&-X3)BL`0^dWJ<7XR?kRkFqOhGM{g1KNo833xl(L$gys<+t{ru1CZ%T}x zUGeu%65<>8^3>i<6MrXSXhmMcoeNH;hO3p|cr=y7R|$_*g`1B5+}69FUYSK{_kaJ2 zGpnCj?&_8kF%idFv){Mgha5493sZVy~tU2Ug+oeLGx zqBScEd_QE{xB1zn-xj$Z^XrbkzPKks=EA5f_mhY6mshv9xA)b@GynMbTw1y~^L*2_Cr_-hBwSmX;tt%_d~sGQ^Yl1`}W0Z$E)o>e7L0Yj(Ax6!PYD* zRa@I<+Oo;IOxw0?3l%q6hv$3t{JHk)6&KuGAKUj-^mXLt*I&GM=vqO6lF{pn%Q`zd zsYi|HH_6Cw-4Q#^YL=`SO?!XG`Gj$~revK>4;9?FHmN^U>aC7&c#-C{m!rna`s?q@ z?NwDvM@B}Z?7uR+avFL%HaB+rsqwt@MvwPNmrc%GoS$>!mT_dDwz`a-I&$R5*K?My zHYobqbmZ}<`Yy-*E8KV($=oP?;Rl=b=U0m;?4(FrTbtv$l!Uy7UWr{pLk$hOZFz2M z#ZB_wKG#KO+*gMB)3^k|G)n8?{r&-f1=I@FN$p{FA!Q6s8rZg$#v zc4~)$>x5^_o*OZHZ*rY&dzoGuWl~gJY&SRKlIuRpb>_^OB)yc?44h=+eS+cH=`FH!!GMK7MY7bh4#m5(|A@yVey#O?y_dn*3k)T;gxgNw0d2oI6gjJ zl|+^cZc<9Nj*bnNIy(IEZVZBgf?eHmrz=7&TaO$&#!PwRWq-sOyVq|o zWnvd$puFEd*yWK>?s&fI?R8qpv_02Fr1HT|tNR0!drVABIHm35Sa^BUF7Q&`>Jc)+ zFJ3#3#MC}LE*rf+L_qC=!)S}kpTUT&77aqm_~~Tc_3PK0_SHsW=e=srg{@n1?#@YZ~p&INjvGbN8;$4AxNPp5*C< zxto%blB|oDa`LskxomKn_tk`Kw&Qp0k<7RsiKpJ(YBnC)C@9F(T^3|8+58k?r0B`9 z?El!>%|>}w5Vsi$ihP(QY})0@Pf70Iz}k?cMfbbEY2Qd!Nyy#1e!VqO-CZR&hX4F2 z>#K`RxBe_MF){H-Gg?Jt`oUIHFXtuNYHCa;&CEvn>ejcty1=ZgtbBKu%{siPFOTfS zM+k(?rP+vrhwg5ri9dL!dL*4k!SzUPaXTUd?(EiGyUNCTYwUhB2zz^bKQhU8#}gGG z0NJzU)w8?*a5Ym`-#$z!as06#yCb~2mfy``@&4ir59-txq15#9?lTjIW{6Ef}mYh_(GA)}XD@9(toM|3DeOhYhy={CKSmX`L}6Mi{{BUs#l9oMovJS^8kvlZjgz!uuT{5t+-%xH&_hmm&4HEV z50ln7WnIU`#l*zCqN1WIE=fp86yY@bsQ553Qr>-iXMOAspIPb`6vXi{%|x6{%;*Tk z$H~RTv~JxxwcnC|^m3i;Xgs%`e9Mqo7TkD;#N%UQgP3sD0%kNMlTZ?sfrZM3Rk*5qn)wTWA^zdmTtQqdmsa~A!loU$E{ z9=%_i@8K~r+t=NFyIY|FiC`rbVEx;-Zy68{O-)UuU+Ktg*>d=7LsAH(gjGtl?XBjC zmwET@T@mtl<9NDx%{rR$yXoH!Cfq>0m(o$G?iv3bb?l*VXMgk!2h%giV+f46Cu}C0 z9t!iD+mFpDc+BOVFg4{DJ^o79@8(TCk&Y~@2hMk9J!$B4$4$?l-%PO}4R|-DnQRVy zcS>?(it>EsG4K8)Q0w;jfV+1&qmlb)sX}DEBe~A$EF2t(&j)dYym9OiKKWIz35E4{ z4Ygqfe6W*Y1BiL-3i;(xudBW`Ev38_=O$F6MWv*U04w0pmaW~Xn-kQQRnt~>$Ng_Y z*!)nKM2)<2?(Xh)x0x@e9wFIDxz9-b`mJXoaPPqbGb~cy=hwShvTf2D=a7iDo~h#8 ze*WXq!}qrrmId)jxK7y0xs2`}aPO_UclE$+K~u!ayF0D89_v53AL};wDV^>@|3^K; zGv?;LTaINta-W^rudZI2tdlrgyh8Eo#LE|*-Q8C|Bx=MWE^x|TOjT({V6nwD_|)2J zhWJfzLQmzLm7JV@Z{P05H)GwV$GM(_MIf93KXhYnaH96TyS1_I`RRk7nww8I4}N~_ z^wMQaP(?*$c<{3NUoj-msad2?l(WD6JQ@9gIJ69e&SWCY7?AcZlX>8haucW6- z{AfxG5oB4282{Y%~wZ%vd_1Jgy$oWhI8(A5@RY%;wU7!;d z7G_{zQ0mBYb6|@_Q2mf_xcK%a4Ze{5wiCmqj^FF|Sbdhc@T1_vhgApfh?OAMI%7Em>X$O!W1XC{XX(wgn;Rx)|8&q$SS|9{>=hNE!dLZD^rHdq z$srUo$qT@B?SCOZJYbYfApJSf{0sq7VvT-q>Tugcu^;@#Kmp!v67CE z_>z`C$J#&Cl6la|WT^ekDvO3Bev0MNufFK#X{l$_YQ0?#cbj<3?kKa$j1V@(FGrZD ztJ6yU*&F4tnqt6ubI8fb(O5A6!vy;B?^^$Df_JQW_N0hGVpdQtAKrWRWt9 zcWbr{Ges{iFVDYyyKeH_i*p~I8xKp#ENB%d+v@~*{HxfDx0`sotzLL z;-1E&d~KNz?m_={_rp1_U*G7B zPW{O9@vHSYMMXu41i*r`jlP2RmlBGzyB%$u7?bQ-W2i}aWrjjxSA$PxB?+0v!n3T{(EneY1l98?~TuL*rLkYn_q>G&D^ zQV!8ATf$nx#Z8P+a8-;hpPrrm!>q5bFY?FA!b0TG{q4f7WeI$>jX9bu^kjRHamX&w zD^FN%+xYdH%M5GZ)a;v!14q;IQ6?=!^g_feicj=d)}hnWU%#Fz#OjO)ZgG2pc;wdRrFby^Z`aFJEJN}-f?-h6M6dgkHja!c^B;_)?fqGeZak){A=y?_Aual_~qFZGO#jjd>2zc+yMidAbC4fR+rJO!^ zAWCxJ`CmGIzg1MV1a*@-MK~Qlk!cj? zXEIbCV6}<>i#F@0D!IDK;mTY=b>rVND*J2ulTCkqeb*uB3-=H!ujIR&=gXHbp4cT) z$p`T&?u}A#duG$2km)osi!vVPjGdc@hrq_Z{{C)UiZ{N;_bF+l{)mKMH+5Q~(k`NNzcAIwY3qKmJ>=r0QF14GTy1=dIZZu%`wQ<_Ve$%dAdn;ondxYCP z`R=oPx8G?Z0y_48T!uS=I9h-ZUz={WFMg+e)6?Tuk>P+VleObXO@{hzy`bP?Z12lU zzeUs|<*tWqHu%<9oUrPZ9LHcsJ}>#uHG8F^3a$k>iO*1V`G9Sh zPP-fY>|_bkp$Hj0)8H8#t+UO~H$FAW@d98nMG1V;5cPxIK+BHp)@?`5osh@9LqIA6 z*poQ_QPk*_SFekh;OC7Ob_Lky}iCF-KuqG zRiyke{%Tx~7~I!SFYVTMb$8c+tXaw}rI~DVxSwXt)h;kp$V;B6MXDoYeNkUr@A>}y zJLfKI?zy>Xn;AehltvNt1+rA`rQb|Q8%3mQU>Ot7pYiH&pI!z)SOzRtlIOr`rq8-+ zRmo{3Zf_*N8qqwOWUJSRlq&?+@U}xo zM+Yx;g_%R_3)WXiJB|U%&3N$O!JE_NRj8}BovEson)`|B^x#ZBa0o~YzisB#LJcAN zH}o~7@$R(zz?M@x z%}SS|E9}Yd&l+b*FU|vyo-8-up@F?7=hLaw3-=!4LJ??4YW87t`oyFSYG85C-}fW zpstwX-l$o5&i*DjQ`DNM!wp+J;+GvaN+UZmJN28AZ0{%W*llXNg8PhL!r=!e=jUeC zn)$g(c zB1J_1xOfMpggB$wJXgl+!A3D;S~O~l+}5b;$hI-boHluJ7KK`pQO@&H31*d2$8Qt! z$&VEy5@a0OeuO)hU&n1vc{utTS4JgT#fP5q^7Zwt$ik*_9hOf$@CLY#NvZTX2lidZ za1%ej^4}T>k<>uwwnlF|B7T?^7*!usuqhFPh9AtdvR#XDectx=fanrw9c<0kNbTVPl|FcL?XKrHGx+LD#_mR-q(bpLls|8reCYO{57DZ= zZ;#-*2gkbg{+{0BZrL)Ii{ig*dPdiMOGogBD2+Qde6+9%cnD zbaUzK_x>0_Q`$%Fmo8hm4L03+TJ3}N;h-3Bt294FHdXHWI_}c&wrPAW5_HhS-Opg6 ztlogSG({-tl3aGr1C*lAmoF!tnALveialK!jv5;Y*rI=8ax!^FZoAW~infsVR*;yLUeV zX5WA4&~5<9;U*K0bDs^2lXa6>9~-1SeO`qcgko8}I^o$Yj;Zt1FJbBgxG{_A!NwF0 z_qj=P)Ufm*0hqQ)P)Y#zq|)_hXjie+O)E*kH_Z$uLc6_SX>OY z@V<7ft1ZWYn#*>)`q1^8H)ySsz{s{2F1&Bdd%oV}+grf$0k+pT+lYE}JFVMD*_!fj zE*pVFq;<83E(6M2Ul1nsW68{;YqpT05cFPh>W9}(HXwn!utP7MM+Ct5Jl5Tm))nOM zPwOc<#8#CGbWVuziPwA#Nckne{G9^>H$e!S4z}eOwrkCv&_z@eqh>Ojo0$};thgf{gyrlixI#BEH$B8&O_wv0V{-776i1|- z3oCdqWt@eKh#jg;_S13}UFR<$*!}4^l5u7kbpk*Z3W~Pj&_HW;U;7(b?AmfwRaJ66 za0rSJK)>|#ED{$N*L|$Z2zsV+d^XKru!(cDhTqU_-RPe`2GebUoDwQMRqMdE8pP~o ztcsB3poZsWFUxt%af2-_dv&4T_E(>l-J07~$Y+3^*REMiFOr;FN3&+zd|Z4y;hHxM zY@5FS^cabP_BZ|k5EXLH;VzFRba1hh5Y;DMyUJT1pO_el)fP20%ypIkBx3aN@c7xB zu`eb0=am4Oik((1d_qDDpvMaVte$iGRf$IlGMy6SeW9+duKy|0<;M@QPAEAk$;B$$ z<{yNI3w{I#s`aUf*JI`iNxvZ63`;rIu1&Fjy|cgH-2&SZ{O%NCIW3x=KGB^ci;iMa z%5`EZB%Kud9QxxSiV2W=CI+%QC?Um2FTO8C)RD}4roHDuuMWpoyW~sUJ->eaBGQnv zv$NlC#W0SMO&d1&^-ERW4t=hZV}Cl6Zi2A4kCd-&^_X-0mtJPa`wCCKL^46f7;(B@ zw-O6cfRfV=q#B^Qs+G3NH;2;~hiWu>8b@|oH0)1_x3M3STfAh68Q`zf2iKkBW&AY-j>GNQcsrd3p~0;VIaz71u~&4SEpb~ZA)SP! z;9mSeH^x7QXmtXS;-y3Ha#EIKi3>_f7UAfsAm3%oc46$QcGI=9Is*-7`Qu z_vX!;iW*<$#j|T-1@prH{6XS+J2w8(uDfCGFrDNr0qA9cqdk?ypFbZrVx`Jo>Z3lU zSuJMdM?<%`puOGLN+m?(zOh;PeAb!YE=gzJ-#=+j1z(^IeJJ+SKrK`#x?vx<*Tu`Z z{mwT%jRurQFiI}E@NDKyh;;N3_DSvasJNII7v)$*8Cw56RCzlWl+_p;{cqxjKt{!9 zwI`lWsch>0xMw!rFFLPF*|tS6X(w} zuUN4n(%x{&O>f;Z_BG^95%=96Z zlkgtF?7rZ9d8MNH40NBZAfMFK{P%84IA#8&;L4}W^Bja=cb&A)ygC!5^X;`Ut|v1c z9bLQ(>vei?s6+@G{!&P{VdF+&*DIUuOjcG`oxgWpV3*U-nbY-g2V~}1x1{tUmIv9| z*%cy&fyAjrZqQ9L=0SPV)jSBJP!KJol=q1Pf?CnsO166BBos-ES( z0q$wEW;b}M{Tdp+k&%(w5hx5|a8{Vg?j6@w(YJSabJE5ph;GU9g6it!gq?>3BXQw} zZo1K2c0XPktf=0kF&$r?x?)7c`s{&imG3Vw12@+-UHc~hp&nm-=yg6OvEdx5AKan^iUZ8*?bkk2Pv>_3m&e2i5O~A;_Z!dP+VNBy2S;2%h|jre7dW-Je^*bU52Dv->~ zO!cWoeV~q#`PP<}$4?9r4y_rPM+v2QFEn^kqJI0mgJNSmk*|=%W97aZd>#YWc7_xs zd;Hn1BqkKNO@)?v?r}KcO_z#4v)-9|2+*6GY=n& zbsiK;ad1RQ^QFK422MgZ?l=(KHBc5_US7|@>X64nStKpv;W5~xZC&GohhH2#GN&7s zDx~4)_Ny=Ch=T1GrFijInek8vWKE`~F9Qv8$~kYX|L{TL(yz0G_^F(78fsq@91>6u zkv#MF(HCcFa8lb|JBgP`0GpGt^k@5k#0j3DjJ9{>%<>CJWg8T8pTG4%G!e*y1|E}KAw%oz5 zyNk9icBMBBsWxKDI0Nj1q=XdScq*3-E?rZI z0QKFo;=J38)5`ChoOZkz4#43*|7Ubm`<^Zs7h9BS@l&tP{;#>9g`i)s{Q|WT2AaM+}Y%$4`oBH+rcI-GW3sQpD;I1d>7ppkWdz*j`Kt$N>aNE zyQmK00}04KGVfrI7>-{LDJb^#9N}5`){i`f=q>Yo+b$x|LX#!po|Mzz!;0kKMal(u z41{Ul#w}L_=1(@%)s3X&cWZtg6H?Sjmfg2!&z{rL@mYdHiGe(_1dJ213LYHA>B|C} z5$0;_AOHO6n!?DaG+5n}{h;z&~L+IXT8IBVCIiGf;fp z_xZM=qL-4E?nLeR37-!=D4}5vhsb$)e9HpoD`OSaoc-755xRwldVOoNokvd6cv4Dd z>tcLZSXdrqk;>8=1ueuwL}lSn9ymRI_F^$bS^?@9kmWe0k*e7s9fbZkJ-As$=ICh? zSUpx!46u}V!HPsF9eIB0y7rr0Hth_0k9F}c09aQ7BPGlg)_Mt+J*hw@L$M8ZSi8AK zabA>SxxdTiY1VAW`hECzPEL+Cp|kNvT@m(=sfO#J+l0B%FImD2Z_8>AkFjqzokjEK zrc$(LafO-Tq)2r@oxoid5dl*Z7zC8s{7{%%r@-%mI45qDTeokQ4B*Y6dmxilJbxqj z8v%FGrcIk@JT>6~=)~O+>4%ev5vf)!jSdtOji)U%qs-~E?YTLW(p_1nr}`31UVOSq zn#vTPWqYon>19{f+!Iz-TS|FRHEOQo#)Ui+9i%IvQG3>2bAe@9=3*k`9mLPl0zP*BiB`#lA>SM`_ezt_i~^uVdqY1KfI zq^F9&**UO#xC2#y3qOQshFz<3a;%r>G*Ny`Zv3cDyPwyKyCg=jq&|7#1EY>m;{kXe zK(a@_kWcFnU!<&VGOrP(zz(HrGisP!`#d)F*pKEx2u?BEp^ z7KSqc(@_0|_dd)V-q7c-4}d3os_BkJ;g^#$I-Ls6XJuNlewuMa$$FiweLswn{AVuI zKRLE5^Md`@(JBC-`10@5vSXQ@IIfMx4v$BX#^FXd1gA~v5v}iU!tUP}nHjAVKPi_# zJAN!MXy1qbS<%k2yhr>z&j*oh2^->P2iucSXz%z?8-kWM;Bm5>C`32@$jSgP+F%{x zWpI61r{`g>W2=BMWRa|_ES3udO4#%}tIIiQov(aw@b?=?=`19^G54wOo8l4@PT~X& z4-YGWcG1c>bGRYRgclewc>f0cZ~+d^+vMv&{aSz^nQl*w-*AP8hf|)AOi0uNARLBQ zps8z%ajsOBO~-Nr1A`}Bk=xI);`Cm{H8^FMh7;gCGa*40G&Zthzjco_L2t&|H;JrM z8hP0`35yRH=gf9rL0sEw$Fi*4+)sB9M8#J z9$$*WOBGZ~)%*7&2n2ni)`xT8C_z+bSh1oE;kcwJRtCpR>;9!#Brsqtl(xGl3J9bG zS?6g6r`gGqY-hUsZroTkKi}aof4yIXZjF<{R zsTz&*-EZQ05ZwaA7JJj3`Ot)+NrJ&qGXh7Qt&d|wEE7}L;$dSW$CDr7G56}+`S8}J zd7RkMQ!=v;E8geG?Mjn{kI}O2NXR?$J^3Uk<9s# zGTxEVo`X=2m#}Zv-;MYH*B+tmP`@k&)hnd;m<_(L;NBop>+fAgch}~)Z230sBAWEq zVIl{{2a>EN=nFP+VnBKoO*014ed>I*#B zBmDKdaxV6PK^h;i+L$9w*(pqn)f9KPuBk%@s0$uA%r|hwAr8DVRsBN^K_Bd$MV%#$Yg8y;)eGJQY| z_?69n4+07I<;M>ZTo#k)&jA^?I7uW#?f=}~GM+kR7p(P8M!owR~+?nveo8% zx?Jq4wEvq&btPlYqhAk10|Ka7L0coNJQGrh{$=;~?=H_F%cy`y(JM8FN&pscU-J7V zIGe0Z<`I-RD$|T|ZgtQ0TO`m+Th$P#1i^w(R;K|*E_dPtbxF^BD1%S>yUK#p4}Nb~ z)1;`T%a&ceDi^lj6XwrO+oN1N_xG#<%4d{X(J&yM>8&Y#di*u<7bIK8VnKiPH)%k( zs-NQAky9Fc@X#T)xtTGIXrLp*bqArK_<#~%ICt)xn%h^T2QZ?G5QYi&X4N|NyFcZj zl~@KWv!+#dB}5ck^`0EnBjsRZ*8I`G42t6(0R|Vx8yuuU<`BUJ2kQ^d@mBbMOi!Ki z1~*9@LO`Gfm+#C2RNH#-L{d$PAspGRt$YBFKVin_B~8c z!zCR1G{a`)F8yM_MyNunAjE9uG?bF7;0$JFW=NQVqD5{L5SKEX4~_i(Fs9^;&h?~ zpc!TiI*P%|6QQgVWu@Qs>&lldy{c$lM09f03kaoaAP5orQfB53tbq^G*JC5*qjOx2 z4i2g)sPx8j%*vJ%BY9(^Ue8bd`)^6{{~sQC-N8*L7hyIoK;{@3>E1_N&B%V>B(o}} zUimA^^7HePGVj;#-z!(IUX1V)F9YnYo@ZV{8QOW!aNdL z08$q@Ex5~EPj*BZs<&q&oR&eDp(uz-`Y!6>mEUu?%Rl#vIZR{qiM4mpJ^1Axv|Aez!?%{{lzFV#cX@B}U)q9)P_ zgmYgAr(6g%qch@(Dfte_;|qQ{RaION8uA%p$GM1Y*oTRIyUbz=uC`)CiV=W-yZ7(M z=s!6MLH(oix1Cn49$WspKwlO8(-0Cvdt&tKyCazv1dFC@|2&dCh*u>>4OGsY{w{#-c`LEIOkYYaFKTRx; z5DN(biRyLIM_CZS3!W32mKW!WueeQh5zGe<)m|X^Fi<)a@xi9&)pI=fz$h%WVASF;IfYP32XV-Lij^4}9SA;00h<+4u zt8hnve!@2Z%1rVQB*ZR?W`Z*0;T?}bu$f3nNqx23ZITU7tQn$nAGA%lszN(D|K6fH z18eY!egOfxEl(2@mjI)pR9i`6g^y23?+W#sfM}rgDGKF3V9Nb`nw|_83`}bu5pi5q zQ&ST+YxnNZjeiAOHS{9|AuOENMN7bse>Z$i3Qj4^9H>0FF1Vji~rT1#*<>0AqW(b_(Fci4tDuVzY1VuB$jGXqn9ku z1gB(JEt7-iJv|Sp8zw2@`+}}m*xtU*s^x{SX1@a-n0(QAUb#X; zynH%|>NFHW`XfnUWfuC0si`jPH)#o?3Sc`!*@k){bno;f@bS61xe&b&OgQc=hq0WLG5|x0v%hW^ zNv>|;evLexs&Kns8k(h zKH((xY+hieRuR5}f`1=)>s!(Q=BF2$^zEFfdMR-_{CvSf4vd~LO|(4#G*`h^ zT2j+=;J^VDNMI<>MJN{N2#F16aIsEC;q2*)biBHd0y7J00QErmF^PwAF52V)RE{O< zihOjNpL;|2`}nzd?PTJ=CmwtFeNo{In8Ez1t7)4+x-skTLA*5s5T}S4%<)G9V?sj0 zzdXpe$eqw%iG7jyjh_dha2MFHfgXr`-_^y-PMkdH1xy+X4GNuiD!RIC#2=MnAfA4B z_ilPZ7SFkVuipr5RyEoLS|X(cse~NAp+-o7 zXL{d&UCNC$6yfp#(TF3F*h*V6H=|=G05lnlcx(ha$zCN<4PRRB`wt#0hOSLR2^l`$ z@S`C)R<{?j6Ja}1gD=PX0Wy6d)(B|3usdAEd#uO$<5KEbT%I1cB5jV~D}&YI3ETbf z;lp4!yGZ5Ql6f9}!MM_qb&BrXU~VBKECeYiJTQ<`N+?F{2HT{GFANt@>+B}p&6H#* zt3tfBDF3c9v0Yey(%ghtMLrm9lTaREZHcPObGrqeioFvYT!Cv zSfyr4{|3JLD|Ab2(QO!Z8a))s0J}64Xr#ATeHsccR0f%qJz+;15DE%ZTA;nEC`FS) z#uK{m6$bJu28tbjB^0&;1Q84>rD&F5yT$sIZVAFTF(ig9W-Tfz8b<#XRYAwNWI4AI zXqd2D8=yrHvI))tV*lmbb|#=Dd3}bz3uJM`R$s)i>L|q^#Bt%b2)CPkyqOj*gFsA8 z33^0$Q`Ag2q4)--P3J4IKmE zPWFf%R9D{%Ds$!9wKssdyTEsnTc|P(;A@1^H`eUa^V7_GgVx1y8m_jH%G>$Oi)^?3o9poqGbt@xON(fn~8T6Wm@Pfo%Pvgn5Zk<>-aq0(bN&KpF z6lmAL^tGTwzRTENR9JAQ6Hl38FBII9ihrB*kOmc&OqzdAb;r5%hJ=3ZO%Q43xygfG zB+|5D6gh$)=lYXqF?kIc_QpX$%a#`hPRl~QOjrN5aU%*UA!?!C|MjEsppmxnAc9cW zoMe#1S(VS}<_gzvjgzfRvMdeJP=yYiupZ!NFKqH18$N*;@x|t(p*uztjhaiq3W-#r znL}XC#J#K#E*w)!%hcy{D->ru(XYb?)2Y*7o7P$J-f!O$c7}tjq@_rk2rexyAT`ku z!Hge!3p#>y%aKb1h@m4i|Ev016>3+K&0sGSkdRpE#YdIjm0U^T+NEYad36_l@Br*L zu{R@^?mCtTU!y9-W(1S!h?6o4se`eUM1v)k7IK`C+#e5gqaXuC ztBm3b3Ux`%2lUF(KALd>EAfG%%+1Yxn&}->1aA{|fcRdCpaf+w8a%8>|3~~*NXCV@ zOSY4vr|rUvtIQbTW$LO5AdJ3{ldbK77#%3T?SdI2 zW@23c^IG9k;{!K944E5lf^iC&k@7^)NqJ%bx(p!(r7GySL#MUDyR$cBrQvL=g^OM% zffLIIYrPVVbflt3ARs~v&K*&2w6(qDOjY!>VPaf@lJ?F3D-(j7MP)<%r386tSLS#D2`X5L+})1erNI*y>g9&S@^33CFWn?ZkFyS4G4h$ufr5yLHdKug4j8Yh2^ zy>N9YL2+-v^9RKwi5{@4vVj8f&B(}Tc?E~i*^jAfpu+i}QoDYD=qL!R@VK#;ZI=`u z32r>Ha1}3_F;b!7ndPD>lH_*%wIYO`hlQPO-Xw1S%FN$TqEA>_27tEk*?Dixnl-0$ z(s0vHAUK@V_HqqYQn=wYV=_4o8H_MoCtIsfq>;c1LZzUgfeqk>kOgqr)50r46#x>F zJ_W33rbO}$L-znfdBOXktN!K?H*187GSE!6u;7yx41VEGXl(F{C?lo8*ieh&9u(#> z(ESj-=qZx7NH1bkqM|F}IO^mknJgT1$`j;?)Qyw+N3$%21vgSk;K~pkH6aX=WdcF! z1BX6NJLxtTxz#X3+v<7X-B23AHU4zbiM`7&0(--CY(<%-NI-djCanI8k| z@k>Z(B_CElrLIOj3Jui+=|TKzfJY@jP;W=U)ss{oowkU1H6L&!3iC3=aZT9u;%7hO zKv#ZWogtvO_W~oQy`lV>f2z>Qc%T-N{NeuzMxU`*)5|mOx$rL^!3|BQC$ytIZWu?Y zt1o@qbEfiF>Y0`0)kG<7waG7pmk0f^=-gmM)gG6XCzsC0$49|Yq=qdha05cldcfIA|kCC72E~w~5<}0_){%_-Cp_o+|(e$>$WIKP>tuCnv8v zF%rEf#qHgar4V0-QQX64rX&y|)Rq(Ml0`qs=lS^iYV^x7Qs^qYU*SzpHH-BJvrp;< zK#<*MWc0A*5?n_|hpzsO7(!_Xp>q!&J`}aS;}N+|@|0hNcn{6*sKgY#l!RK}ovEp* zw!ksOkB_`UT$-cTXhH2^+f-QSiN(!q(Y@QOx;HnH)@d-?Uqul_1L*=s!4!B=q#u6LCFsukkC$r0U;(Al&(o_{ukITE zsi5KO?M?VZkxJO#87Qur%1E^eXYvLkFmBYTB1^ z=KWildu6O&w+Wx!79QM~em>@8i9hKHrcMAiTXkNN}X_8ID+{9%B5u%~~3;CiddM?!)Cv7C{c9QyQsW?2?0S5~#f_TwIT91-DIp zUPH7_wYcH2v0@lPGXqyru%p+;?pr}EAVd&TLZQIQX0UtZv@H31ceRX(!xM>|qCgzN zk?j%L4vP!`WUS_Dv>kSvLXy@Aj-@}~+$K$?B~_7T;LBI*zbx1-5X#IA_qpI~$Xj1g}Oi;8Gb zN_TzP2UUm&eNW70@p+^_7HKsGlp*QldaRr5h4e84C5-g`JbCiut{h?<`TCVJF+FXD zb?X}P;v?Cd6eQ*n)8>uzdnPo5jBt2Sj+QLRgMxj5x;->95UBO1?aDW@x3yCNlJpT5x zwG;|<(w2-1^Y;$2Eu`9zYEi@I^ypYCK<@~N9jV7(>xrneY2Q`HB-4k!c^Fsn5q#ff(wspkIfS8A97`iyn;)9FO z1$A=*arUhw+6iVr=j>NgD?-Oi%<~~!)k5_7;=K7l1R)3<)bx#>y?gg^O4&XtEz?5M zgzYRAKR{-{REBRM#~Mq1Ftxj{j{!skwwNlH5vD1BnJh4Ni~W_%#lxUliavkl!3i&d z(ulX<7t;`ipoo6!N1)zwB0EEnhaCQz!-Ugcym*UvLqh=wN?XWc^~pd0OjRlRFH$5L zDNgMwr#*aPU2{v9Ez9^!++XRI&61>x7pT97uMk(;^xQc{s;@04qb2WKO;pgz^?M8O z9JtVAU;!3wG5V}I*Q`OhTM0`uX|(?+AuAkT11p!Dv+(=?8ce0J)H#t!_ep%CyY8Y!JrY0#j_ zWCb`Q46Jxj-&_TZB1${7kdbXUAj4s<7jaLxb6YLGtJfB&K7I}iJyi5Qiw#S z3F`n9%=BvDvf-c2+bAV4*kMzsec!wcg5=pZ^C?62S{15&((gkIrI?if^;#7b`9Hns zWZn)Wd_8GFjAPNQB_o(dWf}ffTL1p%)YSelS?b^a_wOI3Wd3qqf3w`*e~!{Ggy;YK zmXOBQ1#0X+zxRLV$GqJ^iYwvOi31f?-!~_re>O1&X#_HY09eL+)k#5H*MA01F~zh6 zuCPt;rX}qLU{EG`7XM8KAzs9bQC8OL#M!Yla0Oy7zOTSsWBLp7=LAy0;-ORIYK z|MSvwWp#BD`Kl8M|MMsRpPX0~jU=F1&EQ3m9b6jYx;|t-cmVuyVY^~n%Oq&0!tMUP zT3=;hoVWOH6&2c;m>5`IqB^2kdSu~3f^>%l;SKq!Wa<-m%~aTyMt~tn*hP{B-K&+? zHL);|s_xE}%3ZEB_<@yNTtGoF^Bh=Lv=d`gK<~enHtO)DOEMti$Y%&{V7bywx3J_U zOx2;oiwJdszfSZ{V(9# zu$oIgquT>1g$23eSI-IUB91RW*_0>iMm6;F3Pr{8SC}At5uuZeUx4DbTxMaz>UOCr zD8Nv)%)w#IHSn5{$A2x3;}oqr3q8;e4Yi8ZZ!T))O>lVlVl1pQ*TRFu}2DUt<@C+eI@9{*A_Jptjeu(5?==N3MGr5ihG`2;3%8T;1b zB*gUq!!rQ%FaG=#^3TuI?Fu-+sLMAq-f9D|8v~?wjD%#Lc$&f z5gJ24zkQFIyYD}r&UI3omR$&e>M9=tTtn#MA#QTiO~hmd;kFZsX4MeOKlQdykhF}9 ze>WYV0z`6jy$Y7+(mLLOFVsefbD4-{m*Zq=CbWI+2 zIHdMSTn+RTySoNbBu-x)yriL<4VIG=1^X3(tT(`H(g4~qmw|ELMSwGeax!EBjwKM| zd-m`5+nc^YP%s9siDmC=%@!wpsL%0?F{oY1xkL@BiB@93=Ou*N_+gH;iXSC|j!=Qr zf(L?)h8wlH3Z(WCV5*eQ^KeP|6UH7b>!d*kydKD`6s?(s6^Uzs#|KjtXppS%DBXa0 zI*%PNQGz@=h)yD00?UQPj_(eYjS5_Wr~w9bDUS>raxM{?K~|IAg&Fkcu+S@8nG6mN zl6EG*WTF(1`tSYwW$+5tOGUy5#e;I3oFqK#+W`z#C?Kg9AQ&t^VI+{NrJRQ!Y4d`t zxQf2Dl$6alJzL={Oh1l1IRf*XNPzsQacuk}SZ%lN+<7f5jQ!I1&+Fh$&v7e)ppQUF zSNyl!J{q=arB>nFvCGwcy-w8bAE@l0cEb)%CS)w4mU3=YimnDDApzjvV}$~dw2zX> zLcqHxU?gYi0dgbsFzRqL-@#%=rUM}NdV%jfv*y^5Bb;zNFL>hMNFaSkgifVE!s;A7 zgVRI{M_UoRK?I8vt}D4_lSU+YZfB6)bj0&qf);A4&{i6k57+_6{a3knt&7W+JuO6hD?ILO^{t^1s*B@w4kshW?!hFzlU_ZV}6oow;IOR z5H5|FdqFyV!Yv`}1%)l*17+%sp%Cfrkh{RPcYgnV9X7}$qXbDHB;sU%YxLz7aP*kk zu_RK#%^xsOxFKrC`PC@X1?M7hFW2u2`~vsNS+L0ABiN~SHC&u$rp2`hhu8YpnH>J- zYRUIPwq_SXAx2IF{-Z16LpJ@-;4)Y!xfNVFF-%N9$p$l52x`Ob!eF;oTbq>(j@o!I zT*<_Q7h8gs_*|6-(6ze3X9?-2auN$97%+G^Xvjf~@|e4hRDWhoH|PP7dUT}rKoxKV zEnsMN$AFrd$=)bZbdly0Fe{LYFav`V2Gu200H*{kiY&ER;6RSRKmvG+z{;VF^haGp zQE23k+1k@jeg}db>J?fLczZxdRV+sh?O5$B@;Pl;a}6|^T# zygwr&8HWgqPIOvY5Il8XK<|=P229@=ALwMeN=RZK-@y2Rqp&zx*B>uHEW@~QbkGO93C#|}e2Y8{MuV(E;j)bka2mH! zyVwKZxpibF(gBm7@)!sU2%?QN!lP{PZ^huR>v(CAhAjq9xk;*m!=1>vq-q>mioEj{ z>?guX)PZe1WTXkybKkgU#bN;O;sf4P3r zc`K`%XtjoOj2E}D1pG0ZLK!h&Ql8+RPoVtmx(Hx~F;bnchw^mRmLa+j>5X(2fqO24 zNXk}$#kc`j^b?f7>BllsW+2%-d-hBT?qo7M`b?k!uh zmbIhH#hG+uP-I#~A(rCp6z+Nfd`z10YV3f1S_5`0xXB|p|D=BF?@!qpdo$smeS69S z@qJMRSXwmaO{@YXPDB>ik#HwUaje2xxTS0hukOEe`Le3u z?P#C~CPv2lM_ymtg0RnsatHklI>P+$)h|bcLr)qp{yf4igOmdJQx*wR5eV_4qlc)c z`6C~q?@1M}hp=8Q;Rm3lDBN0bii^-~#?H$6RmN#B0BG=Q-=_tDvY*6IwxMg`+ji~h z z(Sm$QR9qx}oO#TpwUUPK`|OY?|b*wpx4$L01_h} zm1lw#FOOTsyQ3|a2~+5ny|-J6*nAbh1dAPjn8CMWho10WJ~K~A{Gl^2P{Tvv__XAv zuXGV0ezlJ6d(gdp;j+Icv@N0{Q!ALkFJHfgD$=ZUGyA%YcQHM^Un?;y69MtA1RW?d z5(%-&{0y`UTDvbaK>-1k?_+KkfmVFc{X)jUp#4J{nI!Z_c^LyZyL&woo{5R(f#XlC z6_Eh-4Xa0JpxfF(tR)>yAYbvi1*o3v`af;~*7}0b*&O~K2gQU;IfOb)bTJxC6S~I7 zKrGkr3Xof7cc~O`H$HO%4VkM50fkIPy4&_B8GHK$VGlDy*W#gk5zdfNl|?~5VPp^; zIr?OxBgZ~W48p`$Oxp#kPz)#)R-pk&ro6mdt`)Eilw>xau<(K06d8D5h;aG&9dWvY zKuw3ZjdOH@j8%Xw1iEsJ)@6YT9^zP;U(1UoabgAc$5aEv23S66&=zt{_rAo>3+R21 zS0Gc3U@}^dx*R$8ecc>psuB4O!rOYt0!VQs;H}tyY(G~|uT-iX?2?REpdQ(CqsHfk z;qu*w*;Ok;ERYu1(w5x5b&GVA5>gM(C}fi7n)R>PCQk~B)oa%>gO_m}XkkYUymiM8 zpNx!c6zLnFJPDHx(F%>M>#-`t%1T@g=sD2&kw)eRZa>R{8CFD}fE5tQoP&t_2;t9A z5@D#HzLgvT8<}QFOt-{c529&hqAR`RU=>9s#-03Db?+rmA?6sfz)gup3=oQ|$?XCb zf4c4tQltMonQ4eA38bFBIMn{cp69^A;jD-AKG)em_2^M9C~1pP3lL3Uf!!q@cX)}E z!K@NzF_}+{b#6?OU)f`H@}vNJzQeaFG8&3lcM-{AfUmLm zW>2R!2?%_Kpulna`wQVs(E6_TI=%l+DujDLlFJF>Zzu>29_P-^LS@O4B`>U6R}(%M z9Rh4W?%%s-3X{&KHSij0-TD`^T1;ibYQe!!-TKmk8W5o#|YFo6pt z`^WCQE;vgX^>(K zU}z6W0Iip{ri3=dG!!fsQGB1@JSY(I1&6E*NCZ4K(aY@~my$4mzXps1HDL=VUoEJO zB>Ex)vNZ+18|b!&M}|BkLZB}uUJ`dWR*jSn@XY7@i2BP;B=KQ7r7BALIy6U+{$uQ? zju0tfz?57eeIl6D1`iw|X;C%a?g{WBK!z+K26+>H4CxlXVjl*Oed+9s*0MxKBCv?` z&LYM5RUR1s(Z~$3j0~xQ4nhmm0Ml1ACa92UP^1n+{69bYKiGToxSaF1|NERV7z~C% zV>e`p$QGgvV>!ztStDx*Sxd-NGS;z#vqe%0r6i$jS;{h@v{+ISLYqC6RNC(6ahB`* zy?*y~-~V0L@4g?8-}(5?7j>SW^YeKx$MSl;j^pip2Y2=pdF>bwjR~Z8B1C6YC_5UB zF3|3o+JH8juw1$M>GKG`JyZq=)GoC!c(!eweiV`f8ZUC`TIh_5qGTgcZ;O#u#tH4M(|IH&EDaI3$iX!Gieh#9?B zz;H=*ON(R4M(~J|he>Rbo^IKFnR?`-*GL_8LBVz8hUP~{H)b{Jjmi04Poj>Dh@hXV zx`FeRb@tGn{QcqUD~6nw9)g+Lvy5_oI@R&-*QVD=_*-}sm1`XIpdEQvv)Hur%Vw}l zrD31t`F-Qp>d>AIWxnu5($|c5O#t@K@6$!rw}pmw;ZM{^nezwW`ulvC97;L`efs?Q zpUn+^IZeg*?)bm$OuD)MR`G}Ctz~__)4HvU)HJQ1x_j4V#;0U>eXQz{iW~37Bo=(# z2Zv0=5#Gr!Gv;yNX0qftv{iU@b}4?*=KL}$;}mNBB1~M* zHzVyaz@A^>X0cwkF(iZ$o53cIkQLy{7n_FGfBnX^aM-TzKkl3v1W~emY~H%ckOs4i zn&02QEpP1~Eqoc3%lD?!0Kx98nRHE*t}_J6ueu74cNdSxA});w48% z1Vq;EJ!M(MD}r#la-8;r!z1b@HrXs64pEen2;B!c5LE*i_~S^MrUdn|^PjzsuLTH! z?J_A3lLV zwD#rPHu^xIG#67C&z~O{U&y;U>FKow!6jy!enAbde{Chvsh~v&k3~yFh%gWSI%IZC z^k^cG+Sli0r3+Qfg{@Lht4BS}rb(d}?P3Ys6$;5?RX;sBTYJ4BLx;{oVM75dX$8tD z5$fFfU$2DD8`OI@o!N!RBX@#2|MLDj#aY<1ZwsU)bl9{gyP(v6`*5-Q!r3|R(6MoD z!jE(bKVw}Wy}_2^;6*K`n3H{HU(^!h*#GI(I-2bPfWQA z9)PRncEGWT6zYqEAAsx@F|)&i z{pUeJ_MWnCp}ytM%gW~MUm>9!k@xYl27UXQSH!rxyN|F$Xv-%z{{2h$SGeArXd=H; zpwaOj>d+HNA}Gaw&@#gdeD9t45l*`$IfZIJab9M2+_+uw5z+|>WTnMBPi@QQC>I}w zWCEPcMXEtY$BUM#iIVsAKEBpUT^LjRVdDqT8Uq^o~*a$~1 zsA12$_;4MULtbeaY_h-y@AEI9SJP6PCxRpHn%mObVaxig?QZ>@0TBgbheQhY&35p~+8O!@f- z3B)l#bjj9o-Sv;h_$*png{M{Ci9BY*A5st1-36 zBD0jjc;;0k6#|zi5S6#|+9X;gVBnv~VnF`*!g*yNYZHK&#doqW)b+EwTGxR`L=VvG zeG#aHz=guwp|gjQIZ7P&YTfS(S()_|>(;HyE-kIOJ-b&sXGU*~?gJ__#i!~tZ$nQw z-K~7+$`=Ze4IlUsgnSPPnKg9aG+IiW|G+sou7d8;;zjTIV1=~YC($^4{(QZ?`!orp z!_9+n`QQ=t89}KJ4zezxivemHY0!%@>G!9z=*)z?yfHY#a5Odld?5e1$yWYnac4@s zw*y64gI>@q$a_R4qCtg%*yjjQ{`B@;YC@X{vszOJawFjfmJKP0ty_5mlvW@cXRw$G zS41+=Zd|PN%ZG*S63gh2xh(ID)h z2;m5%q(D#?!$z_Hq-LSw>$_8OcvXupj;Q5D>80pP+q1_{p3H+f6soJm*FH$ca4acF zfoc)J?}2iLM8nP4t-sgSxwGMH)GvsU8cA-|W9FrS3ge!hueNn`k3NMI$^xx0an_*j zKtVU7jCQw2)r3ACX)uV#`WoINV?6nl`)ov$6zOcdDnp_s&Kj@+>CX64xS+$?_FYb& zIn(Ty=86xu>Wkb8hGwp`S|0JeLiFcoUQMzjDv}>aqD(@SQ>xh8Z}c^gKzYU6%G3VP zt5>g}%io=qb8@(NV~FC3uXRw@NiM|5vgz+>ODCO4QSUb%qvJw|!;C1rx31_vngb652<|npH#{(wP|W^m?nIzt_Q7wQ{M(2zU_I)?hp#^r+pFV1bKSm@(c#Pmnbcj9^791>G3p7O34wn0`6(Ks zD}ZF`O)sfcNKV>I%ZJudibH7a881!ls8<98st-RiA!7`HgF?ZvU0}tIhhiJf-_W4%QyySL$^0oUkmIHd!yXzvoRap}`WvJQN!=dduE~2DRh7H)L zJ+1j)eIc=W2`J^=f5DE_auTGW3TYMdZEyk`pd9iXYpN0bx3jc5%hCQ3D+Mpz6qVbf zg{5uVv^fG)B+NOfcj%hL-&xdTHqF`1H)-$X(55)uaU^~cqqgK-yhzG`?xx!YhZhoOBQRo;fu(%14Wk72G189M)M;bxVF;fHn z?|0~8X_ThV2~0FoxCW7a_IVy*B!|T8Al`cYX{>s{w|n%{dfBKuCrmv#_Cq_D2}#GQ zeN8ev%%>lnFi~VFOy(mO_dBiSjaIji*dCua+$In0a(3&W{bLcf| z^+e5rt}s2^UBST(s@RzRZt=FVvF`c1D) zEt6{ZL*3>iei|!PDx#~q`CIQ##o>B0qn}59y<&3dcEQHK7K2xn5;uy+2nu|t^GISy zDCt)$LXMUT7uqdC4R#eZZ|1CA=aObMyiLc_p@vH5EE4PJOHC{4UZ-)|N=CV&$w(WD zWBbu=mosa+ihS_vl1iFP2}UU`K6p$i*_AefXb$FBTi5Ai?R79DB*e+hg9RKtx?RUR zZ5jPBqzqD&hwsi=J7nm6GGguhC#AN9%)aZfroSAY%dt(kU6hwYtx@+9dT+UG|M&K> z7liF6o|fYWDRR3BA1!JyhObwOd))jo7Y%j%viKIRoV<1RANW(`JB|24pilKCdfm46 z4AJ)cOoF7J>0@&IdH`UG)@yZ$cBE^+h@+6~&;{{a#gCpfdr$uVb!1H|g?(;vQHiP8 z1;W6Rzjq;&MRTNZC2f?mO33>qwGjH2h^P}a^8Q_5FTTCv+tyUBKiJ1ovPPgFoiOG2 z2HdgKkcztwox!@Mf=G~RdbuWVTl61r80+VdC|wCfKA(fq^9I`Hi|ZNu_CzMa zWr3pz@=Yj^XP+&ft$jIex;t2`JdQf!6RDN+bBH2HUWOAk>4xTi{HQLkBfNKP-c&s&&a=juls0U!S!Y(m2O4cq zHH||5TKlmK{7zp)+ zOczn96FPZzFZlWXh2IPe+Mi!w3R*tjWn1B>9+sg$jM$Xf!aF95B@k%f*0jeZ&H>;E zsqYk|yVPWhK1*Os#0=UQ9-QX|eq!DeCSDsg{Czg8)wv0xqm)z-si3Zji3ma@9mY^B zedl7rQGiG!@tGrpIYPCiZ))OA<6tx&g6^(52o6hGvAS~O^5x6ckPyN<1zMPWHj!od zImjA$aCm@IfyLhFd%q4{-Gksrf)hpS2H#6{8#0RfRG3ro+Yw~dWSI<7O%~V7W6Rd9 zN6;hEY_b6(?s;O++X@IxX|6T8PLn83g6K)QE%Y)^>KGj}z8E5c-paB;JmXy|i>J!& z!RM#-IIwB^*p@VD{xo>suNR_z2rR3%Id1G8ROj93(FikIZjX8SZ2+ZESQ2LF0Q4>b zAX6eHG!GTs1u#CnoPX=HVj>ipKHbzPIx+9jk%yv$7co;hqbWjCy2oB;r`WB^_R(V* z$Y3>@P7#3jFuvB;IZ%L?{XJWm4vp~q_64LHS&gM^JKg_F?@LCL{Fp z2WVmGj4z!FNoWJ`kPUN$#)fNL1)EIlGWBTXqKCN?xm)QI2$XYVa=Gt%qER@|gE|cw zq`!d@Aq0_jq%>>okApg*<2$M8y@&g=*Zx}9cdxpjX`kClC*!L`&Ld}6Zx}g;s|n&N z&yKIM25xRx9Au$IUeOnc9QQx#sPm$Sqdmbzj*wmUn0_`S)p^ldf^o{_%^Ug3n>S;R0tPwQ*QckBcP zlATOqVoWp`+n*LE0*#7-ZJCp^vk3h#?lu`Q(60Pt8-a=Sjm-^2K}#&t#jEYFzskWC zax|d?)>tK*j9~EaoSd93#M3qh0`v3p_u#wmb3OhEGeL+%=DiKw32eoU)YK{3v}JC= z*VXNm)lXMfcNCHkSy@ERol#;MkB-;^rofu!jN4=PHY`816UW41Z4I&9uS%r_38bxN zYMH>$!Q(qpmpJ3-D!Lo+>Ur^EkG46NrM)C5V+hTLr9&fJ?NP{B!44D(3abS!VUka! ztl=@l^xZ0tIDh^YSu-kzA0?~mkDL*{A`4$G{fI|bE@nzOB1hf6lie2-fe7U=F^g!V z1u+El%z}#US>B*gqml4z<^V<_yB9iZ0&^%TP4%G;V0?9?w!3;Qd5Vzx`=TTDpwA&l+(4T>=a}U z;Os9fK2DACVZ#>C2u17MOmaiFk0JXB4f-!=NSTEik}gC-&=S+trgxn8!9fRT0QenN zn{;qW&z(DV{IzpuBRS{`DYq0fVRWP8T>xwamy-+~_)w&(6DS<~i9|;*16MjWN+DIv zvw@6$e`l=Q-VF<-7qy(x$$?zG5*7On=7}s+y6HfL$2xzYm)~JG(bt5u+oX>Bj_5R6 zG4hy`C(?ZA^-lN)Ag@<`QvTTH}dH?-Y$HC@;F$f<|GK{EQ%ZA9H%mD(9=f=9;)qcZPnWGbHV7)fcrCl-YpAIWz{ zei+XLm6ev#iQg>e;ti}%0SXw&G|9;na-^}u@Fou3QTUSlX~4@+dRuqznMs=%Sg4Xu zEd6<)d?IfDbXO;44u2;z`#EF0?~rD&_LE-Q6W@9dStUoj>|n3tyuLUo9qEp!2qBQ@_CBRoXB|iBQd@7CV?dD*zj&o6uO5A z7%j~zy)7}$(s7U*7N~(NX~)H|TX~%~XcKc8X4J4uCku zZk%Cc+m3=`P*>THkal-@%CX&6>(*V2FW4wq5Zjl$Q{@e-98*!)h}6I3TbeEjDn&eg zMSf1LgjD}Yt!9$fUQg0jDY4_jx1dSpLb37_HQ;;E?kEq5oI;G!*M>x?@l&5af37}# zX2W^sc%#z?eSAW-Pmz|30w4b+G+>+Je(TdMJl6Bt#)a(b|Bjv7O!b#`HFMDz)C@`>x^Yi-s1#@5jRW8z3gGg$W^n|>%3dnX>2ZJ_5%TgXf+)610 z|NBJQ@a{chG$60O)k3u-eabn(+M=Ejm^0*L7te*`UjJ78v)_RV(@I})j*0-Z=u6&7 z-z<^461)tY)yU%DAd)oF9MY62J!$K*6^DSz*dyMseP5*-&ASd<;`z2s+qQB#l;y2e zwCAJjQiHVhrYwmMoV9ih{~LRIR$B6wo+M{=6w!?Nli z(zqpxEAp75SpGJ}CuE|^=$rOq(1${uoXNZWLfU}Hp$61n_sRIU3qaqb=fD1@(XwS- z5>}x*s4?Gs=uihmC(Jt1#wqLGJCs?46j3I(<$!JtKYEw8WbsqsjW>7taa~2}nJ3!L z&=891@`&kukWN5I>pk)B^y3id0w{{GzPn!r3%CFyA`OVg&}B#W?Tmwi(DbmNZUKLI z#M9h!A9fUncoWC_v;)nqS3kwGp(ZM&h}#@V&h9UM>5K+2RMnoViXS{ z=EGZb?GX|X(xYd~3$LD^n0{e$S-X@8=N_d#nm+YpWcc){&xX7UIojl(DQA|fdhx@U z>l?rStJ9zE*(KYIigY`UZ1MccX5YM3E8As9=jS5yt^bLL71g+gH(P2qs1@;NqaDJ{E;W?p!;qgPU~!D5x8%k0XT(Y?Rq zpG%unG;<;>mT8|T&C4Gz*)G3QaBIu@XC3Xvslu*)dE)rtT$+WYrRBtWW22*t_Bs`H z7@X-;H2wBZZ3l&=-ac_Edc7(T#Qn{)%OxZH%V(^(QZV}1c>hCP_UK1U*;?}a&lTb4 zc78qALrcr@MFk$UKZoAFk4!dfRJ>lHxTZRB_VIU%9yig;-iyvuGp4#SKW|CehT=z4 zmC=hUk=W=Z8XysH8Tcm#*I8(!W zZfwh`;~X3uqDx=x^f|-i=cLY9aq`^qRZhL1&<2JdB(AN>tgc#XVru$s^<6W@b%(9{ zAA{na*q-t`5q_yB0>k6FB_Ap`W*@xWy{bI3Iy}8uC*`a^Kb*T#lr${!ofGYR*M&2- z@2{3TZ&VSqMKgWD{+SUsn+(fFbapEJ{>#f1d#&MKJ+aG)lNFDbMkZQJn6U5Q@=IEi zCr{SRDq5d;aM^iFrq$2N;nCzGr}x*_8o0d7m_53b3)tR@uI#$KdPe7IDW_uBKI^zH zYxm&(a4kzOe>h(|%y-Qiw3tj|O}pFe*Jf|;;&!LWu=jf_zhvkTVwU9@RUaUa&O4v6 z+*IXjpQNYvYI_&E&-cd!6Hc`rvT}~s%r)s@XCIgpIX$C}W$1?Tj7*hNWaKrXj-q=b zo94KWH;kJ+b*3kOmJN&SeffPvS>g&8_lahD3l3){H`U7x*wiF`_u%K=`)94!nz1}; zL1O8pci{&!tG--_dhu7I{v2w$^h|6kN4amLnA3-K*UWV4Ko#Q0cAT}*deG8o%&b2d zST~x7t?KaS`!nwu*r-o88s!~)$qgjNYRvX!Xotm=#_IIv?9-=?d476tLyqRIZxLHk zIlW@`u=l4&cHmk|FTUJrId@CT2NX*#jfm@}TO?Dn=E>qkJ7HyGy;7W1o49Bl;f^A*QH}Hv^Lw|mZTYmY2mtbi-b)xUc>U|Hn`X)1` zT7P!=ncE#qK`KzNAE5p!kQm7EV^rP)Ss81zwij zHoseP>ht;X?AM4VCZE92?&4eZi{g zYg)|0V81o-Bbypbvi;DsY6Gzaylp@->A5( z4z*iWCJjvg4vp8eEM2inqdIEC#Z66y_Gr7}%Bqjd?iDtZ<(l*bk+(binSHHJ(=bbW zdbis|<#^rYt32cxxNh?B`t3bVMStx{M0~P%y_g>wUi;In#(+s!j35$*#UFDqjBzd3 z_a7hiF8p#*(V6-P6r5y((MoASxXrIsd#!U9utk)(30dPQJ-cegu?Y76G5FcQriP_+ z4q+o?KVYWTnhklmIV$NYlA;BXiGxf*!Uv$>)>Oim+7~4bmMIUhyb#-JMndtyAA5zJ z9wFAoYV9)zCU!&5o;~gI{MJ&JsCnv2p~=Y`*GtN#zsHE$8+c`HZM*!AL)y&!8!ECo z9zdcv==r)VAB!6ss*3z6KJL56;LGE+F3+(a-ZPw|jZQO83kiIqSZ77pk{p)sT`u9h zp}IU1=7RB;PYr&ydg7t=Z;vlMdw+Bk@Iul4n2N4s72(C}$qy2+sY{|`mI2g=v2reX zHkg%cI&}5xzEignPNzQE{bEMjrKg7{8I(Nzsm;J1B-PWh8La~gyy9r4e`r|#l>s*X z&TCHkHDJ2-WBl8e-JM;fvN`0~?cK=>vt-*m<+576PhM^IR@MY)KpuPZE@7`P(M%8| zwceM<*0u)ABOr*=|IEK?>nt^kg(Wf727{lkvzCeCd}i%nUptdE&mHqB-p#HKEjFlP z!HPIv_G&G*Oc4vbHQwx__7A4DWY1c=Y!LR-p z4iws3*&lu`DmOGVw2)`%R+f7(wz_=x!UYQ!O!+u%pUxOJEvI{RryV~sGx}GwHdJ*8s;7o^)O%;t^Rd>`ll)vBg>pwC1jaDcx8O;gaygocE zOndfE9rcn&yUgs_ySKy5xy{7I>g9S{_n&jkwVl(a>#=SVTZou|B4_(V-8AjO-TDfJ zYkn45+T?uJ+oY*egF{1?L}yyh=J)mnP1q3|#r9VfpW2W^-hFD**`nO4YYz?|Inr5{ z_W+)z-b|Bq7!-Tlwa#gxc|}S`Ov1*9i2j+Mbn1pj<@(GU(<78@J{yx`d&Zh~G!{qe z#+`{jNnd%hanb9edy?#*VcYgj9B-hjw8OyWWxJ0z!`Rq!uvMPNM6KH+-(j=mVS7XA8hL$Poy{^FQq*Vz+pB&>a=n`Y&%XP=wxTBpe!e2dk9 zs!0DzZIH)gJg}!xRl%N7y=llv{Bmbgs-i~-1!{pTeX$Sm@$rS1Gb{V>eJc_xnD59( zQ)c8K!JyNkGW_DJ9q&)yY;v3lo3})vX!94IAv<9}vig>ut;txn(bnIoFitaW)$R6! z)1D@~e2Mh%^2&WQb#@|wb|Y0wzVvOcpdS@O!ZAGFsJKQ7=0ziJqIl%ddG+U9$DVOQp?iX_USw|lwY%Gt zMciMRcwj(q?zVJf2CE((?l7d}seL3Ng3XDO=S}cKg@k4{k0ww3gWy@UqKZf{JltX2 zX?WDjCy;}HO;=L;fe?=;-PzepgXQrQT-<-gimSQ|?&2x4 zor|{&&CR$yOkxIrQGc6)7a1Mh&fBIfFW6FP*$uWLWmzk#=b_*DjFUgGWAy z+a5IR!`X>n2p&3j=Z+Rnv9#37jKUp8rM@S;jKU$>6ICx{Nrs-Q%00Nk4AIFQDclZ<*H{*;#IN48A~JoNW3i(fMhNeUoONXGT{5WA6nO$ zT6q_V%g9z15G>sx`Z@T}Y}m<*FES3N-y0uZriYYT49_h0s9GNe7O>TSU$&RcVDG25v?KaVZC9iwIt_4VxunegDVpEnHl z)_)V|ZpR(vGCpQ-Sn3(%Ce%?;MrPF@rCq#Ma7yfqke404Qz&9n;nsadj8L(zf`K79 zs%yO}n>=JJQMUy2nwjsb&T$KQ?2bt$uJwKw6K4x}=%!zIrz~l|m)An9R>);zg<{|| z;?Z;Q^{PK)bo|6Gl8V=D#^4Rsn7Mj_89lUp*FJaDV{(@yc3k&jqu%FxBOCqFhG1o7 zvVPGxl%5Bs&T{Mu%$C4f^wvBc5^`r#Q$za9<)%~^eL-m9P1t|Yd)3CF$2=7Gj3-T* zlDc%{-M>7&ypDlhKE_W?yHP=?ZU%s9UzW4qhDHq73p25R)^2*;y8EvDYbL$B7oDEv znBys%j&ON7znyNc^HuUa%8Cw2hD0RBo#c2r5sZJFJhIMFzD~i{EzEIiS!S2*J#Wl$ zjVYmv_UmljGie;xoAVJr^OTUJQAPDN^#$^;u5EwpdxdM)OjiDkqLPl)rMLB%m_kIy z!AOaTy4Tu0{9|wqn_xVfK{mF2S+;v@qedxuK|i?G3kVJEIoqwB`!CCv@3%0rnH>-s zpMA;tR$yo-E=Ccs(emEW_BfoV-5m^uf}x%`XZu(-CO$UB3F68xGHGZpWq<1E1f9^U zD$RPQ-^*Md|KE7*VoQtEJ6;iurpyH8msjx()|7|j=;RN7l1xR(=8j^{i?D#HVd*Ss zJYRX>l`Hzppbb2Thr8L(`QTqT2^Pj(6T46lAhlD@sSKY(k?n=>O^MBi$f5!={zb;_z zuewVn8J51-G-=W#3A%K%i1reIRdc6jvfr1UPhaTJx|zZ?b{wmTm+N2RM9_e^qcIDF zyp#p{0BBa9OnEWNQUuptZI@p>oZ3(4%BK-7h3@|Ha4Gh+Pun5|$19O!4tCH)MZ(!t zjG`=uoLvKw?o64n%=6^=C!q@~@qUL0_IlrLXFu*t=Q@hh-X5)nsc%|gdV!$lH=?BR ziMOd)OjvT_q!-eZWcx}IR1fgF?gv*yJGSrogCfAZb`===t$P!+zdwfqTq%t2)!NW$ z-SVr&sZpttLP&;}WLW;bv*r(qIjbht%w&KWR;!CVUH|!a_DZr#{v2y%+kpgUoXT?$B4rA!BrC*@+t%&^{k zW3oTTJ0PDzIkBV>$m{wiwUSnVXZ_QeoYPq|N$snVe3xW5mYiC%xWt>>< zJ$iR^)Tj%@>$zk)2}k!B`kcJ*d?+&6xZ}Zxd}1m;tPrqLr73|1!0xPs$2?KyPCxaC z2mC08ZsORnDv7WqDie%SGY)9VkGwAsa7sK5`D4seK$OLVPqH%8^iug^k22vgv@(L(|t6fbk>tPH*XX_PU(&C7WB2QBJ?q7Xu3y>Pd^0TvnNhiud%Ehx3%x6ky!5SFee9 zt@pncj{rGM)S1jrwSVIF{*ob-$QU<%5M0y-dv57m{K!7jxg_VY^`1WjySv$Vu%?nW z?K85^^10KWL&L%n;@kHs%(>Gq*3CjEY|{Ncpd`m>OQg^qC>N!zgmz>?|HUwcsz8hy z-XO>ZQKQ1R{RyC_{atQnZ$6k2l#-mT5fa)z>Z5nR{BZHu&kvk?p&GyVWZuJKG0L6J$@e1yj_> zIRWkB0Duj0vq@cYGmUel6afD=3VW`aZQH44)AsUB2uY` zR=3Cb4(Vs(yJ|E`5g?wPCdb zY`$k;h{X-b@sf=~_Lz+I5#(4s5-?4(PIx2n{47Y1M2qg4y-r=RL#ydS zf}SVViaK$7uz8J~%F~Nj^O1J9cR1-iNYZFz{U3o`&C|^}tjl+<0JPMM+gQJYxA<_` z(e9cnVk*~sp*gM-*-;b$f;~vXqjvUCS=kbJb6mFgeZh=bv+-CHUrc--N$8vCf4U|- zNbEXO?Z0?bCq27yW*4JonpFBaZw&3}{?4tnJ0W_F>taVA_G~rGc&20c(nwb$lbSF@ajoh7IVKJ8YVi^mX|_g4WmS~v=0Nnn@I86b9g@O*IH5uK zk~FK^azUkv#d-eceYJsVz%1LoL7O$BFn$x67ypXv?oon*0a9M!z+ye5TSf44hCmTz zIXW9GBVAaiN)_6gWc#~cNwqbVhHOS~jHAn+e+$S{e%ZaFeL$b~E3bWL$!G*3>-=DKa4U`KL#RPO2)81iHpQvmV0-iz2LI7pife9U}K5Zin&RZK( zK2sHl21%*H;3XF6o;Y#BHW!6!nO|!u!bvVD`w&ojgNkCyP3c-MB`hFH+Xs`JeeB@! zP)Cs;I@4jYYW}IKUwSTG_w8ES+O%X{ zzk#U-Uf&NM83^5}M~j_E;bn+a)WqoBK9{wXQhxsEiSlJwWK#Sd&~JhG4;cKpMrg@# zhxGL%j24muZTV1Prp`QXRr@X4XSn0T;=;)xBT3401#!n>>ZSRx`to-43J=jekKxTe z`_SyJWa*7`uJqNIwQg>r6+DAo{?(FWB-LBoYI9LVvDqdqxGmvtkti6!#oG%+zksE? z81WsZ`rLcyH$AYohj^AKH`v&_5dw067^x!**l${T4CwB++9GwD_3qABeyP3JX@wY@ z5h}{ue#;}6FEL!cf99IFK3t%*PVGLsW)b@T!Q}A2FYWw4SN@+#@Be+AF}kuK1H!4@ z={??{coI;FA1JjxmM>2DM?stP$M3=#KE&e)Nv!5Dgvv}u>Z2BAkOL&wX{v7?oGmL*i+Jmj#WvhViC>@QPNm}yO}9WeYW4=%)ETAO#EZEjHR7@Dn0Va8QiC@Dfl#5tg5M0$v0V+9OIqw?uy(L?{Lfy#(C#||E273EJkDm8~-E69;%@ zIiL#fl?z*|RJB|6PqTA7WYHKu2&usm-#(BT7U-6y4N90MaZYTv&XOb$;yj=}s-j@e zhA@YO@j!6z@rG~E*hyBk6xeGq@`hRKUTzWe4K8E4K<=HdcHtYQ*`=#<_(kUA5n*9q zWsXcfOF20kk~0}}N@4>xj-k%g&lZDVHKN@9B$c**eRvyzeor>Cky@yLh${{R!4KEh zhfNh;K3Ki1hSY>wjpIQi{AF8E8<{BCj%50htOIN8Cml~JxhU!#Xof^0RWFn{y$ITr z;PsKYKDMcIQM)NZW~(!GvO<kz~02qx^iMTEuI8^rB705z0R|<_k zJ!rGUkiu?=WV2)C>oMsh^#gD^6d(;o>$e_V&K?Q1*Nm&Zv}^Uz;hxPqAWZQYaf3|L z-@o#`&AD7k+q0+RJ;w6 z6a$?_QGBmcno7}l!CJHLlSG37EN!z}H`UwL<-)TyrYeD{ck1-0eI)I!Z4$EDipWb; zFyuz*s~`SRgkj8q{!Jpcm!wQ>kwEoMS{w4HnzFIUcEp(Rie`iadBJY<`JC}>q}Ct3 z?$KO+rJ32{r;iu2+>qFJ$JvD+(9zNog~xtGLdSCZHKw@GOr=enUF4J~U>b`@fJrgS;f#J5e-6uE@qkBziW;cf^2JRpc23 zQ*TjLnZLm(**Y%>lNUEQs20*tyx5@YZE@qm<@USU>Gfx6u3moW&9UjD{^Cl8lE{Qd ze|;-qh?b7dDZPcz(cB>|sU#{;swkUsw%MvyDmb|e`!<@!&Qk!Y9TqU$A!X${E(JTI z)(mqYO%SIlQJiD*VhnLe%D-#Iah2n*s5HS_`#<0r7uhMd*rwBLzMjmo-^zsSg%QyML zihxQ@6m?w~LMc6{8_`xQJ$=2N9;`X{=eKrNqFIBu!F9J1Xy)O5#*4U7HZ{B76lPVV zo#GsIW94pMl$XKYxU0pEF?TreqFM+?zIQBu1nhurvfoem7}u_8o%c5v2n>{a81(gh zD|>=42Vn}FV3w6C<~NXbAHqi9Imnbz*7V+@3A0~74~!P_iBo4l5#jd0@wb?B^OGBDzQ%) z;wUu|C)xeGnp-}7Cup+}kZ4;-(I=lQ5Ctq>xvOFrn&z!gc+;eH}9UlC+?B>fP()Sqq-dHt{N>q*MWw2B(&a`b4t~I$t9vg@-jZ z7!2mB85epYe7bFg=-`Rh;1DjST+00P!C(u<@fcn#$xbHJ=xJxq=4wO?uIEinW~G{s z1SMQInxM6-5_sogk6_uIZ==edP4&(+*h1g@V_02D3CNhy$XYfqH9FN#o?M~?O>5S?xrE<_}ILFg`0`w5yQfXLhlW|F_8*c*MlzelCX zviURdu?U1n8YK{aL%Gt)V#?J-Uo&O;;-ILHYF})6Z1m2r-%IKS=d>5q;(QuD6FmPB ze(7zfW}F%a`pOCz9QQQl17DyRsLh+r2ng+2@(YN1%kJ-S@c`%Vjtxr^Js|7gV(R*m z7Q8v2$}-#Hh6L8s=j4%V#0Bl2HGM!dIm-fqV1L_6>pccunj&)0D6W~x^N0*{L|@U%plqC=y=OE4kD256LX17-7t$H;%;I!Z^^c%2=Xw`uKjPzy>Lde-loJ@mNYm zln(M3K~$mh_PE%W^&dgyl0c1AnV5jHK~hCz+1ZLp)tdYM5WXIDf?% zkLIw_p0le8_opv9y88jmK&9#>imfrx(lTnuEZMu$OcGJ63AHAaDZ?;8(mt1>gP9_? zs-aV~>+P&4-70cAF$D<9mfVsw3)yS+9Z`Q)03XaunaI@?riWq9<_k~;aA$7ogx{6? zzCU@c@Z@9P@E+ILx%<|dpgvPqzlb}~uv$4>lr;hYC{-d^g&{8OyiK5IFGG?CTN!x_ zKsI5+U7p=@K(hV-Db^4h&o`ZZ+)M_0)Y-Aax$y9-3uu#&7Up zQ%WI(Bzloz{MK7=^*P5ysMEbHSEECpORtak6zHZ->r!20RPBxmX^0HVnO9s{vGaQ8 z4<4U{dI1TQ75fK}fm9%x4*^HiiBiF zL#@|Gg=qBpv7>>Gth+@+q#gsi)N%FFqoa2&C>p~L1Fp3y8@6t+45DktBO>2wCV&0D zsPtRZ70{_+MB;TCLaJ$QnzPIJ3KmP?arWn6*%Mx!B_9=uobU*^Sf#iq=+c(M1X})7 zr}ufk=F&t#7$HHTrdX|t6}Ct4Y&!iM;zu@!?h<9KTr?ByjP@vr+EE=0QhyyKCKR>z z5_(p&yb}D2E=tY6dh_;cC16ZhwI8x;_49*%9d#Sp zQ<5JE3Q>A3glra2!Jep2iUK6AZZhPBD>19autCPEY$D`1>ZI*%wYJGdW0`~(6H6(4 zqq2|?GFReU!F;6Ff|Man28=ARrJkPChJSkqby&j&x2J^FLj*&xAX0@h*)|&Em_Bc3 z?+B_=vQc{`D!co62pS>LC^Y89kYJhOx5uf+P>ao>&&8*`kEyOYmuqTb5&<-t^J=F~ zk`xmmr|&1a#12Rm+wtEhlg6K7`XCn{xa4!pg=s4gOh_kVbtFbQAtgWQ%Pu@yl1uS> zrBL=|2)aGN-ZZ-OubA_14{-X?g>vO#sLWvOz_o;Of?}1wW zkWv837o!?Bs7_mWaG5%UQ&yGC`oZ{2Wl7@-O-I-dL2pE?Abjf-+bVH#WU&$VN4I@s zIeMCvRsKNDnj3JKyKjvN^~M&^SibBGDUq5HV%@~3;*-b^9l_pO)y6X=Ecot9m#+`~ z`=orun(523cs5btHCEh}u==M=^<(#Me);;tEbEvLiR?txDnNMKNuf&#>S-+z)MJ50U4@X{O0NS zgv#?ykJz&kNfT^qB1Qr%B3(nPB+6@a`JD3oPnov*A%>xZ>{uu zC)FI?SB?K}nq5Wi;FvfmRb(eFBrBERkwkvKK53v*MTu0dz^kaC9ug2rydY7C=&(9> z-~{PV6(3~BnGwt#5^NtIj>aO6T%T8m^#T?~nEUZ?^Q;RsA6 zrCa1zI+7TZ9wMITknBS6;aX4T<*x7y_yQdPzU%$;>s~ncAyU{!DBv!5i~u>{;X{8})qg*qR%BdqbkU73VFj&@b5Yjk)Z+ z1kzfgWCyyrCCK#{9j8^k;^0E^gj>}IYLSXP*0#Gix+1GIDoof!Ezkd0H}^QiE?Ir7 z+nIl6c#p|Boo#T25fok1Sv7;HOquQlAor;}_3k)!C5Lg!sxh~tL|nJQLbDX`1VqR%|IvB4us zVniS)HFX*nm!l>-pldrtf>rXWv+}#uM$W5GhfMxZ7+<#nc@q3_oo_iL*Y=pd8z@_g+J zu?>MJ5iJFPGmBx7D#;9X0PxB=sJ3>`KeW;NMX^E&vT(fF&9<(UGN1j z%{NpdLD_xwlN;5ZNkD!2$&wQ}?2gLew`mJy_boN0)8b0Tv3Qf4l*MV-w`sLsA|mL~ zM98TD6iWIOs3ESQ4quaM227j>2%2Dqv7Ty3%lJGv=)5rj3bUba0)1CMP#UdM-@EDJ zGvZIwFVo;rPqgHC6amB{$?9`peJscAA6uoyE-9c!+MxX;TK@^9>cFaL}n;e4T7gk7$Bjr>0zdLD1&qt*muea z4?5-V5xJ}*4uiP)VCQaOF^H%hF8189>sQ@lk0ddBkmgX z7$QsRgh-lN4KumQZT)L2g#|ud<+bRc&Ey4@_C`8uAh2A6rVOu9s`R zXpQRVR((N|HSm?Ce%UhSE*E1hOh^_+3;nmHL-iVA3tnKNt7c`XanTX6vX)vMWkAEu zKUw04<=k-Z3^s}^ttJS-()!{vDg&%PsCcWQ_$EtH@YwcD{kNrvQNL81s`Q%av5A%~ zcYUd+k@;ygRWj#*jit3^{as12dar3AKQoqr22HX;jVZ<1-`)!B*Vc~d6{TjZo6KX2 zJA5gBs$aRa%3J>NpC~OEF*U0Rl$P%>jy{nzLOAm+x8s~p$7ACT6kc_c;xH4_%5Yh< z+HWMiLnSQou`m1%lP-mev8#l13+V9f{?XedhPkYwcmluKt^@h$wO5keNR2r~ASFYO zu~K(trSJ=51936=&iHiYck02~nK4hRtJIpTG2r(Km%VuZlyC{RPkR)88YMsD^`H2s zY1(^M$PYtSNKgPkwvgQ-S=S(>^@xPN4%q*Z+RwP#f_)ctN8#bZzG@mgauhmsP z+c;h#0CCcyPB^LV20+b95)tr$98$9IEcgLm!Bs&0#skhQx%DSLLIMl1qKA(%^~P5t z#|CXCGL^LCP~PCTa^XgHNCg+*QY{xASn&4q6)9nopaQL$7Gt!2%Cxy#=(InOo+3%2 zcBC$*d!71CWG?C2Ipv6O-b@-~&finUvMo?KP z-6V9vwfT{89XqT|%bY8i`}G0{ixlzPt&Wb_`vl8%q_#ptPq`E=g&Yu~_91Pa1v23@ zSf|7@l8;LEB&K4&m7P*BjBnr7tM?jtKG#dDa}U@gO-7Wiu1%t9PNgX{^T{X~oVdYT zwpJUShj$fhRHeE0>8df5>NQgRo6y6xdWG}^Q887QQcLOW;H@b*ZQjs@NJY^QXuDHd zx{wO;y2)b8ivPBd1#YYsm>=HO!GyKrLgb#dYXec*0df)!we$sQvJcOq!IlSN^j^wB zAe%>)v-n&{Q-w1q8wf65jcZ4dCh+eC=UrGL2RDI`DpVVcE z|4RX}A?6+*KE1i7NaNlMfbf{Rf&pVA45e1vHj%)jTbxrP*o4c7$LRahP(!xj+uv&0 z?I)*Picd&jrWT6REnf26bGe*vcR+*^g-)}rNiviwDDK>0$%FAFj&8{->dWp6Nv0yN zBKO=UmsAS2A@E<3)djGqw4Pl>imoOqq+(0C4Cqut2uowjjZ!u zW>DvMarg2HF=~@y)<8N@srhTDR@pr<_LU&=DnUp5@_jH&U6p9^FRhb;4|!d)ICA~H z2gjKDijGcD93cg?Q~G6kHRIj7nYzLh7RKakuFHP^q<=f_!lv4fDpG73AM;p=vCk6% z56o5gH+9u~KbUp9Lq(Ew1B(l4N()Bq^IT#^$7_+s|fhhR863ZdD(fPb{G}KqfrJ6kIwQV6Y+gF9(+wT zAE@NOkX7>%v)V}Z1;%Y4eL$Gg!D+Tt0{^I_Qrw|rbnP7l99o2~K&Vf~*+0*p9iSN} zc!7Xm(#S{eBjxvwHQab+byW#pX+tV2{E@l_n38%u)DAle57l0AjxrS$*jp3u_`G4V zfHWoY5b4`=dkU9F99sA4Z=o(Fufxl@i`^nFud&l_5NBVvlQ5Q-vGXI;?xlytok~rS zx;%`sz=hac;eIJ1^bj(IhDOuRbon-~&5tv$r#s0O;fk+uLm zVD22$0JeP7TaBR2^P?ARc0~E0Ge>--yr!+ME&1Z_|69vp|6MQpKVl{Sr|W(H|Mh^? zw~1z@x<`8G94e9^l>kM$=G~5)Q(#X!MApk#nW#_Icy?zoF2cl03o=mvlSiLQN!UQ* zCRL$QoGfi6B*&RWpSsT7wsuS^maNZsrf~SK)%k?|W}IwNMBSA%`jV(k>I2gfN{|={ zgabai4~8L!UQr9aa4-mptgG9PzudN}=3X?A4!UmXAG7Rgbq3Mk+fEHyBuQ%bSgxFMMFec?CNYrAc zHJf{;_SSz?>%*r-JaZ~aRQDZ_<}z?QGHop9IF*#tIAV{BNCX!EvL%t-`iyL% zrztQdaLT0Y&!27zP9kKEw0`nABYyS&V(;CL^no3D$&bIYASf%*K#;a%s%*jn z$SyUH@*&Mw?Q=u2?(1ywX>sbP^4rr$RQLW-NWS^)ZsO|9QQoHaeztL}>}w58B~ z6nSw*wu|q_K9f|^%Rp*^<|WiIx1@-`T9C1$gpZcdccK|ctP}!N2wQr5+YHdH=`Vx# z)RQ?d1l`5XTn-A+QHl;Jem2}hRg|t#&;zfeM;L18ptR74O@I5rpDr0@sLP+VWSzj zqCrY8Z*%?F?y?*sFv-;5_tpO@9?GIE`i;`SE@B@+@btcwsbK^7;T-~2`u`xlX0&rPfg#YVY>!uMqGl?!|4Pg!+VtbC{20Y zbo=(bBe+oJ#oe~VnNfw^^wo$HGS?GaYN|Qeyy+$z%wm%5@+SX{bUTvs zb*=U(VS{*ji^D^#Q%qwxFePxrV3B-e{*H(-N_B44wtaNkZm*{N>4ip|NENj;*D$|6PNc zbvRHNL3c={TK~LPH=3En$STgYwXJpr?MylD4A&)XPU?&AmR;@3RrxvI2#!jBtuhp!c$R!T4_d| zKbu&&?p)$m3qqgaHJf%^3j^7S{agqF=8Ub6fLOgP+%84+h-rS(FjCh5`%8G^qTZhD3T>kplj&Ru(70zj#Q^)say;X?G8xsB5mJh zbFKYkT7r0wWUzx;I$8i}C^fKrS`_uZj#9Shl8IZo*^v=Mycdq?C0(W=9Ssppgzi-v z!rpVA@|kC)Yxd2T+>3tIb_Xz@B)%$z)f2he8ID+H2!P=3N@E@V4?|H_p~^iWsg zY^&D)w9Ft#)5Nzvq71`KKoa3JBM$EK9Q){zwQz;}nX{yq){OpS?scNW03I0~wC5h% zYE8)#j_Ev~%vTl3P+D7lo!0F6LPmyOT0IrLSg7q0M&IzOv3797X1OiBKK+7;$V+;a zBRU=+C`gtUYa8vUQh%O9-L|I9;TDf){?+pjZKQbv2AFC zR=naeMMNOG^hVtIBz_M5#(=j|s>J3g!($h?6>0@H^z>ojteF@5bI*x8b=3?mSGTAu zi{TUu(87F)`tV(I!hL)cGLl40mn-dHFU9jwH1KERzhD}n*|mSsf4y&_wHEaLgac#3 zdlw(BUaubT$J^bz+)L}sV-XLe2UpU9(4QSgyRA`K(W5^F!ExH?iiZMzq-cmj-p_{& zyCd>2Om)J|#l}&ytk`MDjuf3o^enuUFGcY@ zL|#!ZV;-Z-L1mJLJV5?z4%%~hiCBa^3dD^rS!u1$zxD5uZ{(r8a8aSv$xXpCs0X;# zZC#db9ZKD`WRm=dKXbx#wfb29x7hdf2Z#=PH1~=kDf-_Qi+cmm%C>c}**e6-%=D3A zW%-gD-(EOp1#c1j|G7j438sCUa>EPvPWwLr?C%}n`&rYsL^F`S`uWc(`ikvaOy+`R z!hnpj60eD1Ju#g^vnH?WTFsosWjxAhRB;bLa$;1jq9JGx_h^KXfYXZU|8i%>n1u|b z^}(|=PufRienG_Gk1{JjN=H^=unhMC=*_hlz~fyMx=1b09T_JrtV5ozc_5y1nO?j4 z;mk=M1p^?27YLkDB$s+tUKH_XKmEm$w8f9W9NC2u$;MA2YkIrgP>YIVe^-YNc(2f^ zSRVEB^|u8Iu=*w7NZrSzSwcF;xDHYGW}#I`TZP^PE3PFZ<}0cBu8T}wRVdlcuL4Hm z3y=aH7tCnpql6YO=apwBeC5;Tc$0(kvE;~{N6~xYUZHqqN_j1iE5g8%akg`p=CRDEGF-nY1M5M$VS zTK{ESw@ha-4mUfUM&uW0Erzk;FH`wGV@LO|T&JdxjpJ}g)EvnZZ9|xq?Zd?a(kJoT zSSSAdzm3Ui@xH8u46u@xGFDJ9RQc2RwFe5${ahC*dOcFIR~q|G%>1MhefK%MgZv3O z+3EH)%z;yGiUC-Btg?#4t4CqkS0?@<*$aY&4{|$n!xLni3CE}|5h!z}F*`I0x5g_; z$gGy)Pk~?G7Z7w$NEX_RRwrxuGbMvbnz9fO5ZANSWbjB$k$KWBW^FoCSv?won(8Db z;P=#n!O}hhUKmIZk+d4c@!xWmbv|`i-|O?gQ8M0I^O!7fl!qY`8?cDg=!mIE0)wCx zanJ~zsgUu1GVo*G4NK}d0j1I_yNq5K5jkaQv+zce;Ni3Ukxe6-l!_oDfA-oz$=$=7 zq;9p4&Yd#yb`5#=Hr4h)DQcOHICwQHAz2t#MHS!UU|VyGv&$R+ICsRar--B6D$e0Uae<#B|UXgn?K1&8JbDNGiLtHd_=7g_K=S8;U~0A8)q_x;?~% zB)q&#@RZzo=aUTnQ*Wn&^)epGUr~icPX?G@QA;c8UksXgwn3XLsz@0oDf5Sz-)AkJ z{a4~lQ&7Z}jM{CFiZ9b7r8kGB0vY^O|86?#W&N&}ormFC+lG9a~nH9=Tg9E(Qam98~>Y@+6W;?+LE%y*S*|z^ZF`QpR~kEOC|71caG>}@+V|>Ckx%hG*oADzsdnR1>!fr&|&cW z0z=#u%lKX3r4@N(^&QTmOj>41JpOy&{~{0{9+})!Fm${5c7LIh^b^Okn&)-&K&OlX z^a*`0Jiuw$$S?Vb>8dvv{u`y>2T=d15RoWEPe=`sfjFLHgiyDFh-*q#c^z2!RGuaF z$P9OPOnv#Nkdx$41u>xlVyBC4ODU|+*O)4W9ESIYFGha5Ki;2@()T2D!eCf*lw#Qx z7gV{$I`*2h*M?TPVtlwOt#?AiOTSN720RDEiOfrf;dD$7%>w>ivPH*fi<=`uKDk)0 zg7~^;S}*1!Fv*y_1JajMMP6qN?jiBPk^W>j)cikVzg_8!QfUu^RW-PKV{o5S5@fhD z&vKi8h4|#Nxc`5|QWu#xa`(BjU+N6Mj5^o(m+NB`8+!L>r5)Pm=$n<+cTCzG{WyWpvKfm0wiCR(r_tpRT<^SBU z|11)A!+$o0#(yG-#(#pr|2`PJD|e~a@fzHI1c0&#E7I&4GkiFSJrXgu7cQA+`U3Jb zG=l!3N-Bmxk=DgF?c1M(Su@7xZE|+WwFJvf>em!%9^Dj<52OQi-qf>a&!$F$GnUgg z_L%~CEq2ZIk;Bw4^wEA4VHpn6Dn4U1!h(1nnc@ddqyqdAELj=bLf%*27q* zyoZS!*7Y;{nrlbdg34r~9_N#Wb6i&M?%l_#K7ixGIWsI^j7K~^0!t?5-$?OO=W3rz zXH2eG9~`ybgrn|!Qu5P)#D$Oc(m`GECUchR0{~~kaT(#Gq2V)8^KLC|0VNGJH4`bh zR6!izd@kT;^$+XD^?ET-jz|6U$yMAKw>TeA^^qdZNX6Ck8Dw1iUj4eUonl?fj&XXN zLdk)n#%Z1)nXs`N>u=k-^*$1ETKw0~N*nB7X79F5WDA#)C!rX?=5esN>Lc&osA0oi zJ$j5oTAx7|SicPk9N?#Y5n9p9KPf3`Tf4q)PX`~^54$+D?VbF7{>AYv*Ud8S|6t*b z+%}zFrIZ#myB##^k1s7=$EMxyx4q1>qC=_q?y^H=1D-{8%^RJ!G-2WB>l?1$a7}U> z>YRX>HJ?Kox9!-GFfQGC3Ix%@yYuLWiLTM97Nl*fQ(gH)_UyzEYf$ zo>yz|U*Bj)<&NFz59-)0ueK=oWL3SSVz&mIuIJ9qPNlrbt%JkMcMhNPol9`lM&qmqoy&jMc&~f++1cLJ z@XmjEb&I^wOS$^aT>W*|q#=dYp58OASbwUjtuEfWeY^45>2`Lf6qe~he@jo+(q{j( z_{-J0S)G1t+y2j-^cE-NnO$+cae7nP5381V=DfOI+}uHH$GYsv9ZF;UzP`X_%9v;R z-A6ZXQ9E>mGd-SGS$6|Rp4_*ugE9@8c<-{c>htoEi@2X#43*@5*AAP5qf=;d_2z z{AefrufV=$JJ*zrEA`f2J^x_6ojC0ISyx#N;iIK=d->O-wD_-7v(FaRI-c>spc34Q zraUS6h_<>WiqppGBBr}#&pnPEJv}{FJUFpaQAi6oW#u0Sqj42;w!utQ5T)sc;4$hi z?K6ZoYuvKyDVhRDW-+ucoB@!@g?Y|#2__t_$DcyK!!gZ`O0gEVU~f+1c+?q`lBQ>u z{QB#}i7PN&tAa33hf(P+i1_M)a5cgHX`1Beap$n%LQbCYp^K~@u|~em&vJi2M+JHqgC7_w&He?J>R-H#@T zRsLTKleEW#={Bue9gU40_7oXrlNj4HG-AL1A9aCyf~EQg-~JIhyw$(sC)9t_bzHBn z5AW;GT^k<#x9*X@$Z56dUtfs)U++mj{%?QTyXTlO3otE}yI?kLInw>p8BQEduoMzH z?JV60#Whvs&-5E;|J1Ov7d?sik}D^FFYE9Yv>f zFj0~-KPS6UP*yoN8XNW|D`q;UT5aoXb}EFQy(T1=xhTDF*gZZoF8`L_0~U28U*!F> ztp@`3+cKUrV4(Fj>{UAZ_s1C5Pvdy_9i0r8A0Z(XQQ*s$iIv8FsJY(e5eFfXj=;{} z$kKXhVsZUqAD>gS`^+M#Udn2GpUFTok|Re-x?@KyIQ6|zXFf2UemX7X3N|m=TJx$;KBm|2 z6u@(YaniVCxM)V76@7-K=~$@|@q|-7(mC*!^&W5EZP2nExC34WhiTlE6MDIaVVwLa zcO6!47KClvpMH8%lIGp{Eo#LhRPk^u2X`{fI1Tqs;7D^0B36G1&YjI5f@mp&C;O#w ze5fC6XFu~s*{<#TEtriP&j1HPObM>IbROg$!bU$+mz9g#eP=Y0z+VAZrc}w@V#ZWB zS!jgUmju+gzWvM_XV2kK-iLBy3=IvZ9v)-KrE+k3*stB|4aG0+reu`lscDBsg<>`v z(QO#HDukQ-ASJz?ZBK<_0>>{)t^%A~-_|r%m%ghhkl=rEQ%(_WEU|n)Ldn`Qa-9PT z{}ixDV2p7?pK*QWoEZ!3FtxG@;@O^nGY;oy8r7E;>&m-sJCIELc^GAFeC_)6FdBE~ zag-a4V!y~Yf@BKi+kM6r_mbni>o;n4Gb6*C)`IQK4bLrk?``mxmT|kP>S{f&FEvjt zd0gb}<(n09s_kpcEi!w*ZJRbhoEL5juhn~g^eIkXKgC%Pt{1P)|L$nR9K%a>UpBPp zWMFjZ(xuP14j54HdcpR=M0oF6M2rg@4ZO;~N%joZRfHmxh9oy^42c#ka)!gYKICHC z{P^P$*zeb@sS#{QxM|&m3m5L}Fkn1;@R@$=cd_r@IaXCw*^=`cwsw4g)y>zX^x5fw z*0+SfFUmZ35}!i^Y>lzz;lg?re!Oo8H*4GO-5JQv$B4^eFg=~dxW_{FpTMcIy5+L$ z*_;s{78246cg#`_RqCae{Ne;(TbpM&nZQ@{`0Q~>5^R@A&{8IHCClz~0!iK?u6AwT zwQB^+a_Yfhe=Vx-Fi)<8RV)$kv_+f=&DCjiyB!B4rQE%{yK1AsgkJSN?M5LpQn`J6 z6UxpXp4Fg!{XM+iPj}Wt)?->xdiYw-bz0@uH2X&Y;v)`4P$;1dT~=-fAAvv9L4FsO z-yaX*Jj24`;LuwQBeJlD$Tn1_$Yfa$UJMpgiS+J8m3L)OyG|s zA4V5?^L;17Y5jpPo40Al>ik>z=e6{IoyUpZzfPF2oYb;y=gtbVP>r-W{6DXuJ)5;` znSsXr)XX=9In*oaUr@?#;>edW{@*aH7ySl`64sGAvyTdd^PKds+X z)&iI$zfYwlACt7Fq^&o;#AnLZxMS0?m67?#g#CNxG3jWw?mnhlYBd2M#-JsGwQ$Vw$uj^;t+~VTZyX9h0f=pE?%gV~oGyFebmwC;> z?zaL*#>nn#*0%R+Qp1WT=R?g-&3Tb{tX`1u)<~Tc|9A#QypSmn7`rsp|oW3Z@>MfIzPzJd;V0G`yfLfi~dJhm~LF#2OQnK#*BHE zl3qVU+i7sa49$yFr|`!tTlNtTRDdFU z=J73o7>^2L))t#JZ_=c{GJ5ep->vd%9(0TKX(tYj=zbQl_Y|4GHEN!AruXFy7>1jM zvk7pyFe#0ER-So+)4`*$eDyPJ610H$L|nVBn7iwz;imxPW(lrNzE)$#jFFz=;U0N; zd8Y{x+R1FNm)FAT1x=#4WKHTu3fGljs~8r;K84=w5FaAaFpri$4O=wa`DK44$I|te zD_DSXHbvF8--7Ji+*~RIYwhG_ZF((|P5hwWz=3HLf-6*SaHPFoa&2spvEE$PxYoB8YRxl zB}uqI$bEvLf+tSk_K{S=tUl_F`Pp|3SFHhXQ*}$$;(0hhisqEO8!kA5dd@dfs4paK|g^+4dbZ%7VK%n_GXb zgjV*MWx3Y^ll-9ys@~_9Ph5X_a2(Y)LHKpeQxE{3fK7a7n~ohr$x-qDAOjTd| zs)0gx;U#y(`jd(QhTc2ExjFAED>G1G*hJm!vtwtf4A#H5l4@XDl8?`iaw?~4U$?Q> z_YCv&^-Zg}Ofst=Y4ra2=PSO!^=$RIAQG`d%u8K%6R_k}O(JZ8WR0P>8br*cFw1&Q zvYwQv8dL8p zCybjNtF?bWkNX%7;wSb4$IL{yUi0bQV%c!OhIAMoFpy46zvZoW`^HYOB&enxaqeCr z`IO^d6>;56C@3nJ>7xfX!MNl-$BrHQfT+_O&Oo`#iKMP{ZeE~KICFB|Xf2X5QK0?Y zt$X_|j3Dq8Qe06q9v@|*<9J+4p;!*kDB{d53`xHNHhO1zaz^^LUAtVg|GIl{_;JcW zTNJC_Gmd!I)YcMt`hZ!sSR4Q40K*iHTM$a@Fcm<~lYkq5*e8`CWK+f9OSn21#3!!cNWwpO;1`%)UqUcsp36Q`qi@iv z+~TI?4U9^6bsZhXJ&j9TH@VKU=~wu{`4zzx#Tj}A2D6#PHT7c$OEO8t zpEUJr7%c+FiOX(5(=NAWD<2`J6=Sk~K!EfO)X`gbDS=B8kfp4vd@9p9+OXtFSG8zr zVP>ZTc6Bp6^k{m$*cDQXZq?M94P+^KdNCvOEx?wsD}}^5$?xpEP_UT{RC{0MR2NXg z2c*`u9`oS2>n2S{V}O=FO`)d+8B?kxf7d$5EApVMXoYV7{{5BJ$Y7QA``5&|ccQK~plwij0WBK?52G*+{y_a&?Ml2f5n!`VT)1 zI|7laGt%%!ig>cX1WTR;!9fVi!ZhE9?B=B z)o;+CZ~y-H6fBNAeTEDfl6i^h^0YGQ=U=x0v2PqK!F+M%$dT^8R&rivc-qF5@6o46 zR!-_|67;8O$c(?;Jv;(8jF%Cy)1Kp`Oo@9Qr6_)Ia`&}s*XC05@9a3}ddE@>#AUgI z>YKRq&8tmfu31lCE2b4m?wRZ8ct#mT#W{EU`0<^_H1EC^QQ7L(ZEXf9_mWA(6Mc%V z0_|kJXbJ0G`B9g6m>#+CIgG(_)v0eu_@oD;p;4SW*$G^yGyJ#Zk%YqZGlImbN;Qp6qLSJd|nhK*+`G$$B%>6 z-BPPeY2hkia{)W(o|w0>exoTW7W3Wt(;LU0TG_F*?gFU8*Q%TQyNx``yR7wPs#B!* zsR|!C1q`;&7SrE~yeBUrtW#~ozwSRM`%8U`V$!JonB_ET1=~ zMNU{nNmj=C#OV(1br}R(JJlPE)X}eBFSn_Nk(jEAv(TTP;Cfca_>y(2$(Q>IiT=tt z&WFE@ z6#TM$(V`<d`_7c$KCFZy$Q7nd7ga9ZUU-o+l+KqX=ECjK*$} zlrf^UH6nv!^tz&l$qKE`x0H6q`aT*#HYC`=%Dqq+BWbxcEjdRTwn;EQHRnFwtHtcj zA2Fx8^SXpvk2ht_=PiHyk;UaC zRP6C?6TB>8;R<*u?PLtlJOW~+ucWG`3Q0hFR$%X0O;Yc2I$LD(Nx zyOG|jK?w;nMmt&q@KgPoCkM~D2wEdhYA4s!)W|P6lx%O;H;Tr!`9?;60*2@D^yG=! znvQXt8nG_n!2XKD2RD>gG>yhiEQeD&ph%(EAYMK{EsYj~{8Ow^J1!6`ZN{hGwqw-r zUPCX>`b}>-a_Z_T)hQCK@(G3RGGb&5Zp|B5vKF`3w{U!dZzMONo8n6w&?(ojZ*P7B zBO5qIv2mYc;{2U2+-|Oi_;?x`z#&pg^#+>d{LPz2{YEv&xy1v0aeG5;28(5is=O-( zsa5e8;1!>8wmf5_Uk>1_5f07*mM$jVs9%^|v18pJIo2fA-qU+*{eddzjgjrM;EOJw zE}=WRzw#p^1m_S2vfAd7c+7B2$#n=XSCIRAUA*Iq-|ie5oh8~(XSlblUt3TJvBRU{ zJPzIErc9-Gpl?c2{*UYp-62CB{r3CsB{)z93^HbI;VT=yDius~j;%hQC1l62oR7eX z4dL!N+cYCT#$o4CwRpnF$}Eb{hW6ot~FYOkcGdZmjClr#bAX!M15vhqnT)>+s@X1R?_^H+Wb~353Eq@MMf~GjQ|? zX9L9Cgh>0LA%|m#)bKB&5?NjhLL|N-)b(e0WB8fpDIWTQ4 zAnWu`-XD57ZT_9|SxY%Tcg1*NW6QrJtFGO=$-=pNVCY@jo<80?vF!lTbUNL6;3@MB|nP#@-ghq?KjU$4OP`%4T5YWq$wqO#M>`Sbh|*^``F&IyFzgl zuB-6FaM~wYbRG3rN)EEhx{DqoNBusdzQbI(`1AWeeih*sgq*5Vv~6 z7RvAi74}^eHi%z3RoBV$kcH z<;z2|{zx_E)}^dL&iH$~Y5maEUuxE;sI;~#qhK(HWdW&|(!#Y#83nHsus;)wbg<=- z?7B7`?<1grniu7^=$OVv@6)~eWkn`-uf^c2C^}s44bP1Ku)q~y5zURE*hUFVG&gYU zLI7Bz>O&>u^5ROPk(_3WGL!7??k*PoX-UM3k*!-&&{LXfq+Prk-mG73{o=Gm_4QWt zI4%YI6xCpd?8kF0^?UUA`X(*EeK%wRR;c@xlGI8+uHq>YSnyAduqMAk^12+{Hg7 z-+gc7oU_NuGe?)X%%adrE%6W`Y1LkL?cNy+H z0h?f?S%gI>bHaX62{#V*G*WTxey=Vx7+RmZx{Ylw93C^&c4vJA&=n&<%49g1fVkw^ zHeKk4BDR!j>8GkHrHZ}r;Fwi+Qz=-PiZG!#BQk?uc>2V-Aoa8eK&lTIHeZ8{_9&pf zbg$(>BX5Y9qfz*bwYP?Xv`yAI>y#5rv&^#hTto;>EuM_~f--mZ@rfImd*Coi%)ay#upN~24-1K@JXW|=)(H) zWd%XTs@%LhsT`FibLQ+bKh)S!YnhBSvW#w-*JbEAFtRAF=IofZs0X04Fup$K2dB>s zzxytXjC;#Z&tINA6`4-cu2-*KU5?_S3Zg0tE?lhKMiL!vS4~zQjgzxSs3gx zz${o13az8uHQ4RLa;3T!<^cuYX{`}4jtgd=s)7Iv%OR{CJm>=KLD%aBRS-dNhQ7aE zX62uhLrN7C|Lx7qpXEN?+{wFhI0cJC<%Wb0C)wOC?{db)eBm9=!>47j?gOo5qG-8a5_I3Q-rn9;cm21u(On|p zq}UL!vpUX#r8kP?>-6@Q1H-z#V5KC`W0LIT2Y=lWqn~mjEosv z#gct+%snT-BG&P5mXx)s5BudKoJW9SB{h0fZS{x3yqf1s$sTzys|s-O*t=eXgQXf? zsC)+Q?PA-2<_+kqE=g#$oAV`Eb~b>7^BWs9p2x-tY=2BG-`aewx>PL`8ijM2qvcG4 zW^T6cddwoWvqwAHi=`??dEw!Atvi(7{F65+$gVq2Wgz;)ph5HLK#^;tq*OhP$W$b z?9f5Tu=$WhQ&hZN@10?dBf!`j4wkzqLx0fj|Lm`sLVl*keWAe%mf?}h+icrL&Dy}b z^Q{%tj95vj+QAXS^E;1nJ;Ua_Q|0x6BEj{=ehvT1=VPw`R^D9M+eaBYP zi5qwhZ3$+-%&Kzxwr$SO^z>_!4;$8*F4S!t6#TM&)2+DT$N5@OyH0hggWJgDl!jaj zre85rs@3*C|1{eEzFyEELEXwI$-=h0OB8cn_AAOODlWib9mq^FpP@C)U8*!yK$&MK zBdq2jG2MZ>S7b60bvdpn+nFOA?`YmwO(>@9{1EWF`a6DQZ78Ciz?naNq8a(4p<~Y2 zsA`wuM}3oP#*)2wFZ~w18%Ioa-neN3V#r|SBgPuGsjNd#J56AA=mC(~X6Z7mo~?lK zPHJo;rHxJ&!*7*(*eR4znp%^GbsGjTzUlzu0XvyBvq*Zlt8#)cQXt|v1g{cln*SZ- z!8zm9`uGMO-@YvermXzB5Ts=YMW_!pl8+b~;x7ls_XdQ`W%{&pzNek~WHOaOb zVA6Sr-M;jYng0kzW<0fH^|AIw{W$jSaGM5Amo0;zqU3{0SWq?x{CbABdBUi)QKtId z-yZ=)XeZ00IlB$+?g@B-?#J~t$(>NZy|$;NhedES$R$As>C*?{@yh#`$xG?_`XMIn zm-q{8G+(QuM~}MHd~(Xdc{sSSY{i@8)uk#s{;7Ndr)dw^Jqwv6G3hk!O1ScYF=HU- z&%$rc@#}4R3VT8{;Y?Vg6<9H*4DeaBOk|_-^73=?xCpELHk=}q&Yv)0!nW|<-Af>4 zruHPFDl9)1CGm(xv_1s=JoTR<@6N>&cT|F?H$+Xy+2RVNb?3&78z94%cE(ftZ!Fuj zV>u;(L^syPWT&q=PBq`zr6V(uKvc(GXMV}_+fUm-CUdbU?;I$Kx4J&(0?o|IWgBs zf+UoA7;h889e2nA|DKd|t$KKN=_Ju%C2xb8+q7*POkG-pJh~vHgHa%LH27ehSaa6W zgQaC#Sy^cgk*ZLtzig3))r74D#tb(?XR?SwDRLLsntyx$EFwlapyYu(3#m*vFa))} z-1YS3iV$2sauz*J`hRD-AX=`(E!M0>i*%8&7`GH?==CK^otMO#C&CEaV}^Y-7~vt_ zB4rwZLtGq16gQ%IS}zmNhd8Ttsy5D+{UCUZV5F7xh4!tERX9?`Kra$Kl;JA|+ zm)xrkCY8cgGly}{;lqbXPscD-g@Oe{>t&2p7-e7)16%GBe-q{mb}WIw#ozI$O%_F9 z5acL_Ui`wdnEKBxZlb&=AQ2UOx5Bs5pPONdIZ#VU2b1YMzsA`!#6Su1+^?A77`*5T zsCKZNXQAIV_66TPm~1}WYxnl;v-qX9ty>@GM4jd6pmI%Du3UK-vuYt#YQ&hwao5E&4Vs*`@-mL6EiTQRue3L@Lr=&P(<$D zqbMX}iA*%>O-hyKz-{!{y=Kn1GAgia*8vEUiywGTWUkNKJZ3f1euHR5&P-n!nRo)Q zbz)rl<(+3;O6HBG^XoI9X4acLEDA7Y(|`vV=q>rEVviOe>nLxx%z5% z9n*g=+E~1razfP170^qv$e~-Uynjo8TN6i#W94)``O1DdW$bXsy+sIhDeOye~l>rr)^+|SAgqo-F^u2@SyiQ zx$7V0T2>A5Sk2-yObHiOl3lZ>$fQ1^R44!}3S)(kp96B@@_$S7c{gIK8O6ThJxG0}~4jkk0M2HMu ztH1yLdk8LnDc7I0?>o;5gQ76|1Wemn1Ml!C(oT%i0#rnJ`Z+33uIzhg>=!#K5O@e( zgOo0u-;s&X65$(ph4e|QojvsfvD#Mqjf6>Ep zijmP7Oy8-tUOa%lm>F#AQ22yT(oPl`T!x*N!_V+o8cLr;AtFfG2C$}J?s!Ri!ce8z zu?bsePv1iZd3MP2vu#1jQ+9;h_u_Tv?gh>LaOGuT0$c(wbVwW{in)}$5kQ-6x;O)Ud^YM&O>8dHtk}At;sCNsFg7AuwadKeDFVR zncdBXNwzG_+xgC!;QCdIHecS^s#~{JxGEj%FUPfTfp~Kdh~S`N*vnNwhUc}s8g77q z8!9*#wzfgYPZ`^1)K5zkPNb>qnz#$HwX~CEK9yCaJm>t>z^uNuIJ9h-1gX7nPLHQT zTcaAMQh}_$w&=Hxd0jf*WdP+tz3wO)FCGn9e5Il9p4UF+-rg^kJdH8QBF;IgT9Tw0vNrO= zGrDj0?%llk>o9d?VnD)6ZRV^tO;)pOrl>^fHXoJB6^Sw`UmjAYCaS%^pllkW_vI#J zupLhGz*09KOl~xIHcY z>RB};b_`LP0q(2}D+TT&E&i4Jg?kuH&Dt40N!VKRw%(1>euv5&R_X^Og6QK|Sfjyr zPsXkksCSi95P|s|6)n^R48?{Qo7R_qF48wH86JC$>NM3bdIcrk>`R0Y3@ma zMPJ7fnR{gQyERnkIu5-loeqxj{mF*d{t*C4y}4yIw2_V7Va`&_Jmfmxx9$s6Y940n3t9ujhyg+XVkWZhQ8RIrHs9I?_L}8R@7=jP@HhG z_AbDrJr6oUMA2o-uv@_R*-FVE8lE`+-jW6pd#UYXR*SK6cue)@(&_A$zOd6J!w5|m z*o^WW*Pz@4C=Lv=*>z6DkJOTNU#cI3oA*2UoPzE&8CeFtWZ`5Kv2-opaH&eB8X8B} zvgfpu*;1K2EZHxtZYt6`Mymti*$IX1_^?9}$p=6;GkT6PBJAI2@O47d{RA*m)Ln4}#r_{UM z0M}DsK3A|`SnGPq-WUeCGbhmXl3@tuPj3)=3~(sSs#wl_?WfmxUAA%4@L$c~sSx6> zR1vc1?Yg~c?08PR#oWcfs*U0+v51|)Lr$mAn|*HC)X~Hs@NcY)8_Zv^aU~SkaOt4I z6H~`odv~XQ=#|sWUJfe0#vEnma=~WI^(KjotL?JfzAysWx+H{@xU5 z07Umz*b@45r@e1bI2RCK0hrg-*Oy)wwafIcMr2m0AU11cMx*wIcYCDT%R%EB+k+D89~MI2oTgqRvWk-dchnvtAfAkMdddy$up62 zwhW_Q3uggbNpKq;2fKnP-J5|9(Xj$_=B1Cey9&{r7F zbd^f|?0RVz`98X+7igEs(I*WIyDO4k zK1zbTRRz(6;CkHRa!*kOE?*uiz798P*tOW0pEB&pjXm@9GcA;_(Z>E#1P*h$(mA+( zM!xgmLM%F3$t8ml_lNSvg3uusMp<_C{{G6{yVHrmy3smA4bPCD1=w@@dhYZ7Ui_vy z(+@oa7#G^fk&>RZ73Kw{w36OKkxHD|)$u7u5;T%_2(c|oEhLqON{CF+5*<8R7c_=m zjJ$dCCUxz*KiON%!qX|+s!Ee zL)h_slt~ziZSoIZ)#|eGgErem-1E|*Lp2aixif--qHU$fu!`Y_3-SIzod|y*#=!gW z^|D8O8ENR#!RRq48y2ulhYqi=6?qr8?%GvfnYy{TxPq}tMM#?pgt0@6^W%i|=Y|?L zGATi^d&nX_MZ#YQCF-l74jzfwuWI9&GiTbYDr^um6nwI0fAHh3NQgwE-u?T}AWT1^ zxNd_IgnM04SPhY6w8T@RFiWVGSf`~=LBHgx`=%aas!D6TKdkBwwE1D+AnQ8;=}RlF zlsD1p{>z|2g0|Ohj5qM%0N0ToEUWCLKXsg!8}tuBA-jH4DV3z1@${Ut<4Hv3Jel>s zT%Y=avhNAaDk`5lEH4A=TWzSP5}7`O3jT?s9)zPA+tkO6zje5 z%+RJ(o)-aHbNFU*=)xhY`X;@9K^NR$M2%;LX;ZBl=#rGYA7?b)n4_sbR|11*YQ59;_ok2mvywb( z*)wbC$i+D@SNe8v@0rFqTO(%7CoBYvT0bzd(r6u3&IwS^a(<^-^X3oOMd~(rWQEUM zeg*y`3zs@$e;b95MvfZ*S+9F{tV#~^hWM&v*%IDio=rndeicWKe|W=)X*}r*R>_)J+lsdRy$MEP1+K@`zTwgU1!FT- zeh(~Cfnmg0CH*&KpCD!J<@Cu$Ml<+MgvxQNoHdDkwc%c8{-(phgjatEKQdi%q z8V1k#WvsL6@U^V&)MJ)kV8 z3#Lnu=e}0>jIUc)dV#j4s%RihR%=SOH+yY3+~VMs^vO%~#fo(Ot4`kv6P-oQa(WqA_obX=AizV&Zz4>u|z z17N1!rM;_Yf`Iu86bg<|QyhBokO|Ck+H0Jor63QnAs|a&L>?%l-?S`ppl=6ENJlSaCFgXkleT)F!%(@&avSMXQ3c_e zWCaWH7_3aZT(J*`#I+t5ds1JY?n-ezU`a8cCqfF>)Q_NYfu5;#JE_|w{gK8SYC4QR zwa5GWRm6?}-T=4Om8P_Uzl$gMNYiL18mY(xd3JcxZVW!XPs6Gq?Vo8fe}fPj>(S0j z`eX>=#wwXU!a9?FjDX>*Pe6Q8vcGJXAOmD^cueE^*HYN+V5DbT{C)ez(vEM`9&mJ$ zDk?s(4w<(&zF_e&7(94{ zN+606LFGQ5@N%cKwAx=Tj%YIlIAX2}F4_3uZBlsy7?7C&CXpG?)?QLiqm7uy{P*y64VMt{0^jKgVnUW}3C7&nyuOWKIH8GxZCfj~!3H!)&@ag(6IW zVtkcUS$-PWhF%r~tsmP+Q&EV0ZLdPS-L_`x5eY#W-_QX`Gg42IQtAABwYkXW(buVHwr|oF zjfY^@s@-rbdePtY0om!>X|kW_-1-MF+-%xsv}&Kf>@4>+`D`u$&h9#uhDQuEE#UGYe^fTqbSxKRuxJVR!d3!Q;-UHyEj81; zcgM)UQVoZ}%T9m+G>tCz6_6))oHTa=EfJt_K$g9RMvWK9_aXJNRx8p$U&=fY<>XD3 zX*f-rD6+@@dpD#fh=x{q8-XCgBbNN4(~OKDJP}9`1THg z6w-adq#N0qAlKfsj0hT)fdVCIK3o3%1je_R(yiHqPas(b!I&135hPHI!S}+aapWr5E!ve4Juh*}~{}`i}^}kL$m%H|V*Py)1_gRyG zAuQwRS;6xCH8lEFQNT&%h3>q{zrIFnQyJ26x%s6ppI44XZoTEJvl}5$dZCGmMHtG{ zSr|VLDtFl$JAs}LqALF(uA>Ob`{ih#b;_G4^99H;W&|RLc~c$|9{B|z2&ox#8d*5|daM;tieloCkLcizd}^ za@TNzB6ow#_i=4{yH>4wFm8q#PjBEr6M^{*3jRS5`ZuPDBxq98g@Yv)&;l`&>qhJr zdJc|i`ZAYaJZ*+<+i!~zmSh?rF&{ll6j#}%)N_78UQCf@Ez0Z<=p`l6v?=CfJmx{< zpXC$}$zxKHYtrfBeoDg$gqw^FdkEnsWg3L4-{I$V)-#%utiJkYVfApE4mV7G+Pr*T zk@~h;BvOMTj$CY{8F3wITN)&ur)wpb3P^OT5&;+p)noYGc?K}e@NibI&Va~WRX z5w*G=&L-}QJQ~&LN8I~-w$O!@8O0HO8G67MQ|&M0mnnyW+YrH^X+wdePoOupCH)RN zA_a1@rcGxOtY+i-mth(x4?|zBbLlCA6NJ02!Mll zG)}@MN!xCxVU9<@j;c59nX3lLR>-Sm3ktEv{gonJRy=oDPMA^FJ)3tP0aq2o0*t@Y z^23|VLCI$$gW1L^YnI;~&Mv3XPE-pdQuD;;(4A^$msWK-kX$SCB%DC<#}+MGT;LY% zSKzSP^1$ow-x_Hr6MpyDwb**&0FZ2LdbCHii?5aRXoHA6Ffju66uS+Xf1ENVm3ENG z>UDcn(#nHgdsFWtwuy5xuKLqsamxv)D|x^(^0ktQ1uQo0%p1sXyhrxO;q-6Oyls$q zBLp|NsAtmN_XTh4l(Q%JXHdL4CSATT7q(RyC3O*)+C9A*q@B)PIglIoRGj0|8I)y% z-SHJv?3=2mL?K@*{qCr{in(}7Ri_sNWYvVc^V18Yn|}Zsr?D_6znw*z5OZCY4J!Oz z__j9PY&dQx!!jd(Hg5isQJ*JE{~+Q^@|ft9{K+PzEWN2Jgx8_h%0FR}$h1F9@K^~2 z7UfaOZjxSqWfYQM!2Y?E*t2Rp5;*%PG@^lq_ntE{Alz0|dJ$_RItec-jIVkkln|lx zQMw#viCRKVnDgi<&`!|iXv`JfW3h7*gMl=M;!6|FAkafB*%VGGbetnOo?twAKdIt+V3snWNm! zOd=|{#jpi?N@C*EO`kYdl>%Xf;ND@?v1FQU!|>WeTwgh6Ywg)>V$wupX5 zh~)_J4{_PH@HgVLxIIN?BF%p;9jj0x@pxP1+;*lNnN(MnW-uxI7{-m6yzScJB#Om~ z(&r0R2Kblv?=PQ4&(Z7*_0+0Zp$MVTM+fuXq0}wXZwmQ3T(Ba%;0sz} zpy9cP0&Q&(+BrNf4RTMUcDUe$0*>ZCXFmx(?G2^z>h9r46b7k?Kf|(rpv3NtUvFqo zMCTV?b=8L{aDoXxD>35I`gWKIyar{AHr9#k>PPr)P63WbCSRg_Lg2~#0K_f^stZ9+ z0v)YPqopfMdeS_S=s1%JP1NaW%yc1PkC7XS)z2Rl7e=JWb#S5rft9kZMKf~o`Dl+k(`Hm)xmrqeqdOXb( zwLWG|bue}FO5(4!W!>bap`m@9f;W*Fs`;wEZ+z!ew_oV~!p=Bh$uWd>5`b6miHMZA zy?&Yummovh?WF*RIm7J+y}3+fhc=QLeYI^f@uC|W895erL)hls+1nxs8w@&!ho|}X z-y0w~AO){h-RLmfIY{X|U-$3Zr@ZREXeY~&#CG->^VB2Py01l|k*33QRatzhx8y8NnSH7i^Gamstw!9)H1&0!5f#J`DvQW!Y-?)WfVns zfFe`p>_;Ns$Si0`Mqv=AX?T5Y)veeh^mMUvgmv&H_?H01e0)SvB~mymqi}7G%+fUP z)r5G9lAy>epiP!;9-MGC^KouYQNhBDQr4jwkI=JLMC`%0srH1@gk=?VA@~HL(Bz7o zm=ScA7OttXgZK(3QQD=0Ld37hdwLxu^MGRJIp}p%JS#0l3lKJ(sESCX#|>i#hmcuR zLDD;&X#?k|`jR*)$4k<9*v#YCTdddT0p5q93rDV0WGFP))y>t$$VL9`*@Pr&;*0NY!iI{bSW*k}D`tI6EJ3DDX z{T8?nDzg|0YqpIMiY}fqe1hl2rAAg~m$|LDI9tRqyNlU1mwEW%7zx)WL=^jd`Xo*K zE}qA5v8z;vyP+v=qfaN$YJ+o<|5dL73ROjxVch!P!`Jp*%V17T8K~z9yL0U!t=!NI zb1*aAWBzHcL9^KYfn&DntfK^H7M3|}m1OUO))ktdnfq0%bK75so}d2F2|4~_XA`d# zD~$I*K3eVWhz#-5p!-mT5utC3LyO%k-3A`Y&RbaI=GhPQ@zvj^ZS?W1rjCKWEGymo)@3A9t<1dvPfU6m7#LiO z_w_YfYx(x9v0v}CnKy6|WT91$G-(5R%w`(*LtU8F3mpMW6p*jaJ@mgPkjJK=Pp z91P42+oD@r-9W>~Olnb|T{ZCisxPCDeh;EkjOK7!RS>ooiwY;0E*&KVyR#}9a#Ren zO1k6@S*p%sa8lUg+K{PszIM~6+K0}Bl2_I(=;1s#EiB^Yp5CUDK{of0H9ERLNoR!J{O>XE)!uI)qr8fMrZk_r<2%eLAFVG${# zFR5W_#p^u9myy%=52PWKiDa$q;@Xr0ILt>4h@D?kYr)%pwtK1oO;tnDH=^;9F})PvgGH zOkSB4!w()(pyJH1?J~-B1|(;ioCo*DQz{LG#2CJqkxHp68_c_ao)b%%#xkbrWosF~ z29+k|?*CKV`G!P&$8lT`2aBX06ve2<;M!PLL+QjEZ8$PJ4R=~Mx5^C8MDtH=nz)Tr zqW?vWv)U?6u$$}1JeBfDHwqGld$6g9X1|G+bxu8p(kkk3x7Yh;Y){9Y?1}i`^mq5W z-}m!^(}@LN}LYYsG1(tI+VhfVz`#qMli! z$&cgIrjf~34@G~KiyuFFlyBZ_Y(dIz1$W)B96E9&3amog)XmD<#~7^wL$F9}ZOOK| z{-h{d71c;fIL=_)X3VEpH?A-(qgD>AV+TjU)OD=!MlC-!>SwIrl;mNxVU(0K%z_#i z#BL_yyQugzPdAgywjP;y!A5d^3lzykj!0tp^28y!Ec&-Gpx$UKSaVGpvP28a24ZTg|&1#%Z+!~)rqbkb%SY%y@niq$)&Y0_W2}T+FG#A zWo8tzSg;qeS7lcup@CX$v1L;`eUlXJn2L(fg4dle$o(_#rq~J!0t-WfmI2D%?|(q` zJl_@c1K}s5(cuft(TL7IJARR>Y`(D1S{d=|n6qegbzpj;*W+X~vWe1vbE(oX;iAsC zL)^mjOcNRm-`i9jH+_8c8!UdYH0Zt=sD>PB_N_oU<7lOSw-oML#?DzM1a46netr5$ zTZxCA)H3iR!wIa1Y*z!@Nq^=xXn>U}yF=YHCpC|a6uB(4y*zN9TpazNZKAE3!jm;U z)CgKl4MCy384P}ym+FKsq3yzW6BxNgzhLv1u$44`k2{wI3C3wc@&-zJPV|sB*dy_=WQwnqUqsInhT@{&I*u5Vzg*-Xmd))6 z2?;+jvQ6~KG75Dvv(A0a)9cPXYz7Bi1rV{3*tw5dKi7)}sxzsjn->@-YhRQaRRzoxt!{Du^Z*#I9q|UH)A~c6!?TOeGl!+ z1`!6FQ>P$JxmVw?k8uIHwSPLrmgybj-ftwt$fd>Rn}yiA%hRHp}!-PwmZYIxl16+Z` zF=Xc+;Tiz}n}1E?S;#md3a^oY8CM)nkVTpVUTpcKz9y%;)*fKyU@HZ23!U(zkvsi~ zEa^|rH|KD^x$gq5z1_$bie=<9u($Vo9X@aI`{mzX;!H(4-3-F_rb^eEMY1Fpc}_kM zo_aU=m1XTq%5GDHWj>3Dh!}%oI5;mXY{&i4Nzu|@eZlkSKniw1k|@;)l;7VdEhUPOV_Rd literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb300/gdn2_b2_bw.png b/benchmark/linear_attention/results/gdn2/gb300/gdn2_b2_bw.png new file mode 100644 index 0000000000000000000000000000000000000000..6f1d9b85f184ba13311dd2627bb9bae446fd7a0e GIT binary patch literal 100770 zcmdSBX*ic_`!;-uWUA1h3{i@b$WWPO2uXtwA~GdKGKNeUlFCq#l#EFzl!y`{v!V#0 zOp#ILLdK-ao%I#tYd$|`J9=f6}8{Y`K+zIv#pIepR1Llla2iu2@x?d z5iw!Dlg`d(on%Bs?f&xV!llTK@82=ycbEOYKkU+G(=44u|NUuf zz09tfeEAXgcJ?gWkKzUf2FHK2 zJYLMR%*vvxBv51DK9x+9m!19n2K8O_`l{Z`&N(^xXJ(|AvU(@Z}x$|UwJk5_EKN@e2PE1r3dd;lN z-Mf+f)~#E`B_)gS#}5p%1vH~X?%J`$HSIW2yJqEV?3Wmh=(w`^hDW|XViQy!w% z$GB`+x&M;otn1c=wzL?m6fs`2<5c5rfBL2I>FIauv~Itio*67kI{7YW1?4=(HaE}Ao8aDPMNJAQYHYBBx3@a7MtuH#^1f@~;l5`+Jx}oX{oVboez;`F zW92|51{Rip7cXQlvaB6h%ilsC7!@1>;ljQ?zNn;O|lp_%^dlOx=c*ugWe5V&#I3o}HngxOsTqrR$46sz^)C{OwX( zGjV6<+vSH-S2kywR+I)UXXD~xR#8!D`S>xxp{G3S@#Ct7$21>JEgp<~sanyV@3cf) zTU$d@Q~6!u9_`e_n;J7sc?ARnPSx0qp8qv9`qgc$pRJ^{ba?L1dAwJ;4{3TTr5Bg( znES)d#&%)!YaPFw9QTP6Cmv`g(R1!P!xF4|-M+Udd0;i!Opn;e*3)36Q_6GbxmaK5 zbzVtbU7d;_(SGUSVup`V_V9Rqc>T=bjxa^M>-&E6b zWz|*zigj*!Ks5Pjw1hR&Dk0rNTw#wMJ<2*i@he8rOvv18ZZ^NX_eiG6Hi|VSCdTpQ z;9%Lz#Ngr;D^?7zRBuoEB3fy^ATo7cXY4FKfN z#&`SqaofmmA)Gs^$7d5$QUqCeWOOyOv}#>=sDNcWGMght$HrE+KDFV$b?463-Zxd} z1!ZK8uGwjOD?TM9MK>%VAz}O3&fSr{zuNQpDOMr9w2k+YlWhu$M6c~MsVJx%tSd-O zO{I2Ac}|`!Y@E-R_n9;9uTStle&WRak(}!q8hW(#c7uOFJCOp$}X?+!=q~*+>8~Ga4=4x1S?&)DIsJq zf6tZG!GoLh!q_)yTAqIy6%!*oJKkn5b?=IL2&hYPx3a z+SG^kWBu>C8SXuOD*3vif}f9%dUocM^4YW7@lhc;r<=_OnjW3(DD=u5esna4DM>v{ zE%k7+qo?hX9MMqwo8&o@dnBCdgneRqbmPN&~**_B<_07$q{q=>Li{|HK&CSgVtCP8u zRa72(|M5U&vy}ai>F@7fa`M~Qn6tx5Qv-uF?VmoeadKXkuzq{exFSX9$ioLi&rr1A z#>z9}Ve39Rx(fBI7{z*rY~DNvs@&>rM?1eYq}0EAr_^wNzjpSC{kZ$}Qc`Qa*2Ptz zUg+I?q?f)LOUggny3QPL7puu2bo|ez_vHf`Q=<4m!Gp1M6s{)TF#E(=XJ=L1!jbA}wU6iP4`3=E?S*M#t=;-K*D=Gx_o4s$FSOf$F#G|t0 zou3e);twS#pKN@<#?8%*rF!wwC5HAwFRtah3a@dekFD$037Zyj;}O27o*o&IKQ}w+ zK(}~t0+s@~-e_?vK88hp{{9ZrV>akC$FbtEyI(h^>O@vE`}ux;dA?_;s}vA~gG2IU zJ%vgnKUB?a!Ibo~7MG`_4{Ta|IG8v@goTej)wmQErms0MG{ma@Zv0z=ps9JM7GuXUyBKDPQGAnP z_}TW1qeC!e%X;gPkw5oCMpeGOPe!%7d-pDSZ$`MljjGoJv{VTmzLU$<+r}2J%gPv6 zZ9TG*Vwj$uF2REh3<#i$bRR?^%X|Kui3*^hrQe)q%8Ql+G&B-Z*Cc{VtO#dE8Z&x>|vCil@U36wW8o~S3Zx@hR#|}kBMXfjm z9;!a$m@K!fD~pFmn&>rQVd3pw)6TPF4F^xQ<=SP8TbrBvvGJ?L@4X=~*%ic#HL`?d zo!pwbSb3cn9oSPAwYS7to>(ypXx*aJ)zNjGP%|DJdy!i$DJv^$+xnEzWuR%#sZ&Cv zK6I4^eVxp6ENyP)K&SSLj9jG_%AFeC*V`Ld6D3;u`SUv3CCgMjJY+r@ zUOPTMjw^YjcG?OUqz6dDZT!bt_7y9%6IB@{?LTw7y1E+Pyb>B({^rguw$-aI_V=Fz z0xSObaV_5Jp%)i_rZ}V|CwEmxZXKGKPzP+dz0-~ft2`u*zq4n$v#_u*Tw~)JX~&HO zzM36p&2}PDvR4kbo>`({nyS!@aMf~Q{j~@-K<_cbCrlLTT>!Rw5eCCatlhV_#Y|*)Y z9JrIBe0+Qw9~tdx{%v)$*<+_&>&DO_Ijr4k?Lb@>>1(Yr?d|Qy&zuQG)$YWnx_i7W zvXpk``7=Aq%F61C>r@iBkziSM{p9<5jMWiak|K7A#!aD=8y9-6%``2#SXih4$T?KH zT(P*dRi>@y-T2pDCnwzS?%lqv7Ce$CS>@#AA3J?LK=74#G>eRkOq_Ied)2pmV-dl@ z7Y8?O+}Jtxt$~4w$?x%FG2nvE@88qLIX7oFHtM>aKi^YzLlCIPFDGXQple8Z2v;fo zpr=Pfqg9Hd`X!!kPeoWzYU-AzM@B&kUejOHD}{P{U1r~Z>7Ch@`88;A(yhSbHxsqp z@ry=;u;Jl^0@V$#9vWs7ERGk^g++%FtsZLLIL2vF62M5-&qj8}W$WZF0p_Q^F-kwO z4VPg7)a8ca7}`xEITL%jIpc2Y2!L}1uKeB0^XKl}zhBbP!Ml`er*37oWfeQM`-Z^& z?cTF)fs#f>MreA9XPul%u?8K!z2Ah*dMZb5=#TL;(wuD`#-HCq{$gvn@cPl7{V=r9w4E2vlY&xl2cz+lR z&&hb_Gl@+PO zjq|dqcX$JaWjfLs;FZ-sG?a)OkfUX-zPW9y8+LuWbzk1WSh+0=5AO$X=nil1FXf>; zTPfC~xwczSDoHV>`1-$pX9ANsT&-wU5+K|>^x?yYH`;}@Xexn#We<+E>3R7xCeF>FujG$45t3SjgbXW_+(#Jz%Z*;Al?T5w+{*zP;Z=@ew5Co|eLga%}wA zJn+zct+kPZR#VeGaiOhm-dNA|)b&>a4Dz^p8kBj@jpj6utJNrlud^8OPtLXeP|8P{XT{{H=Yf)J_V&R7046yT^DI6>#v zuR)KDo|yxzFH|TtK{e?{x(M1KjmOtB4|rXwslymw&x1L1F9Mt45&@@iF=s)zkTcKdvhnx^(QB3 zM6djp^q_)BE3Dp6SpQwwb82W&xc7I-`#2&*j&b5e7K0(49qrpMd8&~CY)?H>Nbf0o z(fxv#FKIkI?VPfYJdM#3SfM^{Xkg$e#y$wcLy8(f!RS`-Q_S6p6B zk3Idyb)20dTt9T3{N*bgqJ5`p7w^5%OiBR5^5yDa@!GkzhS;#|!^6WvqobuET)PN( z1hVS{Y(Ph&9Ov9UM!ql_8ymB(TD3S>brE)AW11c_doUf?_iOAvhaW9sIThDT5nSoaX%P{bG=Fx5=UP2jUb$ zK;pJWUVDM31k#V)ap;NV-lsMV^i54om#$nc-AAji^YX;^ z56t!*g@G?$%7aHN!I#)Ep-A(_2ih%p4*VCF^M(RcJSY(3U}yJR%)I*e_r_EvvASDg zv``!>us(&q-EjPOm|d2qDjjtB5JjOa*SD zt@mX5W?dI+HvZ#dcgMGH-w0t8G0yW14`&6Zx0o6Kx%ueR2jwo7CxAWy4^DIYxPs4a zn(KJ`b{Rl*1?tGE5si)Pgl!rWdCOxnerVb>82%e7k*b@s zjbraj@KAWr<}F!f;Z>F=Te8*@?7Lylx|n4~HLLJfxt!vt5f= zmM&fO{>Js|jxNW>@;mmm%5bQut6PAd*Eclyzj*PYbyM=wZJC)#0)g>gSU423hE;x(^Msdu}c6>T&LyoKHre0 z8^f;^+7@yHkE`x=!LBpyl{?=q$W%YBu9Z*&+!?76)wH?o8M};Iim%)eax(W%Spb?5 zi<+PO;^N{JGqu%dP2iNVv9V_dKg&OA+LlsvUhP9qLoe5ehe%!M^NG8&+z+9mK@3P0 zmc`2v`*pB`W82ZEtkmy@wy#ZJy+v&Xj9Xruv&SpI0uTH-_0m(t#8iF1Cu5wwok&Og zw<)j_ruo^?uski5KGJ*w>+9={E8?qmsG)x0mnS79waR&Wdkgt2e{*}szHc4|cRHNV z&2$aDQAdQL%vR7T-4eIDOnDlA-?nw6W08l4$I)$ZAD^B51Qx4$Z(GcQ(NJhK*ipnW z54HCG&JO4UY&<-Vo_YbYpTx6@HQ%iJApEfKcS+k8)??3|UjdznxqR1sYS*^&Aip?N zsw=3I>%kb&dkIE!Jf`pMaQ5tOKxw`E($dm2)aL^Cq^v)v9d2{8($sP6&r<_)!xg+@ zcCC_f=f_#>KNo1<3Z|hB>*zTAnRbzLAEASq0RewS0|-_dD7~k8{ zTa`JPl$c2I)m-z7mX?x=$1d*f@Z;lyh=hI;5Eix^^~)D3m6dejbc2j|!}#U~hzP^; z_`$M!dv6R)bogw&YCDS#fjjr<1gSE}IkgHB`n@L^=4F?I=Y1xA-96GX+x~P|2B?j& zQZq9%=&u1<0<=TJa<`^BVj$40q~ed}oPtNNC*yl%X5BtCH*J(L<1)IF6MtSh^r(TP za8d+)0s)?WbVl#VbbGbD#~1@xsBZNAM^5esI?iFyQWS7W#5SWNz1}`ky&M2I>Q+a| z^S}*z&5Uaat4N&u=h`Jv}&lePy>a7a~R^s51zVhtarDbuqfyu95zy7+YcfjE^${lZn zc~vz~ngeczh_XOX)gN@ni!11>VJ;C_5+($MD(9O^uPyFv2wfCi#q|heWL2(p?DN)! zh83t2)X?MglA}eCxXcSd)dRiIXj#ya)q(eJ4wqF_q_@Z3`R$^ZV^y=d(o($fq2UTp zAYILQpaaPDXSn6uSWh=StZH_t?X64A$PnqPi(9^XYHB4N4e@q{hA&16<-5!0D6NZ1 z`>K5a*qlhJ;KOCJan)gy=u@v1EkQkZ?Iye6HY z()J|;ca<8-A#4yY`2i-!jmO}I#OMG9(QDhXel%bFMcvnwkpVX@$&=yf$@U{7WX*Lq zoBEW&VR&5^FZBJb_D4nr25P9tCdG-rNMX>fE%(oco1EB&QIpTFa_-RldH%>9$*J~V zAG6i%QUVHMYn)~ZnORuqsGPecR57^6p35thY;7MNZbhYh4EdZ~0G{y44{59LwCdxP zXo-N*#ufD4^Vh^-le$1up^D6+Ii}8@rS=vUJ#{N4=l?FWUMw_eX+^Rx(Zyt-DFvZ$ zU*5Nf;?30i`u6S>_t8G0 zGb~%ChN|W8vvtQ)`;K*Wafx4!BBGnGg|12yn zf%WUxpBnXgp8MQW{)}DtuF2ES3s0i(Z9iU2xw*L=hy1=EdT{liS`bcye#(kIRgPEq zA=GW_$+(?Or#CqHcbNM8N!DKL{*{%wC3`NTrT=J}?iPZR@v|3UlSoOF;c=~J@RO1m z(}~MCcUQ-c?>GcG#XdS2ieG&%(^wzJ2@dqepM6N!+(g0qSgdY_SBg{S{nQy4{!g81XOo$yd%%$+^_ZvMMa8-4Whk$Lr|*?3 z%*x8jeAhJUv&>Y82>`0nJ#m4!xp4}Q#F+O(9a z2J+9MlM_aUwO{|ls-|vL?&eI;nGJ$pgZ4#Q)E+T9GExErN<)F+zlz+d|D?7E z+rsl_1fS0Vt{tcDy$fO8x;5SN%BEeC#{~AoW_-K4xYgXoCKx!0AE0gP?+>XuOTZ5H zfRGdJe){z3M~!!kLPJBl=kZJ*?TmwOHaQP4mi{9md`kNLQ;8_!KDOA_#&{)v&s2y| z{l9*N{-qM7ZEZ%qEPVTJ3Rk@gNk3R+FVp9GKXBH$uXgR^*UTL7KA_j!wNE77$r%do*@<7rQH2KY>P*u!Sg%lWP67mnT zqztQgc&JMy@F^hAA_^b{dR$j$XZ(|8QxCU~%(e=}8Jhex3k!Gr4P|s}gB(Jb5aHqp zDnUcwKXo1wF0jVBBhsYTS{cC$Cqf(E>z38R2EGtLPV4>x??Ss@yiU>k>fvO~S9r3g zmwqs1cTXzP6)J;7qYW)Cq(;GQXt%MtE1;H;`ek|d5mng>C(N*5DTSR)uKo2&c7<23 zvIraJu|Bi!AR&a6YFRcinua$#@qUMfhqa{xe|-&aW^0C!7Zes2)-suS86S@yq&l;G zLqJ;sYwK^T*dxd0yhSwd6apbEgOUw@|FAV@>Pv;_HI2s5`Z{BU6rMDu?w5Bs zzME7)h+jl`jP)nD?!Er-tqw4?kbcHhfQ2hF)7Ocv1Au!lqh=oWonL5^4^+v*Daimi z8y&j>kgPCm?fQQg#Im=iTrZ;ShNv~`r|B}H?y}2XUnlKI2Y^$IF8mnCg*zk{>tGXv z!|WID{)~Ooc|2+f-S6r6-TQBvweR}PdVbYA^LBZ3R8WKJfl=Dq$1KPzfe0WAT%QI0 zU)Qu{eP;94k>nPIyLXe}M0L+YeNB6!a{)yQH6>!?bdu)@_xuIWG2QexN#oUg|G?eL zSQGr4HZ8_eD~Hz7n8BE~KWv>myZXqg}@XbV*rGRZej#5GG2fMCJ>akSFG%kV(zqpGi| zcsT^1uT$`ZEnZhQ7hS0d^qL(bBrBW$(kvf{iDj+0YRq-7pRGBjdO>u2_FqQ|3I=cQ zvJb`nE@^Ayf`Cn0e05J(SAv6Zbbfw*0+3rC48=vXv`)I?PDN25U$tj)IhHS99uOFq zfDP0CA)SNz>H4Imu30-HUZKN1vTalaKoFMdMBY5AdDF*0{nl|g! zty@HqTnaR^9OMIZn_%+`fo*8c&Y(4QKdz0D*4a`7F$7)ywH%z-I*0a;;5s4H?(I8w zu11l3@}|i7+hH2tLk|ta*2Bha{|qnyVtbjyz(-@bjjh$6nYGR%-_Pe5qD zXV0F!pDBi7Pw~OSGWe-reC!xfDyZs?tCIK_l>ZH>yeJt*jOZ1#gsbxq)0w`7@hYxv z7-&pQ(zQ3~se2p9%z@iN0Hx&gWGd(yln>&Jyvg>OaUK5iF;yqU_(MW4c-p0zkBH#p zPN5K&)x8BRi%__6W%V}p0izeLk6JyRJAd0iv3k$^d?r_B8a^^Da{!tn#YfUyh*arn z!0ue#apn8l%-i05=|>(3QLKp71mYcT?%jLvAV(;#V#ZT>$OMaluzi5VPZjAjx+9xI zv>CIehlbMAu@N;@UtK{}y9eeW$Ok?xh^ZMJDV!PV@;`24vxQ>i+-dv3`Iu8z$>LX) zl^ViDSXzN&D22p_5VG(Z{}JuBKL-y{I5t|;Klg25T4jeID3|sfLD9jog2GVKD>2?h zy}h%D&GFxgj>eNfUTCvdF2G9QT8R^vhHK*#6)P%|zU@#mHjX(27D3}153EM=Aj;F^Pft!yZ;+AU#71uV-mkSk zY!46eUbdf}uR^sfM`#GqM}}Ryb`dm5oD%3Z*KXftVrFKZg%)x2ijfYoBb&7oNrn`x ze@h|n2X`#5_l*PKVZZ>(&$Q>!P@50mW4I}7n3%Jsr@MRQiEg;^tn)to_3IsuS#|bQ zT3K6L2aX_WP!{Ji7ff{Os_UDi&W{Vj!DIw(#bepa{pw?y-c}Y~1!F6gEnBt>jgQk) zV?RFX$SyhJ>o_B`Y^^vQFu-!WbnQn+cfeR%3YG|PPgHJXMuNnxsxJ-H_-HoN%_4c( z+1>324AULN>+Op|Cbh=}_7`yEMbttExN$tSEWO8-Q zgpouMVscCg_z3*HsF>@uiu zr9cm2R@Ezo4YG*EqK2M9q6PQvZG^rEd6Kz%R5wX&DOCLQhZ4^gL_k$+S#2OnVm=|F zian-kZoUEC_Y#)*dVYRdTs~k}5K#U5LO{|GbTZ<%-P&%ph}d8BOF1uPKSw?-2^V?x z+&O*e^gSjuv>l%pJW=h~Rkm?|1G5#lpa9S$W&{wrD{md-1pzDp!wdu$NFIr{W~uyc zW^FA9f}`RFjjPo_GJ1;=D_vKI&%E9PBV><8w9{|P(X*@x@AmG`0$ z2vV%*bxCKrUA{MJ%g_^c*Yjax99jy(JM}3Cd50$zyIx&C0p_AV<9CSPko(Y+&J88Wapg>u8dqiyr1b!J5}PB%BqdjZ|89DC zG)MF2)DFS8DcIHtkXeL!8vZrW`;6{vdG+a&yn2TNdWKF~4<4)eG%&gow}0*l&2S_k z=uCZ(t}yMul@a-+kyH_Ykao;(+@VV z1$c6H?|}e47I_&p9hjOGu|n+MHJqHpLISK(g9`8=bY$XxOIxqu;wo)#=iapMW{y})rhCE^k6T*a6ORL3f!z@v8@sHthoJ@gdFvO$ytC`!WG*+S(pMP8eE;o?Sj9wS}}2gxcifO=RtbhTehR07f{C>_=8%rhtPrbknUv zV`Fqf4wet1qodV&iR%V0gP89y3k98BTxcjDX{c+&ljeZBY;)8s z-CycLk)-92&sXrA^u@DE^!_uAEg$oymFolpLk;+zXJ1SW-x;1>X&hyYA zRuR*}tKdc?`$;~p@xis~-;Wh55P^D*R3?zeeFtwniA`E}b}oe_OP0j1&o9&89UYDC z{o*v!|Bj#ZXllFb4}-|BmiaNUH5UqA4vI%DM;hwdHMYNLC?*6Vhq*Jpb8lc+geDLK zDPcXr+%`5g@bHwd)DZ_y7LL0is&}R?@H>ze$TIjuRJ9y-P<8LP%Pf0u@63L#uS@d5 zu93Z=UA|6kx3MudOhrP=C2ii5;Q`J;l~veDy2Dt_=*`Fl1j6VdX*t-c0NDEOb5RXV z6A(qYNw^isur84P?UItJ`1OaLojC@d(Im-&@&$q=9JD!D~ zjl@bS1}_HJbX@5WHL>tCDOg~+hs*z+12OgeuVDgJU^8Z!d!lND6T->-tdW!%>Zx`U z1gE6A`Psi8wd;TTsQdc$G$fRTuS54V*vM1^jJpD}@V)3x*gipEbdSyA3T0pc-P&o# zDT6mypHP1;bg*oYD^pB?*JOcGrIs(CN)m&)5Hb zQ!@WYkMR6ocX9oz{w@>K8;F;50u!Efa8Ls7XO`jqyG9c2-y<*tAMiafvmJH_@qj``=HQY1X9jGHXLx+P>A|OXGzB0yb)ZHC z6Q&=i+7m&Ut}N{#Bc2jacal&QB~OdLgP&*Uz6JFe^Ke>I9M$ zF4jOCuHUG1p)PcUL+wfgDP7D%zp$~{*u2cpM-~Q3Y<~UvMesNtRwwjfaB=U?o!GMd zcm5WYj(jAC(Sz4-+(_rghiJ_@FnsmL=4HBl2;C}#a!W6Aa&o%LtGEuDASwkUS_PPk zgerjFcA;WCIFztxdi-a)ZUt1lE08HwOig(KNpv(7;SZC57rJi}YCDq+)N^7WVaj3n z*C*Lqw{IsRIB>E(PweW|tByR`ckbM|rMP3qDl(pcu>j1qAeOV_DWVC+Mc4k8nSLX2 z=l(+^z>NgBe^F6Ut0x*K85AffDItB$;ONmrP+DM^rjv>Y66JYLi9?+N^|}lqL-Ntk z*4{eQf(a1=7)Xf_4m?0}!2C;>FDt|F&_+rtaF*C%046jPk(&=H{<)|7^`GoA));ZB z-4(_GRQNDH0@p$X=^YG%QKF)vboBHj0oDI8TM*EoY!+8q3R@4^_hn!d(6aTvN#Sk^ z)fN`Y*j=)+34H?+S%TijWP=3r8i^Aut3`|%*tbH?3z->tzKcwM1kq$EV zn4kOe4tfgA<0T03Xx?RqgD-}9y^+|p5CZNuPG+5aw}GfvW^Zb4-CD!4W;;c)IGUQ5 z-wzYR5(-a$LrU1;+X7-WvYQCb!G)`zJo!MG2m7+(;!>`N?*e}}<@9O>KZ+7svIUF9 z45lWAKSHaY0N}sE4r_E3(b&896}by+qx9MRrkN)4KR&(SfJ(<%#>vHX1yW}Z7E4xk zHgT5{l9K!p%rV1`bsOyqdH==}*!l6@1UT`y4d5>l7lDBbHoAXnMIo|2-0~hPlhi_# zAZCH^X=>k}CcH z(b$6y^K)JVjpH$Lx@fxn{=TX;*X|1D50b!w0UEF7)gcaQfeRTL896L;Zn%5T@I3xY z1*u8=+D&I(IDcaY&L**aR3Dq(mR{h(i{KN9NCB;pI?HEkXNNb<@2X*tcQ!%lhULd6 zR__v2mH<>PKy5@QiJ$#*6tr+`lLc|@P!h3lk6T$Ki$8+G^8oWn)bYOwLda*waf2k} zsStYgmi+mheXySKW_wU~N+B9YeBY;`u^W)c0g2ScjAKSyK2=pQLaV$O6@_>-oqH(> zuahhzS;9!@m;=T)U%P&N5ri8|9h$bAKt3g&2vP+BU}7bOuwMy;wQbL1!(UZKawR?~ z=|<13AeOa$z>p+>hFsldo%>gBjRVpVCjoCvDE8PtjZD-?O^qH?W!vVqF)T_MK)up#9?U_I@F}EpKZsRCIDi^)<=e=sr4XHVV`>cVpiENlC)=Ot z-#VK1txr>xJ>8|pYo`Er3FIV1;~vVGmgg}Kh!D8*V(?%sI34`ne||UH#&MCobel9z z(;Q4BFjG&w@?N~SVud%p2~xVGO=9k6bS4Ihj%ZOoF(k@104Sc+IzlKowHxyRIxU~` z94u&jF-p2v2|;+0as_aRPzyfJw;5K%Wa6#ATqpx#M>ZfK%d84j_zjtoKt!< zzFDxMxXV(1wMg*#!>SyD=N8$E%_2&%f{+G4u#9x~o*h;n&cOmc657oV$ASGXK3Hh; zjc+6P;x?^^UJogOcF5rb<%05hajSkt=iHwe2$M&S>Kxv1Egt#kQ#D{3+{mf0r6@LH zcn?!|o?c#N5S;GszeBnkhFR%nXJ-##qK}%tb3knL{nF^Y$%jGoSHSSX$R!JkVVR5Q z(xpqU-MV!NhI>}^1{g8K7X6l+ z2sdE*uM^N|4MttYe|@@w)=*kot7XMQIwKzBaCaGVfzSLpXb}{j$RAcDW#J{DY%ZFd zG?Lx6Z7HFDEyfkzWBuABv8r+4KskiK%P0_OG3(hTTVk_dH)1aaODKArKd-_{CwaO_ zYxYf?dvFj~x19H^fpBJsMcqRb*i+Vi;m_0&3#CPx0ueC;1Tm-=UoA=C7tP;FuY#$n zVCXIbyP7X(G#S1F^>evdv4)p7)A9Tfu_g5Mizw*m8M!8K zG)Qqpfv9$z|GP+4O*tXT1om=?Gcz!dG4r#q-^uVe$l76zgZuZFMQqW@LS*dTy?d7V zag&p74#ldwcT>J$v=q3z)2azH4U%4lJ&&nmE;N{Gai6{ejE*DoWPPCJKRBGsH2dcd zCS5N9sq{eKWTm4a-#5;GqU@C~KVA#xI5YQZfjsGCr$#L!)lg}7EF4^rI9Lmr_4PbJ zSNfYM>1|GHrS;_JCAK1XcdGHh1t^lQYHFDAu2x$sVdx=$au9{33nOccD+I`3By#9M z5OBd*lMZ!YGZ9(u@fl2*VYU;?gys43=aetdGaoc^@n|h8EyFTl#qx)-EB3!9qZ|YP z?Zj)irQjul@)1DlI0VJBXV0=W!1W5kVxy&q)PWkEq#Dc)qwAvT1|?nHm1K~YVk)ZY zDDqiOVP3h?#rE2@YvG#7m~*Wf=_`yWc+-W|+3oP)R2XL8f>8Whw$L1s-@V~kMXceG zpKL&(l%#zU#sa4~ACrR^-krM-jkSBeH-t9L@<5HTfFN-QT{Uk(!YiqyqsRo{@slSn zVXyKbAgyJO@e0ghi|W8$qakJ^m+P-~MvONl|FMH7^6uTc{L6aYSJPyUTi*QOghx)k z#&W9Vq#$JJ|A-jsd__Bae->(xm~G4UTyE9`5n1&X3G&Zw#ECm;c#lF6@eJ|x$K z*Y99RuLs4=AM+>cH*aP@RopHip@Ox7olGVoDBiZM@V96IKFQR$ZP?^f9c7cyV>%@L zqM=dwMBdMija-7x%YIDFvw39$k}m=E>Fil;STmiM+sZT4Chn>W(5iT??W1JmagUPzaJ{WNXyF0#I{%;G6 z^h{N!u=iKb`=BpVyivywx#+C(ZSANYNoGZKX$ac2aYe3Uucp?m?KEH+@b51M1*zYh z>itPl>zdgcsomfc1T$hLWCIi;fDqdF_I1~yESe((ezF-RlY3gaa56@}tPW%i68xix ziJUV94V#97bA#<~kTL$@-rg11=o{9rr$Ju`)D_x;PcA#aRy^-L1m|rHJS4pqfhhMk zYA!nKp$4rzoGPjBuxS2t403CPF;>d~=&RteGbWV&d9leT_%|o#iBB>gH7}q)ah{u+ z`!iu0*=F?gBn?9541T^K9erC0kwP-Zdqg~bjkx8dmNN;L3yvXC!bJk1D`hu9fm?}{ zTLB37_{1AdYB%(!l{X(!Bd$MgU$4kHH)yhP)27tg8^;7$NK@wdTlRvG2cxA3&*`g; zPTJ0?%t}{#SC#`wokbKRsGZF-V!L*!1834uc)#@VEezpPgRqRq>?MxRfHr=(QU~sd z!=EY);cnhRPY@8EN{velMcf`VdKwC6S@_l09|{~pdm>YA?yXDyZ5f+EfFJ?VAw1~? zkn~z|%619V((c3bjyMAX-(hcKRUq@u4i_&#J?Kl_au5Ng738665-IU1P5g>F7X8(6!uGg2?NaQQmz9SS@l?W`26b7*^JAeLM3T}*sw*gjO z^W!s^`UPqvr+?hIu>vDbm*Ji8Z{N;}XGT^U#(JQN_k@un_5e*E}W3K_h2bSwe#dY5f|5E;jKXd)!hMi?dPK33gkmtRpc2sXo+ zy@I%p`Q%WSLB8WA)G&Hn9▂qY5khvs8GOW&XBjON5(ysu}puWnGrk{0q>r_Gs zp+yb>v-o^sVr;0+&|$SJ8w%W=Fl29xvf|vNX2<u97$=xsk{KgzqqAk z4d_3C2E_QnbgEGE43ZZ@C;uV3NOJI!oDK-}tGc>_W?sq=fM@4s2FdKwNI@Jxof+Ue zrSR5`2>gN0_+t{vB(vwE>EalNB4fw%e_bTN8BY1cz6b_PU zBeSvCM;r}=oxmY0`Sj`TzX$R3r}IZ@ib1c*`37g86hBd)?*#0MmL=u-V+~BmAwLZj{k&y~!vm1sp_|Bd~hr*@K(^6nB zTu_=;S)bcTy3}4eJ*Q3BZ}0SbfC*Z1PIFMlk#B&9QmsAp;K62$vu=QXLmjua4#K$w z#qZxAwwi-#&;y5M`>tITlfymPj}&nh4(ca8rTfqMGSwvJKoa5eeYKvsvkMBo)%pe5|Vs&?|I`Ka?`U*cgMx3gP{ze}hUOfcfm zQ^(=lqv7cossq{n6Fr1DfS?Uv?OH4WP4|f(AN_rOsm65u<)G^NhH;KcaO?Q7}|Ffb0Rz&=NFZV)e{?C8oU;Yn2E~CLhY0=!0v1At#P=X4CC*puq$f9#Xurf-v(g``EO^0g z&@te75`+j{HsfADu|-~m<=WB>j=h%jnKY}T${|Ibs~0rt8+Qqv4T6-8dj)U+sG zWrNW6|GFtM>PC?r#>{+WQRu}7$JcUlmQ+_WK|@8?pyguudtHvZKbJ#;S*)T`y6{u~ zxvLVhKlvLO{PHaQQ&Vkd)K$i;!gs}3Sz7L94ZsgB{N?@uoVbF0O!@M#Uz+1>TeI-p zLLP*c_eJ}#I8@OmhQV(wo6mmbyYNRE645@2R&&SpjLBR4T>|f(`JZd}ha~*HpiBRs zXQstxHo;0oFz#=deUAk>>jz^?2V$hia{c>k-p!wG%iV_WB5R;qpK!AF-8(j{07gc} z#XL+4KcamT=<$j3mr6F|?~{XIQ3~lk@I@RIc>6(oDXQ?oyAf?idDF`kgY^9hrH5q5 zfiak5wEq72dvRCIc;mfzIS}IjJjsze?Z_13Mg09r%shns1?o<;c^o+7%Rm43J`2Ur zBRL&FP5BtQkq4(oZB2S1zGL(8WJ)u$L!c>GJ zh3ulDu2orLz3XT+eE1MpSm<5&S}T7Act4K1YS<1~zC{15>AU~dp2k{}?-34b$}8!> zHh@T>2Bn;w0EVbR60{=NLa$J^pFA9Zun*x=9+?<~H6dF%wW$H3Mk(~Gx> zb5DxV@ynjdp8FL~rcoe^UbsMWdG$6DG9^cGX#Pg}eK)kXlsIo?xV<>xEd&s)6a(N; zn<}8AUyF^+aGVD=+yf|KW@$->)9}d202n*rEIYXHY~3Z$7jT6=kTMS6vBtI`M@^KJ z0{q~C&{GS!O&)$9JgQzNo<&wT27(E=jzB_kDUenZ5P9Ehlq(D$I14?WyLk$U(gc8_ zQ@kJlgl_sd{WJu>j|NJb5~9lJ=}W*Nu+tYogeU`)X>7tAk{=iaK_t78K0-wzDBX-4 z4q&&~FgPqs9g^w#YZ}|(l?1x_f>0$w+a@tTJP=5?fe3QpSP#?GSP0@u8?GIPC`^ut zfqX#{q~!SniveduRsVqtw2pxIuouYtVVaQ%6;JYV+`_(e`~}J3xdCC_@@u^*1ldzpr2;v*TEQk$m$4f+ZVSnn|ygVr~T1i7g)14b1&qoGS zP+-lIlH2h)t8({X@C5uEmVqkbM^CQ)z44JgKx%@+5T=i@Qh@1zjd;l?PRd~swD}6S zMH|m#XJ^-p?_%)sh7?}`;7WPMAoTin8O}Juv#tOXtAA#1N<2Q`$Qp-FAY#hbHtdPW zMWS!_TwjNBmv#KrqCh(g5|J|n&f>@`61IS8$@C47 ziM$U~n3cJT5Rr)-2)Q25;mGP;Mmxrodn>i)lv z$k7^$iX;e#fB;&ZK=ej&O9n{Px2=$tX$qd$B*CTVEe9@5tX^0@W^=PsfM^n^mB-D^ zubT!9@9ENFymt*u9gFl*7zC&!ELb$ZGVFPOF^lreUz!^lRFJfD82Pdn@f*bd_Y_<( z1#~3`%XA4SQsi_nOzHWeqC-5Thg?qZ3P>5D5zy{pt*+v*A2qD-vq)7UBu7h>U&To> zHwtr>r2L?PT^4)kHM1Isd=K~uelH6K7~!Byg!^Jq?Ak z5JTqYW|os-GK|p-Lz73lA<7X-p(>8W!lzz>Se)$f%=Y6Vzabc7N9(&S*#JQ0(TfU+_Eh2KbK zc)-gxoN3FpUXDqRv@x82Wau4mAYvXe7%YQcCRtKo?&5Q@oVaM!KL zQ!_J3=m8+VNJ!BTu>k@YAI=CXTGn-?G@5<$K?o=0nc+b#!HI?|p6x=^85LHq(g;Ed zIVcP>gye87k`y2ZPV#b1-xM`nMzs`ry(sCw2OTc~LBZ}YXFZ&}<_~X_AF>;2pej1x zs`vU24QSBbQAvm;+3kuqN1R&}4w$veA(y2zPQ60cCr313q9o4_%N7=02?D#A8o7d! zyJ%91i+1QSjqvD0W+*>|6d!_PpX_3@05>#b$w`N}F`+-Js}2C)lXFclymqAx$PDA> zA=qia1BqB*WXSFPo5RzyL=+|`64;3v=WPdPSB#NzP%fC7MgDr_?tlm*K#)B6fkz>E zA;RD=L(=^tGE!2jNJaU`SY67{?XGxX;jbZbfn}-m?BIYoj4n@I@g*nxO z_dh)Dss@ffFQ z;TT-;8ppkR_mZ<id-$rEQ4NUeX!jh0N{BK( zhV_m#f>1ItV0EOjLRTQ&6_sfb2@D_&G9vLqlnE#(1SeTRwi_m=$ztaxVDYsS`6y!a zCPPxF3lIgl;3YV)0ZGe^p?={$+JP4@UUoMv^dv(S@i@bbiHBX%0ro;(ULF)`T5|To z_i)OSyNjZDWSo6tW7pxuCV;2opvZolwUT6_`+F!|CnZ#w_F_!IsIsnZMBYya@9=84dAJs^u}vRDGMtnkx?0pErsoOvwP>1dsvjJM;5PWT0Kf`l4jyMEGD@%=3j|l%_?B`$(-%2d4njV4)>hax#9xlNdl#ZtdgFSMUP1Z6d1~wy zRqf?uPKJg#t#u8VC~?&Ymq5x9V4Cu@y%!aQ0|^4LYm|fN@_`n%d{7}pgqQqAGzXI3 z#~Ubvog!xQeq-`^<%KzR9H|wGGaR?rqI^b_urMAuas&~}Xg7J)vE+|gX0&Ag0Fb4< z$@ZLbTq$Z|9P@Hehb%OJEm8nT)&i9k6PAyZt$)}GLM-eYHRG}1ME!d*-yf{IZg1Y1!YJo@#@dFV=&<(vwGuP|@k_r7zS>Ohu zpTSG{9MevS5lmez1M{y;rWaWXio(4gHWb)Z@z*{ztU=&}3|X0%F(HIVM&^fBq}mu@ZJeC#N6y_vEyes3 z%^C9 z9r{onZfMfF=E-SOC{BSPA^Uf#(f%%aQHvB=iop@RvU+h@&rI<>+^ZPui-q; z>wF8JD#-;~f%xn;t?nP^uRBh

I@Iv%mHFTxdlE6necDzc}_WZmblBpzN22_DVQ3 zhlhE%$NXAwzitN?-5c!I(+ns5DEfD4sQOlpN|+;}I@}s`rjn!WUPaVEd)q!1% zfx=;#YV4!2lo>Z%(JCsacx#|mG;LN-=4)sh8Y4r}cp9_2@g@`vDlyZS*OjZ(Ze9cB z3!;@@q0JiFbtVqTs0+8K8|)tatL})br2r({XGiIYqPgnSH`Pofmk6+s? zzt7^XL|jsorA1T8D-@EgPy|*-&9r34J(Ll|_(1{9e_C{e31Pl{UXL-1r#Z$VzZS0hmTwbhWOf%2V9ueOA=60j1gX%B`Duz$lwmVL; zKY8R9zdWlLt?b+%6VO|mdT1panuk#I@IYrHik8htx@PeBWX6Z?hyG*w`k$3~_v0te zUq^bOs+HpD@kKjJbRrc&5k{blD=6r?eW@<#6hX>xz77^m#vXE)>^SPWC#hFhYf|eW z*UPBkt{2Z>+YT7CmpDfVjr~;nZQHTO{-(H9i20RPMw_>xL5^$I%p&>A950qr=j>D3 zQ?kmO01tW4=+V1}dmUKJSp#Ng=~8YHZJmn0)jid~4s!^XeK&sMPawWy;u8m`DDI)i z*x)`|o-I1;0o)DF|JS;y4VSN=16C8=`=jpu#1xg^Cywfb>@!_8=F)}; zy3d{N1_vNjI6X4)Q94vAwghw$EXOwE;&)w-vS%9z6DIOaU@758Xwu_X=1vrFYayGd z5jenwX2Bk4P9PFauKW2_^nSch%RlBt_`)U>d6h{jVmJ>`6rcZi$JaZ6FR&r6Gr^&{ z8W=E#SohM6bw4%v`w3T;-{uo7K|Ufxgmivm$8N>!++s-g`qzp2;*%l$JVXL)&atLx z(wFkm4huq=E^Wb zHv-0p6&{;j+I?mp>u72DT1_W=0tHm%iKwRDaDpUZi}?o(l%PPG^H*E>At30(B$}{( zQfYE29#4hO=SycH8&SAmCMn{zzfpqFK8ZUIu2tewYNum8Hs}5O8E66&F4WsI@kT)m zglM3WaLB@rIR<)-Y(nm;fWT^b{)=YlbGG4(!PfCJmoea|hYxFZ7=NbDC7hwXF@qLo zINA5~ku$hGKkFNrpIvwL`z6*29zHW19D>5*&>m0@1}&X;maCoQB9Z{b8Gw8>8(rzK zPb{QkErXWf-3!v)Mg00DbFq9P63NhwX25N0%wCD35z6`LQ66m zI6)%^Q_m;SeVN_U>M*{Blw=YQ*!rS~k=mu#qG$B^!(XyoEc+h#>+|_+OB=ByP^%EC z3SUY0w9}5gGlCC}d(n;oA60FGC14ANBm=l@XO?EH>FY6iDo>ZK0E$@0Xue<-9(eVoAMoU|TlErrR)*i@c=Qg%i z_LOzL&DiWR%1UD20{`c*u1Cjq)T=wFhO<`RF%0M}ROay!WFm59=e-Bnvfa?-h!j*B zNDaqMojQGCP%H35*9&OVB$F~IbF)k2FCRCf88=-z&NeVGxC5dkaulJMsmJ>`e%0Bj zrN0sFRTKumv|PfZccUlw`|9=L;{83mkvBY69=J$V>n8#Qp>TdQchG5LHke>$uM^MW zP`MQ@T{<72@USeIbHVbvLh9_3S#bX92$yE2a~DA*b1d9v~o>{_2inGt)~OeX;SrI)70 z#C%#qANnEc82etO39rEi>5+0yQf>Sgiv1$-U8&;z}$WNX^jkc@;aG|qSE z*zN@E+YvF8R!b)4L@7?;YdN<$=Bw1?aYuSqXCIIEhymP}9+eB-S((e~T6VBbvCpp` zZ}uUpma`SkFM8f@NtMlD$%+>J{bKtcLtAb!`p`g@hm3R1#%OKUaq>z!UKaN!nnQC~7!u9Jj+fsmq)7Tjy94*H zGhV(GcPN09trAg?TjOoo1sJ0O*vsWBS9T;FegKqJn?Che<(&o;qyTyv{dR|Jg)JXH zeA4{Fqu(7Gg1WK~#K}+GT8W453q7dNAZ9C|ix8KNJ0Gd*C{`>Sm$CcjpPmrXoN4Uy zi!+T#M22bILicRYoB!j=kK!{^F4YofZ^n!h8Rl>B1K2=43*@zzy^WK?-+-c(KQBqX z^6Q-y7zWL*3na))+IX+LI@=AtQq){T1}z%pBOiN*lG`QQcpvyd+!3IT%qL#ulp-S7 zBo6AnYm9c44hlEl^Xm+2wH|}HM;+w%rcbH0b^<2963$@t7oQA|(;(4XxP{w87goH< z3Q`>!ftzNBNT>HDw|Z?o_R}JxQM%s|*+}o3Fo@Jdd|kj$;+9tH{Ad0X+%blq{qo^S z4^g3#%1{St3j84!wK9{$b{Ah3P>v}bLPsZD!gRxDxrcK5>e90(hWxa5)3e($!8_zh zpHuSWXddg*8W0J5AZu|D+-MOYFc`b?btWBX49I&o!ANqxFGgKM`EY`{M2R?J7S9l% zXnipbk%|awkv?-f)vc@FAAdrDvPfNlA6=0r?IF1b9XIi%me*)zlXY@wlU@t96BDvV zqz~;Yg7u4dHHdjb3+G?nZl4f$0-fF_+=IcfX)X^-Fczr!6akvptDRqK6| z9-Kh4yGd_+32SL*feul&RHoNdphu(~s?nrZ4ItV(;DY2(6^IF=;rsw zj~JX;fpoh)Ex+si=NkR4$9Q(~(3;r)TeCUcCvE86Xk7q$>AU(jR((j{z>L}HxJxvs zkW;;%7ITKhZXyBeb?n?V1o0)H{(vCaMz%2T=`=Q7SAU)%@Cz|;{L#gzY{Xxircj}f zLjk~psr|Q58h6NAOv_A!*+z=LU|GSb8)redpj#HFOYU=>OXc`H{Ig=kFyIaUyd2n&I8v#vMwIzWIClGxIPF z&uuE#snZSV)u0;0)%i-V#dH!v0&SU8pKy6M;4?W$_${nPTAF>Wj)BBZEY3ksg3r|bQi21bmhvVO>T|x*kZrZZO&SyP zEvRpa2r}0~wmX;`8W4wD7LwD3f@LrY zvjb$H;g2zGim(2oU2-fT)!@6;=k(|k0bNe*+3;KE710eMug>Q3()aJvLRtXdBjKN{ zf@c!o%AY}P(X9|N5~rn7Md_Cb{^g6`J_QJX0)u;!e}?OVN33|1UFQA=pZdx zf}>?#B67~}t2yB~vwcFmwok&0S$ZgloUZ#45{f!PbrbPrXdDjIZ4FqnpFjW8 z2;E-qu&)*^60$GgrYzV#vtMF%cE_)w__C1Br0JES12D;)tpDNJ6-)1RyjLJ#nN_Hu zKnA^wU@eXX3d!I@fS*rJTD;xpP+Yd1HRtTO&l_>?L(vZ}yLB^fI%ekFxm(1)oW*~*&yv=%Wrz#2kp(Ffz~5s+59l=RoBrNL@s0O3DlcBbO8BYtKi{sn`IiYbR6>W|i>TQ(M)j}?v1x|B*X;fE z$0&dY;0h@Y3cdR?CMjg9u)avqygSqD6ki^`npvDXRTg2v2hXPfAreG2V<{IfhV(=O z1W@mqUA*1sl$5RPU;Cs7Zu)H=d7ijWnd%^mJ+62>e-<)1;|8nfj_AiU8o;shF ze5l9&xcf_Je>9x)-oL*oa}_Vn44=OAzAbMdMO?AVQhh7psxS4mm@v0S3fin*gW#bD3$8xeUDfFeFp`weInFkcWq{oWURwrSco)AfyxS`t3!Ew zU#(5uA~ z__zO!D^w*guAiD6oUZlpJMG`cPY)5ep>Tu*VX@-MDK@K9iTn{o8QQjOThx+r+S_4I z2EL#{sG}Lfnlq)v{~Ir7(AFcI3vciR-Z{W7?9Y{AHXNU*vEaa6W_8= z8|K}g>eUZnfr=v|Qw(TPPemH-4@VogPDfpM8dMR(r;sh-Y&G_t+1v~$HsLwCO{aCIfrm>ii? zz%GR`5{$>E&D(GU+1pXiSmQ%)re#LUa0@_^yB%EKzCJ^YkwKV2pfj*A=;RuyG6S%uNH(^f|%GNuya8@gO$u!)(RSX>NVMdlFgd@WgEE#--WKwU>Y)%4b3a|QwGEm_{ zilVmC`U)gfLq^W&HtLHIW&GUZlsCg!N%g~q47N20Fx^q zULQ0m9Gqej9^+e+UUN9Tmr#IK2;3&DKo!XUI*m{d^JL<6eXkv|W$u2IvrXgoISu0W zU9J(}(!R%F8m*;#I)RH8X!~IsYmuWa)UjcMMB!|*l37}0}>Hj_WVmA!5OoJARR^%ot>^!uk%=lAK zSYNR^M0omqvvy8SzmMYH+}IMnxGwh)l|}^x z(CA+ZT=CVz^K7N!%E>c6>t;Za zGm7{5>EcN2sicbLS(;z-pEXbB4T;C}LuZ=i7h#PaReRts0ZXGe?1Q*5jAG%lIK1)c zp=<82_wXUwE9*)Q8SFewv&ksLc3lzBoR8pU&F0C=Sv|zx%_5h7_;Adtn5*l&SOn3C zgv4=n;*=?~h2WVUaou9K7`mYY|4i@a7hMaH?VKc-d$D~P7irDI#$6kb-+apFSm~HkJs*7ANeJ)5r z3lyX&D@{j^i+6E9-u!d}Z{Nyjv3q>D zxm)Z88OIB%FH=;Bo;50AJ2lgO z-dK^>Kn7qSGyYNl@I0nWpy@9uXh!Kp;OY!T%s#4rXj)Ss3PJE5jXt4#>-<3T$JEG* z;4@aM&bD4;Sy*RRS$Ssun}IKX>|b_yO-|e4wY_!@-V!&%Zj#pdge7!BQD53**U|nZ zK6iJ6J0_%O&2BP0CD%Q{-)Y%JyUeEUO*?Ead|14!+nN>4w!Zb%N)Dc0Z-2heW4|#2 ze}#UV?&nlIHFDGWHuE_nxv6{m2{)s|o<7cwnO^_(fr0jE!5daBK3V5mYn=ygEFI%M zY*UN+C*$g)H|lGGmxnu;CnRUwl4%SBMzJwe$;<`XL2fXC%-gxV?8!k4!m29VLu=U1 zLb!yp;BaCdt)oeyJ{8tCj@Us-IE<-(_r0Zb19C>sedZd$voGV$nqa=YXA80<0kWS z3d(T>wW=%``u#g>Vv!UggaQw)B!v&ZL=I(87&BFW4q=Y8d(iDe(MAeNg%}4 zUT}`@cI?ZG4v#EcmB%*TyJx#)!1&aIwyA;L-aVYU^+;Hd&$3J58abZXoo5G?EE?_A zqxtROP7505^my6M?cDiWUnA2_o_BCc4#@8KZs6tYK4Yh*SGMd?Phsr1?Lwnx*{i1Z zS=%@`<7)flh4ZItD?Wd)Yk9wadUkFjwSN5K79fhl5aN?DL~P}`Lw6(^HX5tw*Cd9HECC zv9)DvHU6ZDKe7uxYb50zG2RI?qnrI7NGW8k__M`E3HBoVUL<0=GOwkfFTYb(h zsjUd^*ysucifHkv1_8tRV|@6i<#(g+MERU^Jo~*Sil<8!6MIG7OV^88(X{{aDqFW! zd0#$i+?J2GN7THT;oiK@%J;hmpW3{4QSjYb)f6uiKfKw!t8AaAjEGw#S5VYxR{!Xm z%ks-Pa88sRa$8MLHd-o&iId~DhJ}RC_wCDmh&7)CdgXxij9!1Zs*tbQ_ON`E8-qIr8_6NL5Q8>e5}rpC;ep4uyI zZr0!-w`W==Y7Za2RpV*(hUbkFW(}*Ot$Fs`^G^qyv>UxYY1XR8^|e6*E@K*8T3&E2 zN8Ng8GQFZVfZvc54yYBfwd@Vu2LWd}Z#E&_UlW!VkAZith8^Ckq&Tz0`SY!}#g% z`c>)CBWda(_oTF>A@+f}8yZq@cJ2CjLi#**q1(9Df6eJWh|4Z$syK1keXi#l^;CqM zJ#r$R5x-{aX{)}ig>e(-H7T*FWP^f-7%lux27dMSt=r=B70|(-oq@1_fzVlRR;#1O+jowf10Fl zp)Dvg!wRiJ%qzFs=NWVDlKHs{k0P5CttoeY?jQKZ*xla!W!|#dbr-+z*}Nz0a;e?T z>N>h@PnxEfUU?BxesaD5bml#rpN>;4x$Oqtw0(~+#RJ2V{*1Zr*8We^(@%iUVYcKQfGHgw8OjZNs0cE$v(cQJ$&)I6@^nB@! z%>I{ty|fC`+}WUmGOwTdQ(oNCL-%cHjew=&ek@*+iO9132t7cf6Hpn76X8(`h2$9` zmx#+T5o%w67Ep4I)I~&W&ybTtMb07sx{Om`jr6<11I&E`2qPL4;j?K3u+6VQMJjer zLK)^cu_PKo7``qT2Ask$HV}QSlh1-r+>R=LqBPkMHY5`*-I8!xc%S^oMTyZ;mAfyj z!FrK~L>d|@$I~5|?lzZ!L*YgVX3SeyVM~3o`>{alTjz(&e;ht!S#Y$M$Nuwmi);L- zmN#Kx)TXujX4=o&nfmdF&x3s*&7!2OO}|C+t(JD&7qib(JSmT^I%9R% zuvI&sle7LQz27Nra#!;QB8B72tv`cQ<5-er_oFqe=jrVWFF$7bWa6L^?N-&BCB9g2 zAlg|E1~?ecqYdWls5%g{W{l&8t>DnW3UEE!TF!ZXuN&fSIem|*oAK9--qMn}Yj;6^ zq&ivjC)Md5dhremN>IKc1)a?hQwyGvMzhLd$v{F?jTHYbp}EC4gHAo^#LQJ)>bnW) zIBZWtx;+#8H@CJYq>BH^>hE&{7IpQH)A%&J5@k~TV~f4_%%lg>n{>f<_|N1{ALrWH zRpa1HhC1g>ilJir0w+YJ90l;QFe2)9$;tF)g( zc`mJHhE!j-=fm|g{eyG5SIC&Sh;w$vf1K><;SK>x`+gItOY%xn9L>%rGIDV>AlUNx zhbjV`H0j6mnPg2^+`!-o&yB3__J*}z{xQRPuBeIma^f_?UR!CEZF^21Z=ZH{4?AG~ z(fpX^&0d{f(k?Y5V`t8?n4q|%v$LyjiR{!dF5$}2T#uGxM(-)#ccWx)&{g25vH>n9 zTu;;K$8P`S_>A+pdOun*EkAEfvlB*M%f76eo`34Qd#CB9a~~Ws4UdnF zHBWIcyzydw@X!3>=7#HhJqA8Xd+F`uZg?%vGP{yOp>Vl4R4-~L))ajwKha>uDp)vG zhDQzsIYgJcG{Zb))eFZFblYJo0F-(C@|}iN9xyJGba7HfDKO9ezGb zxw^2<X{XU+wVyG{{B%@aHRjke=!E-qcsS8uRH^o>O#)sk%?Ep^H4~%GMG~%Ht zl_x;$!)rxB>$Y@nZ*|!vz;H%sgT{NRby@x}scHYE7Wo(RPxW*zJ>pZ;=X7R6`;_o2 z(PN4uwf2;ss-E2?Xw*8lw2 zeUx|3tfZGgArXmOqcFP3;*o29MHNyr0>2#-fqzIuaeEUpd4&Zikd0$j{o{R?4aY}f zNAc4zhf^qu9AFFQO)D@cFU@zZ-=s+pxQ5^Ng{RVKP0mtI<@G(?;eM@?uOu35e_)#PE%@&?>ns?nV1t1_)=RV;hP?Bc}K z%QdQOf^%P37-nOIgNIe*dM77l$2M!)w5AytPs3}){+Na!{7JXrC(U8g6DS6v#x~=@Kys;^&ZTUp|ldqBbWuU&nf6fP` zSi9UQ;F8E7m2ZzQ*@E4g{;+~Wz~ks3EokA{?jbr3v~fWcn8o?d0UX#%XH(P(E-9hQ ze}pk2SUdj235b5&`ID?R-I5Q(maT8VR1vcmp16oEn(P33ydxTqWE0x1M5ISvB{uuZ zK#zddMMq$(xc>-Aw{9#R6ZRUpybh?*+hdWNS&U7xD-c z9`_%dixG_@?^3T%pNRn0-NSyz($U|0T*qe=D)^YsPxzA%7Z(Ls#yEbbX=0}DB*Xfh zEsQgF7uyU!*=^V||93~6OFpL6zrVl7HEn~l?hDFKj!4kRxH9fqcz(*J)4pCm%Rk<_ zGAcv!vgM%uMY`I~1<{=rPP}B27P2Mp(ebqz=N294b>^w>l-@1!!xlU{_b|CZqjO_o z_8uGkb)swk{i`_F@Ss&`N9HHOI5Wc}5dwbn+8U_qFWK$hwQJXl)o*G(^fWBFl3$Nc zb@RanvkIQMJd9jUko}q+AHr(110!t@s);skrgjq073V+eUippzOX`RgPHwNm=+1jm z6g;TedA@Px&D!U zE9@^60HGNJz9EX;A~AGNbHmBd^=TZJMFZ94WWs3x7KvN_A=wX)&M&^%yip@t`=drq zSD|%%kPYXgO|Lg_Q{pkNyND(OS)Gy?{lm&imz`#4Y4`Bi%5xRE`wRa|0Q~ypi{raK zOxZYne=;9Y{ZSo#)WCrjemfD(DnI<-H$QmvXe@p7#nlhxad$rpn}3@6%-?AXZ#e)V z;NoZi6*9s$=XJ{DtEaR3%w}!f3mNijWr9u5bmcX% z_4Eh!de4ygV;&q8z?JMrhgbso><~8VsFk-*0NGu2DhY?i@`rFHGuxwMmc)os)ZjZ<< zcu~hGeg5TMAB)@E&CjvJ0mK^x=`dWL*W}+Q`X))U|EJiREt$hIN8Iz!rlEY$;%Jj< zBU35f!qybp%Qo>h-uUgu*&Mo!iE|%v;_pZ$+kt3RUYc31kDR9%#?W%WeEYMGtvPg# z+hdXe(GrXe1U6h#!43E8G_Fwvigr9g%Fzbj;acS=2-)nGEe9zB#|H<}W#_SWMkXZe zqbHS9yyQH1n2`Wvj7Tl_r}SE^oIQX3DIVCV;w6nnw(x8^GdDZCXmHd99Fa~#b`^gJ zjZbE}M<;%+d-L*DvkA$$oZ+rJ3NTjT8ugG1P{3o*Fvz&X~q~hG*SB(7_~=O_%rLMoDpQp7;GDg=2zS%of~OEp#w%~r97Lo%fc%D`L!agrpwKoORrc@>7B2f z5c|4cy`^K*22;+@s_u9p^H%eU^4|LXV^Kj_UrS3T=Syg#r2DtG?ep63{`G6+0%sbV z!n+Np%g)WXFn1Tz@t>ddwOsbF9ye*}^3iLuZ*QnrRmX(M=~2m_a&rR-eC;&a%Gg-q zo1~fy;LJ799@7eXy3&?y+ww@M;1QXYdE&6kKu<@Eiy4z0PyM(P36Rb-gq?TuV`N@X z3l84j;noFs!2mA$AS!AQi7*Lde6;r=6i4?`QhGkgizxZj-y{)zX)=~r^k`#fPevKc zLqcrNVD5ztmHBiEj|uoxktoKr)A8Su{jpo}8B6OuM^aOgV?ztaAL?ysz{1%-r?^J` zkB|fPb#vaos&{|?ZEHHTdSQ{plN}q^*gAA!$)?(|^l)|47Tdnrx^Tz$MoZtuojnla zS*jAg}K)Iay;`P1i{dTlC${R@o+qQkTC5@%+t7BM@9>Gn`WgssZBDj9jrpJ67Etl=l z$$EHn0&Mr##Im;)=|7h#jT$yQG}L=D`oob*1_Nw0*8H-h*zlYro)O>Bkq5CVNpXR8 zLtbM>|MlCq(cf-ddK>@R4zECcw0yfBZo8PxF_d`aA^RAogc9qvmrZqG88|09*VTVAyOGf>*CT29zAei8Pl@QH?Qi=cdFp}eaxd3^vL8Lyaq5;p zP1I?Ma^;Aaj`x+iU&5cW9N)g25t7+3#I(Cn$lB)a?Weu=Sn=!p^5A(7C!VkiZFf55 z=)=(;BfCrvh^*bTr&E5X($JBQ4&12sy%F!mC62m6nU$5LYtoNtD;uZ6lv8`xX1PyL z4!B^v=2y%&=dxBkbWuRJrpAL=Fu{Pv1FMa7zTC2NjQY<1||c`W6`$mQGaQ1K{t_bi(KCBJ{y zvX3l(Im((s|Io(!`)$S??%FCL4`(`i#NZNf0)k;`wfiphHTv~=2shV+%xDc8S$Mxg z$a~uIVvaqf+POKjfRmk!sdC8k3y)lv&RNk|oI^1(2&RlZ<*D`KeusVekIRd)Gbn^3 z{z*6DJ;?zzY^75uca2s|&FOBFn|>wiT}i3`(ieV~2CMAP#=bc?Ws&2uyu>XXOo}z? z>z^3oyfE!{)!Ob4e>%U~?BJbNzutj93Nv5ZL3i)m$&E~k)*G?+gZ|_auZ^o(n0^0n zc1MM3O0A_|r~0HhK5sFwxOLdTUD+SIjNDh$Ugx}e?Q4$Pce;(G1HR504LY9Ak0|LJ zYrY0F$vQ8s6QM2ZluQr_I9*BveEk1TTG7hZGq@l2+p8H07qzmx#<$8Y7fiWJ`h(|LRGB&^o zUT5~h(n+Y#b3Q%yUA1t)m$$y#w$`*<+3xS|Rd7hLX7=5D3Uwp-zmKY8*bhbnb!V$qy^-e3Yt@s&n5 z;Zw%rzW?Hd(Om~*Y7WdbUqwvK;N6dB@p@i0^1Ae2+uv#6l6O5~Jl2j0JK|ngwBe*l zMy74bP};}h6=fHOeypC|$$it1*K2q5i+Da6Fr>$+N29v$qm3y%w_WFB`P)yg{0RuT z--Ab5KHal@%H%8SoP3u2{JK_;Dib)}(&xKdSx0IT-Ktf$Wq;}>-7}qy^QU$4-n$!( zH8s9)Vcx?)O81;Sybez`EKF(zG8g#)!sRnP?lWyDTLeH$plr@SKaNhEXiF znZ_tp>~%UDX^(tkVtU(fsvBWB34Ze%9(E2a_W0n3GrHO`n`Do6U}xqy`?Gd9#T)&6 zTQ7K0X!P>r8#;62J*SB>Oe$b7a-c{*obg z@Oo1sjCCo)7TSIOIl<5} zoXPK?O9FvRW0LD4s;uXimki+^r(9#dJj?i+Rb-Z5Hm&E}#4}ovUddCFzz&0_hpL+D zIr`%)*A`B8PYI5hqTT=N+G#g5+vZPrV88hBpetDqJI&9&lYi~lDm#HWBAWIa-)^6m z`R(Yitk=0d08BaYVMo8EJ`GL@a!xyy`uOMb@FTtRdauYcFn+$~y5GI0tHOR2^~_Cu z)5)p#C+B_d1?!-T4FRnf6mtEm>CW{H%E9VF2yzZl8!MA+u$aDrdt*10n6U=Y7&uXy z9)ix@8M#fh);bTaHvV9pE++cddSv!>8`svCm##`W{-iwO zUF;jpvH%pj+F02#+ISL8#9^l~OQp|5Xko|j-HyT4lu||jo7n;V6(d5+VqM8b5HC%X zcA9&FY12^D`-$TK08|P(U4SX89>tW8L+G4XAJ}MngpeAwYK^Bu>j{JL z^T!Wc<~4RSZ}Fw|z+EWY2%k-NWEhqIS|$w&HE;q;o#`YaOTAaGZf|2~&V#=!m~T{3 zOK#WXWAf0 zPK0(+4w!s_m+#!SFO7RRhZFXg=c>=t3RbNXP#Gvo-#8|{E422uNQ|lct>m-e@xD$r zOU^H8Z{=(3W0aZJmZs&$dtd z(0Rw%PE&nP^jc6dt@^vpDXC+wS)co7eW^iyEz zm@h|X3oh$;Jqy7wCwCeIPmi==?$DmhqVINYZP6luB_lCjUlzGo?ww7oDzi=FFT`)A z{_J8R$3!V3DBQs0O_f%}3?1jwd;W`tmJar6HT|#J)2?V04<6C%wewid0RfEiKyyD~~@g+qS;u}I2aM>w`_Rmaz^5jeAr5c@k^%~CV7oJAYp=uVIV%$;r)MA-jCFM8C0CWI>m$nLiIzj4bLBt&A*+FCM+?VodW*9~o1B zk1xwty>~4)MXfAvHu?I?+IyPyoz%bf8EY1>r}KtAjrI)RGJX32PT;mSN{jsY?kET3 z0j8Gsh&dnQy5`s7{9Uigmqj%Fal^UWuNRU3L+)7$0ON^s zkN>H8IIC6vvZOClt9?nl_~Mo6$dNB2!}!tFIedmMl18uTpIk6)>zFIot~me}hj*4F zYpmVIjY4zx@f&?s>J3|6%oxA=%Xm%9?est2@bCXMA+p)3d!w?V>qSf{H5p&Qw*{Pl zvUz#Qmw4w9s_Tp^ijOxNXEfn2k7jFy*}3`PHXOwqD>Vsbdo_wG&Q^+Hng?Q?GKY<=>&QfKx?^VcVHm(lH+ zl$S(2sqp(3Z5elY-o2x}L+E>b@t&C$vF=G~mLg{(Mj{$odNcC6%avOW(KkEcA`l`y zMGh21c;{@tuW$E#*q^qrXvFv3dOA9H-py2~YvDr`y({(T)4zWZlUwNc9wU|NGpnvg z#)1B8x#9`@Q!=hLm8;$}lOh0=M33QTdOiPvzPG!>n6}zCYto&XMe*|uOkXveWqoDo z+@mQsRZZmQrLEj%w1pF89CjRj1P2mq=OigS8pN+w@l^$LNY~&JAG%DO2$avTY_hZe zwGYjHJS!gXWy$b|?;CpKk)lVv-78>&q1>~VS%dYWHvYJ&rn}p2tmsjn-1?>l^?n}%4J*w`2fGaZnpnh_s^f@-$5-|H~;-Qv(Zct z`afUiGzM4u^Jo73e}4G?%?)$W0U<@+8i2Ba?YsJ2I%zlnUL-gD{7KBrif@(v_nx{= z>XUv|(b5oW#+`7^fgqH6ty&FQxiU6;1-N&RR1>5m={1va5PuFsC`5|+!2e!iPFou2 z97;f2MtrmfuD||bO3JGeNqG+alYmWMN#EX1U&S+ml(^_!wx+vdw6 z`Tdnxy?Lg~=uSpPHZ*N%t1_Jg+Znj*17mf|k=OhzE1}P9D>j4^eLU-C+l;Z=K%j+^wyS{1;zg*Zd-;r1~kw7hsz2ItrQZ5 z68Cv@{Z@t?|F#`$(;vaIzwLFZ9ww`_IFOroVzRGAPD>LI!1#~rtk{rxP#QV^AqXRmOK#s;O(j@v2Grr2@#oz2KpOUkf*Kap& zTzvL-Rm-CKP{#8L5|6o^Lt7m@^@?yPN$|1@r`782f^;>Ide=XV#v&P5t`MC?5XFwt zSBNeOBx~lV#WPt9zV@DPoQ9rzMo7)UMys=W{2lbuF zG2L2Aml4Iuf-L;=&$*^eEy~AJv3M;g;f?lniW&|xZVke;Wy{v5SKVr~Q5PwuUH`k# zjIZvWIiwneWrDDhbU<=eSZdbITe7MeXrKKnbikaL%yh2~t!$2@_Mgfd`L}?;w@Ksf zZjza0KLdQ?KH>JH04}2AlkcaFpK!T2dw!QE{TnTqHoVG=i=0CHpD)*;TWjbd=$VGE znjlW(Be&CC+P61VYO#G-6Zes=Q7Z)*Ij8F{}`m5tT9b=T>(mt%;}#8v>j z6`;s|BFCfxX4TC=j1EAr(LA;1MEB~}Dm{wvyP>~zBy$=em_p8Q@*;}*vq-8!AJMD7`VuuCeMuTrD>fTtGxRmLvNNJ+gpn*TR!GYP3$!IRns)y7z`eCdzSn_%(Lnj^XF00j~ z*Juz@VfYxN>D!58AcQ%hCC#E^VAI=@3{uyOQeF3Y@_#>meg@xA0S#?e5=|a=js;=3xN2}lUg^dQ` z<^F+o`*idtoSmwFwJnD{QAY`x$&t(+zLAhoN?$(w#4}@6Kia+C@7m*~AB@;oDEdJX zS*fHfmya1aa=VVZ?RQE~BFkaf?#nMt>5N`yfmU@=vFm%CIS`&&5FcrsBLgv6WF{GJAX0yooQ z-NM0xH|oSb>in~lCM8}{8ddVV6jiNcJnxQ)x@c;7o08HsBqXG)tYmTN=i9Z5`Tz9y zYv?ph5~(zmADt5yk#|6?vmY^rc7D!N%i=!jpS9`rI}90bUCzmU8m4{b! zYZ-Ppqm|2nS>}{rmfgvuJ?}6uPqo`x_IUH9ruy#R?u!CKm^i#u?4d3KG;~HyFPq)Y05^CPVqt2|eTSswCi?Vc3ytb&b7@ zOimCc|1=t_lc3H%qI-cU+(Grze>|vVl5=^8A0A~~g2x}61K(ds~CaQPe zrOof<((<<0*!uYR=$Ob6cIy#|%!%nV@mx!Dy;qY26$M4gh3rZN`VAwwTTW+5A&zhV zkAytV3c4Q^)hbrM+BoLEjpZ$gjt4X<7LbLj zSU_RDr8n9Ce~Sz!xu1bkE`6##z4wfpikgkOzVw)lF~K+3B;RlY)$et!@u>N~1Lps} zT1|Bs^*{d44=euv)&~BcyZx^+;D33y`%H9rO_b1Y-Fh%4AVT^uA?H5OdA&m?bupi~ z0S6wuaoZ0VFqLs>D?mq-xkg!!jz3NHQ~#zW=O(J5KzxlMQ^PQgo`%Ct?BNMEj$-<6 ziBv8HSqz!tp~VArCENs~*sDUFNX1LIIYeww?$);sbjw8#M3f1nQ6rJ{I<2L8puRmr zhlq3-kG*43U#rpv5QCUNKu8Kmd6m<1F=+}$17Vw!=r@^n1AOU=W5{%RghSV^UAr<@ zbuAZzl^jYT0vgd_(dX;vgKUc+Qc7l&D>+otSMgD%e9_X>loJeNu?T75^ld-*d>2$P zO(I3c2iQ4g-WJV>pYNJT-A$UcPp7iR}g&|m435Cb}>Awk8O}_y<%;&tuZGopq0X`+tFL@F^%rwtfrxoDG&B;4tp>C1{0*Yh zcplGAAYf7Bm6sJo+D$c7eV&n{<{ftv@17gqGaLdUBIus@33uFWY{#nQ#Vu>P>st=z zBU%D$Tdr^-%_K)StrhZu+0zUEtWaI2rD3-YZZpQVjM6SpeXq!a9rGIYu3MuT7$z%+ zy;GN)IR_@~Q$eWe7lF)a@7kuhd-zFF%~FoFv!9MnFmE(~;Z#9L589{!nDlgXmn*PC zB$*{7&oj{O>G*X{&qdEB=3g(^i(F~`Z4K2TH=Z22T5l!@exRB6MnJBO*wh}v`zr;#y^~7ZO@~m6}*aG4Q0~R6+wQ7)CEDW zr9Z#U6_pwgrX{~Y*F?}&7V@{03DW~G856ch=ezn%-y6+^nJPV?%tB z1(B!HifJax{;Ef@y6>;G1C5nDcReaB$F*xm%^Iir3zxWiue~c#`kN&3$aPF07WQx2 zv`K}0OPQ_H7YuEUy0x0oru;>1*XH@vAZpBkC^hltm#;a)8`RbvaHT{{TK%(4L|_Uc za6m^26Xn~S9J~MAGMp`j(wwHDa13L{heCW5LCKPZFlKhTy4h=4aGod3tx@{If=w1^ zfx-e4{(+X|%=_q?z)XaFOH}>ho5LwlnN#UvvTtBPHc%ppo#eIoH|A^K_Efz<;jwR% zUpus?VFH@4Pev)Zm_H)@m}Gn|X3@$AK@%Ue(N?`Y{c-Qibteh5*yKNW{(Ks94_%M% zWM_XI6VMCv*A851$Fb?dS!e=q(*9tiWZpX}KKo zl&W8Atlgu(+vjntO1Zc47zsBWbaaNoyO>E4i=F3P-*U6B}Ue-$Uh}Jh)`( zQVSrlZxDHTbazbbpMs`u4>LRlpn_A8kzhTBR1Fe}kB&9GvpRhG{})?t|23X(8PDqcQSAo5<3 z(6sYfH|x($b)VFePbd8yLNO7@(u>QX*neM#rEpem2S*y3Q0QlgruwMeqJDA z1(F-`P_cz~R(Bo~JvMb}@DQXzdJST0hqi5_8|bR{OB2SbTt!jh#L2K9)a{mQwTT~m zH$!=d5TrxV*=tzM0wd-koBX*%td#yR!4+$8*qk}@Mm&QjyTWF2nzxx z2tpkFo}fB1IsW(d>@slIW=*(xU6}3qc*_$TCeS2k)vxb|AJ#nF&v%Y0hN@EuJ<| z--krEC*GT3J_|WdgrW<)?+}#?XQ_IZ|H>Yin;99|+-Titsat@gQVA>V5dAPNXiG>w zcv=ncrr%>jh`Iw~(EDHk$g2}UOAPQRtBH9>-=%LKON9X$KbBA6rsGubuy;TD&OIH} zWu4vK4`#Q}=IWZrcqh#?`sb7jb4Q8(4|Gh51J=?>qKgBc(7%6c4k}pi=}+_n;6qns zV3LT%t=uzOtpNH(gmP=BIGTO}elVWc3AxU^TCP1;>!`wSh%lNhe+P~hOF03{k{qEfppZ$5B1D@2z zq1cTS6i^!Lc{8%8;(rp|?zmdIxW$_4WUhXc_l(K@2XYPns#9b$TUNxOdyQW6dn@_vc_w4!vc4KbrVw6KheG{$tsT{= zw(6%Av~1n_^_HSa^d8m>gLJS~ml%olc5Tsw2LEv@4Ho%7FMf&2z037>X1tQKq3`yq z{5ucbo#kEW(?6;Uwk(MC=y3}9#J9m|*Wa&J{WsSolRCQ1*ww!&Wg&$UbKu<|DSPM(1K)({6$K|~N6q%?gsJ*1 z3X3YWbTNOLf>EB(@Irlv`kv&-)KD6{V`1u)#ZualFTz#?sDDWH&pD=zd$~P^P-4lB zOC(&QS^I$P|>nkj}C5iXnq*9_n0;mXpcX* z2Rd*l=t=x)^q>d$fEs`QZN`v_Bz~~;!_(BL&&UtdOGLFxKYuT{k>zn3x;WjA6$3Z& zu{9P&4I|1xWhJ*thR2D`D&-$I4&;VW-!jl$q2<-?HfwW@DnoX@!!l;wg^z#0KfYs-Opg}Sc@wkxIiE%(CkS?NGy*T$lgFG!!11J;=9z8m?i_RiY#^VIE zq%-Y_hcy;VL(%ia?nR-HhM=X8(|1nOt1#(&Ob$nbX!F0i6^?-s)FpCVZ1{y&vZJOC zoE#P?0};87>RR9{=1*v?f^evjs!h7G^`(Uq$X~Al(qw=`&s9DW$aT_L z^{RD!7uQc;j)ImPK*!`qzbKOd%D1my`zUoswTHI5?`e4L(~BEHK*l+U6x5|MTsaZ5 zI5#MSX?1U_G&B0pp)tdv@7@i7Z=ZZ_#s*x?e5l_BtlCSG*L_-cv7WlGz843a9E4&J zKysQyeQDr{-zL)IVOH%U%{2nEt8}LVgi|#njj6#-NF(%ri*4tHj_Jw4#p=#;qQ@lf zN+#_se77_5FQUdAK5GJ1Kp5qTj18DXb4b!OYFe3|EBtgVzdiNkyUt#l3A^%nIckHW zuW!EQyam2oVwiXIb8%6~ycvfSB~YvQK|lgU42VsaN)rM|_*-nhJ5{Y3dP!Xd8&{ce zzL^Uwsl($FgCr%1&;g)bwB1x4f^l#G+5>$+;!&M`P9=AmhN{Ehy)&ZzM-Xcd@+-0l zJ}I7^-rU7lxFNoqz~tbH~GDbD>B^z7hEKzo|C0IL9t)WbGh%fi^$g{h%EI<3UM zNl?v$Ixrcx+vDiKm&;oka4L+aslU7J<*kQ!y8oaeX0Btn{04MZ))-(TQjz%7r1zp^ zJ~a5{Y5J0kRKI_PTUWce_sj+pPLm0y(mP~k4m62m=SkPWeSdnQpZv<-UoGc8Hrt47 z7`i)>?0p-v1)t*v0vBk%I46zP6NC7=>tUB1unfxCV{Q{#CV?4SKV_TTf z4+q>}Uj}ICif|ZP`8>-K{0N4qgPx03Yd7_8@9u_%9tR6E>Gc5ek)!$Zd^`*ngP((y zcjG~%X1D-?Bpir zkX5c)HIFcc7H!fM+>PRPR&jm$`1G zJH}XwbXLk~AOo189I&k8@2O~;=8#*~3>99aaGJk2h%~{*ae(^SEeGa+RM+o3b)qy! z+{uN%gL&g4cZXDyQAh^j=X7n9jEe+?r@2ht*r?lg-yR|)nCoUAXT`19A@M3f)UgQZLu#@0k?J z;E*-gvoJh=5eTMV^=Q+SUS1}cHg!hvhQ%chzc ztL~TX)lt4UJHsErx2Y0I+pL@bb?EF@IA@?(K{oE}Dk+d2iO{}K7`1QkJ9+Rd#jm!t z;DZ}*X8Fe_p)sj*RU5&z)udm(l_=vG%dm#cI_Rmc$b2oK@04z>iYiLE==+e-3<2UW$uu2O^u0vO-}m7Dg9oEtym+Bety%(- z$pzE)-W|}luvN#7anbIo+vnPC(g3%v^Wqu_rscK1IDm9GCnx9kN9&0#0nFquIKd>g zGCw}-n321faKlPKnHvOYA0^3b`5-Q%ZW4f!*R!rgiv(-( z#6c31i3JEq(wXt9>)Nyh%)|$zEUpX*1@~a#yN;?4ph)zPjCT@zYe5k^kxsk-H^lRWC1rr%h5$uC#+<`j)LlbSq?#{^QZmxGGTR^00&DqN(&fiU#i{g&GX#`q3JC@PYZ$j3eDW-Q^I zJu^5Br9Pq*-ISi{HTd?)q&0&%*N-t$50%Iaq^l|E`2f^2xP(Khy7vCO6u|10BgW6V zCMf!htHHp_byauH*U}CEkf9PCk>UR5RjvNOco@9H7{^OsAsb zx(EM2im2*!yvmvJDNQ!qz>yFf5V9zjetWXpZ-U`3T5)LXS;beWs!TFG1;ddE5Ms#W zS(DUnVhHQ3r&C$xwN)XExp`Bs+R~Qph$ct&?b}yGAk@DXhN%s`^8{o z+a8}7wq837KP{(I;R~P6G$NVrXy|Vgdq0(p?zMJus1mDDCWDU*bdFg1F3v7@O62+* zGM^wCW_&l6O+KjRL>An8lvqzR4zM)>MEWomxjISg=!CeY`tS5-q3*}spfEpkx@a*d z*XmUU*c2R+PDG%ShsXP;4OinjE*Iy0F}p=kqhmt%Vj5m02s#85>8B*g!SuxLb-Xa)PebjojnvGOJ)pCvQMbW#R*4uAi!4K4Oqvye zl1>0y;6HfpcGxK1a63eaBJ3yPUxg2&Ia`4b-wNzi0ZQqhC=+fhnATbK7Dl$Ib}4@|8}h++uu;t?*83Vx``93!+}P z(`BZU=V}KyD!mHhPI{cJ#Y!{dr&5v37@#R*kB#c0?i3g3(Sh*353R@9m?CQ&HIzI4 zx&VkWfAbMJ0V0KqD?HSjq$a6(vmC1vKqEET;rCXk1+p&`gm%{%E`KyjIWDAK)erst zhwJ}@5ju3s`%)9h_I{m(9K>Vtf3f%GVL9jT|94p@j3xUXu85FAvSeSnDza9jEG3eL ztXWcwnPEh^DqEB#*;<4eO9;spsZ=Ut?5UJ3OXz-_*Oi&?=l8ph`~KrT?&WtM-#L!Y zGF;pH{eHd9*LgnA=ks|wSblw{$!r6RTVXySbS%*U(lgYbYqi10Npp!#zi^`Aay{X$ zCKx%j5#9}u6cvH$VN(kHdXUR-`xi~C9vEM9Mo*@g8L+kUR^aWwzx!>hjL0d9MCiTK=by4i&+`{IAr(i z(R+41a1@%lG#W*89%mM+)VE`Bzv(0gstMqX#rI-|NfC*Y%mMbyQ5cVMYQg{&6w=B_ zNBxF*U7_<35z`4OB2QNOfBY%5s3t?g?7})%hL;U-<>d#KBLG%sB>7%vTg2a=Z#K=&vb;kdyKrv~`?c+0~ zE*H0eD7CB@qoG8aJ<2XvzDxCt)T0Xk0?{kt=M6+0UOXB}3S{h)$v?h?gI0z>5d+Ga7m%?;FR%|xtm z>|j2CggL&u<~D^JAKj%UKMK8z#71ra2moqeiJvV0oo(_p>(Qw@D<_MXBOV-5*l<9p z!a1RikXpa}@F5fU{Ff&hEPU`3IhQ4nk%qi~bg`;(JiUkyWUq231&xLGc3_)kg_tfy zOtb{F6QRx|=u2`5%qIBoGHnv`*)UeeAvi+cdwi zAcczIB>iEOTZa(f2poVea7N*A_22t$BCWW@v%5wU06Xx5G75_BGQCJk4l-NldIu^y zKLvy7k~}LYI7~-Mo(+unxjDxVfZFV%%F8*UDRHKEXfmkt41xQXzl$jNyv66+hixK+ zrt!&>n~G{KmRs17;t|5#-Si&~4-RHEp_Pi<Q5TH^SGD6UOErCq%y$h`bv1Y$iVmVF@hXrJFOgDlk< zbPsS`TxS`v~1eSePW&=!>LFMib%e%gxz2WCC7?@cr}uQi&NJM@<)GB8Z2 z*0Qo}qjsuSmZn-d){gg6vxL*Oa9A-#{kDM3_)|=#vpG>+g=$bVi&31CdI9Pe41OMn zZ<9mi@=I?=5XD=@wcA6ol@rmL8|hZUN)<1zBQeWRm-4f_*=zDrix@ybhUMIdys~NJ z{>juG+=FO&m#5zjwf%$XRBc??o{D(Kdd;IP=zXc0n3 z-%pGo)BhlstbBwxrJ(w(ELKeUCNCT*X0 zC35T~ptZo05*In;>@at7~`9`=eAqD|@N}!-K!vB!SMJ8-|L|6Tx;k|i|B05t7xY)v}%*F3= z&OaX{Gd(FJCSXby1g!6J^;jqM+Xg>(CclDEs8G_7x?VQO?E>h=KeT|i3A8lIVt>4L z#+$K2xh48a`j(dQJ^*5>HiK?xPO*em)cWn1Xd={5$&Z(&1fHw%UsEg$KEo7lN$VR( zRwQtTrolIaR0d`4eua{B_}>(bjzDI#V}CX_!EH?KCSC8)Ttg`l1*Jzcy@c!ljvcbu zS%nKVJFJE0vG?oRt}0zqx0RWiPAeH$--RwJWATBaYd-zQ{rLIY`wW|YKHn2q2{D|? z`UR=nV%L=Id(qG5R$^z;1Q2^A3YweKd)WUVEDT%z z_Gy}Ei78SE;wx?-kYYX+0{;W(q~;KlD1i<}-P9ZcZr4b~2BfiV;^yU~Px%47T%)g7 zwMLygYJEWE_N&=-^?&qRd|j>t>f-c$gkw|)m}(@ufLk25RkHz27t%!&jGN3iIBkw~ z`m{ZRwOP%%H8kvUb!t5niiv>uq;quk|K8!OqiM^QzGXJp5yEXae(eJnO(n;z{F?wJ z0a*YKX&BP{CPA~&Y(4VNI}(Wne&V>A=crlb7B&BmF{Emmm-%iWEf*9>AwaK< zCM2YHu2ho@UPvVSKSz2rY<-JPyto-j8|;exLKugBdi?!+_2$vry|d@+&BsyG!m27D z%xFYjT3X>%$=Ht(uSD%i8YnY3jhX%4f8IUKE{=A!6tRZ9ytTU^a0N zxqgpK!<~&`dg#CGa8R0GzI@@U@IVat=lr(@pV?>5Ab_7ZSli{qvh^*F9#{WH-%X@? zqBym((WE}T>-SGJlp-A@FO_gfvqve=`qKLd)}vy7z^8yx(m0bx3ZD)0y$}~` zin{l7*6vkqxqvM0HX|_B0Nov8VgK{&V&S=qBNmmU&+}8X)eejsq5k^BEWZ7B;7?nU zN=b^;!hm`pkpPD*%5Zr!@Hze|z(g?6xJ{dA_UJ3v5P=GMp`*z7vrthRa45*K=9m8E zc|iU7dQmKQQ5`AM*XcBz$b+1)5_Bu`f{%~)=V#k?_FWVqD(-*kq^EM~!K{KN)QcX| zxwkhA)vHis+hj1(XO;a%eRwb4Ui}RLu3WPjSZVmx)IYu+q%cs9XOStCFx*f6Td@|^ zlqnNrMrf!4-e35pRq!TsFm8&R3+{m}GyyOds@g>~v#O^>MO~F@_by?MsKVD)uWh;F z>n^4G3riPzJzVXZRN122v4y?D^4q2%Pfa|vV1_cCT0xrVEIj6>J9;hu3G(b`FkED$Aa3ICXDKfT^ z`mZcH(biO%uMwInNT%r#&2PH5^Ge&wdPDs(ExpM-g6eBvgqJ+P4Y~jO3>b&Gl_;4r zz27s;X9O|QjhmkrKN=K$5-?*X-Jvn#)Mt%e!{ZhP!j}_gCwM%;WN>n_^3Hw>9{JM0 z6z?zrl*Qs};P+3?z2`riijZQfy(Uw(leg%Se_h7AeB-FDPh301UL%rFsiQi_s=iB0Q}x%B%Lz7 zl9H8oYGLJLfz4r!x=tma8WOxNjN&BIyh~|--*qI_%Umu=n#!`L&UBc`U3y;0V4A0( zY?Vq?v8j2_k#f&)LKG05m?WBC@`E(U31a11y+#djszhsghtBQ1AGBCQS9PAQFn{RF z97DlQ#Lz#8}sh2zLtlrDq)W3In2Z>4?uTM!{VLn}_ks-$fs{184tu=<& zNF%%FrasN;M411ZXmOGBk%y@m+SPf21NGh%-r7t|%=NYL=#EL(=>- zJ1_8@CaEcX3o%Z!jy3-s(CU9UFj|EUQp~DT<8tjvqKn^ra;xjB%Y3F(GGV5gb=Dj{ znh&u5EvJ({V?pwCm#F{1w@x?!`{9QT5h32YxJmtIdb@kBY^q0s-^4(Jy0|EqiCKyC z`o4HvDZ-#AZ`=3(e(&9KO;7p~|{T+9sjSVVh=mtG{w^#MXAh zbV$-9B|z+`8wW3WvtcB?ZMIrt8~J)V+IvSdIc`3V1+R z){ht+Obso7>ydH$e-~loKKFm5rjwuKaWzsxzyw+K$egV z1t|ro6*>hZ%e^8vrdkt1MwFFi%@_FYKHV?@f{!3coSwp)?o7zQH!4_^3PL{Tq`*cl zZg^DcNkDFiuILLgh1$TmcJNcK@H5Km9aN~lJYWFyIB8eVgt5gSPVAj{!C{hlIXk)7 z!^HTUGuG-?Pj@D|gc^XN2$@2_XM(@jY~s6i2o9taEMF*jmM|~es@t|}q#%N|&_SU! zK3drQk;g0Z-6K>~QmtYy)|=}t;6kLSJMrvN524@UJ2aju^+y3?Iz4vmTDPJ7`=6t0 zFLD5>NN9YCo>pE;zS8xan&?Aqs`hvysEY;B#u&{84^6#tWdsgdci2Nqn7itYM~o;i zL=ZtI`+C+h&Ffk&IQIN0wbeK(Siw5P8fE1Ew0-Yvi0GmLq!{H18_?3&cv}MmjXy|j zFP#B-0Bp~VB{~-4=gc_-c4WP>F3xems1J|4I19lA?hDv^Un%WCXsDpKvIeUkRfETR zi()Me{^pF6WdC_;SPXV_0<^K8WF1|}Y6f|R+m0yQNMb+JB$|Y1P~q^~9Q03Lyf{d! zwhL-wT9{*LCuZUax-lx25jsMaWITk`-G{WrC;lN#*l1SQD{jVy+?KuPhO@^yLvuVF zt)@Qtrlic=yN(qBC^t*ABpeU=N|hX{1Pc~4u=}i02I0b z?Z-*veE_5gzB{#-JzEx`}OF+nX!;Y zMaPMuX*cB*lpeI;I4e72Yr_u6fEi}(@6Pb|J;LLLdHao(EfnGruQXrK)3|@oSjLDc z$8(luLO+le3?+S^v&Uw4hR(zgqtO1{Io)leG%!`lT%MQI{)AV_BR&-6Ob|#0NG<9- z5M5k1Q#5~(M;&B-e2Eb{emsm zgjElSjuYZSA!~@<_lUcC{rk=mxs))_#}7CUU@?UrTw_Da)7(RhVsBi%+9_w0&G|RU zEq9uE?xVlp{Oc=qV7T^)j%be-n{2$-X^s}J2hfeJ{E zU2?utf_7@5DNUXH;3|W=IqcH}hJZDYhC5}dJItp?U20Fv?4-N_ws`!T6eaJ?R>^!w zR!Q-WccK%^DWf5N1d_;MBX#cUTjxfRqi?6Sx2FZs9sMWA;f&rf(?*ZH;(gr3>Bna1 ztADnOwcBdebR2#ZO@aKSrySCee(wO$XY65fDBzsj8p-&Rb|s5E*r!%EKH zeQVayz-2!!kI1gqW1!ZsI$)K2r2@pz6?5+fn2QK#7+00EIXyEd(?3B!a` ziPk~_24h4B2V_c_XP3UwDZu^V7))2bAc{yaQWsCY);A!e;|Z=%%0vfvDPm0+vbnD&doVf(sOe7HE+xAp zCL$0JE9JmA9=Npn%okY`TN<8Q{bKhr_=ftTWXX90sGdQ?`8}Q^1z;}%w_&LEl)h2y z8TFdJ=GS#=n^QEHFI~DMbbSu;DRX~-{1Ac+SuFw!GuCG3{X=6(tElCRev5{19|yPc zTZ%CU&C1u6w3C@dDZ8g*GXOa=7*@kR46P0DG!$7se6}%GcQuhCeJ?L?qed4mOvyBW zafQ~#HD+@M=Q(QFOctFeRj|vO!qJx{>}WAPCr^>x#53DZ&qE}WTMubeTHzgJ{a{!u zgfoc%j}q5lQYKXw4t|}bZA1?uzPskt_qG$gI*ENAdrm|-LMiKClCYn>Q5a}5(0J2l zaoy)Ax9V1ji>(3r6%CcJ*of#yfF#8aPf|}s0btS;Oo3nLUzq>IQhjUD1tNacVGke* ze8m&UWSL{!IgwwZRP~||lxwCyO=rLULs$?SE@?Z6YKmfWzgTy3qKzfg$q_BOSx!jl zR7Y89QdqYnusI*u6e5yoje&-M8juTd1z76wFkv#_ef;xuO;GwBic)WgQG&p6TWz^& zX^u!HowD6HzPlzd3U6=}1<_$BW;<~(K)**WIzU!b)DP~?Ok5gRN>bm5q=)nUAhm4n znXc-u2^iTbeZ>%{hUdPO<-q|QGvyza0DNu1w}q_qLrTy>c_L6(0Yem*(h-u5abttF z{s@oK-{t0YBYA=yoQi>%{Nvke=9WNFaf&vEV_PCKKG|s^hhkmqNUm0+#$1Tv!hunU zKJg!r;tr>09FvieN!UoWg$EJ5p>{~(l;bIAp(l$Xmf8&`Mj?BfMr+WG@WxaMGH`{p z{T;d4@M7j$LcWgLop*1K!11Jg;;5U6 zM6R!y{Zn5JlVSR>wmx5DMVc#&Dt?#fws=00*^B-M&`!97f>w)J6%D_CbYsRRGPF6~ zwkjmYmSug|Tf#o=X(5$xW;>C%O+3HqbBd$p$PcedP`*Rvq)_@`&P#gVSzuln({Xaf zO#v2x<4UdMLAHBOUXyYvYVJdL2b;Wl0pI`X zBty2y+r!-)HE;AbM(sFq|C=|%>uxys<%K)TbdhFukye2PefIG1a_x$F&J?n~et(ll z_7j%s)~=oB&o7z1xp#|AqS$6jKqqcT7qsv7?LvtpM#_*2vQ@L*It81tv*iR43mI$| z50T@fwra~Ik&M0MQJ^Oue!L=7n#K^VNhYOdi=$PU_eI|kWM6K4+8K%tNlEDy4qEgT zz)=t=Y(HELfmQ{l>cW znj5B|6FM4~O{<-9GKJp8e8Ll2BOu!X%5MdPV3!0aOSkDKt4T zGKIDBip*G&oYh|nI%TvyBjR$(u!s*1RYTQIUSi4yflGXd;i8KJsw9GNcRMx*)XXNPYt`2∓Of^tQ;6B0BjK8d-yj;v=n(?zFoCH=Is)D3!{I0XZ3jhTIPj9AOyZ4% zVM3$DidCa#l5#cNMB#WgD`wjK%#T_L>seff*vSzcPM?DvhNaSk@(Zl2YY!egc)z}* zsQyl$?%c8CNR>M5P#N*`v*9brm>*awitRSH@4eaYa$J(2^!)bF2C@pNk80bt>=19Ab+La;4yXAe01-YRlMU| zYH9P=y94{KjLz9F^Nv_>k{LcJ@%6R0$7X^rnO}XD&?jvMeje__=oqrMZwB9n4IPT= zse9oP%8;mHwFQ(EmSiyE3Lx^l-Bq>ImKXF~!}Gp6Lemk|)9&n)@TqpHj@QbSS3P@c zmA)lw6HfJk(IK|d!Dhb^532K)RM+tX)7cbw821$*i9;`&<&wE)NL4}M*)q}M@UW!N zR?PBf*#|1Rr#uc`1jd2r^}Z{RSU zsB5C-BN3gE$Lr09vrB^8DJL4O0wr5ss!)Si#07n z2b@li7=2XcSb&e)E_flvvv1$3^Eq09ZY)6tQiamKZ(wwF-MSYCQrB{pwrpAn7FSEk z%M0ndrq5vNa+%T9sPlLOprzEJw!~Zx#2Z8bHypNO^D%XMR#Y^v+lN{SwFHWx`6Q>Z zTsZ6VAlESm<+F4c_Px&s$lA4gw+P9U<4N-(>tDKHXTeZyKWeHc#b&-Q8Z>Frk-Q_s zgGfuKXyY4m;RdjerDEA_<~fw4cboCZ^uYY8oH2LlKggQFIr0!|T|%=MIG$}&yS_te z|JROFr^?^5#w|OI*~OUbtHN*>Zx)R9C7$y5B?q8;{1TW(Fv5;Dp^+S|Nmtux+tBgs zu1xLynmtHV=qIZJrC2(Y3i;Qg=42|IN+;UXe3Lq;LXvu6CPBrSf8p05Yo8Upti+{D zUrh#hFmX*e#FG_MM8Q(xrn)bdp9=rpk{sApC}dQ9R z_)cG`O3IaDrJ3Dtoe4GKAq%9cRM9OLQw|}HMNDHMzQ5hSM5dRVS@`VF2d`>64d)nH zjNL?tM|R9Ke`9;buY?tlOURrYinIvP=x`v2`YY}N3)lYW##4;h0@af)m!p4GuQuFG z+Eg;lt$>9jpr!CdL`^}tk@cMvG4kpV_8{o5rQgZ~6EtH%BwH_9e55#<$fFhtq7<^$;pEIKwUjo=-XKU(SBHp^EN7I<{SG5NTUho;oV&O-|FJP9Y0fdh%Os z_9R(~q#knO37-L`Zz0=rtAgbuMUWUUh;&GPjHvv&Pdh*FqqFu*Nyeuz>5M~7-+a04 zCvc02nC>dm3F%T)n6Lh7uabhAU@-djZJX}h(0dvXeX@Rwmgr8qW*p%5hh?IQX^!`K z>g6h-NBB^6(==Pm`G7*BJXw&-32b-fUZ7QmFm1-UG7Q`)qdk(p@zbLk4^t$YJDC#^ zvoHYruW~ z&8}yhhx9~vFND;q_YR870txcI0sA!&l42^kLORa)M*tp2?)1^RU(-R?RgbTA;r zNHnNY0fl!83=k6?(bA_5EbnSrL2`YqnHalUsC4x$CEij;T#J~-W4lX*3>NR%w6RlP zf`!PcZpBmf6xL0wkcG|p7t!7Ej99}+-%;pa_`t~!1R2~w{Sf(G;9Ke|5$96|+>5?i zCCQddM63}>_=J3VmlYEXEnmcjD$zj8!NEeAKrPDjs+RJvlna9Q(lH1puXm$+_%Gp< z2rDBN$yHy`55L1^Yqfm4? z9Zq2y*(^u*4%DE?GS&5)j{DM-u z&k8{^>CX8hcs+az5yrU(cY<~*BxZ&bA8|YzuMnI-s!lXK`?&99pCiQKkV|=e?Q)V~ z*D*+loRk$W0&$Mkv@*fzq#yuR zbtgzlt4T6G7-W;ZB^QDucZj(+OJmx4HQ>0-;+*z~-wmsef7u*g0X8cY3y;|v-eNXL zQ@CU#B+wh;FK~>@rUv7>vb;EPqa2+QhWSl0(nK~5U{qA9e2C0!%qnC_PuBjG!{KIT z<|L75613RgQ!fMey%%m5Pfrd-%zeZhunEQkY{ zG+3TKd)Bz*>hJ6RK451l^S%Wqp*=_+&YfolXK#McYEq3!9R5qNvVzn!g?|m+xbf!A z?;kWVdx+@62(4Bs0_y?dw?k_Bx4$I!rK0G^z!BlQvl}{}x4N=u1)Z%Roe_W>AP=3X z%%Y>?Cs+!g1*usEi}0S&wDu+exI}GK8(VO_aoh%jlIl*4W%r7yhoqo9m25D-lXQ;? zWA%xU`dzqhmZ>b}r>Oen&Og*MB zPo-lnGNGr<-GyPv-BM|m-6{&;TbBocUmb2*pJU*0SX=mqPxZH_P^e@G7uEQS9&Fb<}8jKp|oBW@DU z>|!4NyUyCbs0L#8KIOb@REPQJ4iFa$$kL$jM-@4?aT>}yq;n;_TRI*)g zZ`qwGeOH{L{j#Boa`=#mJ9-)dq6s`?;&N?|nhydsRXuwas!LWOxziCAqAHyAe}iI~ z|1XrQV0Pjs^TB#GC_L=oMadMD12$m~r|h#|aoq14VQP|)#%yfseI{QKFECNB^GIOJ zbuB1BbZ#iW3j&@Dz0?RE9*x;i+qUSf`~CNRXR0MNNcJu=em$+N zU->=f`<B-fA);jDv)*0$gC6L5J_)l$-C#H8Jsea zj26Ufr>;44n1`9kAXpUd*Z_-z?*BnR3(>l)}EH>AX8+zJeE$T@RP@ zhty)54-C^mtf5i>k=cLnFZ#%NyzSdj4W9SwfRa^-7s~mK zJrq`3t?9>XjP(W%Fpc^4_3O-Yb4PMslHQ!%cBJ0?Ca>Ekh1|%zqWooQ_=;kNcck(Dd&Uo;BO?0ZzR=n^-cfV*b?P z%1o}loa7&N4F+r9eRO%xV}9oHJ$mgnoEUqF)Ah+*JN5VKt@)`|t(jx=LSy~Dy_?Z_ zrTpQ7$Zd7?=jER_@2=^xYUx<8fu?zlNzBXZznKq<_46(|7Nq__vn;DWbg82MhjLVX z_q0l_Z*T6d{+i1FN(lW=`PV;q28Cja{!4MgMQCf1{E{=SZnP#!YF;$ll1~(1Lp%dTGDGoun8bf`rJPdp zs;^dS9Xs!ZF(^&9{Wmp#sQGVW3qIh#{DH5X1FzNm=!Bf^jG-fOFC;;rHW8nF62T63 z*)Kv~Z#VVW$*AZy&MV1snXkNiTQ?08s*f(RQaY#Wm&)u3Nm0C_HJzaWH^t79I;xPQ z+%yk>FItSBfK}Lt?I8j`k>Y~PQef!kqXHhC_TDsEO&_Hv=SDVw-gGeNU7lja% zRVMUr3Q@6q6wHeBZb0^G@>D?wl04(oZ%fG8)!I?>0fXmBKNr|Wtc*!vT{E_lV=iIv zv`TFMMpgPLeE&6ASN(>;Yhw|7C~evt4v-N^#c~;fQ$eqM@$e%{Kt?Nwj*U42!mgz8 zU6!tK-ze=UldI$(T|cCvqf|k(N#^7+0d7`C$?{bUlr)Wj{4gOiX4*8+Q9BSb=Q#iE zZK^LXE1HaGV;bg_+mx&E#Ci+o#G zt5(fIMdN6{hyQ)xk7$q&l)TAvFMvTz0&fd~+Wp{y&ou5pHZ}5lu!X~rydWqz-fyM; z46XUwtF^S`pq652-@GssMWCNumE!M45~`bjK(p5RdoEqNRe_V`@I1cgmv4W;M~y7& zOYJ(x&dDD7n7H#(VY$O{;G42~ibHVPoWHJ~T&qDqW6ttD@V6(ix59EeflIxasL9#Z z*Hgpn56-RZle-jxk=jQK=x}CSDz|4G95X@xU|R~G&AG9q=6A>I&NFC7zc9;Llg=iF zv{Pq;;0`15+MFIggbeZV(-P{?5QWHCJdbM_xDR{%lQ`(bgFbZtHE%s|l|5Xpjp^r^ z%BYmF6udK!NTKM=`uX>rKTMd`5*um>IF#H>niKSqZ1Y%MqQ|jNadSQ1ou}744j$lL z%0`cP?&^4Iw~5d{ZOhUx8IVrsI)Kdv+)i;A5L8nk@F?|` zU$(QFJM(DCkLdsk1HnajYIl!JGE^Ify|^8$kX9Z>Pj}&DwJh`onX-{$TBZch?8-e! zl`M*BcqQ|9WvST_i6QGMhi+Lc01B33N2K>ck~?_tptM8af{e<$09q|211-s4PMDL9 zPf75Kg&qOYv?wDNIQ{yz5W8Y+v{`r2~+7BX)!9FT#L?WW;CJV0iBt{ zekKBLrQzTbb9_xx-A0YVSu#@HCA|~H?&`hYv^ST6+;zctPqG48^3+3y)Yrmj+gBXcade9iPZ&rm1VsUZ_2Dg= z_2tPhYu{=mAhZIT)N0y25@WB!NCcq8uaEVi0SAx08-{bjHX*uaauP3G;hKCBQ6NMbJ< z`VE2KwBO8N+l$-$t@wMJGdPxRg_=76GHvGXG#702r(AFh?QX%C33K~_`Csy3(lK!^ z$`^lYVjw*rnGG*AAAzG(q4$j+q9qqx2pWetwaA4F<^Fu#nqCnwKMhF5=y@L(13OpIQ{%7gAj0dFXUw7EDGf8=vHvuIR-}_H`NtS`qc!Z#p!ZMT=u@<(vlfb1 z>Lo6+*ku;mdt?W=6W1~_-t8#)#!UH1Bn7PGfm}w?wr#CHf?);!@y94GRP(^R#zj+c zmo=zgU(eZ(oWnUYzuA6hDI-rWdIe4lP#b&8lokS$W(;o6t?rb}78SM#%V|oU@CgB* zWb}b)(>)3qIW5*koQ@y(>YLnmSYA6Jo6>hWAZR*ET%7ZTa!Np(!vdy{Y&W27^U}po zV@zZA*ZzCBd;-0J-niQtE(q+u>=(k07{S^SB?Z@Ix!cN20?6eDSc2|5Rz8{(0KppH z^oOo_G=dq)d;Z{gv0ISI7xb;$bSkgeZ>^mtH{|S{x4eoW(3#S_oFG>xm#l_-eLON=s7uOk^Bl+HSeK zUmVGoN5!w&Gl34R$JvfJMbHb%Y}kI*?Ol^)3a*>G0}(>Tipf+Gk|xQY>4kOWM@#Gz z)K2z(dFyJWyM#q6FeD`OtqRGEr~_7?^n69%g&^s~@_eGEENHyH-<|%kU+z`iyDrV^HclD+Fx$}T@mUv{7=n;qvLp)V}%+Z7Kc5Q{f@kp|xOu6t`OSa4|uIJTQJb&NZAC`HvP8ASSS-X=pUiWC!$&8#k)CeJTi>lMla;Z>Dxch`Qp(7V78Dk zGT7@v?^8l736a=q=(_-N42h*RTbSx(J2C5NNy+@;4sS1;;e%f z=_FGj#PRG^KEh%dERTFDi)A8th#ipykroa%!@NkwHt#ev=B^Ki8oPDlg?z*8GiiC z%lDoIT|J>hasBFHc(gLle|cp{SWA61fr}EscY0uOV=1(GPKEsTENLv_DJ&c~VrMWp zhatL0Q2fzi`^4Ygz2M^$+;y^QFn9$+GVVxUqTR3>(zF(Z98|T)G6IfpO|lB<+|FT3 z-(bB;_QDZ%m`18?Il&h>qMD2#L_iL~{r4x=O{j&}2vIiPK)VWuAWtg$c@@|bM}@^i zbCz}JYp2BY8Od}RuR${_PZQpa&4VU+|K3PTotQ0@{z{dq@Y^)8q0a;445zT#<|_@r0;Zx>ec&6W!BxQk^BEsB0;8tFOpc&^E} zeD8G#Ds6F{_>J+L(A(fQ3|EODM1RaKd+zZzh18fl8p|#1f0MzRYZU|g~4Kko*#UvVN8hs+2r;xQwV47wqrljjLoK{HE} zz(uQGGIXK8fN9AnVQC}sgJAIeDtmWSn~u88y5CN2({%Q4~QG$wxTfCX;%jd$XpvX()b8#w{zd~E508TB^DI{ zMz&i3*idU42@R8zRx}kH;s=3q46)M_?{!*)^-He)wr<156J8BHKNh|F`?bbsfYsnM zpQc*R=;&y5xc#|h6OVG#5_8|>lKsH!n?!!Gzjzud-v^$arb|zmX~au>x^MQ+d)ImWPkq74bAbOzv}zwz-v` zVv!fW6rvCCiSYiUw*Vi!ZeCmUI{W0KTV$yt*bbj1IF|q_a7o|75k5HkZQp z(keLfGq`FrRz|uR(G4S8Sxn9ZMdzY+rN_t0dvH`0LLq@krwl6NzFMhBpI%h{HZ2F8 zw{L#lGUQj>dv82=U~YLj&LVlk^SD-*}HJUN4q_P*Ub4V3^F-F3FG)d{?{6} zeAwJ)(wH$u=gww3B~?_E^e*>V_UYrFzg??7gs~p&58AzK>*p!SOzIz=6)UFSAJjJ4 z`_k3zP(40pSAFvRbJDX@;@nUD^LWn@yPJS5`br_R#G5ZH2g7plTmJll^VAamzQ5bm z3mcaDNB#w!w$9%__jo?7o?rQk%cp$Id;CiuiAirYY;7I=0joX%<@9n(Z&p4$9=s-@ zH2=xdr;TPeGy?{KtM0R_sBrWGT1d;iP8m4<*=lol{fidcE~Padenrhed2#iW%_n}b zzh~Gn6;hZvPQ1YrvS5Nt@hScKkS@E^mN1e7@@VPFF>gbpRJ$m&eyy)1mW6jFp zB3X`)JYU+@d_}KcY7N`9Az{(Fiut7-XWV}>I_cEakl&5{`!smc##*)eg+-?CH@W@RkgItU+oIx>mivbGi zQsxpx@|Pzc9Z5R1r=#?B;~`8q8vFk>cgpq2VGcC+wIc%OE4mz>b^tTWR`;ZywR?h8n#K z!LE#a!5(fz-h3EqqI1B>Jc_@*@Xu!@H2IIVP3fx{#Y29a!Tj(4?Z>e0cJ4os_Ga2F zEv?5nw5?h>%5^yP6oDwhl~$-S^+GwBGSl?6ye`+b9K2Rk9NzEKMv0_=*akVywC@>3 z;%jPo?1W7Y^Pd(F{`cS1e*7B%wa9J2O~e~+bN8AH))Q&E;Ep7%n*CPVkFgJ#E&X#L z_6itk8{<^`Ql*uUj8@l1jvko|PDlO&Oh7ZfCVK6!zX}tMj~OHDf_n4WOO>ye$Xn)! zl@F>9x^*Ql7i#?>zNFHs04C^5vH6gJ21EF@GJjqiJ|J1Ar|F;b+~sGSoHjx$RoW1* z;4xDsM|n_b3d>ornwqvX?%7DKH{%ixez0KQLs1qt9k>+7!f1ulvzQ9%EtByS*C~Bv zk|nMGr;lFwep`br3Uu0sku+8D9 z{+Xaq4OqTVx*N1tJr^t){Pi>F++BJi>TbP|g~5jUpX!phn$k}MZxB?i-{M1vfW#7+u+#+#Gq}=b6Sq1Ex5tND>_xE;TrrxI1bd@yh8-WpnKD?D z$L{3)T(T(+cIQP={%Y36zGy!(xTWma&+x zgTnqk>#x!&U8bHoCh`?fezDnLXB<0A=oDWB0XRgBD(Gu+Gnr`(n0*?3UE(5}HzTia zH{OqY%Lu!H(V+u&G;Ua$D?(?%_TV~;bEQblGi|BUMTtryjh2_^^k6NgrdyWP>Ii>S z`t+c!`bwdgbCY1scSG(!zizbddIt4c zq<1)0et@W$@1IJ$IO%*}S!g#1td02r@U-pMR4ceXK3b4l#7VNSIR^@Xuj@uOTHl`^ zdx!H8WnZNKz%SP#x}nOJ4g(?v`L0{rjfa7J)jiT^eM{nXCa|2?MU%oEg=9h>Uq#|I zu)L_JAU5E8nWUku9lO+@Pp?>vyx0j!Zo;f9)e_>R5k@0KI-aI$swr_WnZ%S^v4WR= zBi@y>*M6FWb{q(Sw!op_jyQPSD;9>}iS%&Z2*IJXEOk1Q8`f{-B;W~<`r2NAE zrTH1dC0O83$_EX;PfqS9h#Rp~Mr6vxhj&`GQ7o~@i=;jB>(OeDZD?Ji1n(k4U+F2q zuWBocRO8*qdWXueB@eqPh5j@zzAaN1WV{eN zLI7ByFc6z<>D2?5m-t$)Io0s<%{!MbU!LM*r4pSqskdN+1c6YQ-;VO+Rn%V(XI`qK zlM7(pz2Lgi8NS25lXf(dMJpEP6;*pdr98?4Qi^enhIAYE)91*#7+l4uQ^dP(f=bg$ zwIjd@lO3?U3o!8txK+a1t~s%m-d;}iGfR42_8GKD=F#%lgjf$490I4r2vAUX^+e1`Pyg@*?GECV71 z?-qAAj@X(L%U$(?e5>idO?m@L93S5r`(|j~6NnJMJWM#u=e^wTKTrjVDZDY(h>4qT zwQj6HRcPd&3qbjmHr?wC6Zav7AB+_J?OT@e<69uM_AfY=}N&{ zH?oD-au%_P%9`6ip75SthlL5vQd9cYC%+UwJePOs%$XL-=Zufr+@+~kXBuxk*^7V1 zdNVg|q^d6lkOOS1n`J?4-^4BZ%rfzaUsjx(*{_evg>^I+wr8t}`lWBhS%p6QCdCAr zeHr+i2AX-!>~Q^QYFPd*d>~(~q>=XI*|S-S9n@_7kImK6nS=_!Px0i{tGOVyb6^;{ zd>i8d8FD$T?#yMJBqLRCN=th{Bk-J_nRxRw8>jdDV_2v-`x%ounW$3W`+I!ZOt`Wz zA;mCdrPMvt#A`XCDHD7RHpHIeNa_f5{Kme)BQM0*Jyj_jhEX1zcYUynGF?*NQkJ_Yog2wux_eaMGAjFh`E{Yv=&2pwX5Jxs|alwlyh2_%9{zKu$VBOLi`0;Ey zUW|%A1NWjJlt?04gEcV;+J|Hb&_xXRa@%~7LZq($`}A~+m(XB3Dn%kD8%yq7}D4Y7cO4_`243 zB)Ze-!Sxg3>5z6~>Q6}D8mQywq$I7_-OFt}q3wEuXWB+Z7Z%kas;6TU5e!jKl;@ga zdpzSO1S|@1Hx#pd=?jwGxgQvx)NQDam%dCwl1BgZ>C=ParFSljFMhma_?!dk+1Vn- ze7|Yc7$=rrv7 z|D57qNAVT55eZ%5qFZ)aF>8gmOOj%T*;_AErgLIqaj-j#874askSwmv3^9L}@Jc(L zUeDNs%(@A(*yS}hcK4UjcWxfbs6&LGP?*(CWZXnX$$TWW579dQ{^`nLPg85ZC@+Ix z7_&>KVaJ!CW_LJ|FbXZ6-k2oExWQOw@EUt@)AQ%d`U`4y=N}TxCE3K`T4hNgD=6qY;R+e zYGC-TO3a!0=Rc1WZXT`3_jnQ}zL5_l zeVm~c>*_a-d|qH_$zP~f?{XIHf4^wq!WGXyf0w7j=pMkpLu5Z?-M`&>daULsE#prV z0-Oz)*C}>zEg^jEh`f(b&qk`0J4_DrC7i~2lWBA$0E|>I>0;Wd;JOI`R897&1sG8t zBqKAgMkEla%yZFxM8xBaZA4O8Dw39`22_NSA5*Rl{>C?mLertj*r zLyW;tlOg>@PCach@J0YhSH~-YV7;R#=R`|oa%N^`P&bDxm)pB?T72XY^rWOS zK0k}C}-Q$Ru`E^?Nd%q(;M{86I25oO~5efx`-TRd9Tb`W9QTI7PGpmf8 z_#p-o&6Ln2`j&P5yonZa%r5YN=1Nzxfvj~atuL+v;ABA3h{9n9iD(gI*ICscKw)`j z)iCCye888tgBS$mUe2FI)tl1y!;~ujn>j1Lf(ctVu&sh+dJ!;y>Mc zGlYb>4Xfk>;*Va?j`KbF)~#C>CS(_`#E#8ojr43eOr!M1aIi;)WC3j5g1hCisSwHZ zPU?h_XZ?vY+o?tCTMS++-J1^A&HDEbq5QPZv)N@fLsB*xI4z%6WdXz8wn z@o3+t53ZwTwk2;qe(b}`d#S9s0oHV*qFSzNKR?v>TYbw!!xzvG*pL)VIi$=zf7O8( zVN41b|GD}rQbi!NS))s<#_Z#;EUbG$H1(D%Iol$AB~=zfn$hJ%B3Jl}ZZul_VD5rJC`-|1N0;NWk&&m zIq8qurDBZ3{_06I6e7H*EjvRf}klQnsC{Itey+|#-H$nH!aF>Ou{ z1??~VxB$-(4_j;0EGF=Ldr8Sb;*|`|iW#f^jY)|en610Tg^zH?`Q+!SK?$<`1SZ9u?ER#s%>J2eI zbpvqm@LT>?yC*nT1|V%u%r6&VF!D!c-J~p8hG*-4UpeGPGiPx58Gzd!4_d#3#YENc zOz?u9m=1KfE~sU08JW#?_(MCX`QaYamEOv7BE^kw>w)L5}=>wJRJ30nC z$wWfvY+fp&iqjs2==aZ6U&lQyxxam_4rHwO^V>jEZu+5b;T~nKdUvaj@k7Vj36}7x z+!Z@Waz^GkkuYS?EC|GHzQc98c>1A+)gC)R03!Hb@ToBHAoEFP*gH-uX;ThFljx88 zGGgGRb@E71n>gg}{qFsDW;18ZJ!U9&kRwdVwwExm$nG;=@c68Y-k4PoT1ToVxi;S_ z!D`h9 ze*(x^!)DE1CIUtL1y`!W^>AXlC6D%_HNe_v*rB&baz>Mt`zhoTK~R!*ql<5;j1Xme zE_RdiX;h5nuH541L~dA9QGTJ~M8Psp>nDYUR(jexQ#fZfDTL|ho=3i64Hgm=JeUYK ziz>szdyP2sS0$l&NTL?`hO=vyb4pml&2EW-wi54250Y=oX2EdX#h# zY^JP2pPCJ^(s1VvxtfpHu{gv~X>J%i)DdnXZt`5X(1b5E<+X9ITTO#(ShY*_$79%- z&QpKszwp2=u+yEK1e5bm;2i6oF&{8VqC4SwIv1VVFD8dw8TUU!dmi1Zs8Nmwm4V4*FiATXE#Ui!XN#s^0lBebPX_vd7oPU1 zI3HJ>e{tQlK_tz~rUxh9)cNVBW8KeuU+K07y7^98B6-$-j^5ob(*i}DqTTvms}+Ns zXDJ^6-w~{~16??zp5#gH&SBtC zW}k&7qg6Q!SCU`sBr&8)Uu2PlT(H<%1fZzZwBdTI{P0vijp$_mjq0U4y)ZTy4Nl_` z|E>S_`pZAw<#)B7*iM5pIC_2gI6^x34o5qD1(8VJ-lY^0@Kg>GXeOrXV|K@8{}!@wHOu_w{|E1P~&8< z%ULJ=Z`RQLlbtu^#oH}wc8Pzsg$b0`_%=BWk?BQ629n*6+WIxQgWO^c%uNc8v=$-7 z;QVm|T}bkhjYccHwk$KTFrj?w1sB?sivPH3)s{|v6iIT_ja21ed-5yyX3W#wb?Dnf z@WUcl7`AKii(Q*FyBrd1RG9y@s~@ytHj+fJi+u{a#{mb2K=0c@yI{oGuHIDgEOS~! zpNyJ#X>*A5VKC6Rpkl7?L}==2$mufttR+q_i^#eRygRgg){`*y$)I_n`#c<;IeNQU zCZ476%KGd|lc?3i*7cV$RANT;h7|Cw0?}~J0*RmqySbw4co*M`3~kT7s;r_j0=P47 zxF9xEBOWPavL2jB>{#5Rj?Q=S{X73C>H9`mU2vo)zvxXZ)Pr2NBSFx#&*~P=l2B|4 z2?^QaBEaOHKiAx={qxVgxzC&5{xs3~d;aHDoJ7`?w}JY9^m_CdOCoI?PlaSp!Y5`jaUL|0gb%3bOruH2!lI)D>N}fVJLxx$HrhBTu6e%N zRqtXFsFdNfz3JKzPOSKvOt4JRC@Tfm1X1d$DL;n{nOiccQS}!@3Ibpu0YnP1G0^mO4w! zY(#c~O~*d4Ec+KK{xJA`(dWL8CT+tXQGgezp@`PKIACQC0KF|z!kzt=9AVer?|H#@ zT}re4JY08vwP{#B)yN&9$YC~=hps%lFcZpB2bR9;us#V`zGQFT(Oy;cEC^L>50cJE*LL>w0f>@1G*ygrj#n$J1-6)5!0ug!vtD8zhZt( zI+MCqX%$PkM4->7x-NYUqY{n)U^i+MJBnjCLehy&-eN{+gdn zecyTS(ULWM62$W`82-aYd_fO{1P}? z3|u)4)C013dNRk(*g1KQ)K=aGgChlxk#3*h9+1ifGy)Nftvf<;!VvNk z=WhOQ^9KnOO~DykP=Us?1sxnoN;`wf4n^6C497$C70Ca8qwY*mI#eXG#CimX-#~*)Cs=Ea} z8@kDuETuQiRE2E?&&b|`C2_LNZ2W%nKMkm+r8Ati&S4&tU%r^_d!PlG_mIY9bAj~> z`%mGP+#*s&mEj|J(XW|t`}6gxua6^|BDuRUy#MpvRQISbt>?^{6Is^ybI}`~M(*jk zkDl#I{bnW@o8S|2&skJEIZ{H%)^^A%8*XNqxRNzzt>o9cddGg5ucvMC zlZ<^Ey;QOzjNW%*jTF+L!$|wUQr27_O3c--R*PTYlh?0#bo|-X2_F(SZWu7cFQI;U z`-@^bP`Kj>@ynHF#L`2Jj)fuIuupyMare*`$$WIU7XGA{Dut?`)iZNqcSn}iNEH|0 z-yBtsKsrkLL|a_gxh@n7FMRiF&wP=;W*v#@ZK^`xSw973LH_XaP=L|NdjUyOQ&S-< zB=@U`L20FFc1~*`M%2vd&u# zByB>P*OTM&ukI=FjEU7ARVC@RPgNZpTnca#NOvh;s7e$b`-p>OS%9$+sW`$?XwG$>IA+hGm zHYwiGsvPBpaOJY&iqfeb?zN zjK0OLlEmE90|<%-3JTJscqQ=vZ1?J~d}1OqNNH>zUokX1Gk9n?hZyx7z@zc`3Jwor zFw#EcZ&{~OIRJ@;V^#(E&!&d)VvpACa9!-Ja?8l!CII0X#;%~TPQ_;ANZ-_X)z(@` zJFW$u=Nul9x0(ilq1OZ^BR2$x$T+{Wc#8BS^XcxsTDI@35Ojs+xlXx|#1~5~bfv>QaBU{FbOgjRUC*ThbEiaD6cau4Oqq=vf>Gw@9G7&VW|Eh;Zw5sMUH0xy>r;|E;+*jmt3&|9+Mk z!q}ILvfgR6g(Sk*T8JW&U6HgfBx8vhW7nWtp+aaOr9`qt4Jt)as1&jmvP2S6{lCX; z=KpzKKhK+|7xS5NFV}UQ=Xor@<9GZHl8_?_zi67=7Icu5CsmTi0qzBTGrt#co&^!J z#)n!<+L|=1K~fVUC#Pl#4X2xuOydb~E9w2aY}eA%bf*c-5&S(hsnFnBhK5ta0tdcJ zVqBur@i?oUiP_|AQ!ab2qR?j{MjLrJjqEA)mm0vRB(~0nBu2r*-=%scl@2`9UH0W& z5Nl$Sw8$E2w@hx?x|6${3M?sH5P|j5D3`4z8aJW+(T%N$bG_HU>MC(}9+{GaHMloB z+;q%aSb@7p@pv5VUzkbkibn}VD`zEioKDzIQEB<)Q&;696R*M&AkEDn%ALeNPrZUz)}8^sIWUT($mdx7Ly5wLoal3^26pY zta+r?I#_yOvDlK+h~x7Z4!2(4?${xzPOO^9y$nDm;frCLE&Ty3a%ALaa_zTqTfTy5-DZ#dwD9BzP2Lr(c!lrri=PPhdjR;?`3o0hJPH-*x!!?|3dXpn zttF%$$T_hTNMSkPbjXUHO?T}wWQY@XFrnjF5Tz+HP8ALWkr&{nL-vCf&pNB$7cM`` z6f#2UJE9UUNq{H=NS7B?OIL1Gs(kFDBgA~BCtV+>Vv@$;0=F9P>S~F*uqA*=V8q9* z_w}xA6{SP6U!yb^Y9jqU-#}U%rB8^!RNkOjK8qgR^;SE>X*Qfx8Sh3v>r;(tnMd*M z&FO4~C93Iuxwro4rD*G4y0ds~)qyivgvv5dgrF;&BJ00Z$lH^vAOnmBODdG)1t|P} z!`AHc2rxkev&}jBdxfeiTDv^sXrku<0Qry>i~gjVxL8`7DWE05yAno+LPVL#+qP!s zwrnJRX*u%2WWz$&96;8nQB9C6b-h5vROqVtkVP$X;ffr>1Ev><0*yZ6L4fx$c-L$8 zt48dvC@(*G#iZ_Y%v`CL9U&h}vk&r}CEdXIRO2FPjV75E-3&>Jh10`HkHQ5`?d!qD za;(>1VJ9$Ud=`zFGEZ0`G{_HATKneDoWE~L!O3~ts-0hr^hEH^SM1}7H4}bLdR@DA z$9+sW`LaRUWspF741(y=!lLVW*{b2wnFJLwnm)uMyNUQj7bZB!TpsVeO$#=X&Is@N zv!gY9t0OnzYsmU>DROQrbwB;baLzj?C;jVW`KB!<1Y^uAh3!71(oTeePLP_0or@~DU?yt1J<00$rW*DWJwGJkD zapSC;l2ycfOK&@V9|u+vL^)Rd@9OyDGM z{qW(Rl;}5@1lipp&_4W3v9K7)N?ZYlj16=xZuWbe)$;uGD_DbP1tZHJu=}W2d5(ph zDymFE7OtV3X#AGQCW;*G5MEX_6pFY%p?WZ;MxloV`P>`Ni8j;A+rM#!>|hUs{nAsw zXkqjV9ZW=nh}gHTnB%ub(Uz#^S0+))qy#c@JwT`9lOi%FzrVZnQ}Nu90OnBIbUHf8 z#MlUqRp%XQTRI8JlT=+r=-qE(3m&!FwNA!*f$!TUn#b-E=olpBe#F$Su``oa+REm` z#LH9)nbk(hwz8x;_vpkpFl&7ilP^JS+|^o&w6iM{#H9C{{JVH{s|PxV?M#Qf-SMrZiu)eZQ0L?4)xKU4MF-iQ~F^d993neI>jjwvtojWvFnX zM_riglbrQOLxtMUTN;q8^;1EMS)tG{i#|1_WFvvJ(hrnP5((0P_`Unoiq05*DXF?r&Dn}xnK7Yl32XOZRpS&- z_2$iu1uP#}V#X)3x&7y3l-rcmK#WK)S-22l!itz*x2$2gjijukyeXv<@cX)v)sp&r zV5Xk(Cj0eiqD9vo#635lhKw0pyoA&!YnAnQ#L^rULHv90 z`vR;@2&oab$MQQ9>P($M=hrJZ^#1ed(Oyi!@mWPn-_m8M@C(xGc2mbX_|y4P_0Ge* zRfM%m*`OC-uK(PE{N#r58>#4?1E{HNm~jzRx19I8a6eCap4l>PL+kR~?#a)`8By;W1Si*sc%2B*5zL~cWxiM-Q=;ZZ-IKVW&==T?5i9PA@-&q z^!?HdXC&T~oS#mdYp%m3DT6~UWf$oaQB2tXB_dfz=CR7AC*C)9pQbsr@wH1F5{JXj zaHc^x{Sf@H!=V1@B_*L0qJKpbg1q75l|fCr)wZ%q8AtBFvvQ2kLXADZy=o>m_*rTk zSeiu3->_Rj#MXr|2i1}?IVDb#Zg8|e-<&&={TIZfR4;q?=UrIT{$yvpEuI7>(qq*l zW97<~`-JP401R4ev2BX7npiDiy$RRelC1VdlTs85M=#pBWlLY-aYFI*TiS&>%s_xw zo&Xq@qt?rMnolIN&g@FWdUg>Pe~_b^AJ-*%t=RslhZ?CJX|w9iD5AS8+ABN?;YW9N zO6f|uvl*?6!G3jBTBl>iZrISBt+H~>nh^2PvUReX^<)re%5sJyz&|bWTO+ybpvLv$ zf~DEfU^Uw>J@Nz<8u)S-hTgq9$s`8YV8Dx@7_YKl~pnzx;7TZ&_q7CpwKT zN?3GNv;XWx{he=e2V|l*F?t?_a2VYcLG(YRyUy`XZor}}*labg1qWvm;n+e&5OK2k zqNW%n;ZaD``=N%U)|qfpO5)BeObH?-pA9=~@dQ)sD<+BJ!NgyBey_fAj|;j zLIvdTzGm~i<|@9oZGNqvlb=Fn-{d7i)WT3tBq&QvL@P+@CWWvQBv6B%A`_UbuZ#AXo@kZQ570ZB`*2KO`@ z;^(&9gFxMut6yu5!u$#&eNABFf&(PB(&oXw5xR~tTqv%v!A>irD@P{ugzFTi5|qk( z17@wH%^TRDPxr{do9G&Ri4zc3vU?&tJcnc^wF#q6^tA3$%oXV}>A9>O(_lr*hniQ3 z$E5|KhAMtuB6TlGv2iCNUfCOdX&|j|RQq?a4P~MMJy>R!nXq(2av02wa8pI>E8sI0 z8`tdjqI)Ol zzUT~zfE1r}`u;@9aYCsf77SIW<|5LOrC7vRN)g|pydeWA%$R#jWyKzn3T924F)$BT z`w(JQl3KLkc_UhMRzUOjTFCl~ip8lOEled%*ZfCoZuFy|(n+>IH zNwOvlN7AWH#_1q)evxvM=_S&r{m;GnedrFE$FLC@=K(5m6hkfDf1>S)#*!E)DNAu0 zkep1W>otZzKBR_|xCxd=!u7$M1n(=7$$YYi!s#TwI>J(N%qQaYa@gmU5C153SZhug zivJbD&ISoI0gJy8YK3|Hv2>{|7=cN#&)0y1iVoVM37DA|>A|jkpuI8wUkyXmKrA|7P9Mz&F|0($E!B zrc>-)@i&o&DAe*)vTU^*^m(zUv|7pBmEUR76Tga7U;Io5sE)PrDepsNH!y7dvVH<6B%H-yl`B?Dl{6-3>P zs*k2^@ABN%IukUVy8gDU)?0@ANR5?C?^w2=L%k8XlXY}^l(wj?4(d-SUNxY;y6Wu0 zMI(~P{Wl6rtA9Gf7>)qnO^X}FjNqkDM=uTS*!mRS;w5m_ufx30$>bVo8ih_HiFAhr zmhWY_9yzIUpHoVi4pC;_v(+-#mUwNo;p=D^Amt+tHIzOb~LuC8w8 zR~bGp-A5cW&)VH7t*m@!nXUeGy(FhOJ6z^M$y6Kq)SgHE`E%F9<#;0u?x#^En*&Z@&Js7sHu<*&^k5ltS+|f4RG7lV0$s>llZZ~Yy zX!w~+yiVk>MXDRg3Q>B%B>3gxa=kz@>`_lRGSr~m%6l~0OyzH|Oj8+`yDDq{;QmwJ zSHJ&Si%a>$tdUn7YlgV`edxDDtvsoF=$Y^DvjcH!$2lit%+ll$X4T;7-Q*?SjUg6L zb~!^;6iaNyy`1X&;x>tTBl{*v``J(~sI>|pf{;@b*DL6T?Ked4W<-}8!LE7f;Mok4 zpdId3rs5gQ?}+07(oyJT{?aNX+J(;+m(--a;W5tC*o)uZ`TX?r&rtC^FDFw~mx)`1 zO$qetmWt-KM)H^L&nva@1>l9 zhZ(07bpNy&Llp`0CD@UkQ~GCeq#q`RO0BdQhF)-7&bh24C(z~gMZZ@elDx)irA?YL zy+!MtjnfQz@hOBb_5x~CC6Fg04j%&Q_nULC_+(erX<594S6tcOx8Y5-6#>0@(G21>6vmqA-be#aay^nLJVVTDKV3@r{ku}8Mf)f08uij^W`Y=gLJ&N*# zvble_&m*wThZZFN;FM2aw*Is96ky;pYQ2geo@NM1tJ#@4pd4#3r4#+;(Af(|$-6(b zZ0f);9ISEdm6zJ%rOKYC{!Y6s40W+vcVCw?6l@~VOEVK?E*K9 z2UU>xoo^OeCIHUz#xROn=qy5@guRBxb{6!9M2uXYOqx6J(p7{{QKzZc}pgw zm8&%`*hyhu`nlj&of=txwi`Y?A4X+mi?izA)8d#hqpcNq@;VT3s6}9{>FjclnC`)A&7m z>FC-gMGWD_{S#O(8#rmOsA!5ZEsjv$1qo|O%zAYtd%#TUVfc;!0tx}WF^g$J6ku}p zK@yNCPzpEF9%B!@Z4M=|1S|tnz(xphg;tWRua^|sHFbp-D~{<$*X(>2dev~0?_}L@ zlgQbg-Q!MMx;A$S-nz#nGCz02jH$JIf}`3+M64Kiquan6xqBk^#26(Es4#WXR$sKu zNxh_TVblQWZ(8iE`{aK44b`WhpdjVwQT5d>7ACDa zEo9f4gPR&YxG6)t$PJ{BH9+xr&pba7NU5YHR=+9nyj>A>>R+U_LLg-Q;tMnZ%_0Rr7S!p={@Qmc6-JX<;nq2=)&pv%r z()w(vMHVT&{?X{MJ@x5JjS(CBr@f?ngFh6NT`jFfx`LPK_U=7FGD!F<0ekjDk~#+- z?^o}fzb0mq5@x>E9}f;=!f()YyZSBBhODxj3Oc`a`64C#T}Qy4bb(7ut70#{o8;)& z{TFq$H@*Ayvt=hRc)a;9>gTjiru)vI6+V{c)mXxA0Nuf+&23y4Em}kpu!~R%gD)+~ zsp;u^FUa>NRoogPx~`O8W!`Cr^Pjm6@Aswi^EP8^6egeDI-$K6(2@nTfU$QMYbBUOnI;#)VOg`md^3 z`JLW|y!TKx<5&w_E?X2qNyAVYXMia99^o_HtUA&r3Etpo=Wf?k$6Td-SsROI2aWAX z=Yp=b_86KzY$r`ph8K5g4WukZjRo`Po0#a?7;pSVT`6#cdFNH5sWj)3ho6u?y?-Bv zfwcNoQ`wyBQ}fMT1Zh!HSPXy*`x6l2?5R_y6f*U562NsOPubK&+s4>_kiudT(VA#9 zKur?0*}P2+UsdHF%Oi4PpP7Ovjxpc(*KO4}m6?B?NZY_#8Vndmwz4QYsn7 zfv$}*tevdOA$?BP-833LJh^*F z0}Rqs|9$(8;aex0Cb2>4a{;zj=#L{@ustG`Mu35ZuVEYSfrfzMy{$h2CSvf$~F;dv#IZ9~A^SG}N zX)E7l-{5NlMYdqC+}`CIaPb6vwrw0xz_)FR{S>khxF8sKb&kD%-oFs{qddkW264}hd1k0r~kiS z=XUt9V*mYq@zB4+n|}PE;3w6C|LuEI|=@$t&u+J9catzOwAZ+4=F)2EdUu zH?PM&EP48pw|}3tVBtb=$k~Ogp8p3d_$$(tLV7Rgjn#MZ`>XX+=Rv$j#;g+*+S1Xj zsij&wo3SLv@;f7iXS#m}`QG;T->h!*C&+{*J-aSm4FYX7VS;m)rfQE{%5fW^F76$l zmPt`mbRY2yfpAlyCzJorTFtN!k#RIn7i+Di=Hn^6kY&qG5iBx4)+!=ATvtcOoJ1mj z@0@i}*RNZE0-2chWE0Q&eG)rDdm0UNidPg$^G;_EbJJ>p+73QKl4a6aQ~l1Wk3Clp zHoFXMR)&k)RVc2l?#0Y_uzulMggs@a6=>;js%_STNsUtihRu9kD4oQZa~tWR$6|5l zqJ8RWg&dLfRhlkoT09IqIGGx%j85+C^L^=o;Xboz1jnk`dV42%)nXyo1Oj7)gyv(> z9SqxR_w2dHponNrRtBOAx9D%Vc{G6v4pQ_dunf z2~>2*1WGS4#g*OL)oqj=Iu_y{aRPU6xB8Yyg6BdO!uZhAWGj1{TxN)GDzP?ra?PG9ccR=jK$jSKu zYQh!egJW4GHDa~w*+!zPiRh99c-#E^?qXGHm1{oRoAQM+VP57@?4jR+{z2&^;5vxH z;tIUM$K;ze!k{`#7A_4rl7!mickS1_Fh5M!|8&6@K77dWIQvj$n7cp*mi`Ey?2#i! z6g8LU%#EKG6Y5!6JJQC+CO*5D2c9@~JNwDlj0^w{XHM8=%ZU?{-(Bc1$2-U3)K|eL zc}7#`1LajI;eWik;ya}HvW*~_v{|n>7-I>U9v;X|oty^aXdXVgrg5qw3 z`ImKp-Q9kM4N-guSaY@8XJ0Ke1F!Zu5X&g?gIzbW>xPc9xG7|DD&Nh z;d=o&lv~_ftDq~Oa zLByZ9hjtx1Mq#&RxXBFgf`VzN9!l{AdzVL2Scz(yUXLDO*LMG86>WIG$=Z(@IvkB$ z7CYeJ!C2l{cE0rBZbeP$@H*QSD-yllktVg((&|NYDEem~K77c$e*HEoS}_9iVFyo5 zGxV{HO}^4*;?Tq3%OWNx$gac}Bs?_;=ZWBiaLm~7g)!Kk%WO(XOnEAL!-mZU2AWLz z06Z{d{yPrGau^CS+Zp=$Ju+OGuVPm9z;ik%p#OQS7($QekE#UeJ37V8A(QFjsfsK| zq=I22LL#lqG>ALK=lx3S|Lj#S+KKvF$BrG1-46QHt`a)~2SH>smNSilo&HkhZ`j!d zOqx8ol6<&*)GDAWFNp{UDkxf<=O1Lul!%xKo2b(6`bYh7i6e=?a?x){fE7yDiv!)# zlV~>JX%4`+cK3u0aR_W^(%FYALb#X(Yu}ln9a@qdD!jLv1?Mwm-C7JS_m!##moOdr zw!O1+WNvP50*x>a;r>XP3Tewe<}<;%R1*2+sFYE-8+_UnK@gP61hjd}9za+W$qb3s z$mIeF8E@VkPtFxhZYio_+q_p0lrH!ojo9w~92018L7q4h{T*eS3IE=|$Gze#`lZ%h zdFU9w4`i`@S?qf}epZ_%O)h;dbQ`O5IQRQuuNW8x{4Z6m_^(iDPschu+TA)+d7?ag z1L>b|avcY5ybpyS#)$$y4FJ??++!=u7i4(#iWsc-1Q*$_{q|I@I+Grrg8g?WB8gfN z_Z{Q3QKv03`=<)NQ=rwbj3mR(i)Y3XgnbZ$5*$^kE4ydg{s0>`0i=ICe|*r1guyC9 zzk_D1&YpBzcKYLws_tu=v00pPS|!w<^3FRrI0&DgUWfM8H@=PBc(gaYVz)VjuCB-b zV*mkWrk0qSo6iWfj+UkhK zaA5wKfRgYH)Mk)Q5Xd8rjtC*11nt_jtNSx9f2?v9*S7th;k1*0c1RAL^K$28j#Vzg ze8l55zD9!Gj+tPk+g3xvCAQMwaBc~S!vGWFKwr1U&6?dsKMk(lU~Pp$)Q)6um{%Lf#Y3{-UFum%V;C1JH6do;!P3A$ShndoewU(bNQ;V(A3qVRU*}Jh_zV)) zfmYl*Iq&MSI?V+cE6z}5u*Wf`riCEbWht!O@Z#bk_VH#k$-kr~M<1LU?xo}Z( zik)*VG1V~#{#amW=#q*{yIDO$<2+ioDGfNh->MKEn{r)U*#_wriUy)wd4AoFvMg2X z{QG{N+KNy~Y{}R}hn518jEci+%d{a;g)NIxmrR^2F+iyl5f#Vm8~EJ#_;_V)Rgz|x zE^!2?)Pcw1LVG9HCf8QRB|k6K9hVqw_T}+kscGM9YT7Y_4trF#$WQ38IDL^$p$Pp* zZU%%;8P2mO6NxD3JPIc^#l~i?rw0)R`ApkRbuIg9QBsP6QHkubn-OM+V88#vJ2hEe|c72qi`71%4+qNVRVLOd0Fi~VMrAC93`Vq zpO!ChcCLM8+Ac6!Aw<}4kvXNfx?)B;&qGO&?_ROm@|jow-;gTHUCLn; zl1LZCYB2NC0EiDw(frDe!cu^$C@PNP zqY5hF@ZOKPnAiC#*v|I91GQ&3w2eItPXl*bcA>Orh`nW#Dh<)!%P;+MMYXjN)tpX< zmSy@`LrrbRXh~(YUlO1TeG`SMSGX0_3zVEf@!n(1&D{v>l;MCo)7_+NzUKYpiDnUB zzO0n}{_52$?p#I)-Gibp`o7aE3Y>OYDg~R9><>sm$~sheMT0Elev>ZC?N3aPz~k!= z95_Yv%m7JI_gWexA94l@pPZ$#J4nWwfqm%SkOR<l5HE+)>JJBCe zONksY^J3IM?gIh?BQ|VkD9?7-ut(0bo^-wn2y^Vsn>Wi4tUh#?fd(g|NKeM*4os$4 zGmnAO)5718@*20_{vz0mLo<3|TxE;7hswFnfBr#~E4mLPYVTt_ zyl>x=Gl_%F&HjOCI669hx;5mz)V2knTzv7ZF%2-U0pcVV9Y`$_ge?8rj;kX{dr`c+ zC`~R@lgOH^@&2xllFrLXlU{!ih`g^jKFKL1o)$S|5M|-`0Ir+?(tfz2$iukeQeADe z)X{PB+SLCjY`jK8<1V&tTT)s4(EFLMBlgbDC5A=iZ@93DjLGn&U-L`q&sYT9e4nfMGWG6{`s=li)}k}I7%uP4ckCer*XD)#4OPx zBhRhdann%vK~SJF_p zXJt$>)m)lAkJHp4z3@nK+f%xiS(LqIClxqbW8EqrQ50~6Y$uSK03}=@W|uoTUQpK3 zK;Pa{Jj}Igqc4?TY0Bdb1SF8xkV)u~aA6eksg&HfbgReCp+^XUULRcZ&Y!R4W%tlE zm!=U}(D0{gnsLvIVDv`u3I8WRlU{Z45;LH#9@@Eamx@bu|Zp{yAoLm|lQF2z!TzDE3X4m(XLGfbwOaB?y; zGjmz9fi$QfnrBZK$C3A1K3?V8$_G_3xKGc>Nai69{1B zLsyYlDn+G|DMN$EoH2@KROWf8$doitDMN{nDPu%Pp{S&U zQ09bWmf1d6dcVW(xA))s*!$S`@jl1X`^0_UpS7-aUFUV4=d}X04((%Q=3}NPiuJ&L z6&;FV6rm_OSH{KoFWV=V8Q@n}7 zpt48LCGyudXG7ikd4@sf($&nu4_c`|6!#w`)n@n5vwcsDPepyHYm{k}dBT6r#5C2t z_61L3V!$%3=1Uv3^%Tb|Cud)7p5(lx$arvc)@^lkoqPFgbLWqByC$3M*D5j62>o$YiT%^E?rua@V365ry^9q zKQ(pJI`z9+wQV2w9qp;9o;$H@ z{tmCZhhCLc^ZH$mF(GSTdy7#>MVn!dii*mpP+&-i-?L{sl$4c+#`{g~-n&=&;RDCunQ=y-v1662sb#)C54NLIXV}14eCWpK1ueQw=Jywsnq!X`pCS|@a zy;xXMa%q3lGymk|WIGd~b$%o56Cup{{wOy=vw2#NacXEx44BwY0`pU+}e3vd=(la}F zU)n}#|Na2A-AQ`P_V)IAX{IaE($b#h=GNu0Yj_r><6H4c9S^819Xsth8J{~3L??TCo*wRSeHVPm z>;3)v_tyvs(J-*BQXKgHV&uhq?9JP^udLj0PEb^o5u47T^uS|s_=2OOsvk2qvqQ_U z{oc=gQpp9*0~Reso|I3LZbIp&Pnn9ARhq+A6F=VHm$Y~t5X+XJ)j4jQ_%_3`;==5d zBc+JFk#res^$68BDI0TrS2b_i&&!|tUKm)PZoUTFbolV$MD0h5?@8M*V`D@5heter4|P=Jx&M(Uef4VS z_qW|);`+>vy|sHre!S0_o8w%uB5>=e4}!9?tA70Wk!X;<+Vsc~p&R>|Q)Vkh^ZgK3 zqAiT%$Z{$%+K%<=*W_3k2!*TBQ#sz)F7Ceibe=P3&WQK_`TZ?9<4B#tzJ2>7z2`iZ za&yN^-u^RvPtM&bH{*`P5o3xYm}m20vFV`-0aa||_Hx~aO7t1_-5;72j-IvfoSDd| z>pz@oEJ87^U%x);&uo&*pWjhiwr|&${qXs*MpcGJgw%;eR0clOBgr5vJ0wFbT%6Wt zaBxsG?#*z0y}9=$DqwP?J9*RjubC&V)$@3IdP<&r$9v=Y^#}F>l*+Sn-@+QBb~_(( z=&d~vw$Vr@{@l3_Q9I6m55NvE%3#q zhYwM^xijeL>A4Nn>GycCwiLSBYy0c8I=oo^WZ9}!DGpxayDFH}cX@q3sHVnwYn_J9^tUuDYx~q#AAQ}NkiD+jqAP420TtlpjNY z4vvlaWn^SD&aKmoPRXMa+rFI}{})k9lV)Cgt+n;&yBGxltGbkzw>O!5%(1Fp)nSrv zCxRcoYS&fSHPtXTJ)ls=KYr-YAsiP!q`GK%&mhZ+YeL($am9F!uP`g}2o%$O-1*%5 z$o4az79m0}r>5FG?9a+;%(q|9xojCCPw3X$r#4N}8;?BaTD*9%Dt{wG7cx|N)3fuz zU7uKmgoF&Ao#R*6(6Eo}eP*03&WiUQm0+a$9lhn z&-M591?wgp+9Mub`t|Dy{HIq~@SNRw^E|6pUvuhjY-@Yve_bv9g1x=+?9|vXYwN3c z%*b8#i}zhyrQMXf=kns8wMokkMalTRzrVxsS3^dA;~x?)`@~(4D0e#j;-yMzYq@Ni za#gW_H}2jIu&PgMt4lT6>F|>S=axx8Kp^USLeJ-$cdQ=DU@cTVJY=tigxrXXH1)aR z9XX7!5UBUxWRiQd^UV#4TDEN2=+7FuYuBy?goKop1@qX>&w1iw>DX7x`hIh4K)*tH^(KU#itFLm2bPjQ zsGh5dRShaIGk$h%)6C3FZC!9+;8KOz(OX-jrI%AG4i4Em`O;e#uU)$~0HGAbDWb+m zm45uF<>Kb{q_(Q@PDn@yCAc?qb?Ojr=6$i3U%wh*VU+P`%JCa_?li1THqf-Kza`=9 z)+?h+dnWzBt#!0M6F=^s_eJ1Qg4pFdHup7cAN#idDi|nQM!mi+q`a-}ZO^%$(NPC- z^2+*Z>g#vZd!BCZ>)UWBa;K1mg@rGNQlZO`u)$W5xP*k@lke{_QLz?fLG8%u;*L5x zI7dJec`ch^V@|gP6Jgbvbp)}zO^hYEbpRr^UO~F+1LFc%CRvfDM_q6 zR3H?I=b>uw@_^vrS13w@KWo;6Od|(N5O6_`OZn9mh_RDt+xRK~dY z__k6%mOSTy)yR{rKYs=w`C8>1cGA(|lrYWn*%!JRxq;?H)os6mg59fDt;)2)3avCP zaAHNub{YA>x^d&i*7o*G*wizouf}KdP(E#YK8Y+^yjU2u_uRR2r@uYVsS90tJzqW# z-(@W)6A+#cpi7F$Gd92hm13`1z1|pkPd*xF)Y195*`Nq1pLb&(-ZMk{Qu~sY zadG(`yd%+ubGv2RHWg%qnp_)`kgfh3jk6h(l9G63UACO9O9|xJY}WcNVk?&1CoqtS zZ^yZ-0E0h0Hrj14NPCS`SCe&0GbBso`0?W<6%`E5znk}1TW?gmBN2kIDlIEhwV5q^ zot_HtXx*H*l-gq5uqA4`=JV%QzrJRX;G`6f9u>s9rr3KMb>+mcu9O!2Jt*ron%G#t z%*x7IlWMXf#B=obx0NCXZ~GP(E2Np`-%O~OQNDe@yQaFD9jQjq(2)1I)QVsI{WSnk zdMSq5@#oCUMzEA*Pw+Z=^eBah%|2a2d!jPz?U2jx^q5Wqc{T?2)u(0aZmikY-kfhw zQV{AX6`{U3NzeF=cJI1p^>+Eqk$5)zkY*FRsU5Q#Fc;S zT&_*iw(&89vv$HEej3Hmp3nPi%)h=5d%!w*uUuhn!ZNK1NwD;Ze)1x!t+O*uXMFZg z3kyZkL+R(wx;tfLo}_f0J$CFyzcr0ig!;^Q(|JHvoqoRJS65hd-5DA#Gq9@wx_?O< zS)!=-MDIW(fK2|g?NCa&DV6E*em@Q+l8Pw~H#aw7eB=1J zt1Gr<9(iS;skzx;ER&gyZTYw7j-T^#(;s&$dv%Xu8OpNHeD;lwj#itNwrLCip6zqk zXg6H2b$hC$$M5D{+Ns9db3C`(H0m|KaNl9xQC|mS%~k*KR~>)dk8@`f6ckj|QIuUQ~2H zsU0?+baZr*M@y)1Y5G~WNsd&r7sj^Mu1EeHiJFNP+f#3qFVTJEFo6AXplop)oW+fW zFJ5H5c;yB3TiV!Y++Xzm-MiOeV!C!7*7A!c9XD|)1>nJ0sP<5WIS!=&rTbz+o2xBt zuQ&a97J1y+^ZMK4r8uM%2Xcable~zCDlZ>+0}JGZ{O`rmF_J(lB&nm$U0W5AJQpMD z8k@OE;d{58-C*k;)#*RQ-isI*HdJE0;&MIr?!96m{XD&RPR6|H)R?5lIU$_G%FSh) z{UQjAk0B}YXU!wZr^s{qtYbxjsaMJhIk(Z%@9u490fyh{*c(>9yE#rrj5MCB<=jYJ z&vWYMLoac!d$tPbmbs^=hoVeOnmqb;C?M)Njy!j`SX3l`WyuO!9CqI;SK2>5R3^X# z|003rG#-G8XbAvM`2dW){Pv9>DdjZ^g)Lyl%=ADpBO6;eV0-vUc}8aDO9=_OdlOnZH@xi|B0cBJ|`-R7^SjvX6%*?fa% z`SL4>K3ZyOw1CyHz@jUD{Wa?pdOoUiZp;C2GW*;&zI5~AE9eN& zWn+Em_OIn!L{HyV8NOks$Hdd}4$;K=Dt}S*jMUNN#}~1%1PfnV=kA(+yU*_XTYgy= z4g%&;9;~gcKPDS&HZ(C=BPd9%KNP8mmD6v0bEL>)1$OWX2Q4x$!OxXpVwdmU<@NRT zZT%E~aHrd-=-kXm^(D84VO1|P7UzNQA^7oF?MHj>%DUQJERWJRi>}brcdBOyRJjCz zZi!SvVE+}?l~*!PREbymaTw-ZAl=)sW5*skT}g>NJ+h($vA_S|K|1QHfP%citU;2F zNWEm&*hnRo^r_2`1%<{sS#8OVbIn>a#f0gQ$5AlOwGPBbi+`yO9XIl1ULabs%l=YU zQ^zMKCkd#a>}D-(y|(Pw;U65#f=-Cd+`PrQ&uQR0f2wg#Y5c)ETinO3%P31PkREhC z?_O>{awlF?^u~=Fi6+k^4n@l{q2VhVKUE$5&}()IZ7)A?lOHG_(VXc&E#BNrOjUO_ zS0Jf}FMgYTq+qeLv$JWT%Tm&3q3ul-O$JOSQHSc4ydY<6VwT8=4rhRBg!*l-!Ne6?8)JY<*e? zm=`te1Ok80BU(9Equ%CEzcp_12fT_R`45rv@diE7>2Uq&!0XG4tE`llyhHR z9iy=C?M)G|1BVTbjLv-9clhz+$9K0|-L4zz?nLVwH1B-j0tr6S#l*{S1}-8Iik8J6 z2lDK%gVEO=2foXxsH(m!FJITvzR}v%b@eF}eed~M0~_IpJ9l`%3SC2^-0|kdnx9{@ z*84QShz2>c1;wJ!YnBfX=jF@G3^gf6TjOPrB*m*O57Upi{p0!UzO2I@oqUN@-#tEn z#=I0yM?<087XKK(GSmHAq2Jfl!+Z*Nf6sp?X6bV;(3lG+K?s(zjlUG+JG0M|GvQln#IpwJ%L z9!Q$GUEi&}QlPPKcY1cd{6>@9B{elQ4LMdkPxO*5A=j7G)GP(N^@=oZtLhHTajzXU*)!o1!yO})$Q)0d`i-fG^s(#LIaVont+KVXB}oi1pE~S1 zi=0nM6*nFKv#apzmnUjtBgvkJ1wJpd60#dUx$>bN1q6LW5OC|-BM<6WzQ8F7avk}R zG=o)9ONy4L;zj>zKyjeb@=XU_yS5zk6PU2FT(rnyU+NtvulV>-$1E&_DNr2GxTm=($0{M$Mn~tSsCKMxc~5kX+;WYo9oCK62Zjm(bNAUa9Vu`U z>veH=@3xxRm64T&h7*irR_gi~wp|6qeA~}53JAo=j%lN>T4nj+{b*fZU8-&^k}rxs z>eB`s%ylO}M2ZpAE_%K>FQf9)P9wy_e(gWtkam?%3RQ-7ORm>_tiF_beaGA@N$2tV zzB725Aj1#Qa`%%xc}DwGy{wM<-0ul|-70%hG@k?e! zLh03yq~(3wiqcAUS4cxVGK znfP5W@DWX{?6(_WyAKjN1sRC2rgIY=>`~L=qTnYlXJutIN};q^VKI-&EJoz#G27;7UwPOyx*~K<8HDZReVK!CvW_BzVuyJR zf2QAv7Y|5GQVl1H(@F$T;>sz+sRX)7DDAgC_kII*o+8 zJ~MK~^HY7BQ;c(%si&uFg!A(9I!#)7kNlWf|K_}_ z=jxmxp%+CXYE6C0dU{-Le}0Dm&zF4s$N`+I1gdNh?G%pdtEQ%v2!~fVe|8?T$&cQj zS~B_P`mZAP$?0h;k8^n6RHu`nGcGY%IXPv8t|Jt6zNLt07q&m%tw=G-T8wsAZ=>Z+ zeee(z-d@YHSng{z zL+D7_aF#D(_Pfz}Md!Ijg2v_@|k zQgwB7!g5bVJ)c4z%5xhN_jz~mW6HdSrlvtHI8pt^`t!xJS7N;9R!B-pel6!&xiZ1jhjU;JRog;{8@ z^X)P!LcuWUQ5A_w+a&i@U8AF;TIptTVC}?vi_-<$HPzMiQaUzj>Qo*;yl!V}J>Qhe zYVI*azkH)1S_?w%2-SOb?wEWacYW8Fq0PlJu~J`#u@Fz42h5raU6+~UTG!kSyFO+d z@wsorlislqiRAN{p8JKep#WpNJI<{Fq)KvOee$S>FZ>pkpPFdN{`?$})g9knAt3&W z)31hyE@tZvMJ5hKzSnFlJm=is7%wR%L<(?q0gffDPkG!HDiIwzL|Md6umZ>_gP z1E1$QvJU*a4~V-5kRndkrK~_kj_hFfzsEJ6A69SG1~j?lTr(IA1(*AG?{Xb#o!|U7 zEr!)Qn|1w|`dIAk4c&{LgevJ@57JT*G5S0brU8VC$1eIkdi1E=SJ)9Ou8{Zn_rIsL z0B*V8(kPBIY1vH+# z28fh#WGwREN?I;H z*HRQh8oj}`SCmw>_9XlL#t+iEdc$s;pd+j!s$3b5`2*y2Kki&5vzQ7^)pKd=h=>w` zcqzbqZMFTBl=m+)tR+A-J@$5H^s^@UUUeC#r&lP?MTw%EtXQ!E60B~L?z3}8YEw-N zdUo2Ysl7R5CZ7(-Z-M?3+Myy+o%FfpfS%6@8!Ed7jsUiyCDNTXrhBkcvn3{K&Hq`% z=;QceH%X_1Z}rCM$^0dfiYBxLcQ`+iFnmfO4Jlr@u!X>Bs5hs-NsEiC-MFrmX1WWf z{zui3Spm+;tLz|x?(TL@J8aW=U?f zv_4UL4T%c01QqC;w81rd^cWbRozk1}M!nx&2j=Zk`+Um__XKG%p+3tv8>u^{b-QIp zbhA#rXO1)1sD1>U2#x(U^aiC>RR{Z*2qLDdzOWk%+?xNH*r(1y-_!Z_*5J@kP5vwr z)9|*kX~Z32SwYAi1?^>+bu#3GCwJ~bM7&K;y2n}6nOU&+nN8DjNHV2}2HUn$`iY5& zlFy%)K_@~8Z6_Mi{T=6j^iCaV=+-ey_glSsH6>*p(|ePnL!UVG z$Vjd+fO$K2Pr?KEygMF-rtwTlJ?P|#si}6<)%?*Hdv z0D0A{uPP~_pgvUm{#KCTJ?~ZMHuglzTm9S9W0x(|zBD(VFv0Oxzqjpg+lu^s34QyS zFTc@5oUTvf$1nLq4SZQuwV`G46NRIgJ2Ur&!?n*-i;Igb_`f0<9_Wv< z_%qN_9iy0)VcW5ninyFZ7(>I&a&r8^%QuyVtlR-p#)E2tCYV&^!w7KQTQ`r}(f~g0wpU0gQkqKMr~uCFtLYsaAap;cwl#b^F!S zlnf2;mb(pi+}XdDZ{TBfqx>@_eO24vKl~nd4}qJZd@j;4oHf0w1DMnV- zFTKB~fcziq4dDw?`2?E50+gG}%!Fj_xo`K93sPC8O)Jx_jvZ2P0Q%IrA^R1<=Haec zRo$(9E7rraJ6xjPKc(u`nKjSGX!_+X9Y8n$0B`^F`Ev${1f_#&Z~D5qCv~g`sy}>s zdQ7=&-re0jSwkFO3Uwn>ZFc5#jK0|dn)bB5yMU&%2a2ZI;FnOfvy;O2^24_}Yr1Tk z!nYUh2DbI*kl;a(`k3a#Y(2$Bgqp-eQ7~z;6;q%yfa@zj?QtKJ_PEN#r1)br6MB-% z+{`XyhJ-Orwvzd}E8$q$1UX_V85TLD0Z9#YhK4HH(c*C#i+j>!9w-Id_!4+)5V%4- zw<_WDhcoY;YWv1Ci<___M($}n=EVM04Ir%092ZGvrxzx?+7HeSG#yi z4wx*!=S%77=Zi+2iO(V;&;T0H`#bU@rNGb#s8?V^FMz|y(G_A5x-c_gi>~&V#=!?~ zym=k%?52JW?5rJ?txVUwm!=I)?1PhLBSqblkuJys`$*;(t zQmWBO7_7*#gwfO#b=bOKM%kZ3bf_fpeb0;q}TyaN2-8ybG8iUvZmcIl_ z=OCCK)0|Dft?Bipktf~1W}jIG8xIZQC$F` z6*VU=_d8)zd%LfIJtjbflyoG}hC_J6pU=i^9kKt)i74CM2!dXdpC85zh% z^Z?c`$#=Kw+#fy;E`1Nm1@!=ccCt}ES+nP_ETMgE5pr$KdZOOa`cQ(6xi%Y+K(^HZ zH$NfWFC_@M(?p^4v{3DrAvi~_ks!l2OI$I^t9I&hX?O%le5l#5vNhyxocNam?xNA2$i27hm?<_ zyXT;YsH^rBL9>X0N1zNnjxU(grhRY-feFiNZ^3sCvYZ6@uixwL;=&lH5}RoM<6VSl zMbs_OOe_@n0`OpU8mz#PdG`JT2ehrxP3BXAC?Yp)+--VK96!#;!^4Abv3qwg zpk-p}o#lE05U3GuP#`FuFiY9-o46pA*vErIdHOks+sX%-v)a$qdw}!dAwJZBZ zc7lGSQH1YDP1bWtFX80WPjMz1w%JT+KcdfQde4 zV*?)!VIn=!cDawIK+8Xz4c|+a&Lobv%JJiIjfMhpZgCUpIVb|wt0UNzy^gSaxe>D- zU;2~b`pp1YF%|8?&j!xB=F1m)P9MH||2~u|&`9FxsoB}vhQJ_-woihKwD**4^!oo1 zzn?oKfj-kR*&!iuNd3OF#!`dRrxSHt&04y_G6{nuO&QAY0Hx7xd=k_V)D^v6>!w_d zntAjTs))R}f7O-k;J6pToG6s#MZ+JL2CAeu5E1E2qi9sO`Pfu>Ep~i-W zqv*0=F8M6>8_?+}I!!QRZ*tB2+4u(s+mtDv`}bF(8a&BJ1GS7MHC0#EXV}(ixFb|S zNj$u_klp6$s$HDW$|?{lTU>{a6ItW;@83_evXqsSXdwU*)qP@e@*%XopP%AcC}Q+O zlCD77dwQWwDH3}1CgW@$4qAdZ(d`p46>`A_ja~O+IuGt}==nr{^5n@~bptKM3Lu~Y zVVM$roU!;XL5r53AQwPbZD3$tUS37<@b(G;Z@ujL$I*J?WuQ;}5xgC!`H4pgq~Y{{ zb&&^V#CEI_9@GaqI;fw^bPQ~recvJ)R|l^HZ`+W0f`ynR(MiRksQ>&{umfe*ADN&P z>_$H2hQK|G4rt{raPGoK-2|6@t{HGUSMMbVVF@NC#_Q+1W=DR_F|+_Wt5p&{)(aNITIl~=&Tu=o%e z;N81-P-%P{8Vtk7L9nZ6X>ssvJMDuU8H-Nz`u?>b`BuQ-axpG$Z9`L&<F0QV z=zE#6j0i6rNOpa*?=8b_o}&6sxV&b^42Wp187;d9tziu$GQCt|ZbEy2#2rMUpcFyk zZox8`3dBG?hp~bb*nfVcQg{%Csx}vwK#zjZrV84gxT2tbwBc`pDw4p`i5g|bHOf1B z#;`ClD#B4gVshK*kCc;>6AYev5!iTqNGsTIr^CU8xeW3}|FI)w@A_*;61av?2||5jPSSglxAn)Yf|Zq( zq+_oh@lu748=YG6=6ChWmwRRgU$M8NWbj$moP4}U$aexFX86?M9lDoh()4ZSuCNP4 z_f=>#b=;nhkB?^(VulpGi=CYvE%BsvOfc}7ev*c|`qiHe8IKyLu>FKpT#nv|_RV|* zC+vjr4fO!;Ip^x*4lIKg2yVrmodAh(c1!R>;uXmnazeh4?6^P`=&zce_J-_O`u@EJ zG;xEZBW44Ub<%q^wu-!ol5;;yWEq=ZIQU<79#lKlJSpe=+muETAi~g+e=^E{*KB@% zeqc#SiS_iGes!ijd-fO?muD619w&AncMYj6TR31Kri-=s%yb~?Pc-Q88VOeuhWw^O3cz|Wte=>6jIt`vK-PQtIHg5$ki z{s-J2Z~quQU^X^O^SgI?tFDq+wW^SV_i6<_S6A<<(7wwY5zbvrO-;!*kDwyUPygCF z1j;iiR2xl@9r)dL(6v0=;(Bk!4Z#IV9c1P)@YcJCgJ;6(MX1yXHjY_ur#Wm0qe z5Tg(Zmbw%o&YHbfwVJAd+Min0OYp8-+0k^q_)A_fJZ)$TOJUxD?t|mC77k)9AWGmE zD{BIb2`7cFoE$G~I+t_BCeu6)P`AVYV%%bIKQGH3W+?bR*c5+6Jp^Ji&e`ogPW%rK z6B7J)+5b?~)MTdweM^wpZL!U-U%wUv|KIE`RZ z16U4v$%azkK>U}jKhz2?g?PD;--$c|`SFCiC%xVLJ#V~`gjrz#)Hzf<;yHmr7T-9Q z!O}6Na~CJw4>W+)5p#St4487Dbr+#HHa~Yfn4I$?tMYx{ILc78-Wnnm92UOl9|)^#%WB}vx>TtZtv(LtQdDFxWz5i(x1ImKYWvyRH{+6% zT0L%?YE`_VYg3GtAwVEwrFX|hrd4>JSc@BBVbK0r(1|@b^ZD`F^m)6g;>t z#!^ze(JVbe6*lIlqhO$z`I0%k=@IVk-#_hOY;3~F+%mB;?g#__WUfqbA>W{ur+g~|m zOLTCIv{$*P>C9jIbevjoBHh0^mIpxh4Y@WeG0rh@dCzhjq0^t@m}DG&mR$4{cS3(V z2&>g2(opj=d$zkscr{L}cr-Y;;P7&b6uU^I(Essb_RapU_Xc|V`-%-+_LscGb|D_A^lgE)Frz7V1t@WCvaDaj~D2H1sSZ39~h+)5`zU~!r zE)9i(;PdE_WzAx}*E6%Tgs(F+Hnsros>;u{bg6bO=DD~yBO5A5%m>6zV95+pUYTpc%_q^oE+M;M`+SWvVhN-MPVl^ zs_QYBKq)l*MAjr<015TV=AsE&&_bXYC3-Jlw=&EAT8sCW;b0RLFSS=@_wKF#iS0yQ zqXMT*NQ8H6_?4BFQIJ=I?%x0kE9{%LwzjD5<0Zg%zR1Y%Tyg?r>GxQoHNlJ(1En4% zqkrOC6`0#Lv=L7;GG4w8TozslK+uNX1|kril-T45kOn0{+)noIAPyd`y#taDPF$ir zp!P!nhy`F2sGW-ZyE-Z=6b!mPFr~ENi6|dPCOHR~lo(NcSimzVVYrJTQ%$fuZ^vWg zG#a33K&^+~!-BH#5cG#x~R54)Z5v8&;P)QaU2ri>K4K8s)|f=z%HIurIt7!Gi_S0CxD zgt?P=|AvN!NMl>z)Q^r}(o{hj52LC^;_*c{hMWi;g`#;sITu8E!qcZuwX*Nb19lPT zQZQ+Mq+z|&uSHYAfp4#7bAQ_z;or^qkA8L|*Nc1BKG8p@O}awyu8xXO9_sa}7|+Mq zo@j?+0oufeXbDkefhAB{cdLghs~NGP3f4kzY6k+0eBxcH6G4<9Xtq1a755fmXg?2h zd|2>h91_cA&|zzNc)xJyxv4g(sW7`1EBnPOrVbGUC;ZaDz(Brf@4?oK_`u7J<5;P7 zv`KnTVs77lEcyqmS$G_(>_jvfz9&-lbT`&@G zQDod>xT_&+^(**eh#xb&Q7v9a_1m-aEkQU>$)lUWlcC&|fvkbsrM30JkmNcXD6I>R z$=GJ)4$=#^mj%;M;36)o2J8O zrqO&MW`fZEJVNXdD0=+(S^}X+bBG}gWhmx9K13xCd$cg{dl6dXf`fx^R22XB zBF|vxFJzVjQ)MOS1~0&58=ebZKoN%Kr7KsmVAK%iJ2(>nZRr3?UlFj2H%jf0g9Uss z`|7J`p-GOyFJI9Ur>Ee);Dnw`9IiDFl^NolhBIgd?Kif0-Ye%Ew)cC%9q6277oW zKqN7?zarG}P0mg_hJU_Ay~Uhng)SF~hOWiG9}}bO5SyReTuWxKJSOelMa!)uN(`cC z`yu{_Bk!OcD`U%00Eqj6kY8eHkUqpj^cHHyQtsv6-rg7zGanj2BY-2`4weNwqzDTM zVaCHxuv16pW+n-SN=Zr4tHcvs0aHa9dXmn-Af<{PJaAwSPAN7dIEZ^gI}#&2zR^0< zct>~v4<}m^ncq=>~13w6s(!`S2#}eH(rTqbbVZ1-Kly ziHQ7k`(@+@3p4R2LYd`y%V>vFy&laLC}mOYT!qVMLjwQ6xmVO%%}ghN-q; z^09d7@piU2W=-4M+Aal!U*jr(&ca~DZaB`#XvghSuqZySn@ut)I`;AseUQkqIoJnt z`T<$F95W@v@?kEbQG{T3fSl2cxJ%8bMO@QWGUX3gH7cUVcp-FQcL2iaDao_7qB&jY zAyKPtR_)+es-5xjZ~YcEA>$4}1Yd6FoUTzCnjhZHCO-i-Bf zW5P$!m%6^|gfImcx|XGhhNL;t5>deG>nlhm%supIZ-^F@a1>grUon5vKtQ4(qHO^a zm@r<10705w^fD@_NEz5x^%(ghmI64}m%&S7yRKt)bV7a>{15e{B>_AQ8>w;n_cXTb zK!->MoKQDjK}JGBQc5>_QCnwIo48c%_QpV1(#SMYkP%zfH4!vCN|IYG$pmU8aal;6 zsO+rDI2C_pV;fv0j7&^46i8eh8~%d_4-%3LeBh@B1p>%?{~dBr=dmbEU+WIQii-nA zJ%GN)*N0Y(T_=ePT{;kl_Wdxecs1^jeycA}{9u9>mXhKm3y71m6m2oG5Fyctrv-{l z^35D5S^8^Ioh+osxPvw_QPb8XK9~? zS&c9l!3X4t@vW^|xz_21(zKiy&5s)2PnX3nM-wj*7`z=}X zIPzckJjLt*6KdZp!0o3FJv#AmzK}nOTb^{fyspZDMpsw57GUSsFpy0}G@_yH#3 z${~(6P_yv3m};BBHixENbSwt~09GOvloy$@zyMilW-)5l$=%;9oH{EK#b}fz3M#Jt zd;CNsbVk5tVg|woa8+$jgISL}dPoQ>DbwJ|o@CF$srTi>TETfqxdNj_cA3XsEuMi{ zXAs67Xc>g5LKvP&*{r#dj94L$>6gP8_x?TWZs*^=Fj^C42CIsR1B?UhO-e6ZRUqAI z32X||-YU}dh9#H*y~zE+G89mX_#B}S5Q0<*dYA2WBe0V;i)iJX)sGq%5!gb!3;?c} zg}`(}*l*J}GA5TiI26A*1miL2D2%cy+bGfcth(d)=9-6iInIc%yu6vE9N=cuA99ZZ zUV2*aFU>!WixbTSd(SQ)Z`vO^?DJ#bGgefmSDB}-u5PFIoGdJIp--P~K?t94SH1{# z59LeH*C!z%!6HS_O6St^3}48zB^VM~c%)2;zU$vyXewaJO5AzuukEjkW%AA+JjNP; zI~#s*##DsvHfZcx39|?YdT^H%H3K#Q&H=w$x0XQ{;CAd}V%#Mn@}{G9TU0yFF$W*t z6WvJa<717L7>yBvQv%_tYtl_H&7VK@t^y0sq(a|?+DAzZe1Ddmt!G``GwlErg6Yb% z+!SOeco@&0SDk-OY^1PLeuSe2zlS~2u#{4)Tfq)PaWF<)2+andpi}!+=C)wWfjxIJ ze1Ba#vXb;7s1-C6z!W$pSYhFaUHE9ff2d_MM#7h1bOYvb*lCx;Dy!Co?bll?LKxSM z#=yqw?99Xo>N6b0I&6oZghB+%ta_5hVGM-Tr5HWBxu1-%Kosv;wT6OS{UN5+DIa29 zKtk6O4+Xq=dhFG3HansOuh>gZp&o+*rKO09rnhch{sr_)JeFwq3+OPUQM_j)l>AWH zpRlM(jH3fB=KcKh_-e+pPtCz~lQ9EAHX{8Zfd+%oNlM)yz(afo1UatLxF@n4@-1fYK)S8%#JKh9~OZH5)V(%ECVQ zT=k~l@s`?qmG>fSP>+C(35X}q4F3MIR`L_jq|@wEFha;cPA__+hmfo>CBp#u{UT_~ zUoFJ~7?Z@XQlYTxf3WmZR8&hqv>qK}NU%XAQ-LgBDIaq6D)DrZ*(ox=ad|Q8M-Zr(+h#{1$I&f^ zob&+X@|!nrR)xMSEv3cz+>0e7*HEBgSdKp`Lud-?eOtk{Ekp#?4i@-B_COsXgF8Tq zI_o!Aq@|(^9fKT~XWOxu%&FmAZNuFHXeBYqv<1eBUAI9QlrjHn;WwHU@?tUXDtOCd z=nOvBv#Jp)!Y?r#P9`3JW;)L(7@d;Cl>k9vye;W#G}W|r9HhOTKi7dt4B}gcwTN`M zK~f-J$z(82!3SL(CAbPOQ4om0z$ldB88=m^!!t;g#2$k&(1#=P3t2Y6ftX4#+mLet zUPL@zDY7b8m1t#tUfxk0X5OtQm!Mks1DWpAK8bD#Ql-$YT`Lr31}`@jt=xWg5uDdV zo-;Eu+X)R34dElv9a2)IK(%ROwz&C%GvQE_67pYLe?Kdl4ieYII9_XOXQu>j0vV~H zzOKh|?$AGeBvZH;M;gV9uqQ4R9}EVxk>3-X)O|6bue@fd z5^k6IcUu3&R6u~d{HItppa1&53xAM5^nc5BOjD}-qytu76g?vSO1^@Fr%m?weLlAGQ`^dB=CYN*@ zogi0@_I_bOucFq(x?~ACzK{lFS}a8tN`m-&q9fZy`+@~7eXJjTCx;mc`fl*IY6D?+tI zZuY@eFLxCES9Y1b(Cl+lGQ0Pkr?m0h=L_59LH^!}* zuA??Qc71ypy~O>iGxvFu!T&i{MbSj}jU+rBVvaB>E!ByE(JUrphrleCN%h3aO+ zXF^_5wtsmo@~cyE@CdLWIb8x~$7FSq&N)I<@7L6GX{X&re6~YWdW%w`UZ08vp#?euo8a72(>d#KOn_(o*x9qjwGaQ&E!vXZ9UXMQvM_Z# z0)@=ihx zjNX(Fb||xzR`th(;{N@N_^4ODi-a$D{7p-VdAJ3ojQGIz z>5S~``!HY?8ygF_{}qzo-;WJSf=&tB2m@!}+6v!Gh^U9?lZj{w8bKTFKMHm$k|Va3 zc=~Z8gxQ0kBAjHH8t^@pQSXkPJh?b2?9cs=34#C4RT;xA&Y+7}1uDYIkZ;8bZrg1t zs#Rdwu}#3gwZEIZrYjLrigBO=62?DND5HLpPJJbC@+A|6X-q;_slSC{!ook?s zF2la_fXJv{c1TN?V`e9i%(6&AceOtb$=-l^MK(|0T_Z zFN4$fe|2qN)6Zy!nNm6ntD#9zIl^S$1#I25?T6o1O-9{Y-P0|4gUKNzl1>n~I z1Xn;dZZMpBpdPss{Qx%O)>{s4Za-uz(w`H23X`{4trpWh zB`+e|(xY2Ca>RWV(eTU5_x7|PSL3^*(O@`r>J+(7BbNcrC0yS?bOG2G&@z!r zdVYN?Fo+LDDpM4`LFfv|HR#oHV~+q@O4|QWkI(Q=fhkb{*BEhNnt-Sm=sJd4U(%&^ ztQX&i6h;&;!c`DA2(lDz(ZVec-|-hoqwCX^^;QjeBxo{KJ5PlTdeq-|AZ$?JWTTCL za!#Nf65g^!3xiBSxT8+#TFM}q|H0txPTN?E`|9hK!I-`r62&q!Ov|a)7=8|GoQ2hy z)~8Vcp@xB<#LRT$vzp8k2SfUar&!M$A`@=UgQ{d591y^PSql6jNN?hr4$~k!XbbUg z;R)1ciC>y(&#IPRhGFcpO@!c4!>F^43Ix#Fx>RuQr~i&AqhGR{*5hJ{sqFg;KZDBQ zEe@}~d zDO_V`BzG|Vb$yH3c;iOEJg8@_Ri7>psZkbllLjsE(o+r^8) zg zY@bA@Xy7oz!fqyLzD;l|>i&y>1SmF{nayH*KAWT8eT{fd`1BijWI06J3Ir(*dkY<|*c1a&3~z*jV@h$5J;uV5bs~}>2@HZ&Eij0tHrAr4hH;==(y^1Z zM0L`CQUG0rMiD7X6+lp#kkN5*aRH!ShA@O7Zh_hQ!`V03&r%%dVK~2k|K1Ka9J#Sa zJzCa?;y{j2tDXlarl-gkK`eCJ^C+yLB<}Amud#(tl(?wZphj^?8u4D^ zrX#haLuiG|aQRK5zN>9%oNsB_(8YNXk<(Lv+mVL2stkqM;B0qy_j=5|3*#avLEH!^ ze)HBX&@6FKrv1Syq)>v$^5Kn81n_-ecO>QoGUg9LFc=be28?dxfW}V^tQUdOVN7^K ziUZBm;h-KgoS5@4#Q0huw7Ot02tmkXL%9EpAb4_hksY0{tQd_C8YvENI^3Y~iRFx& zXs9>=I3s`l7-WMifwCK?lQaRZ9+~bf!xq1WT;>P=e5?Y>j6dOUq22`mUx|Keee@Q` z%r`YP747aO-=6`)JpUaE9wZbL%|})-3*@zlh&LooL=N^Gw6RCnJJ8ZEW)KNnxs)iY`(-IksF&R zAAf)U_F|(1j}_9^yr?X=^fvUDfY#r7_zBlKb_0N20m!8Vw>e8rRlE!YS~RuAhf($l zKtquS0JauI$XK|85F^&d_~+!x4TwXFadjstOz4hvr&iCy?MOqB>r+5!i{;*rh=84t z1}E(kZv{lHk1yPv$&@EJP9lAWSAz2j>Fhj?830z|r?n=q3O*V{dLZh>F|Z`Wi+DB} zy(t;F#vOwhka)C--4mKAp<2pZ%3D5@Y$p4xS(EmTR#dWuEhtm-;p*<_{S){$oa0p2ChhV%IcTHfbSS$y^rVP;_abp=1 zN^;Q9MF?rMTB`-f9XrH42xvj>3CIuX2YAON6uaP62!Sevr=ff>P(>Y`7;F>ophK4r zCXV)jmj{{EQ)6PX1M5UTSUf~S0q?B)+xCc$_5b>X)w_dluf4|&7d}Hl9E1g#*!&4Z z*V4MKqjNnHCPf6gBxcZs1HwcH{^wq&8Lks~T~;Q9nVB15gJgsdSF#X7_}%;WE zre(OEE$JI7!w|6A8Z-Q z6;_F&FkZ+I@QEBrWI}RbjVF<&=M*r$K>hXBRJ3<>(c|t4*iuE`UlJsDB{B+8$3Jx9 zB+>wwMyNk-khp_s>`&i;F8jSe=5l$`MjT+G!hSQl#z<5^u*R#B@NlS&gfUiAFb1=& zGn8*Cfser^8iy~_{vEM!xgQXtB87F7? zTXGyg4Z&ah0Lcp-OGcy2@M{{!4mzU>#$hRWs#A6^c!#O$VIbS6!I5f(R}lK(H@=Hu zYa($-T?6^xhRZ>gf=&<*(h3~Dx^ULWp)2?ubV5f7*pdvCfYG&p{yxzjyqS#cZ#2qW zgjT^H`4hp$fHCZNCy<){9JE-+awP4K>E;S#90wDJN?>pZy?%GO!vDeEmq%0GzVGhp zEzRBr^;V*ROhq(^49&GELPDkn5<&=>Rhm@PHYKx?F``UmXhakxk|C6-jG1T7b?&LF~S|m``S*}t;+fMG<<(@13l$)2mZpMm%W`4|eb{s`#j6t;H_AQ_e4&Bb5(48E{ zg2ufj-$-uu^yvpr2V#CvCnIziuYP97i5a&OIDq@#TbzfVDDZZ7okz!;Sf`xgw=f;z z=&FJgaSF8fF92R1YRS)=4zSg)ebCL{Un8Oap|L#r4Ve^5$!+6hVS+K-N)GJC>_V!I z_QEQs(c$$4Y)om5<-S8Yzy2fK)lJ&3fc{w;&qO&G=wzZP9;}N1;P?o@b1L(I3HjZ- z4d^Vz+KlHIH~*;3ht?Bhf|hx8u6EO)1o(T$bgEZ^Cp?cX3dbFC5&Ze4@J*toXh_Dv z(`By8*se`2=s=cHYqGY=lv&Vl?KplsBBB};0|_bdj(!!(Pc(cx^;iSM4|Ns?TVPV0 zMSL|zc513Cyw~vA$fzih@tV*X!UD7?vqn3|)>=hyo=5e8v|YyyDj~w;+lx9{#4}vY ztw?7`w;m7f9INy-+GKfvlBS?8&T3V`G6-WB6U%r}oO=!`6?9G`hMCYz1e9)v?n<@U zkQzx2>Vg@_Qviz{1=EsInS(qUvalpD59ksehO3?%?u}6m#=3=&XlC+?O~^+WrJg%U zY=fts`qVX|np!mxVTnuuPcj+SiF}v%nYwL}+eZOgBfyx6C{XO>scYSr|JTJ~>;5;j zOLcp<61 zH@@dkJuw^^<@JEp<8=43fIbK&{t>pJeBu?9k^EAsYrMaqr_y;er=_NV2vPv^>{gLO zz1{AS__0ysT9=#!cS&I!xR4mFKlbu3Xt{*P$r&-C^|GX?5D8KovRI^|f*JAzP-=qQ zHN)S>#~l}elwumn_&6A-_?f|2;j4g}uZzGEsZ|$Z(jAyaXe26bT*Ai|0EI$i#JDxY zv&nh*EG=7P0uYkCfL%fn!n4yShLLOj0zFud7?m_De2R?cNR9|tgL$Py{a1f~E|wQ8 z&s8WMs6WJyipNpkpGL%iWSPRCW)29>R-wl|u;X9d-8(=Vq2$3oYW!NaH){JJN4v#h zO~za#^&jwGIB5}|mN9@|RPv5|T4UGk`Hjo}jeBPl56Jw@8-Uq0oe@EhZxx`NwT!EP zwWH0biWbo|V+9=WL+ijc%zN$jP`DbD4W?FZ{+|7?l8fOGxfrhzTlrLGIu0nI0qu@@7k198U*kn|JSEJ)(V&ooPltK>@Ag;GEOh|eWd znc-C)t~5V>oYihg^~cx+0w6ZzgXqMHO!3h7{fdeU(7{6Ond^c2zbQY3a&t&HA_(x^S7Ui`6fipvW05lSE@cn(hV z45(ONEk=K5trxG$;3O>7D?m)BnJ`Mb z)VonCm;d&@N2e`*j%Hk%sLxAi5f%Gu_K1sl z72+h~sB9xUjXOheRUTsdZwOx&Ebx};Q^qUy(S`&e53T#CP8X2yQNUW@0DmX#`Z+GL z)ZGt&GndtU2tK&svCs!_8j-J9P|z{0iR=cLy@=RlaQ>Yz)~ul3LuaQOOWgCMP2X0LS}z!Ip_HJIPjZQpxP&a3MTnH z(w4!t$HiwLD5X{o@Hw<=19Som2CzGY4QmSu7tF_2FGx?-h4r7F720f3a|3E)QlJ5u z!SWS=y2zjn**vR0M%_r91^hR{5M`%W%wa&X zJj`HI_J9Z_`u0sK>;Nhf=b%SnqUs228i{$gocpx`UQy?irCOIiJAyo1dTik=)tHU? z53arYgY`H$KvqX_IT|P<2N6n*gi7nnmo0hS%FFc^Km9fm)8E}X=HoH9*si_HNo@RF zwy9UMV^%El48Uvp(!_>lCjNl@iS2~ z0-CSNk%IdrLBr^%jn)FkfL9>)QhVmt2M&^waoF%h+wULJLFs>(({9IIdoqPv4tiWl z?oXXQ>ssBmWgj^(N9^d`V#yG&fMf^Qqw4K5x74Im%)7=m+6}rK$?S3}ULMl z6ykl1It37f`l`bndcu%WlLmOeDPVwnTV|{z(|-c;rp&|fjs8%^8Z{X8+ujXg{ z=du^O%G9Rh^<)j!4<%%A{&W)i%$$KB^(CMhK(*HqFh@W_x3m?lnK>Ygq7)C!5k3$) zV>Q%08%@;hO?F~I?&=kScYlTWl{O1u1U&XYyB4P5@=h4$6`&81JqwSn_<3(d*_7U1 zekqO7`_2pFHqLjJz_(~^5x7e&6+CgXyGynMAb(mn60>>GWjLvBC}B=WC-1C~&dR!> zar*TpXS0)7YT6pi*4g?lL;O^aMxBKMZ!rw!vQRO`%^;5r4pyXq8o3|_oa|Oo7ya)P zIShM%_selBz!wnn+?5hJ)zmhg>QGRtOnSlN7N6BTH$v;kb-@3(sU3!y)D(J}!6!P#>gL2=)hr_ncn4M2BZygO1{jaJp+eA+dFG-3>6l5LfQ%nCY#srq)trxL;je4ESSO^| zS%r$_!L{D_*ZB)4I>BxsUQOx4l8*1~1bg^`iWu@d-@q1=u7t5(T%1Z(C@l4rJ8S&= zohK18`;Hani3|!AA6HQlY7(yoL>$>V*RFFIsyD3g@RcZw=t9?bA5mS9Z%b(FBw>w& zc=i9h3$fR}2}OoY)CuA*X1%4N}z7|_#sxS*im;A6x9^XgXqk)ESt>CLonEgG5i`=VmL2gVFjk@1fVx}rmA_EC6l`AT1k&8fD z$7S;mZdVwI`Qxl1{wOOpoE5>Tg*=8$sztnWckkwEpSPq7WRuXMg378@Yxyo*`%~43 zuU@+PhYf$@vwzJ_gV!Sadmml$$$4y~=r<{TNzY7)fE#qceeAY{O`YhveDnA5u_EUF zrldXQ{ZZv|S*yx!Qx27%Yv3in5l!PAc1(vrYvsCiS#;gmvwOjU%C1?%9)#PU|A*pX zSLoW6f0dM!9BU2Z{AJhQHlwbr2aMyt2ORNUgc38}SfF0Vkv0rTIG+IfN{k6ew&tA% zPwgSYrXp(@`|kVs$5*Tx{z5Pv&{>ZhL>D5yvsV5B78eepIKRF_eQqXW_J30 zfA@A)hI!uoIlnQ)sYVKe)1jyUHpOK;8v$W+ihUl`|6qnw3<0@q0L>H|t$}Gqg$Hpi zoQTEItcTrATFUxl!f^#;7HD&_9!g~lGFX=5)XY9b_tDWw@Xp~+)~#DN2blyRYiLU` zgF$8&xY{pN6;WYlhKKTEFvIk()WrzE+rq(dc$TTKH@gtFGoGWtW#ia6Xyd8p8O2Iq z_>baO^Z;0m@b@n?8XyQi`bdf}^s$^krD*#AV79MeP!VnDExdvmv+IqfB`+^6Ym`Mt zq@G6aRgcb3-45u!hWxN+Y%>Jvxt#K&3^X@CckW!n@Z5>rv&MI+9-n`@P;&}njc3S1 z^=CZJ(&5&UcE0V4i}Ww48^hX2C?vJ}+vmKBJu7r$!oeWn_$T#@ZreLsP3Ij+F`l_V z&3i|M0DJvAKZWYx9o*pa@C&jp^)rP_wiHxUe78U4t~7-cPRt#VXr;kL7z3fQxFadC za1CHV-C?Obxuq_Es-r5ZBY7Y(r3pkvqbvpkPmXOSv}1~(Kp_$lsiBaW7<%t0xPmPt zx)MoS6ouax;AHQ?_+4W8Fc^exBbA~~c+m3QKiJRQ)tdxxW7N_Fc?`9Fr@u1pphwl` zM13zLC&%C}2LYM+woD*%Kz+|P)GsA|!l?R)@V**+jguv7+-yZ=7^=yLEpx6Sd)oDx2&(BnmVhi9NaRe(c_~<8Qiik_=xZ?5O(Bk$r0*;huBu z((_Q^9r}r+;nn0+@ghb-hc$_y#LqmCZfx^ATgbB4BL z=k}>rtWH%%gdf)~&JOl%Ij5g~S)``?V@BZCcm;oloLO8Ct5QFVGiF}B7Bk~%@2M+h z6K$_Fyf2&jR{vI6-xpcK@5I;J$G2WEV*3@&+i1XtlJoWOpor&Qyn3~>^}ai(TG9^+ z+a)$NU=S$BC(s-<^O)_VxWz6Hqyb#-GGnLadCPi~k~X>@2q?KTQMh)u>$hWKk6epY{*s+_{@OM1yBkg2%(J8g z`7QDXq(%$3e|cR0w|VVsx5&t;(`Ns?wsccm?CUS#kC*j#$lTDD9Q_!5O)M&k_eC{E%BUo<>zza{Pkf7;qY8k}+4$v+w0E5C{laX&@Gc3-^Gz(6MQ6n7) ze+OwIj+=c`!k%(Ic4){&<*h+MLF>T}!>!YH6h~6BT3va&x@JVzhAAZsICk{%59(!I z>bjS`J-_EnX}s>7dEdk)G01urH!kyd+P*dAd_V>^I#9T$R-XE!GzE6a_<`VnwB3@9 zb;ETphvLo{*WD=je0Xp?P=9?`o*>^EKB4xWr^RObmkh|h+_Ac@By6tv*WSHeb3^Mt zzKRkS-+b!OV#b-W5r+h?e_zBFwaLzZ6L6oLxgDS%P}mzf11V^73=~lCBsY0c1S~fI zFp*c0N=3^NcO-O(qpY(m7+=yYSA%gtBg`d?04DbPTcThCjQJAEnUCVtp0Tu4k+KDo zMUq`Itc%3-;=kyDx>s^KgqIcW_oH-=EsQI=jNwW(-wSwSbanO^Jsi&n%NT5J9n1W zf1Fb0KJ|ByH~KnY{*Q~q+`TzHsXM2-=+KI0>{AI`o7yQcB+5I+spTUK4``{TFXONc zg6hJ}n^tW42_5q)Y!|VA|9-2ldBEuy+YE0RAiSbz8HU^^6}{XQja%MA^BTVE+I;lp z^i-$?r8JF!a>YhzQL;R+7$-m`5`{&gdeDHc(8&|9#AJp}mID{wN*-i(FQKeMmOfDq zj61UHJcMySON)^Tkp7Gs>nE8ZSR`>J3#_+BXo7(BD>Z5o^|_0aJG>K1Yp^#j&{BcFkmTOl9Eht+# zc5D&@TJPsjY(qFltR|UZ8f(E>?nZR(~Ivb#N=OH6{g+$ZXPk2So$fGM)_If_%So+UW1_SZ)S&tVgc}o^0 z&vfRQ%ZS!_vpq82N`0?RSGH$E!$JValNj9G>H}JpPZyfk_G(nQqA$SgIkUUDe|vrv zTLj#^T&PJvBc3?j{uiLCFwyhHg~G9mBie?Y$Ht6@?vA|O1!3&%*3F$8*00}Yl{{>` z9GHs9XAB3#=1`@vQXv{b1mOaY*eIy~5Gf6B5WXF(5ET%M-T6U$hBk;Z7e`K8n zA!56pN})3toDuDdvS^M?xnas_vol@A3(yN4p7AO?H)} zN|joR0Z)(&A2Bu?Cm1nCy(A&c@DCV(#hRJj&*VzS{%OPpK!A@p0R={@MR^`~M6Ve5 z-;bEg!2_+$%^VDZv5K7kp7(L0p;1&0e6rePZklq2#ckAllFUAB;eNI+6|7Ip*c-Z0Z9X3v3sn&BC&x5t?RC2yeK=)vI{Usljx#GZ1^4Q27#ca!7mT|Bk!X65@iqvHrSP_BNEw(KdT<(kn5O@Sok#9i6Sz zM*#=&ga}*vS!k%S8~JvD#*@7Uuj3--MrsP0T3mm0Yr5zXyn)`^gNJfC7)JrQkT$f6 z%3YzNL1h_(vDmwH-Qze>`a$SNMVIxV zXwA8s4HZs|i#>+9sO#$Fz4NP4@<*w|iWbS(i)j`ST@5OKpLE(fY08z}&-S5v=Z5U* z@w-{48M9-2Iqmz4WP9m;1;HjLFOdgD^mE4ARV{vCWt%FE-VPARzrcPC{al~O4Mju#K;7=bK8z; z+G<(IY~p8B=z45WNoBP{LP827R@T<9V7h1@VQQ`f$rkx&Kpamhn*qjR9>zQ|Lu?8V zD-Tf6CZ4VHR4fG`XI;`YbAjyXm5rTk6)Z!BX>R6){cBE8EuxC}fcOngP8P%#D|x=eFLJHV@P{YG;jczsWStO+Gz(w(ExpBgxvQC0NPE`;_6um_Si`^2 z8iwv1&r*mIY`(sD$>*zJGcJBlL#$7~#pa>^a`01Iwl-%TK8{II2R4!9Gr;jbY6^(J zgr+=($h-BXsc@WnoL_DL0;ky@B6iQZX?ZfOc%UAd@8{GF5xJ4L|l8e z-CHC2qqI!-ge%*o?xMz6yptfRsrjnOfC7ne$AY{jly%UIxhB7J?yu8xl;D2sv0iusMgeteC#n)F-HfTeN#9%I8X?%5a-H8 z@l}2N%-bw4EOnz%!MyH4q`hPVXT%EdO**?;Ms@zLzI{O>FM_y3RJ{(q?{mE7Oh zi1P)Yo+q&&Ika?gTy&eA0a6ezMYVqfqgcyJNny`#(3;@ei0rW`k{MFp(DIH+YeV2&yH16a>h zkzWe;^-*?Qa={3Bg9Q1NL8uI?d+G`m3{7-grONS7f*$Z@{B@3l@jhb%gPJYmK^{*FYfSV z>UGe}S&fbyeZlR9*{DWM)TWFJA5b?yPzvlD74+W*aI zq&>~v!>^9!WI*q(*Z8Ys24MWUWpjB34hUY`8| zR(Y0;?eC}asGkyx7v@QT7p4H599rIP$9}5>6-r7iMF_gYP%*6XBk8wo7(cGML;OEH#kz>jk>uG&-`~-|E8=|} zHibboz6k6ORocS)keH5$2FOd*T^O4n(7wGI$3jASD!Gl}`YWq0))yn1S@tX6Xu!1N8zyW$?DKug~4ysVd9_iRhP#2~z zX<0yV&V`}hoduEmjb)N^;dCL0)H=pPyXl}s$*ZnO?*XvN|7e}Cn2vU|v*By6Kv*@7 z4Tsnp1@%R$3W1mTm6zE1t1!7UrSgiXS!qRYx$W7XALsh{@!NFl)b`>z=~5hfI7{_> zr$`!sM`)ch3JmxU;y*lnz0kMSE7^qA?k9G)#j;QzMVr(@*R&VUb_{h!?7iLKO%CR` zIQuS{s}w3X2iYy!DhTF{W6|B9@*b?Tf1s2_#_s!oaT@K^2r+^%47o4r5~fM@$0!C4 zn4wW7f^bZ0HEpyen}#3_`aQx&fqdkr=1@fo5HadYiJ1p#v1!{L?Dd5AG7mct!_Mt4 zp*EkyZ(=A`qe6y(?X@^jvP|1%8`rH7E2gq0x4& z(jMr+No06N#w54O^Z6iUXwywiVCQb{USEPLmU?O`FlV4O&?^+=FKh;X`Eer2uC`0d z*KXpu{F1dxl6Sp6KXcVg_3FylE%T~vOf8Emg;S5jl|>DgmK}K*mQrb=9iJYnnN}A2 zu-;Pq_4$0O)iWdVL*9aedT?OD-%Rrz`6FWvT_zo-K|Qi>r}oPYRhLhyYkfF#fCEbX z>)2XC7-F@ewCuiz@kzN$xtzUZ`!xX#g+5HJj4cU*kmV#43qPyUI?H)ngKa*)4YsQZ zK+4#BV?)tC7zg1#Ns~i&S9?iq**WfHrzDf&y1A{=e#YL|ilB-9?(2yL-`(x3KN`ww zeZM1qFVC<*AoS$Do09vZQnrfro%$)Z}OG292 z-K$n?l+yG*S+Ch9ndxP2KUO9_bgHC*kEx5JHL{zlMS3@s=I>lUGX0G|5eqAx1{q+K6gU9bTD+1|^>jnC>mM(OjgXd16=(Dv| zryn2Saqj(S;iVDq-O~s+1lcr3&5B1Ox(AQ9U0w@Ct+$K#C&J8;E{aQxTUDMciS-7E`j{+^o zE_&|qjtRmz6}P9=>P1e(TPrcq!?veTaoV~baet#CSBH+Di`7v1vM#M6UZK|LlU(u2 zk&s(chIzbOU97E#c^w=QGP-k3ydEEY@e&)<>KnEmuq%#~&ymi_YV!J-^>va|^SkR3 zuHzT@-j40$eZMTx>*3&V$5x(vk=1(2tl5q~eXE`GreutGa#)S58)y(P((_p-`E{M^ z&?jBrA~o;OH+VX4xTh^}q7T#Mf7QxgS#DP?&Wz3Cn8aX|ipgExAAID|(e3@!*2Qa& zJP}OH>OTUVlpkM!oU_9?24QhS5PYw5!n2O2P$|mL&ym?U=?3(4c+2mv!#aV5WNOE+ zStB)JjBziM8$O0Y{|cbAp8#6Pbmy-&hLRR_I)Mw6wycDtZ+B!80wuIra!!V<6Oy+F z#}{AR+L^gwFOR*!<}XJtw&hHWXP}8kavqOk!zX3=hu!C{%a4BJ^wks7e6#}pc-j-E z1+23Yt$2p^l=s!5)9Nkst9fhcLh+>|@bZTI|;jq@vb*34`&$EeI_hGul(DtZJPHA|VlC zvLL!{R(2g$+g>zKCVL*Zn^CdsKzwO=s&1f@>+`OR?vhrCwn&ZS_qN(j?OlE}1NwX= zlZ&33Mykif@*Wxd$SL}+YC-M$)yiLQCoFRwO1~86I`k~F_vKaL6Rs{bVjFrBIU7Ic zjEinRf33r&#sE(a{oXtM<;s{1lB^pSXl{xr;8Y_J{SyYQ3&+`Q~%a z>ZQ)gt9lcT zl~KBGEC0c$j;E~&cpYmrbZyjkuUN@M_l)4xo?9YTG@~@aj3s6_H=1n#iseR5U(4B; z_)zMt|BH+0`b=A|s1MWEouGou=q?Zu;d$`^C6791Kkgm^)h2Q?t2xB6J9n8y15nX~ zUd>B5@2c9NB0GsQbA-plhAN9uHG-sEbG>wJR>Lya#|T1hOGs9PAGeVGE3gttgDU%*1F9lf^s!>xxvY%F#i zw^ouL>-E^Sqq}kSHzm2Dh6_PGaU)X@`P!$92?P&kkv-ONu`K(_@H4P)ax!cY_*C-r zK&x_RFpwY=8$i$()bVWYN-YbsgBpLL?+K*mYCIxN;SE^ zl4$UA6t-2=yh{`1W_eA4+QhldUb5YXDic3$CI>XDc+Ny_x%vWN^wzEn2IKG~xeE%x z4%q$6n-6KW-mzx1*EVO}heWQd-vY%pQo}#)ZNMmv)^W&dJ~$y-43qbDI_o|#a`j<| z1OpHn2p1nf9|FhS`@V_jk4e~J{n`HrzoM$A7t-A=7TQ>Wwgrgi`n(rRv@gyX-0d<{ zsDH9OrMw7c{0?Ad^}i(1x#m#P#qHtFDXU44lXH@U%yWKaF#~bu&D(uZXp*cvb&uaOqk*MpRjzB-_;QxmJIuYb{du|}Qb@H9Kogr1 zT7WJwE~v@!LAs#+bEI?I8}J&cF!`{UaRpy6yoPCCZM997d&zX~6Mgz4LI;&TQjo5S zRPs9gdLc8?_L!@7q;g#AkP(WznXFBpHSZnv%quW@nDV&*%}>~!T;vXo1td;2`%GMEu&!H1Cn1i8U`l;EHjvi@T4z6`| zuLcU=OV;>_h-Ha754R_?AAnhsYS!u`ozjxvsjsUIuZ3-$Q;#&JW?1Kp%FT7OJe2F@ z{Hy?-?`&iGVi3+8#w;y zfaF8p`lRn*$kUELD}&OxDdM_lfraZx6-1FMYR5Y4tjRqf*D987jrWEM*Z`FeKLJ*Y zrq^9q$i<~T^sEBzGCr-wft*ZBoe6{C#uaW_E+(HL4=S<^If^288lL;Gz4G1rl>=q^ z&2dL3T##kH7ROtZnrNP4_3oQ|zFFFA-3Rqp+ zz4d=K8F?1JvfPkC zAms=pM^I({4DM-)425%g=e}K{_rFB^_%8aHzrzzH4bOwD3WP+u5#_mYoqpDyT1Wvf z!26R(G<}D&pUU5I8XA0!o{=alc^7jE?2D$JV|$9l*=!D8r}e;bo#$?;bBlC2wUTLD zF}AfLX~1^L*|N?Jt~JTB3PFKqKS@?|5BZ9h-_1pPqP;=Pj>gGN--#R3IOM{+v8tfB)vDWtR`l zm{;0s)MYckfmHl9*zuDvL6{=+_Qc3rLv^GXyFP$D9_k65 z2;v*UZy^ZLHVXuwRYlXYV1GRap}53s2gCrH?UwSeKGncFnAz1@M?nTIkv@B>1~v|< z`g$zE!OF_R`p$hi7)L3rGru7RL^SjEa^J3<7#|qQnHb4wORx9jY6o)tZep}}0+v#g zj6ipAr;`ABj^_hz5pEB{;5uhg81+s(kC;xWU(cr?6%5QA>#wq}JI|3^uIJcTq9~?& zd`|61%>adO5**xTwz>4)<26KRfLSxnTA(bGRB%r)`)Bd@oc5$Vo(!Y}kKZiPHn_Rz zbN&(2hrYN#Ld3)i4qgf^Vce<=rFld`(W5XiE5@G>;T{tuS=qKaD5NmaTgYHMQl*SW ze`2_t@{*q*zm#PANNAL3EbESWT z!JsTC;~7D!vk#7datBl?HsYvaZ)Ldv2t(wc0@HagESf&Jx56=Pb)2)#cC+^tX2roJ z5xaY2y1s0vc+#B9)egR=Ady2i$lFs+WG}C+!`v;=77vuh))tTVTx!ee>=-InE7rKN z4)@y0p&xvLb7K2&+u41dKPJAOJ@V{?T!;+6Sk9orNhk3QcJhOH{GT5g7vJm&4o)j< zoY=Zve}Cxh&t`{(9L5(zp-y#TZr%LiJFOahmS5 z$qLQ>{k8J&l#Tp8k30mmgZK)^bJP2avqn9)e7{O-D5LMQb`PZLCsCbH3_BNyO}R3U z;}vszWAIOcnz4t4N5_DXhO4VR^8t%K?#f-JA&u&ev2H*nMvyWIp_UVp@yDN!m^NOR zC7KU0p;~*7&`4jaM*9g)3fsVhu7$)YwtMy~IDugDe%SCle_dL&s*ukuK1{)hKtvvL z1J%~bJdSNiZHPjpu!VQeMc&b{f(%Z%!Me6bujZl??>;?;pOJj}<0S(R0?lK46~A^m zHE&gyyY?t|si^@BTLyrj9%Qrkkt(rg9O8z|UoU6*PUfg>FiHX{reVJ}@1|tU9k7XV z%Cdcj5KsG?rF~6Z<+I*pbdWL)n$6k$Be{}k5V9zQ#vXg?as9f8nfgoyqi?r!wmh^( z#hD0D+2;pD(umXLFVa4J8>T1e_A^Z^SQRYueW{ z?fvE|M4@81DRmgDgW?NjY={s69*v0g^3hlhxTd!^02Io+?VclJE5w~sc)=?go z%WT_&5H2w9mAJarBCe7c^lY`NLRMO)u_p@k6H;BJosk3PwMCEo_ZZf9=!)dJAVE#b z8EY3FSfvEyfiSRQnV$W6Xd)P&T4}-+?w*{STn%qYZ}bxqio~%RGJt6j-4^q^9;|>W z>o(sKS%`7+kSQ&Gs+TsR;kme=$NC>ZtWUo?&Irg zhz&r0K63e~T6;ZVI_Q~9G&?hXl1Xz^xGs06zX zS44T3Ho?=sgNF<<3b9{9X|=AungD9yf?>N>!PuGw*Zy^c%h%VpvEJ)J1Bax|zNDO3 zMfKDz8RHE?LLwqPozOM(T}Ie`+0)b}e`54|P-a)<9bXBbM-iFBEpaG{`k*=mao)DE zl|GMV3JM8zU3TRL;m?nZnL7mW*4ZDy>EZ6aI*8{mzgT?G zZ^YJ{9BuqnR(ok;bOZ-I5*{fYkORID|4QL#3f)Q$v12!6~ zeujnf@f%M1cm|=eAYf1h7NYDg2ap_u<;o#Er)8>@e&!gefrg!3+%>LqkI*t2?>-wf4J6VKFh~ z?#@mvT`NlbS~zpQCU8Im?1M55c^JV$Bu}kRxiy7+e0<0T+2~GVFb;a%cA=QM(?2ta}Y zGkEbrNZiVymc{L#T)9A5Fq{Y2uJLDWi!7(!k9)NM3cU$5 zJasj@QkL$<7=P3C2dY@AMvfiGTmZ^rW%&8|D{QOGJ|WbHf<_Lc_gTu*g~Q$94P-cF zdk?|@_Ox5LZUw{r9_uR<@yc32a*nh|8e;xM@l9oiwbv0E0}rtQb-iBp@w*{zV_xZ7 zX2?GzJE9PmDHG7$@$z&^*hczaxID@V}OBEWY*^gJEKVctHvl zpY36gbryWJs|}vB`Ab;D1RUIZBuB#U30&&_u#=PCn|=3NfyZBMtN58TMC3gPDM`re zF3Dx`uZ-1tAZ@!36K!+k zI{OY?afcgf&R|Mzdq0XGtO&?iFkt=0@GCqDIGIRa=aWxLWkpO@=y-1w*Lin7ExWHa z2wcnA=?0{%J=gU|k6+5Nu=5JDUATBLO=i--W4LHzBfV9OE%G9r@g%4{rU8Z#F2Z$lcc$4Po=T9<(qi*9*rN9?kr{(3akS z2_=Y+nGBcB$HcyafrOD#xC8V~+pq}8vrdxk7-2J!IS{6Rn@ZfW2&edE!BdO>FgE@} z@E&Re7CHUoQ6;g3%>XRtB4E zoqG5v$uWQW^Ffe9OAIX{>msRUgZi7233o6n-|_Phtiohr%DR}%p*%9xnYmSvPUn1B zr+g}YV8jc8qD(wj`Vw)Y+Pr=H2zj8y12|W4mZ!wWRELFYM?Lxk%Ztjh3hes0x2lP zT1cs)XB_o+m>9flS7R>#)c0J?x%!(vNGJfdzeNiGeZy)}Pky_!*rmOT3>$%FT#2Yh zyz4sN6ROCagJ7cxAe>%8B~IqX)rrul@Sd04UCdrtfsk{@J+4oT1>7WyMUI+fKq>So zd90kC0+GZ9|C~|7?2)gX4`2bgoDtJlvhD*mHU1o5;gK<&o=%{}Oz<1Zl8cCnfRhVX zb%c64ZVvyuyPHH{T*u0J2JW2jjOiW5YSnGYMAbw%mNg@K@9;opXk5ZmVZ`e>$w(ZU zf86s@LSu*x=GE!dXI{-Az?$A;TV_uYm$l2_E8Zq7=0fB;cPY@5>MK>F2%;KO#eVSH z6*W}HUKf2DJvR}t8RM{0-)5g*z$Pgq&?0mJcx4mVLjCX7%f0VobTa^N`h9Ul>Paw- zR7ZM!Y`Z)%fKMQHh1<(gXzWE2fmq34FhC<&KxgTP2v5DG6LsqyfV5%O4?TH^Zn^A~ z2O?WT^hre2qb59zN~XJk7THqx-_XowF#C*XF&ZcueTt_6eBg!KHw8pqo=>3F&!pKC zKLO9ZN3K-@cC{_JP_)dGt~yt$;nvdfIXFXU7OdWi#Oz^{ag10K89%)HPiRv`N3&Bf zpi9EmsW5(o`U1i*B{&fV!Vt5*=;K~qw%weYFqsC(1q1Y`;$&c1Ws=LpNUgw-m83hS z_mQol`0Jx?6AX$c;8$TUE1kO({zG&yHJ&}R>+sEk+7x{tPhSV=g`(5hoh-rz?ZN15 zfGPvk?m|c?0{2SdT=aRax^eK}OAr;e0Xa4xm?87)zonx*XWYg&6_C)@v}!9wuy^|T z^J(Dm3DYzA74DwM^Yu&#Q(&FtyGqlPS?ibPt|8qwNm3=?kOe$=JH+`?Jp{w3Cj9X{2be#Lj zwBWS7`RK9cqW`q8HL5Ec;XXyQPTbsN^?VY?+AeNrA&L;i`$OLR0BQ#EbQ`cKIvo|T z{@4zd0m|bS{2T7MnUq!?dv7iv=*QSg%E`{qWf4_P$G-<&K-Nx3QrljNf02*Gsm)mj|W#BT{f|eY4+>N0ZoG!OMDQ#T` zwC)(PrZ{2>@)1mW^AVdzi3J%~;_GNo*QI`0wP4MD&5!8r&uf9){O?8@b~ ztG4`!#i?iC=u428V@HM&)R*ez|Gt?+ih`34Pp05iAk>LPB#H@PhpYx<@$YHIizKA4 zVh%wxIN!zUQS6*p&FIoEzgW}CcbTSY-KOhU6nI%A% z2_^$js?5SF-2#2KEo`lt(WXC7x ze|%gMfqcu%o~XPjr-vfojGZr$orFUKGS1H=8W}=mBY3$0X&VzF9f8_Nym@1_`_DNG zX!P8G&~ak~^tUF!07LJpm_quz3iD^}CeJqhKGN;NJcY3B+L`Ap(F48Z=?#Z)Xnc@* zuu~*Dgc5=eN7wyF%AcSZvsl`;_V4^PgYb)V&Y>`}4uS*|8=4W!v`Y z{C|#MOkK$iU+LF{?*Dc0|4*Z_%UE~uK{c4Q_Zy03yb%ekCaSAuAlpCX^O3m9q>0gH z`3=WkU*+d9r92f1iz8;WFp1sZy+n|kE5Z;y2F7XxbxoYF>7N1Bw|&EHSWi3Ql!4ZAF-Q%Wpl$qn zg#VoKXqmdj&KZ=thd|>(wR~jfC}M&ubP8zdK#D?~mv28bQ5gzVj-6Tl=r(r}{SAm^ zqcqQuC}&`w-3ZFr{&>7E(G?6(bZ$9C@AQddIV@B70{zb8o}#TrxEm!r-zQMshI9GJ zp(AkJ>1W&qy1s|3&$G5@7!8bC#$ zVGG60KRMcrZG!bKBd6g5PIZu!4D3_~5Gj*|F2oDtDZmc1l?A3qp}zkh!TWmOtBJ%$ za=8yqxdb>Z8Kt3-+KKGRaR7zfh9he_d=4%EbYMP?H%F3%pFk{Zt1a`+0_!eF*U)#~ zgs&Bm3kOOYuYgz48HyZ35borqhambxGNPvvV1ofndp(X3)u?j8Kr~^{ityrvE743p z7?0NkX05o42lVGumO7Q%D-C zE4(i0$J>zf-R8q?EAQufl_5Jp4ZB^yjv0XLqf?!cZNt77TItQhIUmU!Nj)6^3ePnl z7d>Cs(?S6z=L`HPC1OmwcK9$9Zg)fyGbM6{%M#g1aa|6%OOcXqS9n8+&y0x*DIh!^ zFV&#;$44S0J|U+e%@OGzV&91f3m^F|e*JpR=psLV|Hbjdf)16E9u0 z0g_=aS2g3d66`ARv{K%diLWDVrqdN|?IgyUJ#Pu316VCwPe*rM%_E9geHcaigmm0T zL)6M4eDTArP3d`(YnhmUEWS{uZszVPijMe}Y_&a|Voky&#G`}=D10H0rkRn2c*s{T z2gJkB*G?Dz+R-LSsrnlmgey^b6GX?Fe|5O#Wv*k9)hBW6r90tz`i^>Dc36{PD~%6s zcYs9KdU4x%V!7tYt z!dtePeZisDNHGqX(9K(8ayB9fd#7vn)z>s3V-0djZ5E-iOcn@K(XKlPt&~}?5xu6x zIrsOO7V9fkCf5=>3el`&13fDp(zGiZdT(w@D+em6jpl$GO@?sE-w@9 z+5#;xq2E2o^hsbrG5!QF-$&ZU;IoPFmnG!`~ z{ne)2kQB4P5H`RoQg(-6!JG#&0q`DeaA}ND$xc67+0!C1(+E9c8Nfc#oJNmfe~=|& zHhQ>wHkNipe)JeU28l#bUJO_B@wZEB;b#MMVJ}Ru*$qtk(9LG-F!F#=h6_B8&8sK3 zN1y*aq4RHq@PEYq{MW(%$%XybX#8J}#?_LiqG6kfF}BF;+EY+i_!wp+5`Iu|IfZSA z%4*&M6)i!1?rE4JlqjpDoPZRj3@n$+wLAy|Z?c7^(JsGdH$#)F45$^hX6ix6D96U$ zA%XVj2LiN`js0|y-c%G}Z8iJ|I~wmpqAdz&UWp{=3m0qD$b2WUDioGaUQ3fVg+S?Z z8yR}~9OXI$`wDTTzByE(eEJ=S*F#bdWXJSceJxH&P#lwmozBR%0&&-3OE`c!pgEk2 z97Lo#kaUz40e5bH3<0}_aVHcn@5RXf(C6@W0(cR(HVouUIOBjs!h8@^7Nac5e@ixo zhAgQSVn-6_hkgg`z_MQUam$*otT`L1F^HXwk|m+^Sa<$sXe4rmhNW%zfth zq}o%XaDo0f_8B`GwQls)XhP~!0EX%o0&w+Z|H6Qb>;NPT(Ffoaf8rL1or?!}Ugk+m zD~X&={)xahqsRajt^d!pp0frYI9pgX7*USg~rw<0=0Udf-sVTRKyxSJ}N!^ z+|)zlUg*=Oc#)GIwo-s<9?%DuB)1@h(B(RfQVAsQT4JP3M0P}in{4@)UY&6{=2;E_$c{Zf{BYzFhg}3>bY&?!yu=v!Rr_$ zy%EmnPx>J!s8IHyGCr6m--IYK=3>jA=Q`T>`KX-&8iCuA6io(H7yCg_Y^AZ)99V5k z3Euf*l4{w*=~@91YedBI9#QCMN;4W$o=s_2tla+RWQ>-_(=yavQrQLKE?H9hjW@fF z3s9)-*SdwkTvh}8RrLGr-_OE0>ieP2R$&l`<74<(_CRoH=-h^vv⁢1dq+fL!BB==U^d>piThh(&1#GcAPC3|qh!>$bupmMmE;*{e(h)Hh zXz4XW60aZO)F%PFDI&5?24qaKKjQtc6(kG2aiPfDjBIUemUG6Rs~_wn3>tTDg*@W0 z@kB+j%i*p*C~1vBru3jFR7tv{67W9YcbwG*pcXG=5@UbH`Frs3XZKOaZQUnYt8ksoK z+KNj5(kzEBgh~HlS5eUJ2P|8fm}6E0(&4zxKGf)i+on{-{`)ln;S`JHw`)k()cw%H z+4NE=-BjPTf1oyFdwEYp#Ty6ZzX;I`KlO;rh0d85Q-9XxxWc;kU@53?HAf6*j|KA>imwKsp6}S%Re<=3 zJ+_FJ(6n8G1jq@f@8Qy-CyMWF`%OM;lHQSWBT|+sF1N!1Tf-zltU8JUDAmru`}XDo zTD+HJyXnVq6Jb2_*`{pieX4B%M|&m#g4ZV&wJz7{>5kA(ehss~)^$R73

vx%LJG zvk@7(h-MZ&3-1gzia4i;Rw>Go41ng-EcKV}C#C?3XS$=)k6uhuA4Vw>iR%QI=*^^m z%YiPRE&T8|PJ2G2MNs1p*Ig1UD8e&UJNJX8`!uk6l(L0Ig#272m1?CYc zB~Iz)V0h`${F6mN0U!g&(d`ZmBc_m z9S#GpkL{Jk;;lgHTmtnCTJ=$#OA!po;YUY?``ZJl;tnlI%lhrwD>spZ8+I;x6cAf6 zX#~Jtts^wT7A060$|zm$)4FM=@DJ(aKw$ zuqzlwSQJKbldLShc~JR4&=zr&;TFQc`cMH)3$ZoC2d{<3k4XhHq?jYzfTAbx(lMV~ z$%H_BkW(s7B0w6dDRZ}keJ-X65f`zv)8`Cu0zT$kBy~{z0p8oaSl0*1Hu_ZwA|`k@ zN3!}B=>m{E5h`WkV1h4(@o1OGUgFEByD*DfArVcKPm_Wbn^K6wR}_L}&Wcc^C-zgj zxpM~TEg*VJ2Ggf*0*O@~DUKS5`z%5Y>}Tm!;aDe6j-j&3}zOOE;b&NS9@RST><_ z97%a2GR^%cxwHbkPh<~@B9*DB-)mVm3ZiVXP+<_oTBtt`qI*DbT};!hrWRZ&4w~|mWz9E=H6D#(}glLyIU*i9&26S>ak-0c4MY(Sl zQm9tC@8*sfm}g8D5o#pAJcSDSVqlS@moWsua(8|P;g3#eVvx}o?H-Yz7%HYd-WA4C z!Ke>cH7QK-Iga^?T ziA#JJB()(b*!Bo2kPaDVI?lww2PaCm_?!V>+IUSMB!8Az<}IM8$q~*ZVn?iU#Bg*9 zr_3!r%n0XyCnKW8uN0g$A?|;VpZy3_VV1xJk9JOsTljW$2;s!r#?K9sI<5 zA|V7^Vs^n7xSq*1Jm7+IjS2j_>C6Tv#%P2}o3D4ciAA~uEE!wTnZPb@gLsv+{yR$l- z#h6wwD-9@Th_}3rgP+H;>rM$TCA=C*ri(e$;XxqY2`+=*4^9>pYj53pUZ zp%7}hvKnAeQN-`eVpH{SchGo@@}qph0wTbiyBaiS;#E5kB=YII)O|y-cjBp9B)C>D zX9zS<4ncYsjYjq@hvCFdkWZn1ff7B*ynTZO1fj7p>H|u2OngN4Q=>Dl=j}aYA`c{< zs-J;NNzz)q{(Bfz?;U)JTm?($Vaj3OZ1km9UrHC(n(}D zLu5p-@3OuWSd|ZkzaQoBjv1sgg@?~k(MQu9s!uJ={fR%iIh;n0JCW-QFz^60X0z1( z#(55!n)D9-n=iZZ$9zi0J_w<*3(Bfr#>vYW%7eR zqjZtjF?1E)kG_bXptO1_O5wAnQc9}YOO)Gm11Q~06%v3J10JT;NRzWg80H7MDGDov z^q{7#XYrNi!Zf?y0h0J{?7ewh&UyR)oqaMxV`;>Y=uBiS`&#zSERi;mP_`1vlq@Zb z8Owy6)`(W4+%}P*?_?`%Ti}0NQJFbIaLP3_0ePpEE07aw72JO#D*Va44lr(7y{`FY$Xp9?MhcfoYNkiH!{__m{(Bz6U<5nv@PpPuYxo%d&0=PAZm_NeSO~I{1tk~o$l78o%{UU z;(r?TJ&0EU!;_t$Pw_Cz-c)gh57Bt@IlsGqFl~-b^H548`F}+>jm}M%JvhsTU@OG1 zLb8H@&_f#sqwbuLSqQ9(P~hV%)%nNSzv6DFpclckYcLpDrM^a$NqD*cFV)vC)8r`} ze-}Biqjcs=yM*9IN^+mz0#RO-D(Rw$Osfgguff4stdIq+_Ig}{4E#xpJ5eoEE;#(h zW0=`KZ#>267PUt8Jq0>Fvio#1)vVif*?2r*7bF7NvhCDHHCH8h8~ik0MP6MP`~qFD zuX^w73Q>!cFU}SE{=wQa3&&Ks{qsLk)0#bsnAkB4nnk?lL%zQGhj!0-!3EN1qb9oU zis4L*$uX85`M?5bluJ@c*zk`t3H2)ChqOTJ;k6xZZQ7iF6(5}vQ`e9rr;*y{Ks z9h79DN#8+&Xpah^xCDfu0^alud-}K`#2h@*&sB1aM zXHjD+VDUFyl@zVNX^;so48WMT1sY4*I1`{m-to2#m3|*0oep7|@|r5-#PKNC))waH z9?oWH@<-{m?8zl{vbB$Ti+Hs|3WF!4aJSr(>cwAz>R%vnL=HGrRv!I4?(w+-+{A-~ z7**6TMxy7@bq<;alDb6PM%dXcuo(AciZJAhHi^a|(K)r6~KVl8bkb&;B>#O?@t zCQYFdNQ9Ji->WO{?1-$Z7`4TXyM4fGo=+N~^bQ01Xi1t|&qe@3mtzi%47&U!Z$j)5 z(t0HB&XJ+&mq?Y4KrTG(`YmJ7$idG%brZpwf03^X5z7BPS#43A7?8u&Yf(Z+lR9KX zij+#n_J}z)nd2@8K>9tgO0Bx~?ezlwjKGk<80ppGd@qEl%{k@1TOvMu=(uUHD2Hx* z0e&n-^gY6+kp08`swz09&f?jkGIMorT=Uj6dj0mn`^aiBAyUSUQOWwyolHQlvU(xr zL3KsMOX9yjLf8ssWWF@Jgwnbo7E&l{>)h|umRY0Wno{?m(pPOL;J6>_a3ty$T(@8q zsX5NQ-&2|=0zWMVNf~NaoV;D(n?*D2^XRv`iZlVC&Kvu8fNXR>SAMp177#W1wt7*E zrTr7TcSYY_QUf(Vtiz|+rzb&*Y#@`Xro^w%Crdr-+sX+pN}Sb7PZGM_>K|2k%b$Fi zrdFNt2qv)IjB3}v+EX5|eIIdslFo_}>7&=!7r+ZLjik{#vBM@5oB}y?I8v2D2;D*! zq#3ABA>+!nRh9adE|(X2NAF<9Tv^fM^94+S(x)dB#K2j{Pn9S=;fLN04SI1Mv5*e# znN|}pd(T#>7lJ;H$TG(MLUXXuXK@>XSmr&i%cI^d6zmsYfs&ag z6Wpt@tncMXxJXwM26zJdsi}*%AC>`er?`bx?IX?l7Y)BGtSGhm{7Qi+M2~3{fA(YY zo8JJzY*D}pI6U8Ztb#KC{KLikvVd9A{GJEXag~V%MTyP%yM1P^CuPlGy-7-O6RB?3?`<4Mt8Sg=kE%6@L}bD<TuSHuR?gzmONHUaPWmG9`1u6w} z)np#Vj^*DE#+yU+-|sG?smoA@?sA!;Baxg&x|v-EZk+8v3sA?{>xoGD<>1n%F~yGLU&mKI-0 zhG`Fq{B3W3@Hd~QMh0*INAzCtrC59+*g5@+k&b=tMN_S(^u<$0-vBO%e6jwV%v7MB zRl{kfHOZRD*S_t;)F%{ImE+rrNXH+mk}6HTU|$GsFFZrz#DpPL-hjh|Wt$usTJh@C zg9VaSh{-!-zX`9$1k%0Yrq0RjqJHV=*&^6$C+T)b!v!?a{fqxHX|94AJr-2KuMNnz z13|JR()e)oVU>;3I4(2oNm1?;lOU3Vch0;ct8U7sKlhmAQL!Osiwn1K{9z==^XN#9 zkFcGeNsqBp7f*qH`<}ZwZM*AHVBDLUq~Kf4DkpGGKzc$HX4rO>zp8Trsp6iOa0q5A zwePvS6E8F#pw1$BxJF74-?#D%T>*M%TZlR>+xG9<%*R|3a*?8c{6kM#2zdR z4=WU1v$JZ4PyAPWpRW8jM*ZI!8TX&;(^&9@Y}zMt$3ynsMPbb8r)H-%i7^Yvl+|}v z9DC-_nfr6(txumN%O-*r_VrJr7Jth;2l|W`1}eju!~PkaOs*D{iocAXNvx@n(mmu5+_1R7WKDUIWc<-bJjAGG4Tv_RXX zfJpiJKU3-VpX{)OU|sT%xH=EI(>+r*Zm*wdg1<&GyK`dJ)x>Usb2TVRc+QlN9k^ZG9 z?7TtEb&FJK^Sbe9PE zEs~Th!ZAHYw&B&n*+^ufqqzVUgnq2TgIgp?+JH*kFw(JqhbP7H{r*t$mVAyD?HyOyvtp1oK^)f`Z9LG1+zs9#9ImL^10xI2AhdRx>cKZp(8?AdsWCT58l*5F&Ebg%?IKw%HZ`u$ceJGYYa;E@iH18g4KL_K>cX?AQ7?_Nc%cB zr}t8N$}E+#C0wMglonLGSG(Rk5vZh)c9@1!rA{t-j|xBE9WNarY{P-L(yVk7ywD z>kZU#XZ0ih^u77=kuX1SVW)vZd0O~pIqXsbMA!L35fnN8K=NU1#}2rV^{8x zv3er39jX%Jm9(c1Eg~=kR9qXc4$r@vaO0BjVv>YpkcZ3#z|S{Znw#XDNckhwhtYc> z^zHd&_={rn_$yNGM;CTkAKo zU;u$K$-89x{c7p~7ml`+8S$0ElBpffr@`)7uHPO?fQ|BZO@k=vAQ?>;S+W}{llddj zLimeSI^LP8#xy@@c;ikk>uP?9#&L^I*vxI~%RzuP+Kd9E=N1(^O5G7(^FHcAR-Y$` zY*8?3=Z;mXj1}M`Iy5nmz{3|l?x({wg$;_8P zGqu&M06&QJoYg$)=2W6W`8GUw#wV@i20(CkN|14oXCm{#JDEyznZS}`k7hjU{2z|% zzTMTyo!7FR*=X63)na9V`?Z9N+-O?$$u(7QRRX-0y8IC0hC)gTWdZdPMFi^6TNU+G zm?`<1o_}xO5{onBIcbi@@H0euBqQ4NoPHs|;uvFWLu4q+CI&sp&>>Ob)WkL|?a$qA z_WgJKFRzv{N8T09o5!OzIeoGOxJV1XN+FRwI!i!?yjVa* z*w7(Mo&uJyQu)M@84hD|8PnR1>1A@m3BWh=BgnQ$J@QQYKqiRAU%t%FW@7WlwL6ZT z>h(|}nY0zi?}Xh~^XL!+g?tnQmeU~sX0<<7e*SpfMkZU|@aDU`zfaulA~k-k zNO@fLAH=lDV^X5KN2seYo+}A*xAxSQ8AFQq{CR2Fk#@0<=b1+>X&C;^Y&t#Dn4(UI zCjoPW`8_fgOzdYPtHtB5cC11I_IbGIA8f?!@bUXT(*CLU^s{$^>`QwovIaI<_u}OW?|sazYUdQX9FA$flG{z+cjh9YfIqaR%!z+YSpFYxrc>P-WZdvM zT;FwZ2njnjPX-#ClkvggU?R%BlaU?Ne@71(g?ayLedO0q>Zn%#H8K*JCMpbl0#*V~ z&bf7AsI+%<51sfxI$OwrG~hp9->sSJA0OlgzpwKj*5Aet9CuxcN-1qH>fT~}e3cfy zK(=kugjx{(kAsuYe#~kI=YRb$4{3Z@4K@mJcrPuV0d*Q*>F)RTFRA5fyjGA$*QFdp z2s0IbH)f_?B4~>*i30CEC?Y+}0% zemAPP>Ct*wUrxk6wrlji`A4y#ddCFVTm!VtnLU%v<*&s@wI4^=4&%eXa#h<$B>?gNOceFc5D!wC0r9PCR5Va$ery z0;{fNm6K=Lmpa{C=H&Ii`Ny%)_5W&f)R>;BP@_!weE*+pT$JuRdGp8jUq>OAX;&lDVlcXVc}&$$sUn+kS_bQn-gpgx`3_RcEBYYlkj-ouC7H8GtH$F#R0>cT zQac;I&8C8~d5g@G5N}IF**FXjGPUk8E+C5%;d)>oH&dZTPnTX@Ghc(6Dyk!rjwz)5 z7WGtae2n6^A-U!;z4{7?NImu#ggqAFv2%fBH>`V0MBt?Y?yI)_(5%JZ^_8$GeiTU& zV%BLhttD?ZYK8j22h`7(I&1vElC9JqXtDTDq+Xsyu_28vCS>aM=hUNt0sk_0S>!R! zdi2h_v+_MHs|@ZI4xLRf4;YH_QQ-MTqx}mfil(*XpYjrOEHdA37&i zH|4vy(8{RxK|u;y6Vw8iA_24_^E!uZ6C(73O6^a%P;JM~R*${uoFo1S5~MrBKsK>lOU>*j?Xb-L{<3Va3GbqJwlW#{OBO~(-@|K;x)t$#>;0v~k9_&Du z8&Hm7o3=?=5&z>wa`B{T)26wL7IJEOM}V!r+hH z&g{!E8>SM8gSj&|8QTlRRcc+>y>sWQ_@1|@8?=YIscY|1>R>W=kuVakU>3y&DPup? zV;7%OeD9aYtW-77_1R}pEt-+64`v=quFPp@D+9VRMt`%ZnV<+UQRPV*o7I!433+o& zzhvLLAr`PL3bBxhO1=K0Z~_3s@hWLQMfB`6_O`5udXNPFqR}~>cN~YENp__4eTzpY z#4BsEO^SMMJLVPpw}}|x)V*!IZhx7@&P(rLE%PZ8uFGSWIo1d~e>Y4PgWJ)z-}b&U zKtQ_ARWff{Q%c$?F;4O4`?ldu0bM%PA0Ti@Ibt?C87wSOKeDo^H(&XD*5`efo~p?IWd z)NK9Rt)WZt$k@ISi-LTaRZeucS1-z3P8kSzM^5z<%&vE7pp-Fw+j3-7&_hUhTDPW1 zTcyLX?>||TR6l;wMLU6hc$(tUL&da|ejj$zKpv40`^30*%-Qe|5?V&~fo(*wd&hCk z`rD@F4KT_qdMHzJaG90l{@RLJUb9|-%=RcgXCCw0CSx+>5%-~iLzo0FkG55{WwwS=g~C2}6 z5yT0Oax%~+MQ#gFssxeC4D4-H(DzV7($Gwu`bXFA%>qTi(vi|lSfA&`qJ~g-sMlL~ z&upT7=ZYZ!E=3wIJgLM(rYV5IMC``b=n!q@=ysLT_r886^#`TfQY1NQP7h;H&s(#y28U!b) z$5nMcJmK4q9Ua|jY{zE;>}h9F(cs)q1_fe3l4(X#)M65~li4SN&LuM+S}M)ds=TPU z8|m<&p-h3F!}5I&t^OP;b9U8kHl&WkGm3A&!aiIE>vT>>gOjFLHHRpceWHp@*^ceo z3uG233+*uPu>g7*vLa(u7bCkLhI^zqA20T(JiGH1Hgu_+dfx3j_f)iwZrfpWJjl3~ z7c?qd|Hdp*`vC!xlqaPQQus$%LW7GLe16P*i0g=tbniW#;;gjMXO`0(~@sEX&;~GH?eL4O)XFLwz7(GOPQxQU@HL*SV zBZi%D0=ALE$)g}JclSq$Hi-J3GcVU{PxFoTY7IogDvZX$^al?fRGc&4+}>#NiG$KL zIB4m+xeQT&df$wPuUe@xvOc;GqR9>VC&D2jR@2 zeg5+0K|HOIQ=z*u@p;MLoyCbaKBBqLvcM$=JMiZht@=$Bg(UMy1pRGT!<5yJAw9HA zPv%s9{aOZAbmc{Bs#eUP_s{s&(YQ36D(C}Jd8aO2#tt1iG-A?`jb7SeifH@TVca}* z5282!_3IeeKv#tB@liT!O;ZM1HRI22$R2#R)2!AtH9+8MTgW;=1ny9D>e$hI@(GvN z;!xf~y?XWfAF!@jAjPC8Dzvw6_-*0V;Z|AY+*oZz{47IYxN`maIDAQa@hTi-s-IGY ziTnsRnnvP=Nyp;C>OBAB?Z%AN{OMr1caZQ!@MmFEDB*B=IcJxjzH`K~yQQV&qqEEJ z#rV&idz5>)_E>j)+K;+y)9WYak8w|SM})vl@s<-N1aS-8)~-|fB%E1vy(T((Jd0|n z@%-7}T2YF$fBqTXty{Mf+4D_OUXs6-A%{eNG?|s&(Zs}xWNDFSvGPc&&uG_+{MpjY zally|bFTM~hsh0qm`h!`4NP^Auv`SjKUpzq)F=S>B-u7`>Ghh*#18TQc)fl&89-xu zZpCNogXT@(noGjIv~=#}@q7deZ^;MS78)9Q!X<4#4ICYN_3E`*t&D5vEtMfZmW2$T zt$js}r!itBp^Php2l!X>)Ls(Vlnm&!-;g066MiHn}C@_L{ejU^|v z;zRKfa`I>j5Ni?gb5k0L^N=gJ?R;wEx;{=$*x1%IhP8Er$V=ZlIO*X-pX`zEE`M9piC~RcH}^WWYu7H(uQCy4VIRwH zu99O{^7!l&Zuql8i}ZT?e5Aa2ic4?L%nNtlzkNFqlbg+v@oh50Cyi#t!!0V%%3HqJ zJr03ftYTtfcJ*I;7;x8-`y1r|U*FaHE^>BnI7tZ=>&m*Q_o}(>&cP#B{n)Ufiq0>7 zwae*B(6;ewWoISNZ2P87mNU0CQ0#<3`QwNJ^UGj!)n=9BVj#CO>=j ztW|%{gX^1j+Rw$r{bFb8Rq2$PkK<2QCR7|kz%8Ub(_%}94ewsu?9K(%k&oDW(h|i- z`i((ogYnK-?CulYtaJUQ<2f!In?r^FHy?8faGue!f*t$LxUM1KrtVvt2)SUyENC0`zGasb7NgG#c4W6Vdf6?b}>_p~aYB-F!yp6g|gSzzQen+&`iG0S)^Zx#B zU%-z?xlLE%4;<9Q&+qcRo?p^rjhr0LE{m6NVBh;^)0+3%Z@7!gK_F9z<0w@NUESBO zng*?Rv7M##xPCu9yFY+`>=4yk|k1m z?4l+6;>C+I$7YNkIdUFgvM zc(7G3duFnJ?-|MZv0TM$4=WC8pqAgHnEAY?v&NJ!9DC5JVP$^9zXu*Pd>{W zle~RhLLtc`V3XEOX!O*+W(^dln0x8?L-KxS{deitGYejMs!k@AAFy8bssEfEE3J*l z2zSoc`g&%>8f~s_-O;hPxgf0ZA-#WijnJ*?AX|nUl?f$za?!J431`l{WPU-A zg3HLSB$m!5W`ZgX(Puay&iY`FE=#`7?Xo1qv&ZAM8{Y1%`A1`ob#gsHtAXB8@RV)! zx4z*GnyVs^n|^9(((k|^b5nSR}hJVj=F`XeKA*%M0Y6WvB>EvgTJU_p9a>9 zbPZBea;Vi~-XERwouZ<*tM{HkORZFS&z^;eZd?g6h{TCmjGbT7Ud9p2wuh8kMo|)- z8m1|=dwWwg=cl0>BEJ|*3fqjlT}+w0v1af=z#1PU6YXDqnSI)w#y0r?ig512&1x8+ z+KL7$(VL#KyD;_f+zy-M3;7el^Ui+zhuWkRbnUyjD|~#|us>$5ZmPSryVE(H2Z*`h zl!BuehI)wUR{Pej{XtGE!zSvltSQi*2Sp6i?7QIMEtrjyUd6Udt94++tS!mR2Ji&e zG?D}s7<-e7T1;D)mtG@AG1_ft4y41F3GEjU!Eba^Z<(j4&sEElKXwMsi9~%N zQ@!Xxo`-f#xJZPM!#8roi2LrQi=GB##W+TCS^Hc*o6(e0QI5-}Pk%bx(5QBuI&G{L zuK3T>u6`Ny|5znhH?!cf+)4O|v5@QA9Gm7y6q^KY72d;_rQ|MGYaXfuHnL6XeYwDA ztyiZMz>Q+m=S^K1A=RSxv6uJPCVMW9@1GQ+D0yAkahtB+jqF+f^?S_LZ*v`{DN}B_ zow-YAW;7TL!-~ky z(v>{Daybp#4so+L;A1+#t3<5${BF~Seg+m>*Lr>V7A6h5fA?kbq5+C7nQNM69dkN1 z^!NImoX)KfA}i3F8-FF1N7x#_rVo?2{h3qeRgh6^ia=&Aqwa4_xoXM3W}Nd1C_>7X%W54j{h` zdH2>2*1-OBYW%8yTO^IN;J@O2T(q6VIf^)%zN>d6!+u+^s=c!Osv9XrT;&WiY_FR# zhuQG|6n&>6WPKAXwuPId+_Z zT+VsjAs?Ba>ck|Cw?nd+O=O^CiO23M`u88`Z#Ayt-#YTsTPc!FoX{0+sM>xXK_J%6CdVPLY~zr8=zZaF(zTBoag z!KD4ZPS@_*~WSiL*$NAqro1sw*Aa6@d;SB@BJttBxA zj($gSaykE~O`5@EC}5MatA;vJK=OE{{Igg7YrR)s`w?+2qdv^qp&i-eFF zAE9P|0}lT6pzolIur}N@-knOSn8AGlvU>Yg{I;!|qXSS?2#aCEhwpLhpnlCo`{)~V zSh>2Pw$WVZBZo7K!Vq_&jvwzq_S9*=06V2e%zXxd zC?o|w^9c*XQaexFKO*V@SxO6p7C9kon5AQ!@u<{IZSBIMUZ2R`Td}BM}O`_=eyTc=?g%PHy7 z9zAmW`|rOstsUR}R&cGQcE<^3Lxv3TsOkcEi2)9eyt<>i9mYuAWrGSDsD7E|q}$ft za*OHz>8o1pYP)E8#AwTBFs2G63hZC;kKmK2&l+lKPF%F;js9>=kJ#+_o=cW&TC%5l za2AfM-Fm4Og|w9$SX$bB{ye18#nScZi{e)eXV$#`Ei;l07Y3FaQuIC6DYg9b?cQYL z$Dw0~ZJ=acatJgqqw)5@!LEK!|2}m@b5Re|r>&FE4Qg4nx5NMXLqFvXUh14|8hp^i z)U4yPb`Kv`RlXfDV#K9v!$E_ls&w7%JcXFB+<7rbZ~w8e&#LQ%*UD`ZuK8boZe03m zmY!1lL3M`!lA}^r1y5jO|6H? zODApKygBw`0&1M16Aa=yo0_5RX|(~g9c`m_Dmv-unWgO1DS<55tjNEyXLo)VbrLrZ zJ2B^`-1%(8yJe=x=)jK+PLIB6px)>K{!Q4Ts9fVAd$NEc@th)^5=@tS6ycE1>%N5r z^AWdSRKW@Tb^9a0tD{0OLTE!skY?Y;x2t*$)6{xaP;i(HYoQW;1)Ln|4(+nbWY&5` zI931n@NY+EBS%IG6HY?bN2=l?x;|qo?$7UYB%_9hvC1!CVG^lET`#ZK78#U?aFFw9 z)UTiYm-5mb3Pio}Z2sRozaMb8*nNbOXCM+p^?B$K= zs6oRYf4nO#o{sectOF+0e)F^Z6xIAb+I}K1e)F$Eabs$?o=u;-@a0A}5eFID+%Kv1 zg|OF;&qfkOg*TJu8F#f-(@csy8)`nX)aL26t&&;taWFdk23yLvUi0tw7&5ii&)s1S zCQ)G7Af!P2O%cLJ{YxNirL4ulTn`u>aDI>Z?Lgz4xH+3-8WOuWO(*x2DquvjUw%0Z zh>v7*G;G+=)4PAbKlp~m6K5weYD@^rN-Q}yx^=RNZCRCGJ65Plu7;bG)OSB!*S8U0 zYO-FB3$^NN=|;h(PgP|^P44Mc`q&Ygb@TPvi#=zVUEB?&E4o4Py<<}z(8e3ZlfQcX z`t^0&YOOm!{H>rWF;EK40KM3`MT-;Z=1aRxJ+lv+coFr6_XD@gh(pvOCa-G^IweIU ziL_zk#-|<(kzD-p_CHFSS{q}ReQ=f9le{b+ozC_HG~r)C<;gx&OLGA$wT#A)-@bnG za1(m^q-Vax=g*(}$C~JQVC)^qQQNNA)@8DlkXYNhOb(f<9rQ*r4+d2-H8UD;x0wXF`axg*#J1N^?8;kRxxwA4*VOe_+$ z3O-|xqi#?~EJ}tX2+s8w`fG)npKM&K(|G2aU?J-*_zD|WtA302;SPv~lz+(&fKfu? z#*FQas$3mQ^}RuNumdc=!?_g+*!IJTh!Ds&@=-}w`LCL;town^&M_*c@5eBeqhCU+ z3myVc^= z2kSNqnn}l=okN1>A(XIVU`sb@*kuqG@qU%d5)u-OO@r3TG8ud3nJe#5t!FS_#w`i> zBHz!a=-i{n1ibNgI07Ly9fI^0JWdYhZ75aCKE9fF?-vfUpiGXpD<(#vEFsHRC2ZfvI&GVT%~F55ony{9?7M7M_K&I?qK?@6!vJx(G2pE zJSwwG<1EH$tGD2i(rHR7Of0WXEqr)pBB1Xc4v=jyoi;I3CvyPR548NXqoH-fL1&SV zuIT8aL;)Pra;hLABiMPeuLvzNHY*&-Aj0ZU!)99M+i@p3|MBOa5zwq6CkB2b#G9*7 zc6Q!BRBK(gWp`%KcZIq;6JE$GBtGBfU<*-)()u zMQL+%!cadyKiM+PjOqntj6A;=qs^6WK|ky09AymM4TslB!~SS!1jI42i-{tJ4Tzlj zG<;-esi}|Y+ow;b_jb)N2dUqqZ{q-j@U*ro-mJ9P+A)j`+%M8v?=h~^C}LW;OPVUn z$U$qSpGLS6tt{`wizE1oj?$mo^(Ur#=w8M!%q<)2JR{91tE!jPQ?8V*?T#o~DgFA8NYv-9c&v zeI1b;?Vh2uav$`UMSHdA80Z#%stnd!gM(Pr2T2FmLm(4@2G}jYT+}KgV`G zEHMNQtmy8MO|k%l)^K7@!$ys~W@novi$k8@A{+A3^EG!Gihk)>3t@=6;NJ!f33YiQf( z(i7tT2E`=g`({m=%%IVEzJjLLcRTTd-z{pZRSjY$K329 zw=MeAt2MsYva-3kuU{_!qq(^c^mx8Hja1d!^l3?nJ1_T&D@H#RKT0~i*@X;0ma6U= zZAva1N?5m@l;)A8wCrwBN+nal{{fa z2{yU&F8^wNak1M6z*r2&wcWZef$ctHUZZyldH6O-^Q3>Z-{*=7*EE%ZxMVVqiC?3s z_ZqFX-WJhjE0!XE5#BvcjqX0N?VgYgr)FLK1&BTgeZ_mOui4D^<>mJXwAw6+J3Udm z5Wr{pIvpO{GXGle-In>aEc5uUSdY(rI2#!m*=PR!edwZ6U;tK5FUak|=nXL5qoU&H z&-b%^l`6cgr$(>KVUAm>C89+To#T>Ip92yRlD^{#1L&h6JY)YF{_G!bosqgE&xEXp6 zUF9ebef0MFiUBoQ_Q5075zu7B)&_%OQ45kv3H8UIc8nk@gs#ae zF76DL5e0`!T!KilndRRyqr^2jAr#()opi(M9<)f&krT_$#qJK=l!eF!gRI1caTjZV z7r9>$=<7N;FLOt9pR|M8ZN$;mYtKvRh5~Q=dQC>y@CIloHnfJzCl=q+*=1fl>P{)f zBJU`@pS9NOwB9W_mBMr&wgJ7)Idn5;9G@|~5>81<#NkDCn>qlqY>!RjJuVx=Wg4gO zpf_c=b`Dt55vz!izzlBpyTsd*b`~*Cb(CIwgN#N*H9fA(| z?+C48`jK|4RuSYdQ5HPCyy9#Bptb&-zc*u{n@0Yop}a<8NEsGJ0dmGUu31sxz@<)< z$|Aa>5`AD)UL+aOxMI}#y**3=4!#D+4Is2WdNRK7iB=e(pN0mqeLQKG?JD>(R(Cj$ zQt31`cW}4Z>sleE{?o`(w^;Pv#N9O+n*A;kRk&YUPpSyIi2}X$Momrdgx93fkBqsxPC|@jI9nlzcN)QnGrP|cBcd(l~&-fm4r&nc#7seyeGAE(u z;S=*b|&jlEJ66 z=r%QI)22-TnBps{l|FROZhXfDR0N?&GGY2={WT2Nb*<+*zt3)bqXYACTJR&c#Q5{2 zyXjm6@&Q~1bcbQsJGRzSsYY)oKQqE|zz9R<19ddR!oq~U>TTRWF$)S%Y-d99iO9=5 z5G#-}g>{*ZZ)fYHdccyB?6}m1imx0Gqp4?(i;0GOtUhJ7QMbtVuUF5hGw2}^*80@^ z$%4EoS8YgpUKBJsmHQ31Xtf!ja1Nvw$N3)THm0V4OJ2G-wMApSw8@w<(8~{U%7`63 zv!<^qD=khV#y01x_^X+jS%7WK)K2T}HB*rGO01wGwlQByoJ3HKRw%_nH%8v%P5^Nj zp}o$9y`pf-B7wi!_#W$z@O7$#)Oj5WCTZwC$PFyN|8*-T+ z;Vw}79%GEciHw4pcNshXDfA@3BK2i)e*Pi0!88i9*xH&faNF1$0M-K_;ZDe3vfLl0 zH88hmudkm;WhCJSxHth{oD8^SA?o~~_2t?^pxHP2!3ONLCTovT|Kf8^0(uJE@L9I3 z6|{wZ+R6FZcUG@ilbTbGo27?pEH(ResG;Z*-l+-Y&nQ(e1KDS8YdmN(cISJErdE8% zID&d0fz-?L3h!yMFP`Cr#eP>+*QjpCjTo_7Q%kEaY*1Xvmopx@A)f*R&J0fTsXj(p zYNebF44KV_Poa`EH^Gq8hd{FKQ_=XBzUrCVd1-7S^8;!lAPDN_dKAoH*a>q{WsWc z%A!Tb8QSs3hvkd$aE;#HNN=62`DPbJUrbj7!&L9SaN&aem&fOPADl@@Xvr~)Lz8y< zLWw+<-L*k+7sJRUqc%S6sik_dxw4*|B{=LRTC`NskJW27OK;sf>W>*8|AT>^jZE_iygs`p&wH6mk7tA= zW<=~YR*hxHsUZx{I;vS5@N>YkmKvM-%(>B8Kh4L-XD8FHQgY7W<0>9^6gF)#F(%WS z^l|r$hqK!X#%X081j|;O5AIvOT+J!MuqXlwM;ggfRY6kf&{r2L|M5zrTIDCfcNBJS*(pM9dGG9@)N6^q9| zf7{eCkEmU{cGO>uI8EhOrpr*J1>@dP$7P^K+ytimsyvJwUcl*E_w}cM(O=*eqTcVP zYhX7Z#W~QJcy_!XSrH6KnDPa-cPGaA3k{da^5b)`_pPj~ybg8=`LH#hpZa6qR2ikh zn%Gu8hDZUMT~_$|UMJtx930g(%&D!NLtmb{Ut816%A4}OxOcTjYKb5^c6Zc|55N)A z%gbj_jc@EaKwE!nfb~3z_EDEe`Bduvu1ShN9kXn06{(`Hcp32m=yo+Rxs&MG z?o@L(W_xa+#84x(YlNIvIry;VaI>j)Zf>qKrl4`4`|DtirjaY2t{YV1(>G-&y?;}b z_O&(57xY>1aIPYlR<+#}(+}c(DN$2q#|4w-cfrlM4ZES^Jbv$>d6b|d^=hS!r@10f zA@>mKxZ~XHO(oh(z`OJUToaF{%*~zbueJZYAg_<5PeOO**D;?-`7@T>dksEpAp>hG zuS_4tIShA9I`znHo#H)B+`$xehJ)%ny-j*@yS}{g9+`QAqIh8*7s9{Fp=@*|V!J?U zFJRktvOm6$tF2Z)dSb8FY3Z)F3j&SBmG$VW29Lq8@509?yPX26Z*Y77MKw++KV>?~8Kc(tWKFSRkAUDSqZ=47)J>@J*=@Lu>=A)5~;L41tw@)rNWxDRNo`8r; zT}(6E@oWGI5Ak5_W*&rpicYgRJ)SXPB(AC-EWX@Sk;%a12i!MsCuE3k&gA?$o0|K^ zTk^?gHp*e9)RnxeJ8m*4?xDGfIDp<;=;?)zS^Kt%=05v{W||7UgmsPOZoQNR%K=5< z6A%0;pgbHL%birlxH=A_%Vn**&z-J6DD<)+d^T=&!4fp%v@d0#oPz~cVIai5ayd2@)r0(X5@cuW>*|B+=+Fp;P_VXF%o@prWPh^si2dv5&n--|9V?g6w_TQ$w}? zrFLUMzdG)>BQ-yiK6z5{B~5kN{Kj7I)uKGa%c^w8==e~H+;y?-a7MS2%miL!( z_wHS2{YhcN>cvTC{*K8v6;TrPo@tmtGo=ZyS`?Ux%d83{~ZKMo!q|t`rU8d zal!@6pHCh#1U~U%vPcgRzxWK$G=5yCPMvYFGcU7ZlLfL-Hu2WnzK&u;j5`S;wnG<{%_XK#F0wQbLKX) zuMaV+Q>5Q8H=&~^?$i@jO=2X8cz~BliVhRh2NgH!SZ+GfbL)iMN}k!+;9fPO1Tnd(X`taCh5SCZg#=@IFDR1hca=7;4#_? zuE8Qj`yqa1UHXSQc8|Srz`<@65f>YnURc&+=Y{nS=C7yueI|c%Md&pL$8JVO>2%Ls zGdHQ%uz3$aLIKe#1)AO5gdyWwwnWx{ZrCnmxsOl3b_*0^;5zz}sZOlghG1qa#m$1z zcc@et=hlY3*y5c$ZF3VA+S@bV?CXdp5szlaO)kyvY#TNA?2=9A9$5pUy_ScPLI%d= zS#vLhRrJe4XK;-#I9S)-T~Y1_3}H%K)Wa{U@a+Xmy5 z)qngE%rRyM;VBy*(`s#7+7^=7J}sS`oI0?wGmtXRH__7Ynp*Gh zh;>b~mhXJI&zt@M=hdAt6WA5kc|>(-X!Fw`IvmQ!qHmW>`{#fRrW&ORpLRA8K8M&7 zVI4jb@L5s;xti6g^!AK;if%1xb;;4$9CdiCoUv8%+dh{ZvGI|Y096k~V{g%VzU-9(e0 zyUP`ogq4$&ZyVa0K8}IKnlB=dq7ae!E)S)PQto~VNnOlJPL0)2CF2M$WFLE+nsaUE zL{9hV-EHx{ulbPp9F)#diA+Uu^z=D<;qF@ZWV;A!C4N;~#{X#l{PRADyCS|lo&MKz zm^rCgFBE;b*7K(M^A(zDEl`a|vg;b_4m!&0i*$T!{$MYhU=!=rt^1OC#}<{xKuPVs z_4@3%fuqX423()L<*y&xwrD>OeQpv*IDWyS0dC!%4u$0Bd(@o;GcWysivd-p>kZ?Q zqM~%4B~AT`%soUY1Lj^$O&u*-7qrenTz~f~XKX)+))_%Ff7Zc&@I#i7m9ET=E2KKP z4YJNFEDWdMD&qfoq)e)%TR1V-H}A^k0X}c`aR-^c&~zlxy8w&mE#1`YBM?WSgykK- zRQJLG9cS9#Bi4Mxb8e92`{k5quCvlqBO%uz0?#)335P25jCkzClvo9l|~WCnKs~rfJjtV7OxGJ%BbA zaZTvQb(OsWQr{hY`3)k2Ax{)$o1X}+MlEsqP?;LCS_J!_RPv(jiCmWR%loWi~gP*k~E=C z&~dJso4(|HYJ>B<*enS9x%bXf&>kk2%?55}osippPvpRnE<|Wa8?3)sF}ToY(^cwM zcZCG=MM+H+lQ~?yRD#^9ty6h={Rb@n7}W`4{zI$K4&z!CKdDpyr?p<8z+)Y9&wC)Z z!vo=5L51&8Jg|={ zCmtxjzP;;iJP!`-O`B=l_=s2hS1qkg_BVGl{H4oW1SjWv{(^P@sS;7!wryKHA^! zjz=7=ZRXjO<yRplYdODLXmw0t2Wt_u7{WhwD{cj9Dp|4wjvmzf#Oi z*!;6Wz3V>9c79@Z5fkxsN}gyf>#F6Z=FU}nw*J*PMrpz(HQ~j2gH@@;(I|FKuVWQP zWzdQi75y}>Te&)wDJOLsG}z5q--YLQ$_pz2FCSU^;|2Are*OLdUOh^>C*aK7MBL2> zi&h#!_qHrfo@SN&yJp}Lg&ij-(JRK3nzEU$?gG3Na%-=%A{bP0`TiW(m8|6@RiJSJV5ES%#(Jv8vbUmJM1jwCQY1DpH#^ZR^ljBX zHvZ$+=@&8LDn?~rXqL3RJmy0!8-i1`^0}dAhaQhNhm1UOVazro?cDTh7Tb)1+mBp- zDbWeC{ln?!e;)&?4gR>z(`@GYu6VoYp5QPM)^Z>E!+jiv>(8dl^!@X#-hA#u@Zh zU4xo*cBfKJ?1VpIvkS%f?6c1X`Bt2ii?rI?qBnlpSU#5@?+MlyurV91aNPvspG|3$ zgKC2P12i#Qr>|w}vD1_F6Ug1YpqzwFDPU)ua7zMhq;2iAh4B=>w?ys^CsZt@Hn~on z5H&;!I2`jm;IsP3AM)hMFZ)`U8{z2*?N>)LMJJB7NBuMurpnE!GTWd(LQleq&nFI~ z*9-b75rKB1HUx8IJJ=Rpa85!530Ya!Ij}!=c8|r2d$PiPkvRNb#!6^~<9I&R zrq()Pc7LKrOTRG8uG%rd^YX4YE6`G!#^TQ~xa6aS^TlPI*(x8}TLt{}N5+ruI;19Q z0C;qWbuvv_$DNUaYd`G$o2Qf2RW~57J>`@*NwdsW8CQ?D7;(P5KZ&QAQQe@2Qk(X2qvmN!mPM_V^nK#j5{81`3^vn&tgMN;tAcAXTKw5jdKab?M*ZbqBP2_gPJmRCr z$29s9z%Pc%3aDWrdq;#fGM`J02M_V;Ozo9*WiqQ9;oWG`;lTK4ia@c6z!)&jB^)|z z(7S#_o6*9{(`dwlH%*eOqwbOMx++DvrxE8MPa%RcCniML-`}dMd=0ICx&$-6$qyr) zJL)zg zT>Ew}i~^>J{>ipa=fML`phTENFLIQ7_vAOLaE2(T5vT@7Hfq)D7CW%*vepviiT!Ti zs|uvdl2Z%sA0JBY*sdS}Vrq21LlXIb2_aqQ04=*O0h%{Dwp@D}$o&!`7RkURszjs} z1=~+a-O}n(*dC$ze zutj=hd6>ELh3cbwM_Q8Eyar7=*9mHk6VVpPDEheb*vF<+_XfLK$B=Ik$Fq_n))<}ym+_$)_~>*=rvqMl+nh)!2s-W z(uNfx9H2qvz*sGXO^VKj*8Q!hV~uloTk9x3{e<8Ma;S98 z9JpG)=?D&u6aZc?Zw-~;8|Org5|3r8>$GHha^q>Y_ix@DAD=_>avO*4LHA|xuw-S%ON#c` ze*RU-tW+3?)(MbdxjJIy>R2oXVpOQ$(B!_##m5N+Hc^_FnLB0xgh={WyIw{|J$N6JwiNGlEx-_X*n$I7=Gx4Ot7!8XXOm8FC6QA!7kXkSA-&sW!PCQ=A1Xw<8h zHDlK%07#mKEqhux^~|E7aK?6I(BA%4D_Gx>u)jl4{zrmjHpNy_JPshjbM7B?wG23@ z!0SkOw_gXu#B0$F zS~>$041L)QvAS#C8z>qulYRokjy1RL*$ad@s1RRd`u47or2nHcFRzzQC@A|6S1PN1 z9V^gQURFYjY${0BHco@yY{lP|5jV$@84GU6`&5Xz!|m2z4K+QF($>mZNI>Q(Hc(bq zEznSAigqfJE0rbDI|H#?*X=yoSQ>`}-9HYe$n&R0o2UO2G?syH57j0JZzoFnxZ)+K zfs=5@LfFGylC^`cTrrRI1;PrC0$aBj?dg(Vs|(JOaUS*SPOD)8zQaI2K3QEJaa>9w z6BZ;Dsi;~9Ib$HKwomBCgu zwqIyuaOZGDf8;Pe$V1dOH(^n|r6L_epJtRt2(+10At^5b_ZG)p(wgQzdp1KMLXNl< z)J&S#B;Lnvj3vzqIab*XFrG5?yx*h0x7E=+KvkDDsFdeD&feZWd1FuN=$b-?2-acI zwhMQE{PD-0gx`&C{T|by4mdB3-Dr%)>!z!dRwt~k;zuk!C#P+dqGCw(*Tq6_qf$_Z za3*p-Xf0tNz zdl=mh*C{w>GJ;{w*%l$B=RjXZ&J8bqNQ}P6Jz-h$=hLT8I?BU<;7x7*tQQi7CU_V= zK?rB9Mjdo`An)RxQyYuzL=jAHVldt2Nm|h)=g;=3vhyJJ^+2`*h1b(IzfTlT-o!X7 zALqDik)|Mi$6aW+a&?oIE$1O0&#!k&L;+c2A>nvPpFZu}w{PEQ-4}rNKJd}PMFYSM zni!=@L{}_i$J-8b1)O=iBYuNjxt;vy;&YT4y;+aAF%D$rV8(#FU8$`~25IcZa=8mm zWsu3v=|6$%Zo+#-b=D)+y4>sImeobS(~Whf8n{1B&gNh>qh#2sKf1AgT9niIAz2@J zyMxO{yrM<1hvfFnE>exqw_bSEYUPnTsDm{MCC*ic2fzRR`?!_0>jfT6ZJ6=Bl##Pa zvK@HBMMHUuR2Qw?lxsZNB-uQdYuyyVJjc|;Qpl?LthjcrYm{I~$g>Vq+s|i_>Qflo z#H$anl~9}40Up(1Xbhpx+nqseSbJEx_bk>o)lq0@>X4$a7CRoHW=u`?F}|_%7}fi3 z^s*@}JX3zjsH<8ZQ_M9>1Dr+)^oHA2xQ>)}(u0brLi@#m@Tj z$CREudfYNMk zf6;&=J0A6Gp<_EgMgb(9;!U-Lt`)6@Hi=m|rxz~7cr+V|%DJ^+Mfwz!xQ$X3-rR z#KpNhw*J+RuOGQ}J^$s&F}y1&{UF_zUn2yM6a5U(86E4(3>LTLW$xpuzT_7a%;v>s zeHcjT7mitp8&7^2^QOHr(mGtV+2xmRQ+3?Fs0_vg9@L54U+co}B!pS8F&mB@TNo1g z1A|a9NLd{`X9eW`v1ZM$Pz>*RSih>@9LPun*qz{$+i97#EnnAo6i}v1ZixMI#-rE8 z=bgZEeJGf{Q^5)A=iQ5GLns-8nU!PC+D5s<@y0GV|Ne>Cr5q}yk`k+(u$PPJp)$1A z8nhSs5_yG86MKpl?QC-sn4a-@=|MVE>ndjO5NSxbTn3$fIopt@;OQ}=qkg_ry@F#w zQ%Wm=bNDL1S;;fOvB$XYU^Isgf;cHeB%x{CBdfdvrF!zWLMv$xfeR7PEha~?hH!K7 zEegKgxTTrcFc&AC!wHvlk34Y09+g=#6iqEH>#XeyqMZd7tvU%+ro00j5O0zE1oM)p zpflJG2fz$)?87j2-DV99lthtxgf%9w%%MkYVDUVSO=Z~i!T}|8#*K?AxqZGt@{tB_ zda&DUpdZCDDt+ZyL-1@K!h?>nB0uVLEuM~a07xN4!3PRza~A&qW+nMt>Q4z2>Wr~O z`tL}4S!`?Zn+kXliwasvujL=-@9DpIl(Zd*!<3Bbis0s*sxIvqM`Cd0#gm6?@VYc^LfN zruaRdS8m-m!6f#yeljPf*!mFio8FL8dVTcfm6|~m@9+Eh&gs33U=jBDLG`!378;U}M z8y^8~%JpyeKGr!PUgwj2G}6{kW{clhXq;@rn9nN_@+CHG?4q^k44GkaSlz)Mf z|Kyg)22I7pq)-CF@1jaFUdI)3c^ntN?6Qrr4?|+;%n?}znZPnb^<$Wyg;MkCl$5|6Xj^C_%?^5#F?OEb{aaAg zxT}xPc>Gi02Un-I#@z`xGCM9?YDh+!nsZvSAX77F@S(@UmBYw=D<_JbATyMAP}lZM z?Dq6$E;v{5fN=$D$-PX8@IZT+w0V|6uHKa#WSLiHq2d`91d^xU9WO5<$*5e)WzG59 z^e0c8xYEz#x1|~?O~yBT{`Bcmw(f(k7kF$k-vdk8mg%)sy>T!)GS^2sbL0Py?#}(M z=lk#D4I8fQnynl%Bzy?XAC(TEaXX87!@KG-HrqYuhCl#P*H ztB5I?J`Szrl0zq~shoA)A0PW(*X?%w2iN7c{X)Gz@6Y@FdOZ)1=i~8s-C-d07$AGE z%>s+N?3d(mErnz`1pMfdMhJit(;%^!E!yE+=}1K@Pw%-8?v6+9;u4V(Gm!C#KPQEo zB;6r$>ZAbcuB~@5hO)?cKRzFo` z{^%sAsIp2Qpd0af{O*WX4J}YFHks9f5|V|}(I~4GT9IwCfGBNYZ$D*kg2Qo3huDjZ z$xA8BgK5EKSyEGZWh5lFMK_`$S;rYFJfr42>wGl@8M=fvW8RI~P?B-Ke*Ur*RQPz! z4uJqMY|rZp`vKjW!^Zz`fKt*Hbr7*|-PxbJu41=gW#jyZLTnVkl6-a++&KNF@dEq- z`=p5s&Jq(RCLF1L$sA94aux!IWGEUg5v$-(4j~Ax2k8Ohb4t30mxf{Wqn8Y`2+6~l zl~m)dlS22juIBj-?(9OwvK3koj!9H8{r608hgEBU=p`e#$Yqf|W3g=bBnnvb_t4fk zW2!B`Fh9IguW|8``(M%D?P4<`ZWzb^t(V5ewUv6pik-_s7m4plk(hCLK3vY=z{~>*RQ%;ckPqt z2KIr@^$nq{U^s%HXC}z0abc(+KZWg;F#fIBNjL{D=#WEs+M7=4lW*#~17EB%fg{8< zEs5e7dT;4XS8vfp&H`MYOHmM^+W<2u= zbqo+YvKsD3m7;M6yii|rRc&ownxKgJSL_16=ZGRJ8lbGE-N#+_1Bcq+#XueUaYFh+Etngz`edkPX)hW!vJqnt5Q)JjzDVSd44o+FmYu=Z z*aVW>epC~WP1MeHBId!in$~20kd}~6r(>15=1_wa3U|I*Ps^WK&n9I;-!3H~if6k=jcFnEN08O+T&;K*9e z!WJouTuBYVg(_L#%xaU-BuH8I^w~36c^(TDBTTNBhSUM|Vs-{HtXA1(4r%vN7b}}? z!Ob(xZe_%FWfW)xY`8>B_}Dap32g!GyC7GHAi3R}EQF8Eq<=qE&rC{MDu9uscFpT2 z9FHu6vl@A0<|7ub^q%EZFI|U!A2a62?7j3{oi6mYb1EiAjKZCfd^nTsDWI+(0&Je@ zR3qvjFpl}Pm7`T)P^zW(&jvTFMi3NId}zRcGuP3U$Lx&5a!I5EAX_@dDTqmAF*fDX z4h6{}soR8V3aAEa`n^fBoLq214Of#P9cY|Dw~E+wdoyg#IvasAK=?~3+zPn$_|9-x zLK&%>Ep&xfUbZ((2Tz+99zKHBQ>Wh=%`BG_PyZSsxHULKsA~IIyK7~t5Fz}uLpD2$ zj<{EL&%rT4?GPadb&(ttc0W<~g$m7;!Wo#efbqcIJB)mU@_zhmHWTWBWL9`2 z9b#kar3ThBA7@@ZgG|hRYwZ$-b+U*rJBfTVz!aqFNjpvk3OGEaUTo{u?sI))#-Y^q z$69LWAnCFUsuAS#!7uFqn1@HAqe>%w-Yk16@^=G!JCBXrV9L@ui7Q`|by}~Q}&NW^6h`@gj_EMUKW#IL3?H5XU!>IpQZv>kpxih@|F6p|p0$oqpTP41Fuk=qQ< z7?b>4qegOuN~r2{&F9700U!KzVA66oUoouqz17gXmWNr`EwMlN(s54hl)x62y3FQw z&w_F#|4ao`I^_3~#!0q6_vql49U#ND44uB%IpxW{rziQ=%6}R)0AOe+g*8)0qfC66 zCfWKQN^d?9j2OkH$xLVyS6{juTyY#+wQ$KY7J!9!aMisHuOV`h_ePDc6^P#a8PWO_-a&FgG{d%Ip z@w))LW1(lF2N?_C<9}nS)mnA7feU> z&*sdhc7bat@5b_c=0~)gYyZFe*JT;~KY#v@*|Wb*U-C7qJn*&eY=;jMOU#6ZQ{%>gq~% z??-g=hY7CCVh2G?<`*S?eD|xCy?q58&}x)b_KM?>@*EdPoFEhNhje(5T};p{W>>q0 zZ(UA7wiEbJ!%C{+i)=+URy82)_7xO_io$*X6 z6)r221^fv=U%6sMXWVp^?l_ZfD`k34)=F<3Gh=6G32M<}!bJv>+o@FFVS-LZK2LSf z3}ge&JMe7)rXz9$KQx4ZyKu(r-0s}w#V{VV*+QW+gXCo|O~;xw8RE)};uFUmT**~_?&H2_Z0FPdDBZOo1vO~EPmHq%<7DYV zmI@%B{#8?hI5G`8jx+8LHo-!men?_;+nMgp)^2R^NMTs7%m^DMxESKf85|3#2vePQ z6Qp?34mKSGahzv`+`Y(8iDOG~^KM>|7Q#^xrn1PHHX(;q_ z9M?O0A6hZRl#k#s2=TInxYT3ninKB)`Lj#C;fO;N{oAT=IPeG>(%sx@GSXHM(ouWX zY4g$qxzefg0RlPSS@Ljh5g`~PEp~HZ-N(ZLo%Y-O(-jW6T!m};;>ljdHhN<*r&@S= z4@azg%;!H!6zpZ1!Xw4RWGuCEfmF2=PG~DUtWor{JE$RL9pVpHiAr?`kuJMmi#P1; z+w&lxuywMAq6-2D{g87cts^ge-El~B;#H1g5@pVA>V4fL;IM$F@4T0K#z57vYKYD?lCuNmk;Ts)sE1MWrRJ zsr^!xP7Oz#cck{!*I$ngs{{A^4a%*1+x|pu^;;27Bv zLMN;l=CLSv7&F*lp`ICwNvWZ~z6_a=Mh>BCs`UW00>0SYq9-Xc&qAE$KKihQ*Dgq8 zF~>{$9EAbzBtbQ|ONPexDNCt{{1#-O@Vmj@XUOPZotwJ0CUsWdv8G5-Tn(F8b(xEt z>h2!9cl8dL|6`@snZ+P2@vOI2#;I`nQ0Y+>HNBmeL}6OBk2mwffj~vbWHzSkC={Hu z!v*uA{Mijo4*hOq>fQSYy^<-fS$cAlt%hh(!InA7YKM0EJvn|RmA0&k((>f{&KOq^ zC7grDaBJ1bi&;e2zr?J<{>G!so$1pQ*nsWDuMm!oW=7(}8|CG@=-P*YqaSz-14*nG zX@B`T+i1Wl2*biOfl`uzHqK^cb$q|vzmdvvF;6!3m^gvKWC6{t1`ev*WFW&#o#3@I zWPfKgIA6$Sj+|A_i<9aZR!{d(lg%LM1cd|y9sj>=XxHH7svge~%~pl>LEPrWxvJ~7mda@u3YT|mMSx-H)ihIa7`(U*H9KD^kx z8wm+6zLnRa7LnXCjrLf(7^V0e%IEX@=1@FIZy@_FcDa?=mRjhGfmM#dPd(rdnX$))}Ue9O# zjjK|+MQ(2Zvp_32^Rmm)TFK?EHjm2s0CLNesD!#T0ppw8H@J3`AQDTtc(`brMVh}V zmlTAGaGTP)MXp@_&4kDt$?R+t2b!L2*b$WJ#g3svfwi=K8Oh%QGH`P$p=<7Ct_=3? znf)!Gv!hX}*;H$-<%Rv)br*=3OIVvb2xo4$8j5B{ybk+(LOPZp9ddrEv5rl3y2rK^ ze?>Cit9_}UUof|`-zRC77tSm1yqGkV`D-e_;w1YDV=??##oDdY$%!hy8DB!G~% z74KK=t(d$eZ2Jl#1h+yVpPJ55AB_KafvM?(_7A@XK*fSY$w@v!*F*VkO$W&2rvWqE z9=G*ca(PUb4m`+E(+lWm0w+j?%v>8l0?b^wkt6KD^z-xrT5-^I8jqN zN;!ucKPp_P86bEx4_yo#XD~`&mCjVn-%Q%5`19zNV1(^jifsQbWLGz*Vu}y?!f>x> z3N+y6%5?IhstC+u4)616)Tq&Lq!-$uVq|596B?_7@=C%hw3F>J*D*6vPr`;PV?bmx z3&k`nSy17^s%<(mLydKCUFl~^W&pepF?ezM5 zN&Ho(i$r6`LZ`moe`dFu)0r{f>-#IF5Au!%VGaE*u@o}9f@3P!LvRKJoh#X z$PQinw4vv@yqknG^Y!Q6wJ(^BiBB7QOny=K?EHoqnr0M_Ia*Y%)urT&^@b?UXzT|e z&r;XF*tBNN!T!ZV{qNTg2prodARsR!Z=%DzXlE~dMEz&~On4XP@*wulPsfjMWYQV( zV0A@ZU3gD#R#YS58h-7O)CsjPULcAP0NWd_jc;ODbmwl>gW~-fnfD8>_7dB$CUu)j zA1Crh*DiM*s)%#hmzyVEz5WY6O1Vcv*M}_dtW3r2=F$06eY52QRu!E9 zBaqF1KB%%K_gvu)ypti+KI;4QMH1Y@uMf<7=N?+WHZG~R8*Z_rKm79E)0m`+HD6I7 zAdF;%VJTLy%Ii51B(Xf4Z^S(6t=qf42!B>r-!3+!5&o31E1qKCVSetpwVG1aptC{D zoL|t%vo>5KWh3ClY*RXw(M&?NTRJC_7|rs`cAO>4eP)zXKINrVjR4Fv)=g;lm%IJ$ zeU~?Y3bN3DqeQRaEx1>PbAsx7ixw^{rBH$@IM<}0*VpCdg=<+|?t)M79wTI&>Nl+V=}Gj%twL)~Bqt|_{GZ zBgZbm$MaXPDR_@jVRf0iDkiU+W6DQ!+vc%*eqA0w=GuG;{7Em5b)o~db9R#_xAU8H zGXX%hRtg=dz%)l>h9Vnxjv(3tTY0=$gbq4yquFNp0sKG{CVMm;odajb70RT*8L>hE zSqfkNcM8}}#`vC-m&<`pZ5pmOQrc-URQ3fMw<4hEd00!Hl%5a>ODF1|gvuQhjs^th=pbVWSxoaSyPO=QV>=B2qNW_X9vcZfybM{k_H|tCzfk8 z0xkdIiKxZ1E+1&;TSwW^mWlN6ztv-6coL}9R8wim@uB8U;YXz5Vbw>k+egiQE=Mcj zs8CZ7Aan!ecE#G24+~_++_}}~dJ+YM*&_NbXW4GlxjMW&e0LU*7{d1)3HfqM+lSP0 zH6-!5tzNN<3AJ?;;>Q_0tJS#KloW+?=-pt&-QW6T@5GEhW{Vn?d9?>2atDnV23Lot z#T9vRu)yR!Po&+Vo`==N)(IF@$mT&i&D#!H>I;P!xe29WWqRJG-lwouIQ7U8@5HF` z#L?qmHMM-Nst|dhy1r=}g@=^IkF| zpR!UntGZw(U2b!mVQ2Q)v#!2%K!2IXY%s^k^_u|uJLD-GpW~%g*7l!w<4aiFmj*s) z1^8R!#bbI0m8aWdny&i*vUC&NSHzP~si&k3x_pW>M7;RL#_fqNzuN1Q+)o8zK5L2Z z>G^&cQRjIFL+NX-FEJ_ZEEwNJq?f_Zi*eGsQVuoiVaU?w;pe79^83TD7p>!0HB|r= zZ9|8-z_62rsWtTWxy_dV^!X6cf{RiCFq)~Qc=n=to4g$i!9Jgblg1bhZ=(?s1@esG zP{AeXRR**AqDlqu3~XmIXJ@cMv~D8-?MBf^tH-2Oi)69t)B@h+1UAAv((PL{+6=(2&}>*gHCK8>W)$b;OEJniI2loC-c74JTdTD51@sH zFPT~FaK@f{F>{$+^mT_Z@QWBYc9l^f5|n`PwnpJv4h8fu@YO(+MM01nFg49#WXxI= zS6>N_U8%z=BA#gI!%Hrxd;UPIm0+$S1OPs;Oyx9|l|NvS7aLL;LO=+Gj$l~6la;`z zvLmDdXR)t2KgE@Cxom{>N$goA44t+0_w%8|&KI&Z_nMCoB0k5A`bSL-w=#P7D~5>i z;X12K*qKmdH0!a9Q5A%QGy{R3=S|s%4l6OTLgkw(xzerqdUy<&C6>lX=xdAPXARgF zAHl&A9P#51e}=zlcQX9b>;qGKFhV%TWTVgQ-`z!eAyGs`yf-&S4GEVQfI}bt76Vsh zF~8+H5=Jrvo{$Z-JG-oY>;4f#`66*(Wa#qWe~5K|QyWUgDS4N064Lw^-^StaIOZcO zhvYd_G+~1rqlWE!L)W!)mq1L0iUMnJ$WZUXxk&q9vXp9Ue>KgOsZPvqwu2)4vY96( zkyK8>rwXzCzFn6>G1PdL*;j*sQYx+k)+H=c>E_c*s!xpxrJGb>QN^Xu+@ZBFWY8V3#gVBdgA%l`{3f7et1 literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_batch_bw.png b/benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_batch_bw.png new file mode 100644 index 0000000000000000000000000000000000000000..2f3bf54ef298315e8bab441f42adccb38cb71d0e GIT binary patch literal 100841 zcmd?RcR1Jm|33U8Bq}N^Wu=6MWF$KyrDY|Fgvt(C*%~4l4Gooy2q~hF%oK`BM2a#q zviBap^Qr6l{Jx*x{qKDo_kAC?ovm9mflpcDu;-`~MRBeqzi2F?s&ptCirTBT zOYhu+o?mWyY_*LH0~UW6W7ny((HCDZd(-fSgFQz`e0NBmnt8Q>S#ZfcE}NA(4C>}$ z;;zNCX7p+2*>|nbwsnqwd$v)@%gyI$4)eZMl^ssYH6o6+%gMcX+w<$#Nfkz#b^rYp zOZ!;2nuc}7e|=S!UWr^8_}^cNTF-T9|LeO}tYBwy)#U#_K2PP%!6j<{{iP@TUF7_K ze_9|Dz1)WXgL~95)wAf%*=er^K_H6{r)`h?Ii#D_3Kak zNIxZf_z9cP!3PG_&05PjHXHc8d?`87UCR<08mi8-oLw-;^IYojyr!NWBYXqzpuEqk ziQqw>foy*)%^rmaRuwiER`RNRHzp>|8NG6#IrLC|*Z%!WHN%Bz)~s10Y4dHVtE($_ z`L8@rPS3F(?K+{GAGBiCmV}EQW|EecPBbY{w6(Ky3U2=OT4?`WzwM{WEI$^U8#G(L zVMB~+z;er)C%5Y4O-)T-xep8Bby?S}xmp#!@6qu*56f>)S3A47I0f&$Z%`V>$KvGV z^sMZeVFpv9@4Ui;2M-*k$9m=a=XKNU>ZRq+4N%lJ%P;(TNje+qXZGFMs;;6!J2mp> z;%k@gb~zL6D(R9Zhm%{H-v&6ixs?X45Nt5NL@y{U%~A8@u&RfLEU$t`+|co%fdS=n z=U!><iyS@RcqET+uPfVa4cTD_~n^jhOOQyHnk}-u034K`4ofC{CeGHZNhi&f=t+?_V<2w zU0vPvLPBQqbF-Z_$w$uD&I|1H7!gh}%BH0poSoGQd}f(WDFxZ={yM!qxjxIj74J+> z?SCM%A|CG+ewXQD3va&FSD~e=x5p*Ebr(D9DOUa4+xTjoa7u@`l5b;o)>x`|3pPMF;Lnt6y2ULGt#d_XSP@0s^nRr=_uJ15=|m z;Uc;WE?w2ThTA`7%*_m8Q69*;FyU%$laf-;^BDc|*E9E(qrkT%9*H^@kZYxd+$rx+)gyicv5l>uPE%=_cLOebN_*-ct*oJ$Fk4HQ~WqF zL%IEjio0u5$vec=P6qJ#yiW_x$jG4hGqGX(lR-B!#bsq=7^$LPzh)KG^`Do7@rhA^ zJDu7S57^@>g^qJ7C@6@Z{k`Y<-Me=?WU0Uu;~ddj!$U(WBV}Dyo<4nA)ZNLcS4~6X zs`jJZOYpt7JeDq7R(wrBwfc6BYwtaZ-?}nk)iG?Z za~qgz+@oq^rGx6-KUOqmUAx9wnPGDzzE-I_Y`@SddxayTz4cklxULy4ot>S5 zbF5w` zuC!E8SeWrhiqS3u11@a+MO+he9Ij48htR=i2!JFEi0UO?>o&&)iIOa8`0!x~!h=eZ zPW-??v+7y}5AyDBYE67Y?ueO~6?x+1yS#Mt6l+uhA#J( z$cGQNhO~GN{a*6(*RK`4yvq+AI`pjj+>z0)np-~O@e-`-i$;!%Feto>Gt2ULq|0RF_=ISI}8Lx@wb?M8vC6yBnMmn^-TVh-H za>=9JA!RlBN^10uyh)wG+)_+SmMqz}eY=XLrlxj)YVjVQKL4%(!;eujGv3V1%x`g0 zXs8XE;mY-ycFoxFxVZI3`ChAQQ%y?WTwLtfTPHO!Iho`g+L-Guad|lp6SYlRdiQwW zFH?K_FvR%>a&F7zWhR)d(xHQx!=oHZQKY5KY>St#At>Szq)lGsos@yYpVXdKFk$sGKp`($ey7W^&m7 z`t|D+znQuDt*VR&aei@e?Wady?8ZCjP^=6L3@slcBt$3Ezn!wO5}}YDc3a9{?m4ZQ zXq=PT*^rm>%4772rlEns-NYlQ4<)-q60RF{{dRdwS9ScR&;dG1+@(uT`-z?ijq=QN zf5DI2lWZ26!k-@>irPw%%VzWO=2y!jWk`}XZ>FHV1=qqLF8$UfbYPfAK^d4FAyyBpa*fNJ<`p!#p4@( zy?v|7eM;#>A(i3sy{?XlD*F8SAg--&{HfEYH7gcV@A`h_3_Azc$(z3`F1~_IWu{b2 zOiVT#JXZ#&Nc`l9=W6ci5~Tuhn3Cf--!*2Nmxf-!2}s;B-&M0IzCtX>>CUZ|2pT_w zOxp+@L*!JXm{pV-;&x4+U_sl9Z`wbjjuG_5#AHFxTI?ai6DRmje}1rpa&U5b3uq9X zkTAC*3Hki)PA8%Ki56z(zMne}O>feUr7H(cLO02C?Af!Y5TPT@v30lC++>I7 zB%;|BWPju|8fpO1Z?D8D%D#3u5~=UoWVQQ9=Zeo!N?{!pQEv+iDI`^vAL>u*u<@Xhz{Tp??A>_)^J^4iR_N&l%bfSt!kXN+SeUAED3B_$;~Iy!zu zMcz~8x7D$A%{@I&rG0%`YV{2bxuwtiWTd!&J|A{h*3=ZgwEFtoymUEL_$$Y)@|9z& zeC?gXKHWc_n_iSG8piRp)?nx6;)(_&;_e3I;r#gVqr-6f0i)N>A||Pm0h@Pph;Q4* zx@*_2b5kQC$Q#=EUhaTV+_yvz>wLH$H9m85;~@u}0C~@`rBhQ=1hL?`PuST-whpq5 zOvfiG_|C1xWxR0Z%93wSk6xr%#K7jV$NhG_XST|jlpMDW>(@6S)G#hv=AV`(4lvNv z)%9d>wQw?$?Zm`H6M)7ZP0b4#8B&0&l>jS9ke1#*)sy)9{F&6Y!0F7H zGu-9J3=PeY2`7fi#jfAG7nI}HAL~BcuIYYz^HEyL^4DuAp#%5g!u8Y4)Hj>tuLb5R zYAe6vIM#FY)$7+BmhMeo-OmUhLRJ}SkN*vonkL?(w9*Il$1OAs{>4{% zRi{7R6`&Z89693H{&6*~N^1IT8=U_1$SwR561tm@W$q8(YHZlcU2u_MnT{OoW?|ty zFYW5}du1yqH9!fh=bbw5(;?HioUvY$Lz<^LKHX0k?S8b)u|Qks*KlkX>c9rgG3zXDM9*VEH`7^=QYTvqoh_uJ#gkH;EjSX*~n*S$;vo^?H6lKO1b#)EstV|u>vd^`Qp z`NWB6wy!80hV=rZ&&uhl^wwp(;1PN9?4g1u6J-XRH2EY!7!i2}OQs&Ld8_((dC+KS zBTvTtJL5BtG{V%7No(3`Yil`8d?w>HBZ_ zT77~2&Z8ODxAaW*?+?~7Jbn7Huy*Vvil1YPVO*w{we^#2xLnjbgmR}mU%Q9mMQ`8w z=jZ2B{7C-ylBf9O&qb#{BArP$C10fGy;evs*5&1GXRV}P54M(aT24My zyxhToGZcG4XXf>`RRqRg8yRt4yYnm^^(EmD-?QCbqKU^=NoVa2<~q;kJG0`?pFc8= zEtg7GoWbQVcz%NBwOc>AF0Dgt<(73Xt@6iH2aqvNo;*oWPoF+zdfIToVSdj0Rd)8S zLx))KjRBF7JP+mFFI-&Ag5<@!h+(PU&6_vtehA_x(6F$uP|Ba~NmZs8%iWELXvQT5 zdfJw0`{Tm%=iAPCdT#pUHPBfp_+Y0K!`RrEgRATA;vjZYOUuB40f)6pK5N5x6)xPr zzlOw3T%N=bIbhB_&+*GCDO>ZWx;Dwk$S9kbDAm4L9~U2QirYZ>nZCVDLvc&n26~Kk z31CqHW08&|!pod{zOQ)v_^|;3+R9a{Ds$ZItDhY^qIo3v%U9_rl|6eH4YSS=Ac{=N zhwGepIGF?Kr1{UEE4b#ZNa{U!6yiY5sJ7 z$MdwbUH7H!&OXSaqwY!BT*ejpxq+Qi6j|am+pe2YKHtF!18#1d%)Zmn_F7lQp^3_$o3c51`gBoM)k;)1IHmNc7yVmH!ye(O zMNhGjh4Pu5+QoPtC+v}K!ola6nY)dQ*5brFBIlBm>N`7n#5B*AK9;}VU)^OPvOmGA zP!uX*uqEL-_x20DZF+Mt&9&F?sc{YmVwFnv*|s=m0lM~2_XAL!fZWjRU3zq9W&7$@ zo zD2-LSzda)_v;V&I0qYZ+8P`bLQUjTeTT;t)*lq)bMPo?@Mn*1$goM0Qkd$N_?XE3F z*_(E@P4nsTJj0})`0;Vemtet)9MTk2xXmL6Id9QuT9j+%AEAe3Tc_08hs1U zL;x0YJqgM>uZ^~rx;1(S;04S_Ix9P?lWuChm80Z+X1EGw$JPK~r?3r+70g^P@ttK4 zSk8lM7O3ZT#_Z$4vuD#^F5QE}>Cjaz5-)PY9*1N(k8Bfi2iNwKfhFVSXIg8&Wjl3H zmP-E44i4K0yD*C@6^bj`f*R-C%!D{F%5#@){k!+?+Zb!;qns*4MU&_enc&S21EaMj=130PBqY8X+=*5F7Afb(x_b4Jef!F3uMT^cSGYfrci)cK z6q_j+fea63iUEYtT17A5&=Kld&iGBd?i~}KD~XB12FH%I;((y^iO_k7%Hh`V-NU#Y z@(K#g9UXU)w;!FIXbnS*S4$~Z*U)H1)e-RUAusAssjaQLchfx^MTPT6^fwla59J!U zQP=L?4Fn@00E~irU5e^Z`|A88+jOw@FIsX&%zl zEc@zIIh}u-sq46bUagr=#UtsUu32LPM6NXCN{W=I)vk|?(oZfyX_aPEyA>fb+H<_` zc~%xVa*jZ3I8=+l1bzoKnIfyT$#OW@aj-?5vRArqX*ee;#?<28hFvjuaeA3wR{tmo z%yOvqHHom|9rW0IRAU=LwOaR zge#?%@A&!JWgizfJj(1{;EHU~9M7lmLp^)DZb{uPuOAeCMfo9@VXZ~c+h zIp68t9W=^%smG1G@A{vUi`+Tq`=rY)A?Y)B*5eJlXCFCre6~*T3b}G+#qQm^Y5ioJ z+Jo!TzkT~=ctGLlfqX%YP{yX-bWU7Xz55$QMD~9F@nZ>PhGeE`74+vPD2>!iTbyFy zqv>%ae;#Hia2zP#aqir?JslhP`IYT{puXPgzSZcJR`g)UyONUK6f0imlXR4s?C%~Q z!ZLvk9y+W6*2*b(=Z>$(QjK%;%jywyX_QG`ni+V@VpbL|5(_Rz$D_3*!oHAzoy`QrTW6%w!wSsZ|Gi zOCRm-F`ettSgW(^7xUFF!;k&$)w{Lx3SNl5Z(EZ;Kj-;JD#x+)a*5a3Y$HHgBg4l6 z-G>lE)vyi^&A+s-euPz^^+TOkeRt!wZCbq@F`#ymM3}Qee*k{q-yTkgVsgqrvF0` z5FM#q+(evk1sgaH+}mLvie%e@1aVi3kVQT=D^@vu9~3dEYrBt7)K`6)RWLQe0Bj;U%w?=4YHf-W6Z;=FJ<_X4zHy zmr~b&?y;K$&IDe)s%mI>FC47;wTOr-;P*i9YsH`0w{PD|o7#gp7pt29+hyE`*89zG z`4t^~cc*E`=g9A`90dWcZl-#-sil3q0>D%v;)-n0+}?iKrTbem%314#`QEPyVRh*^ zoQXQ|Otgy@9sX>wOY5U0syFlMB zwi-;Q4?i=M8tHE|nKEWPPs9NbOg=t7gss8Ni|h^Ta_fHoBw8avX#M&<^I)`PA0Lbi zLd`A`Vr%r$a)r;-pN%M*=}~tUf}*JTb~w}a=%0D05u#sY8nbZzK)MYK55Gk=AOQ^g z9A)2s@#y@gXGlXdR9AJq|hRd0+0i%L5+Xb-LKq`(vQn^0?7vNr5P&0&i1&c zA&V)?zOf|Vd%F5Ddv%_CR&6BSh$^h9VZXKMh<;8Z$yI$nv#q~A(MzHg6BEMkNifru$w$Oc&n^L-)cNDZMzZyt zot8v1;oNGZ44SN|x%qnQvgy$Zk)@x-PlnqPPC;$F^lJWyODDgO-TR40 z8dl}TO*XAZzg=!|BbfrF?G2wJPfLwi8*<$rt=#8INCB);695gsn#RM&>5ChrwS2~Y zSTHg(s{$lDB6HiWS$DN8hlf60N769i@nccD3R#y7tQ!b>np``^PgC~0cV7Dxo-6EF?0W_)H*&@xz+y@mGq67E+>Q0QtHvn-r1Apqwxv}q_*}Hdd zmF(xBgPJ>&j_18DVVJS2&uqSLpZ^Bw#`M%F|1|T`_Sc~o9$UU#+;Grk2269({*0N{ zFeNk8Q$*R2wr_ZvrmOPJ^VGrq3ODv|gVHwNHXqv8_`5^?azH>U{(^r(Lc$2Ip2k-N z(Tyniskt4!&d1rlP3Ix9_q!y;AX<=bqaPUCx7*yjbv(1*&1yLRSBA}uM3(PeI>GzA=VeOSo+E7?3LhD-@@f{=;)}?>O_Il{4U8ZoiF4 z!FW+@KF6>pZ^mGL#a`1Np#Pj3=lXK`CFc$-YH$t8b^fyw$MpiKR`=0T_ECpXEeA{v zcbFIY(I5fWevjG1V;3dP=6meLw7g!{8RH};>YDdrVgC!~j;E!k>m&(o-n?|t;^prk zV^zlQ~%CRLsGO$)dDos+|8?FDeR(p%1HjEHa5(wR;{u+9fUtd;UBWxyEdoW zxZ^e}Elw5ffNS-=*9^u2bR(mq*B?9x2Hes#(PLvMwgJ%u?zeF5u!Xn6i?Z^`&5U)f zbJ}qQ6%+fQS}t0&2-TdGl)2_Xz|TN(Zr)v4WsrL=tz`GyxJY+mEog7TcW!`$qKe|g z0dyzf>O@3DG)$5W)Q;8+viU*@W?s~cR37gP#PbaF>=e! zUYtC=JqI)$YG*p3Ll1-3>^Rf(B}P@oxs%l|t2VLa>H#~8`y!%~d7K=h8Fw3pCz8Hr z*l;o|*cS3@L!)4d&C2G-Fe10HuO?#GaWpOYINF4CNkM&Qn zMQ%r#o#xQ2LKR~B(wzV3+`9Z@WMo8T`mFAo`ua5}?P;k8O1``xL@T7{OG4H#XXsvN zk+S=d9&W-h~hTy{#m%lrNo16-=o5xY+*w}Q_ zChbUF%fk~tUtd>e{qYD`HsmUk0{wasfo_{PJBzLUPX*FV!=tF8Lad_AcV@8E=)hoKU*GbkUYppI zi+5w4OtrMGg)2HVDc=&~RH7sGZ^vuhpSn^m6FTqvcz3#l1>cR%w{c*cj0&Q3;<-Wj@``7Dz>j=fDy6#g@5(1Zu81)_(x z(9_oh1x^#z{M^v*z2~d%xEQ(=j~ay&Rm3n2qQzgb>`f5D zPkrQljiFyC=<2z7CX%HmV_GcZv6YzES znc+os3g%=-5U4bE?`FV;jyK3|_&x5fp2k+6^Wp`;om2A=-l|CBF?m(0x4@y9FO-&}UCrB`sMyp=(|H}71#+^ZkO|QweJK3W1;7iT1 zuYtkA?;z?reVm7B0HBsAk2i1IPOuhi6JZ2e^+(77Ar_3tV4Zhq(Lh0l>8b4zB~}$U zjVAn#)+;$V0|2QPQ3PHBR-C@ySYFNyBua<}fQk)vNA@Q~e!bUQ_q!X#F7kdb-mw*g zCPajv8BWKz=+&6$+a_esZ0*c_@#Ja@a<%Xn;uY2t?KkG)L47UywE+7P0fCPo`t`ZgF*|R!(?~9vXruaccO5JWO@GYSD5j^5=v^O_H!GQmV zuw~aHB9bygK}isK@Xt6jv%RI;0TXY{AXuvi-Q4Sr9DS?%l~X#PER8bggR8iJYlp>* zU+Ynm)FpbG?f6}=L}IdC(KuiooRt;_F#ptpr$ zw6U>i?(R;$u@R;n5LyPOJWUHs6eWX(5h)LaxT~nByhGM6#xJK)x1nF&mcYr`P6^I) zeDGa`-;VNh_esftQj6k)nMPUCzQL$1yB#!C5i9|s1u#0rkKl@5B?@zh6j%z6$A&nV z(y1QJhqB(#s?iG}W*zkvq1y11AS@F}WCk#On zYmRTnS%c#v<%R`B0i9WZRb5SH78Xue;b{FVf4)kqoBf@YTU0u*5E}KgWENbC-a}w4 zL-80NBjq@OYIcFk)dnml<`|%lYadRYn;hDJ>MB0z3aa(WL~X$kmlRlCP=r%{_ znh}`>#>ek`Ysq-|l3d9z_$6PgU|K}?)F^u;++O9>K0B6W{jnPq zGBl?)5nkoVS5gsDHXP2*&O^B@c1w9=*$~y<0IUv@vgA)yJdv?&ZEZ=-jT0{*Bg2J~ z)4|2XA4&;TSW?1Z;x$M=H#=_fkzmO~57i*V@?R>%Ry)p45k~mAjhFe(gHaI{qX%5%P%vB{w%Wu3P&isK#4KwI&5V%aHmf zrlx2q9Q%tK_T6SP2!>NbQ^FAzh&}W32>Ed?x6Wq?-ieHiyj8XXTL>mc!Uv?_=A-F! zfY{^}N!be7MpIBmX}U)um{-BjZYj!$fx$r<%4@Xd2))w@b>Y+wM=$^e`CdE0p=!e{ zg@eDC)QfOZfwR8!>Xi)aGJ6e8=8SRS0O}WUa&qQ+O|INx_|gx$r=+?IJW@pX`2O9% zicZ054BQlpnN`5d&+=iXUlI525kDP~jd4e>0*F5X+Is%{c_B`Q0|FbA?!qr$SO*6O zMcWegM_kYdu-U?7lV5V za_J39)&)WgD)Vw!RquAUejQu=2-`$>En?yXlWdNoJb@1*K0>``hU_rgfz(ieqv&9i z;+DLFxWyMEA|mQ!ho{{70A4zON36WW+Jf_= z|K!-O{Y!$hlj~-XVr6Pm?`M`io7j8k5T^=T-~J^`_^FWer0bp1;Fpcxj?)*5^|Kz% zvaD1=nqh>NO~W>`e+h8Vov{q3j)Tw2rL3ypB_ugqHWVdH(%Ort=E?@GZysV4fFLJ; zt>fpJ&ryPS7YmmR(fu^rhUCr~)^kaH3XgV}8nMy_tCx{R42G?nMf%NukArfdrz%rT zltSFu1X-M>tiC=Dc?DiR)8)78<4ZU!RSn>1f~w0r7LLezq>yc{a4jCLnfblth1pqjFsc zDUWzz+YyHcJATwVh1uLg??Frss|1n$PjhI;UZ)xO85!DNEkSnDOQyxXpA^7 z49LX%cgkx%Z;iH7^hefzpy0^?{&&y5eHY=$fsrN*dG3iZv!Ua%vA_0p>u%I;5=BKt z$bf4yUIbXKY6aM1UA5{W%-qokVAkV;cgjh~BkT!uFK9m+a4~5Y_;_bbGKs_ zU6wsO?ZdYHsmgtg!!a>2gsLH`Ig(g3`8*I6b)#vGrPFPyu`*FGbamFIM%Bqnoc>6P zSCR`67i}`W<-R6(Di2<4l9U(dkeY^@?C^vy9&-D2Y6~AXHxuloGCOy^!AZwYuTn7j zrJ%-m9_zx7><;ajEOSW6GQwJMOIe?szUP%Ve}R0&EM7;$lpU0FLJ+1m3q0qq@BlAAORzEKANZ$~IPZc7IUUq$L0V3AT@< zy6FOyU9*u_aTRd52S@(sl6(eem8ca%3l4NCbmeE@_~D8Sg@)e@Y_UyTeAhod@@E!j z%55gY+7<-B#OQxpKq?oXXnA*qEu&K*|cyRHhsEPgr z6)i1R;*Upj2%C$Hx~QPwqJp`pP1Ffc5CDE&D>QKO!sWILToUf$$6pfezi{y)F2V9= zLtt+?AYS>ett@>8Hka_$#0x0J_)j~PcOqEOrqBd4giSZQIejymcqA|{ABu!`w z0K=A?{`|MNtwFL6*4?^sgBE2LRS2Q&3pCYrq2x!pEtFE);+$*O2E)E#eQW%xSpI)n zv|Jh@NAe^=07qGH$(Vi=Tft}`W7y8KfuH&s^;h0h9Vfg2Yd7l9H zyv^eC+6*h;HUfhY%4q;N;Ao$nWgsiZC$-R+HuVmUjkkoM20hG53+oIixXNyCNtX6i zR;HG(m7ydgB*3mO0iK2asfs_0ee{SF2B>CRknjmIUvSQpLXi(ajkXJ(GNc8qy-EPg zq_qc>VM}sqzK-U;{rd^Q$O^E5_j>d09UXK?6|lqr5aAtnJx9AXv>iHdpcq1!3sSB)tDW+bHfIr_A2Y>tOQEl=GtQb4?Y7ZvCePL)*$c1 zh0nw31G!R(qL4O9I8D4_bUOY?>L4#j7saIF7(%ZAe0m4&N$!F=bM4wS3N{*&SV6~^ z95L~x2mTV72eaJ-b6sE{Bgs2pxb=4~$2n*TTw z2ITJ4h_;oN!7B%W&kuG!df9#ohH3^D-e4Hv1YxahI|q;B08A1mpwNk0)xCS#8SVgs z?ow_^Mt~hn7Xw&)QLBl%5NjpAyq8{LrJ`N0!Em+1XJ$~tD981H-47spQ&0}va-4B- zaZM;?OHjSn_&l~pGI8y1SO#4D%nSsAIy4Z6<|2AnONf~9rLvMVtPz(EF(R~;d-3t{ zB}K;I?ukaO1OS64L0wObh~5IhKXRq3K1*z@zmcV>H-HrIumQdSD+@qr0Iv|Q{5ik* zx#>g1lz7g$t9l7`j-Mc2(@JNRwm=6X>j4*=|S^-sfA3*aL>yP47 zVCT-&@M{t;I?-Se6p6w%(UqL~Dn;m@)py2w5epYZS}Zi7U)PeB3;?MZVC_VMv~@td zgXlFn=i$+Ux*8mbIxg#i>suNhj|h7)DvFPa#_1>M5PmXp5&irq0>~j07`kW@yE{B2 z!O+B`K_M%3C=40g=s)X7hd*)MFRn^25S#uLL}`@SAPH* zQ$SSVW0{*TFI{~BH%0|_pzGT66K`m(gg%tr+VluK9qeLsge3rtR`lu9meN}Ecy_8n zt0uAOI@s01vNBCOLmV2M@kLbOUzcFP%+SJNkJ!)9)cc+^!<~R|2a-|=O$mG`Tn*y$ zFI}R8Np8W@`~7*0=Ifg`SHVoN2-h|H!qLG7Li^Y zE)qWxZDZPNHvC0FQ_oSroWRPFn69R--i*xwX%+xqC!C1ENLEk6d%wgi!b4s}xJ!yt zKqY`8{ta|4A}ON3Rz<`c6(zziD(J1Gq2RiphNc}W6BtuX9y-Zv&+)CYE?t*ko-INy z#pzK4a~AJh+(goA6D$!p9bGk8)ARYjWq<(@7#Iz^O(&q#|3uWy*?y zz$!&`a{?VO@wf@R5WMU{QtLTHT=d zQIM*cnZ?%0NBPcWJEx#?_Qt|4Mb4U!JyZC%AZ-Q8&@^b$$+O(P{2 z-4!=S%8DL*h15j86xIIGw5u{vBasF-615PDi1)GQR}LqFly&#H*(oWw{rt$`LKu4g z{yjxm{Rx2gR8UHa9fcFs;PdU^zI}^GldyP28XWzs;54ML*n?or%n%){E%wkgWNx(m zkxq}mQuQ%RQBjdqB^nX5%3|*6K6$m>yIY8yFnK{{>?v+VyrU4Kx~Cc%Q(!Y)lHm>; zU?^59>3|@*?3OQ}XRI=kI7k7xdU+JERd*(WLa8ocS#hcmjLu=%4dGjk>01T8bIX4H zdLbgwRX#UWB_x2Al~w2WG(q%tS3=81Az1`Kgcs8=Gg}8*mnPAkj+n@aq=%z)$}<@3 zq;-}B=P*8(hJq?sV~`5M*!tVkD_|o|=J@`-_^~V@sJyF&mK~+1r;kC03gYQIZ~>%U zN+cd0CNt281_eF}u)L=KIs}`N!}Y)fxBF|P7pySLHv-b z6~b#V&0q9sMm!I*jWB7L=RdVNS}vT}JsEW?J9{U-Hvr{$Ay#488tL%UyAabNNBT*f z-`g94aMB9g2cLOBQj!QkN_ve5V%yNdmvjbXFa*H2)zxe`NR%I8)MaJ4fgP>Q%eI~_ z7C$|H&+;be@57hgPZ?QR0(S4px`-k@hrv*&m#lw4bH4q){c}*Dlro_@URvpfZ#uHP zq@)Cz1Zc4$JRXhmABjm24m1>cT*{nmq3a=-08@MzEBB^XzyB7OGrm=;me+B-&lm*L zzQ~o-=%V#Q9z91-)$-y_Fe*<=bN>$nCJs_Pd@j{;t=2{x!*AQO$w5fcEdq&rYqt=k zOuD|}L_|8mlN4Hh!S%J%)h^=hU4Cem z^<#{g>Z~iY9TR*iNm`n+Je;IMk7!d^&+_@+$pc?cufHH0^m#Gy20A-SLZ@SZ_+@p4 z4mA&%ssIP{8&p1$q5C0_tAR14P9P0g%dID!1;8inL8~Sj#SwC?Ki-6G9Xv*xJKx`3 zR^t;b)(N8si#apF72^v}x0Am2-bhEfF2x=cc;FptP>WVd0Xt$>GUw-|4uW5NQ(CHF2SRj=ENaE{zg3%-F_ii#U!29b`Av#9ZKmXx zQ!tWb)%7BW2Y`;c#5&QPS}?@k2&Pipu1*px9dUN+I*+{=1eHu zPAw=%$#zoFD*jBG`}CVG$|=LjRTzyH*M9rzp}VjcWHW z_cDP?g}Ls-1j|#zjx?OhhtvVf1S|C%IxH=!M%vO!9MI&izIL zORC}`wF=0e-#ZUdq)!GEK1HF&iHxJrnZhd#0-OJ~`OgN&Mo06bAD1eu7wZ9Ev?X5T zv>YP~%bnZXwQwtOyPs5F9@q+(u+`=BxMxj7Zm_UmZC=be#zClaA%NqEg)b)NZC^g z$v{4_YFAe3yhcZ#vvc_LfiBeJa72?T3IHbvr-S&s;k78jeE^5B2ulk%zYcD@inKG| zpVNTwC*>xZI=`Up3IZgV^MN)geyldIZvceb-~YKA>H?rOoZ3HX*WJEx;{vYDo3CG2 zAsr#op^}8^*%U$!ymqAkUExDSpZeQXN*MJ= zj|72j!iR_x2F|d6YTm)Vry=x?J$1xaj!MEGY0VUN6$nr2FrS#(+6EJ|IP$~uj0^&i zhI3gEW@G;X`=~@*of8nz4ARW#0bp)@+bFqh8)?fQMAJxzrxM<^7=M>0H*zo>ie2hx*ExPo=ha|14=wK8x81S}-tO zF54f5S5pV2MUB4ThmH)NGz*G;*oWK3zh{g>kW<#sSPUvy)zUIi60L9Kx`NHPecEk3 z5Kibxm0w1CNzSxIi!NS}hw=mKQ&d(adZg`9Y-|%|SO9)4qR{b8<~fiS%};!+tbXu2 z72*V-7r88L4ucnn?n!CodAJa-{DumYeQ4~YrwXCGoIr2~nczs+6 zl_N(uLDQ(9VGhyhGIpkIXFGb8AzG}&`;)4Ku-d48-h$MD3*jy3v!<4o*i7w<1XhyL zt_61mfiA;}G`69{A|`Bn&@Ncho^Aw&2Mi=Q7;Z1<7R@a!v|!y)Et$f*NBLp#3-R;p zcra)Py(>#0w2%%2)Rsj*eyoNXA?ohdTj!5BM0NqillHNw7Kzs=v2aeqPpiKAD;LmF z5aR>Nbgolzu5`%cGb3LQ(fjn&tyR@W$?y&?tihfi5?1czIN7_l-f6|S4tk0VxIjFv zNZX2o%<`AH`&*L#{ik-!mk&Eh&)om~((X(9|Jp35yx8SUs`}_uhAo!F6wv;E-$*;< zQa}LNSh3@|!d+U7w&s8HUTn*z0=uOTL)Q5rvjfj=QBdH)AA*sbRmv3q zOS@`2-fEO)z@Ug_#hw0O7}y-w|AP>+TYvx7s}|7W_}`x_y70;WmyuZNX^|q$Q0QOz z&gYkmV183HD$9QHzX0oh|Fi7LcUMgr%rL?rk0f!3QDZWokvxO{dA$GpmM)qZxsnZc zhMfKM$QSGXA&CFoN%m)EjUuKf0Wm-3pmqQaBtm6I?P^n2?=P@bpWJ>_J78gMkXrFi z7%LpmNd}A{ATGWFae#QtMUSLB=zfhfhMMh+<^KNaq*XAq-j8x{^6zChb-1_ww%?T!PL7KxP&Ah-3P%cACGDCf#W`#*G@4BE(Ds zzVrFZmuk3VjmKE5@C5Lefyc|eGrPijGjhyz@ZPwHAVN4?WEQqdFPNMLG?0^bY{=b- z)r(F|4I@z>1^IWVpiq3%drwAx01KuKgXOJWw-Nk{GtcHmn;lb)01bLKDY!9g~X#Tgf6jSVnZ?E@SNKB6TK1O+;H?7 zW^d@^%p%J>I5>PU=mWN=DAZu!R7C9;E00_emz2D8>lQm{9EYpy`5V85^LMrw9nx4s z>#zCKeR3oX!*yJQLeA-zNQ|&Z9T(VH)EoxyDp}cC{<#>Uyg_yemk?^nPbM^=q@W50 ze-K6pk-Qbb?R)wuX0*{83Qu#Y@Ne0&tgt?#J8+L@5JM&;QbvJ>`Wu@Tc5%dUmC7Co zF6*vW*LVOLFn~t@r91KYAeoz5SRDCc`h-=5ALxYoE^uD>&OggB(!2>brusmFB@#Uf zK|yrE15%WtO**41+@a$$Pe#4er)VnIZktxqjb&=tH(eN3Npme+EA3sA0BdAI8S-EB z*|xImtxDMC_(m5Pt(-fx$Q(JewF)P0JABg4{LASQjdfFe{{#>33G(7Sq^%u4P6O#p zJAn9n$yfgyP*#Ljz^E!V*p16wCh_4h$hX)JW%LSznchZ}S(88n_24Rfwizx0Rl81;P2O@KPn*clL=8HH* ztsqC>pwgDJ`A4BxaVpgyMUMXx#+L-wy(B5u@FgSl1e9wk>PWtn4ta};@NPm8I$|JOqi}I|bqU{^=I~xm>s3ur^ z4QpJ;iYqo#jnR1xN3qQSBDpbxCcP0Fp*$^ zS4aafOMJa(HolB;B$z#bab7;*MEi-@|9Hs!5rEpM6Y+791!L zYZotHONymRctnW^Kte14BP^zY@ENeV+~rfiBoRh|xTw2#xd`r&vaT)!suTx$k6nI! zVe0}FRgl-zz&0WnUjq2GsSzL@kc5RKZl?HN@KRuLbw7w*D#S5Lv#8*u>=T9(iX=S8 zY?HhR*2m?M&BT-#ZW~YMclbgsL7(3VbC&@?Z=AEg)hJ;+BN2umanLNl#1CTk!yVT^gTV>J9n{coL30CUmS?8m z&xr?|nEX&@Lt+gQP`$k8t7SVZC4lv-aSPDKOnUj_QN*N>S0`G@(xppDt&O^Z%={7^ zMF0d6ZH9cSjQI?t-4~Tx5sZl7&+fvYN_vX=d_hN%+6yykPN1ug+>hSM8t*{ZWL2Rk^m>#2)1=wTv3<5ku?UL% z>(|pE{JuqsN;_4~fn_G*G%P2@aC7S34Nx;YmlZ-LE`je>f!cDy!eTKLD#DJDIU%X3 zsU;rZ5(fZVWAM?!#;+w2PtQXdI02zq(~kbfm9Q`k97u$mZJ>$bUHt%!V#r*I=g-l~ zNdt$aAGoI$gmcl+&8W+EXIDOlpK6&4a9S>H75*q#!_L z!>&J}=6<%dWEnBbf}tTzOBe%n9<R}<=LTb6&cas-3lrbRPoGQ?f}m4l_2ag& zG+)%Uwa|=a0Uaw38tn4kn?f`c(YM2WFhvL>7BG?sy(DBv3o;SgI;o? z>f@1fB{JWS_WGArtG=ZeUqvi}0*D!(R68$nEVzHoR5yApT&)AJ;cy9Y+I~q7nfLJ6`-e8d>(n3fZ6lH(k zxyi)`4<6L4ht?blSV)EmlIw(33xM`7h>MC@V4^*51**!Bt+shhIG~4t9Wo-9qL&OA zO*OQCGzF)bIOK`iR5b?XgLHV4r@?Rx><;~o9XrSXF;E=-?!sz@$3Xazkb(cz49rTz z-)&~^7~uonR)5y!+tcg$7e879-;nf=2C~5La5l7JE=OTg=`k>>vFqAyxG7fRm(G2m zJLSc=v*Z_u9p8%U?9UJ?QA4bwkNyc$t zyWbI3qYN7X6EFJm%@9IJU|0vD3>jIAOqhxVTcA0K%|+7-4F5We0*Hi{EWbpbr}zmq zj+PaXwmi%lMFwJr+ja%8^KCPa$)R}E2`}JY zz{VDwmX@qWpH+|qbjuo(bT`r{gH_sxsbMH#F&xYv<7mmG7@e{l7(XxzBS}}Ir#|Zt zoCZ4R4G2QW6GsOEX7mUmGV>#}$88UiUP?hQKLE-S?LBlWI5>pB->#X5>xN8AuGK2I_#<)J|M>v_36)XObFf%vFivB=m3>P5E9#Ki2m18PhAoLzG z+KFS2x0?C}91BXzTGi(WhCU$Z=gYWsg_EKRQvotdazS4a84Y#=4waW`^sNxaY5hPN zV*G`ytf5|t(NQF@D==aQEi`?sS-El@MVt+6yI-l%n_)j-HQrt?Pnubf9$P@}VlIjA zTy9A&QXAQKIAPeG6a9t$QUOdq0ltZW1Os6ybTi$`%z_vSYn67{v3xK0rE8>@!u&!8 z-Qh$q;K3++jn!yo<)a$L_@{sp7yg3xt6f01d#NuQL}G;O}*PqURwIBzyH~;b<$877&ohf}g@!`S@6N z?8uU@(NObm#dVXxH7LhRP7Wl(1MpYT!X6aZ(b(^UFpM$H2W$irxkDs7l3HYNS(OgOUT@_LZr$sL^50G<3+RpBDBmb-KBR8 zn{gRSL_EZV^&_L$aILw4oGa3L&~%Q-f0{I(Kxg>ihC^2jV-DIW67`?mE0HI!iGxC{ z7#|<()U|tvmk#@2FiMLoJ;a4pBu>o`2Bar4I~Yd$Q^Y2OgQ)3(u>P_%@0 zn#hclvyxS5Y40*hMM;B1Q=*8Xr8Kqg=W(8{>+|{j{rJ z7|-K*JP!h36#$-(Q_%D6`Rra9?}b!rRm9{O7pWA4{1ONb^ARk}OvXA$fcg)^HtvSc zMKLz;ULFeI$PF9__DAHtOF`kR&cMMm({tzq2*G<$NqARXT>yY6!pz#{An)A9QouCg z;08ALBa;CzGJyh2)ONA+ypcgEH6uhP7=hTCK%iHmam~zZ*B8sh9-*4&kIGgf@rW6b z2@ZtIK7yV|A`KHW3z4p1ggnHsP;D63Y5!otgHMkh{E>fZk>?<%iD_YY30qG6-(wD1KQZr}?U&An)Tx9sp|KiR6Jq^u&eJ zH3?zld2FD(@^Ve1AcVjx@KBL9R^I+W9PRg^lZY%BvTf2Z01M@M{<*G-J?A%9e*m z?q8)-Lz*~`EU&@Eu9T5E6tR7L=cCGJ=BB5Q7MXE#%}}It8H( z{s)Q|wSkr$esFTZ1QgWL?Ra>2lfGo);gyiQ+V^Fb$P3c{o%<_M&U;Avxa0WkiM;$h_FfWtuo1x zpks3}zgV4ZuiwquRi5tex`*B8q~ph`}i@L@OIuYtdyZ z#tA&xI5}V8JoZ(Lts>)rOZ|CyzCMNGegpQcR1K4Ve%YYveOixc(U}^idD@dYAcEr59u+f_C z2>e7e1%$jZDTWzZ0XPu{l(KB-0ByeFC8r3l^b!JR)H)psI)`~TM64Q6x}jr(h89s- z?}*4qVY~Y$-aQxxFANd-3zS<@sju_P3umcWp`S>Qn?fE4(CK}9!;379#22Y=r`c`R zGuJ*{neqt=I4)S#^MC$05{nESb3SsBE*xO5aNI`;1r2DE7}OPpE){MPBmy=N%fZ2c zhSl+S_7~;=8@a43i`5#(Kz#FE`VureC*hgqBfx=~n38^~1-bDA2B53uc01JiyKz5F zPGc>7Va=O?yxvV5B)HtZP57b6TNT>&li=R-@x|1Y1R7HG{O2$n zS}p2n;@=zB|3Ums#2tdw;8}M-~N%y8P!JmZB3 zhe0}`z7Dz2h*aUt{L(_bh?UKt%{gPq<~e^M z1^}c;7p(xr`8%N_eHt3oavV(f3a>^)@PnY~0nSq#w6>)Uhk-!PwBUI`KX88-` zY7u?ikW!%;9n_XhAqBi0DV>0urLVqh^-U&1_rvfq1nKFenXb?;^;{H!a-QH?Z*4Sx z`L||+Y)2{-SV;y0yvrjhdZwZbJnY z=`@3Zd{Q@~iEDi{frz+jaCvI%QS~Bo2_Hy2E>tblki+-8pWb;Pa9-4Nglj9I!wRWf zpNBQgVA-9095xwSd(OOh=RvDq33eC&5&~!uXcHjnKZvIU40__1_KYjU_r(fbh7}-j z)`O2uYy~4Y#mfw31d7@KcY9H%;okK!Xy-Z3dDs6XH_9b}p9Zuy!#Cy&)l>G_&dUW5n(ntGF42a}%#{ z3gL}tUeh+qY9$p|#5s;Ar`TZ3aB^$IYq*Vg}c-3+M`)Gjv53Q$w%aUAKTet0^3) zc*@NnJu9%tAHet0GE@HsND3$tWCwg3SU$B3d5B&p{xL8AZv7OE#tHNf97l&#N@zMr zMIb^4!%GPH-vbf&qYGRd)k(&mg^FsL#-gW(MA78@z>1e*-2cK!%rfyqm0-+P4`I zT!a%s*$8_kNK~FkfIR^u3AX3p$pZthZ@cX{{3jWrsCY9;TSv#&=jZQ)*3nc(1LEgN z)^462xaaN_1icd2K|Rz;*C1{Rs$bcVF&ZzA{^|T$ri$o-CW5>~_>hJ;6+fQ)+-kYe zwZ+Aa$^-%DFlhxqY*}6HJrp+*#*b z0e%N%l>+O0aPreUEy3#hkZLj;4J^>~7g~FVpg}rdFgh0yyi)1l`0>fm)5f%v_M z;?oOiB0w@ji8-V0M$;LGz*fNUUb49e_PSo9_&^LNgomVu(bHC=sa%fM8|>M2W@UU> zQ&L-iA})p}%s~?W(8@0m-;bWqB;%raBzq0UH+8$B5l9?g>Tf^84v&vuLbIoe$crhL zK$QTh&(D!$kfJi$hDcT%2@>Vp6pKKUOR*OLiTRS9_m)!Xi_#Uq9G3j~;lpPECm$&b zSK910yxGJyq~9F%n%fX3ri2hm)m-$^kZua+*FmC7Wedc<3foU+z-Cccjw8AZEI!b}`r4;s0|5C=@0@+b zQx?ZPK2=2u>#KgZPBJlNXpc$<#PyHboKgv>Jw?|kZ6GPTwq+N`sJa_ zgF=(?F%J(9Dji})xCX^Jce{*n1`-qv@L#uPv}98~A;{Q$d1zHzcdAatPt4GOzaE8! znE*v0o}v{yc*Q!`LCh(%egP@7 zfqz1c?L|y}Fb>wl@1D*8Vo5?87dg28)I}aOLUP3|8O zGcc3O_QNmVusu>qPTz%bGYigmjGJpF1{wisIqQI3fR+>94y_#6r}3SA9f_gkhf&n^ zD~40$LHbEj%4zL5GE|cXhXY>xCT5#~O`5*y?bR!Cb}mhCmde?gvp zCku_*+K$uD`ji+v+WeK{+Kzm6oS+K1*$}y8Fc7yv=4u=$yw$JM(jDGYf~C^)P2PG>OLVsUoiTR1Go!ldvd!mIwlI+d<gs z2f?j@h!;q(3fArX5bW+3bUMK2Hy~rDt&7sZ0RLLoBu9AT?$%*b@&ui_@m6Vi3*rNXp-^;yd5tngc&&`IY^iSWSHmqD(EVg3e$+{Uk zoSb&4U)DJBv$ZwkzA7#C?R9Fl#9)G%K<9=2{3Cq8@)56W*)uIebAg^OCp|$UCp|qL z!sS7Gh-fE8EX>IOULBGqi&PS&Js~?>^1?&!$cX;^;ho0TLEue?kq{gS0%QiICZuaq z7zicL;;e<@7f}!e_CwEz9qHQ>N~bl7gO8L?jPMlXk6&Tms1l0aIQ^dm{}7w-9a^9M zN|tI!+x}#rsL|{Ix(e*}!RYPh=Zpuj2Bk|)!472CNHR!Pg(QqnuX+asT}UZC3;6T5 zvx^n{%*RN){C()`LsWVe!4;>XE1ZX{4rJq(+cgpg90B@itij4_=dS$JECZ%Q)@ATB zq6lKCxA-jwa{!v_n^7f>4}-!DZPka6X~NtC=2F%`RL^LSp&Q@&&%B-u+U#g-)iRG49>By8ZBlreia&nTE3QT(DoFvxIz0w7j)0EZ)ez|0CLHy==2h^!(W)mR4@&&<;efm0M6S1-*fj8Q|E2qWd0`n${k? zL_j%(@wK2&#yxtAXdFd34YOK={60)(mw_}t<2i_C$taams~%jiv)B;d%0V@ksWaR1 zmE5vtfC?Pi&}?LCgh*))88Qnyweh$PM{MoGp?^G{ttDr&?TMU@O6;DIZy5|1|= znGN&%duPrs##!r)IzKBS;x&Sz=l^(c-tx?z`x44#nbFZLdk!3Xeb=b6e!6<%R!g&k z+qSKo#k=|3#`c5HKy78Lv(Ht#zD#10QNiqsuKvm$$Aa0XvgSzSg>6jG!>{~EJni7d z|FgN+lkd`urwZ|gxj@hF{16jES_2WOoblHLL0|+vBSv8=M%9j^U=zM2O*98e=M??| zfoIh1dpdH`1=v(B;kM!lteq8qL??#tk>%*Z^*{Cs{Ue<5 zK`S3-0VTg9a%NX>m;AOhDZ7L=9i@Jr|2 zSBEyKV!sKoyR(a{1ICdJbV>HwdIwxQ>aQf2fs~(-KVz$l-tXORt0f|Bd(t=7RnE0* zMoy{zjU!=0VJ;V~KO`UH&CLsH$mG2ghV6U1w|wOc?%_##X(z8|KJGC3ohUqxad1=n zMek~UhQjX08-ETwy{8pfpC7xq@yvqv?~{{K9w`L;YXAD_!HtiuI!Wm-`bsb@vq36G zJMsLv?KwM&m=Bf3dANUu?af@UIjLjw-`0}%Q?^0|=Yi^LN=?Fc&qj#8jM{Oq2Bhj1 zUu=MeQyfi1DJ8vb7~XLiDAWV2DH`i8pv?y?;qeSqkk%or8OH$FNSXjRDb!z1j41FK z&La!d0_vCf1qualH&Cct0`tke`SfxS0DXdj64FWmQH6jiN(28u4r0MHXedFYS^Fxw zWtsrofw|rj{r>$V+g{`0qOX^iZep2ZPxMzJ6NosrumGhM!1O4L?uqTnUYJNdAa<>l zZz|oseKYvhd2asZj&GN(!rZoDAp|Ur+_)XJAF(z4!f{F8qpTOXDI$6q&n)wz^dGO5 zwlll7Pyb_R#<#J_LT~P7Pc`i2xf&jBEP1l!dZDiR&-cF<52qVv70bDb8RWlAUQe6b ze6vpeVCBJgIi;pkS7?0O*{W>Kv2sSaMIK86f{2KPZQpOALhmcVyag2sphwk)gMC(eDJfB1(YehdVvW zG-iUJwgX*-G^7!t445+LBhAM+3*doBfJkNhGeD&wq9A(94;AcW+x4E1)Z}D!Go!-R z=L9j5Yzd0G6Ud_-MX`~vZIT2y)FQNT^{fcxdyQuGjCbh;Cd#i@b1z!lm~mtK*cnhd z86KD1Bhp-Z96XW-#bz%VEY*zG-1gXDX~e8)j4x$;&S8QStJSs+Hcy+SW)wl#u^&gnO~10Z{%! z%MO_j39tX z(-#;IU37u8G=uT}{rj>pn^lSHWsD;d(NznT26ZJBWo3=4hM#{!x)RxM%n5JIh5_#* z=!Tbo7^;G(0q8@8!zA;UIX)wd!}9U_AWp{ zli)|MR8g1il1APkr;*8_M!0ryZl`H^LTCIF3eZ2Qih9!$i#EG27g{)h+pc?5#YBA=u z+<+d4s7C|Ma1G|K@7}u?25K^iw!;-A^mTJEDx!7g@3S-i^3FZz=cjc2bFYH^Zzj*3 zJY#gch3<*mlEn^(kD8yqankZ#x6LN4c%4NS8%1BF@iyf~Sk10DZTmd1wjU~(*Su;+ zHSWr;*c2Le^mOwu*PBsG<#D`Ff5DG>U6SUu6|e?dCoLAP7~E->HZr93;uVLq^qG5e zjF2@V_k1s8Sao1RN?B#)PQZH9#tDSV%Z{diph|FYPR%B37tB@PkHLNbrxL7O3(?vc zC24>mT0{{u?@oknH1YmxiA_vwW1KiZf*LIX)U`@TI{xCHU=vUa`kVK}y6(FpF$wX! z&PRK|lsKK%wo+Wu=qZ46HF_VqQ>Z#?wP(Krcoc+)p+Nv`UcB%~RMcOGnOZ~;Lzjyx%~lR=z$m^0`#-L_nc{AmZ^ z0(9r2XD?K|KI#30&ThZ@IZL@bQWi9scZ(blUwHgDe?+~duzx{J&ce_mzD`1CUfspm)y2Y;F;uhELC@RMJ*jc-||B*5@JAe9ZkHTSK7+R<&H%C|(1yH!0qwta)?@a56R{XQUx-!zot)lSeb0IJZe*L_X! z+jm|c`JJ_8hB`&%ueUEXRCcbP$iR;TPsqf@R+w}4bb0{q;4bMf>`Iy5tK*xqc0K2* zp2;bs^)tj_t6UU>tXt}Ha=;4df2X#M=-=sfZ6AJz*vPfndH~~O8aw6!E-YeaxSih0iNXub3p65#vbCYD{XYeoH7 zc*nyxl~cZ5j;M*$zV}ctyVPGe@}u3JAwETi*^e*F2_MY%E50S>-vsjC!pgYrPAA^< zrkvJmnTsPAl{cCFyprA3)#dW@J}rpP&8hq!d=NnHP2?_2c;(HTh%)bXWZSnQO{&~I zEG2n*i-i-T5MB0(PO5p1f^<5Jkk;Qm8CUwwPCie6fzSoSvG z{_Q|@N-?UW4`=-picCdjwh_fkc>uv;J8O}Aml$SdWS9eY<&m=fzWv9a{@$7Yyqxje z^-%r4f53m7IPW`;v3$NbkC*bkeUBU#9e`4o$C9N>+w~{^`?6){)y6Z7=j{qppEP|) zxWPI+OQlGnYiOvsVc_+|`TxF5LU7wOhK~{vi13=pZ@YXaxEs008HY(`IfTQGPvigh zrN$Z;@U6a9r~Q@$&gc_eHGN%8=D)vW$XxQsn*N_ZG8m<+_bD@$ee3+1=ZOJ-@IP(( zo}ZUsXaV8}YxI?(F3+z7v2AH|EusrUJ4$T@#DvViy@)}Qg3bivAD*<=8HFzPCu_r= zWuMWCv9)pzIJyuTMf4_7!|XT&)F?0RWpXZE8V{={2u3IPL)gD%NK|@2A*fP{nw(|8^CcTA)}<-}i5a~A?A?mw|7I8_@vm z)pMbk=#BtR zqh(EnAx2J(`EGi@f2gYV=*^!uFWqd!P)Psbh4I(8+(MJ9?&oRXNzgcMcwlW*G%7LY z1u=a&5ZhkxKU<#u2+~AjqdywNH<-RtK*CrJf=v$wHhhdQpo^{AF7RU4^<7~c3;J|l zHY#RT-K?)yU07=Pmh;|ZUe}(RG=p_)_=h|*94ToFS`?%)n-k=uXm7be0d2J}Vcv3O zNML~@$RT4=Mo~^M$f50nIuglBym~E9_d2u;rGwXmOUqf~mUSwt_ZGA@(m}{wv&^DI zN+cPT+!1&~4b!QsHXLtX;U1a_-G+RhZsC!3ZZ(n?E%gEq5uVq7z24^{ayZTDc zW|oR8iXIRoFGon!_`S#dAQP*pi)+{b`612`l>QoVCgy`YVDPXYdu}gc{qK&Er zT>Wa4&&$)_mR}!u4BQe^gY*-#`tDkPFZ``#V{C`6l$$ROt3g#yGd%!}Kb4WH*lr1U zE-g^hTj57b#?Z-Kw*4cOGR(b?;M|fgLWwREATgwyGz5b(n@(50>a9(4wTNoj76AJ5s;Ez)RvY}B$SGthP~jj`r%SxA^f*Eoa{~o^-TcdNb!Gr#E%0t z%CPxy1E@#WhXDlj2{?7pFuw>g1E>sibS4fr?PJ=;=D2bpC?J7s5&_5b(6O#=)`dhw~mO*~3=bm*_mqyq_yD}Fy(YDaH!I=WHXFK={=^qL*SjG&8(t~;V zrFLfc`~p>r_7*Q@IBbq~c0<; z21zAmRtIVUKmfZgqP3TQ&6-w}I5{Asgw9ShZXKP)3qce!EQvJQWUQ%7qi78*0KR=G z00nIGOt`m7v|%Xq%YEny9vj|`bZs?ja~fM+nAfR>kfiG1sjLV9x3N(F?)v>3gpGm< z=oG;~?_*XVHH52Gt#|3(>zRy%B^jB#EtoX6@r@93K`BoB?S+@No&KJzpX~9aNx&u4=A}9FOjihv3Fi-Np%_1AlX}4S5#$$ud>>k#Ib} zA`tPgBAU=(>p^TG+vST;{Lp(eTE<|9LY&wf45bPXdp3Z&24Gw zx5EZ}yF?kua1eflU?UOfxcu+ZrL>tk+oaI)REOng|D36dSKi_S7WEUfKpj*RZGpug zRNYDhDqvy3M^E5*RbsL<^1-!UcQm0cd>duE2WTCGdZ#8O3IlhprKaw>&J+H4(5ozk z-=ZR0v$f!DzogNqW$mrcPNGrUJT=GjHTT8u?{)27+Fom;<{#^)tHD%1Vj}j_D@e?& zGpWw=YIaDYO0^<$R-1h2;_6e%%$7|t$7XR3XFH~YcEqQYoa|ut{hfH@o5qMcHaQwk zIU>q86g*A3#v^?=?FO%G1c!g|PxJ3Hvxa_U-jz7!S?BoeeYGsu{Rhe7K`8{rihl$q z(c0c#2Q$oJ{U3rOS`xTL7r7rui+hs`P~2Ru^fA@M8!!Lf&~S7XzhZwMHx#JCaq40a zh^r3UOu+*BNX=1xzCCNOCH7;=v1{mBjKa=Q%IH6iQ5gdp4@2b z+rmDbd6tgPadGJukPuM!{O*!&coeHZt22+vy%cYwUoFQZm`o zQ^G*MvOwI-!_Q=Kq-RjN=qqc?>WXDhcA!bq6 z7F@oezWn-AF`<^bFecBs9V49Fe+``0;_U^OFc=r! z3bHwtNQ7rG1Zly8446PiZO8vi%z7e2D=0`)VQwc_ANeQ#h6`p%}-6lm9D_(dG*Off*3kdywNKWQGu zv?e0HEr!)a=->~Vj=+;&39i_w?GFsDVunL*F340Eiuh*51<R#rOS{bQNoS2`8BI!pTtrTQ){qXa%Q;ZWsZ z`oUuo{KBhODiO)=@#DvUi2uA}T{{YIcn-|aEcBsrOr5%}W2?+sa3GbAdQ%Qo*f5oU z`QHAHm5?`fF_xb)<+7ywP4}^V&)3kzJ|(B`RxTigRY3`c_TI%ymmXTwMJ`&Ugj3{s z^{P@ONd0b0rL0*&9!g5Rz*&F7thAt68NPm%gG2xBx6TkD$Qo$62SxKl=+O#!yjZRz z?c5g!CA7YG^Jguy1ruDU4DT)($DESYnLB;s|3Pqhr&i0k7H98@KhFv|g4~h@1YCL-(prRXb=l73WtTm~gqL;$Vl& zsPfB*oP@Ht4!4S)8sCaXBi&7Xh6{!h#@e^sPmsNo=>Yk!?@@*eMAtvguNN1DsR!s2 zjDN=7B_oTqc5v9#VJTCXmzVb}rzZ{jakWyLI9>CBB_R{Tqk#f3!%=nHO`)V|4#t+= zL7n<1NI1neSGh)y4$&w*j~5SMf92A=@d)Kc{}=&`+za-ia`iLS^}Cpq70KQFQjh2o ze#6vjnXDe9b2w(rsyaJvf)@Ir;2D&eWiH!AKWECJjHQGMw&MFAh}EM9OV4T zk;M0#@`-8G%+92+9id5tqdk%gSc?OkillXkYG4?WGTGiGbn$0Rc7qR?hQ@D%6yaw= zgKbZ_Z`FvAy@b-O?C&YOfD+RXTs4*k8v^)nH_a(KrW|EcZGU&+aB8u(z{oqdTQ`b) zH$Cne{9ZMXbj?go42YH7pyLz1W~VnZU1gdqTnCzME29ndE6rv4-(`(c;`sbj)xaP~ zaQrl;ZNrBX$ z;pjGX=pQ;na9{C@7h81`hU@jc$g`H*irUi^Cvc1xsf6%nLs!DcO!-YEMk_|v;5R)cclMM(c40a=Tm6WmPjHzCzJ=79C;NAI+2&fP;Lqg8CCW*K!w)sPf`G_r5Yie zMnz{5Fu=kP=+p1TZD<22B0rBDLIL}<8i?h0j3@G9|a|mj+eea*RFTH>CK@B5{=3+N4|Yz@_`Uo8KP;yL z>`gW3kWfqPcT}o2I%!@cCfwuX9njMlSzgjAV}8S$OWC6AySJp%%Qe#WJ$^hJ;#Yn9 zyhJbg=tY_vNlwNIX;-<6KGARyDqt|^W|*odShtUu3JaJ$6uT)3kx=vKqKIRN*PXy( zLZW!|y$mg0+ckhQ{EPVoU4!2|F8QcR!M?QH@Dr3mBhmLTt-ry1P}hwvXl%grRW}Jj90Nuh#&YwPJ^W?imC*9${={*S;DPr}p&f?=3Cnzi5CvIqT?G=x&rO z8O!=rmlaFLHVYTjHrJb$^XwX5|9)=5WI&)dL;Ca)Y)HhyJs&L+qXDuv$-#1JQu}|r zUIY@|pE!hCU{3a;hdnYwuj}GM10R?zq|c3CzAQ&Ig-c|=TCi*19h4mckNbq2d3`D7 z&!H9z#^B7B1kE+2LW1H-0L3F6t3tKI)-3bA_2b|sozd}b6lT?pb*{A;($y@mDE&3^ z@y`!Ky~oE>tT9AHBIMo_@^9vL8WU-)Up!=9jH!885QNWrv4T|ui9aK0U$qml-JeIw(l zOeJFuA93G#=ti1rh_??LyLAfd(PS5Q^8^m3o`CwKW8 zZB!D`b}6~}rRt4+HQyXw&3nb?`>^kd*0ycAM&8CdoANtGe82X~wSIFLE$ZAdcH&F_ zXwm-e{!#S~vzxy^7$pTG{)V|!bvw7u-qk(3vmj#7xXswI{;@JR87{+&E#m+9dmsJ! zM1VsE0QB_myH!I-@p;A`_ZM}J*~l&aljSZkTqpnk`Tsj+-sgPhoq*J)I>2ll$l~a? zLT|Cq`rf2I5@*=CbEhDbf@pCb2cTMj4yI_LmJ?(OMk#6=OeF0{IC^mO2%yCAM0TR= z35@wQN(Fd+6}Wa1I7E*~p@G=i?Swd@h0WQRRo!EB8h2J?7^e?E2iQQ&p*FA({hyU?o(MIi^on>2(3T)s%RRQ;^x< zGyzc}kWCwbuQ(;J{JNg4t8j%UbexZ`NAHt!0G&fdDPJFViX3_*T&0fP&w5=r54?$9 zbiJbxM!(kCCNQoB&i;qi1BG!I=_~W7!{gk<<~r?}`}XyZu15g=4hDyQsh{m~00<~; z1niH!E&nM?oqcmdZG1{Pt_2|A>*{Jac=BdTCL7w2_#$Xky3s z(bqLK2WToZu#GpY5*=L)YH?evk{rgJ&B7?GRvZ+;1U<#s$XU)fIu=6y0-u$EIW3+) zkc0^BPkI0-H}Pn`XES=Zo1U{GBKD^kuetAc_CzaeGh!n(=)m*Q(=E$tT>=yIy}9{L z(`K}a<3kmh*ywg>v_VbhZlXQt>FfKm#_XVB5EBb-^glme4iqR&Q!P5Y?siW0SiuS7 zwXnL#5p$%BDWg^D-v;@u{w`^RJgAG^vK8A8ZP7qZ8}#!QV#gurhNdk1fU4EZmnhgD zTgR<2k!v|WKR>Xg`u84VYbhU7uF<{!m)MSW2XvJ(5%R+X z3&E2*2VAm60;Do`!ILxH520umiVldv!|Md3qz*uls;jfp2gxAvdJ0CAkV2{hp$Fhb zF#{fx3KQs4IJiE78#n5pDwTl`;aDIdfQ@1k+$M^o0bvKzw&O?PCAzq?ZNJ>!$>Qf4EB{KW!0SE#gx;aFo|WmNvz5i!VzPCSjogaYRBqLjTF>7j=!# zY34hMaMD%{7oM)gMWXGx41iOKB{mgFqoWD{p%YGsZ$L@3+XfGvA06d*Y>1oGFmGlc zZPG9=ZXC!;Eqm`4a$?+x9kx?`ett6~D*8l!+_4_>ZmUi-9+giqS&d;Zg%a-HP1vFT zr>Xq=pW~>)QR(8yp+J;B_u7H`FAgjAD*2`D%Xa)^#%)&OF=jJ1UjdVfnuOQmUKq-;c+J;xwEVf}^<74{HVdswCCf9f`ts?5RrKQUwW!x1z2F|2u5;`5?I>^)dxO6U z?_N{qEq6n_D7<5EXERs{aO;m8q`Y6U!IBMgl#^E0O+DcK-c;-B^YflhmJIxP>%-UB z-(L=*RqBp)tTJoo={B+b0RS%`XMRkQUvaftg13?69Zp_eF+6Cg&6_`Dva-8U#%Y> z{7Q6GkWK{(|CcRaKIHUppRyyEC)z2-O8z1TG?1#NIY|B2<}tP?-6QZ7kz+MTB>u3- zY)AUfD|)V9mhmiS-@EV*C0;pYznY_Ul6c+yd|yWnlk)M+(}H zA%7vFJp^Zmux&1y5KPgy9fLN_j30aU?9qZkDl|c*ZEBIc5dUQa>IhW0f?qT7A7ou> z;mMR?rD=$e%K;T_Kr1{}qO7iweUFRbVJVlUFw+v6%H&uP5cI&$6W~O~-QY|k*CMZ1 zwxv>Fkin(zgA3D3I(!b}G3^>&PNQOhGKg!S)^aUfYMH0Yy0ho0!YaWe@F~ zI?62BcK*HsJpKxKr5!yO@#)NGP9W5?DJpux#2w9ZkB+9Z z?)dpF&jo8^kg+IfAm+??h29NAcwdwV)!}(HWo#hVixv?+CDega%;VWlY2ks5e$cL- zOUVq6gz*(1497dHS-(3QhzHjS^+b4SU0izEh94tpHy}CvfeGDl42}y1aHMY2q5AlNaLKPNGnlo ze3%RJF0Cfep6X@CSO@0pj4o(5n}gs1eCnp$pD!M^-r&(D@S*Iuq2vbu7r{Y@!|Uk? zus*(K&#f8!kRMWJLNWHwJ2})MDnSSCt#j^ZPA!#t9aYA9nZo3u@4l)%FQ)KPGOd^G z^7Q`wvYgKl5SXQ;0XKiQ^jky28$)?+HAn7!MntG$ERPUbq+A#vR=?d_2B<hC*f(Q+~vohGHHT)L; z_GxGkL4&(*nM+m5F_m}_!chP9qB{Qzf7YwcUOz0!Q;8xmAs2|Ys!{$C1sc63C=x6_ zR{icELXpWHEAqRCA~)1^umF5U(Di@0PLA^sXA(NFfo9TL*6N@%>KKDk3el1X{=RbM z9ncpIne}(}AkDkQ7mZ&msAh4y5XHT@I1h1&HcC{eN7ZC*p1NrkjMfI zMpVE<`ue~*Gsz)Z2%twP14GCr@(T)t!4}o7 zCzFRht8x^VFqcgj!2pGh#G0bR3bIV;pcu=apFQ#A&K5!T$IE0Qq8w-)+Uk)u%CaVR zunXc1a}3bw6oMluq(lb@X-_(A9u-n`+*ze*hVjDEPfF|n;7d(q{J~9j?HJHLg2Suf zGKQ(O1eRD^D?1Cngjy()Dk8Xm6(cfbid+Hvu_EGOtt8{XC6qou9SC;3pPHHqqEmTg zfCAfLROMx!V84jH==1+Oa$%g^`_D(PU$b$m0Q;`kfBY~pgtdA9e!$`XdGr4)7=!UY zn}qRyIvAU4%o{+t0EW_xQXWbdw}5wYa0(obWrrsUt06%cPWcmPnN-$At(G!(z!tTc zlI)W@MQ1wyrz~e(3{8(EZXY5PD#S()et#XHijFHagh)jA!5r2D7Hqit@EW4dkc=GC zF-U`vdaAW=$iDLQb9jbFdsjB!VVOauZ52rErrAo32F6c_qY$%X^YPYGu`8K12n zxJW|6huAOhaTL*l@r$As()T2rMd* zADI%YElAQ@#JAVu0t zy}|OMR^qcRPl!Dj6>zZJjKZ!|CjlA|7J~u4#N)83MRm9W1URCFKr&q&?wFbhEW}u$ z1f$Iq0(tae@G_5{972yXgdf&fLQ@~~4lTq%$m35H2>t!=6nr>BMP@j(pKsR^1s?Ib z86H2Cu|Rv+fN9YGe)3CGNprB)h_emlHU7_A_>X|}h5z{y>WL7d6^B!B8=L(7rRI3A zTJG1p%lt|JttB6mzX$h83wxRp5hdbf$7Qh(a*{8u?I++0ju|uj0s;c~AfjiK0V<^2 z;LlZ+nDYuw!L64;>?N+bTDg51T6LIwpff3zA?C1aEan2^;vkb4J~mreFLT^D>@2Uq zL;+9?bpQ~Qw$5{ApB&@eIx>?0IL>1WG8 zzuQ^cQ>xUrgKUZUfz6j+z6I;{bu^L}I`+r1Ra0>LL@VIKv)so-gRJ=(*xLh*S-O0 zKXqJX^n*z;I!qC4hC|26UwQv5j<;|68gTRiDey=2Qis`t!_n%mYalLf_~dj53>Q(l z^_wn1Sa9nlzug`xs~B8rBQXn%u*xGl`3esB(FU z3G`GS=QCNqd;XSclw=PurzJWFplAyy9rjTitlxg{&9C011@(WoE2Ng47SS>Qpn>Ca-lp)g!z7XN+Fz=syc=JEaI*5* z85?3$lZ+L{8&bU)&?+^WfOngYgr@@Z=Cy~Or~&3q+x40CUMw8QV5s*T08K*!i%w-N z^gZIeLuxYwrNL;h zLk5u^P(Mx~aCPt#D$tvT&3>|^k@fDgflo3rVqoX*O4&RV)z1irDYAg58=~-P@CRZ+ zCMkpnG*Km?E*OUtqBfJom}mHmi{nxL^Z1`9Le(b7ftTDC-;PFx_b?w77_TvD;>0MN zh({uXSlH6RZsML90?F zlKRRgRz_%O(x9{vuxLps2pe(QY=e^I0sTd67E$fO!!X7zRbVqhBm`+Mk@N(EQrSo7 zd1|Rp$OV*Ii?Fst0f3>Snl0W;1uRs6Xn+=x)hItD<^gp`fc0S^^4GHS0bqy7s2Z{! z&^*z>Z*2x!&kV;H+bDGC-i78NiuVsu38cbk#bsVM)A~^Mez}yg*$)dmxwpA|H>^qk zI~k&qPBZ8#T4SRlc9@Sq1@XI>9RZUfioJzA*aG&xDvDjy_)r9Y-iwn+Wf9aSsEq+K zJdkFPp~!;*A?J>)NTM9zSl;5>dw;V!9#*;yJD^)0wttB-5J!d$)Q_5)gPDVtqb#7gL( z!l5O?#r&j52)#IAC=8=@e-}EUsz66*J0#0`u!RCRsMwbL&2^J6AS0HMl_k3L=OwG8zf(KwL9`g!Yz=;j;_4OqL2}o5doR!d_CrJox4*xo} zj{A-C#5Q)K>f?&rd;*yoKI({QdIYz4FK5=&sfzD&fZs5~5kDUM=caszhQ=Yj!^iqQ z9T^RJp3^^puaBKh7T$mrMD9XMcIo zF_y*KdAN2Yi~Z)#Hkb$2OaA`LXTkpuN58+Up5JxGqp!E2D`#m z*4BTI6+`A0+yLIA3g|0!*IL#amS4lvxg1!qNUUh>U(Cbj>H!yxT#yOk)!7(J;WkKJ zgy4wSaItUOk?)%7-vi9PIK}Dz`y70KdqV+2Mfu;iaVGyw1g3z&rTT2m7gW_zu>Bad zlJ#LeclNLKdxz5Inq@{3cjD67(O;4WjEvmxAO++Igp<^AvKLx{EB4~NKNr~7Rilc6 zTF)n&QrY*n*R->EwTu+e|9(L!epm3GS*uTJ0sQbhrIURK;%oZ zNJe%Dtcue&uT2<3m>kJ7hQjqvf|>w8iU@9JlK=h9D*iCO{H73>=pSf}zz4ZD{6_LX z)BqUM3do@^*1`m8{76i{wrv_FYrlhG2u0cxgP@5x`Op>S3wwY2xI0cZ;=7idv5`db zc%ToG^f3<<&gLh&qRd``u^70HXH;o~s3_7TG`gAtCQ77Gw&?F+n5_={JsmS`kX;4j zmsFQp&zXHxW5U44#g3>3se=TI{&%sM*xX{91R>Q2N zVPYWotICwMXrYEcxgJ1J-tR_j`OEx7&?Ze?LCkLISh{2h z6*Ga!nIl{#mrPryvG8UdGl0D`qrla6Q}Rlq#?Yl=4uTPpllaI*!20PxDaQ|385&4_ z;+RksbYT08NgSN-z~`c{{%K~WyxmeT8VKt}-?yl5Q8@WX*kYH4GaFCU#i3X$+j9nC zR-CmqJ7uZ4xie&<5*e@a9v;1o0fsGb6z>3Qr`i12H`3ugrczn``n3we2C6U+^@ixG zs#zzqT?Qyk9Qh$XrVi-?qJmMRgyBgz73B!|HUI*IC+KN5WN#72C`_1vtbiRgJ;Jp2 zKn}(nt_IyIdn&8z>kTPI2l7sAX(=h$ky|fu8{}X>vfE1w{WJ@9T$M|ihVh`0Ci1de z@VX=8{~q=rW`4+pA?qJ+S~C_;vUt} zFfy$NgOo8?K!^eHqw(X+c z)jfOsP{c*cg-+c55C9HDX(|4@y`6)J+r7I5$0y{y-)sAsv# z>#mTAbR07>icKYj5SrllQ{MqXkT!eQzh2DAz_~r0v)Jvp7@F^1F6U+)Y{lj~R|gi)m`p`=P{}i-kV~h&A!i6Twyih zdw!gD^RKxV^|GAO5!gnm$_?yb5+BYSA8(()ia5{43BC^Y^SOn6t(eg-$iB(5yUlPM zVm5+&KpbkcxK^PQkq*`~+L&eSMgJ{tm^6i7m_nI2lS11qy6o^VB?a{&MJ7s_?T6SQ z!~xA)GJ;=$ikX*%QyW?Y>IE@KB$0ulxOo1SsD2DRaSB^DZP8_$`jiyR3RBX62WmUW zHZBMJO14ew?>LHvzZh@`?z-Xw`?YMa6z!;cAr*Hq4MxI2@t;e~0BfzDmv4b=E(~)x zL6j@(Hp?;uU|gJ>ehoaAFh&h6c_5-Lue?pD8<@m?d}y)|2vcTR znJ;&a_Nr|IxeH$!Nph@GQUf<#gE#v52xJdWD@Llw8ocVMtmI7 zyyC;_WGaVQkLODfF`4ksd+cZ4nnc983og!sD&_+)8}H@-6b#@SL5hnE!k-T?(N0uV zHuMjm6bfIk6258mPJyrJw+hBc4oYmI!2BT+5HPPaLbMFB^ssNq5TVQxlwrX7(vRaEWD`WhZ7&o%#-eoffozsRRc}Mzg+y&m5Ea zU$eutEt{>sUjg>z+mwnlfd$-xwu}AEhx(V6D|#~$F0i7Dr>L093U}xk9)goU@0xt*)U(>1w>%b&??vaCrn!@AW#@8dk(sm zddci{_PKL_QyHwzTj?#VuVNh70RXV(uQWq*!{PR6P{xV?>~r7~dm9)p1=kueR~Avy z_gAq0Qs!XXGG&Y{(>Z-QDc_a#NeWdMFOvYRS73$3p>nLh`%;5vkoyf8t}?;c>z)2V zg8C_#7s=SmPOXhsELx^aBis0>cW2Q*K;5c;(i6A1uJ;!RS&dzb=DD$<`Y}Ewa#XL% zSl{w|A3BPn9O_+xvVj?JQpG~`3s#q7=1l+p;CR~GrqP1w*1F>Sy#o$n<`uzux@QA1 zk0H`&3`1?Y-^q@{W2TM;>~WJ_e(al-7#DkC9t@T7Xa){+^v7Q|XygQ882klM3la9o zY@G!(U ziu=C5$Mf&+`98<--1l*8$Gz<}eAegleqZnFIoCJwxA64^RWyL_Bn*AUvSOb&#cV(`6QEG#WulN*Q^_;C?N z$Q5t^h-!q~^9;_7d`d1*;aF`uaCyaG=k(G8mbt}CoLsSL*AoYC|JVJB3 zAF?CFP*)KwU<}lWt6;D{KIa7>Gs<%@$cGZ~u1DF)w(7SHD6waUaH=VkKqZz%pV2<| zTJQrhZR`D5*Y$mq@93=_k(zlVRw@dHbv^bov)3;1N~k6QjyZi0gCJp(;yttc-4p-W z`^#2U`xj7P=Jz+wFUmwlaQ%DH^zcfxnchaymM66K&=f&|*Qp zsUh+~s6^I6kD)F6K}zhqnJ+bq^4I4MZ*^)gpo84U5TJ!OrtrJR73X2yfJO53x0PG2 zg%m9!vlD_1CyZIj684=3T2_$*wg}M#&64TG+w0K-p>&8s`64GHJQjGPV_0WWa&fxM z1v4U5Nu8z%Z;!@O#hd}_6%8jFT|}4EsN)$#TZ~WvquKE!dvB+QA9?U8_mCau8(Sad0GC>sAHOJ z(jRt1^R#a2*LqsBfJHwncp?YkT!>a;^i!hw+4|elDf>tf4vPQ)^N%zdf^Vj3{^MYl;jd8h)8>`SZKmXlbF|1sX_uL)`ed6D3d~?M~~#o zt?CkQNJgDeE2HN-YpYo$kJds5rd2kcaTi;g_qKxmoWTsJPtpS77_QxkwMS9qR{+UZK?g~DHR2JlKWk;&?M|l654=;#l< zy%!hh*6tvgO2)-yq1^G-%Jx$)o`K^OBv~t&2%GF=iAw#!G54wUPxQ)k9V}-Xh@kui zJceXHA!{B(kL@Ky`hsqnT&X%OL~&yFQ3!-h?L#%XBdiiDyzM~cAAl?bM_e%dKeOQ7GF11noji#F3q_stRl= zVQo>W1@RT%yFc_pR(lNIv&F<@cZDW%d4rp@QA(heo|WdNl%0%M>TrytafBzt83)qs z{hB~m25R6qe&f8zN__h;iMqe#Fk2Q2u=xDdHb73YL+NUrvRkY9ticn_{Khcx_HFv< z;qc#8xKbSAln4a}m&HV!{(b$m>2GIguA~LL!XZdE53yF}&u2=34Q>!A>`SwXuSGOk z&s2VFOlLVI2_%y0*KQ>fkdSMnPSe~#hu?uoDR=#7^oPq0mi0xV_d?cJqWZc#TX0Le=q@GpQYU11}Jq9yZ%1yMowIWCgLw9Z=k>9pQWz@e+ zdgus1WNFqyQO<(YXOxEO-yXNTz>C>&&oj_jx9 zA@$VP?$Cp(7ypuW$`DY)Jr4@bVYZd|<8yj5vx-n524OG3NAC6ep2w8i^7!%SQ6_TE zCb_SUF6Zihoc{6IB~@Fx#|EmuLMweEZDe+wpMnmn{=HXBW7)N|puJ}2y`R?(2#(@e z4D-m6kEXx+HtnZ>PqaQ#?WuY|ACT}vo&*eLO==aptwksiQmNvcaQ>dq?E8e{8j*zq z;`O7*!{UA~&L5SYaV$;gfJz+=0Cj4`1@-mmuQ!>v>N;Eqfs}sH(RoO!gbUasYY(@Q zH^Kb9?T^N}bgjjLiQ@h>(5_|i6F@0!sASAAigBXdBdQ^9*)qHZQ;i*&Ab z?cfF5IuhvrEfXDcokOjiQaYB*9-a9E8d&=7NpJC-B(+G!dEyV0L!J>a@Nagx6up%!@d0gSP)>kjShI6#>GhieMc`b?NA*YQZ)cJDLP!;u zjN`OYS)=8nBkPcY&JepnRQY277{^n;!mT)(zJl&Z>a@y~=NgP^X8$P~cA8`@odm(u z_`Q&(kG__f@obd)wKa`uPcLEQ5RVej)udEMq^Yu|kac-_aXcaIZ*UY*y$T%y6gIM?GLzVr zxw_2dN^$Ds#S71uHu;N;@($;Y92Wc`76+VEJo)sf8r z{!Uvo|AkertGAC`kTOtNQwt-kz1J%u~AjOd$p+XnEXuVuM#>Q5qPJ;g{bqQ&5TG|HqG7 zyr%yFsysfo+qq}Y>VJYP`2*ecy>(Fg@IwgBSR0))`xLchi^b+|rO}qa7fUN4=e9s9 zDyw%tnSLsV`OylPx zW~NO$&3bV#U`oPwJU8z77fq(1`PBbSoyX{%qqE+=El;*5J)fWVL483nwFTj!b}~(Q z#4i)`WSbL}VDT_+CKQ?(Tl)ms;%>USqI`ug%vkS2tAm4~Xr;M3o-n+0>muaIu21P)elN)BNt0@x0V5`n$_;)s1vZ5E zv9Vo5&`xAY1?wa-FIaR%bDoQp%2f_y4y&Ik(99Gsh0!YDk%ZoYgn{(%{#J7 zkTW5%i9a&d$-;DQXlONr8<9$ihBjdC)=BpuCQD~p&M>;rSEQjwoDHVn7e^E3_a4M% z=$!^Rj2R}BHiz4FTHrcMnr;Q@WEsN6;&)Fc3weQ?9t8>;5av;oGpbtk>hfg15bVeZ zTpqepTRsGi7Y|gPK?mZQu#h3FFJr??IKGG^aM@<{?$Qs4H1S(#hdWRuYVAmY` zVl4{NE=jAHM3ZsZhrfK0<+D5w6`|9klLr+T2c$aXtM{Pi8DVyFudK0J08bVzD53N? zCy!8V#o7X6y-6-t#C}SSFD_5?8PZtn@vQBB_urMm0TLVP5SF|YwfN42)-xE93lb?g zVc6I{>NS$SEoG2Sj!ui_B3bJ+?XoSc&O}iV3a<~ZO)F`IUYW9ITT8R{Vy5W!XP736 zDDZm_ObrHX8fK~-QeCHBaT_m|&3r`I*=@!Ve@YjTX=mZ&yCD#Y4FlmPIg$vff+*)F zx{Oj^!KwOfCyx%KuoF{pksoN*`Me8z6^=yYj0;P|Rp|)Ks3XoJ$@r4y^#|*c#duG% zzi&4ZtbzjNIE)PjSl#h4TYIb!%9N$bnqfO+SwU`@%+}~>+u3uQA5i&$s%l(uStETV!Po8sw#GQLdcEzqEEE~@4fd*_bd+n{g(IZ_euTj14i)eFxy%wrF|w>EeYV=yLX>oPjnWe zxXde;h{9*+O-W94BUOFS=wA}tp6Zw<)>bkOQc?Q*;TqsA(43}lu<0kLqVIR64wAlDMGpq?-*M04)z!T49p* zW~`>NofvrQpytFb93pfaHNFG>Bm|V%(F6%c8Xrw2YGQS|jNS-zvy6$4Z@=Hy^>?8g z0@qf`6=I$Q(!Uk&R?@fetamYO(t6H;&LY(J(X4fNRntR*)P%- zRA54SGkyl1R18-pmsKpT;+#|s>({rhJ$2;nt^~P4~Lxu-dFr+HSQ*^m7E>VVsWe$r#xu|b6Ok=d__vzhmL7T3BPVn^i}%97hm0_ zjU;|wV55!{qvHOnFa%gZgWKwuLjA4N*Z!(wAW1segRFG!phJ(Qo%TEc`WHG{h=%0x z=nmz|8IoI)>&dk5b|?spe;?`%PH`Mi#O##!{{42F3lNG~c)Y(>BRHerF0MDC>Js;V zA_pm5mh?G6}Ebfa1e&II9SR_4K5+HchN8j4#8m2 zaHy+>_r8f{<{R9K*GxzdFR3=8_a2A2l`C9Xg%EE<=tXEh0U9p?7vY}4qwx3bQgh;8 z9C~Xi%Fjuw7A~u}Dp9Tq^9S~hs3MhCREFVEn(F4ZHc1Fnq;P>MQI@e`MGituaS+d+ zxu`=(rSvt}{mJb@$^qN^!lY`2l8)Or8cJ%dKj?LPOF<*Per3WeK4gxn%rVbU5m ze9ah@pvPUfn>kAz8Cr_>1>a<7ch%i6qx|bvh>bg}%gtyeastYtkv6!Z9AiI~-|PzR zlOyH?Vg%?i#w@U*jfF1I|4<}ua`4iR3^Ug7q0|GI@3)aR3OlGgf>c_LJ344*OtvlDDTo6-@?5etcQWw0`rB z`=$dh(gq0^?jIRM$bhv~ddU$KLW#i0N;Gx~F&q?*!G#KVDejDqKA zdc~p#YZLULn$e{#Tl41W3fJ*Nqe}|=?rSip=Fm<0E%I9YuHRzpfV`nWy^gedI&9pS zIzzM_`~CdqH0>dqXCCZur8OI=+Ruuh`(-s2q|OM?t`dKLbXM7g&L@_1>%QvcIrld4 z8K(=3E#Qjmh=?d?9rx_g3QM5-T^PQQNR&P*_iQbiKfv#_if^U;l~$}lX`qP|lRdD- zc(wFiS-AMxR8?9LhodSd1+|A=A;&4yjC|2{Ev@i>frFc%;aCFib#GhU^d@-XVUcIb zp&V4Eca;QUJVotkd@SoZf0grII2dd~-=%9ZsjnwLfbC z$t=C_@WQozBMM-l>8no!QpK{Bu6GPRkJc%qt?SjsGBAt; zQaWIgu%1{@OJbN-^PAw8e1lk5i4h?dtb-^Umn@xV?uq=Z9o}$~8JwE)-~_)fZdHwg z3OSKcN7!&bWN41?d8uJXEbQ-7PbN%ohsSe-?J%%t1oxrhV0MjKO+{cWIf?8O84wcS zvPCaN*Cd1RC}hffDNX1~jVAzXTH;P~LCj&DL(A9{EZT9Y-DB6UTjyu&Y~y?x`V7UZ znmSR~(KGHe;0QbO>6Z*t8a$L$xtE_YU8K0(De1H;6gM*rzD5`IY@Ju1 zqAR{lW{|=k0(QV_$tOUGYk@F{53JTVIFGOq!^L?S`~g^6Y!Vk;lHn}pinv$&K6En! z(g!u|Id6AVRrLf4Sn9|Glq&f+vgbT1z7n&#Sw^PsF0ZLPctopA*)LoJ5nJY;_&~)X zN_?TyDhOK^KsmEATMy(e`4#=~iZlOywDwmhAIm>hIJgd4?%{r8O%r>yP!pJs8AKw~ z+_m?arqVLGO+qcRuEY~w_*;;nK5I%-6 zKknboSUxZa5{t)cSvA_l| z3JSVVafw$2MN0glJ_FX>>{XDx>P#|gkg-~(5=J@ncF7%8^_7EkVjIhmCFFgw z&m+xyFV2;Ahp2;$1T{nAB!nuZmGthOPu8HPH6-7V3WOuG9aM2Sl4WuZNs-0HHLo7X z$OOPUmOOo*(E>iK4~;$i72dvFJ77lmSuQV+zSN>^U5H%-j1ZDsJ5GC0u~+hNg{-Ob zx`x#{(*CsBX5%jQFGvfY{M)NfY13AXVuPmunjNCze@&j9!2}{i|NgRM+X0e(D=)PR zILiW**3P1B`#HbZTOlhlR}NRnu$;Hd&JpGqt8GV5mr0VlisS|)<0$H(U?d2F?>&7` z&HVWJoO0T29D?B{ao#j|Bvq2KIPcqKJLqx(W1|f421|%Pue#RS#4P2@59B;`??`8jvP|40BNKKI#_vV}i zr?0(~xF~`Umirk;J#0vTCG_?5^u#%tQl)L$i-g%wYab`=)`Z%&ej~CSb)ng zz!N9VT8scxQ3JkA)B!ka->#iG$(r=z>nH05X&2rrBO$xhI};%v08R36J&z!2&|Kgx zC~8rFcDv;c&m~levw##FUbv>sk46BVUWainLyiKRNI@k;hS3DwG(J{f)QzM^=l4 zN~{{Bf6ji|iLzJ@a=42EfvF@q#}kjiAtuNaG(_Me8aUeO_ikIFHSXxh$SOMW+tD4s z*c^{@PcXk-GDT6N|JtP6Trnb8n59+8@CvNLm!u-0ewmYVR7i|~xNacdVi3jG6JsYow1%BH5~X&LG^v}z?W~haw7-S2 zF5U9O&k0{ql!8)uv71G>sZ3orN}D#0RN+aBdSM$<Gt%LtzH{iWmo=7#Nsk%Vwet&SuKe)A z7l(xu$Vnb>dlSA~B;Crmv?2l(Q3MQ`Sq}oAwwYUWy=IrWR8HjFyNR(o_@xeJdRpeS z>RlOk9aXx4lC$xgV*xFn;tI<>JvLQ(###EJ zI3A4x2CQ3Jv0x>7hna6x+qtncoNl)9d4GUYy`K4i&AWOpnGKI9xs8v$k{4}WvLb@y zMpBQ$cVB(!5meF9s^dMRK*RPsbv9~K-R)XCg9uuRlE>=2)U$!&YOz*cMXENDuW|n8 zH|A!dRz#qCoT++ub{icQ7Y|aL$>q4FkL9`yp6a#}*jWlDi8vCVG2oH`n39PxGh0kg zg?}ODM?zhdwJzpc%F-2|+M!}DUiB8PR~p3^1xpXfhKL~IXn|qiIql-A-t}NLfsYN} z<=g(I>9i`rdPV*I*NA55PXUQzs%tDDhRfvyN8fN#ehD|*EntUYEF_gvxXO&(-VapWV zn{7Ghd)v8Ad(~yyqNRodPR+=y8k5xFVWonZTjsM=#N^+eH*C>-=in2aM{n!g_EybW z-Zq8(3kq9lX>C#}hYYknIH$$b>h(6QtZ-FdwxXs%gO0ZEBg+75{H(txaWRC_nO6#Q zb#ykJvs95Y93$IrW8q=h;cU;*CTDxDcI#SD|IgkPOVy{V-u-@7+7(*^+b5K5;G)b& zsXF>C#3sJ&YfZKijET+M%LQfG`H(o!{JMoiMi~VTO<2V>>lcq$+HMAAj8Rn>(}y+kE%k!Z(y~0pf_txT?hWV*6;xxfP=?UaFeKU=u zH75)+2=WFg=hX0;R_&f}UIfUdMwq#`oaBr?MV!|Mp z5?O=b`{e0uuvLPrZoyG04n`t`5eP#NG)e3ld@o)Y(3362xl%>cqGAmaXLcw{68I!d zf^!V@+FU+{C+)|w+?e{S`W~q)52+2Y1iz!?5y(q;bfh6(#_vSwLjD^y=Rr$}A`;Yr z3TH-}so%QLhhC;LoE%jUb)qw#4}enR!WLn>CrrbEq%**mfZ zxWOO%n4`eGc~l91bthtw+Kcz-!HyG8RmsdC1_*_F_Wiw0=5VS@ zMn|=An{t@8j~tXLLA&^0R=Z8L4LSnMtCFeEW+m({JZKM!D*vKPBuK3YrmhnZhY2%OqGJo3l}Z&RiOVeDEVG4Y+7lz z|8Gq!B{7||YmfTY47kHrXjZTK-?auznX>)$0G{>8kinXJZ9S9hQD<8HpLKL1muRk5 z|A>;6PEx-8^%(Wr^mqJRtJW;dkJY+KK-$2aXggHhlF-s`XgEG3xqgHWd$i4>@1MY) z7c;7V?RiE*Osv$XdcSy+`|lSOleFnSF7jl}|5Rpm{!bhCOaE!({^oyp!2kEnG5@_` zTK`=nTK~V?7}rxCR#U{1$_XhJ|(h^(lYW0{{s|VMnc>coe8Ezjla*GN;%DnNvE0 za!RL=!K&Q@g{eh-QO`ZBZtG!lA;c7x=rU}Xe(XqV4H?Ze)XJq@DEVSv+Bqjb6-ks&i7o}0)U&=^x~=hjYX^_-Ch|vKwHDCWRV5a6)CE3Iz6DevGP*` zzcJknxol}-NO_!6{LRqRb&jSf>FOrFHR2CKMs$RZ^msC8f?#vXL}sLfV*nw5o~X>oS7mCa zNyrfO?|JWO-Atpl)_M@gQu`!HFt@HZM}J17{YEjPh4a=X@`&cN zIPyHMm*NOFA})P3&@s7LTmMka@mSa!4xs{Mapt0uxHEBHtD4i*y$HY=7Qta0A(bSG6odo|)&jx9(bDA`H4kSx~M(v>(?9Mif$MtL4QOa z*PSi_g^AtaU42+vcf*5vI z5+`vg0n5naxF2vl^tXYQyFq!sZ)?XT$o9Dh?HPrcsKp>2MFc3cy zQ4vb74b;yN&lG|E?>WoVb+R7ueG=t=%2mkT$cRUXJP}5naQ7+lg_qCW5Jyg##=~A$ z6e!it^u3>X)WHq-S0Uu8?E*5p_@-jwl&t*+Kit$MAlsx+_gLSyFWR zx;|w4(^h|;C6)+$z!u}H;IC@kD3o0O5biQ^UtcW0eYO}VEUl3eM~ zqJH0_YX7QG`HShl{iieC)c=0Cd{E|fE9Rstl8@gxP) z&E6%Lz6ts^Gb#<1nE;S%B`T)L0asc7YPn-@3s)<~ZT}4yONN-JjiKa-j6ijl-&`LVQJ6EqKG)Cd~(Im1yGNK`?=~nM|L|XLZE@35tCV;VR zNkR5%dSq09H0>-t>{;7Z)ShhIJJf}OGI9{Wqh2X`U|N@vu{7|f6kIOaR=kWwDej@5 zuC{g&pi0EdUC5#pkR{Dlj;?NgfSoLi3JTt_AWqUIkp>E-(x#$G$I$FDS9oJFjErN_ zBolJDn)#2^|6v>V?R0R;_G^wxtw6^O^#ZMsV|9zYS@zQoMH71>Y`95T;xWqwMazZ^ zxjERonn@z1u1JrseEo1&7B}P6MSUp?IWm?(a>IGcX~)SlRKQqUbntr*OG-wqqIuW% z^K3AOs#G){aDEPfyTP5bphv%{=*E%V`#05)UJTrAW!RYObn?WFk5g{Lm?~C&(o_eP zc}TnU1>PByxS?W>i>cW2%Xi_vKIyrph8QJtry~#V+&TOH;|U1!<^y2c?R(tI|;Xg zd;psjA%iRfH!+FJWM-mBv+qSyK?{Wek+s^UB2;d@PpkDzb4?DL*d4a9agzr>yPikX zE3_4krDSP418g5h4<^0;z1d5E^r7MAKSN&z(M+b9XLwyg5&|H26wRt4&QZ(zCD<~{ zWfzP?5t{(7BBY&p^DmP)CKtxe&eVkv$_?3MTwn1?w)g$ zc=(Fv)VgW?QC!K06Hw2d#LpEISnS1scpcicYo}y&JHxh+0JV2tLA1*f9PYt<>-8vT z{rm4d0QPL7^9R$J1r`-Vf);8T+w_f<2D#R%vwqC=aYvzzNK00@KQR7;vr@SFynw#Y z*phPWXc?z4$b~0lesN9PwLu)&R>C6X|y^(WcBHaV= z))1>d_OF`oOoqdbn50UpZ5n(tp-e0*X@aKU%*qE3;lv1UY=NPlJ&EC4H8TU%bP zS!g%){oXgwp@jhqa~#y2>#zj7O(YD4kod-x?f$cdkaXxvp4BmOhpZ;8MJSDeq|4G2 zBkxzTK-fC~@B6yVPM90T{k0%xlTlW!?bJ^jFOQu2>7XIbAgi($r#cbEc+VXOEX!Vn zL4ep#0QQZj1(Amny-tUdCCop-N4iUDYRP~R|K#K}%HQ@5_=*rz?i<0gI>k*s%J|vI zF3{gZg^f$NnJg=w#wEb`VrUs{SH6weJ|f}re&Fa#xBN~7!&IlceumqP3$Q0V#Qw?y zYl0@W@;~p0WRIe)Lz@>G&Q5`Sn=B9ccA7NRh5Ny8bxB%XkZDHuT|5T%7}@_RU7h43 zRV_OL+rp^n=|Zw+^XvxxUE@H*PF+}1LXjZ6IC_{HZR4t0ZKnCObm>Q)F{~2FZoYQz z&o@i51ma~F?_5naLUK;PyMVK-^bTkVnQsk5#HmnvJ5Gfcj3NDKfff((;yc^#mr4Fa zKIpx-v`IIIzr`{3vPO;IITh!)%`)#oH7_t~FPlM`NKd}ymkb0n^g(0Ya>ABYyh{ho z521@BXgkBjo+`qcn(q}}Cap`Jsm)`S#Y==b&X=+)L^>^=s}Or8xj>qhVTxzS3-~lA z3LWEZq4!_EpxjOCF=nl2-@q+d*Q z;_cJdEzkH#HCSRF+&oHz(JHa(7BG{00&8W?XDI#so^Pcidas}u zW}hPg<(Hp-KAFE#%lj}YT6s`vwT2kpqWW;YS4IqxE~Y@k_wvPaM8p!LV?Ez;v8ljY z;Sm$CSC^^Zz*tZ$H2cRi>1PAep);@ltlPNuatDMMD=yVU`%todRS=>FqF&;~l1+tl8-wZb zr*`gR#6$at_$7om=JxsJE}32@CyYFnvrDS9JOZ;pA|fa6VWMw7Xs!*s;g6Wgo2%sb zrYOzW(7oX4rDGIUH!oPM!^B!DPyP|Nd?|t8M~3rr+C1*9sp+!PZ)Co(^c92Jz`#Ih zg42anDCPffd>~qqn(?Jhug+`*X=ocW@fb|?@$BK%W5$xl z8p9M`h7@+Au3jn~a_AiD>@9US$KG+65z=@}>B*ZUi9*VC^j@LFYcT)GcH-1UI2z9a zSJM{w`e5E7GzK|1AtH&MOwJJFqbxxY(1@2wl!&rf=;u}@crWemUUGnSD@Q}Zsvr}# z%s8nj-)9Y!_Tt5>zComNPS8L^hnCU^Gd^evruU!fa$(0=v68;x05BHHI(E9dI==eJ zbenLi(O-($FK70gqE%OjUqXZ(rPaHQ-xyl@wsGuP-)U>gm&q=p)gWE<^OfEYo=+u8 zxVpKC_r(`l)#119i1QOC{<-VbB1n8CMpNCTe?vme;aZZ`R1cvEWMa&Z)1f z*R#ij{WlNKs@}yP>4^Y{C?4fWhoAI{SE-Cg9paiyk0eLt_Y zpO4okpZ5QJ@_C&XGj)S&Bcp=Cj!T4oqxW||YwZdT8ib#vL8!jDrjLI%P}J`ibp7;S zmwwjMT~W6m9ljr*#tiDDtz!mt`)6=S+>MRamcPCGePrWzzq?&|Mjn0TCRP*V3{-$h^kyFC9Fm*+pP zbF0)7F@b1k$Y9#qdkHZX(m^h?t%a4Q3NhP6=gKLW2f`$n1I{htK0<1G?5}L~vEgZr zvoEz3hhTRKjk`xZknjjaH67iO*y}CAQTz0+jD|t$!lro3sZC%U=GmI6mZUkOV#QdE&Wa6@Gf8^jRfZO)DWto z5K9r?Ec@_6DWZLnS*Z2;g)Ve`k_znOsce@Y1L9K2&UXS}{Th%L7L z)-oDc6_lSMBkncwnp7T$TKq?IL*~rtxn0(DP8j|s_~GLUlbOrsxw^cm*1NaCmxY#B zaw{sHIhV9e`&_U?%c$A927rU34RiZ9F8Ws3wLnj|=v!XQP{JLjky0)nF*JeUS%641L8tIMVwos z=GAzNb2bCN*Uc0+k%W#Hs6{Ux1j@z9Z1+7e-cNb}LD0$`0%a&Kj zYGStE_W5UZP&TZ$!fh8&2Ph0lTjR&pQ?CT`>T=~1b+L5uD)RL%>8HsIPwr?xSchG_ z17V+3Td+{#UPpuJ6_31)d6Z&>#uF=M^vujx1<#`U-~UF5lI~^kPP4RGKi&MYz^%Bw z<|@kB2BB-{ijaxg&{aZ_a`8o_2I!HJbaqFZQBCp2Fo#{oVuZ*uLPZt%ymFtYKTKcA z3w#tv>mFTn9;gz!xX0JRI8_jBi9SIZnf-J<>eKP~))^=TohBxPT_wwQWC4iv3~Eyf zd~EQk)GTS{j8awZ!wrj;(S8&=LTo9Msk0HM7>Xhbu9ehZbbG~hkTz+`!&&O?ji;gn z=kr`Bj!A@_VrII9fN%)1ye#QDl=%qQfR8WFrs7~O#t~}R?TX5XrIYR8WwfpHbEW8E zg|RK_+=dMel-NnzP|_GPh7nLa{#rv#>u=R3XwxZ(O2jwTsR;fmC7Ur~-QIm4&##>h zrY*(^ga7>XAx9vUM`pRU1u{OV*=f%kEJf56K~>NCOHhF7}Ksjdl)ncf5Z*kIse z7V)+vPoVjTMZfeT=%8AD)P$7**G(J#M>odj!^u*8_1e+=@qe9M6Fd_BJ)ETi%%EUYIyPzBr_fsS#(^s%K zl4^Fje1OWu+*<*KwyZ>tz)CaHy8IGyTuk zNyog26Jb5kok+JxXl+$HRDE-dx?L=#6v3!1UNb=xsY+f}h*lZ)dWno*>%nXw)cocu ze3p$lXJj{BOZh2&4r@w8(tM_r#lUBRdpw-=({f{+%B~dM?+Z0HFeT^3iy$^<2mn~^ zuwP@6+Wh|eAQ9ytrJ0v;8vP3EIrqY=z$Ig5d6qMxU2IWAz%Jkl<_2m&1j6iLi>hyO zh(;o75LNze?;$j6B4~kz0{YZGQQN$q9LS*P20{Z)?W+Vj ziGPeR6#;TZ*Wdr^`}J_S+u3_@P8Z~U&@vtBWh5PAZ1!4-n+t44d5uua;f&AynEP;B z=ej4iFkROf`B`3LNvvMJe0hYx^|gV{z(~bO_Ksk>w`SlZNjq9XaEec2IkPp18Z>iA^vF zq2Zx=*F^#Qau}v+_K!Q6?tnZhC{(^U)UZHWOszeAg;;=+*zKSLClHQ+RbED*Xntz{ zpw3fvzY2Wr|D5KxSfLRW&{dH(7!T$M zDmO8KZx73z^In?Gs1SzGDDqxbqtp8{LbeyF1E)nE!8#2rJl*Ig%Wx76S^_pS{r(md z8L&yI;{DRg$JJ*pkV@?nAJVUl)-fMn=?)|>CvUqLy~7#ESFE~0@J8{3Qd!L{Yz=*u z;&~IsO}vXaD~DO89*S^>eZGYDN zy%U89#ofYpHb$;h1fwchj~!s{BAXK?5}5&5YH!#=%3kxIEVTB2@arIMHT$8nQUuVW z%D}bH(5n)}p8K07gKva_jim^~dlnBkG9Yo&XZu_xX1GKcDEUTVVi0G00fXC#W{Zs~ z*mk1?pAbRVRPXG1*l^U_2yAGeq;)qkm7b&QIKHVt)22-qbe+L&ka5o~rWpE|6izMB zqnE|TwWh%>>Q?fbc2ov#^yOt1LJjL*^rK{M)w1q4)~tCo5^1nB)8GsPS9g{Uw*)jw zfm?qbHh()awK4toc^Tq8(-3nzImJarK<^jqaVm$*)z){kRX1rK$ec9s3?zD4ff2fV zVc{H?r^`RJL+UK_5|a$p17nD`_S)SQAp)Y6Yu9#Ev3!T5JazP?iYuA>H5N=_kJ!Fa zpxbTE{V*&Ig@E=`O21}58|aVUbYUSIw14guQGHD#j`%bH@}NOJhwKIjq&g4-RP!QA*L;5YoAYq) z&Voh+t)1K+CXzN%zi4Vzl}QEnU)TNT-_UpsM3 zIEXDp{sJ*PzeoEaERuFu50w|rGE$gS4Xs}96^kqN;x(4_R-;(IPLos3yRj`IoAgB= z+&Z3q+AWM&`W~31Ux~>d1#4*CjwIY~kNiERP>gRC{v39bT(D&5hH+$1=U?H{ zgjWsKuD1ZT)-V;Oe?7$l44&hkObG3!8cE*m(r+@w<~CmkTGI)1!dloCCv734m`NbK z>EulKVcQ)h+T^k=Q^|tFn?)32B30na!y{3roeZpdW63q23@Oc3A8B?uQgev8n(Q-9 zmphLiKW;jKQbLLgl}xvhPI}(!eSrmHEPsGDXj*_3hd*=59r;_TjRtKOn7Rj zFy>`&=Q`l0lAkXQyBNU7Pi^`BCWn;ERgTJw%v?VtW1o4I^lwD-hM-AW{(Sq{SW;X& zTAkZ(7WebqxTWCBdye)*`V{SWk>rsw#0*s)9r`fNM!xq(Q?%;^?dg+>Sr+yts^G=? zd9vxndr=|Nk>OY-Q#lY@T~I$J&z{F^=iCY+NG?3q(JGv7j&z+1OE&)T$E`_rU1N{a z7kX1~-7y>T7A6SHJRK_|P$3RZv^z$ZOjLrc?^F$Zv?i@_b={U>1b0JLfOK4+Jb5Bm z2H1!b?AMctVu{h65tRX5G{dI)uLDS9wl@8)?7l$YsSkn_a4g_K2n)1u1%UOzK1 zbT3*mhu~)BUi??#HNdj8eD6$r&c|#l@^(@%dnt*>0I9RVxiU)sgx7Mj|)jM~iJFyY+0|Do%GX{{I7~VKNc% z&yo#Q-~6v@Jk8-cHEW7nJC0+XwjOW?7+<>K1oMq)z7lyy(|CVP{uAc&jIAN{=B8)8 z&0CP~i^OK?RrlUD&xipO=6&{I8pCUL0yR-eG+@nW7b8720 zH~Bo5)vw~T>j{Mk`p2ip)DE70StJr-FF=xY9GAhk&rP;K{1Ud(>DaEEZ25I#i3VIn$b7g6TDD1En7uNt^hoPGr(G;Wi zDltHG_V6-AQwwF2=>PR##8w@Ho2FGw+W1WOdWia2jA<^V=_+=hE<9f8ZiF4mQ9eNk zT*#IOghjF{)J74kSYcQJEy@&9A%BT5o{Sv;C6(ut{bbQE=9pD~TO~FcVix!z=U%e~ zb>henn0Sp1u=(D!GI3ntzG~*8Ip5Q=iXhQ-Ce>+b=C*Y2HSN4_Wu0))aIUf8{OVT% zagssbp&yB@=Zv_`Yj0kl=93~8+^_?@S2?QX5m8fd@>!@P{gIOu5KLyq5`(g**nh&! z#ldSVLB{2sLfdK&G;tvT5Lm`@;7dx1pGoh5y*fc7Nt(K8cdT zN~>BnK0#QFQ6|vQfBx0WKwm#5!*B*qO$JSQ8A(niCI<|A7ei) z3FDl4;V^|jE)X7=kJh*Lbah8wH%wJnTx_fw zNwJi`(#~#viHe*fCt>$P1{Ys>EnCjNT3#}lX6t-~KWToS_gWk+^R>&DU7jsA5n z)kto+{nq|O1z!j5#~h(QD*Soi0Z{jx6Ig)>1f=anJv`%1IVQEq@0|*l?~H%+qTU#y z`#NxD-vcrUE8(;6I=~+@??#|HyCPl7Dw0wu07mPiG%wxBts9dB`8q%!n0vam5pY5_ z=1QQ}2^5Q}!}GE>Y1gO`z#Ez=V=*UWuE?Di5B5KEi-7HKyaa&8`74ABFJg8}e!n>t)M~v100qZ` zN46GXD8!3lDjkN|Gk}@njg2W;JuW>)s3qwCXOcf7;_*t^@638yL zMUNo+&3k;d)BUt_PAPZX4iC})DOkn@k0N@oJdu;uuUmJ9mr)L(6QQVSEONm6X+jD4 z();rQ{h!Qh$+ao`38!s78MAUupV$Ok;zY$et5c)o<700iVtuf!VO8_Dpe?iH2I5QA|iYs8`@OS-EwzRzF)Ef%#Ou#WMS69?|)RG-u=AU6Q<0%?Up%0<3O7N`XAtH zj#4e8RzU|Y=ktnB0(nG!Ar+llZ65lbdw^IjUkJZ*py#<{<6{iwGK#dOo?mtC=6O8T z*n9Wx(QCN1Cn>fu9)%5^Gw0m)xhAG6k+CELF&B$A4ya-kTAI1)!Ntj)+PA-RGy?Op zqi`}~8{4C$9SOlRr57@o?B~x1@sO?Xt1IsNjUqAfOD~rU2Wojoq!6=6K<%UKuDGR4 znz#DB9YiG^g_f%rfiza;5dU<3X~R0olu(vaI*Rq>(&*D#zjm{GJ!EvpZs3Wy*Rr9C zSC~*2_UzjR>~0TdcL2F~Gm=gDWt_kdYU`L6l3}u$NK*Dt45ojCbJ!GE%Q;h6yr5NW zQ;j6@l)LErcqX^cDe@&r*PzOZixKT#?Q-I6Z!zO4t z(isO=^}f~srv^oz4)2b>P*|=0><59EO_`D2u5io-?;@S-C-n ztFs~WoX#9<_UT%vQEW%Mt53|k$@cGO52w%j-qe@uYKwQOwF)DcY^~#QB~3f(!G#=U zwq)d>8ldN=W1ql)_F}3FjOTmq;2u3^jC!OiPX^o0Zm4x4uYLnIA(f83b?cTNVi2rA zOio&5#mh*XVXBo3;+t74H0n=fWdRs8?U56XjQqa3d|e_a4gs}?Y9t5$%wl-DiI{cz z6NmgEdD+&iinJ{1-gy$oCa{^S=$NO#_?xBhN0F86bcrf57!WY3zBvv#vK7O6jUkm< zI{nsw=llAv$=vgR&CxyfN%Xk3IVW18jwy<5wqQ4jv~4zKo^P{4elo2^VlyAGe}+f- z^sWa9zeh=_Qd{rvOm@PcV3b^WlJmlhTY?B7lzawL&rguW%CO+9>q6U2$MZgzNYB8xwbhdw7~))_Hia}Rt+vfsR!2N*Id z%B=CozdrW@-~d%sj3(JGKA69La#e&OH{kUr%^o|COp1)Bu&~u%>#g6C`KAud0`zo> z+TUl{or?p!R@xNQ&~vX-&}hn=rt6Zi?_b1OOz0LfCCHF}6_30m&+IE18kN+o9?IE} z#A`$Ky-bb#NPF=xuf{VJ{^Vb~1sBK$h;UXk0p-Q#y~*$$HM0yKGG`u7pC$okYpmBX zT45>DN{}q`%hVf_^-Rq^9n1$54}V&h9A=K!g*DxM6hNZ2c&}m;W`?X5^o2%LbW!T# z^zF8j7?{|5MYgQbXoA*Ov`4O84edjD1XuR&+am{`_vj z+#Zp}=%jLkY@;6wYEtfg)AU3Q*GZB?z#g?-5iG^2DRw<#X1i#fKG(&xn6 zb$qYQ%rR$<`+A|}ryuk?9r$4NmyeF(Cf^kQs3j=HZ0+q^zC6voebKH-%(!_8yNx)E zI@!b!UtXsc&t?@hw{GFHRF3>=^(9@AYEp7Cg+G5c(R=7-ovv)WTc1so`cbI$qt511 zNnde>*8wu?f~{M1x|*Kv+vWWk+A4oiMsHg-m3pbAWV#Y@mt-F^ij{l< zu6YyXY`QG9LmHc(N70x>R%z=!z&NMk#4An{7fyk#KtYCM4-E;6`$;mhFQ3md4s`L3 zZ50a}Wod!@0^|E_YzN;)rtlFtL^sqf)BgOKCL~e}I;&#kFtA z-Gx4^Pzpwf14J-|rA#!!!Y5jg919i8L+zNv5j*BXquPo{BBF?LFuPCWVF%F??SQhD z@-3NTpGJ&<@(#ArNKEWPx)%8Ek-8Za@FBHxSWhB@kUj;U6fgOE^YI<{(z0+>H4 zHa1~Z>Tf=Kn%IBj+MS53J)@($JKR1X>R%>=r&0S<4oAx80ql&3SP+(dGS)NscUm{3 za>=3@p>n@PQ; zs3u@~J4W_9zKlUxfa{2`I8$WM0{&oP@hnDj(q05yb;d<{n@L0cYhnQ>ZEhZ~P-;Zh zMXGqWaciC%_Ey7KHpzhW+&nE2cL^X(f$iJ^0ns2%Uq6MoXl?7$P=DZ{!GnvD%G`Rq z9}0tXcF~0i^Y^$=o~1CCXu-guiP=2=Ia9mNMsD4EL=9E4RHyVq1c;Y@1G#+!MPY`Q zWco5!L+qVQOdkDmz$}-rFNU*(QjEbmoLU_F;t%Jgh*V>@;|41tL< zs}1Lzux#?ZX}EFkq!lCfqaCBFnoouCWK7A1og>a>6EZV11ulu=(c9W1_BL3tVukb8 zgG5wmD#3u4NxVWf7wDToVL-q8o*b*9cm!}u^F&XODaxO51R;@%by&C^PZ;spm1a8K z&j_Br^fZO_M0pvW%}F*av0OJ2aJ>2A&I1PRGg&l0nSAo(gn! zwTq4g;1Vc0`>h8QPC9YxxZg44@r`_sF$g9)x*9~I{2fkyi1hpa>25i`N`dL zmfk9sQ6xHheqSYTFeR#HDY*G?g z9koYlYgZ|(xiR)zpMfd(I>;I{E4shC>_Hr;^L}3k>GgZf(K5Y?5_M9MJF7jGBtm@j zQkNf$ZCr$UurbiC^H*}(u&eBV$=;_$v$r^~il;T3F!3^Mx5BGw;*5-j>+H%=T)-$A zk71>AQAp%KJvsL!nu%52-9Np5(T|7Ep1*XdnQ}IjSG={^ zdxx2w>)7}E%R@DN5bfOJghiw}Mclr}2d!dOGr)6(*`;$pgRJoGuqLj5WJ0=mH`18- zVQE7sd1;qXb5IJ+h6;I<%WFfUh<*<`&kKa!s`sm#@2*i*2U6;a5E3L3fg9HbyUT0!-=ZlED*<$J7vk&se^$dU(`TI}_L;_mT=m{rvG z<*PNI0Pep4PHfXXK*oE(%ulbWSe*p-C?I;DZuKb@`8>^|ppfK^A3k&=aw%iGYqb1s>$)O5y@IRO#gW8Wooh`3A*$-#Zo?H(n3 zSU_t4dl# z?lm#8c?JkOigRzPV!PR@#CtGtm`w-wKV0?CK0;hYOC=c zR{Ny>eLGhz* zicC`BPfif~nnWp7s;aOtb_p9Tc|OMAg>8L2t;NeXmT3@DC#Z{jQGrMOY)2Yhzfq&v zNwk}0Uu8@w?#G8|7q^r<7&*;zaKJ240{w4DyNmHuuXFAfN^ROQR%=qu1Io-2!$GlZ77vh z$f2APk(wl@G>k)uB1|VDBU6)7sDz~FeJ!(p&+GO4_dI_+`;YzmZL+%8eSh!k@cDeM z&t;c2wwRn`!Ke3cmVeb@-i^f;X&_ZwjlQR5F<^~nW$~4RPrlW%`UB|c zbo*%~KFDB7SRSU>3$^1GF!B77#|x+sLSLA5v>iYneAf4m?e`FDNcXn> zA4mtK(wmSjlRYkuJOb=8ef15dlXJja%J@oH2FFs82`4$n1PSMi3&ayMl3^65Q~z=-ZB@#D za^@+!$J=zW1QFpGk0>ST#-BRYeR8^fud8>t8^-|5We81{QIr@ECk;jGz|}euvTCXQ zdi(4e`9LI9_}!ny3yw@{b=MpNa0hBOvEU^+*;DhJD`)f1B}OMB4E}1qjgrG!RK)->!FBuSuolAa zPN4c9;H;zAAr&p0;ztV3I=IdBmj~H@j6LvaUg<@Y*+`_XJ}f0C$QZn$9CGQ+=SC}J zltKwo7%mk39fx25jrD0z2+>c9z!M^=r-w(`&i1|23)w7?MYQ+hHA;Y_%AP(H#WPu7a zwyo6cNwXvoWK%{AyQ8hnqJ^&mFA`0vBUxNkx0-26kvsAi;#Jhj>6aV7H0X4)oyXKQ z%Rj0Enu1mbQ$|Ay_eUKdgO|jPNeOSYQf+Ttz6aaGtiG|5Jek9lFZt-!cjTfo%n6JA z?BJSr|ER(`e$m2lpxi>O+SM4v6>>HP@tF3HyvCuZ177)?QX2X7Y4K12JQo7pnbxBY z!0!a`3ZTK76^wq0Y3}p33R_viBw|$_a7cx}ma-?QP}EPs=49kWmrGE(#PFjN)MAT5 z`S!x>TiX@l5(6|T^Q`@5T^{;$xeT-lv#rGWd`|p^r(Qi$ez~~ez(7xRZYdlx!vtf6 zX%>nU(9a8f8#g{YTo;UNk?<>$jdX2gx!TLT)2FDU#pF?DFvUIzd8QZ@bD<9)Dhyo* zy?!BUH^sE4vu#)mu=)hGIOo59{rW}UZ)Zpatj!`}%tkDvEXkbK zdyWgCLFAy59)ros3NKkUK_5Dm^InTe?r&1yxs9H%vquZAwvC%cy;AbZXZ31XAg56B zVoXdx$NJa2YwHfKf45gdJYB%oQ${*uOSRF*=(e`yMwinopQN4#HSJPe05~DOkfdUH zV(L?4x@XGo6y(p--{}XL{=-BdjyoG&L}cB`r~>9PPh*#$2NHWInfHdWk_C{AzN+>g2MFicTXZ^#_*WdFQA)O-*;nqCU9Cb4HUq~4^`zjSAN{dC)JhK}Py_cE6yTCzvAp&RNr~TiY0SL}_xmPq3oFYGn~HiYz}` z$ZJ2?(!@62{8{0@l4o24JT58t@pS=JQKkV>1^PB;FjdW35$f02rcLc_MD*Gbmnv~+ z2m1}G8483Fp}_OmX_3Gmml~{)U;6riD9maA@ew%i|Y)3meWD zj0mB`cvN0e0;)~z+`2`Z->}J<@@p>a|+= ziKZosUy`?xnK^iz9MZ3=>mKatLh{U?%;X@1O*;!piv%#X$xlb6A z{Dnt&2eAYl$s&~;;$`%qe6a#Sk?@E@Hrro2rukR4%|Y1)Yjk1RSNQRQV6>%!sNAV| zIg_1-t%zyezWsR(Po%*k z4F)-*5hB{0;3Ab0C7NQGeX})vao8$ltM#v?m!Vswqf*>qjBe zS}2*ZULvR^Xg~YWvOv-f{ymf?-yB`_IQ3rm)d!tgE0TCyIxkyR?V8`fJ9qA4!nS(= zM?m@1{Cf%P?kDs|ERhyTl_nFzly zp?kS>E!ZJ+{{H*puKAxN(&m_xvS9z|NFQCCt}v$BJsUaXgW67*YSQK;-nG;1Pf|6^ zwXmVo;=NAy+kvpell@;{S%)>;sU0`h&}YC^l~f&ZGtMVu?(X+^^rqIrQa2x_wj;I5 z2g#z__z}M$ol<_*)iQz@qY@N|CYjB7x&Ywq0G)x)7639lKhUuJLvhpR@xsZ05}LAI zNK<}B9%`>DSwR8f$X0Rqpc7L|wuQD(Y()c7QSVFRmo35q(w5iER1%!!sElnN3imP} z7}e*iu<3G_NoQyQXhP)6ATqfPGFNMBu0PlNWn(Kf^h`!*|Y_X+iarH#J5{k<_)>_0+D?VtwQ>2A|dnXHKam#rVE z>HLowR6{?2gAAZ7J->9plDQwT6vH_nslfAD&^jI|4| z&LQ;o782Fe1T67kn*ZwkPjmw#;%OsdX*^3Qg+kkE`bPNG8LMxmf5M)~f*@Sd!ZcRflF2y15iY^3$e#H>l}$)STti03EG z<~1t$Mt}2`Eqi5nG3ZVEr{;H-jd3hwu5^+SAPHK*5#{LpWjI)6BN@ zEdN)7dS&uT^md-j`zDN$>6Akw`I23_XvjSTh5P@Q-+YbEPmucfAk&dTJ3{&|Eu!sj zZ}YzUQCEk7D<+(MZEqMO&s5k{H&ve;gww+v}yX}gDkgl z=%(b66Lf0|{?k--p(79g;F}#VQZ{E(1`o2qn=uFUm}jWscB8s@NjG!bc7W_0n$$qb zo&=#BKU0rwUS7bw%W4_|s2oILF@2N%phZqA_O!R!&C)NgCw&v1y(UKP5JgJ9w&5~L zm{LktyO#szj!$SFokQNIi0j?aWXebkGq&YcQy+tby`(CBV{{$yHa1`8w*= z&mgfd6-@V>AVJ8;fd~kxD&qLQVgp$xf?Fat&0G{5?!RGo8FIRM|5yu+`6R?UWs>;^b?;N8yW3;;VS;^qHv>< zRUax5&Z&H76UPdL9t039(6)i)8!BAp(LmsQ@}K`rci{x;}8Bxr{EnERU#y^s3nuLz#-&Mz)@w88#U?+1HBh z)BO1r3MFY(E?uD{W!|Ue`ZwjD435B@<#8gXkpYF{`q-9*v@{TISOLZ*J%2YAiyjO~mg> zNE|kD_y*}NLX=z(IN76C{D~Mgk(yq zq;fU3@S&mY^Be5Vzt`2h**tnAoe$p<-F63$05CSu%f2eUQXJdJ^pY&;E}2=R?GO(3 zDr;fLO$oX0_%$a1 zd-S4pfU>375?3z~r@fM8^{{^Zl$=`43TX(X@w-|p9#Q*E6PZ`W=2pC&*`@6s5&UH* z4{3_|I#UmBNgp%t|EsqL?&atr?QLoro;5At1=}b6G;_(p4rh3!aR6{b)8QAt zvQJnSKCNK!jQ&c9$7CkIVt6XE_+FVayM(#(LEw$05k)4w_ye=*tCT$r-w{WfgyT4KdVL5F=%g@vXtkLuf-iPZj z@!U*%*uTnR^de{8LJ31uOb=x)h)3B6$`w`jsQ7+q!aW0@+s@n3%>E!W@bu7%a)wym zRXypW{7Ai^dXnRIWK3(Nj7x^6jO%V{{W5(^PEHOT$cV<;aEYzxdAu9LZ+|->aHfR) z();bm4AbzB6ljS(4ORnlv|>6T)Rrx&b?FP?Ys4g(7vhNIVZVsc|M*U{a%p2V4Dp%fr*h~uX6GnsrGxFz7>x-nQ$OdI>Jm1Vr6&5)OKXlRo=%&K1T&SjX=eM)RG8u z)i{O(+~|b24f;1~-A7yYps*PiI$b4tU{Qjj-*D@3=R%&ol;q0ot43>wb&#z?d=;0S zJ;!3Lm^O5EabqINjLxdX3H1T8brx$5=tHh({_)LDAyb-HzWjacZFnvu(Yb_>ZhrY4 z!=^LoXsf0ro~LVz&*pz*&{Q-Il*oc}7GCOVPxrV~5~57}fx1pA=I72Xc|2H*{1`St zRkUV+JYRMKouGV6w}gLnlSw*R8OdXnC_)zgRq1t*#fuOxyA{m(?<)JV$5{4MIgxQ> zp{BUpBB_^|Q}i#ht>31!C#mV%o?xRFM(&v@?E@7x#Z#_r`gybLI0sD#>D%5`CXiUO z7e;Bp7jT80kWHZ3QI;-*Bv}S_6fz@==%-K;nvy+3w{I7t0``Dg(-ZwjLh>E6_Ns0L zj+azIxm&CBqlkMltRrYT0k@ZO--v+UvLlJ4GEJ<$(QD(*BZ7QU88bj~QWhTZ7#C8I z@at~(+SHm+zB0#k6O@bb$mbOyxGDfx4PJ}%>#QVVt zM5yybr1+5cbFHqS;j2S0wHj_kTc;ufO7Zd#(I!)-o__BtJrxX<0Z@6b+;&w}NSfAk zu2~dm%P4%_=DZOWmMcD6x{fM4i&r&^1ib?-gj@Z_ae=&f1ykf zEzqt%Kb#nV1IeUq9gMRGx!Ll$ioNI@B58&c4?AnEaR!&oU~~*Qs7OP>F=!*0N|7)0 z2elAQhKBTVzk8st^6~k53$jSX2%_e6!-Z`yHJVWLqH>fXn=}wfr0c5eO$tuh&@M2d z3fa)Pr(T$%{2`ME&b2c4rXjb*7Q!yKg?%i?`ir1lGT>fv zqll$;x*XREeVu9z#s zKFzRH;6ro!zHu|(^DE08K=w`Yk}uQIi*_Q7&ZphUwrkd<>kF1lj&o*wK@YSGifa}% zJ6*|#=j&(EfCx0tjg(p8!J4sbIz6ltdizt$@8l-Nfy2oJ7{eQ$HCFFr>nsX{e2%Tr z@U6h$C(#ijv{D;(&?=C{N3sxFN;nFXR+frqal#S+BKP`lQP-9kM^flqUh1+K4X zNAtlU3XGe@QB}Ig7G-J_nUoaOk!W)SGna_=TNN0LjFdu5G~u0S^wg)ZY5Au&g@x?~ zmnGdaxb8CfPQKgjAV#Je`T5wfV{yFkLJY`ehXPfoTjKor^QEyM5#>VnE;A(I5i10z zUu>h|>-#Yef4bN)%)9!qNwao7I4^@OjHXlJThzBy1^4X~l&sPkdsqJ`?SDusR7-A3=Hy{#JVZ( z4H_7@tpY!z1K7w~~f6s|Cu-@z!zT;muES zJ&Rg^hdYVDl1u21uK|ROqa^DCM%Q`7R|dgHw6T`y{#}4`q&L_Fv)^_A$CK?^>lIN> zz@q)TyxunmEft8X;ih#)wq}vK?8Di-`NrtO9YV@y`E{d~LIM?~>*w z+kLH9#2iR(VroS_!CU;TIn*cdC|L;0DpRlu$@bDgonf0K$Wos}knE-Wga5!*4&m^*PqqdS8oJS*gn$~I&xjm0R zl5Pxo-=w@g|BLVV+v5y=$R`S$4%=`Nl5oPriQx(Y$wUbX79hJthn3%sQ#%^b2t@O1!l!JQjEFNf* zT+X=m9h~pr=qRp1$^+&__G{M5-3e=T{>tqhGq~^+)@;+J&*yA zSAGc!a-=g*(7ey$#piFebmF*)5~3tJ1UB3h*I4@EOvWOq0tf+mJ+_PORiM=chB(B~ zhx;YL%@ul&Iy;tIXu=9NV8hkE3i-qf6>|Rxq`Suu6WgcjOOGT0+1Yuww1eY~*4`*g znzH%J&Hg-w-SmKNE8oAK%g4?xNcISg_Iqh6^WeO!U4EDoD3xN5&hpWE4|wGhh37Fg z-m+10OQC`llzL3Vqmdqc*XvhNkJ^oz`o-Gn%O7}u{eS%HDG~quN%_A98#Te; zzkl@XUwZ!-?SA=_t%poBPyV03AF4f8vy9}Q|M~y)tG)Li??@$w$b7BG_%9#7I*Eh# z;p4~QmS1T-Ga4)3V0GBuy>Wsi@wPhCNr+803m*u=|+cs$k0gSqHFD1Is7rF?e4FgX1_Iu&CZ(2!-$TjeQ%PLK@w2E?z4zTtk z@X9HAof9a11pCTFa|u57(Lk;d{MgOv$f$O%f$yCIs8Wpap9eTx4r zYXn1JA3{YiBa^#~_F0x`->~kFyq&8kF=(UpOm8L8qx*NFa3Hpc?T`SQIR9Ki<1Yb` z_+3sHG>z9I%J9n?zE9b*Fr80uWM7Y5z~;@`$BrKtXBko3bi`>0-pC_^1M2PSDoOH5(+Rm%H8#66wQXq+CbHG=42 zeK4#GyndhX5cm^nrI~7=i__~88v#0HVWT)L&}j)L&Cpo_!LcB!3GBv`6z9j(uY5Hi z7}?`MNw_1~_i;K$osh|20CG;Swap<2 zi`C*<9Ipw>L=oqNB~3ED5BVo$3_@Do6Ls{O=dCHOUiH7|5pXOxxUFWN73?&?FoU3R z^cP;VZ~T6TkPiS~dv4CVFY$pPmJ|HGw>354RGgG6r})dqPs(@{QVB~rMMRB*UlMUj z2VGtAtJCqxhRQ45Tse1Q{h-#vRwC%;yvE@K{E&Nh?KP2PRy$fMXlTAKM-@j|u zF@*Jh?;LM5mA*<&GrhVjv<3}KN-d=$kgEzX%QBoVi7BM;1#iN;3jzj-&}mU34-ncrew~8Gq67}BZCLyB&3Q>1?!GE(!!*S(n;}<*kz#atT`K)b_j({j zCjEueO_i+@CSs?1{T-@2)@0pSuwlW1SQ)Q-@_~1E2yfazb(V}PdEtcs!z&P7Ho~h` zy)q3L4xCa**vBS?0z-_a9?u+tWw=2Dp6BNI>lJ(H%sQFnNAZ)lYzKhJt=)GpBElW0 zFQVcz7bc?N!CvUd(2z3Btx9rT5s$<%Yk4&dxg9Xjt15HqMD&*>-I{(#*Y|;IwtRZ! z{rmUL%QO*3FO4UO);02n-&vE@$KGqefB|+;#6If>_oWWmwqr+%TX_X{e!@tED+H8 zR?mG`hS5wHRj?bZTf*9mAIjnEDPbR9rn1}LCO7X!bIYb*S{Vm#Lr3>>A2m`(}0C8{OLwiU1X zhRtr7x%(BUf(vlBs4<}S1$pkDR;yN&k7)W3%0S%QDA3I5AT3f}a4pjoFOKJ**YeN2 zn2?cSpT3>$%vM(c&MU1En_p3ee z;K2jEu3b-2f|>Q|H5y?2{9em7ujjCjXPOj)#AMp9LBa~iH8C5p$ReMuMCtkrDr?ZP zo~qzFsfk9oyH~y0nxiIyL=ez%*@#MX4O7gd(HF+wQ$PZ>tE4+ zE6dBvrNWkm!MJztmwP>;liXxsxwIoE#TyCju>1w*Q^fML!>U(0Zm!(BC8()iq8Mvu zWOagcQ~@U;Q%qlqrg;`jgC|K!5dVJmS3mh!_a=8lG za4w%iUtw%)?8)^>|ChqPHCW2;Gam8za=*nuSf=SISH-DKx{Y<~E{m2T${%$Xan#Pu z%}u0o&!0b!pxP(KOAFg=!~4V89$eAXbbWXfr8@h=HPULE11+=y*DarAO|}XM081N0 z%)0ZYYgMP9o5I_UU$vfy^2*obmJlq=wZW<6CGl!}VQPn!qJhc2(=^rS3QM_n=b?L*T)c?RZuKE_lz?(^ptMme&lO`Ma| zYIVG-&`<}>afIqKKn?y&h`1{XXzms&I-@A{!92~tONK5Vn)aAMS-jI-Ak!f;Huz{l zzyZF2b!k$Dkp<~!ut!^B(2S>=6L)2NUIEzqa!~SBhfLgVp zfEN?um}qZMV_{)22}R!G;l}O=uLi9C*#GdSQp%GNe^V-6HRQTwl#SG>rkaj{w(#U? z(Q957>lR5jKoaT=i35w#g(##&p#Ki%U4Sb3>-qDZU>Q3MEuGcz)mE2rxW#3!JAYJH zT|6S=GdhqdRM$FkeHEG%CAwi>KFdRRKXoTk!!TJMZOLahAQcwU!G9ave7;DiLSlXPF1`BdpM#@pS~}p zYYD(;#Lk_Ec}8L&+V0*oSNP~X3Rx9MJ2LF9z7V$Y@qJVMXY}nvQ`0NseYblvW16DO z3oOUW%pMYLolh1ozUT5kXv+Mx&mcz7Z<#=?(b48IEuqI262iVjYV}Hj7<)aJu`x5t zZvKJ=W_|ihv zQ_*=e9be4lgF$+OUFJUd+@|a5;i_NZ>CT^Lzs-u^rZ>^HA=au}QgXe$o?S>RzR;Cl zDT9e3Q;=t;R3ooRY5tiC5aglp;l0dRJ|EuBpYQHo(9iCDNkgN;t^Kzv{D1k4^ZjJ~ z+wjxf5p2A`wUMO6E&5z;Gj>;>A>a|}7%I+kW?_8_Y)}fSW4PgkY0-~|N(6ZJ^gGZd ze!!D*BG2MKm1Vgkm+6*mKkuy^bkSM`HJs?%O;W8oyCzI*#$p80TNs z^PS$bgb|MeP&bLc*7bLP1~=349&*HyKB%SW`zNSVHWtBTlTKvUQ5{YXf0bKMCCE-K z+DcF8nqdjmyc*OUcg$xoe4~88?9YmyEXj66eJmZSV7mR%Sm{Nn4|IG2l;A+ zueveGF71ou;)6LkQz~rUO*h=Y2%>NP#njY^>IV-W8t&Gtr+v0Az)HoevJ7?3n>SBS zS2z6lJv8RO@utfLdKu;%me8L9uIiFylzIHDwBZ@9@+IHd!1Cb}n)=qN`2NKe;p@|X zMd`lPCd1>P9PQh;Z$F^oc$HAX*M4RvEofK}ViYeV&7usZ*E_;`8B>zF*e}ujVPavT z6dY((HftVF2cW&F*}h{(F6D5Hy%pG>;mx5mEKW?ih)W+7+HV5tK2mi)loixwXLPlD5oE}#0-U$KcJ@4 zJPf|FcdFCHN>;|5K>~U#B0|p^&QiEl@kWyxiP}bHUZMJG(Rp=RbPzu06tWm>51n~n zDU8;0)LxSJr7;8G_gFoXIZH)3#m@P2e`+9yqM4L;g2r(MCNsEq8ZIcVwss{++q8G@ zalnbzZLwsraq2?0i)Gs&EIyYS z%Snhu?KMt43<3P%GqO?3b^y@pEOY@qWNk8x5=ECF<2#B7O}*4L;z#hm|b9(mc38YHS?8c zKQoY65_n`48O6%_csuQ{+cohB#mQF|~mK2CT1d zq^IxW6)av}#W`R6?lnaNZHjMwG5Fw4cWI6+2#^k`WhN9*wbf$xBZXu3J|yxlBD?A7 zIi29zUv!KD|GPC7%Ij%B)B3u~LPrjoc+$&k25?y%{ivu`X!=#_Gv=N{lD?2@660so zE`QM!Ba}YDv+!f!?yN8}!ULpUxUdi)L>4tUp5I<3szrW(07kWJfYZ z5M1i97XTp~eoyYcM5iZ%T4p-u_@76bBHx%UI|4EEKTrV&d+Kyy{PY?|(3u^^bXzp7G`XaoHt z$me9fHDjMyOFhG_oMvYiEX@-1ouwrQP^}R7OH1!(dY4mNAWNnbBcKr|i)0VGTn79# zNX{RS^cuj7pAPU5qX>FfPVB#FyUc*{=|CNz(n)kjh1OM@FIU$LV26J? zIn?@HB!HVBLQD}0Vk6`Vf~*6EUp}p!8ECgI098~+zC Cm!&HJ literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_batch_flops.png b/benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_batch_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..2f48c4a029a7f8440c027fd2dd3469eb175fc4f1 GIT binary patch literal 89658 zcmeFZc{rBu`ab%YGF2qfWGspZl}bdW22rL+iBPE|Q<=x2fsi7USu&JFArVq2^+6(` z%tK|KXZCrkwZ3cZy^mx6vH#iocl_RC^{G|f_kEuGx$f&auk$>w`=;iheM?wYvrrVZ zB;5#_-)rE)uWemtj=6=FtI&NX_#EHzF>9f!Z}lZ`_r~| z=d3Pn-7F=wSxSum?4?WAcDp1bEdTuzo2_ikC2Z^ICh#fD*82_YD2khd{Gu_BuF|Gx zDC&TUqOM~^&ku*AI<@nR0}kH~D$;YRDB4`T;k!ol#n;!}je3?RYV^8OF6;F^%k0Z~ zc`Uo;)i}k)^~PtX^%c(>af|CVb&X#yXg{bLw?AUT!o*bgIewps znc3CBwo(h(wNum+!yJzE^z@&NuVq(nzmU{x&C)(G*fuaezAyXYkIE#ib(Yt41lJ1* zwbUAW_~hisJW>yj>$Tp56eG-(E% z#BwHh+h&`mR|w)&vtEfQwwM0~UTSx|z3C?=Cj|rrY4O#PQc_;2sp9vgEfz5{GG_Mn zHNN)yk?XMO!rlGr$-m)CFIZZBj!~rh*3i(k>TRx@MwHyDjto}1tB^SkQf!|=N{E%3}g_Z9W^cT|LJyE*zeE&obFLc-o& zy)+~L19yer)YYxZwrbiuFsHBr?<|hr7f4ZWuPsr|JYQp*T5zFm3vNEvp}*1k@@4T$ zyDh7>mHe)G+=k^F7#l13`8mGH%v*Kc=FLl`&sXm`b4FN6Svkb$;F=v5RU92(YRmHQ z@Pvhj#~#Z(KQJ)h84?mw@0Qfvc0$_X>$@1mtHD_Z6STkQ*fDZT7^z&kw2k8Y`TeG! z-&!%s1o;|xUE-sGW z+w=A7-EwkrnmM+o{l}ssBmEDCZWh?NbES=qjpVj%-dC^E;a^W$SR79>qZ@J$j4gZZ zGH#74&T}4R5fKr2_Tq&~>aonq7w2@s!ord)^@BaSc;#JhG-O+~;g%hTJC~G&Y`X3| z`a9NXxO26K`^|uWl4PC4fu6dpWEI_Kr^qU*2#REl@l`FvRh-~D5>oq_1m1Og3{~wa zkMs!$2%utyN^5G4%nX+D32xo0lRR56KleE6L^Re;jJVMnePd?I zbZ%x+^yBfY3*lX(s)r9BzO`!W-t`+caKB`u;_&4xz2y-y>vmoG$zOI){ITv1g|71M z+O*1IArQBm3o*REYNeRpG-Xv2wIdku;m;mNUT)vDF~Y3oIOyV50nr=_jVpBJTW zxJ{379h?_ZJ$f`ua_iQ+3g)|=M>b};eJBriFb+-W`IfHCt8@13+4N%Fwb-y454N9^ z4K~%Uw1^e$A0`jpg*^D)mbcdzE-KKYobRTJp!3L2>{dbBUh{@5%c8Ge^-lBc+O^BH zGDcBw`}Qo&((qcB#o^V}y6LWywpxgeOpo}*OO_NjzqwlMzh;N-m=>o;m;21%zR8@2 zzY|T1eVy`~4r*v{E#GiRce3v3p~zi=l9JE9)qCvkKTaN}FH5KuKfaIl+1by6pQ07` zw39hGZqkqbu2F6)@i(!w^v8F}xzDULFOPVWdOUleBh=XRpZn5GOiZ3ZK|y76!W#G2 z7to2UTSv#t%*@2hta$Pye^F79B%b{fgXfDL?6BI4P#L~ug)|bj?Hq^bk(A(mvxZ!U zw3($Y=H})|CWYU>uMOl=C<$7(4@JSti;l6i*pE9*&dD}^S|SNwa1tT#EIr-HYFaE6 zk>V8`%=z=@&po0crrlptCw?VH$E2mD^;W6{r{zzvAkQV({<=0WVEyEH_EKSCVW&f* z$K&@d@ySitx^-)Z!=S2)N?}98nwFN9o$fO)OWg7vBWtk~Fw)aEzq`4-^7CgeKR;HI z%q$zS&e>bf_hZAHt$oVV*HplUB=h3jmqklfZ^PX$!+l!+t`=Fl>(c(y4cC^eiBrD5 zbf--#{dUWG?{u@$PKzcXEiJjghr7*615~xOId0v$b^G4EN4?fboFbnxHMyeve00JJ ztV~QycG|WxQHAf`(Hj{VC7sCKuC1+Y-BHG6Sm4gXxO8n{rbTUYe}CGL+f3&_52S3{ z_fwwf=~AyQcQ3*=TfcsN^UUP%ONV~mEUTt~+qc;#rl+}ozK(v>FBK*4=G*w%rEO}o zoATUpx-91O$9qXy@ryov`ozw|1k@mJcQg*Pxs-Oi~Tf5HKWUicN*CnRI zQF3H4&euNm$8+1mC=GEwEJ|uJf_Fjc|hX+IH)o!PeoZ~@ z7ZbxDGTPeKRvf45=Z)~9_*+|BIm@KOrTArJ44$0G)e9EU*-No6TC}LBv~*vxQF;E8 z)Kmsa%6a5$X6fw)b>_{RH^-s4wBpOuPGe!?kjaK}qlRAHJs5JOJb2H2i>ik=DgOM~ zvEwEeFK(u8AWgT|cbY|r#BO^aZE=s{$HRS?b;!u!p!w<3Hx;ig`d}PLLqjv!ryS|& zN!j##lUl~dC*jTBIyu}md0#|e)24&lEWWB!>_<{`m(@N!p_6hqDr)mmKKUnEfxQjc ztj)c*q!SDf1wL=WIasW)sc1cae$RP6Guf`r?4w`b_e|nO+{+c6Bl#7o8o^n}FCIf$NoGzRiCW{ zy?x#%zHuXyUFRnn>eBCOW=1w%@2IHX+50;?JHxh`FO{F}*?I{{aL@EsDXBLvZQ3eR zj%<2y_A~9xn>SZVpXcP2bBwVsz}w%nf4G}|?u)Q`t<4y(pnd;M1@i`iLQ z7b0{&zqB!Xn3&jxdOOkTCo$w^$gqr^-4g(!sH9|3TWO&FMp>6JvGQmIHtdI6E2U_| z!^3wv^sVjto^uuXYv-ZQ$r%}-}klaDsOpnm2v-}Lwk#TxjtuE9^bihXM26$ zXm{;P+YT0l?2eUyb20CD=V$Cd2ux6jPZ+6;GUAScAQP7a-|e166!Pe%ig~) zM3P#-T$`*FUcGu1CS|q=DPt)`?rQapi-yf}wxSmlPz*zxTAU^ZOhO#ERKZXk!1Z-r`A8z7U}8fAx~L1Sz9&vqKVNkApTB| zIXPT6GBg`7Cwcx{qPp*g9et-p<18zp6-swBxzg$A=wNfNxXQ@()qX$2y(Wtsb@%dI z2O)rARAtAZcII3D{=fr^QOq;@y=>JEAHJTJCeimJ_d05UZjvU=*w`2WEXm2qC8env zsy5=mpF4AohIOSs6#3MYjfGvee!WU>rZq6zPP@*eeR#i2M_9eyVtae?w*A4=Cp##-FlgBuia;lRX^50OtHUq9@VbXzqnQC ztWwsaK|tW)A6YS5g`OPCRHN9-%gfbHH@S^RQv7necl!he23lXbqaW7Zi>R?-_?e-12r`@|LDJtKp9oR9?|`p zb~HV%zDwD@YXm!GQ~unPMeO6pkE7*&eo=ZD&sBo}(RJFvi|_M)QEaBOTWDI%iXc=_eKO^arV^ws)hjrP>( zQ0yN=CAgJtXv^p*DW-KW$n@JHxI%%qbQvB{yzcz&Qe);mn z$IHu$;{W=@;BNn{#Srg0IXT0h-QBE|DPnl?_dMDZwFckTenX?N{w4{l`M$0|RA;9; zWQQYJ73JmkD)x>84Rf8Fj*$2Mo46B35+s*nQ*s4;`lP;A-gPBand@NX-!~Dgl_`9| z-Jf^&>Mx0h_o2Bu6tOdQZ0Qc*9WjFp`X8?@zr{KbNV3hc{zc_EBn?Ql3OP2y%76QuM#@0f%eOp&&(9yFDDZ2z0NJAk26ahQ5 zvMv}9h;V%=Pm#j>%vIb$#BLWolg_J^m6{V16NR5YFS~g0Vv=Fb7NFc^yu1O(;{wQ< zk-IJ<8*^7aJg_{(=DN3c3<9jOrbcysZnmYpo!-rD*0ti!ZERg&R$2F+owSU= zI)ID|jd`m`)GIwWtT4-Y=FFL+Sr=B_xqJ8XV||I2uU?gXJniuF+_A%*z~*ObQn-xr z9BJI#+(?zX-2GJ*k2HW!Az*sEKOVdKTaH~~eeT2~jR)5f4@VcF$t8FvDvI~1Va^pl zZi!ZGIa2SIuHAXN-|X`{^s}E|yEvRWb0+RWT?X1(#-85Z!JnVkt(9|R*MDlLh|b35 zSCvpTu6%!MZ7}kjXLg09;#c2$L23y@veTdb`RCv#9w+3bmu}PBDRSqOC)(gfY$^>tCF^+tnjPRX#D;x^|~EB`+^uy!!d}b3<~x3hryszFxh4oiRl(l?D41rGg)I z9IaW&SZ~8JphARzvS1MPe!lw`u;bYuucVD$J2Tu=p+)xHcHtX4`UC8cmzREVEajC; z8r$jHJsPKhh%cI(oo?yt3qY@bPBwxT5I2v%EAkWmR?510&;A7MqsNaU0C=BeWGGo% zO9t@BUPGSSlo^!cI4B5ugA$-81teCPk!1A2!9gZAwo*J~YgD3{x#|Am(PvyY z0|T0M*8>8U9?QCL6S;j;wS@1OjN9VBS-JQjJpjYhSq26M!VXYFv!`ve&<$RF@L&xs z9UWlGZHpgIPvpiS)4#NCp*`~{On~pC7z=7F39gsZbPi8+ z@GYA{Gr53?f!MtMnRj6ifYv6r@MD)1gN@*Pv>%D2WJ9#;6@(}0$*rnZrmC*$--LJ%M;SHBF50C^zkHg(Crt>WOxlQA>% z_)H-1t%$!zn)^`bn+v^;W?$qXWI&dLfzsDVSqJvz%jqmEx&{4)-8Ow12KLgO8Lm(e zUH-w<&Fyohg+7wh3gj%M@xC84?fs*{AZU;uTM;oynz6J<$b<!z!YhZGroWVLOi z*<)+Fr6hodHCfjeogV?f8}*+?YA#mD&3M<&?1UVG;C~a5%O-UL0+(Q zfB|A~HbzU45cg_fyYvhvF}ZW@agQVHs)OzwsGdi2?ZrK_(?-#7 zuifp;i5hmcE<*@`m@wi#i_A|ox@UdpbwiBRaQ)^!f@<{eUmakL(`v#&y!eR_7)q7_ z2keVAz(c3NWQ~XH#X}ZD>;tGrl+|Q2}I1ZZ9?5TcY zpwkxS<0u;`Gv|@=+aWQzd{y=N^Qlo-)OSATzhuDyHr(s~XI4rQIV_JnA|>Q$=<;myRtqU|j2 zcjdxfKhhh5@(}HI9jh}ZEbxU3VqiP3-V;AAS|jpw!WfmD%4rk{(?Oz^#FL-wS7IUz z67oNhBA%c5N7AY>&#*tqJv%VMcWUz&bhMN6X~qQsRt2*w6lO>67Df8wx8-NbBW@NJ z?jfp1llydPS$NFvb&LpwOHU zVrY{lU-78(uW#w=KzOilF4ufEJU!6z_FKN2EJ)ycUFrMl zcXRRf@rkwVDBEe@bMz@)Q2o%wxIKq56uU=8F8TDzBtDhL;%qj~U)nl*ZHW83Qi5S8iaO@R}EljfXDh zA!8>b-$VWi@R*&*h1>k5q$s}dpjs9i`u7Tsn0g$?>?4&(H|Kl)~2n* z$=xQYd|Ez~#YMb&Ea&LAF0JML(=sBPA-#keXKzpsiy5<8J+NJ52w5UYFLgCKEdgT= z%5mE0a!ZlV(HCczEn?$c3|1nOSvBSC6tD7FwsdxdGFs=Mc8Y(Teq(r?V}YreS$y@r zqeoX>TfB-i8F9!81Vj~0pRBW3XReeknvt+~WMqVsgN3C*KCvjziiiqRm69S}h%NbJ zG6j-OQzOas&izkQHy;no-S~ddjPwa(P+G4*5boT5@%xSbf_dKAsnL()Cui`WE0Oc* z85!R;Hu9pGIN$I1CCh=aTROzbcQNJqs;B%m0 zS6y4m<9k?sQl+7VQ_bXQhmTDEklhl!(bM)IUaSFO2(1bJwh&OrW}sP-vZ~Ok<9=|z ze0#jO&+8TErwzq=(8~}626Ao@fF5GV1YqfWca*HdQi60Tq>#M=0yK`(r^m6xHuz9_ zdis-}<5aQB!%ET@f+T$7mMt8>-DrZ3q_qA1&7!Zb@6+~alZe0$AE$!WpFeMadV2fj z4GnJe+cOio{K?KWPB}eR`lhCWTefaJlK+)@U-8Akyo*r1EW`OeBnK(K{oCHuq#5%) zdh`gKX-SSm_4N%2;R4ObzGltw z^|Lv-xowbg9H&M^(7Jm;FyuT?z;Gnj-g0ujx3{tn;5jG1{P0Il%g+rv^-+rkeD zckx82CL26IQ(Tqys_QcZqQlYhs&yHsTe`a++b+5dq%34?Z20=%Sb~|N`LT_MN(1?> z7ZiASHQ?uV8|&n`xVX?cPKPC_vELQqzUJ-y{^Rwj2CV<-9%`o()`-Q)k0L^vzVj2g%7w5y50WAgTz zfUN8qazTU~vY18hPdPrgRR!iXb2Kk6j~(1Nbhl1fVS=+R)N!CqR0aY46h$2Ppvy&p zK`U&0lfAvYwT;aifc<4_*FMqIdwxcP^3)ZDoVoIRby92wTip}Mj49Qc|C{tNH8u4U zwHqMcXMS#)O~N?eHO=@DKHo?_;kYfwyp{2L{i~9GAWe}hLPU6^ zh%0GnRySv=9zHvDbq--m7?Kp-%D7F@Tk=IdRa0u@0mm+S83 zW0M+5RV6vcSN@JF-K77Nq98^j+8?;{{i)9VyeJUCS4k%_e&o(t&?q?>ArLX^ zLfxIT+|;IjBWu-3xs-SyYpn_7I7w)KD95C!M&D%_Rof8p@$EUmf>D}(8qMHX-~tW6 z1xzsjCOR+sp?6WfagccNSYMAA;NG42rizNgr?tIwMT^lahy3PN{l9MYB$Q)TJX8s! zOUd)q8+)1xjC=3LsQ$tZAIi0VeB|*pbh@37XM5_Nr@PNOql-9+UE=gLFd$$r;UoGR zcTe6~A~eW)H?A3J2&6x0`_aUFab#v}_ON)Xww@#NOVm6dm?}r&F3>#RvaR^2W{8if z1@5!9sUE;~z>Q`=TNAV6O&txVzntsJJd-Ot3U22b!nKWP0@&Nor?$xi3&#ISPx^wu zy3mld1eB@Thc#OOiQa-?=@@NF#B*;$(u7{U@y;9=8Jaweyp*Hqra>bHE79KltV&=V z>#4g66rK{nV!-5u{(#nTdbM6Qy>E}7mbrNs+gyP1<%;;8T?u`wu|FRhCRTrKAFa-@ zGr#cd84X$$;LT4i9vbbB&^P8B3W*GzaW@ICUA%@kQ&d#wNk>6Lw}^6_zo~Iw6r7l3 z*9ojsx@E(5NCtvpVvAv;P`iwhn`9#<1CDt)I)HZ&b_T!)xn{aGb!nTV`P9pE`wcq< zJ2h1t(5gfE+#AR%Zk%C)hqYSB6Wr30!iKIN6%}kyZ8Z)0@oZQ|hn4S+uf*q+08giz zyj^sm{spt!)QBRH<6o!Eh<=(50b$dOEk-rwZ+QRV57qk8X;_%(*AU}HGZilVWQWL0 zQE-=Rt^)0){)hG&T8|LEA#V$rY8eHmMn{Jby6u{;uU#iINZ(8GKROhdqB#Z(Q8o=B zDYk43lmzLKhvZSfy`eK`zm9M#E-p^)u>gLc^#olsRg1d6He2+uUMdIW{rvfJ?K+1hw+$2%QG|NOdmHdaJ3G6C40N3=6cKfZ zz=5q2;)a&f4^OqsBhc6P4&O)QpOD{3)~qmFq}gE)yOu_Vg1BL&ioSh23JX$)fd>Mb z)^nqAejX&7P%G4l9B2k$@hx05exEisXWg-52hASfgB#F7LPoLQykS77aym)yJ+Xx( z|G1u*m`L#hr0U14^kFahmq&%A;qKkLsy-}S?Sl$vC}U@UPljwA92^qdS=V?N9g3D0 zrEmDv>)nZqgt|d;k*K?+XwUFY(nIeF-lTsBUXG)&JiZ+TXiOO`=Py1PeTxZ1Z;@pXSw%N2I(fDPe(Y196aMbmCWLNO-tN{`{Pa zlbpiLfO3A|zAxh|kwD@Ay=grwYiqele+rPIXRt#`O3QV2w07tvX>ZC{rycqrn6!c< zJkR$LVgEkuG{Q{~no4pxmbGJceyTf-~Jhj&7P z5XFzT;c|&qLW>7yF?j6waWgE0rGHn+WKwjQnC^Vd2-_eo94Em5AM7 z2(F-8CJGG&j=9)8+U*)X>?AslolZmRkG(icBS4Q>ypF6%Fg_##oiN+1jUVn^{8^p6 zTwOzB_{n^od9(=uz2H_{S6fgM?RO7bPY#`aV)S}7eqI^5C>AO&B%@D}viglicb#Ee z`j+b;L7F3UKk;ZTh?oFD`lPAp74T>LqM|JDR*)_m)kXiIgZX)9vEMMPodnfHe3HaR z2lO5fRWwOAnFC$9Hyz_r{@uHIK>1Hh_ZRqyUDU{g;{;ScarF^jmjYjn6XL6?x*Pk} ztDr!Ew7V2&)3xqs3I+Y7rI3j1qF?$HArpy64XNYgZ2-!e^^f3JLtBaJ%9cdq^sYol-LuvPgd+T zDJSfR&=Hfld=O16C}lqEv_RRbX$qj0W3}|o+t^A6#cXS z?%!4C;;N9U3Hhw;I4Q@n5C?jpiKf8%SL*{RE)!ldYc1g{YR3U34TDJUew z0He{HUtbbC0(H=G5>YV3ZN*5xAygt*6d91M$(>|nWfAEJlsLKLzV9-pPM=l;e-G*o z0%1l~KI4+r-h>|@mZ|Dw9h>Gt8sg27e|%6$i56`Jot2eU$n#^IS-S^XUN3aSqT9qb zs1nvCWF-`a6`*EUAZUkV%^GLZGcq`E6UzYakDQOTEAjm94#4qlyvaktDBm@C`wERS zYM`PE=B5tDS{>Oc^uF&{O}qKTK+B~2_iei`)*U``<`H|Gqq*Ac!LhLw@F(QRb7SqE zD?ON)a2$kbE#*m5oGL4!6QOc%@{EfXX#u<(a_fb0u+3iM{3gu0wxk{XTbdHKrv?Y9^$k8h)hXmL8W`+(8Q+ zw$mn#xC=_-=&kk|99MIvcXoG&Hnq%7$T)%SlYG3fSHvuzJUgvjqNzYFVnte_; z+ZQtU>tEVADB3azg&RG-!=Pi_?8a&Th(7!Z^bNa4W`TFu_(!rFE?pu%nA^s0n_gdY zo$k$!gLCn`@g`8us`9M5Hk;s5bExFs@|>Q(hc5Fw{+UM?5)2Vb*I^l} zvn)ZTm(_LJsvS764B%4-eo1sVt=PlG^>fNjO{8hi)m^W|`9<6ae^g1(b#;~7Y?OBc zMCZHW;y2iQU%!3@op==`wLW(S5&u8-YV;Q<*)W$d0zAMVBUG`{c1+r?Q+?&CRmo36 zAs~U=U4~6gH8#?8Lx0pX2RHd1FT3bHZ_cP`UjCjF^iMG9?Eq%}<$FpZy>f(t2QM(> zN-0zCGIuat&p=_}Q%yQQg`Xb}4GHmqag84OB&^f0|M@^41}0_T=H|}qMcp7;Gt9B1 zdLib5ns~CImjXCJK~*<#GuCJ=|A%O>gmAU$Uo5?K>zAY-h%Oi%7*Cuy@hNKe8mu}6 z?Ut+rFfYHnJTDqLQFHFf7g3Pj7L&4q6@mmC0}%z8M;Ybr$WsP!Sw^BABELWGu|vXV zVr5lEDC?H<#vQnYl1ZrRR}TFFphZ%?ABMLHiToD%0x&iA?%gBaCa9z_8SYxe(V;hP zFu}#Gc=#|IJVn=Ya&ojIQNmO5SPkqWlTkVVNTRy$f60(P|MdxR^_^SWlOO3M6+87uolv7RuEWw@PMF;JcvVovJIaM|8QtB z`9nd!1tgO`W?@t#zs>1KW4_xOm$4oiN9kL*g(Gsr3kto08?`;|s)N;|C9gs^*Cg!OD0OUtz^lz^VLl7{S+eG z0cxc6-oCNzE0{WTi*2}hD54Lt-o^&5tSAoTZZgX=c3s%A4{seH8WCPXdkhWNg4^(d z`vnA3QB{?8gFwj2zo!35)zu-pGFBN7=^Z=9^13pgRFogzj{H)`f{6t_Y=3xE79DX* z;}ruo_}i>LlC}@bXn(+;hyn`>Xg2 zaLco2&kEt6e!;U8%IVX8PYETRx(T%kRZcpkARKXO+C{H-@*V&i`ZD7K|RKH z__K-lUP82OMTeu-aQX6OlQU(WN#3-%Q&;D5)(+7^csnQQ+6WO*XpX`Mhp`u9{=h#+=OG2obOsu@x&DDf7sU)Q|6 z-njVQ!rSCk)YY&N;|xc88+Oz_q!V6GKpeqwhrrP&qw>NNa_SUmFg6^D*n`$U@5xaw z_-uN<=jixpabN+?l`e4GWk*0d4WP9-C%gFXz7sWt*b=7TkzX~2L^1NahIL=S+BYC& z$&5sGl2+R+<9@6KAN|7RY6TrsQ@etjCEdxbTZ_Nd)x|=;C5W_a3i|KS0{7k2NpMV( zl9Gj3ThP6MVPPwY#i_5a?^}U~0+dgZ>=NYRh*9|d>M|>}@P98$5Y-i)*rNLS)ysv| zH8*~I|K1Y~4g8YqTwDyG3D6bu;3Kc)gdGcI{eC z{6jF!qE}CNc1u+;-~{G3(0@{%%hv5B?jiKvuv#eMSyg@cB8C3eXFJTr#58s1&K*+F z0LI=lHp({zE?KrLNROFp1P3|S2~CQfdI!VkgS0ICH&=4Uj?nDT+5PJ|q)xqO54wFj zrn(cxh#_ZCb+~xtSj)S@8t*_OcFe+5s+JsxrucpLJoblL2TK$y0Tv@Od9Ss&g zt|wwd6(I^>3)9`7P(@oTW(R_kNSO(98tUrrz;JB_GbBdckawGE?@kx~6GNg*7;T~1 zvlt&@8A)14g#x3O){4ol5N(Re%MW#QodbB1I`e5|(49MRS@YnLK0~|>>0N?Z9z^<6 zO$G2UMtyB<@zqV>YJD(BH#2RR&2~cxLUi0jh%!@z>*zI82dve$engCHqFYG& z3uTgq0-_}3C|Z9T*ghkmMS+CvN?#yk+Q*|l_bUhC<(Am=nhj}HAE8X)O_abS*>k|!x`9UZBQ-**xo1Zz!s z!fVkAJh>BO4e&r(<)Q6*R36Vk$Iw^l-bHJ3KVcRE*~*Iy-(kcEX%lmHGz0dNx4@?f zLTjYRbiuZBRrCiA9M}m47N{&VtrQu4e?!;;2bEL*+GPh?m(+|Wm;;y?`ErWqBVi7y1hXnZxi+*1~pVLaT;6($2aj*AbRu!L=?AM z6YD;Dc*S%4El447_0z=2-F^n5;!@vt1Cawt*so||$k4z&F@q}zqU6Y3Te8E@E?TJU z+xNDph&J(1LmlE1^Fgq_F`k60g@^UCAe$XF{6A~_nj_G3O%T@MSsE4AbnFtR8;LJRFnmOnP@lt zqD5uXEkjHGRN@i&_wwDHV{M(C!sK(B-4^r78=mXg!zA!E%~--XN=i~vvu#dSyy+R1 zWAYBO$LwfmnLQ-3pdN_(HSh@d9!m3S>hD``t|grQ$sWAIsYT45>q%N#gGYE@MXj= z>@(!hf8o6dtjR<@M|To*|2~TVy=S(LFl`|Ovzn|U+{2JhZ%0NhA+B?tUAm`vz@!r= zEdu}i^oGBwc-rXz|CQCHKG$a+!9ipAN4%wOx(h-sD!L!F^Jk8COC6h7m>o!AO7&2ID z7a&CZV)9$fK;Ltg0siZ~*bH_3B=IM~l*l4dWoC%C2eB@}pKqC4+v#N&&!ggI2#Wy8A^QI^{#{mjkE@9@Q%jL#dIq~C08031y5a3ee zy)X+3j)-hjGBBQsNe(}Blr7lX$c#_qLi#Qnlxq0lvl+;Af%w}e0U@bPi`0$~!QnJ>~$;rtCduHe%eI38o zg%u2S1sT0Wz=t@IXL$%Q{}UE7(iopV?~CX_OUg{R5P%`-Bm^-+$z#~1KCHS}Ot$1Z z?{bZ@n+s*+Mq@Tq6f$Q@#&XDPhR;K9z_-8Lt>@FHn~5a z1R^{VTW2RjKrgZ%MM24!6!Xo&21u0?p`plh8j48_1bsqJ1A#NbZVRTS1zZ5=1c*u! zR0pXUE{B}_g@9&ix4!(H9YXcTQhrO##{ykd{7@tl{3Su13WK2Wx_3 z0$8O3Bl`}htfK=E-3McXO2)8Ms32>?Kza*2Z!G4-)TVPWPW<$Xc;{aGTfaFZPuaSB z`g8j?UKIcM!9%>};9^neIzCE2J(kHr8hqFoLFffyD{ac0?aS@gEnm++y=8Z{0sT-0 zgyETiw``Cdl(40(QTA~u%9d^hrsQ)lPLQze9!0@I^9D7K-~^mWAP5T{^b>G?EN)z# z-=qcB9AsDG7ZEwo`^j(ZL4LuwDwTZ=G^m_fzs&IzeYrN#QWw9!jLkCA)Vw{j*$aWr zebzXGh76IA?!L5?k>jRHOSwJ{U)vT+W9hQO$O>29D^YX1W1#|JeMOMuDgHDLN-a-PxL7~Vj%sMvi4%w zj+|uLWlt6>l$qCRsuo;l}&FSA&7&8ZIjBqT#+Ot>jMh&bh7PHa#jL4v3V0J);p zwBlm<@ob z!TH=Qs$znv#M;_=&;I>PfFeuwIOWAYD}9=b9-YqdLGUx!C4Dk5*nsl97V_W1#eJTB znAl6>!NlNnf|Btet=oN$o)=+UFiL}`8#h@T)wb}$57 zo*e2RCVyxdHwjY+aG!MK@d_a8K|D9q1u{|zM)Xe+jT`Sk)+9agjDr9@W|FPZ1Wbx8 zg6{SXVi{Mx8w4U^D?_6gfMiI9l8DBH#6!-~Ab1PkjH;(!p9o@bZg$-Gg+=XBlA&Sk zA%=2*IX=+FAc5b4ZYGKwikw{Fo4@K2aywDI9G#q6FmDKXElO#3-8ZjS>GaK(twp#2 zlZ&pPC~2M z?Jsk|rGm@|Ve7~}d@m*!X)L}z36hB42zYl5?YJpIgdE#3H#cqN6b>UK)J$#w%ND4h z`gPfWAYdY;+-Drg94b;AW>$jpzC)5wzP9-D#7=@w;FF9&%Qi4POax*Z1U5m?My1Fq zn8_`KDo3?P1ygyilLfo(aH(GIxlbqylt0N!KQ+m02dsQU9U?*;%tQiR7zErZ2VUW| z!~z*3CDxhywK*BDPBPn`v%pNs>GyCrknWDXA~rl5?shYYcVTo;36_Th!kgX_UsMptCu-B zu@L1M0&?)ZXs#9!#R0Wf5H2}PA$%_@fM!~FT88I^_zqC zLz1S2&PeQXc#l@YJ~V=#F^Wb6Q%t)_ZrP%QSu8Rwj>+&g3?1Q4qw3}1AE84pOMsBO zyZcZ@JBEue|B^7X_4O@u&@*-~rCEd5`}APP;f^pS zf*aQ@Q)04YRQ%H${QJLrygZhS#JKREFFxUaGi`s?0`LF-^!ZEVU?qCpCr6ioXHU{j z@Xb;terCMxu(GjThnj%AmB~}K&#R5@&wc80aM`1xde%mVGa%@1|DdtyD(4a3N5nq7Miek2UB9#?zG#r<&o_$RWRaCG zTp%z1Vugj@{@?KOK9Ch+W|k@K!MRP6H`L@XYI##74u4#D`7SACx-nGp_xpRzw3S3a z8eKt##l)dOLo9p9YSapFE>}Rj{cB-Fi5Hj{D`3e%?;8uwRgEElZ6Vj3;|{gj$bdE( z<0l`y>Em40a1;Ut`0+QHJS+w)6}bDqc(DzwY#cN|Noi@H zfTjIA1j?3_GA`WXi@*cS342UU1*t-u=ztlvV*%HaE|HUtFj0YZza19#UXgL(0f>&C zYNJEG@`T8DFJ3^DXQ%u^ct3PgPn2l!W{0**OMwsWxZ5Lr7hJS}H-75RJ9i3ryM9;% zQ@Z#*W!+c*zNN5W&7zK(n1|?4iud18i;hTJwRG3QN@<5tD{CN8*_!jRM>!PV&LCD)ULhDkyQOWD@$);x48Hc zS^J(nz`vTA=Z_#%35!mYK3qo47y`owW(;y93Td=(n9gtX=%4}?VLetC-@!2Q@nuI9Nq1-Bfj?w}DKe==?T^X9W)j=tv&aJQ=FOp!3`J@0s8$ zp1=fNO`iu^Jsftm3p4EG%qB8l3m&U-qx-^Y(2{q-JM`__x5DabPSEL?#L`cC4sK32 z^%xiMHaW5AZ;)0;d+NVXQ%pS1Eg(8+Cr02+;b2cXnl6ZIB^r!?u<+hC+qf$4E?GBq$NVp) z-d!#<%vG--bNS*|iWnIu#N-^l6Z^9W0#XxTFkcCtLZy6{u zf8}FOPHdob=MZhg2Akj}BrNO;<|oCn7SPyd3UF`*MJ%bV$8~js&%y*k@niB?9DG1t z&jYJ`t%ht^VZJ>#4^*0OY{Vq$%AM9Rh#K9xzU(GlDoVEr>(*|;~2cY$4C?myK2ya{v%;K@z*&gq)oz7EJ4Q!)JIRWZ+;Y1IXh9(@D zhT#eR?>~Ova3mxZpCRx|NwZ>0_Yhl0#~dV!Ct3b5f+?M->l%lH=R@}CQ>Sjg^vgn# zqjH8UuCpukG`>ELjg!*r0q>OH=FNwQUV;n41T6T=G^5>%q(dd)-&^=YsdGLz zO7;O}A&QYl_b%}D^f?T@!a$g&ol9I+w|I9S1Gx1?-+dFcWDqWk^h>|AVFZwXf<}%x zQhESho|TFNqhSiLM))zZp5I@)=o83o`u(L1%V?8^8ZhEDa*Prpm8?0->m#1R1~nPzR1s=G>_6Ce^c3Wg4LE#I5Df)R9=n6KYPqaKA196%TUY^1V}}f2JdApWSzS(3 z(3n9$xE!c1gyLWL@a^06BIKO3aYltIBu$YY6sAPP6b`}9KZtwtg;wGVmY3`(@&u?G zWNxFu@YN+8#s^l<6y_uHF=Up!6|{O$aj|M6-(8NMZWyo&K(-Xbc|0v=cr`xWR9ha^ zKO)sKd5G_B;H0ZW&jb}}6{Fn^Ig%XVL`Z_Hz*SopVe8?Fy`f6F-Y*W{YEBv=m{m%# z*d3!7K)DI3eh>UVXi{KX69g+t&PqZr+Kj4E0+y<#gct(BMH2rgInEB%nX~Nl7wIS& zyG5{-s^Byz;^)CbYQr!>F`T#Tu(Y6U*fe?qHH65S_zo4EYeTp-av}xE>5vrQ)HicF z0qSP50AUV*!$n3I0peSTJXlEtHkI{vf!VM@$U&2^h`q(33|x=m7I|VCl$2U$_5*L%u_+6L zSv-mjoD^gvgZQ|7hF~2gb~<#I;C(8;e7TAn#LC3R$I}7dVS<>Jijs4RtM(vYf_dF% z8vFJ!0t--{!0a>>@dhDx!LHl-yh2{*cOnP|oNoA`Hl1Lxj*;H%CSqVbiVqbdFa*Ho z$l2v9SKff669dm6hNSKVvf(5wHqX|HLUh>H(_@z8J^*qm^e;f`Py!Sf{9Ut&d;qaf zrnJE(NRcz=TwM#leqBM%@&Pu$WinBO0l{G%EYwiDH$FTTWRSJJ{a$dcdEaYlxC-Y> z7^!CL+XaKb#$rPD;Y6*D?r%4I)nFQdR71Y$uePj=gH!Woh7S6d{YO4QVqr)mG~3We zh?CH>iXfM3#ws!3NDEddqI>XH6xHN9yae?%%7RaZv7oKBm7>6WO$HQ0p1@(4;>TWi zf;HEkgp!35k15QOb=AW(Cr^O@&n$a|8p7gMQd&w!0h2w<@&Jc08A?lv{^V%ik0@e# zz@au|pX`-oU%5E-bA181xB14V9H{w36P>U z$7?7I<0e#up(dCEf&ippM81{Z1V;II$R=?M6&rR~!4XjX7u6D8n1W0jbinmQwV z9d$(ze3%kb!XOQQztoj0w>;Rn80y4$Fn~b^Z-5Dyct}1b;Qlb-?{u&G=fK=9(cOQr z+X9H={$A?RuKU~I=+PLo3zX*qSxNdt07D*3d=gO44jVB>+~GQ95@^>E=p=4kY7dx_ zd^2(m4Qxcjx(zqQN*H9-q-d=eabE6LuD+j+3!w}0oQ4+xj+i_@XVD}<4l9#rCXOXUf}m;k*vk05 zz=KbTiPROdl3QM2pj@3xU`;|r&`pkpV?q%L2{}Gw!;Z5lh!l(wuLK0}0=XRrTR##( zQ0N1+T%gnbQjYyfYc2bbK(Aw?3W$g>yNvf85mf*&Wqtq*N}S+wcs5852b>=YkHnRn zJa;ZxGYvotCn3}%?bbSgtalZW-HM5OQg}&22oQ%SP8FgQ;=N0Ve}W9p;M|5=K)-}7 zFKp;gBOESNP2w=XP%f4WXR$_ZXY>1cmH67I`89lh57)p+po19wCPN8i25qMNAvvyT zVVVG=S~x&-74X^n+Bxu-ME3=%0wcQ;75yDbJlAh>G+|*w0{>}9pvc-+Ft>yNG_ zB04oR5wz(Dc$`GSvG+)hD9rV=R7e}ad*rNz`~d%?Btjk%s-7SRE-oJ|5-DmwYt#7F za~z2JPqfQiF@D8Nn>J(@943U1g*yh1o`%fJpKw9~r>7$R5BAqt5bbRUs|ihtG#91PjF4s;q=C|?S*3Yc|MObx=Xu`$ z@qBoX_rrU3`RjYzkZ{-8gH*#PrSPX*nFO>Tb9v)jRXQ5yXx%A70BW>Gt zW9rNW?~?uQ-X)zb_zaTWSiV{QC&;En~AYd4BAN>}x+6UD*9!h|ZJ8qlEaR$s8n9yVf zrm2pW6!``zl|A&Y4V93Q8gBRFq!=0`G>-@$0jfOT#X%=IQPm{;u`gw>p$U*zWEqvvKwQK=y{kdFTHh z#H{-Lf1~8R-Hwi152y^DJJZ1EXc*}uB$DH6k^;3p1|zLl{Dq>utSbzThZ+4w<1~CD znH73)lX=xv%s;|(Na%aH`EUaUwmbz;%5A5%NX9X&Hz9sMjBP{3g4Wqy^VZf@a?Tmm zLYH2M?o_lTVy<;&pnFNjap!Ra9X@w-`23B5E!Qw|N67mCM)y3ks5Ll>X2inR;HtKn zs{AFOekMpe;u|WPMC$1uzPH2`ookPFrePz7$Be#NX!tuL1Q}Xy zIoG1_h7JMa=mM&5j;2A2iOqRxQ7dOR?#|1Pst4mCn~o=-{cV!MhQrVH9ZgF;_cvx8 z|6MFMx}TxCL}mE4Q$ZX*Q7s`Ms-Jo$p^-1tk-&s6F1l7>#PA$P#{lIOc1t2f>oYdisd!U#qJ- zf=c6EnCTcgeMsO2odomZ3OEGNS1LovhK5xmob_(wpV{}E;4ZBk0_#VPFff-7tq!wM z<^ywyYH$E2ew<5#NdZK|6%-anQKcz1AqZe(CogG^3~Dh$HwP_%n^l34dX?uy#lcP0M#)*=+%4o7E@C^P4I?fju8Q77y0Mi3vwQr z4(?+o66ui>Cw@~>Qaa!LWjQ<<5FE9FfBAIsc>kHR^VIggZv`YIY+;aEsb~cF#qEJC zi_}!g99Z>d(@naKAt9w`NyYTaLnf);sM5geuXjx@{i_%vDoRfjm(({-9O%vJc})Vq z=?5b;s*ifjz(0cH5;-rQ;$#pSlG8J(^Us~GyH4D{vyT~iMpiG=d45lKMDw+_?)4_T z#kPe#j$-2%GR-_%99nbw^sNMp92Mf;|B2=Pr_&?SInflI=hBV-H`JQ*+n~k0gyld@ zO%2k{Zh)#jO=XRP7aSbcfPyed!#J@q4*oV9?oJQX0WT56!R!s2s(ATN2bX)=^52&pyOw-rX$Hf#lD2o8qh(q$zk9$ zfq8`Ll5qcU=7+6z3Q2AnJF4eJtnpU-b9w*VdFkd4vd=x#Ve^fu?qPiHg0=Q-ODMkm%fDU1p5!#j>(-Q@b@9S zy4V{}P_{5$K<@R<%R(P8aN=TA@)yBDBIMGTazw4{?Cdqgma7k(O_eS@XaAP>vWMt{ zs2s6ZmO1Oc{WIgQ$<7}Fd0w{X^IUdO+4iz;eAml9{dqFA^7CbC|KooJS{%i^S<~Tm z!`0e}`ThqhA?^MR^Vf06f%3QpkFL0U`XkXi@m>uPe|2hDTBPiLRsF;B$$SBF>OZOsO`V*Ff}u91d4CCnlXi zs2c>M4`&6)475qv9Wyfc`7!Pbvr5bHX@CAW?Z<{AjT3WR=y;GTog}(q`;$6PRuU{) zTGmZX2!FfKjs;Tp3X;LQRGaf_cyw{)BB2?QF+TtRT2dbwvU2Uq8jc6vxZbI9lZ` zyg&p;Xxd=#esPDLpsF;I%E3^ePGkUwJbgiH;#yl#%H}{9X5qp?n#hq0IMyB!?1LqmN;MhU%o~H@3F^jpu)$l>4_iZcVEoTNO_Z?hJEA_Vl5Pu4+O?ifsH$@1W|_zrE;(3^;f?D$ z8>SG500XAsgMhGa!tSt^1)lu=L++@vAFg;k%qs2x3F#Rb7_!yNSHYf|wPu?DpEzGM zgE4PU@%wQ7WP|1JpDO8wcD585Ts5+rx&N+=Vy1addhHRN%w=z#wrw_i%BLaVpj&V+ z^S2lozgM4L4`pJN6LSOv6|kwr2WHljMc475^*s+~X> zgC-^ut;slok=j>G`!XSVX9veG50tIwS}5jy+Wh<0kRyhtFan1k=u^m(&ddo@jibOK zD6m2A`5X}PWYEWi*6T2)nl3AaHbWL;SXCx7(8};W!C2*rG`h$y!e~l{ohzO3R!qS) zYu3=+0-E-ZiGa$E^dU??(pd-i@ zn(`f_$EjtdF@Cn^I^LUf$UQO0jM-)U#%NbECt;!Q*yNrDi`wjh0*%a!!`3|BPDOXO z{0YoDquQ}Z&&|mwZP3SS<7o}~=tBbw#p7y+zYP2op!wqt-t(d{iI@#InSt!}1QYwb z`>*oydw{dcbqU&TD@rXIs7WXtf_K}A6$if|+}k%{;R7!>dxq2eM}?zIZ^fRDVWk>< zjj8sj!}&kNgmfw=TW9f#l`UKX{Sf<9;m(`$pbLn>f_RgSn@Oq-O(vY@TE9(BwXYhk zK4d@N`5%fXLW||RdL}Wba>r9nz7txrW(tOIyj*DLcoA9d7&z-;Ma56?CK5v1HFlOR z;P?e3ICOi=upTSYe<_fqaK58tflpI@M)ei;M+1@GX!V*K2X1_b(A|D|+<_Ne^?DB* zXLBQ0$R#1*45_dY-rs*$P*8Bqvxj3(eV-!u;Fkzy8>xFE#d0dP4o3F2wH^EM?L+T9 zUQ3~UOU>VxdMg+zU7js)CplRH$98S9>pcN^EYJAk`@e7o4>_)AET-Woj!xjl0t|D4 zQ;PNn+tCz$YKWQgrzrQp7#JFWgO}g{an|`UmwGRS^ z?0SUm-oKaUYcX`Pe(-M(`-^Tv61}0TqJ_P3#@>`Rjmutl^x)X=@A>9=cG1$M!K{R< z-%1{4?CiaIIw)Jd`_|`b-nzGmM}X)ac0cF}*Gx-2c%yTB(#x#vv+GC7b=aG0_O(1^eHI7l(0N@$ zWsaJjoMG19`|W+@%;nM!!QRGO>{7b#T|ZSFval#1gh_qb}a?Naig7R{X4!S8Uop z9rNn01kT7pJV7Uq^3&v1g`hbNTct3zVH3tp-S5t%sjpC+hHp%PjgE9LBARfphn5(;JlFy`NPuYmcV$;^Q(Hu#SI#1WavApcD=ScJhC5nansFepsq@bQ+^J z!S{R4Tf_HBPQna@!>0H1>;uxAF4a08=JC7sQBi5vBWBpK-sjzQ^@n-34{`*876*S0 zi}^5Kx@xFDCvnL61Lv)>W3j>Ot1-fLA4WQFE^r#L-Z8IY_Mv3Uc|LK&JAy6zW@v2` z-}LJ8oF=!aKVwsf2D(eWWvgd8+F|vnAN9+~aJo3s3nMLx5z^xru9%BYUQinHhgb}l z$%^g7h~X=iO}z$F4hA5x<;|OmX~~}!foB|UI}SS#PVa870?Jnl?}Pb?TAGQ8#PAfH z1Q4?x+XW!p0Q`%=}DH{E}0=xfV zP5f zPF}+lYkkv6j57vVH>cg4kdRyHrttn|TDWGI{Y7K%3uienk0q#P?dZAr%lke_VZvgH zpinUKraM@MV8|6XzA^cXNRY@X30My&sEP`eP|07Js?Pds@_fZxNTV|Nh&`l3| zR#N>H&uU>^yCs>t=W0Q8YD>)|aTw5_)HBSS$9NZv$zMRo6ZBKKW$qLP?R{jKrx@{r zQ!O9wYjCN6A^tmhImzurK0W;FCJ0j0ij7WRqF%rj%>wfK3J3#{=naUo>#)VlKjnfY ziSp%kUX#e*UwmN#3x>y%=s_^BPJ04#(bOQ(itJIQ z>!i`^w4q@NV*}8~Dn0ww z3b$iz1wG#+!i_Hu+MVLCwpz5#&WZHC-lf3ZpMPr{fjc_v;ls;{N#fz78NHtey43}A zmxgXzvaKJK9d+_Qw8z&L$|o@sayiBx2~|i^KQnUC8Yr7E+?%#LVR9hNpF{uTA4KF> zqf;aJyMQT7g*5z}%s3ITfD<&YA1@+&+K|R}WBM5S_;CP$a7Prnrf8lJ82O&DVJkZL z1}ZuH%w?y7FjT`82Wq`aqETgV$sqhjR~;7=Is>WUEhtZ#`_jyW=0Bg3n{?yFPX*UI@PjV2_LHo5d9uQQ9apPstdYOT=c zbsrw2s27_?~osl7H^f{pZdgH<(=KoSCJ6M6F0KnAduA znM^uwm2;KiQQg0Ww&)Gad>!QH+hjlXE3U!v8|n{HB5==NJA^b1Y`;5|eE5)S^n+n$+b}9-+Rsvw#O> zJ@D<7DoLyAmb4xe@5e!qfdE;8a;SWOrBQ=EFH-bgUanQ%V&B`yf^CP!R*~#_xZ;S z?=1W5jDMC&pFb0oB>xLxy?W%v3xCv^i}K$+?IF>ia(j@Jy>2sP;X_Z3Oah(oxKs7p z6km9=(dwMX$@*2I;iOpw(6Dy`13j>-AL7KNzj2r`)+UWHXe+OsjS}MQBtyrZD~{`Z zfR@>a8UMN!hRVx4OqFNvbM$nx^jT0fJ8a(tr}~Sm+WDtV_Wk^P zV>r6|&Jwh=And%iS+x5ty7N1uXZ*FBwffFG-`P6P&jvfDoo>B@7|ZQB!nqoZBiJM` zSNc|r;$XY?P--AMt^`?44m-HpG-LUG23L(5psM#7wtLOp)gDRPj{a4>>L*{e*Ntcm zjYqIqW7QYvwPW`j2Yxfn5AgS|Yi!gg3+8@_U%z~Cc#9%F2ih^f~Dr6P^} zr2CzpKNITSVUPPi-y{5#tlf|KV1f1u7oV(}z1s|j+1>)(yhd1fCO`#_qN=F_!lvnA z;Qgp0(O>f1WRSJ)m{;qwW-Wd8keZF_zfPfc`v{H5@e;%Yh9R-=nh>CPa-WtpmXu5Lb!7M_z$vO)Xr!4FXUIdR{>tpLOP4NW1{jGoa09 z$7GQ_xfyE!v0zn&5oHHB(mHg&Ef6_Oz4cU}k?l=aR~Oy$bMXOlc;&|!=)fW!UAX5b z+u;4zGJe(bu-u@{c`}{MNmh7NF%W2Y;FJ=o{_XSg%imhWRD(md)hzj_E;J}>1z+O* zV6}!3@UF^01$%f$mjt9H71|!$>B^b)d&K+$x?A(K(?Yr)J{cM8zJ*In=haYG7-}Iz zotfm(151Q%A?gX0Sx?jVp4)UTIe%Vq)dpHzYOFVp+zyIjCk$wI%vs)lFzVb-rBCCZ zMHeu;?>AgLas@Li=KV%Tb;Dc>YKw=)%##1ik2=MXL#Q=i-Srzc{=$rsC$K*Q+9s%) zm)*mp6u7;~($qM5!J)tH%Hl02>p=Y#*j9&hJlqI@_6mmZk&6blz)UAlp@?x(XetO% zrS1e`A>hgjS1b96PCmGMG)zI$pzc+OOu=xnAR)uJuorDYD`YN^)3YwEJ98H{G+)(N zVl)4;DwoL^cqz$$;djKz8BOYfg}e2A4}XexFb}Q@PdJ`0C*!i<=B5VsCe8)lXqfz_ z&G!Gq?oQfWx%a3)1{T1$@fKxFQ-d_)37X`{)d!r0gz04wgs%pw?m9FQo;7=Jyx;kn zn@}E2Un}5c`CoSG;gNK)!f79G*iDbnIJ|n=a>fHt1u#`-y-Gs3cbph*p`JTbHcP}$ zhpV1Bal#E`^%2}Y(0oCAM%VB2v)#RC&oOKgH03y5taACC8uXzgtz$CZkp+2AgflFT zM~sZFAOY8*`gBpDNmBUn*299cK+goh`#Fbjk;rmFqkHk81?fn&Dl9Ag=Qf89F@%?s@FWd+1=-ASb#wa-S)&e!mWDYiGO^WWp$KlOPQ#Jc z@W4?Y8LT2S0hrDhxbjkfE)9}EC8%&ZaT4_zb$?>=$7Q%Q(>nlIX>{zvJ=gMQ5^+qX zCa`yX9kuXeKreNJT^_ZO8yRVW^$ZPpr?~gO{CI#<-zRqU0f_KzvV5_HX_Ln+>Uth- zEYKZ`$?F{&lDKezv#F$Xi)zNzH9}Qdw|xT!Q`|m%x^TkiJr`a9rgT>atVZYe9tw6aSP_x^e7^p1o<_|+ko5#~m#@HV-(*rjwLCawb}lw5SY zF@-^$KSBTW;yd)7TgCX*eACo01q$ZgD>E!^Ww~%BzG2E&tU*TPhoSeJr?$0U^}TI5 zFKVZ|Wh6hi6msd(jAP3+h2Lao$2io5e@rsf4T>z!Ih^=0a$RgHUra(#SXI^G)9n{D zmOWRP`rEmczpq~Dce`=WUB0xLwP>>8v3Kp(`YExED_SP}&!$&(=Z@B(c0M#dCUEPp zgh|PxWiYu8S7c3iACUOG6*6TJM(Nz0DVDx5XPMQ0;8Oo7vn5yJm|rVp$**ogRxaN! z^PliDPs()~{G6<3X5CdIwReTxiP)6p`{_SQlqyg9xnu=B#jqJ)1A7yj&(;-suxUy4 zm#O>OrSskEbD702CxfT!VD5%as1ux-QW!PS_j;bniKv()MMMn7%byG9AdvfRPRvAgv&^AstwK@%?mS6!a!O*+t1c^kSr29ld7Eq45b+GAuB+m zD-d(;xlXUr3j62B!)7t*_*lbV1~Yx5VpD}!mZ05@G2u3)JvJ?}AY-I0st*@_8x$s; z*u-nsulJP(O>E^^eqOaJr6sTCO+I3bk(@3taZJZ zSn-s>iS?|KXq{4A$NV)0`F+KiUQ8XmRLO3Yp^f+;g;Fh?t$X=x_pvwhExNwIyE&xq zYGc>-6^iIuDTPXwj#e|NAMbZrp~lj#ep4r3=r6SeJy=DC6TQBtQ<{A4$vW78g!S7} zEVBhQoAb46*DQ-=2OASB43yY4R>iWKE5`0_Hd4&X&lm3Ijm{XT@(y-`XRr37iTz#lM9JG0pmCXS&B+ZR+v;Q z$U-gcwU^&%>+apVrM|zIVwEf4TXx|F2c${JGOL13QXNmSd%rkzi}Wo*cRRg-I>xx! zez6O)47)~a&V9Mfp^v&2868Nd+X5@lER%r04g*OO;fsId*%C(!)2b2?U&nX9_8lmO01u;o7TY0e538!AhU zO||({o$e@y$@H}oRd=QhnD=FuPHQQ-9-B+IGdZa%tDqr)hkhw%f_W}%u zS>YHy?aJLpu{U&{9NCdLI*kS3?2E}|Qosy{>PHw1g-#)ZfjR5^ku>DH)BBV~;nmAi z*vUqkUhBTX{?v|-r=h&C8_~=V_moqjBvy#kXVAd^%J; zdfRU7;+5T6ZJw-;qoYz^r{I$tBKLSHiKaa2T?TM+7u4!wkTyyIX)>830S5wQ>_R=J z&=(l3yUc5eMc3k`t{m|PlgR=rXIcM`;DQr{ab3Ld$3A>JF;7`~wV5@JnJ&(rSt>jW0_5>@zxKWMs6szd3C@@ppC;DZai^w!MNu&6#N{ayRAhA8aaL>~}hzr7YFtMKKC@`{+%poSRF1b5=NP z2=lj8G+Mc8c(BVCk+;$^D4UI|^<^CUacEkC`sBp6DBG!BSjSQ3g@Ag3EN|O+6Jgkx zGJBU>Yv5Yk=O6#JH{ZH|@1Cb)iq#G_XqNE9wc6F{1FfZkv`ePVUUs+Jv1Pl-hmbdx z{G3ZOMc)bd>aiil7$KObVBrX7F9K~6Sz~^LMK^+JcvWx(vXh2}#-ot>d|Nk|A7w!H zwasloQr*mo>^z1eiPC_1qT*e4J0*%Vvd`FK^QhWVCW^pG4Bn$tP?|Vm1^iv z@9hg36LOg-txc%NOe>Q-W7Ohkq-J7rI4RmWu3oqLM{!kqwD-JQ6D9+dF3G2pv$ZO1 zMlCdFuHR7TsWK*>&r(LzL1huJ^SX`m95 za@HfDurkMF(TOb&)TV^^8C9rXGJY|xDmJ}0Z@-_o$nx!?oZ&G;{CvCowoP386hQ{#d*Ny?g_x=%muDSRQ@aCWec9c| zDs1pFg2RLS!C9LTHZW|k$t*@2|13YzbISzoU99Ejug~GaD=TmBe5PkpX;g{S=G*=W zcLCUEsrfTC^fap;=OAlo3@iSPVS1}68&cFlMwv~=-}u^h;2;liN<0%o4N=bk&+Su8 zmDtdI<@xydQhM`*699Nb2Gg0Xz|vS=TaLbR6Ox!FBqX%id2Hx(eC}Gq_6wW}?W_cE z+1a@6i58b#_&(Us!Z|izIyt z>0QM(qOZYj%ozLNmhfd^J4cM<`)@5oeHmZkb@fm2e7_PrA z7P&kb*A6k{9jb7>$^p1V82D9_^YJy^ zH;auZnRqdU_XI%b@WfzN*6Wn~20R(pyvOu&)jS5n_0=bU*@T|AOO0`iGZwkBcP7F= zcDo$T1v4x_RUuSu0J~B8aI>GGSq25?3VmCBv_NNt%00WyoMKxgTHXEJIyQCe?FPLk zj&Z*qx+7KX=%CPCtb2P?x5liQj^dqt5OtsTwm_#onRR(!U%pRaU*$6wo~>2S%seX- zTC}#GarfwCGlTYz3}(0iqt#fB>zD8RT}k?_tA%;sEQgq&>NBZnZ}rztbjtz_0q36V zOspunDe>hljuy)s0B^JLmfXmx*^+gxUS3`s&P~Lp1JAA(5h=9I8vV>CHw|(4kd5L4 zEo)5NwXHd~*F&oLE0@r zXWAYo0Dn58|D#m>-)k&^3s2ya=Bv1jl;VUO2|z$&4-EY>jq^QJvI!?01OfnV2edd3 zgG>+{?DAm93TaMbnEd#9gkfFL)Ndh6ilb2ZrZlpZESv8cp9GIni8tMXbLDX;I}=Oa zg{rKkiRI;uzI^TmdBFs?5eYZX zQ$_AxSfITS;dopn->6KhB96(xvh$xTGiIFI-SS73kvjmfOg3j=zj$kPRQ!0kuVL}- ztklg8T?tm=pH5}a+81+K5?J!)x>eaZ&g?M@PC!gwMNUv&;x3EAX-i{$+3c&>h}|bV zGc)2p+a?vg$y#w=XG_*TzoeMH7Wr}6+QrWg1YSsQo3c_#dJh8BnPuY4z5Klom)U$e z3c#Wsk0f;x11u>$(XGRbPxO9kV*zzxQ2|EF4R_gQ`5nI{D~=(^MSR+NcL3HUwDr!t zvE(S5^?o{jD-?I6(meUIR+}W?*w^0$qF3eRvri1mFNzyU)UFA3UvZ!Erpr(^e=lC} zF&0%NmYdbPRqh%dQX}6UmI7^X7hi9#U_ym~L(}_RyoCUuC2)ERR?=Kv|hd(AUDaTPV?Zf-nV z_X6CZAZ;2dHVOWq)T4TMkH@lmRIJ>R9XJHd-<#4ofnRT{=U=ZbwM4bu#LpSpi;3*F z>6@S|v6jXmqd0V2mO=;&eS!E2%|+Y0_an0tG8ek&A+UVSuw~9=To`ebwc6-7yThdV z^$b`rG#JD+1AKHZ%o^HF`?3bImmSJjq<=rE=BQkbl)3p7OS1gAdNy*cFNQ{InDL|d zp;wY0Z&gv!=Vve;WJcr<@{A00`eE$GcG^7@;|Nw$0e)R9^N`eI*nAdmI`-F@)Yiu` zJ+J3gTIP#bVV}47=$?rfcs8D#*&l;6@t`;1EZ)8pg#<0>l>U#pr7_jMam;mMVsZWn zb{x6&aqEgX<;JC7hFisyi^3&pS+#}Yu_ei?txQWojQWrQS5Q#zShIfWWLR^;fj}-o8^asd~n; zsp{XpUPvhKi9=lOXIq)O5(Y~H)*e{WGUVy;&f{`sfbZV)K!c@&dd-{ICoa#b$p3E4 zU+_6u<_<)jW-5h2`So4qA3Jg{?&%nLt&w*|c=(Q%SzoJb!%RJF6q3zKQrkujR zDaVIGnYaFe%!397fxv!(*s=u0e22Fq;F|7LdsFex1@$?G;~%V{xs;C(Xh3jb8ai(r z&FXF!78cI0YPTu$QhB@}^L^;Il0w$aFV@I)9kP3O=;BiugG z;xZu!&j1d6rfOMZ@v?Pz8R*uV0r|Xqd>%u;YK8oO%&fjNOud~V7^Zzl!)h=#x7POF zhS98`kw=gi&~%xna;DvD+AQC(?Q7QS>fJH(@Vk#0*fC`bLdsTDjP_b(R5-;BHnrHt zjp}vhTWA_|D~+b7uEmWl=AZcPUV~U^*B+C65G6P=ktCR^3m)6Mp2JWVUFDxO-f4kD zh4;}W0bU8iYLfcuHjhXbvHrf?-m@+bY`lDZjbS`!EqnM?k+U>5dc9}Z$()$MLhlNF zFWJG$6fLs|7692}Wj%j@minbVi$@WIfsOsygI^rP&_j5*UUHCUqL33PD$n6O?Xt=c zldo|2++uI&96Oj=+@Cp?$r|M8@KVj$hUPSWmm!w7hKu<+pmKGIWm?G)LQa5^VzJRs zsnyOYvd#1yAE*|iAWU2bcAYVt8CYN@0Ivar8quRs@OG(%KlJHF1i5_p!U0`_Fkt+I zb&zt*edI*61bautWu|_#Sb%EHy24+|Djsoz8w$J!P%5kvOaH#JxhQ~0nFPh!a&1HX zPHR_G*6;rQb4@R{u@T6wE20kn)bJnLLQ8&P^wO5iZH2a(cY@t4?jw(`(bl&vKR5-{ zghJ;$#Y0nYqWBA*3F$Lv>5#&q;-wG(m@Mi<0;VcJ#e>wX3@by_4K!G(%OP~qzWs8A zdyo&$iVF{t@{!QmoYao^hV& z3ld}%$@CRa!Ut=~;#NPH=$q2uu{|CbNyCosYg_Zgi~C>S9naFP7ARXx0n^+U@3Vm? z__v{HbW9%^4KZsl6pS?SR3L$(G6I9s(`A~Pn(oOt*;DzBTU4g}5>K=KWXKt*kimzi zjlSlV3_HMfVsKt7h6>rmrnW_zTE(Ik1yVR@pz-*sBNMS0!+9X{* zxLLSjp#BLgB=obWG?^_0;o%EG{d~u#O}W0^P0K(+Avp8_ah|}qXR7F+2AQRxes)c5 zx@R27VlT7*7M#Cr0-^aG55MDWJo$8DxmNiI?yDcDk){CKFS)R)Z$`S()Dgw|mYKXs z)MtEeniY#^5C$d)*G-YWGlXzD^S1o$f`tolww<^Ma1UlQ>CY_3#VOMV(55WPX6l+J zYEsrYjRe3=p9T({%PilmvTO3zjx4w*ZEeaH%jThqzUMh~m8i`&CJ;B&8!E6Vw42ND z5$cw*)_q+Xi*!llH*tp*mfNCl^!D~*=L@n>IZ@yQ-es~--V%<0XHF0^Qi?#_(f`5+ zD#Pmiu$R9Huu}yMD_A32FzJ{e9~?xHiG2|>Ge<0%i`fbj4W~!TjOLM5%93e-0f`0JC7FceG6u7jFfxI8|rLM~yCC z+FiXPS1q37T7y1WM~pHyR0(#|*3;jDVq#I64lL*gP&K2cE*IwC@0i`=SiTL7wylS(UH^jgmjxC}AZCPvOC*)T=+Y z5SXK}_N!ayVp;q1H#?CUbn5--d*Fi74W{n_QEMyQ<}gAZYPR2?-!$d7Ccb}ec_{Ul zXY*ON8n}&uCQ}_9vrd~$&8Kn@1`sx0Xn9y4I-u)Vba&Glz4?&t0Z`ig;mhR46xMakRS;AM{_+F}YWh?#fjr|!er;5!{K(i|oKta$8M z*H~~y(#_@DlZwXA)e>12F1vHOwE_Y<0oG=0(p1EEx*UaL> zUH~0QYdangM!Vet=pE{G5Zy#nk1RHJk#T_a$6VqowGHe9ws$<)E`m4~)A_6Dcq z%iiUfstkAU7R0uEtl*61bA0wK0k$O(ds%ItYAdzVu3bCP_PXScMH?R^De0W~xy0rv zhOePz@QY~JIXy}hMy7x&%O~RaP!$p8J2C|4LLy>=oEIjd*#IAMnM21@%@OQ|K4c>% zi;!*J01nBLC28nPde2+aU^gsHDWW40TXxtg?#pznN<(#zAqv+Wev8_N&6M=%*!8jR zS3OMLzi)XLN(i7`CC~>ES+ncM(1Df-3L5!&~|mXo*8aUA41K- z9LK&2S_|F9gH$~|m8I8%kIlt*2m*uH1bBY{$1QG#r^_GU(ylvWkirhB0^U~2vi)U5 zofGGr@N(MH#%HCZxqi0)Rh8;Ik0H4wN3oem2Aow-{QQU+; z^n~{W#+BEA0U)N_NEfuA6+&YR3e$o(NE?&|V>h?l-ZeY-&%Mi@O6TG2T%~ufyJ#Qx ze8=hH=lheUOl2@WZlIKtKr0MfMG~Aj}%F_M`I3DR+YcuSpC9QfJdU8 z;JDa~K&K4n0g=wld*w&&=+DFleK3tHIVfa2D-^G8r24kvegg1=5z1X*)!lf+ljfdY z3H(k!4_3NiZE$~FFU?W>^+(2xouu4wf7bmT{M)ZT{$F}+3qQ9Xes5n%)K}b}@Yv@`GY_e~ zc@q`;W2r^@CYI%^>4MSm#|eYrN5M(kc-Qo(Jg^d*9f!VyK3N6IfipKp9~+$3$wwqy zO7cW0_o788E_3IvyZXQ*A|cM=kS5h;#I${TcuN_{&PTEfT4~&vGganb)vU5-Ni<&M zS@8~NHeS!HAK>Hd00zm0VP1`4+}WQWgE4e9s$#nUN{41tH7fCq+7e(FR)+=H9G)0R z&T0aAQjC4$jao+*2BJK~xZaFzCoyc%*0v2xf4xYXn|+4*D}rT6D#)@(3MaXyrahx8Yy|?n$Yp8~8jqaSRRfc-tK5Fuq-WIIX zwJg%Ex1JtqGFgr=o&xG2s?h0MuzN3FC=_g0<+1A`KF{AG%#jk;g5w~R*(jv>X7oj1 z+T1o)3B)l>WnX9Pe{K(|d5f5DaGG#`UMz{hCQ7QY%!f2$9&sGA>kc0u1&!y0oegw& zGNKRTQ+vm|UF0eDc`|WOhYId9u4vNEZ2%7Rqu_K93mhfWer$HFQ)l27)jsuwy_3js z`cbeZg!=2}$;r9@Z-lS^N22Kee2o8^+D;v+MxtdfqLm4S!>#Z0$q+UZ6sKbSiwFy% z{ihVrPE&scJxcB8)+iHJk?Hx1z>$D6tR~7z{tnCgl5%wm&TjjeuftI0uU*dJO#eGT%C-;(tK zDKP)pNqVl?ulH9X4j0@^q!Wxb=~bUFT}cXjd^eSwWS1ar6#M{}4#oTyJ0Jl|e8P{f zbnq%KB>cGL>Wj@}a3S&IAHl%8U=(wxqF{F*oq4AT>N#{KChDxl;~m#v+$(M+0M{yB^t9>gH#Xi_P$PxfPgGqEH zk-4S~pseoT@g${z9gO>kqqeUjRtmE=qb=-G1%+2@R(2H?os0+K6j{J6DdGxmahVt+ z@z<_3)R+7uvP3i@fI#VR8qi@s0t0LUZsBH58ww2YY~DyvVpg9)S(bg0gQf;7Z8f!+ z?n>spfA1*DzC0wkAQa&eC>ryrzC-Edk8DK5fWGZ#Z`>PA4?{Q?*pEp0muc2MN++f* ze1^ur|GYQC(EIaHokTt(O$3PEh|V=8TM}^{n=uNS$pX4(Z;(B^5vn7%MD7!O1t#N+oD99snMwJf-r)GI%NQ2=pl1g{kF$v69mZpnK}56k*_d&XQ0JFmWf z@-<7s7k#95A91z@GZ*Epp?MM5nJt)FIR9rY5*4OOpLfm2zbhn2kHcIH3bE+Vy9c`W zqvRP(_aw)P&jo)>#@f*T&s7}uL8LqpGPLONIbzOt(p$G_q8zQSzaLck9C?GyB+|rt z0AidVTo;94paO8+61iMd0PYXhI)ZZY=rvq>SLw*2F0b#%kSEYFNQ`|6$^1Ag)bC8H z$(IWaHwB7FLWysE>?hjO&Q=nQIQIoRlwj>+R9Hk_1N-|Bz0&)bQiH1PY2+nHfL6UI zgt59dE#7W(8f^Sif@?$}v6=G2xhjzbN9=Yh2@p1J0U0clAkQCE7gpcdsGDSP$zU^ zQ=p0L&K%6>7OD7zk>D~oOLRZH+r2{#x5ChON828k+-IJO%qFU^NEX5QC(#A1cxgry zS*Wb=R?Xcm9EUH+??j-$`C3GVEP4*zqnmvmScNzkZ9A>vKEsxn8Jsec`c`DoE$Oy!W-caO z&D0XaQ|PBmvwySKSzzWz7^6`Mj|`23IGs=^ZrAZNC-SBP%sZY;axVgwtHzLXlf{OU zSRVOc-RTcMUxT5VQ&w3a5q$%zFbvjf+LQ#@F<<=zP#E{dqF_t2tf4Z1*$Q9$%^WHQ zaiW($?I&zxl+5`DD>}j4-Rz`I2)R=G`{`e=U*{KL0LvJhmQ8o|uE@pni3uVKkVH|d z*MIJ1vWjE}Ye+x#ZdfRC*o%o6tW&=!tLXfKEVWLE|#vTi~-^RSdkzmxpy{K@c%amI#K;q3Ax}v%y_@jrVYp(hU3Z9 zzYmoJRr5HYo|R_PIm3|o>zkCV+Buz5i#K5tGZuTg+OenZ z*T;o^Vf8xsa=)xcVnE5DO}7mcf)z@xu~cFct?u24BLZ@@bj`g%{E@n@*-Z2-twA=4 zx5E4X;~u$hxb`Z zJH75Js*ZP{33X$}|6bz}J~r{lQ#o&zM9pm`1y!Qvu8XFIziqJYtlD-&f0}RiRPnf3L>re8aGAYEnK%LqKG6IPh?!%U95Ud=UNOJ;>H`u8CwXI z`%7+u_}GeWVt@%RlvA70UmM*cNoq<0wpJ6qQ!(~D1x3ep-P$}A=U&ff3~r11k^bv@ z966u{)J@w08MQyLIT|5vxU$$eX=i&U(m5&I+NZd=8c!(O`#%tz@qeEp`5$oqU%~zV z>D351YQFGEJdvPdeZ|J$YbjZhW(B*L*ts#FRH6!6h`_+GnL9-Mz$tnFLC!k;<>SmGM#1THNb92tZ*97g-biafpCn@lN?7$15l`gOG&^qQ^9xSWujSoH-O?Irtja zP7EN(p#Bpwh!GS89q1~kRqjAdjE>!m-1VR+VldbZZC#CU7^wj5oPlLy>*+(|+SI)O znQt?h1}VG1SR#VK)9=W_f}GTdB#-t061t9$Lg*4U)*`QFg14W)}dVIFb z_ZMVfg?{e{|CkT0EMVJwczylx|IH9U3IX)gW|q_Hje3PY-+@hrdvDsF&r_CMwNU@O z8i~>*Fp%RI7r_fM{ThOz@TNSuFU0*255HWUOQ3@YAyTN(>&yTgER6cc*V5zX4Ur z;2(|QhOmRnxD>?_5ojrGkx^R5o9GoJ%JNlTk;Fnx2yqz$hhtHpk98Q!>jpPKH(ANr zY}ebUsY6x%llx=6zF7qrPG{9uC=nJOe(IkFl0KNZ7!wg~g{wqhf-gWagqCy9u9c4)))_Rmxziw5N8 z*teO8N1NaSa$+~8VTTIE@!{Rp`Gdjp2i|Q(7bz(eqShUt+UQ9T_*a6ej`>)N&r3i) z*Ozw}p2FaKd_oj6xLA`FaNpzw0b1W<+M!zO%M*@cvWRuXbdI$wF6D|l3sE%}mDU}y zo?>a6CpL7E0hSB(Kt4MY4QPa|tCHnB9uMh)Dx<|xmM73plJy9*DGoV;0c+X8PoBib zXhkTAWo#vgNPb)34H4@O!qJWS0dtp%M-OW$Vs12>38C;%a}~Zv#8W3+VN`@In_IA4 z74(T9q_+i%34%=tevzg@!GQAzjn=_jJDtEfyAiC%J6hol=b(P&<*Dw3n3|Bg)~^wQ zK|bDB+q{Fq8SxeWND!EHwFo3*PsnxAJYLwx97n5b4Vs5=ahW=8EgT3wOi}^04&1?G zpIi-3a5>NoBY^Xdhx{Ve1&GM1!{bndk_&oRB{;GzuN!uXKm;JF7ReBW*3P>qk}<(c zmM8@9-083Sxtq@r;ZGS}DWdXy!4+8+lD5u1f!;@@WqswB=b+2Xu(@Z?trhymN7*)1 z-n*toqv)dXIt}d$eICJ%XlOR(QUwM${>=j2==XYfOr2i?mMaKm8BfkcHz#(%Y#PeO zX5tLW0u-fp5!AV!eHd#?L1>V;Pp${Trk&66p#5y3xu(ifp%$)m%`6SSV{JFZB!5w4 z>%n$biBJ>e2#D#T)Y}xb3NlDFMOGG@9KkT|u-FjfQmZz}57k{Jh)R?$q%!T&dw3y* z&#K0@Tdv3U?~xJuL9}Am2LVZg7tU?jd8gE(~j+6*13Fmi#*59d8~n7yLCJ00V{Ylnn4H^ z)2gPhOBoYMG?P?4VuXImN))8q=5wVmS0hZ)P@jO&r5V=X04;1U61?K;EIDtFiLQg%>VN@PZ`(x3Q`#6yv z_jLzXf?JtOWEJQgmx$wZbtutAg{{P=IW%QHiemhMeqP*X{r8HUmhL*5?S@7NL;ZYu zwAtUV%Y;6A_Kd5jGqhePy3a*d++E5VxYyB7Ib;?fu@m{7tTj??;SVW>z^8_%ADTjb z7c%kO!IA38LjLlF=Vq zPpDcLtqhip@5m&w^z@S*-cqoN9H5@AlmEV8T$eBcz*EIhI1$*)Rxm@}1L%Y!dXk2e zfie#WAv!5GRd6 zKjI)Z53tE5QT{dyc#st-PfRGIOpGXbVu?x5q^?HH@uZf>6WC=^Fy^QO02-w`&}vZ~ zj6T2aYji?U9%!k)S2M*L(a08ax)wRd3K>>=DTmDDta#+nn2r)WL1DQ#RyDVu4&wKJ zig%PMsEUZXX{*8wM1fd_L|Utf_F)f*F>^}@{3k(=C+aVux0@0D zqkSp)iHOUiu3n>RGBn+C8Cyg4!(K$?=new(FP5Bwf2tDSr4Kt$+k_w4m*VzGG-jG$ z>56DlN{gN$47r&27ZNETqojO)HPZ_0Vlyb5eQX&dEg}rHQupD(M+&U~tDArj$yn?> zl+E7^jIgT+2!hN|gsgD|G^z!-5zr*mMv?5boUvjKlSK%N7-|-qoRV1Y?_j#S2O(J! zu(>!-n;T)&R-i?OX(;#oEYh%ck_GAtDG+-5}x5 zBZU=bHJ3d~PfxG%oC!VG@R%W+fG2q&`0{DDS$_h(Sqkr1TzLfo#!g>aU(u7Jm|Q&l z2%}&Pqv8Nz7H=~xi#$YMG&~#Iu_tkq%2tRJ^B=FoUmo$+iKzJ3!NS;(tJvxF9*Y>& zQtbPh`(GAPDN)8xm=-P$=*HPsqfYeg#a}*np*1ou-4B047{pH7!8MdJ=HLC#SDUJK z+k7?|OVJLqj=~p~Ndm$&*`lu;qC0fxGB%AMA$Hn4D2-`%!l4$`8ION;<$IQAqj(4x znF8I}r77r>#FQiwoK1biD&>p~|}djXY`;sf!zb`Ti|d0@`NMzVur<*MWU5*hlv?qwx5bT34UGMiky!4ToXf+ zkM8CpFI;JLa;>}IOIsj3aj6>0UM_(Ay*cs|Pkn?bLiki51fD_f5HlA+lI+o;9^<(x z&TMe&TS!nQ^($WK=MpEi1@N8*F-ReF_)1e9UKj^3!sKKwq-oiW=oMC zmz{sVBY#^9xp3HH-N5MorgbXX*^S{3Jr2kAbgjoz6+R~%40eM>W-W^-d4Mjl3V%YE ztX_nq*n*!~WU$=n3deC)V+plfiyVGNvn@z8*~v}o+BYR2&61FMrFqRdTIr#d3;UnCmTqI zf=3W%AlygAG+B!qafB$+Vo`cBiIDb{az@fjL8|j)f=p=t_ghnY6}u6K;KH%nx;cl_ zERn^k52U4ECV-H&j}5MggUSQgl9m{jty;r#Y^SZ@Eo><-AmvKaSc6|FZ}xWRRHXf+ z158LY-rmJs*^;qnVnCmi=?c5XJCICUZfq$!ROro(+mT=MRvbZm;3g~K*ypQB4j?g` z{L5s)CT*WeV7ft6ERZxn(>Aam8b05`Csi#*G!=#QC;$Ir@6F?K&fos;>=R>|Fm}RO zRAVW!CfPf)r5%w;wi1$(Xfczqg`AZ&+X(I1$`WO*358T9JE=z5NkqDz#~E{dulsS| z|NI{J{dnBV)#LJ=OX@s7=jZc&AIIx>y4Mv>Yq<#cho&c&WE}S_x`(3Gb>A{Tkx<0-aA6)q%r84?#90 zP1EOBUK_`&EYXix27<7;kJ(xoV!sxONRA73PcE2k+Y__2M(!jLK4{+m{mY{ULmf&_ z3aSx=MN`N&)=-{J2Y%}CpLX-Q|GMl9M+8v@PRpTt0&dgQixIW5(`iSTA^IE=+yNk5 z(liOMhkyLPM5Xxi@s)8*AzaafHI!&9sMO%tH~d5ODSmUlOToTMPZoRkHQ^0)q8 z>Q%rH2>VQVCHabawa$L_TMGRh>^=kT{j^aU-@fYx;}YY&3#omLzH$73pl zr%;K%HyPAjszm?l&kw~y{fYE&j7=e7$zX_tUx#EFV1s$uPWq?r9*}pdca%8DLoT8} zb=m?ZmFT~a&k0>{^zMR`cpHzgo(R7#@T|)KkBc8HD>(qXxEkTf=&3`rhzLm}lOWoA zT>9zL@+N_^1T4RWVoR9P{Z~o|b6#JX0;k0MDA#4n^Ej-K zB`PfygvORnUaS28k)X({L;S&FpU5JcG0uyP*|t&^yfkzw-%_e3ISTwyNO#VLD&SRo98;w74NT-QfAsa{F0sw33{8*Uk=wQo(SicQUn#($tx;m^tYyBgPoj#AvNQ`E$Nd%Yh%aevS4SV#z9QV;z`jw<9R~Lkdy0O*ZiH!-8XS=| z$du?dA{JdSq(qr617$?p;c#JfdkPO-{A3m|iA{#QzwsQ`nr;5%%>()T=Z@~8$51#? z>Av^r(S`5D?PCecY46e^VzI~6&u)45$5LQR#bVjk?S1a!wU9?lI&@T1W)36N(S;FU2B_YDS!GCDRNfu&E`1>?s4dJG zm!yIcYf;h5hQ>A5EYHpIp%`EUhZFM?U&|!@p%jh&=hY9IA29Q?b*!=X$&b4ov^{D) z9Owbh`Z1|kI&UQCe6K)68{iO%74ZXkA`8~7AL1sv;o%aBklwT#$b~t%af+Lt5fe{r zkC^a}maE;5T>jZM&gjw|ke{SET z-jgvN7?q5t0$_=c{<+um2pYI%;nj9*EG<}O_Z4bmZ>|AymNS^*VUbYa*nwl;H%0=v z?dUl^8H(2R9bH?yWHOQTm(%}XaF-_K3^7(?M(@7l85d<)5=UBK{PQX7C(KuPxkk9wVbZzbxZS`*@8#6_(CPTWdG@MhzH zM}6owlgdTV(QcSQS+JrP&@g#+%5_Pasm~~i7?geWr&SX{gOm_GCV)e z-;|esJ>KrydWxF>0W?O4CJ`saqO=K`U%^YQusX01l8OF&nlO#{N4NMSJDHoK5dRs? zf0IsL;0f<-QsYYgSS^ixip-y<&YyIkiv0$+M75GvEG;cEf<(?;J44o6)k}zr(vWap zKUrjE)5-zcC6J$Eac)Z$%-E?{i~y^SV3c1GkTt4(e~#uB&R+Cj8Bp?Kfbi=KKzs*; zp2OM^>yv%SJYMcx{~c;Z+OpSKfU``)iaA~d*!oM#8dmp}WWs~4Qp!-=e*JgNhouic zGFDnt+umhF=^myU_Z%b9)N-ks+tet}p*9vhL)zhCrI)H|d^|+9 zUT+*=H%F<2DY)vm6cx}ox!1e~vPz<^ZfXBUvPjeznE?g{=%IW|L%;3IkQ*RzhFXY< z@(CKqE#fm{Cv=n$3rD87^1FejYr0(?RlD(V@q&d&rOQt%0iCM(;8UfFvl#7nzQa6I z0=FyXUlEYv@_l*4k9JN_Q}0ps0In04QaQFMQktD`7tu{%Ti_4X*G=h9MJYhYwpfsW zQiZ%Y82(ky?6Uf1ihqg!N5k&W_tXSlxj5Y+?YC9IWZ5%f0+3 z&B~T8W}HZasJODpYP*oQ453=G#5u&}9wIe{KNeB{(Zg>J-C2U_MFk?q9Gv1LEmNSO z5)g_<_y+($F-z?__hy&4B&9N%B~ycBu$thqdWizROWS^XXfmnqE!x%7f1QJrK$1q` z>$Zrv-(k>_D39sYU#rx$JujQ}A6kcWPe{VfTOs?|vj4ZmlnvC}4FU4S?K)#HZ6^$R z7FpXyK7-pqdbRkk`pcnDjT2V>Qm^yBO$;r9)N5P5YuQY#|H%JqHmv5g<^TWx)en#S z=BVa#2j(TRA|=XqgbtMHtIr;$1!NB>43OPS^WT1qsMVsx z*lM;B0bcyqh`wRFyzfLGJ&Qv%$+Cl6L-nGPEA{=S`lT<27@dTGut`nrtRydqacvf1 zLoS+$j80*BQO+ez>ul_S#~%H{h^jEpUfXN;_qAy@q%ZOM$MyqyixraEWH$`lVT)>^f{At~ z-j-0pYA;OvG@G<@{PfrZP25gC?<}jAuefCPNhKkLgT&K$4^yjL^}WeXm}11JBU{bj|Fgw}F-cJc}Oi^r04 zc_ddH#DTO7bGvK%!BClkt;Dr$=NW$3_2h%-iH@&wyCp0w4V4$?UdMhY54bo$>_Fs{ zt|8|dM)$YRBGsy3Bz}2U7FefAl+i1g^68}?#d5nP`v>0h(asVXk|L%~VIk4P{tBqN z9Z-ZT;VMzaVb(n;inq{-rVKrg+co^xmW$3>@A>xbf1o`<8&^)**sFB6@4L!BXe}50 zYkML7=VSUqIiMsU`W>W5T27K0ujz?vNPj5I`y26=i&A){$Be62t}MQ0nBjtpRB)2p z{&5aE-#%OO+k<1LisY1d%Q#ebV*CrMz1pSY)sRXw_6>e$j-t-=p{i^T{4(CC9)W<` zW#`73MsM`vrHgRJc=g;N3Gad+3cO{05zF)%A+$)>)UCyTJ+yt{Xn#3RqnCZY;C1Wp zuu1M)-)-m@Flkx8q1F5E^!zJ^wpB^3pf;93r z&|)fOvbr#b(5Z3I$`~eLG4AweVrTg8+h&*M@?I7}T3cdrUmcntHm_988Z!;Hd&{fg z@``b71O%x)`ZRTr<}+-Po6?adQ)Sen3y>$IkzR%gyy%c+x%I!+NTBV0YS6<9?9<}T^ z(JRkZDeZ;_>4y_LQ6C_|o$jsd)_q(4Pkb9Vc$NBfe+-l&D2YC_ z`-&7aK#FcM=;)}-Arm1I;<*oDCU0^-xe5LA#nmOexdcM^R|6u$?X>&>sSGv^e>LZi z)9L^1?R*afi7Exmxhg^{p)_xR7}28g^WnY)xE27}WPZH53zM#PquWz+whZ~!gSj_B zrrmjx40=|4B+5kx=9&F^>fikyXd65IRoEi%gYM+FJvGNl!=VVRtS{DyfXMX~NP+I}@;|rQXqbEV>>aF73k?_Pb1W827fQOwD7Z>hWSZe!CL3_g&`d zzO{C}67@5XfKe;IJ?#XCkya=+Iw-e%E&W3r|7{M9T=(=p zw7zdu!ozjZ>zEYGuv1DVQG?Ca#on`V_aS#e93Ro(E+X!lV+}H> zBm93Rd2lx~C-pE;d@1eTMqZq+(g@#E))#;WB7Gd6U$8j3(89uEF@xB|Bg5^@)?@=o z<3b0uhQ+U#?Xv4NLX*cjecv*nm@lhi1pjbuN9^>}R0+58)!pQKmj8cbv-qXih{;x^_wwukUyp%0Qfi(tO@F25LI+^q6lI z%LFvYd*p1Xi(<=Bq`OONj>rqHexjd>ULlkH=v2a{Qp7Yz@%1&dHR1|#Cx7!;iL593 zhpbFmTVzg&RI;DQxw~PFJBhFrb`D0eJaDzZI>O(i2# zq#;7ISv8v}yn?!=U4|4Bw5P?FNWt}R$v0)-TxdsosUA@d&|0N0L`|tey-EYC6YO zTI|CI&9m;#@8#AEh}KXF0F@^(4^lzxU`h_a+$1v+#q)~+CGpM~2qmF^lt#egJiA`< z>-8FcbK~8S<$&@sho(MtLn?zYY(79r87L*yBfI7eNCQ78Q-AFD%UI|w*ZG~0G9ZjU z8H>glxUk<~9bQqRdC5VI{tG^L8v-vcbQl{8oNq zK3>S}y?-6v@9!DEix?}u>?99{{{AWp|H>Zvi!VtBE?%y2LXL?XAqGS-fylf7xq(tV z#b;Y{nSq09Oo37T#j%H*T%lwuo&9nU zCW9u{gpmGdZ#IQxZMWD%=E)SrS)x#^vgK-t z9zsoudrX?=k~}ye!dvA+i96QAquj|BTokHyHFLCj!>N61UIQ~QK{D`8Z+t2;TuB`4 zqEa$S##q$io@ZWPkipyGh0^n`kfPnyM8>B{kDSa@m68%aY*Mb*=kHw`w`ja?X`*K6 zd6SQ`W9$d(NJp;d@8ZZJ{q+)2N3glf!BvU?SwwGORQoiJB2X}UF;i>PC)S<#eGGgggA})q1M93=q zdMA~s3Y(U0xVy0VJWdfqu6(V24^HT&l-jsOtIukaFK=`%Af|tS(upQXu(Rv)od`cN z_DzwA>uGYq0|X2iyDP)aSk@4i#1n(N5}j(>{=2N~A**J1iNW_MkhTQJi^c!0fj}xY zCU@)5{&NkYX6md5+?1U{qY46-8L47`_&zLFfRT`q;@`hBP@FUJ^|5|z|9wI1+o#f# zjpBwGTj*=CXNc7;CzanTA`4~V;R}<>QsVV2Cu%P-70SGURli8^l&{iA-{uw^^|Dan z0P1EhnZU>(H*rznh?ygWC@J(7dVSjx8O|zbMGg$8PW$k3u*tg8fX^S#KGWP6)G?$< z?Jj3WkBr%y6Kj$9TVPLM>}&r0I(je7e&9Kp7VZ-!Lx$KMaUNCv^8QrLe;fN`j+$Do zxoWd3A+v{Ts=^2qjPvO>>~nt9q!HNd%*cTKI{(iv|9|9$UC1gR`|}l1$UM7Bwp`1$ZDTmPJBqs4 z`vvEsfd+S_sf9AI=j3Zf;DLdLE;yr6gD zCIIjdWU?DFp1$MB!?00Bm)6y!uMn*xR_cWuIgDSCdBI^lrwvl(F!4Q?li^v-V>mxK z#&X|pbe?r-+O(-MNAyzAd^G#^XuQ1r?;~J*A5uvZxcxRI$!z$uC|9=+FyT}Z{QR(8iKlo z)ZF2~3~I`?#T=h8<%sjDwe|W4!^W+6#4Ui+%DsK4O`qGizt-j)lICCNY}iWXw3Mk$Te$zNLp}z z3Ow{p?CW8qn>kx*kYZqqHI}ut@}LEdbGYlYH2-N3qmyv7gi#93*;;Qv-NFv@G{611 z40#u4J4^v+AGg@cD-Q}>M@hetEwYu@=MU~x2%wl6&W5Uu?4(4C(cGd^Ovr5worC0a zK6*4mo=$d8%|)%#2j^&g`?C(FdMr&UR{Uwf0>>IRtmsO(HT9H^l5~56MQ*(0A4>+u zW!y<*CD(0H^AKMD#f_QNve$G+07R(Tj~i}aZwFN)5xJXveTC_Nz6}Q{yJf0$?wt0D zJ#+S~F()uGXV`4Pg6iG69D{3xv>hgkG^GlW|9C5y!8>ws#1*kVqhOlU8Xs_zba%`@p<-GK=4HGFKq7God zFNHrD7ji>a{lWMY|aJRhxMA+b63iuf`S7Y^DGh^Pla(sa^ljuDDMgQk@9bj;uP z>$0L^S5NPOnRj+Azo|dueekiXW^HM$IzV_cR}n;7#h&GXUD@1qb9sFof=s|FnLk*Qv1Trs0EH+VlRNU!B@%$lR8Dw)Jb(YO5kztdAOjVZT`p*T4KW zv1!KJ#OMZvRgYW0`tSeI2>A zQypLN>8-;HgpHk0Jp16qziFtChJHs-CL`H|3ZK8Xpxc42wnOVuCo0dQQ)LF?HlE)$g?1v3l{vl4G$BM7SRuuo0{Fq0eKF%32;d<7i z%gb(WJ#pfMKc7w>5^kwn@i)Gos4fx$T{eCLd*+^4tfDw0F~uF}GY>$ZqlAM_YNW&2 z(Q|>Gu6L8;2_*ZWif_6ur<+?&7?S)oZ=F~*G_?N}pt~Cx89neGIyBAU?$Oy< zF?#~OPUa3DLeJPwx@*~EicPIrwaB6ye^C$l3~aIf-wQ#x+5E_FY*_bwS~?aAXJhp} zh@9Rt`(M+5d?#|kyafk>IAU_Hgr8+^BSpYul|Q4$(w;-8o;m);HO(Rc^}Cg#d$*swHT=k?Ph7xoExuZS=C@ z!d~|#Z-gplkz=wypmpf=?rllmjU5r`Jzb3%cmLz%HJVw3C|>-1WCZVRO&qj2wYX@& zXAA)6UM}Se)8}|Nyd*+!=%3?i-bDSsR$+ZRS^UtXiCxWdj-e&%YF?hPT_<9Dy}EV( zq07N;?%XI@6T~DpBDrJ_Kt;LvCfh$pt{rd!TGMe=!8Fw>Q7s|2+rkVDV1DKuFSkMJNPCyB=?l(92oV9Q%)^DAby~r@b6YwUMl3vfdb15 zj3lzZxCeTT=MERkLQz+;(Qow#KWN#2HxQBL+-@V87LMmDX48&wvvdCw{BJ zhU$;1p*MNQk2;*~Je<3c#|Jf@b)|(D8sM-D08Oxhn>}`|!}oGiekFRujz&7|2P`tf zSEu5x{|T!a1NzW3&4)ehp}fMiJBO>cR?Rz@#XNrUytXQb4Y-e(UdA7|)~_S#Xxb+@ zPO`MLjBuA=Lj|i)#vr`^dHcJQm)|$D*gD(J;)jL}t)Qe@{`~Vnl!a#$xRtcH&Mc3$ zjoor3+<+q&!L*XSr3j!vzY^MQ6G1y^&h?J*PHEFsJos5j^9ziWd;X;Yd2PL6J2Z77 z7&t@SO{^^7z0g*W)cM{@Ohxh2xH6xga4YysPH-+lg~}AB03^US8gj%m4W_ zY9H8cIh+Od94hc1&!&AOerh_XN>13Q`*c!+@pzdkIlhxZAx0!L|xS-jLHZqU8kK|e0jlV>WRL~ z-nt}l^rkXgosL3Qczph2_21wyq_=45U^zFq5>Z`#F1+WoDIC9GiIhUTEq*SOhKMzF z?w|Cy*ic~;VSPCp~7aTD>|JByuNZY2vY~YGkHe@aut~)vfVe1{;0lFLMFvmE^!A zmrSj0>egfpv$q#4G@C9$l5@Xza#FvY<6&%$xN!};IOw34Qr+Bgr6x8j6~*@gIRDj7 zyK_2DfnSt>CF58e%_vq99j~@nZ?9ry`2<6Sj|EwmgP18T`pLyoc3d*0Y+X}0wLl-YyYHx0C z-omF@a3=#HH++Z1FT zt`OoQ`igqFX^+wld#yoT60QKPWwve}TZIEk6$yp0uILx_4l_T%)QWw87d@z~F&o>j z&GWDRuG=g{L8l712tPe?k5W$jsE@n6Aq@Ua*xfKIPK)3^RARmyEt7`TL~|9 z%)0d3{^?qLuNrrE4z}#L7ufIf!K?1&M+)Xoh?WKnjlic@pkftEVW##$75xd+iMHG~ zJjU_V(c!~~A75UvKwS~Cae8#KS68_j?W3`TqvLGe(#t7<4!r3t4eY|GJcw8a*;4#% z9Ma@c@9pZ~pcve5X0V6%Pnq{2*>aQI>_Kx6?i4?J7LDu4+R6XZ0TUCGh;Q)JydgMF zjw+F1R-;CaG*wtpS@)w>UAcA3`hZ2a6?ARHH0gRE z6gvaa?JX@s;8M#8KFvp~zx}Az_wrZ$SB=$+JpP@0Bj3^X>hgcH95I%G+sO+bzI=}J zrHS{OC!+`}gHyzSpqNRJ%7fsDm{woo+{z0VFM5@i+RoWpOS>`DhRHcd)x3mhgTs3{ zO{d&YpO-yN>rrnsP%C=JY!6tpY~0c0?}8$YCSE1k9pWb+8;wADlP;r(X-z^RX+(N%{IYtGI8`K3fnGu6XELs z#C&@oVKJ~`LZoYYYf_txipzn#>p3&{*x1^taXvmLahb%SE$R;#puUz9u6g>P@^KK9 z_DsQfk4`-xC9?13**;tAJ?ZQ6S2I6&J?~B<&z?Oil@6>y`m$BEn)H@AV;mfziiEj; zhTdsO=`ib9L4;$QkdI7Nh+&qP%<9fh#vwD0SPpIF={(CUr@Kv5$b^Uiot3V)4sMv8 z9ii8%)zK^Feo*6wAc$RQX7Rp@>pLE9&gf+=lcUhtG29d4-jTbepP{Mq5X3|rS@Nx#svpzt$j(Kk|l)r68KEFsjR)@41y7r;)s#cb|0k`cdGBq^dHaSZawW;B9`Q?-Rv7E5}<%fsXN1V?%at@HI`Tuji<7%R*3#t zny*XqfZ1kMCf*!hyGyV$N~ev?Qo_sGEBQMYWV&NIjD0fCpZ$ z9}VNx*SBP4jwH=82Rx5CUBWCgV!Au*09;QVl`{Hb_c>d)&BiSEbtbZG+kuNCu zbl`<4OmEfkQSi*Hs1b$w#G7CbTC$eM=>6P*s?`cz<59`)S9)fCerRJ*M`H z?J-*~hYg|+DuYL};bR@Ms26e#!;6x8Woc#>NcXEjO5Rgp@Ib0pFJ2tM59Y#8ytrhs z^3ogjfw@)037tIl;cg=az|#48Nc0V6watHMYz7L1sN_} z;r=l7{#x7G%C5qewrJlz4oyoQ^UUBQ3#Ja?p@HwuyK zm#ym#Gr6|}YYP=eW;vhmV=)d|@$vA7R}T}x0~RPKw(x**UH4CUW@v`jp7@gSu14v9 zUA{aDvL!jWn(kBmX{V3lmQR2-lXLA#IS~sHGW}8=t4JWK4XKbA7t1W^X0c~!0aHaX z8l{`pJTuk$?MwbKOJm8D23gcnEHzB*hTH6DBZcG1MZ*MzB;8v|D6_$M454Mqqw}^qb*}mPfe)F} z@C3eGQhw>}#6$xAbei|zcd5t#{`@5XgWqVQZ_U+WQ4&6SYOzlR4`B|?8$XxXprI^Ku6 zEL>;o&ywp@&h?thJ{9%~L}eufkYkV6ADvCFcN*+H4!e&^#?C&&T%S)wk;5EdIHv)H z?76K=Nam7I*k((n-_%eY*mgUV_DY96&AENCtPLqHm?x%^1WJp2cR9QiHJ1++i-jdwUQoEaaW)U)r|0_fGl>8k%z9uV&_;E401H@gOUj?PIt zVesbd+jV)_JdPcKff{c9+6gm;v(rT^{|AffBp^v>Y`u;Xuetu9s#@w#@6YetA8n;* z(TwYKH+H{cvw8F8N}6@vLvRm1@WAG}#^FAM%C4E_MnWOVxf;Q-v9X(75D`atQrN}5 zBGb<=zmE*uJhr}8g7f8dqg4#GKW3P|4Ybi(NfIAa4AXCy#tmvmebqJdIX;`oMYS~- zqqxd2AfLO$!g-4A07}KLwt?@&>U)rV;C+|4HJh*#f4v{kxgYX_ht(>x3!_Gl-T~Fw zPN@sPCfxNqXui42Xcf&?hBy{?<@IOQ9*zC?%#CbgY&@LvG*tT`%BPl?kzI@lU`E`| z{;F}PVdFuLc@;z*d8N@RLPxNCiUp`|qg8rVk!#il2lwR+WPW z?Tl2{7`O1^(z=yFnm_&+e4*|}WU-if zRDJy_m$*MTGZ~?C%bq=Z=ub*yS4^$42-lz0nax*nkEiSrp0LRsMt#@_IPZ3djq6`R6fe9m*_tsHjy}JPta4&yX zUT(|SRvp8kfnj>HzohG=ep(5uQ;G-7nup$XY#o&&xrb`=`GQRAD9g^^lB<gM~eC_C~Q*eA{rA^5q!p*&V15{t`bw-R0KWGh$9gu&`@D z0Ie>!oyfpje*69R&`LdxP4j^``csZaS#0?wnr^JAckkc7Z@l=aIfBA?Htr;}r4~Q^ zv=0aBn4Ud9IE6A5`C{eL3v((j(HJo=hXvBtv`q z-AC>K(Y^3;f2ypE0mg6qrH%Pv9<%|6{D={uVT<3?m~s8aqxRHiKuh zd7XHfwjxb(ZiA*BK`76jtB#s2etY;$$35GWtGBt#AE({a+a}6-}F&PhY%nzul4h(ktzYl`5B&mC>q+cERM-6{mRO zd-v|uW>Xnt&-m^AtFlZYA@Hkx$rm%WAJ{~H?d|3YCy>@8d@|u($21WkQB42Tpi7ue z9DAL%TiY~$Kcc;ic%*W>$&!1q@g29sjXF&gNuj4(WUbM$V1IDM$uCR&wRJ-lVO0v} z2rX$t_jz$HblsXYH*lmz<2rKXbL=phwGR96ei)4O(lW{pqghwxax?6-f2EZWx4}(* z@I8kj8t~Ky`Mlm+KDmMU1Pm48h$7KXlYyyt36@Y>I8-^;{-J4Q_Ci=)95lIvop^xm zpyMz5W=^iypb-&GW=g+x>z0qNZ(HR-!lW*HAUhR-FLbjDG|yb3rnCDVM&XjTNhc`4 zp)pGy9bs0};QXr1@)N`&7pj%`<-@>D#W2;>J}F|eO+KY#)GCgg+yiiU<_x* zUz$_*(kqYj&jEfc@6jqeZ|f!3ym|A+P$j5k+>#|rv`iam|Ju{>;tm!iLWX%ou`~cVb z$y2A=5W$bJLES#HT_h!MP{>?~Nc|2zzW@{acI?l$oWOhPthT03t>IlilWNniuth&NxISW(AT21c_n9u*fVLz>v>9db*XT=Ua`m`^;gB;<`+adOX z;wx^D*B$1i3%o=C1{)I$Dh z%ZU>w9-DE#cI*pC_=9)>^FL02P0s|*mMuL|2dU{FpW*2HNS6n;q%|FLYKYOR`DC?Y{DmxPe(9tz9xz~E;HG)9gz+TE?4h~NGAlRzYiq|V!zW3%g(MOUh>({Nj z4YSvvGQC=F)E@(M7p%CI=rmM51|Ia8T{Ay^`m~)0dGU@Z-jJ@DD?si?KCGR$-XKIn zF$X{C@#*h6dEML1hh>y}!Rf>C1aa?IO-bb5^KZ^;y{~Shk}TX$J+}B7D5g`}aRaF6 zyz)j|+r%>X#EH5QmP1}IqG>5VyphKNhvns^vm6~AgB69v#j|mz%)Ffn+4YXmylvoH z3QJ2}QHmGxShv~~)!9^}%t$dzXMW9J78kV4%`A>#u>Lib)yQTmVwh!`(k>>epcsZ{DV&jvcWGBCB zU^sNtx)z3fjtN)Z)>8DjdGn?&*|!*bZU1f)cdwY+yR$-ZiTP9+FOx>9#IP{l+DQ*U8BFp~DPP;~-^nLxTdbea>}##(+?b7ar0bNr;vhBlXjLY7 z>WqEuTA9h8SrUH1phjq}HXHVfe9^AXZ zQ1`)%3oMB&y3c|Qt5Y+2PCj~xBkXR=7((w8iz!LAwQPb-@h1op<9!fGw|%Q!GMyt` zCJuVAm8vJ{C)Q5$<}q(7DmLYPVDq=bEl6vCXW_GF^We&(ZK72LNp4#Wiyyuk`lNDc z9mO0WMK(*P+RPJvIEoIMt*q_+u1;!ds*I7b+jDhmuboU5yu#%8tIP_^fBE6Xq92EA znje!JK%c!%Sw(gI=FTsBO+PfDLu$nEbcv~f% z3snZc@qc`wo;6&8dWS+_cFjb0%xs&+n)Gh&4x8L3$u;k2!Zs^h-^2>8-i(v9fXp z#LJb4p#cvv^J;JErF$GlQL2Nx;j373+|dOk4)m2|uMcq^^2}$`nl-)EhUdT*^T-JK zJ(l%yN}ccS#85r2!Mkyf?!f8M8a&uF(`WheW4`UJ;%E@cH#fp1csLm|(#6vUM!;KU zK?Ar_l`K%SvA4HB_Upy^-RvTg1?9t%YZsQ3%&FM-?(2-<*NmnS$#K+}T>yb*aJa$) zb0hq&N0Y(Xx1_A4Homd%cWPt8-Q-SYr>ga={=C%o70Y_olg4^7D=c{R>eZCj%Zge` z^nOQg$>eTRD;_p_g^) znlwq)?MQ+D2tzz9kbTPs)%EhL&eyfe9d1!UlmnD-75Sx$c~Y~P6ivlzxi@ev$(<0= zcfo=MycKg>A{>oFdapHn(6vjKA3VB_qHNi%$0leQ_l1#J*9ZU$?r)4F#)=y}eOo}U zr_liG%`P!^@ug3%tPEOv$~Gly_A%3|)c6IJ&pUdih=^VG!IWx$$?VK|j0cLGY&g~f z0{QO+%i-GNG*^RCFP~~LAVlj@+=tGYxe3$mo^BAS2sL)I^Q?S7Z)UI|)=*t&KS0h& zkNu%k?0?`$UI5}rzkdCC!!9yyj*C~@j>OU(e|@0YV(-#iJ!?GGp8PlE1(fa41mBNL z(2bkr1F&vi)|%R}w`!c3nVD61W9@pUDZEk7ywB^-3F|9_8@evwviCr)=8nBKaCCpu1u_2Ken9NRZ+W8c9xHKF*<* z^6DrveetTp#b3JHu~{>1_4KVIcTTkwcuQ=63luNB80JAYO%hxAXnAa?5VD%?KULQ3 zTfCVuvV(Xv8M-D)ekA$DLd7rIFzZ5@gP}9@jJLKSrfR#Opdf+Bm1KbD$^B{k!EkT` zW_$_zzGY8Ghk=WqTGD#u+~HzY)_4_;(j6^y>S!+JL5W#Ws9fle7k7@%xt;(aWUiud zxlt95MioiLoUz;l0GQFTk#<1}EEIdK`NHK`Eo{JvDW6udd8Z~COrJizVdKWOR5(iz z(pKMYWU}^T`f_N__moWC`4kNsHq0T}c`^KYR;FqcoTVL~G%gh^5_;*AmQ_zI^i%>r zh+BYX9cGw*XL_82VFwc*%VEQY4f1#rL>y2k+nTO8$?dyu=v>@@|2jH;Jl!nQ`kU4T z;GOJQt3tR|{;l7Fdr_SFnu+-xmFdmKdBlbEl+f4qV`BO+4^#a71m3qFKIAhSF_8w9 zyJro@S|!gfko)VF^D7U_m*Nbi>peN!jg|ZM{rgmn8nwtKCOKCTwBU_5o6d1`dD9jE;men=IG8~`Mq>g~teyC_=QCbOO( zqr()JD{E8D41ehVIbIQsbf>f&82EHH%Q|M#{y$du`4trw?uQ#y*N%Rlz2j10;*JzI zK;Vdzmtp0@a)e5$DOb0;X(y4Zcjbbr6;0c{B7@%IypwWrb(yfNEZsx!1JM9p?~JvY zX^T^Xp3dion*O*G_G!{o)^_92>= zrG2h=)v3gkKy93mkRW$BUs%Fz=y-YNMqj^vm5NPS-Xd^}#J94JF9o{O)6iRWjcqrx zym8~k#xP1pz7CP}<6SXkLqUJEKk52Wei^4R;4V*T1IGV|lCexz?<>SW*-&hyGmjNq zqec?!@73wfwoiea*|7uL=j*TEKE-dp{T7ToNr)_lr)!eX)_fbPa%=X1H5P*AKmWYt zQ6Y;_3xP@Oq7I2~C@<5QLZ!~GYdrS=B-$xwV}0_zoA#-=U0Jd&Gm^$wg*7$VA_0-L z*hZ-fKOkjgAJsUXOX2gOq}#HGqtQ^}?|vFRX3S1huc;4fNpYou_$+k)<-@^gis zFjqa-&C_Z-4$q0sX}{0OUe&j-;cuhRX)`l;a*}JfhnJTWnrl7N;BzKdq;*6mYtwK3 zHa)wzZR%3-8oMU+?L3W5Pt#TgLDO#&0@^9+OND<(`j^3at(rTAx(N8>G0cI3a8Yb$ zzx6p^KXMp>zMrY7DL?^3)+va7P55S*AY*oaa`pjAYnKYw-l`0i6b8-AD*$Y zyv-q0m|}2_k)XA&9hi7q?RlY3t&Y$601U#~CeMU<9Bt@=7?5$=uzGsP&z^=8Daywa zZsKn*r);26ZRe%ybt`CNvq5ju&c3wAFugqjtqI3f0Lp0%DgRA~u+Uykrh>4wj2^r? z;q%5h#h?+L(xEM8)!Fodh?L*T1|23zcWtL`+s^rnvG_T(b}!;boI#&hn`q~CE%dZX z{D~m>%P&_ArK0qIYETpy*}@*@7~Oz#6x)$*(^INRa^s;k<70yq+qum<;iDqEcQP}2 z$6{%xJV^9DatGh>c%U-80#zI?mj%I}DjLRYTRV}fQ>w2HUkv1EgTj|D7bv#VRGl64 z#~-?*o}HVyzRh*Yn7(unZt9?3D4!8$sOC{!k1-f4C3%~{rR%!}w<<|R|6Xf6^ifgK zEa|kb8(wEqw*<4{!Svy98OU32URFbC?PSH=UG+EY#Fuje>nR$`Qby$9{&0k5pjZPVZ}HUPixezEJyo>zQn3u9VbZ^4TWFxU5cfqj{RD(N*1h&xGS2!e{6+Vv0Hy-&*3cw5QFGJ=NP|=>a z%eoj%oO;i3%>-Z_g zLLyLT9}#BBg@%pw{X=F18v>{eLAtsa6GfzIYLZW7GY+5onOJ@8XCVBT$FE*3R1A4t znnwLJwfSy@<_Q-{122WQE*!!YZN_@=L;_ULo;{gDHY7hesIKg|wf*M*J%g(98q~0J zl1PNv%jBl5)6KNS1<>SaY9rxGmCcsBcdXd;+qd^>7th< zrl#2n&y4ypL`#wGL;sGhQ+Ahvsj#ewN5`i64(@!uZu(%WKWR-AhzyT(z`>n}vBA@y zHlghx0kqZH6Aw}*dDPc(u2IH>tABFkb);k6NG&`vTNJ_%KQIM|$aF1*QiL?xTX!mw zYOU>In7$m#SbKZz5cAY8Z`FzP?!du|8AF-FJU-RQ zK^l6Ku6?qaF=H>pXW?_ooIjE4)aq$kK$?s<3FBGla;ezE@N*Qup0cvrP5OKhuq^Oc z98V@Q%9Mss1Tdb?UO3c$2sTYkBv0aPVM6VOPvS@V2`2Wf@43YAO$k@OC-x>_wv zlZIA=3ZbRh+@T3(+CNF_p5*2u$wPkG25ncL?A<-uqHgeMMY z-cL|I@=3f%ekkBqI4KryRVVlRr}Ng@*4cFZ_HBE0{~Js7<=fYHo+ht~dI%Bbn#$gy|cK+gTnscu2&cl+dI&GtU zQ>+EV(_{XdXcuL@wxofS>^F}&j4`P0U#hBD05ik`Mm;%&oFYo9$ewSKr4RrW<-mX@ zAtNZ_8ctkiS=(v@0Hc(tJJKS`&}Z8!MCUgj8vpP^$op59OhX*o%_=L6dy z>V>dz8(i2SI*&&pH5NV3Lyj+8aXtuTB!*S*i#@@Z-ZDj0Tl4mSDQgPa_uTFDyBSnR zgv(=)UkAAl^eE*eoP|_TZR=!KDveaixP#kr;K0@$MUJH}bOdLFMTp8z1M1&No7xYw z;slJ}#P@|pJC8cRqhkp)m?{#EX7HKW+MV%m=f!Si_DLK+aJh{nC1XZL{_(arD;A6) zy*9pJT0nKckR@qwJ)(qgIr^wagiLEhZPi}iN0|}yUShzqL_;xN{$V~|rSg^MBbW-L zq#WCnRK@$tFh(dp{O$11~FW>Z1KH@MFn}H|LtuOKDFZc#CGu_5$t1{nyk{ zuO}Rq^f1nJ--Iv>C+kelokK`7de$EAUN6O&Vt zI{qA~N&Rqr&%?N>CwdOc5PFPIfgA(*)cll%wg*ozyYk-R>WLf~)9I6^1y2^1mWscD zeIqD;GKUbPakoRM9q#)12SfQV82*pEugpH!&O5|b(Y2)uVFQW6+?8FZ3S+F$$lcn{ zxwh>lGrot|bt2>ZPF-8$=s-VdN3VpMM%&3asItu)kAR$n8LOJwoo0bkw4wiQ&_HVp z&oO@414^(RuV25e+hGoSc(e+Vvp<1C`BZ;pw^kiHE}$B3QTFZ(MiZcUe%B8B+4A`& zy{+m4ihiOJo^iIjF^bg+=&52ngg7O#X7?f5C6#reU7ZHJ881ZA=d|Beaw=_WsMD^Q zOwheCaAJL*k<^Gr1hPH?9cQTV(0rspuHyCU-r~Dp9)K~}#fTJVtP&vK>Dd5*6S@XJ zD71Bied954u0!34VdD9cU0I`W1nQl_>aJkN)!0)=%U8N~kOMX# z+Ms!UH$YD5;govs^o)JDW{TmG`VdQxnKs(Uxjp%&o&4T!_FC|iTZl2wUH(v4qo9$` z#x%KDJNpKq+!{8)8gFic`ptte#}`SvZjB6s?`Qm?MThTiwWI4XAhHmFIlTolZ_H|2 zMc~E0+)#lF$KZTM8ye`S;Y6z z@_cehQ)ouU#$-{xhwfc$qQ{f@Ajc-^L^4@t#_bQ(P;HgPzc)2qC{VzS1r+%;3!)-Mp{8}(cIbom!IWfA}5UVJ04N9S~{=lb%& zSvL~TY>^p2k?kiN1-ksh<>fcjg#1muukQg!px{JvEWB4Ck|L>VotLpp!VvyiRjQ|< z$wa*Yatn}BUz@Ap*A8~5TfI|iQ{}D!9v94wxN}9^tm12Jtf8c-C^i$9%g?R9D%2k0 zXs1_a1=O0}C6v*3LZ70FrMt1j89qlJP&aRc<;nWjG{qu;LhAG&{CkH&i+I z$UKSkWsOZT_AKdSwwf%V2AnxfZwgV#Q`&%k>Ez>L*tw8LnenOMDYych4vX^EJmWJE zDho_VjG~_tRhE53^cyg=f>!NZ7wm)&s`l^SpC!5pH$wbXRpzxEv3HP2m+()B<-;5C6jU^mgZ3jR><`(Paj9$zaQTQJScAFAbV-E&K$=X)SpRb0f&Q zq`UsOxp`whx0dap&-&opNHJv-8G$DO+Z1UZfBQKz9~-iW4{VOv$lOQnh-sw3^fh0c z_$*huCPvmNq0V)2=hKf?i97p@-vpo%y?v&EHY7q zmbU7F3i!+=F3e2bSiP zLdVnS>C@O}`A{&@*Crhk z2pqXoEMgu7$hmUYyfeh?z^Pi*aJmBv=V@pbzj~F(ic+d*SJzVyLVyI7nH_#q*QaU8 z7tk$r44h~whDSyN=qSa{BjZ*`*_qTYlOZ&tklbToPJCHEh4VLkT6%7duSQc9W2?zw zBgHjJ?eXOK#V;CG1{hq0y4=y}HC7aGjxEmm`&hK3-&^|7De+Lwu8Hs^;|7E?CTh#T z%4yhS;hAh;#l<}%wpu`$G%|~VDu0DtTa7<_;nD~$MhXXzjb~1uT3QL5(WLsi(Eky8 z$+h6<0Z0*sJ2WTwo6ECLHTZrH95CWCXUesDK*QTpPP?+4ag!L zE0n~!bxT#9%(I~qA_doTbE%oa#p1|46*5}PO~@ z>7Gg{*QD&0ho9bhL#LtB5df{hU*Xu?=**ZZq%g}8kz`Oe`^F}_rGt?ld6PV5)F`D@ z-1YbAtzQ%z%h;Dx%X}bdOQeP<2sFxa?iA@lJh)x2)@h%BSY_9JSY_Ib;TV~`-W89R z2~jZBm4cicbV7P#qPR^9R9BV`6Vk23<0K%M=%xn~dk5}L?EE~=XhL`XfIUEVsfnMK zuDk>r(ny&@K`u2jSizC)k_Sl_LroCHW20#;6zbS7X&!o)^R$z76ifLR8yr@YV=T6X z{$?u5&7vE39{Es@KEJHk{{(7QmvxrnF78f}Tpl>$H%;@KfcNLR9U=p69hIl<-;_~M zixw@i+FIYlaA%gt>T2sc6~^e`*84Dv{FRqyOVD9@{MKpJ^zxgl_~uiYIVAlpJYX+U zvmeJBh}l^LGMSs?U4_Qf<;S~ zX;}Thm13JqzuWQlmvh`m*$`aw`WdyYjw5Z5sK(0xyCiqGmrA6T_=hwVM3KV=?KgR} zw)}a4H8>&%Z6O}It&Bo!i$a)9Yj*sL%JgeZi6dhG2g+&skD39jtZ?MWNKH!oVF0=# zc8#kNDV@xn{WbJ%6=pXzul8?Xg-=I1ur?n!f1cS*ms22!@2f7$-DLgHhO6u3^l(6P zTQiHFnl+oz<9-hr3v`o9=)rA-kehsBaTE&af(K`w5o?2Fa%x}yFFuXi53nS&tQQL+ zSCvs(!LKMQ`%h{V*#fz<>HJ>Uk-7Dql`>Aj3o9k!+W@;XBC=ai#5SD{E1&8b8CeN$ z!-hFvb-*GTE756|)rbnYM8)Ji&t}%vBq-2AMFxM?eoFtq+`_bXbqrSq0edsAUL6ZV zKOwG(j}cQ-R4&s_9+pvLBs&q`@Y23mP;?l&W6qX_ioa#-8pP6Q@WZJl?FVQpnbW`3 zxqSWhPOmX6RBSOdTTB&0VfVzn;`M7wSQ{3Edc*~NA1Gst2^tfZDxX(T+#uwP#ANow zEiiB_$;dRPB(z%Yc6}`_w9Cw3BbUD(9rk@JYxvJLGndf!R#va{tC^hlErOZnCXdLu zhM`U1Es`23R#1aWiI?P2I9j@)5c3~3ZWBEmI81k32LqR5&X*hVZey`p3db25Q>Wu~ zR2Qa!Xj1N1tOSLZ9fGK`P)Ufdf-jbEpC?X7i9r-i;t>{s`^M>&K?9gd#ooz9;t^R> zT!W(UpciKCuvv4xDzr&O11-Sjao&}sYoNEgcA(9pua+08RtAAXU3s^AGu!J;f%UZ6 zYgS7_V$YBpG$tcAjAw3>4VP+rc*c1L`|vtYU@?0Q9|yJOgXe%Cx+$BUInXpDB8b** zj6rniRMh!hlN$CW5vqu$Cg9x|D2N=Gfftn=12P2w6i)8PpZVyYoRt}jlpBIPq!e>M zY3NTF3TqOxEf-3&*7*i6KR;AoC_0N9^C_DocG@_ido8^w*Q|QNqkyaQi#TBs>?nR+ zmsaNgpz0~28SO_{>qd1uTJjM&Sn?#xF%Q7~o>PLO2q#AA|A=Am!XTu1#wx?uOqCH> zfIP|p#Q0zi$I@rZkkz7C(ZYJs5M@ixh?rGbLpuy-)=#W3_okHolOK@DA4q_=(HEnc zjbeZqs1oa(F#x>5%Yf>)!F9DXOxDt3vCPM(nYmd^OLzgG#oc;Z&c$`)`7AE7ps3xz zvo%+Vf0g_H3@9ocW4M_m_wr1~|8L!$`CrxL_Qgrfu2~lYF{e=DD3T-QK#7QGdXBR= zA&QnL3XV83Bv3bRS|)bD86^b=1V<7z2dEgr6i09X!^DY6^Q)+U%6+e+_tp1b_BPGLu;p}o+t?MPqpPL%RoBU)k0Sb(=N~!~6dc^dv_A~1 z>eHn?Jy%#L0JTLJyNEy}0$N|217&j?x?*Lw7u?{p=%U^rT4FCzi`o>J^hC)tY9LtrduPyf@GtRry z_jc;4tMr7X2hTQSkJ{~>J0Ugf8iKvZDLiw3vk9|3CAQ|lT;MILn#!W1EN z{b-cY2NoCEGZTm(>ZQ}T)MO8jk{h?nYDGAHk4{3#CI%T?4gr- zem2wL^iI-iErxC+_HTi;ol_3)ZZA#fiCju@l)%ELU2)+yJN{EAll5C|=%=|$15zHD zZ;YX$*09Nt{N{M>xht>R82qU<=LH6iK&85X65C`Z4^jG_KoA&{TI=;Ujqk6rx}JUO zOzb2sODd{SvsmITnVknjbZtd!_XY){{a=3Zh{e)=IzzjWiZ%-CLo|VJr^DqWt^|8; zN){$v z3|>5%?m~n;^$hVINzoAv?^C&=KB$adaLX+%lBcea$ zV91+;LmKd8+%wY9i*(|2G|9JAS`WZTf3g`^1}>h}K7ZXN3mWV0ZuLbSt6?{JzK|!^ z7SsPZ=^Z25^Xy;LRFA>Z(TB0n{jQI{spmd#F-t%uJWQ&*6Q5TmmE$Wem&S%onj~;% z-6d0%%80snD@TYt<*awF5oRfeX$DRJn#&1lI=AJ^*$y_r!#n@nI8Jj-#(r1ZDFc+E zY|R-YIzP&DP6P(unSnQvx!oZP-frPGfirOdq2WCeM;1+yD`eSzlc}A;56HZ6p;k}qL>zkKd(GM!s)c}bQ=&4 z$5CI$zw~!BPJZ%+E%}UNaiQ%&8uCQk4NZE>*gSBEOTp2_!$@eO9)(9)%qLf^$*>!| zD*ja_kn=%(W7ojb*2eP`hDnroSg8!4v>AG1*Y&r$+a}w~>L9{ETd+hCC2YGfEw=y= z=5(w-cNu?N88&z62)3?r*=>M#1P%U<`+>7$Qp^KfYzFbG*oeI0kIx?tkunK%M6#mf zK^``*{2bi9eOHVi?$hL?m0%kK3qE$R`vCbi8FH zM8V(lXnTFQvAYO9E?#{!T;_Jzhv{r*S%Ts!u?OCgKWT8@=$z?tRJLO+ch}?VgW+uQ zse64ms%m2y(p+&g|zIin*0echq}*36i@AOd_G= zJ($^e+S+B+{rdMm>6Z)+geS{Ij5hS+xt#kv5f14CdSrRK;PN}56=emhdshW}nazV1 z#MWbQHb6#}l{Y%;x2{T8fv*e^+JTDffV6M~bN$9iODa|?yM#3P{mRt$S67uKpP7<; zMH6oH*!=ikrI!bcueC}RLYXIbvCKZ%5^_vn2G3c*xEP-{o(b@CzxyQ@BNv>%$jK_A zYlOC)X*z;Ea^-*XTHWEra^cUV z8|8>efJo1mrhPNFdN|oxPDeXdbomj>g?8S|W;U5A*S1pHdFR8uSP}=ilhx?BX|%mQ z+{#!BCPmRiAr`0t5Ft09-H5GU(YCS$!06-s9y~(&uT$4?!EGTC7-Z&_eQ~+ko?cB5 zso?~uo5jUWGWVrm;P`#FI9BW)m=@{R81gA;rb&!tO|zE91nb_b0Sb%~tiuQktwLT1$Y(L>VXWia%2OknB@~^8CY$a?!5vbrv6K)phnV`?sy@X1OWbuu zG4=oP_S^Jv&>)?0!&QX2&JNk4jrrEGV|7ATa{bn?nWxU4$vDkpvGZw>|>z;I&fje;HQ{r&dg6g<|S$DpNg zdr;WkO|EZYU`y8JEt)j(Cs%HC-mhupT3^%yGXGmXTJG>+y=g@1;~+Kqi&5MCu)7_IqSuePz-BGFGuq( z1L#$d+!#3GF2}?B=%U>?IJBo7vR^_wXN}19J{GiJzNRE%CFo}5sz77voQnH}lh_7I zfnvrsgi-iJS2h`_n)r25@o;OMEl8SA;cK<)TKviJBZd3h&ao(_daNp}d9i~vAI;ZI zDGjGMq{19*mgcyLZ&EC*isA6ftDs0MhR(z_G~w*Z(U58t3|*E96z6|5yLf}}> zwQU!+{x=uQ5V*+UBttYUkCLqg(5GzQ=^&2hE-Pl}?K!|5-*Uup1MzG#u|j%wh0~V} z+it8!qydI=r;w$&jjh#d8*PQU1eT5iVpQ+6U)mOTu7(XQ3PUcVJ3MK|D4p!YBm;dp)cz>X*VZxo4+pXnQUN?tPcY#Tc1L z@kk~*Yt}~BV3fePF*GP`&SiW)q|DLBsa30kO&;DKSkHWl1Y~l#?Z{A`b@wb?tLDL6 z)kD0zyka63rhb)I(K0gG(}P2e`1Wfa-S1VaMgo1s znXRsC+2%!8rp1niknx>GGjg=SiSEyv9(NfflL-m%=e@&Jb2VtO2=7)^uop`%%a;o-tTO1+$`>AC8d zll9H|vxj4eAn{L!rSGN1YsPqwfI5VUz3h_wCykG`8D{`!&6ZvP)GDWd8S|c~f7-k5BoWIBi2Z23UraS+{2+7R zHWMuO&~gVC7cWBE8Uts4==qvWr9+`SxXdCK#9+_~nvm&zywk@Yf1ERa09qL2I;8}| zz_9G&j70OwxA7eBk$%cr5tWq+o?a8btF%w#oE7H~{Fa{0T3%V>N*wZ1ia%!Gt*}$9 ztl36049&yoI(fu!Gk;=)_!>1;3ZlXKOF1spT3flBr@;FfO!tpHT#1W_0+O&6Qnp;c z0qj$@lafqFilU<2=j;yz26H_9?9=IY7Vj?Ko3Q60>P`>bylgFz56*%Jv{o*ElHMuz zKub+?Y%iZALEx?WGJdiiQ zddG|n(QFedE0;guAu9mmOTufaV;9{kNB|+cOX4ExyHTf!7W+p;U>bWv(r)pF^8~#9 zkxVokaS58 zhYnguKZCv*@A1`q2;0PPj@2?M>b9^v67@j9iN9eTN04i2m4%I`rKPb{Zq9dZGdFO# z{FAxX?s5lWCW4v$DyYWolt!0N`y2Z4B&|)90Vl9`*b8@4%BxXjPmLV_h(ZQhyPmmJ5zKVJuJ=UM9_iv|^V% zu0(WtqadrWu#jS!)Pb2+L0hkKP%z{`^^{TBLkQCMQ!KuC=JAKcUb|d?e?fS$;1WFr}WvWypnZngWhvfN&l{v%X|W@Ow+_z z*cJ=@G#D*K`C}64O~k9^2P4sh75{Pk==W3jDi!Gm(PL&{fRb6+L%!1s!$MPe1?xD* zS=EinIYMvzWUnbt*3ZMo!t5p15V(t;RcH0su^-tSb(c(qv%*0GeNPu-8MzU(HZ!=4 zb(FxMsY!ykMOl>6qmo|8d&`o&xoL8kyNxItIbDJOq8su&X>R>32RfR2i0P5@t7s6h zYuHfpRW6r50kNM!CLHqcbdS-DnxPDrW3nufFQ&{IzkA|wZz#RVNm=No4v-Laz=uOh zQyo!0Ui7R>X)}s;#uzr!+!0PEY{o{}$Pw;7pj|oe9yN!JYs#}0(Og4u z`1h#Gc6O=&Tq<7q!a)!Olf;~)S%9Wu3z7x`u18u_O-roDE61>#1S7mLWG~46^rWG) zC}}T@n5Zj`D#k&(IU0$u=~d`EMIvP!8Kxr*s#1dERMXC%MzAZ}5TPK;63`5A3}X%k zxO-l*i>3JRryYTU>R)W@mUueIiPvj^kQ3 zIOR3}S#H`Icm-M2vE%1Hb*$z#-{_olrM{KB_l4B(a2yi{^4J&fy1ZRB@46LfAmcl( zUHqX>Z4CK-d4&p^w@~cuf?_RU%TuG=)E_13@`MT&P{Dz-g+*{#x-nVK)9$~>8Bl#D zpe51Vx$KRhoY9L|IkqvW<2|Y!@rj?6qDK&FXo=KQrU=Dm^M{j{0H3_wqvdN>*SlH< z0(tIZ6UhB7FeRgog&VMPFL?|UIUS)^U`j94@Oc^hei!Chk-0UTNp7ygp(~w@A{0f>+zgR1Li!a@Lz=Hm_OvuRGn?H--iL`^UKC l@D~mK{@4Gn;~k%de_{PwvhRibR{E}{Ploo}zlh literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_seq_bw.png b/benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_seq_bw.png new file mode 100644 index 0000000000000000000000000000000000000000..7e4b6f5f7e212924e79f52ad2af9501676ab5ff9 GIT binary patch literal 97872 zcmd?Rc{J8-`#$U2DK6_Ye|FPFzd#!!1^}f&Z6!(39uIs$c^Ei*=IIn95wKbR-c^D~* zV&1z){SZab3scl$SNbLR6X{8I1N=k&!Y;!Ly0)h;IGfs^qV}6!IDgjm!dXi*L8nvp z4wkm(wn|7zNl1wcT3onr-a%ec(&j%uAz^EOM$*1<@dVz5;ryN>4iv@3M*c^0CbH%b zMMF`0)m8Oe!uy(?4Rw23mkcPFE?+^zS^PTXs7|)gfblpWO{{7-)aMBd~_m9P_eWT5w{_mIGx-X)a{{0dD0kjIjtp9#D zo!ckkY5)ChYsJ|5f>r+gZtA;y9i9nsgPkrD18W{`v-COs+Ktm$c5-N9(423F8%H3o%Dp49z1wo--PUfR0GN^$-9^XI&? z^HScSa`QDn;!f9_Di~Xy3Mz&<=m$2 zmd&rE&$Z++EMLAn<6N_m=x;nzw(IZ{%~f~?b93`Ajg2v(Y>{HCssL)xm&`|{<*n!zHsttUrDE;QvjFOP_b*rTOIOSz9X zJ3o8&tjcO$R8%zE={FBn;MVQim+)Pp5dqJ#vOZ<@Va4ovKJjDw5&es%rlt~(zm{*z zvblaJPSY}Te#YqeNrvzCIr@=)ToT6K@$nn9_V1UvIM%l|FnM!HX=x00l=6;05ED-s;_F zYAnF$=27wS_qUhT&rj|9^7U)`r$-7o-Ie9#MXxW8xApd3M;N^8vNt?(B--@t6^pur z6&D<(Z@$zue$f)2oP7O`@V*%R6eEjoS(|R$xM6=oRpS2JZ{K(;Bb7A_4Y{{$*>W)J z?2&CYje5PUCWW~U)oUAb~)5tX_8=+UF5Z+z&AdK=OXmtXbuwGh8QtCOTBYC81& zeNk=v0qN2q&ttzEGe1a;r4`H>QGz3XnvV+GwB~!L)jruzm#CMtef8F@TTS2J5mq~W z`t<5Zsn3SveP3f8b8?;g1G-xBJ;KsFy!Q6>^%eIdBieGh`BVK zDtqC_K^pJPPJLgNa~x@JEbuCzRDLvOCO3{8d2!m(G4N+o&WnLxpOY7THnp@&)RWk} z`QXIZH(NTyRY;zzoHLfi7YX8+h)#^J<_@-N>Oa1u$Fg#9#UDbQ(81zxEMsQ&i;5CV z86QuV_n68!_~P{bz{-OB{QRnPbB*>t&ABD~No*yP!*i3Wu6IL1gvAY?scgpYPEX-8 zm>haOaXx?kyzXA>^HbNWk`8ZtQ&^}H*Eif#nHgaE;oc^nxVSh?*F-%rtcINwT4Gj%SQ2f}dZ4G(kk12Yr%BJTo0cu)W zHzvZnM?3}{+IxC7T=>&;FLw!pBkpCnez#01aWBImqf9*C(Dx<5Y zM~@1Li81zkdbCqdkHd3z)IfdLF4Nxn6tS+#h#lAQEsF{ZsZB?-eDSS5MJlI_&UsD$ zmKl`SO*|w#@}qh&@?GKV^tj#68sW1od8^Wnzb<;P^^D%`d7Nn8)vH%i)W(e)W##0e zky`zOg06&x@uGnH70l1^E?=!!h<%8PkH6mW znwXqyAL{y$?KUcg@Vsq(3?(*H)^6>xWy_G*X)fJ;-hD(hRnpJSL*&P(4(|r-aFVef zZnp`*)>~Bcw6|ZvrVDM^!bY92u_3iIO3jb4s-Z!{!a@YqYQ@TxS3V~jn19Q%k#+pV z(fHCfEi0deUtZ+npF4N%_*7LL4Bv5aB|dawuuR}~c=!#Rpv2(jTxZG0heVAsEG{J- zPL6S(9NMF;eI+t7a>9eu8CmB)eDaRJ9v@O#vxWx2=rT1Tdco1LI#y$OUD9DD$>Vv2 ziMsLU2Rn9QZQ8p(hHaLSksKN7EDd}cv+Mf#wnCbU2V0jS+Uq9r*vT2n>G;E`io|S^ z`r*Fhbs@tfy~MTQ%3jB9@^9Q&W;*_-Sq)W{juM;9cp#;+{w@Q;myP^P_hFf-jTesv z=twd1Y-2sS%T6(}C>RGy!?n`c;>>h}JV>yFhiq_0i{l z)rm^wXIeKT1p|^K9!xQkCaWj5RUGO0+E#W8F%xcC-J%}E%b{n_pR-Z;CM;GOMg^X; zM))qDla$V!IYZ<9?Bsj4LB)1023K7abk(%w2*0}BS6TSCc^)_fq#7-I^m0f@$TP2b z&&-0iSoZT%BbL!dnO2J^Q>?p~t9|b2*LX_;;=^R&jB0 zT1qW;cVJz&&dj~4+FD&#x+edtxx>>oSZv`7!2CywGF zg*y4}g<Q`zUQI@ns$w;6&hQyU_+7nfj!H#&n|My{vdp3dZtH6;5P0?KRj7>ZYO1Z}^~G$D z>FsQ6Y&ee08#Zjvduk}vFpIC;l;f}zkFKJown#HX%LQPwjoYnNziYr9O3Z6fsre?qA=zjAE<{{8V5^D9|p zG5&-9?pP87=_rJ;-EjB*;^N{c^=oWoC5>L5OR?M0Hd7kO#Vd1;mRcsDq}b|66^{I@ zO+WEw5spJUIcZ57bJ4+Aje6D=1LXmMEv2v_G)TJ!VFzk|s8I<*t>)++~^r?9Et5+RJM^wv>myhHfeG(HB zCzPJQc(HM$am9Z&irhRrVsSt=WFG`s!%%rgt9TSIQ=#7>?%)jsz78~w6n>gXg#Mn(>1S{=@?ZeT$h zQtnuQTuQ#jS|K6RbLUttUc8ucX4-XjN*b_ZaHwk^qA(cyAO3!KvvdB<>I2HNzF)o= z>LnbcIrAx8^%|S7-cV>*n6=}M3l0u%>p=&+Ja>#_>g!yR5&^e9i@oceNEdxK8W+`Y zwJ2MuyBn)?-ofE=M8xVP%U3_$ep8U1TmFIwPANV;=;+IH(a1_x!TWhIqFuLPgOIE&H*)Qv^5NevTdl-f6Lf@btlq)nH9N|+TFKoXn;F-i zfdWO~6V*1|Utr*>oN4oI<<`>`g{YkA=tO`cIk1y{`o~R+{Z~x%M0!25ZB?3??1{7* zCai!5Yspt3AEc6e<79)hl^Wl0uyuOv5fx&wIYaX1g*bo3GpH7y4I3thd;Fx#OX(@! z@{o;8KY#udbL`rt>$ANv!%`@yxH5U>Q}FrmKgZIW&}N0yXh$k-qF7O^eUa>=4=3x3 zS>ONl>lY)?SB0Ixh652CIJHSmziZKuGfJA|>xtfd_)u)5x8XpiQ5N=$tv<~p&B^O) z=h)~dC;I;wy{wDle~grs8IdbEG}gz*#WB2p|Gwi@mV|_a*yK$a5TH1Xrz+37cEB<)kSN(wyyxgkpR?~b#!|53Fj zJ%4^~TC!ob!1l{C(?t~dSXYVUz@iAzHOf6t1oLuJL&Oei+Q^~BH^q_VIg`yst7@2i zKDu%IJc@Jy8o&EY;xS{7sHoD`0vV&Deq!HqN zR-cm?&}S*0QLfWsnDpRR=5)=;w-Ac&-Kk?N(E8G4&dx8*obbk zuQ^v{xrPR*f!VSP%oHaRx(-SO=cX#h;Y`YhRXjXN{HhULn~XA4P*vN0{|>~V?GAMw z>?rvH7KWfwBr>z9k)L#p7-~W(Sl@(QOQ|eyiya4tM+HIwQycXbB{CrF(`aRii)L10-lWlNzwML7&qR9>p! z)JIVyPWh_kT?S4N;El7U5vk-6JmfX{*#H~1hGn&)KiZ0DG{<7GEjX!5v8O3RqJ)@1 zVMN7TM3FK!Ha3Rl;f>e)>{W!ua;*DKcX!3Psb0yR)ZljD{)XltfsKYoI#n;%;!KtA& zI6u&)8W=k<%u|qct|^-)$Czd3&Yfs`1P`2Q1@f0i!J`rRVo_G~#PH@TRF+$N)&YK% z#_hY$%FoXX7?5F6t(H9McQnI-h9cBaOP=dxkRdBAoBZ~tYs*tlkGxod#d$aE-|dsf zH9I?titLN3jxrR3(oO0l={P%z8E&lF`lK+@=2y6!!;)6_VP>!cCyf}C`(iWkS0;gqRjnFfa|mATyd z{8Bt;r=GodarypzE>HAJU%q|Y_)-5BFbSSz5ryXDVeeBV*U_m_Yf1%GPP+TO(&aM5 zS2Yrm;%^~w@nwZ&#vk|{&RzoQ!EPH6X~O_UR?^_Fdb*;aa^1! z-a>487KG^lvj1WVRO6*{&978VOavOzOv(_}Rk_Z#fl}B4qj0(?{MGsM=e^O{+2p2xaw$$r@lS)7Hs79XqdN<1fuEr83>BeLi}8b8wV>k84}ITzp|^hjDNEmq z3N6Appj#`;a~VLzjPG zUELwq0lG(xc~|Jll5m3oF0;1#b`qP5|3LyRimo$8nZ+qNoy9 z##lj=avfx{Gy>W4E~3T~Tq)v9YCdpmwq$GR#- zG>OO$iL~7Pb<#^IV#mdM@l(i1CDukdUh!xpcj@cv2f1>z4%dnP$jPy8kRXglX=$nE zH#*fqAmGi8zw|)qF#;H0PDx1#{R2#{JU6kJ+I+4__H0wOj%Ctl!Q2$ni!+t=+%L^| zMj=DAM|#ca?7bNjwDSIr^6SfIk7r*Tm!Jw88+pL<{s2K*{(R@|-Mc5+uL?Xn->w$e zPj}$Jff?j15rfq5Mi#+caZzcp+m%MF57pV{DshaCN^RY$o40N z%(%@z40-r4;2&xP{hunz>@z?QzTRxFc_r;f@^qBTa5pnD6d{HH$xZ7LbO`j)T!j-u z^~<;SJjOBizj1?Om`i`MkK+*=$D^GH7yp1=Qk8Xtfi{T(t!DS|J;L9z0!)??!6RZf z5pNs2K2&a5vQhsj>j&WfFV}QIl?oa~7z4B;H}vpymWfD6Xe-9IdWAlG z$hgC8gcYSPX-r!D-9@SC>9&FPw^kn>fnKW6HT)n`5hioM-3I5Xu6uBJn7!5OYkYZS zzWe1@U3+NTo9#IQ6el07ZfdRN&^cyl&9?X{rgGNB3}lpUjD78Zgv9YiiA?`Q4=;NpZRzU;XQzt;oBa zMCOv5(YbN%1?J^ooF-@HK6?fqK~%zBovtZ{$2Dt5#wJIjTbTlepCZ+(f!m~|d}3o` zH=p^$XA15~^Mh@mfbwx0eoISBH6%(}st9dpNKJs- zdQ?b=i0pYSzPxsxIkgHFb_%kq9vyyI8WHZ_ky0+&2m*MwbLMB=(rWwAV&?;r4CJx%_ z>gongj`g?R#9>Z4YYV_wdSjJt6!-H=r&~K#I}(T}=g{4TwyhMgwfR&zN8G8ULtb3;3})IDxaLDCnetFSzxR^_jtxlDwLF{B;0uYvOm&qzUu@luGXd22|I9 z&YQ|&Hv1sxg3+^^96GgZwIVl2jUp5{>4^DXzuu)jJH3|@l-NK+q2BF8u^O4xtlCz8#?tcI#Dog~Dwoj>fjfSFblTyvGy(zwd`j-zGUu8KFE3qw zD>&E}`GH_W)PzgW9E123o@@(rP;I%6qC_`v?y)-#>F0aBHE~R5ZEPI)c10TCCus)G z{bbo+JJ|rLcl6ii%P(Kb;N!2L7fAkIJ#yrhhvqC0e_iS^g#&{CXDon|A9Yvx&Ey|& zcXtPmwj9c2QB4gSLRAeFc?BDr&qjmP6If|kuQ^4e*SKx_4w`~1;2?kjCrv9taTVHO zUuadKo`0I;`dVHeo$=Dt45)WJ5W1OWes){{tR2|MIkm9ry?4QG7ruG3rc4p?W+5`_ z=O>)?Ig=37iIS?LqqArKe$Il8U>+79d}O0Gc${y`N&gy$$YxM2(h|Q9gHS;mN>5sj z>nk^HtG{g^F!efW;_Sf++VkJvt#F_I{nCtZ!UU>;(WC{>QG$$e`1JOpP=V1i1b~FJ z%1h-8SvY(_y}CNpV^ar%8#GGBH>nH{3*CKR+Il5oR`E5*CnE8Y)-H^7gHOO2HlJyTIb0~dK}=87-P<6*q}2+MI@tI|2i*e2q8Mf4X;B-qD9^l$5}LLbcO2Ps-kVYu2PhKC}OTfhWjetw<$)&>t02 z`OrZUnk8=UDb{A2bObs?!vX4?9&fL+S77^DZqJ!P4FYcg?JGx1wNw>7XSfSspLeGQq4x&8Ht;KfTN9{qk(_WjHJ|Iz0dAT`{6Lo0@QlrPA+nddea~_@v z7~b=Z*9F*v=l=15v9aCIB+sKqB0}ZQ?wBk`j;uaif!^L;!t07X5{9Bpm?^?RLT!0x zC&ycoxn<3oH3#fCPV)4H2TPi)23GsvkOLGG1M-4udF^#P&*c({VX&kwL*E%7=m(*8 zUfY*V(Or~WLQ6}zG+bIg&C&UF?>aVggi!|-geRt^7E$O>9P(C!yix)biFV*iUV|O_Jt`hyCy%Iw{t6v%-JvAgk*p+ z0EzR^uiz*{r8gB7%aBWFPuMN?8?l9CCw!Yb4CX-diphz00ZIV|I8OB{Fv1a5-0?0IM+abHMSm<}LB^BdDt`1kuA zqaS~FKU}+Gdh*sWoszWd$%t=fMH`9X0&-U%kz0}9CuYZ6L+ZCkvV2wG!iFt}9H1Z8 z-OMTuR#CeHc~DsR zNFM@@wF8|7(0QthLyTa5NnZYP#+~Ruw`|=?qY_TDRz_fIc03wtFoTqo6tv1R2&T#Q ztHZ*=%o|dVWxG#qfqwD~l!>@;_OpZLhA+?Ym*3lTopYmp8}OYbKc6)M8x<@D{SOuO znK>4npE*hq5aAc}h`Voa>m&@W^h4Y#DJh+|w^y^W5<`Wp%Cb2Mq`p{q-#uS&)>Io& zF-Cseu5l)KZg5`ib%VVoFX|4A4$r_n5Cat&(=RX1ET@R7O4zH@U`a~{)63_+M#;9e zoH=!h4lKKIzWb{2{#LG+_-|8m%_A6V^iHGuw%{GNpKJ05`_w))+6&v# zdWdYoPwu0mrHZw`J#?-@ziw3_-Ou>WRhKJ${^iTS4(9KX z^@Bd4ei)r71TM6a?Ck6W{=JUK??}lU+J#oo^9dnFdEb{bF$47oo0k1|`Z*BsYprMW z72`p_7A5G!W{3?-H80c2yOPs9V(^%2G`O>0_rjj?lyrlm0S}ISx0eG{3wW)oh957! z-PiTuenV&)*1x0m;psIrbK{DZwYN9s`eRmR2eGBGw4ZnRs*Qn*B`Ml4jtDE zZ+LiaY9q()9&S2d+#f%FKs7dPE4*a1DP>@ezA8rl)~apjHwK!WTe}h~@Vpd-T3eq@ zd-uVERYYA!CIT;^V3OXYE*3)coryoW{pZnWAzh+yUxg(B?ylmmTebz(A4W1> zQvdXbP|JC^AudFh9pzwjI|rgM4RB)@+;9ZZWu|o@F${p;4w;22@~&%$ul0J)>*e6E zu0nHY1B{;NFPJ~yTfd3ObD&_AXGhpkf`vVAR@GnilsxJ6DRc7dySbe6{VhA13cLi+ z?!rlTNMZP9Of9h+6t8H)uiUL!#yol0FwOiNX?DL1xp|2E>NOwhXO{Emu>$-Cy>%WA zF>q^~<#KmoFpB4d7s0VZXwNsYF!2FW!IV(@z7;N2gd2`}Y0NkSl56F}w3I~u+I^a4iL5L5$ zU9N0#SOV}KbTesHN>~_n*<228~XX7y)iWg@lq)u*Gg{Q3R z{$L|Em3Od$f_LdE2=lfw0Wa~4S4{g0G3m)^9kZx^cRED-tp~aaCcAmTFn2=s zoIoOR;17v_;}K0!C+t3(w`~K^`Ecgz8XAf?P*RS{5KIn;SFRgRSjzM@2Wo4UEoI?* z=lIx8_4k)_Fia2?GGphTUj&DP6MGqjRK!seWoW6laf2af4v_W4{5mjjo=}g7v-?4`kKYAlPy+5I^@-P~) zfUpl>pe|j%e!Vfw0xTwn#yhTiNZ3V{mCKRt%E;EW4d*`NJh>!r#@&Qy08}CV4vn$Z zeGSl5fnQX$Z_n@CtEpKe;59=-ac@51-MDHmt!g18*k^W~yFqTAZGJU+_t&kcgSQ-X ztB>rdb@l^*O!ys43!=KD-luNUR`eRo}HI>>crI|Yk zOi7C_qg$wyItB(@aAa|u5#k1Y(&&!m-wj*d02Xj&r1k)`P4OXXJ2$sivM)3}^K*%z zXC@aZQWHjBNA@#M8*>~s=sz`#K6E_&YYL`XpO-bo2UongerRkVM7r(hFw^wIw~ zLX(4?P&lESo;VSe?he+-a1Onfre#Y@ix1)ZRRAp)IJNpg2r2CV1e(Nfw2orMBU!Hj zbt(E0Bn~tHuJK&3MT%vQc%ikX&jGA=KWxQcRm@}2?4}_GfiQag+9Rkd^J7n6pE$(| z$|a=p;&+I(MWCjFv*hIE87W_oFc#u6V3i#7MGx&?O-;?`kQ)vRkUm#s;CNaXX!8zhMimqXAI(RTa$ex2 zXaTkdxBi+PzprL*{~W#3TH?w2h8AgEDST+rb^=S5ERpj(?x}?iPGYq#5`o_FRv;%P zXo{S}#^Bi-G#_ox;o@Uz`dfDcE2hZ z&&kPQK>gt$g7V`hPr7sRI*bPA&ODRmp z+d>+_HN+B(GvBPc_07vsD!qo@nz*56B zESqOp(y`pETOlEFcQkZ}zqqZt`^ll>gL2Tn^fISKEL`cVt*v1)gJj7!yigE?KP^mU zl$5@I&j7n>DIg;1n>PY27Rze!If)aYgLC4Itsqu#>=n3GtTCQ9QK##p^R-1)G%Vv1tYG414nudsQ7~5%K56tqcyky z3E0rxzJb(p9z2_!*tz{UlX|DCfMHAiVI@!EtN{^YVX8iF%8N8Y4OBmL=+1;!$reW9 zH47;F^oa!@a8S$|iZg~V+A$9D1{hukhVzkMpPA5o72|l@!QcdQ6fsPr2f1I$4gc+O zid**liD>?YtM@`f{Sl6)KWpMF>YuLSkvX>vsfXz85UD$iYS^yP0-^7Mt2(5d>-%fy zYrBCFh_VRAaW||2gh)qILf{ofB|1jeZTR)+%{UM;3&0@+*irz|D!8A+6XyiXjSvAdoj=njm*|by_Vgt)cPno{Pu`#R{R_CqTXuq7rW8AQGYAT+wh1#0W{6O8fr!NC7%InRxJk zSdMm_b{w50nDk))eL<|nP*Ks0vYFJ5O9Rm&FiYXJBfd1_TqkA}%*&{dgslRUuoe%4 z{nr+LZM-b1W9U$9Xwk5uOK8WU|DC@1QKf2m`qItM1g@ca!AG?M^vcz{ciEApZ{hdx z5oto~W1Y5hQwb&oN(g|Pcmm?s z2HEcm3q4tJl<2o#zHk#8=G_ek;2W2ah-_ND92W*g*5-hH%69==dk6G}`?Kkl;U6Zc0 zW7(WHfxhT{4rXi>7J-P*h={J|zV7GeM`ZT6Eig2bUNktupd=P^KShXXUwHMB`3Lo? z9*`R93(p9uIlzSVG5laYyN3O#lPCSlq|A!xGs`@Hvg0~GuaAVL=-3I^x)PH{(Svd} zU!Fx1S~Sj{?vbU+H*WZ-LGrbOSc33;xzUY455PJjo1B=4g}*_1LCrvZ_A!<)2-A?# zJQg-S2#VVNsgUwk;owb2^^<3lc}*i{W#JM?%W7&c@!AB}w6!=dH@67L+cywcp|O8- zm=U-PD-Q#t7`=yfyK1LJ;;p?rTdyO-lf8o5Bl=DO7|bH56UE~?+FOM!hT+*vJoFWs zfPd_V&pVK0{lJood8~F1+A53*MC2$&rj_EX_>0d&t&a`osMlg(oRpKHQ)Rs;X(I8d zJa|^;)dkD;wl|mMJlB&t2$zlzLQ!)P{T%~xgYGoX`VAYH(DK(#v_SqZdUspM4^>A| zF}7r*oZJz(`(tE#RmQFY7z_Zu_aq;;Ijt%3BPX=AVbEOvKND1S#FR^^7$PgBx19WV=kqpR2`BT3( zZnLaW?-WTQ{+3`jCu9el9{9gGdA3=2BWhK}Lb|zkj=ch1cm=edbww`97;-N~k?B70 z;G%J8nU3;V-V2+l2#)%hEv-`#%({BDM9`Bdn0kWiJ4VOH%W(3ggPzm2q+EP$-=#_Y z((pOm)7#4g1s`iy2rd76r)?`J@MvxDeIbHK70@a0^Z;JxL5c((T8faZ z##B?>3Z2H{M|VL+bwCBAqcDU>PiUT`@W@C3SjgxpVoM>@VHf&{o`N)j1J9r`+qbB( zdvQx(U7lU%63G0wgM(=S4=_#=i$ktCt$#55GWwTG*q}!9Z2K-oMMcGfuiYG%@ov|$ ziD)uIb(SUGRWJZiU{!b`B}+tx;j~3^kHLBn%N+W=x^Bs*6Ao+de}M0enRdyN4w#gz zG#XKW_jZbJAaDWWQSgM`>E!7MkiQWaI0%KXTeNiijAoU3vlheGQ!oSLcLqjBcVUVU z3@OvnrAt9@3YEE`;UJS7#F+2}2Ehok`7bE8itZDvfW=^pFrc7F9CxY4a_G@HFp5%_ z6p@$D#R-ab!Sn@u;UgFY0pUkoB0Kb=%`h2YvT!;3daXjpPQm3}7g+KSNa+?$BlWdo z3I)^)iYHkjFm}L>#>$e)+rjjpaxWqLBZNq@I$)7u z-ckV*zU|&cuXPUE+dPz>#K(7n{dAd|k;B*x6TqXatZWn#k6gVo<{dzTnIwByPldT>`=4=1r17~t{YyMgj?j8>-|0E977j9>PoM%Z>WL+ z9GsJ3uL~HiHu#=QOiYSPN-+8ui%|zc!#$L?zSrrRB+J#hAw<&@f;d2E46GGAXKW4< z{DCq|W_>0mT*x{S*o-~EOhfEvJE{t{PecGzN>sn98{S=J%VH=83PO5PNJ6L*NyaZ+=$}S-RFD+gg zpumSzG*#54sCSE0RR14aWuVv)xQbv#;n2Vkvkr0~vzfWM6m%=6J%Urd&@R08?Av$w z#fz=Bi*(aFawbk4#gAb2uDdeQq$%2c43<XE85eIxAvmVFoI z%5dr5bngrGn zIHv!5EI#|Ks6I$KnA28Kg~Pn*xr%^Gz*Qgq#d zoS|F1!lUJt)9(Oiv}PCxHM>laJ{^F!th~9IAG7#k--OwSY2_da2pTUKw|NAVviUID zvlIxVm{5!0)VCsRoO%pW7d% ze6n+Mtv{cu0%Hemk`toeJf%ZE|Gw`+Vp0-Url}a{nmC`mzO<3F^rCb@DK>%lB@1}I7DH61ltAJ{v=HC4>2vq^*ci#_ zAgiumT2u|=>$SD)p!NK(U*8E>XliA}h`|RkcZ)7sH*q1AEMI;VylVHSN6B|Q!9#+Q zV0Zjg_Xd(&NDVP!gQlQex>N`c1~^OX+GMl<)MB?nH)H|-$=cdlqA)3XOmP4MkuH#r zpTA_3_Zr!3e}AP7Xy1udm-PIk38}!Wp%26XA+)*>Z*X@6Co0Sp?4xC7B$$OZR9sNI zD!B6iG?YL7JQ|dSYmgkQQ}Z*!%h6|?ICaVoHgZzm2@1QpdIyF;;)9hT>L=@`ETTvv z1pS?&`H1{NM`vd>DRS^t6qb3SsGvQKhUy3vQ~{HyZNGj6V1#Df`t@dz=@P&og(~A= zVc}81m@`I4u7L$fr*n>hqZT#oTnGu{_P%N+m^4L==4hI8TuiG~7ixUSLg z7Br2p^A)4dx^?f~6MIE4mv8>@|AcI_G1I@GNp50*=zOWI?L?eBCdmVd<*{!6KQSf8 zN=rl4u^fUcw&bSroImue#7)*`&Rm5`VBKeoH@$*kLOL{kdedR~SQ5~#tYlL0@Kf+< zyCG7w0bik_x5Lr`8gM}(!~){&6PhZ(1#o>^WmW$)u|^aw6&es;-o_FdnCN8e`hs~w zf7E_6oN2;T!Rj?I_WP*RYk*L*cwgeTMQKw9nO^Sj0`rb35nMPPH#z?vk7y`x6d5z= z_oP*jmsf*G4R}8=IQRxW+-oSxL_fFsP~bYe1mHjw^c#qWYs3Kuw2t9kb+|xiC>X8? zKn30?u?MGOc9piR`Q~(=aubPI2%VG)JZ{Cm;oy~Z=w5~|dKqyR4N?V7#!fQf4qgQK z1kidJAhDH(!uOS1DHm7QX!x$MUw!~~P{%G~c%Tic6Pa;dLc|{Fzk&bb1$i!X5ES*w zxj*%eC*0h`2IkmXUqbBX_%#){CIkg3au){X8Gxsk!eCFvJVC zrG!Pmz#1Ye+o6X=ZRgI+U%{-HhJgc)xUwJ%mjNXed*C966~wa!K&mMxBO^0tiB8Dk;63fB2v?=ak^BU0wX))iCv4h`7BCt9mAEzupSAwg0Uw1$-m>);s{KqkMU zz82QKlf_7HWWX2ss|{UI3EI=czxHWrq8+Bgh(I9FebQs%uLO0AkUb1@js>>`F5@8H z0W_Lg54P;m!lthWvI4Ka4E_hZ+_*Sl3FB-EBRxRIBEL5xOW?hC5Cb2Xv_;mttS9!- zp#sjSomf>IpCx!J%*N_y5&4v`oM<4VD<%GB^Xjrq;8$V4`^)Z_n`0Ef4N`>wVQ8_1?QLz|I3-0eSZf(9JLfXkK?gbEV6+;Y)c#`^i;20f29P(6 z!$dV`$3-PHSwt5EkEy3QJU-54avx0334A3;h+MTqhDP%5o@zp6@~9&>5KTQMDiw9B z*BmFp>;bw$7?t6ov3iPei!6{R*mBKpAUj0sb1=;q*o9~iqgWBvu^F$CO5n~$*(M|d z#z@v+h*Qb~qyv5rzA#s})z}AGc6N3eZOT-zF z3QLJkBQI=~3Ol^gP1$yf|JGF9Bf5vy-aY|$8P5k|{KX=($14YG#_fTZ79tF(!-Q)^ z?|{*Wm0-uKvhB>N`7mv%=^Ey*?=hN<+Z-?hvl~q@_>U4tNoL%>05X$Im|<_kPB{cj zBK1+;*aqvSaTH}K3}~SJu~a-q&^k$)w}XOMFy$AYu?gdnWNsMQNXl)*f=~VeJ%*ev zVc4`$o{YyJmm_xyqDrEE!OlBNCcjaDxVgD0m+_+taGX&C=xcfu#z^YMred{A(0-(Q z&bno8UAL&eksY%m%s4-G-5N;$^giCOy!sbRzK|RQpSl>WW4gO3-$f_%dDXJ)zA-z@ zur|1MNI&ch#zSPC`a&DG;VU1Y`8N!L-wz|odDqf@&_@NuurDSQOTqOhFu>gx@7}#rLC{A-DaXgR0VI*J4;*lEiv$iCb+DD5+z>DWzX#l;m4n=^ zd9JZe7qz99!VpP;Jc>#`PFm$e-KCI%KHAOyt(WQoytpwYu8a5i@bK_XJCI9_yc zJGg0Q?A`D;*rw`WuI9zK6pS=8q(T$FEUt}Wt%Z0N1grw)t^O$k5;N$Sw*cQzOK?My zVS(okN@RBZ0dkWRhN8B#rk4~K!x%*fJQoKO)#`3kGO^WHUyb<#A{KuBe7G-|N)Qmd z-D+Mb5kKY+lN{CfvYH&sfPYp(H%6Ud&~{g+xBS=I=l9=VCxQ5g%@c_??luEO?g>Iz zlQB`k)YoFTv>LX>MsG*`1MWLEgTuouXTJ{jQe=x? zmPaqHj#~m&aC6gRUqFr!>jc2uZtxaVo^={=7n0q986|y3$E-F3ot%@+gV0bx=tgB@ z^jW)D0Xem2WA>5$;A;{L7GM{=ot49--$y+8xQEt}asEo^9XoV0N%H?E?h1|9@|j# zo*S)qU7g-wKi+1s%S+nJufm3?ZFBni;(jD&BqSubmt{c=BsaprMFI=z8^pJLMViG; zQn()o(_p)7mnZ5m0)P;7R3^QqV9jF12jts(jP)2btRQb5lLPLUfJa1jVgI(NPA6j{8pKYLl7kh5;TehoparX01W-g# zSy@?{`}8R~1t%NP4v5&S=vC^v->0xIA(bWQZ}XsnMqw#MjFCV_`aH~Q=odl#^nr>- zLy=i9_x>{Ykx_i$uLul&@~vKr=kGdDQ7 z1w+Kf!|E__4X^k!_ep!mk4s^U)xa;9T3GnwyBC2!fL#N(T;L)UwS4nt_;rD~ucG&> z1LlO!Vi)k+3aI}meroIQzQJ@KuA-W3LFY$aI(En;MS!ruSzs30A#xEW7gh(s+k2il z{WgNR{hBM8ZY6yPzzmecVjPO9DAmRIx}B&6q^rf)$zl2NjYJ_NFrL6(vWyUg$fPCc zZdNw7MQ8(H((gd>dX|~_aa0yG-x7o-@jc=*yQcQ2_JS6MM&Ztn0ONKfKcj|$;9N?m*s7{3_x{VU zC}l5<|5kJT!nq;7w?P<$3y~=-J=8orJjg|W#JJ6r7m1Wht`ebIKDr!C*54F&0e?-T z5U3t2hm&6b=_B3<()UYClf*48B^g+6;)@Q*bo}J#Iw`-Fk+FbObO7b`YzRE3qUXkgYw$Nqdr-Dnag0#GvPzjT`r! zdQyfbBUU!=sUxhg*hS%LKwJhQH9yVhhOmZy4AUbyv$&teZfprf?s36B2!HyshorST z^uM|_kw;?=kozn#eYTv!RT+fcx*I=)YDjL}0%J>2C z9v#a!*9LaMfgTw7Ns2si%)`y*i(4?q{xo9&j$>*`)g~5MioB}JivLn=R<%%4;H0{S zKzs{t1@4?M-+^3+c7RN?VBp{%<%f)zkur~ZMxB6P4fl2&KB@ss3vU|ngAlg`u6hB> zuLdl4cnC;^)zj0H*!Dz>mc}*IP%W^Fy#@C7nUE zK<@UkTgSkB6U824h@I<)m79!+!x}yliozQpF-kPNLRL{)|9^7fs@lVgD@kSluUAao zLKFL6e*#(Z|L$e2KUO+Z3t1KaH#vwWCDbSkkhm#R1$i1h@dNjU4GRN+qz0o`Ld@r> zjQ(Bd6)XO8b!_#9ZdlGr0lwOMdy_MciG{>~oCj|5h)`fBUQOVSGB_NHZRF+U$>awp zGjfTmNIJR23@;DMb&#z~_-X(C=Zf1|L;@vk1U8$H{LtDjAzr~>&``e`(ryAVr~=X^ zPl3**!Einpx+Q3$NV^T0qaC;8;RZfbVN;yYU9Jf1g;(UJ|4^kd2g*^et8rf#X{Ifw zkyXfzASl$t`$0ww^hZ4~vIf-84Mxxof*G!uT1=54L^3ppHXE!A8RZx;C!?EqO*Q@3 zeZ4VxkVv0uUZtfhFpPj>ND)TI=yS-eTi%EdG<_!kM8WD%h-EScL_?u^+OAR8{`*x& zy8g~0EWEKe=)Dehxc%j6}%WHm7N%{6A3r`Dt$(a#&=@FYf>G+WPJ6Mo@$xEaXVknv!<0ey%M3^wh zOh(82M_dhtdF^GZwy5{YV@!pj0Q{|LIlqC=x*ZY0vUu@jzTVop9CB5No12?a87xHP zHIgo3!Mj*N_`8af8)%D{p+f&GcRBBVnKI=Kq`iucUIVOx7$qE`DM}IUT3Z9#LF!?C z9C%7FB*~=x5aupsFt**LppWJWXFkwo**~A~UUcCTfX~URdIN5-*c%G{c)|-#aa`QM z3;}E@8E$~q-eYtX%o&9?RLAC(S+1U#UwXP->Zs+f3j@Lz*#7xyhcsq4Q-4JXzhqi-<-L9#l4bDJ za0TwoKo?pFS~R4F*yqq7Q)qP;GybGs*g;(QB>t*|zpzK0m6oU-rmP-y0PpMA_S#+j z3&bK5fUV7e4hRiRybff%55S*@%`5^+Hz215i|8yuq#qU|f+gwZ31tewBe;fKD0$&m z9RsAoH}JSr$L-_7aIqP#Gyv0o`Ozc(sj=VFZ-Q?BW>Rl7qxr1znSZxNy_HbZw> zA(jZ|+qUUF5LPwCY(62>a62L?$;dLXxC-M(bFK`$4gnJhEtyiFegNq>C-b34oHZ9* z$O#jBX+JGZ&;dUT;GLguRSS>(3NEgtz!mTit%TYkaPm8Cb-Xy*W?a#;=fD9CbSfA| zI@YviA?7^p*-!7^C!l{WO#_#z953)>#ykt|TK2pZ#Ht=CpCz^MFdp1=Hs-iTkd6@G zlnM%!&hwM+&4RA{s3pcy*pZ3p4?A+|GR@ymo@*#2Y%2O!4u@^(kxTyPjd*0ADo4_) zs_fpq#M{}rVa4q+4(79^R$suAqNTvVd~S{(oBie-7W;|I2&Ol{O;suKsWAy?I>E zc^mfqYZwNNeFyqo^7)+0aUAFIG3;@PdKdcm_~hVWt!Vcd z)5g&C`zCnt&GmIbJN_1o>z6lX@mBb8Bfug@v|FWFzyEryG&wYKgwvCTxY1p^LZYW_ z+XShOxMS;xN#C!c=Bg*iv%>=wbXB4{6%8U!Oy^DidaylG0V>dRGL#W>Kq4}fDH<@k zG_}_CkF=HNdblhZC`6kga%0s`ruEOP3F-gZS;{vVEN#Ug2@V5?sU2~>P9-P$j4=G? z8X7VaXtu`v6rSMsyx0*+KlLBc38{uSX&fN8wNYxRb1?y@a+3P zw(?Y^a+HBWWQ}(Cx+(6ZkEy>5EiOZQ05zIbO$zlrY6ac-S6hmhgMEt9eiJYa#oB(m z*)&NKGAJimZ4HFWBP3j=0*?0?*RtW~%Ip0k$sxhO>^3$2dpM6YJNyG9oh;)&5P&{n zkV(5r0BTTeW~0X83@H=cqIiTs0kgQW{>NqrlA>0A!^l9)xtPA*%u=0u)xSG@PPX>i z`lk}w?|$u4Wz;lieR?yUx*onj#)y$jiR&dyYs26b22K%>E-+R~m+gpmH%F?XzQL)*05Kas5#7 zVY*$s#!?Et^d`-3>bkb2-vus2s(!LsywAb9<@mxNN9T>=#k;r9Zgv(XhUsCfArVe# zJ;ay^1bBJ@-N#1QFoBK4220$!!dFlSp=v+2Vy5(OW5y`Ni5&ny^{H6k(<3?IJEp8U ztR-FOdh)1ZE-*03uQe#po^Q}HVqPq&TZAa{ptQAJHPqD)i7ExpkF!5EYGjMl7)a&{ z)l^U=8)c6DtVxrU#W0`tSA5&`((Bghd);%Z%0sTLDsJ?1r@8czovu??s}B2JL4_tx zh!mUafh|fd==Z2eY_i@9*%u9w5?2Fyng&82RN2=iSrv6}boLp9>e6x35T&ygvm#8p zhQH?raLv6WtzIpptxOJyfi!PNFTFjb0n-YPn0FOS0N3^ImYimk-ORxTb#QEJn6)(=;{tiLb2AL6Vhk;5vX>+JVakD8=89mi$ zBE1SV)$cG>j6ok^DIm@67G{3a?m2sIBRx+~Ev+U(qe&AyT$=$oJ9eOqs7f2ls8K6$ zn66KLmzQod5kHVhN#AUWCs+RdTa}?3B6$Xvy`4_5Pf31W-Z-X=oNz1>OE#QLp8Wh7 zfz>?vOp~FXl;*MgSo+ef|82642{j zcacQPDxj{D>Ol7>Ph3nyso6whfu!7rZ>6BcKOZ886fh}RZ0nqG4ZY0Z4vw8!P;ek?;I zVG^@U8njf*McoG-`v*#C5TuPHU4b}Q4q_mVZ{Y#EZQ%5BNCE5^{vX{}8Y#R)eZ^z1 z8%ce2PZvY%?$pev0%c+l_Cpspn6QY5=-dk~&!+%nJwoFSZg-ho&>w(Eagk@Rv+|dj z@jo$JEmtmEgf$C*VFwH)J_AJm%J+$FJQt&Y%b;b-LQ{>&k}@Bglg$rpUvt-TZS`HOV}dKU{|t0%GQRbB$BA1EYTb#R z-VasWJl38HHX!CTgp}@qnPRgb20VYyp#l)y0F^vAy(T4LO!lWs=+Q-(Ky8+~l%22% z8*)f%VHOXO4?$wm6^fM4m0Poa5*5s2mJ0gzjWSW+n6teQ?txj2t~!~6xGZyMp0n0Qg10ET)#p*5J@1E zxYn1D*y*udxQ*R*s&r!gf)5;+%6SP)j{HcuXklW*5EH>)T1#dGoZNvwWY(ShClM1l z=EiGFn(&>e@S}$#A{r~GOfS(9%e)3+;-07>*gW{7iOyE>h=4|);%%J)K_w6$tM9p+ zs$tq|5L=Vie*z4Gdv3hfs9GPGcoM}4L83t&Knjm6!T{FHTl9Hj(QeSc2}iA{?*wXo zg82KlCP0F_0X|)x?=VKnA&esjXp8!lc}9gyDqs}n)HW4XS4lx-Yg;JIAmbpQ)z{w{ zMNzYZu1HxXqK;C3$fCyz1!hNs!A8Mq8c=Q`U*$g-Pqw{yA*y2$Fh)eK*Wp7i60yYW zvkdIAi`S~20}7`j$i^&B!!(071xL=m#ikL`)0XE@jX~iKL?XK7Vi}D^g+zBkavIn5 zma%WAMyC_+PB1(e0Sl|0pv%e=W>*}H#78mczDkL+9!1cP@aRaW#A?#E^A)D_lr+!k zen#e^@FKiKe;IW!5DZ$jzo1QIvm@Ev#mpJ~2b^asyUfCL;Lud1W-kj=)_i=Uv(m^6 zVs`?mU!h=d&-T7xRYX_ZINEM((e;FRi>wR{cY1gxajov}l9Mjp0#Z@zD|khJf>5CW z`svZbkJ|{Rb_~P60Ns0bW3M-Mo1yhb`&{cl1vwTwqxB#i^Psgvf<$-~`H4>{P1Xct z9^#)v#)&>~s=xcwr_?k`3W|x_E`5O@xNs4rT6SvIwyj!YAaMSJ696_C_Y|RrmbLeO zRd#p5qPcUs-HpQf4E`|x)b-nM>#DtW9Y9WG?3eOOT*_sPN<1|%YjSu;f_q9uC2*hc zLy&edOiWPwbj)vavNCH~fD0IVg#wB0*cB_{_vl00iUl#!?QtAOaqZAghbjtG;E0qI zGA(c+)!@S)O?b4Y`WSUKSoXRZU=VYNnI(%i>vv&ma(K+>{_DPaY+o&wj4w({rI3{b zz@zpBJ@QfO2sZJSc?fXMX82a8N%P3yU);bJazCD`Qbr$uPnL#Magn=Xxn8QsF&(X> z*!9corVJHPyZ;Jt*Rmnz<2KP4UVB3k{kFaD5v(6^DA~RZ`xe1?`R+iu*m4%FQ z@i1k~I)=#_n<^A};W&yi1Pa~OQ4CWRf>{VZO;ITDzqFYcE{#wcHey6{?vcfIS8p$` zfTKJF&Rh4S7{#N43SW^{{uMP&@x5fMx;PDjt4vAhI{9@oT2&s(rxOQX7_6Zy2gK6# ztXYw8daJfr^e}L7g|Zg^6=jmuTr#F@Z#76j$y{XMD?^Jsypcv5Xa@;=)AiYyx_YD! zsN|nhU>9HrE`D26m4PQ1y)wBmoo+*5bWFR)XnU?|sy$wR{&Z%^K>53YX53b5fX-07 z;f2-~IRTIuvkWJ8Mv!C_-G*#zA4CoRgWK7!#kF91-GAv>2Q<9bI>q@kl$fnLc8nU5 z#2yh(!t9w9u#*;@W(4o9+5F;qYe+qkp`zNRvo`|@3XwFZV2Q+lLbqR9?FF7<*e)_R z=e(<<;v(T07nq$9D(KCAhTmK9;*JtUb1HWkw9q9Ano@jXTDJ}zWH*0aG=HQNm-Am3 z{F>a$lYxxj1snC7AlhRsHjuW1kmPU!l$1Hu&Fo4$?CoTmQx9_>PD53|;)@r`J2ETR z@;VbJp(K<}DT}1Hp86IFQF{>em}tZJd~|iq6rv4KN-Vp%oTTvl_Ccu)ME?k&PTG=5 zCa0u~SAa1Xsx*1@@cd>d;S6%=4xASW-L||}_=V}dy2q{KeqeuOP}XH3d4#dwY|kdz z$o7!oeXvm(NOr^I92KE(`m*RyeWZhgHhrJCt?TG6iVC_b5%giVJ06)DN+mDwjOYXB z#b|Z&3p-cTbWSUJC{#zURYjj6ZJR>s9yxho*wPvhB@C5+M!=;Q!;gb{_gmTe)^TdM z0i1GB7bd75xLFk$-&M5SZ8nTfTAB?YOQ{vf#xh1hGi~`Y+A_Mm3~*Db2|glORAvg` zd4+5u#&=lcW)Z=>lgto!`Gf27$g(&Mu9`etKtB^hXSmgO}_8%{M*R*`YnPSxOw&X53_`@J zQ$;}NgqA~0Q=#UrAg>YK&1aIV!=eESuOFMWFFx-VEavjxx$g>^qPm&398qCPr$ia@ zV?iBvn6-)8-kYBC2v0(?&F{bW7nubOYy$=VB%eh0qoj}sLpTAR6mN1lKMFoaTCL+R zdxrMA0@^6X#0mD)g!f>{xl^UOlWhR}m}q2jzC(B-?x)qy_AmdP3Oi_xWOPtlnZBW| zwlwYv@#d@B31I}gz(W>O=o`U$Z7@ab`uSd6upB6v{YZ3lGe)4KjOQJK z^KUY*VU$!(7!j$Bl9CdXu2)jhD6xhhhK^qv(GWXXN^G&Fmggi0BKrZFLcAYZEi%7) zlWJ3pk&(|G0!I}6C!LKqr4u*WUTUewsDe+P>@OXtOcjA4W3Dw*b5u0Kxs7PU1oRQ8 zR%jJj&LaII`!@Y;v?S83P1#t2$;>gW%ALHtcDol&qZ`Tq)|p37wG3LMNZckxJ|i>0 zz@PCaC<-QVW*UAKBdIPoA>a;Wf7 z#A}fcak7cq9|t=i43d#u`tUT_((T&G5WRR2T650Juwffs*d@S9#67iL=~FUeyszpO z8jmdxKp7mF#Ct@t-~gZTyyL0K9;}UB8 zAggT;^+1o&pki6mF;P4F%xh`CwY;l*b7<)9oaHlXDnQRH233Ep;|#Rd)WfWGRfVBV z|GkqMo%?ML1WMB=iKSgF&|#J4 zpO?@7)^S6lP4ABWzT(W)dK2H$(HM44eYA^p(fldb(dE=Mo?$@$`<<;p1rlEd0{|A&64 zxoqsJP@4@yFm=q>x?>yPj|(tT+VXzd*DjaJTgr`%d)?;E6_ff~{P(}RvQ6p`ZFw~H zU;j(cR<_lDef@7ZywcNIvMdgjiJrA*%wAI2c4A%HXu6hc0@q~UVai~&q#hcl;@l!l z{K2&~QD!fFFo#D@<5Ts~lVY25J+`4UvfWyL;N}a@Iyz+%rROiZJ2)y6(J6jEY1fLq z6Rn>^j>e=tb!x;-O7{Ad)^jeSZ>*SahC0?=h<$S>AEVG-*MDqXOM_*&kqXGe^6{VTCQY= z)D>oDh*ZM$5C(1c8-pxSrfe||)Q~~J)js`3%UL{9rjS30ZH_z2{p#MQQRl?k8HQBC z+LoG|PHui-yZy@Dqsr^YUVfLim?yrG20iNa5;MGz!aW#JnYqeDqMQ?U&8T{%<&Y7) z;YZo!mmvQ^AsZ>+9}XcWHCZh%D7>+_w3$4JwulPltVhRuIjzy;0pQLb@$vC3vbB^H zfRBH&@0-!EO7E?IRsXvbKBq!2URQ1kF4OA4ea_r8)Uv?`#p(d|i5G#IQCTP>k;5;h zp1F)ic*Tf`ZSmC|3ZZ?1hHRABOtT>>w(JkQzsTI_OhXI&$q`!}&h`TMr2wREDr;Q2 zQ=vdE&?XuGCXsLPfWLLCRhO-yy%{BuObHNZ^`33rw$0GWYS*<7AlqYr0no+)`&*dM zs(J6-JL7Qr#MK#7nv63Xj1L4wtunn>f{Ln=(6U);{Dk^ z2XWxto`OEB1yx%nX z#D8MS=A9nDEL8@{{IeikQm?qbG2C*Q8*;*YR|4?Npqp@zd`?|an|_)8C^~hh(-T9M z*ZAnaCQq8ww5qx?8VW$py`j=R34iybxtwgZ*{CO|h#)%TPw-Ho7tq`s8#HYA1qRwV zgTwIiI^^=q&ItO4&0<|oLtYa5Fx}f+1(T3BdW>#=ZEyTv4XXbxeQ9)P`Xs+64v|+g z`2}(`C?*DC3V-kH@~(i03EKN{){sJjl5%xnIpwG9Qi@SAeN`E?W)KZxQ-wI05HDIN zuHqC+G^hZqm-HhQC4ZFeHoOK--B)IeSsolwqzy$0Jy_t0w$-QhGoHyCR3$;!m@=YC6-uW_S3urqVp0s#uW9yw^ibg|Z7vwa>fu9VIfk})%_j(V0-xWK)T-b?#HWAuDSzyf0kt59i1G6V zH*y$Y3Lp8(zru-*r{<8XEpG2Htq?)w2gKb@0364!t#NZh40EZlQ2%!AD0CO7L*;}D z&|x{8!q)T{1=>??O`!mFl94$ETOzIFddM6LWtryWIB1VwMGglpBC-q6Yni_m3Li|v zq?Y=4m2i}W zNR(KP&+Jn~TMwR!bzhz7H2ORNc4Zo#o|BNd#k-YgjHqE1BB+7WOnc?05c5QSs<~7a zBp40{5Xmv}IKu@884;v#8Fu65&8e|-M#)J`*l?P6R98xLv}4_0WCd2P!1Ri*uG!S} z66C?$<){Xnfd8QVuv$RLBc^kJhEsd#;8F8>Knn{G=4nebIig{~%U>yg==?QqyFwbZ zkmD99tC}dpBS4ULCM)DHsB^p4K~gEC4qII~RPnZPqV3WNt^)@N(uW&>q#_Oy40V2m zUI?X2UjacxdDvt1Q1dV#4`H;ePW-lo-#z2Ty6TE~A{ixsHd4GVFX~Fwc#-B#upF(t zv=kytr2Om+OL64Xpama>Y~(qVl&REZ1pRo{VIR-Nw=#K) z_>?zNZh1C{7l2vb7$%5X!orHSkj5Q=(qu>K!~)JJ+}RV=NnCHTJ%Pm|F9DVpdUT>H z=a);L$CJSTdbi^z7y*UZ%*Go1P&xI3cTmVdI{6|$0yC67FOnh7Y#7MD#|{{5E6b{R z6NSta@=Gvznial`p+4~{Mgnl#ss5R^@_Z#GVnA=-7T($_ub7-~8zCO+l0C7`^_rw{ zJ;8%-N`BD6Yn6BNwzZ8a^V0>-Cy~yDt~Ggp982fuqM^S(8WU&mL&XW5AG`{NGy<(s z%&pgGcn$!-BE+KJdNm0Cg8w%jgV>}BSQ!ogxq zS*Wn-hCZ9^&|&z^PUD|46o}G@dy0mBgdkB(%$_LSVKtxIc;Gqpa8H! zZ2aVhG0mZ$(?LV>?jGE`SF20!`a92ho_hE08M6YFw%(l?G7WmU)z^1C<8dOvqt7k2 z-ULlc_^MPhn3tf7&hOW#R_zoZRy@&OXMWX8EJ{bI)`FbMwMXV2z@F||UBzB=?+bL5 z^@Wu>Fsf7GP=0$?TV4H##vG~5?(~t6u|hSDDwm9nC$5 z1=a~kLj4M%S>l+BW`oWSe^*W&ASV)0Q)n(N`pc;&N_amKM4zP``r~i%?W~zsXiw+0 zyO$ohd}T_L_rJ=Ru^a?2!BCK6&Wjodjxi#-RETLS5&7&@+aDFciwUD&Viep^ zfqK|C`=>t<@4LCXZ!P_VwxNN-Wa6Yc%?X+A8=<-p<3Dmp`=DFr<#+;~|Hbt5@phwD zu3RZp1|HyB-=`;?cyr4H_MyD0yNKr@9xJ18FPT;u(-&U4E@9i!d)@5lF!;umCn8GGQIJ z?%UTE&V6?dytUIT_0;^!9pcJ&?rfXTv#n?~?CqNZ04@1SC%5|azySjQZ`+6fUqq`MVmt=CR~Oq;nH!STNCPN-H8P=%J_oO$ zOGNXX3r(+XWsr}dc3Y63Ki&JYRPh3NMdhw{^ft$&EO;QJZv@VSPbp9nE5}wF#Z4RA zYKKYq$pkI$*k{onHhlA6vht8JL%}#P474L2r}8aV*DRZ zq!^QO8PoM;=wTPr@%~J838J0&m%*oQ@vFbpLG@YJZ`|M<4gI69H5;hSt~#=1bh}f1 zl^in+Wq`hPVd3t+5k>LuAIEKf{Iuup>%!1;N|r4oYMR=V!|*ucXprB&`xeKC4KDij zEu(AiK=q$`tiN|;W$v7~HtzY0mvRie=gjEw(JM54LXQ`?$2&~R-4%K!^uv;vRE+?m zTY4#@KRYL9tIrR9`qRe;t|jM#^Eut5{7BF8=c_~V%W_Qf5W%=_dN zpj!osx!TR`-iY9~13PVB^ZdZ7;_~w4Zj$beD^sPas;dw0Mj3ur#XO!M>7Eu z2^RgK`Hy^4Y{w&6nnl0ecm3k9Imy6S_u#9-rpX>bi~9Eo)l`b|jfhy3xM<>0D?YYK zh-ZVk-@7D&kzu$R|D-D?_Yrk>0KqG~X78D#F*~JG=$xFy5`Ul3*N1it?|Oa{?f%rA zbH}x%Z2I@h{KloJQJ*&(ola`t*30Q?r+@!Jp*Y#Z2chx5UtW{VXs-YBwPbxL6P5pd z&295P{9eWWLz+te{-Z*1ui>~R|Mka<8o{sB6cwk~a(e(mjx+dpe?;30di;^udrdGI z_P=RBTl`Vc+RQ?SG>vl7lD7Z&jML*8Ah@=7V8nr*jpSE8DmhKPv)bJ~nwE~|`U}ae zio&;yBKg{mo|_2AeT>foNy&Lk$R#2Lz1aQBfiugCPEfRK0}QY9l^Y*p!s9U*e`Kmg zVCD&3$h@o6=O=Vir6xfDqHf&C`&W95YXQBsHtq0B-|5ry7x`xH9~#RX=#$@U5*&h@ zi?1Fg_lGT>G?wxxj5SdfXsU1QJoPeTJjFPYl8`kxsPDN^lOtJ?^3P!LOkf713Js+` zE2Ww^HZ8(8`{?9oqCym#TzmHUctN-2uqCRCGZPc#mMf0+$Y}Y1k(j>a7{VXn%#;YI zXw7wBKRf@vd7a&zJ-Y0kX^3KNXXA@@248@~aFWxo*qoBJXM?J~hc!MQSv2bbD`EkB z?(!Hc65X8IX_|F2wmYsj!DKdP)dKYFV&YM56&5N;y!EV(n76XpXtEY z2xdmBjap-YQu}oM<;83YZP;12)Aq`nL9_E?H{KiEynZEDg&TG9_;7zwL9*usQoW`z zmL&NA+^)zx>s!FYbSB9R#0uOXtUT;spkthQGf$@tq9nr+`_ibL#Hqj z@Dlhhp!_UQ)wRIhetdsmaopYDp1nla@CAyg2Bgg*%&67GT4>om-j@!6ZKs)h@ZfL& zZR?YV_yKZVr~5tU%R_Z5@weL?O&*sQEcB=GnuQ(#OKB?^emi>E?P^^U`=Q&=WR4 zPmQ%NbUvAL`h2G5mph{jH3Cb+jZ};RFa`$lY!&^QRa65}Y>AhE4sx&MNmW)cWoSp6 zHffT@#}1~BFgzX@toyB9yLRC?CHx8=I$b>922XHUg`%;ea-*cT>7AVlnzhqz`}{#~ zsEg@H;s$d-O?^bTXmV9IRSrjRa#p4+$j9$7T1+E2w1nqo7E{I$Ko-v;p7B9qcb}WglylQqI?z00-?9Q@T%&TMVh*Sf#euVd%W50jI?XhG9Ui{M*5-)Bvb#|iIJ)5Aa_ zfjz}*O9y2bT@{4i9 zQ6{-pAn-pWfGP$7jgi=<(O}sf-@y^5Frkx4oYNp0Slwry-`dMFNgxr=rkbbE+x~IQ z;>D4ar?z{e|1>vQHx%bB=D;uYS~EHY0AVv&kzMZy?4ma&PCXj3Vp~Y*BG(X%yPMnaK@h zW_f8*@@znY&F_>`P!$phCX#eK-h5_@8UrDu1Cvk`1gbwwa-U9Uz7W>C>&uLho1SL|PC%L# zz=PQ0>$^)`Z^Vc`6n)wlb(d^yxMgq8bnpY&&Eb>xTaoaBE?jUAeRF&MfKoyr_{Tm8 zf$WzYa}Ndrk5Q;j1M0Q{ow92efKS|*g{Nmgd}R|si(kBm( z)p1RX?jRwNbbgaF3sD#B+P!!bdZ0^_bluOLVFki0Alq z^U8{fJCNDr{8$3x^@k5<;|6+%mZL4|3vPa50KaIVmgc?0)u&68Ga<(@zA~pcyNf?h zB$$bDw##$jxb;6*80c7^9K%c7iK7OVk&zz@-A$p<9}Y}8=ghhz?o1&Ln?qo?Lu`Qq z&~_EwvB+QJZY(^_0OWDZe={ndM|>H20#b9PPivS1?bG*T#32Ys*m!MGU22XEp=3P< z?mryaDoIS;7>QOsWqNMnY8R|$On}$(>G1MEysYi$lA~F@j>sIh%->u3c9PHUuMsg=gGzz1QgD`EZzI@129<>zwSIuGmcJCQ2bZ#-=D!AwBQ!$JO|$l z88wR^vtpz>M)is)dDBXsJh9a>@ee6hVb(L0?~R;$Y5n)dmZ+*}GG;vuT?Ap%5!`_3 z{9|FgTWlpYoy}as2HC0px+rDzl&NDO)aiEm$pFL42WMk_+UwN({V8DJGkacwCHMJM zU0INKIolO^swF^o-`BK5)~1t0!>mrx{Wb0R)GYjqW2$<{D23gX|99Yc{Y>Q}&HH>N_5)R>_UM(*oME z@BpRu$IOVc9Vf4wuE)=EXE$T!%g=CNZmc*Fv(cShT|M7;WOi)B3Cxr5-xL^ z4j^IC?JfuOT+&Z9Kt=tRIMtVCz6s!DZc83|E=@WD`hK@mb$@KE%khSJ6aYzIK0J_= z9x@_>2yDSb$?DasOX<>gsi~kch#`S!Qv>ZvytD4%Urq-6{B%q`SAc7?B&L9AIXpa{ zKw>$D6fAOnX249qc^L0r{rbgPpW8rg(tlTI>!Ts0S6!m>vbgd1)9i}jVoFaf7EYg^ zx~5`wU7`jb8iNn#YyTB9hoI&y$PWVse#Jd@NW|InW(_e{2$9!4Bbp8;ged z#HUUL_oeXjaqK>h`ctQu~wr0$;dq_QNxIz``4mtflb9T_j0djGo6k6BM|L zC}qu$WCC*XwoYd5yM0YpMY$2jF^y+1ZqoZV&3H%BMh@72v&l0RxDJwvz5T@fLyi{` zdDM?Par)1XKmKS>B~Wm3p0`h1tx)!o=CEPa=j=c?m!6A_@Z0cN>W)aW?o0&fUW#Mm z$=zOH=q3Gyt{PC@gZh1#Nso}f1G6R$9NFojh2~MjHO5TK^<4VGw4EkLZ)+ll*>4VylQ z_{oL#AaRWwH_kYKQ!6X~eoN>Kct3yBpOkV}mkgQZ(~`1^`d%{n^|O6L#C+5k@Hxmf zVZ^Xuk(4>ieJ5`l|8$_=I=3!xmCVm+xQ~?Q`zSAtMNHnXUTNgfDrC1E0@RIEwvn;P zHQHG=$ws$SO27GPZtj;M$AH~r?gfLM>F^d7AT?aBC3?ikxqoW}8Y36X=ivnPv(Yah zl`?0&tHoP6mI8Et(XBYnGUm99m$VF&To5!hzyobm-MP|p=g+@H$FVIhgz@K+t2?d) zcpIJAqo#FLS1vHB^SCCPr*qDSdghfYPxpR7IQZnt__a40tP(|uc|gcHH5*_W_Wt~; z@**4J(&XI4%#J;c;V5^xv_JTa7(qmIB!Wv27pp*KuO)}_oCEgne=zX*D2fBde?nPg z0t*FPn(MDFr3$i`iNA01jv(0@Wd{z_vTJG-au*GA!;JKiS2K3+aay%%_x&Yz`R#2E zD7M@N_S;FuoA1|Ht~+r!6xq%pOK0j0Q%{_?=tjaf{Wkk=E}YnEow>S3Mg_(AwcEE% z#f3{OY9m(%4%B!yQTc6&3OJ((>m;U2b&`z3qdXyGta)(62SmZ_w*#OkdG>5!=F*Wj;jRK~OimDYja%2R*`OnuN^gDi zmcIq2kz91rhJHe)#b(nHJQ1c~1H%S$W?B!r4t7=Vdl^{UV7V7A!I~RYcPK30wj->^ zI3Mek>ec=5eIULY!b4^8M5Oxithm&eUz@`Q=%rQI*ts&@mjsO%eQb&Ra)ut( zBRlstu6c39FXYqOG(zCWfVy-{Y>PW_V3fO6VPT<`wh<|Pqco&Mb`AID@_a8wA0cJK z%LHM>-=*AlZ9w0QMxTdfKsN`euY22l`0$y;p+G1x5USmCQ;^$65t;XmTrIY6L6PSI z2c`oS&Nwwci|*iQ(wtNCjiF(tWCuRgV39_kVUzCl7%b)bb4tdXj}t+B>>AzeSD*cM zZg*8#9655IocpqXxw5Gr+xFz*XC|#l01a5GH^5*`<$@rnir#vQ{yR!c{L4@+Hi4Re zg>FDExBc|EIh5!nw?PD?siZFpf6YQP&281tbpMD|C&((1WQAk21h-#3pfHgdf!s00 zB5j>>h8$%LCB5CHABE5E%=xDhS2{UOh1o0ysDa|OE+B%%5UDfv4%Nu2L`X8T0MXuH zBH1v{Hv5i1tX5W4<&n$ml0NE&RDH!{>S|8TG&~ONVru0u2CSh%3H$29hUnU^Towua z=JTws_Dbx;G=V0_r01uS-A*Ucid6(WAeA`+VaA;>-8-O{aYnOF+s6}bCla7@QB@Sb zdiD9nSI{PZdj909?->nEp*aWzn^rb=g;aMqP|>c6K)DpV1PrdGsC} zHw_s_*p^p5!-Yf9XV7Y(@b-1NN&NF&zuA<&LtSgi6gz6^HB?f;wp+pqPA~EkAOd}K zSbaA6kT!*2LZcXu{A}=&duF6}OKOfFJ(q>lLZ*~q=4NL8>ZR;9OD-!9C5-g!w6s~2 z#I#!gmMgSwCCV^Ash4hP=$vGa83L5@lHR9$`(jEzzGm4HE>F5;I<+>Qqcc0^z=eC_ zni-q({eJZC0*oa5+^7(kV1x^vK@{bG7gu((Fk z#;(}wx zlo5~y(X<(I34Bgp3Od${0pEF-lh{DMdjH&Iq&0AW)Nev1klCU@wwk<2b)#CH==&Hg z9QId6&$ZX@0yyek1$K(G7D-N zg=t&5EZP;>8ON|FP|xIqX=xDQju=PRr^eQ1M>6EHlSck1G=3CfmYPT2EmqWiSSp#U zt4De1;3Qs*)dGB03b1jR??(H$B|_Lkz&zQAEiv)_tqBN&Hwsb42YTo!A|! zGfSrV8E!D5j<8@IQQ~biOquYOC%b*-nI&2sI_z$HQ((@E$(kl7bpxjcwElS0<3Y>; z4~lFi03!fPRMm$4K8D6aP&CRQyV7Z(9#WC3T1Vrfb+vPZAGt7kV``m}MZhJB;l z&nP$YrpO`Zw3iY~CZ_=PpQf6jyT9(#z!Y|Lm;R^XHWEcy=ab*=sh-dOP&?fT)^o9K z*Got%Dv~?sDHG;hX(Y`f>XDhiXfXquLuYF;*hr;qQxIpj30<58TqaGiBWlbVkf{Vn zLn%tx&1Uq|LTS^ozc}Y&Bwd|U+0p{?%{icSJA0dUAw7RPm(~G}BnLi9?wrFjex-iM zqvptE9*s0NvnEJ$>R1-#_JPTiiEqDp{>j-F{Q6D|CJwyaRFD*f`fP_=y28WB#gh#a z0BI{r1@|S z8#~s;$kbVlx-J)OO}n;jkr?#`V=|Ms3y%5uhI?m3HUi!8bM}NNf%*hCV#Q{>lp(8d zutcl_Kpx2+v>@KWy!XV>?$nVfC4sd_X?+JdG;hPfwiIY39oL~PXT?egQho`k znzXfppK61nrUy^R#^Sas-@lJvHLPC}OqCXCoLXk}!qLvkAKJWgH;nQvo&UR0Qp$uq zlNYLm)Or=+yFL-Xh1?J@SF`k}qORd`sI-pL<) zS}y@q-?6D``WT?lS88LAiR989B?w=!>dwkh2Wet}(Co|moBs*RyO>JhLvslNUO^`h zcC7op`v3ahOHXK;{MSb*hWtA9hyVJseDsp5iw9|IYhODtdi;^B!sQ0XrnAATUSwS< zoR|1stW8bzKcyDVlb^cSXN#)A%T*wAkQ9#e{U4f_gtw=E#?OUw#w_c}AFm*@opXD_ zKHT>_otc)Fc49RDCuxweH}Urxr71!I=anmWaYRhh_I)FXtKcuI*8rlfw?jns;PE3qrgpx&A|Qsp-Ai4ZAe z^y+6b78~tNLGR5GD%-~sR9>LY3J9aqm|b5UOaq^&BAGc5{)hsmF}ge_+PG{Ian;>f z%hUD^L=rq@th%0E>q=s7H1|hTnI2F+RIBWsvu}rRl9yDnOg%)Ss=8nJxcs~Y@9 zQSE@=rf>$v*@EnT558cemNYc+z%azt^iFnzIt`|vs3LTt^o##(;QpK7x&!u3a=L5$ za!)mWuCs_}r}>CJjW>mT>7}tiZPu-~J*1BBG;rZIhUkvLLOiJdXd|@WCw0BO=l|aS zUTH?*s;((}mJbLcAwWP+Bh(5w@*|1|a-mZf|I-Vu9|hlafR5gB!rv4~4a@yQKo1cJ zGSnSEm*HK$awhs%?6v0BGmejf41Bu}@xC-SQ+NK2JyRuP(vrx{=-4FxMtQMy;87^2 zfG{>|uQdmXw|$BQq`ba(&@l>!%#f$90V7r|!=t|};^5j-t2E_t#XRce5te#=K8c~} zPd}M~ER>X#ghQz>riI*9MqU+I>B%;_#VF|#UbxCiMTaRg7l&^b+0AzO{r5XOglN{0 zdQIo*%#QDDC_Mw%^W>h6&l#nE&=v(JfTK1VQ(((|B@a3V2)r#_D9K@0Lt{okb@D}A z?zle_?66Fj8LFq&qc*)SE{~ujytYU8h@df0FbE@-ZaJT#HSOpN($3CJn>NYPq?}Y6 zy>_J4LBeG;=~AjG8pj}g&o$C~OJ%(37R`|sQl?AC2h>Hm*xpcfflo(REp#hcpMrv?~0=G9wqdcn3kldNUh|t#_B^yg68dq4$ zjNetZH-+etyC-Hl&d9iZyWf;9nqnW=wr$%mNS@KC5Bcgh)Tg!oAz`iA?b=gAH=x2g zj-)S|yjk<%!EznX*2CvqBs8)gjmz?m&zc?(&qf-S55yb6b6$*wIX)zn=|Xok}P@{OTbLXfE(HUNS%mgC>8u+2*c@}Ua)}kk)kBPvJZQ=3l~nh6gU*4 z%G#gh$+!*Pdvl>6BdAj?sTo`;ykBz!6>SfVP+(tqFx~VfbVe^RouF7FQ)9o?Qp&(f zY5c6xzgyP3UW+D4%9pB9n+pn-UNEO|SC2!ohbYx#)5Db<16?u+uwa!;e4GwvVwc%s zWRmwqqHy2Wl+iih5rQ2<9=dPYcV)?tx*gKN#^W$VgSGUMM8GIGO#oyQeqF)9aveOF zf${HR;?Yi3H3mk>fv%|$3{1l}qui8RxPcvQpNvc&q+ZPg)}=!dab!Pu#KVvf9juj%(8 zr{d8+30Dv5Wd{Ggv*(27u6R07A)l&)T<2f)|CI%j$J#tNEB9c4t}e(UGKW^nedp0z zUOO=2EQfu@;8P$-Es)AxK3m7r)>+B)d27%++u45h9}#}?YzLw1qaC?g*=zfvo8liL z=c3l2tIXpRGO6f!<};b&qg)q&_)(HzUJPj5Wi^ru+7QT$VUq3=LtQzRZP+<)`mn{k zC8(OZXD-yRr~~MCp4?D-x3*4OqA*Xfm?ftV9K#h%z1oFU?!IZyF=5*|jmiQHU`};G zBk%5oA6Q*cq@_-%fx<6;TOBY^BOM}vAX_u+H8AMzOLNWGCJDTJ0aWehlx8+?6*Pfn z2a<5e-7IAqb7h@XBoAvj>r@@-(*$I#eSzJ0qRI@&SjNXOQJcGh5M3~x&B zYW7E`wV0GZqDUrWK%uI^joNX(l46X$ zm@l1rnJUGSdadC)#8gYmTTV;hWNTxeck@=*?!Ea;-EWIlr>l?&>pf*mpMZmDM&GP= z8jH5QFE*$_HjrxH4+f}xzd14BSCNPpOt^Q{HQ{1AG4-83OsWJV1mW@6!t#11v-Wl! z>DmQO;%_}Uf!NrA^bLI36QrBw<0_@-sflu8^AoksIbDi|4x*W*o9=o2gHMwxcdN&j zfUZkQOFzlOePl_IR7I`$k_m&`uRF6!_}7e2-&}#92@$qmIL__1Rd>wh$1);_;D3?{ znfz4xgB}@=UcHJTjm{)6-6htw8NA3j-RRWFo*BFCUgSjhfp;}yEc@~)gLza8a5|R^ zZzj=1lVc|ePr!qAB@tkJh?~jkfYt}?`a`-`R{HRYzyX#OChmQ74{(T5V3KeLE;lhd zLcl96n(-c=d~T~VGq>|g~42M4!RRSkY|kJZw*G}HY9k}?jAWXyH~xYrJZO(NOg zoRYK=L%5HCFkW)%xs?zX;l_dEal74hW<#=NYa4S-gyL+j)2en#ii>3wfVxskF{H9% z?m@&1u0VjI>fl?5rf;mjP=EYd-JozW*T?y$p%OnGB^7XvD!G)^c6h`{BuLgn$ z88gqF16=CLh%$xI;Hepy@17jPNbx=lTSOg2AlK8=D~0izY_TLeP}KlH!IjG#rneEy zTl(83zBfrEA{t{xa9BPEgv`sdNi{NIvo`Bktint5%I%lep%;TII)zU=YfK4VTz3XuvDgy3aBi?s4RQiCo$RXz^a#$hTE^WkxVBb%@!fS|HY@kfk)Xy z{0@u6aUFD-xXwjNd7M!QE?t13iP?^h(O)ikHx_(FK$p1zk5(fN?RWYMUFO8_635_6$qs^#w<{87V%z*PMNLlx7U@b4y zl>Yq$+F5DGt}ipzwNW+Fl+CKzr^Q^fD-h$>UqS{?Aj5G$13uJ4Quo{7ZZfI=GpLzs z^9R_|v&9v7uV23c?FzfKZ12aK{yi9AUUX!WLq^U3N%Ked0fp1O@uo%J z>`Lh$2;mOYj&y8W&L1q^W}h3mW0JbM@52bM_qByh_Q(`{U5rDl-@6txLQ&T$*d|(d)F0g_qFo|Y?LzmA% zn>6ZHtCozMnp<1%5zPnDRMy|izt5efk7*fQbov?NXW@ds@xJZpo0?L{kw2KVKT3xq zO+k~wd?_p`jmqG)CHA-D)sfAFq!MKxheGXr@$p59@GrCgY4?{(h8^?rok`nX9K1Y( zw>&BP?uvds_tp1KUO3`QSl5s$m^hHOmHcc^uY$sGQ@ANr>3fC?XRhplwdh&y zI_S~RR!CSDMa4_yN!9|OMO0^;mMAw}#7=^QWK**8@e^J~$Jm)e&ShLQ{6Awd- zITEX!Y>gH-H zD{zX?s2Nog%b;SUeD>0zN(whM3>#QaLFQw|Ty&{rj$r1Q9=;-G5Uo;n;DMbow}vK6 z7Oh~XkT6RF|58#!nL6AQ#q44+C+t7hD;W=f1vmFuQs?Cp2HtZd^vs&cd;0&%m73GW zOR)_Jnpzv9YDW;*rD}kMp)52o52?U~=@AMwvKj{W!J;>f=5E&|f9&w~HVA&$1IQsv zb`E&Evx`e0y))(#v8?ZmYx5=@7;cIEk6-{~)AvNRi2?NXgO?TzlKNPh5GjX<8PO1k zb&rrrP}GZ3meT+Q4O}$kJ01>$W=`&Dlh}`mS)%L=+HnyZ6~B3Ng3uhrJ8s&vsX0te z>n~kK=^)z^Ow#;5^j7M^XKg;cD{HaE%AzGr&vn@ss$zcaLGP_68re*&yy9v?!Al`%UAB+^ zon(4Sg;IN}Zrae>>z;$HY_9s+d(ot^Lb`~uhx$1vHn!ZQwzvAiRU4l_birtfTJ`BD z{qAut)`3}zcV3#i`Q4nfk6pe|zLWX?fW7Gy?bd@C(vOYFqfh8}ky?cl-vq3Tgd+$D zqj1Q0unw(;2%lsJ12qUtOAtC-==SvI*SPOa`NpG)7CwX*$>U$Trk~p=Bp3jjeU#EhF{DyroS_Mm}-(+r*`@{>q5S z6nv|nB^Il^>iz8FW^=Wi%t!uf*7^0{e`EN_k4h;9ysfdZCcb}s9{bsRpPfMqz15hYH(@>Pf08QA_o=6AUrm9=(cN$;w3GX}6M$S(wL6R7KDZg*m%MZ4CdtYg0 z+CDX=v&}5?uF41VMjuA7zA6ZticB6@D!{L$DwFbS<+%yh>?c0!bzX4Jp@}0(@d!V>|fxFKV zpomiY>46lQ8a8PFk#8)Zbz=IQkE#Y9jvBpv7i~9wF_Q;a4Dui}-+RJUB=O(4Q*HdB zlIFH5${QDbn0Q0=?yfugzIk?Km?{c_(193UcR@U);b9cV{G?E>~%(H-LOd|`CLCtw&v_rk>fLV(}W0bD9`Wb)BxLY@eblTpud2{5Q z`OT8$Y(Dk{5wH|+aT**nLj;j{xO`&do{o_RWBNYJOBbQ5ggbRNPjdS0Q5v`JUeyg- zOB^QkoqeU2KCvHpQiOuDoAzc4EAEXy_T{xOX-)+|X@hX(m111-Jl$Sv%bosrhO}w& z`jA)tL|eaoZVz`L%uV!_;;yV}o!@fIUjsPDTarH}LnibZlH(|64nt{4(N6%3N65Y` z7&k1qpiz3nPa)u|D-M?oSBxpZvq)On_=6)>O_9QxCm&BHfbe^o*0}M=#uJ*2EPA(c zwOpxxO=5uQSn&4U>Y!PLxA0C-JaVmsje0>mIhnW;xPpr)=0G(Oqg~Md_Sr=Wfm6{U zLeu%tUqCAYG3DG3H}~ioaA~>Z+LEVF?RM8HdF|eT;y~zg7Q{zqrLYypg zi9V?#3g`BZ@%lgvY0pVP0c}(+L0#2YRl6FbbQL7kBs`x{?FM==>0H=ShLov7>$Y?~04s zq_VbURfs#@VkpNQp_k)nw0_1K{3{?6<-EB?hyohhJ ze^6X31RX;D#`I8A_}uGpEfp2#0nYMLkS(H`g9qxh+5`4QRMOm)HXNm3|B!b(spdKx z)~*X#`wK;Nv5J?vo0^L$vd&mx zqt(N+L&Xl5V#gj(hEl;QxQwNe%~}unv~{%!vj~6?FA)4S#dL~nv*b+siep$-59pD085qrhUa)hS^dt1BK$SBkFUIB2m{ zv_pKpX1>d!7eWr)1M||RZM+QK9y%*CENd}$?jaeXq*>^7FP5bJl8!rc^V%oInpKg{t_$rn*{1K>k-AA=s1T4ZW{YpN8EfT zlie;D1&wPGuv24lrv&?)@eFds%~B?uBjw(MesD$AOUnysN@3#}n3G~P=?Y5I=8Y$0 zW@#m?yMAj)IWh82)S7#eKUYp9s>%@fMCV`cBB-!UT1g$vpCv9!^d>Fm^-K92+GW(5 zk*iJs8Du;NxzoIXD*Q&n^qF ztELYD5?*~XSZ^A1Q+)h78Y59H5kh2J0&FrIG|;@E3_ams|bv)0sjS;aYGh_vcb4birfl*EsxSDh9337@D+;9 z-k}ZqteM?qgS-*N$q1H06k3#-DQlAo*Ock#aLxc$cUCsN0q{h%qAt=etAnGv7k5yM zT*`-tx-9qu2#84lu`7b!U6Yja)>kU{r|}{Z1}bXet2Cz(0{-2iZJ`cg~?+VO$ zH#(b>p8Yv(&K((}Ayw(gy843GcNa1MqdZ-}xgBag71?}JnDo!Q_<@)7C5NM_qDF{z zi{+%==`7xa#E|~;)=O2zsz@!^p`@78(06&=ohhOO>~9B`OHmR{Y8GKsXw%kI?lOnR ztIu%YXwK6_{G)(OG6W(Z2$v9Vo7R-kkQt(e;qjNKG*-A}q1EeSYI=LvXoS%L>Ld*K zQ4O~Z91^l05tnVksO8FYmy38-#7l$hvqhC8=bW<|4_*i~BTLHqkaC62KRzCh>T;w; z2|z2N*jWt7aLhVRHuwCpSxNzTyw2OF?|EJ(L>VYceefI*P9BHo0=XYY;t**emw<(i ztI#du{hNogB+4^K^Yf9$Uez3~a%pZ0*ddUc&%@A*~`B7x&b z+-|Y%Zf=VytKTl$(oB&#Z&0TN1NOdB8}CEUbbmy{#LH;#XM55-?8+x5%T$$!a1alx z4_n5?HdR)Zi5k0v-GnN(Rzlk+qVg16T(~|2vk(4m=x@jj6>aA?DI#CrixbJAuj>~c zN}i}QnWm5it*FQbjP%rR>zI8MOA<6|(ZHn~)-@;QNLTR-A+gZfSN-|0jUx4*A>YI^ zRH`7r!|POQj5hF27xY&VB_ZN1^vU`@fh`+(70V#p_o2`cZ6Ae$j3!&hb{)O8ud&n0 zFKz`uIPq($92kb{KjrYREI!+$WJW3{X|(Bj>q)pMh-K#@0J%#cD#}JhZtlw-m|HmS z(O`U>L}?;Bx=H({SUMZAH?!bEf)D|_)f$c;{}Rk{I_U|tHx}_;6xsfavYJuA=FLBm zEgV?^aRpoJCl(}_i!||f4{BLKC9aoImM>-uP51*!`NJ||!`5O3a>#|o3ZtL?9kH(X zL2sCe@06%ogYI_m_Y1o}8eGAN4x$4PJ`Z(8>sLePUQ}=b#kHcM(U{7J zKPs|N!YDLX+_)EG0%#Fm@6GQFl1Q@v-O=Eo4{+ zg@rINY(d82HlHmbVOCM#9^=96dbpF8@(BA-Y8i5SkUG`hj1%j=>)m-%RW!m$n;$4C z9wm$?%YXlJv&YQT4CIixgZ(70&d-fDBGq`w^l1*vwkCWPt7G8ZD)yTe2V2XIR<|bt z$gN=%rfw{;m_|8>$JoKez)gp8lmf_o)4X=T_|7XPl{{~*bh3|({&-CmpQDZg=F8_j zNm`3ZRUNG1&cm}?de#k(YC*){i5|sFsJp&74G@PU!*S(e&4)%PC;O}Dnb8A_LIkbM z5kwP=ecn9PVC**bo78!0%8s8^Hx|kps?s8U@l;TI*_fnEQK8Q%Uix^j?VwI2`}^E+erq$!XhqEs2jy2z-=ytzOqGmM|{ajUP&IDQ5W@cC~3Z((;k>N0loI-QgoBTZr z+7EqVHdwzj(W1Hiv1!{o{HxHh9h{_#p`&OnLDI|1YuIZkBGg4pT0iD7EAc64N)Jx- zx?V<{i6)R0K#kKlhzc9F;J$|2n1Iar+0@`K2j864JnFNHN8r?&!|t2eyDZCq?P3-% z*HWg?q}d@4zho4-$p_9d_e7PM2qqp&y%*r`yqqPH;)Ky59>i3~o0(Xl+O2QRGqu(`A9EddJWS^tSwUQv74 znJx7Xe;^~#f)zrk8f%+yHc&2kOa!i}=guIjl_4}nF(V+W)XMrA*ZsB%XUrk~69c0w zGT$73bbY$kBx=(p^}kiD(*z$t^J7$bkB0RniDnJ1e@}mVfb13D#lx)oV>}mU=O-MU zE??%&{$J{Qh0zbEYIwieB50OEm#Sgs`!FN%?vP>$mZ00vp=TRrOo5tUari@K1v`R? z9uMnnybY1}G5Tsus2{~%rJ^-T$Gnz2S#zAe2DY$fVSZP<^v8qrTPpW>9xaoP`rz!9__ zH57WO-$YkLLudQ$-anV>m016W^%J1x|K;b|VUWpkToK#ApAjHu;}leMwedXDIRmQi<}!^iOL(rtlGh zlEi#eP&u+q-fo+1?w2s+ataU%{6DIET#X&*Q(ikpk=pk>~|>8Ch6?hTK0(Vvx}bO0Zu5-zAwKGOW{t zN*N^3v$`}8QVK~iE8_7XDJ>F{RaJXU=)Z+AsBKkLB4xOjNx>RCpYu52qvQ5;Yfn1rI5455-F2RFp5%-4I_i`j63<|6d`~Jut-_!IdK;CtcWKdUuG2u z&4x@Xnk_FnfY-|j+~2>9lY^{@KtO+W20?#C zlQ7(~CVqi$ozX?6F?!Y@8IQIRk-mR-gF&1(&|%%Qe?CDmrnTPv5%j?AL}rV^C_WLj z%NZejpdDrvAc{E-eR9lbM-Q2|hkK+o?I(Sfj0|P?=5enrCH&D=seCeyVIuFWv>HuU&$Sz)nfw%4ICXso;80pAL?@>n1SjZnoRt z-%m>9{P*ioUMg|U|EK>>Hr}u`uhr^=mv!GarvdBkq6;GFPtUHM0M2=(=3`ks|D#n! zNpxF{e}C1*XL4uHp_IyQa-4E2EjTR8vz z&u-g?-Q#8a=c~fuzkl|B;_6eE!2bRI|A}n=-+z-Dt1epp`#1j|4Tb*CMXhs~c1P{)?)=nY&KC{C~0c-cePZ zUDr37Vu(f)Y>5^1Aoi}YcOwT>6cw={QH%wRv7p#VG%;#aJfOrL6+{FDvBZKcHn59| zy(P{A7I{ms3Px$pb==N;d8zV9339q;29cM&<~?6dcEt!u5h=A4WA``n(XQ=xl$ zQ4nvkzsrq}8i8k#NJ7PJryLXH2V-k;t=0+m5`Vb)lXMqw$z{mGY2byTgA;n#!6YLF z2JN~KQ3*joekF4xs!+}PlKE7@g+m1Tr)kFnwghtMRJSf}I)2oww94Q)i4VaCxS(h03aXm8 z+HMKwdqYOm_eC3d1ix^)v z=qe?>mvjquHF-|$~-FvGP(>>EAzf#)KG1YHv$5tXg3V2fI=g8(U_2EpBzQ?tf zjYiEBT%?!_z+_uS8ZJbJA{PJ}%YxguKF7I^`SYKlk+kK9o2`{U!MEy6ra^JDu5DZs zrX%*TG{rl#p@ogD?bw_u4?_sPKK*aFGl4hPZvJW&%s>f6~qYGf4vuixr zSmG2@O8kCf8$xB7?@BGC$4YA1LHQ_&O4$7^yLRm=oQNGin=tNeR}X&){To*CRt~HCy}<=#5@yje4eKkgKy4Y1OeVNztBnvKHqWH zEGJqQzu)ga{Uy%J_a$4W$Ox^sfyFgy)T|cS>dL_CQ9qeQq5t?FHq-W7Rr>8tq03%J+K0@6D*;<~w~5>KIOq50zfUQ9W@ndkd4`C+9jM{V7f@$%L_CrC6Hjo7xdQLTXi_qJ2NRI zzpwh0jdXZovo%*Ky;uR>yUZNZH(%u{c12_SpDm`)~i9Wr74Q=Pmx%hCOf1QI9 zuMV!)>)nI<_g`LlHJ2$?BPOpnt?0yGF1ybil zetBAYt?#?j8<-vXR|*xjtxt%v71~HxSXRM&wI9L?a-Z^tc`=8EdEt0O1-W^%fNAYo zuwBA?MeAbPoLj=p)P1ZIwK|mD?jXL@m+LK7K2RkY26yXd&&a<9RtrCwGV44`XSuo} z+~okepCS|_q(y^e!n77|8huNH6FC5>R+~88tAfmftZ&adO~fv3i{F%dPeRxPXATZs?n7Hs7$?SiCZF@x@Lh^ z%LD-^_%w?hkH{zY=DuRF>5XCImp615Fx$$*egLZ@PiVg|fNvm-*^=yJ77uPETmw9V zI9aXoD747sQqGEZ8~zN*5>uXjd%k}dX?%;|Wq*HN*11{b=1FXGg2gl*g}NsbrXhki2InXyckVHB#mtaj*1g+ zQHwuvUj0#dSz234MQ2rGN6*dKLC0FhGN)(J5xJy(;=$3XTOlJqM)#>Wrx@=z*4$Tn zKW^u^V$#!@WDi4MjU+Mvo(u5s7D529-$5MEIt4?qY>cMO^4kv#NjTbmQV2e1})A;m#A_%PuKQ-BN8!0N!L!flA05fYgRFQogA`kKe(lT zi0Y#Yk5RcPgpOVn%nCcpb{T6F7S36wu1N#H^uFhe5zwE+>0W)pWFet!37)GQbky!f zXO<4HB=uLr2ZUb}X-60eEmdVpQWTJ1$XA0ky>*;D)Vs{TwQ4&3m&yP`uvNMPa*8-I z`DwDNcp37u6YUO6G~pZbIo@!=xpRb)FfAS;Y80PD7SWxyQE0+<#{tfUA{4jE6YOE> zD&{>|U8J~+n-?N6sUZP6M`+W)V~eJjApS{-+Wn`C%0Vwwt-KVtDE{?ItlwD8eEBBG zIwG{LP`?ytB!G94EehXFpdPc0`pA|}H3AUL#U_1>mu3CbGqTdnAzVxsa`Jkk(>l<% z=}3y$1EBUU(kZo}VA&p7Tt7yA2_B`wOKL$AytwcqCy&ICton2SIC-7O217{$jrv6x zf3IX4G6Q)E99?=73FDHC&)s{76C|r1>sU}93B!4iC8dI&F`gVc&f>a*<}Utv#0FNx zg*s+V<(H2fZ76hl0e$GPAevYa$E_^WVoI#c`v@~b(G%?2Z~l6NwCwW5pFMjfVzRg` zGSihJA4tj9{}baPZhSNvZZQd++OA57WCzE;OSCd^hIwiXd+>8Hm!{Q;ZCdjpam9UZ zyg0ZtwS_k)z?N}e+TShy(v#2FPG?x@64X&gTu9kXBVCPu<-aHfIZJYAQS}PwX}rd- zJwOxZ7i){HUei4K0QK!aW+G@H@>dh}*zY@rFD^I4MP$hTy$ z(#RvzJ>F%m@o9G!v4lA!xt<AOQ5GzTFMWjjUd(kI}YH02q zzZPPdNmK(ZNj(9QfD}$;6ugT!Ib zQi&wc{-QbCGpK@*zntP*FW>;SdQq4v(_yx`5ET8iB`lR5tos6FZpYJkjwi9eN zOERSGl(bv7o=(l*@PKdmw7_Exjo%1)jsLiHD6m<41VYJTD9>r{9Hn!Qm6fbWn)gk&< zgN<&fY5sNj%-bc+|7U*a{|@%#hA3)T%hVnFz$c)(eJ$M}YqFkjymN9zKf&T%+8S?m zV_^R+tELl!Y9tSMUxNiw9(Eel=?_+i-I6_PaeS{Xf6+f(U%#mFGB+1BhIH~-2M85f zJ=pm1-lKgNaX#OzmR!jTpwA<0g#3N zA2HZ`;90kplC#0Jbbr#6zKVjYR~a)t1`UZ^TWKXph4#3rF_v@1ArGil>NA3bbEE@7 z%hUQkc8b9!A~8NIL)oM3E^CRYu!Y5wljRWhJbp_|tDC(Z<=VA#B$hMs&x;Oy7<|=T z&rURvS+z3gq@*OD$w;RZ{W>BHFpo-3DGqeV#0Q0&%DiXvC?jA*BKt z0(H>ba}=p>WE&;765tm#8%0x;85m&8#q=7+z1utcMwo3M*+F-j6N*P~@iS1D7E@mT z?T_$xpp;DKtsv9QtG|`MQ0;|NJXdm!D)O`WOnsect|_chOwX1N5zGWR{DO8RM0!%V zshJ|_5e3Ua5n&`e2&qhuq*44{UfB;T_c4y>3Aq64e92gH)> zS4((2#9ojjBg{dcx>EW&$j3XyJ(9?Q#?5IWLLesU6e?BdZmNwc zE;?Vc694+VU}@tr5}m!TuKL5?X1yE8X~y+L(Ta>OY3|)PjTONA(Fb&7>t;H;RSC~{ zX+obF`s`Zsi(0E{1Autnh>5TJ|8O7t*U<<6txm;X$;Cpy zY1dAwN;&j_x>a6%`pXe}AeCS4|NgKQ(FJrQ5G%i{d#3jLZWAMV0XYL;NR9o~{HlqA zy=%J5A_3AnOZVxO-S2}%#s#Py)O0eJYrd%;538~X9o(p|%pHU#gY;4YtJg0c$B{mW zn||r2@eqQ;iGAMWqY`_{GF@wBdDHQCZhZdYP7>hNrE;x}U-T=)$1?zGy+yoliAlCZ z*-yNc@Vih-j27*`0Wc#);>9Kw<5gKszrG3lxrkU^mT2I!)aB;Dio}TiPxDZ}MVi~x z!N+G;rE(DMY!TvdOdM*y*rwfF;NuTo8Gm*7d^Xh=+O6_ddDEkd?%$BO4Rn zhlKwQ&jVJsSeY}Mtg&fHP9s{z7C;*e& z+0lr41laYSVU`-cd4Y7c!5tj^X8Md7uItJg@8MP&k$B;p*r}A;ShRoLX(DX5&sLe& zSaUTf)9=T>3pi+An%7LYF!qGNVdgEr$HrUO|7&kdaz;Bx!cq@l>aJJJhZ9kTHh))s za1IEx2s5blCvIVp($_yRODLoXF*#Mvu>9ppR`y(m1|D0 zrx)#OH_Af{YQlgz{qYSe?i&_Q^ssudDt)t4jGyl9#PP~($eJ^k8h>|)D;!hm`i=DT zMQgWmtDyM1+sN6!pYuKs`VdqJeY7nC_Jl|8NsWF4W$`FG%KoUD$9jc*9@;I)w>R5* z0XhazVL2ZDJoNtK$Zqbwx<0txp&_%sg~nK}&Bv6co;;GS)GEF~_L^DmlLexAbj4>M zw^`c2xU{T{C)lPm%^e}b;(*jJ+QF1d-(BBpei4uTRoXdv_9jYii8Aw!n%m(9P+YWw z?!DDCT5`DI(x08i>v~Nzw_cX*+4JWWUCl3l%dJb7zes<&Zhi?DW6`Q7#*c3t+zEum zBAox=iBu+}eE|o5H9z%e6^Rscc30K?s|rKUT8^_|By@a0eC%|83{G{1+A9R31-IEov1xBAH< zqXdp4Z_WosAN>n>Vz#fFP!+I++mq8yl0YlEq0B(8efMBKvuu0t7h6I^7Ynh?axIgP zGv}PSyd$RI?01$$)E$Cdqech!)UII9n|xakI54A-&f`W<97sq{TanVGhyR0lHLY3J zdA>a@5A^53^JF}1)MBL{=NOl5LYE=06;P>{PlM?*P$aS8`ThI%vve(1d%_g7?7*HC z{Xd=bIz38`X4vRO({7LuMRRayfUM$_(5>ajlb6lEP#vT`A`$Aw0OlCxV{k*^`Abud zeXu;yDCaV!6>1(oZkxH@KH4~;OKW^+vdPqC|D+BRd$erk1P%`w@I#ut{tt{Jq+T2o z>3>x7O~HoA=lp2e2t*Q^Rfju$VeHP};>Wqiw(M&YA4}iMu3j!lAeD+PN9J3cE->^0 zW)C!?PjG3{0?v;iQfiCaB~b!Ya=B#*c|j1^8`=Exg60iQY}o)~e+x+3Tr=mrQC~(% zN<~bwKGOMMgO?RiP^Tyh7-{+P`|s2wA`dD2pfEKY|0k2;EftZ``#&|mfE(j*GMf^6 z9otW~>{NOc(R$A*)9m%40n(W0WTEdA&ZQxFC_7>gUoe^qg!^Ty-uj>1I(QlR>%RT_ zE1c*VQhWH!S9#Dl(Hn{avlkGgbw*ZLpG6O(Tj|<6wbeM0x1hq8TPcddZe);w0O3Aw zHTs7GiOR{n<}F1izjkal8KKFCKdO4|7M}CW(bwqH4Y)aSdG751^BWKr^E%wf5eXYCx)SpSG|z%NQ~2*r|1x0Gp4u;a~ZqSPyM#)Hx5BRqrpbXOU3 zJ~V*(v)4m`g>Ih!a^<#r_qK%pDw z!4F)RQvb+3^x8Q$vQdK^^{-PI_s_-fr)1Y^6%T8J@2+(;0BLYM|M{A#(pb!T-gP0B zVX%=mRDGW*;Dz5kK4n~1aA}$fDx}z6p=wE!7FliZ>!s`95@o06>+759Tg<_)0>8FE zdyY{fRv{tpomy=X3h3! zKi}-in#i*a&XK_mFwH5%clDVAEe@Nl+p4yXN5hM;$rs+J?Ppx@jdOR1`1e5RhQ~+rqE})X74| zVvCR~VWmoLsx~pXu~?Cya>o5O?wvX=;AJ&$d2gs+~8CS?c05hg3if62CvG9!6u#)GD#jPz|*o$7M#j3 zPn(5Ps7ZS>r|>PpJ_JR=-g&QW%mpePx>f$QzSMfx7Nb44a8q(w?4)-HSXHW7a=|m2jV=Yxbtwe=&1rJ@CsA z{QKy$=Ydg^RU|aI+O#%D#>I^98T7{g^tl@W24OGpLzZ+D0DWZzC20Tzin7nTQ+J^n z>D8pYJ@wc}gsA&A^OD6h((Dr%%w*;;2HUEBbW|qv#EAqqQm5 z1Q#noph4gK4qjE|dm2A{)P?8A4A!`VXEN04Yvb>g)>3Hkchm~3l@{-q09 zHxEOGEIkLVXHgA>L!&JL?{w&p7LbpOmeVU4qM<%=-j@j#=Pc_Yv}jc$D8w~kDM>nM z0|Ie5`jt)nNxB*?R`%?|ygw_j1?~Fd4`&Ta0~9Ah%LGQ~0e4g$@0FLE#Z}XOd9br?v#nxqUqtR>Rg}h+ZbW-kgZgzZP15!6!Rxu+8 zX?yZNM9WbPzcJp`w>aHNgiUL%O?uW*dj;BR3adz^f_TMsLRZLP>t@OE#EQ#&VVII`%@7Emqkbao`N7+&b&ovrVI?pTrNX4Z6CX zzQwEeGI$C)^O}5PytHM;ccH7Ma}aRW%csIPOK)xO@AQeME6F5(f*&@ebdtpap+@UV zk%CR~ykJ!;O-{A%xr?4M^@M=38X^V8R$Z`r_gj_bHu(HOnudyisRhi2b2kMM&k@H5 zSLJ(Z(1C3BnPcg59v4cDs>wzqb-)o zC|tx45GtIu-FTap8DUj>JX#r$S7xx#@w%f9(CA7EsKz62O_yr&;GskGI=if+S=+cr zHHjyP1rP~?Y%{*XW3wvVHu6!ICYpb(jMQ>5LW!?~L&8sdYYxQ)vBQY~f`I|sb12qi z{d48T%^78-4&*vjyqph8r?$+yR=`{RlVbWhD(17m@zA zD1D?s!{3!EUfk6xWR#Fp-l}?nPVl#7o+Z1kz*cOax{Sc;0pDMT*62I+ZD1f{9B2u2 z<4`Gz_L1JoeY$hq2@&E~fUh*_Slb0EFJsfm<{nONA!Qy6tHtaj z5jNN;=YS=KrZae6e@bqfn!6kK!AhX-mVU2Yt9V8HD&i-il>msFvW!^%x({`&JPty) z?0J4EAnkfok>A{`0Oqndro|d@fw8h0YvoQ`z_V{nN6(tOmhJ>#ryP5+FG8)575wVQmn0|=FLt_ATPiah$k!-r?O{o99fINIb`N&4KNXD+{Qu|kSj*xW=;x7~x@OKpER98;%gp-5H+4uTJo|B2EkyQP@STL;d!C?lcGDnF ziF}n?vhHDPR>VI92ZfZBC^M~WU_(c%Gm-Mr8&apbu5b((j@3qGHRr|{#CLL_C+=(w zG(Veq{)elzKFHF1^JTJ z8w-Qst3NfqLCXo7f-1GL31t%T6#tp?QgM|jQ-d@Z+ljui5N1`bAKA-z?0#*SH2xab zwru`0lo~@IGiEQcQj^LA=Be7g>yMfdAw8+e$`7s3Yiq@sK!i%Wse`A<*Bfu6=MMv1{|8baREx0c;K+n!o0PS*7Jd zlf#B}ptoMH|A9|Rb~7=IlqIZCS%8S~eE-zS!*gAfcRDycJMlM}&*XS(+g9CLrfy_x zI8RO1B@^y61d?fOBHR9jixqN7?q6q-q+Gg2pjj+0 zr)h%m#fvUeNBh%o7}mL*@rm&t&7R>TrJ&l~SfUg}$>1u50-~v@fHoJi~*(JpIHHTx>%D7h|&RWt*78lxk z5&oUQ%)|Hj`I)(ogXP;Nom)dIb+MLgFf2a%tnpIkX;A9AP#$O`D$3j=sdtT^2N(G3 z*2?L4Rc1h{)x2Sbm_$sU{-MVT^FjHyB!?E6hQ7rr^4tqM%-(89o`yaX{sII*VyTgY zLoaV#Pn#8wo@+%8`)(g~FvZ?oX%Np%-^G|t5iY6DWO&YUN0?mw;std;u~%uZ)=WMk zzUJP67%aXsx?8ty2`0WBvgN>m1C}?hTEO)niAX*KYEyUFI`i_je)ps^43UIv^Km;* z3%C$pF2MXZ=dYUdHnI@&-fAAK1< z#4Z8je)yp?!E<WFfmrtJv+XwrZcRAo9gU2|9%0YOOtJA@p zU{ccyJ31Ku9MY-U?yi49kzr_HM+jb3z*N(ogT{+lY?H6jk$w9zmeezTIe3h+a!Kfk zg2LCq0Piv^gTz2)<{dYSvX@y3^(94XJ`Hvk+E1@1EX4Qpcy6d1|^jTFUX zbx)+HSrQK$46lXe^J_$Y(#pHrKrEsKT+nWDcXR7ZuUoZ8$sT4!nlhVBhIz8hB2Gxc zpMQF$^uSVA=Ho)|IR~mwet(G4s!oAU1!w(xf?utkbce2z-hPcntqZ4*M{Ztob9W@r z{eWwZRlaL3ns;u4=;dN1%L%nWv_1Q;IJ&J&XJ)=})F+>Kw9nS&)`Z{#i_HglaXuGD zL~wt45%(1vjsg{}q%&dJHv-MpqjNyemH+bVusLmVF7vf@BlEUOsBW=>J=-kI?>hg_ zr#|IyO7g4q`}q$*{#@`8QGQ#o;x|tAHRj)*$RQ(Qo6PMjubK|KP)lT9AQ+p4gyG~r z?{q68V;$;ACOmXdM~fpjhx=ZY0)gkMVFenAXpMDr9xg#E#i5L8z;-*Gt1S&XH6s?D zkwxCb0jL2=-p4p{VKswX;aqjPlgy!bk4%n%FpI`{!C*Q|K9!T=QBxT%CY_qITkZV= zYNMeHSoWR+dkom;n9hsDwr1++lfrJRH(4=pHP_YL7+2CuJK+m<#n_MQ-Z-%YS19{Qn>NJFa9zu(zO}` z7YG@H-Yvy>UCang9bmO+J}gOW#-c&XH)cnh3*bjC}yWmjfIrH%Ob9t`e>*&7&-p6Uk&SIjL_%W(1nXXLqRT zUcrO`V4wW)FUeDl(5P+NE1vVZnwL@_f(GhhL>KFiwrr4dy~oc*Aa+0y>E`D4tEDm= zSuUX~A`TZTO(u=5mmtH8IDX}dO`U9JX$~P@$b}HHENnoC*_2@NYjARMIyOLNC$y9< zuWPHTNE%hGn0v!fcYc+?hVSG${_!=*Q$YD?fOjWci9^AtxM9&}%!pT#-7Fdh5ZM~4 zvolx9lr0-abcz9o7&T^$QqKCY(U^gJIlh^Nj8HmWNt)krFDGQyNcd zEZ&@d1Fp+fG?2zg`J8fnj|vn24q5LL4y$iY%3 zOwgABuO(ylrg9VaZ**@SIeAoLG2!5t*7;K&cVKub$s$=g$seC9-{dmEO1 z{HpyY8-W5YCtS#jl1LC!8|_XcQs3?$s)ytpV13rf5v|9vMj?>5Ttve|V022O`Ovg* z?#OxP66Le}G3YmDQsKP5HM?d?9}^acUxrDxtgv*u;J*nU6GWt3o0D4;?$NkY25=caE&X|yy07SU>xT|ao4I@VjOm0fg!tCkN7 zjU4u>w_xVy&%J5LSgd5#=V&H~d8}tl6L!Mg1w8}<5TC3zkzPUV+-!PI@BsZH=GEI$lAIA za#uXQsP5sv1#720ArhY9wkl5teH9y(f#(4``Wb10fA=BRDwiY6uyq639nI?N_eaYg zEw?96nq;wZetBftW73?6&1JG4pUmxBD8#35@Ia!%-T7~4?kwl##fgbY-p=-OYf^iu z0p}5B1QfSci^sX6j<#7~G>u+@lVENqzwqoLGglP0H^UZrva2E@GGQ>Le)w)D@4xi` zR{H#DnHrw9%*>A*f}=2M+SI}AAm5qfd}|FOrqn4l`)4Phrxj~WVv_$97@5{7&k2P- zvmWL?`XD0TkJJBZmp|k6md?Gy`+srmqi2~%A_G1&dWkn$5M`L|{iv{==y>cgMWy?b zLAf)wd@z%+H0Euuwbx!%-1DH$s~0aFzZvq~p3W{7%jfR8w%}a1(Un#LlrCIrMmPWR zG}AqIRqV{Z$=_qzv12EE6UNoLu;aISbuRAED;$#IQ%bovq_%g-F#hha$c=K-$KKbR zH}Jn!2yOp^z3KDOO0(K9U+Y(Ny8X@=e-G|%UY7Zf4i4U}{yQ(e8ItOVH1NsU!_3~P z@y}lS0b&>une^znfv5ohKC1UIbHk)x`DT^+?f$!K*?>BW{)u}Z03&o~(=X5Pq2gdk zA5ZwmexOu7PQ^rPT+j(vxXT*zCN7$eogS9pFWVk2f+vLgIrGc?R*_$yw$B;V4`w2V zyGAKhGpTtY2f;L~HoL$^`$e`=N?iMVWxP7c?LkW8eH={L;2u?)H}@5tT>TqjNdO1e ztl6_SwVh-9s`E6)eE-OLy36cPQvM_cA7lM4l_5KPcwb}Cw?mS%zAbuAWdP#9bKq=J zxg*nS#fWZL#Dl(ez?mVw?RxW_KVUw~RvfTOB8I92{wN#-IvDXSiYTI4j%o$HuP7UY zrFH&keEB9#q`*}RU!X4;P9J5%*!QFf0CSy737uKfVbKQN|wfVu8IcQ~#@p zt#5In7ZMVfniB#e?{KM3Gr#`vlo0GOy58(R-mp1%)? zr%r8pw_$V?ag(eL$P!heEL|$%)taCy;6%=}SARA;01=0PYhw5p*)R5j<}9ifX;lVg zpRAnlxjN4~kVrI7&me=b-Bu;6%$qm3H++}&<=azgg?oYc;lOp}BIpY>qy#EPsS zYK{OdPi};fo-;b^^{b_c0$o@P$A=R~Q-Kv2F7(v3%wtkl(1%5DZr#4EW~kEt)LjPk zb#DdA7@>c_kf%uDELO5c=_%;cx|Zx=R_DX^^2*|@))(F5i`0mkS;&xqgqcF|A9j*0%RUI`7QplytIrQ67nBRQv3w90i& zjEYnjdm`jMy3$RQwJF@RgO`rdfWVn(@a1*N6nZiaPm&*Ub zK=dP4gLSos0U$^Rw61=mh7H$}tVt9>u6QuBgt63uE%~QAeazTZssfsgQj8e}U}WhY z^O`*;q4LN~pP!7N;2X_w1-*z<`*i%@vVlP|0Uh^ziMz)2mbt}uP-aO5oqZ`FPM+L` z7w5B>l4TB2(?H8}n`e%N3!c;S&c=^;d}M?g*X22>RKh&DGXbHYIu`qYFT@>`R*xg_ z+w@W9y%S4;Vo)YY(ajGbTBi=%m{LFNbCP6MZBwQwPxkjIDe{gJQd?OrN;hNN#QS5{m zigsHB$M(goZpL@=8_oNFD6cq!eN(z|H?FDEX!}MQAWo#bciHzE@1y7{y-Q|-@#2;r z4+u|nbQ;&@cW7q3b0iNmVtXay-$FJvYS=XB#7$0iHC#3SLsP99Z1~jpXa9P9uC(?- zcqj^lQme$({*jO!;HKBAZ_Ibvca~$3!|E*h&LYK&8xz1uQLst5KmF4}JZ;C5#|do!4X zE``*i8bvnWAcVEL;C%*?exSnEpAPTbEa753ea% zfN^5h8*~3uc4K=(qny}#U)F9d5+sS4(3^{P9cf6Xlj=o_c+_~_pJy93wR_L&N);=$M-;oAe5hB4CGlju6o* zOSM3)UgyQ(0|y3cxTjptsJgAHw_VFkh}tnWVKxjFZ36cPB7ROpP6Am6I{r$~f?5Fv}^5cTBtC?uYgSYeA^dklxi5IbK! zekGj(T_4<*z*A!8Nys-)+ z!CAUqWEl1F_a}s2*x_&P#S`SopCAaeNZvC6wv&s_5^rHKENC$T0T@n`0^hw6l6J;i zC2{r7D&@XC%w9_^m?Y~wtl(WIcq?(H91b{yWMjM~9_ZOx2+&>qq0JW5B=xU!m zCiQkR1q#8{EF+u!0SuJZvh}J1^?xKU3ZPGt?rVOzq=sB1ZjxaZMC{JiH+%l{uZDD{ zm#0~f)CrlfKA&4|@4}T>2I|pmu~!`%BN~N;umqcCZ$h>f*~$LvD5oryUrpG%=dzF4 z`fw;VF{*V?(96OvfM9$wvdwG$n4>oI?UkwYDrR7v0eO>1xv1bd;~qHl-sDrM9Glzo zO8Coe9~rMmT8acC=hTWfWm7fQhM?SYdmZyGf8p~W$2?$-zdJbOE~IL{@X+{M@PjBi zIkq4e&&5FGv9yxz$2Z1Iz2cH(@vP? z3t~smwm|T>RM;HI!=*nO+do(Th{PtolD2y{8fJ9SB7W3#h&ZfvYLR3o~Xk?ecW4_zixizORO}hw+QI|{sYclHApe}I=P(z*>Qv;K0hNw2X zSha*kVUa1VV^3o*bXuSzjKc~Y@0rp@Cn>4WvM)vm3)=k+U`m^kr*C7X+dNl=Vz4 zQ@r?#ZqKL$1(Sj&cSzYyV@3Yc^x013ubG(DZM3lgLas(OGPp%3m)vcta|mc6KO`TL zYowS6kZS-2|DYhyfQuo`CeMNoW3keaj_sai?`Z=5Zy-@VN?oQ>2J3*(VaH@3pn%BQ z7lthg+uNr{YYu5VqCiV(Ubzhk(+H17 z)7iZOJJ5Od8rEXs^H>1(A<0jTeUVg$on0Y)-}jD{KXP&^3Ns15Y^sW68A@;qy3lW3 z{yGukibz89@f8mrJXk)VM*6;KMQj>q$0>613WITaX}Yy`2M$s%k^;<}yRye+&xbZ+ zoO+?>YlT+ zXyLO$*Ix$2I!2r$&9O9_)VIP$Xp@<2*`Nn8<%8N; zjNi3JzRLLwBx!+V`}XaTLni^USgceyLz-=|q=TY_pch3%C9idRh8;1@73j0to{D~s zFCJeHw=gdy;zI7@Opj6~jmxVha4(jTR>;L}=UyHd31@?MQ(udSd*s?RUx4NgIaSTi zZ+Xe`uFr}~Il)us%>jgR@94Q7H-MD=0o0kbbDOhm<}I7oqpcBoLb`#Yx-m6zc}Bm5 zpL6ue(Td;3EVty3=3*G;3|4h6GY?tmroipVK@%oGTAJ`n7}H!xDArM_l}cXx{os_vC(!mq zu9*<4Vs*=T-%td)+`4)}SvQIt76i}sowCDZQnehriQgL2w{ZMswZ?3C61*$&rf~5W zPKR~Y1y@Hn*8^^-gnDZ{SI8ou{;&(3#T=ws51n(Jq)PfoNmYUE&v7N$f90>K^h>TB zQ&qLf?W3Gdq0sWZs-P3m7P20jspqj&&c2xZ!TfCQ}w4lFaNo529}> z>A31w<^R@P%6g;sx~Wy5)<%d%s?hzZ2k3M0^5q!ueL=5SbCKK?*OD!;MN$T>IU7cQ zLMRpxSGa%s>3CmhdK-1yupNE#C(Nri!^vrXo^(ZWGT)#F4M&W4JZ7yC>&@f2LC?ZL zy)d=_TwA7-iUVfRXH*z0d z+|(X~Fp(tX_;Bth6T3Q%oCJy_%5DTm*(B>B3FRo3WMO;`{!V6*xx=b=eH5l~!0e98 zc4)xw14pEeZWm zdQq@gi&d;29HwG}+8Ot=G5r_MS>>hyFu5LlB7Fe|WBm!)gC(>Q-3%j(ZuHcxg?#hf zkHh^&^8Q(@wfFgx-cGFb{+5Wx%K^H5; zq*Lw7PL5DZZP#FZeta-vLo3y(Q^(p{j8tZl-3ppIbt>&uH&PT0D<}3cJHGJ2!-qYL z9uy7L7+&ukSNdY{_n#jvS2J#m#Butn%P& zSQp*LaMycRG5@l5{zr?V&W~hGhJ7H3qWBM2cTeuEQWmVvMdO_`nF+Yc#R}s=ljDFwm@8aKD?sNb zwfOO8Z+ipaEx=4qh4ti`1$vhF_qIopMf!bN4>*4Mko6HjwA#pwMqlZmyd1~iV zM2g=+u~D|s3m|j&cCo!$B$WMkB236k6h~`;2a+!>Ted9c%IK3W0T`lnIF~pte{~1X zX`Riik2~43Eln!jFhU6%`QFCc0|pF$mf(W!WAUF)-~IXWWsq6{w5Frqwa)K0V&le* zDz=a~b`4yCh&-JOmL?k3w=IX#@;(7a2JG~SV>{Q*Y60W~6125Oh2!cu_sv_U*1uDD zQn0n3tQ=9i0&+oPknL~3O}hV*ud3V_SXlA0@JKmXjhX3d%+l!BmY_2mQUdSX#4c;Xp2Pw~aGX${{;gsRmcQPVCaWA{S2&bElAoKP4xI6LS+ zF5LL^$2%k&XSW3On2ZWjNC?YX!Wvmp!t!uy@D8eo43#umM+_8r(tNEFO~%G*XzE*7 zC`t@v)~)Tux04-=!YtjM%Qrrqe~xzF{h@Aa=T7d`t>E*IdxwWs?6T2uUDM|Y5FtK3 zD6f3uPr_0jzpNeX$g`Xk9?b7uVa6}NF!ZAzCroELefFV-kvq!9Z-Gg!2D}E{Sd#H< zB>KOmJ>^wuDoQ5kSJ5QmM$1%P9;9vI7Byjb*xgA$%?%{+C?dejeg#6@1p?qFDZaba zi!Lzy^0*n-Ug(AV*f;1V4QoIGV)Ve>kN@OuaPCLL>#;>}5z45qyohg{Z{oC4bzj9D zE3+;*@Smp`T&+R{Va!R8BJ zH)U@sO~i66L-M`FvhS0m`VAodg!g7bc1w)5F6x_nXV$|ik6C=1Rajqpn z*y2=f!W3l(i!j5{{p#i6oDaTg#nBbo>?T>uaG~nxVK2U<0kW3DQbQ|*>H`7lN|5L* zfL->P9ioxNlgvjH=J_C4dW*}1f7mp2|D!iF&>|ahPM?#vm1-q2f($xzd@wk6j3t`5 zE6fZUPb?6o=FRIj>HZGR3CI&1``i2I%R?g1PnlLT#>r($s~nP?h@fL6g3`jVvZPg3 z^ADo>iVW7AG>F`OTv7X_Z3XvU$2y13y3nG=m%xr}=fqIj+RsjWGIs1(jojty4WDu( z5!H)~A|=O@Xi`oUz~`A`cGtOaQ*R0rX3^t-^nH_8Y>SzLdWee-ymz9sRfE9S4_ZF{ z8x+&?v;FQ(2NlU62X0WdA4D8IYdAr|S>Zz-m|VzV$UmoM%Z0XW$H$V8Y=20^v4!0# zd`w854>dGT7tr5q$QVw0d8seIt0T=)B zo!1V|F*hPT--pdGwmDuk%H}22jL7HtYZE$-Z0_Vkj{)<|@`>w{c^F26&65i{)6Mo~t-B~7 zq+%MdMoNDK7GfGQGQ1pl<+G(~qH_8aEm2}!ao?a;61%<%a@c#OoJrRODf6hz@eiX- zhJ{1eChQA60s*-s8CIKU#|m%^H%0TYjl7=fxC-Z`)!;f|tQcdCW(e|c5z=Mbjvdrc zBLU|6%z8^v*xPmzG+W3;na2H~=4mXD>@8MFwf9U*#Qj@*Z#)#Lo1tC~ z0f0ndJ9oYr@J4Uav{VzLmj%7>0|1ZH?A;=T3$I4Zr*VGsO6?hRvWzU2iHDjWr5t$L zzK~&8WS0ldv znit3_l#`S4D2i<@J*4%Jss@SESRN2X4u@|}3Q>{OejHsV9%BrVurH5QAR#Es(H2HY>ZD=ZW%LL|dwF>2|nz^(y(^ z+}U~7|KBo}ihGGZJ>OR|!POqn3ZYCC-Qn2qxZSVUQBST$mBU28uP@eCl1>xx6@+x{ z(*^!9Z=eciV4vQD;1WAAPjJwLVi&`w?P@i5(ceceZtLLNBBkt$W)A+%-ZPUDu&3St zOWx(;y5>B0?GLt$S5sKEyXSKn0*O?FEiOlPm2Qo~;#ml?eEf05-u7ezl)5EsJ@R60@d6VG!rvB4~-Mxhn*Z&^fv2OEpJJo@EoB z94zi)C0`R!kd-qA<8q0|33WP84bvUfzRG*y6=&}>WHv2-8T1dm%dZ)oX_ja61cc%^0yLo{C~C2=VZ58#Tn7VncKs_6OsyC zsBCUq=h<%{cdB_V3X$gOD1{SLO&`q`!P2b?-w*lvd|HO$Tyg+ucBB*<9C~s{l_-<$ zp1{!YkHF*A4O5T0le`;^j%K2 z=mGkl&CRS*qedqXrmid`ucb}U0w}9jh>huIK)}w=-p9$1&w?2G#aUPRxx!!1-l}q_ zRu7-Zr0{2Rtl4f@qqY2}7IhA-U1APMUwq(KL!62iAC&2295?B~@1J!==Dc-}b!*l4 zGsi7~m5qOFTI~JO>R>5Xu2qjVtpgvNSv4yD>kB)UMinc%E2nSk(8t8pavlupYrLWTzmgWe-|prQVw~THf8+15KfTX!jK9Wz`tQXl zW4Qk3#s71|9R6pKIQ*ZwF=llr?cNa33#Z!r3d1-cYLpg<_y{{I#taL=mVCX7XKj{8 zF4eJXz`U6=H?}f~5zf1Wkkd852L64+a2*GlXVa6UYM5W%obenp5Fg12Jj{-`fA2*N z!u;wi5=j(}eh2BM0rI5(y}ZK?f)}JhkK*{x1OT*NVwOoHwQ|zctIf`h=FoJ<7h-># zH>FFJvPYn-usXvltt6+lj&=W?8tJnb^OQ-yi%E*C*I~1CM?Nc13GmNg`SWpdyYS4p z(V8wR)#X%Evg|)r}=Q0D|k6`BF--FnnF4W?0XXg1?Q=~Ke^ZULcTVZPx zLu&TA?X%}7!(nl3Zhg>rrGLM%Tg{HVgYy=3L@An&(q`c_uA9%vBCnnaIMrIx=f@2& z?{&KCnMYxMf{5KVzNKdczr(+A4-*e4^Bwt^^j|ckk>rHN?jb!?N(jjC?7--G&nKbY)DREAbB&5eIh6Mm8oRnK3N{U_WSAiy;^*n!f)b>TypGJsM zpuRKtwCr=>7Kw9XIjpI-fO(-7s?t+err_YH5CQb6TA(`=awaSdVdsdIV-dlKr{3{T zLssJ8Ts3*~%IOg8s#L4?VE%worRqGi6Rl>fh5V{+KGcA8ptuw|)B_&{fO3o> zyHl!tGwU{$^;!~8my0f~Gyx_?qJAoEsV#x&Ep#q~jHtu}SD#^q4d=`wXWzF}CX>9X z<=}qq0_A}04huc3DRpOfjp5NL<{i!5v?h{`o*xw4Fsa zXJJ!IM+2!=p8FOX-8+vTuZVq%0FR7+cA3j|_KtrMV`KXj<_V|L1eaNU58v_asI9Fd z`@n^x>|GXnRS_r3^XsX*k&r<%3`8afd)GrX)fV@B%keRr4B4%mAYu(aQH>7iOtB~xH$DbL0&Z&x=R*#)S?{;Kg3mUD)xbBCpq8udXm}FSyn*plj&m2a`9N(~ioa;suWO;~RG!;{#Z4RG)ie0;VqVG_5u`7G zI$<5Rd&%K$u|B7F6wlN2N1@ATDk!Z|p>|rVDrw-Qc@=Qkwtf4e!&Wa~wB|2wd?3x! zmDD{`r(&P%oV+!M25`cy?hc;cQO&6V(})Q(AgQP28^wDi`ZtE4BChM?nn%1rzl z0+4$B{+9z+z%iqGl#_!-zOgPN+MyeT-**RxhVMNl4YazsB@LLz?`;4&A71d5gafvE zcrJ?9B;SYLBX%8Z@I0wAJ=&s0i!O-JbfdK0?R!!>!)qv8V)OVurf;W%p9lNZzY-B! z0tk`F)o+;aX!||XX-#PEMm3z0`|hozp0Cam0VQh`vBtKU2@XO$@w4cTh@4_JQksQ( zE_vpSVInzOZ7nU@vM+km9FwJbbD-=)*?YVl=D<`4x)r?iO0-?2l2wN3$ zrhsT4E>jXsQSILJ^xlD z82@A^b>pm;Xx6V6vkK9G&S+n(Zs~knPWbtQA)T>>Skz&S<;NcneBOyTT6X*cE;kz> zVja$D*0j(FpufeJmrs1u$XY%uxWH}bKb`40N{5Rovg--*G2D)eKl>f1d@vEzI`=Xi z0uF{Xbp9e6KED0heh&1m5a!^#A6;Q%RKnxKKd>HI@^u^{jK%AE)QH8{mQ6B&4nXMU zl5@V1Ymqp*G?WuhIE?n8Mbqtw+e*y65Su!?FLl+}Vv5W-d_yDA5EW$08!$h(6?5s(H{qFtq zeAjQ>;~HF`LgU80U=;zTd8RB^lby4C<6%s6+mk$ZAr^;j9s!n00@8zsD5W06+p8n>hG{Tr z^k|P6Rr8iP=~qjlV%;yDZU-Sulds4|>ev7+V;whyu-bp30&-);cB=hrUDWhazDj_- zyyezDV%h#_n2wr-qZ0znp;Fh=a3_Xk02twJ>J>GBMj1#Z6O)(ez z-c+iSBz$R-%F%-MkMoAG;!&*L%Q#ZIbizBl#vc-PLniGaAP*c+QZz~@p;Ul(Tm4_{ z?Yac>eRZ})BW1E2hxmEWP%Ht;r|%2ss1DzBCzw`|@H=VC1X|NT>D4mA(KU{_O6pwb zSrXS%aux?4R{y2b@}|+E_5q#sN(|aKKKw&#iN<3FC$xx?Rh9bn`^vrQ_bu7#dhUj1 zO_P(JlFuNG{hn_d2fN;dkE57;p$Dp3|KU8pNp~jA%X}Zh=XA~+Twp)Lo`^fZx3Yk@ zSY>|-XXbd^m|tN(ogQ3oYR@HlI9i2dd^x9HaeT??SEgk$lyP)%-jD1sS}xKMD5{Xk zN|qpIZ$t|!jpW*x^Qk?~mR%Te6{?xkoE&qSL5*KW1~gQV(hfWN!=hqpwqBAWM@!ClV&hfV0ky`+8 zd)l&blDZE)C52hv#||HW0S-q*Eslg&>WJ|wwInspjoDUO-pXw+*UuB(g?G??JGh&` zRx!XMJ^$sle5cA~KJe-4{qm=UH3!Y_^&jB8xuX1oraRTyCL?Ot3VeOtJRtW_|A5=M z29zn``dn6&u(oD-f}zRW0JOClEavc_G=Vp6Jz{6?9sRYjRS zm}Sf{eSW~3AAh^WusWHSFX?|BnbKM+?^@Q&kd|ZR#dYwBcnc6Q4hDQ!=4aPK^RN0Z zs5$u6sA8U6lJnsXlTLbg^{E(V8nQWjRd(g{0uMAG9nz$Pu~vCrb%(l*T3%Dfk`J9+ z+Ni6%!#I%d-nnD33Q0fi`Fo2EWgSAsHmyDPLA6a&TOAGGWIhWAzwtbbcpe5}>geB% z2+V2@5h1%4aEayivdB);^xI5aQ4AH74ARl{Tp^ziJb$aO^XANv_BTSST`nx-N3oHJ zGuBIRa`20NSmdyr7c>lto5SjBI2=-}7<51)QEaVdTg%4RTmEJHv4;xZe*aDmTv9z6 zPPpJ#95S0~P92|8j~U3pE*Xkzo=tYwJkE~dc5yPizA$BPN>G2(IY{r4bWP8V!{iKH z=FrW{nvVC46kNaI*v*6uC*syD4|!V7w#zo#ndLUd4!IiehB;TCAU>Zr37 zr&%D?R_G6DT7AHuZ^$eUqnCbMIp`bca& ze(!hv2WCph54P!UDO+}qi__E=3M1L|D8*F*way-gjrXfIvvcc@zpM3SJy?(OrQR{} z_AfGCh6pzrPRm0&$~l(99pElPVK9g~u|BoA4Vg3FAL)9V_#&+x7quiKpcnAY|Do_Q z%Er{?xTWAVbZ_OeW-OL}{qb-P3z_>bK_I-Di#tp%tnoW7E4HjuZW(l3=GAlt(;wc~ z1Xs*ANdKvG>&B#>r)bZWezSQkVE)z`t{qqPFWhbXgMqzG*B0|vlWu(C62T}E_%Y)v zm8%hY-d-K*U){?7gLIg`?BCIj1(|>Se^;6R-;(wJ&x`;6G*S4UyZwL1-JUZv^kB z97Fdd=VW#O&0Yv6HI(R}uRQtrGou0u&IhQ~vKJ*ydjLhOyGUNK5F=;70dk(Qaa@8@ z4iz==b&uH$Y2aLjSWw<7-aPK1F>rU*k|`@tLwFnBz`V+rD?04#+L}{eR?9d{N%>BK z^(iy6CDem(Osz9N%FIHNe$q)VG?mozJSD%F8GAPf)gD~i_ztQuB zrN5?tZmGy4HXJEW`oHCvu~{T71eZ;>Pcn&LCA^cgL%yJ3Tw$?$P<_}UklY-sXuqEk zbp}?tN%@KNVm^0$`S`R&{%_^|0m&HZFGR*~rwE@}Du6Lahr}2TSAag;x@6_kSa|FJ?ZUdguDBaHQ&cwynvXdZz7~ z1zEZ06JAccyy$hWce!=Ck-$F7Y~k0l=iS{WtGr0e-t*?&%f)U3pL(3m?dSf>n-Rai zX@2|W$%&b_cb^Q}_IQ2Om@ChmUQG5LNB05VA}asc6R#`|hSzS9+qnwHAA+(|W<=f_ zFsEG3%etR<{=^$3R|zN(ThphXcjDu^m)_dC|8RX!hbu2q4^Ce3A{ukH5ksdcWr==s5m$r+L7o7Eh7y?0md9rs)wy@@>r8=l#j zTWjpURp!r%A=}D0H}ff+gk8jr6EAAT){gjX&*<`7wmt4TFZV_Bk5Y@r&V5rRdf((+ z|80*qef;Q+>;CHxhr60rdjp?9>#SL`CfxrWBSWkExfV;Q-FmI5<U)Wk{>n#i)utJG*YFCJdau|^HGK@$upl z+C+=Vpkz2pI=f}S;eryypi&D*Es8VWqB~zwq#4?BV;1R)(3<5lS{53U(>%Neih$P3RNW70>D&ji-8LGZ*7nY zVH4BbbWVb`;7Z7r zDOWjdHsd#n5o94w8WU?J|ff0 zVaY|PODbcDvln+1X_}|CV|?_sZ4LYGns83P{yn7c47@bNv8GFrNyW6ahGYCX;4+O4 z0pD0lH*{d;sNgYQAn^mRW6V%{3b(ElNb+5}w4<7HU!l%8t8kp9s*=|5WK@`2YvgZP zOyf`_R866AHWVA>c$qq^L*E za2i78u)f7YHlPg1#zR2;9icFMw|&=h>wYl}I@-k@pZI%%lZdNcaN?NIrBj&Q9IXHwJ~$g&mf? z8H}8X0dsPh=nmfX6F7hPtl~AlU-Fkj)rnNhnmKI67+*a;5b8_Ys_dO0;s6Xtb(5YyNju8#(+D#JuPs z1-|=xJk~Bg-G>c}h10bjKt>jKKnv%Q;4{jsnfz+Npb54Kfs2B>6Iy!P&PI8b#dZ&k z+WgXmLk7Nu1fXHgM4dH4193`~6An?(Vnt%Qb>z=2UpK^CnU<-{pM;vwUnm~}_leOk z(juxgsO~?v;c(%uNl&FcV>M&~mkN^97)z9zvh6=j=%I->0)LMfnQhS&gEKLRt4tV> zH(@iq1c?>9@^u*bQiN5CzF;kR0N0-am10=m@*y>LL!6b+0$Uk%Fny=I8kpB{QGwj8 zeuaq@XMja2V`GP5_Hw_x2|w$3Qrb$!BNPpUGecSlR{vR=;j)L~U-8?DpT*T5rKe5j z>>R*k$69%MUShy@ASZ~L4{{!XFyLZkmx*xr>PN?m-=JFOn34OSR31I2LOP%wj#OuO z=hDHkcVsIA>l(7B=f*;%xYttgB4QZ5H!aF$j zd_~&cL3#fk^3jqni2>sM{jciIJ*?;Z@8dbIjoGvfnbXg=NR2{lR;xL*zJ*fyq8z6j zDopq}ZRPaihi&DXB8H)e919s*PLr^iD8jTkmokUaP`Mw^virL3-~ac0UH7hQyDn4T z!{_tvHPG(89Sv4z=s z{d%f4+@r5c-MiY?#eJ__c zH`Gl$#BG{@z6bE*qIm6?%Cp_*U!1wUxL?>oo|Fy*#LHYJM|yK2YRqXn8+y7)B1?R> zvHNM%F=}3L>8T94-|pwW8RwfF9C%OBAx2H=kgzoN&IEE;B+;)2HQ`fX<2yKYgI(E(|m65WM<9%Ib zSFV@iIv@$nem-=JyXh{y4&YH3=#&hUomPDTm0!!D)nXhaP1FJZXUEu9)% zD^wAx0JX1|N|F}iKC^7?ABwGLSU}o{Hcp>TG13 zZZ<$j&v<;=UX}~oJ5k%U?iY1rEXp4Z*S~xg5ftCBjg`hTGAd;zcd?5m5pl(rx3qhY_w))6&$8 zYO^!T!~%ucl0WU~R+0EzJRJL4OaLmYfo3_Jh~qEy2hZ=t??0_0ER?xoETm_?jjBFx zGe#z%XUw&N{WL&>dlRcKv71X>>R`27^{e&+H7<#|Y~7pS$m0;M&+f+sZRr+y?Xdsw z&mFb6ZRt1PNU|jXq9}9_1WxAG_G!-pAj%2j?rzvzOG5z`o5L_ybY}$;zR60^LywnhQATyygG!K$p>}&0(1bMVc z84siV*}ecMwHI5S5)0+KxosVoX5-Sd4YXviz9x1uiGAGC%V?u z>5(T%UdO!Wz48L*MJL>kak6gTyVn5-J}ADykUkvkI975zu|5t4!4$Bam?fN8pT`{+ zQ`=rS5ZGoSzp@3@)&xcwcUn&N3mkJ9 z*4&@adkhvSLIZ3=%x;XewJJ+}9ChrF_dVhxa1}Co?GKnlAtc%%&4kp=obr8rXC3he z$oBTm>Ct7lGmusoad7rMqbYe=}}>yuA0p5y*N+gSn~r= zK)eTj_{P4T1FDmr=0xtVoJmCmlJJs9M>RBDz#jtJu$vv{weQL{7ft2mp2CM})x008 zj&iX{gk5Q4oPBrgjBfmdoW@m?@)HXBDi2_17EZUcNKGJwtX|*25HojsrzzhY9p(6z zr+d5!x~_ccvTIvnsYsf?#QA3LOZbSX>?AtjyoecbD8->Zn4H=vxM;P5X8-Z8rV>ZD z7;Kri4l@zdgF-MHwd`5C=cqPxgrdBMj4+6YMp;mxfi~BT`0`QTuPDCo2LBb_RDR2j z3!K`}9dB|I>?jzUOI?n#$m7 z0_2YDD>z>7#dYt=p$Bu%{?2uk*FKO0kG;4~B5FZt?*F*swwJGPQ_*KDUu5~xy=bW= zSkr*$$F+j}5eVHG+?|VJDwl9b!Y$|ewJlF=6RU`_bKtp$o zb!pkl@>GoOIb6*qp6>e}$%0TGEe78EFX6^F#(UgBAhtvp+9dKdj-Yotm490Dz+X_i>KFUX--HP(CH1 z5_{cTN}6|@;#e!8&W5_dL+C@OVAAwCh9a&e>$0lJ4TT`Y0(1*eCkZiC(`pz_IP$qDIfOta?_msUTD2@9tHNM8!0B z>9pz#aN!a460+ZVTS)c8sFL62eDTeUR#}hE83#16Z2NZow)+ot>)WAaK#^bXjoGi; z{1$hhICaOgl;aP!z58)@%N)z;Lsu@nvj6hxD;qm437x-m$foJ-ZR~nEzB|Rva`~zL zhI3Wtr{v`?w2iwEyQ)0w&e>B>D%KS|sl46!?wWH?D&}U1kf}L0Ag?M)XfpvOldaLq z%gc}J?#$5mIn^$W%eLtDTp*AG10V{%6e`$LCv-sR1XT{n)-*j1-xwW`=T#O{_Na7S8^* zhfVbxhXzcVv=d=r@$hP8phfcCh=_=bxN^-|IWOzK73kGJdU@f3S68(9j}{g_|FJH} z_LU6e)nBjqTOLsJ=kWhfv046472E&%FYo>V6B)_@z^f~hoE>Vs6};me_5{_~BL4oU z6jykvNpUiM3vC8-ha~ZBYqv@1;K9j0`f|QL6)y^2#%Bj_E1fcZdK|MdfhWZJ{6lBg z_y6?Md6eGTS|e(-SQcciE#MLy78$!d?NmJknA_9N#pUqnRtxAZ1jW$E;%{^6dPr5c z`YoyBrLdejRqY4uVImA*H02{;$mkX3&W71#VNo{BXk5~zYgcuH3TDllH;;C`OGg{h zC4s#vse#G6pN5PZmjdpg#ZrJ!gM59z&&lzV{|uGq#q}>M#vuy{8s3$Lqxqr@^cq*U zRJ$Nq6dm9PIygHAu=Y@WHP2tGIlqu`xO`d;v(I;QiMP=>kc5u`dW0w>d_|>eSwK)E zVh=iycin$*EG>1HhP~;=(tOTaSX}HX(nIn%=s6+H7TrcNEmx7kty|XRP+`JgbC|Po z{8Sar%pEY`vah^9@}(b3#`px{eG}`0D1>ChlyRn-!twopAd0o5Oy3bBc9JoQz)@q+ zar^oEmmF+#^PIebW#j^Sr&V+U?V!L7o-$>R!XNds%11u%{uzj(E{2VrBhpzbAHdo& z-Z@ZPAPK5P#jy?c+1c3{&03H1?8S?qZ_^Pw@X>Q2nHLAt%nQn7&?yTV7PI>o78VLN zSpQe6`$9&G^0WYN>iUJp0(ZPqz*5z?PO<1=gJ$~%|> zz-kMmql%%BmK`FW)N~N8+Ec+4hC-jH=droh%>aXYtY0u>vvYLxyXPAoFG`XlC)IERcKYdX}b`r$4*`0`IDe8|FR9;~~Kk0ICJ zDlT?_1uX(Gz}7zNHEY*?07}<6v%=3TxH9F>tMFpu_b}R0xvl+7z`Gg~$o4}B2FUE< zymXp#zyXQ5Rsi;JaN^?QO(kV)hV(K6I8S`n^Dif0F74@Mw)8<3u40jUvmQoARcy+s zBxz`Cu(GslzEPmjO^MYNdZ~_u`|zwaI*oS;_bB8>SC&kUiZxU|n30**bN#AS$-q@9 zW0I1RwBbi35!)X-*xGLPC_fCViNQg~e_8VU#U&(cMdfdL*6k;5Ole-w$n7VCgM-KO07=P?Mbdq*qwG2;YImJn=I$E1VnuLVD*jj! zJDeSxCQtxe^?K-Gsu_IOH2xcYc;-MG8ykqbW{ENIK$Ahw!I_UXhE#x$3pesIWAx-O z$J=_*=vg77De37MbJ3}1(8%`u>iJ%t=oMUgdh$<78`bA~65e8X>;r}jNoEWwqqq2v zQo;aNIkF}Y89kZYQvE*BBz1PqjjXzL>kirRXI1+iJ+uuZ&%1L*aBg@4poTDT&W+lR z=d9xn@blYv_G~{e2vQ1WU_xROf5JL)L2X{t>vrO;Vsjvz6X>87izt1O!} z3zOA7AA1_IA2CNmLP8Mqm_x#8WP+P50@<@M5;jZ)8Lj}3MtTI3hsPrP1XFxIdDcf+ z36PG{zZsAU2dc3Fc~$44s{F;~a(PWHdiWo9IV5qv;oEWb5wTzA#D8 zg92(tBF)jD7ga>f2~-ZrP3e!vc+S3;MIz8f=;bR{4lel!`r%~HE;<1zH!&61**5#% z%a)}~>pyg80yF2)o?Z4&E*P=nMIR3jYxCEPoz4Tor}rE@WXSjfhYqy>hw@+mw=c{s zvf_XfY>?x0=|$zg-?|n2wYBr<12h%6e}VV5Fn`T|-6F12iMJ5U#}{ba&dseLg2Lw7 z!^e-ifarcr1JQ9;OK2T!vp+;MvhPxLHFo6OL_mz_VyA8suUz~$hMF*(x4GWML$8tX zBo})7w+-so=fjnjmU>-!pWY%S7@1=;Xr^29z_1F(cy^&|G!qhUl+@y?x60K(6t!nf^-r{TS#3-1x-JZ6tv;fK&fTWvM^KLAGPV zlrXv?t#Hv?6()21mNtAkTjS}|r?)+=^Z|K3eAopT97phEk9~_5E)r2|tyVOn;I&DR>$K41eE8M~DtO}j^cqCUvwM(ks* z6nZW;E-q>ATnkC zuETR0M~!DOusJ_mDz<6F2{LuykM|<+0CoN`nmWc{_wmPvo-W(;Hg z2k)p-!{<(Q*@_kKFwNYUv+Bj1;cV*ObK3PEtt%@2XdIQi<92nu@i2|a?j~-wCh4+e zl4EQnSZE4>_w(Yka|qgO;#RB}GP@yRbmyb5ie^WH9*q$8XA_W_^#zl7%D6KuZlVp0 z+q7v+l2iGdh7dsVqMtTfH@j%-l|VSv{{-o^nA`pYh+f?C_Q`O8Z+gJX7nWT%b(jtmw@b<6Tw@m?pv(FF4OA%;_6NX)m@8 ziFZKXzI|ouNA`9eeXOqP4l@gnOE*pJ&ysPWQ{kp-qX0=W)}ZaKjH=94m+V$gi2MO+OlLa#jD;}w{B-myM4wZB~EU(VoGSW9%z|dKIOrH z%fheJdRo}LeY*p}Ph5`Z@9c>_$ltYKq=}Uk26)eZ-NQQRU+Et-XUKqsqMarycp2Z=M zQ(vH!xVxb9a)Uk83~W9eY;&4)J~+wgQKfsn}?FL!;gepYWYazy2ea--8MSS4Bf<5vOE##)L_# zIGvk;cyzLbCeFE6YY`jwBH0kgnpttXv&KH$C7C&?8=IZ{ z$EOsFxy)uY3OQqXtFab^F>{bGL+ff~Jf$qBl^Q2b(D$4idmr!UEm|-Oj-my6w7+qb z^t5T(P%rrS0wBD@Razq|jhfujEh@BCNibcd3Ka~w@c`mn~n1N!XjHERBU0b^Yb AXaE2J literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_seq_flops.png b/benchmark/linear_attention/results/gdn2/gb300/gdn2_fixed_seq_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..91f30b45afd03a066c0d5b72c29da86cf4fa2f8b GIT binary patch literal 86945 zcmdqJcR1JozX$wQnPsF&gr>@t2C|Y0l^Mw>BfHF$T|^5F8AW7eQ=#l46lIU1%w$z4 zvbi6x>YQ`^&i&_oU)Mc8*LBXhPG5Z9*7iL6-K*5Cmbni4LkWC8uO@n4T^@M4lBxQU5NPh z*+pw*OB?;juu$Rkt4zCIr+?Ud{Pp$1*PPbI!vYcAs1!SLLPYVmyWXZbF}(mGpw*?2en=cm%_=Vcb} ztN-~aQsoMkQ~b}*#mheI|M^o_`H$JDEn5DcpI$~5<^S~^G19N(8UOQ-DF$!d!++_2 z{kfk@=r^%)`~pmA&Po-=KbX-PwZ(9 zDn{imrf1>h<)xg5nh%5u88Ask*yOrR2>l-J2$FOjjJ9q`r{%8R+EXLD@Vf2!IZ@|9 zEnPjm)lPd96)$cV6eCDR%xBMbbYA6`nw0cx-DUH^J2O+{=K6h{WjEFc z_uCcevaXkYnQoA;R_N+H_U+@d=u$r1^<`BCJ|;;9a{ttYh53E6v$G>3UHc1M$C>Ep z==k-r_y?O(=*`W|-S%E6%ZiR!U_)jy?Ne!MRAXH}0^UNtt*aL_tjC&RF-HQV~aXg6(4Ow6g%r!Q^S zPJY?cv}*P0)g0xq=TblCyBvIfUt&DjWp3KpeY{DB@^%?+AN4O+y>Rj3tE#G{RP9qU zl^40WX1bmeKRz=sF)7SVkJmlg@6Q#wMQWZy%m4-tW&+=EuD5r)bHihPcBk z^o!g%WMpJ4>f^4p=82nEcmC|H;&e6@<9PS}z4}wr!#k}S8IB%3D%sZ3R9=C96ZM#O zN*j8(&&{tYRJdofr-Dtvbm+rHJfDJsf|35ZjhnV?iLv?k)S@<;hKrj!J0wchOU844 z%6xXDlvOwLlzd3e__q9uzqXuhFSr&Kwich4uU^q?RtZtW<(@u!*08qAQjJ@dgPSL= zcyjW>1vj^zu2TQ2K|%iU@te0FPi5G>d$;b16J~k~b2Ae!?0e|kJv^4LSwl~SiJV?! zV{1F!>S=7eK}i3FcXTvwTwGkV;x%?V7Z?A+LXQOJjzTy86rGF>0s_mh=0tYwQo6#l zsy53~bES~JZpL@>mV%4FqP4bk9T03Xw_c!ds$M+To?#(Uw zQa;Z2xTWIVSDLUeUQ6V9I>vo^z6%k>}Ff6>Rk>nd0$!*n?d<83_ zv{r3m?tHR!bHy@dX7xlhKCN_pjw26tT?r3A;U~&}Px(`x^U|R8`%18uZ{4|bxx4H} ztYM+6tJlIJ8Lk_6wHKy-Zoa;9+aetu9j%n(f;EF%w{DG9xWaT{Zdzi!^abzwc$J|- zL9Fr^X+BaDm_9T-K2|o;ReB{UDT#_6C#zueSB>nA-RIgZ8d;f{n@$MN#oBe17{3b> z?Z&f^G?{vlcIN$kv$bv&f!wrIUADDOfV0&3Z*1d(O)DI7^{u6wo12Z>3odSukdU}h z6Lu!GEyuPYzdhT!CC+uYy|Ah|XF)|@Uq7xd1y3bBWVkguh2kv_HVwtJc3l2}nuq(Y`1trJ zeN5$&+O}<*G4^6fWuisrw5{o3u%FY#Xpg6h%M3=TC?$B;L6d_QN5way^cyR^75GW@86&H zgf_v%VpTlHptx1E#h znf#GItcXA9+>Aw;H!?Q%&d#W$wDiEt@9){o=h|}8swW*|=R3R>1gWb<9y4cNFq1_z z`*L6%R#BoHxAXRjxT;@YO;itVkrcbH?;rSM9pLZo z?=#^0oacP^hpsyR%BukZ2M!%#K+R=awd#s?%JI`Z>^`F354jz;Xc zohJ2hpR}}HPdPIs_x8@_x@7GgSO}e{>w6r(E{lncB@4F)YxF~|qj^YMO*;E#k9)jsmj$a5JEWLmX7`t0XCT5jc# zZTFN(0i$^5Qf`EYhqoFGD#x?e)zyvs`c;~C^10pDcN{3ad}viDDD^1;K_2d*p11)l zT~>-KQH_H z(;F6hrrB>zxOZ<=dB8gVT*p4+>WIA`vMlemx6L)8ywdLXTXu(uSQoS9*GSj*N`FM8oLn?$)T~(-cB&;c8Ajv6$jNp1M}jsh@Ilb5lHUpzHOu z6<4ocFA*-Dy_}fHpK4TW7(0)WZa?+Y42w7#n=$jrlUHx=ZXIu*FS)Wbe&}Xw>_#-+ zB|CQPn4FpEc^59On}1&LvX77bw-2HL`CfCgvzsrdgjE@bm3fkk2eG3>OB4|+?O*lgwc=vb8OoN z1^Rk=e6Yj$#KhRCQ|HdHqVxp|8E}@DmoqF~>Ju8e#<1YRC9|rK)1`jQdmO$n+~0jJ zJ-cwVpkAVx%VAYjMP1#sfjqK-FJA0XjgsMJWMm8zHxE?$j0?qLiN$a6>!foj-`jr0 z*LTShTH3==Yd(GcY~THw@kM^Vb)(nJZ$Z6mt1B}T!=<&gY>|@A3cZ#0Mkhy2xb`|M zd3-EN36F+6;}0J`d~uz_pF2%nZnpKBGw;GK;Wr1|+pl2@{nVEax{l5gvyhHwq3n{m zXYbx?*sNB9p?B_V$T|CY`}H-#LX<^t!&Vb-g-2`$1G$!?RcoBEb@7Tnr{b)m3cPSd z;ZbetS*37xKNjxogN;cW1G)BY2v&}7%|m^w%Xc|vc@m(csnt{I>4R;T#&(}BSu((K z0X>#R4pl<1RC1wAl`-lWu*y;@#(nCiWL3Xo=rnpp{IAiB;`z$s>snE{u6oS;zBV#q zXBi$FYZ77@YTr}NNR_;Mr~2)4zK%7=v17+Zr^l^<8>BC8Fck|Ik&tM*f27NijfY3r z>{_Bu#={Hqv$@Rgda1lHeP$l$m*zShy<1_T$Ho4coRIw)N1^xMR_h z9_h5+y2{mIcGAA9r)T4xyLYd!DR4<&P#cyCM?=|Rc66{l{zju;ONBU znxCB_H(-p1&&d&jUx|OWByX^f?u_!J19lFRl#~>W95%6KVZ-$M2fN9IWc*gTxt5Q* zdhyrS!xfC;n@x{k(d{jiK_6G=5!B17o!WbNGcT`PW%DU>^R4H9d`cR33nOnLX#Fan zg*DV`EJ=w<)AR3Le|-%|Jn<}n;zi?7G&YV=3dbWGcz4S2*ViMbt*v)ZSJCUO1!Jd$ zPMey>*~?sXb5jWy-J%+HYwLK!b5Y~h43TD}JO5lPPGGwFr9+1f$t}#!JyPTE{PM*Y zpr)j{`l$7ZjT<*EU_F%vgXJi^jy|moHxyF)3f>G}O$D_1~n| zH#1yZu`oX&yJX3dQ+?HuYei1Uh2-o#|4p|s^~4I47tW#7w6rId4O=2)JUE{`c``ZC z;Z>KU`S{y6y^t&Z{wAo{l=q2eXOhfRE=-Q_;TbTZt<6sM1X127J1=W$SkHA7T`4Tw zr}zA9fS6gOX&(0#g-f{JZj}BD*eXnGgl;MYa6UT!y_KU`<>=A!dpmU4*x4^%y~+^w z;DL%`k)E!uMcOa>PYMh`@{3zCj4A*<)zkI&jt#XOt1<8QnHcUc0rH@{0d_a+-OJJ5 z-u@xadBRRVPgWY_p646 z6?}YrYWyonvBssQIX>({VecMjcnl180E=ey$7iXCy^eIpqGV{PJyKG!fOni6;$xrJ zXy2q~;gXY+qXwJPZU`CVSp?8hixw{?Tm1Fw*Dot8nIx>6N&9oErjPhM`OFx&i1NO) znEp&YGZnVSegnbBuPZ8g&?@&@?I^@k>US<0a2RS9oSmEKpuExl9^oSOss{4%*(h`> z|1C%EzpSXBKbd3mDtMdbQ|@E(Ko+;feiAg$S5T>>q-254o$%dkTwm+3ii!_f%8yUE zi#tmlhkpEEQBhGDkan=Qe>*<0m~YA1j-mkaKCog5Z7FmeuZ@;p{HE6-;3qnqZno7b z>Qrl%WoA~^Yd~)M(e48U?o;eJIXM>X1ykk{$J|bxigud#9V9!?psA^uJ)V$|5Vp^4 z*U?BxTIvPZ8fZXMrRi4dJC)PpLyxq!p9Zc7u)Eu)j%vW#-&jE7ex>VE4JsuC7bwH4o(Dg zi6{t+x*wr$jM%WKo7O)v4~I_diI00&ti_o{mP z894S=Yqpsjou3)jt0H8(_r_(eUH+3PW#)jodb&Yhhd%2&4Rbd^Q5Pz*GEpCxc6C=u_fK$?jW|=QQOaTu?K0X z(;#4^(gX9Pin=5{dh`hE;k;kl9{0&ibk2z8++z=R*=`4kOiQzYSBuhSYJm!AgS)FIX>1N~#Y$kuIt;0fMT@d(eE)W|2aODzFMfPB1&yM)uLosYBOF4( zRqQ0o#(Qr~jbC3~zW2V7v@CXq%Jn%QGQFy}1$V&T0M)j4m67iPI3?I$`sdd>e|%;h z)Yu-c5-Rq2zJhMqT3+fZ;EGvpx{QOAro#EaBYZOYE!ZKc*$;DbbMba^s+Z{~Fo?SS zwn^)=Ye+b9rMS3wY@)*D!;X8(zXb98{{7n+UmY&9oAx`3MZL_IoheER3YYnf ziVAMMHLVwYU=+^&bVkxs=&6H8-9e=m|AH4Xo;D~vi@Z@pn@((*lgYp<7|vP6bftNC z^|byotkxa2;#;;T4cjKai`c8#H}h;Te+gwj+`cbuoM)fwrh6ZO+|!$n)wH)8Qfv@; zYD(IPO4TcLhA3Szi|j`_;XXbzAs@->LGr0r^5dIqAox5 zLAy^3MIHsYFYB+1<@5C}EG*o))fP~Zg+%zJ0sLu9G6Y z4Q(#g^z9vf{TCwOM|ShG7Yi+Gkiz+HN2?%7sNB6u_3+6&ha{YuBkBi_E{D z*V%gy<#p$|b}sZwdJ6m^2DL^+MC2tFh222?X40XF0x&<>!5_`r((kf|eT#1_hPUCY>ntm=jnib`6{<+Cn&G!fA z##@Z`Z@JrrqV}+ems&6kb0i<^)S6RUXBUY z>4~D)y<7Xb(HQFR5DPqJR-^m$w`T7kO_I>e+DH4>ikTh?>}bmZs#)tg)_*BDcqNc` zG}th|X2LRKV`IC?k<-9G(SR`{a#z zvr%;BKG$!#X}~T-$Qd7Md1BY4u+ySePVnR_V1H4sd3Q<<-Bj%T;9c!=?FBSc@OEuw z^YAeaY1At}JVSHKbs*l<$bd&e?Ut<-U%{epK?>Ss+ZydOP*21rP*?z5AAeRJV{D^C zKu6GbByyyzn*{{)vKT0}C&mY|&ULH@wHf*SyBx%qs4tPC!Y6Z7D;ITs`?d^F9(T(N zDg;f$Cn$)ONMlgc`ntvM%}DRtH#$G(M)CZ})Y*Oy6hHXK*m(yB1#|`R_p^&A+1ZiB zsN|PX*3G9byuXHqx1h~o!1J?by1%!VmV(N6sah;~*dHq8Bt(PpvnPc@hu*n|-UfHt zX3|U+rMPvJ5G#+&YwQCWIqMv{rasz}Cr@gn>1kW919Qbb+P7ijMp7BGhmY?~i!$_~ zVDkCWGW;iRR0rKnq^V9XnuZP z_o?YMrHU31XHn1D3)|)fp_4Xhy!7+;k3Dc>HG#KVjy=4IBBDG5vP!NIkKWM4%D1sm zYj*Zm4Qaogv!e&yCqBFAd3hg)jB(?J&*R5iQB6X=&w+vLMNOygENx`oPGnfrC;OjY z4xngJ`Ct zb!47i?8nLz2qgaqTCkp!8zir~6dhrk&w1M4$M<;7z8Lc%>PP&xWq_$3H`j_Tf{etk zmB@B9N+xY}$GPG;KfvxTw70b~?t4HzoV<9vDwpKdv?j}0%t7rWT-!=sOUSkQSp0gwy0(Orc~W0in&gGmM9J1Pc?3egczJ2@Z782mZcc*{Sq(d@ zgb2t(r6K@)8Mow0Ag{XCqmX1l>eGMA+{4d8)*~t$=WF* zXFu&BFc(YrAe5~5@#p6{%2DWjRvqxW9n0K z)I%$^%T{&_WLsb2?^I>R1{mX6$;~YoA07drrP_gL$B;alFW7u~p`NCHK0$Qy`}2-C zNqj?NYcuqyD{MR(1#KTbFi_o(qdYaNx8^tOMg=&IZ;_|iP{1pXPY)GD6w!F=7JC*p zPvb4Fp?smiqVZS_X(gQoz!hL$u_C)!)@#0i;$60EnWB?ZR=fRK zMU-f9fyc^`QFWz|cE7b|tMsj8oX(=zr@3=T%A*vrMEv=orj)JJ)jOM2u0x-@h6jEf z0E7X$HpKu51Kl~|v>V9TB5y426A7Z8*P(=M=5FAT6%Z6W0LA<@R%>>V-`>4z8lvdk zCVn*O4S#uiS0nrrt`Hm}*su*+V44|}g1BG89?7#~4ZAHK1S(!-ZBS1Hjoasn`aCAN zHEUmGsBl#8_axAps|3T^x}el@4qJ1)Rd#i5+9e2`0|lX33>t7`qal#`c;Lp(n-A=? ztXHJiSXfv#Kz$$1JMl>Q%ENp9IRon)9Bc;nJKK27?mYvX3&iN)P%pVTluaSaw(WSI zecF9PWgXXZ<5hn#(sQW7sMm$cjXE#>h-CD>BGG&*>7Jyhlo^6ygN-cP_i5>-zQUDMI2SPb|ZV2||_b zg5|l#YhF45_Ko_MjP9Uo*J!9h@f84=r>DmTtg5gx?I(U1MC@~04VA16L@^UjEwZ9G zXIFhyngAmeH?WE71b~9ER{<>3WDJ_bg?8#6Dr{t28N7|CRL(g%z>x)>bDU5ya#x#p zKF!W9gF<9C^V=4( z=4{Z*ux-oXKk?Mm&!x`13G|S156j!HBe?*i8S6Se_`-Pzg3}9c$m15lb>oJ&Wa2&1 za*|;t#ZLlAkVb2k6sUZ@<}O%FEYK&k=>4}h9O(SxHU(llVx2D z?Wa$l!~`6nf533GGzdLe@fFwx+AK4~e?AcJ>C{`dZc%dLZN7v0-&!)FS{0hpbS ze%FS)_9@qq9ul4+L~`ex+uBneLW%9`w8E+1-`o80N|cB2jn`33Ot81HH}=u7a3$XU zXak*<7%8yw6Mp4|=r~ME4#x7PAG>&24ezveNe5OXlZi z&IRB8z}`2&ZTB-=dh_NGWa4fJ z(Q9QrrE|`81OhT8o2f09F*T)b4P-0aTcw~a^L+3zSp?KVRPT^u*jIlDs72C$0}NBu zFuJC_rg`4WG-nveW*-%9n*E*Ig$Jb9Q)kxw`kL+&V_Jy71jeW8=Zc+dJEEepzu0q* zcs+0oyP?TQ_Av+^g4QM}#oX{%_xSt6%iP26cYz5|#C+ZskTiJk^4LQ1*QB{U22;Q= zqHy%2-F^#{PJ+&43x$F+^IrRg-od4ypm6HU89&dtY0Fd|kyBK{v8X_($gkSkq#Ah6 zICTH`oKMNjh9xPlO8Y*wjD>}c^7ZpmV4w+CO=zyJh>ROZGOQWPFgts3c2X3GQD}fu zi*oPp-v?bsYu@DIs^Y;t^%L~cj*gCzn!>~}hJ3Tfsh>?LkSpnTPfU^Rmz=Srpn!mY zCucuP2E1YisxE(Xd*c(UrtP40{G|gnS}m1;Bxkk-M#R0UK9r;pFJki{Q=lb9J7qOM z9BpUm)Gs(35mGKJ=&i4z26cY_9)NOkNb?1?S87d^^usi~=*!^1(S zU(ttdZx9g|-)~`&Xc%>bI)#Gqq2QwU7l*xIUcTTdeR}Wm;Qg}8&QF^7nY9;rq&TW6 z0JcFJG#Tr!V_;x-BjPs}waS;R-h64(RK^jdp#;_N%ddRsO|U}uU`e+=KNkdzHX55P z5$0*RTSn2(8T~^G?X|F6vS}D!`qy};98W8Oyo~UIKk#58v$EQes8hDK0jqW#|8nw^ z+>e65={LhIn=<;{Jl+fqnM^yXx0nkpUc4Bp_(s6^(qVsYNhh^+JHzTI{#6Do0(`z> z+!>IGUpVw$uO8S;#As~7WWjugUM3VMT3K0Hk~J~0%N!KfQC3#=8%;`g=J8UhvYZ>M zFMbUC6!IrzlqmN%Skx;8p0;+~I&^JX@OsHAk6E8gE6=qX<#Re6D zI{~Mg1yW(PlV?p(e0=;PFt-oIUb3pHs&@Ugd;xpk%C7ntt|B$ZdJomFCFSV!D?VV6 zk`^N`=6DT8n3p@Po0nk?WIla*$Ma2FmgNfQE)a{ebIfgQBEIv`YC-t`l+>>RH*L@> zZ*O0ZYl0)H`H)3fZQtVzqvG;ovU9}GEawiRY*(_x`hciRhX2E(D`p4E`i`By*D#tt7jF$3zMu25lqR}M?)YxfP^fH zJk|mJHRv53k5!oF**^9Ji%iu^N?>*V$S>Z>6p_lxu2 z?~nB;g2bdgq|e1Q(~wvNubKB7RnI7|DO>#2tJkhwdh$eceyWcrpyvamVj}CK%#FgM zKSx+qaifo#0O|9Qy)W&555mFnqH*`(`tw zat27v{;ps-D7OvfF${%8hW|1<+)lYhwMD<~}N`u^R3S5&{?!g5LtHEfxbUHv*SQwE}f5p4$> zek1yH{KRk2wc6+BjIdSF_TL}vk4ha&Iu3`G(ami}btta**N?s^FF%bWMeJfyf6`nu zxPgP9b?2_gTuoLZpaw+&(3hf*iVGB<>yROI?PBv(n(I++Zf;SFTHeAzcV$_>M8nc| zo?|*`HABHYJetghDpQ~77$v_GLc##zo$!b}Blbmo?oIGgd&%r9K5 zH|N!H-Fw(4^7hT&+oW#Zyg7ongG$mX*zj@ki~um&71`-LbJ`9jb~mr*=I#Z}iB{tu zdB3}(Un0At!o~Y*c7$ndCUB82jThcVWaaWesSi4(3$h=S=Q}ZhG4@hSRO>9gp))2@Zm!SX67>J zBKEzNs|Fl>|JKonpI^fT#!{*%}l zKq=`#=%mg0=?aftV#!k!wnt$z&tq&bh+eV!H*_9Ak?4UdlllC41sb-v0Ggv7I;t@W z!&*^e-z@!({OjY}9#3}>)J`hNgj9%Q-ahSJx9M5eA9BgIiXB{*q4r22Feg+ydD6i; z`o{L}dv1*u6it=Ad`Tf~b15U^oS$rZT!)u|OAUmxLmwACotE@CYMEITylvy0(YOic zt!Iwv&Jmvr-PS;z!)DW~G7GsbvqOu67c5fk`B?+RDE76qRtxTZ8?9dCJ|(cw|M-52 zbVjB50rYVRjjxMeqIc4`znN}8hB0z{t7@1xy5J$%JuDnzn`(^T+wR@BOS|G7K(F)s z1y^X%0cOx-Bo>|gD6DAzodB5ml8)2($h`I!=)V^vMN6A zqzTJ*9*COo8yjAlJdqn0x9Q!3-NL2c6Q5T9G1DA+>A!w*=N7-yk#YEJW%W2;Mn(|G zYJKA}4v(*Bsq2utAk>~zL;TsA`twhB%-Kh3GhI=bUZ@EFq~)V+RO}f5!NEH6=`aw< z)tfh$P>m4fbcP;3mTZZPh({CC z+Kw@cczzg1ikT^g3{LddEk;rHhS^!PFh8R@tbFY7VR<4=K@R+fM~%LCFQG(s?o@>M z^#lbc^~AF+Dn9HQnws%}(gvF15EOPaNCh}rfzMIiP@Am^XM_t81Q9SQ;)2<9T{o{3rxFf$&DryZnBRxeTun={c2bKdq#?v??|D4(^BeTAEetHKL zP4rUnI}Cgxh&rSi7II=aAq%#UO1CNT%a;#Y%&Y6iPjsQ(>d|kec&f{=rB4A3Ob$Gb z$~vnvzL$3gu22LjJXQ}Y`Xtuj;SjpbUnX*iY)isM=mdLQ$M^{;QvXWavT>0)UO&V} z&F<%TAq&Z-Kku45F}m|~`Plb19qU{e1~G(W z!cIR^9dQjWpm!>J68m1`%)M{!=cN!^9P>gJIolcaAXMzQZF>O+($5c+JesoXVGh8i zF@cMA%vlC0PW@tT_~IKXmm*_vHPIV1`#55_aCi}q=|pdjhB{e*u|xl&M1i(-Qk(8NChYk~ME zG;$dBa%uL#Q6!<^MPtB7qWBuy*?_pv6sXWV<^EDNFEflrpK?`O?l7&VN zlC=WF7YkdWVHl&ET2*mzlZ@#TlY_VK-%rk|uBbQ&f8$VFq^y@`NKf@Uu&?;v!bU|A zmKS+=c#>v8=;C`x#F0`&We`ivIG4M8qY`cw3097W8zF^|=$};;#P7}2RSxkp{i(Z= zkfD(S)7aJaKiShJ-M$|`geV4h_4ea~+S$(7dhFPcY?R_7g)6V&DLP2Zzat`K9^KUR z^q|0Qq|p)IMEcV_oZJt%7mXa!i56U?(^-#%n{vf~J5{TSF8)&b@bM$!QlCE?AUvI% zCX4Mf#)TA*nz_`C)q?Dl?~NNOfjz zaFvC)*YOzbj*{`%UYJ0=Mq7|*eOEoY0?!v=p7@Expc>(OsA0r9h5R8bn2$EM@)*>* zYFoSs(@D>%ZKJ<`hx|^0`oftpjK4@)Tbvm5iK?vk;8e=J@MeLvg@uI!zB7$>WovM6 zB=$Xtk@ClCZI3rm6AHb7S7x497g)HSG1X3|v%q*_uOV7k6FE1~?&?;I=)P2$}U}%iTYnHERZc_Xx zv=5^6NaQ5lnb)11Ec~IF<*X1l|S49}u{E*FACE0=pu?Jj!P(#gbo-F zH+Y8p+}hn|BeIN&XMf&kbu}_FTE22+ocM8o(*g10&?Xi2^zJKhXwP8hl{7SH#ug2F zI3cv6_xyIl^mf?ji@ZM+dARxY68P|w{a>QX@^hZV?;oFUKs@h)#*K(CGc>*07xpGt zHAu@Wfrt1CFy0P8h=Gsy?pv_}l?h42+Cu*1|wSl>3XjrV!vscrvFO zq07BQ9E0Y>2dmvYSJb5ZkPGi_MEjwam7_Rihh#r{R*E$J(^ag|MICv|NZTE5PY1b2 zZqx@G<`iDchkO@dP$nM`YChQIfY{_4(U3^t-gO1djxZ|NPJ4(Wj3j1BeLXws&1<}Q zWI`^39Pvp?N**)T`E030W(#aSh&22Ju2BNyfvd8VRB|Yvv?Eyp&c%3R)bLh@0UUIv_qV8+p6b6*REaTUEAbtuI(tBX(DP8uNV2E@W*15*<$bPbYD>%S9mOQ?iftZ-;fbQ@!5zd6 zgS$x(0_?}Q+&D8a4x}0{A;zE7RP#3Xh3mK#oM#HJgd_-x=BKR@7UhGa2#`nOhEDt5 zUd=HXG1;sf{1WLmgkpoRzN4WbBa|be94f@@;^IQ0x9Ef9y)rXj!AiniBQ8X1thV=MeYKgd9cjv|Gd2 zbY8y{d4owjEr}4fix*GB52qZ2BR$WfUqeurhWJXMP!MK|XXFPH58%@d2~p6{!x;k= z2*3Iu1=$7uTnfX}zN>`xf1Go_sHmvwJS2UAs_7wG)MlREzeiHC1fwa4a)2GPV7!H* zkoi=r5*SGW>scANejn*+v7itZ*tv-OKuq<9f7b(1=T9cmER0lf$Q;NCCO zcN&KFpVj9PZvYyEt#XX`xriTRo$;aZ#$W8j3g~Z2S%dvcC>j}yf%k9#j~9uKK%lfw z`7W%6hKBWHjAzfDm3;mk8Mdsfodf6imXHtwi6^*Es3f27A|fNzv#qpA z1p%WYyc82B@$Z(Sk9^3neb77*j!@yHwHWlRtk9ozWS(kQlclLyjuqe6UxP*l3QC{;WFa&4iA%vvyYgXA=UV|Fh9P1XL zLL++H;UGwuYv2y*RXw+;QxF`gu7JgpJdxvExTgBPgbKw*)NFAMvrW&%HYb|)eORe* z6`2y16XEr;*H;T}N3JKb%k7Rv(n=^Lv0xgFi5=)q#8ODhIE^S+e|uqmwWAT5|8UXF z(br{VW?dYDy4Y97NbhSjUm9Ztej0^zawf&VLu~VxG2M4Rh9Om5# zg?Cxxelfu;HYVmZN?&~H_SW^-`!VPZ-5}O_ZED;%IQGFmEyHF`8UOVj3@TpLqjf#F z=bskW^3%ZZU;^PE5LEFTvKjhK6n+ zWuxqJtgW>*2FoGwz_*?yh>bW z^N;9bWcq?q`!~~PJBK-tUtT4}1|(9XOjz;?nQSW-v_n?NhF1pFzk= z|BD%+0w)&9%|u2I(9+SBKsJCJfoX?JNXI~nyn(%BUHE{3B4HfNPjsPYlNbllQ%DpT zYyk+R1d?MN8fLas%-A)ljYNBrU6@@=5rF|dRE$m`*b>Q$Q6f(LY9J|kwjII-dA^X< zv2qsGJ~WW)aCqr2B>@S?Nm`}N*kR*MPEJOn z1tB!<-7FSfcA@hn4ard|~;BkIC?5tv>AoOS?kLp)-?ev=<$E0`e<mX#OnMo9HzgVABS*>*KqIar1D_hG4g3T|XgL?tEqe*;NyOpHA&_aMG()9|&I+w? zcA4L}aNIsgfS_dp((xPQ?L_F10ZwQ3)Bly5Zp=#q#N-te6l|ENI(7tCErchG*3641 zj67+g?voAh>Q0%N#XoDWa1k?CY?0u*^?|Z;zsDPNZBi4 z(iW~$LW|p-WKUDmO^7;=H0c<9A35;_3#gWi|`BA0sZ zTvV5*XEI4{LG>kxMpTYO_+(_3=2t<@&A~kc&*~c(WQV*Rc#?SMbIYh^@ZBL5G(fGV zZXS&l&_`=gys*Cy4qt$vWrqY9vGmZNP*zD?5=x|RM$z>4(jF~m0Z&z-(9)8UMF{Wu zq@<)&r!tcuIUYj=@c9UUABtYd+qaC&>vk!^pdV{xrCzuUuO|;09r_V0A8_p}&z?Cx zY(9462bb|?u}wfD4uJCFp(R~r>AU+A0|3E0bXbrf zfwXWLlP>c9V`Y^VYKzp57J4nL2S8&}=!DJ3!Sax=Qbl_8LL0jJ60m~^>5J@$MeV?> z@dXk-MMk;y@i89AF)4p$jbj#G+&s5x6N=G>4bvs#0}b>jCgoVN$o^^8cDZYW2pa|d zv$nZz$&GR3I}NbIy(eQK^9-uA*DJcc+R#uCogZm9CcDas->Bfr7t`6FRl@MON=aU{ zwu`jsSDDd^k($M4y9G(I!o3{@X1&bPd)FAw6WKbrSz2zU<-Oj}RL|5r3_#b$-y7qp zFc~dQ12xfoewG`K02#|en_lB|$OtAf8+!jTytsp)Y)q0|zU7~f4k z9!X)_{tccddOHudbCUod*sZ+30)Z3w1gHG>PY`nS=kwFI;J+@L{N+EN7~vKFkIbw8 zzj~Pd|F0J(BG6Bfl`ua{%p0%TmL=QKb=LwAzpLip@wF_r2(+9d+imCa?qx_pc*S;s5%oi z=4BQV8^EW?4qgvY6w4jp!lK;7p!Zp{W^F^={3*MK82Q`03BVzi35l%>0(7v*@& zoG0U0fq{V_q<=40>~AJ0j^qSjucaHBp(IrQ_ZbaQw;>Diq0*L$K|LTow&%?E-LPT% zG)=KT#y0;8!m2@k6611uNkalgEi1 zN{7i$O8q${B4*h*Vnv{V0{tR?@9-*rXcNZ19KXJ+(|riZj-(_fM|(azGiUu;6BPu5 z^Q)IP{=nFQzXwB9I-?fic3*?_kOU@atC*ETE6i&CiRz5Wh{eaSSg zgSt_Q9z&)!0|Ja6 zv4nhr1px%9<(RDBWfP`lh?IF-+dAkCP+vF!nwGk_OdPZFJo>LT{hi(yNG1^a zOi^fW9gVg zwOGpl+`0w^t|8P?fJTBB_DnAlJ~<&meYuD*rK8?nO5l{Rnt13VbVJL+X-L$+24VEFz=&Ovh9Oiv7F9EZ24Vg~t75TUHNIx>SF zdoXAiOqw`BySYdrYrH;;RJvaV&5G5R}jgC`2Nz zV-_T(02oZwf{ByF7BYX(5_D1?5AL93#x}dF>KG{;{cT`I!ZXaw2jFZFVGp8!+?KmE zlOi5Ua4qe?c_}#*PIWjaWEv7u3|xQbkcT`VX_w3YRLI~**tjGOrPm^H_S0I(Pc#(5 zZKuJxDQ~=q)A)TWOJ-P5fgqqPC?llA?!3@tf^A4_EOiAJE+jP%o>}<|f(;5RBl-(G zl+~Cfu(HSQP_~t${R?;0JsBAoUgCko6>Dl};LTlwV6;(Mn(M&zl_b%&GhrX)4bT;h zD~Ai;r0as(glRZ#kS{-E;Q_ScoKb8Jg32vlx9$dFf}MEqiP|Y4dUHVLNLwAv@E}zd zLnyQutHnV9R;pgLF$b1fswuc)0V-fViHNbFcqI6V*=V18_wH#awM6@V`{CvalL}|D z&|n#FW>!)=QRKcC6S7RO^&x^j!p5o3Ln<#CGeCHBSJ&6DEMH!?H6~h90$8%{@z%L9 zW{!Obac*xKKzNYig^ZXczOM(GC$w^$RB{{BdV+wXC3sHG<=K$tY2>t$G!9tjBjBHP z0X_qwzL4*+zWjH_Usi@hnN>LEv240Fdw5{t5i~-{q0nubN;q=FZ&_X5JZuaMkNP$2 zW?Noo?h{VVVdCEpXF&n)IxF4!(~Bo#UQOKepo+lE;mtoW#z`)O4KEDw$;2id^x>dt zT^u{Y_{WpPmgVB&qGlH&mm>G|8Y?RE;;%CpZiptucSyGA8DAkO{-ARZd`W1>mkskS zs!P1V82eLGc6`N&GjfmA2ZJG)FHbkj}bi5sv4%#KJM-&SqBTe*_$225Fio&T0**2F?AcQ4Kw)lx> zbeOFIir7u8;Ox{JHcy{>{xbZ<>9* ztmjRPV=boWSl6d!-$dId2W&vz9I%NZHMr#8?nXpD7Fsz+$YG4PgOXCwKzx!_P089z zQD4wDk%DfP z#yF8N&OZu2`Uo#3#W6BHj6@PCa*_jj9L_j_*YRJBLcNHLV787i50GIsGED@QdivbC zj9phK3L-08tWv-o&w`fTp5cb%fmQ2}8~^?S6_yOm;UuB85Kb=DO)nl}{QY$hzp>TjUL0WoEl`h~Q#H0Ka; zF|o^FWSB#19E>wJHonu9qnBlI2|nv#A9k(CCCLIz$zIx3`xO*MzI_x=I_ru_51f<0 zt)ZcT!=z}SkOT`Gt%vFh6tsx?H#F$2k<`d>H$6;yCalgerpEOjwVEYLy-i0EH@e>> zGD}T#o5>ve`}fS~WJ%@;Z4w9}PYDa(56Fg$pM#;lYHnT)ahuvX!|w2Oc)+YV6b{!P zMMP!>Tdv>n-x4tyDBhGJxh4x!?fY>Sgi>VcAz6MJ@A7D07%?P&hdJKmh||W612W*) z87*sTEdN7?55K}N+7%eVF^~c{+YpLW_8V9&9@BczV6ljnm}~XleRGxRIu>S+kwA%P)20mCx8REY- z{JrH3!hd*A|1WdALY&z`y+kc?4$1eJ$#DvmLw@gRwxG7Zyq%p00jqcgi2h|ut*JLx zQFC5oq4K=HuMg*9NjfJt(=bwEk;+4?tmovSe3>w(>ZDUD{mGcb%$+$>VD z1afjTA^Vc>1VwxwB%Dh=e&m2kc(*c~-y=Lg$)gO`p(Kaa}cXFQ|>+-{Bg5*Vg95EZy7QjqL>u1sbHXxMe0Wi ziRvT=cL2UMHXl?{BCg!~y8nKaB_L{l(9}!BYtwR$j(fP`JAof{POgD$1N4+)cKlEV zxGg^pyYOY}$2kMdQuBZ)W7DULamNVsaB=Rm@CF|tlXjObUDB&co~9)^StJr;fmW)yVoMqf!Hn?%p`??Oj2~hDX6Z?LF>E*<9h=ipZv5`A@=wy ztP7xLT8dbffGCX?=ChNdWL5&lZ;-g&Bg|Lh>>_y_P0=_-e0wOA8`_=-oy=L*-ri2y zDoS`QM$3tmPoN=&wUH81PuALIIkM#cgx;>O0S=d8bUIFe4jqRaMAbNm2Fpa|Bgm8o zAQo8FBg{|35`TqvOHK{PHJ5mUNcR1_2P3L4jWm1_a!3Xb5m7 z6ArWuas~%|*zx#jQHwye=|MZyRWd9v*??z zB$FVPoCofaa5g|88A^o#GXgc{&=3wE9L1vFhyz|^=f>_f=NRbgL*u@IFvCk6e+bl2 zhMYnC_?9hOh#}EFa~g^!F=*lIjf{@UgUJxHurXO%WqZ%JZ#VJGH*Vj~Bv(-G!u|>S9)|G? zKk(UZybUsdw33My!wiez<)b%u1GHcdy#4t<%-p2>CX&?p_Z1)&4Tb$vZxIUwQ@;zr$_GO;4?oJp znd9gI;-RAmla~go1QeE-T5#hx;P?|b)y=J2L3_uNZZq;J%G3X=^w@=eKSt{>%G3Xo zp8xCL6pgn0fAU;Z64O9V3w0zRdOw&c`P`BAn&(1738+_~Uu}NH2ibo&8x<4rn^i=! zWGnCA83S5wy1#Yo_I*qcqE@VQTD!pUjpwiaA9GCo=nQ?>wyqV?d5brQh-7*b>PtS; zUeIr7xl6p5X6XNdgMQ86hIyJl=WYD`k-z`@|I*2_MAGHA_GM#Jz-1{$Nn=5Vs~)dC zOFOJ{K*~|jx7XNvJv_U@XGAke!k)cH(U?$0xF*Rfs5-mYcsE}q!@rlpw4Q#mJKdkN70ID~h^#Q};f$&AL8n3Y_X80saGMB> zWcl*|H(k4NgPg}_k34R6FG*xUXB$B=EiElw3~;J*60u89QuuL|^-n=QO`tfCu&W3% zXVL*Dn`G4rF4!Zv#DDDhfwA~g3#5lHSAW%GH+?LEZ4vMnMTF=Ragyeco_KxvJ1zb^ z(tgb+nDoaHn#@?SL_&eQa2;XD(}*+>!<0O2WUZ2$kSQsJmw$uIx}tBu>+&lumWAL# z12F=|Y!7Vzi>sfpDcpbhs-$E?P0s$Y&Bp(A6~+m^$MmO#mBF@X*D}`iZJpb@(brpb z%Z``-brtZ@7E^z^HXRQdz*1s6LFh$MYZ{!L>Wez0tb92?UkXm(X2A?FWGni1|NV}P%l{mrys!=Pm5Zr&QL;S1S@l0LmU<|B1vz(4@R+J9QPycF z)aW{k8fbQSSN<3U!213r=JanD*06xKnw5IfEC!qC?CsUAeboOR2L}UozIyeFczJ)` z)g>5vD{zBXpc`pr7;?kdx3jY&G@0N9a+nAv9U~qM~V@ z2hzo9LBQT$s#!FOK058~;Dpd-JfG*Ej5YnPvVVLxv11DrGDrGOJalM#?NhMW)Q8%$seiJxYz4z}q-uHO_d5-sa{&_m~V5qge>-)X$YdFvII`8G5 z@ATMZI-`-|IsHJD=GvN7@*i*b^2IAL0Fb-O^4G~HU2lL`0&jXwG*ejAG6B}5tLRtcS)B8zN}xMC!{0v&v~M#Q%9B(o}$b8Ej&g zrgvDOS-`=a0-4EEJf|fVbIKbf>>JADp+}7rL})rxahkk4&# zkLmw9(2KG~StF^1&<>L@R*+7OAV1O7bu~+KA|uy2p1oK6Y{EyR8MmA~xedXxb$SOJ zjgV;`%J}M+P3^UsnSslnxU+?sVtRtu)bN-e?DNAOCz z-hoJkQqWq+C>6|Xu05jQAtA%q003Nlo_$$q(FuTIbxce*92?~uMS~37SV;k<=*ou| zbP){_?QpB~X0Wl+1cQkjBwmQ)5xR7_dB6jzsru#Q+&aK20SV3OHtF2Yvy{d?EPt?I zaTrToCwhw$j@ywS6ZiUMKR=I%o5~{=r7|%B;3sI(Sn_WZvw6zyv$Ho5t0$3#v1F{O zc=z~pUFwKy+`O=-KPfL*4W(ma{oPyCcpv!e<&2ljjj`Mgl#~6Bi{LBiKK#&5aClW-tM2#gm?@AjZ2HeO6Ma{R(&+mZK zbY2)siu&-WtAIbAqaC**+Yv>@MW2Mw?rM|SL}d1DTdICkI9G8gZd2Rp(OIwd0r(D} zc5x+)a8-UElD?zuR4B-!S1Dagvz`**(Y=as77Jm`+O?Aa2Nb@9p#BjK)>Fl{0J>Ll zoV4$t%%ZsE!H5L|wV3mY51Ld_Xa{ATb;`3Qos!f>eA)@##XGVl{>2878uRVr)1CIA z&1*^?cnqR9cXxJh*p8b23iUE7!MEwd zc`{WLaPbB(e@gM{W#O(Rqvyy?{d` z)g>P85Zqy}#A;G;jSppVYA%ybxl1TVc8~w@P48Q$puU;{NS5g4d!J!4nX=gAR|Xk& zG^<3k(h!6mo&{hU13K=kIbp)HCw0H{H4G_;3NmMsU!FEfMnmrjU5T0C&!Y5CbFn-) z@(-TJ2J)HB0(2?jsDVmy`4-p7<-QB4EpMOjxpe7LzxnST*|ITmpT4GrsG<-vyC`E6 zCFc~a#A|Ald1OHWt#%_J{8`0~QQuA`D(XySfh!e%5Z7jS=2P#$q?yeNjeQ5=m1>cF z*2^;FE`|*7b~G`Yrvk#I>+8bhfne}wEUWl0pKLx(u<%JR0ncm1j`c4?#~+h*vLz!F zI70&zbo}F^B&dYZX76<;9;#fDvzdAG0o8Lqy}r|o(;z~Ki(+B*QS{ttnKF`cdUoAv z)v7gV+Elg3RwKnodtJM@xOgoRQ=zX(C`Q*$f%glu%FS8J#L6BM_ux~ku$DyYbpsBH zf8-=Lx9U7!CgA|pFeIGBlLwjJUpg8jii0VRokD>qPA>|Jp*9s1;ynSnB!w+#iAU@M zW=e1*GP3zGM4s#RhkRy}Xq`cpmRB)mE;-MLZEE=rYdx&Nd@+`H^PCO9hYiLdmz&;g z#^pUcz7@F1$~hP6f>f{-H10W{GQ-8P<4)JqP!+JE%*3PNL^*ErlPL(sCX$}D_*Qq` z!i7%xx)MDJNitQ=Fhp~-9`rPJQC@U>wVHHRusl%$`j=E&`t?KYe(^r7I(DoBfV!3@ z8Ugs8utzdX1Boo!1@Vo`i0I9#Vcr}Y>OgTIu*3I_lW795Owq;drHLP?b zpm5;HY17N)I{qkpK-Vma4$#EI#^o=!_7z(mloG+%z>GdK=ird1w59-@yK_Gq+@jW$ zAD6xbN_1)Zw&W?o4grHsoH)@h@jvR0)22-muV83knFj^pb>;r0mWUk*>ycs1YDk&V zODa0V7T4v3hvtZ9oi|=bi^Q@TEV^iPxbDUwIz0~@^=aj;+kbdTHQ(Dmssr`}MM;S) z;$>{pW4WRw(#G*4DG*8X-)Cu)&+netx=Hh?bgSvzpMHx(2mD%9@q__w#Yg6>WZaj2 zLh5wZ?mEi&T$?fqCX+1jpwf7h-|8|mD^`orGLaOVa?!_!Hpt^zbp@5I)v#fX=R;n; zOic8eFl}w#hm)w<1E)sE_-Hzr?^B_f@yo9xW_zCo4nUXQnRG)p+!Qg;C8DTlo^H=M z3YoQmOjxmY=%Ao_j<2u6lUd!8h93+ETDmP!4K@+4Ks2??r0RGcE~!SIHc5CkeHuJ2 ztd=3%&K6QJxYdA=O_5t64mw>etI(;_aU;($4;RwN99+=nN42+JMa-c}_5%fX7?Uvf zeLOAns{&u#R|4Lm{K-2fz5x|>8Q?cEIVi7 z>~2nH=1lr6Cf1BFX5a|7Z^8|EYH$7FTzw|dD)(IWXV2tG+~;!nRY|lw2cyDZZ7LvF0~W!yTl65bD+zPf1S8>EE&ZzV<>fV(AWNZ;Pr1 zJa8fp$~h~6zMYcf1m2}m5=b=ykC1kglT4aiS|8EsvpTQiJkQd4LXrS)@1GT<-|5}6 zi@H>yWaZ;og(N6frn#DQn^x;Lr{ABDMa@yAp@B4&?pOP36=iNfPK5a=79Oi;q=e=H zqR<@9pb)l6G)%;!)`TUZvY5(JUx%fP7k&br7b9E;5f+X2+#S&D)cc%3(v)xO+q1OvN!S z^&)~439r(iV<;% z)4oT&(nL68G5V9Gf^4O#LTC*I8ATus#t$!3Ic&i!SJ}S3*uA4eX->xy7AK1Rii#Us zb+<~t%i?1!)0| zJ2=&nPNrJKSkr$4LySA-&p71Sd12f4oPY(9u~ixF zMLKL`6k=Q)lx+8CaBQ21sBH9c|$*G}QOvC-bw?`dC>Zw>! zT09CkwHjVTLXlElcbT2tyLrz!{buZC+)Nrb6jQPfBw>JgQ#{BjMyR{peJpJso6TGr zHdo!_X*avDi`^#<9=?AkmpqZCTpBra zM;x57o_#WzKf%YyeoIz+iNQBLL1`ZD8FGP5qp+AVz?>35w7>Q1SESP89;!l?i~jlc ztRZkaRBg~*!K^r}o~u*g96hyy=&5fTT)C_yzmo`dfQHGJEz7=MEWdo?#*q92@R-}X z%BU}>;ztfvYWHjIJ(OFZb}jYnCjV&o;Gv}ewrcV!jh5?@LDqgmt`IR3qOdrm2$p$};7v(`hR?otsJYj|g;hUJN-IbNMx;DaD5MZG`P%wFQ@6aS zs{wfLPAT%%VEX;O5L}{!we|jYq}TTa|D6@oq$HQZr(N(}>aC{!voh#aM*{lIC zZf!w~AyCM-XXRz<36-zEfBE$s^nJWPlvZN6qA{Jtkp<(x7eW!s&7;g)Gj{*?((Z^; zuVC0#nGrRSA9vktWHWczuhW*8b*rYJxwJgK^x5*ywM904@+ZINRH>_!f?-q3e>^K6 zNjWdJLN+I-?qnv7EWet0%d1bG^k#xXb2H1wHQ)Ymx4nMDIY#3uc4`w`(`1Xf&h>ir zb@m_KvK~=@CJ;AX)m{r*E6WuQ~$4S;SPqM+O*O6`g-TegZ(px zl)MQrS=%ozXg|lMP5UuRT!-TeOztPrEc}JRB66IGz}XVtX(+=+jcZ+5oET{|2=^5) ztv9NqoA{Lm3oPE@_?gp<2BRjcfVOUXJML)`uNp8E$$2n1P zic$#6$_i0c+{=A&YR$>lmE*2Vy>`^3j@p?T}1d!YayWMaTazaRzdd5@N zt*)O?Z(Y@{O|(abUk!Joh@|J-abiO3DpkB_+Vlkw!lDclTpW<9T+-d(h@ERUAkq~zQ91! z$=COVnH(e_#vo95q4UV=u7-_?+P z2wFgID#CH^2`d$JlCP2{65Sxh1P-8+gfBRWXgxl!g!A(VoX#I~6}}X~IK4JQQ+|c| zPsT{lK2}tSNq_eigMhgwO`R&!mNuo8n3PE^5!#FTLUc+391$=k!KJ`GBL7{L)m_H7 zD^#i~ssnKhV{S@3L?ZgOiDyGrP;lr&%!S|0dM-}|TbJV*&HN@LEZ15)z*b>v>(tr4 zqIZ3&j04-sK1Ll+xI(Otd6Gf&2fJ@IwYlhu@l0ATs#ie4-5D3Tn{sMN@SqJwvzsZ9 z5C7rot8fo>LZEx&I<4gu(JaDguVfF*cz2#FlF?EiVBrN)3u!JUbK`&8PlUaaS$6Qj zQ1d>bLgb<+v4B`LCtMzbPgVOkX^k@yJ@ASIgBMVZTVMq<6lci==d_3sWZ*R*!OXYA#>c%xg%KPyo zVTpaoJ_N~uH#gk};7dC`K^I!BoA^b>dA8&@KkUaVDrCfu37#t~qK!c8mPb7i>#_%C zJXxs$1B)Wi82F4t8z8u=i}e0N7zh=Hc{pW31>DJbmY9Mn4y33sm>Mmf z+AJWCTVU0Z0ZXGXIl^L9?ygBzN1>6?wY0l6SHc8eui~i1Ba7>X7A(HKk;f~&g;+Yo z&gA$cVc94Hw!4=?3s&rR>PK||uh0rw{l1h`ySu;7pYTtU#F@k1^b8I-wCg&hl_+g@ zR8=Z`VbGYBi~CYnp<5y-mOQv1Zf%Cl2vWKD1z@crr^itC41!VzDKV+~JvE$hi?{yt z2s=A8-iWwaH`dVvmv^TdgD#q~%n8r05xm-GFk3 zv*tlFgAKj_r|`HH_cYFy!hv^pGh)s3@vZygw-TW!BKPU}(^+*tW^$ySNO~E$bj2Fg zBVRB{B(FnbszoOBn0|U|tJ5Xb6bBf_)5*sgr53#=&HPoEW#12F6$oWV-=n74|4S)S zr20%d1r^W*Wl!K#v}~F~VrcXz@z!qMJglsW)lvkRCBYIVr0MDPel?&hY^z`Hfyyi+ zw6l$vdziN*!+(REdwlk8BJiW%zI_x`U`sb9R`G06ou-VlHeA2Mz$lhK8u2Ui1b0dS zqN>NhCHi}}<)@FsrpUq-!3&MA-`(yG&-OY#3yCq559Ic~Ef_K?4*O=8w(TfGut2Q< zoTdq4p+{j;VQ>PvhDI&_X`W4ugfapg4ynK?}g(69k=`@B4iZ1bwQpGxcUWfO3O>|mbb)-Q>T~V)UN19 z%8YsXTgsLaK9k?NJvq~bZZ`a6N0K%!cg^rT-ynMGnKD+Y5am8@!bbNdSFtGi&)Yl# zEcYdHn3uttOLS?R@?AW0`VPqe7$au>UW-SVkP`L`^fY7iL&m{a@Rh`PoXpo5+>188 z)0hW`Y_o5bcs?H#beKPu1EL6pff>bwce_(-HpP0R*WgJ&awY%SS?^a7b&W7&no+xD z?v?B>Sa;9itX(hx{GDbpP z51M2y1_=KC_WQq58;QNlm%1FOP?=q!k%u&Anxz~USUIo>%+>7iV~%aq(c- zt-Qk76mP2gxVpMZslpFmB{wa&yZbFE77RpVJ!-Lh@v$4_C#fc6Hcu2auu(GfCx_5b zB3|Rj2toFkllYIc&NRR1J5FF)iNMr!%wrpR6CP>Y@Y%lwo0-Q%1ajWwdjI2V0aUo3UNkvI6 zNk5g&fQgK0itYhbx=c?aLOFbAt$#{)c{zxd!^Z4gT||0f+8y|VQ*q=gD2O&>B5H?k zCHcqG6J|_z7~Yt&9t?ZTOY_A`8Eq5mwXQAPnm{E6|qI#(~g%_o;#$ zLx#c9cD*?T^ew)<)J8JVk}<*#Wj(;DsP?a;O%Sttw(yLi8E>;*Gtwx2jv6(DOq0{x z?7f_gU7=*8I=kxyKVNshf#MRHe?`3}O%9(N4))5vRQOVa%qX3Fe58KtUGrT*IrEkr z3alEmm-~B=@z_T!w*KqY*i}{$8PV*7?^S7U@!%+2Yw-&*c{LUSmFV7Og?*PMO+2#W zex4l}s&%AE{mnqs(23gfdj_r9lyKbcgKPPvaChRN94m^5W}@4-sZXAD!$e1Hva++Y zK|&HHmiH^OgXlDvuQZ+(dRiU5^w4O>V&SQK}Mlbv)87vuIQ_A zjqxeq7&`O!o7+aq(3L?Mu(DhA;?drr&z! zFV|Yl!};}}S2fO7u*lDCQNi|ph2?X5uGIOjzms3S()t?}`jwaX@Bfl?7|!sY7y9*! zn+^Vfu6}>2L4%fD5hY-7UEqgW2si+C&mS0QP>;wfZf?JkM|%27GWRi;lyPBcaSfEu z($G;Vqy%3tKe;F<#n$otvqYb6xhwnCXkEEY{Q7pO=4-ahAMDq0aOj2_r?q#q@u{F4 zTD^MksyX(9wzXN+FtTsj#j^C&(UB6af=KWdI5i9XgZ#>3HTBiYMEaZJTjg7U`Jn{ z6@)$;L|da6Id^2fzt$uJ$pVfU0+#TU}>ZD`rPcm zv_;|9ucMHB`rql&`VbWi+T}Bj&xm+&uwocb;Syz+yHRzQ6f6$eWc}Bw*`sK_9QYwe zoSWNkxszm;$_VpVcD574yA0hQMksWcV3@^uB)ypZJ{asi_&qf6|)2G z92DU(9&$b*yGE^A4s>WngS84SeBM&0WpDkEZM9LAay{CEwVd}QPqFLw*Qy$|wPI$p ztWx9k!yV84!YBWAkXe_Fn>RaBcIfpntS5||FjL^TGpG;@?Lw`BckwKu;N!A5bkp$} zBRgpXv(jhfr9s(C-UQCSu6)CMvLiG_YEh#>G&04=#`3Z0@qY1W)4sNSP-q^*5*gvv z&8JVFrk0^+jf9hMIlu5|`Zs#+k^xk=Y`@(gyD@nbTd)&5xi?YQOsO z0g9J#XXg#Z7eUkxMW|ZamwfEg`UZ^)dEK5PS&o+cD9R1tNxem8ZNt3;>I!EjIREJB z?4>V$ls;cwmlL=G-8Q$0d1JMeu5bot972xnojT4^o$#~J1_W;$^*N^opNzfuJRqc)SqkHm1?)!DFy?gJ3A?AGw_6|$EU13t!y>~`j4_>Y0 z-VEoN$9g=*8&WDbBAPoAjYyZcNXLsz%rbHxB@7BsN5r||E|bgN6XHw6@)4}Tg%D1yEfdR)SIWI1 z{vSfe8p^o6a_I*Xoy{A!Y{@{kd_)|k93NWQ47HQ6!)C?C4+J-e|ksZ&;HS%4(LY1CO-~Y8``Ry_~US2Zrj%Ij_`e zHtHgZr&1otQl43Z_jAM&l+(SVsEfA-UPMn)0+A-??SM1du*>7JvX0Yk)W9%sSwVa_ z#P7{y9JIU+>Aij?!*oq+Rb9XwQ*s%LhNUr_%8#YJJv=;IX@Dy*CuaI71N$t?uv-Hf zcbvk;dTCMVhE1EcE6sZKy2-}mDV#F85jt&4WuNlW)54ILnVaXIeY4*lcR9NSKB&Ta zApt?9(FTaok%Jlui|SD1X0nf2PY)=5>ualFhen|&ij6(ZJ%MPNH)qaB{ zBF1HGMedRBJ|puZbqULFvEHL{iyVD27#|p8Z;4<;l%gkI_nfbD$cz|8=GbpWBCe)sMfg|*)2bAmQ#vf>96(m^>ujW%0U}MHa)cOeSW-)2{5grl6^UK{W@^HF&S%>xu_|fmPo*1 zR$AS%#n+E7{_K>;MMk*j%BT1H{!LkT;UXmO{J;pGpP$wfF!kXht5mCobHLm@dQLLa zA<=qq0eP6FIKJb@k+c~bGiT!Xu<-L|ds4Fr?lR!|BM2kWMNKykfu~ooOCO4AWBIp_ ziFbDf?DBbx4+>jc$^3ED7h*y@4qy~#fzV8?i4tH-Q50@_koJCfc9BErehgOwNOZY% z95m70KwrF7wHtkL%fHLuXA2IXg7|acOiUWLosnj8nN$vLsG*jAIQyny)20!;V7~td zixm2X{V(EM^j$HQ`uhRvtLR-~9FlUlNA}`@5;;Lm>$ixP^P2M@s~?&dTnGtT zQDWdI`zZ*dN`LibznpF#usJfFbfq#m@Fwd4MAnJDYs%SA-@ko(OPFxru%O%1VqP`v zMSB;pY86Y)jdD^Q3h=;ga{~NGhd4Bal2$E!*IZ62q^vb%wo+b_I81>)in^S|dt}i0 zCO;o=ZE185r4-yukM5xl2&V>ki>ghck`a%KB%F^bb-U7d{Dnt(*SGq-(XLl-qJgyt zf0Vh%0m6R;A#2<1it}=(+3SbkFU8lfB#TrAp_VnPaf-}Qr2$8m!Z?SKs>MkM z+QuZ=tSpzLD(e>*XPViYu3=^rQqqBd&Cre62Lu|T7Zp}E0&<*WJWl-;%eOvpGc)aI zTpm(n?I3XDbey|m)OcGOpIDS%eZD^NlD9(O8p)`%I_OXvHO)#>dC&sqkpGj`o3=;)yt^b=S0SZ`>s{l!v%D)4xB`MP&Lw_noTvhJ7}(#K=Wyeth(sg>Sd{MN5u;+8)0 zM6#*wX$xs1t5mMcOoVO3{%*!ywAi-(<8=K(sr+2xKv+K;g1AS%>9Kog31CJscg=(Q zO)O+kI2DYcWwT|tpc4Q`K;*?WR9e|VLCEHlXaO{;x&hzh1Xhs&=5Aa?E46&#MO+x? zMHaL+Lnm{ju3Y1YE(@)F$La0U27|mh^!zlmz)I)9Vc$j&!JHroOtJK>jT5@WiXls5 zy;dqoleSJCaZ*;U^8%NYv++y3^rrPA4RSNZvS^L~8qK^DuX#s&I@HyNyTA_`Fxr3rjmd-EG^%VQ5( zUb*_Auio;*^*SlyNhhVsy>DsA9tjvaP|Go6*X_6M-&a0c) z2JPLu7m2TW=FFi(H*Fdcf{{}FVkyRGN4LhWu%4BD!9bPk605nvy<13slzVdRw;>^2 z>>pMm6oetbcpLvU+n!`+Z#vs$a;RluUksG`*e;_+nh%kFC(wKpr52i@)yRtb#6dIG zY-Due-8a8|gssEO^5oq=xElGGezj`VRH?LBum1XLTV8PmRn|$@^{}$L4_~B1QYFl3 zU0N`8Rp`dP^|P1>d5}k4?qohkYKO(=MiST!ouflpG7F$ol~K8R$@lJ!NG-QmGaNUm zi2L{Nw?8)?GY{@oJ4yQQtCqjtJ$=6<B1n3|b|W3a-*c6SwFs?CCZN^Oq# zV5K1EJmJy=^y(quT+tVR~+$tO~_MA7H-iN}PkOG1N-l^7->;n^=oh2AOa2 zwksPx4=-BoX`*AwiHMXm)(kPI(l5Ag*cM6?=uRC#r5JE!Wo0JKDcZLa6vN1DRDwhC zn!8WLEWX}&f|Xr#r%r&jBXMWR!(%D^0#b?*Cf(1Pam^i7vP;lc?9E%Io?Deb88^q`;Ni=KIp0uowUUi#JEdvokH zImw>nKVp7)dA3CtrZe)e+fX{kn=zjf!c%Cs=ONFOqWc`@_JzGC|Lmd14?dZE2mG_{ z*)wNG*JxX2j9xSIM;=mnaxfj^taM7}c|5aWa!hmg?xyfoZ9`IaMT}V{OGhy?goYG$ zVxROhy!)79y(4w4wr&{#pt<$c9dA_@9T*PwC4(N0A^5YMB0y2Yzoo8HN`B@;rXyUj zFCF6JG8w0V!%0a=gRWPkLdP8^=4Up3=o0>G8rGgJS6{~l__hUhuH94;V^-qBm zA15`mTQYQjwPim!N`@m3_Hei}HG=Vbrk*6n0~YAFZy!G8rS2QzpmfPQG-O_;yevUyO!Cf#7CAW)`xCX6lu~ zamjHYs#MyQ`E*Q2N1q%(zmi66QcR`OxBF?%jmukI*Wy0bEOy>~-8bZ7$cu0+gwiPz zg*6AD*oS+7Z}aa_F@=-^%TN$VZA2Y3rK#`)rNn&c?1+jTIDP>COnYeEha5{C$sNt# z+#~xw@2}~Up%79&)la@-3lS*Q3q?mNTPGbzW66>wxnRTeggZ!l2a)=;UowG#AIT_! z7j!`5EDr@`bTRZz6j_g6!6)*frOKJ-2Si==$kR}R4~*zb^(##@^G=ZzPUF3c9l!th zkxvCvLUn1Ro5pFeb__r{Ask7k1eNZXu@va+z*yK*<(u(b1wj)rESbU4^U;a^G}(5v zRwTqUhgUtIL+A=k9W7ggnlfzIz3}75`xFI^2uo~!km*cNMoMau^~Z9ROhrxT(&FK+ zk2A}C%D?&;qMAhjkbY=bGa6rK!b8bDpRy2;>06`&P>E5L)^VJ&9MxRC_o|zT2Z0B^ zdxwBcJm-7~abRI_M60LFoG?&6tT2H|EV;|lpbIv|={FLXy+2)VY&aIXl<@VjQsyu( zkVZ=|AE#J_;?hEW>!fzK+o`NeSI@mM!Th~cI>Z|{Aciaqg;+>Y5Vsm&Y3fh}hdE?u z#?{==o^XcNX8CqPgOp7k3Bb;|JaOr9rkLnZLUfe7&gvb--Bb4til(PiwHzU(>w8q6 zOxBE;-+hwHWOeFc+qN_b(w>|)w3fk3l6NRC;6Yx#>EC`ULWg6N(awQafuCjy(gADi zM2jo(L5lvH4@}fNPpafBr>dU*d{((2bfyh9~AC! ziwefWeCM5fHbJ6Ik}A5eMEt1cb1VqLvwlRJhpgdZ+CUEqsemr5$8_`q#Yoa;5ILmd zkx~{2RB!3y1TrpY%&Fcr&!0V;%z;kijMN^_mWNP!=-oWX-vS_Q$cnF%iqr}wl&5jHwjk6!(QA0Ovu5=ZzJdk`D1ljLNBA=BN zFOLek$Q}IA`C4Dqh8{E9{59|)b3n?xZhPklI+ z`ZD}Y#8+81AD>?){_ZQp#_wIvjA~s+UAb5@vZ&DS@~#EtK4@YkU5E%nIaqEmqc@?A+ezA~gYCB9s_k zUsYM|N0Mr^Zc@t{jf@U=O87Q_CmyWdS{vPDFjJsie}B&xOWOh6pFe+osgh6k&YoE& z%fkKia;Wi$AEy`T$g6xzo8;YU*<#KQdLK9-pbK$pS-AXpEA2;7f|>vQ^{Db-t-pWG z|1Y)ExBjo+D!TW#)g;y5e|AD^WBvd7Yu{m%Y5(;#b5*eBBmDlOi4A@=t$%;V-Z`<0 ze!tpoceL#1kQ!`sMM+`8@XMh0tPVkmBmz+I>@XhVM5gegoIRQ<*tw#WO^AN$W}SBr zWlJo)bJ@-0Cdu@e*_5f&K)`@$&1PSa^N%yl1H&^YIsvu}hVAOypHee!Q7?Ic=))i_ zm|lD~*z)P?*AWie>oL#>#wD$Y(|fYCz{>RR{(qgd(8{P5y?*ETrP5x@fgkbqOCtNw zoRQsw1d3e7$O2$%m!1i&|LnGVs9}@moj}UYuh6dbk+ZF%zn&hL8gjjrrKP1Hk5p5X zMS{W;5hTi7PfTpz3>V)^gS^380x;wih53hFIm;2A37&!c#_#6&pSrw2JVzdmXudPW zYsxz6xO}XN(gYkO&OA16As@>*ZSU-a2-26{cgamEJEGjt$%065S_OyUj0B(4u~>el z>~m20;i0>`O0mV>4`H!9ez{P2(!y^$!E#JSHPXoT9<#?w3rE0wz`JQTZMu8rP@Ke| zALMp2O5hZjUyJP69ab+XuwMc_sugz`R$8y^Dw;@|49}(-HWk<** zF0QrEI_U)<(v<;M9nY>`Uu&UZmkm8^ELT+DnK^TXVfH#^VeN^kMc_Z+lM72F(2@i} zd!4DsAF|PRm@k?Y)dUwM%7%Pm=_sx$tP>4w^66oz_5HWggGrr^-ehM~RDOeJ$CIa6 ze_b_V$n9;;+{^=_kzk@?>L|@>ckjyKGs?%q+WAu}>JbD%ot=xlkxKQN`(n-hC9j)G zo)TO$l;H(AwM25_jV&#rP()_+E)P9n;Wzq9%q$WS%eP^J1|h&Pk{V0+=tY>vkg^?} z?tRwk0wSA6t;Gpdt)k2tWK>k2yX${VIbFm;_YbTDvD|*>$FN9!sD~sY^7<%Zda?XG zY5&V`DF^%2cx2FeZ|5d@t5YV1cUtz$JGFlHI;i5<-l5bRPE7Bp=2@;HbL&%IFC`Fb zD(|0>~w$gD2(kNK5G9022&IuEJfyTBS;gm1&%VO#%9Qqko>bMzK4nY`$D zaKPF${9DJJnbTiApTnJaPY;Nk4U-v>iD0dUm$3VTDn~7*ngp&3fuWNxNNHN|iEi{_@z&kR~U7u{?A+XLtpEWTv($%_Dx zqu5-m)SVS8R!qC-;{&bbvU%?O`DwHhalAtMVcUVcf)j8JjYhQcjVIn=gr6`{O`qaUSIr7a=FhPjUAw*tY zU;`cF<8T;)B$yJ^qmeW-bn$MP2Tx8(Rtv=mM>S)~se6P#y#&M;PL|^u>5uFuB?}dg zv1`j7>x_N>=7Y-n$$Rlt|g96-G zQ)QsL_M>u4t9H!ItYtlM$1?_b^Ud}w0R&e1#N2R?gxG689}v8ZrU#Tg;IYaw6RJS) zPMO+d5Zw*O1a)Ye!5j5^^>R@&n4~}0(y{};mgIU{M(1soalDd{h@hwADCum9o1?r< zrIjkW+2J^18?Rp2%ERnB(pNP6vm#fMj)GFc6E1WqNS@+Yww})t#tAbDl4tyKXghyN z@hp$L^|~u?XM6*=Yb@m_FJW!6hy|dhq8(cnTwp4!eS~aO@w{i(zXEe*_ql`G2A~d@%gE*a@jj&s;-O^E;gpi? zQ|`T4x9@lwC>xX6>@r}d%O!HRK5h@uudq;hCJFJH7@aX@wtODLkq3OGmBEhlQO+K% z*~67_aY29n`KL_iu}YQS4-K*-yPy;u!Ux}Z@uE*~0uQVMu~t$lsInmD#ICV)lrfuB zk6vAx*m-pJR1LrJ^i^y$+uNx~Z$aMwBP>}>isuF;LiXb7-XqoMu* zN<=$IQB}uY%HUyYEdGSmoR~mtqdA9;GJPXHR(V_@78+%PDqmce% znQVUU%OS^642g`RN00y@{fKaF_bh%K3AMaeUUhZ<)bcGc^Whe6PcqRleh*ZP@Ip

yEFm_YiyD5buxCk=)ZyelgWBrMvw8s3*e0ky8 znriQTmVJM<6TyO`R9zHelOkVR+iX%wtF7KnE+6Dg51aj+dlzj z!f^D70}Q?7uuL0|U)*19x1i`U@++kZoa&?p46dC-qu$Yz>6+Br-|GKCu82ad6;M`L z3f-8Pm{(u1&jkKI9JcsHQ2m4nTF?4YwT95-yQ6EaQK!!E=)MFZkh2UlgzBI5m^>Q* zt(ng6_P>4BUg6gOPRV&ot)E|0YDery`>?_)9#mPsO`8`HF1lDn?Gg`ib*`e*=D5o2^oXX`}>LUn9l z3vYC%Ve{tu-tM|xnS|=VS8#C!kSdAkKr1O!;>PbGF-M-A_v*+ECyMgq;tIJbIu-IY zFvNq+!GK-F(^^#rs%KXW9pbXZ24D8bP|G)#?XFMk>{ghQ%;ugPxoCzJ4}$g_PJc+! zMd})RlGtvhE)EF}K>BA@JS4Q+nSQ0rcffmK_t19oc^DcRNulJ;G~z=?Wu{F2XyfPe77u3xN8YU7jyCr?#r55Ff@23*-|`RjT313^Sm z$q1B_z5Meq8VwcilfjZRa)s+UO71uet?2ReTpZngBvy+yz%CgM$5#VR%QEiAITwmUo5 zK)WUChn>Y5t%-r9ZN+zj!GPf!?2p@ZqR;ETQLn=+XCM6edFItLk$1qez!R;Wl1Tg; zL;;rCK4`f*&*MkzHKg+Tu&ab$mtYHOaMU<9BH1B0$k4_(X3cph8C34N=3hy7+ORUL z-66n3B9>;wE+EJ|ylSVS3jpqrzDCG>m^~>;S;iYTY*5WRsa#3qygkh;_(i4MlqI|P z<_#Zu7^sX-D1Uw}Eb!`HL6KAkZbu+FQToX4qQ?+i<5r5L)|fvzy3$f05NHHKK+9Vh ziQF%pUh@wNbWC@dOG$=$BP@II%i7Db8!39H*FUv(#H^ibz+G-KoP}Rd2@$eEaUzg8 zK$W_s_JWtmdKcn?p4isV62#WcyOrr&)pzceo=R58!cAOpHc#v5_;?qxzRfRw0F<=- zSukf=Y#>StI13XAvt5-M4Mz$>3B8_p_dG;$poO;!==}y^zsM$LW@foGXAvD=TPMZgmgr+oe(CxHA!(k)Bw6FN zYsD{!7dIMQ6)as+jmV{FEuqomuE4_tmR zG0^OsXrB*S0&((WX9yu&RL$z?4eTc+__T=B)uIGnZhlEy_N^!5W5M>~>$?p9{ z2i95oW{a|Ku0o;k0R%kIZ8i#@cn;bc!^%_@mr_cC-#_GbG>G@?22ef>n@~A}jABB& zWy@o90K$|5w4dBm6Fi4Cv4Am%sBiRb$z%d<5XN64X<4P}i#lDmWe-bXp~3OvKiz0X z76>4UYSWC9jO7k5r6E+uA4h!?MfoC<2*wl)2M-yN$BJ^GjC@s4Fnh_~ZpL`d9U;I- zGIrlzN#T1${4KJgr~WArsj3&kM@&u}4)m*VYZsugNX__Re_FuorTNCSTl8`eS8hRt zh6VO=q{Kh@&{%QF{ykKr6Wd7YOKwMd^AC8Z2Pi4$E=Rw=sN^~Tv2E3%)M=sL{xVYU zB)AA>FAqzOrln&wG&HpS%}*G;6oW!qMR!yVOF24MM&uB-xNIC^H@WL>a_|vuV1yvX zI-6ZcCBc)+e7qJcNQW_uW40jbrMhb`p)81$^3fcBAh0|DdzTNE8_)v0ar6Na_-OAY zx(LlhkAsNG=@b3H4&+Bt52H-Xr3{B~*$>LP?d{anIj@tbZABu9PG_{p3}vGr3le6I zr0Rg=(?7lw6>|LNyXK;&65Bqwj(iGd;Q~Q04zV>3<^AyAPFZ~mK(8>hCIzc+c!R-O znS(AkZ|sryIJblMS@e2erSJs62qXVRpw=y6t(q`LacdV(H+OC0@%R7il%UU>-PzS; zARl1pmtm!}^Dc(Pl(=c9JjVp~oI5~LJ&lgtf*aGXEI@P`^mfI5lS6RUY+mc4byoyR z;{u~tLN^1=PqWv;<(^ZrInB1!{**hWvZf>ahf=rEvCE46^wZ?F2VdDi}28z2!+$zg~FQen5-;g*I4 zm7%>?HQytx$op~`p*dS;S>0#T2>*nvH4--kA3hlXxL@FO%N&C3Lc5bK5GIX*{V&!y$s}Wm<9V48@P*?x!OGHhw8Y zxUlsA^dh%}1Xil(lQXzHc^IsyC|Z9U(SlxGEcLOTRYxVnIB-TN;yD{x$8a9p;8;*+ zZ(PI-vLA~E7HrQQl;mbm--|%fArxz#*%vaSA53-;8x=6KVUgoqq}Z9*cip$N53Q^1 z)k1O_I_Pm->#cWdp$d#uMu7Wt)j67PMHgSA>bDL?CQo20TKQC%uy*X(fENqPPEG8( z8?niegih`JlcP-!rnn6)>SOc1clmytjW6eoy-E|M5z)NfJ9|E7%2Jm0UAWxk!^e{K33Sa>EHt?_B@1Y5BDrRnLeR5WPq`CJ%5W5cNsl zmo20H@!(OBo{fAgKRh})5NK%Bph0W!x;;!n;P1Re(WvZG^23XWNeNYB_Qxyj{wKzy zc>2n|p8ql?!XhmT|LkCuyTsHSBi{&#Z>;S+rj56@z=h0FR(MwYveS;s@GugDPO{;+ z?_j12QRo*oFT2-0=DTg!8e}bw!&P@a7kn=({kbP#SI*3*=Nv(Pn21TF@BE&)WyEm1 zdk>N~+qiqSHRT@U9NYcT`{$dBuEG%jv1-(9B;Spnqb>Z-)-M}3GHmDiS~rgTC|I@F z>q4DQXA&Qg_OP+;O@S2Y3C`i=~saV@AOGnajH9Y~G#lyp>+*#C-w1?KY~8 zp$ANEO7|$nCn9p;46G%=k7CF62&>!n-KRIZ##)r$eMNIB%rr-^opC5wSV7UxY5u1d zgR4}j62VFpeW~LFT4$hBh^Y>={?(b<@`2$LlJKTYGubtkAN@ zy_c_TY=+Jgu>$J(+>osnsq6;(7GGyaly*x>irc)&?}@2@i3N@t>nUIIU2AGcoJYEZ zSl`vFSC4DiaNAsZJniPqXBL?3UrMKzO24q=Y6FGeQPjFopyNVl3q4O2WUc1a!-1~# z5gN*FmBa)9*n!aq@X%8;i}_^U0veDp@=!@4!V^qemS#hx;?hMX&&SiO`2;ti>Zdk4 zbk47JH)x1HKe@3BJ9SL1vEuuy#tVB3auHqKBD2!|`n|2f>r~NF_`dtVc9)V)>Oo%9 zC5tsb%WS7e-6-m64-b=A~KtvypHsLRq z32!x}#qx6k!}cCbL~_q;(b^1Yw+^${bj|yw_-MrU8gh;Axio&UX*2%+_t&G!<{E)! zcXDy`_1*V}glscbYeWeOMX1(btr2FcFHig)mpnzClJvS!KM#7-TgS`4m-8fcU!Ky{ z;J}>zYuByGG~Y45!^qATk5yhja+gA}u;|`&otVtqC5n%g=jD}LqGmNgsVgKb0;4pn zQyl!fUYVe=e`)r)|6aqsJRwRuX|52@ei^Qn z)OFsAl1tKRV+J7Yc(-Y{@AFtivoVAFuMP!{cGjN3`Lim1DyLEh#p0WvKDX%SG}v}P zBTiI0NS+leni{-eJ4J+?O?Kjh#)5m%CG&pIi3z7**3V?t_4>U<3g@P;ikxMe zK?`7e*AeD$iUC_pEOy5X6A&b+5hRiWlkvP~Qf6jmcCNyC$nY_`OlxgHjVRsIVSEzy z^jUUFzLCxKG{M_EkeXP9%XxB?klWrxZ`@NiYCfE56~?%-8RbHjETx!wyj z52rqxL5f(ItgY}{DmQWmH}ZPUwQ2@(!xVl^W>i+F)qFthj$=ndHwAPjQHb!67D54GK@8R&9~l*bgrcmOP`QN6m={3kX*T{1I(&hxQ>O{4O`IH&++50W&Y3g4di|}P2XDX9~3mP{n>bS=w(}((t~|YBs!myu;B^dTmprq32NUJcfv;ch>{8Pwqd~I<$1{(weQaJkryi(?I+SWl zzzU+WOppn>4W1WC8E5dNFNXftGlXOwKcq${dUFk4QT$!3u}Pe3@t3i z;}~}e>SS_y<3^1R(S(baos3N_ppK%IRLNE0%iHqtFWqy^kkvWQ-A*jhai6yF&(n)) zB=rY?=>T;vcqP}N4uznM;6yv2a3twX?mJj3X{dAQoAcM$%_TygBAMgiXZk3 zg#INN5>JX{7%fxpk|Kv<2@G_iiRcZvv<$!}BTr)d1Gb?kI1|6nr{3PCx5yf+wl$&- z`?fegQf4XN@(l0qoUkH-6dpxQyk`6Lm)Z9zpLs2AUv0=5y*RHL8s;Kj@XEFLK+t zHuf4H{WgeyTi_TExblq~H>R>@O$Z{Yt=Ybd$AXLVm9&Sc_5<9Jd=cgYE@(Gw7z*G$ zddm#Wx15MKI%`k2s6C%JH0W`UQB+|9bO0O5BcyT^UL&B4sSXD$G`0gr_*-y>1%ib#m? zJwGZad_AWo2i8}ck8n{lmoBxkwHl&ihI?Htt(1cU*OQV%`Jc;mMpxrejH^SrxnUi+ z|Bo!8?WEY{)=>-&uiLWsNUBTF>phVx(v6@5Oi9mA0k(-uHn+ z&Jk}ch}6%!M31wTGi|lmY~;@ z(s5*Wy}w{yoRlohqSOYBjGDJ-0oh-rg~jsGYKC<>&}Z7pBqxqCA}?`i@v`X5ug~dc zM(b0tMq>vzt6lN-s?!y#e9pI?Q++LRj?h+PwDqKP)akP{ma`BdAoE5S1Fu-C5wxi{ znNt3Ec|BmDM8**z%nxZk{0*7V0JFFe`93Joi~>Fvq$K%9-(#B4rI=FZ|Dz&O7$kKc zti`De_rWBO2zm!VP>3iaZ_(Vj+lt;UUZ2CUp;o|?xv|$mbKo*nr`n%hO+NUh8-&oBF7X&!9FK2E|)~vs9GVRO~ zrO4vc37!c>c98nK3J9YgU&8EwlJ58y%F#cm^6VTCmF^UpXm`ec*E|!;_=c)R)Ky>0PS)S%q zmW|-OF2xKipH_!Tc#xH=-1=WvmNQ|3Juhm;UXWA5}(wjG{X#f3zrW_(1Tmg3}0@AcnmvQ2rj5eya{Gihp zLT~hnzQ`XTDXbv@oX{xCcrC1YE(uc|TG!%RKVXaKJ#CHUny@74L|zqP5wdD0Z^%>! zaF;Z?HvynHlrNpdVZ?!SGE?C34iJN<(4XN#Ww!?!wTE0+zwSGyF0!_o!$CwMMNzGQ znSE$@7aE`Z`Va)MBcJ=}nYjaH>+|VMplmib?ba?OF13Co#qR5X@YEKFbyR7m5x^o< z$d^`#UPWRd?y7E}8cr_(xDV1Pp*iZmdq?-bZ-`>v9}*ymG1N}q9?gY4-H=7Nkns~i zA{oTp2^o%Smi;~ks_-nPu-cAVEq%#Xf*m<6l=?L1_3J|<^7!ws_kx%DGlMoUk>dCF zmtTvi$-$L5gqKWUGR}(-_x=0#%-$UA5k_q%ft|Z&y8PQ5`S>M>)3q_ppq!$M%cX5g zmw684Np}AV4Sj3v)Z03nHf=ng61=&qH!4SVxIIZEjdLy87ZQz`FSBEE5$ml<1rGmS z&M+w-G9Pi1j!Uv6h=E`1k-8e~oV1Q;XL~URi?HHVZf*pvuK^@0_ZSURe7yaCJ-B|f zMpIlxgpZyH-E%R6?e3&Doo>@MDdiXP!UFEn2Fq_n2Oz)2%*lK@z2~nl-*=Z%vWa4; z(xj#f&DElXF?Ao0UAC*7v!Db&8AmN6!f$as5ScM#_jV- z`-K%r>(Fxg8qH(@?CCi-*l{mC$g774fF0p!fXRg(X!u43D=2xU`uG0#9(9v@g0jUnM2;^9TXA{>c(ifq!?%B&Ha7orTjtr$?SSFks-7 zcY)UDwDosQ7`1Ak{?PyVWiEX{9muzsOqsQ!Y0Ssz->3PTc=g+Lds?^u(x>_j;+FR| z-Bd@tI^=S;d+0z-WBB`pF4^C3E06}~@2`Ge3}D2DQAyB40&6jO@*E8hLy{qy#T zJp#^30WT^6^y4KUOhO5;skC}RGB|aSlj~`(lqw2V8To`x7NQnm=nRxW92s%9XqjfI zKX<<|oQ=mz{X~&@4zO%cVC90m10-y3I z(X0@*q*4@2Oz=8}XcnGpt9jY|uhPY8WsBSP{Ignon zyiNsHznGZ>=Z($iA*SxVrnzNF!GFuc&2CUn$&`soGk!WHsh$9S5NLk`>7Z?Q4PV2( zsg=JtXg#ypjQ0K`HDb~+{N-KE=UUuL)6((ld7HQ#l{&>OO~iBrMA5T>neGIAVn!i^ zaZnGuM*?=y(pcL?SJ5he_b1a6lfGzPXG))c?sr=xs%mMiz$^2hMeV@6?^1*UISJ9K z4h652rK47opH<}@+^nnzOn~sz-Fp>2O8pbn%b;@~YRk*w+44ICwMzW>@#9Fmkn~iu zzPCmY~R#0iRM@Y)Ks~zhHELZrA?!k6A5-E<&7+4s2$9Pm;nyQi%J?1)x zF^%UL-rfH5*8;TuX8{hU^Cl@8$WRcE^&y*YJJW+mJ*`m1ZQr_eMDQ*dL!-yp)5R4+ z_19Q|f&CE5&`}6B-_2>PxtYC7|9MR{$0efj{xPTv!z(GY!=Bs!qxqi9t(qjQ`L+DV z_uYnn;1&7(|6|nQ@9idkNvHL{bIRiXzV-jT;hI(V|L-u*|6OMPFUzd?`#{aDSM-}E zH3Wc|dZrIxe-Xi~edZ!GWzrkU7$0nm*!0j}2nKb^S`&+WjgUiXUcfH|Q~D7M2Ms^< zEIr*c`3rhqXM%b3 z_kBr>$+AROu*I%%qM+EYA}YkhGC^qy3KlFNiXBBnQ83YD*>O@;u!7Q5L@`z{CJIPX z1Qi<=ilT^$h$zDOyklar_ubFE`|kVPbI$(b_&j^B#K4$ij`91w&>I>iiGb&!x`SO5)6MM0D|-=1nB5ad8r zEE!fR6hA+|&x$ueXdvodkIENQ#p)rZA8|Xw#r`;XkkTIc9`gTvy0s{&66Q^m zo)Z|HZ_u34x?knU)aMlfv-M9bwLo}QJ?#p(E5R8J`qP9`adSts7laB={LorPl$ z7YC5Lr{_eL$@r-OS2vD{krO6_9q z)J#xku&&?SVmK>q<=f{`t(?4VL~13Kv4zb~6oo1SX3gdqcO34muXgwFSpB>LXlT3h zDt8lo?KTQ*T^eTQD5Z}`fnW7p|3-$N0g>ygWt7t4BTR4b68V@WoF{eLQ2VI;j^TyV z%pcSSwAd{m92Z~LmsnabT(e3g`~?w&f$om=kTl zfSUu*0%AF~?4>*hvl}9PQ`;haK75;n9ZgX^kAh3+xY+mNM`j0N<{p(bw0!KrLhaYG z_j|limtI2fIw?sZ@@~?ai9&_NHckeh84e#KrL`)R!Evdl5ikHvM@L3RPH3U0@*G&e z)&j8D(Eq-+idvydO)IEaEJB{i(=0ih>Ig!WeY_6vU#Q2J5a#R8r8C@Vy!4iaC`)=&Uh7B5Qe=}#;US>4DFYvbloe7Al)P6C0zXKa}ZEZ0zsN)Ssjj^SvR;fYN z)kZ}u&5hf(Au0_9D_XEy+oq05qBf3Hnn4`>_mX^^IJFh=o9D(Q3{OGtl{0zK`3Tm$ zpvG9kwr({%s(fv~cDr}hCNU_-TmxFxa{)s5?ZAZ+#n8e;p|iK|O8`fW3zJi>ZA5ii zUq(d62UslUq>Rz91;fc-V>E)p zk-UiLjxs1h9G~*VNzBi5!q;$0bk+>I^jD7&m>bNcNA-99hAk z5B(Ol{`(eu4fWDm>M^ff4e~K+zf96_T=Nl{#c`u9@VN{}EX_NfY= zFZ>S&`KIFc<3&$oh)53SL$Y~?R&_RAxHi9wa~_aDE_h95Sp zv#2rF!LW|4&;akKzC4W-vMZR0_nNgk1z8 zk#O)vZKBp|-@-OeJibDok+UV1gT>D-(}HCenvL=FFhj>JF^(nN%f<;8jL5BcpHXOq z<>zJYI*ZWoe6Ru&>EvC2%#t8VS=X&(7qGDmA^@>QQ18T1BT_TAkJr{Ua|2)OU&pAk zX%f^)=glg%FkJZuD+@fM&Z}lcF_PBOrqd4pDdlk25 zDq;!*IC3jdc%ws?Ia420+Cc@oC!N`(D++yeNp{30yHzVzoC0L)0#3p<8jT!QDnzOE zFlaslXIr$gw*uVg#K#rfXs(*8_&KR}M55oYQKRuBX8|qA9N&#Eulp*&NA`aKT0{_p z7qbE2I9Y<&K1`{E-%MdR zY6*%mVx+xV zI6%8q7k7?8eiD1T*^TB$rkvgn2z#g7Ka^*h-E->52CzeZD&n(V$p;>Wo|>{a!A*?1 zDgz$>uuqSFiu<#MsfP^)V|#))(tW1nc^&5#PrIAHWWb#8izVQlK{7#8reG4Mu_?sa zrNgo7srk33r~4}(n({~_x@u~@0PfAQLGADx57Cl~i->~3@RAl+eeg8s$o_S(1~N%a zY#t=mg>#+GtvH4xu-Ss+`f7q*pb8T~cyYFJ-A+m5>`>i)1Q&c0$EYFUPafgr-om2-2V!h>~PAzxfux-}LZcmsjBK};$rVK?2 zdMjKY_|x$=ZDFg>ZKO=^ql`v4U-x(}$|7KoDHIk7z%Y3*$96t7t*x@!|B9@mQOPig z9Xq=z-+Ij1Wv%qSZTzM5hG17Y#C|brGG0%H<6gT3D7qT4kqFZ9I$mEjSNSc!OWc74 zl~^tsn1eI{kvwJ5Qfrd?F=E6X$+$J?x71I&{KjnJLsXnAE?F?(Q3x~m=0XRy_E2lD z_O>?d{tTXv44#2{mytp8@)-wn25xfHSVe-P6CXvo6OC}4j~n zwqe!@K|=ioLJtRvm*jiGhDH@==wOR|inqUmnZMJs-Y8$e?8`AXEgP`M>_8jXu+t63 zjFI!G)#69HzO;dkSrb1XQ(mo3(E!8 zEc7`NB>b+0yzt-1=TGmFdbkH?-CV&YB}_a6Em>Ek+dC)^B1Pxv`|z$f|9${J_iIP$ zh>;N&oRKP=XeBo!j+(+skz;dIZ+!`0R>n89%Q~edJ7N5YpIA+E#0Ii}y3qWjmBzg| z6QzKg%b$gx=@_R-AC4whELia;KB6ZO-7tr<7$9R{s?i+Q;qr8mU0cm$g9sIDH<@EW z;#X^A)-&JQ#at_N=~T)ldPx1oyUc4=A=9zYrLX&9hfj{sCOclmMqRe2jGvA$c$+_&Q_TG| z$mx)(*^7n-g~aT+rVKpdWWfW}|1#EI6VB@aR)cDSv(CS$|^)YR8|Mm_GJ z-AZpYos)rv{5r@?V>!!&`edv~$e@ITgl2-pGeqESjCqi0pKPPKtU`@UC>PVVq)Qq2 zB0U_+GJm)vzE=*T1Y}lO;vnsEdm*5=lXQDicO|O}9q1<;$}~ zqA20jZ{f!rmX98j%KX|G7WKfN`@wFS%@zqVt9u!9CB$Ve25@}ZNFjUBHLg0O#Urvi z{jpJ(P!uZ@7Nx>sxJ^zOBN#A#mw*One*XD~QBzwo=l~U1f@j00v-`KHI!ku%)C@(4 zOxu=*8q^I=j{z+-jpC&u(3JXhmY1e8CZ@50IY+we5!1a0a92+7<-ier*s=A${`w3= zLWZ(~I~~TKI)_Du-XiT^7WH&?>&{+!uJI^gcivj+-Fw~Yr5*MPG^4D9pG3e*;T3e? z6qZUD#qOeV6cHm@zGYbZgh~~zNd!t5!#{M@)@`;u?~QB#4@tijhG#Mgo^Bp!D#Ie; z+ms{W%3#nfT;~niVexqAFA%Gj3Jz4E2s?{muLXZN~q*uJXsl-b932&C#e z7XNN5d5uUpWF_LClG)}KNqO^cut1n3zF4ddq|+fQ-$c^omxiz~<sy=4At+@{;&){EM@vEdFKXlZiFmOc?PqT&wUx5{qb5Ffk00U#fLj=^u}3? zU5%s(bG)Q%72nXggh@@f%=;E;;*wZ2nBvOH>aLZS4E>mG7w}I4F)g2{_Qh`)J(KRT zx0*y_y|PzboqS59q1m-J8g@8DKOJ<6nWwu7wGDxhCBb50eqv9xOduA=&SpMDB`DtC zwV>{e7P|a$H|(+GbZe3gxQ%OCj`BUszEn@N8F6vR*9|uPBWin}fsD@h`!{zln<{mq zL|>%KgwzH9_P2h5j$>+}q!Qlu!R|#^%z{-+1{?TyMSJn~1(whF0c!v`84M=1&F0I? zdd^io+jp$yNVgS)Mu`TRHzJ6eecbrcE3k#@h|Bbz`V^aKF;K;cHZnp5o7LzD4_JaN}NP=59s*V-@e7FaCY?sV*RPeE=+p?~cwL1p z#!sdY2>U@_9@P2f$LjJd59F6b5p&?gaqeU1&ySUeOA<4{OeMt?kw?&Da;BwBOW>=3 z9s6}QR&K*@>bLzq{De{eCJ`Ag*opMSx`yLX*RdhWkTYUXYomPbN+zT*FR4>rm$s^> zMDAQMJ)ps1M*akJEjern*;Ubu@;I~l!%5av@TX4+b6#1eoVsv*CgOFJtvTp6ZQK3E zBo1{pzD^WkN9pgCM_hxDGm$h`|F@svp^#VEuNe|C&{_ zZTzLc)*6{|yQ=`VA34>&@Gl{l(H&;L(n}4%m7Tm5$;uIx0rvrUrQZ4qy>6Rb|6ZUx zP^lMvW7qvf!(k8|MXhxm=u||sjd%2uGO|c{9i7gqJ#9uznR2(;7RI$dJ+D%3KjlkK z5rD>DShfD&-K6}OM_Y*w!4nwZImf_o&A|-1A z+UQf{B3(6rlwcUov5lJ;R?Uy0|yIPW# z$8%aUY~W z{GDPD51V8p4=Mm&7`KF9wHo-AT3J;6BK(#Q5X|C@4C4}pP?j?>P7ZC+a5~Rn7$voa z!$7|qJ^FsF@?j75WLJdG@89P7IV}#N<9?Es*5&;Gs*WQ(E9h3%gcOf}Uvi4j;r9$U zi(w@`VSv|8`tO|3qUx<`h2;ka(%xD>qKqC`tn;Dxy;N#Y0HQZR!*xc z0wa#1eU~VKt?PMRhiP?)??2_-3L9D(py5kzka0WSe*Easvd5RdrpCvO*zKNL8|t4Q z;)g)Ht}?S}KjGf-_8%87f^8FzIo8N;Zq+i(E4(W?xJmO|&V?bT$_OjT8~BsNi-94t zGa}BhY1rc^abi_kBVh3 z;7wkLj9|d@<>EM8%ot$sw(Vy1Z<9-RB7=ea5&@<6^2OzcZ^k^tfivxQ)*Xb-|0BpboFOG(9Z?8^-{0({Ub;l+~eWKs$_vll|?P0z5LbC)5TaZD*xoZq{J%7x;d|0=-wYN6C+0)wwGRTZnJ#FVq z<1^X~7g`F2iBryd=S*0Ou;25vKRtO2pwhKD&RXV5CtMBe_B>2^2s&}cmbK64_19<0 zqXi9fR;0J`OJ+^3k=W%g+1hU{wH4H@F=v*WbGB;^y{iYp$IVx?HGRQ9$&^>+!MANY zPdk%Q$9GHkygKcq_AY!2{IDRobChOict zGrRQ;7NL?oSFV?O*(dsk5V3X@N2n7dtrxgEH`_%-++Gm6~x1*J$>qCR~P}!JeGl z*qYW`s)nbh<#m}Y*8oD&!ail&lQ%N{dRfGtCc*G1dC#m%Xl=Ipd&P% zj2zOi84U_E!zKd|C;S;eBRTDvex1}X#&)0uhi^;KBXuj7+(!E& zQ+8~~lk1zk4LlMiKmvb!V`n0XkkcMepi6H|2z7Xm;(ebCAR|Z!96imeu!Y17YK(+g zm*v;|Dn9L5-*)>T17vEbD0{ixx$j;+l=<|i$x)}wGqw$<8Wm}Rcy36|8xr~ zLFuv~Z+k8vbjvVYLaItljS`SIKS(7OFx2zJyT%GGqvL1!Ne=8gKVy2IClIMp2+Ls& zfMU;tojbL7AAN&L&3+!wFi}og?JM&I!FeQNhxL;k=o`DOF3IaS&toQ>_)Y}yKr1OA z5_hig_xE>6=WQd9>G%}+3uEZ)I1Che{kgU|bZ%N#sURU{ChG)=oX?NaLv@|4pagkX z?ByGSkBQ?O6mQj`?F(U5VLIMg%45_qYX8@dbmvFvH}4h6A#$q@X%=cv;`?VqCAb$1 z{`5%?q7)&v?!nGW%hN*Eo2Cug6~3dMqz(gfxuDy_*=)e&IX`ZnG;_br ztPL&3&Ag=B_uDOPCQq1utp1k$`fq<~I4Q34IBT;r^_C`?R`+%4+Ucj2r_Q=6f?|y6LZsm?Qjxf;f9336~hX3gg@4pQy@0^Lh}c-eEO?mjvRFjZC>w*n9(b#h=oFAU#G?!^K9=0! z!?dirPx3u;WKC2(@5v>y{>k`#1Hvf>>rZ+1cl@F~|I*pn%;~7&N&TAn#IfCJ|nctgOf1(nENr5$d+CKPzdg5#(Ys;(mO~8WOv0IAOOY3KQ};SuplRox(s?eB zzTm=Xz0Mk*Ej>9I^2>kTk<&#cdz1ylE#*A0y{LK5m5tmBQe@8LuN_uC`Mew-saiAi z&D9_BpMD~3AN24z81V9P+eIoh6EM&BUwC)L+Nz9!cMR39QKcW+zyHzXzpm;X!4op< z$&q1mHQ_{$Mq#3@!g1+G4<-jJ1FwCsFE+fw0L$}NUs0h&P$O=BQ#j|Juef1c5_tgQdDzqEbq}}s$grv$55BTm zgN1T+Y2Wf<?3%!d9!ydFmg`ui=HJ~J9*_~x{MC1 zo^QwAXW|;>Vc%eU-DApW;xYTsI=VoWnnzfkket2;Sn26+!(DBd<8*laR-b!K8hU%y z)>J0-n|~_~e6hbPq(xjT7YISeEt*Vf!9j+@_l(84L;S>x@-jM3126(VgL zN2w0Zm>>{)-ts#O%43{9cK@~g@<~T1;iM{Gwy$pCx@{OT-h7hhJr92su|JKrq*aIy zQ^WmL6e3n|SBVwIhLZ>0UHpf90rJ(j*SU||;mJ+}6L)5sVios49$7i+=5F&2C)l67 zHFHy_=Y22Z*C|~rLhBaF-lCX(5$i}<#`9v2jgsB4YvGx>YpYmyUDffpWPJ`Topf^k ziEE>bIR)hS{W4S7db$#)vtWnII+LK>hG{_5u<8_(-`Sg0R$(}#r z3(u8jrlIOydd1*Sfm7}KiEN-lvGG^B&t=lD{Zx-!l~W7rqFuE~4DFkwIajx?5=L(6 zsW%j!)}Tj^nD^zZ_HJrG0EYp~iS83kic`95O9$Txqem+C`=LoHHxZ03rT~$cm4*>b zM{oanJ^yXHpq@>KzBH5SBFyu?F$*w*(|AJhst#Aj6DV9W2HZ>s4v5*BB#V@0#x>B+ z-oVcJ)8g1i^h(?U_*lp*$#e;x8dPy(a&IanOA*A&b0p+Ey?3^<(!EJq5Q4Y9@&Zn> z%wvyd4yk&z%_o!}I^tGyaM!{s`K;A9$WJEEKQ^-FU9xErMi@)LDT|Og6Be)W*q$>B z6RV00nm4rYo(2YOIr!na=wzej&)NmG=N1<>F?k+$ju+~{k~;oqV-NdXW#oC0j>^MA z0JUb|t?55wJ3RNJmig1saog5rvbnF`u`AtU3|}v44C|B1lKrhYr@JeEa@LqPFg#8R z7EDaFO@5aNITtL$@sjJ_#iWt|-AP?3eS7!tnAZ1k@*2ky{d|WHE$$ThZ%gjpweF`p zw%UH47S1I$clmplR-$t5b|kbu2qN8+V8JJ|(i{ z*S!6WS3YasrFF9Cx`i#Xkad zZPDwr3UAIrg6a-f>`qQJ!4lbIvIV?S+Q?Q@CzOU_94$bxWXXb&)iXA9Fz5emPQ|Pd z#zz%J9ol`xspfUC?o?@y49_ZQorfWzagoo5JF#abUK0mzv5l7%=9e_C@Qjr;2sKBNP=In@(leZ_d@d)u_fY8}^9xBg00O zoMITUp(HhC#vYdeelLf}3YO3%f-0)jEb|_RPxS9CizsNWvdYZD-Z0p|$hTgT$brjm z&G=P9P>j9dJ0xft6=7X811;Z+ST##(0em;rZSy@#Q7b-qN91aGONriYL&UbKqf-(t zV@}||1VvX|Gp7jxOEKJ%fcKZ2l5=`fZ?@Fcgex)j`)#;+0^P(>!pW8#Qa>0PTgP=De>m$!Wqv~r-D!Y|l|V1WSi~twf|Cv!)a^zPKzH_5!Nwj(COxTM`&EaK zrCGy%yFCdQeJSI_i}(KAhqYgRYu}t5M8mxYZMAPKfVeW7uh_c( z9BYZKV>#~y_%@ZgBY01BLAN~8O{5kM6Le^yHm| z22S$kavIky%4~3>>PM}TQhKj>wMlQd=i!7KR|ediGJc&j-PR_#J(Op}KkMvWuDucY z=l@!{{a>oh{|i4HSzpd0ai(NXXS)S_J?L(q;3@9vhU-Tp%&PfV=Hp668Aq&)CrK3{ zNb7CmShDebgqGMMJb0F~>JaDU4yU3bmbz)Te}&mbM7KA6$xhu6)GDlG{f zB5z;*rvS-(7Iyp>B~;KUfBHdz@zyl`Qt0FX~y*BFbdyx-$ zJ@n%KJ|CYsc`<)5rUgL?`$`oNb~hoYkOU#6Y$o>?dm`IBZ}A^rTLkPpWA>@R+xrU? z<|Glq$W#^jhUM_%fe<+^n|hv%98wgY)+@~|=5P=w@gW#_FS20yg?ddTnCOkQTu}T< z%UBZd?+$qu>rWloyAMZIOsh!R#rRi-cN)ul;_(g+JAkDOGyauh^VdTgFok5srzPA< zuJLsEY@_k*iXi15O24+x&cd{FDlF2CixU zM27vxhIq_t%ooLRe>6$^vh94ou;sT6;CCO7*tQD^w3JzrTX&EmY%Lq)H+>d%=FgR1 z>G#xr*51qyXu)5nIn@|Z)%~#W+^fs&RzBHh{PxzYr4*)yi=Q(kH3T4aHbgLYZ}`3A zyM?iyCA!br?|t)WeV9ebk&v7LVN1}|sK{q)cCUJI`YzG~YGbS1B_gSlqF~CBOD*yi z{C3a0MXSVSbvH6hI|Wb2LrTruuk6dkTH)XlXcwJWLx3&c4BLq!woHgzkZw%E$&{rFy#U-TNy z9N2`>ml@*l5GGNNBc#YOpnmGal>@8T(|dkyG9*&v_3&(5-F@0J~w?I$0;eHTaUo*$YRhnGC#I~ae}Va0R=w!1m>+ryyS`m)TGhk6~m|Czakvg~|E zwlP^gUq-;Tu%Faf;siDF#06{U&%9{BtDu6u9;u-F_xBSjw zH*cf@SwA!#79ayoX+!KDKTh6erI9wsVd3e*pq10!_4@Q?wQForB-BTNv^k8!QhF6+ zGg2vZzyKzjRjol>XDVC!_zlN-g`A#{3ijpB)OfoEYa0(338_;~)6%O?jfdVY)IbII zJoCpPO5NE0>qO7GhrCkzkPUN4n(;&)aFB&+2L}hXRXj1en7QBSgnB=flmf|u*tSJ?PrcG@-!x74(*o13^pxnV3uPpy0$OBMYAP^LxCE~-?Lq%F5l@@>AF&=l3h`iDtnIeIrt%u4qv3E;xa z)~0K`g`wkEnYi2Ugpfy*;L5WXq=I1Ow0~XBNc063!InK{r=PO2XkUM0-TQQFs=cAa zYfMEtPDFLRIn{HVxVAH7=~oq-*6xr{*CRG=1aytf{(zj3^e(OYy|~^On?y-)aNv;4 z-v&NQyAtd3_K9us^wzHD#|KX;(@FSM8-g~;aZrVG8l`t!@nEgp zHm_69`!%7Yg|A>#|n+tr`h=Eh}8Dfto(= zb`I(#Z| z2tvJzc32olJ2GJ0`9;GO_?3T|o|>op_}-Sxt)507X3_q!iL^!@@)L<&l(*;WI2#`x z!@nHz=6I$4{jbBz9<-WEUjz(fmVMn-5KeJ{NUIS&NP@{Xu`==`%zg8!$4 zqGR3suasnFqpgAECr0l3RE_8~U;n@QhySjo_wWDCYUryUI}i{Bi`vm=#t13N#|^sI zKdHK~_i+w~@F_jbVD)LVe@O)(M5&M)$1k^;r+|C{CPl1}BGr0nU43tkcF%~3nDsB$ zQq~nSD=L%HeiyN8+1e@=NuRIUEj9x*vK#Q0=uklN%aVF2l;&<^hNSxe+eHs3eKBCy zaY1sWN}wd%!)q7FBU2J3T~hJCx@=8BB^VvnZ%y^Pc5IRz1LoPUd2^s+A=$)JN~(5C z)B6I27#9YNmm}al^{?7aD91q7Df>KT%L4LS9r0UP({h=id7vZFG#Q>T`J|9)T9IN%s)Qis62-tkGiHUpth$fonv#aKPM}w zHS~z*Zr@#9c`sOH{50G@x?zLz44?Ub_)ryl(+TE#PT`j$uW4g3Cd-*-!?#;w*_oKv{%^ijfsjV(HmC(|A@mN%v{wi3Sn%gr8e9bf} zLW#k&N!)1U+BzUYropG!PhJp+FsqR2C6!2Z_UhvxIG)(Jiw&jGq^8Kjv*X6&Dh7{Yg_HoinIm5PE+t}_O=OpZzpA>FOsp3I_ntJaVsJl3A3IPW3Z%TBZ+p(`G3P-~#A zT6OCqzdyagLMafZ&!xWR26R`)GmoqI)~sz)yWixhIs-N+LB-k?OjfR^HbPAduCrucGUjEE zwa0Mg9Fh_LsEY(qQpwA0VqD-S=!pdIjX^>NyLDG0=HARX{Ab1}=Pnw01d6cH=D^_K zVu5-ABzB65g8U)v5Q^F1tR?i-PdD^vX;GJ~>O_`gmT| z+lExB-wPc=y!1J?^&=>Wgi2R#JVQVWk-@436BaDLHN9C45OV?DM~|5^`F;5l=~WUq z8b2@W6aFfRQ85i0sIw*QF3VPHbGWKX`{O#(9qhY#ny;Wh>o@bye*wjH5x7YD zW`VF9wW}n=xL^(w^h@CS`bC}u2Dy;e!}YC#M$NCgPNYqVv%$k{KGlyupC#XPA&-ji z$s_`H;}}gWhXgfR9Y*cl)5XPQ$51bK3<4Tj$rj2sl#As2=y4?ADuUVPC$3vMuvq&M zpw2O0#*6v?OKH?y>G?U|@dw*!({`(N3|!!-5x%?8>ci}P={bdlKxO=POT;>I)~bDv zB|(tS5)GFmlUe{^*Q zNkTM~go39DF(arp53v{Fevv_)BSBDHSkE{qEk~Qaw~-qk3fYxgB+3c! z`yr6PQy-iIb0`K=%J}}P_5nbtcBPSiDjHu`>2)EPTMW|F1Y@iM1EwfC zS|CT^?z0|)J0zQ&0B84TUe-Nl|8YXB77I*kQg+TCO4H@~xIq~}MG4K1)~x^?w;-Nr z$Wc5JQUeFQI|o@Z|F?U~^A?^h-RRfw(?Z#^@-O&HC&%)yxA@KW-jT}d* zb7i?Qv`i1rbh;o8(Wf!(6-r-E1?P`T+*xSNF58CMo{4+e-l0rhh2bVGRMPwp1Vu~p zT!IS+B~^_R{Wofz;X>NVQAG8c4DLjgwo1+pkiHsdS^?QReta^TR>d_QY&vMjaKmq& zNwW8K3AFwEf{#Qo0mP`U+#W-?s98`7ct00WS%d7~E7MjUI{6GOAUKC~Ao&1vjcfLmqcr#8 z(9Msr$1b;_)An|X^_1rmWM|&MENl?x6p2Vc&?4q_l%FFmDPaO2YC`*Rs8$7G=ymqF z?Zxny_Z-S{M#^RrDRIIL#~!2GE%}j+lT?3e6`03T@=o!urqwgv@_V}B>&kwu5Fq$M z{N=}OeTS5NW%};>Otc@-M*+_(=EtXY^#aki2x++S4^WpgfDlF;wt0@x$|5&VP;*HB zqdWKXbG3YnkSrZHGF;en&aJu=Q51h)V$-A>r|Q*z>BDrw5TVKBRSL2sk2qrM7b9oU zrHjuGWmO90*Um0m2Y&}&YL$C;bL)Qb$sJY|Fy@Qw#2&Gq_UT#0O5j-d*o2?#<-o_^yA|teYNt>Pq>G=xa(YchX($CXuw4_arZ6X|1N`@uakUqRX;sC zc7Mhz>4d88m8}VuhC_&~+;`l_sTA9b{LpLIwf^Ar%MxBmYW2LkWc0w>a}LZLDSbm5 z3xl;sCA~_y=KgOw|or+ zRk5J{mP4L~3)hN{>`MJsQ~ZQuwM%Zdxk-%%o8{NhL6HcRy4#(t^qrvtfjvBMq5L zQz&(*B0oWtV1)h!v)c@Vsh{{J!yIXW06PcJvM-jf*|^={J}`&Vgr|}a%3?BSAmwpX z0R^p+-i;K654mwT%&t$9VQ;S5CjY7@brwe^g;V1=txBgPO={lk`&TUbFa86%_G|VN zLX+S-Ww{%z%oiwyPd(zq%2P69s1SRv^$GzD2cHA{t=9ZIM%=Kq#b4)@+-!AACN z^Mk?K^|uiYy{7PtICpNS+WqSHU%4dY+NVhc4CdHZ$_ygQ`E#syQITkzpWixm8leUK zq;FX`?b58vt9L#u_OSf;I^H(9>c;HV4m&EcI!rD3*6?wxr~VdE%CJ~p>xM0*enQ7$ zQhFkK#0X9Qc{dD&x|BBdF6^FkS7->%%401mA8*tsflr+H;S8Suk2y|phRfk;V48xC z&Zp#6YeZQETMQiM5n8%c@bZ$K?+ab#TK{YsemL^?JcswOo|8`6p|t_Aw$jMUZIoUM zT0vE`;5C28I=i1LCC7;_GzJ%pSgDuX5zRzU>{bDoGPaRa_5$IWyfBAZTLwtzh(&rN z$U3Z5@U0giLNw^;hY67hQYb5iNu_5lcmu6U?h`DA3n)0gZ|(ko3xkG~ohx@|g%p|M zdY2rdk~7xbMYW@Za_pyLL;7?U)^8!i^cxWHKQ~16nD04u>df=?bOJJYaNAXL5!13$ zYZe`=GHH})y4>alI;r(rw-NUkZ}nk$XX3&1))4_0z6%52In8iYh0_8OiE5Js0^5(n z2L-FJn}wK*&kp6*Y+cXJ64FCwdfo(}S`}$le~Bm)C!N&Si?d!oqBDra6zTs_Zw+BH zMi0!V4e*AI+A(;^O#4YQ)~ix6mdLT!wtouVv41W}RWq{er#=jhbW84a1%M#o>Z_w# zSBINX#1}0~1lhvh7uNISt?%Uex$mSeqqZ=TEhQmd=Be(QAFsyz;71Q#%tJGn;l_Qv zt2un?(MEaZBE{lA^|roir2&-crXIRRvQ-Ueaiat(-y+Zshw0umUE0ayDs+JQyah|4 zoQ04SGKb>i2+Q@l~0DkQ}a%^&ooK`$sFEK~}2SgDd)?7bszODK6 zeMF0slalU9i=kK*8*bn~?OoPn$dkQ|TN|k;qcZ?nCRgbHA+h{WsJpqQAlbBL37SFc zE1YMf6PIpd9=ZPiO1DWwcYH3uARv~%N-748blZ02RKyu@-wqWXye0wHT;MF^%*dte z;0OXbOMR7uHOlSpeBS-hF{oq!A7eIdZ-8~$zbEea>q)c2mRxSl3JXs#mpwO) zE7H=;+ZRvXRpH`t!*tcMs9F4wwFT^I(&U0(t5HZ!o*s)!TS%R9Iyz#m0+`Ba1R~NA z{-y6@c^O+G0<0}^d-t~Zxy=;wNOm<(Wzt(Vu2+7E^aEi1y{OryCKmrFAMxLt-2dNz z(`bQG9?k!EW$}MOz3_j&;Q#jL9ApOUGo12P;PQyA>>-_is}Pn(6j~r1wxcDixa_pO zI5TAU>XmECcjYTpY11_?f0n`>9neY<*V>L&bY7ww5ou6NY`nP;9zx5qd(#BYM9myH zM64@@{(=U^F9f{G^C}MOM~S3L6-rk0&J>{;q7o6duQ+t6~(iK4kjQKqz*>B65zF*^6dMfzM`E5MUTN}V!RBeLYA znu?>+U>%|e!I|FMMFPy;lq2A4s#MXIado9^il-ndl0qv%qMUE7mBEZ}1e`&HG8i#u zmVoRRs%Zmut3=R#{=@<9QpZ9Lh0>^!-d*b5GUNl@X^_m9fSiSMOwZm*>8z91KVsd; zZ-;xY&^SZdJ3Z;wStJjtRItuHC>;!Q1rofF{fe*e3gjQfL=zFE)-R&I*@^JJTUBkP zq;KD&hVRrScG6U-P-}i1Q7l88__!iz#=u*X-kz$es~_4K8-donyIrMcqu*SkIy-gX zKo^(%szURFuWOGy?s&+@{=m`4wY|DX%Dm$CdcEO+*h-hQ>Wbly5?e?89=yHw>}Ho2 zNt2(g&U;i=`6&K=aL*g7mN=B9wS2bvdqD}Q;LO_}C@9MBIEOL;Li^O!S)$_Sn{|Om z%^Q@s`$N4=S^2#Uo5(e4%#>gDNoo~bkvRX@{Y_~rolZ_FYxMp|}_^S$?Njc5|r>DlU64rK%OR3yHOt=yTl(#iYAs<}eL%S$~Kh#s- z?Wq5`PtwbA+TZJ3`PIfND~oqcD%+H`@@SRvm%*1k&ztTIi2DcPgK-)<=z06fQR=xZ zTvB5ta}g4g7JIvv`;`A#+dHSeq7Ja6tP^TGAbZfgj$<^ue$^76yWtwO5cJwNguYyVdph@+Y#YZ9LP-5*!8dBtPS6UR8=+ST?J^PsA6;L@1a7X+Z>?Q}HOn*9u9IVL3Wi$s17^X@D zT%1+j`x%IJ_Eu34D@$~ew7SFO$&`5LB;IxubmHQnlePsA*Co#gd#o@&AtQ8hIxBoZzq|HuR$IX z?U8v=X4oz^tfICD+_Wzd*F@adega1;?8AU=3~FL7Q-hs9db6Q_ws|4H4ak@RAr0B{O;GVR6>R{UtrcHcH7 zAWnuJUrcfnxE;uRplS8PrpG~53IQg1d#LnxitDDFxv^Fd^jRQot1;3~b$Mz=3 zm^57N=q99nj^0AQYbrR2V0pTFKR-GJs&yQ`%YOBF2qC#E@vNAKG)(jXK)Aws3PX9R zrxpm6tO~GUB=pI>DF<(?0{1!&YC4l}Da1mLXhvD8Y-wS3VnT&)wS^#+4 zzz$ z8b#G8cqlZr$nP6>96l74xWSSU`n!cbi#dGqiU6Tl)MqnpHjlkR7-l&eercj-9n&6} zfq=5It>;4+ieS?iNVMX)68J5!jiEdKJ+ZqJzE^PD1k*Q;!fI2wyG8ghrq}tmc@UZ}vs^&ueeWEBPDO691BD?WV_P+lz!g!_f+~qtR^V#$_TMB< zJUscgOnG9$uz+^B%jiavuGT%K!d?mJG9GwG-j=}F;@)9WP7w?8O{Gb~eQ3$uoGoF% zD#;L74;9=E5`gKnXIDA6OD_uV%~g19ObPoKkr5BKV7 zWl`IIcv~uoO_GBbsvz@#=97Y^F8;l~c-&--2_BaRTgN*M6c^0hCCL%Ur9~;Q@7E5F z3lf0>spqJLa?FK<5t58RvvB*Rb*N2)`{;SE5;Pku_c)q?0u(-x0|&un3f?OOiQoc% zelxm&z|nJ@wZ2B|Ti}YIV4#M%MN$HbLrPYzK<1>|ZqPhQ7T@@fv^L*s&skk#MTKc+ z@1WN$bnyt(+OZ*Khdr*1Y~=!OtvCjd;NpA}QX+L?&+j7_NE*f4*$)+v)11CDXaj7U*%qA2JmGOntIU(h{vr!JX)k#MVr zAr(*>?`5ax3{S(CNT@D?|1y-jtg8G8p=mEAglyKis7MSa@{tpAM^q2pDo-y%i6-2$>vDK{-#)BF6!(DccDn=b|)^{Hz zyZAN~P8ImWV2R@-cQLqW#DI|^v0%oq>4^fE2_Tu9Z@`na z7dnFPcm7T3oyvJ-pD{!I%E!;#B4iaw!y^R0s!{W1yes+4Vv)=;7ThAVWzgN~5A{fF zSFmt}8jL?!z*RCtki`0Q#TTRqiiB4pB+kV(zXwpi7sjc4^U!vQ9l5p@ug_mq3w^Fp zN#V4W@kgpub+@npj9AWyDiq1Rl%Fr|-Fe(DsI$xZ+9N?`?O&oo7VAWY$S=v-CNeUc z;nziY8qzZ%&jr=&qi48-o0<^VwB{xrsp}MiQb-yD6(U)dge2dKGbh%E zg8Nnxa8C6b_Y0qWRlra#SLx;?#S)60bGMvUwH~%|nLV;`@xP$Wp26T3 z+lx77NZFOsB3?Gb?b<*uu4@~C4TI{B+oI?Jr%00R!P?$H-_)2L9rVxAk z7rYsB7HP23YnG7{6s~SNgs##-SM!WUwBEYYj->D9cWD0ypE9E z8w0TXvuu2rrU=K*oz-IAgB%ORYd*1SEwM^C*5f5U7FJqM3myHeI9QCg%Bj>Co)tokFoFWv$YT0PU?d5$uZ%6 zFsum#UT~lG-h(7NLUqM-_%MJ7E(9{$ODsgW2ii5yj7*gZdkPpz=Wg%d~{ zeXN>OGs~QZ*|qB6hk8OQN*yL{OCMP+>uah_(G>}JpIYlOdI7y{xY#=3Vs%?r$$Ovc z?IJ`DVuH$SR1GeZhq!HKJee!#QwF)?T*_n2tVu3Bxh$SxKYm#jMegzoqN$1A*hpv* z(ah)EH7kA1O-VlkmXd~lbx2K#^4ZA*nF7{L@&N1KKo0a_AAyT5nEzZ5x# zM4#Mh)IyWc)VInoYGeyM@r_f=n)rvOq;IYO$+VTfKqW5?9p7D!y=RrU%pToP>!3GN*p(^ zEf9~e{UVi)I&g8&N=hx!P&(7hDo7s=+33-JhN2q~ebrE@K{d{Js#4I5`s4U1%C_j8 z!FLXofdk-ZM^|pHcR_>lv|BusnR4q1V$+7gGoi$dAMtsDEryTr`psWcTG_SGt@%)M z6J+H0#fClLYa7zUc!~&RJ~XO4P*Dmf#5dV}H<(>P$gHiY5a4ztAuqRx`vcgdQ2fl$ zg*9ZggFNU>LlreRoScSDC>9iX^x}Py3fT*`O)6w5p6u2SFt;*TJ0xS#=JBnDQf)~| zd0hHvzKRvF(uY_=iWhFoR*qwQZx>NOrqaN~Usyqm$G}m$#q`AQ@iVE76y+GivMY9l zO?G3)B;Nj|Qv7|S!I&+T6i`%~V;O$6v31+Xi0aFOJ$@$uWphih6o90q6BU++m+|`T zR{!H!O{;`^Y`QQi2xm}(M(OQod%%>q^DVIqxVYlEM zO_1UYS#UD%V8}6t5!6f$QpL$0R@JOEFI`S?CJMTv?{BX>H1It>Rq0@4riRU!y_Q5` zlh=3hYX&0V8K;@gbb!+&?c7xK`KD&q%k#mz8g1c;DNJ(u^L!GbxB z#G#z!ApH>W#f7(bG$*FG!TJlD5As2oL!Q0!4RS1#T9A0|RFP0&xu8Fpx0(?H61z)k zT6tWglL`VDwi~7^hkY2MH(W}X4wlasgSBiEfurSng9uyVoo{tE#G2(KEEsZ(bk@0i zlq8-jZ06qrj`&`RAp9^*J@z@vFpp2Dt{3Q73dffws+El!8?L7TqgE!Y+a~))h>W>T zv40&g03E-ck*k-JbdB(8MOMVh?Od=9X!hQW0I(rM)4}zo=Qnrp| zL{ZG;7R}r`D!S^R2Kf`lhH!tv^K#ZmU>`jIGVA&L?FyEIh${m0+gq4_M2qd~GD5t| z{I#nH>;XXqptm2^)&m4Vs6yG<#W*s1F0thzdqXm@rH8J0|5OYR=%55YvCAO;FAqus0PU}A%=}^F#ecI zVe%$*Wwv=|VPoX*RzL1v$K#gNVsbuZ=Y79KPb`tYdzmEplVC7W7!jQk*%_AnXGMlE zeTNarO^DOhoU1xQ7^0jR(rgQpY@o_ntVk#gMWY@HDLVQKUkn2vS=@ak^Vj*T3~g1do)h0r@TA`p4elHcI z%vkU#zPSru1;BhTbx*;(ZCK~Bz}}L|pSQ^*#G`t^ctwOXH(Pa#R36-(`8hF$TOo3w8)?(=DS!a61}Bu=f{07 zZC`cC#H8*c`|Cs=b1UoCNasSOoVVAgx$bTP6_TJHg4hWdOdIMf%4t@~c5p&FljV}l zx8d$mrLr0)msSM`aGNk|4@Hd(@{-z0LW_uaiIG`AyX6U0LN3bp6AdDG z>aq<-Bsy%IyS{~eDU&OLv_!3>#M3c89SSvb2e--`3@`OEs zMC0W8(|!XL6Njw)f=~c7^i!crPS$^6k}gI7F*}NKSH54My`zB?vls^wZ{4h1@S>@7 z&FpjOKVr5V<@54aYJLSQdZ9_^TV`Wql=2nr~O02_dXWG)FsgB`1v3VIz7oP{q zb7{7^o1R(-4YzX2&1ur@*UwzV>MTd9ql|z-K9}%k*(#QMF{Zc3fo}mHiUpG7VT6s? z7k9z9?RM zQB4*J$0S=Eux2>;%$#eldBPViio8=U-wTvqMqP2Sho5g7c$ep7#QYDpS@J|>qJ}uv zr5P2mqu$DVx|ZJ3S&&y81`On34yCF0LLn+HQW4qaW=b1-oTj+|_CLB!TQI^S=U!hC zfC5{`t(oU5!R@~*1mk#1_DUCi!r4q(H|tQ}EJP631^vayg>%Rm8Fp|z;)mSRdPM1J>h#PNJc z3&Aa9`j5;wqnON)T3jWBBGVT`FTL*|FA4~K9P{M%omiAVoGN>sXdCbHyFk2mXZrM} z;dUy0UsS6+H1>_NNbl@#oubN=;CWD}MS5#_C-TY|tEraSSQuv*&Zvs4=TMN-^Wp&f zCf@5fM|6eMW-`hF4ebgMNRX85Ox6f$Mne%ONRAjVDKGkA;gmNf$n$HQ)~8EuHM1Pt znftZSprt#1J1e8yq8^jSmxx4-=BOXHK3+OBN#1NOdF@Ldto`cXnveB;@?T41qW2U7MzXV$AV^fDFj2Yi0$#720dA2%w(eV z_q#w~G?M)b+fYw*$BL%Zn17M!6u?z|fL0eunJ4oYKyu|(2$d%Wd(Nk zL}KUS&6aH$BIsNrbwf$4GhO!pkw0Wd^oV6UHuCN$ROLmPw$|PWq;iZcis3th5|Q-W+pE>~b;NgB$$Cg{$J$<2xEt29T~9G|hoZo-5DD3~#zy|0>ndQ9wx7t&E-tezKY>=D{6;P}AraO{J%S7C6o3 z63c}E^^>P3uNA-#mzo^&x(7Ap#E(PoFoK`F;kf}|FO{d{uQKMzx4IQL!$v09j2oEe zy!Mq)#HkuKDGi3wP>EPx#!kXKR>P`f3C>Bzcc$!2BSQRk>*%0-88RTX5z)~?=9joF zl|n}&m|vs29V5X3a+_#gG}WYwKFc(PO=l@thN3GlefZqJbK1D z_buBGQtL1`BG4T%XAzpG)$*p;+;MI-LIy~l5MQ8-CZgYM_0DU#S{^Wg#a|^`NjGfb zX5=V@b|%r){zo%Y{6M^7W@Q+4SgpEM2!O}bEBo&D#<7fq`xD|0@6 zrsMVxt@D?pv81s!p$S?Xhp|XVq0`t>bvrWd2v2cW@B@0%a zcd1R3si88dLBr1Kv1#>v(Y+x&Fc6JwZsn__jHDM*Em1x-TAf)?ZHgY^Rps(w%4To5 z+~~dw?V^#EzeQ3Nd4Jo6~ zPC;2#F3xCyD_X4_t7zeomc9F~tf_c5^OGZVF4#s$iIQEN+#jAmFk!uZKDPUMUCYnu|7n{;{p}#N|t6yqDYp zvX?VgXUVS!rUSDFnscEts0Cj$agoS^HTBDQeZshQWE_Bmb=ixG+${{b0<+{coiPQ# z?pvRRzwq<3y$l_1gbrM(to8+Qf;5`Iy9o#NY&aRq&QC0vp6`F}KbZXr#EVH}?q5+u=$CE?TvF|D)< zNih;liV&%*w1h0N!ALQwF-kOxBD$!Str8(rmQq2ose#>;c2+XYN?mqZgtE}^km4bh z_WOSjblGKh;>O(f`M&S-Jpa>YrozgBth3KpDhx+b^?_*K8fjj}t{F{k%i&BF-7m_K zS+96y>36jXLw80OS(VonoF`oM=AHh6 zIA>m|5nL?=CRmb?TjA4U0*?A?j#ZR+mzuJ62AOW(l@C6{SrVAKQWt1=U%H1??2Rmg z_@EH$dlvf98+dxzOR&bt99`yk~z5$e}$yZkYpGOh2GlBO%DD%3frIeWsa}|SK$Je9nXdxDzX{KYlkB| z*-Z<9m15(x5ha@8e9Zs8Wv}lX$NFS=`*J=fUdKkZpK=rG45iP`*_31->3jCGwgUZ1 zwqHH*S%cQ^z}(e+-X39U+R%(Zw&@ggOi`GFtal?#4+zwgbdo7(m|N2&#D|rmEHG*4 zY&3dsMvmNXMFVum#ya547W6>#5@NPJ3J$YyfD1d6yiEye5}X7rF;`k^OK-+{H~Z9t z3+y+tR7uNA6j`p~eI_cfT0OVsQ%Pe+R$V$^r5a`3@BT_m)HOaJMuUUuA*GtBI?1pi zD9sWIpqm)G=?J%ciBnK|7;m^BSh*o@>SB~L2IOhtm){K4Bc zCW1zaTiS7j`s7z|%{`Ku>6cgu4zZWfFh^O$n@Ggf_bK!hd285dQU&d3s8+~@nt#qv z`Ao%WzN9&ALE@M2!!DCF#Vz4=8QXr$nDwABEt-VEAKuUhX-L%!q|wZ6aK+Iv6$J+Hl=KOV2O)@Oa{?S9|)bzj$c9_Mi!=jnI&pvDpw9u|tC zmh9c5u1!%)>nUoH3ljtWWZUGjjGYb|53RTjV-eSl zgA^)PR4@pOiu%WRCY_2EW_sx9;M6;N?apMzkc60{N_4FppY8d(KYwtY9r}`Yd<~N) zwdTKn9=dL5bfnb(`{%d0aU$J+e{PK^`;SoN|NfXdz2|+->;L_er2O;v532O`(@XN4!w(#yQY_WKKA_Q{FPhHZlsmH4vUEq{Lz}b)baa=lCLG#9QhC4 z9BwkY($V}(bm+| z{2I({S3Pv-kXZEH{TtN$IM<&#e?Iu}(I*f0-rlfz`}SH}tL^98Jf1w+M8!A`eC(@_ z-xoaaN^Jdl`c21DDC&1#-6HXmS??d~@88cFE_Q;JTDko^-;VWX)+SC5P7HVb`txU3 zna`5p?mI?3a?ujzyj^#4qJ%)8oP?GS`XfBzb&H)r+(Gl*v=w-P@iwGoin1*)rR?85cT6Mm2UiohQeZ800%# z_4C_%HEcd~XyL^>s4kq1)Ik0n$0$L3r7aL?S_T+PWJ?kBS? z*8gbEP}{R_UxU2N#*G^@T<^;}j%3}(vSJ~P-6CbI>+{=gZ#-0P7aJSv65Ms z#Rp*;kIh!iPhDRp?rW)(b^NY_e?RroL)BmG%6)~)_|hz^PukW~nWoiQ`4U-{O$VKS z_h~)&e&h8S&2GaYR}t3LJ9P`rn$Ep`ow4>#wf7QU8K!v6Fzpv#WUO15PM`dxiK(vy`$gUkrZ@55I13I(sqX=BMN%iC)K2Pnu3||K3*@ zXHnf-weT|NRoI<|+C+49b(2E3Z{NQ|y1&2b z`orHjS%OTskJ7=`+>XJ)ARM_E>(-p-7k{o!Nlndu+-0Jj ztkXY#^X5%ihdy>H#&vRJqBB79>(EdzB8_FXHFYCA+=pZB?ym?`lt)XpWt^HXo3KIt z)`xL%w%u=*>la>Ho^Dv^dH?=>?GtKhYBl$^EpB%k=U*YF>tRwAY+m@~p{UZ>+U3DG zZ+_C>x=$v(oBA_gJKA$#<*HReA|lB#W%J*@{h9gl?A+3Cc5>@DnaNqkD|z|;Jv$8z zjqJrCzSTRODeP80C3ilB%VRns^WR=C%d&1=fjvuk9K0_dt8$Iu#-4T0ugoh-8034k z=3a<1c;WQiZCVB~A?(TEho04I<>Xc!N!WiiAYie_Z>{Z!F+P<+ym1JT&f`P;SVFJr zJCa@49nbB$Sr7)ik0c&UPL-6r(mOJhN85iyc>lev%+%p*i=)4XKJ#+=(Ip;=-hKEm zd(mVM@1sYLNKlzHoI6r*K?rF|R7ZY@O9eYZ$Lh>BZBJic-&1GLZlKsLJ|qe4yfU{_ zqkD^&w>NV`!hvP&&B@17@3Ajm9%ib)m|KF2VnTFfpZeXumSV>Z-Wgp|vo3e-`Ws_* z&)OBMZgToR!+kEMys#Tsl9Q7yM$JZZvH}7EoXL}KJ?@yajDtgnPu516iJ6(>&Nduf zjcL;oPkLHP#mLC0tJS+H&0raB@k2tP`|Qd04GqiANT0}9ZLnTx%*4>JPMN6HoiI9?O%XesLGhr&7pHn>ZL&>pL z*z<;BemRZ@eX@?IuAbhLN93p#xThg3c>4MIQObRdPwpT>uS;0JapU6IsXxmorVSf5 zSZJ(&ZRw&i;qWg8UR}G8=DX|q8xn^bj~iiYG2_@&=GpgVM%~`F$;rv7Im>(*&sH-J zzhxWO$jYu%nwwY~b^m^2&v9X{=4?xD>J(z*(~OLg*S9y?_PrNMF)Ci2kdTn~IYl4$ zQG%zUp`M+6tIZ=Wv~Hcc>pn{?D;BV<@bb9L?3819c{w?T!NI{N-{z$p7XI4Rbqzal z?S>5uD^{#9t9wX~O~-&EnW3+#$#TB!`PFl^4@k=2ymhO35YM*vX$Cuj*6v6OXsxYX ziWIEu;II`Laro`_cC*=4rsdJMXUBf`ANld=8DmjV5#@1teCYIWXN9KA?qEJ<>SXtk z?NgO6F8(AZ7Z(=Vwr%adXL$BS$`@ z>TwUYWM8{_Ri&shdKn@eJ;ll|&vofi-j&}xTh6i~{DdH_2?+`cHvMs}X=&k6^I?7M zIQjMKDcc9rcxDr91&h(_^Pf+e@E<#NtnXtg@4;xrAVj9q{q^yA!HG!?PD(!B-W`A~ zOp6z*Sy_p_2^G194Ep50<&MPOBWE)*GM>9mN6#khx4R#t)c zcU&w58rr{Q^X8Yq{5xzqO6kk1s!T?E-s!~eYOTo>hryPkt^-4O8y2I_^Yi^L{4TgMZ@A~cf#p_KR&-P; zmfk+g?QXZzBd?8z9$fEMbie$nq2b8>#VjnGeEj@Jd+fqA<4dt0>Ah8BZTYz+^w}+% z(o&ryTJV|{BW>yEo&WWM$GK1Do=?@u!3H~dMRAo~ ze!GPJ^BaSFMYzU$d84y?_wHS|F49R@0SPuCN^`=my6wUuq>JTd$J?(gj^B6Zn8<~5 z=Qg%Iw_SU@^_EcAU=7V8hNM|B{Ejz}c5-T>ex6-vteWqK@M`Os7EMh}&cMJxJXS>j z_qLvfG3)-oTt>W)djs-WH*as>+go$@b~kTOOH01v6G}PBu<%ma!24`q$JMJ>dlnWJ zR*pV2Z%8P;#<(=}&9||r=q=}h@j_CD<)?=AQs~`^xRzJAJo0KYthTprGwu2LQ*x`> zd*%2j#OSB^J+bzu<9KYI?dMaR3XySb92~g56u8_T?2WUWbQI_Enm1Vb!f8+xaWVL| z=V}ol$?1&euWCnk2n+9-2^{;`o__1WaHU&SF#ooXH*KTaanSkFjan}U23n`-Bx?r; zA8=f|>RVO2ob|`Z#@>6j^Gas#1%n(}d1dxff*(>&Y-z7PH^KFP3YGQY;(BD`F?o6* z0e%I}nVbsm`}X>_E%xZ6goM(!Zn zD*j;RTVLPx8#mZx<>X>dJUdIWDPE=r4ohclySp0T%x0I7Ghud&YF-7tF$xmxUG7E@J-fNylP z015n(#WULLmr|*tH2rHe90oov!Mmj>Mn=Z0^PhRJ&z?IDtZ4iEIm517Q+@YtV_eiC zx1zCed25dKwfOk;B-iVnI3Xk~%(U@fl(Mt4?9|v`z{{7)fP>$h^)i>OTIC1KC?qDv zs-4XBIp0wYl@Q0WW!HdRzap%xls?1M+1a_-=|{7xpON173t!Z=wQtE@urnk<9UzL9 zlvisHId&&LJ6B7)P>r~`xfNXc!;+Skh6>Na+nXNfhz%u*)qwe@%=MHY?gfdmqGzx# z>dkqjFNK%6b1$^-#&>I_8!C{f$l^^~Qc_~$=;$31vQ*xo&jVY>KU(WxZ6d;^th{_D z&=}=GG5`<{>OT!LGvS$;nXkZ#2sGJoAGtnEs=D01W9f1}_yy~#D@zggmf)^__ct(Z zv;3%zRR|VP3P#qo+!?LpI^h8vBDixWA5wJRm%<&Q$J1E7Rj+bO8h%AQdG659X>4qK zINzb4cCu|`pS!_goY!GM)|chwbXVpk&#CyPW_@WJs;WKe?EHMRlp;|xq?ugczkzu-&ynLl~%6TfIs zy3tZu<_)_|_z{dN+ONz9VNa};kvUdS`_6K*M)A(ykU3NNmr%%i>N;sRGc#ICd88N5 z@nF5)X_-XJwB%TCnsXocC?0l@`NEfi5}*i5c)J;*@QUKOsZD8yh5oo~+}XGHCV7s# zUk?uU@%3c{Ufw9N4|sok=rcjfDF!c&#UClf{cb+rCX0|pmT`Lg=ka?zU%>$E>V0>= zV_+aag4m}vH5UUoQ5UoJIQXw&fUu=B((Au0yZ69qR7 z4iI^7ijd>FF713`=;(&He{UY0B%@ zFB?Ky;02WVFW>a5O5h6hYI1Vxg)ckxk->3L!4p(I+Iw3kp4n)cv z6&>j76Lx2w$iTV?;UM(%^q{WP>JQw&M2p~}m9YP=SnjNex0_gZZ?;YSqrKOIf-102 zj>;cf@u=X3hPA7I?>G`R4TaydNbK%wNL-2<*H^%Of`*j-adB%A%P2uN3*P{4NiaD} zi)Y7p5s4zZ{#XK147O|=9HD_VaS$mTGb%AXC0@lSsnAT4N3fLa!*ED4z>$2dE!m4q1b)c_#fZO zai6_J<8e4{_mZ$If+Z*~+?UFLqv^5t4ck*rJlkzOiM#T5QGAeT+>&KZOU0^svy@_o zH-;CxP3Jtn;(6^_cCa4%-1bKUH`i)xtyH);)W+ACe9&;LHAyQji|3MVq$>c*vi%1R z&{L(Q-ix>O@Jbnbp$uq945=Rp>}cB;E+zu7oh07+s856w-u0i=dPLIMy|&+I60JI^(i5o**_ih7qxdCFmxxw(1g zi06YMq4a4j-sqM|HxOu2IoXy?XYY=WbXOCa;b_il)L{s*+G!^5dsds^Wp_>?45Y~% za&;|uYF4)*<3R z*xFDvz!ld;Lnw;&XlkM&+j}+T$(h%;2#Epwqj|EDivqEtwU-x2d8dLx#^v$zVn#)L zmoDuZ$objTQL*KW`;k$AOjK4J^H*@&)Jsqa+lrQd#5(Id)?u!VR;ohU|8Q`T&G?W3 zAo?0nQ5IBOoq9*`)YOgcRu57=oUP70t}_;ipxW8hMf0dwCH=Jr&7!9le;N*_7KAwV zy;tda#ffJwLn^9>bscR$#znjlyt!v9o12xTWdM?6*ZWvCl7j$z5^g2NbcXo*(^9Fv z{CD=OBTQp`CUW9V*9mrn;WF$G+h1Q*kO!1N%{}_)cqZH^ZXMNMCV1|-eJ>jjY!KeL z=>14is{tx-@bU(LHR=Rm$}_O(NCGPILTC5SW&dkYGbeQ~`30otJl$VI<(+ zBEi|W-4hee@i&dLI(m9G6xq&yzS!RF&_HpuO==fjas+-6J}yNgY*urF-0kPvi&7hZ zGqec*FcDEo^9IjD>i9F#t5Eq0-F|Jyf8}7eqmo6}5&7bu?ZuM8`aR#iJsLjw_^>dc zu)+D!P`@g;jbd!ODsO1UgWdH;83^<*1yE~8h9?3pUXu)k@yR440Xq94$W|5ea0 zCNmSm5!)}UMx?BJcJ5FVQcaZ+yZKtI9;H*{H9j_0iK8TV`}Jl}8)0@Rl**Bn5WTJ< z6vrHjR-9a|XK2R5vvYLkx7V*_&zZYRI zB%lUB>ci7FMTED?Ic?#JpB?*JvWW8V^J50t$E-ZtyNZK@h61#`-;(FJZE)Y$E7My> zi)tiQR8%Mru$HL%wM~i=4n%fhe;Ln(EZ3P?Rdq9G$q0^47yA&3=iSxe;kQ7}K}4wXyT?_~WuksgiL&06*-ws*2pEw+^)y zJk81~2a>VVmyn#60ws^SuMD~28GqV5GP>McZYi@4U904Cx*#^2V0wv;Plf`vnZC zEE?81{j(T!8{?(lePly?v#jz0%ofDHq)wTd#)syZRIyRYpE6A|tXpNWyREIQacmcJ zbKhChB+}6(SM%(4Udc}sHnNv~o8_2F%R5m0qu-r=e&I(Qw|U#PU8vCx<6I(taDvjJ zJh19rh>VAYxp)S)Zrxg=)h4h>b6UJ>%Yi+6UV_q1IePc`bJ;LSJ-d9#(SQWD9Rk&R zYF&4pId*vXoNH{t`Ow=@k54??utGxr`p=?iHq@h~SOzWpb7xZ|_ub;nDf)StEt7p$ ze@m`Do}!DxI6Pd3*Vwgq#eoYQmYEmY6{B`sl+ZBE(t2_%Ro_wbm8R1)ScN)So39iV zvpa~#3}pi9dDN<`e(Ih?pK8fsrZNi-n%}Y&yB8VRdx_VRpl-Lk>p^+5J7gh?!xFmQfUmt?}I zn&Eg?Z)aVcMjsFuh=}(fM}Uk}Osei?Wo=EtexNo(Kgb@p`~2c!Rg|3lQ{<}|s8l4+ zKAWMPF#$VxW=MUkvtlul!`}5F$OPJq*HYw_&DSQ5Z<3qT-!ebw@8?&i+Y5HBqJ2wK zSks6=^`P5g1cAEVz`q)po%v8(9^{7zh08K&`gxv1pI>wVQ1kfic`E|#Yxp9HzTt5% zFS_ByUWm%(d`{cSuFTK+qRfm>X7`V*iS7p)Ih<$LZB>op@Z^l!>s#wVjd)+Xwg`zm z&gr=xF0<2RWSvpx(Zr;|+`!roV7YE@I8gsMFz3Q7w@;vM=5)Q%+QO$MwS9X+kCt6L zcxH9W3=XUO&(Kq-fTc~}e0>|a9rbNNrR&`tLT9$I?5*tmZOfSQgH=G04@a*?YnAnx zZBx4q&00c0gm?Zqjirx}Hucy11UwaB#MZmJ@{a$BK}G{+7M4;(2s?whYRO_TZM6dK zgaZBQQM)$N53$K*$9+=L>YxR^%rL0}(M-Q|=~4&|C8(!>21<}ENnmnzcK+7KCy6YP z-lTf&+&QYBKlWIVqgL%jg_W-SJDkdq2XEfE;X!I5t+?GMs~(|FLogAVa|aPeG@^m2QPtMzuqgL5 z(9|A~00o&knFPL{S*?(aeLHz<&LppJ0g6mTWh^MdTC0h;0K`u81wf6u!pj$<@Pdux zH&}hnUrxoOr-wid@T(?OQ*$$_a@|v?9#4EEm;}5#-{r9%B`B1ldm|hw+S>SJWMmLS zlm|YhK8xC}heUpXcF)f}56$O3Lp|$}dzbR9%DrJs+|+ucG0lKidCE0!w6~U9BT!h( z#jQ);M3fZB1umnz2cMQx1iAnlXmBUg40rq19JzgFTb1o;i!TLoP&+wRtVobL)HB%h z+w_{g1h;Q^ZNEE@qN{W8$iTgX*$CSpQS5>iNuR4J=~^vOhgKP}dV~4|e#v`K8Y_rO z#I#jw(ak9;Ctgbl$Y}`)38ZMl*tP+4`{G-Xwb|-^oJmKY zte{=A$o9|oqx+*34Xld-4}QjLJ7dbx$G4@FQWAt}58apd#+mNk7j+LW5KJj`V>W8A<(s{oo#)M)9L6Rl8CT}r z19`S-M!tQuz2$ts27Y-54;@jkm>!UadfwfCa%-cswDyd2bzPm*cu9|HC9mJ12Rlnh zDppXs#Bkxlh2e43l0C_3GtxF+l|i0%0$n(R?d`mUsM4w|4E?b?euk24kc9>}=3STkb5>_?zBZJ~$F> z=}RYgp=qRm@8YVauc4t_KiZ205P6ThwugE~M@MJIf5-=Qw=r~Q-OMw4!!K-o5*J`1 zBH2{>WA|l*E*5|&V7#ZNN@%H9O?9VP?3B(5r}d5_X?1|KOG*+>EtX}hPaJ;rT#0;q zYW!!qOmt*q^|{fTsNVxk zOObd=0eso{`BSAIYeCoqAH*i;<`py}5$(eJA9lrcL7usJtTLE?6U7d{L4u4M1Tb2P z#^e3tqnn5VE+;C<w~t;( zLgJuUu6e_T)M^En5p^+c9-hacJ^lS63F|j(cq|?r5h2_&JTjuKN%-o(pdi$1Kxao4 z+gX7pN+E}MBdpn;h}#`RN2wyOdjb2$&mbgnKp3s~^X`Gdjw;)88;i$UZ>`^lk3jGs zM0Z9(&ScU2++`HO57n7FV0qvfRdAg+4BgPbJ`)@?6XgYJ$U-a^EFz7^*RNl146H;x z7p2%4SXai}8p9D~rs$xeK!8>g5eE&`_aXTf^fp{< zcY}t9f|?K}NpA-R2Mzjq_i5d{-DSj1ce?K3E>iq@0nPO|8_3CB08_GQXgXCd_wl3N zv6By)x}bu%<7P~JN&}QI^zdwzp#2h32(S7l7hAf38|PtqeXdN7w20! zC?bl=Ch1_Mt+AO|glSO$Ux3S}kczzr-(3KgRRWkz32wFgxE~PobzQ#nkP<{jFYrxI z9!1O9?=H|V!;T96zJhM>rxcwEJA^Tjp<+k?EJ8v;7Nf7b?K(jIEgzNNv13zwZ`G`( zp;8k<`0$AsY$?t0-=tp1NEs@URC@D&CChJvcTy1zBc#T!kr4uRiINi$8M&!Dz|T+X z(h)@|DGu<0SHU$y*}oYc&LB8@ARqj>$lIgeJ3DDr*iW54O|x-hqLs;GyeBk*-3LQTZW@fQCN&B@M<>T8bSLn1AX9?mnS#WUc3}qEXbYEO~={5jfmZ79rD-Fbjl^MB0nP`%J=Zg9Tl@Z{NOx%j4uS z<&Z;uAr0-1lH~xm`Zl*#y>;cYOxv#AKp2DvhrL7<%v;^l(`v&9azRv5FHt6CMh(0W zT)A>(3@+LXEsQv%kOI!$HHtl)6c}?eDk{*!g95NU=(@-q9C~MOsDR9cFDVZ0=U6qs z4k^StBhZGxk0TnkO-7~?8I528!sj99C}T4cje%T3-jT&FkMHBhkIk8;0hKFps25W! zcRJe?S>H+f=A@Kk*~A7jN;yi(g++!GM^r>rRa%m@xwyO#<<~&$9lVsS5Pd8~_bM;~ zAJFz2o5q0y9WV+MIYq*3lP+ro>Y*C(V7G8GUZ^-q5Y(d~x z-~xhSno>Y9w6i}()Igq(u92)VHXJk23U3{qMg&D9Sw?t;WF3DvYffS5c$&Jtee=S< z2#W)|@fS^WE_MYc4xLmLsz%*&TNAL{>FKh&Jd%?|@JJC|*43{SNLEFS{nWnqAlZ1> zr@K*V)+K4JCH51nDBMm^Uo&O-y`YPH|CstDr>Jf2esxG~HVh=XFPy|tRr_Nv#zBhf z`}|@%XkMc40wPi`!F4vyEvdGgWR0$0fCXBy^|pyjM718E$f+7 zOM*jhlvG#mLj@FHsSB~!8#a+-S9b#9&PdG0Hcu^x?&k7tG~zFbU9Y^!?$#LA;C0WwNoqCRe@Oxdl;PNzaV7Wqztk8@T-x9z~e zwWwZR8QG3#vYcXqtSwq);uE-i>9c9XY=0YY(qaRz-tJ+BCRT4Td!mqk0Gy9b>dB+l zdAMW3gEi^Oh>B{yZ)zepSed&$;2MxNG_Pgt-)1Kgaau}IGHf$6qF0%=N4(Jjmy>YH z%q&Uf3SJMfW5i2Gf8WFGA~$zLTf6*swEL7u)xf*jo1An);{Xz?FOL6g=O!3M`y=o% zvc@&%CG!(ob~^uxX|Y?Sgsjgwhb+(M`Y`{_9k||F0z+>!nP{=w6$=DpwmoQvDePoe z^~8xhyZDwr?S+?*!`9$kX*V`DRtja>uF#DXZ%!lMUW@&V-tz7Rq}C_=C4%WN(G1ux zr&DPza(e-{VVZVjP0fB&FW(u7`Q)}-!aetUs@!GX?tEHKf&hZ#geC$AAAt>@jlcOy2gTSo_6OV#&2Z)`ZipS4L` zdex`;Cl&s0#2^>izV1%6N+r+&h`aZmgFR{ajgc+^qrc4NFw6?Z5f>&7GKiUARlFsOCRDp3!&i|Y zhx>SleL8_M%bx49)-8c)jy>~y3OjcaI}9)e96oW#6hvPSZ)$=!qZiS7F>uJv%VXrq zs8f{z$qpwT%22WXL;`zc>_#-|}m+InZ^d>j9oJg>IwbWvEb<(sL}id}gM#U>^u!#Et4g{vJe{64pWo!tZ3z&~z# zn7AJ7YFWy#CS@D?Zus=sgqX0dLKf;m2}iyNFW|I^$u(?W)5*XaO~v+;vw0iVt@}DN zIm*Pw<|D|IbS#wzfrJI9FZFw^_2i-M1$t`__kaA%Ngl|CY(!EHEawAe$Ja8VaoNy)-Zu;PA{`g>86acn0t|Ph}LuG1(Rav_KNgYIUG3- z4lSv=Y|G;gV7CAwUV;c9M&uF5knlIrL9;$nrnr0u00*g>+^70C8u+S4%dHNGmi%JE zCGln9+;$&$SNUK)LN|gp-zRu6AYA!^(KhhaenhrE&;=aL_tMma-eiD-Avg8(*|P>m z;oo<=GA3NQ6n1xX{LKMhY^E3%uJgbCN0dFP=>H8n_y13O&Hw*5N3z8K_Cv3FsELN; z5H|_3`H%!bF0lc4);r4n`&h*rIBPjpuDp(O8lw@iI-gA=vA({Z=={{nf0{GD20r}g zF4B-nlL{_(^E|PagJ^3K{TV89awD%Nqylk&TUwGO#3z(sMCVBqQ3sgvl|(W%R6pZ`={2BGh{f6zt)|cw9w}qj}{` z(hH$z0==LTHD}hF9UvF}=Y3Dnful2PTn(7PD7#}v9O3ramy_py{5MKa6f)<#%m$hr z>&SADzc~q^2S!}MITURL?z24bhEx(Ga7V{gAiPen6HpXBoPLBy*ZD`X(d>zS5DUFC zpADXyg&!6_(Dy4O4O3*^e)_a(*@peZ)NxL|8p!KwJ1Oy}Z1>{uyW@>^#Z4@DagS+%4aOXYiYZ zTZFKiJE7?zri0RDfOb{kvYO~k`0>9s+*M@)A16tZw6wHadloNQLX0x1YHA$Z+>EGn zU=-lPoqn`XfI>ucO2Vmq`*ubrMVAAAz8f3Oj?Z3Oo0HT7Ae41ao-{_3o513L5eY(` zgQ?I^R=3?o_WnlT(}OJdk!9~L{WM(B7%vU z7^#*#$|FKN7)aIec~U0E#zcVAFL36-Ru*kNl4EsZ2GEh{kec^oq|P~L5_cx4B;gH< zmnYn0Z}gSN9}Zx<;72Ev*^{Cyj1+C`?TMi)WXHv!Bymf0PM~5n`=L)dGJsy0QL`MK zPWke1*J3_-2PUlkN70Yi#1A3#J-_rvRP^X$FW8Gwmv^G7kahkgi1m&iF)e}y)a-uv z+1Udy>p=M`1D(ML8xjm)(ndM6SW8*kPC8Pf5MvUo70UYhQ4wEIrDhx{h3UtCSHKD( zIk^+v&tNgH`)&_YwL2WCMmU|eT6|zKavi3p2vP)@m2^D)f$k8T#m1oe3ONKv7rCRBFDNLW zC{XBJcdE+Ezk*B@*2@tiI0of7IX^hQ#1#)SZV)J-GEf+UdK{lBSSe!tLbZ)ocAay* zCaJEI4#(2M!)=3(AoO5Fj9%be2n<>o9i5$?AOI<4ymOs;H&<7i-go>To*Z9_GG~`J z%kr4R!r17nAPtBs4>7|xJzbJCPw2=b@!p+1>x1_U?Y>b@sq%|bOlI~UR{)Pem-!Ym8|6x;+E&cnuH2Kij` zbJLO#{DM$gCVv?3C}SiQFR|(&YU$Ju;637jp6BI>Kx;EoCPxTJ9^$}s6*fJTJ#t!L57H7D zz?b0`Q3)JhFAjnTg}_U!`BRHVQM9IMiMT--BhD+NhPjEZrNrZ6rT2;y5AkfY_fTcT zST>~*&owwvo6d4(s3_O_S4oOPh7iK#NK;Y6hwLqZE2UJ|7lPi>9PT!4-O`I+;y?Ic zCo@X2_$)y6fN9gd_p!}+_8VDQS<8RJBDWao*qgCu(-oPm^XU}vK zz^{QYL~Xz`fd5IQl+D_nXVj0NuJm`DBaC_VA<)8;f*TlVzMd6@X4The6(V|6Yc>=x1*Z3*-f)V~dL`)TSx(*!dzl^LZ zPP}({xq5atnz(QiPJ=D$B?A2YIoEOeowBstM0zqP!NAoDCzJJ&0>DN?PqX7nOt7vk zSOP5>$0ed=1e=VLVnTVT3d>H*PSW}`9+sPXF}M?As;Jf9{-T9M|4~+3wNjWF%xeXd zRGLR2tTC?vWxm4YjR;RfJ&eu8gv9uiW8qxc@gQCQ}Q z3fKPn^^|h=o3L$Zm*Mq}*y$pz=sFQ&`{G}MWx2h7LzyT2;?QuA@|}oQBybT&CpHdo z2_Wlxwa7z~A;J;aVKOo+i0Mc0;Os9b!0R?{M0jIYp4#Goo(bv{RHMA$*>b;S6L7(h zFo&K+qKaB*DALupGLQZEv3|FOe)rUW)-nKfaQ*Kkcj*}z z9-3B%6G{+EX#1mCD*iVR$qYOY69ehR$ep3xm%bEVkt|{Y=w%WtG%3!`c1-J3#M;lz zj2?pRRLrnoCA!GoNMGqW$h=D0{U;&~WDl*}^9G0jg6NI`_k8Weiyd?-9@*L3Kvu`; z7@f~mcm{W=x!CQxckjUk2DxNvl5J`6H#26f;O5>qcmC|z1SzAQtQu44q|0i8&yX95 z{`>s$xCBDmdw9NVxx7Gk`OT;4X3(BLe||xt2k-;62hS-jxFpR#gCHHjd@LI{kEjA` zK_JqZ@`l?q6j3J#6coYBDUm6J*icXbog5{K79BPr;eCKNpOXn&kgBF1=yr#w(BN$XYsJh}__m$DeD*~47Ia`NdS5gMLrPMT$wFsw{Ylsb;7#D+ z`0zR^Ja7L&hab&aKSDQO9MX^9zH*_lcJtP)Ua(T33@QH&EDly_az!upAOqz(*?ngM zM8oMbXC7r1A^vrO^vm<#X2RR(`0|z5liI)q?20dY-PS*kd~2NO1$VBW(VKaM$%xIT zJvBcOlebaLx$t4r{Dhk<*;1|(Ht7Gr)>^W1n+4Z5yF6L(Yw}Qr)$y8#(N%=F7z-7b zC;>RkTJf68ss2HB@Ebmnk-U{50+KmfT-(r<(3&O==g0nPww`3Nm62?(V&Y13~VeSmt2(sSqJjTwSK=zVeESlGaweE?!nG zEiLD%-y+nhj#6)n9qT}TtwAky+?IB}bvIO>AQ!az4nyqHPybhuBt13V-@!x?i{MXe z2D%%H^w;twyJ^f^8Qw*(bRa)M!LD(%0#dGmQ3)o%UudFwSy8dRdS=xiEB2fyjC{(z zZ2UkI9q5+=mL|oXhs6#x!yE&>maHvj!gdo({kId{JQXb#Bsfy1aG-fc<#y~?L=l<{ z5#XDJ0kV}hq8w24Q>Vd`I9v|!BL3i_nMptW73@CBiIDnC*1NqP}s~Cqi33!v9?LVrx$kbo13r&P!#dafXx&x+=j;Sa0vr{@S%&S zv+ttr4*VhA8R+()Jm7eyIecF1)yzEF=@vm5fN9yRB`aC`0oZlFQJpls+=KCTFVRGS zin-P?iXv@p1P@X~wMXqH9Ixd3i-M!W;K4`IGKn9Z2sjBcj4N2Eexv4G>(*oYt5*&R zLWYTYBMAQl37i!cjA^j;xarZy(|5Iq{n9XGwP zC(?l$aiPWa?I7#D79o!U>lZordnUjK(qKW#T3F1qr*IaDUwmgOS&+^BrqK&Nvn!;M z0aoq83$5s^2pIlZJWm{QXapq$8L%-Gb7g++#Jk?4AOyD#oJwR$?)m7uFbn|=mj7PJ z5=V#ZMPOZocao5oIJ65*8Gl^>gbKim0|LPdE+<|C{sH__U6I>m^nS;ou<2_rR)S6x zf-br!SzB5x_Fo6ft3UF8Bf|BOBdGpJsh%Z<_D2+uUg+c3*475beF~)<)I@-bywi{C z;0{XQ^#CmNf#ZQk#_BpaJJ=XVTO^`_PJdQ(2YA2=ES?R#21MS&Vbb`A^igmC>6hXH zNJyabs-1R(OB%7=-R}d=o)o_~s=Ep6r%hy$53tUlsKsh%KtVrkY562H5zhDtv~g89 z`Gjbg)Pz_!?M^+J{nVyIRZRamJL2|sR6y_H{>1%uL2VU_b*>k?e5d!$I|rg_WnWC<-l%$8>UkQ-w;~Agq4-JS-*nOHgagD5F(DD#QxEey%-eq8+3q!FCu*yE9LCgAp;Pvu~aIEd6H$(5m@t| z0TeoCNJ>4Cscv!!@igQ7XL%HD8s>S33aDV}(v6hR@h5-^2q}AcL-EYWK3tR$i+fBj z9di$0(;zJ{Ur~4x_a9=NTKrDJD8tm9{Usb{P#Ky7^aWydMFo%`?TH$bpX7GsSz#v9 zi3x^=WRe+y>VX`~0i+Qh@b*Z(B(~fBW}wVIrmP}}G`M9CiEDf59J|&a)YP5p00BZI zSn3qs8|qA};5Ql?8eRBR=iyFzinL0iR}Cs(e9H_-b<(Yc84~aYr&tXfg2xf~mxh98 zy_|R}MhG8(6{V+ub9qR$uG@|WA|aL(?Ien}J(uF+<0IN4!jZa$MjSr=G&i@_u?7%R z7QNA{6Vok{41w45yI-H=dR|4(QQ1(4FDj0n6;Uk)}77^_M-mrh#19lRj zS;7hceAQ2#5+rU|Xg{#xc74)=1F6)DSu;)u{rz9jCr5cGx=khRr(ssM64)#)T?*7a z;?|n|$)^VK))uN_f$OApR1T`oa2$yR_c6JEFiMdeM=XCo%m2-6k{f3iLe9Svz3>m= z-v8(T|LafxpPXw#V)9{#J*2l7VY~}A6sUF3p;cfT8i}9#^z2+2d|{N$w^tmYqDMuI zzW^r9%4JSO?~({+ivF`*`AKv0pJghs@i z3jgJf{4D>jMInjMVM}2Z-#ZFVO2D81YXIa{MhdhUfho=--^pYK?J7f<&E6gSe>)zFW?ll7-?~O+0V4kIABrbb#b{{67dhi}<~9``v}v?}%3zzE3)d;NXK!UhY1j%o%bV|e0_ zlF1OZ^4{~A0Ja2RuvmMxS64*aAe6F*uRFBiDzg%4&TlL7)R<>lp_0T1YscBtVu zYt+GiU`+8}xkMf(G>Cuza3EEX0Z&<2JRY$5_f`H&oHT_?QA`Ljfmq(N5q<#F#c5>= zgG(?TB}Ffn4MwF2%%=I(dZAb(K0X0#0J^*APU~cC zm1qP71QbMj0LadKHs3voL~JuRC!`Scs)a?4CgvuX>^czDct)KsU81EhykhazaL}~~ zL2DHiiq>N837TB9y;XDxOM*v))4@1#9sFM-&BErtP1&cEmC+wu#B z+n1j|^MgGtM}1MzO|lofIE~@`b#aVjP7P_L6FtfTiCX081d3|p5dgr&R{ZJ}%&mU~ z1b;;=jw*Z`*T4_GIwKAKwd}(OE;J7`o#yCQx3CZ`Eh{6Y7!Yos~v1}JwL=g>|pDqkN zQkO%u6z`Pp4TKO-oQu&lo7LVt3(UEO%}#pNtLoVMSN!Km8S$iR=u`l;SUi5`{kqO% zyhsm_1sg+sO`Yw(BN2sZGMs1ye!@mIKmVhwN(ul(@baRi6JKEKW)oXvm zW*X4ljgArjF7#+EIvaFaorwQ%>-n~o=(te^Bmx1|^CxDa5Q*aI~c0`~i#$IBX#XB(Y z1$bEakLZX~RWCx>W0?vy%oYHf4;U*s* ziWFo*I~nJdZ+0%h)x8^nuFq+RhLyrXk6yA8z#N*thcW4C{G&(LYj@#rpS;u1leZCs z9-hpFIYih9<3IwcE+fi(?9P(=0>g`)1DhLR>HXJj+O#QtzPj5E?(h|0nsx#Fkp_sx zix-oOF+Da|otZ}X0dS+F83c#mI)Ds0G?*b%bSwkh5#dwu3Sm4_M;#vF1bUn_?|}aF zxO$aF^NvIi9BZ8@FGc@g2EihHl=LD2wv&$fP)VcJ$V-IJMQcaW*hBPwf@SbT7b#kK znQ(?GfS2HAUYy#wksRiS1un=uFd7k$AdChO_|EB}Yl@-W6%%0IzJm4 zQI=pSA|fJ)@D`zTrAV7s`4|YJkiR?h6nd~UYscqjMwb#(0z@!K`7hC~4=ksltn8tw zfE?zDy$djfnIfdEp40SAC>h~}z2*rv>xlu;yTHP1I)4#Im^A002S??}1fjCB>M~3; zz$(XpZ~()7Bxh0*0|UUykxpK?rii%?A@t}7Ce-jirb1J?;3SeUC0FPnN_V@h#WjnIThf0I)z0u0>sq(6Aei{VZ@8Iv3H6Kn3|k zUzuBl$r$WA)}f?01>+pxZ5eh*c?4l>ESn^V`lK{JvjygvMD3rbuPKU5o`TDs$baB8 z;L|z<<%?{=UG)eqwrhK-TWzLsKxD#xJhoFAzAT z?7r+v5z~(*7_Cycv=x~Lf=B=~#V&+2 z0IMoQj9~(#Dl0csuf!Y-7Dh%!FCU*yoDK-16q(yZBssvruUIe@Y*$$1Noh;?0;GeL z3YW54PW!{V?}t*4kS}BuA6lP-(PBVu`JFqkq=CZl#74l*^+DWz;^2-PeHC;NQpY-pNH_vO#}i$cOnSl#V5Dqy zn{kr%nt$;A*DYuEKBnZ{_FiH?=FrmWg=qvAGsY&B#o!1eCBiBbYz~J3!UiLFr$CJG z(BKmxO+pwn_rbAGy+ReRK*Feq6|4fFt?eYpyQ3o?VOxW2QwBzj0OX0uNx%E6$PhN1 z?xiTu;IP-ebTIuNMlA5ONl$2bO3$Rn0z zwy}}C>=_Eh6-GNc28K?wI%-PB7r9PyBEWNNly49n0>UT29G!?{cAREh+9ZxK9F4>d zjyXB;F9aREFumV+vRcxR3JkbyVYI%FzFJR5Th2qe;jikW;2;4Q+~$b>+4ig0D1_fvwX!nh8ux& zTMrElp|3NkxjzPb55uL_W253hbFD-m7jljpq%a&cT&8NHv_bX9%k=1 z!U5n@aHF1>KlUbEd@)$ojKQzSy;Nu`o3xL>P-|{_OYiR#qPP=fABDXw8Lh&`Oh%BP zzCf-m*OkuYCvkWeouf6Q+w|D-9qZ0)-n>~EXpW+w#d1{RaIV06*PhO4L@Lqw4sCgj z=zh{32q;CgCc-oWLIp+0IQ`JAiQFz(HEyiF>nPrk)?*zJ!VlsoE;q6|h!_ud5NURa z!EgilVD0S$NRl=k0$hVIC5KjtCyA-5#0&*0g=A(D9l_i7 zkpP{FT@A{v9AyLzh4V;;wuvW%tlGjrpfxfGdO+hxG8f&rV$BFCgd!^G#W&vw={cOuy2Dxk{X1#%*xN+|u zi{PyLf!M=+(q|s(??A%Ar@#LE3Fx#mW9=V=O44aS4gnG-rjKH<8qmMCi{gY1DON=A zZSbAYQ-mhPj%GtNU35dSNfQJdAT0$cPKEs`n)bpjVS~|ANOrZhtRyW$N+tvzhEMqg zaos(kt4mSf3PZoNOic75CdnBV=l?Nl2?lK3*?4GSpSz6Ps;MGNRrH3j0#v)KqbxKl?T*sIVm!74$G&dEdFi z4bzn`2rkiX^UwL)s8e`ta-Ylj%t1DL{B{vlTEWG&?%>*mL72^;J|NzT&7m>K{HTPY z;%efEL{4pV-+RkI{$?7a{s8`^gv8#?OA*!$R0bJGK%IJZLkRY4%;yjyBS$gb2E~yH z%mxdSyC7PWf`1|2JS1(BOaK7;tfsH~-zL^BBsM&u6CuSY5@LOp3It=a!u2Ccl~q+; z5OPHS{KY(EL>GiZIe(10WI^mB?dv(`Kl?*6^}8@m)8CK<=0$Mp)+}Y-Mxc3rybK~V zk|r34Q`~0V!iAv7a6?h_4`eFb@*+&wTm9wB7t$dE zbWKr!t!kJ7Hert&U&+I>2K61^fvbD&;P{l^+yR}o{`zW)3}{A&5-r6hpcn{Ny%VB> zGfEBU1|6772t=@mg7qfZp#WBXG6!}S87o#eofv$dbAh=c^UnwlfH#&>hF(QI8lT#Z zGzY)vQ^$d01WJ$zO0cjY29~=BZMnT?UBoua6&}AQmu(Og!_lTq!dY!04DeeP61_N;9|hXpwW9|Q&UeN?h)jG zzC3svy@5(#c#L(ru7NdtSSrXw3jvV$b@8wkf4r-qln^F_3|Sj5XPEjtyI#*qkWR%!|0%Y?}|sfs8WvXDsak)G8rw0yHBnRWv)MhcT3ka6VOl z;5P%!xc+H8j#gT~cR3+RvSf+Tv$0M_Tkph)SlE-`N{O|ZnXrvgy0V(OVUFm4v> zPk^lGA80>|5g63Lu8C_Ys7JZ` zYVPmgrZ8oBGoi@;DrQv>;7MaSnUBy}&6Kwu<~NtSzO1V}f#|xh-vL-(La;0!AiS-X zI~lG`)&OV_FpH6l9j$-l7jYgOAKFwXWk7k-SP{r4$BC7#1Yt?+vfwZ@tUn_Jql)&< z$7aNZOmd0C}<%;#e@;W ztP}?aA>0>CY>V_bC8V_oJe~ty(yBkHgQ(!K8d3ibao;!h7mvvcO7$DUKC8sd05NTJU#pv}9b!*(q(%c;oSnw?6${tE3UFk_!SD<@(mDe!jY z31+0K)M#q?z%+Dy<)I_)M?3s{;-&vr$7z4xihI@zAPo-e%%}loAyvSn-M@_)Gp4fE zp_)E!*qD(H4nYslk#2g|u#yRVU=ejT<_Y9QI>oH;Y@ga6)-bGx+#rS497cFZHK z!z4P6t8DL6rgN!JCJ}^JBB+yG9Xt<{ib)`ESD7MR{r2tKqUzgeaT0{J*g|N)y>`{6 zP1P3t__&GQe-f>XO|I;18c?KBHg0Fxx~JtJ)G zrGb`GfDymLjvmx<;x_x`<1-5~zSFJ_lfXJe{{w2{&j+~z0tU;C_O^COud(Piy@x z*k3^68!Sx5jappXR3YMYIq_v9vG-x>j#0V4prmE}(<}By>5}r88ejv~RTIH6;2{T! zAHKW0uQWO(MZZix)XwAK2v)P}&l8lO-UWGe;V$B4zSu0i8VXy6wARpOQK(IvIny6m zKvQVag(c${Pg{XtVhWlx5#b_&6j`SjSO|a}7UmR|N@Cmr&q2`FK@0vM@6%11Wbylb zN$n=~ybMh<7Y;+Bvsr&2{;px#k%c8@9U3_l7Z*Dw)v3Iu zYpE+m^7TnQH(0q?wk1mVB6jJ%o6XT^T}1ii^z@R%c?hCvp+2mJrGd=KdZwRB>>O&` zlTm3DCF9p^ae;(rV%nkYiA5v!Uxj(uyX)1$-u?P(9v0{^F4j}Gf8xn;>*|+Yg269B zxFkN@0C=-YOO}EY_Zf_?y&ZhPEYy2{_=Q?k3%@;g=%iamT}+bnntn=eW$Ib!y{T3g zUeOGE7pxQdRId-u%Qw5`m#?u$`X+YVIcI9EdSBI>+@!L* zVf{whA?seWqY4EWD*cj}{;Xjc_wn=hJ!qTT?O?mlqocjEzJzt`pY82Zb_j$i!_vX4 zkb<$%u1`f1bD7<8%Est+TY8vUZ6{2Lo;%vwZ}#uNKGi)MmNu%qrcX|_D)Np&9fj(m zq86#5$fVKrhNLgS6-w$tu<-kRHV63bw9VGO@6W6+M=tE0cm8DA@fO+f^p!H@fR|fu zYD_#C{_0(ms@b2_N16HJl0O}xqE}pQzh6U4%R4X9<$+Y%wP6d-GxCi3PBtZUABYqI zZi%(jJ;vFj@!p_8?Pq896OgaFZ;y71*HQRLvGas zJH{vH7j{JGSDsECTak&RdZzTc<0!D6A&!asM1<_JhxydkSUxc16_)ccT4Z zM~G8S!AR|O;LP?DFDsA=`_kdY`GY2}ODC@_klRjYP%uj!82sJ)Pt-QPa-6u~PkfleAehwzX_8mCo z?E56`E%8+oOR1X6ORO0;u8C-57|a&5QXx_jfxyXRCMvK_-b8Se(K}ZDH9C?> z_-n##?j`~pi;G(eTnbK5PQeWlagAA*Kt71ARv2L13{tZ$-n~Q(AoY^S zcE||gJzDfDCtMih;IBi_^KdN0ePz*#cjU4 zZ&Zzly$(mlms0m|yA^^gFm7e?f{tnn10R5Olc}L9*6vy>sc|x4?8voS9itqgAN{4} zog9XxpPuPYw3$8_9OX<>X+c4O95fPrcdQ^l^AMHqwbjwCtlagi7tk4Tx@EX1gtD)DzH+phvZNX4YM3{2GCr(_A{yS)=0CSi~|Og}Sw`r!Fy zRovQ_m`yOK6t~f*+L}K99A6WD)^@7-nX8v;<*=>wwF`3#2gz)MU7JEya(xs@6U2Ry z4wcf9&%X|>EVN+nge^%q{r(df`g|X)*@|xif0%6gb7X_AnN3hLE4BK7q%EMQ3hPh1 z0q)dZ*nRNpgTSZ}wMvbIC5xR)30B?_a-8ZmPHf>9vCYH35qOBQd(Ra zo2&GW4{@9ja-0<_qH?Ces!(4^mXnB#eg`^fZ&n*YbqbdVxXDVs6LJpbx*aPs)arE4 zrG<59u{-fV=GQA<%WhI)gKC?5?(liyhB~2gqk%TY1AMO*ja;~WWF982SLg;)4#?t{ z;hJwf_+!xW2hmO(VIUVKHW8F=o3nO+Z;9ZH>5%3?VW5<$(U5#uFY!h^Q3PcrtpU?R zre-_M-E_sMfsQPv>&0E_L}w{nm{*xa@_U>6#oP!y3S;=oYpT@89IwinKIM>65UO8N z@+y{Nh#`2&FaIXpRw7Zp%y19(Rx%-k<#!DQ{ra%z48BQmj-F*rTn`C(=j;54TSZ^m z+bkqzNCsq}_|+^yh925O_>d_;AX&H7J&Jsg|xn+<<5QZgL}}u~byV71{B0-t~VLYF5SDZ#QSQ z^3x%1*wnpkP!v2P=hBIjZT#x*IlFV@Z8~y_a(9cyV&P`;0;oUsW6-fl*ZyqMV^JNJ z2uM_KmPzd9To`6??uAc-sad;L!##z;!&M_M-NfJ{XzR!*`~IGx^Z_e_gM*V!@GxLO zqlR@iR9xosenBlViJIsq#eU_fp(N!QfI9*%jhO~|RMHfF67l%@(xdR9qG*+=IxM?Q zJ|)QL8_msYp9kF4eAUl7>D4KcRx=(S`N2cfD?x3otej$o>HydvO-3oZ0x`6hfTNa} zu9wUL7F`k$vj)0zNUu&c?*~9NiQl%l11%qAA08ZBw|bMVyUpv4gbvy#^sjnYJ+HSejtd+fTqmgkzns_QaSMVIPZ5;xC&<j+KR-WTWq`m>JP69{j0r|~W(r(~PIJ!r=OD4pxR69hiuv*X(;~S#i+X$#{zBOYo<++D!xzUK99VY}Au)277HC;irf0 z>S{3bU{`oc>G8xqCHg#r*jy3N>(+e*#IJhvJ(N=QKQkF2qmwMi{tgaxWOSG_1|UGo zi@r%JZ-uDVpLhkdk8Yt@^83SS0RZ_m0Z0fBmzf#C3dV8tLEeBhRSx}xbgIPE6;K=% zdJo%hl20k^Av9Ldz7ObP_`0nFpS;Et>BH3ZeF*#yYskt4GCdub^_C&X#Xo&If}nN2<}rOa=xO-UT1Ny(d0EjoqZqcqr585E4xq z62PlV4gL7Yct5ta-QPYSih=~Q4a+G6*OD$2W^|p19HKsM{P6Y1kNYWb4R(b!Z~dps z`J!LGa<^ksi{4pQ+c#K}&8SC1_B>&FjB>*uVg3gUl9*MK84|@rnfJndkt48zEgy|D3?La3znm*4l(axK15GF_L_lNTl#h?TraU`t?W$F)ZpW3f(S_NSD+XOGuV^`V zn*u4CoY?>!+BQPhUz#(Lh*^v zdjC}9(tkUSmu=`6v-M}^atd(ziUP}#4k)rir~Z2T)>Y99OP)EgB@{)*)dIKe>~@h+ zp$+H(m;89Pz-D+SIFyRj8kt`PrVH+FXzrQr@IFAHpk|ha8GA3synEXgSelxeCd|ev zphHyEfZVz{1_6#QeC5HLMsskFV|w*Dzx!OFTx$=gcYOHrJ$np zVsU-tIu?e*_9Vem=Y9!qOcN;fuPp4~yy3=mh-xH<#c|&Z8Z~I3Pyj8@soR(AFxSzH zhY?mzffZGh8c?qda;Xt++s_SXKpeu`w4Y}&{8FY@nJTw#CB!(tlDrr(+&~7U$QclI zkzTL`?IW+#AL9PRp=&@epuKnISN)aZ1>aGT{=uY8Th6AE>7TeYRhymG6EzWgk>9tE zvS7;k@4l=w*4i+k-1f7OQOQN>xj#g4B#xbkT2CyAj5~h3oI=JLi+UK%vAEiI8tYK8 zoSr546ptA``#S<^By0M}!Ao{2|US4bQ5=if2BrOP*VN$>S z%!@f{{#ec(Q3&Kgv>Q3`_#c=Zho!G8g+?u}AldKFZ`qeT#G1L0{}hP5ykfse%d!iz zsw7FAw&&QQ0{2b(%$TYDgS)blGYD=jdjH|WZ1mLJm^yFk|5&srcgVaQ>D9$HsBB*P zEK(;T)VHTz^k5X%-^K}|#8;4B)RFSHVkqY)%fOe_lCEfA{#dZ!hh=KRdiB=AO-_dA z-bU#vo&`L?oW7@mnqhzNGCNyycFvH~I1iy8V8&tB)E-op{n;)2(6QzuQxgtkf%0O0&fx?|3V%5B|0m(|aq3D2PhczC0X>T&`I zoE|(<>vyx~%=u&aA2ThGZ~?>EWq3(LNsW~i|LLOKyXE;0>$=?CQIT|p`l55{JA`4~ znZK$!XwaR1UOXstvW5LhdOqPfP@V42CWa8hryO}#Op7R4fA-J|RY5#@i0+0U{rcR; zBI+tDzxjASMJ^4z!pcR(R!*_0S{20u@{|x1B$a^z6TW%KL0=?CA(+JH05JTQZhUt@ zVG)!)jPo1|N+~W=(qR&czVJpeUMi&kc~lI&p`#{J^Ag7v97$bB_K`v6qVw9N@b=s1 zv#0dHkj8puNM$V9Ni;az;E`kFh(=-x!MQOji0up=_45?B@x7R`i{}Bpo#98nxJ=`G zC<=aQp+A5BJ^-E0Lq?Sa1`=j(Gc5+czlQ`KNfu%~;uk0l>;8we=ZNDzT(HcPmCVrS zJ!(B1=<9h<4$nY{Bi7);y2#O1=m%a$-IF$+j;dV1^Rs&pr%HAQ;1Qo)8i)=(``oV^ zCFjbEZLr*dHHLqie7B+)^Am(9YjQc#aYqsZ78znxD^Uear@t&%|h;UvYJ)DUX;U_-7PA=y^8Nz z6@~~HNqg5lXh|~52oxOM8Jd&FOMHFjd5`_UE*P3ya-T^isw(@#Y3Eelltnl9||$CZ)0eng^+RM z*997RSm-w&!qEU1uU^iH%=Ax)|L%RhzK`Ye*pWf$)qOu$>!j!JxW568sW<1i5t&T& zbQPQw+0Ir^_bWQA-w{po{Ct6Au@kNMf=DP$a9ftndb{EUg$*VjUxSTC8zXaUBkRhe2z`cx){*P0b zA0AxKrvSB7X!r=|pk0;=?M;>f1B&2HY%p?i))519^=-8Gv5~Die{xanvKgmLbD0Z{=1EkuFROIG%U;flaD;K(6`T{sb zAwZyX#tLz`WwL(M-?g!Up^n)DV}x#DkNr|ws;hF05N+!oJ~rl1a+VhtE;(c!? zQ|QnCy%Bpe-$5*kD}mSuA-XbvHM~>H6^jdv#G6+UOd3>SGj8WFcBa`J1KOAF9N*Fk z@P0(a1G~Fu7V+2K$MsC)L`b?JT_1WZg#r;m=828>bkYkUY()A=#9%Te6L5=WbO8BY zhMA$FBlJ73v6_P-mTs@^sA96%YWyfxl&NVbHtI8m$1&9pOh2&*EA{MoY249PFj}Gj zAREo%A&yH8tP4&gncANZ~|h#K>+%9U4QJ;2?0 zuzyf}%J=?;FEH>~$5zW(a7LcH!(|Bt!2Z;^y|ylddK=J=r83w?&OmYePBV%FRBZYwx@H-9R4kW~HcsL*doNGKCc|ipHPBII3TQxxu`yW* z#39rm(wZ^&KKJ4z?Xt~JYxU8y(ytG|Y?(P+gMbiHLD1I(*rZ&F3A8(avl9i;a2wo( zuh1k0rMI6mc0g_6pSyK)EADdxI89JcLG{OE-*1h=m`%KvkSt6BqJXQ(SJ7QA^vfL;0oBmvjGY?uczJ zllYor8YreJceIp!dVAmG43pkWrpoC&cH05-pt*IRRInVLT z&@7nJzOHZm#&t3i*)Fo75x*L$||ZrWyl@7i<7(cJMU21r`M$8 zCL`-jGHC19FJZOJD^NVJ2^9fW`M=47)NqxgP?CewXqy~Giy+NvY4MM6`!T!)B~}ei zMIh#yfz~tKR-JpdhSl=g>tyVA#(|qdL(3tlVNS0)*9JS=%QQHK?$zKt_XQm|cu*r! zm&s(h+}$Vq-zINWtN)-C1lAcj?)&!-W2H+4m1?b7pQQ0)vN%^J7AoZXS%t_hotqUeVNz5fYzc` z_JD_rGk3R6j93(qn1AyA&kbj>FXARCB5Ir-rJeol_rbgpJ`&yWFn@>O;#1%nyuVIl9lT-RM=+_h47KVdX zP)J4%v4uaOY=nf8Vh_{qX`?4lSt5OuF(=OWZSMohh4w@yO{jf0gmy-9Se{9qU5p5b z!Hxn!kOUlf1e*^n7pB#xX}?C$E2rS|Wfmg7*(e>Xwa=G*^!5TSTNo~BN$Dy`S{4fc zNOQ0Ec_)0g*d)z}ChN4wypBTf0hkj}X`idA;gC-4{?&NY-sZ1q73H91>isn`C@MoB z$WKa`=-9tRX0jo{HE=fnMo^G|1HmT7XoI^0UT@~(Po@)deQ{Cdmo{;#PV75djBRDb zV{LESCu&9;!llx}t_utWb(U#^t@+bdQ)8>!WkG3y#`s`e?S*PYLW#H;hE{6{V zrxlzt>ayeg*^d{b@T=Q;)G(CfNubjXXiPpZ`txu$b4W)ixAXRgI3-X>J zOhKq4uxi6zz?$0{xAYQ|d>Q2pkgA*xrk)AMPPl>ojZ_#Q@h^0(SbD+54FMoE9^8_G~*y2G1W;ix$0P zeZ;bl9Q^eZ3WcqIsBShLa>F%-aAntj|2mJ+=B1)LH zYc~uP$V1qc6bQtz8Nl{V=>5EGYoMU-XJ`}jfUxA;t^uwq21r#^j5JMM`Tn;o5RWu_ z91R|Jd9h?4m3`0T*pV?KhN$xil9=nL$Gpp3jn^S^F*Ntw<&XZ(!nt? zF*F5-NJ4JNp3*VML$}mI#WUDWPyzom_k4ant@BWtVBIEH=0I*vy}QGavdA8tk$u5A zZr5iIYnxutzMRa&vVVV@F_CZABUb;shz$856?omojbi}$ThT5s0b3U2`l^WNL z@t4@#EgQ75v>XQ;w#W1ID$WVAlaTEecD{P^a<}oG(0#>}G{3-kx|H(r~N6irkX3UhC^?x(?{mcUI zKUeh*U;2GJ=-okXwksW#ZIe%<3$Mp7D?WE+t7+dE9PN7(m4?L{hZPP zaHX&L`AhPq({}_Z?B;-Hp+D!7JZHkfnln5nU240{y4`~gzuzv;!~ z<8KlAKHk`{qr;h5lO#%VPvWjG$C}Y$kj;7@AN1tIr&@I#${KFEv*_!mQ0wq%7am-@ zmbLwH{c;M$OG~Ap{#^^7lJn2Kv#J$*dNF|!y3o?6j-FTy)UoxA=0oc6F_;Zhrc(O}kdEkVWq5Xa9ER0^Lxw+Q1N-? z+Sh|E7R(4X8UE(@@wQ{*2D7G8NVZ2)ofh96!5IpDsBP|le{NCevSk|`wMq@~W!gs!cgt2rl&`AfW^7=)gG6^W2El zXq1Fw;;*Wa#c?ZE9@XN~*TYNMK|vg%AvZSn)w$(~&T3`dwoc>N%=XsS>zOxD?t2(X zSbj^8;bU#VotCng9NLg^a%=kfG3ULHlQg&HcX?f!!66CwAHCPrjadDzU1IU*qQ!$4 z#=lgbw7YEbSY>FM$IyT}s|I##v?HIi~8jTcWYk!ID>q^VpRlVw}8(`p}&W6z2u1n|v^QSu~hoB;)J8 zn>MBJ+=4nyA5oB-oG|GNO=DlWp*-#mhcxfzs!TPhiqkeSwtEo*&M`1TFg?i4&pmwl zxs+yxJ(jq?1aVCU2Wi^P7MQ}EhXq*{q;DzAS(T}Ulkw@#f1DWOmn zK4KH6AeA?|`R3E7qab8PJKDXvWhJE~VuT3bYRG2$xUXQ5d9kTZ$#Qxh_b4qSa<4VtLFah;#fu5-lq6DsIhc%Ik3|kZp~E@TbUr>v#emd{vNm`?(j}B{b zO>za^m+ArnR`U3C3Kniyz;mC{rDNk=evK{)Omw#AN=|V+WL>jyqrj_7M;+V!Y(^a@ zcRN5OlGWDNKWh5BkSsprRjU>jGH5{45})RNqj#kJSlhZ@m3Tw$oU*)1CwF67=J|tf z0xcGBMV|3ZEr${|va_?NF8;vzS~Mbkx#dZS1>`Yq!Vb6Xg?u%zq~m$-*(!VZI0w*r z$-e2Fm%`92X>P728N1Zz?Ec6aQ(~KTpZwZh+%|FXm4P<^Ok7Bct`ypZ^aKpEcTG7o zdidxQ&Qcdqr^1`I`~`)}@O&XE5UEDSmO`Gyu>bQO?77zB#X7KsD4^@U>_o4xT&T|%it<`AO zJ)c2j^bU z2894Hmb@E4c0=aSX2OIQ{rr8ffLY?PbkSq@Mdm@=Sik6kHr(p+GudgK=f`(zZ;p4x^5iM!mqMs9Yru9tn!&~ z@pgLe?(=_t_k0HB2soVtR*(-89K(|P`t4g_c=&{s-(?5}ysg>j@US&&*AjePdEWP` zSLB73mk$UE*t~6U%aTYNwJa>cW1s&zG2u2^0N zZ^;9!@RnoDR6OL7lv^Izvi;_z$`QWG6vDWo*O)64OtgQ{t8&NfgQ*YRwRrdJ>C^XK zU2pB`_H9bSV*4KQ^N6}iB7{AH$ZznghAMlCxK0>~B*c|sE-cMS{0Ta;7(@(lschC1 zdJi&KG`S#}yR^AR6JF%<+-sCf?y=o1HqN^;vE8kS?Ydf2l~>QLpwi)rb3T4FqqHgp z;6Dc$S36KwGB2wt5*jnE!@h^Eze^ZF8iot|BPi$obUF;1jJdSz0vJ}{(Muf=MDm&qt9*LtCF&n#fg4pe(KT@QQ;*J4 zXg;Fyk-)1`BV5{Z`uUz>+5Ahl z-&v#gHx9|9tWNG{-)G#@YyEV+X37O6naLw@OCpcWM?jr@KYRkQW6SB&rRXj; z%`Myyo~~U{c4dY2T&WZ!3=^#fy1;9fjlUaq`!QJWfeygBKM(M^12QI?%sR1gjS?s2I~i(fJ_IE&Iqu>R{R>t2_O2?VQz<5@(AqQM1V!<>ygcqZq&h94rGa7 z*l9YAGiB$_eP#xBpdzN@&Kx^5qLoUgx-O`J)EnhO^i2%(+V8wOwCBa13PTjt*08e6O#{ccGyb zA%Nh>=u3Kdf9^#;dMu?B5J>uLWfab5XzEgtE9LXOL29ubp$;=)?F`+}jB9ioPNHto zc?2O*dcz9Yq_4-+3$e(&K+PvG_c5Fhuh;og*5?(6vKnj}(0|~-J06?196Eo{e@fh# zx_WwT_B}M-_weoeS#q}{$`@q^4Bd8jT6UGsAspK}{LwD zaiyG)Xa*!`{ptL-&CIgY-nB5#n$4#3(N7MjXDe-pD}A}DhB>O+P}GSxCih7{fnIkL zE-8$~JO*Frg!97E?`bY^g{9S|go zn0V^FU1^U?f=4<%qzaE^hWhNea}qh();@Vob@-g0OYuaXK!GrfFjR20O53g$l79%s zb=0~^4fm;rT4uEK3>%)C`So+>dpkPCcv{?*Ohq%`&S?tJKdz@kC-UyE$*#!8V|a9` z?xmkc7ZzpEqQzpaz%%#aiR+b?)6SdiKZQ1eOqK|#iQP}+A@3hefB^qO_+Vqgwv%6GJ%GC=J=-1@!oUn=1R zee0NH-l{2~Pch4M^UPk+W8%FIGen^TXBBt~u$|G|-yQ{0Gt(34=1b)&d++r^HrD|p zWqFcjz?47a(Sut3dR<%AUZQO-h^r6fOf27(PImXb$;VyMY{=izQVZZk_3?hol*0&6 zz$8bqr)dQCvmDaB2DICGSC9?hL$}}0sZym;&HsF1W81k47K9KHx%NZk@U**Ic7?AY z*}=UXL5Py}2BVIM@jJPkR?VCD0dQXUew6i2DsAfLQIx@lHwZZgdm5L&BH^t!ZG z%RHK!$t_UaZ>81^aF>HQrK=OgJh;gG!hE;loSvl*jl!2qp8hSeP3hffCz+uP;}>u2 zW`Lo>d>)`93`(bEMcyD~T~t{;`0(k+r>rAGguOkY;AhD&0{PDuf08K+X~1Q>QSe#x z&JY}fmNIredz`!numn|Q#bCdP=7^n z0Xv${eRC(oRy~hv>i4ea@An-qv?OK0EWea?`KYOEZ}qG_z1eJL zEwW4cW=$hPvt-Vo7ohrJp1K|Vc6i=7-6JETp zdP?I)mTf~ewsaZz?S1QMZqr^gJ~66w++v0v71q8GGZMF@+N9?RduujVvqWARcmb^= zmJS39DWEz%SoqYx`~8z3QF-aHF;oBtz%-Sc3i!ZClM?UTY1Mn-#EH9G*Yo4N-svNq zINXF&3LhVl*=%25-=yhBFKnH-OQbpg3IISp9!(?~Nw0fsV(*2E7NzSQp2g-kc<#XK zuXr&b`iZ`$lhA&LGSJW9*|PmPtEHRhdLR770V0@9_4n3^QYKF&LlgitQ$fq~n8J##;4pqk7yAMUY2Hh@HsGMk#IcW8Kc z__KFBbd{Qtp#}9bkZDHT9v^ye!ycC1U)ixk4nn|as!b+4>gww5W$=?{tx{`xv3K5E z!*A?E1~?Mjpn79wixdj=J4!Za@0fW6)(Z1wR#zw{h_C6utO7pNZeBrtQGcl{y2{|o zsv7?G-mliwZ<=Pt13pKLT={jN-j&Z?X+hP>l&Il3TRf>|%;DCCpi-H6pIUW9+R``a+VxF7;d+ZWs4PAecud{YX)J4AH#EmC!O%Jh2gSNgnjC_@k3r55 z_mxCw(R=K`lGeJl>7^iVX@f5tWJ$2R3{SFbJPmH#h0V6tsk}}KYoapc1Qn``snfq9 z-grj~mr6eoiPAIeHdAGL>En&zRe+l~M&>k0K6v6iem!k7whQw09_Ap4p*ITWT!8+i z9`W>J8THo6#!(@E@`?T>6%uV=3@h3F`T1W{GU#2$a9-o%V7w^2=_e4rQ=#pHjcl6=nkUdCuZAb(^syC$U!+Bn)6=3@XWbWnKV((( zxv%4qacOZszZp~D{Daf!)NW}^tYEPPZ@ih1^Bin(IvrXgt0ms*qdB7s@lUmvcLEjJ zLoxg0Hl6Pw!M9ksQWmoRVq}TEzO^NqLx7*nuF89+U^8_{8h+}Syl3&J;mY>$ORbKAifZ|&M zm%rRFMYCQ`qBbASN^P2PfCks=D4hNwc1dvS(Md-v_4WBRl}-L9DNGfLFc+m^M8|q3 zm7_zu{y4dqjphhtG98puJzsxL>sg?6U*aF+Vx;s!{szz;K@0;&st zV%1GY-!`MmvV;vT4S%S9T)*_L=g*hr*>df!ctvVoT>jai9(8K1z{J-_KXuKcbH>HeoJ+OJ{(yhwf)2Z_zA`a@=1FPN zy!q9{hYGj8zzxd88#mVc*X!x)&6(|`_}LS_M*#DAM01rrVQZ?ny35L(;ow`)V+Oge zE=iYC_LWuyX#d4jKT=(V>6+^5wW* zw!-p0*!m%0i9|YC3+VvLzV2fdpVWX(39YUPav`uG{{Ow|In+WzH~v2^QpH6Yi+bZxBs$O6 z4`$;`l7d$R7iGh!%#pqcnm$&0aW5BUX1-S>?eOneG72%^Bi;+1*W3*@qHRnBGbn0ZJ{P+b_SK3rmwR$2p^w5?|2&g^D-ui)*&Ya^P~t9JlGI^&{r z>qvdntN&~N4`n7$jq8^{j0DJ`C=dFdNU)$++MDg4-n*Q&FAUeC?-zh2vAB)~CNfO_ zPBB)PdoVm7E7}Nq0hP+W0&1#s)FO$c9+SqLT<$d=mlt&esY9wt2RH~3C%JI82OpTZ zzH@W3eR;WqJV2xL8DzZMfkytc&Fb5={#{g;OASnVCxP4=ssLr+D#>@Q;0bi01}5c( zo`gcbQ1E!{B2*Hbfp&>^@)&VUwa7 zYWro)M^*+L3DgBcQ}13vOCy>yWfXhROQq9mj9`W=61Kx2H6}W3RkziKe#$7CDrtRa z0P3L2>GIsWuV7lf_1Z)encb-js71C364baOIxW8Um{BtO;zj@7;+HRyJnd1Y$6Nh= ze=;F15v2OrJ8*_~C+}MgFDd#JXlctXN}-y*@H5lzejTIF#=u?47N5>fS&UM|NF_@c z)4uFiuMCLQDnUF&wDjZoVp~dWK_eU~pgq%0+;}`d10Nzqj{;_iTHOBMs|Z*>Z)ea; zhzJTLE5~qZ;26?`ZOTE04Qx}IT1PY4;l}8h|Dg$(i9dUXNI8-@C4arStXtdSPyenz zg}v{?wEv2Wisc(ko_!2bM08@y*7JjaQSN-`8kFcf=f9Ix2L%{;CBTx~^+&0uAI8RXG52;dYnc4S;NMTA zY^>5*UcTWFb>a=Om-$4xG=W2o@D93FN2>X9Nw_r@^3^Z4k5qjaUew%^pGYW;na@$ZnY z00V+d46WFrcZW}lGH)vJcXZa>?HvblLU91%5#Xw5DrM~6Jqa<#DFyS>rytE_)}p$v zrkY*8kz;H$FV<^qKCIL&jwbk??mGDS$%f%ezHU(Fl8hXPDxTRAkx&^#f*raeE+F&I zI`Kxc^7UJAZeQ626?Zf+ILoZ(UNTA|;_;d1zYJKpFtc{85l5Xq9P6g}{acvtYJOee zy^?cz2dNz}lRJAx`PBOa0D-z`%eI=IE-$_IrL3N{oyGC2D*Il&dhNw=2~;=`sy5TE z7O0=$ssrCQ)g-9B4R67S~h>h`+?p+K5s8pfV z8Jc}d^Sjm=|JD4A_Ef4x*+S)b4quW=nRX z-jvDZMY;P!L>)qbU<-Yzj-X_5fOE3G(n@nf^oywyH1FVSEmS>AUpI(Z*Nlt)whq>n znco!J8DKp~hAW?)ncY_n?bLAT=PQ*&w4rWhuKDAJRbi^-OTVwEk14S}Kj{p-<8>6b zk|PLejg0+6S`4gy7voNZ9V1*_Bf&M5ynLnpng*@c&A#^9Cv+=-AwZFg9L&pl_pTKh z2BSDAcmxHDZ))$X_p%0|+UKEpONHXrwtsH5_8b4J-x#+`uh*Q>herVU45Qo|2~!P% zdz2g2GkFx9v|yzWOLZGHil%Ig1~<9Av&+#{`~It{*3TM5?Jo5+*m5jGzt7GuOder{ zvE0F-Hpj*smLgi{FVNTc?nq<6qoBMGR|92{~tU><2Dz@$v?nBWHbM(gE880p# zX0P=~9!1jR{%n2dDsmfYEQ4(DKY3Ts$^ypG@&IZ?Sv_uC(ucACb$Pv_J9M{L?&64t z8R*tH=J8WuWazQ?yEMJyY7R&_TKHU;p#p+|D}@HK=n(dE8|H`fn0O;Hc)hG!c0=V~ z0mt`eM68DTL5*QXvpMAZD4+`PvmnuRoa3_!n52#%GAS4TS5>75tDoDMLBWL!7pl~K zb@l)2CB2=_L_E5Tw+v*yBMlp{Z8l&Hv~I+sc_we9bcT1%2Brdxvnb=HI^bB^m2=D- zv9}zC2?)w|6XIr+NE1*&M5mn{Ad}t7qH;~8L*(Zt$ivn)Z#R3@nLjM(WpLA z9?yG}Z-O0{sJBEpLTKIbIzlYzm1CBDeA=z936{wQnuQoJ18?;4kbbzg?Bv8BGjtV7 z*Wu{7z-$K!J_FGKKe4N(wyd2MUp7@)sq5h)bb9 zN7D`951%y;(hG-~v5;yhjPj{#uk9_MzsX^Y?h&3P|}%~~L-Yo8NdHW(=EcESVP<>Q3I zV>*F0lqM6{dGBi|gM^Mx2A7T!u$GaHsFPYjrv5YG{ABci4PZIk2ib*C>nQJ%2@lyX z-)irmyY&eBK67++?I-Ut>b9)t45C6&v0V6iyOq~`8TAs;nuV226*^AV#`Jvz%el9@ z5?rWgX5C`enlJtNhOJDlY==)QV3TDyg`)>o*J;oo5;#{gK|w%pnSK%rp=x^F9r%RD z2if3}Ka#5#WsQhOMTvnBUia_OqySj?w)@SrQD5^0%d6garP4uBO70CIfPsa0W&@0; z(3MX;K1z5)kk)pizj18JbZ9}H+apomL3~-O2~TbzL7!z><)NT0JuE+UI6m7j3iYk% ztN02S%qUhWV-msWD@E3qbgf`Dp|g)DIeDN3FG@>BqNn$o6QTKi{j*KNf@Jzw_)w(z zNvs@06;3dAOmCw`GE|AOfcDk_)W{`FwDlkn%RF|Xuzqr38GMa&pnJo|TxSem?GcDjm4w!9srl_$igiYY?5Tlj zPLj|cnalS4m zciefm5RZEs9g8(Vfm&g=G5=K zdSlRf^3NJg4~ZU^6SabrlHWInHPXC6#B5nm%tp|{MuKVCsF|$T51%x)*^ zwTgO_h2}-{Tg>*hUkSdcG@)Nsi>d(;awOHTIs#@G?rehQ_Z0U-?%6wauwHmrnF-aE2G;ME+T}AS;^WPq>dm8TP$+dygt`v(qp`js| zrCK>$ZMwEObZ%1J)}{f8T$y{RwD>6DRR$-jHE4Hv#3eGm%AT#(q(>*s3tDIKgaw0B z)kysliT^xK9w~UAmi&0_KUY@0{=gbf(0tt8)onPewps~AIUYvrc#03!U}Og27WN&~ zCh^?MC`9To$N+qE3+6*J!!&X=GKz|S{)sO7#f%?i)wxwSOPCCiJkz5aj#6=%w^(X(&y_tzOJh?g#kD zB^Js-Sz5^!RPxlsp*DhWVo4YS03@T80z~Ws6iu9bylZnS?XBGmo|VVYMk;Et*NDKW z7DZv7!lDk<(=IIQio6ujc`ar^kzBV46I?Pd#l|;5#3o1-NWLQ&_kPIzQgn7vK}>T*6Axuu2NbJ6IqNkqF~ zf+kq)b)6ppUE}$2VC3X}rS1ln)m^NTqV}|#n z%wtF~0_MkMot9c?qD%w+1TbaFV=B<6ibE+x@>-B+6fP@Su&oq42t==qox%4vfrN46|rjUL8b&!C7lhS)p!KH9jd40I;{U(*V3?P ztjt;7v-sqS|C?wk1nd$O2{a(JngbNCBYj3tRc)mdVMh(u$^Fp-nLm(1ffk{VbQMGQ z;!eehc6=4$s7}P{e{O4-oiA^Lm^WNtCZ1F)b8`oV&<{H;{?G{>O7qx0n%l2%8-*rK zCKm-pCyqQMGC2yd4~1J+sV#STow-qU;rw}#c9LRLXn61aRd#8Uc>m-8gTCe%5;mR$S4BNbqk`tEcAu zt+VjaK8l7}0DGuE0VWxUi6BWzF-K>cg!fep(DFE1%0QtIWogcu^ICiPXVsxW;%Ke1 z=M*if)dR%4~Bgm!(?M2NbL%h@g`4G$&-LzRbjYIWo>zas|@rM&0K{qARRhv*vD! zn2hwsRsYNw^G8`wkMNPuBf`BXP9u*y0ss5t;(gk?@mj$yP9|!Le`0!OYBg%+*7QVU z7Z#;oxs+N>S+*2!S#!H4cHU{IVwIl=F8;QdMHm15%h5LV2F)o8;%*3hb}4_yuJpNB z^5fi@wPTM&yPy2P6Tpf zwzQ7#Uz)#(_;a11A(Pp!qJED$JYwJwZQ4pYMjo>)NbfB>jqn(xQ})}H`R&iIwK>l_ z2>pYwezqw&ZW-ElIh8O;rDJ(kdpS+_^TR`0G~sg-dgy-DHK`={=Fp+q`>$V3vZK^&48#nBCtH$NB9mQM z%F|7x;PY8$eND~lN1SptG(2DYJwv93@KrW(JUqrxnl0owrYF;Oxh`gOg#*dn3StPm z=m4H@c-&e0nU{*a=3Lc&0JjHK+MeHyR)tL6n}ZFAOp@tLGJoo^Pt-RCd;7ph@fHH_ml(*0mc-eEHIMS7Oe z5)`104CpC2&~9)ZXTDh>4S+Oj`)j23Atwcq)AOX6W&lOPmS6y2k}*o*wk!Yj*D$cm z*RNjfVNmrgx3~M!!c!k6trImlN5`PTIYa;d`fr)-;p@KfXqdV>#1d(DVTMzgFgisw zAYCQnQ9~beri_#+r7>|HGLURSd`k?~c7qYKCAo5Ms3f{c^h7!%b?-dVcm+>6Sp#<` z8SO;?u+?wSi4ZMtvb=V;_icaECuv441QG!A2<_?PB$x-~I6#RM0isE18JK9N#WI)A zi-1@yQ*=a1)?=iY6N|_LYAR~c0|6FUSPa&Dte3q^zMdgA_=#?Xe*fV0&zkS+hu+!M z?WANo?IZzp)VBXg=xq2uVwb2QG=Ez5kN<_X{XhHhID2<=h!bT&2;Z!XFmiF#t zY2zlVYp?)zK$?td*M9W-_wN|?|Iqc?a;&TGwLo3E+)fiOD`e!po?rHHPnGtHU;g4O z`$IBO!nrg4o+CNOh#~;bVt3ZCWH8Z7&?&}nWA9r&r`VZ~SL67LOGYEwkPRy(Uc*fu z)>`!75~cVgme81E6IY-T!BXTfcv+$x44nMrEGb8hvS#Kg3T{nQ*pSEAI6ze*p4*S~ zqMNb3<{my^+>dZNT*jQBBLw{H*YiQ!VY_>s$VoppKL*NAw3!v!<7z(vq({|MCa7gh zQ;v^-ri?!a$>%1c5r8@;rf7n_rnS0CA7rS4JWXa%<-UF8$tjwLC>f@+9=j8%Ja;CD zd=2--`{vVFXo4rR1D|Z()vnQ|MHv2ydWxCuuC148t~6{QWUgp!r*_zW3aCJ|vL>BV z=_VLq9RxxaiH$|{gi99%h~jVGA_tUlagezLr9Y@-PeCWu4(tLXWcJe2Qv}T;q)Fu= zhaX8bmZ#Rdne_sC_?o4!Ik+wgJi}ei|q+?#N@DG&q)ysJTjh~ z9>3tIph&VA*zZB8{r8z^cJ}g|p+)CHrEXUU%+BBSMrpZ0a=iPd>PVZ*3`4c;A zZ^8xtQL(Zp$3-%Tv}a31#nlO;8Pdi$_6W8kZ?WSQ5Ee(yJ%n5=BgbjPg@13)+Wl5@ z#oku?AD_EIs5ph1zu~3<)T7t72I#Bc-D5wTXmBYjD-Z*3ky9Y;%6oEr-}q3b!bQrU z5=DUCJZ&rGT$Q%gJ3AN^3joOM{0CklxfA_9t46dOO=lurRm%)bhUdvqWCe}dOxqWE zcLwE6k%%wt0#Dwg^L-}T7_b_6S zS&ayhV=i2wNxuPCx@tF#fL{YPHxccu#iE~eYU$~D4%)lk=ftaUYUQ>d=m-FC^RSG) zrujSlzN6V?w0E+v8JyAY|AsS(H%6?h(O4_kh{|ZSvF34EPY^XfDY)Ikb5s?%l92$i3z7!4H33J~Nf)={ zW=r!QcD$=sI;=PBK<&f+ZT&ZFuw%}PPa4*B8p~0()%^EQ?D&i0!HqW!xWI9?g&cl9 z)x710U;h7%R4Y|eH2Wl=DRAy9I=V$3#ei5Mn14NZizye|LdE!OPKylAg?>xS7sXAc zck9-gztU$M(`xnyjw5uyN74G!M>1pL=2vzDtx7sBygd21((RDwrUaxA#u4p)3@HpoL&1&g2V9TM1)hs;=)9MFll>(J#&sY|Iz#fXuKHa6n4zb|uE^v^cUF~RzwF}Lt*Y&?)-X}6fEURa zQ=77uIFDi*%W?1= z2_F8wbCfJ|T4oUriWT+lW$?zn>(@)ML~iGRI6~DRL7Tx02yLx{VZ!5LsuZ-kl`mE=8y; zTjIm!Fjok1ndgzeAesp$>9Dt-j?kT#iofq7trA(`Zm;jHp}h%9#==Qw+CO0NT1S@LOQy9g2IzLnHhiyFiD&uW8n10 zl4|LgecVV-UIj$+MvxugE#Zf@X+p~_*Cy(q2MC~)!3@=;g*B1gapDc5-2xKp29dzW z6Q4u!XRVk-1)*xx7GhE}g$76?vDnPb090bk#U*FyZ+Dfws&`X8+^OgMIz}__P*Su|Fe&QWVJub8cxB0OVtaN@j?3vmea?d)fgSZt z$9F{Zkl>blG7T&oPe5i>qMj*+)-&UbUcvw6^#aU|2J@15#{ z?@Is&P^s>OB_|+ck5wk>5*Wr1kSq4s2#ekFF$_dY||_$MUzO1zPR}-f#x7lokT0be70-q zJx00l!J&%?LGIPDk2V=se-jX~8ZhgSY~q?v$kG&ZrJ2Ar-Dr$K&dUV(Yc%eruPiXY zL^(Se1XQ&?$7QQHJH$D4j0U5p*&r#gHiyKw+a7@?q8vHEIu%LH;h%3(^&G!cYzNjbM$b0%ky7l?e)?A3#+W_SGDD*V&n0EQv#HU?rBE=ofwk>`nicpKR& zp7b5Xn8YB0Q}E8ian|dtr-}&PAT`u#B;qQ@MXF-aNIt3W;48P%){|7~KfMnk-!`H+ zxrV1)Z)m5eTxX3*i?o_&FNbz2;PY2DzGw5eA9>70iAXrT!Ur9jJ(c4CaM5n`H0lVT z7KdX!)q5Px_^TaY^^Q#C5XqXVJQVNuO9#u}+@&&zG$xKCROWx6ij}T-^I{iX&(i+kD8x1-qg*P7I-oQrgoxz z%HV`k)RB?{i>a*yOi|5v?Tu8{ukQ;&`nT!>az-{vJJH^0OQDv&Tx_(ow6tHZ1Rxk~ z2Qew?a6YeH=(<`BG9#d_qJkdXxMtDwLmI4A1*iNMf=n_BJ3;hRetOZiDL;57{q<<3 zQdL-Em{@q46Icr(5sL|=r>lJlx8dyBv+ZDGPu<$Ttlw{#HHMKus51GY?tlm!_9WyH zwMJ^lKW2#NN=Q!4ylGsDic)qy>ytpyNIy`PyM%NUH%}bRZFT8EoErVKoP4j>jb?Cgx4W zvQiKg;RumI&7M7IdyRfZ{v!Je>@LS3?Dajqe600``FFag1;~D&B|7x&O~Eh;`wY^& zCc6qKR1z7vi|(L){44gW2g?;iVBzs0AFSv0=F7p3E>ub!5qvW}HHWYQN@=mVqg(wt z3wMwO-r3hj@0t1GHe5{)-!8i-xBth_lABE%K>Ted#_`&P4 zVeUbq>ejg@#0F&GZ9q=}8+WBPEwW%WrxKga%|eD3_(v_L{3|+Qeb!?n}^CK>to0eYW7rrWO6x2n!{ru1PM2d~w>4*W7?8bU-d0CVlqU5F_%5Eol>DD0`ZxZr3;mrip|_hm#mE(=?U zkftqb;WEfnku+KjHf9E|72ogl`z4#q!Q|-yScms;`DIL*GRv#Q?A&pyHBDN$mTkt& z{Fft0N|ilJhp702_<8=Rl1;YhPYJumpLTjPHahq5%&|~*u5GW9qu|Ut6Y(eG%AQaG zZ_3tcjTu&gghrDfEpb}&Z#Gc7kK{;^%>*x;Y0ukeq&1I^;vE4?M-gxx5dF*O5kc54 zvhv*acZ!5%;a?a}J4_tI)@|Bk0BE;=T)D#`zLU`s~DI~ z1k;{JByZ^9R>i{ngC?cm|XgUOhGRNH~~u|DcwfM3Vr2Nya*m zjCNsJ<>DEtFA1B}+T}LvU@@JEdi<}Sm~CC{!N55o z%grKLwL738Y zdGTQFK_m5`2y&Ox!b~6O!}3m`^1ukx{_Om|SEbHAjrnWGOQM5DrlP*XO`wbZC6i zDOlAS#7Ey|WX;zwdmUo8x1YMd<)2(EU5v)clwmlCc&AjVQQ2ZmPN{=UW+@0%5kn{p zeDcl(kwS9m}n} zcmULvEPLdp;pUghX4n59$WxsH+H>6#j-NcleR`5qEsSDV#U26O@T6QV!V$S$38OC5 zC*a9oh=#QYw3c8oLQgpn6fe*#Kq>k8b_tSxg+5N=VOoRnQe$3*6Oh97m0-4YL1THg159 zVpxc+PknN&>vfva;77YM`Lyzav5&2&MlRYJ)Pj7)ME6$mC7iz!NT=NI_3=GDe)8l* zp0-xEjEMmXyOR$m?i)L>?^==~+U!Z&Z|J~eixx9Sk^m*+8ZEd1uf8qpU^RtQ&~`ge z@#c7Ho=Dt5wV;ISlWRRg7q%5x!-AjMCa~w<&;8W568$zR#BksN+f#^d19V%yzBbq= zY-N3sCZij3?FPrt^)J3wB!#x6V8E|0Dz+j#0g=1q4Oneu`mQX72W-booC2f2m{5Sh z{oitlWw2}HO@sk$MpbWQGolfMTclM(d-hO!ug~5Z@p<@K^dKSae}xD(8O(abSEXD# zz&mdL_?8>*eKL!#RcgWOnaiO(a2*dHJrDh|lt))EIXcSgecIGbSk5C-TS54lIJfZK zo9)hGYv;15i`}cxXp$fBFyO$@efMVYTwhiw>Cu9}51wDJV^P_H8Pl%?G49a?F2Y69 zagfoaOVtGS#~xP1x{`~^x#B$Y#K;F}v$wC-b#}oQaPz-8R&W+1vS!bo$;E~eG7V*Y zWA6xJ_=^fQ9vyl3qLmTot4%G%6FEzJX1f<(V;je|snH(0a{41HuXxS8o$`fW$^e|d z<*ovb$COm716Id3k#8x zFc>(v!lN+Y*+N4nQOs#%b*V^GxQi~rFK}~?UxTbGd)iW4 z^xy;&084aqB^#UK-OHzq6n%sdz@JccQ8}I0G@i0uk$dE87|Ou*SMy#$EfSH2qsWwg z>*u|+PNUZKO$7mMcBsV~KZ+c40;SOCQo~faBpl~39l`m6BiG(6uVxO zt5(f#z89F@rQnl}508hPh_f`R==X(*R>+m=!A-Y6yy5EV(r4A>ufmQEa((>6lp(_x zKC8N-T<QZ`i0Qu68t_*efiiywW!6J~%CAdPOzi)ijfnw!*2JK13`8+$|`A5=Ly8 zv3Ys3SB^iv76)3jFFVAmZ3{W8D9sQoNe|lSqp_PNq@CZ!j@CKLv5FVQarc)QHq*%7 zdV(?Zzru6$`uJKw64^Oxz>Sb}2^3Mux}bi6R-1EtFNsK$TD(C{#VND$V8k&uhM2s9 zvnvQYxB9f8wKeC1PxJ}Z+b#^2U_(ttV$@kI z7x@Sk-1tbWeK5bPG(p!g$(zF9!9B^#TV*nM$Eo|0IW|fMnoBx&rOFH7qd*S36Co`o zNQgDS52N~r%byr}42_@5&(=X@!_fG5`&&%uZXx@=_>q^$$yNoInp7nHR*w`z{}=po z!Sykf(o172Iqt}a5?8a$$KS3Oi}9AoJfdU53FM6!6haafIhmQ=p|yEMJEDh5iH9BH zJBd-AljQ{p(&UsrIO-)_#qFRi&EkWl51=`AFLveW9@wQdGc=nTp}+%h;?YLsNbA9? zvH07Bl29$ukbr%o(>Xw;Su(qmLyLt?olonwKY)+0M|QWV>D7w+d+izm7Tex(s%W6% zCzWaB;iz7lB#PA^V7dMfiiG{X2 zYZmBD>V-PEpZ|{6);YYd%PQ zIuok_Lz@bYK%}!Ocih_79JV1TlPkZ)0`4Tt>X7MjYK!Bdi(xmc0MaA;dxy=%eM<~Sgpoq7K*ha@)tj}e@%di0zEIZJ zX!?ttP-_#3DH6EEKJ(pkVwTP83!`JiAA{m&N{OMDtExb7iP#c4>nLjQ)~jJIb>X|d zO24kmKs`+^Jblr?j~|;fI03gMHCW*5x~P-E;V5S%bE$%+%!hYVMax<)gUt`{b9(cF zm&;{@#F;GHVO4(#KEB^6+t&^z`ATGJ?~JGCexUcKnl3p`(A&25|e3yF)@Di`pX3@@g6|n`1U83SzCe(SOi~CaB(!FgVV^vOwMtG zInRWg@s&UIoC%0YD<1FEt)vxEPe>(p}@LiRtq<(LO2=Gq+i?Q)IRA#d{OT7FA0{HE+4b59T;gXcWFVGT*{6DUyC|YKy&K>+;>D_hK$J zrARufhU0g{v`vrJO}V~n-P`BEIW{BY`iyQuHfO@HVPz!{#0RHtZc3L`=0_atrZBkf zLW*fhV~W3pdyq=OT(MMoPg+G1+N7o&<5@6WuyWR-f@>K!(j21q?^Qe2-(8B1H*9t;~@#`D@#_>p!QhYn2L$(V2tLdzg>U`Tf6@m~gLU_~}LW-o}2j z|Hg(TubbYvIdfLMN6&MeT3hT`oO@Ygqc89L5rNDlYwH6ehs~s@sj}Z_(udlISQe!A z5@?PkxMI_(Be6I`Jk)Y_p5!VR4pC>1O}(8&6F}DApr$z|rXKxWgmqw}=-tTGUcP+k zkTR1LUNjcj22(Mc6R75866Og;N9~XZTR>=^SUH?S5~|ZKUsdMF7xqq|&Xz$R-|$f? zO&{@L93k4H@%6hNr;7$|Bcd#yetQpfi0z~26wcuQJCq6utKnvhq%07IjPCJ|nmsEv zOSjMs_b}x#euDpf{O&x58xypC^=g~fX@z-W zsH#vNnG-lrH*>_P;I>rE?>~_aar6;SlLzIH%)c<1bW3 z8KG%wmS9S<42^Rjw3KhkK%%m8!u@VU$JD&57XKf{4mb&vRlk1yV=LQ=!c6q}p2w!@ z2=kO4Jb703Z^5h@X~Se%&ySQ&=!^x3(pVo$FlG({cXiqR9Ky;dlqZQKt`|UKO#TfI zEhSYR0~X7!nMOqq)|~`b)%#bjw|mKMoAd8|Z>duCYWCI4^JR(DM9Mf^6zRZRlBiXj zsm8UM1W`n3w{qIa>6d9s8(E*JmJPD`sL*WsAi9#Y%jFOeoszX07Mi@LDUK!>>E+}DQ;h};jXAcMOrSnOa88$ zP2C769A-|BAXq#}yHM9$W5A*P1iZ~`XXDr4ZxhZRM)?fjxZ@>rDAY+uey6JFMbly> zM>JR)(0o>rU*{HU+_;SW$?->ychS@zxZ0WImKq_WB$d^>mE|QEm@s+k;ioxewCac# z9=vRIQuu?cW#v^_3+Ik*ro$HHT<-0~b*q&VBBle|6vj~^gxH`8IC+2KTJW#qWZbF8 zLtCHf)ote0&kYu#Z`u+_c!KzN2UpJjdKK3}c5WyaqSO^H*qsheWpJepVARHyj%^BXkd`%$6o+srQ*wGJ=ipxmSp zlLIS!kg#*fgd-DoG`=Ce6X%mWFD1oPh6;Wq$4`qK!}`u{L2h+7D=X`&Z;mlOieu1j zo75DucMAX^M_U-RgIrVepub-{elzadFYmg!|2g#JlYExhQCs(64sr>mG>LqB>u35_ z^7K-i*KVhoV7AybDZ)_C<{bX`%>SON-V&S063jBt$47-8E?{m(hMouvXnfHD9?OVC zMbUNh7qXS#n}=1jdHk?_o3g*R-+sPi)!yyzJCB&}JE_Xy8*DF+wiwD_RqmpUhnyE? z*sg6AYC%~gDt9H>hT`zxVduiOUU1x<+HHd3^WL{{T|?HM37UF|Eg~8OJ?{*%B_+h$ zNB}3Qb4Sg0d0A{2op9}Ta10L)4N)2d7W69R{8idAa{AEEHn_gNu(xQ%pIp~3dHlN9 zs9VnsMUiVmrzcL`KqHXm4N0AsLqLjZgp(p-tUkoqjtj$z*tgU{FQ|{#BK0TPg1d+> z&*Y*_oH7h^lQ{~ca0C)zK^c@}qOm2TNZZ?2$72*`Vkzy;TA~roXfUp1u-#O=sON31U5 z;!Y8nv0`CSQssTqMxzbVoFwRv@p+xL7BBzn#M4D=js}%)K2s;E{G_-2#lE1V?n;zN zgzDeE#kMgOmuotmtY7>_UMq3PeY|AGsPr< z!yX=e-;YX+nQJ!K^XakTl|~|s37ho3&wyR)B*3x<=}Wuz#(rjQmQY6NcyXoz(_me98#O{A%>dteRI$F)dbc)GM;YD>2P!S%{b+Ejo}! zmyh}xFSJQ>rbJgd;JuS7l0?iEdrh|V8o9Y*?Ic$Mhy%)T`nPCgT&wE_OMw43;K#*= ztBuH`mmxWYRYz!_dlT0+o6(0Wl<_mRyD@j$FHP$P;OJmh9s>q~&3wt+4U-F`NtXhh zy>@X{{?ke@9Av$!B}+TcE^s)8ErV52GobCKj|LiK?_tvczdB9Ix(WEU0S2C!1)9&| z-8N_2z#{`F%o9O-$dRvJDq;0*3$_UgDqbESh6Wu?lYv(pOs=KMR+XD1f&Kw_;KbZZ zJ|#s81)~(Hdh~G7X?Uzcc{Xa+F8_%A305yht4B zcB-VRgIh9Li+a!+UMKLq3ybbBdX(p2F6uA$UiXVvE#qlIUB)%B(PBqM+3%T%Rqud z@cgz7Z7%#Ni*HsDvOl|R1vE7t4p8R6w-m;a{cY~|dT-9Q#U zCsK^vVDsfK^!dwuGsxGJjxzg_w0s)UKwlZIF%ayXTpF5Y3$(!OfA~RZ_-9ZpXb=X( z%034tg)!D{JCXnq9LNh2IM3kf3l|wN-|C!WiM`d>p#*mPv_da|(pP+0e;R42W~8<4 zv~*IOR|7nzof|J1ws7gv$=(JJ%x{ba8w;x==l6S%^E1%o8hE3ib!l@S=&{-ORt3>x zdWqPkQ{kG|2iejNrkMuB*aKu&jTF+dg!qe9)={j)Z8+lFqQ&e&k(qq}C05HqR4kX< zSDQ^r7GA2l2|zZQjc~rCF(Lv_F!IaWe46EM(IzV>=3HHO#(cH6Yqg<^t}Sz_Q>}s9 z*cm-eTZg~69K1!<%;NQYeDj!f@(gFoAGxoczNuGcWqj*uD?wp=eyTPxa8FC)R;>3^ zzxqjw9fgu0F1%j4{vH*O%EQaws^dvXOBbL+=WMjk99NAiTlKj& z9pURPV7M1&QzXq%w(IFA2dmqTZ(1iH4;IxHwsy0q{=BI|)r|eRiSWpTsxamT7(q!7 zl#rFc96_Phb&}qVe#c()c;aJ{L3V?^Pf=4kvL+*iWHO%3 z*=!SBiAldi@#qaq(%kOrm5MVj&e+2`*Z%y*!(4ClPS7)gP)IV zuemz5;fV9o+ngRYC2#$AlxCu#jo&U>iL@xq5W2cRhSpntPu{6kth z5xXxCiR7U@Z?K~knrQd;bchk6DwjCknbYRp?IF)*bugZjshymAU}A)+aV0qb)s!rH z0HA85T$bhno$=ef7AH0z4G_8a-fNOt)MlwE(K|n62H}=2K|)uX&V19$;@@cU zDBetWHI%6fUb;OMs0(>WF5m#N4u^eC4QDk;9SdvZddhs`vt4#iV2e8@mn5o&k}sWb z&ci-zVRx(QbLVH_^X&j8J+qvKit#6m%;L|X0yHfYTUmq#y^wsz#X61Yv_-OA?MkY) zhK(E7v8*K>~7Lx$I9T3k8x zL=e?!rXlH95H+it#T&OqLik9KP4TPQE9F98YQh1!g{?e4ahZ*9ufrX1S16=rOw6@D zO;=wXQz?Zk*+H|$_ic0Y`}(wj&Faq7`lXne0DKTONo}*5(YKr2phI9XS8cOoXE8gI z6IY`=%E63_K*y0+jE89bm+9-0U%Gur-3%eZWX8m4Ok}?BjA2I2Kb3Pi@O~|nkEV3G z)A6rZbzcm(l6_I8!pw8dN-YO?9k{X#njpwUr~9?mdYB$51;jn%thWQqCq8$DoJL=n^yq@H_4o!1P(fh);YmK)SGr`bm50B$=bE;591nL zCghM*ixr^Y-%^qwr)hapniRgzp+_%I-{mTKW1JfEKX~d z-&G>nFv!uCU}&i)r&S^$U2PKKjI-<1x1a|Brgr;p9+9sx+(&Vm>6cX;zLpm!-(cJ3CZexsZvqYcC{6OP#VVH4r1Q$e)u|K@u`%~&g9%X7s z8{@YIHSJN#!MSnccCU^kmLNhonbjc-F8J7%squ}jzdi%kuij68Z6WU?J+SXn0CIf(9G)7Br9t2gWn|Gi`6TPOG6olQH$@mQ)1imHLTqn_;#Lct@G-5|h zZ%mjxNpfYDK*dqV=~l8)1={$&?cG$<&fA(3P*9o7;8?P^j6}dK9EFu>VX!LPpwf{W zK)*cM8nomSNtET~p{j{T=D%mp9vVAgcqyFtIfgo;1{`bnt?M;LN52F6BpH0@9vP8X zS8%f5?Zr1Y_RgVRakHq+V$Qg6aONt9M~TCi39nzI=D=wR&s@g#n)~*-s5FCnMgexeqqdfN1^ohRQ8Jklqxe zesPjgW=IO2L@H%LAd8ice{+||4pt>+7^0`!xO2uen5K~M{mm>4$LgVFpSy9yBu$ue zjLqDQB31<=ac$qk5aW04H%3%+^D~uQz2zd)%iNk{`cd0E1a^JjRg>2h`rY6aMON$^I6-0=W2lMyNT=}inGvc5`f4nqZV5)JEymU z|2Na}yp&7=M6O^`wrp}>vt_ah^cckil{}ec7f&t={?+u?`Sbe@O*TH@=J#c>LZ+B; zf&icZj08D|Zk3xwm^Dl=X_~}m*$W#Al3)uuq$0LsYt5`ca3gbVwTem{-|2SE_*xWubt^r&L6WwaJ8Nel>^qn_wc|Mv zB(jo4951Q|O`u8S=^DhRoh14e=+@jv)<=KnzWAiZdL&$wK^#taaAGlQVY-KQ+@`iJ z+217?!0Px)ODdF(E4zL81~E*}ozIPod>`zDQdbu#Dhcm9CLvHXe>RxfQSJItlD=a) zu(4=d3_Rp%6{UrlAG_I#gk%yq1rk>h>(j6?vPuR1rWr?Htcj+FGlhEJ)%2avx#m8y zTCp6ukBCis+$@Au@0?v@+yi}5)`P6bM*@DUq}SsjNJ!Mo3DExUnCcYCJf<2mUggTR zTxcojZy4A&QbIq9FU+qirM;9vCFzW0=W0`ceFTF`WU%~Y^dh(Fh7EgXnSZyw8e7|$ zp@%4&%ysM33TD0ILaBzj!n|23G5;J=7o5jf6t>r_)+;{e*1>!#RjFAwsIF>1(FT)n zLlJ>quaZ%$DiZ5mA6BpTb_dhllpi_l?zLG2+&cissw-Jhfc!;qS%abQHwXa*=pp9| zFER4!#K&s1HtA0gsAli&zX}4A-xR^^PXF(Wi#B(e-O_f&lrri{B=n#Wa`riJRF_lq zxcggj@%;SH$}WrtW&4pd70QVp2Z7d#@P2xOF1S}Pe$w1fH{upt+#)(>eZ{` zGT+irQ97S1Tdv&5-p|GZ!nuJ@9`ZEOVEcyVagLZiwzD7rt~I*g&2{Em#l1MQp=y;X zb_YF;OMATL<5HzomwDFb7&zK<^gvOq2TWYB99@X3g`=@L(PP^Y05{9+n6HlWh8PA^ z3wk3&u=>|08`+)T)%}O28_J!V~= z>|#BZ+ws|nr1hN%6#gcp4ks>GDqp_*Xs>?vQ=w245-nvs*Rf+qnWM5&L%E%Bl4Gc< z*V?5u{;yQaH|TbuC$W8OEc9nnIYZA;!w^3+97wW@U~3 z)e0xLxa0aaWVL5;{-O&VAvvw784d4rRSeuSO_R0+Lw`Tv5kr*Q3TP#x(aC@wjy0K# zqt(f;iGIOmPiky6gef1m@~{;avf0^y^(uNkkwbPcNGLJwP6U zyy&a+MC} zTKmGwwf?u)?9Nunp!Gj@+O{a%KE{6^t=7-3(Ju_TIJ9rZn-}9-Offcw&48hTVd3?a z*)f{X9l?9)d!UN(7kdnwAL^KjKq+N^4dbVSJZ(!lL{QT}0Te8P-BJ%V zkjU7&upq<~^t@yZFXg+@Rer|6U~vvrw+r_rs>Jb?A_$UQ9ByaLZyMb7|Jznna`kv+ zD?+*ff+1cL0ihjD;=ehnFedwHX&-B(nZp#80BvriZzMkAUI~|w2@gjwfsKifNce+(7Grq)?gDfiPk#VEGbHX}roE=k;KTWMA4#Do^ zE;@M?ys#%B@;Ti;m3nNb^yot@jUqbyHYYazf&qhL?FZD@8_vpyha^_n)t z_~@g@e4uo5GD`)n@fi3(v=vucPL7jV2(Rv8cbTUW}?D&fD{LAV#$PC`L2S?m$uEvbAGVo7l9>)F$KQ$?;F znAjGoWCDEvSssbYNP>x0anzOFAslHB`&-xO&fUf4+ljwSxbh$kNAf0Xdy#`#aow4j z_SN^+do2Dr=_GS0J-AHu-Z6HQN?4|}T5nVV|9s{~K9fAEu6ZkO4NB7`;u5yg4*S0` zuI}cGnFVjBh_S0jL63u9Q6CntAT_!$dVd)4XYdPXu%*}uDr^k;U*CbfI zN6sw^W=c!06+ULMF2rC4bb<|Y#AK$Mwr65@_8< zuGcocblj)li1DydL^LBCyq@rgN{z0YL{Q!l)+g6%Wnv3k&^|3VaXgy-{G=7(oqk~# zh?(len%&8yx9zn81UFsCZOcc#90yix5X7(EZb{w%>1!g6E~V_oGB8Fk7&SL3b* zeMjt^Zk(zdXnhfOsylrbkT|r3@e4M^ClThwYfv3TIF%%pwr2eG;nr_!b$i&hg|cN( zg%8q_w8n~Xnm&W#$3XaHO3ty!c!zu2naO0)@QX@-l*9JfqicP6Z{48{sW9pl)Xeza zn@6QN3veJ`(0UX@;j1 zYLA1{U+xym^z3gtVqUqC!|hHgG;u2G1WKrh7(8wdKr5lRa!_K%Ks$?Q+;AZt0DO6G zFJs&$sCoZr0o0vgV?U3uFJZ(y= z&+}KV^k(TKp&pU>vQ`l*WZ#tDg&)w*ZrXeRO6^&yD7$IasB*Ljy&MH;#n;1yzIV3% zBS{l2gUfL={w^jqToH%+?;eOx{+k0DPpw`H2BABrG%Wmvh~4h9`5P!|sx5teZ8UwE z-M^l0=a4V=0Lp(-u}LplB`Zv1&&R%swwgVZyvaLKen8jQ1iyiY9M0hw_uIEYCRlko zqom5<|5PP9gFdAmD72R7%zm27Txgof@XIIq=gEqQq(kB>0H47JqBpwy^4=@hYcLkh zzN^}teMw2f$$eN^6a4d7EuO>+A)fFNHo`B&gq((kTx(b)QEd+xw|8_AS<`P6Ee?L2 zQF%&YM-)#KhzFXhypw0qna%SI(Rw&d~6&SOMS?`~w1XFF!G zPxnr5KJ-+HwZ-k^rApeB*pPYAiD|}8wQtxE4u{TSwo5yE&{f>82~=ha8Qo78)_1VC z?i`aNCmpN6?9RCo6qyzmaQ4Xz-cIR*Ys-B(Z==d}0Bm(L`2b@)DEccf&73vg(R^U> z8=%kq`_<22>Aoe z4^sjMw|LaEJJw|$w&;+@%^S&|@kLFX(%s%Xu8r4DHg1|op_UZfYzwZR{ zM0G3M3hh@i(uU`s0+?k&NSB^lf#fajuB#f>#f1XqPF!kgs&{eYF&p1T6}<9hfL#@U zj0XX))fCEpt0}6+a_GJOM<+AAfCP4;G^u1t9bl&}yf|IHy?D_FRB8PYfvL9JIpS#& zE9_4Hn#RY*UZbzGseh#pOO#l$x06NDAJ&Y6+YIBUM;k+CaRS$bFxjI>SCHh9mRx!CJ;g z4jRfl&PKX{?p>-L(t9GFp)O}Zotss&il*ueY7E!aD` zu4}Zn|90y(^G%*ZwY*?CRybx0M*R^(t5mx3X_+#)2`_=&>X_Yt;z_qRk&R`5!-z*E z(2t%dFlah!Z)b~V#>ZO!Wh!Mm!0FZoa0YfaWS=@T06Ktqbz#Pz0eHc{W)19&n@J*V zdOOg$a#?_rl5TRn@^gR<2;1 zB0YqIS?-*J%hC9aKcz`27QFpetv6 zoSk8G2kjf`7*OMYnyc}tnS^_I%Om-UILyGS^t84Ry^ht`I9F0UWqLf+z`#JP>>J7r zUa*9Vi?68DWDy!hZp1y94ICS%r_GY1Bu}&q`7U*%OzgV6y17+e_=9EdMO5bo< zaxWCMY%Fk`z!c|c^9Ba*bAAb*Pz zp1;T$Dmko0ejPrEMk9N?Dm~?Xq(e$F4c{ATaI81SF3Aq!H3u=Q9i z=|u(6$&rl;Ilr`+xl|o(rj0Fj`A>qP>?_(?oDRXH`|LEAx|~9Ee0H0v`3dBsHI% zp~Jv;_3sSQ;Uxb<*{T?bAV$FgDJVsDjk!Y#$lvviQA(7)mUKd3gpxuIUDe6TU!eh3 zJ9W!u&9_76(dm|Z0^NDXQ!cY}Z_oQhgm9`H*kC{HwN?!lUuqROZ&I5Fy!->3l6znzkVGBK_K3?7kceGz{ml4Jj)+|SIl6i zy}6>vONhfd>`u9@AObY8;CaU6qv4AVc|X3;cF~E=q+*hXajym6EwEM^gWg0_aIXg3->SUZ*K$dxA$VdLqf4 z^Yfy4lh8-<8Q;3~MQ4Rr@^HD+$uxYL=f4R0pk(m>sdAyowtP!uYzZLIb-Uyp6D_8wj) z_a>(`&;Dpu@8*L#@vMM#7bSc_rw2maN#|zdk=jD{(!g&uBUPep*0MbWYejBw*A*s> zNQZNEk*fRq@wZ#0R`~Fbf@uZW%x9|}y8qhOgR=_F_K(gaDf*Lq$iYllUUu=tneprt z^8=L;eoOuWh5$_vKUJ9Fj3=?xr7Z)q%AXiLYx`V3j7WMuD*muCgEkrg0{BOfN1bk1 z0;Y{O%KpARdN{&xVzcQZ*UkQ=;PF-xvxch*FTpv6ol#{&*RPVJD^$sD z235rP`lAJEQ&I%9jSkPd0V+VG{q&wxio2GHu0qrrnl<4UAH2sG(DW=9vW>K&;6UVa zupobOUJcn|4JH!O-@3i^J&ZD_4dl(g+3nW8BcbLJGbGIkvpAbZR}s}?Blo*a$eFJN|F=tJV1k*%gIk9S&5os6O6kIwA6o7AD7PbChjiFyY=Wu z8l=leg3hoxTYq_>!Q7F<4;7_H2nnj(>Bcl7U!kYkp;80PD}hZ)IN7&4uV~)ghYuqN z2bq-6;ziIeD1Yy{H?WX|PiHSEt5(rZ*pEAYT$VfmIMba6HYTBt9B(b{?RP12;$}TNkZ|1==T8HsL`5^Z6Iw1&u`>7RMe~-njA{P} z8q>3)VFYhOwuu+?y^a%(UKg7KwL}|_#;ip$$InBv)&&OMg~Wn~5o-e3X^?elQBVngS)vDyR69bo09X?V=770W&)&e5C2jl*{|{~ zG+QX*br30_aTvhFRQ051G=@>+)l?D(j);eV)~F(9r=y8kJqQ zZe7r}@>SA9ekuB%t!u41=c^rPG|sW|U%wOvVBfc&x*>7<1$fpSI%Q`T&@;T9M#&HW z6D_q+%m*0-rA^8*mqcDe4AwWBv6av%xhIVmjpIy=nK8|rz&a25OEst>6%-k_hb>#y zV-QR2<$cC=gI%6i#~My-YzBCW`#Ek9wUCHC^0KEWh(l(lNf;PPn8Y8{f+rC(nI$R( zk`1riavl?0J1b8>ID)+ImvU)x#ggo9>?m;P!NZ5CzF| z>4`?=ZCrQv3+ymq^3A$!OLi-~W0xU=LW|M4{aA*@F=Nhz}P3y{Qq4R@#C zA5DwSNLlrQDWXvWZ{NSQkU{$csEYR{gDk)wgHYV(J1wPiA*lv#}`c24WB z8{-&2DN8RRzQev+W8%=Tub>8BSwJt zwDtHH^$Pz`s}8xU+H_SmIaD~ul;}7Ra)0t4FDDrCodZSso^pLT1o@pxCvD*iRvQ zIoRqPs}HevTKIC+VD^V}DKw{m;vLK+?4$^bH9@S`DGA6(axK9@ZI{c7bybX5gq)9h z(IKNxEci?_4|y1^t!SS<)n-A}9(rlRNVO^IWtpsp%u+s+nAlxhuO+=1e=YQ#pcV@b zrzJwZ8c?mnR~o4TYH*2~-NCH&0ZT}1Hd2A}^;Cdf0!nGs+}g0vmmTdwZ1FR32OW42 zN#G_rJ;XR~9h-pe_*+OedR|3UQ-8B==5$dKHuvM3NoV+{%D3KK2(a zykR=pdBzKOWpiLGf3k<65M&@<+_@|@8;MGf9qPrq+{dIdbM~{5!=GsOG3GFTDfND$_4!S(V8Wg<+2d?4xDslvAb|yH+sysl0aER9e zLKT_GU@}Q&mh67mbkHXppa&)lyLc`&Kd+8(CEubiX%2~9h@}}vSqJ$tLLA$n{utve zWv0Uf-R(Z-Dv*Q2h}SMY=#x{WvK8{IMq;{4#nX(FlVJbZzYj5hk1B^@CqB5Rnu@&##@6Q@L-QGNS z+uLEcS*wk}@;PQ|nB8Dc2-L?c@J*WKg>Y;@YCNy^I%Wyc12+XOsA8S~%cbww!=j0$ zJhJzDc))q>-Hf9HmzCe5*Hr!-iE8AC3l+g*M#AIo@=8XCu+ClO_nZ?qkr^@r*qa&# zrAaj5mkjETi+gLHpQd4uFBtfIU&-8Z)4Ljo+Gq*$C_Xd=eo4}5)C3ZcKueV5P-70! zKAyCjstd3=x*ma-{qb)!<6DJqA1c#lweJj2ykw^xEO(=>kzn|4@tNMT-e*iu@6_pn z)GC`e+o6fd@Rzz#Z14=JTFg?K&mj{x5 zo5|Hy5Ae9o{?T9wHIx8)*X^ugobIk4eCS{*O|hN~dT(Sxp=CcFcJ;`t8wtM3eJkE$ER5cYG!6X9C5q|Kaf2gy&Oq#@wS+ugmQ-h_gL1}m z)z3H@S~WFvi8e!qmTR*li&n|1><-ns2HqDl(xk$~wyuH7`3{>MU}sqfJ41~$je?b- zE4YS?57soBad3Eh&?u45h5oGC;YtSInK%IFoklMlQH9C!hV(L4nApOZ z)2&ab>s_}GS$m80jEL?~{Cv`rD+|Cf>ZgNI%W&WWTnuT`Ol9OIMI<7@eOi#f#VFQ7?xnVB4-J?QctikZuhqK zF3lfroIC6UHYA{HVa?v=UM%-I86UmOD4mZio1jSGRaXe{yGJcGzO`3NGUI#HkSz0b zjz{dsM)>FR+hz-u^f~8q&w07x0h6HYTK2m!S7X58sY*)u=S4vwDa_YY9Ri+Unxq{A z1r*7z8jgK{NhCX-et=2uB+gsB8ZI4TV^e6x7ZRGmjG}%potmMRDtD%|i@he5+Se66 zH@;=r{LZtBG7|~*?#?M++S<{^%eH>M*%(|u)*y3G1~i8Lae7&zv7`N(eCjr?Gs~I!YGCN8yZ(#!#9qI5qHpJsbKRC5 z8k1em_$YHX0Dmaj;B=~cN-Sc@VX#m?>6|^3)djtyCl7yHck7lhf57#n z-S)DC7oBhSXZig`F629O9DNzy@16Qlw77W>`8ChofCTo!-{_IJ?7lfH`=8vAt8htZ zWu5P1T#=2<z|Ni5-$LHIDf^0%5<|LICIvrW^ z(c3*)V`3=|VjyKi!bza!!vY$YNTa@u3*3-64(#0K#anJtgca%~o}~KCu#F zV3q!4aD&h7{`Y+}C|}QiP&rLMb~I;j)I{b}YPwnUP{4G0u6eB~46qdGIm2O5r~i7WH(kq5 z2$@UQ$n36$Mg{Dn-OLFfwRh@=GG9o1Mp@vHi`&+s0Rf@+9Un#V%d zw?Sr^f3av?>ZSQDAA1I9NQaH23LLiauf|^8zme2S7z3&4^z`ZA2^NzeAeP^}%0vle0k;^A`^;X9?!$bF{a& zkL!syn@J6C!F(Tmv^!~V;iu^T_yso{=J=zf*_+2L%?UUx8%HPkJ>e^vltEK)&F=79 zgS@^&B8nGuIBlCQDI@T?qCIl*1u1rIbKO5naTLp^ZS<^h|N3xKkve&G>YY6BAH-;^VpBBx=ubo_YLQ*W7%(d(4Yv}t``I{xb; zT@RM`4k>09ZAzLRSgn}eoy9wgwy`Qg`p_e?pZxm|l8ir~H*9=2GA#KwSi-|SfktSM zv(ozXXue=EtF30OTJ34R+HNoVpTGCLdtzftX5I`OJXfPb=%Y^y`PsJWe}2K{)8-}L zzWAr?<_C{oRk4;jg}?NF$FcrjB542bKmMO&wEw^Cf!W6*xJjn7^Nj9y_+LcUs#4M7 z%iQC4gN^SSyr_`E@4A~hw=NXs5cA45)Z;x#$VF=c?Ryw-|7WDHY>rTH%^DmL;)f7< zE2DK)LMzLycq{mI^D+mWb?N%nE>^6VkYlP{f0Gly+*BY5v8gE6&LUz4s9k=sA4rmB zvfmpqcI;RI_w04?+J*aosBEFGjdHf;-{rrk>0o@cugL?u0e$R~`{yKq%;ehCwgHW) zswrs;XzkdA9w4Pe-w+B2lR-R&Q?~-mjtpm1+LN)-%U+E!#%P~b#>YBWEYvU9$J&A> z(akfNISqOOIXalxzONqcxuEUmHqvYhd?GjNAB8D z1%5esuego5Qpr&1%nA|4@a52HUNx;z`{tH^N{!Q?-Lx4qD}1k+18PBE|#u_3laU4GGBc0#kK-!*yZ)=Y==kMZQq#w zsv@)yV2HP~qJh2zN27l#yllyypr47C!x}3aQdeXcoQF+e*k^TlB{nk%;RC9h`1^;T zfXOwT^2~4&r&NWmn^n?CIebmN?-qf@Kw2cs6!n^RzD&K<{1(we8c`CnE>Y=-9@SZ( z>DtEQ2=<_IJLD%};IBH2)b`C1a<e8W=UV{4g%g_VbU|hPZMZy+-?X>e47P|QEb_1-)hWTB6;AbMmgvB9q<^;Un4ZP6D z=K%<%{K|0d-##)GI#O$v456XFe;z2?+2rKsA+K|vJlWmnt?@8T?-$k=E|w$)FG2Qd z)N$w$Y@ug>ro^Yv+Stw`?t8#mFj<;ae|I;=U)xx!{SWG*{~;ESBRRCyu9JE5e@mxX zVlfF#p!I7cic(dz*oZUmvgQC{2Mfq!<27jXujG*LW3CYX7oCYf-ARH%Mw7^hwkqE% zppH`>6~Q#=FJ{o;iGL+=MF%E7QtP^t*l+7$H>;$8cm+jIt-Gx*iJ@S_!hv^Lz8ftE&%*}+tEetYZ zX03mHw5fH;X!k#Xi4BeYqNE>iIW?;rO~HJp{*5C=ZC8+HY96>b0Zi|9*MDDEtc6@t zw$c>nXA!G$MT?ULvLAKK2)yOm><%4a1-V*i7q7T!VIoFd3o?CfzThIy0Brh<^>EHavL!L$`vVP#v70!3vY@rxiObE-A zfwNa@U4Pw>qXbX+nw>gzVyttMB%6QN#S4_~Q);g?;Wu)!QMKI{u3c+7;I%VeV3g=V zz_-F#Dc=`AjrwoEsPHZpsmx%{m@R~huQPvxHsNYX2<=o5_cR-zgxoQi_quodoi>t6 z&;0dYtI%%=33layG1H~sqMeD*&I)7ozDX%ib0w>~D@|v`-GsnPR@Usyn{J8r!eUYhf`CDI6Xj#z zecMu{_D!Gl{84|VF8y$7ye{4+-uel~zfGk}R?xe@hzm`9N^SXPQJ%AJCyZbCawj=b z5FCuN7cYM27EMWtIb%-3TbA_s8if{8#;5p|cb?=}*#fhgzV)+&@wlFL*PrfzM{r}@4 zF#tO?U4&lKNgQ5oo=gQu9tT3L^QnE`8esD#=RWAdq0vvEE<#Y{6Pj-T6tLr@H8C>{ zCy2a|y##3w^F~^ERBAr+kVZ_A|6QJJZ3+(>NN$J!#BmycgFX7vX^wgGq@Bi0h|b+V z&G7nu!0yHfAec!uB2lsUQ4eU*yMPBR)+hxLpsVjDjWF&$}*i@6A@Nn1A z73lx&8A$4UwmE(7;T~hJ9o(yVTQoB~IP6F9SB1Z3-*DXJcWzWvQniY_X9iah0&0kT z5Nvr{i&C4*7syAI%89#-qt3E~)%Y%m5u8xazm)64m&T$dJE-SB4nCGjS_oijUq|Lk zq77eFssawy(lg#YU16uDF!CLm_uV7&DN@>5N;36s^*SBv>h*eH>${6Sd3GemVWrcti&r|FC}_Uy+Z+2<+}&T? z?&Xc4u>}>Kb7w6-^Q(*7s#;DLJtx@*X74C_eoCsjM%uv>FY^!Hn!ayc@PzH{0<)v| z&#AZiEq?@hs<}Zey6;#`o0)FV{c~Rroi*$=Ewtg~PPHp&+Gkp`z5^S();KcFu3am; zm)HEtz37{H**SUOCuz|`?@V7Fot)EeweyXE@25?>IzFaxv-8jQPg}RW0_XoNj%W)* z1a}x|&x}{~z^F9g=lpEj(6XZA;% z`1Ho-pZ9!va9y@J-T0Cpxbl_V&QD(W^jZ(+8+Vr7$l7tLPufAahV*Z@eRsw4?(pnq zRa0>Q!H(rc!mQ#+>+3d1I_T}^Kl8b1o*IwWuJ!JHF(p5HwENK;eNLY|ke}gvqe9uV z*Wo{3)5pH;J}&0SSLdgkKJ_wx)Br3aaI70Q{^u^r4i??&${nS$W z{1j95!0ctQoVzF5rbO1QQh_NZ%P0TE(leQXq8B!+WURQIJ60b~UOs)p2I#%zD%~5I z`On&^=kJHfF#4=a$_k`FSm{yapWRat_UdmBtYx{)>qXNGW|E?pdyO+r#o_&i6xR3? zZjoA{(?BcR#e!OW>&Umjx$ZEfSU6!wE=;eif#hh6sQG1_+#k>!RckQlSpkzqZRfOo5YFIDLHyLx<)j%~4 zW;GAQM^5j{%ni(c5bXz3S+~711}{%(&-vee3MjnsM@9yx$xqU35iz%+6L^4f&>)qcU&6LTXiZVIu14M_jKV>Cw@29zRw4`+E64ek#UJcfu zi0kw29viP5TyEas#S3;&=Y_+Rpt3nY@@o<+hsdZg$N}^t9q#~ngvj{_J@rNIqI47M zi}X26$F+}iqtR1#gq)+LDh?e8^u4=fCVv)7UO9@`CEX@%+uv`#yW8@O{6AIQDC7x` zOwozy@|QbJh1h{B=?O$l;v7n^Ye1i=%`<*OWIviFsmK|}}Yx>4B{8hXLnA>!)J@wB|>}95V{KZ;lY4yun)^wk+{+H z#81mU97^jTB|3Q6NeZID*;Q<&y+V{Bx~)^KT7MW!R)bz%%v&{ey-L8+stglE(h<+4sDL zu#Utp0g0?VnvK)^&Eq*pmpx9$*_b}3nIg=9!6oHFnxbH6G&Oe%gHlF?#@S?k{Lx45 z$W!F&8{)d61-*EwjHDczw)%r!*ML{$R4gs_ib^Z#g?QotBA97icu}vhWB?fqxi!f{ zMkDhTz*5cdNZ_(po?|6Pp5)H#=7s3((tUdM*NPrTfArO*9vfNKe>!@pY;dHFu#T zLan?EQGbOH)n#o`%+AlSlq4O{coN<{dJN5cP)wl;z1no~uwx7X zxA-F&Xv#oGAemvhAQ&de&~0lnJWk47Ftl#MSy1JXX*@bH{ zitP%PB9Y%s8iwMA0h;;_m(uVCUzhXTk+2dvx%#Kin&WXP8@~ z_wHbk@j$OKfSQ2`bJJbToJFB`6oVokj;pC(d{6;89}(o*L$EKUjmbJ8Wpy zACy}&lY#dqf@*3nB9j*FR0ce7+^4_i{5@cJ zwU6iWtmV6GLGg3ZF^i?247n2BS6$?DhhN@bt4W2BlQ+|@y|U5W{#KQO-wq;RT6ND7 zOlG{*C+D`Xa~aCBvIz|xA&`Dh_R>v1D^FqZIYM*WObw7RrGqOi>{E77DeX9(5|ptr zp5NSpGOKqmHxl z<=y{Bb#ETdb>6=Jzlfw%XtT6PlA?w7lqlI!6E)IKD6LZN_C;x-kc6Rq$TaN{EmPWr zXr(Dh$LQXX6!o8 z@g;M~o*>2?%McuS#4wDj-!-)3Qbpo8MDcyT#p;trKlF0qVRC^!}G9rrDb zm6A>bIxt~Oi+SHgzna+BHq{MUlC@Z#2$+`jo7vR9=qUzj`UD=Ew(Hj#d!4Y#?td4) z!(Nki$m|Cg6wn3|OUqupn(JgmUiYE4 z&9wRe7Fz5zpRbOwCL=)qBZ@p{)MPp|Ntf%2Jnk<_mbZcHMF7eQrN;lCg*SJx2_*bU95ECy?8Hp7)$3jZJd>Z~qaY)3)v0 zeO*?6A~tU?d;Hkf$%^hhe}+FkkcnpPbIq9A709y8tIRCFX#l;j-o%Fpyzr1gaFS?6 z?Q`WDAW+Q3$8qOvU#5IUI`G^4UXWB8JFjO_g$!o4_Go;Wrtw70ob3na3fUufP7@hhu0PtT{QL#0&h02HH*zbl$r>y$1a`*&5 z7SAP_nd}r6({i{3Lx3}1Xa0OE9Q;#ZcCRw{mYYW>lbem(j&x~U;8b`0*6bm?mC7YcYSQ<;Z>Xmnsto%0)FB2NxWibS$ zmOG%6uDveuC8dqaZ}w_cb*ZtO&+gLDK7Mbq#z+#Hkk>My7@}Xsm7yp@S!_giG$$oq zy(e`F^!q5j8zZx~ImOnyyOq`o=~0XBMJuGV4S}lb7QB-c%zT!;|EJ^DTodBrs~zhy zrQa7c?enrJ&3FqRUg)94uCSB=4Wf78Prh1I)ITph1X=DNCnj!p24+j_VasAP&Ze%K zVy(v@a5=3cftiR`;^E{Dl1!4RU>rWre$2##UrDPdK3&Z>Us~Frqfu@xZPU`-&DH^j z=gW8o(o;zAsgyZDV)+&BU*_0mJJ^i9$z+hb9qYZBrng@3IlaIR%l>wv5ReCi{45uV zJ||e(?{ng+jmt~s1A&?)&o#0kkjN|n^q%Y`+worycaLf6#ea)eL9nf?jlc6;ZFU)n zSUlU}Jd?esfJMv5gKTD!eE^?7%QS@vCy&4xShOmWpCO8Na(QUG`~?z>%whnMvgZ6B zvlS8!!8 z|IxuvjfHmH1p(tRvo`-Bg@bQV-q9U zNH!f)c0f#UN=^9+yT?B|o!xcPW{9R!fz0>e-p;_+E>c_gs-Wan?MCh#xWpZ>mcLTd zxd0R+1G8X44sw?J95D#p&8L|om`Hz^5Nh;l&tOH0ctQ5ITD6>jdKk{_xv&EiL` zi88#{M3yGdvm$fu#c2(llfC%o{w7!mP=3+_?*?bPwH>&44iUKbZf}$SSIbxbPM)wx8hLYVjnI-Qkm>fJ;YsQDo!b0a8Ku?k!^ujLA)Pzr@{1j7ac`mfwP1-1LLD(j7`%ro;#(E_qMraY#J}!|7HDlfNuoNj8<B4|Qm?8>Yhwo~mUe@$FRmT)G~12I7+*UQ6Jews*w# zyHUPcZ3p&CanYFUj5cj?^P>>rZp=)%+B1$5GDimUSbYE~IP4m@#G-=c>bddPr*-Jn z>+g7Qr6gM?E9#`VIUQ7|vJ-A5E2M7C|Bmb7ff*B^pFHDD(C3BT^q*RM00zXw*x-sxOW*qsB9vM&qEu;Xa_3Sa7+qG+{vJkgP=~B6x$A?Fn%^X6v}vMxUVHhkpY7aCP0M zduud7iw@vhi}Pm@@@FyH{o&h-uV}Z=Y&Yp_91+3qw9_$&rs1_55sP*ciMj=z%o$6r z#yKoN&Z7Is(aA1;iEIR!AGfI?xzI{nX`YsfqbvpxG?vZUg^vYZmjG_zTKZO=s--jt^$^$c z=Lyz<<(KV`v<=-3ZOLoN5!R<&AvJAt8|%CCzXLCU$;|P zrL83Q4?mWD{rYuS7RS2Q6@73`Z;n)BXqhCc;y{;XsvqXTWn?lQ2>6+Q!!NT(I7Iol z8y`8O1_aT_K7SC{;;cg&m|V!0$(C_v?p>RJvbszc@ezn!yggcBn^&^5dOwgKE0UWnq%nn^X|b0~?yY zxSxJi`nqBs^~5vi*6nM(qg$`su^Wxgen-DR^i$iVa_&^*yX3a@qs_U%T7uWLk8zPzX)vCzqF5CJH0OpoL z{W7W{Z*F?M?hEJxy85u;kMF^G6dYt42LQ6gqQ-KTPO)yIO&4|DxzJ05K%SlDhrYz< zKN&y;UhYheV8d_+6BCoXc!J7=$^N=d!AYIr9KwfRp)s)7iQ$KDo^B5qA7^$QN0;UN z$4uk%(dX9v=R&x`KR zQgoa*>Hsm#_Xk6UO>FNq2xn0*6MY)$1Jlki)U*R>*#Rus_24afFM8iUb;a>FZLZ_A z$Q^?U1`iJIt(EzSCAfSn1J5jWZ~64)i^J(N%SlaPldPhQ*m~Z>M4wtWykI3?Hiq^dJq>4k;8${pAVA^___NvJmKV z9$-KIMg;}C-ugG?BX?anvyvfk;V9#J1j5vCL)HlHJXpW<3h^@>O~-TP_vt;n0iOl7gJ??FH#}TToLiskYsIRb836Qlan@H>QRTuLnM@>$PMx&UdcNNdteg4( zl@n(-%fR5o@Cm%yWLuiSn>>S8X<84tw30p_`Dbuw-v-yFuZ zOsK13uK-1oUm}CVIsQtbETZo|{mJ+>d;@mn%*xrgwJ)&wt?V7iM4OV)=4s!0#%q7> z4aa2dp?UcxIkjuMbKe)t_+4NJc{6f;{%im)ML2FvwRxpcFsMFD-kf58_wZV{bc^m5 z9d?)1=62eB`LCc>2iIM2ldjh&U#6TJAbb37iA(8f&)1?6)ZKuTEv<@8^a)DoK6h-t zGSjrDi}rMI4R0#Wqim|6t`#EbIayKUMj)CL9h|5%IRcyKAmb0}zV4DmcSMYV2diTz zo`n3lYUQV;P}f)gX+Zy)9Q*5oqQfE2_`^naGCmTZ!5pv(&THHU%z2(3j{PXnQSF|j zOtDnIK?j?oEWEjz_ZP0VKGHxUFe1vg4tlOxQ#AX;+HQCQTUAsXSzY*0zU9$79YXp^ zG2TAcp9v#RbOud3UX&UQw)*^X-27ArGZp6&2r54(E198Y_L66o`ZblRqv@)kvSY^8 zi=%wmD_pqv$G49WxHiW}2XE=#i$3JJgiVeBY;V!&nMN!5L7^r9chh7`PW1Awa()(l zCVzGPF~#0ym&Uvxutqj`eD>*O*8zQ35THO+8})wMCp>5ST)ja{gUVYqI=hCOcE6IU zF6{%OB_`~;y^;#Kg8kQh1p47Y9cF0wxpEf|)zl$ZiQYp2YY01?ulfLnA?@su*}P0U z57g5=BDlb72*gkXXV?&qfIt9!er#a<&Q=#n%gS^re2iBYPGS^-U+!FN^hr=Z*$^Fb z)w zn^tJ=bC%>3iBef4kg z@n+Q5Js0oYf91#SqXf@!y1KfTv$Gwr?Q6AtZm6%n|7*tLFEul!4cm9}PrI1wS=rgf zH2V~NApB&Ol(gkQyPsh1oDS+kGv&hFyDh=nL+NwXHZ+{|_N;xCPMbCnj52#Zhv?Fp zGN*}hj}eGCTqCG_(RLaUuN--QRFn@NJ}{As(GtX-1tHD+qCl%6NZgv4eve^}=+D7_ z{%Q8<)2C7b%(0UvHIx`LH`ST%x8H7k?ons!FNC0Z4A--FaOhGzrb`RTjYx#PiKphN z5b}ox1@)p`sR7**AuBw`k5^SJEG(?d`{Zi?W|}~DujjpOK_b30%g|+9n}1bwB^oKmERMaYjbQmiYKkQb6Zo*o!q}=}Xz!^}ED2 zWL(rGmMvO!=;rp<*`@Cr4%j&|pcr5SI9z z^QLf0AK%>FPHYp;E1o{B&%1%GtC5?Vo6E(bVpxfde3*(NGh8|sh1ra8w^MyRZN3+! z+0vIynl!n0c1ce*$7t*6g>s?B-MZcI8Z4Iz_VV)cTSp}Jb6IGd*EVQC8`hg?;%kO- zsmCAmzk5iZp)mfpPJ;K3ojBp~w28br6yTXRZZsk7UE<08?!LbVdpc>~@%=Eo{nDeK zJ}nM!W8IQTA8U>u?@6A>XP{B?t!(ZuGYT(&aaI$UTQa`j0{X}?_|1O(`kB=(`SiMd zaNj92i|*Zvf0C}9aTEHB(mvU@n9B}@)bhbz2ld#GWWN@D%!|CYbjn%Nj7R-{aVWY@ zp*9N99_6Y&snCuPJMZhKzziFF@OFGk;M>Q}N{<1Hj^SLsoR!s{8~!I<_kZ|y`l9Wu zu1UL*%Il}^elWG;*7MP@jLht7bz%FG`VkI3L9gUS-l_^>qpB+P(1>=sv=Wvi*H7$~ zz;emdYk6k|i&I)=f4&C75k&!ye|F zAGqxwGarx~@*N&g)mC1-fgOH%eEnSwX1Z<)JV>8}PYpxe1Gq>y$Bc<7iKh^9 zMu~}?)q|5zgnn6Y&No*#q1O!eY3G|Ke%u%t8OM+4uQ+;eOqb8tAv#)GTK-xqSI5^L zv16Or9Oz1O3@qBHE~hE1w|(3i6=g&PuB%v4kZQR)`Hq(s;z5pPJP$<;01W zN-E|8SC?OEyrV~2Y&{n>Vz~DjkAuJNvYYGe-JY>3J?z{Q!xu3q?@7WrXre8^IG3qY z6SZgtSFX?;qm^x9YU)B`zgln%=d_`>*&?%LCuYNGzjp2adn0L#N9Gsn;u3Y=g&VJkjkiJ^*W2V$o_?sF;I55k% zYX`^3F}rqh$$lBe)l3fQvfI46rHQtmANZz);?8`M{>-H&2b^Xxfw&E-ALX5FH11K0 zcI~>ug6Jw!0|L5}Wb0nGqg;AkfA8^)nxRo90|`}aNxhxmdiC4ITv3zd@z=Hsot>DN z_?U2vM%YHhh&bF!ZvjhIpZu|bO8P2g#9p3xUaK_Df8R{=_AW(d)HTTiy1e)_(djC> zjEM{){5|J|cJ9Ahgr<^-mylRua*wVp$xc+uTNvHt1XF7#gh+>lv>|cfcod-Njf{=S z7}0#M<3_7fcXDtVD&vlMt+p%XT}dV&sM?a#A2Y?e{tD?@NB} zwl*gq<=C+(Muse(=e=g+zO7WkW~-)?qDzndifsK0Y$CmwinC6$pmA=2A793CLX_}t#^NjN*quo77~?V0_A z7k}!E8J)Ytx`^B0);WBJys{ATiTVCD_j?;exZ-cX$kSfAeD|(Lz^}%MhkTiXrfMtV z_j1kJ_Ukm!sAFVg3I2@k%4J5MRA-a%EoQuOO?Iz~afK)RuC`P$qDKYHLPeg}upr>* z-kx0Pkw1rzi%eOw)`Ut_TPZ+=^v!w1f9q(mzcTEoi>qNE$@GSQPeZJ^bgbl-Bzvk6}Dx? z-ok|o*_FAZ2-oy9dsCOK=+{X*A%9yaT}@Qekie#48fBu@6O=jLI_0P-Yt zP3P#ioh6V;8*B5cf(Qj&IVe_R9gU2WSjpV($<;fLi9?P6c=cdbQ#*Y=*%QyrccxWe z+sM_>$T!;KhRY2Yauf2gsx6j4pIeY7%?VAg2+d9hRYmhnlgCY*xMTi?7lZ?o0Rz5p zH9-S*!bOZdbog*GNA)c$q+Fdw##qYDRUU5-IA`f^2+ahbzl!NZyeS3TmXObBa72u7 zImqyU(XyZR#Yc1dr+e?r+}hPiTW86e@^$x$in#3S%i7I4obUe_1Omvuv?SGHNJnGU z#Myg60jK7bsWRhdHI=R2%Vt1kYM*i(+ng@??}oE( zC0W_u-wd87jO+a;o$0fx9E1d&-^{Ly*s^2b5?xVnOO6tiE)xd!cpOY2TQmLOWkzjT z?aoxO2Y90^_M88>f%H<%UiJQ;7BrIu!jObm5^%DRZcEF{qr;rT3Z|pWQu(!z**M;~ z4C=q6S{=qO7=?SO^?;x&TAE>%K05mPns8;x1q#!x-#)&@OTn%d15TeM;)j+&FSBN& zb^XGrzL=oXkn(4!r>EZEGt=`|LYfYtXaql|aIhF5SC#&5{KnM3U3;5$Drr7s>(*xc z!*!%I%t+15+!ECgZ`WX8-DcT4f5W@?@3XMPM6@(fY)Rx>(6f*c;-6L9$8;lsGk4I+ zUJVv`_=zbHn6jBwxj@R=lkab3ZtS+c^Mn%?0OJq_nRtsu@=-Mkx_>I#WRk4vfU6Trv?b3auopC z3XpgX8p~^Fu1r1yIr<_m#P#@Xm6>wy;mdlDk5Uwrc@@QCd>TvYH52qB3c`kI#=L}Q zSG#-n?k6U~rC=5El?ik}jFfQwQ)YPGXz&4fttSpVKU zaZbru{fRZ4Yy@SY{MlJ2`n#XxNtoMbu7PiU|2!NNh7(o9*o&mE}%Gt@gPi#3pq;4;@^jj+ZO7@<3}s&>wj=^#sC@C z@1H&%&ZSRQZZF=+GohO|=WqdmSd+%Q|K&&zZ|c#?+xwa#n5-p>jCL^1>j4y@ow6Hd zvxbK~w$DC~IoF3z?H5$3jry2W=)nX7#TLZtd|WqSDB7Jj)e&sso{9%t*m&((d`}xTSGYUUH@ozws9PkE>8xP}U)`40GYDWXm1q}5KgR2+i zeVKf2c^F=)fh)g$%2=`0+?Beb^3Flg>Ed7b1c#+2a@T#xM4qVb8^1w1M-n=GIA2TY za^GRSXY+)DnVLia%K+^!Miwr6EW{`WC#@$T0|4bi7GC<2GS30OE!4&)l14diki*9X zaMOw{p?O4`1hWzLYOwAXS@(rW$0x0aH~;)kZY)ysD25gQ39~BCeOkHi(8z;#rgD(j z4CRO|UX&f^pzc}LOQO$Ch`+`Eta)D8)Od}Q?queK*9}7pIvE)$5dRk;KO{|F+dX)2 z)_0J_P>M3G6P*r+vKc@<*2ORSkbTwr*JVSI^%}x=uim`5DLPSuog+V%1Oi?hIp@FU z`gGxy*2VmW6UOhBK+a3mDd<|vl z1p!h$=(k=T@wfgMVR&eAh5;op&v`zDL}Cqt&iaUR78KP6{yT@FVe8B1rKQ2BL>Gx@ zd%DhewTFahdetRnGWc}}QK#(X%LwSxsk~bI4GIzQSdLiEAn~PzN=%lfTg&6rT`4Ik zI?B?)oYK3`QKvfLCU2=sr7B{K-PW~h)spe4Z57}r?2c*TTdc|byZc9GD zO}0Bv&R99bv{2dX$&8$LAT|MmZ;loN4Z&Ds#s~~N+lr02zwP4mY4fGkR z>lXzF85$B2a`E$W*rb{pQ=%78eXAHTX&JO>)!tIauP@QE0Ec^Xbdr0RzShUIZ(bCSYJ0!_ZrSmi zVMROErK8Meu(!A0JA*C5pZ8G~rlh6CATvyx=aN`+duF;-P~|8RNBw6t&`u-n*g!v+ zD0`{+O-BybUq|RqHW^W0(Z+t7fPp}Yq|fCup%#~Z17h>ePSUTas_IMvjR8Z3#m2@e zQHZ|r&$6_WS*Eic!7ZkW7@pc2o zN68?wtfR=)OMNgdCG&FUJ(E@-u_wCI(eGD$CQHkHTrR*Prhm8L!>=}P)C_W}3mlAF zNinZTj2LG>pS;6M2nY`7dA4C<3_NX~v>Pdp$xyM}xWDR(Z;goys>FucdKtVos zln8V}wY0WA6EsDGjp<|IFw}#;GM`>QKFX{@vQXzPukNKJ363X3A9=8L$F}WDh-g9+ zQ=b(qf7jFxukWb1K&ikl%@lrY4v1)E$^|d-hC6vKVOfnM?Bki8>Fd zrDQ_`pf8)TK+_o=N*|L)`xYj2}?6Vm2lq2qpDkP;&b_>jv55qd-lo9-Rw`JJ{6CW8a;b z9f1mOvEk)%X8Gc;|G8tXPX~nOgbB|x9zA^UKx^)&_IX_-k^6PrOt`2_m+6@95Dy<; z4Lx|LOW$KDeDCcz0L=Qf*AuP=6K(FwN5tdHut}#c9*3;lVg85343dJX?d&;o{D^5w z=g>t$p^JZO%uDmCo~uBBI0hw-`HYEDJy?X< zJ%@?i?hHFkV5plMJhnp39(Bs-?X;qYd(X_4<1wOO{u$!w-Qr!y39t;Y(mk?h8IX58 zFLbO8dE6a`k)JV9=tk$3-L(@&6Lr;t`Lu_7M(n%o%({$ZjPd*LkEJ3@3wShn>eQ)L z159=Nd=QYgGQPNh0`AykR}0Kj3l=L&(Fy|DRf_0t(&Yl}ag|coRX@Qj6MevJ%9xI0 zQ%tS0JXX^#(&bT*x;nP@?ZQ6N$5vDaA6x@!@#aMr9@&CNZu zp!#!PW!}GUK<|YeOK4o#J7fw0b|8hQiH zBW)?`N^0uCV}7FE5d4C=1s}`A8oNW(6(j z_^A3E<}P~lJvvUP&{N30Aq!(&))CeODF0e}9KN3O*i)YG0{IRYQ6&tEYimyiYlf7l z@&~IoZEA#E<9{I3-n4@B4l`rB@3yExN-$EIz%eBlif9i!o>DS#*0^!wEB>P<75m%YXgpzuS7^!gcH|K<8lGyU$azh(XLT&^G z)0Kz)RR!Vf6XJRx3}(hRq^vFKnlW9afKpg9C-7al&~)(EV4{mCCHIbb*8}>kSCvCiYk#9PBVFL2=B}_YIz0V`Ff;e5snbx`rhC0?45ngWVGeA`}TE`xR>;jOHwY} zH27jBuWgqu4Jq%`73Os}!6>(Z`-o*^H5yVGnt&}Y5mrX8r)l=>y3Kgy_25(>jW{hK zwKq|<`ydLA+CN4>IG(`uobN<5SZWnViaYYc!IQf|E~y06Ifak{T;ug^Cp}_cMsDxZn0nEhv^Rquxgo$LOGv4tmylS+ntV=eH@$1*Wf5!=!_ddH{EpY!4)`1*r56H^L#Sn79 zGx5b`=3@tZUaH(<){Q%y!ujuC%%RPDS{u0iRa9JDfq&LB&0mLgTcMmSVZP*;0S2^2 znrt=Q6fweH+s~0#K@Y7uxbN`r`f6&jT2>D);$%BjMZP}&cJa3_y?Gt{B2U@Rm^OX- zZ?o-Gy}mBDF)>j?Dj7yeHacTq2c2&xlY6aB=+*u8tLfK?uHvz!B%|9k)HUzUofdf8 zZf&=Aa&o!=n1~tN$<%ZzAgFYZe@q(;(z;yP+VkfJOAp(jNoQJ0{X_(B9QTQh9>ZY7 zRpz}4wrPuA?B4cfPn@q3sgxd=#X=0NXe>~a?vu}6`8(WRzidTc6J8Jf9XDwhZG*YF z^3nOhHy5N@ilVF2tXa}W{|!V#v6RzYqYnBTZQ*Z}_m4anO4}zxScN-7kEvMv)dLRo ziCYsix6sJQJ`2VtHl0o>FZDj>sIusIC&sx65qOUbRx=1b5#{lQu3syI2-7TCIx9C^ zD)U>v*Mkfhb;#2)WF@|&wH^o8VXMA)d`wvCMo4cj8bz5TWlVoq7|40w%m-(ly+aTA z6LN|^F|B3GYJJ;PIM~OGcfRShvi6-XdV*B@kSO$Icpvdc=6z{sAgPNn(c-|lveUXR zxA~8OdB5X(n>NxVrVn>Ci#pwsE;8hsW^5-|$1)YW^AxOpy(ishGiu@6bEiZ2sf(GJ z^?rP>J?ic{WA>|3AV+;A#IQ|hYBYqZq#rXlG+|nk4pyEs@6)HHapEEy@g;JZDu`*k zavEb@u4D@plU=+uKA@Ee#fdlhqH{^?u$Vi9$$AQb4chM*JIy85%PoOLq%A5t`E2Ln zWuM=6`e@fvTNszR8`dG^dLE)5=7UTKtITi8*0FT<#zO7T&wtiE^h=CeoLoH*y* zB=|3AZR~P#(I(xJZcgm0$l=Ca*WYaRP+k&5CS0LaEliNAy=p4+35A2uMMu$=vcR#U z%Sjxm^%Nq~-jkDvZKMM@r47IR)(WsNCvvfUyBC;bhJkh5C%!2ng$u<<>rohJMpYqv z?-*!u<_rkn)o1(S17F z7p=uX4Tsjo?y2X7kNWPgeOXof6_OsU%zqWzMi88B!)jJ zD{HUpMZnZGFz7<^-AWT<14YG1D(ur8bd&Q)##woJE{YRrsw2QA>pPWZIeT}`7Oq&a z!jC9m#ejdomr4{Mz}eN+RS1s1YnegwzT^xvQZ%eqLnV;v8Kl)bM|RxEuuP(B^x#!1 z>vY%H%F0UDrRw%IDUF&mX(^(v{W!WK|GYC(MXH&nkq$BKxwTZz_F=k+yU%WPr9LWn z{*ED|J(D}+c`~=HIhEJU^zqW-M1-GDjW*-fp4EzWr%s(hJ`Q&Ccz^!Kw_i@j(U|jU z+0oBOC!H|`S2b5)HR$zdkCU|}EvNxfuqsaEKC8}y@;B7!quRrV57);Tc!YBG+6jH=e1T3U?b7DLX?-$R*8Q%(?MZrhBiNs7)Z)NKidpgPaz$V|d&CZb<*f3$l#*=aFR}U||oiqE> z>oV!`xwsHYN#e}Jj;mAG0j!u<*%})OCN^65)EFMH36F9!1iIYFP2u5m`E?M-L`JN` zuU%T4WBTG(`nZMT)gZf4@Bkha2Od)XslbXl6&@NO62Rpn zmoJR}1f&NYsBJn0z$XGGsZe706d*zE@lFivM-@T&@)LmFE9O_c29M;t~ zXEfg7lbfkWe&@92?H(8PI?x;aw2}Cw@GHrTqn0MO97oviao>NfO^Dg;&<2#P^?-Q#Tw~WCS{+l_k|k2mt6XThY!oj z%cs8TXTQItj*bdeSzd&4!pQ5_uZiogHCo9x0J}QE2;`FyXVdwA6bt0ap^&F;{3=$T zOerd%()}u#>d%(xf+BbO9N;7 zc&n~BgaEJ+XzES&&O0I6Br!Fcx7a%%U^BYJ?>tG*s|Wfx?*K}fQdF{d*JAQn&#F}bOd6D0NyJ>N?>of8MJ~WN&$im!mD(+2=gu}e zcI>zoA6=Q*(bV)(US7is+x_%8is1x}J-n8{$xh(g`ifK^FcDHIV+?HpQ0+5)wtSXx zqQ$cn>onyt1J16;NAXc6CKvACAOHCsr*sthxIJH-)8?5zvjreaMWHrdPYuPZPRrC9 zD(3Cxef)IOPuJt%>FBI*rLA!uQ%Bj5sSiQtz)GQcDB(&#LtWM#G{uC%<4Ne>Ie&w5 z7bivGJ%mna16sZWhY`C*O(95vXu1dY-Mpyoa*d5w(C$Ffj6%!!JlyNOX5H0-7FU9) zOU4}X)Ik7EPZ+JcBK~9}U1blh4Z&NOrqL1p5Zg^l_8p_?}KT;!S|XV~=s}>_)?;R?i2$Kj>|si8VG|%eOZ*7njfKjrPwbLl3*5W= zV2k`4N_&dbUQ>WSlktV zGo2uUZ2CJt7rJrd5M&$ic#cf0`@T9EBID&_eyBUtt8K;`ui>#?KnN7$u0_|baspx2 z?mx&kYc_71wIY8kKj%tOkv=i0z9QefD`hC~9H`NeiC2U7K=nP1tb3L`%;vlr5J?WW z0^tPPbtQLG_?1CT_}@F$p1A@W!L!?}Tl!hWhK8OV`tv0Z2iF>R=ur00KQlS(rm|hF zx!EXXX75%N3@qcXFYwF*Zg|~{`*<>zPkTD zp#S|aHb$M%u8>N+%{;%}x?Q3-Ee({zt zUOsp*0nH)Xw;PI*O(GJpSin4a=z5ziC4s!>=rm|&+pHV@{I=gTU|B2B?Hyq(n1>*a zU4b@Yy%3s!md6E-dlw-^w&QyMrs}>&;_S?V2P)hp8wcE%{`b>5iy89dHx$>!HI+IH z4vy4pW@>AHEhfrlA%7?f)7d<1mtu$~kJkNF*@nJ8iFw-eLLK`?jut*qEa0LvbuwB} z1GlJSV37Bjt(jK+F$37!`>MhbW#f$Snl)<^9D6B5A?G*~VL&{(IAnDQ7nX}$|GiaK zTOm)4d-ZCLAR?ocMQ5irsGwOuEb5RmG7xS-jtM{S()ZtA0UM~IxNhsEDa%MtVD`wQ zW}!&UL8!mhV)MW6#x*ktTb6Yw!5p$ZcrIOJiJD|}{7P3QzL6iZYCcx-kkx7^c*T}u zGkEa}8DUPWz~tN%*TWV1CKy7+Mw*d&FL&eQ#@L$(Pkt;teHc}Turz{RKLH+b3*#*~ zRP%$WI^rDWFelYw4q4}te2Md05IcvYC~Vq*ATvs6X((#X4mCmJ%q+kc^1}Bwje%6` zI5RZp%er^(-X3TuW@0o(HHvG0{&@J_e193aLRV)SbXfV5w7FX{baLRb57+jr;`8~% zf|XwspKL0^GtL##YAM7QfkJ-}Ph-)o(+j3QzOmB~SD6_aC+Q@VD8^fd5u)XV?O_r% zh+is-q_|j@D4vo((~%++|7o z0~f!(P#I5#OsB`!gI>*$YZc4zN<7S+lGLIF&)*XT9BF>QR@;|xaZ?Vaa*waY!nkV8Gn67 z#vszX8zxXDuS`{)fv;k(@R&vF_z$-8l5YjVWU)gLz-EWUz#z9+)=COxo zBF%+ ztcq8S<6)1hK$n^?v(m@UxSzH^|2us=zwwkCDKa9;Y8uWjRa=&p&gnIjKdTlzk)%7R z2+x^?vX@5@&5)a<3gi*5d0*i^Y>FxrDOZD@xa6X`s0T9ya_@mv%qfw@HANN6mM+cX zHr8b0bsnNzL0+&=cGnLFW>|iDIE5vsl=d@f^w7IoDsQXy?cNyb#Dmy){ye!NxxdEG zu;aG_SD2>qnc)?W2~PVBlF-Lkk5@Fy1{|T;q{&^wcbwHHp4HzWS?nM|FEa@sAe`F% z;_^$yV#;QE=))pTFqJWH0EW#OHGh4_H@eh^5hr7_q;ubC@Qu{Q5#O6^{UF4fj!qH0f9jHw zUCo?_Hz8E`oN_1FZ_!-NIF2i^=B;ETiX3Ppor`qJrjV4esoblKmy(basbtXb8>lou=Wt#JgV({E@IMg0mK~7mlh7e~VCpu-7v4M!+?o;u3c;B689(FBOEqa1?!GN-W5z*B?HtcSyPj zka|x~b#W_9uJ&XN>FJd27m)Eq%15ENGS2bdYwOfuDvA4{p>IbY@=WRV>?cf341b3* zlX>ZqDuG)X{_?S-Bj4#WJ+>9POn4-w-GTgy4RFL$DQOmec%jcz6(UMVZAcEN$bBX8 zV!ncOG@E#;$HMB82m%dZK_M1p@rPsYw7qTKwJ4ebL5~_Gha)R~HLmB<&6`aLF(Uar zw_aM~FWP~4jkvPd=SBR$RQL<(W&ib0T)Ia%gy26AkC%|y!n>1t0RtlNi8a&KR-q7^ z-lGrDiB~)e6 zu2uXKbJ&yevNG>^(^o|Az0;h4+RfOQXFZh`06H7yi(8cf?|WYFXT>%_$!pVmn|=Nf zt1H8gZv7xsiD_FL(kFO8Vg`a0lg@#qJ5DC|9_wP`o4l5CwV|TVjKBO3R0w#5-NnKG zl4Pf9OTXHj9@7|asA?-FIlp2AMYUkW_sl)#Jr2fVy{kMN6Lpy(rtV+1RSsPMZ0)$A zu9!#&g=;`sUkgkt!y)f^W?@&+AF#8v=Ov+q@B*+|IlJF!Fxwch%wBE8((9>nJaJt6{f8s(x(!z zt9mh+3=K<6SEe^(Cv(Zcp{0bmW2vd`k-LLtlUcCgQg|7xu<=8;X?%hbvLWCjuUbtZ zWowjPpQMFu-$$=*^+87D-%tPNn0_ppzT2A~9J0>^!i^}sa;9gd8$gm8%5b+s+vcG? z;u9&SIuyb1R1=T=V+6?s|2U89tI2wggTr7=L>daI_*iwj)6mI|<5M#Z(`i$`n2RZ^ zstUVz&(kk93u4ADDSr=vit1^7?tnBMGF9~Y^{FML%xbiTZ)IY06G z_gTOs`F+BuL)&X8{RRx^68n}=zBVbTV%f*7y`F+MZ~6CfXw^1Dm2Q51e)XV{)1J6QqU0-kfcf{*W^B!kta$N4OF`Z2eSg=3g+GCK z-Z`8EqSRd5)_Zt@wbNfs-hcSu;n8II?!A_|ldQ4zpLaMvX5@@lrQmFS`O?BN;#Es$ zb4!Em)jdp)&(Em-L|(i=Vj8iq^^G>#I&L00u~HviD@AyN%Gfr+({m5~cuuaamtp+f z)0;wFBWJ4cP$55i`wtE*SL4oVop^>6q{u$>kP}3DX%<{sToB#0E2ofoAx#c@Y~lu- zKe>tv8(Cnr1>Q50>@i~D4t~-+$MS@UjAD$4Jf51Ws)){DVfYi$B-}W&Km#DYp1jTi zCTPN#11p$wa*;F#OvMk;SL^=RL!RD26I~W)aJqZni;J5*V@uEuCLtO1=+WcOw-Gzd z`#UetfZ;gkeDvnlr%l4IH?0a9AcG<*`_*0bGzd#U9%TJp}6 z)mv{M*y<@_gR>-sczmyAfshfe1g!HAUtbfNTEfYNiDb3KEc3E@%ARL6g-)vGM4oIi z@@trT8bs8AK|Qy1ty;4fI4pDZr85E1(u;ub`rW&pOIz8GPfgCMGiWn5rA>V;y;(a_MS zseaVXcnu^{If`30I6%8qs|`GQZ_+dfX8Iq`T!i0r!#^iY%pl<$(+jP@8=`@p<#+U7%XL1(Pd0oIG?IX8`*G_mV22pEhB1)n?W*2Kf=hUAA!c z-PTjmY*U=x1|hH3`{R5!1MM;=|R#}GDcg&ILWN&kOfeKD^i0(MsA_hmJeXmU`@OMTEPywiSg z&+0Ic6o*z06cPuH4CaS^?>jOn--*;Py7g$9?J_LkPvc0~1}}^0nN2EOMQ~zIsZNp) zNEOMpPx;fQag=}3dY6innsk0&oh}%*abrR4Lo&449SJ#gr9#JFe^d(w($S#~i6e<8 z-}{IY|F-q|SmFw{F4sM0WKv;73pb$u9 zjdMkI0tjaTiDJ)`%#wJXnXrC0-RzMvdfKl6UStdI&*Ocfm&{8444^)d+{eMy`;Y*3&$WJeJG?e!ob+jtomB ziC&_cA4zB(-W&q!Is(z&+J3MGIns0l^(3Vo!ROD>`9NCFum5}#;zCJP0cXwi)T4TarbrlnD)>PZa2@Fe|Z;=u!4kuaY| zI{!;}PJ7GQ_91FF(9*I;lI1gmYXk2ZnVB^Z*9y<NRJ3@!|n8 za{O!2eqxjLhywCg<$1@Xd@oS8s5^Gsd7ik%RCq!?O`wCzUFzh9al5!0JPCCjQc!nm z_%SgINM{0~fkfX$_CF)Pu~m}vZV*r9&Z3h^=LyEiBy1b~{_{_BqWrX1q41>b{?OI2 zT-4y-KlMNV_2tvNy1Vi}|6l(5{{dq7C*r94`~L;b`g>EY7g2|bQYyE}iUj9jBkhk2 I@mlkL01zMttN;K2 literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/kda/gb200/kda_b1_flops.png b/benchmark/linear_attention/results/kda/gb200/kda_b1_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..3c2505f81217fd67ae452f143c14f6690a42e5ad GIT binary patch literal 91525 zcmeFZc{G>b`#$;zB}zz0lr$lc$}EzpR5BGsL@8vR$}EvoNKq6C$y78LOUPKLh!Dz@ zDJf%_oomzgdw!qqT4$}Z);a5(^T)A1$-QKriR*bMs7xmqL%O9 zr>sp;44Wv5)|Fu~{)za+NNCkY{rIwP2TVi6Hs2o4++t~^#eTr_ zx9OX`E7?~uaj>kS7xWFH82O*d{$7_at5E;d|3p*Vx0wD8?d~wwM?2jAR7tZbve2=x z{^w7ZSaLosAOC;+#EGty-Sj{HqE`ONM7sa@Tm1YLoIk_%DF5frgl{z4mH+$^Fn+4D z=s&-ewTsC`{eOO|#~Y2M|M_9cVWN$K7yt9qt_;)v-}`V}KV&KLVv`Of<>j5}1+$B& zst7UK(rcUzj>@CCvX0%$(u@n3mn>PbjBERz*7ooF^!2%BXZ{-3r5s_RUBX;O3lJYC7K9uF1i?R_@A;8!WqQTDW$*jaHwR8eOf`Qrusce93EOvQL($ zV*T3kqH7sn&RN#kSTW=`*GyLg$opQJj48=Mzl3k&_1PJ+|}gKQf+2Zv87h8`Yr9NUi5QCvIEER)@ITB)JaWqRD6j-Ec&=(XGU zpJrRf=)*A?K|@uMI~#NCH%$H+nEK+P9wFl_Bqt|l_oGfQJmbI}kvEl<2h4Jn`4wy8 zR7_5ty7cJKy*RhAAKd*7y5)%vwFSMV1|%~rw6wHxJSKLmmG@v!WN~o%^)NB9xS@eF zWbN*v=4@N7XQnEtdj%GB zgIe&(>G2`s(}{k0QghA;QfHsP6j`OMqw~yT!hza@Ppr9UpAAcpfj&v9O8EE#u zetp@KQ~lMkm-O?GJ~^J-q@l>-mu=g6b?k|8fls@b>J+&YH{%#Q!YulPn&wlKijQgL*)E@Kt$B|8=t=@AdQm6Lbld$#+Sh21< zg_)L?Ho@Uzi~{FkW*&iWFLuoJI{bQjdWXr^m(F_w**4XsWgkgBy4T4mLp`^yPB+$L zV%Rq}HkO5l=keGiV|<=eaBgU8gGXz=Cp}d(KGZpq;N|%?bwnYNy{9U%Awgzt=5GvF zbNBmuBfGQm9_9qB;E~d7js5;A>Kl{z=g%1yI?o;l2JV+vi^=>rAl3b6a;>z(!;|OE zr4AVxC2H`#DKFnrDdX$wOSfgqmLO-GH2oas{-bGGJ71l>Ue)sTt3N*Z?3Cn>!S-Uy z%AV5J+`PtgUp@w$B^Kdr+m2Ki=<74s*w{=Qoju@YvmPnn(~{u%gchbo4xq@1mu;U@ z^`5?XQ9Loyqm_DeTe!z2^)O9ti}jl}(GLv`F)%U~Ma!ONVPWy{@u8+?{8V03w&z+A!3HMvPYfjW z4GfCkyivrH!$|v{jUvTW8X$La+)ZJLAgD2eCNK&GBbH z-oLSLqm=Ux_M)Pq?Ur={4-*nhkW^faW)FDGhk=l$+b?!SouOV zJKy!~Tl(;I6i;@#{X@Txk?HB_)E-%B4K;rRc>xzLuW_nHAme58GC#C zqI}OOzqfDq*&myk9CMZt!A5XW463TC+^r&c9uv=5I5`s~29RsWSvz_1FVwzrn#2IFb^!-MVR4aM%sn+}HU*K|%i5Stb(` z6CsOio3CtER#tS>-Y?J1{4+9kh0Ec=@%Wc7U-k_M2vDEQb?l+ruwes@&vu9J`*^c| zZLq#CM%O+!D=!cn^YWvB@1VU?;JaH}n6#611#})=S9dd-j;^IhEks zpGVyi-`Z6XUOYM0ua#|c;`%y;wTJJ^z4@pxw@gn@FLR>HLE7b4%TfxJc2~viZP#vX zIowfrkyihe%eJbjDuBuckh-4af|yZ^VjH$piJqd8>w8lAZ2>$ zl!>XSujS{I*hBXuxYb-VIsYOl*dZkHyk@v1BqX@REyL=k_OTX-ln?aPCN{oymqgN{ zsPn(Sl@|Fkg%2-VBe9Iis7SOK4f6A&yWsA=K~9dRudgpjGj@@TQ?HM9f_n1T%a@j{ z;NQ5B4rxFY3wq<`P2X#rq8&I$H{P92(u!X!qr6AC zr5!eUO>}W0Dc}n?mXCR)9E{w-waZ*HxKhD&SUIZ43!A}(T-@Zb?RZADzFM+&LUqCc z&L|mYMgWXb63RS>O%ey}uLlR0;lo*Zd4nE4+^B!-SnU2=n=N~q>c`wFgarkc=I7^M zm>d;E0oOvYt4lXtmwB$?gNwm%r<W@l*GrdsO(wKRUlREjMF1cgpOBw8U<5Lq_MCz8{jSo_P z&VJ}}DL@dh3JMAe9M2GN`PoRba^*^~_l=EQ06*paOq@5OqOLDrBXJVv&-M7)@t5}v z2;v=%%6C^GAly@=F34%{cflMZP9%~LlKLYCnC2dCT{}E7va%L=&krynwW>=fT-`r%=UGOF zT*FQo5yKq5y2r>nC;`ICN;$nGaEd5fKqd-^C>)IJ{;j&MI!cm0S`w z@xE`j`yYl)2X5;M8mAdbxBvK@D&53EU)mh$IrZnrE7AD4I6qXSq$lO|^+%=b+I*TE zq=SqNkPY^R@~hBMMF8gF-L-$LElQg&;h=x8=WVL)I`rw-i!){us_|J8w%>#)mXxDU z?sm$IZW9fT4+#k&Q4!RjQha&2Kelvh_`5!Fg>&`7vF0^#+`r>v@=NcwHXJY+k`4*G zQS0ui@!?gKFJAGbbAL+M&Ye3~)~6fCbglV=L*pACAKx^lo}e9zhb_7%Y5V-E!sKne zzk3zO3#LqZK19>we~fG_P*Z>V-hSHfd)F8*2gl}L@u-a{54V2Hv293;E|X#ZIDs{& zbDe3DKP8ZrYg&3$Ci>nF)g5oI6*x9MmQ0;VxiGtK`$?ZNnaj&o^W$ao((m_lt;+ZN z&rB(uVc)5GFj{7nFCW46QXb=W_wzT&*mo#L$)0CM{umq{E^2#o>Bz9ax3=F6=`rJv z<`AxE85pfv^LaO@UVr=I!m#$#p8?j@s{@&NWgck7tM1#kuMjagINGN(JygCfNjrgY zvqn^swhchs`0Uj0^|#&ukp}(x`YImhc#Rx5ek^w~9QMZ$HlW%H&0rY-=lG)po zx%KDIpC`@C{Bf?vCnx!l*c`j7E*{)mj8i0D_7zz}Sw$uG#48s9yHPGk9*E91Zs{r4 zoxk1wnFkDoY^CPpvhAAVLe#atYM;h0p`+vR2%@vpr;57LD{~7avKn^q@$zbq=iKwFw6t{&ayA{ zwU3YOr1qef1eQ>^mmTjM{?#6zE`el0gJ!cX?Zg_?_y9wZazuF>-U53ZYu%ic<2jX; z6XNvxKC&jMRU}3k$Z0mX>wZssxw1yVEa?1kO7+ihg|On)qPc zeH0#5S*-&Xqk7v^|gBd66<;_&7((;$o2^xNukR%495Gk z15FQ(jw+){s^1ol6rI|Z{{8do?LtC~3NxcCv2{bppFQUr)%)M~*-sxYEh)J&rza`X z^fq*ZJzC50iHUX;hF)|n+Ak9mx8%KkO|z$SWdG3&*D{6`E6PzbOPujs7AFcM;%_J4 zYVHYt{HOob`QL+s?Ml?1)T2*!)z`dqKKJv>izTRkd$qN1jz>i|4@Y`#TWFwNgBF+v zhm-GaZW`k&di5e=1aBNunuV2h(&z=>t{+x0Iko97Ey67W~=&PjIS?{VB_p&~L zS1v3U^%Os}{AQ$~h+fkQU%s}n{ssb!(u`g+1}gi;cufjcI{vPekr%gVk!oyevV6V# z*nSO-GQbtkXlo=&(EO6_9IZiu2l@YLCthiydkb>@{KSA0QMPaY{)-_YD=sth-rXYh zL&+xi)GL<(ySZ7<%-Ss_B_$-ak!I(%$cd{s>NM3eZg*wRi=-LmE=tzDO6q2j-%=r+ zZ|ISW%F0x&ClKx^aHL@V(B1q3IfG;D$LC6WSI>%HFX`o7HosH1`PC&Y{hJ+R#pOGq z#-tU>v1?z%En$_Un}xjIamOdd;rw}WGK7VN?fUDtg?}*jD5Excg=%@qwKY!Xua&sYmi)$d6}hChnqB7`51gpT3QNG zEC6?pZ)Txyqw&#va$Gz-19^&x^26@!?{_x4-C8U8_;{we^#s!9Rag1w=cec_fi=+3 z?=>}zYZ{icX<-TZh(weTY``*WyRQM=oW*tbKR>_7d`{M|32sJV$>es^zvr=Z=~Caw z$Vh6>_&|2+uI?$+4B-81*RFkXD7E|5-QB&{!y~WhIpM^lT3%!|$5~8XPD&C`6bfgg z{E+Zcw@pn?I1X{&Rq*z@UltP`x!%+o%1R+7hzn#lSru9Kf9AVwk z#B9sEcCAQ#bCMSN6wOBmSDXB7d{%@8L3s4nSA*@_x3f?Tgwd0Fa@F_s>(|J;Pfsdl ziE*g9+Vi|{{PXje{awGq7LL1Ld(9l@`Q={!_?XNS*@AS=(yXpgoSoSk*XSR;rzRJl z@RpCxU!VPV_@}wd?;Wj|&^@JOS?$^5!}T|@rxaWCocEjj77`z+?(HZF^lhC^tMSt@ zJbfR(SFW|jxM|E29gbKQ!scvbLqh{07<9gDUbAM+o7&o)m8d)9QK{NMis2r^&> zI@Z{M4pK!B;J}ZVoxtkhz$qeos^2wb^ug+;JPcaBbFr6~7s0W&g^w*#DlQsp{M$F! z!5GDNdJ;fzu_n<7h2u-=))m2;3wJo|Maza~ndEr2FHY+>3A9|v8 z&1qBLg6}+yai5dDf+WmeF zwXW3=6vt6+LDe-zMn)ZL{?;ZQZg8%pzPFBPXIdT+)_=;UCPpLRWUJNB|Eiy3`~ve6 z{bzfvj~jihbl&|Nz=iKlr3oO*dZo)rPhJ4L+S%Fd*|+bC`rVG-jwes@0R_Z<9$nld z(Ca8D`paL`D6cTXq88mgg^rhti|f0T*NLt_QkIfqLw^PgD9s5rc^pABy#5z2(p+A#E=?Phx)))Sfr{+9t5)aa zOE) z0A17reSLYPtA+gWnW~73;GK#pE13Yj1dQ|ZnkL!JRWbjNEqhI+4NiFNEBhcDKnS9zrF&qFPHNeXF)WQr)kQz4LS4vE};O` zKhnPZ7e4rM!fv1^Oa619Z91&EGbu>4Ytn0ca6O?75Z1da>lgx5mO%yZodhUD3WA=7Bnn(oybMqI~#eU;7Bi3Oo8AL({zdw9V5Z(R zt5Z7ueK~@EEY#H*y=KPro>TycAML8-I1yrIR8wctqjPn%sz>Pzi4P`m=lH1J6m$lp zz-{-Objo@4suLV<5~%RB{`Wyu##~2JboUQ5XMSf(YfR}D#7q5!y?|SW@{;ptd z_8QVupRVc}am%{5T8Vy-PoL*eyUukH=!1qr6}e2d5KTN`3^uYY?da@$<~8F+y&aZM z2<$M-c>471kVJ34+m`nZ?xG+2Toe0ukLzsSyqUllujwJRC^=U)8X6h~1_p|PRul)$ zcpU6>dv9;jXNkT8HoyJVLe`dI!L*XKH$@jjd|=Am9c+9JoPQiTF*J2bNZWSpMf5?u zXLsz_@sx-wg$V!8@|9?MR=qg$z6hui$i=%2#qq;1)8Ju?k?WzkE59I6G@<8&%K_M_ z^DcY-gGyL_B1X^YOM2v-OKExexyK`T`;~JsxoIu8=X5n<=DCjiGYdjum=b>fGExKr zHINY9=Ni(a9J{ZVt=D$$>gm}sTVPT9sAEn?Tf0P*1;m>;^Q#oQ$x(}v(o(@g{)4&i z@($x2f{~5P3?88=?3m-+cH9qy$S&(9_OlI-^kJ>VnqSF|B&ot(?pw41& z!__I*vutNL=~RTjc=o056YXtW#T)qCa=c!pJNA9LaY5Q&jinHWo<4t0+WwPBlVZl* z8VtvR_Qo|Hj&Uih_)s0IRD+f-y25-zYK^3WQmy^6rAt|VuUI2#Q}S^CE!~Q5ug)Ky zc97b_ZY*@hnP+s9u<)|E*(uxLO7E$GY=6rue|^L5xx3nYkX7NmE8`(}+BpR3l3eiG z-B(doA$BammLiE4<7n6eK^W%P(a6io%be(>_Y~>+W7PZO2V>hQwWRQ^dMRN#2eU&% zL)Uw`w&0PR`aY@jbG;xOv24uL7%w+>%4q2X;LwX{ei>omgDo+H?ke-Zi7CduhJso| ze8fVhJYN0~MFRbF>D<)hWE}AK@rIZ+P4elzFO5@=q%ICrUZluU!YW_)PR-ow2(eR((N8}*9K)=jpx}f6LiK%rD|jguaxq0((Jx<)86p>% zY}~Zz&8JUn3bW&M$|Plm(52l~k+=FIMoZD%BCn|!8uCEf4DHJM@`K0_h2_B; z?{>F(qVGxTtWr1Pk+g2ruHS6@{;oKUN0jK_JH}qDYS#q>ay>hbi-oJ|>awH%i+9mC zrO--C_=FUzaXzdX>hiDBxpi8+++NZ0)5E2p(^zr0tditaqwjXJwz(4JwOWI%A~5Nnfz>kqqer(EmS$B9X1W@c67$078QL)RP+eGFy1s`bGs z_S|aF^g?=R*U|EJ;=Ak|9SP^c^!?c3dp+S%KN~ZN~3>He!%uQq@?J={^Z zQ8f78vu9#HKYOeiHXEZRX+GS?ipVfQ*5;D;$dR}gdN$)L$DyVyYo=sf##}>Yw7^yO zcN=W_i0-9C^aV;_%*aW{JwvJ;=(0m^+_=#>QL_qsNa>tjlID6n=dN&s_M75jT7*}Q z_nbVa+xVM3kap2~73_}u0=BM1^fW&Ip6D|Uj=RC9T>c7BjnB=lmF==l+k%c64LuzQ zN&OdRmV@?6aBi7+g7Tt1a<~7-xq%zh_g_Z$?~Q2Kuh2Mfp!nNscd;{7T;XzP&6NP~ z>W2h$(~Wr1Kb3;5f9?@~jMp>j0A#f@l1-Js5p4ql%OSf~#VD+!hMk-ursaYeWBSKD zA&q6&wyp(a{WN(E)pFO`t39_!jyxPAZ#m-X>KZtgZkYRAU+d<;{|fv5OONXP`bzS)s4?^T^XH*Z95>UAhrq$3;Kfr|&MGnd(xcmY{P;f( zdcodteD!X(3~CR!z^hH8KpspK)S{8@_x>d%%Fz`}zI@7RYD|C_tQ*+^BmSa?Go8f3 zmuBod`{DE&M?1TxT(=4e3T_(+F#t-hs{BupS}V#fuknj{u!%3TVaoQG1?dWUwP15*>M)v<HYE|vCQp%XSDmr$2Mh`*EBa9P`!!r#`fVV#V=E^_8GN6fe6z>|!gCRdw6oD2k%j^9 zxtT3%r%+B(ANGtI-ESUA14755feF$>D(s0Vn|J{@24tG0kj!PC92lB5 z0p&{igH1C4dmTIXg4=USF<{B?E@Pon>!$1}aUmh~Y2oU4wG|0xCs(YKztZ#kWlw&dK>FY&xopCQ^wFx#yNfC|ASy2c zUxo?l5S1R(W;=4=)2B~0D>&D!OCNnS4Lx88{T%-367)!r%BjJjLodN!3%oz_qq}>t zA`2mH3H`9<7SuExbN?kP)}BPdRUfIe?8gg0Mt8pLiU+<9DWl&oTO>s{+4t+K^QREF zjjzs&!iYeWBi$#*LDaV)O$S?Y+(C?!5k1O z7khjO%rOw4m9_N{h%}GD-RZ#w$cbb#{VrdQ1Lnjjp@+&zQIIz-f!olSKojr#-~nHh zg7;davdegEke|!XojXUAay$UZd8C^Ka~}^i$u;Y6Mv&N6@0l^qMT-`p>@7!ZlQIq7 zY-OvHJ8Ul0Nl4MhK8!`L10PWH>b&jGudg@(s1zZ2yr>erQ+Y(Snh5j|rtE&TNDnO^ z$L9u435r@ZK2_|SJi1KH;Nm8bo2=~5H&rX(a+DX z@atFJ9g)k-D#G4CQ4lY)13yC*LWvr&dNsN!vHTgLJ4r8n?XPdIuK{?ULS{`eE|3RL zOK5Ur4&5Y88o(dRhSY(v45vPAf)b$iy#WjxY%ul)-dGLvLwJ7BwS;Pf=E^xP9={@2 zdXDCMrQM`-D0Cp3kFW3+Z(3W|0spDiJ~~9$JnV5NmQc#&Cl|5+!GSKvFClH6>MRR- znw_nNbuu}3?iyg!1`!eFwQJWB_8$?5KpAmv?efoi$=R)H^P#kbr@TlwvEjU_DSuRS zGzr0d3ECN!pBd5N)aKkLY7F*Py%M?m6599<1QhH+OUOp5si{$WG&OHXsrW=juAZIj z7mdz7bm#1KqH+`#Qn0raBFPXMm>4mTuhl?^L$_gBxsuk{*w|i%0n02UC6)9v9hB^6=<>i+g`fGBGny zEH#vkPLT!Vfj{B5pOk|uUQie5hq|*-o?yy=OQK>T>A{7O?XO}GluQ7@@1F-pT@NGb z%9Hw-&&|)k^$`BB$>GC?55x*m_8bM7?fUiWbg0=uwzgi(!yQtW0|Ktu4tq`3;-IuQ zWuD{yo$mW!Mtp8|#@(bT!@?J7vT2$~tl+7`Dm;=mZ2I0kefYqEm9;(Z?OL-o2QzJip3F)p%+$Pc4YQrxwG27e}DMQ5V7hH@ZEi33gXM#pXlt4 z$t!G4dj;Vyh4&3#Hlo6^fYF!FsyXWrNtS{-c^B-%S=SD~oW3(WzyA+c{Fj+jD@)_P zTp-b=P&UTf1J*sSav$y0BAyDW8gwi|Nk=#)QddajKfH>2I`f`Bd-klO_#99#x2Lw6GN;&sG%Q!_03jr8U+Sm|p zC&;zMWo2a~VXxio<_CHE0fZ8n(&+GMDI4fbzUUi`@-AG&Gf_X(IIAVr4ceCk!(l~Q z|9$G}zTx4kV51?5O{OF7I;b@3*RLmbVr~a+0L9f)A39kwAMX^^p1}jXvJATtwOIVEG<>pGy@t$eMJ52 z8B^0mTeohNl97pnrvi>9%MrmJH`EgdE8%qYcdg`p@tV;`IY2)Fcyy8~;Sh|2Tgm*( z^Mo?VZL6RF1fZRcMa@Sxi_mz+xfx{m#gM5Qw3?{vd$FQ8T$k|o;!*aFeKTI=l-oTw z^;_YeL&YI?-tdTwH}8$hqj!GIp3xFD2P(OvHH37KMA?2~aHn5FTznOlYRSAJ3Ij`h zgkFQY^o^+z4M=GCHdNjlm0d~t&-i6reqLeQq<-bE-J!oXi0FuKzY-Xzh%~7lCU6nG zOgv1Gc+F$K?i2|m{@r^BsAIv?nJ#e{q3=%;s{j(Kwl>@B^vK~S4e`H%W~)eHf?R7q z`UuNcj4*KiQCH)}d-wF;>#-W>Cm|>K_I!-lAST9&5(dW|)^*meWj4C9f95V) zVzhRhXw6vYXy8;Oh0ny&GUdpO^tQQ#|3V%AWlZk6h2AI@;F2(MsHvm88nHKT-@c4x zB+}~nUoDF0wa0kbLhBJ|emHifpC0a?kl3gYDse%?>-G3`t*7Jj=PXP%#yH!N_@nsa zJ}To&Ylss0+HDn5^CuXO;dU}$fqIWp{=y_iUiSX|`+$xN0K|#EHG8*~I}Y`ip1tl{ zIxh^wwbWSuJU0Jf{GZ+#d(Zd-0dl^3_imCoK~qdZqRf|`X}ehkyJx`Ohc8f>$wLX~ zrA6f{#isLoEKOe%BgV+Qz}Pqh)JhVh^}qaR<+qJ4Cv@=t^7H?MfBXN_=JNml;TFQ} zzyGs>^YSFK4*?Db#(HZu5F7~`=^NnhF<$0{XsS8e_0J|porD%Xx%?4})Pg6T9?*yA zd9Ce;J&Qy9489M3^Wn3S+NLLZVvvO^Bw}veS_$LMQHs?w7huMp5Dq}nP$egs1MpYz zldp~bML*Xep=lRvjd1qGK?6co+=MdDC?)eA%IuY3UfD&|_>sli(Yh0}47$5=;D-5U z46T>a#ymrfUN`hb<%0(gW=(89e81|yhZl@};5QQnz-{&bV@Q1WWx>1#losBureR?` zB3~Qj|K0Q2i??p+(5Nc?J2CXJ$av)m#fBD&qG z`|-Dp+2+S5Ce@{-rS0Wa7gjee{ra_QOem1?&*k8`LSI)7I(2aBcfpJK`HQaA3o{o_ z-ksh-QV6o{zJmw-P`E(sFk)$g%!tU23b%}V7XxH#yxTI!IcPi_!C9R|F(B49str>K zdjKiJE0)mHx5J-9$xB3?_PR$)v9Yl|g~m)30_~dE*rbJd^?cw(o5C+6!;NYI1V~K3 zLb^#eIwJsemYh0uO3WJ;VGmDF&@?_f%qkv$e@=p<#pTtlh0RK;226MZeKsuVR38VDU!}F*$ z2?Yb|(g9$GaR8+Nei(d6E7SKWK@84~aMBX0p+lukf=$>iE`CJ40PbxNf|{_agLhW{-#TcSzAWE(6>~1ztgntAtKsg-pF74=%Q~wIvV;Zw)?H71>(1NfvJ; zzm-hFcpd?~+(NM+Y#!@KA3l7TIExoi;W4lzFk)ap&)`E<)j^8yXKR5mm#oWUKKQ^Z z!0$7H9|9$T22@v20d@|t--KBH6v877mzeq;ktZ1*AU*f`Gp?#D+X@x{(1Z!=C4E&I zyaIab3x9t_Z?|jPli5$YY`fm-^_VX(F^R3s-@Fh)6$c2MRY-sN@}*@}ws{Rd{@lt+ zn0Plbh_BFlZgwwVT%8wl*3+jf^=T*WnehOBa#+?S#b)J&hlfA4sNI|}4QRzu`}xaL zlS|+t=mVAcA4B`Z_YeO4yWnIlLVW91oVM92>$so&$753L{6bn?u`(0jmZXv7rnz&es}J0PX7I!uKNL8%XGn=J9pGyUmb;+CZ&XD<;qJS{bC)uDxPL$ssNb$YRSC> z&WfZmxVLjI4Do{p(-4E8uOxCO9^5@#Q*;zjowpfeDMO~#h>;IMKex3Mn3w$MM!bge zO!oXQI?Cx+%WC+j1$S@QKuhGfgoI7VW{YKIWy6Gzu_4gm>sLK}dLse_)CBQ#hfii( zeen%kxdjUxaqnI{PzN#X;V2G%f47pDmSD1tkg|_EI`jdY;@nj0+(~nDVjJ?n_yr}e zp=SYw?EClA|0rG4NGuoMjvd<5U!iF7%DU*L=nWmq$GLRP8ers=aoU5nnYfZKFJBV} z?xADfwf6RQ^Va;`VAW}{gphKH*hW|mJk@ru=?lb=n`w*aXI%3yZJj9aI>93`sDq5a zC27M$wV{LN5;LWPUf15%=EKA(QV92Fl{-jJr`~EBQRDoQE36yrz_esKtE#A&;AHA& zSw*TBtHLW(qOmj*Mm|RPGl5okkdVnh4_vX0jg41a+0AxHa!~6M9f*)X&IsEz7!iQV zX+%XuA>9=NRQE)HT`FZJg2d6g=N!@P!n4f|b4z@d(Z9;&DUA{Yr(S96ty{N-{-tKr z5fBggQtwUApskQ{8xA_ABDi?VsATu*l zo7eVRo+9M`tf8}~Pm}Q`Q1zG4*)y+`zYZs?KVc4rbVRMUd=r5`KhmT zV`J-C5Lcwfy%G`<0RQ6gEUV<3XA6C3xD)pi`~3n#=Xe1`ypo8}<%3QxE|&uXmuA_v zG9%EIt=^$5__e;C1H_-6!Filu_~)*|_XK|rvIH%KoR>73E_2ru-y)7~jfvtS2K(pG zQQ^F2A+1c%*vbWfv?>FrCJwfRj^CikkaLhl<-i#E;oP(}K075;yHSY2*9Sd$J1`4| zQ^(T4Zsqyp-RDjh`L?#;_PASN21N!_gUVS)ejUc@3_0VV$KwX0MoX@ zi)wh4E#QZJ>kJJI35pBv0uzO@$MxrTYY36qKp_dmT-z-tM~sm{F*B=nc-ND~ zCuxi_2Tv2+Y4bT{6|5OrXzDRhDR5|DJQ-P!;7L?fGLC=~$HK~<&!*KFy%NmwPSU|Ek~A3ik!`+T)Ve)C1@$HW-GhC4 z-{Qk#m9^`T| z7htgkPPZHJQZ!xd9FSPz#f*=Bm z3=COWiwC`98XO60MC{xE^6diyK{yd4$DZ<1S>tO;M+csm7)L;~o8HH(;y~k6p5qB( zeLInL?m*24XdASco3p4YcOl8s@Y#HaVU`TFjPfZg-1DsSO(J@z%^y}Izl~XWb;WEX zhKy~NnKx%`gPMV+qEut)4{j#pC78f(1d<*HRzwG$#LP@_SpAtOq_Kb|ITTnJ`MiT0 z$TS{)bgO}DO0<@4)lK3Pxl`HIai{C9{JC|~4$4+*-X7!+eE`D<p=ALqqfEVB{i-uz#o$ zk)6(h^5aC`Mymqs33BOiL|X4MGB1v{l2&u=e#KZIN=n}4_V_5RyN8DyXs$c`>sJ7X zEo(bfTnvsU>=S;r!(ojUNKUUGT@U#Y5`fJzV!Mg&IK2nd@zPyzL_?sUrHCkiDeJA_ z@UzO0`Hg44q#e5pafmU@Tb|wObq(R4`aLitq#P~9^PpO|1c5FHSO8O5gSn#2n4wK< zOYWzph(r_|Ay+cW1x2PE4k9J^8;8Jj#=*r0e((*5InqI*0|Rv3QT`0E$3gHF zpx)+#5k5fM@sLyY9XeDB{6+koq&1L{ktzSLrK~uI)z+PSEOe)mm6J1okbTfeDIYYH zC!txWcM)ei!Bj{zq};!B8DNCBl=OID5EDS<4?-7XhpH`(_qGZ zLU`AL0F8Z~-(vMDOTwzr2XiE3I(eRA9{b@>vKqQj*V-1mtd z%t9%iXbAq3@DEFHu1nF)w6(R}NF58?s>h58fMQ@tnxx-h)hM}-PrTwNESKioxnfJQ zChV*{@*Z5kQ@dg2R+t8%V0stOhC^7N0s}ogSiC|y_=r$=2Xq92n+a~kI9(gw$aE5} zC?cDI<^Fzwcm513E4#jjz(?kDYp=2k`od*-3Wh_X@2KB{1ckUBgsY5HISit+)mGdQ z4FTXa1uEw*h*+oKB?qV_2BljLK+f#$B8XYxT8z)N;U69!A1{QqN1PT|2>aP-SFk%* zU3VNVJ&6%bP_0RI1W?#psygc^cb8_Ji-%PVNw5S{Qmk-AWq3nK9Yi;}0d7?+IWCg93YP?!Rd)e zPyXx%T1ALiCn-(r0pb|Wa~kUOm!)wS9J%o4=V}z`Ylyu<37I^W( z_q_+Kgd@)j)MA3~+S*Ry6^Zu|r&bldX{`kHRb*_1{7p>9Wi+|W=LP<_X=3~Lx;-1F zE98$~h7IW|nDnil$#cm#2PPnPiB}L;J@_M@;|c!-Vg~#dnGb~usorV61g2|c$N*&K z97qys1`YXAOn#Cn9IiEMwsvJ_XE$(34%$zuBBV+COSSd)U&Dt6_94#^&o25ott_i! z?Awm(CIl~Cx%rBy9BBqz^9R0IE1g-+y{i!F55ST?5;M9Y-4xxW=s5k*^+HR5xVi*= z#*-gNoI=7IF}Xj{1xYHXPBhEdXJ(Ta9@1@DuhLJXpW1Upn*u{N(?+ zs{dDRcA0;_GroL#Sm-gd z#&{V+zhR=}4NTPNh9;TGbQGG`>XXehHFN*ou%;)dR+4>tsk8_xw*-}`a)__ z9(V<8%jfg^7CytRJc~ANt~iU9gjL+fb+Vh-p90G*qQERHN8`U+P3fUAm^uvj?*hFv z2=}{$@WNYOv|ged`QP7O-nWMF1|I`4oKW-6d-dFUa6Z0#z0%wZAveJ|L%lBPj45?~Zw1$Jc$(7!7cqslOR2N04byZ=`2Ynk(t=r0jqjt4=%h3|EF8XUP&3m;y~zjICIIJGAPueyqo_$ zA&rargGIL>vmrRa7A}HHNbawIM)vPTR2tFk^K4`)j!(%P23$zu!kM zrI{az@|8KQx_nq3Fven_O{?xnuAqorWVf2i{OW{BK6k%o0HzuGrbBL?50LOQ zfz1FV5t6oR2^r*rUB^$%H4s7=7VVabzLIZ+MRU`E^E06pxQN_KL;Mzi|Aa6=Me;{e zK<RPKqd*U|2%HY^Bt_ux^&XUllgYAu+`zcVRpEGJch-51^FH`y0pRaEC?G@k}9* zXyp1CQrn>s)CqabFPzW?LNTFOnxKy&zX#3${+1*68ZZZmUG=|nM~_Br|ETL!#y}f< zfW#n4dMi+&7zKX-Vw#YAupT{rb{_%K1}~0(eEf#0_wR8P#YG^Rlc28&NqzJg*6aa# z5}<-Y2=^tVc_-5cz|%_)95_He4bP4Nl9VR*B47+$9)bY^+662X57l4b=wmK|UNLl> za>5&wrD`xwAQ2e$h4O=ID}+6z@k=01&!`AC52#WFhCv{|L`NA`&xvJYyp$&|?&|>< z$Jtmgz~5M}6gEt5d5MtseEI1)5OgPW4k`CP!hlkjF^3_TP=G6hNS}ds55KJdcz{cb z3NbE>3zL$CPhG<-6=4ieQ6XZRESwt32Lt#i2X_+xE`eEO))>=K)!DYj;WF!BMSydT zcpnToczCX(grXf$v5>9(^r;#s9YeK&m0%^wbO51x;Uzm(c6dHEC&e%ij)?1k&7VlZ za8d=3sVkr|^$D<2B3@;D{O{5HqJe*Q_TyTNU@XS{QG{gEzK@2e9cWR#5;^SYxdstT z$+FMR&cbwcUwaaK0qGFXi5OZ;;GEiZmMw=uP<2;)Yh@!5(qaAOla^ks7R;lFu7rj{ zD#j>r>Zh}A6uIqzpjX&m-+cVI0X}0RS*017($T5oQ^83}|nd zgg42TADKCWj6nwX$Z^3_ zkoJ!pMNUzplNi#n?Eji!v62vxxGN1JBWWTDNdVv&u4+p8pf%)&j}BsF37GBBh=?nY zn>vs;O2LLWHFlCvckPF#gP?fa-F7$LiGNZCRIZR6G3{WYnK1p(8SN*)C>)Vbf(>vr zI#v}-0id<>Ly3vU4q!I$%Q#lQu)3P%+qZ8?2H9!)Y3RwxOwVomC@K3z=5@6G@3QNZ8C}ASFTD-cWDB|A2-_PhpWxfgB{+f=nla z(;C{g6r+4Yt>7F3oyiq2xD!JeGu$t}#m*mK-8S~eY<$^BG^Yon2X0GHrq=FA9M)N{$$sm(Mu;H6dqF_8dRnCmBqs1$zZ{P)2vId4LGA@8s zMbZebSOoT)_9?{ngdM=|s*oMYtvB|rSX;EQ0jTpGh|vxvE<|xT>KPdS82krNj9ma&y5i3dy9wUNfd_roqsXcr5F1j!8$p{>tz`T?Z1S#pu zoSP)(t73&Wxd;n`UQg}X_sV%qb72gS#KJ@EWGRIiJ2jqB2lE7*St}BMIKJ z?4O%~Ov!`ES*1V=ifhB}-Mqj#Zr7ud>~8MchzqEefv#EtT;=rp8!wUHFr`m<=gccI>cGCNESP8VZKvgy9aFL2pqW2HzV!)=S8Ag7?R?0lqEGsw6-S@V!LN#Vqbc zkiJ8%#N?0Lmf+IXzTFq87Retcx=sza^(^2J9*UOw)mpHEB779SP3y6~7^Xy5p@xww z4&cM%V*o0=uf7|X$Wat#Ci;0U-MhDj%&{O5Kp~<-rWF!fv@T%p!NZ63xQE(nV_*&Y zk>^Fv#Kc5tx?Stkk^Ul2V$O2{W^3G0Lxjl8KI}Wg(xr;ncN#U-u)hS38ly@A~BUUD-dzwCOU046oeOD` zJ_SJ8=0l}JAGOL@o!$q|I$|gP@(6JTY9*-xX+3BKe+LfxPOoauL68**tLrtd-Q&I`9t&pZMfrwAbUb(fqA`JEQ266qW6Ks z(2fb;BJ9i%=7h=k0SW{87@)fw0P(3aJ6wM154A$h=>TS)@7aESxx~6b^?Qm9VkgcQ z?cYT}YZIhr65}W>pag~h|E{?Lp2M6@G<7&Pi>s?OJ-YcPd6r|pK>jJ=sy1>F%fToq zRup-?dwTq#g(W3^N?~kfP&0_n79q`%Xf;1R@%}x{v~lxAXh|3o3&kvR_wVXh+HVEk z>Dq}{GwjG^j1}xUSHG-0WStruF{E^%M4DnUalUbiASw+&-7c#}7U*1M0FfnqYb#-& zMdByzA&e7+C#7rTCTh`)F0k8R(6?+l==<3=rGy?DzqWukC zTyh0ybnU)4Z7F9t6~+oKne)xh&oP&`)x69)WrAimlNaKYd3MyK@sNxxy26B zDKZYZnqo+eSLKwl#(Uh73p92aU>%4qIqWE~2bsJ8X;|@@Xe3vL4fgDzu5xU<3RFi+ z;Q|UV==!9K#`g!H;YRyZI!%fv4%9@HQj9q0TCsi!zV!l#6xX`qE` z0||{)B4#gGMtmWE;#RK^FdfjfPLUacN&r-a%E`=Ut|MoNR|g+5e^*fx*nX+w#I@CT z0xpy1(^a=lpyT+0j$;)UfTiwZeO5Fl^B_!sLPI>isUY@?OG@a-2n))lJ#L+VETCo` zjq5jn%iaLVROh)m5@EHjjvd}4v?V*AqA9fML=n@^v|NSvx(tFi7C4h!9ZQk>g~)FL zaS>h=UrcV(Q@pg8SnknL3b;Ie``$|$>JYiq5PBgP+G1dP*v9=Jf?yNwrcRP9!RYJn zPmfWVu*+q^81x3RfvtVL2otVg0t3<3-F+GTi;%M}NRu7n;wn#$XA+Fxg4yLhrWtZ4 zF-jY`2+Rae0m4fccp6t<1Rw~|r$5U2fb2;QIk^e~R0Wyc3=_~=g#7{;5F{m}bQIk_ zO5#OC+bP^B*0JjbSR;~y5aa#;M1@e%QP7FE3TJ+(U~Y!!U{zyY$%eT*aBD;`4pcn6 zeB^dfobjiK$0tqlB-+r+@8|#iN8DDMXaSU_?41X=JPRe8hQgS=k8XwhdgVM3{I&dL zU`o`mnZ*4yJ>D5Y?w=Jjw@)(E+T6zwQG|Dd`pPem|t$CY#XfzU^}G z3I08!U>xq+UQHS~;eMbACon|d+vSMnutt!4kq79DnuP$cN@ zFnk$|Y)9lTeF)ItJ^awYT$?~LxdfRG_ev3h9=8H;6TYwLqn7gIf8C%RwuQLm5m#3WLb${hp`})aJb+=ztH{<@ zEI<>OH>B}`9!n3{*LQ9+G_SwYn>N3?eR*`J&83@_K0&99aRJKs_OxZlkVh9YwJBT5J5gFbG0wDvG2b)|@Mg9Fiy@?MUAr}rfF zo!cZ%EnBu}l6eEC)XEJwR9YtCp`jN^H-&vi3R?+P*8ye=K1lR%hml1faaF#r@4(Q6 z_rg=S_P}&0W#2&uYe)4{6D50BWMoo%04qspgax7IRG9ygEhnKIpoqu~pgs!e zm!6tcau8-XEiH{quAwHTl+8;#fnSApn?&D zsN407!GE`%Du-<$GZxU}EDoM`aPXb?1225D?o-^k7lgw|Z1_-Xd~*!>LkS2c{2T=) z9TrIwJSrY8H@|2Te;f~)^(ABUIfzc=~bb+6g9{Ml& zHzBzNZN3_$HPWL1n33!DOpk;0A{;J&B+L%%NSkHQLdQ6T=g9|F)-H7@TdKIO0)0v` zjH-g~TUrc%-k>~wK7`*>>*2moUuM{M9sW1=-aM}7{QduZGh=-4!Hjk6%VZ7NiIin5 z?<^%MrK}|)QIcpg#Ed1pOIcF3lvYXzEp|zTN(fn6$Py)_?cA@o=6im(-|cq(J-2gi z=lEl8gS_9b<@vm>$MtwTuItHk@FoC7At`3xhXMOz?G$A42@{}!gRSRl*3DXOE;eoS z+gE(Gl>YQf3fU?G7?15HyaGf;12`LU>MtFJuNX_BS`7(1JKSV?-iu7rNj-hW4e)ncin$bM%vk%kx1R*9|9qTXclZ$H$WnY!ZigICyyJ5QmICyudm0nv&y z)lwSieLe6F&bhR15NP62pN)-uvC~*Kn0sxE$H@{yhsX%KWq(fAwYm9Yo_^m|f&}>L zRkTP(mco_3qMJkF&aq>QYkrw5dkrJb>%1hr_9IpX*v%XF>u!2UNTG|V^al>Dx{%~o zd*NIl|Dt!7Mt>dte39ak_Q1!NFh7q)Phyz4p<>T()m<13biPPy(4)Aa$AdmOS2(mg zwffyuLRp2~=z0f8NPr1ngl$tjxJ5IV7tooiLOLeVQwBz zi-NcQ36}dgX;Mfdcwpnw8`rNd#2F^FG^spz>(YM|C+jU(&>JlxzN=Nx4;`p?khs-Z zne8)SOMeEjV?EoJ3d|juG4i*2Hb>d?`}92A{an~ik)Q9I(bhybEh%5{V?2fHJ#0#% zcT-(X!tlK5s6aBajR6`B0FgPf&ivS#fBJUVHo~g4^x4WpD7#0Xwqwl4 zVYI&VfvtRY=^s2W0b@3teZHwj_wI+x=FZ*zb&VJMl7WE}t2&6p{7R+^F~k~>*&OfD zQ2S>c@ib_D^K4RQ)4$z|+gg7WPvPx0TAQoitl_(qx;b(QoCb~>0OgZ82g#Ad33na`u6R^#ge-? zO~*`{vu+Cf_snrJBU>!$gU>n7aQ*hN9-U}c*r;V)V**^*BZ-wCMsN1=dea{V(Cx>d zUI7Y}wFKSz1WECQZe`&p6yqmM5Y1RBT-cn6ya1%1(tP{Y5>MW&gakkiS{#_=RQbh@ zmpLUmDA&!+jn28H6!>rX_@mKguv=n(2^Iq33SQuW8@{aIXaDX*38(8Aw3gWtq94FH zN*n8F4XZ22d+jjd$6G|@{0=O}T<9=+Eo^{{gF1NDDy0P-*~WKFk*D`AoH10JbV63y z3PxWHu`%lUXSkSY2ZlBhE(*~LQ%@VMc@f5s$l)?JLq8RYa`p4%dSd)T{ZNS1w*%J+ zeCI@{$2HrudZ7vKq^ep(FrdS3PGU58bB!r_4(68*$tJzobR-H(@-`{+wMdRNPS+^C~{noP5D0;PTY@eMrGcz2s&aYy|C*oe% z`xS@3wV1RgnjLHItS>=OvJfkLM~4y3AOdqnogi&!ChWImj4YR_SwPv^9_s;cy>D;< zM*P0zt(mWZ4`Ko<{<<|el%&^C4tA*j2I556ThiWY%;03e*0)Ce^Y@eyvfk*0$U0HY zR3Z?aUgMFH`oBMM`(W2lsZ`3{vNn$0f7I2{a+@o7BRg!CO!7HJlLO@~{DgVYj7Hdg zCiB@c$44=i7_|3sGyG12TuQB^uHf{ER{m3iiyL;cgC>yG!cC&TK+dxe&_BY*j_Sd5 z?S4pZs zx1aL4#e^->Jg-LYLBZ749T$o42!W%s{f)GlFm=-;dfrYTqD-(7-7NVt_5pO)?4wnDMVTe9!dd)McYl!fs_|Jb=AL zD8I@A_>^g&_u|JeD?bzKAdHoO`>Sbb(=AMJHcAasH3lk&O)-f#YTX)CS|6|Oseyt{frli-01@|AaJ-y?$Py2Mkh=VpLp>D?89oW@)$&!R* z*D*(i0r(;YckI+j)UN+_*7+TW85+~ck-^sh4x&scJQ1+(2P}l1x=P4pO7C{&smtJ2 z8G7{CiE{Z@BGm|3c|+VZ!WU5d>JQS_Xd$XB@MtTv0PAzj<4N7S8Og@I0zgcKiBwUYimrkt^V@WcQclL7v(NB|PdxsI9f4%Xr$Uv|dp&RaNu z&5(zz&U+x79UZleg6$4-V@MYRzI=F^N;lePV;0fh8){nkSWcaAd)HO7AHcpa%%>Z} z`(C-t8{ze4UbOoT?W5yg!q79+Wttg|~ zkP@#TmbBJ1xyGi!fZ~l2vMclHlW#8TBjz|osO81ssPwYOk5Q*Caw9`bVsx1J9$}xl zVXG9kbaTC`!!h!RglHqK3111;reHOP^PAoyo#p)pY9i%%&az5QOBplKVD93@ue4s? zjQXp;)!_Y!&YGl=nTZ!@B0#1a$b3VXr3Lx7_DG0)oN=!hO8PL?hTdi|$7j5yq>*qU z%};X74>z!xr4JXxc!W*dgUq*8c(So&RK3E}KCxobiqTB<(@}JVY9>WrMF*y7+?(rq zMJ+71$=g00lg_4CMCy?48;T;7b3L`f^$HPA8$K_hyrygS?=7Y@Rs1xekw?t87yd4P ze%5~FR#SE{#}U=Tkx3@A+Z{c{*Y4AHJdy6yi6(NS)0cn$grvMNAz_Tj%xq!nxw<@O zHuy%uLpl}%DvVdqV8VTS?mDEQTkRm6iD-62AhA&GwUZV${Cb|8rm%?g^0Jg0Tu8l1 z1$XRz?(ZV(G;Z8t@+97Kp93T8y)l3FPMh0EL925;1q#G5(u=lifbV+qjSOv zZ@!)-Oxv6}`|ih3m2W>(<8&H}ft-uPn}zUU83~jWc9GJ@O0N6Qw>2SAw9akZN(5NiFTrVt;e@rH&5$(E{b<0;DUE!KaQ$BET)VA{<>bDG ztDKy?p!Sf%S0hNYud{ze42>)_-x5u<(9j7p)WdB@hdePje8aLp6#WOi6(a#z^1 zQT`Rakas8=*(ZXJ9}it)Vwf`pHZ-eQ?KkyY)^l}@qpz2X-9=E0jLTNeB_HE@@J3{{ zGMg+DZ8LJT_XOEB+%~hjo}K2m&(6%UQ~Y~L<8gO~8tOB?i+v^OKhAyzbBvBrB*H>( znX-|fMot)FEUIZ<=nc~SBHjGek^(Y`fq^3bHbj;$mbzoi)4ZU#t(-}&kzC> z3dnB+aTdzAr_fE$fW;Fjy_@TaQd!+6B(1IIDGc~bEZZ9iRLM`gw(Y;;Z64BcdV;)@N&~K&Ssdm zg2vSUGN`BuEkG!c8!o?Tnne7EKcQ>33oj}y^PAoJBNdHL3$PyBwbfS$;fYFm*ZjTcE&TiskX(*j$?LzE1g>P#l zBlj_o**%hZox)R}wd(*WH8koNTB0zsr#x!irR~h6O=XBoO@>!k@Z|hzSor2-a2d8B zGk;{hF^bSYX2Xlq1{-iP+?|XC!+J1&@x{q&W#e*08B)`zUFWtCN=Sqo-uW8-4VMB2 zZ@P89@rFBXk2?$x?zXpZ#jiUdB%}}8ykJ9ZO~T_dh+l{C86cvj5BA3T@scHm&#DuH z{7(+b>XhTk`-S8Jg)qPH>xy#`+1y)<=dpO+nQ&KuCNTaPNM!oFV zr=3{4^73CAd-Y=jmVa{0f-Mj!hkc&q+}mY5^G#&LOT%W@;Frrg&4^m+X!+}Jt$k^x zhM->@Hh0PTyzbOrcW=C+4N$Yqu`_?UthI zqFqIoRyLlMTW_*{Z!4<6#L(vDUS8lw2&(cwo_H|Ex16ch5Ud&kl34<`;7olpI|KkHHCc>V?75c)}zY zniwz{rw_NsZ7(Q?N7XS}>xYUe1T~|Cd|$1PDdo+{pajHfFO!P8s;X#eKXojfeO^wW zH0tS-LVIm_`%O6ZH*Or~eS1A5IMWC09*@X#-NXj|!8Ags$TTi(?^!-%0d{Ufac`j! z#-^3Lcu^0B%9)i<&3mL}X0{{*O+;8@_R+C3ZkSw9kT#)F%V{$2bum$jLM_fWE;4^< zt~#c`rF=cR=@RWJqb)A2n_2Y~h%b}U00cxMuNA2TJaJd*zu~2U7>#Ui-cs(5kjMa8 zl@)gEGPLBq5Q8pE81KyNy_Dg#A<`tupGfeQoep|7Ojz%9=xs_mdt3g7$GrW0XIj8S zNt8n?sY5jjMhqGxVwarbaFqJWrE_!jI%(Zmq8<_qyX-f{gs$It^~Q;gM~?;;gsxr) z+er7nX=Q^XGPFsTJ53(5+ogYnCg$y_KaZ2S6sMP@wHIE8LUGZvJGrz0j~34kvkApa zL(UbQC(JSf4pwMrMKU4p9#yC7GwDp>*Z2E;`$OYX6Q7j%uits>2br6*;pS$X=kh*i z%9#wgLwgyHKXD7ov`~(oxq=spVbmv#eo=_fL*dao`wc>vSlqgHRlqi+u5LG7+Ps+c zDyl3{g3fIU35!VAp)ExCSfx(g!(SQhBIA-Bl28<)W zZX?z3pb8u~#_M@>y#Kg_Nu+7`))sL;rZ& z(YKb9N^c?@Ud!W{Ht7xGhF`K7pm{e12L~JN7Xtx8_!Uy}Fz>LBCj)#dvcUNaBHT&A zUKDyMO3FGMob}E8qr@tR8&F0BB8@z79;x0#z6RLie;vHLJp=Hj!MG=f`LS_;sW)8$~%X0D&L>Bm-@GS}#*MrNi5(%Kktj+wp z{zp8QkW?}RMdYzble*;k(unpK`tl1tizcM~-~O6N!7urlC%3(v)GcNiWK+X1y%9IN zgAY6I9n56DfM&6{vZcgT97eE};+LwF+2-okFn{2(FJ^x?zCkf~+OcScbo_~LPs}a= zi$2rCrKRHY``otT=OiJ@oLV0H0tiyuw*jbEQsS}f4^vc0*Za@o++HaWqj2jh(B?Yo ztTTLz|4vkOq}ksvL=$c_gY6j5gLPR5Y#BEpvubeCjZzyXxUSDV%SESZxtpbOGso={=zb!Wh!__aPgb!#R$Hq*6x4mNPDp#@T>7AygD-IM7lc}odf6HOnp8fIIYb35SM%% zij01|ZLIv>!D%>_X?45G7=YD`q>n64Rko9Uv6rL|F{JB1CgRG7WhPHIfJE?ut!Gz5 zWreH062)IXi5~)ou!GMOvjzA~hQd<%^9%*>2+POGdd%AGg~wIXkI5*e%PzXmJ|a?( zw0yGGM%np~;&L&biEu8*8p}|1J6-T1HEIJu9y|9=`Fiy&hJJh@Yy-ZEBU!nOKsp2t zy&tv$DNar;f(jR}J$p+gsdU!iq-M--426_7lk0==J}?p`K;M$GUC8`zmCku?4QX;@ z7L<9965iLZOdOyrEjmA;zCz#@^UvEgmI+3v;Pm?T9&m8zGV#1_AyX8sQ~NvF1wjWP zy*Yz#A_!#(-W=0cSG%zIbKW zuZyXhcTdrsktXK`u#Zi9l#m)^kS%QO6&`N#%|4nK-v0sKo{x`T);a^#OuEYf2hVZo zNVwn|LNkXK_$`|6I;y=);HI|H7W;lY6vWBTXLD}wMRH)2w52jQf}WSqhYMtODbM=z z+nna`^An(#WF3n`R1S(2hk((1a;&erKORuXI>O!T(&6~E*du>|bH!56X?dsXjcrSv z*H97c(uR>Q?89C76;}T}e%!ByjJV`37Qyh1imsa!_S$g`hGww}P%{lUu8 z^Aw&(phQD_6 z2yWL{h_O3f@n6Zff*M)ojJs*Wn-}{osDTq9BUXD948vc;3TJq2AEKPhC^mMkh6HcV^^zU1xgEyswR;`l&0&@80< zM_b!(Zr+eF*Y`9TI@7g{>6r$-dnew_F@|dr`*YL$5K(sU%gQ)%HthrTcFjPC?}CJwq80dZ=^vSn%j8gb0ZoqGApR z(su00F6#QB@{(?wVzI9(3}FFv!hISC;iy}L{IqY_@-|fHJqCQ`j9?@)3h4g`O9^2x z7LBR=_dc==3mVdcHeT9Nu!D=y+PpIdS*y;$BSJ7}E2Qt12`krF@*-z4=O}+9Cm{Sq zz>-HP{3VZ44)|miY$S8hJ@JLfvEV%G1`2WIWBg49bk80&QcgNRP#3!x`M|N`2)O`Z zgN{<*0jybSzSD#rZYn=T4*nUsAesMMp|R>Cl!c6T4J*H>vI9})tBi9vMm(jUw zwDsq)ah-aeyE*>g%-$r%#@rwUrmb-=H+LDvLFs>$Taq;8uZN#lBz+93GV25BqbYEL z*=eEovmyl$e6GlRKWx=eS0gk}2GRhKSDSJ2)a{OcBS7jWp>=kYu5!;^O`Ezb1OUst zijTnE1sD~{xwp>lP+O)uV}m-ynKKfkXn{Z(q*LDK@fJTKg&56AFKIsK|IIEJ!H!}k zeR#nsT>mjNtU>`l#}sfc&Ue~(;@gf$nS{CsbDzQ!5N>tQgL;Z8?gq*I%)G5p)?%Fk zuAF?o^e(&xjdm|UgfwwXq`?8xYUtYeJ8}F1C7~LE^v&0=zi?h_7^y+X9zLcIl#(_>Y`G3%2N}?eH_!JUKR5&UjEH zv(*IJ@`#sN7$BexqB2487ZSpyjoyzXJ9FVsj>V+gV1d|_7&jqV6YGo+S!C#&*eB&g znE!Us?Ksu1-^Q;0kr`bQ^*|dh!6UYBjd!${VyR`z)wC{2DarR;XPdwwWv*Df9PJWV z7@7I{rwe|(5Dr`Pl`-56EU3!GnrGJxwtK0hs7+)MTr3&x8ITBf=riq zE-hNqdQPBOP)2G=QLEs935A|sQti!WZd+}PpVj{4VRLHhir-fMH8MHDaQm^96R*Dv zTbjM??c1|XxevnPZ0Gh`+3~t%+m#*9hdJ4YZd){Q;;)9|%<8GCYCd*PI^lV=o6D4S z2~nD+E+1B>7b%-%1(dmesN7I;$S{d@(widWwr=G+pY2+S=gsrNqZ1FL4bP1zcB1KI z_#-QyOJ5!@ce3cAP{d8|y>QU%d7rKZ-I;HZk?_iU^pgG7eifHjU;J=kIU&us`~qyI z3uKpLMS*|OKfMB{(84m<>elX7ty>p(wVgKw;NqjoBT!gm1HrMW4#em9l*W1jU?bXCRv{0|+)2?N}j6nDAZ>p+GJTeg6V!41q zrVOR;9Pw-)hkg;7x~KMwoi>BRL*HF0OJ4@G$gV;R9C-2l*|}#q(yf7e7xtZh+j;A{ z*(a3y8P-=Q6ip(J>L~kqES=LourlOgzG`YrQQ(HczzrW#{@A6X0`n9;amN!Fw5Nh z0CCg$dTY}=(?>k@4XUjwDueKFo`DKfc=A1UK=flHIjMHofwH8+sClb6hNVa~bwJ82 zgE4!Zax6!rI3L+e`ThjF7SG+5-Wif%MM!QpaKTut2R_g^Lo}l=mFg5dwFz=_$gy-T zKr=SLHWvH6IJ}4frqOGI7N2{%Hub>rC^NecCUfofn$C0SL0?~9+1&}8b&wJ0!grSC z`xq1++oh|&>`iTw_4!kJ+bP7X`FG(Lw)$vb)=OW!YmDFW=!D^4pEw#U$p}2Osk!d? zzP_u94mtN!x>vuQul2EVQgu1tppcu&;F<>K)eyh##762vNi^ty5*>XSOZ<>k_FwEOIeWc^;xQIc%&aFB7fgv z?-X{wEb9bIcNI}vOEZ5 zT!HuK5550FAgR3gue>Aef61xZ_3A=&twot$`pY#X$@-0?K~ClR{ms7Fv^kvd ze))3Ih}JCnrK`M&P>896TDNY^#nxyxm~mieA8y_ZBa(6Ua0IR=>t5}QXD_H1-65w3 zCy#9_GA7k#=vF*Q*tC8LL0Ns98%h#1QaW#Vw?|qm_Dt@aa^m311J?eQ!(sVvlUBmV zH?p?2?xUq;K>6ClY<;8zh}gmd%L~tB&d^-bC+jcj(SrH4-%TN64*3}9YNt5QJ?j1Y zpzgbLzb~D)?uD1UIrghboD1P}E#}Dif2-;#LMM)Iw0*xy)gmeDCxs%p`- z>F7`o?#L(Zfq8gwIZ@x?a)Y*3dyP#_Htxn`Mcw(XpC-!Rvm0G&DvFk|#k1G$ZJNdl zpC9M;Q&R()c?Q$fWoySvb>y{&2RPjBqg4IWTxa0imGRa*pwlaeL-7@b2g_g;mcYb# z^=_x|G}pj@q)-f9iYq3A4HSsVS9I%-T^0*^FJy3I{QI-HM$CXByO{Jhh<{+dya1Ej zR5->Vu;x14$)nXbJH0LKZT&4XqVwD=Jf5`H%3BqgBFFh6gf2dCt0kyH_-)|%Bw}_9i(UJ@&r7rnOgVlPT8L!Px_suIlXm?Zr@@nR%|<_k?EJ8kEsWqWrAcR`yF3o?Dh) zPWp#;tH^SUPW)~=AmQ_Ud8lu)!*jlRAF6CrPT*gcSPf|xHN2U77zG+!)Y8P}3*amp zv{euIvY~o=>Wy@#AwwCHe46|}RW%J9e6MwdUZL*K zQ4tY$xR^q-;((y2n3%z=aT7x9tk*`xJ16bbkL8uO)!8@DCf!+YNVk8wgj~9$W9eU_ z`V^#Pv3uKbNBEv{WeC3_9|&qWBK$&*x%E;^_Hq<;FOQ_sp*@HeL4gLb=YX zz;!SGc>>W;Q6BAah=g=TvA4d$Fwc$GRq?7z?Y^tcM)zamn}upSPH4!UEqhNPv?Ok| zo)tQ7nce)mjT@&pH@Bot4B#Q)OdOs%;NlAXsZSHlTCo&ZZvks8qBK5PRBkoy*;4P2 zPibjstF~=dHhjqKSy8PhRcNJ6dQ|;~sMD^tVsl3qyO35JZq#h*d)p_q`TpJ8wl(G! zJSX0+P@$!XI}W(kbEAi!?oWgTOoc{Jj!tI z$v#RpaCm)W>j^TdcFCo(HtS~~s;V?-yQ^@mEYkh`(CDbMkSsldTuYUl2e={W?>ZI;0ugTXdT?>@OV#n?XFlp(y&e>4sZiK%2Q)anBtvW< z4h0crhAL$lfh&80<6H%0gsraq>QUMRlKc8yd>6x_HdW>GXMrUF{`1!NQHI^f_M?1QLfXiXDK7I9uB@-tpmEBTShl z@Ke#V$hQRNvth)Qm8q<7d4&IAPZ%yHrxB&+FQ|Qz7~i~_4(>{UZToEkQ>f5_j7=@$ zY~N8OJErzOr=eW?Wc5w+i0`BMvb5Wp(E+z_>mS)X(F)qtj8N@x&A5}Bt!p>mjFWfv zZZ(hr2={rke`U{{J^Rd@h%B~9)f%tdU%e~(YH4La!!`L{?lxk@w}mu4-ja`#g>~dc zG;#D1`NiXYJW(StVJXhz9*_C@WC%N4lJz`3joe)X@|gk^JOl_4aiKWu(Za2rir*#3YdwGd zVNUn&eKj?wx)k)A-=xAA?2PIV%Xtf(`HHFpVH@UM*ey>h83-3gy6O98v&t*I8z{N90 zzStbF_VKd)EX&C3KdU9l=Tj;(N80T^q>2EScrc`_(w@p@+WET!R$XcIhTODKljYoP zU**O8TKB!azITQcCRsY9^e|f%ytfn}70kGYKA7x4xTOyl?+T{yihpzBVBvm=!ThI;t8w%O3bjInMoeyHPGAU29ybycz=%T=~s)yQTLpYsHvXOS-zQx5>L09P-9R)CT z;3HNU%9oa56(%cteo9I;0xZGi&w7^94?SfmWhV;_t(cH3KvKPK77O3(M+9%4Y+f9b*+bdwz_-<1hDi$=lk1Os{>1|DI;=f}_!q+7PDXCx1 zK3Z}x!0Cb68$!>KsXr?ep6joJ(vGO3Ola5nUX3SI?T!zcCmDG7lT}v(oS(V45YkGs zW*MY8PyB1=@W;KeP|kb({ulS+fZFn19#wxxpqB*`LG97uFcGuVyBmXM*<3b0 z(}$-RP3^x|z#xwDTa&xE`D$?R-x^uQ)#W)EL(ZWWQOqQd4_Yq3JL=+x13-98?iYJ!usmL3a&5wh z%Tb_?73W`USqAeVM>pbADafm>aaQF=o_nT`ODyL^{B zmuI2<=~_!Fhf#U_HF2GEeg50QHkG+#p=UVulogaY2H^? z(tgqTSKg`79@THV=jRw%3vJZJ4r@qBkQmhea9Sn653iGmx~}U6T>5zB-G#SN%V49yr_AtrpM+Y7I@4tzop z)qeVKlRNGS_MbOO5>QRe*Cd%As{J-1R^|FNB@pNtrdRv*EyY&YS)?(Ww~^=48??39 zeY5k12;1y!2M$~`8V{Y49lvdRc&*RiuNyukmF($Z3M#!1bBLv|t6t@!jKcUOHMu$LDn@4k2K2ChEm!9@P)`&QV{QA~gQ~__kxO z4qA9+GZ`?=IoQ+d)2y>onX|A$bsT-8?7ddszPIPsl=`pWQrporM$QT8qEl$F-{-h( zqwiAB1t#*1!34arpW9F*WRl8l?eNYTs08$-IOqAt=N;s`Ie9LB!`t77+0K?EV0Z56 z@Lf8kQ%vT$6wpN9&p@}rO2C&_WAiHCpzBp)xdLq$RG3xm{gHMD*OWE#8W4PAl=B;3 zsbQa8{&5dZQ+jWlcmm?;BtRS!LEGP3%&|QgtkNLFM?RPbUeZeZ_HuGinNTZ2iDB|@ z;H?G}vU+9rOYj_!wc6u$09?RvNew8MI&Sc%sc?#A^bDRrDKmZ#!rDJ_^`f}baN6=w zkZ!RRx_9Q^4g1ycDP`P#;xaMt0>*?RVb%0aL=Im$v$kKM&C29O)J=0VZXWw!JbSiP zRZ&Za{J29#7Q2E9OnrO#cNyG`q&mJ)-3{IP(zM@TN7Gj4!FX*(%B@KtL_eJwSuLcT znc1UVqw-=ibPtkBmt>6jdPiR~@^ivst@F?SLhb!X&5F5l7lqKJK=G8Yci*2&(qo#G!f-G z$+^IbGn2KI_v!BU8r6Aw^pAx1%ssnkYyWD4yo4Z8Ha^YYsR$p(uf8fIL zG>l38Te!L9jL`PZY=j~LWCnE0;Kes~=uo`HgKRUgKy=ZWY-VQ0-QP_o>cZ3Wmg2>Utb7b>%!-+0Nx*;)L=eI5x z77p84yCEGUDtX)tFduunZ+N~C9Y?lyHXl|f!Aa;B!K$pQ3`QC_P&8<*YT!xTkZ8Nt zlN_{!iZ%s>z!4^?;ABXkK9XnKu(Paz(HM?vXOyrDNJ_`)UkJNLvP`>$@b)iv&SGG2O6W?Szcw%i1{QpQoE=p}{O)4eRw&XJU) zm38{Zn)cjc^DN{@t7)7}Ap4hM(eUml`M;MhgZxbUaJd2k$(?IAE|vk=9qca+f*e>J z_k{->0ZSz=U+Jx5DSm{ z%+RM;2>&j`k0c+t={RkGc#Tfc00J8%^bm@_k}-LFvq^b%g+0ZJ^A}*S%VRh`%lB?o z`+}}h8x}6hg-t6X5o#!Jr>2INb5dG4MY&Tm-EYrJ+?xNm$+1^^_@4TmF{LRsL9T(@ zQ1Bnjw6M5zH^_IsZ=rK~kkQSrX=a9b2K!^XCRM+;cLa0fv$oS9-6}S{T3(t^bNa=e zipbjo_4EpCKIK)j4^v%=4*BzzHykmI`SB%+_MKT_)2MLA)vwoY6mLJHSCA{EZsRbp zdOME#9KW^ezYjg7<%G}%dQO$wxBS_)-j6^0nBS`IgF^o&;wZ*O{@^|2Z*08`$9IAb z2>b2`ms?MB!wU*x)vhm~JjjitGYq*uWlO&_j2TSm(VW-yZt8_kc=FIHgb>Gr{ zABpAEafw^9SuntKgHF^(k8f<*qSYO+=n!&_Mqc&Px{Hy0GwQ+V#WyAJ^P#DL&DpOV z7MwPOvP61mbK$=~esaNYv2GJO2_;0n&y-0({`I$ei7_6vm1o~)O`jh>nB3hY+Eo7= z8%@GMKIFX(VZjH?tY-a1FaZX-ZLG}96MFu|mJWlNV>J|ec!w3I#tHQpar#DkCnIO# zUu;!rY6#Jih34ciV@^k+b^nhgbzfvyo1JTTg%!I;k{Sn-TE~4co$oe`sAMC_hCy!q zAdSKwc-$FuGe0JEz`>$9f|&qU``JLj50(d|{$%#=bBI}g(e0{$6|JPI!HgBV#fSQ# zUlwMY&QBvG$$?OXZ1qFET!p?b9752$B$>2X+#-11mKLgfq>bsSj!s!K$-@@Lm2$0s z;t|)cUk7DR2z#8j{%xd23Yy4%T z)&XVSmZwjzv=y*eB3&k0u+-^9w&`IvpqYx4Dk^68#G5ieQ0jUU6F(}$Fxxn<1{C&1 zuDR&*hbXS9uQ~t7dvKAH6D<22VNMpq1Jvv%?H`xEmi! z=cmJzh^sytDQu;->+ZnkY3%QHcc319CIR-9f0-E8ttZbO(-Z#NgJ!_NIrBEyNiijL zGoiR7sFSnB1V((Fz8)i6uYqL%&4ZvGa$soKxZ)JcdPhCp0_epi|M6vXF zalx0S_Ln|QQdv<2Mz@)=Z#fq>T>uf!5})cYrF^c2@)_?50A&-X9i8ySTxC_T*EGu9 zxbBt}-`*5k_dbI8U#Sw251yv%a}pd5I8O3J_QPzGx&3+ft|C`IAu^ZUkzK0N;L9xg zn>4Gt!>Qiu+vz_fY;ac5F;`NTwh%8;CzmI%ZAEKSS;eLe4{zKd2oe_>3#V&dX;AKlT=0d{P|Rsn$lVx8CU%MechFh-85TkLL;Ta z_ctL$#G6}EXHkJ*Mef7mm~yIT9wGV}v4$m>;H(v~VB7Y#0dY$S%maYM7RWOKX%7rtrs7Jo zEbrBQ#;MJrIY&0f%b(LLR28btG7v&;`N{5eKXLfk%DlS&RrGtd8FF6z%hSA@uc85M zEWW;ee(7a=PdG~;#P$h0JxEP|pjH!<1Cg6rowBX_M_oeB+-;DoT6!|r?>0Xk9*4zR ziwbS}vNz5i{VA0`P=yaPwOBn-iUfJf!b&-?VcQFK9oIfwXly6iq{;$Z5YHMeyWFsgm59qTM!)xE z>O$MF*+2fJXP1dwr4-^!0-F_6TT^Ko7spx5N{1_rdL0*GkyljnJtwJhE zZ5a1fM}F>oDm>~!U%8XmETgH$aTJRE`!gw)y=R>3QQNNW@^pLmUsLatMkOr35GpFh8O+Ngf3yF86UB4UFBU+B2C zwOBPy3b*nHbG1aB6CPU!xl`DxdVT9+;kQo_y!};tRajeeX+Pdx@`i&aj z$Z+iS>uoga{-FJUBR_xJ1Ezx5p80yPZ3J<&M~@y}1L{)3(&Wu8o=@(L`j}I2Q3|KV zmijxLKHTpU1{w z8@F0%WMn>@FFPD__un8s)$`@}wm)tkZfDxRwb=iTjOxBnVW|1}VwKC<%TtPvj7295q-lR0B_FuS=~I7rU(yMcQ1EL|mPH8j3_ zX4&Uz%tk5Dyn&)7fXT8&xRJuOs5Dh8`hokW>uuE}9#P7l;vmablJ4EzKlWtP86dD4 z3UeEBxLPuSlSk=P^TKo^m7)@yz(v_k+rcU>yZL|%ua8=|R}`R&q>0iEIolOMIh{*C zKwm=5yAQQrG#idU-;t8~lkZt1eaVnd{Cms&e9Ps10Za1Gf^PcNZ zjg)1TLThv-r793uNo=fMdPVBcok!>5&`+u zZ{`OmQ|18mq=-%BInK+j;lDkVXTfY0>A4uVd^7~Ky>Bb9uj%IZ0xmp%c`xJ$2=V|q z&4!zE;lNO}R|y0%8Qo#_u$3?}N)-q>?dwQK(|O-uVwqRlXF0Hm_bzc4N9fChzQw(G7;oMjp*D4BDoqICby#SdKy<2`?fOKJYJ zKpiRP-B&_+3klm?_uB%YEW%ckLzF6t)CxJsS6ivNf8Xokq0+o@f7hNp+*&H&!`)i^ zkowPH(>e0^t%#GZ)lA$iM zczr<{DU>Is+0R%M8Y%Rb6^^!FS@~o&Ka2NP}{>r_*C zI;)P;x)nE9K6+@^x(8FnQ?ucWDWGt~15V)z#N{%K3d&7og?bxNoV#FCQz;Hy!QX;r zjrf^tXXx%is+Sq!AA`8gO4-0?dTu!kwBmmh{cN=oJ{!4zy_hHJmk2PZE~)wAz{78F z5SFyNGKS@81K}P$+#aK(Y(@vRNFf|``lz=NJ~(=M3+JoKH}LpA3xlL#QHi|KVoi14 zJAC;O`bil(&<1keg*_Nvj?uOcNNUXt-HLpdLc7z~+r$IsMCLCPkOd_dAaC0^oq}9k zgibSc`M{yS*FA|?5Pu=0cLrWLA2FH*xqUGcfeSjd+wzFhll1lV+l+KZXS$8n15}O{ zKY@j)uktY1G`d9m& z|F`Gzqr9oC*)Wr`IT@fclm#_I=m(UXqElC!iCt}Yp0gLr(bP1dSr4ft#65|d#lv{H2X#DJuYbOXQ+&nT$P&>dsq0N6-wQz!6<>Hj zzBhM8?DWW-rgcwZ>R-Z1P-w?6Yx z5oZTZ?zs9-JT;bf=|MRDG*U`f#Q|t&#_#9a#Tkp&j_S6*a1vr39p&=xGq}4uJ9lr3 z0ZCP*8Yzu>1sL|b9_4pwSW=G^OrHV}u{bgGn*7xi& zg$i0q4pEhSG#YkXl{`Mph2W~vfRol|B39uWXr4454G5DNWT`gFLv>H?lt#FJ?g)8L z%KPXd5!7;0!iwH%L;0>|gK_QTAKuhVAAvEfP)s?>m@ADn?|j`yHe|Yv+UO$Ubf-~k zzCCM?nOi$gO^RX&(<`v6gVAF*8Bt1#{YoWY@`#~A;`5*crivGnu5e&hL=?%uJ{ow{ z3{-F;u8BAMLHP64gW0E4?&S?9{9EQHe+~CPB1et%P>L0j_0~bVz;{**5?e5ek370k zR6T7NhdY=k{7A$-l{{HAEZm{>ca1(5ExxHaSE-SUaua_2kx*0^r-r~j zn0b#X4h>7h(lqqoiWsF3%{m{Ge-bTBs9S`=9!iSU?G>FJOY>7;wx;qD8InG?(*9qN z*#12D{o|~vueLu6Z>1xyvKd4H@ykG;4E>jpAU>-0sjBUK|7WfH z!X_Z!On5-Z)kq<74K=7`vF^nUtD(#eYlyviz3|TbKA$2lz$RU+!5R`OZNpd8z4tjs zvr-Mmt3-=7uC3&wjN^9kW?5JEf?_aFo`9`Xs?Z(P#iOG5ki#~A9 z{@*>)J`_sz%c?NffPf?s_ z4K}bP5con+3LS^JQEQdFsv+w>K4*SkZ}hI|6w@*9Vp2~C3e^0U&)X6`Z)=OfzVPd0 znA&57!qoy_!h^TfJ$6sxR9)SuIJNFa=9RyK(*Ady{C}cwJDNVe5j2S%Ypu9=4G&=K z81ZS7xzFCUmItzZ&dv$q>UF`|Qi?8`ajsT^@8HyaKB(Uz2niBF#60xi^BMKO|AaHC zNF38x&I;cg)OA}gwmy1}Mgg5?&?o9xxZmdc0(tsFe;wAW#}gK`WCTZKxb#eCNjk!g zq0jngFstYegFOR;#e`8GHf9+A%CNa)fB7e6f-y5kEJ*gW;CBEem_vePg;*RRMG~S? zWlPwO;})EsI+?1h$R}YBIAQ2BG@Zv3Lc@Uge3h6bBq2cJmf?`l6lgfzK(=|mLRi?n z*kHR#^2DQI6j>rkZA-IG2jGVDH#Y2iOB{2xl}WWzRl<;&nDdREzZ|}CWHTDuaCRB+ z*C6f-AgdoSH$_ZEi@mjbN&=S0B3zYHO%N<-^bxLWiIhHEYbIovt(*8DRT{!3X5t+X zdI`qa1#f?x(5<`1e3{ctU-nL$BkjFfu(O3f3Xu*>9ztBAKwqT_M~v}uA>>Iq;NR&p zNftz->`y z@i~3S%>-G5OGoN2U+*cRFK=uu$m5i7;KJicfYQ(=y%e>p#lf$U;;Pq3eRKVYB~$Uw zm1cO<4AH~G52v13-j>@T4J$N;Dc=W7}lghXVvGQJ>e=OJWr0aElp$J_1|xPG`M zY<2G=H2DK_Y}6o}LSs5h568#$j%zr(YCu%t=xbCUvG*o*C?FHNNPT%Aus#1cw4sTm zQ}zx#_@pi;oCA9xtWS8<)`;1>s9j}BuOKTbw~Q?vR_-GQ)t@b>x1T=yRzoRb5}rha zmSz>tRzoh{0fC+BJwtFo)*tk@-f3qO@M+U497>xcQ^n2ymA^6UwasH44X5f~$-B-? z8Q(~Jnt-m|QZB<<+Dd6D8n~ELTWwXB4eBkcfyC8(#&wmZmV%@!N)*W8DU_A)XMXq@IwDkOtmrzLP9MT59Mtr&GW60Gr!{ zj6-5dV<_63=jRbhEZB)+9O`5;dBV(B?>kVK_{ew@k&10G)y$SJ(cur{d3np%H|CZO zciw=SEv30QXu{r$e~e&wQ2fJE+|^}WQVYM?yW>1jLsCINB zNGpxl=gK1fTXR#dX3*RfrzeX2|FHhTdyGV$Dc1L}y)EXE5S}0^l|^j%h=5ZBt(3-Y zGdrw*vc$HPzJx6yd^F_EIr7cwMJ}mEO!P|Ch7a)}g~kE}%I_1aRRRp9d5(DwL3E1^ zW+)RrS{)w(nY;y5sZ~cvHZ?K;R$_Hnj)Z=^t<>^Hc2A@h5nv_QulT~BbMl330hF96 zE{B3qA$*zS`iUnzo%M1^{1;xme`|A+Ol1{c;1|ALxZxJ|SWuw4^Nv)PQ{Uv5xMN3;cBf@xHBr@HT7ZO_)~5y+U%Y={S|Hv{)ekFPG*+?f8*({QJAZ9P=sJj`gebE0 z9ioZIaXuP=jf(7O_r0t}S#=Gfer*$pov$M;Bo(ysl;MIgi3`GD|HvV=e=JQYtxk;! z?Iii#oo2TpvAvN~R2uY;X3!#9MpkWeW_WQ@`$%6y^w#`~)&~oBFUX zuD?Sj2+h;wK)w~w4v$vErir*}5+~*jL7yT^<8w8&D+jEMuz`s3>f6>ff(8x1!2$`# zVtrYLUsfePqj+AIcz3Lr+alIUod0_S4v39yWB8T>(i-F_kyAEMn+sw_61$WjA^GuV zgeq5z$@2f`9Zlv8Dxbd7M1NnnIGzoL@db-Fx}i)MiAdakTc%{O?eUn&GaNk@(5}?v z-BfA6$c=5KX#85J6E^fAamFIu&W%fKZE7(HFSE-$*H70qQjmsKYW|M4&b|0qAGjnK z&XQ;-?nKgNKZ7ylv!#N{=GeDmYUZ)c1?sl7(!<4`96`if+)aN=`-^ysWe;0H=saQx zg_pkv$L$}w%qTmloEZ+&u;j>B%ED<7>i6At&*l~)4Kc&6cyf6;hAD}=twJQ+w{gs| zaGWk2Lzir4TFsU~F~wOjLcYtt$zGgJ7&fKZ%SU5s|G3#-a3j3q3$r_4!2lbsrjG7S zY#Yo-ZLhu^Y^}sUJ)cKp|! ziCE$xwE>_!f-rO-IWVo~c+^y{i7$oFhA*2JX(#SZOr(b8IK=K%RE0KpKoy>@}K2p83#;EtA~hj%7W?g41R2biqt-lLx%^sZ);F!j;0=T!NCL z+L;=5103lmMmoB{_fc8Jjcq+4&6Uk0Z5I?lvG6xVfB)eKt~O=Y8^f z)_KGF<&Qvx0pjALA`wvI$C}W{vn&(rCK_?rX)#{N>lHQI^DkO|47koCsiAG%i*Kj* zn`cqhBx6|Ks`DS(dM{6`3EOLk$Gl}JdqH|lg_1Qfvu<5ErJkZmfz*OxdZg_|JpH%D z!+r^>N*vjP!FAH8$Kp4BMJr$?R#jSEPkS0)HOQg2g_k%+NgHcrKT)zV`+4io)-0Px z(8&7(28p@h*~1LnzJMBgDWy+>9IXi_bw^s`S+v>1l71F0PkMl)r-v^hsb75GUh&+6 z-`jWdkf#{?!4*`&dcDYRboEME|f}0o_(45!kNN1#YJ|IOe_sx zjD%VzK5Kd_9}Y9vRF;?HO2?8|YYU}HXGgp^VE?@5w0*n92q0-k?!?wjsv9jLQ~+u8 z4R!p+zCU|j7f$Gg_rcvBK?$zGTVaUTfKBaJrPYp%s@QVbDe0p{ zS$R(PMm^}<9w3x4#_YXeXDJs?P=qjbM4xm$n4#rjtiW;TAdqe1YIB-pPDxTEE(xj? zyNsA)cpK74rqUos++mDcCi}g0f^M08AtDE;ks}WEj*~*=AZkjqeDv%7dK4%GRAfwyWr-q zyi?~tc}}|EYN=4~#+4z`R4+`b)O~B-Jsbr0F0Cly zB(~wzC{^1BTF=MMu6B1DxrNKVzU@q9+R62BoJfT&zoJ(%q z{B7vDK({ZCq!rmXTI_F9*}}FcRqrrPOVz&RzwJ~S6|{0WUTr!;=$O7!68g38;`(vi z_b>0&!jheXLp8g?)e-|XO0N^cF0WlK z@{veU?+_$p=SWQ_85E{=a$^f^g`$&`thENSBs9r`^K*H>-S^d8oY@v*<(X zCG|K0=|QoaVTe5Wy`-33BC|VgiC{;x8%BA&QtcjuE$*}8#mAN+$qB-h2Q2d~0KSjF ze4?KBs#qDfo5!dtr4Y5G8nKAcP1KX;w@>~fq_XVu;ZN6O>Oj2+8i=crmKn_S27)xO zG^yQexSIEKWuvcOA6BE&AWodlxVdVR;?}@V9cLX?@K|DKv@byY5G!{0_lA_b;?ovF zN!Wt%zW*H-!{ry6CVe-~ehVBJEsCjl-~_dq(K1@f=nIqbXnq$DHYvleZ!H~Kf2IK5 z=7g^9{X*7<%7SpCcfRTw_2MDA{|Zp?(`Ef*G>AMA(r`|9StZiI4|Lzh8jKCA=0|1- z6@r%u|9Nj|8RF`5ax9N&LF!~)H7^bw`9v8 zt4sHB_6@bSPqbD{tdhU{)<+i!&JtErT(;N|1{-vJk;F*3-eM3G5=2bI0zR1UV7@iG zba9n)*KW$}TL1oHAQ)`0^fP*cK4TC>SRNJrTbtn8gu)QAr&P(pY^q$ME1(?IDwjY_ zeg;fGS;|z(C-F)!tjLemr)mR6bkSucn7mSG|Mb#2086Ix27W6o*TuUQ)j z&IKDP{mpGphV}0hnkSPc1U3L)o0ON#kv+zWC<+UYlKNiD*U&(!X_*}%RP~RB9x6d6 z35eoZB}$8~via@OUr9YO#lvb-)8V&(_a>3{(qtnxWJnV>_Luj8P~`+~ z6fJ2#`@0{{LsB#ne;TkNmZez0eNE+5*(S65XFU-amtT_Yk7nyZ?adi8D+aj+E z{FW9OKQ>m!)z^Mg8v+|g!_ZQ9_7Wj7{;s1`y&9sK!8xD$^E_WH9%)D-Y`Vhgh!sQV z0(ITOF>3=Pr$q7281Hk`JKS1n0;dF&=?2}_e@!35UKVBjmRXc#ZQjtZX>P+VuDj{6 z?J1NXC0&5acko~M_@1D`8A8wVLJfLvTptcOBZ*$H9W2z*$G5vWO3SxeaQ~Gqy7xZNn1p+SE26 zto4VFSd{(qHqh(UfsmesD<2K~>YjXMRmov@PRIRI-LJ@<2!JenQ!6E+aQ1zDKZ5tI z1w)Bp_`>Wt8qy5TU-ie=B77MKa?FdYC0b39+(*CVaeR}K+XU~3eW*d$?`+QI!{+r_ zhz@*5X4{BZiwFMuGY*)qw3BHZ8Yu)LQ}qKVjPf0rBOy_;j#xZo9U-*S#wB)RGdk)V zD4UnSeMmoZW8=Y7&#PYE_w^mxcTGDzGi61l3uX5~d!Tyx6U_3A)T2xo^oi%$UI zrK|8bEc;z0ood<0MG{+)Y2@xlg{gvG@;7Nf>A@lZcGfu`=EktMNA#b`Tm%~gkz2yl zNDJZ0CnE(tfhy)EhrI~7NWnT79kL$+fA*DO4Lhwd6#ZW~IcWllVeLx;yW}hR_9{#e z{iPw=aNV$)zf;MP`y?5VybTz);($3K{cC2tOp1~11PRt#=#NHT1c6~+iB~ilxVZ&K?Yh}ukT38bXe)Jb{U=UjT zuqa6^HxBQ2YMpQt*(M!o?LO!qkeM?g3QGm2RQ(_9y?I>C>)ZYvvhCQ|JM$ctq0B>s zQs!1c%J9( z_4~&S~|(TWyc;#kKw zHS7+&r5Kb3GT|f@U~P?^Nx)K(6!R*j*+-x|^Z^(*>pKCqoqE*ykczlItrt zE@-mI!YGW7&)pP#)CNQ-rQ2_)0MCSLj=9p+su= zB+1QG?4k%8@3ciX&Ma7w~hqiecV8qTWy{ErRI*mRxs1X8A{-7U2eP5Fe6muq!% zMs0#Yf++(QC&b1`RyCz0E-^)0SlQ9X9CmGP(Kj^nkV)`@{Axh&aWE)ui|Gc1jh9tj zeXCHRlI5g{pfQ-}Pvz2C8rCz|gZ^Z_SNxtGp3mp{v(?Ko!&hoL~;R zlORBx?6TK0O{A(}4&6W&Jmu9Mg$Np@Lif5JiyhB+B7V`?p#fvXD5CG&*vC5PG#oB8bV+q%1pmt=>io2Zr= zzSoA8``BOCE@m$2rU>-~!xq;Al|rhlcc3_tJ(wR6wF7a@B2@E@gL1)4Gqa`*&Tek1 ztGnOw)$j3y@Z{Cn;k$(3IftG^u)D~Fx-cKu8-#l=qXT&^lmdm7ZC<Kx@*dY%%t%@_U&3~u&RpOq+%$_q*kbCRU%QH%h1&|i zC5BLEsIc9{D~_@zLkb9#H#W`s0@G2+)b985{gqml9e=-5e}b;)F4ZP1*?VCma-aEk zek@aI!jVY@u2N7wrqe1135SaHX;f1WsvT!Cyo}DQ5NW@T68pa8&sNsz`gNR}fGi^k zqIhV=P#Q=VO8}c-m2j~xsee!8`FFUj00S6ixYx0DTx|WB3K=vwvTG`ccHweH0WYZ; zTDz4w9sQ4CDJ7nF6-qY_KO4Wpa){b2jZHQuGGQ5IrHn&}Pr~K;T!+a{@a|~u&w6+b17+5Z4qKS3wyB7(|s3wr;>5+$QF5Ca+SiX53;TrR^#Z+1$*K#ubf zmoe9384ldoDK=2Xj;cq>++%nxeS6(0gG9Fgge>?+!UE4XK_m~*nm)VseOzj`an^HH zRIfOfjIGQuIweUW(CJU!FnR|kbK#Ma3AB=hbQXlc12ruaPY2iZQDI}tLgZA zt(%-5cTlbLm6vKX9QuDATj=OmzeW>mK!gE~%tOt=#M$y9Xfa{$hb{2tt&@9v5o;3^ zL{oE$X!>uRDo-kAn~K#%QJ5$^L)<&7#$HAMT}rO)vgpS{`NPR6oYi(UQs?sJ_zp_8 zz^ug1^3s6+(@GJ3q)e~KYGbC2tvFcLclPJ}~N$H)e%6O?p?HeFS zK{7lhR7&VbSpqdfhW|%1>RbIE(s1R^f72hCHM%xPEEqxgzK%#&cY%vupXfMRG>e^I zNEU9jtRw`EObjiG>xu>ivb1j&Z-zpRQ)&gzjU-)o$Y8TV0!p5UZ!Alk8Of6HK0;O5!SVfeSF~bi2&14Y8xD%K()s1*iac<*rAZP$eXNk1_JOZl<-LZ zRTSUp_khS^wF##b;vh&b$&N>J`y%UCC_#x|4)72Wy|E0%mCoTH=S8fklk=;yfK7}# zK+dsU{!x2CtZU(Z0}!<*wi{k6s}N+BswYprb?ZFd{(Jr$e<+6RxuNove1A`WNU5?* zX0-)TBRd7-1F~RW51FG=EhSc$8eNxmSUrR=IM@w%tW<5DC!g`RRy+C4weQQF6#o$F z@KhOXr1JN<@KksbMZuh--HmY3*;;_|;5HSxBy*|l z6E5&hBM7B=DycBTWANEzTO3~|yFRl}T)3nZu%rc-zJ$IZ1={&VlT+-f%2@|K?1e|t z4@aNUJq6DMw-lX(D0o^e`=r*@5wSbahR>rQ9G|S#c)scOTl_DWERzWP6ld^TYH08d ziLHQ;mSx`|TQaB^1NpF?V)oPLvwBBD>Doug)Lm1(-yboaPElMeD+*7;1=}ofw6gLC zwQhsx+C<7cAtz*WQ9AD+xA!BwBv?zC)SGgdf0(T5g>DcnOJRqM0_bXCy}DiA$lpv( z`M(WfwOxO#E%^DOuPCil3f961R$8Be1)^^M-*xtFw;Hq|(>OQ={5vR6+kTZ+J45oC zPyAfaiIRkNFh&$@JAR81d5rF|i zzCQy5K~bdO7P1>o&_J3XVVDH9l@eXloi9Mg@D5{<$WO?is4tT$c&NDAF#4RNQm^b1Oz}_HJPoNc6{{Dv;pzoRyUSW$Q91TA^IT zSBomF^wT^dpi(8^FfBsDxPJ?=&g0BvZdo8!FG@c(=Ls=TFyJ$7oViMXztZAKEPN43 zNZrXE6g%JxUp!&=C_?b97aX=#*OrxENTca#q{o-}V}bl!3NNm>H+54NpZPz5+IA=; zfX!PD4k56mLI!dUbgKXRjR9lJ+z+jmTmzq;0*QZY>{Ibus?@e8rxnk7c=|)#mh=8= zY>>Ae3yVNB&%|tN>53(`@Ufk<_Y#WRDWRIWt8JCTz^_=jwU&+g{p%)QVif){!z<$J zA$|pWL5g#}zgSyG#*0MfED(Yi)QCtf1PH~(*)c}|Q?1iZnFArR@aZCPz-A+@Gj%+SPk z*D<{3lo@w5AeJ4z*LEe^QeMgy6i)%^%kXSH z0$S-2R%9M^Q!a?N4isv7~RGQbk{ zi_ty%wDY3za#xO-un#jeWC)8J?Zw>RXd^P7&#Z z=qJ+qyl@D#RvF?K0^=+a7O^0g91E5_9cDWfDm+M@x>U|pI<4D%54nP%jO~=7m=k@S z*jkFNxA10&yG$1L-PuTm>X=8#)*Vi(Mq@dzZ$^(YokIB#LquBxi;D>aPM zSkjrkvuXFepnZE$M~W$`;G!z^Y?&bCTjr1?80NKK-!t^OxHTi=agpCfF|=P;Z-Cw9 z0!&9*YYU@GtJg^?Fc~cnEt%MBZ6)$owXJKMEIgWPop^r?Ed zwyOxLQbpM>dIy>JCOZZTJtFmAPN1}dJbn>HN|h=9j)+28MUQYc>NKc^9vY zTz8_|x%Y!5>|iS&t!JLO7gOoDc5 zYlKuW0BtN4v|d_*3pDMLJVlT8F=|q|UW&*5jto>LyWwpjgUgH?ciur`9drLg!C2e> zpzy9@JH4js3h%4>0ykV#jUtIQ^`+l@`6a1hnJCV>%}=9L-7Szrov4*wFr@!or6mQQ zZ@XY=mKGmI*zA;Sej_Pk;y}>kHCzWc+8{EKtHutC`Vb5GQrA{ zKN49XVmPUx#CA)4N(mz-;$9?1l(p>iM@F6*>7DoT==YY9dB?j>?bgjDNNkrbs|=p=>Jm2Orh+uTag9L2qQTDNlG% zq9u<=T_=a1v>rlbN<(rR_N#x4AO!M`p1k9d*x*R3Au<}7PvI_s(TPxwHyQ!;iq?VfnY=Wq)S6td}@*qOXBbyxeN$NjK7#(6YC;=={XSZZzvsV z+5P+8&NgNmO@W+T)Xc2N3>g{|+f?giGAk@5K@RgTRadHnTcL0huw%4QS@z(a0;lp* zd*j%unL6eu!+=tjuy=PzS4~S0o@)#jg?EG1c`O3WWB@mPw{_m3%JRo#RXcxrO+ewGp6Aemfo!nK~j6 zR{O?F8d1B8DpkOL-2@(M$-Yrf_k)E=#&DfrSou~B{kHzcGzA_L39|q>QqWt{CqGME z7rzateCbOuYZX1DV}RDCtHibPRZen}0H^w957Mvvm}Ab`W_DGjnHPRg_P)5aN`{ch zPmz&`L9cjfAk?$nQwn9P9)&{ERLipW?@KA$H}=>^@?F z2P4@F*`TYNNQNXysq2HB&Jq9n{{P*DHmo&RYoxEtfQKL48tI)6Oexbjpm1qRlanY` z)Y30BusPNn{GKJ6T&|hAIOXnwdNqaplAmLL{@Jec3bkn%NG>klg-d^M$j;5##3#8| zMW=fPED~;Nz}&&VmVN%H+eKGzQ9t8*o5wd8|08WoB|*UF)ZfdubCD2FwI&tG-bJ^S zrJtBX0BK%0QV5%>Ry7KEJCUQgS)k*#S(THjzQZC;XI3BUAx;+Jz$y9_>ECbP_Uff8 zZ-us0p_6G8GFW}VY$WZ_)$Oh#lZ*pd6Ypv1s}aZaFIXT0Ty^@t3Xy|0wssO&m!FW( zDDO&G=t6W&F zqgA41vo2=8M>jO{q!VHYGdWBS8*yw?`MWe~`*9y8%i2ao;4}ZBj3w7l6F|J+SXjs3 zrX>7_$ra~AtFJkb9cyNiecgdfc z<2LGa)G;5rv3`|dyL~MhbnZB8>`=`M_5Z2zdRYCTR=a&yj}BY^sj2IXnPK~0*4>x3 zO+R#coYFHXw$rYjFVo&mSEabTcxyEM&vs#xj~_p-=hqF=C+XQ+i{BS7|L~l#THE3n z0*T-O%at`oJlnM=D*fJb!xQVn4R%(%+8meqUtX)|HCQXccTbf0y9fjSzk@gaex&MG z81`>8U7Cge{M>H;pB7L4>|*QxW%(rky-r7sQp&%|PxF;(EU)rEG&JV_`9E*|f8>Vk z`;(&tH*VRo*d3CYAH#9KAB%7A*PL~?dw@qQ06~bunT09&`Hrt|A4)9o=9ozm-!={; z?Z9ELpbF;TK3Sys9igu354qj2@4wH!@K2hHY(&m2U4u6@>J~{q(BJ3h>4dkOP-%Z` z@(NBnjUW@fMl~?{O{IC8Xqc=XwlEF3wCxKH@Hfb zDjTQi;Z7M%AYxJtzPrkiprl{KGR!llQ|M7+LjNk!B zIGQgtC&!u=^C0Kvc2(-xL4 zvPPQ>HsmNtte8iuKG91>Ka9oY8rpm|P!MR2lN#ZA<=*ftZRp zfBrBX0~EuC4{v|JWh?vlA3k`66t^&#uaouMW6z+< z3M;;2&RT8s8=vS2W^k%={{>H0&wX?*0^i0cm<(^4u$aU~+Hp(1%tbOiVO`x;0W{Pc z4tfA`P!A40<#gpY(rK5nz3n>1YZ#*qC-=*YbYo-n6DVvQc601y#FSQt%o^mM^XfI| zfe2WpUi~x1;fw_jVNB*PWR^2P8^t&wpv$BS4zy>7P}&}X2JaW|uKpOmo;$ltuu&oG zJvsD2wW?L$ey{ujKSu5hUCqghe9K56vPj3olQrvi)R{MLUh%^-mUK9o4DjTjkmQS- zaNK&Xw+tuJ&2~_}xqWC8!mwCQd3-}BX=7sI%ac6}5}BHjkug3|xKRy_y7TP97M@lrHlgojapv8yk)8)D?#$oB6+2{>1BzuU)@xrYf7H z+_+)G@m~cj);@d8Rh;VEl}d3Im%dM~7i1xv!!~74k>Cz;UT_ps5#EpsDvuFzX&R>2 z9QHb1Jjeo2{%yM6ma`;hHND1Vj!!-6aOMEQrqpZKhVko9PDoBo9f^i7HtfN+h3dCz zL=*rQJbU-9D~B#WhtX2iZ_;FD7dH=&JWkOr&Xpe9Dt^Ye7DnDyY=eNKM~||tkJN6~ zrz7`ZcdzMN^P@MbKdJDM1(@x`%{|U8e7;c;;hdt%{Ya6Z2j9wZ?G`6z-_wq^WK z#pKB$n|q{K3YAURZ``O+qcdmEJ}TuReP$?-J={O;v<^olAI~&}#mG^n1&oj_PHKhWmCN4;H zi7BeXE@pR6Oxr})5W@zT1<>U_hhU^qOqx_MG;i^O1=~-ZI^{S?Uj6&${j1w)qqO)o zSu__4jxsN&ZIrL#n8=Spw|)B;u@5*?^<>eKZZ+yOH(m5C-{oJETKR}Vh-eoxJAxDv zr|Mpwo-+V{HViE20luXQV-a*XyM=?9*%oFh&Lezv%)WA2seiqvMcDVTmwA9?fD4^? zQ1_2bS*2$c?l&Vz)o!Gj917IIY1GQk7cZJ9Eq?v{_WbG7r^mei`v<%kdzpcf?2~gQ zJioDhh6gVBKjL50M|fGjpjO-_7oriOTB8=oag8{w098f{Qy8gdJ>0Ez4eI z4XIwMVK)1OWt6q#v&Z7a)|_IcP*NjjqJ)S($qBPF*cYlWB2NEk7xfpY{J(vQxwpC& zj*E6ov>|luPxKQ{s|KZv$HKs z!^g)Re(LdGKGC;>N#tMcI5F0)VmZ>y#Lx6LLHS7xe23t$8ifk~JH~qnoaU@l?ccv2 zDxifNCss$z%KiQv-wxyW3=UMCg*xa?cFoj)e zH&Rnm|M}f2so(wnsu|&|^Y|0tFd*9!wh$@xxCEu8cn19YV|wx0J%JYg{qc?8@IRH( z|5S(nQ%?U+3qeTupSJ0LTE+jr_SO&gI{NndTDbLwhKBQV<9>W|t5@eh6gj1QpOg-ioc$9kfjf(x9lH9z$AD@K zmD0FHi{cYz+O=(SlVuoryMY=$8WXOS72(rFFZ&aP_u0lr)-XOXJR_V z{t-2rb+p~M5nG#Q!Pk|6gBlz+Q(sKsU-XgkO~M|;>78`Vx#>D9JA6C0Nj)>%z5Ceb zsk=OEDla2UZwwWys1G;{-Z}1G@7Z^|C(8MvuMrvndpn-ZZt}eHJ_tZvP+wd709Sgy zy0#*MSGnFhaY%8!@)QEk#$!fxoPuB#*uk2%d?%t<&0^?3{O0e)jkq58tZSturNKRl zK82DAOl7>0&)dlW0RL%o^XrvAS;AzkNJ$D5S;Xz%gD1FIsepXbplG~>Pgz1`nmU9CNM$dGnQA0M9$Clk+}y_uXmf*zu#me!eh z9Usq~-DiTT_KUl-`|Q-S)>EE*KKsy#e(pceW9(4S2;Dv2O`$t-Yy5{^DLYf{HBemb zQ88atboynW$KLowX(MO8JU@_cp`CpG;GvyP6U_!B8zrA}THv$eqavqA$-8&4GDn|& z-QvuTEiJBgtE;8u6`iMyJiaRaQ_CIAao%V@W{mCgo6o)`#mua}e1c_PE)5nw##v?5B8A z()IDnl^wK_jrR3!Ug~*!e%e#z)dh8@Cl=hjbY*ih)8?tVwM%;Z`{&y_)J`^(7e?&b z6j#f%Yf#Vptt)_{+GTyNZ2IYLaATUiSiSIk{U7@{)u8!Mg$Ye(t5h%YFmF~{DAm6z zjHY`+hRc~d1CFj&2WwzL>vgXDGL!ky|%32WKMY4Za2e^Oop zvcI{X5S1J2IIBzPr%&0anPSk?-?(z+%7&cE-9CLnlFxa@kGr(!xA3Zc07B(0Esg9M z_+Kz}TIHt=9OKZj^&^G>7eBl4k927~L9_h>&&Q2a-+c|;8^HUEfBp2r^jz6M&&qGl zgL?Xp73w!DpKPf8FC?!1gXDz^+$(>v`ad;v=Ct{L7B}|0e(mbj8Axnzams?&{7YlZ--ES0r(V5&{SCLv8U?3Sf~$4y<_2SV zHn+Ognjg-s=mhQ)X%n*>EpxBwyQ&%N?J*oQQxFrD*_b9ydmXj?-lF*I#DW-Aly}s_ zUDxEqpITa3;#q|tA_AVRsj<(wCr_U2b!_Ns^#-ZJca+*zaW3XYsE&6Vdi8wiZ+Ztt z`w(gkT{=&1q#|VieFlk&i;vs1X%p{~-n5)EFZz4F4~a{yqH-KHYBO^z2N6@Q-P82_ zRRh34ft0Xej06M&$J!Y;()-EXzie$ht2$g*I-WW0J230hPMrg(yMN9r?qPm@uvL>n z15_L$4O1wAg~y@Gw+7hhU|Zeb7BFyj%!Iz>%;Myp;B((_IaaX_XZG4P@mmV**+GBK zeJj|`4W0y#Ri=KQ_Pa7U06<=C|Rk7NW_u4;5-O%qq?#N{1>1`Tpgjdu$Nu>IDm}eV?Z7H zVDUM@{TG`7pWQ}Ha}Zs_C}86|guX>z-%pvo>rb5qT7Ugzg)pQ+{rX{K$r;!Iqf*%R zj9Lf>g!zqH-wtEv%_0Qo*&PdUfA=tgz5O>=m3wrOF`&FT>0WFlceWi-M|m5);SnZ4 z`<>~p^X=-k7WF%hH31Q^Ci?Zu8?-xN{^P-dgabvp-aQtIY$F0=xRENvhK0}G?CZQ* z+wVmgr%m(CESWyyrCrgB4HcXRbhMjNi~x3Y(o{WSbg+%GX)l+}pqnoqPSvRK;QW$P z(5y#ay_&(;vh}WRlXBTW31HICLCmEDLi-Tx*xs)*8%odC)^>5pBS%V~)`7mgJAp6a zu(3OK;`vB|o-1GV&AqrIjM7C?6pnmi*y`e{OFAh<_3OZ&?}yZG z+}-4Q;}*s~5ARK@s?_V)(G;dUW_F)FLKIjWGV8}q3WXkB}O0W7 zS+nMC=0V!Z9bf`Bwu_;m0|y+~AVa-f*N-yr`;(mKHFJEDj8 zTlAy6oB>0X0+c;{gLT!E4n%Ri2`8gLqYkwj8SwWglidm4Wvu_8PkR=pO`a(}UoU*k)*C6r+Cml`BIS1oD^62FmzB zVHH-LAZJSiA}SmPN57vSPOD&M$E6&Hp`UR7^n!ysA~jXlxefX=GIf-#mxvaOTPEOV#CMz@>*>byE@wQW?QbhjNuxC2z#BG znl);SRGmA2p1l>mitHYz3VB}_5t6}F%&brDD9a6)H&3)FZ^l;qG>Mc`OR<9NB zQ8H2J46s^d<}c)tW)>@7R;@NOh<2fB)vC`PKVFbG`7%ZlVcc|sGZuwe63A4t;X7?^ zuM`b-&uJSed_qu8jV3|Bt?aa7QK2Er8UMVh>EkB_mT&To&hx0<;Gcg&5rylbIg{{j z9bwT?CTBvV6{QJUH!lR4&mEA(htOmfRbY}c+yJAZ#V~&8BsCED8@>ZvZG#sPN&I}&7%$_|rrgz|bRFP4X zQk*Eh1t|RG+l4DeXfNNFwCsm_Cx<32TgGxzX42+~78BGwgt%zi&dfiPA%mQdLDgCe zTzWTXS~b&99s(2j-PWvIH&O-aoK4LrEEM&rB!0#cjsn3Kp>d3ylyreuP@HyRMY2jA zY7u%z-!t|pFSCUr=}e$*E+OJ5Ryko@hE_KbTH!S|Ga2uBbu5I6B%<|th5 z&D>Sx_Guz;Q)PQTUj`8l(Urt-AR2HVf!HQ-Ss7_r^|} zU*{d&zug(p#?;G~FGu?jpYLrl zeR8;pIj{5eqW6&TJHMJV`A)@IM3Hqu=G5VKAfgE#T%9JU@+MdHz;%3HM zT8}<3Ct>qe$3~u=d8^Cg%WFL5Oggjh z&~|4ZmYs8S&H1tHm?p~h1_pQS-Cv|Nv1CWidU%miIoj^{W#Z|!>F5u0zsZ>W9EoZ> zh#75D>T}-CPd1EA0VfIWp+j$`bb&z{9o@fp>S4{WCUj66x>i7s!;&;(4r1o>n>!_4 zDh@We<|Z3= z+`T&p03~6;)8X);LpqG!n<@0qInM=^B?f?u{jz0CmwrM(s(fNpJz@d{V)lVywUnol zuTcFR=Gx_>zcX*eS0~h}IPTHQ@xs!6_hwaZxog|DZ8Dmec(P#5+sPC^G(laTUNznE zv%I`r%G|d)6yDybv|^UOObfE4%pCpszV8T<|I{xfRKCN>pm7@;c99D2HGN_ZUNr5X zlgnxTat&508g#ZBOMkZOs@H{sl9!%xAqxDQPIwzY}xRui-yGss*P6^)wVcue1f7X zq8r%oIcnnhvnS`~yO#ezZDxUrLZM_Y*vy_Cjnzgp+r7hxdl%a#qCh}}aDZS6`Zdx0 z(rRebnJ{V_H*e0S)_wNk#bN%=KKXm_lZ;YOtw4uE)l66PnfE9f4olH=6wpsR{OtjJ zpCpK~UWH54@9x#%k)_}6p(92}jLS`^2Zf42t_)MIs zNCL^di`cgW*Q+&K7i=h9n%u_U*Q#6Z-c}$`Su3j5U>B?-w%)D?wjz7l-Yzb@6ifrw zd*L%vr3DuW3iR&uffdQ01TCa@7#BKtja7(AmFKsDComYXVW;Mz#fx+J&i+hmQdT-p zx!r-~)SfNAq9VQkZU`glw*N9$qwZUBodp}qpOhdOP5cNDD#UVx$V0dDUsqJGTeq&V zqLHbA_or~wE8*WtUk*6ut`mVwaRfdtg#J~%-Cm5U-L$tU%RHC=X@4sF@`bOTSf6Hf zzt49iA{Hd_{(sbMDf)xRO5iaMh5zf-Mpaslf{K?KlV5!E!Gj6=_U+5nxDr z+}~QeMA!5l?PoSU#Ixb8`uwx$%F49eI{L4DhBxvA3ltqdaWk;vqOLP z_jnqQR8iCrPmVh0*DajFNW@6KavmG>5Ji<9nO#D{yQVL;eknM<>Z{3Mx;2~?qG|Q| zon54(q#=)7Uhyk(Z0JfsR7E(!7zJ(Cw3U8aaZVz~x}^JecfRqr02nP>wzNOzd0aBM zC=KXza`-KKj}PFa6{965i6#m3&!82ibe|j6-bP6vrH4pdvPyqayuM*Ha3^8niou7h ze@s;CH*V}n0BDb0!En`r#f$Yh`QVD<506)wmrQ%&%Mno4{M`EWf8L=UZ=3S<`}f^w z=-b4FHLeRHlg+*0RolT``HFtIn6II4$ zWCYut(Nw;z98lKlBqGzEJj*_aLpOgYE$yY+j4T1z=cZ#`{u#F;+nuQ}W)hm6 zHrCgjstBSA-%a9e|8%0pwwDYH9DkHrUS4iaQ5F5=3+hddUfE>Z>7zYhj|Jwh-6 zazsFFo$3!bz-GJimKM&b0qJiaoGOVee_F8WUGqr>ZZ9@RSRC58-5HR49({}I;G)7% z)UH?h+sx^+C&J;(30(wSiwBNBI?>3Dm>gAsI&BMLrNvvykY-VroME)&HDPDFGhK)| zOFPb8>-16j-wDZ{?9)+TdND88zf9eg@skMZ;phRpGhFqJ9Vhj|7ShUgXO`3KEv>5Q zI&$_4QxgegO_bn)qrj!j389-6aTsf87X8&~;6sindxWq#)3hfG$%Qp;^MhNThF#$f zx4ypXPJmn#_1a2IdGzJyPwvZgm$)XRnK)X5S>LGgMsc*k7KQk{4LJL@L&H$bFfILA zxxG==^hPnCG_U&A(zE(8I7yEmhaqC{lgT0=YmnYTyrT*l+-2N4{6&ewjm>}JvumoSM_s>5c zOo|>hdE`ZfWX_n&D1!VJ#SSF!Za@?@Y-{VZ+q8atcfzI%uC9kkf?sO4znWC{+PZ`G zCuh%Gx@;M*HeT-Ih62R-QMJNL=NWq$uF|Faq1xD%u#_{r-CNy%A?7o^ z#Q3LwQQo_-vq#^pS9pm?k^0@F3@|PsVBWnG*P2?s|MaOhV#Lvl%D)zD!QFSSNmYv& zV0?S0B=SvM01aD)M~i9`&wD47U8pE9sp{7hzJrHJJ+Y*4@so@ltf6#<;HEb1J!3L~ zYO_K-*%$U&RqY>&g6@+qYV2Upsa~^YKTIyIrYs!a$mYYO>NV?khxr-(;^vAPjnGk@ za7lMn$e>QU85mL@q`Ex*E96(-Di`3REGYz}K72}HMqs)gQq|%d%c7}LtoXF5`-Ny@&U%zP9#eN0iHhnrh3QNCnOM~CC?R6|qS@BZp_ z>-GR{HC4KxNphb<2i8j^X1V74y8Ifkw8hdl@Jc;kfbsiF+qtJFA-bbtX)OPX196)5 z4csZZNrRG~EU0=bG;6f=_w7A}b;3rcsNj+|BC#n7YeF z!bJaOeP(x;E?Q17N}~z*=2ug9;?K^lK3vMF(4%}~5M^gs>$xGD6?hwt zS+r=;%Fo2@psLkt${F7Dg5U@**!J$rC@`l7PtZ0sagFyXbt(E%n3QMyp;t#b`P-ge zUQPC-yHJwM2rd~qeE4ooOHOSvt=hKdj6>wsp5-C6v3Me`%p63$8l5~qmVh|V(Le_B z7z*A#JaeZ>;`^(uc{-2f*LiHK=o{GG>H1%mAjxf0n6OSy$*H={g(@}I?ftJJ09o<>TD6VKEH4fWG;@Z?N(@p z*|Yg_OItPS>M)~=F-W8r_IA;U%LH`*rD;+$c+>QOZ4x?xg>84H0-n;Ph^BdQjR&}& zI#H`kDfjX3HtCV~)w&uF)4AZ7(h*MDf7J8&`C-Hkya%5a70q%lc~tz&b;XEMCQRsK z38q?{DDc2V5k!;;0-~_@Skr4$8;dwf2W~W6wAN)!2!q2Qzb|kl=-GGf!(*2XI7oi2 zW0J&h#M$*sP4IpC;odgobYdcdMYrrym%~XH4XgS`w4ioc^DWt(X-k{d z*tA?>4XP;la{FB4s);ys9)=#s?UzAxEPfWbe>OO+i(y%|cTaA(W)YuAEsxVH1S^^0=OT?CGOu)p#c{>?y4dg{+tv|kYLBT=IBKg0KAW40A-6J@nwx_CR0c6 zGo-peUTBy4PQFGC%)=$8?Y`MlAue)Feg;2&M=M!W?cUd*p`a9V6hWj#gD8i>_1>p# zS+}+R15S5a+^Oxd0rIwgTjW4O7L+QaL z=?g$G>TkpK$gf>e$5VLk(EFpM19IpHR_4mO6;Dc@`8)Gvr`PVxgvx+ReSlD=nkUrK z_#%;p5u=j3T{b19_W3o7cIK9D9n&_nOFdMZDZa>jTR8IW%!f~(;#=nZZJgk-)b~Wa z)6gknduTsxqtl@K^^OX~(CdMTpFhwP5k(OKw^u^4gEtjvH|<5YiiD5Te#05r{_=( zc2&N~I=GJhstpS|VdfoV#i+MEW}MfsxB#ItYgFM}TA2oa|2?@}3_+;K@w{-V!NbYr zmb9?x;O8&=7U?H(8kqk$I>tpsxn0sB#2}t+DHeq?1oOFZ?fGZF`9JloOikC*hwL~{ zCo1q>Q$x>VFeWg_Q4j$W8+Ds#Mauzn-~M^V{z)on(g&}pshs*&BSPT+s@1ABXxw&qsz%+G*`ncBqgspy225UhF+IIYRz(eq1Dq-uj^EL4`Z1PKtHhJg4^sIE zQ$`&JgVja2HEQI@LDy~1$9~tlu?G8VDT3&T+kaE;!6lP(8)Gd@oK_(@5Lh80ks9B z3g;~c*$|OLX_-g>=#G?$1mfYBb3LlgKTPluQjO(o!$>ojlNHb#`GH7_(n4kz=+(!rsoB8l!cs4hD!g%uyJvO6St;E;Re3oy$H_E!d_2)Jan1J<_Z#22GR z?>SiI^tQsn$(Cmq%}ms-|1^p2K#e0G4%k!)0f;|&ch;u9Y80n|#=N|8m9)sp-NP<2 zn6=CI$HwJ83#U#J7eYYio7LvL#E=NE9hF$n%71k43ziGoV9r7?_w@YQcHVR%%Jua0 zbfAGa=Gqm}f6qTh^=_ZxwaCpauI2bAPoFvg$?OZM0D7NpeyOo>S!|C}M=oXf>(Xq? zNlVWjRxb?I)YJ?lHQ8nKiGiQk?#yNg4k-Zvd>g9;{y4?l$Qx_uS>LZgB?rEG_3GK9 z1{%5-i969>{40ETIQo|wUu2m?CtrF-#bDZ#@0ifCty}r`#dpqmx$V>6RKJKcav0)t zL(9^xUAs!pr*-)Cg*|xDHdY>H1=fje>S;)SduoZx*Mqy+Ticx%zDP?s)7D}q!HPn- zSyA~TaE#Mu&(aAIOx>NTeSkThp$J0ee|PRTpv5ieF7yRZxB(7{8iw((G82;DFoy2T zGTKZRPsSdd=>d`Mo~KD@J#}9oo}wNl6u)M-94{^>0dLHS9^b3z-Ut*K?GfQfIf}){ zeQzh}`8RCd9G@G(=s;Qf#nW}FnAVnhP!Yt%NGlfvhU3P-Nm1Qcg=Mr|54R^*H!PVe zM<3g-S=e0i1{>MPoF=ZcK(R5YxgN+KXrkp8pV2VF;=(GfY$V+PYoF_sMoyt zY>@Vt(hvxHh22XSquez%loM&}<22MmQvl~_Avo5y$Hlevi;`BX=0=smPI5ybb2h}f zmA$dPgWYYB#~~u)VN&{akO9-4B!6Y^i`8_BYt^bHhr{DvsE41Ur_pF4Xc;Fk@BOS)<>pg5^ zeXeNo999lw<@CkzY;V~J5WLWoaedz3;5SZF1hJOw zJUom?b)N#zDyQ<;{CR|GwSDj2*_W)VDPKyfLL%L;wk`?i!PF&(ZF9lNl&s&D-l)b0cm zg2gDnT*nEtH6mjv+qd9l!F>XYvto=Z98fjrcwkDzbcX3s%vm862Cc^Loz1(^Wxo6 z$F{_C;P0o?bTvS_Oi3AiRrtXfA|RxKO8Da`XI(gVnG`?cA9}U^bo#li?n`NPUp9oz z{7^!Q*01%ycv9DysgYLfCv0=dSpD~}o*YT%=$jqfoQ?Nr{)g14cWb}#JVrj}qScwg zSj&WN)yno-F&IvEF_#V*L`uzLJJ*p23fXQ?prYrk;$X_Kv zJuD2UYv|8IQ#3VQ{{`Tas6^lNB|nz}K`_QMw7LgxR`)X#-=HtcW@pio$26%Y%3-Ip z;^YCN1G}#P6bPog$T#hY+%{>}ercBhfcYqD8?F1s-=5Wb`sB&frhR75V4Sq%qU(y} zzRXXyO%eR%dfsmb9SB^};HNRb1fxnL5{i|aJEj?QA3e8a<%_2AxY0fm$2RY1$fIk&M z0#TekbLL+D@y{JEt)`{=d$>uYR9H`6zn%&EJ?9h#vrpRt`fM23;WVMEd8z460i!9e z_Ri3`6!^fy=WJqPGiK!^q;;_RQ%9d$61d$N9^N~{hkBfVL7|&Q?W&6C&*}sq{||5L zQG5#J%B;vIS64=DZlROV*kFDL+q{RsRslw4-B&oZjIgG3P1)IL7bkPa=bP8O{&3%* z9>322N}9x>eDf`?U3@w|(LLq^W0bx9CZ|;x{ec$dP0EDEs@0o(>xc92A3uD^qF#dp z{m$qZjjyO8(J;Iw9FMwQb4Az}V8xLtQlxv{Kq)FgvlL2L-#n_@;`?#VDNjyUU7^U9 zveQ1n3~AJ{0P9FACFU0Egcu^9M$oWtT)+PGOLq=S?8fl*h+VY|u7Wt|`~37|`DECO zP>9ay4KRqz6N(OtB2XHK@h6sL>5YJNu^)XEw)8tvLlcwo_+2IUkMhjmXZ9;* zV$-E{zsUd-*B-!bLYzk09Y2up7Rf*+-&nNcXhdqa=>O@$gU5j*T%Q*eMS*M2z#VYm zf^SLNO8d-x7*1Isv!TC!{rV2NDFQa?Aqxgzfqd@J5$?6?_kIQhV3dsZQS@v$*>_g% zC~m4XjVo^sICt&waj@8p2tb^cjm3vR8-?!;k^ z9&^w^#$CTH=X^P|YrR1>JhQXih4zLe&Ba49lG1PJ50Tyx<7~lRrL;`5u3EPP&x>6i z0ZzOxgc8zThVc&^JeWggvZQ4Ok5Ux|sntKyN=GRJ!?Z)T)UV5aJn9lSAAeM7)nLc6 z2m|;C=)B&G0F0fNDVF{jK4j7yzt%%Ok>NX0oV(AVobXq`FBwFSz8J*q4n^4TnkE9` zAP70DpBcy0SaUb&ARZZ2+*(+BO}4_ozz+7*zeoE)4~XX02Zm~if|C9|k0hv_KU|z; zDB6_GznHe5iKP<_2O?q%=bx&`U#az*i~a}pFe-b6R`9#B+8n3?#kuMLZ2#6bvy0rFZ+TI%4H>g0y>6)55yrwX2>Z@q7Wy^ z;kE}J2%W6h>J_#WRK}7E6QvuVfH?*OcPKaBB|cidN~_!XPtxWc}k0)cq1D2$|QK_ADORh(7&sR^5HDfK)Y#4NqI@7Esb0B??9XB?!MO!HTx# z%Y~xyzJZyDAK~B(s0!YGkHuXdSHxT7_?tU>Nz%Cwk~E4@=gkUBh|On@Y8QqQDUIEee+b{g zJ`V^Hc#A8Wwpp=b;4<%CGF9}MJ=Y9y9Tp{>&Spo1cj4nM4yQAzwHhn0v*ZMPS4xF1 zlA^ltTL3G9Kn2Dp^%X@E{LbR}&rNQRKf5U6_`TU+dGo0#iE(uGf_OyV5lBC*UgyYip;L~n~UGPW6 zb^MO&rY&2{Z#inTG{d$#=4n1f9ks*w+12Je6(%n>@yXHi+CR=$v52G`<;&l> zb4TFX_HEic*?tmZK2La=TYK-}*3fH1l&0zyl_FB=Nh?c#Qlba)^CLD17M53`5`$an zFyxwPP0;VRvNV?!?ncWa*k&MrjD0FxPtfFGDvBU>1`s15fmSmaC@fuY*8WrSfnDl- z{=;)VMTM{=LPfR5Amhtl?H$g0=g5oaX2wvns^Ruwxyblc%RBhi5ccHhyLWRHK|J!^ zWR5^6bTOP=*WV{Ys-FgWLv^Y|SO7hW!-krp|90bn(9)w2-E3CeVe8WIo7%%!Zfl_E z1n(vXRiu%UVhUEliW+CS3t+1V;-U3FJyk>Z9xXx4ikldn>;`1nBZzyYP>#lX%&}Xf z=*?)`&-8nVYrwRIl>f^AXUIqo##>6Gmi z%4ioJ`}8$exS}x~-d2A<87F(UwkgsN(8UIlw}Pl7b`yl)@`$N*MNl zF8boMsjH@<>1yAx;|Uh((RXt6=ZLviD@y8DRZXB(E30D4bjT>m&Kwa8 zJ|$W}d7Cd<^l8~CajoDMyCP#q4i3IpPY;Qs6!E4#0b1e`f4rt;lV-=6|IAv9<=ebc z#zNYiE{f!)kjHwZB-FatcGHFpJs1k;g}-`ekn!?g%T{Tz5&MeXWU+_WPmwaG1C_j3 zZKTnX_e_tT6-Q%ldkKRgv7qMOwj)2VE*Xx8miHErXD>x*Xb zhdZ^pOg#5B_JP5&@9{w(;~b6`HXfs%wqvGg8e9?R27$zM;^n(0qfWc&7{hpUX50`~ z`zAR)XD90)3cz<2i`R0(7k?5QX(S-!!?RJ>rSYMJJc ztNdTR{R1%l1-{ezyz-RGQW2Epm!z^QV#aO-MnV(i6Y{!9V|q~cDJvKEEd`XV7k_G2 z9$_5WqGw1kLhm~gwZSobv!7rs*-J&Roi=L5j2SCGe=WUABU`Yz8DXFYRy4ANdPptL zjZ!&qLt?6YORQcGsB~H*J0tK)yipI!cjrO*97q~qV)hYd9B>@iXRZ5SqLZ0N+3+~Azm-PB=7Rhw6x>8V^0v$!XJ10#)YcAwfp zdv-O`nL_F-g2>$ZS)UEPJ_bVb8fKm(t&ZYz-VPi&=;rW_RZ=3U>**vp73@GUpVUK@ zhM;c8nVvdcS6e+Lll8eCh$fCgtvfh-FQ{oze)s0^sZHCRk-E8fgi2{^iM@xWM~t#3 z;-gV32gQ-jgD6!Pfr-=y9-s9l-;=7j2QAE)PhUbPMcf-%Q1a=EaQDZT5|6>6YJn~k z8E&6F>F>O9JvLVOEDZ?xrsvkTvkbHm`o5L_KY5D7Lj8vNn*g zW}jzSQM*0%9)SQdLENS8I~rQVd@VWzw|&1$0ib_b$F&tc=L!2-C+k6pK>eGuscl$J zXmMC4<5LNuE=EmDk|u$^#(hVQ8zhTMegWU8U#NHCrD8ItX;1mx^V?212VJ@cP*pK+ zifNy&RaIG}t{9kG*-fM3jaU&XMWaS_%)`w29XrxpEa_g<^RjXl+==NKv0w|~EGg-9 z>3R^JQ&=Y23}MpVpz^pwg^<9(Ez+iXSBLiTE(yO#UWxvXgSuj|o*Je?#u*C%A`R)yfG&IPe6Hg8w+Je+>IswHDq0wNUK_4M=EG;* zL%U%Gc|uoeCXm2k;y}Nsp@qj$xsJ!o$VOBxACZL!tIF|eN1~^Zj4eo)&x?y=xa`y` zSK0P^&LIQuLUJ&t@8o0j9rT!HQuh<)g6#ms-Ywo|jQFSQ z^_>xHu2Z${vngKDcdeeMp_qU4i$7LcGeLv;dwmSUPHS?Py0vTPP?I3i{@@&`J%2NC ztrJo0JFBMbdb4?Use*;cn6{eFldsI09UZCdX?VS{OfU%Wo%xykG?Q$F$>@my(S)O{ z$RcrQADDJ=!_{v-ir8qOACi{o>@(vSFIy7K?H5G_!Pi8h+b8B z!e#jBdQLS=i)m3ynwt8}-T$)UOX($0DQP89gttug(Zi=-*UZi?FzmsdI@h=_`(-i} z#tTL;Z~UPuN|5$RXl{=G@|8|ef|YD)P@bZ3bueIFSvH_b*l^hvNQ)z6P?6^Blp=!6 zqV*jRd7RqS>~^f_VkGb-_XdJ)%B&S*7B(iLsnGgfj$TCdYu^3gI)`eq77;bXa_gjJ z1_ct5FP*_aJ=6|-UGiIrLdZ=k*!FmJ~C{Wq5o zZExbXNgkMHbTgFZZggyYPeYj$lyN?hOtYdB+5J%@M#ZZ}#_sb^jrgIVYb~0nprIuK zAyF;~BPw4-PJCtT4wR((ph>Wf_IrU9koZWi`K+YmC`-{e)0^+(zQ1lubl;*U=g-*5 z>3=0gGzatUSkc06eO(1L@(kw31+@tt-_lPZLkmi1DEAesDd7f*{R8#2ztIwRY2~Tx z+!iL7+&(RyYU3ZT_JlbRpR#sJPz2@Yv%bkaXqByxPAuQ2TP%nbw|^XqB__G&PY>_g zKpVAN8rrEov5g1BPA=?GT#z)0?UP3XA3rU%VKH*<{mXxyVZ|@a7}O`_mth64FeRgm zd9d*H`ngvzX}aL$lAFORR5 zKw`!Vi~4@i)9bYppUcz*MQhtV>4#}321N8Pd^~K#h$s?>H(JAYUg<(pcNu?TC=aqf zWg^?E*d=*;7;bf=35zP!&cMKLUfo_X;v^Ywr>^C2G5;fH$z&R(w>xMMZsV-$FjAyLFOpMNFDViYK5yjs0U4JNo&_3*ds1~ITLMsXk zA+pN)E%_Y1BeGiar#r?B1bBv=P3Ks)GUPpMrw>}G)wQM@soFTO~BuEsjo zPyWk-^aH)eKvgJ>TqL4+ zWD6^2dSpStK!i;ftbwLE$-cX0$UuENC4jCNj6luCQ=aG0Z8}E{|EvL{U)|ak*TDW@ z75lNG_P^+1K$&rb&As0P3#aJq+x5ktLqQ~-xszC&OOvn#j=M`6bO^RLmJ-$H!DudB z_ViMlec$P7+5FB7&+mmO{o~R{A>UARpul7wpRQ9xt9A%T-j#}0 zXIrzj%Xp7s)0D6pzMAS%zF;S9!HgN@Lqh7=f+a%&qno4m4PFaq^n#j-l9KF0+F?&n z!uFhWIFwtpSo?0d9HKd8I;Fz^K*l{bKRF>jK2vJRZoZ#SPEAeiKVo>t%a1#4IdFMs z*dMmdbJA!7r+`K(wgj4PAjeTac;Pa&Tk$Y*5yHkW8Kqn|=S$^5(hL;CX>#K7018h4*V0NtS>%%jXoFgA!69*#?7r`oP|~h=m%J&t&`_0M)sz6?Uh?Jel|O zc%@(w1w0%tIN27(XR2)WwVk46b8Jw_B(%*5vZv8eM$uHf_0e+-r8D-VHV`pPO8kVV z;Zl=YFi>dkCR2?aVcwpjFBUu)l3WsGOX?=Jp;XESc@Uu?VK)=`wn%Y<01JaYY_>%1 z1x)^u^86&*`{}D~-agCf1q2}kilp|)`o9FUs8^33n{0=7yg~rLL~70$&ZI=$ySG)Q zexJjYBPHZgWHg`^uD4rhJN*hNFD4)@T3L)=z0~rXj7F!eNB*=SA=WqS{KKWv9!FJN1$1|8MLr=mjNd+r)~m zoSP@WebTKypZPAJQk%#|RLvO+FIl6uBFvL`RVJ%VWvGtrqLtX^L?FX}7QInfa0|NI z35?)vz>c4_Y}pIJl1j4xyQMi(2&bsYZoFHRy2>=@7YmZ%opuB3#Gq4cK`sUaArNf? z$&%vXPYHEzo^yh|O7y-W?Mdk^j(kkK6keuE2`CZx8J6`W2V8}7W!|p~qaeFn$ZDxb zWZ@3CZ60{_Tl`9RtK%Sgn!ZEEm~av#m2YnP4jE;LlSFQog4WfAk}-!8Te1$Ak3xk= ziG*@88X(Y<+A9Bb6jod*A2P!X91XjD)s!Qm2zvjS9@5cRh-%bzc_Mm61Y(bkBR^wJ z^@}J(XMeHn%d23RCY+*wp3LGsMS`CYQqGUiBwd>2Gz9Cw4H&NuaW1`;w2q2Rc5KMu z$y>XR5?@oP$$GjgabsT%Gbj)Mn~SqbUaZjT?n1nXt!@u`Fw4b-`?-*Xge6Hd)qp(> z67_=NB4t_0OmXv5w)pydPR0_|hVXqHBrq`DHdrCDDpfW0uwuHO1f^hLSd?prY9M^$ z#*Jc{JI5&;)Pdov-@nP1p+{cF?it3F3(7kWswBw(X-O0_(v#otPBu~3*9-GVtc|a# z>6jyx-JFZR@Y-TgS0LwC1+QDzzDt*~*XBW28KGa%wvL7H(J4YY)?3MiIj)STaTj+OowiKeWh zl+askHCa0MFR?nxPbJYndA%;2CSnh6c=?A(4PAlxg2JP zUzDo$(`wxlh5SF4nm{v0wLB=Oh0Qk<=qcD!WbScr<{p;59_X*rcKu+KPG=V z2mh0i-Sq$ef5pSwG!ygwy~UQ2#BuMC<`x3mEx^oj;@)D(R8Qsz-01$luwdE^m-U$v zLd-IVHCbQA%ASUTF`hcHr@48~J>y4187tqjOA4g=>GSq#(S}cDTt*-gVzBgp7A0*u zPSlg2F4y!Qzt(+Y0A z*V@v8;!2@}2}$}wVY7*+FD~Sss70Vk5Z+je5uAcebhHObY1>Ch?~70)k%w7|2KTPW zFQ5)3$eYD3Mg2E87XQpADpya3{D}Yoqy|LEjN|$`Qzuhjs7Y{Yr;To}aJ0qJK~tX=>TY^qWsn zpF6$OTcCO z5y~K%vyfwQ0_YBE)S?7E6!d(*S3v>N%?}<0UVW*C%vCq3mzVBW)E9UGV}T_zO8!z@C=wg|S@}p}J+epd-XgWOQvYr+eyqjAgB=P&fF(og1WT z5*=-GIDu&Q*17f(J!Zp1R9z^=A5a38ala;Q*@eJnI{9+&nO+$<`R8}h7#Uro^}YJa zARx>4x$p2f#eBm@kGxu{6?j}ko8kq}_7V^H!y3w}HMa{`r*3-$+%7lvKL^(R;Pf)A z0$7W#xXmgitbZ}NH%s@x;Nc8{U4#n$A^Xh+-dbV8Dr6Iy2RG{=pr$agnG|)e0!}As zDW3r&`9I+e`F?$q3^`lf7_>i}cNg&?#ThK{0|6t2xPMuq*5w z;+H1r0Ytep05DG}k#GZ)PcR%gV(!5@`GPh-&G*d#|HcL%L^d;+MM|fsPVcx@`88Qe z#a6-WljD6}!lp~qilpWMeVABJvG7x`mQ_ObZh%oV#c{M5*Yzca?dGwHz-HR3IlV(H zF7np6JmEC_ke|4cmdK~hEofxl?MS+#Vj)xv;4VyR(_l2bbz=mj(kZb8021y(0-&%{ ziQ7wq`yi2cl>w%2bWP5n$}VFg$rAe?)%x@s$o=rVv2}sYxTJriLlC*WPWfWrh#k6J z4_TrqjxZGey5oSRnuboM-ZHuI#dbCIv;F4>2` zs44jer&qCp3gsCj7Sp;s^*OG0c3RJSu%Dia9U@bIVZizW09-cDzF+2cW(CudHyn~* z+rvw*_$m5*1A*v?Q(y;ABo5_;u2CyOVFXh_PZQ6%q~L%dEL$&jn&CNh|eS? zfdM~0_#R#Hd=KL)-R>E|At`T2wP=Q%hVXRimJ_9hh*ZrqCC6R*yZiUn&PKf%`sN&M4Tur2=oh#$5fY2kE^CHn> z^Uh`{J0BRSBN&fldx-PACM=_k#Hbu7h@wqWiE~D`lhI$apVCe%-oNdUa;4YgxCo7E zrv6-J8?iFTHAyc79NdfkoNschR0Dje>rHCaIG?s;%}bvC3@s>w!3;a)&IilcBB@NB zg+AVOV~C~>5~#IMd) z79U&n2s~|Cqd`D+Bry3%L=bDK@COUR|`{Va1gK|)1U&h{3G$OCT%4_hk zL4DII)|R#OYoMMpPJ7VkKZ}m7eb=si5F-DG*E8FzytXRf>$a|%gK#b+QFX)%6~($! z0RAAl%kMla$g4Z*_`$34(u|EJb%jAjiCYhiF&N|BtQ797B`qDW4;sgJWnw32F8tmFmw>M;l<=41#`E-)yKXp+{qC~NMdD6O#v1Jo& zUKIu%EKMu$jQoCj?cT&Cp%W3=7}R@_PiOeDeCT6>z6eFB+IqUJmaUVK?6c-)h~hwn z-&dLM(HbdFH>Db!`$4&u`af$e4pqHE}@;+DFNL{>KPUO?hgS z$@@_}@VAMk%!_VJZ?l)gQ|)D@A9tE3^$!-&^t5D?i*H#{9OOup$o*yP^&$R&jXhEZ zZ+@4wj$&RNq&DS&n&Atl1{a5>SR!U8if5edZ2{`F)?a=(_RbKtiID{!uSo3RK1weUFfGf};fo2XhopEYQU( zfU`+V#sRQqrCy+&Qe7{kix7mI>plXgfo#B$<@ChfPW7{(;U+LAgcg_jtktccuAVEq?wFKA|77AZ!WsCf7pR;jew34rB{^hlPJnl$?)V()V!p; z34 zN~JO^_6uW=<52hL?=zJpye@PF4xVdK&6uM-#%jK|L!9S6OtkB z@RF$9yMVcHNTUxCd_-%8xrsU*Z;A>T#n_c zYq-bGne~d!!dIgK{K$MyRwvFpXPv!Ic@~Qy);|il(|@pdh#g9iR>t7jMP8Y{avqF? zcQNqBeYcDbVNRMg)9@wK1U#sTog9Uj{aYwykt;skjHcORFnNhGzQ%BlV+l zLxe_i{5rp$v}0X2Q>d(G+osM`R_}Z>+w(mYJ#ILD$y7afC%p}NNhsdaYq<+}tcGKW zNo;>tKkj{Sqrrcv>Lx^IX#Y$i%65T|)$H}df-&cG1NUyM{vl*4!e9a;$Psk)e)DM5 zBsnzdG>3B=g&RA3D~;UgJ=B!OYQK=inKt&+zBa7{sx#EVE9gp>GMVj5pjrX`+2!9> zRZU_i%L7D|>Bya&oSuklTn%L{b61Ea5_#TMSHn%MuYUAv-$l#{k&~C@i-(>}-bDB! z3B&~FCZ}%$xr6i1sb3up$)8FzsOWT<+gn)7Sc{wq%8+0_k;bgvGlq4VgTuztiF9Pj zvM-yFqb2qYUS%FdzN9iA$@Ln@%gm1DR_NMTypsWV;G6uFTy+$C`26!7$R8Z5JaLwe$#q!VuITa2Ar%O>n z6*9ROwB3`TRs~&P#Xbtg*t%7#31IEI3CjwP&*m{DN*R-yp56=Um(1J*VH%LUDbj8o z%&qMqeIK+|9K{wEyL~W5nv>Ql|NO%B{MuAz=JSX&UD9|CBZU@rPdxiL$hyhFXqd@>CeLGeHR z)x)o45B{G&fg_qIjeq{4%`ob%|Ni;E`^z7O=KBo)@y21Z$8xpGQR7D*8}1(bA3ik* A*#H0l literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/kda/gb200/kda_b2_bw.png b/benchmark/linear_attention/results/kda/gb200/kda_b2_bw.png new file mode 100644 index 0000000000000000000000000000000000000000..3e0e81d8547b31d4b385d2d8ca7a4e396b1cd18f GIT binary patch literal 105633 zcmd?Rc{G;o`!{+aGN)N)6`@kfRH00fp)yp6kdz^044KN$MKnt^Wwq%YHZA0%oIhj?Ne9L zr6@)rilXyiWWaZ%rj{SUzjnH)8o1~=oOC&B?qo@6o4cGj?cj3S`nZ6brIWL@gZ*}K z2?=qDO#)UfE@zx~ZrNh@pPvwSa5}Zc=`-CVUWMt5x}h^gtzJg{ra2Ycpi9wE)IJqO zefPV4U(f2VXl`Q|Jax8*Q6x@X&tjlhP3L**@yo()77q0W#}i)h-A^*K=o5OiYPszS z?l{g2mQ^Z?SX5PLRz6^=_MRWK^m$UqQmA<-_T3I1NBgKCqP9;o%&8A2aA0A01LI^_gQX+@boaz_piS z<;p;&<-%pHttIKv%AY-@I9H_jP`66*(Ug5~8ja%+6b$ZQE>SYkPT1>8!48NqYL$)1NZwl^6|<9{pau zr8MB$wQKqc8v~WjAAfo2`Kwn9KdW!46?=}ESy#fz7yo~)nJ32bH`u_2v%0iZ|CLXJXhiZp%8Zm6Mlu{PB8ecGj~~@5-yIHSO}Xg#GjL zckuA?)<4*{TtC&=(01( z@bFv-2v`)c`RJ-pE-9rz=9QA2zH`%3o$q49Zr|p7a;(@W&2^=eM+U$1y3%)3#pfnE z21@5&b=Ti-D~t;$E8DY(o?dX3SQptGu!C5XCV*KHf3>v)bmT^R#u-lc$p3=-^uX!cw{Q9Ol{5R>=A=)& z=AhZb%DGkG!0<|8lNMQ@S$CQ}c$|fPMmKY>^6nC*LXxzjZM01D&cyF;rk9bCah&-5 z)wnVKpoo5zyw9w`Xm9g@f|HhW^<5uz#3HfcRwk%iT*lShlAub~ZTw+zAtBYgf&%7s z>()8WjN8RDA6fD9rB6-htU}khiQk*45Uimt&AC8xJcRN$Z}#<;JTiQiG&nT$CU@)< z-*$)CTX*Dq{}|geW95xJ)t(X@;c87YEiJd#DA_)0l=3uHSo@B;rM9;ANdDNTbIs!x z_VXn_UyeO*YcrL!|6*Xv%f>CeqB+&1y3wuECG1+`w~AoZXD4f&X1TK++7v>6j&uwd zA2T_dd-~JbORKk}9&Ua0YVYaRq8)8jr3WT9ldT_kew`2puI1-7td)3+%B$|6lI8#uwbCfnw}Vb>b&vhgqVXSi{N12r|ri1PV~37n7;b->{NZGrRJABM>=+P z_MMGG()q@z!}C~#eV?8!rrJ9?E`;s=!`a%}>OMXCaDNK})zj0%DQU&wHqgpOr5I(4 z8Rgou9X)#V#fum1_!Zf+zj+c8659Lvp6E7x_^<*s@A(l^??AlI_gLRQ^&Z8lhmQ5B ziE*~%It1%J4A2r*zP~rLF3-{8_MJOxw{GRA4CAkSdNj8#$L=VP>>0|Wz(usLQ+MgD zXxn&U?1##imsgvgJQ)-b5ix83=iG)|-!`Xn-g##S_>6K+U&O|z{0a+q*)%?)kJ1W? zmT~A9?RlT)^pp9^moI02R;|>3sQWUm|HDtdby89*zrPBt=&X&VDO%%tU(uf- zX*;{2j+ZQ0?yFC|z55)W+Ht(k5S4ai@Dlze15^7y*nO^NzZST1Za>LN#d%E)cQ-xd zKCx}hRh}IeP{F;1UaodWx!D`W7t{5#t#W1+`eFj@q1TE_$&whj(zjOwYjkN-QxmOU zivGjv?vsP@$9`^CRZ$5+nOAq{!){iTJ2$?m#jQ4KyBft2$}O`B6=gN`0yTBCt!xu3 zujqM~Q?GqyIMg)UZ^i6z-b-_S7F5KEAt?K~1AB z_Tt5hoC;ofj7&^AnuTutch0!Dte{kS8WLajHm8kv&GCqSo4ej(YkpWzP_VqbTq$Cc zp{~|Fxu|tQoC1Fz4yVNN07~G>l`8?s$;n33L*FX{GcqzLfgm>Sq^_po_-gqej`^9H z@cem6Z?&ZzwcX3USB9;2$~Tskl}${=lqde`2jmw4}IF*l@ky?y)EAlqgomPsJ0 z$U0QfHlLZz(x*ShN%-#F%aCLLm6^g-%WB*b4^<9kH-C3e-k`*5$EMAj*}iV_fAR9A z`Om6I%1=};Ddoydj#aCKixZ{Jji2URv*z2YP;Ob@KXPbB@`}?XUY#B~t8A z9a{YD=xFY&%~5u8o}(h-CItb%8Xw4yaWt9F)ig|S+!|vp1f*I>WswNwX{GG*eQ!TzJE7A zbt-soD3|qJ+oNqGqK_vh-GK@iCI;HnG&GovO-yu>_w#;8(OFk%_at2Ta)y~96M{)OwP<;KhWXh zsm_e|n*#;uMW-3(ubq?6&B!?a?Adl;1IE>>SI1PX&MC@Ub#j@K8MY{jc{x`0nfX6+ za(1?EPKErJuYdXDUEM$St?B7e%J1gpqX}42Hm+F?ZG69f|3>GOwQnu5KFIBwU->cD zfj`F%MHYxC!6hv9;{E&U@PR`WyrJK@OMjiq^93rOK?H!$~${^e$224-U^UIso(`r zQMZhW#m2Dn?o@a9s;a7LUGu%CN3^(pWca7%wVOAEu?x^{Hy_RE&h|E3vs1Tx0Sq{l z%Kzx?-Iy2{5Wt9Ko}F*}(<%SCvIbHg_-Q=-l7i0qDV14ZL`O ztG1(`f8)lD8K*yO3;mgygIn8@;4#uEW{GN@I?{DxU)!A9@Q*h+}_5=v)iZW zxofvltjxY-?dY2u9ImIC6dne2+IUzp?`rwY)id>#m1>fwKOXG$&?)utcI<819Jwv% z-pKnT?d6@FoknT*9ZFx`Y3tI|S+Q){TDulp@<`F2B0kaQb+-9}l_rOQ{8nY#H0o|V zWo<2b_^DB3KleEK9x78~-m!dRaXC3TmD8tBUyVIrUj5iMo@2g{&#k*ok#VT>`}$Hf zjn}n4UProjEj#GuRG@I8#IY?&oBOHjiPo;Oy-nH!W4)eX*YazzTpN2oevrJbcx*Q2 zy=Czy@6@P?dGDF=Mx#nU8oEW4PQKGmxh=6h-gyp!=uKj*JfegkyxwEm^yK-Ai^~NA z_5dhmV57@{oI7`plwuLX%=Jo2N@oV!6;<}`?YMD3LGj>04uS_Ka(=AexswaT zOc@31htrOoZhawuqU*%P*%vS2dx0J|^yySAx+YCU@RD~o#(RHtyu7TwZ{M{}?YE(T z^XtUKn5jB^WaGxjTR_wdRQKoXjaNcLm#$n%4<;t(F}w&{SB=1@bn^m@YA1n3YCHE0v5WA2KczP{LT6}!`;OOWJw4}NMS6ky!ACu`Tqw=x3 zy1MP(zMUWGdPhe+(tk*M_Uze@A3qAn$??$8(ByedZ7Z*+kh*Gg!;~!XHETrY?7O?w z=!ImOos_Q@YDM`jiB<4swy>~p8EEB=+<0W~i4*JDd1U=AF5h4yA8T6d8HlaE4pnE> zs#WA!f)fW{xx$L>!ici|vgdvB($%XMA3fUQHT7j!Y3hp`r}pt4@*i)WLFlH-^bJ|U4@{@t-Ftap_uPY35rg=& zS2!hTH4>NMqgA4K97@_@0Q8GBv)I?y_X@WRJ4Ay+qNQLTE&ys$0_eAsSzO(pYgT6zJ3z-%z4c0WBAJYc}m}PUDw$XuVS6J!8XjVG;m|JT3AgU}^iRy}iBb zv1(M*)V_lml%Zmbwcfm|bypV4ZfSaYdKrk)x8J|T6Xzzshjq6W?|OT8hd3Uqd zLdWb%Q{Uc`XKAlhHnr39KM{HDTKw=1MnT{kf6%0FDBB(L`mO@u_c6Q90y4>@yAK_o9dB`tb1q>Qa9hFjR;|L?Q-j@d#rUBx`-0=7dG*LJCHyIX{53HB9CRIua^7dC~4&Hw1 zEdi<7c1|E};CRw`Yz1E|fPl4(+OPtpI`?U5t$<|EIO#n0>riCT)ZAI|Kny=#1YB+<(-IR{H4x48Ge30BWO@mrb>Wmt(zfT^(gU*B>yG&B?x6=gytwXu%U zqgc76wKm%U7!$sLvYBpGzm{{&%BL+QxxuU7MT}|Jrlg2Oh?^Kq&*R2Y)ZFyg`{~05 zfaeEuP8-U3O>&?{Hp;S<0X%+#j)?yg+_A&Z(^gDgo|nq><{o_ZPNCYo+Uc4+wsI`ad=H`jKk)a_)5L?iMt!8CZ zs4G6v3OrafGqX4&qt3Q2C@bI5k9TT2#9b!qs>6trmp2sRfpYvovx*?LgCZ*!a!pP1 zzlIfvXmp*m@A32o%rA=&)k{DAl9?*QO1=WIT1os?MOzy`x-zO!8CXV!O`}-xM85b0 zr(x-?6TF{uc*gcMCMADk;%YDe?iODA@L2 z@@SxA3n~YtME1T_`?VNo{1MPRLDZ31|hR; zGRk749{63zou~JMLLONFs2d91pi6eq5bjcT(wd!>_DKm@ZT4Ra=qb|g(10ncKXS2h z9<&qyZ{VwZ*X(}HcHf!py(Kjp0q`ga3hu|27Tek*;0>rgYE|<&vo^k+jt^{ui;Tsa zcLtn0b7t!uX}e2cO~=dpXk=Zw!&_Z{WIu@+M>B#xaAcP>`C|wUW?=P$!^2AM?y_LH zQcbrE$AJPpvw>fEjVA2u@je#i0Zg8^s=*c@M3JTjMlN0XZQ8WN>)}M(-MBab%7IUE z4OiTCAUO!HEPWqq-(oQnS^q_4CDFnrd(%nR(#Wp{_(!|P2 z#7|~Mys)fhr|T~(uiIY}HCIDbqdz;)stQ4^KFeCq-QAt2lanY^hkh>})gi6Sm}P_3 zZNI!cS*@6zhrTPf&qlP&rCd&M^69#XA_dNGh7Uq|7P{0^x7}y^K1x8jTMR@H2o-+S zCWE*mM_uwOUwiVDc#dg&*B1eZ6r)-9iGGh|)#aJUCoXfwwY9a0ZKkC@PqRy&G-v!> zK|!qxD9-!!Uq|VLADZPJ@mt?d_Ti61NCq|WqzmU^nl6t%=4t2i*N7VP27?4)CHRd<3$q!W%n;Q}oa;#E2}F7sUffx* zuONDsRqX)kuOI+K_tOpL_Be%IlM3krLKQih^Tddcg@q-O^6#UKbBQ0AyQC0n+f-`e z=;&zF>-A?gfAm~a0Vyi~b4kRZ3X>}L>%&7?$gWt`0ay|?ylW4cqK-8TXy1{%=5s(p z<2s*!3bYYkH>vlqoi3xCq-X^WroNeuVqddu5%j?73@vzLiEUkQnzFJa36TU~Z~3S(;&v0eU~+P@?A^O1xFLC-V`4<>Mx`o;9+Ku< z6c?8G5o*qw!nu>%jyqRmiu$B-Mc3)FJ})<3gVmM%>C>Jy4;39Z*2II=3pr0 zJtL)gZoCRUk?w+31hcN-p5`lc$Ob`E#FH zIrtO}_3G8DGiT4s{mG3^Z)=%dA`p=zh#m-&(Q(K~9iat6Nlq5dUECK(Zb>D9K`-$PR2TRv6 zYuLBty}hfk4lEik#1$;W;D>rn=f2&`MX~C0Nm00~&Go)cS#fo^l@}l`;KOHizA2dd z%qI<5!`jNjd*VGF^H`PX$E^O!8rul;%~y*n_KY`+n7mD{S*#3z?mj!2@mx>#2X1RP z4_$VqUdC}%e}1FZ5^rukK0as{H~Fqu${aYSb0gZ;!~SYvjVaK|6GcF+6y`SG;q zWS-dPHQkdu38Dc%gti2Tkdck;B_s(~E?NaWa~{q7{8cB9Ex*dY`(A&_7L7#R&#C)v z2>&!ZsC4YNnjQ1V?Ay)CsY{~n-|bOu+XjT`HCnHhqL<80OG}&XF>;dVGxSbr$<=e0 z1Y>P8j=wyZsId}O>o<5;MC^R!6{YwUy0h+@Z|ib@KC7x-==}A;SN^60@(Gawkk`5+ z$poAPD#_d4(-ZYr=2>%^X&9VP%6)rQroX>`#LXS_dmz_4-@SiC_=Y*w|HmSakzKqI z6a~`;NQOHdXc|OYDb1r~dslFy^p7bJR%j}b8~Q&8ZyAq<@Z)#4>vXe_Db@n9n!vwE z0f*RCZ@GQ=N@`1jHt?k2t!nt?=K;Jdrbl~hJQD8TPYgE1?GBP@qM{tuw*nKRalRhg zItZF`(`O5uu+(2w8@;DT&*L%kDR_l|j1tOfGK_R^FAS!E^lI4R^87FN{MumWmNf~OAw-v0XXd-gD%cBDt`m}NIzH8fO+G<3KD z=QhwS^ZQ~_?d<%;@|Zu`Vkj8zao}(p544C77~L@~Hy)z!a>||&fH@x9&s}?G#jA{n zXj}9B_lxh!pIZes3fy;F{+u-Rqo;f8R6hUql}%GIM}v;>11oO$8cTji{@nP7+wWl5 zPtV;CQ)qZ>SI}xvb5EW-)@NFjTqGDOSLvVW$H#glRQ6A7%+LQl-d#_pd2`cqa@&_L zzc2kr2j(Y)BC5TXk=w&Wp;y{VX9w;@xPhPXw0OZfeyEKl^WIT2P+eU;yb2syqss~H z_(l~X1dks~O;2lBzj^b9t9rwR4VoUav$OgQkn{rb^YbZz>4|}~@u`!S!opTVQCRKcfHtHg~(D8@jg zc&T^5bgHnJq4^fMI}ZJ7+*sUOcc?Ren^~D3&xo*)(8X<*Rg4r--`6TA925Hb?%jdg zQBh1(0#>F`uVJCvQ%V3t<96@(>YHDFt3eCxx!acSoUD;QI_ff1E7PXlb0;;e;NAgK z?=d`;A5Qs`{kd(DoyA^LPUmB{1PTfW>;WEyE&xcmR$P4lz|PLMF&oL>fFwz04ZYvC zpj&Fr_&`dr=?OYk?R@Ub7-%)c0l}mLm0HXin=AqTjP56&x#@=wW~Qev5e?_`DP7&` zR){W80>E%jT+dy)R8+9a=DNpS*vKcNx5!i;w;%srs1jT1kYB8DqLp}27uTs>O-baJ z_tY;?v#L)BJm$(0KLhVkiIq>z;Ir@C%*1NkMy?UjNR15Nc^}aM6bt=^kq#fB+*n(8 z_qMQ~TR^)-4y-@Nb|I;lfr2recqRJNZ6&<81R@+dP7azAcL~l#00(Ut z|L(mY3&`*IVWF(wKqwtW5k|Ch-CknZL!R7L;L=0O&CUI#$Yc4vd-uSKA-R=){AsR|w*lS}WhAfmLpluJ^B(7FHR__?Ruk5O+ zm>oPrQ1|pd>peTE4I-hI_yo{4=k%uwu)x2~PL05NU#2kiV>y=RI;0NZ?gt@_fH)44 z68E#&K&I#@L{zrC%Xys6S-}7G%a?^ z$oA+V|Rx+et$)*VqlHn>})9%DiVdl&3N9@Qdssb>cS{2HeCBUY3WrU zdbqh|msW1f>U3zy!GBxD=g932dM^_Y-#q+<@9f6IPs;EVh)@cw5{G(3o*cQm_(#SL)#5=)KTcfW5v$9w3GTiB+Kg&{Bw}B#P3)0?%;zsZpG(9eeHyBpHI7p`U-u zciuo0VmPw#cvFN@0^uXW0D$g+L;UyzsA4!x@!JA?SV`f0~_ z(enJEvp@slShkFonuKdF`G>9AZc1z5{a+Q}yx5O8M%==Lvq9qPco~N}`=e@dJGf=+ zHytZ-kJnX5js-`!gw#$5(08Qg`}gkyAu1qe0R7Opu&X9&U9tAP@bEajs3+@$x`&;y zO9WN^T#X2SgSrO-q;Go^=9>z5U86#&;j@!WsG&F8kSyZa>7owBUC4B=#_QfWX~>tM zE#AXF_C=zyYaTeT8WuGNIC%4P6uerK{>`6@+?|bXNuQL_y}+Rq5Eyt7M9At%WqrL4 z;rTRZSi{HR1N;)~#D2 zrX@wMoM0Arpl2FYE1VnG;^gLL!RH(6uRW`HtgjM|lDfli$=rMF6vv?s)u+uz1YLG_ z;rd^Kj0K`%y_yBUT!y78X4#s1Z5dF)*fjS<#gg4AjYHF8wxZshHBp|qf?L!yILZfs z62l6CB`zVD_9#+HMrJh@UwG;_2UBI_ex7C+cOLkZY55X7=yBx+qe+O}ANJn5w#@0= zCT85y`tAvFG1UA*$A*4?M>okcNuFMC1J z9AT4I4Z9qgjpTf{!d5B_o}MybKW(97UTI{Yy%9D7ck{%C^`n#J;+=* zZ5qL?z;ZH3o<2Vx8o{x8^@idhDa)!Aka#scu)ztG1=r7$+NR5&wGj&BCq-iZoLFfW0$BK6%lVo=M_yyoH zVm$##s=*8&vpMTFY*9+TcyUE@^J@4wBuzxs-e>8)v&4P-@k-zMxnP(eL{D^cb3+*XJBNcF@wC8zQvy+8{hrdtHg66b`~(i5zXD_#-CIbH5h{2m%n|>4hF9Z zCqUrF8_7=4MHCclbI&8{aU@qYigO0 zM#P?&ovw9H47b1jIedEet%TQgw(-(Yl^6fUJfBJ&un&8wmH#PB^;*1d+i=Hp_#rbh z0f23h&{q9N`l3h`a!T1YeemG@{KF+|A(=(eCvrZsoLfvw0umF2NZg2ljqAh=uVAik z*YLEnHo_#Z&N#TZm<1P!>8JRil9V9^l|0S1dA{VYL%EXSuS1EP;SF@``=Gg2VBNs6 zBtl#TvZ(`s2k!J*0Rc+N=7UI|@Y0^se`co;7vh6=b<06{czMr-%8ua66P)yy|03`F zk>7za%K^UO{;b>S((OmuCagJGryuI(GRf+Ox%Rx+4qH7(t=#(F@9IK}-MW2y1w46V z4oA;_&zSs1GRPhOWRSaM#y2gz*wP+Lv_lBPi=orwM^!I@ZUGNoM)qxmn_kFh(?h)t z#Og$vBk=2r>9Qe%+4Ld4`|X1zfn?{OmG!3k`$GTu=TDaXZU27>mH!`*i~rwp3IB%i z`7c5JA`uB_O2A}W`Z@{0#8-GL~T0 z4JJVqbD0bHPi8f!_HMtwJk)Ci&WlIOp8Aq7ag>DD07+}^NNN6({bw#h%nIZU`4XL@ z;y2!y+1s;&YZ8GFiigZp=j0A*3wX5xqG2ou;e57 zrt;bG!6G1_B?fqBkoGjaRuJC534eT`Zu$1_-|y@B?twm$VP3(23YtL~k>4BG__s(< zW(f3rV&L5>Kr%n%SLPB9ifl!8v1eY1kvP>zi#$ho125zcW1|C_Hzdd{FK#Ydy0pIV zuNt$cy>^rpm26Yr$$xI24vhq)dk8I(Np>I8Y0w);whpNj2Ctd^T>2JSz&1~Zzn_zJ z6`;$OEn7(LLy0kB;}LcM8@S%@Z?}O$EN18N16)ngin@Soh;iN-nm>R3goK8wg0>$- zLm-g~^pF>j#2SA=otFpxCH~KK*}SX z3Zx2pDpHSKqAhX)NK7#T+b|6!vM+f#SC{keaAzVj*g zcl-#8+bA*tSKgNM*mnKK4H^oG2;JTXWHzFxxcE@#t1B3L2^Um8^Xr2s5lZY1K&c(X zm|sRUgjE;6EhJfK&| zI`fll4#PP+cnTX*cU}?QsPV!w5Qy_>CDY8nE#26E_fF41uXjYK>k9vFJ}}yHgc*Qj z1QPMwk1}hJSPp4gz)RWhP6EO1^cKe^Q>F1=hL5i9fHcNSMCE9TrZW4!eWO7V9S>w3 z+VAc^lk1=@4Fcv{ZJG!7bLp<5fuch)CC_nno1u(6X1=|HS4sRvH0 z#f`&hm-)%WP==-LSh3|0vZ3h~vj-6T zpS#w{W;;c4|3EQH5Fx-cg8)j)8XH%F7QhbTm>%n`oRax_`@3T@fOQc>XC)!DQ`w^= zoC92#qH~`CvIILU%@Ze1kO3431|(r`lx;%~%!oqyFhYC({vgEg$;=Ri)z9@g%bL#C z))rid$ON~KNC|of z)mo$|0rh{AK{^HNo+toYF$0r5eXQ7%9{E$_BKM`p7a!D2!~Mbakk20(7f{0r?Z1Ctg%f2p>GIE}VO9fxM$z6dXUcdU^G*wM9^xh! zK&^4_h{KzmiT8SmfT0d#n(VB>!80zuzpO$0p&U3p9o#+=U@rMPz#hb3qUUV|=6mQM z_#XbCKJB6`CRSDzh%mT;G_W}dML^lO)v(>v2r~kTe{JbRN$ok3u-4Ud_*#&yG2`^C z-*5zi>Nh+sB8BZ^VQ0TNu8E9vhGq4t+qZA4_n>7}AaUh1dm3Bl5-ZA;3jY&0F)Y-M zKc_#YD~jpCJEz$LT}jurcp+h=0?FXeO@~jJs*jxUpGMUOZf{kjI8f9N8n$AJh-MEW z?HUmNnDBh!^~DlL7X}{)#`pAQCmG-dOhqjC9qmM=q0bNSHfUptB zp@e3&KSD$YQwvbZS1f4E`Iu;;j^kn*GVzv7QCh-0yu2?#GZs-2Sig}@=&J}gfp?GE zBUlz17Yjbj_Fl)SnyT1ojKb7aFS zD(D;$0M_&!TbgRU!WvhLecRsK%Yq&H;?=81weq4j-qhtfm_Gga_3PIR&oLY8+ba3S zSCP&zE^uK&+Scl8ffPp5h=Gd`WgM0{^+%@fc+*4fOkJ@^qHm$I1)?`SNJ`QevABcj z9SMo0Bn5zH(O7k|xf3;yjzY&vcKwP_PX$0fk)WTpXyei1OFw)CGd;%Ro@Q>^v11MV z?B_6hC6MccNV3>>ZfrH`Q&PrNxG$vVqe77mhs8~)>+L4k0W}TLclyE3UC`;$Xea;< z>AZdtt%YZK^~aABsM!oKLhEIR1qYL2&A_dZo4CH$WfYl54qDib*Gj5S#dLhbsz8V; zM_*w}j{M_Qn~kh^nbtmwjfb_5bVuFoQY%G6K zQPDwY6L_YCoKuo#R}rNX_pTCT?%CH@BqL{h&m>w|S(%1HVzKTL3iK&++t&E3i=HXVQ|Sds>UaX1Vd_Xm~dhGRHq^icy(=qp_0;t(ZcNP>?SY6_n9}} zJm_IN{{+KUWJ&}1V_sB8C^zRNnJ*L;7LE?UcKn98Ieh63>^w5O0>=<3XXVIg;O&bv zbPVdQh57mMsBR*MpKxL!QD%rZ6PyUg<(0H$b7zfR&F}5?LNR~0a7qdpRl{lvMQyV{ z-60cWVuv2BPh7$!$w*ck6$dZOs^I|>NI09bv-9f{+mP2a0}`X+hD~-sVW2PwYjLIp zNphoz5a?httfBO#2VbJew_k141id-B7(B=8G2r4_fD+PCkW9Zs$im+v>VsI zOD6^aIpi?i)>>Vc+HAYVJInh0US^Hk3f=`(+~~CkeZj&Y9Rp^Qt|zor^Ais|FOQ%V z(NLH~4ct9HyByhHo8Fd8OBNUjY&N!?(l*Rxuf+=AUHSKF-*AokLa`JTA@fKL%2TAB zAV_Oti3vz?`P=WrM$8Lxt}(kxoN3>K3TX}`$hqwVBc>`OH~R})Rz(oa%yIsYH)%Fd z(HNPTm5{|BtXJcwaeu#r6b$?cNn+{lu4qwY%KzkMVrXR40r2!`e0zEr*hQ<*oec(L zZ8nf8xrG+$^}I>g_Kvv2Caddfyw3dovIC{41DFB~<5rUTRv8&J%u-x~VuYn_>+Qd= zGkxb3C^N`QTy)DY^|N%%8ta3*&d=Lf?AhbjB!`hV;P$J}d;deTqV!~G_c%Faj?SPf zAs2Z;@x8JKWmHOcYJcEhcl|nubKGBY0Mm&EP9kipO}&VTdn$VOvsgfBO}hwyPSidm z_JL#ZeixBSpnlDl*eN*}ha6wCAkOmUMGEia>twneA34Xj1`7@nr}T z557OJTk)ZO0-l5J!9|0mSITQ-WEjlgX7m+-@p!-lVC8lhCrv#i+Mx>WR zJ&33dkiVvgo!m;Y(cLf}(K7*RpheK04QTYLkPeq3=ZB|8N8OEyIh6E-a?0&5s5;$@ zRZj8&5(oAxa-vwPGW~_NTW$zz(^7Zg`+^@F@vv25JWoP7fEJV=GC|x$e1Lh9R&^A$ z5FmOKLvJ`g-{Me;YcIz{IShj%Z@&NdvA$Ms3K6VFu3YY(p2y)3+`FWUMXXc6Et>u!9^I)3`H@uskyZ#Vi5W#={bJmJ`8qq z9D9D}m>dBi4}z?Rp*2GO5x_MP@cEZk|ieGALh(^QRgy)DBpx-UDKL!8n!6`5|~X zW@8XkUgSGFv>1?p0RtHDW53~fLOLL@9?zEy_4&6HIDX%Yu{CB8KLW@-$NQL|^P=~3 zfTJN3FJe+43CrjB;Ld*`jP74tn4Gx@vykYvB!qzJ5H^sLLyr$X|MJDS3Y(7`MS_m7 z4p2#=b$>`T;l@B2rbSc`{Hux(y}v3e)*5LTav>x~fb5k8N`=+?BsK{4OZoWNamim0 zJV8|GjGNnA$6e^Ci;%^{GBJaFhR=3;tN8^DKFl)eKQ&s3cBQVRMZEHF-Q6kIE75C+ zlKbJ^;JDQlkBt6;6x~E-SQGv`oIBe=n*z`>iCKvXO``fR_WdtkX2Hf_J%&Z1qj>xv zEfz|T9eDC2V*SLc0(?oSv9N;Dj&l!|!DUxYZ6=pW?sQr7(j^#mNw{v0KPH3D0Mik} zwwY$nXuI?lQ0* z28#HTBtdbr3#^v-5tLbXUA)MV)Vod7h(ORGP=FDA9-^!RG?HT;kf(ogte8teLV|dY z(CV$uM1vGNVqzW_zZkFyirEFcG4UZSD%k@90?LrYu;}_vg&;d6EG+DmBz_GjEFQp> zjDnrm7S-grZQC|Xu@ROE+r0uIJjj442jgiW!la|4QDwOIL}Ute#h5NET5d8i^N-<> zYj1+}dT8?il!2uyRs`U-+($@?tG9ag_vRaKNSTPN=OBIo!bvm~dCJoC>#2vnWk~F_ zd5vh_b0`#V2|omH8`*~-GKuj1Fo|KKZv8cpxTKi(@87?p33{Er!G9JIfyxVCQ%d)o z$uRkUea-z()K*|p=Xe|Z+2sVQv zy;4k5Qxmy_it_S=^k~v#upmw#YaoaXh+>RemY`HwLPzdfLPFSknH7RqLx`>b{#x## zUFwJXgs(jh^;f}BS@^oNC{Y&z0VBhDj=A0cy!%RT9?^Kli-63K9DtzVx{>H|{$0m) zqvjfb+;H!8_4J02{=|4jL3%WKQGAl_e*iG{m+Y=x6(2w5;g9sKKAv4LVMguE4k|6Z z9v)6nNJaUDhH4Au2Bn;VahL#o9D~AYi0SWI_4i{9mX`RVv^@XtAuIf-vVPuwUZdT1 zQZ;}>=}zYJhK5Y?N0KMp(i4-Dl^JzjKP7oD%&i(LF_Q2MMYk;E5C07N-|JLh5L&U+ zuvF5qiXlN}1#{r&u7BRETlv=5Ii02fO%?ua|G5EP)BkYMzn6x8{y&{YG&)ZVMD)r9 zbzaX!`eY3!7y_nA9{l@=R7P8Sp=F`*1OFZRs9kS}rY3mf%^(0S*7L&Z^p3;6rvC-; z8hF@dH(-270tjoR|2`w&HTC1i*CKkk6_ORA4G|81{x$RP0Rj`y@i~;#cOx4u z>)5df4e$bJHd1?q>CtEd3txd&sL;Bl=h421ln*Qbv?74040N<}<}##7;LD{fZjsnB z%p9I~>;G&HAV3Z7udV(28vN{nD)%~@a}^SnL>h|+BU^*U7z8_$_U{JNF@MuLpc?dH zpZ-DjqAoTDp@Y}-f!reJH%4mk>~cefU_uSm9&X81Kt6K70A}xA=cMBMr1FX!db9#~ zNC)Xh3?HwMacKSUaI+F)&D|Z#5fH`%u&RqoR)iadad1oxg)H+{35oMhpKc~@COp!I zjm|i80nd<(86c=h1{u*-X|b%yc{0QK912DjkA-OsnM1aX6qb`@V4Ye)ai^%{g zIwTOlVscOsCff;MBPb4X%hNIFgJuL&;WQbH60;-L{^8Lf7N`xnJ#??%f*jQ&N{^3z zg=dHE&zU)lOoS3+xV4oJN)%AnD=e_3oSZ3#-$BxYOHY~`$p(N(fS8a|m9Qp*(cNu~ ziRX}XAXeONLXF=$0MiiyqN4kf$H{8>xgPlj#4$*~0N2J0#TN_Fw724r$p}W{SK&+{ zKNttdDV;|kVa1D14$F-g{L;K7PR_3|!`;9QGY4wUuS(MwBjPEvVE|2{%&Y0^>-#up zTr=1I$Rg9nLr5H>fS_*O0IE{kbTp5zFI&s46b-197 zBC$OXQVxWL559EUt`GwpTdnE@i0DCvSjjOS@Oh{*tUCg&QE5fz!CMHLxGU>Oi?&;i zl=g>TUP!4yaSH-&vi^kSm2u{q68ipephU4LwW~Z2Gr3>vYy&tV`vn~Z`t9|bcrMI9Nn$Mvvr#xV15U&qPfyRBev(vgAUd)m$Q9A7P*h%m zbk{#UD)aI4XN`4+J<;>GOnsgMbzi`b-3MfZXvRRy3W)bBM?053l=YsLJXYdG@Gsd1 zNZ}FUftmeFQ0do1_;$RwNMIxR6^Kgh*s5f19lj#7=LmCf9IYNCY$9WaS|gB-6@Ce^ zW+=TZd6x92Ao9@qHN}EFU4~(p5;-zHPQuD7=Ksv3kwbSdYQGe*5Au-7$x}+gH!fls zr$kI4{6^oC2LOwd-<3#EOSnC)+Sjj0lJ$#4Dl}ldC2=Ce0oab+wcy=YUWua1s~JG}6+#;*QY= z0|b>J1K!p*g}0VEuBkH?e|NQteii`3Wo&8Wo8B9DEn#9>3JP|3>f+_g)xE;Ye_nj2 zFr(%5XR3?gC#HMI81``xbtT65Vb-7&5u1J#8Vx4>@g*t>%AB! zW?YJZaYBS?M8{x9#bgyh+JSDAt#J(2J~)lMsNf; zx0+Ez8iIoKI3hbeJ$!BAHrPo_&{2tz2fP*ln1(qNE;HN7K~CFl7fMzT061zN4US#W z-4*OR-LnChf*gr}w0FaBn8Gw8$|+jI3Sb%vizvHyAG}I(xCY{mbQIxyh!hZPfLFn> zivDQl;B*V2yfW~zda(M3S8-+pI9FeV$H=9S6b>>3zh;|9hw}nkhJ;0Sa^2e zaP&#_=2_v^?i&jS23?kc$_u-83ooTuBiFHG7tv+=0$G`dNAbY;L;?DU-OX_FrVl)cAMrxfWD_ zFm#U&Z02TJDdqh=D*Tx+Hy(}h5ot6FMR@odP@P|KVWFXsRmAc~hdIQg{u=tDK(;5+Vgn$Vu!G3~SVX}7dvKH#LsicW zU<wT3;Hi@LMEi6QLo5kTC7y#f#zhx5mu^7gK(i7HEfKg6XLsfH;yB&+bxI2^59P zMGk+!8m=ooch-Nn^5x5KQzKn@2vedLEX9OfUS8f3F~8f2O#17Qt4shLAn|Cp4FNz? z?GWg6r=b=T_kI>T9N`pYoMmRj3rDhH;9czQ#S~tk`|C}Np#zLDA+lFDouqrR-4AI+ zgsP!OV9h_cdR^$ z^gQOi$eAxdj?0j#$ZqXXc~I5VvUiX603iB^<2iX169m5X#}RFb5O?Yq*Z z**eElE8?i*nA=TqdB53HyyEOh1)k#=H z32fV@J0R^f@%1dH^l46b0`&k*4{xk}3{pZzq2uVces-TZh1j@fn(2GbEITzc@6=xe z9#yfRqVD{889U4e*fgtCQ$s`doSCw|KK5liIiv_U=rC$B{7S3-}?C|-e!V5 zQF?VSR7j%C8>~CIml=9v1n~?4(r`vw&(lCV#2R}AFo2np#fUCsN0%TdK}R99|8zk3 zS0ZMVsMZ?2sT>TaT5~ax=6K=e&6`1J0rYUh&SMT89fWX$r&C~wI!IcP#*Un~LAu#_ zqkDLs&vA+s8U)Gm2C?%#axSCO@i9s_qapGzHf{%D5hw*1n-+EDTfAsd8Q2h7CjCMz z2m9(d1jwz{4eLPw1#tZ7uP3iXV?r_!S6@dG$Yt}>>=!OvpeT|r!ma=I>zA1J9MTgZ z=wc5WM*v*uD0V)%5LC^c#vpLNa&&U?vVaK^IQIeS*}fihJ2fl{NcE}PmYq1Z5fFs% zcl-{CW7+heP8_set)qy8Vb@@F6JG4cAT%xmaH){|gc zWV{0znx^*I-E*VMQ;qX4pwL0_qQ#%%^hXu^tojDja6(K-)Cx8J`HL5e$cn$O(3ob* zPlfNbh@F0=l$;wAd!4^Dsb)*DMk(G=yl0!?8I)MkH zfx{29{bBq8&y!FGNmQV&SE9J!2*eP=3BiYgV`EKI6>imRF+^7B6&@BjHw`ijjDQ3p zEC;)<#}X7{MCiQ?l%F(25)uG}Gsro;1`(;J5~VQqSDd~x#Ub(-IG5CPFbZ;o!0-gp z!{Jz8DibK82EF4Y&X6R&!iOh@%gJ~+ECrJPpfIZzOkPH|*#7HIf<_=SD1m4s#@4*O zbhqK6`ZE$E!LI54^lSrWo@t^Scr5qFhwm6Yhc_Zm6Yb(YGBtU=^Cd>!z(a~)3*lIl z*jyHPu;5yPNG_2xd}nczTEpwmI|hb^-w+2OuSoKEKyI4rXx^OIX6-WwYu^lokF;yN zJnU`9xoOu~tg(h_#-QufKPNjbZ^02<-s5Ws{Kb)54_3HKAbTIx|yA8%~tDxv#{Y}Q;uu9sBe;MYS{7N}tIW{pDgBqoMD4N;;n*$@;MNRO%2@b4T`DpUO+=omS8 z1?*0TUeq%ggPG=tN;EI#K&7r(GM<7?MrOvcfCfnz1TVzm)xsS5|4w4}T-}XZd_+_R z%P@2bX)sPHn+Qq(7Uomky$|&-;g(sMsA1UQ(C|ENFs+{fW@M{|F>r$^P(jGL0i~s- zSzbVoIP+Dur5|ESn#_yr}# zLih)y-0)?uvEIpF<{-^h%}L9?qJa1X*(am`DeKWl0}<#xgAs(pVd(!o(FJ9{8;zV7 zS#cHE{uY7@UVwXFfO70jV$MiKPg_|n#yTtpz-lsbO4hv@i5{w>cH>1U;lRa4)MeNx zlgKA(80Ku%h7F3xtFRN+$os9oD{}*GzC`8J&>S((Al5hg{A51-(jF*q>wo?vB*b^U z_5$pl*GQGXt&bgY7DD7fZQnPd`WoWTmvAfa)h}=-r_n;@eZGRbAUZu$(LHj-CBlcz zPLC304PXi6n@pP86^Z*KhOd5KyFV5UmlC;d>I5N(2H}7LZnVi)jTLFd$d7 zlQ;-jHVLTl@9vLn;c0C5Vg`cym&ZZmP?H{Q zBZUvi1>{ZeEA^WplD8&ZnB?^cHyf>zn&(sCP^W(!xviUwc)+Nh50Q!dm+)S64y_NuAhv>C6GnD15VHc!x^1(*{% zQ1UlWzJ~X}*C2m+4ilO;h{sdGfu^T;9`H3~sHfSLG~3RiF49`ike}}w5O`y;cFG@# z&ttx!lPj(}Tt#WYt`043ltBb|cua$1e*}pE+VAYwk%o*Id!5o3Qh8rokW`3n4UT3Y zk#zb1fers@3cdrKJ8*WDBQ+r4e6xh>&dr>dvp;L-d}eFHTF#DfVcDuxS@9q%aEN!o z14FDvZQ_rh8ifP`F5Aw9g2GyWzwXBWPhf!Z0FEn;hDDy3B><%`lqRI~GO9}Aa~l0~ zlIqDJSfMBkJ%2h~M-mC>ONncIyp5SP&Z9gYdjoTJW-#-X|C zCaeZjh2QI@ezbReW#4-9RaFjG?Z`i`X{ltbm5SwmUK5l}7brd_TLbKfYz@U@`VPOR zxNPIpL4_n(AFl#SwAuu^HiV7rcU=xE{VL~cWjINZG!L4i zaoD201mQ>|w&fY19r_pE;(4a+Pnd!f-mUPxahvYP_d|}ayD$@4$ z(+8pD1#D*OIPk#2X_ej?G5e9zazi#;9{m+l_SNvL3LDO4X)_U~caB;fNN%_T-Ssp4 z5)~v;J0t&L+3M5ytuu1=lnZ?}U`cE65SpB2ZDMP{J+B8Af@LKI4zTTRNf{MuvA`@R zv2ZEehDz;`zrV2U?=Q5$A3!Be&9ljUHvPF`nftZPDwn67n9r_>|0oyax_;UGb3^DN zb{-*h7$%i)?32VIy+ABPkbh9uKaA$O|41c#KTwq|KzIuNA)3}XjD!&GXjhCnIqs%m zBq8+q^DHKIAw~u}dm!7v7S1THUUXbLxpvz4VJa|yvnO(CJnKs?vTt}Z8#*u!1`SX_ zVE?d@sjN%|V30k5H}n=ZUqh=jtbtDyaN%ADH_33pFH=9(b53~MRMEZR3(QC2J@`?4 z`SO2v|H24+xdh1uVmNL}+iFy%4!LotjGfBkMT) zJ;~M2YEYYFY}5H%T(_vo2SO!is@)a(PE_+EcAT=puxPAL`GgZ@LTu6#`tUTP1N0JQ zd{7MrJv7GyVa~(;D!SKNEk|d;d zA$ZJX64r6G?WO*^LTQmnql8fL<=e|q+aUtOC>U4pwJ0kh@)*D^`MLoZ9cs#Z|Ck$7s5Aiggc?%q;l9Bwr=w{_N}3zKPUpA55%(6Bcp&5eX7(t0&D*D`}go9 z#HkYyNMAs?3sufAKxAx+R0_I^=aVb!QX{b~hzFFl%|dolnURU+HWydpkl^d=%c$EO z)1l}{N>I9d-A*UE{j~LZ_kCY)&2Bk_-Et{7IQY#GSE^3*w&?0D4GZ zUly;c*(8b#Wg=>zo>;XmVLZpA4yv2T??&`NjUd5w`WFx)FJ}-$IVGC0Y4~xX;WBde zDGSg{iEwie-Pe)GHe3_VUh8SKV198xdR5X*Yw|n<3uYlad_hnH!VAQZEgBV{K267kj~*c{A=HF!J7;vD1c^;{uhgFUmm(Wg z7rkGaZ5gqnx8tR!m)FtG<#R#F474SG=!#PWRPh>1-3esZ0G{pBlQY!8KmObw1mYn8 zRmuIiY@6#-d#a9V%+UNcsOAe&fiK&*gDJWNPWu|^#T(e>uw5wpz6#fjc#)dOe9Cdo z+1ZPxSqQ9l9zXRJPXG@~4gUo>yUC(-?5~4qJ$Jv4lZH7Ea6sHEGF*xXe}Z(9U&%8W zE2t71H{Y}If&Ga1Y>!(jWILR`)TF%HBAZZ_F^=Ab~1W%JzAr&DsDs{~mYc`YtD zs8+NzxbH`DQkrIn`qPWe39ZFBi?j93+zq1o4Elxzo-JMUymVjlyj_1~W|l8Q+-$CU z?V8%Kt3_+=)zRT@WwM>r8fV@_7zPQa;oti(zo1D=6vBEivcf2*d)RLWXL}Q{K?7R4 z(qxu1Xk$pNuIc?ioSt6<*qshF3L{5-SKTKDC_d4 zJ55HFk5iTA)^F~Xb(48Bw_CIRTJgx&?}s=f(#LK*@|eGDto-z+s@u;4E&Pr>_mnoO zO`1Q^;!AGQo`(#EMB1qP=EF@B80`|NZ}_t$m{$4&9W#~e?(Z~fJLh}$(Y~frE2DLD z{Cw|Q1-%!#IdHC(bc&&>ja|#c+SIhP7poaRjER|{ld&!DMUK?(FB88+zF;sW^a==# zTacvH!fjLMHNJt}EXMPLM>>jbStzi{qvp$MRXDb!R!4(3yTpDPkA;AUGQD~Ln6!Zr z22Eh#Wx>>NF~BZS{s70ZhenKG!EvqK$DQ6n=YdLKjwh$TPDBPg70Pe|_Nn?1%9;Xk zKw`IWQdXh&4d9eFU<^K@$MTI=7q=nNdLGcc@$%*9BO_=?*V*5{53)&E33tG5%hhk8 z{14tt-7gmn)zN7K(UWblIy_NWzYaJLAfRg%qr@Qi2*fxHIFGWJ%sOYykOONEWGHRT zsu=3pBRPjlY>s~LtUDW9mSeSCgR(Y!J9ZyA8N0{a6?qRlEC%Pd?XUi7kiN}v;Z{@4 z-5&4FKZc%LuAVTMvvl4Mhm1vNAt$SDgljtZyb4W~;V?VmZWq{E7B2AQII2(rAgOGnlNH}DhI zPq5Jg6U@@Opku~FDxdOzw>t_;A}%$I(!0rw8xPoi5>pMK>rQ+dH4Y2A%t<){OpGcA z$!MF`pcVgB;4#z<0)^!vJ$v^+9#Raw5w#L0Wpu-T%O`6D;u5zG87>^b8Oq@&AZ{}9$|Ewu{K)CBNjQ3WSGCA)f6R1P%!{E z4+i8Y-pGd-~Ii@lXPQ^xVz z`m|G@+^f2yj+6~-8{wREsb{Bgp6bByX5>uelV>llyxldHHMH?&VD*f`V@nt=GjB`3 zTskTHbxgy_;sk?rx1SaIotwazTsM1Rp#ZKlcOVrn&DaT--9K7tI=3ErIn^*-v^bJw~@3%BVPgd4&85|&D-zoHV z0p*d_hk6n~TISn@I`tz|tF`gyL$Z!F$M$kA8{G8*_JDW-W56eBYUtd}MvMuB*BA z%cXZ|`2T8Gk5t^ad0XL{^b;PVzX34Km^!y3D#@W|fJZCYVe_1khL{DuA5-_)3vf?h ze6G%9XKw1O(8G{~1!b8eHAaxW!EXm^a)&)ZQnE*(W|)eXqW905>3py3xI=wjbJKClxiK~9I6{Qa+p6>m<73Cz z);&CSo+Ey=WS5Ca^;px8bJKV-2D-6Ju;UllICKDJTbkNthJs1MCAb;+l5j@@2M4kQXRKJZw6M zB0KfbF=P!Vbc0c%tF(I-C01UsSa=GXOQvgRvoBvxvCQ_Ve7=^CcE8P0sU(Hoq94^Lr)7zzv>O7|Im*~_Q$eUvZ; zbZAtQ7XM}&KaPz%#90)cQ0LLq3c>~%&tVUs6uIGGx{d*&)R=7rjb>xiqdrpb0XIl2 zXkGQ?x6v#MMzq2vDitO>!*l*m7vJb$t1u7%h1Td#svZ-Gt&BU?X*bDY3UG~3 zrL5uhr7YEP*xUe8Z-gg8XGWwLUNlE3CnqPBF``JBr^b#(QJ_^2n&k@1+>3dcn>z)< zY?Hza%jS!aLmh*aqo_2+f&PGqpV~RQR@$N_oETw9h^AKJalrJ;>gZHUj6HysZRi1u z2igc;jtbiB64&- z>y9&5Wf020ek)GJsY4mX1ec1NQy4S{1R&Byc>XY;rySk5>7z(fK4B;+X`9xLj>WqT z6M;#No~qxF=)+vPfnfmmu#5sdQnkPYh9b6r8k}X0P@`knT2WMQMT35*Cx^5!vdUn{ z9rm;HOWF>nJPPnOjR=zGS=DwXdo*h`9DUf=-OUmt;CPemic?JfkP11_tTOsB67mh` zx^*3V>{pbYl|Q?wV-*{^H(pSz=N7-X^_dUJX*osjKC{71yr{mdd%9uJTx&_P?2uCY zZ)S&5v{v)#;k4#`?*>eyKXX{^yB)ju^iYK!3RBFgA0Im2GuiR=o7Z16JC45_anf07 zF`IQly#%N7Wv)i%8_2mobkL^7qGl=Y$vjoh!r9A@81%JOAsZ|e(63OLl%%?L5t7`C zn5>3TMky7W5T#I!)h@$2c9ApRWFbeL3V5zm{ML6+!DZsxX%Gu){fyq_pdmN%1J&bI zRNjoDs^{pWfFmw=VIrW-H$XdSIs(d{p98Ie6b$8JsY1=%Wp2R5*VB zAcQf- zo4AM~N863_y)BEcF1GK_xeUpCPe&aWj@I!*3n@7k4pAmEb!m&HO~5f_QxK) zJhkJLySqCNY7{{Df&ke8E(YZjm^#~xqX9|6j_DVWh)^?H+fps+qJ&vpP-mgn-ZUE6 zG1Glj%`o)@MDBm&=+O(1=#gBVfeMawZ8ho28*;!=5ilFk{@>>{ykjG}_A9wCAR8ox zPDN#AED7JoyUe0KFa+ue(5@QVwI~}z=(U{7u}3ALTHzV08ri79DbBq^%oun}YF7x? z*A?NkN_8=S4d87kAYcQ#w*&4D7KW-y5y9Lz?kA*o6{#7R&y`eGgElbGCaI*EX+t~$ zP31nm97HDYIfNw8m!0rHPzGCucUttJh+Q{rm4n4*Arw(h)XojD!eEnCYd0=a4u1Xo z#p6XSWe<-YkgCF1UNMIOFi_DK5JxuN8H37=bcq1Sxu-<4(AVOOR*Y{(sJX9!0PX`=5vk^gTpAUClc1``JVjTCtRPBkfE0krNg-09rh`CZ z*9WI7NC4j=(nME=fp&WTsjV1WpFGloThJ=T~8Su067D*GU})-c7hYJstMP1O|9~59rINrvS>>^FZW9 z45M$En~UR5?i=pxHVf)H z*$WFgwTFGuLIVT!X!SWa0xCZEhqwXvqP9_}cb!BtKvdaWb4=zVPW8=PoDYhQCBw%jg#?t@L==HS6u96PdA=? zzXBlN-iAq|G*@irj6M1ID`hT(%O3v&w@#W<(o3LiPrC26);>OdemV>^4gfR)NVmFoF72l5iXM%*cG#gIfC%^ z6%?d=W2o7HTJaYC^JD)Is|c%hF)AR0jt6#rfBa+V#F_HrROBJvi3_p$+-c_^fMK{` z;4o22RL?_h8J)BN&27DWq~~J{)Ox2aY9{L1BpzhW%C_fPJx%#ViGEXk_s*=d8hZow zoO5>HH{=(soe>)v5TBg7F1_BW#We8g`FVTyPN`8`hTDPP`370jYeWL-B{&Nk!4*0D zY4j)kNAERgdMgzl&njdMXK;^{R zlnSSIw9trV2tj^BwAz3y)x@GD;NIPrt!3MXZTc2STbss}+OY|Ar-eMXALwY?V>Ws& zVrOHomGA)isOv{QpR~ZTI8k4QMhXDTTbnXUUFsm(T)wAc`92|hIQ2MTe8hZ>paKX% zsC_0K;7jn9zR*HuD)z(TY*Tl_r|2%!B`b=)pz#oHD;)$Tfbu@twHib&3sg!|=)lJIY4t-s4Wu`9V&NOopa)$@t z$T_-6zW_$}=jW%Jsp1gjprkcm)CPKJyQ7R6UDm*qyzftAS@x`T-)HVP>f7C4KFo9W zeHb$BmGy}Ch#&m~ zkTw{<*BUsp7F}jY^$Ze!F?IQy%C^h}*=xfG8I2}Qipg;ZTxl`_q<_@hk+%M7jQWay zPgyq-7Yy&^`WUORj=~ZMlQ=4a!L%b5FhJ1}F;xT=YBpV@zI~|t$9t16l`^c(ah3rH zN2P8MRxL-*9C(- z_SOcwZAWX_TAail=e934=Y1N7`hP2X&k2d}^*b2SrXpx>QN_J#dDTpjMKeXzSBTxY z8m)i)N{HEkv{~#kkLRyU>OUN@qtJJbQK_VJ=N1{$S-xwGtGk>y_uX3k^oQ*>fqUzk z9Q7EUkEuu32dVx*gcuP3EKPv{+x1nQ>_w+QId4oGNkEOf5#Aef(7L6JiHZuxt4)Hc z;g``;j>fz;BWAkN^7)Wn@B3*MKkG+V54J~X2yh5*bsRPAe#0rK|JX7~VyW35(7eFOw7}9K71sA6tCOZSLq}){3oq&Kf zK=p4u`qGWJsvqh)liJLu)KC%o7i!hRG0*zkf~Xx|(AuLXu;Nq|h1iXci86u;to&AN zH4G(*G7;j=+HThqjkG%s(kwk}>HVr0#CEpveUL8FQ$OA>PtNR4%<8+BnhlV{1a(rq zzt%Rt5L%4_uwo1Y|1mHNOT(x?P0A?%H<&E_5VeZyoJWVfSGJvn8ta6|J8jVZC=uBK zu|=Abkn+daFSG7R_fB)%{$qL8$1tzYU%srAIBYV=d->}1iu{I?>N!W_d95@P-CBEp zS;WYvcHI*nT6aAETY`kmXmN(?K0#3-A(fhpG1=JtHI=tZ^K_Tn2>Ar(B=Q>f^|;%e zte?Jky|2E;L(PvpwhiSeho?36Wv#3Mn=(q zlA}S@slqc?!Yl})aw6}h*CWn>2Wh{Qqp#h%m9AW`9D^EhQ-BFq3Lrb{{uJ)!mW@h_ zkC0R>UcUSxg5z~-*6amfdoXmDyCsyY%(?F$jHRR5#2WTiKt>$W4o|GoT@j>51Ig4u zP~;u%h7O_ehqH75pcdo64fFa3$i1}x4O>`|1ZvF)Im{JnWnvEB+~j-+q0Jt!H}bs> zJM5^*d^vbzY%;YA6Q{(DO$8p-oV!%tm-hyS>%ikBui=AgP!={(6*rUukS9peRu$}l z{FYycpDfXU1i^6xE5`9Rn2G3ijcCt=`qLNOpQwtStW=zZE1k5@-o4pngG z<^^cPoj+=>m9gije_{b|U0z@MH2u11-6+0_$akQfx?=S!em)>Ky2usNq;t?e4it1a zq}{tavWHGxO?%Pb)05KYc*pG-$`WL(Goc6N*ByPtx#<2HBy3D;dATcGP6IpPSN+iO z?7m~4wym9=AW&TIMI0Q0Op}uED@au$aF!n;xHUm5p6Thp4;z`z6&5? zD)Lue^K;+dh7#}7k=cC{9b03-;%R6nOAo$#^BoPZ_N#_*E=Q4f0`%Z(LHQSV@L}#3 zH0rAbCfwbHB1)@%Z|C4r5}28)?n3R?qD6}g;{&#zi_C_GgH!VS0fg3-$Oi6xud72E zZ3GbI%4j4lgW(};S$~igR#7ifJUg|e&>k_rqB2@<&uk8vC}uZ@BVk=<7t8Z)Kit(3G<00#?jonHyLg;O`L5+2%i_sfx6hhW z0u4Op0l(^P?mTmEvq9+0Mem{!Bv9%!hX2x&awB6}RKE)rPP{KCIRMMZ@veHoN}3l8g*>Q(KOhLHRLq#2qr z^R{}lBUFo(^IBCgw*t&(1^m)hOm0P#L`@N{-n@Ab%>=>c$H^u>Oi#D*wnghI40!c1 zW5bd6!utdi6pqGXDxkWD7CO(jgA|pSbZXl&41fXS_7IhyNRtkSrOmMMIQM;;h%4kF z{8c=SIT&=pM6?Lo}%5{cNcJ*loVM%X!h^sj=nWd0Sy| z@x4MqYc?G2)b;LrI_$7W>sxUvn~bn_sIl4)Q1BMK{B)8M$BHbJSv+tAvfW)8+j)QFVwiiUdTwg1o%4SZoD#lyTdEo?UBE zn>lm(^hfrY=To5N5x@)}2tE{iM}j}!Tm-^v5Q6ny#5ygAZSHL{AbEWh_IJ6vyTh z*A>r9qI>n64?IBuYMxTKAz$DThBcDi?>2jnb`-WuMhf!`MtETnP3D0}RvRz-AaWN= zVA_T?x2Fq6_66yfyua=)sQpa#>Ki3RF0Q)3uAwCggmwT*E2vg*Zqb9-N@jFVw*lbPaY+7!Z?(z2I~ zefi3>{p5SeJIRLm-nY!j0dkMnoo}%fw}x$OM{z`zcyd)7G$TxJ@WB4jzmV$;QkPMf zYTo8bXuFtme^uPLdNs+9H0n^Go(U6`)1AR}7EGYg1`LWT4QKi+NF@;v4pICEOtC2g zJuEde0rhvM{D*nF9;x3`=FBy19>vpAB^SQr}ba6kkx09*3c@XD3U z63z?(CxqAdUg+_pK10y}xSoKo&hJhnCp+J!^&ZWELCQDU(t8r>JOpMBckO?==A7ZnoUC4_N7vUpc;mc%Ywo%5Nab>l z+bI?wLU|j7(~QCg2F{O&oiq;X;yNMhb#4DQ`YF+qMn@Zc$vji2vt+*0?wrvSK@|yI{aI)>_4LUT$*#ySaA)gqcw-O7R~zKfF7b3a z{x)bqSQwFkeALDgp%i&F!;Q?H5NLt>NdTK}?9}Lx4}jA^^S8Qft?` z0e?`0O;E#jEJr(UcDAV-4m~*lp2QEmsA}yyvQjb?hMf`mKpuE|(TWvuz^CXQ?g5L| zNk0`2C$3u)f9R9?beOLhZ1)^JpgA1kbhH zFqpsSBQGuvg4q6!Ze(+%2X>J0L#c_NW^cvHxfHnO8kuC4yT%d8x(LJ@S*3l(PKQ zHf7}Tu^@5FQ!%wK5+%1>QW8!tZZn=$9&dnmQMHpId*y4+n(6mBdvZRk(XEO2kvg!= zCqEqT7Ll+mQ24%!f$^cuBJMPNKFsYB8s)cu(@v^=+@bK}eAAb$7OkJDo~F3t z5pcJzuC63$8!M}F9LZkvUp7ILy8`s)A^imKToVLkGco0px3{fIjgSRIOuYb~;1m^s zRMKA;21z0FGgvHiNGv}SZCd1#vg%(QgX>C$SR)LWvk{8;I5r(~1NzB_>W-rOQo6_n zAQ<(5c92?^AsZ}2JN{~Cl(kY$fGs)pv2Hg$x*XNRL9pSEb2f8t$j!@ZEod}tktx!D z!#1qvl&O7hxOywT>jQ}(!$pCwc0F?m|N9_2sH@O4{5~O2fRL21rAlDL0@rGf#i!S* zqL=5vI6wfVm9apJ$*@z(VOaSN2y4EXCa`nc*%NKefB6J91(a+4k{lg0E7s~>BwS?V)$sX(;FNr%v5xnmZJ z;h+z@-u?25wiul_VX!+f_ue+sE20TCkuJlwJzv@qtr`XuIX~HG)owqhRTJ5hiZ<*h z5QvX66`MiElnxwh3Rd<$QY14)`A-<{23=1C9FBBf|5Pj%x8r6w^(F`TRBF&^N`fB@ z-9be;wvLcSoR$iT#^gj@?NCvt^W=0W`CIT9d=e+r)qS0Y+P6WRxns$-JrLj@gwm(6 zx!L$!CTxv&PyMnhW0+r9iIQSICVG;17T*~#oDIh^{AZTn`t@Z$#LT?-m0g4mOMX~- zUT)#7p$wOEDC4K?H0i`(;rFhA%{xlhoeSLKV|Ta|3SO=Zjdk}AXV(4R@^$29)Rd0i z@RhyVi;X{2q`LMzdZ@kkv)6iQ>BEq=dO7nfWoLUol793>W^ixw}EIyFKYN{O@bU#c(nmf4}PI@$&usGyT_tTY&5?cEqHO%h(X;c!#6QHvhg! z;d5s=&q6>+|K^$fL`;JIfG^4WFOGO31Si!CkoPH*C%?^gkKnY2oX?+E``d;Eb*3`j zxyQVEoO3pxyn5}L8WK+4{Gy^8tlQuY`sTBab)6kky%3b@JvqG-l{w;$gDJ_c^0zF= zHp;Oo|D&IBauKHc$!%jY*7z_1b6br+>dL>*;=;BHMX-VgLR`QE#UWGG%~9}ZzqwLM zaBNqCJbztsl} zV}<%z^&3750Ov4kmxBA!geCGemx1ydx&S5$^(?g%XuZb20&fK^5V?%#3jSYlBFzZI2yuS^g?vZ&YQcSJM*_9O;?F;f)&Cb<9Pl@I-OkPM-Wuh8%3Q*5LT_v=$eJ{R4jHPEr5v9U1% zsU{fJ-E%ojvB?Q1IQ!$B07Qd`C_g;ge6h;R4|%{Eg*!(a+YqKfR>idI%urZ+&c;R) z5l%Tqdjv^4iWjd)%cOa~;tm6kP~2!7v~KO%wJHI)_*y(Sv}}&wwdmQ{P6VGmn~JvQPewSHmA5cf)zD9 zf)y%hrX8BWS}gYE%o%cktpPys%cGkyv-mvL$%4?YBS(%TWlW@*u8=xbU}>!InjAr!*|o-} zd+U9q45|!;`UICEWG{v8xaae^4>4F8o#YcYP-g*c@fN5cTN@y~Z=Exn`VptLAsC~{ zrDWYuFeWG>QX+n9Ln0}40iFqDb-tWc@e~ca0;3g?Kq7JI0S+4Rv!CTJl&PW|PRf7g zlJ(Kp2CZPbD#7fNkKx2A3fB@=fnv#kUQ@nX2 zJTvWP{fG{;!5=;50BO$o7!RkKd~T@9wOQ= z2$D#rx^vujOIClaPfxwvH8=C!3#|p^#9qEM+M-hk$-O!X{)$i;jt)6VH3?JTFTAlH zap(Aa`QG|%2sjI@3g@ZnxDxGvfpnn12VwV05DIN^9(8a8_>Z|0zNiD*f`WHu3FQkM z9QVF$BJ4jmH#Y`abEuSadXUqYB95(oY~YcZnVAB>0SQ!zia*P_?zkkYI z2Mm4?*b77to`!+g1g3zQo&k|q-GI(?foRzngA!~3q;gko9uz~%tc7|$@wgob+Tyna zEyNQ(FXcg}J{U};v;@qQu={cMBrg-pCAKW2f%GJlKjFYj7N0D_K3OLwrsbW2>d@t@ zS(}IZ9FM>RV;Y9+y>k7`h)cQ0Y`;m*U`~P)b~Fl5mKjk;*hjhl)jUN-E@oF>Uj<4a ze8MGB|K=Sx>)4ndP!HJ;zz$b2$)z9#ScQZIsx8C-MbGxX#1`h_*!^w*d?d2TqR@7BYblB6}Kq?9-I-L+~9#RbMlmTS6=V__jonDOqgyw#Kb8xDZ9M4 z`Q%NA_wR(AQ7h#S)knF!_gOT-XO8(l`E3CkSV6ag(HnlN$TS2wR>%FILd;F<)os0E~x2bd7mfH{&sSIM&3iVIC`aP8oHIn44DsN1vBf!nE!2Hy5Je0CiPS79sYhiUd>3UlR|>8z z5(uMfqa*wv;6Ka$UwWL|&&t9(sNuCOOr1S_`bQXsNEl}m%wHit4EoDzu*X;t?dRZS z3RWA?uxK(S0LYqAkogQ1`?zoaHlf+@WhJgt(vYmQ;(_uj5|MHuxh2pz`Us zPedR)oK-4B{2ssTI@ivXC=rsDt{oKnYBZgGh_OR;*QJFPe(%mKi5h+#)t1rkVb_t> zpF$mg^m7hVzy>Rdc5-Wb+iNPTBVm|0#4V$(_%D(>2S{Fk9sx-Pwb_m}&vt{h_W=0y z087;jY^)f7Czq}7i=5t$I+&P@^_zmDs5r#H=RG-lcH)Vpl#4ZdCgm2h_A_T(^ z0*>%u7=a{WUYn_nP3?z^;@!mNjE(9-QdYd$0JSoeSPZM~ABu|wRArIxeT4WHsCGE7 zd2mnaKqA=qpH(M=F`6_43REEdzR{y_Me4BDEA9N}(_?=s#m#9z(yW)`qzC#s02F_h z?8rjt4AgUOt{Oxae5N;su--B~_;1kl`+mxoV6YC^O-Ot0D%=Wk zM!=9?TmJ3Gk4SJJ6pZHgMnryi<}bzEjEi z7s2ns*#qBJHx0tZ5+TYbfJUcj)@`4>!X?p7tGJ_+dH5R2?RcLqjNWl!Vl{=Od{v6a zV7@5zMty4ETUK7dut(`YGH|a@Bw_^8;lAv@Z9;yqruoJ1x%u1Ae!t8cgKF@Yn`H1A zF7GafyRp4QueuWV!YgU6mQ#^@h6#^xZn8LgT=^dN@94g{%fV2uj6}2(tCJc_0hHoh zk;cdE=1hidNw$!Xz)%zS(=%NC_XjLSprip}7--nfa%_*%5z#+H2c@&;&Xq#Zrh((D zenTx@XDjlZ)=gH>J1DCMds+rn|7N|7w{ddvZ+9T7QCl$ah(6dGOS`jIlF=0EO1}u* zEY;24CpQNDh7letR_pOMgP~zGQL4_cNyMA?{(0F0yi+q`uS#S|TcHZtpq=#k9&}Ir zrwI4-?&&HNmiTduuEuy9A}7x%(UQM)@9jI)XX{vAAiM&$z6^C{+Uu8)r2zI>3FqRI z(h3$d%+MWlLoGOaa=*Ym;D8iJLqEIBx3UL-{K9bGmuAmFK zC|vp{CHHK`5!?f!v(cc2;=1B)r1jeH4hOL-e9Y?*aRU50h!CwDXB$0hMa$!~)ldOs zePuc8O7H%?lCPkKJOV1aZ%&}kG=bZ2RlNb5feY<7H)7qtq{j94a~N}%u}y7lK;*%{ zcVajf($~R(I_K4_A7K5D$W*X}{A^QQ;Ub7NMIQn|c+&TCX3SXv-U4YEzEfZ*B7mUc zmctk1KE_gWd+UhpLUxHoL4%KB9>^-0)}XzaMq~ATM9| zYEV>_2nwvwI-BVY3v+$1X!6_ZU=XY@tKD?fOW0KeIsybV)YZvUNyGBX_fWV&*F>6v zl5_;DtMTEswzfq1fY((=6E@f-|0Vv0HgMJm!S01mb7rmC_{WQ^^j=UhAL$Z39^Nwo zS8Hpr(;IP>FF=mbp(j-uih?Ax?YkQ`3JUKm5Kxemekrlp>H zvFR}dIgpd!wec?(^?K1^%h!e?F*xwZ@oysoO@4L3b);YHBD^tq3xO7U*iM**^~FjY zW1Pb4*DC;2e(4(P5Ji| zuDgf3SzZ@D9z2JJ&moy7k`m;5|KoOLV2FtJ)DYq+q^Mowr7#n(M-w3sa=s8k<#8A1 z)PMf(YK%@f;7v-47Ay#ZD4-P+LDVqKPqnOx{)7}#18;>dcAk`&EHUc)W0=BG-ks&UI-Xc01MotHHzsO(~+|GFN#0a{ZnRDk>BK5hw`J~d6DO2*x%gZ;BgyZj* zzz)@k^)yS^q&lT}bq!iF2r;z*e{fahefXg4{Yg-fS;Ea^GGb!?T;jB-oZ~1#&I32`pRYMP|NoC>{J(g?U*=X# z_=LWtz7WWMZ)jMpto%X!#u@ZN1fJR++DpMT&?L3xtN!y&m4ny_H{jsPVbgw)gu z+HR)ziz5f%k*P;-FUrJW>v9`59M05&IIMbiD}=bwXulcI@Sk&2Z|ue6vjA8ph9InK z?S|VHgr|E3QXB2}sCNWO?e3vSYT>OQ98`cHyaIEfIS@GUpPRWay7%NXPYz^VkSCNu z@PSgyNFzTJr zvf!=PYAdu~wL;H6_*my}quWk>_5|~cR1y9%25?0)0^o80S6k3aVbKj0jQlVxgk(D& zhE@8XjBxx`3}x?{MA0A!BUJGV>ZZk^ieXUJvHN@e1F%@Fh&Z1)jXXQ``AH}orYSJs zipSK*dz$R*e;;MX4_{+z`8xzIfhY%3O0p0jpUP!pP>3n0`td8Tgr!%)A36-Ujg_)H^i;Si|%Qlgs$Q&*l#2PfbG{b8;AyMMcE!x6`r%*o|QfQb5X-% z>>>hRkw_6?OmbZWrN(GGhkYyWZ4I9$XX*P2QSf4j0f=r!Zd%a=Mj-|;TMLS_A>E8Y ztRRT+33+8X+JB^?2_OQT*(=00ZV)1!(?4G>3Ou{l@EyV=SiBK`-uAj2eZ^iHrWZj`Cf$yS`KhM|&bIC@04`tO@sd0CJm! zoL{(^@N*=!0l%CIZ9j#<;E4)+_-_GKiokLSK&)w<^`C>w5Xrz5(Gws9>6_G<93Y<$ z5&Gm;{23gKh4_OGUHsSoY-a`&hxG4b(SJqw|9=<6`2UdOJ$*X+W^i~UR?8n}LFl95cG6##q>pd>1J_c&x|ZI~YJ2AB>>rYtrzpy`na zfUs+fA=~;84buD{JwhTJLA9wJYrA;9hoDIVd#fCoW(veFNFvr4FT@BOQt80_At3ca z2DNb2cv#(c?40VZ^XH#P54CNkd<;c`-jqC$odwJQ1x5)GX*e@L*O}lWiy>tB>#~r0 z--h{#Kg#-O7t0ou{t;qLEPU>pH}c-l>CqV~KoK5PF6FRvl)s{7!%u)t1f?N03xg0_ z5J)c$l~VS2HsQ%3%EyN`M0-(HUQQt_5Pd@1kcw#{kNp1it5>)q(kN>Ei|?6&<^#hN zNaHv9YweYZiP;eP&prJ>Q5T1Z;bly`-2ih1UJ0dgNEaGIqM2cuR| zko+%{s1>-qtth!lHLEI_p*emo&5ou-bc3IWhLbc!08Zp7REh~uyYdEjax)=jjnz$0 zKs6OTAjuN~MAn&V%gJ4)W{2fGwQ#4n?(Utk>#d`75V& zYKdzqwMZ*CvQh3>JL}>u?_Oi&X4q_12oNo@Q!)hk=DXGOAhQIiBp-9n)kTl{%UD0i z9kdQQ2a@GMCQP9XqOFgBvdYoC!gP5Bk^ca~B-`%$enc~BW(j|=Ll}&e35ej9n)Jj} zY5a(cdFXbTdK-e&(>`3Yo20MF~7xTN9?EC|W1SDXR;n!Dt*yWO#0> zk`mbIO{!Z62>5i<5n}997~e-CmmwE7Ld1d)?U=VX4EdgY`y_Y-Sx;9#U6iZt09qY7 zVowrjEcI>NUfg|)3fSJ4vxI*FPaw^0AWICvV8|JW^7B!#4{&NP%-D_x*3wFf1^hhn zZ;M+4tPAg$`U4RNu(}vj=Kw_@M-ID*o=Gjr$FD+h{&jPfhYs;bJ-A) z65)emSq@4qfnatRnZC>8k{`p3K=V{!@kxzExChQMVfj=Ga~oY1$ao?Ih4lPe+kx>@ zItd>_(GYQdD7c`I9ka({&_{<1J4H}9rj+tqwS7{-IsAw)v2Fe~)>FLYh6yVJw~pFL zIa+t90S?=@Z~cNC37vfILV6ktBS8vS;+#Qk1aY#^SjMA;0X_>xi;irJhPr`t%$N@q zMy><-lP1`K6vO-l#JdAr0O$~d;5-&|oxRd{JjmKe5XUJ1-^kjvdw1&b!X;VV?25VHkp)I&hd>alhbn-z$||05TetkvR)McMq(uN>r3pJ;3Y)kUU6r>|Ay{Ri z=IPUqFi=n(Cvx8<)m4N!#Au8DPnn)G8fue%hT$hUJCqb2uQ1Njq${Zn-heUFS!x0X8Wlw1ep*zS#Cfz z!pEx6@i(M?_RO1bP`3oKQS}A5N{Q7hkeuOp6L+s{k@@=)B@KU%Ry9_4}`13B>KgEdx~Y=0(Arz573p`WTfmz){x%UAE;iT4{#8pf#p}&*~f9B8SXks$wNRT@$BF+lqKIFmRQWmIr4E3a9H81SFijwvL4{A zW!qVYSE%l#Vi$zfMB$krr9ozxdh)}{+d_Ie7*28nJ*Pqlzbau0ijanDc! z09OoVUKyHBbuzQa6{-BjdkbD1mFJLt7%fENkUlZ7lBHmJ%(^qlx}u|hdFbwGP_D58 zFof}t)C9!-CUcIS`jR~-!Oe;0vC;GIIAPGOG=o5ItdtP5B7=>(y*NTuT%C0M8Yr}K z>aw>%_aHII$e2@*ACOZ);FW6iH$o#V1Cn5`o+Fj65LVp4>_RY86$qY4#!_E@1h+zR zezH-}@IRnz0TN=-q9IrbruX%|4-mj1p)IQPL1~hqjm1F!doR}8sh<#FBac!>Z{8Rj ze;7tTXwV~I=1h@UR)d_T=mfz5DVDHZ{d$!=!2f;{N26|h$u&W}j>c*0nH*AfDOh{b ziva7?f?q8G$I*k{z@{McNVb!T765Qd%1B%hvUn+;n}*rah)3uz=!w_Z&|q+De+jt! zIJh$2KkSm{AL+drXYn^b-Y#$y4j3f<`tx>n1*N5GM7&~a@G-$>%7^uV$M)w%Vcy${ z7}oaKA#Q`5o(ktM0nrXEETI<*!!|JmRsIYGM>7|#4Z*h8s#y;HK<8q-=`^SS9`_DR zwddnyJ(&x?0m58aSvOvRjT=Ae9E7cFZE$gMk!|m8EaYd>LdE}ei9$!egrMN%?3q6~ zYP<_8=8R^X;y!*o)peXNuMx`I$-1QH-Ks#sDEOFwe3Jb}hJHJL&{8o;BFf{poDAmz{_%rq76V~sE-7b>g8^40&Y|@2cBQz6M zfsp`2@uGgP976%X6ObSY(wT2<<7ZG#vn@^S;)^5a!_KmPy|agu(@Cb2=dM$}g&8jQ zSmijB;R2R|4`HGZ8tS2INCrakZdwudS;&>VT zEmx`@!0Tfcn3hE!bsVM&Adup=ZCX$Z5srq$#5>*@QW;|Ij$^CzPGGqy&lzVv@~|G` ztuJZWPu{ov;blG=AX|@Jg}I9LL2nAaH!bcCJ^)oFB9gEFY3Nz<0vqU=IUQ%mgYy8FhLT?sxg#ijGu!Qqq#&8LsX824df6h7$l#LCqYc8 zBZ!7DNscgmDe(Uk9T8QIB-IzfY{!w_O5^2{thIkME5LgJffo?|n0|jt)LJ%zQvoJ% zl&Ca@BYJ>XqZK~%*_nn*RB3?tg{cZHYfrUlqA}W}KzmxZu*meW7BtJ{=pRC%A^b7b zD?zm#i&Twp4SHQj=o1jEMj5>WFcAaLSR|mRrTBpd9V7&Rz@>>rEQMlIBl(0PPRrMP zCnoDQB_qfdK_y}Y$d>9&2nR)Is>dqth3e}qK_p3r#juHs#~~6eizROni;&1Lz1D6& zyv(^D5##$!JxKt$3VA)C(?6ZZhJvYSFmiE19dM}hz0(LICt)b0Mp?sMg{T1C7yH8! zwHS!}sd_Rx6QBd(M?gUyz${?c!Eu4wFZBpu51;@67fHk$)F}kslmK!7x1~&A4D7w? z;?1nPP=1IA!UUF(1oD`q5d%wRgl!2zTgN*MUPlAGK^fK)Ad0Gxkf^X8>(?0kNj3aK z2#ui7-N~`V$P;u1$%jFoC?ZAn1C4;ZS0h}`%kY+!+>Nm{yu6)VU8*<=RO*6)uXVW% ze4k@59l?{=3UyX!Q{J6&EO#cd_YA%9Gpg)^+aw80IZBIS@wy42MRa`?)8|lrhZM%= z8|Pfs>D=+A-m_*l$$x;5;n4158`DCIklREb3+-v&y?Zy_1`ky=VV(lh5sO4l8!o&V zPC+SS+~A1JLPG(>sMS?R#~*Q8p`|@{0yZs3oCEkB$^yfqm8seQrpW;nJo#w?NSrA+ z1F$C}DiHc#SFWg{Fn%4@{~B86J5LJUNqq_c!FfeRMJ-vc<@wEE2@VG@1t0cC%klZl z5=2era~J*ln=Vm1BC7Qw;ukxq?lTKFi%Cr~sXvPD0aqqiQ7TTa#; zyzLRORDFq4hn#jDH0GErYXzl{N$eY>^%#H9%gcmmQB;*dwEz?zP(tqp5|`#S9xJde z@my{7t2KNM-9_6d-J_laxS?3+rEaw+W1cYh>=+VqwV?k89}^P&VxS>$jn2crJrFK? zMUN*Kmv$LiOwQp3ISEP z{|9^T9aZJoZGB@&d`OH&O;MwWaie1I9mSX^8&njqcg0>}1F>UCjEM!esIf=g7DQ}- z4GWg2Xe<;_R0O+-iVaZ|d4F@?nB@K5@BQ-|=R0GZjB)ZD4eWiF>sr@ZbIvta2;qoB zB4}Rg0TzFXnDO6@H|}PQKA**4j$`Jqkm|`r4M?EQPR@|wye%#h2am;Sxq1|2`=!7F zPKQ1_+L|tCivb@Q@Ji58r)}ThGQY_9sc7$LU@=ewi)nF;Z-$Wwxqf^qkB9X98bpv9 zwzWlg;pC#mx~a1dPp=Tbz9wUC&4w(z`DnE9m9qexDVQ1l1gl$GH9}a;Lw?eUdBZr00Qwc41ieGGuHUU8dDg8~a|X-s?cZ(lpzTx;F8D z{j}d_<(fB3I-o*{^IZ9qgvk)rrjaTPI1-vPF(OT>D}cNFuorGyi0BofVc=OKk~q$S z!kGT(YD>-y%Zmsw=#;Kp{J;u1Fwwt)w24Q6bCQeMMUE20{sfGC& z)=l!?KXdZz+210KNWErk>&Zp&O#Kna@7g_sg>-z;7-XFzH4Gb$xJ`L~Ibi2xuVa9D z^_XY4@|y8&e^o+~RCmx{7ttS%7-oE8osW45&D!IlY+{Vw#>~7@tbkkBrlZH~N=q<~ zOB|^$aZ#xC7I{O{jTP`$M5DzFp{$ zWjrk$!@G?gUI^|cmR;-g4~)CaIvcQ}?hga)m3ignWK4hcyYWn{9E>6ecIiIbNL$OCu3@%k&qg$_0a9SuU6B2DhL!_VzWo*sHD2+@LLJka90l)8 z2p{ap2?fV;&OPCL-&*nt0IOYw0YlyIj40qgi0c8G?NP31>3WsT29X~yiHO?uvb_oj zdMHDfdX$GEKmB5-hu5(q^O*vCqm?aR3~amp-~V}+vvWVXfEu(z?J#a-w(aQ$z7$@2 z6|!OL+qW>Av;+CiGhi%heeyKGfqwCKzWVLYP48`Q&@rb8j_mD*7XszHgka#&*Pq+| z-fbLfh6A=z(sKZ+>U|FP%lBc0OH|z=yV;!-bjnHG0LA9p!rg+iJDpRPO7=u z!;=B?unDaD`1*b+%OVC2vt3}&c}9U(jPieq_1{n+8mi(Y;_l(qyQYg^2g*&S%`O`s z%=kJzZNHOh$6A*$rZ6#Z%(lTnjxJs239aD`ZUS}t8?_b{K|Xty2%tC051MNpd^%I} z(MiN8K)bd7)et0I`7L>IalWjQ8AfZ+?o9i;%+7hW=EE{W<>}qE>+iU72MZ~>w$tBs zDX#bW!7(uXsf|V58R+BBWL)1rAmantlC{ro*7VCCR`SWg{4iLAhXu%Z?TDPBE#5h% z9qnjTRjFxSn5EET$l=nS?9*dUyJIWz=hZsCQ~W*MnQVrU5b!|h#$&DQkj=Q}>4!0J zyNFuD$qX_751;Vj#S3pfX5g-_;;fW*0xDese3IfGt)-cSbZLyQ7m4QTvPSx~bJ1^y z3O%~KV=ZUb27z0Q3L?e+-qL_A>xKQsBNt=ap0+#u_DNGEgR)T+=wrRXBW8njGe<3op0OQ+iUrP%{3hsVEfCU8FZqAlXD)}Gm!{&yX~tSjTEv;saenR)T%GhVk+a%>;wdT%%5Y7b%9!11t(p3Ttj2Rc8? zm}dKrjsqwl`hkKvSINtm*9tS?FKCMO@LF0t_lXgP>QrRcGNX6-dIOVx`ZA<**b4q4 z1Qw^Fv&Y=&S;hyHTFfp2vh?Vf7g(}=VUuH;v>ZFo_{JA?u1=0-#hv;)ntWTa;?D!} ze)?BYR#ZT;KU2Yj=oEXN8#ladzZ_V27&WEj|5AKTeD2G@o?}VnjR#a{ciQWp?|eaP zKev#`XTI~l1fEny)A zTBpSJ#M_2yIXXU#=l+SkeJnpB1BQo03yYOtxP_pk;zLcXA0F+9frOKmUoJYu(il5PWTC_vBgTyr>gVr_?&C#w#LU;Ot5;teE6y^R<~lD zGU~De$pK8=yS=Qu$3*B6In_ouc#cuejp&(Ff#Ir3U( zel8Z8a0eJifCXVI0zQ=>Qxx0(OLlUa<`{Lp42ij?X=ek*VY`N{+OJGv~c z*vKEI8aqS-N?7WWRu|d00 zySk2en793RpisFQ;g6wruaQ@F9-WrY_UVE3277yb2OD6iUX%)wREbvmO*BgqlBOIO zpJoyJ1uYc{G$SLbzf+mJ=T>H|8(v7UMz|(xh;4k9D5)M1FskaXe@)GRe5dut zp{$pfM9rhNIFz|-d#`qDYVcB*00n?($_>o-nrWN!hvdwbZ&8c8TO=oL(ye4!-)()b_I1d;?Tdze-}49xGvcsf?p4T>!yqqbM$6F!WktST$%rjdetv;WHE zXrisyrxh67i5Wy(V;UMC+NGxSG9NS|g;RxTat>zFV;yP(N_z2#p+6K|?F~hd-Q?Y5 zG?eoy$9P@aIa;aQU4dd)YaIXqGDRLjrC{Un;gR{;p)k98z%Nv~VUbbny^ z9h=(Tp6Xr{DD^9in2SfogOj#tm$#+%ORNOSE0(OY$OXQPVQUSN3JmmxMbe#@1teEeEHU#|)p-h)c0At!mDO zneo2EPNL9-iIRi&*6oTmK0fFgxN+b zIG!OwMrDwIM|(Tcpl0jQM-tMkj`PtG4btg!?#xT(AQ@hLZ`W#x*4Z*=P+;(=dB`g3 zRGux>-8IA>8CO2dSH_oumQFEA?7@d0voaXQIyaWlTk-?ogxvyGOCOm(@A=O>gvPTN zhb#}r+WJ^~dxY%m$E*dwAZAxKzac5hL|s3GT_wRgq6VcCKxJ*I2f};&<=R~j0xvoj zEf(f*lx{HMcO`pV+nwL8r#UuYpxn4gZfD^ zDzc?Rhalx;q>tt}C2s0&BS#igXcZ31CN()Z9)FgZ*)C#94o1a3WO_NgFcC{7DinBB zvsSHIv2UjW*DLo_C;;VdT-zwT?5s;$S!~%j8TC*kI}%~}HNBV1&;4FuQ=!1N8ssj9 z%UbBSms;avQt35ZT!IA|H)r^3WN{-BB%lZnj@84_+HHP6a~l z>pfz`2ulOfbeRGF9nFN3e*eP{A#7O%mT(Wp3eby|%p`|JYRgr>`^}2%t=nQ>~;%4U-ciPr; z1|pzVI!@^kTvf)oZ(%oCMVin*c=T-2a8W0)Vpl6% zqnoek{BMl|ctL;82I=`oyo7*9(o8P_o@G3PQ*!9ko&;fW2|1CP6&joiXBr$&$&+-l zBOPy%2l=uPclSoPanRrR^kO#n8OM&}Il3Y`SR}KC7TLJv0x$6;^C}E=TVN z1u|*Ex$F#I@Ul!o!Q$Y+!WS6iCm0SA$jRl(mp7r#+!O9m@5-8|?y&-vk}akk1Pm46 zDdd@s2W>8l<7Lw_6iKj(E~vXm3Q$pwQa?lJd{wjeHhMlE&~?);HXyJr(#akz22(_W zR{N;mlFZBplDn+SBp=013c>P>UzFxu=r@>z%q>JdLt{wc>s8O8&H_YIA`FoW{n~rV zvmXQup`&LZgxBkLtl(56LO4KdMH7U>mXY(#Cx!<-{V1CJZ99GniQU)GdZY4(c&$Z* z=Kv_`1)mG!uvQRZO<0!6H-&QloNteP#aVF>etL!VAPut(7Jg>80OvO0t|G6YqOe|1 zPA21C`ryf!Sw|gf*4z}3+x`VMM;*#@*t}gD@mrZ9aV3U&}<0HQoqnnS~w_yUtY8Yt4$-H98LJfvLh*T1YEUOs{W3WhWsW z_KC7~Qr8tNQi4Ly+kf?XAujFd1O^2$h_?-%&EI@aqkufNzkt}n%B3rs3Jj6{U;}O@ zQF^fd{+h+4n$z(2CoxvAuvoE2*5=7$=7jpICyoW<|GZQFqJ_&ff49yL!2@0SBi8Tc zZGr)B8rCd&IdtLDJ2~%uB&#w%M6GERyE3eqZ(%Ymj8pKT*;BDw&g8F8fMQVN_(^$( zAH%1cPvn73$A@I(1IQqB|6tiRp%&Q?;_c5r2W5S;XisGf;7~*ibLsK5-5P(A`8e5u z&@WvhX%fnMpj4Sg#P zt?VkTSre`pQ?bUe^-;i&r$IpK?@=Rt%Mvih0WD?Xv?^gV*`#0pmw^dLO6DY@340)? zT_3g79YU76uuM3+N_a`HTLm0Wj%ZNnKvcsG?r!<>J2=%nm@;uk(>`3flNnL}GSl4) zYx`KnLjVw%N)`Tj{1F2vxAQtr{>v7)%)-=)RTqH%D61@`7`WIZhBY`F>XWpXf$s6z2q;@kXzbKVlmJ7L%_e*~&xZ#w54uHv}=WZhk;kdEc^yKWkU?`(_d zEvY)2H2os_X26l-y)h@Sg+Mw+ISnzO-KD{VKaQ5)xAH5`S(rcY6o%b;+rzic z)tu>J@d-IsN*;PMu-D)@Gsag7*L&Msl*;xp_ZY@JZ<)*J8aU>+@s8AnX>l;sEL^fy z^!^#fLAx?(s#6HOWa5i6PKsN+jw6`J3;|?rD_I}ce2H<(lRZZ+mYbK3JP!Z$&jIs7 z$*=LvTrcx<%DAS^-`RNzNpR}%KC#D-A>2qzBBHO}9_S@B2ld_+xKQlFY2FehNmGp-QI}~A# zI(rpM2+x}yoHSl^(BZ~+UVZiTvJ=NU(*O#=7+zu1?v=CaR?iLner|DfUuj5hP*Bhp zPEKW-Pid6&e9q%P0xP|sRpKFZi^bvko1EA-li;2JqMou#Il`=oF#X?q0XP-0m- zz^N`|B##{$29@U24e-tB@#CS)iwDR9<=8&=41yU!dy{8>+^qP7`22Q(ZOEX^iAAJ> zpr@dz^S8E(O|4uitH;cGWSf=E3QdbE%&_e=s%Np$pnzVhf_LJYNGa4BfsXu6>N~GP zPtKYYb%&bV3K%sU0+q$KvO>)Qp3Zy1PsRTnHS|MyA8;V4x^sCV9OXSHz%sgF!tSc# zL+TbrLJ z<%yJ4l#s)wu0c)SZdId2)`C~cUQ43%P96F+Iz0?WXz8mI`@~OJwrW5EXWH_?YdS(Q ziodGV+Q3gA{aM3o53U*ylwyNWCXK%TMlcO!pi9>EaX3xw-ovjXrswM(y_lHkJn5A@zquI&4^tF~T{Ve99E#k|j^?~PNigF947dA5|A1P7Xl*WDB zyzbSgVBQ5YLvJ&!cEiA6G;0IN&r3?6p)szWYc(902xejca4=}AGAkh)VG=zpzIa4>D=B$bLO0Put!C5>mw&H4Kz%Bt;Bhgr1~}7$#+O<+jF`P3oFg%!GkZ~8 z)CUwy#yL$@ERICm9)F4u4^Uz#F`CYvw!haZZ=Q(OiREoF1C6=|2S2R^e?RH&k}fhf z8$y(w`;cd;-9oso*koz`QpZWmZAMoB(?o%yBkyxAlQ&l;50uNXAbc5QeOBQkZY}UH znL#6DvsejBHWNka4Ur03z_J@**T}cZ9LLVxhP`2F#GYib9zLV$ zEU!1=Nl)5Lhw;7YY9#NV`Ou*g&50M9y_nZ-*RhFtBR3wfx|f)U?`F%5X>UjBd&KIS zlnIdHtT3gKmU+YV@i1x*jIsRLc!pktYsE4DSNXqMYJC21%{TnJnI^pE^^poK=Y|)u zpmmC%i%ZlUf~;j^!8l=+pFTa;XW>LndetAWEQ??P1P=gMp1D;2_LE<1Fh}49UOm%8 z%bPJ6k66mFe=Pg;k-7Qo?elT4?jG8k5(QM?_1tVwiq;_G6Nen)gH#P2@M~PvrNYWu z{buUC6CJ}LV#B!4VT`T}Mwxy((IMC%^fzG7x6=t>0>z%<}pK z#!Dat>=f3$6E%F~E1I^#X#(3!Xw{TR!3{){4OCs)R(kh%im83|)uK}_x*0pKij?zS zw<2dwZ{b;^e6w}c`kgBuiL>DrLZ=K2SVoC^Q>KN2;B1<7kP=QUN5I$w7a!uKq_C*Z zhe^W-Il!lvhjjP8Zu@NCitIMKq31A|A@fc=Q6TN;7zcaSx)pV4#cO++IjRyR@(@B^ zaUbF_L2#6M$Te`zD;t39TSmQ6kj`q*Hpd|`8sYB#T0q*au2PRl0n_~kbsoohd-Tt? zo)#i-Qt2kPt62WA+5Yrstsu_cW)X)uYl?=QMYv1Ip8x$o@^X>dd|~6LKW$Q&(TAs( zyEHgdnp+4Kd6zLnt@tH8q_)8Q>i{Rlx6Ol=bNlbL&Ym-}-RY%21(ZPzcyr;4U*^yE z^S6Cc(Dsqn_tj3SKXcBK%?-IUi%A6)tf2WJv45fpQz2uwPunikE#-=HzV@1M@OP$Z z`+En`3l=L0k`qkGnu|~Vx1=w&(gGVq9{BRiJ?m^J_Nx_&G~0*+)|X#>wRLXBkBee{ zA2ZJP$*GeM&ecqB($t2QyG>A0t7eX6FJCl%28V1hQ4dRIa+_bIoJYU6vD*mS*INfS zn0~->)Yawl3F8^yx9%mNk^!3-v+V1iPsd7qNcdF=$bA!?cERK13 z<(VvIN4z=i-@*1VjvMNH=tl^AS)=@}TddQV#7e;s+GmrKUBZspazV#r2L0H6KD9Ot zel^l=HZ5ZcE<_#It*z~^-c!T{W?1 zz9GAHGSiW{2H3WSS)2iis^KHUslquy-Q-rm;a6gQL+t-Gh=(guil8M12s;FfQ?hi) z)jNeUP0V(bITpY=)wn}*tFZr(FcG@EEz@KP^Wx4L{^R8pqA1ZHxmy zZpAq>{9^bQq%rb6QO~*vtC9w^;aeR1^pa3){Wm1rK;;h~K1|(Wd$NHW=m^`1pjbEs zct5YS|;(lM)p> zq2Vg<)OJg{+D1q=k8E`wtv32|`fQ7Tf!~7?8WEBxA|sKpA}CVxJ3z!AdviSTFNI*A~6o(14z#EDj6n-#25SE z%rS^JvVsvzq{!>(9>BU!y&ezaH!u7@oH@uLL~IajY!f9e3SA`LrBeg!y6j_n*Nz2L z#;HL=%^Dm)ET~CO_TNt+$C2(q#9w$TV_o|gLRf;*<#hkcd)v7JG*z8@_C029bA9t*zwnw6&>^c`s;XWg?5u%Cl&p?cyX`$ zS%r?Q+A+Miy=kS>&71?}49RMw(y+%rC+nD0?d?Ow*>{+kQ0F)w#VpT*X#f8mBT%c`Z4? zrbe}l&w8&uIpL?I;V&PBUr@p?;PMW8=g~#m&$yO$Tk`mu5e-(VpO`$i52kjWuf~uW zg*CH8XX?$-1jvLB-CVBZ=aW+R7ihTGXW~+0l{D)3>yhO9rx)Fxv7qir>&uO{ z=n>tDlNsP&!`U*QV@cZB$W?;8LQwjXdQvQ? zrWSGBD;YiKA2bZMhN}g-K3MPe55NngrYTTsO;_Y1uV0#Tz)?}>N6 z)X$I{Pw?+*`vk}N`>dVYsBIiTc|6ZI;bC#xfAIYrV@JtsYV)Dg$x_HR)whMu~&U=0xzW(BAmiB0OQ6G}%%p7T*9l$!_u!|@4^kc4=fMG0c zW}1cqf>r6d^AC-9B=O($%EPwh0>?)~Pl+ecNLFxqo0Y@b7DqIJgA%I>!}W&2_d^HV z&LezWKiN3R>sX7srx&$CQW?u39Dx)`XQ65>0do#6lNvoYwzq4O4&a+paZ?t9px!*4 zR6Hh?-k=C~^PaxzCynbl-(0J?UL<8F~#_*pdy;z48PbJkf=Uto4P<^U4^oSG6iHdZi?qrK)%@zi zJ6nvG>dvYxj}$<)jvNu9o{Mwg9QEe^gx7cXY#0o7E_V@ml*TPH)C6hK5X}NAq|W!h zTcToHouy)a*aGfSYnr5Vlv9e@b>@vWzOIV<7_VbcL}9aMvw*iyPRIyHU8-c)>6E9J zcr8fY+s}7!>5ddPcx8{R)r^zIE^Y=&p>)=>#*MOLHnIlO$z|kSe+dLle_l3 z+Ci0E$#Vt?^s(G}6bSg_VUe33{Cd~BXfxwE)9X*o9=`-6;0V%ibdO%yV~zJ$)*b^e z9z|ud+?>>`Wi#PfGwGqTC|-+gHYS0q104ygzN%{bCAX?Ce(MGk@RG zXJIWG`i#pq{#|F&7dwKVdY&wa;Hj6;Op--uRdtf!dB}L11*9HJSJX6qD{eXTy?PR* zW79|!{#Y3kfhyg0TdCz$RyWQ|l!AWc{`vtR%FqZ~lQpJmKGf?nHc|c7y{4@ww2Pa| zmH^ed5yxRxB5ZHc2z8fej05TNrZMADH-a#^cS4oJ3L|S{5FXrBH9K1pA2yc-CKFIoq%U zl+UWtM&qyBezX@1)0KlEXp{yY*C*~npjT-bg*5dc1C7)nkD{-VM&<&2I!~Qw`_`vH zO_`1TbKz3onjc=-wLxz0GASLhm(EmWp*k-9XgLedmV}5RMkZ&ivn<$uCnbq&hk<%1 zgWA_C-_oMKWzOZQv>i6#GpE5O1t0zp$E5D~0fh<38v(HMdTBwKY%>(`)$XJbDG%6{k$9$uk4jBTl#>Twt(_!qh zl-%E#0`T>+J1*ZGy`C>{PAu61d0|E6{%8z@2(cekdQ}{fP7gIVePsy~W@O*NyIkDs z4OqI!Xjp;MC2wCGXJ-jT#|~Y5kvdkl`BnAqr#Ux4MtaZOBQbKV8=*cS>$wxtkZTlE ziod{@W4b zxKmOMw8IEw>^0rvTbi?@-p{Lz$-1B0!RQ=GOr?HCTf7Ab!D40fgXU0NQte2>P^r<$ zbB}X3^{z>3!|O2XNa?i_I7#K0Mm9;zdumO=ptj5~Q5h7$^a0Sr$_OZ>@VGHGEBX7y z#Z7OY!Q%QWo8_(|hlfL7*d#MY-ZwGZ2Mif;WoJumH&`p#7z2JP)1~v%Z@t;R!OC-^ z4srn#-rUXH}bhqGL2qABT z0AVQu5Dbyuk-EdfC6}*VA;;s{tIwXDo$r)2v=A7&c|!OpTV!xO`BiRCi~TP?j>|n) z_TAeMk7iS-Vni}Zv`GU`2S5Vy*Tbf3N5J@{0M7KL8(D4llV+-sQsW~O6evwKe~b0i zHZ(q;WjsI>u-F!2U*~SbD0yuXq;9-;4P>{4Q5A~loxY+dk~)JA@HaOO3|IRgt9H_@ zxs|rZ}wnK7v%w?elMX8DB+aG@IIuM#qy}OE6xovwFh`UOfsW?hd>pGA|m0f#k z99af%ua&G}rf1mKn_<%hsfCoS5!7FNJWj zJR5(>q>}X3BtZ2sKe&g)fWU%ALBSQ!iA{s!R_7 z)%f3q=^wv8?5QNl1wqxAf3l^m03Ru z>1Zt*#lAuvxI4?3$X|aO!rGZAp@s2VkZm5qnp}g#F>fzS&ke8P%;)$J3D6>pSVCt;OU4wYPv2fGeFK`Y5!${{b58?M%IuFh&W8tk z9RuBw!CKTe=#=R5r`<;bOiH8tCdQMduG$V`duCPRa7>3rhCSLvgra9GDd#iOAV^C& zyD^(@=o0J^I^ky60f6|?)odn*Y8(&(z(v`W@|nDeBws?>f&@u)V0p4&0*7*x%ZU=` zB2zSS(m&%GYcH0YB=lD_TWZVcQG^QDl1Z<6(s}Li zKjOgzMlI{a_Lp1(Qg7Fro9;`r7AHnC^R$_iOMDf zY+cV@r4hI-ax@`ye%C3MUrIYbAoR~sY|!4zd5;@)NL|c}$K|(p(e|a9ime>W zabrVIh%xZ^J3F`kPvOEo+(~`;WbOlFe!vV1Q3<`K3)4-We~l+1Eg#k$70rW3A5e9%RDKSq zK-lxY%fAx+i{sIL`Y%9}0Jn7M>_J5}Qj`A#WRCfOJ0z7nXQW?l_2*e~4 zQAM{*&Q1S@y*EaM&a8)P&Czp}HQjXIzAHI3=6(|qP2QX(&x6pjGGZ&H)L84bK8plT{1ORb@pd@UTcFg4q9R^L0<%A#R9fylf!cz%@D6QTahjggBwFx57^m1~v02P+l5 z?J~gjKV@SUaR!BT3vZ2vBwjNx0jmYF*aqa7(T1qlP5$aLmX72)ly{atj?j%UlvuB( ze==zNoQg?sp|QK+gMm$iEK;%*hUJZ-h%ay%K@4}rdRKDyVc3B z0iVA6=tcUwKdY+gpwY>u4o>qXIA)Ue4k3`vO{9mh3+2tOs^i9u3-9dpnhL35UN^Nl zv(chwf6twEOs2&>{>^V)#t-ocyxwzUR`b+iUlsdu^YaQ9V6Uxqw_VT#kuKg3(tBs?Z@Mtatrn^#??o}m*$ zZ3F~g9blZbG#YQ-cZBG8w&Vtywqs76sniB=bt|iCW~J0gQ!gH*=qVgL9Fq?aAFa38}Q?v z8bGTG;mMOV-8UP{F0})XqFJ?xQ2l{(@?FlzIvF`R`wm35gAW2Ig}(>r ztDnIk8oyxmFgdx2L}Vioi&23bDVV8Y58RBCnIyJdx`DhKKkl~d`l_WSa4tao==y4}y{stme_A&2e;z7K)M86>A%r zg6*e*ZtU`!X{Gjdvw+1~MwBRBdf4SxN6M(0e{SvE!T2cKza1;|2O0(f3^u70k6rS) zoYYWRo4s6JU00kmFh1Llx=#9mPgP4JDr=c|k?WL5$hfkrm83`XLVfaeja9Gm{R_hj zD&pd0wwH_&6Q1=#?Y0aDUNfc9Mk0T>UdePY*-Wg0&J7wj<&S?|m#^KZ^F>ly;{UJq z^TZ6elBFi$KvEQ$G@6azzT(n7VD30Vxcu{nfTH^6h~gD7NobNKI3chd2Rgnb30#A! zXgG<4XO;eaUmq`Gosg@(WFUg6axUEBODbuzg{2eo{oVm?^Jr`YHn3`!a9qk^y3>7{ z1qp_sno?N2>Ax;~;mq+XgH!35=E|~LTr2qT$5(S4Aiw!zcMfZhh^b^~_?-5mz@DJP zbY5+eaUc1~>>GVE;S+Z%3}sj2WkA$<~5VtIi?qpUb?yl>Tlrk z$Zoq)fX4#-K8Z>Zgm(69)s}B&ZI8BHfop3GiIy z)I@&MFm{p@ie6}ED3I>lTl&t~f)g{TMOB~*)bHxv5(OrFLR5U`aso*s-a${nFLd2K z!1mdx{Z6?X%&p_`x%1`)o9OJS)O{@lbb3lm^2~Jr1u`2Enkhqgifi>tBy8(eR};t4 z!65S-ab}k7w011eZ(~8*y$0O`qxP^+SCirv>Fd}8S2tzG_JZL2`6-7FM8i-+V;2{HN^fnFP-Jd)Da9VA?S(GQpwIu+o+kil1Il!SZ@Cy&YNJcCB_FUT6V@Z}N0? zo&pA5xs<$rTpP_4F?Fk2=0p@7wyGYfc9WL0VT#V1?t=SR^-gARMECiIiu!b|4;#8v z9Bu`I zVJCoTxO^w+LOBYD`Wx0w)hHXc zEr2?p*pXG~dEYMhgOo@0yEt?BV30m_Jcp3M6IR-X8lEqX$sgjO?+~ulhuU!QC4Xf9 zD{A%oZoMB60J8PtUUd!XrjGnIjoUN+U8mv)jiv3+{{<7Ef<|Fv^u4qPRA`$t1eYq` zV>e2j$(``Tj#`V=_m#-GWK{z2DPpAe)+p~XQ$KOtEiWM{j@8_Fx%cFDWJB$DKL)!i zF)s_@^R?~m@Mlv6N*W~Nnagy#-h5#AUfJ2*xu?v)^~@?f%cveM!(1bg*}*UN5QH1? zm_4Q5rKsJc^fDri*;;5-8VhMFvU)(Ru!YUD-9R(EVFLY_Nns<~$^jNJGUe4xtiug3)8#95AzM)N5>f9Yb-q@EBz@<_0zhbby)+qWAAxlp|Nh-X@#oNy2>>#YgU*$9$xQHO zoKhQ0s3Fv_vkRF)Dp=|~UKVPd6Zb%dFCd4BT%nK`m6iuPn)!z83zd$6b5w!xQfnl< z!szcGL{Ig2xakt5afy)`2Jh)+kujMj37`uA;O@lLXLKvt$Qz`_5C4;>-M~jx!Byw( z$)c%v)E>0}Qe-Wr%#p+l8p@1{PeUU##ska|%FqMMI}s+<%{?&Qss0U`m4b%6h2(Y? z>G7oE9lTk4G2~d!6*X}?3_)eLp^-o)Gkw%n@RbcLRYvU^|IFzh)}DKYB~ec(=|_)& z&ph+LUk3HH6}1CIZ?Bmf7m+B0A*$)ad-HeuD|*&}l$8+>YKeOabCI#6jQG2k#{#%+ zx6Hu0IssWtyQ?+@ciz3XWDJE9{jcMhI%l4+*Exm?HTiiacAj%br4+D6o;dlIPA-ef z4I)l;ymX?Tz0WW;IGI<=#pUOGud@ogA5ZmD+y8>KAUVSO+x+r(xP;*-saC+@5o|UI zD{IfOpY30qjGyHAn5{I7?GvoA``s6ucduyNQ2Ha;z@xu6P3i zLlv_erW%a8zC~2LMc|1>C3L?r4e(CBzkcsfNBaT5OB`o;NrMgYhXI=JV$y65X*LOf zYyB&IWV*R=S?@SpFT!d!Bhk_@KScjQCOL&@Xx*jvijeGeoADzzo>Wp|Owv=RToo7d2&?u0S4UWO>7$ZN53T@stf-QdaLy+@uN`}C8~K9l>gDUCKK zZ@i&QQg+58gxV-|o?3U+^oISc$bs{XTc(!74P^^p$W|yYeLiTnG_zG()}gBi4e8#i zj0o!hFS`iOD5?)IpYuk^fcU_|%S*RC)JQGHzs=zFnvyj!1-QvC7^b8@V_L#!hsoNqQ*etJQ^kZVO;P zdJ+)4aGj$Q(UhFB_%{6*o(^1md&^5?p;xOHbh9IN%^l>rMkzBk*&d~qc5 z*zg^PHt`VCK-9EwSjbLh6)sd^w3gK{%S)#>T)s#ww%xDu=JAGZ+hCGVhtYdftTA9h zu3le)aBwzs^%GRg9^U5kkc>AB>e2(L(qbw#sGEWC9@*)z6=byIwXMk2tWx zqTr|hnMj}Hcp8({PAX=N%TOOq!y5q=VqYl*e|q!yB}nYhYSYG!1LPCiBvCCcPD+7n z9`7t&&X!FvZ|uyBcvz=7lq@uxc#*VT1HjP1ULK~on37-6cBtkTF(ROy!bXt~B#^Ra z!f&`8c_0e%r-;VYXlR3$>bhg|A|87-`C44Td4G zt`}o1nD)eXaQ_l-*o*P3*K zU75l%F1g@OJo;8(fWfelVEO73wCR<`PiTB+`WZl6S~m90_-W3sbLO;JAL_p^DhdHc zt(Ft}H{ADRvxD1bkmZ2xP;1CkmoV94InHpFD zrp-5|$XTcU_y-+eGu)Voy@Fs{-E6;R#eAgn6psm5iTv?7>f`D02!MR}re{ z>;Jg>tIa|MY=AY zKwX?@j=wc?h^w<+)Ab~;AZ+{JN-DqMY-P`dFYYul78U7YxLSmg^XCc zr!a03lQ6|tra7KtdZ-m?e!7|k?6n5(Z-nak;s&2bx+O%q)A9!-MD zPK4;s$p4Wf#mWMuX2USCyg3_N1#!UI{mTAv&ryNKN!sM@^`}FAnLer6q4(DNvkiDd zSD4z;n!cpCoE>QyPg){*oQ#V(XzTSmHrqE#ZTcwqOzF9yH4as#3=?5Og~hbIR^-(F znf&ygGc>-5k7Px+N8qd5>vat=XZoY0*`8B?*ptTO)mXNN;?^bZvAaFo5#1XbDE7tL zKA;8_Ndu8li9}J;3W3MZP0gZTKch;MCQXiB07$wQ7(RLd@yE%{Eyx7?gu?jJ_P@{* zwft`8_=ayBr}<$c4g=Dp+9=yM+5Y94Z5h?Y{WS?X1Kp0iKX7B>{j#A`822J84SCn; z%W&~IIyE!>)~#2YXXQk=UC^l{#(dJ8kumfIpMg{c}BbL0iTz4<5~tf}hVBdNVQQ zv402|y;Tq-v$FIv8WAcTdJ;2m@qfqDj3d0$UGT^WCp2B^Aje*-%n-SjTQ_ozzJ>2O|^0wR|;K7)nw{E4eK0) zIgbwC+W&RKE-SCckMa6Wd-R{1d79D_0%{g&i69fXx+zKBewHsB9M)`W?&9JigG%iU z0jdN*jp-AQe6pn26%Hb|iT`G^ntdC0*WLb^eIa^R%zM~R3wm9fj2p%*#8-X0$Vo>f zCS_2(2ECy`LN{@L%~aT&{}(#SeOXl}vHuoES^Q+WwyMSS8{0>^Gmy8J1@rb9Uu69q z671}vH+=_Rx+)^CB zm7A|i#@G z!HU`_kX^-KW2aZhPOp!BY@sy<{A+xEDbEAFoXhv7`buB|fp6qgwCULGPhf0UK|fMA z_27EvSy~&v;m|Fux%16wWPSVhyLfGe2LA9*V7xW`@xRH~=;Ocrccn@`fV#%L75Q7Q zVEgmv|NK{jl7Bz)R(~rSY=2eezb!ZQKM$PwKL4}K{#y~hP%(}h)q+SNHx&PdIwv(> z9(}SG;*oN{zPD}3z;P0jqtm+#wxa2A;HwV^*G#(0Y87j6F{Qd4Xz%`-TvDb{(6CpQ zEJTtCymbksT}Hx(4^pqE^aJFGgO_N2mipr4+XiHMJtW7Lg#@7?di;$piq1Y~3OM0y zoD0&sO7_WFr|cDuCoRv+KX4p22|aV)rsQg(fmHBtVrtCSNO1M#2|z`-ay#$m1HQ0c zTT(o4@>d9isALsAdp^0>MK%ROk);`-tRBK)+&;|^B{jNC)gG2R6$n*}CikkjXGmUDyTnpM@-KsT@U-=B8@VW@hXP&sOv zvA-YHvd%qg?#iNW>BbYYWeT&Qtsj!;Y4#7tV2K1Ug0pFPFp50fR0qh=6*;zrxGvzy%cpr z-jF9qGv%9;^GlyyHv@xZ`+}zv&=W}PqXXciB_UR1K zmV4$r_`YOD(XUTN-S!RMRipNeat-E9I2BWQK(GD>n-;&FbMshNPZX71tH06H{35$=gef!?sLyNW_&hu#)Ydd=kJ;}9bMZu z=-M1k>Z3CX{iAeq*tCBZN$cldeO1v+bL?=-$Xn^u`SAyjfbHz@S@~dB(ugW(7Pr2< zJ7VLwrh9T>kKAM+dhg=pXE7^lzt26rEIz52=|q5M^OJ9LCSEx?xpnDuF1X+EfC;PS z)k$wb>hpFC+&#RE$=Y}G)fRSJj$>`;!8 z1CKLtG>fU1k+(&Bhe(ZWLzxoe|6>`Crb!Le7Au=M zscbmS=c0mZBG5((y zeFziwkQr3c(2{MNEzExbFWsmHZeoW2W67W8Xr(|&4efp1xb%R10#5;8c_NJgO* zRqvnk6XT^@i}LjeIvJ3qaWl-_Xd3H}$#X1Ng&aSz9{cw14?l~t^h4^$|EAtNzGCLT z{v2KY|LZKup`}E==wQN++u1d`!ZjMDdRz$TLK=^BtUt2TLefrzx3YWsAGrUZ@nCV% z$m>0;F&@x@83ob(Kw3AU_R%m;VaE(PEc1{WrTgJ2`$aP#{9~?wV>>Y)qNC|&dCb7& zIiI4{WXg$T>@GM834vKvws>G+wZ{PkQV?y`3`f?yH=gmMf1!HQi({+E4<(N-jZt&pAK;<7aG8c2ak5z&G{WOb4l0{shp}3o-{&ken6s+Pn@>%{Q}lR z%>L^V?6-cA3|T9oDOdN zxLEyJDlvhAR+TB68lbih(3a|E#&5;KHUQ!^#+$@?n5`sUhgx=bRhnF*bW(B_C72hv zbnoh%Z()Q?h2`LlqFm6P%x`ea^+9GU@4vxw3R$-*f?z7GBa^VUjRVvY6j<1qdx!J7 zI633cW~L#~^(+Vv>@QYEX>BVODAiF8di3!@HB`o5j-#pHp9Ov0xMnZGgZIyu6{VaF zkd=U)RjC2>5LWSpjs-A9;2IK0hwc(jMWBdUf?yec_so~9%kxR)^M@#%PzSWUo8V)3 zrDIGc$5zC!RkC?QaeKPt25>?(qsnU!3S#1KE!*uS@;q$Y8 zZII(K&XQ8x$GEh=-hUo6A|BlX>#Q`Q8}M=!ZXj$2gO)^5W3CQ zax#-QErDnTDOnH09vjm!f5>ejmh3tZq)Bqc;z(jHTajYC8*V~V!+aq1VXrsvgH=IV zcvZ2$GM4~(3wkExFVzm%RqhyMjqLz~FA-|H)y`MyJX&S&h(en8%XcH5Sz~hM*Tt@5 z6VhPBmDD?x%YC*jIIvA3UxG}SS}mi94Q^7T0`N$DK~b^m{<`j96;(Y z0ksPxR21DcKEw`D88YFzyra*A&c81#XwgDN6JiSCJoCRR(@4;DZ5(yzBEIoK z(kWXa4TLH0<#_j2i?%M!nr$1n-=xPSMp6ynY#*7Ks8x|Aa%y(zxQzf$?gu9C>V}O< z!Avr%rKVm718TRsoqv_QSERItI~58iGd7scU5+d3w^!6fVvBx z;|tPTge##Y!Z)Tk6K8%P%v~ncNSjw{#uxih0yWQmP+|Xp1ESkRe)XCP>#1m}X*cF# z`G?BiTJeKDw$!)(G0?dPgehF~^&_x~U8~-E|8~+eNYw+_kFnl2(}ShPPJNbaMF;|w zzF+xl5o^2iyN~X3L8F$?9@IykVN_uoXfoYGWvH=|Fi8kW>u^t&n2)R;yq6{8Cv=(( z$>MQfviBb1o5TlXp4&NI6gSZ2C35U0z^znjUHJMD&c>5A%2Ox zcSP>|u#vsdZ-^mk{_bfjRBSDQNMT_$aPPriK{MX^vrvW2FX_^ISC4%6*W$nOd&B4o zoU)~}DWCD1Tr#BpG59($hFtBuIa7k3_#%xZGA#3*?xJqC9#!2nT0!R(_^DHJuks_| zD~6VPv?1DV;TF0~j=I&c{DqIkK=sPpzIn5sX*qFVIWMBy$gD?613>S6P0NYcO==HI zJ&L9`(X9?xZ=I7R>vH&1h^aU7Y69AUZjZUzp&-liOfSee8&~a_I-}>ofLOeFYMK)l z;x+14hpv3j<6G?R?*opH;b?sGV8g;kXLuqbdlfL(p@gkV4#oi3fYF5J+GGi>Eg`)R z9Uzn0`0VA&dX}~{F`lD3-nIuNNPQ};))vm41tm6o9CYI#^k8etm~rFkqQ$J|>5>_o zS&B_PnENOj`k1IN)KrR23o4sOC$`KeG!6mk@aBEN15TD4u+=A|ziBy6?pGo;OmMyl zZI45VufM)Od&rDJds*TozW5^8HwS&n@J|pK{ah^H`Eclm3Na%X9aZI1d;8jG7I&^} z`WobYRY|9(sW52$Or-$wUywPi#K@=Q%5#HE%Q+yElE1-MW&0vuhPj=iDp+08@8k)0 z`v4QRBb6#sb?leUjEy6CrAFbvKh|Xb%FCHhu=6UysH0bh|D1CosJG3=LoC4%gaoSb z6$9BSkf+y%bz$NoD0e%HpoGmQK514Nl1mZ9oQUMtUB=Om&KyDI!ZKX5dGMlj3~72G z6&eu6qcd#hYUq#Bv(zGj&B@TR4x>}3oF7ly(dWov@s9|ihQj$YnfTnQbONEp5M%*W zx<_oevd6 zV+4Z4Y-Q_31wtyx`-*;Vn|GF!iW-AU9xbzA|B;h3A39NL5j39OsTQ}I3@#K4eB0es z{JmkVlUq09H8i1%>^%RUqWgZ#UDLEADoH6-k(%oK z(9r@7nR%jw=MdO3|E$*P^i?CCn&Z!GaC6bTqi;|s$u8mP)29XfBr*VU2_x(Xa#DMd z%rudkISu*d$d2B~t;>|H^E7&1ZPc*A0%!V$E~1MrloCOnS>rjQg`@Xd5jczj(TSl_VQ@+eCaTE@7}#qRW_VZixGaqL$|{0XW6XM|oyz@Xkw$Ur{=g(Molh$^ak&D=FUYjVqXDfX9Nge1Z0@uN?h}4&ul_Hhmi>*{rZm>@B?Q9K^iY~NZA82TPWyq}j>e5& z)LDY?Myr?fVyQK%++lK(MI@eDIb=Q>3)ueR1ylrb&!_Esb>Xhfg5^@dOb0oN(vu=V zB>2=OqF|LORUWojIN#I#my~ZBH7|d(~dy0pDlQ^6~A40omxj*YhL!5Oc>aM=`szQl2LYAdd^ zJW%bKGU`Gd6Px~q)gh-|ixpi`lX`>f(mZiyQu;EK^lfg%if$8*cqU=${?Y6cH--yb zRoaJzwesBp6+U+Oi&8|k`7*SV{|;t9yheG7ssysg16?{7ut5zk|FU%H6%Vc-&3&KM zqSOFQPmQWnc*dL`40DQYV6U>&N6X)d6m>CB3ack<;;h%RXVYce9*v28=PMsMbk^h| zPPvWB9#B@>)Nv$f-)Ai1xIW(9xgS-A%&Z2^lePLfkSIYLpA;#Q$U9!0xH;eT#kE_r zr7(C~{gS_O1|3I9;db%-pz!1ulOVh2O5;a5n0aThXOku`+boQ4u)r=Z8q?Q-rsZ5_ zDYFcd?vXXm-IGirxX2ex`9Q$yp-ad#uHN};dAwU46595m7iYYU!vtvf^c_6?m4EJER$OG*sp|FUxk_O*!M9?xGBh`ur310fp3vq*g3kDd+l@o+nb>%dvtveH0$uD@h% zCP*%axN75z?WPOF*f z6ZJ*D#MN`C0>YlN;8s!#l{RnUEMHLM8`ma2I4LidJ;>FkM-IQmllp#Z%S7}PkMDiW zS{zIt259XdfUu#(8jSA4VbqRQInOj8JgF@wPdm*r%kUyquFJ#Oc@;U-^?*r}N}J`@ zFo+-EoM*ODzo|P^l)oWltoCzvwVYq$GMx5~VWv{VGKK^DmH7PgzVowqEh{fxyl7fj zlhQ&Z>fGOctGnai^Iy4xg9rZtN{;(q&AoY4&wKa({VAovSeY`UNsi5_=>$>iHt^2?GkNdX1>#S>i z<q+x6XuPkEnu!Qg4^NX4}!ng;{>$l5Z zQ;4O#8R+1UN8!KtPKD2^9xEus2FK(wuW=lMtZW}&+w_$>atB>chgQ&Ap7Y?r11C$M z%Oaoao6KXhqxSNWL;v(qAY#V1yH^KFb9$k1 z785X1|DnHTT(2W+4!MQGKK%hRR*M9$o$Y;#Q62K@KmVWyV&?+0Oj}~#btg-B!%_;d zQ#YXCXGD?{{y0I)j^m~ZtP`KFt`AX3?nQF5CVBLV$rEl9jwsS12NmL4v(97ke0b#f zC4eNaUx7dS8YnG^mqs-wOW^0QFBU8+JiO>VRB4-g2i%NixeHGL-09mmqLtk;nNTRW zOJ<-v>0$ALVmjyH!-p;kx_Og2O+1_ad0bx}pw&h_r(HarHqQGn7ES}jkNM6x$mx;K z;4B|^SXxn>akJ|J1G?36pyQn^0T-n_k;OR&6}2);+9h+Tj6Pa-hYKG@shRo_6VOzD zj+NsNQ>KkF{92p22U}&CaUF!~K_Wi%6ML9Elzwcs*4WsG)!m;@eXIykc|JLSc?NIy zzM{}Dej87L*eWO}$SAj&>B!9UK;Oc813oqxlxgAQVz)K`>x@yZ>yDncm{w4-D_jE~ zZZCfCi64r91q&?T=1!T^tX}5~t@QcTGe|z2`-|Te-f}rCuktwj*b{%rLj-fo<}HeZ zcsU4qIDTA%h5`z%1&bH=>bav!mPgN=c#@+>i7k(?l&GLl%m~{j%Phm>;mJyX&&|dY zC?-CIvh*dQ`bJoo^y~Kouim&t_{C(Q2dvP=|3o93fzBI zvW0h+cJ=N73|G-NP_l~wiA*=%Mv64$zU(1i)FQ|D^ne1W3vKPF57`}G#F~$@ zw4aFa`a9tI7LKtzsS9rJ9x(3#J4Nq(`~JOX;vFQ81;L!#A`$y_H-rZbavwKkCO>8uW3G&E2uzX8E;IJH$xr{29E&CS^VZ8xdtY3ma3$TZ6E zeUP7}bb}(0zFTlvSK&0`<*B~wYa@o4SxkmnEBq>pI<^DMcF^&)gO}4hcmJH#>$2dT zB8{;*-*kLSv(G#Z*;F7tMIf+c-+~akH%s4VJS4dJT#r&&#P2fUcPe(gfil8 z*OhOzx#NGy4XfT*b*D{QPOB5lK{U$khxk%x8$<2D5TJ`NbLowBouZkMf5XXAD5cKN zPWW!>gpEBoc?}S2+#IazGpL5ALp)4n49~o+u^QK4-gU*+;|c-y`Nt(49UjJ9%Bf+r zqqt%_*D+H4@C^J#8AlROSp<`51q9p2NIJ=JCM*pf98(Sx36J7FHzz`uDw9en`0S)Za7k~T;>^XQ`T%6R+i(xuH*t-BzYGHJ=ZR{d*t zxCRYRmT=g|PLAX}1K0?0#7&JiF!fW(esb6IcmpTf+FxWAe@92n*% zQ|@Q3X~4Gkq1hnCh-e>dVRhU<0F7KdVj>tQMF|yg;<9SgZRo^e29g#XdB^XUBcVr|WjQZ_wx2WN7qn75b_>_DiETeyuf z)$!E1`g#GN#Iz$1N^};;m~Nw7IJBJN#Ev8;Pv}1Icda`qn z9JgMF&;%;?Zdlm4CS1*VAEn$rXk`=f+!zzDpP{AT+(`EHMEmf(N?*Zn=+tJf;U>(W zMH8#>g}CpnS&e&PYkS*e_p8r^J=>+`RDn}3k0F1D_dmoSJQ)DuOUq5d>wl_%7CnCh znJ-z&UtfUbt%1OmMx}*1Y%m}ZFLG&zX^1LMBIQT zb6a~$JrO|(AMHcm((B}~)yt_r>6lA*PD?pxQM>rUB>G@oyrBu zHcM&15RDKYrg&O7Zkw4;Wqgn?=eOX@@@-d^3|S`I9J!uh_@l?LFyo#$Z8=}lK{H}J zVY~`M#+FN0rhR)_WMYiRj(wP}+2-^-sd{9gJT&|p{Hy_sFw>93=Rr;NCyylkmtSnO#6z?@8KJ@(ya+*A>^~xl zM_7ns6`qL4-o7HQjlQjA&TH_DFkMf7EF!>x|IK#xUQpMs$@k%brN9piA^i`%+r(dO z$s-;}oh$~TJqjn%E6e(J$K5Lof%?*d%bd7q-Vm9E&pd(LBNMr9>p$|ItoDzJUA1r) zY=un8jKymzS4dXM=WW$~lC}uH4Y(~eIRsr#)NmLa@t+GBE=4S6Xn(`lwLBDPCU%==n3^HRC(hAp#$IYKVpPJ>hpH=s#>9k zn0sZNTP;>tcuUbkL~oRN_GGSeERg5|Tt9@0T6Va>Eh1L(px1VP$}+vtwAV_vJtgc7 zM$1c4cusn7qymSV6Vq?!7w>k<_A2-|=NRcbGC5Z7Yv=w3_GkO{sid3q@q^{dm%ruk zZv~)#?CF2Nz-=rxCr-q6e9j~}D!u(M*~y=o04#E+d^#T-_VG0Ez(0Od#vNh+=zZqa&A)L*@zsy^L~{6m+-mx!%aQKxQ?6KhWhx?> zp^lkkt4wP?*XHQQlATK}lY0@IMo~=47IS33aUAXkpo#az&ANDwCIy|QJIv5I{C2xM z*2562glpjX?-@fcWp^|-r{$>N>DOAW?QupYMl{Om@$wJ-2KxB;B>Jlfy-(N`8SigO z{Et~i1|5&ySM-Y_ppUzG_~#GVs%c_sdi{uOrksk9lLu#%nsSzBH}wIf(3SIaXG+mD zBaq{Dwc>r;Bay0OF5DmabJ8oO^_gddt#gMyn}Mi7-+=|%v5VUuys*|Wu6RC1bwZ10 zXvSoU!pOQDKIaT2bRNh>I~}FqP_z%S%G~$SzIji7Rf&R0lHxsUbVuQIk!XCs^)vJb zDXRotXl&83&WVRT+aaBH?|~yTA7Ii)Wu^u@p|x>7R3~|Jdoul#izt zFn+?*r(aydGl$Dhu*nw8WPR4p)C8T4gpbP;c@0${`%KL(BXk31tTp+qYOvKTn^Ag9$ zZ`AFP<76pI3MPKX&o||k{R*1=W;E{~H5;?RK1yaPbIzMC3uU8ya9eDi{)0WfocY2##xcV$_yx0Jio zU8G#vEwguaZdxdr-iC2_p#-??_%_3C+m8Vgv+o-q;RzCK)Q>s?Aw zD}?VjqS4;U;srrs9~?A~iOo~gse|9BY4*-Urarv%gqYTdXa3-WPxnWS&~=_hf}>=Y z0l{|%LO$POeMcUQ%=D*w-t%mAZmWEo#{-dB1mTg9)&vot4+W8qt>9rftrSwbnSXuo~n~2 zO_ig9s{xriNJ!(HSA6W11Y+<7H`-Hu$6>V*0!;!5&r)zE`{x(O%V}R z;^^%}CXDt0WX?Wird8Q}9L^jLLd&kRpN3r6ogG3m*bdV4!&i59KXbAKzNarwYqTW( z>YbhN$35s_Iw#gVs7j8!YtRgdVxI9fGRL^xJ6Uzmoo_Jw(8$%z9&;@jeq-E4vE@ThCQ(s39H_;Q7Kw*& zDbuxYaZB6jcCE}L#~<$K)3S@F54iF}Ukf=BbUr;rX3kBY^SF@6WYWk`&3y<=T%_?Q zg1E^6AVIuD(C_q74#PnrcwZ-PKaE9KgcK{KImrtDg7ZH$n$KO))SOL>TY0-e;fu;k zX8HQ0(8kiT;3wXIQk*1tn{AC}=d?1?Z^x4BL=gdDmdhojq|1yMnWjj37w6U{h55^Q!~E51U&P+YW!gaPSx-`9 z)h0_&uA>gg3KLlJilQbZ?aK41maOJJ=`DR|A~FAGCG4NrWS}El<_jSKj8A5&PV!_i z+aP_*gdAxMyuiS04M8SJ$Xb7LyD_-!)G<_vatjFwF{SE?N#^u-{JwN4Euf;?{@b~Q zLv80;zK}PL0yEGAfIy5XOx!gt=2I4lbpeGagEbPsro1D&D%_Vmg618E#uJ5BL|S4) zHFK+&5G9BHPmCpA3>5PYgSCgQ=V%PRhOpjg93{ zu!Oj{Et8cy_4Iq?6o|T8&C_AAQ-;cCDuRa~V=MlGan6@Dv|ieGSQ)3;FB`5y-?^G% zd-LqgplDf%PCX(sA^6m-V_qm7H0x9H6mteQIm%cjs3IMlJml-HRGx)Xy$Xl^W>j48CBC&%5PcqztkmF$|0Ep4HXh1s}Hzy+8&5$)*?q)@aF9yjlk+oMgSrO}S3c1l+A$VH`o)ZmwW5B49Q6A(S zSJ@PT@vgITHD6Ck)h&AsLZN2P-lI=a#-qssb#UuepL5)X34nvm#$)(vYm_$U0vXN? z)bCJq1hQ5J?~D40$G|Z4ay@t95#mBVjri)~`VE1DEes#hHFf5z<7a5Z_W2{Tc?HE% z2l^rc^yL3%2e&zzfSBr-4lbW$zHDvPJ_d3+7s`de9K;$khT0<4K{?9)YR8HT&?M5B z8uMNj?qQUb&Gfb3cQ#n!@#~Z4@u1oo+IMHh%vr}a6>27Nd@>PjMswyX^DEi8K7DM! z(d-Zc9}`oTBgzebJAd`)znP^ocGA+TM`q*b5$6P_Zx2+!pFrP#K7Eqq%bzdR{u&XJ zKL3S-ZwS;*(5-gToIuJf1g*0LJJ>2hv(q3+4RF(PqhrKagH^0Uu4txF#uul**gGvE zU#REeIs0Wt?b!aq@LIJxWOq0UIgVhy~%_n7kS%Hml^*)uT_ZmZ-% zJG=GV_c6RLZW8|3_-p)Z6jNXi?@#;IHFRAOuetO=)5>2A#)B*$E)zfarwq{8HNCyK zFF=*X7Jq%LJV&dDRC9okCnx5(hid>L#PiO(eBJnzQhmL_-1)k31!!BfvwN}RPezi+ z9KENSn%e$q*B`wcJM^wS zAa*o5F~*%l&dwnsVeNP05$A(Cnw)2c#=pv~jC_(ZI7?U2i7KeIOswEH-$QOV!dRzY zzCVO5puQV6Qty)CaHt?U$lGdnj&dg+OQ^2`K+2+2X&J&kw)jp7Zk(eisAiUy|LoG4 zuSmnhD1%bCPT35XYMYaC?xz2p?bbuVDGwv%i8?K7|j`IE=JbQtxyB8V?kK`_&@$ABsbsK<6jW?|Np-g(6=CL|f%zW|= z%3v%2wOHF?7EPdk{1u%*ytj`6)z}ZIXtGTZCkCNb&H)2B*%`B1YH3XtA<9@6mylEw zQhYs_S9JU;1UjIKtC|tm+U|JU0V|)(O&ELFBY(oBmA3ZhgfnNfTpbyFRirRcoj$Y1 z#TvJl;9Yok^DZNloVMVk5%bh|@259!-aLK!G|~xTQtk&|%zgnuJpl6lUiaMfGs*a3CyG&<)kAcWET;mW6* zKxa62AF}08GK?Q;5kdE{N_ z!C8457*yMYdRG9&q6EVIIHo79F;a8CCc2ohF4i%YAI&CHf4BSflY5dg?Sz1Be39N=4f!o3j$+HrvqdbY(*mUzwQ&rf&iV%*{RUhM(8`nkMbcO%;MEoAN2 zt+NXRG@ScFgS;@wg%+Zb@FaxI`z@Vva071VDstJP<1OeE`(aHF3j<^0Y?th4A4c6z zhBTOlYkErHl1;5F4BwAXIIX2Y0nHAMe;OQ>2?a`92xpWvzQ&f<@gwrcl|!i*b`rfm zVX)${QY+=2?X)AFl2K|?9EH>GK41MqsqGB3nZh3nNnLE>_0A)U&z-pA&bA&M$NkE# zy@d#WGfS(U(>!V;S2AgYaY&XB~ZgeJ!9W^3N~zKluL2 zl=_JyoZ(^8b5cf{9yfQDp>M23im5vH{>>XRo~Ze+OAk2ZKS4S5zjX>bz9lBcuz&4O ze*yM2J5%#6T~rkx z;pzYOc1Zl$18(c!fzwbGQ*fXp@^lM^T`xMbts^~2@!Pj+8W|-`c=mIk>V`oYRk=)f z4qiA}CiCU9tZcn`gN8Eq3#au_w_zPN=Y|kwb3SM|?v_;if-6UZ|CoPSCBw4|d~M1( zpB@)kCo#y*PBCCl;)nu#Nh*rTGiRD%tT>ea^-7Zg>WZR2e@@Dn88>>>m0%S!SIUUw z#Dwf@$JCX42F#&JOiYZWZ4R$qXliPzLSKw+?;Z3KgJn70j3H5yi!Z`=YbX+NL=@Q+ za_W*D^b8ZfPJNQ+HFatSu}|ecr0#KT&@k2|C7xmG=}xsZ?b4qFhV@@$7U$Be+oTDu zyZVh2-90H(gFly<91tHLKmF&$$3;amo>mU5Iw-|j`sQmWp4QcbHgka8*L?2-qs_|e z_iSN2a&iy$z?Pe+OyrIl;f=HWb$;N%(d)Um^7?mp{tHs$`#V$kG*yIb+qSJYTd4={ zhW1hDBMrr-dQbg%>lsy-UX1%nZ#U&YOWr3F1y=0A(?55Lb8%y{Qa=2JYgy+wmj$3$ z3x1hat5&AG^at;pcKlvSC}=Tl+O(J0X2!KJIbUl(#?SA#EM$vk<(mhMMN6t)M;j#S zOudOCT~%QL+)cfR!6+Q_*Lv{8dgfbdVogZBtZMMroO3%CZ0*uO`#@Umqq(QJwXNSH z6*uP&w%|v%eE3rJ&p+Cn>w*1{oLs680vxWWr~)l&=*qsK@cRYl3hgm0sZfJ6ZFGte_X27AuDTB zjWnkQtd4MvokvmBC2(ZixXHop2k$Sb%IAu*k!-KRXI&*=FKTgJJJ8Hh?{Vde+nz2d z?&$gM4Izk0B*UucM-j7Kzm-2d-nK#?pJK1vT-xB2SH)Y;+A?@Km(vQrQu>DAxVX4) z2wB!Oe<9Eg?`>n_GOGdTsPpS@jzux?@usP3%FP;HWP*%BpUI%xQ%hvYjB9*%gG=5q zyG-q>G+1k#MSU?6_1BzLVuzk(XR9C&*i)SCyVl*p!X6&rE^azolMXk6$N)}5hYnn_ zq&HbMgi<1r>XjwL+I-nan(n-i5>i+|06Y#( z3Z%O3J^#5Pq-k$#RQFi}?g#0h!q7hV@z6NO;J{D}ygA^iItiw`)tLvJo15$Y7|rP} zyRX{P?>BSkh!NL#-R2`rGrIF~R+bXLlfLWrX?U6WdPbxnIrJM-R3aOGq_&#*~v+3XEt&D(1Sl1bEUZM_dYWxr#{Y* zyJH;83^H8mY5GL_DsE41v{bM1u-wmfOyv^>W? z(M82s&_L^2cAF$Or}cXu-H+kNr}Eih;o+;dZf#k= zaoYnA{$MK3b)Vzr&+c403h0{-3)X+XabR>C8-Z-~Hi6&Qjr%vz$KN>mB0nQ&(j_IJj8;h$V39Ed{$f2 zl2acKHl%>iFi^qAwF%x5?sN*jc;vy+t@`*G^D)T_64>&SroUPH;zrsoz1U@2u!6=;6*aXJ`m6iTKq+$opjyyP#Q}TU z_R7~!joH}DVcxAag)H109<)JPqXiLqTHg6tEOxN$;z3G=P zUz~;yA54Lk&Rily;saxk>NV3;eOXchwKF;Mm1CdHq~bFZ8jl(^YUeMIIfIWfF~C_B z?wH@CelxUYTt_C4%zOj+d2 z;#0GWg2|-y6dce6yy?`P4@X*r`@IK9%=h=-9N@a-8N#jghXXKubr1B-gU;`soY4W+ zce#GRaoGY%x`AH2nFs!Js0H(O!#T&#F%!1l9Rv~Dipp#Ch7Ao+U%d}pcRUmG;Kx!$ zT3XsF-Xh_?@-d7J!rJtQce`zxEw^mH+wsbc8)o-{Rh~ax=cyT5sxt(9984@vf;02{ z&oRePoRu4LzI;}s(L#JsUrE7RvZ)s_1?*%Va0NxvM>Uz`uL8En}ckkaH zBLJrMiqYM0^=u2nFaV8_k?VFVjm&dh2II4A+NRA0arQPdR@gDK!pp}eq^fG6v9WP0 z!>(>eFwC2CL3Uq%d=hY%LlRyh4cY3R<*csUFx_0MI6Q}?s1!je@Z`Zm49Tk~Cj|NsoZY#e%YNT=S;6cxdGM+*M zUPsJF{{#*<$VpT&vx?cg*;?=Yd^kWqAt@*9spWV{9OFOg8fO|K!@93>7<3_m0=f+KOCmt_2vCPX?Mj8JQpU&($BjtPmjT zQ_6k=5WBIrnWR5tS?m=Yc8wBZO&4R^T>~%Y#X3Y&w?vg)1A-dU&yz|}Ur;dj?1a91 zF2C}gr3O7efByVT$S)hSl z^+WD18E}d5^ztQn%?6>8of+lu`)>dBNA`+K`dRnx=}>gv{^y8!;)tD`ens62p{fDi z%jBjlpq1d!b{gS+(|E^m z4AduXZZ2{bf78d+lp1#{QmjAw91*ey0!?CYbcdM z)~zey4!wSOv^{>2%KS69j)S<=Telj)O|IB9&~fRP$!?c_sA`}I$6HQaZ&0H3_cpE_ zj20?|OK=EoWAKDwQ%y#zTOinT-Kw~yY6Ov1KR4h3H`@~@hr|C_D-ojED^GVM5H^3uQp zwBz!KL_elazPzrY>{&ObR=5qS&W#9-)i*O{S(hU}>WGam?ZgQkMF@jAK~FsvS|zTS z?CaZ+{MJ<==UU#d$QoSj;EG1E8%=}y^zN;$NaREt;$o540OOd0qNyUA#-nMk!yJqJ zXr`Myd$zeE49Y3|`t|GP*Jx$;l=;8a6TengHe~{JOEg(6Kut4+BSg_4KR-XEUn-78 zry!3apFM@P3B#7xKyt9d`M@7NZ+;&>eE8*}qH)rfgAg=a(G-5$uv4dlt5zx-n!SPU zqAFwN?Rx&H`k4t`6=Bp11BV#hfIrbvUL^rVp)U{`mBlKjUtslpyYjGq9wM`ax|-195ekEM(o|f&al_^9*WIs2eWpR$>7hHnulNjt^Z{I$CfikAStL?N8IMJiok%w*aKI%E-z-uSRWhZDe zgZr6bThG;G^JHp&`Sq(~w5INEs*Iypy-_2oyDrq~Bk8k;7HW&gacki&&(s_c!bftk zk%57I(owK6ijUg6ckim&?KU04VO1qIn#Q?Epe#A_PAo7r-%6U`nm4@p>?zHOnR$5) z%gf837_1JfG6b1GJ&DY_fC;=GqU;83^h_O$lSiw0^A7K$GE?6kG;IULBx^bz%=PwO zL)YfcbH*e0f~nR(SsH>J3kwUwHf>tBV@DfNy@5-a{`#fr4!#~s|=wG}{4Zm6-b3I}D&MPI6^K{#n0 zNlC|O;ie*v?iiAZnAzR;c3a~P9qhYqO@e6fNT04HCXClh9E&U=L->LEcv`b~sQG6X z%{ss0yXE*$4FyrH-n(a)+JCKhHUuMBC7%C1l!W)2;FWfJ8C12b!_j5gioR{bNhBfCVWcqcWvzJT=aT+l~O#xe_pxz7Kym_nY3z3L>a}ZE{`yM~gbkcnyJk&2 zf`IHh?_f{gCEqFyqH7&%UHMEaTKi_NeWX_DUv-5N|I}p6~9oun_aWMHO=ZsHINy*|%(?%TZYI21; z8XXlSNe=$0dBHd4qdbSj084ex-ekWlJ5hyq#xQY3)v?PVWG?zl8jcw=roO9ltiQnG zW5+C3Eq-j-ljp#pcHs3BD2dBLRYBFU4zz1zW7C>1wiY*ufwzuaq*kC`|1t6NEB>+- zU+^k2@(1bzO84lCo5I3Y9Xiw*QtJ8Ne}@vdGYbl|74yj+pi;Z8TT9@QP5<*}?A|G( zlf1AMNLtmsSO4K{jY}bs@Ji@jcpOZDhfG)xZl{G z8JNIeP%|fmV*L2=@au{tUze!Kmo+pjcty_T6Ekn$Zj5Kx4uv@?CPrC7ARWYg>yFr3 zZ@=-4ZC@K`EB0{A<}Y4+ZIx2JG$!1vXCnCb3v zGuPgW-ZbN~2=R@wuU#{;wtjHeBK#j8iq+o!r447#nUhg%*@DINZ8dFL)ZHz&y?YLI ze*v&1@wd~~(b1v;jdPj$I^OZjGy)A;hBqv2RIX3!K%V!KFYj~}N80@yU-_|-qx@O+ zPfb!=%lm)+rvXa(Ku~<6_z0&U6!C6+=RNp$A+NMfc^TJz@f(+aZ1TG}$4k+gDuR9lpw$#!29*$l_s72tCHJdgyQF!6IGI!Fi z`@m{-GKhf5>Rr5t_vcX0!uG70(DfS9BT+z8eS@~#ny@O~pZb#~O#-x0Xc}_10@=Z- zJL%#I4Y*I&gx>jgqw9V!tp8FP_gj%i6vV-NlMwOn=fg|@XaU>}NXlzB4pI#nr1AUf zOYw_Nf?%*lA_0-6G2%)HB@_2DAL^b0Z1aZcbwBLbUNRSp78-|#hs$LQRbLG_%({A2 zRR)_=K{8M#qV9ET-CRF(Z#~{^Ev@E*$Xb$JBy6uwfAz{oqAQyI6}hq#z9R+>sk;}3 zX3dDsVH}M_f@j^wn{0joJ<)sdhqlDbP!4wzK5e;tSCf#YYa83kRgG>&06xF+S8G83 z+Kmp$V?hg!WL1N${yqBFeQ;ue5=X$#BUxsIw2PaoG^hi0SS z=coA~%1w#VGC(K4rmB1tP_!9lO&Rf_AJToKgV`I5D3LfepOeVxiVEFZ^80q?t4T5j zPU!=?5B9i5{DH3sD@5_HrL9|jzN(y?GibBFBiQMju zzn>qOT!*<%-UpV*&HcWYTyOv|O0_yH1J_*@NJj>?+gT*|{zVVE_ z$bbhXU9ic^%M+~IP}fq0E>b%}pTXu_Fjv>eKhnIEpvYu0)Tbr*od}$u2G3o`38D=w zw>5+$9=vd&6_|3B0Nnfcn_(Tu#20G$RIifzR{7HxsNco}c#J z`QYd&6v)$y(>onbO+|-U52&32=nKQ5-ul^p&!H0Oxa<(qGog+7rfeng^7ALuOOwh^-a?U!?{zZ1eDU_}25Pb_fILF_HBd{` zu3e4!5HQm8h=&=wD*N~7(L;kz$*x zNxQONi|P#-G92Jz0jTIZ$!;dxxzzufj)K_1Eww0Q@o^vTzVre4#yugV^)^I_!9mM66dz2Qbsx-hKLLD5fBc8#0}Jyg$kt zqcE;lZNF1kIEI>HR-TNhy#4vbSW1Ra{-m8eSr{*SdgOsZdQ#7CEt`#mp}eZ{^Jh3_ zhq6V#u@M`6lgAe)}^9=AT#YG+0YMjY9&zSh@U97GN6cRx z5~3n6KK7H#e7aT6_HEnj)s?6(6q%S<^h2U9;RW@hK)72yh;J>Cyl~$z90w$UZR1S=2FZ$EFOT zEm#4MComANTO{6ZmvgqI|KX+q4pn(M)`o?t8+Ytj0*hT)RTTnjK$Nh`K{L}^*QEVm zC_4)*bLGpv@B<<&aaEtXjGctEx`KXkg5Oo-T^{`Eae|H<-NsC>s4?7vLQybR%Dz9^TsgDnC zkj+OOef!UpVdN48Dc@xe>%b86D`CjPBG6 z9(~E(I<${91$-Agij8eHdG6e&xv==ewOcb3A7APVJua z{(`)NfP4FHQ+T)*?3iTiq^HHjo{FYOP4~B-U}Tt*9SPTQCbKlP6^`i3=@M44mQ9S@ zsm-#i-eHIPo7|8um6-RtfUsQvUyMhXK|33`zf-6Um(|x&&a~zKI5KLA@}e+UsMCIbYx4dQGLjb6&|IQTV7U3|PV|o1 z6`~qRX;plQXQIfYvs-a~`9-NFc|aDj*^r%si2i5sG*STa!EyBy(P!$VeT!7njzuJ8@${kID+dBy{8S)ON95 zC$^rM;aMi%M`pVf5124QspeF`o>_rT5J!tE{yI~m>lefF57QoXr9eRsNO4Mcxn$(u z(~B$gXjqa!hoP{Zg8uT#(pzm5t6Wlo`9IIpwVtV4PTsj0Wzm)Q63!VCHv~|7+rO`k zO=+JXu6o2nwQ@yr9)G8Z;sr+#;@Bfjxm29_$E+lAy!ze+HBIO=_`c{uS1?#kPEG-G zE90)}D`pJ6+GJ|su8Bpr>Z&0}$7=6s)9U{gX5_qtMMOw7q@}AHPEf6P{ilspiDy-y z?!&zwKYmnLUJ6J*b!uI7wA-#vWhRe*96uKz1)=_rx9yhRiW^(;EG+s#@#-V~$k|sQ zmJj{9@=)b&MX~*G$y8GZPkyZ+?AG?g{XzU-&`o!q8i+MLJ=W~+KQ1g&9Xj;pCcTc{ zw>eRJMw=~aVY1>|TRTO;l!4P)KIZMTxwCom>@$<6PE|z-rz6*>Q>XHzvQ7skY~wGL z6ib#bXR<@PWqmhgWLPT%#rpkTmR9zN~i zMmVEQx0}SB@a&8Fyb8z0>Be`4oqTfrQMlW~W$U`C#=Z3}Yoe)n86WKD0KFsanrh|m z-&lvHbu)=`N_Kx*^|G{dAnR0qc|V>J3zp-rL0IE;>PnpDbwB(dh!I7QZ!7-4)NLnr7WVD`hCU;94jx#=HZx3oR0M?&O^e!EN*cE&L1{{kS_dI)h;yP-r%tC460kLQN19ix!g+1w%hK<0s)MTpL4H} zu+7LBp%PM%-eXD|>h)`-=)SCKx+uQ_B?w(o#PB%Z=51MEe;!hw6j@)gDw-0*cbmXr z>|AXBimP8iQ>tPWT%0#n0|{A3P7vgw+@3JnA+2`F7F%^C^5^X4^CfPglo zPwe=+`U-3f8Ok(_P#P_sLq(p3G~mfJZQp(qj4ZT;8a0Y?<)sWMk{h<~-R>|jW%`I1ETk!NE2Jb{rjV77gZxB51|L6LBFxn zSc|iogdrnhg>_>j!)j*Y*dHAI5|k;pM+!Ik`AWG*lIbS|iXD1W-MB;yg@R z1GHGT<3EWj;*3Ra-@bi+L*BY^+WaTB0$$os;q(3Pckj%p=u<2z#Q$Zv&HUz4oGwxS zs}z7`hX1V;W+m)L%d;s%^B+r_n5N<-IAZCIlb~6gSFNpG%4S)ybxz4nrogs-51D@J zpSG}Ew?-W~OiA%Msdla+nWDF&g90$i6%6@HbAhCgF1J$t27<|{fP5DE8z;)>fzz>x?e4PJ-#(2LPmZ8 z`n{geK04PYkrRME#{7t!1oGC5$i?&LUmqoiXX5Emuusu?LZfk}1_5#1ASIBi+mabf zN9}7UY#BOi0v57mEvBsHw}~$Xw}$B-7yG%ox_Zr;wE^TRZXSS)RYF2S@<*bT=raHb zH`o~rh?!taaZibx49oD&@AT`00r`{Yvg?j3iYlMe8ICPvW6-}N8jh{75*iTu)!EWd#F!MM4>kASD3l1O|F`BnNb=EpG zf<094m{D8(UkC}B@CMYWWlK#=Ohn5PN{GOh1XkgT4aU_&H~pxe{2Ug9{PG;mj!Y(XJA+=)HfGtDrjZr)Atj15vyyGXz~lB?wrl~y4x~_cQC=R-iSS}` z`_ld+Abmd~If;(WQ76l0Lz9G3rzG(Kiu8w#0nh6QS$h~wSR?uqntNB;fAn(A550$H3&Ci&UHW27zr zI#)$SrIn50znq)yNZe{--;plkG;k5}4aS*uWa|=KMz5JOw%15oEB{iwtGYqMcGOP) ww?FIke*enfH}Us>|9{5!{(pXHS8MB;O>x$YXcW6pRwy_R8t!<+!7KRx0JQ@)v;Y7A literal 0 HcmV?d00001 diff --git a/benchmark/linear_attention/results/kda/gb200/kda_b2_flops.png b/benchmark/linear_attention/results/kda/gb200/kda_b2_flops.png new file mode 100644 index 0000000000000000000000000000000000000000..752d39291b54409d4dc8934dbca95b1b6d084195 GIT binary patch literal 92041 zcmeFZcRbhq`#$`l6fKgXQb+?ysT5_8l!lQKiL8dbv#H3)NEs<3BqT&3vLX}>B9s*} z3fX(z$5Yqm`hC9l`0xJXe*7M{$8~je@%Dbdp6fi1^Ei(4>8*51e&w-g(8>ZYb`JlV%>_Me%w0kDmoD4hk8ti}_hyoI^ZK7( zJW;KOI%;@HtYf_{qtKobHl|Dsck^ z^RtY%Zrv&$3tP{|_DI&7`T5J2vT+K*N?E2C&S_|L3TIY??k;X<*hp2ryn6bjS&Mj? zvwlcO$Su0%>#%pP{qeoEa^vqB0Re$k!<`=&QNA05e2y`Ed-u!bu=x+lv?bpUV~JHu z)v`~CSIjB38|hYlrkTy0=Qu8qd5q!S_LDcVvJS4|5Px>Iv#ZN<-{nsUyF!m{-Lr4s z&6_tDZ4feae|jd#uuuCuNB=#AxQSLzj{PAX9v-bl9?R)ht-8guMXl_VgTtXukz%)( zGH(9y@!>&pTiYWym#kFEwlwe?t^V?5k6E*5uH7K*&a=06pd##F!%JVB# zTPVr8lnbqY{%Ac)>)3%6aGjsy_!#<4M2yvN4LiGU#NlhS)QXiW1s%tFZ_umlD6yC5 z?C3x563+3b>9r?Tj_c5&4b;WUmpzxP-0%p?m;F6*+`z^!`q+sRD+DxOxg{j z)qj82+|p9I`pHS#Bq$MNV6Zh3zeO17gsIHNY(bv_?&#!R{U`T4!F?R%gx-RtsT|J5xAjO#h~ z?%iwcpKQ{QSw8kPN$qpB?J};n4-c9J&F<39*X*)o0jU|?VjCam1*Y-dzB`{BJ|C-^Vqs0ImK&}etShYA!G7k~X8_xLgWjvYI6 zu3X{4oW&1M`NaHZ&6@KZms7eQ?hDd=DG*=5A+8y;Q|(kw{!{fAPX-mv=+^{gZiDk{*4(4@7dI*~k?0&< zSRZWFT^kM|W!pNouuX@okM6(vWotu$vg7!lul3>*bDuiu?5|;I zipt9slJnx?;_6>n=v~j4zmsyFO#aAq?rltne;xNQrWn04f4jLE5%x`+EZW{O;E-QP z*824&?#vO5qY4VrnwsqR;FYefu9`X4450!SiXI3VJvXWoP({3`7{$qK8~IgDHH$Gg z&(Aq)<~yzT+j#I(%gUPT85u&IAHo(YulABE-qK9V(4gTX|WgMPcNJCrp z8p_|_|4Qw%^-fMsOdB^QRcF#j9yzikf1+)9@Ar)5)BqM2OSP!Bw$^U2jc#UU#(ZwZ zURF*n+G%o#&*bEZ6MIa)?>`Z7Xzh#3pG)Uvr){*%l!G~@1DBQIL~-4F&lz|WTWHaK z<9ep;+qcuNSn+nSz06|#&xMyJ4eOS!Vwc23jBFBEYhf?KYB|W9z4+Rn#y7vJB( zlFM(lqXUO8{2KoMcA8!pEyZ`{sk>=Yj?|XG9|Nt$^b8DdyjFAHK};0XEB)y>%{Z9j zfo+hW^h<>I2Kh_R82aBuq=o|cxzCSv&*LsQ=Q+J5+nxw$zL<(`+9 zcl7x2it$ma$L@^BwLW`v`@_OL#Q=GRLIUXqB`+|A{-BsL9!ygY03=Mrbr!5ew zhZ9e^1#-(5H8*cYDtU|9AH>cpDk|EN^)Ps+g@whrOyl*`#qTe#^2Kf=6FV7H+xzVW zrq9dGjnd4uTaDKsb4WcrU}F8iN?jndz4PP4S1z-MfCDZX7huI>$Nsz(MDAsX_wl`L5W1rh?<`M#{XwF=+7q2WH_TRADD@;2(; z?AXMho^eyo+AOn{CDh`@i!c6Z%;vMXwPHOjb?9sL{;ATQy42S8_M1BtBRR#yHm&7T zm2Ai~!TKawZ2X@P0@0@^@pSyaK <{k^J5XICQed4`3lr)0{lZvUSsWrH4#8%eK? z*;ZbDe#pOch%!~_2FKYpZtOCe5lM3xHKYm)uP@TMa(_BNtJssV>{^SnCvw^1B}+PU zqQ<{{`_?-AS>m(Jw0yv3-mYxzTsuK*g_JZcgwc#KJl}03k2b8_xh&I7E;EB=nq{YJ ze&ui!7Z+Q!m#%d8^yKE^qA@5Bl0t6Z|0yG)di?Yy1_lOU$1(oR5-z^j9l|0aZ$5rp zDP&ml&i4GWwpo>W0j*rGsnH%I{-pC+0_%2b`R>l2pslK^dSy4bMI*!T%)8Z{r%rwD zSSBL!r+LS{_vT~0>W%r%8!fZhKYjXir6pgCf7dP>pGmw$c|jL2507@fXk$}TYj=0T zX-DUZ0ITR5Oti5_{G?S>m~(S;2POvFo&1D$+BXHr9n+f6?--piM@*w6Z?D@`87;Fk zL;QP|86)Bpwj?`()cZ5Oh)BInuUXG$8q-iG9_&A6@0a2Y+;jHXx$EN2Q#|s_?-F~F zZboe`WLs{#d-v|eOP6k)e0bo^+qbgnmyC^lS8hD;$7&}bMw|BClf{| z&A%sN@>GlZYJv$$Je~xu$~0>czOU-HT~gsPj$E7(m8M5i}RD>5R>oKj!M%)ABXAof6Lk}BWY z>Uc#~%WTb_L#HA|dBtZ(PWxF^9k@~{*;wGJt@j$?f2aHlyZLxP8r{LRk_`<8{?kb0 zF;}af@R{JOJRRrt zC&Hq_=Xz2L+%qyVG^TNcZUdio8J=fhQM^3%r|EpV(fW1kwjZ+XO-PHf>?rpJG%+#` zZJ4kLzI%7eiIXSkD2v}Uyvd{1&KeqnBR||3)>t-q3Rn(p{iUYu9K2icZmL$E2E_zC zoBc9O_UP*KsA^Lu~Tv308=U z!GgARRD^!Z7oQ5z?vm!3JIrC*T`5IB*kYkliD+yfv5r+_69doAZxI|Uij7&OO&UfV zK0Px?K^=CpIa+!=nK)k}@?}q+1g;or?++K~`S{!BjOP4!Po1I>2M!D~CHwX1nc=*= zB2V*$^58KeJMVP_OPoka*~A!dU=;}4j8|MBg`UY{G%c|>?(_jhfo%i}AP$J7KAu7wt%eNzT0GV!haL~*j z5iC}>#D~x1T7R>jN}NLU>G)IT*4DCNdkvU2ZSn)8XnTKe2SWG~{k`jfpyP!8Aa-ow z=UJ90zo&!0i21B&jAc118;Ev~*QrOA7~rzcd!O<~LQDy?Ma&rx$*+ zsIHEcsst$9Z}Od`w6yeci&<$h1a&o zovV>1e)RSEb$4Gmr>-82Flc`5+ULwDpzTEzDoCcyo3)H1{XZr4j5TYA(;q^g58C1|^I}%%TH`L1B)c-JDNlP&x1l#)c(E!H@8X0_eiTRI08WPNU zrYDh7NF2pCFPfVApNcpXjg`Gq(P1|=!fV?2>N<`%H!4Q}NxGq&j?kGythYQobg*E1 z4jwE65?*%5IC!+vR>I|3`e6wGes+Mv!fdN9bDUvd#j|lUK*s}vgEUlf-46Atjt%YD z8Uy3wYAMC<-W~0lvss^U;$E&+s{UwBzVr0DH*elhZb-^TeX(-4*G(Xd4E%VrXzSLk zJgF~+zl*;J=vY8*JOzS(PX}w~=cs`Gg89wVo{NQ)@wfDC@&~7_NdWEWM@f6;xPsl} z(6+hRac%19!Ts|7?B3cglXSpwWjB{HDq*uZ`R$Y-#b){ot2n{dKvpkjByBP{zX=jj8b0UQBXx~DH&USHdtWBrTv z^y$;V9ecx8+-BWfn$_ZRdvdr-4;2y5g`AhAuk>_v&9SdUodv@m&Yn$F+4dpgFl%&l zG^Rwp?|a7ATUXR7Ga_SBQd4&$7o=VOym_Se+v&I&jPc0)+-#af`;id@jdrHDic#Wn z@uwo5<0U~n%F@|0$o&HY1Cl6HGff(9A!KyeoloC~!U(w;B`7;m$&L8<`1Ia7ZCAPR zKTQ_Mrq~4xsGlnBs)geRd&)n5W-NTY+d1Rp%DZ8j0&4b~+?h!IVA`DPjUXeCMlyeroj`e z!^sZ}3L;Q@#b!}qZQUBr++I?0v;)j-h*w0wlkEBwv6_j=4eZ55gha}6&IQ|S9gqUR zCsOFyi@o(}hNqiJjkIdjstpaSHPhR+Ze84vW%gP5;RmDu5AYdXIeFU=rj}A*NPh3{ zbul#<&!=nNc29rX?yjLm_G`;dKRF>~Y|Q^GIoV?9=dqe3HN`lR>VmXs^ZNk>@E~qD zWEgm2i0^iHw}1e~@*v)O9hn8G&jaFSL|L&B)&bp|OV{_-&^3@MuW+f6Q+9IgI%C`A ziLtZ;=NBR}SLHi9f?HaGoklxgYUqhb`KGdxsbF?&BZATk_EhaqBj4q$0z9sto$?~o zCN1(}PM%He0Y2k$r|#H%ald{0w!+2|>(%xqQQthR7gB?2!iccKB9yPnX)*U*P~-4; zuA4=;7h2^Re#Y@aycv@;f&4U*E7*mVl6yZ(%!6kL<~vhUuJQ5;7v&YU@AB{5ln=`s zrfzAw&Ysbm#zXP;}XxV{M>n? zM*NPA|5VM^-P0!BmC=V*iak4?hjao8VmXcE%s?@tMvpTx2uO$Kyw~NHCFXy&9S>mv z^4`#3+xPuUkNM{u)7zQ9HX?wz1DjB9W8`k{&-C;4Rn7Oj^f@vjb@1}#iGfyHH{>dl zkWrRmuFIpR)Af&~=@c(X?%Hr+isD4I5n|Gqv&wAt=leAV1*1L45AG2W5o}@(FDE)5 zn#l%i7IVLT{W`^|5X={6qp{Wl|7Dn#xOCV>sEasFJWub(2!0y1EX&C^Yniuqow}79S@9ya?^T7< zDy=Njh|)~tNtM#sIg3FAtV@S;olRNB(t@suuwmDR}u4yPw2C(?+Gl z^Ode$>5dniot%O?@R47>e@0CiG`d&OB(QF*f9Ie7S!Y=u69U6)5%2ojZ@tKDkeO5fg1_0DxSCkp0}CV%>)*365L0p*nPf3-`_I1F@p zr71^-qWbnI<2cImd5Flk`g+r&JXX;oS=qAd4C_i_J>D)yCJr7 ziuTS63z2A(Ou@|1?b7s_L!n(GuZBlQJv0S8)6xXNIX+JN49HOY_3K*fU_%3+g3q8e zim}IS9yCj`dN}iUlx6SPzI_QI7+yewvgYE&i+6e^toTx=m09I}%|@u@L~{JJ6aj^N zEADtGErs}b3pq`aFSrd;B7E%+J4%C_sM>S@c14=IP~+&&_1>}9fA|4=gV3>l2H=d5&7KAK~PvAaq=zKuzw+`fknU@$V0k`(xiJ%Fhhwt%c}9v{0$fFrVK3_!}{ z=JQ%tHf|6d7X6u+m{{JjX3d(_8=%n8uUuJzWuTD^QyQ2npay!s?F-IF7W4x#p?`R> zg_BcmZF=LUv?QgVu?AEXG*oA2XZ_2oYp9Q@m#wY$Q$H|?8@g)0X70W|KX$p(IZPX{ zjG|7RIt8p-r2*LKfNhqQJ~zFL&!xoTanDNoMAfG@^y8?kA7Q$L?T5Bvf!qNdI$s6M zgiJokQ{^kkvFWMt_rsFuVOGyQzes~9bMyK0J*alC_9|A5;X_4y^rpx9G{#M|>g($- zw!QThF%1o!y$?Q0{lz66WXW%<>Nq$!qS^~Yq@<*%X6LG)hQ+}*9132&dd2cR?O`xz zFA)2ik#G{U3&iKKqWpt{+F+c86PpOV7EIY8jKV;>bEZ($UtUq_;XVw)QIbrW=SacSui!f5&H0@`s8 zTTG1go%cH(egEOZq^-XC!BpHGj=G&;zov2?goW|CDjq*x{aw3r_*~}!dz1P6My)Ku z!a9HWt0@I7hq!>0P~y9O-wy-}lsoOhzGs`q#Vc1mo+TP}{&2RUS5$kj3&JhmB z+`>ZY{oQRk`udBh`ZT?z;3?^|HB&MaK{~US%;v|uY+*^g3MiOAKj#?nJ=cEtNMA)r z_@9cIm|Z$u&OSpQ%49Cltu1I`^@$_)tL@T7v0)0>z+UMCZj{r-qIT?u{yKu94;kKLc?XK{G0 z+<0q2s(7lJc!R_PkY1=}K}~txzFmyCMZ;M!9CW}(uj$(bQpX*2ptl@B4YV3Z z37fJF^zX*8qXh=jhr{zV`>$_)qaN^VF`P2!sT zsoy(iCfWlg`m$OEJ|1k*pQM|Zw$3hn7cM(C^6Rwl;loK^1rHunJGSQ?AB_U5NS@aF z_dO3L&D1nC!)wL0^Bm%4#H_Uc?>5D8;0+-XEu5#UCnhIJttsp3I*S)yHkMr*@VvKWro9X+8Um^AxnHy0OIJs@LB!}#y7I~8Sem(tNC zItkihQFMppPHTPYuowBm;(X=!gvz_$?0r9fxAt#X5b1o)HF3BrO$o>7WwD5 zwAcq%56c`q>NPZK{Pk&Ytph+zDd4Qv|O z_Iz&TsKd7SR3jiws0WuU*Z=vjvx5fiZf-d`4&@U*DE?;o<^ z!%6o<>Dh`opS*kjIpHeZP*97Ci3L}nP`T~yz8EF74)`bau{59*01sVkbTAA@BK5wo z9zTA(zM+8|x*h7|kCx5GCsDG~{}4K_sOSgA z#*RP|s67;xrQHy^67rbHOQ%!g)mUdMRqd9SeexwD<=6-9ti_VGT|5_>6@7N!LN&j7 z{ra#TUiM|uM~@!O^wv-Z8h$?-8iKLOpy*&)vnF;%mp{q=`|St0;T6M{jl7>MfVX5- zRPNSB&(F@zzVIuqtzG9lJthZ1lqPp0ZN634Cn&Y6*YTfwRLwN7Y)Uv}$WlNxe2qVc zL}W+8Zai7`g?l>`KV?03pJ@#E-aQOhkS!v?b3~-VJ3N!r;rA59uRnaaab|X*sQmCi zC`l=0)H+-1{l^}S7+D6wHzOGdyUvMjKl$KsTI80**Fe#H+)h6;oG;mDn{r__r|$n# zhX+7hphbO#P1lYbjly72ZeHH-i2fKNkvU{Fs{hv6%oi_Chwwh3x=|u#T8$`dWhTWh z5E*~%F$StK`TRjUdwZRx92?Zzs{WmmXFo6|nY>)ssTW^F2bw68tfbLOKFJl^YmQ$ScMl5=zwXXS8BnWM)Ii>Cwzx0o`D7X zDPFIsBpL`iIxe842epl~&`A>#Lp*l_i| zu%>J)`AH97Pfz*lZ#7h}Gz5H$h@~!~Zr7-=wwJJXa44#&Vd3TFMI@9MKrQf_{hE}O zg@tlhooKw}v_}UbdgYUo8%15`a?fqwO4%O{?do`SI{YS7@aKAEt02uQB!3FtsRpq- z!Af-5nWVEN*jj>S&AI$hS88QB%2c#Id3^I5YwPM-3c!G1auJ#Vbtg5pnB}L7YGvK@ z;Z#dUmxOa^F+)>tQO!d`uQ}I`LbSX^V)7?)+mjxlQdAvPS!UYg6qWj}C(mm6GP3dA z9|si2O;D~$a3^U@E7vaiz*A-5f>rFo)Q=|mjgw8{!v~!B|6#8cNcIjqb;O^Ym-2!> zW(4)dDVu^gh8^QIF)_iO!7A+qjY!a@PT1+I3Bwz(SIE~~ zB}+7VAiLj2_Qg-2Qs;t6Ds%2u7}Q$7|B6zk&Aq#KWw9kO5|}yyQLd@D{Q9yD`<;*S z0stgfNl&DX>nMxZl-+J6PnQ9)f+)_r26S~1uJSRqx#1&c*QVn2)R+qQXQ6X4Ae zs5G+T6W*m-MQ`_M2Vc9j!ogl--Eca`?R()f&J{hAQWmpB;asqoxWIkVAtVi{&#ESi zDMwq6p6l%$E}5Wdvmb@m+YmQ4^t{*KI|xpLrw0iwF{rC-Q~^RH@OH=L>)-{RYF5WT zdBPW9aI(&MvP!E}=G8)=`m)r2!QK?4xcyW(A(CL+X~X0b?h_$%6;epB1EMmYVmX`* z?I8O>$d#d_g(gb#qv@VCg2CCaBx@jNCh5ZR@@1e0J|5Vvrlv+rL2xgWfoNCl4E_2Q z}ONlfHh7^lWDk8#oqgon7K*TXvvPaI!cIdI?z=oS*b*u@-J zKqkIR&j)w7QHongh?KCxE;Fx&7$$&D__X6^p6iw@3m(tX6>fth8*FD-`^+0UTEa`A zefx;#sWS2O4kFZ^d8+#6a+LFujT<*YplkzzBZ>rI8Ny6Psd;a`ASDSpblBwe$hNF$ zV1kf@3$G8rEV0M@ha_lVI-nHjr^LF18c!)&nvUqf?G85161&blr_@q3-;~^D6?B=g z<3FFdA~G_P=gdhq>ZTAZ%O+a^ z#Uuf}22g2P_kCZB^^%M@WJ6I9oX?(GeNji}I%cE8uEEmMFU_>+5Xljca7mRz>I0cs1TN%k z-Eh$Knb0~M)2dW$iHy4b*Zuix5Je3~D}s0KKrO=q=3RvoTHLtHLl`zn`@|trDmY%~ zG0@ld;&7c&27tWkeDcDD3tuB+J_hWFxA7KC3S@R?B0(hv&#+dmkI@^mHlJipC44ZYu?i#?}47Wgob8KLw^_0$W8Lt_Y-F<8J24M_h!m6b}# zCQ%Zu0=~Y!gn_J$KY05l%u$MwqRQg$r{@%LRNf-Nzl_xOsGJpx3J>Q8a@um@#EJd2 zW|fbw6F>mrJHiVPDuFzvo>~OZJSz1rH|O z*{RW@FJDxWPr{p6bHwgl{}@NP&2j<+#E-f&(GoQ~C>ab%8&sUOSoP0?(TJ2?R|&;({nYHDdCU-k|hP5FSAT9HIIFVmjSF}WOxSKYoc2Tomy+vyg9dvaLa5X%qpR53} zOQNQq)u#S$FP36QKope65($|WqiiEexA37uhT-uVp={V1K1ph+MPU9g$<;C2o$&H|44YYoJz&=uXa$-Fa{?e5j_Pfb$ zsXC`v<%xg-&(fXPiDuWksvA#oZ`0o%T48})4SNc)W^Ned%lz!Mh#n&c-EmU17jQ6c7Ik~^!pdLw-TUbS+qTicfk8`= zc=qSdAB~>Q5zSRUXJ6fWuTKhFq+tNH*4|!U2nf;EzrK`Py~?zDM-~jeS!_5?aq-Q_ zFSig*;hIDp@}Q$fg{5OmGr_M-SaARTw?Mo?weNe<67Gb~g8v^|@DuJ`4qF?}Vo>bC z;9cj}AyUbHd>~|_<$2)S1-&cGU{OHCZ$WxS5_tanxm#YI7>Mtvj?3=3a}gE&bwa$n z3_DJR7lL$yRTolN6)YrxdObC+rVXzDkfT$Dj~Am*Lg7nm3BjJQEa2=*@w6kGP6gft z1qfRae89AGb8`vGvA6PNwtV>phGNgPKbpQA68VLB2wwDwtO)-;=ud&6Z(~WSb*0N-j;$LthvS`D^iQ> zjECBc++TN%XC$yaZNj2m=tyhpLI6;1HlJFuFxLO|2Om>-KCv1yD87xLaCDeMvUC2n=6fnYpQp zu(Xik)@9L`7r>9b$SP;gp1r@Ld_?(OS$eLKO^eCz>;jsHA295c&Gcx`QZ=T0)C5y!>+Q1mDLmQ@Rw>9_=CST ze+-cC2>B2Q90uxT_4RB>w#6tEkYO~B2I4WQJiQj$ErQ};gt!!DUFiNdaBPWbOT;gB ze@zM&z?QogUQYiAM0-kz*13# zLAo*qE~xP_ZrM^~4Ym6es4f`JRN{q0e1mrr5! zVfcMz)3Xi*zEYa5+)G5bqaQyXBQXp19_q-EBgAb4+LxAsSGneX5T|<~65AlWBX|Kt z0eKTffB?>T8yMIY<6-4hRar>B#QW_xYi5V+_uitBRC18-X(;?Q@nfVJe&wSIQ5Qiz zO;MQ#?+yxZ$-tjz-*NeyzW)BD(AEHp3FVz|LND;4NoYmK#9$c*4Fy~w1LA)WDixRz zPAn6|EYcpK`h8-0n&fpmOfldAA%uVdIwvx)gTL&Q!UK;)g1}uy(Yj5*kz`dfMkIX9 zNBNi!hqvsOC5)T-A<%#Qj^gZ?dOyfhQj74JY&>Ah1FL#+Zx>1%LuZnI)#2rzHSEAv zTSL8dqgkXH!nk4o(ICDv3cIwZ81$0pg4K`~2<_s#MA-i2NZT26A3uI9LB+zmT``iM z^1?Cc>dqkY9=^&^*cj3;v=?afgo`+?8q}(-RX3qt}3xbWBW=XPhG7{QL6di#vR)6z8dM;pp1V z+q?6X#wLMzp9Y)NKbNAIP!YZlHTj&o5>vAdWsJ(zoxuvhHtlU~vVc=xr$QCtbU}UV zqSgT|VryGFwE-z|?b^RKx}lH6i33wLyy4I&QBf44L?&e*;C~u=dq|aumKEg5ul4f8 zg0gPiEu6Y&o#I|#QRR)YBB{L=$phOM=Q?LoppE+Fr9rK@|`9+F3J{{>ngrJ}) zXOs_N+;3pJ0eo~vIc02Y42^Rod_|%zGi)tR!z;0w_c)FjCpRSTxCuDiMH&Ijp%LRk;ovAg+BSBS&Jn2VDS21MZ!a^{5>=@4bgJR!{SlISt;ZR-~E0 zcbI|YEiLzgKJ$d>2HNpo9T`yI2OztFc#*FKm%y?|_KDN#|;O6ZUvyVk*tR0NI8YGexW=GVprNWgO(O*wBs zMhJxn!RiOxx0C|W@fREtJ_TgC%KBP!$dGJ@ta*qn2|J(n9&0lzq9L9-2Bn3eL9#|h z5n`x$=ZE#dj~AEt-6SmtWSbsjhDVQyg=Mki?2#fYS7o-<<&3Qi*_!?^co1t7k#8~L zc$@Bb&vYZ9OvJ>->cAKdU|ZDGbYWWx2;_vVs{hGLAqZ@su+=8#5hIC768>VckT60H zz+{zp(vhOTf60*=0~`IU?az0sM}B`@K0h~2_{B=F5)Gtg_lFN3 zDoo=Y!w=ag#G9Nea1kr@-?TjJ#4QXq@uX{Ij*XFliu5I-5ZKuXKUGWhM>uOoWo0+q zxl$p7Kk-@YE?%yzhDh~u=hz7MV^%PGHq7_Y0O`}foMYu-zSy{A6?+2z`E-2-@%b4m zc2PSePc={ULU>hg^`q_ZEd(!J!eemzZD(fgY)qea9RC|aR7P7U)EFwSNjZPKvMOa`^a z%gj`zXmGgjibki+pnSn@^&y|Hoc3m+OM&M&)E1AVq@>bx&{BM^YN8f<>;^904-32c zOaj3AnRdZu93qH&w-g*b5TG@U_(3C4ZjaETCMO3UBTb>Jq5#f88;TF&LLjyPnng%^ zMnaJTR2@|bJ}bCtu@w@49zy527nfGTf1QTjCa1A)U|Ms~%80TXn$-eJ6w7`oKa^?h z+Bo*GG$_Y>9Vsa(pyVnLl}|L}x!<|N1deS5wa00~+|LI2zcNW}w{hbu<%LI`vqa6X?P=|0sKxKZU<}By|(C z$SpJ|yn(?6n~}s>4V4>PbfizK+<--I`-%H^`dF4>T{J3Qmdv1mgpwqkk|6ez`ObQPIlxS=y#1gdm^PNP5B{?!=H;sKre;rCm znrPhqkj}>qBxv~eN$kfuKW^;k$0L5x_zZjrO#_Vrs#ouYd3zrV=`n;g3HnFk2?eFb ztxCQzk0#X8X0U+hN%2lMxkO-7ng6Ch&=2t55zK^Q_6TU1iwm{}7@$CS5QU@k@vciH z2QH01=vF!Pck`&$<9K)>zP4kYCptp4RqrxKO1S!m>^i>%)*^h7<}yT5cC#MYgr0$1(^ z244U9Q$OQ?NT3KWjhlvwG|9tp3Ts83CYa#-ITPXvQVz-f?600O4p&(XjSU<2UvWc2 zUo=Pz!=Be*GeU}YU6M=l`_87z**~xQ_xo>sG_hAKNqKSl3zu03;*Y~%;Il1+jZz7Zl03TBn~pTJfh?%` zvyF0t_=rVgfE=Wntg-fUl!T}37OAAO$*NQ36`4KK7Q5l(2Ulyj$v$u#N_c{B;NSX( zZ;qdz|Du^0BZa^aQtN<^DUT<0h-HRA$;zAH!bAaFK}^E~!lx*pzj>FG#TvJ<7`D?Nd3eRBk+RWPqU+ z-UmMgZToMw7xW`*xss;oaziE2%JR=a*X9w=ogb~!K6>m})w!~Yisgl`xi+9~h)NnI z-3B*8dcpLnggab_yrg;P;v}eyA(%=ZM zOiJ1(L^&oVraL`0|H;Eac;7{K<0JXO!NHdHwo&oLMMdUNlKe=^2Ah_GDzoly_JzuR>;l&;0_AzuC**$QJgw&4SXNx<^pV+=sv)rYRd2(TQa3)!u` zN^`S6s2bF?$WAM)|&o)pp__ zgJ9N5oB>41;^#N-C|^4&)O%9AY%a-o`Hw_bBne0m`Mh z&by#@yBH3QFi}!y;Y;-#QH{X1b&LW4Z@2*K)=^)F_PEK@%up;#n!zEPI(RWF;4Q_b512>tEt?G406 zx)U@aa*bc70B!kzz-R*&g8t131L%J+HNc}RAYVU9c;&RB1umleAS47?oGw8OkpvKQ z*(-${+saJfG@{ZC^bk3v9^l#_WBcq9jQB9)R;Q2!Q2>5Lw(JA~vB+qx2(!s9eW{EyF=; zCo}qTKwOl9MIrd=2Ra3lRD$BrF?6jEz<@8Vmp z)%e>I!gQZ8(uT}k3~WI)F|)wVn7q5yzyAI1s+%QG5{%mF$Nn zPo7Ntyr=l%*DnUBYtWl@08}-m$%BCZAz+Av&&jaHZ{OB~9zX5!L=C&K{BtY9ifT_P zq!KV9rO5LHhoHKxNx86**;`iqGW?ZLvuG%^TFE8{K=$UVL$R9Y)H2qa$+Zs?$}VEL zm>9>QgA78<#EC1eu3m%2PI`L!!f+9*{mpB@i0~P1m8OsG=lmyg|K~s5#oS-7QGbu{ zzkiPC=&I2EW%U02b1nVC68`)D_>1!Oi~nDFnT+#NNg^yTuUVrEk=oglTn&P+$LRXm z2UiJrBaN15Cl))`e{ON{iOFr|_;Hj$PEuEQ3tG!ZH!b-)v_K!&TeoiC{`;_Rz>1tzEauzSF=|MP4A-=ymawOhWPYq2}i)T*0D z&9*W#Y-RS&FNWTMuP=w2xgQ0r|L619a-2}$*$)5tKwHW3g>kTK$$J6W1rAHZgwajQ z7RIR@efPaS!IC8MqDFT1lzFU{D*@nvP%dgdbxT)r`#mmE(QpO^+4Ai=dc9FglK!+7uGPOz;`S0e?VSe? zdC z_q#B1xXFg~efwUDL< z2Tf6Hn7!tr+VbwemBQs78B08PI3jk^^c_Jf(Y@;6e=Fh2k69Y>-I%uP#~9=^I~i5+ zcn-eq3El5zw0@jiN`AIwiVcqcV&08j&8`2wI!f1j;RFBu{b!oJx^F%QTEDF@wMm-wI?iwd&qKkFl&0@jJd4q&Iq?I4}xIV{bCut7v>Wcr*50A>9Pw!;1v)+Db8JR)wuRVul7p4sJd=dFXshNXGn)qf2W+dnZt)b}dlWF1l`#^#ILn-SNt~HrfgBz9wEM6wE34_QM=fn{BR`f5?2%V56}w>(;bM*139NjFl@WxNJ!djIe>0Zf%3j5qb)zY_zb zkWuYh&=nae^PU_oZntkQ2hIKiUgi1Oepeu@I$oEB2`!fZje8?Fcr7R^ZB!9{kXDO7d{{x?cZQhqy5<52&V|N2IIosKGQy7uTXE&f zV}{z43rQy?s(unw0KXGRc53wZQs`)wxrDm`q%ajamo5>z65;p2 zdLJrNRAmEswG{X?m;>wvDvzawuOH5kSA(Do8qZlLpqbE>;BDzycG&rZU2b)mdzs^A z`Y(qume$|DpQfGIevgcQH5(g`{$6!1`LlK|Q@?i5ND|#7`-DI!3QjFA070L<2IU~$ z;Opn>o1187q>*e7klnHxa)8qLIv8qXiMF5F^Cj-K!Yn+@3{(*cxorNudsX)14giX& z#br7f9lz!2=!?4A*MP8x8xPdtTzBu@jUGa_J(4J#jEu)|6p~&ZJ9&~gjw;X9ef)Tm zbWKuBu(rJamBO=5JY<}XoXE8jpnBIKjT=^M+<%30YOHT3)CxIi=^JdKcF&2%o$iE* zjzN)$mhclG%ZO9Z3%3xk>i&FSFlrw>#L>@4Py7iqk{z>L?o6;!5a>)8Sc(b4_F3Kn zQXzkCqlhFFx7Kti{u(Lv$ztF7s{lJHCk<)rj0F=a(2Au;)Hz#+d~ow4&jdo7~?Ij@-U62`vOljT&aJ~h0^A^Ny7osg;(CHX4_kT9hiC( zXd)2Ghe73e)YPbwY>2J2gt%mu5vx|uHRvB$rzJqs0Wh?JJk+_(X~;Z9JH zJ5Jzou;U&OM<8Sfy3RR~n>GLuzg+8~_MLxz5iX1e?1W(l6ebsKuzp-lFgcve|th$DNb`cxW8OBKSpMSau^si?qiw(xvpr zmQp5tABR7D_)z)cvLfYn@7@}6#R#I}o5Dg0xLXEH0x>7fE>Q?#{C98J<@S6c#rCf@ zVn;$k%CYTxx_3tfqMo3`$d%xnD5oI?XwV*{gwx1a#wwsG3Gsx6!u3H!0s!Yh?xMl1 z0=x}pU}Qyg_j#{BhQdlhGb#tta*5wh4BC0iNYr_1Ke&M4iiq&=g@lGzpL=D_1-ty< zWLH#DNnc;zN<=zJ640OA%+rPiK?(2%113xv1x$Q!jW#u$!!E z!jctiZpf>#Xv0H=LD=Z2@qThG8R z%~yd&00J~gV-_+fF>yz3*;+2~H_tC5e4DaBz^1T0I<)Rg?f0TVa$xhr!$^ZH$r-Fb{QC##ag)!pT35V#5Cg+9l|)thHLx(SQf}lv5fq)U zNhuvWs?_~GV<)b@dKha0vKvVB#wdCWh?5FME7{uMrGujwS+v>sBAGe>t_DhKs-dJ%HNE8J8F zxUk3a=R5Wi+rc(Da=FKDt=vNB%{9!-6zSci+;A`6(p(9GrLWJR~}!N`3CQhtQ^i9A_|PrV2WAtX(1Vc~P?;<$Bfq`!rOIDP&7vEsKh zIM6Oq-m+FE>JU5t!s|dAqxG3Isu{2>_CR!!rl+A$z>t3`I!K!CSOyb_VV{YKiBL-2 z+qbgcc2FHPjA2pq_S&YI0e|D}5pZ&F!rdTD6aW>{hAb|Xta^NG4T_N?VQzQq9~>;{ zN4j$Gx}VT3xNn~frk@~oa^V|rJn79&pCPto-2X_VOfGbzcWAlYcrv}n*4EZgQOKYR zYhyP%B}DoY5#p~ri#^I-Y3<56EOsNek%LMvkT0BGq^g>%{$dH9plnB!Q_NXg&=-#Z z#*v-s3tT1gCp$Na$sP@I{jYe|wE|Vax##KWH?U8rAJ_{NDqs2;1Q=q=0VeY#H)oM{ zKdyO*;AeW`A;?f}VC31$i2D=LOCi{1#4l+Fha3&x!q+<&0r^+lv6gxy5T{M=H+1c-b(=(rKqtUMwB}M)F3|1F$NX6@U%DqH?Ig{$U}Hs%O=^jvgB~y< zEY!mZt1*ca2A1{w#hx8_<}Y7?#xCmVEdmN&xKM*1lB0UaWZ9cHQj;}T;AH^4eI3(T z1ockl){559ovQkT{5zBDd0fr!lrcFJk$S3Get z!je8XJKjP#YlQ70+&py?z&7#3FG<;ccv`86{y8^XsDorq?kyv!7g8k;u0DAD<^|v7XVO1qX4WtGX3+5Z)kxvmm=jBBX6nxCU*`MzE0#^u7b!9Xzffy zOEkzlldMw=+6Z`k3->&dw43e(^W+2`YYY$+zVvDXXS~>uM5W9(lFv6S@drz%4YB9oFa-x<) zO>z!Nid?XU+HEl%T`Q#Mx2StEO7+ki2Mt>XmO0#Zz=m`Z&px!;Zd@No4_qpC>_XcI zB0l=yHb8A6Z-o0bN#mAio&W}ab`CcSUV2+v+J+LG+{Ls16F2c(J!S-sRdvbU0ROeh zB|J7X@8BrNCJ>iz zkf{q_w>P2Egauj}ftSQ&;!OsAC=aY8Nt^gdKn|ibmc_|_ZrP#WY|5-0csD%U7bTMK zX0Z*}xr-?4?n)XQ_&f0Jle;Kprg~Cw9UgoN%V-xZB3)=m?FxEPAljFb+uRs{?~74E z5i|<%Mh7h$SMaS6L1*YLqjNdAV|^8;E>Rw$&749 z?&%?(9b)Xjc_HwQ+;?DiP)vs05s`!C4e~w}hOn`WBJHgoBE>eMZhC|5Lpt5j>cBI- zl}{nlw8;lIR2D8&y)aD>QXJu>80Q6-;4bt8;WCi_%Vv_g)0~}?vkY~kp8*;_Q3?3q z9l&HQ6G<-`h9QH2QlEn=odyXj*cOMk?pzk}et}`2bvuWKJB5dU`J~Y-xd#`Y-d1py zlYq`xZh$-EoNy}&ty=;b(-96;Lq6UY#z;4j%x=<(Dsa@%MoBriaJa0t9%+UIbav ze%CRNnc;n`l8CB`Cqc=(TTIfKs$i5s6yhO-N6t*phpMp=fPI;}@>31UkH3 z7QnHb%l+u7Q{)0F;yGypbHPNtMXZJ;K`lkT!PyJJ!Dpc=G#o!I&cabKz2L_>in}yj z-MF_Y1>&~96{zKiu0om=H_}~%_ih5E66pd0^;d{(4(_6icr*}r{G8E3!wf}(96OS$ z@%}}yi-BSyw^3u4{5EdCdvxmrs3TGVE(E2A2h~=6oq)!O(u0*CQPfVpetyct^Pha4 zb$+^K-n{R-P)B%xz5N9k>8?YUibefIY?>%q_Lw#bgR?98@PS`(mKey_X5wjz-oHO) zAp9Dm*25Fy{!kf^Q{;{b*ep-&S{~u$CGJERPIL)CHz1x-;_8TV!X+Qz^Y$SndNOZQ zCaJ0aCvYBYnx`%tIS6upVKU#ARw^NXIK9~Sd z*-JsJBz+_u`?X~0q<{hhLVFJTisEkU$-9O3zIOc|?7e4Dlxf#3+-7GCjE>4kkP$&i zVk9FN!3GsnauP)(gMxr$j5s2qO%h3hIAlp8AP6QzvI!7GKqV+a5G1FwcH{Had8^K; zI)A=8b*f(Ksiz!my6?X4>)LzmwbtI3W~mqGYUs2OS>KM{p2MIA3+is%ZGOQ9`_5e> z`r3c>b@m8<_~XomNz{1Pirsz`e=&64(>m#WG;$4_Tjno44{l%$%(%Ea+TzHL z)Jopm)<$C^c8UbiBu8ppgOL?oqlv|rzwHtXC~4M-=80Kjg89Rdbs?g@y=FJ^Un^-` zoxhq`_xE*m7AfE_xG{qlJw6$KthvKc;`1~6-p1Y@iytq(Y%@EP&uJ&$=x_P-ZYl00v7%1O5;L-)QIjoDopgdimPb zqS>h#*H?UTJ8Zu)&Hl7li+ygRo$MbszLU;=_3A=+hLucQ0A`_a&qibBfe(9g2cySR zL>G>`>5cz5Q_Jf=^x3~S?>jEcOTN)jJMfzGFS+MNBSH-|uSt81-Y@M#sYAO90hea7 zEdZ@REC2cP*&st|4&VQoew~Xk_tH`ipBwYS@7{=)cEvUhSeOJHw;RO>a5o_&Xc+o|#hbJySc!e~A5`%Obu+{FySnz|QJmiYknsz17-B z2$z8nhH9#j@TIl&B2ANpu|cG4{rdIk7}^e2i5p%TZ*ZR&aN=D!HOY;aU(8*JuQ5j) zt(2&AMDENLChsJgXSTzy5_+92d+aV+s>~iGj~)}saVREtnuhLpx2cHG+ZkV9+@VrWDzL;f+Rn?45xnBGcdH+O)MC zZ3q{@3qX;{Og;?kZ#}k2QnE6Y$o4C-z?V?*tPE^{3BW9Y+7k{z1h~bvNub2H{!f=j zh5$xPhm;NQDn+6V3@wUZt@4;*-bhZtsaWN>g0)rtD#6pEMmix=1E6@qU;XtKTc`IoJlpMPTM zPD}%yWg4+)hG$iz#Ej<(72bdR=lF%#%wu7JCt>4DHhRPbREZUF!F~kh|E+5U^W8_5 zBCNiA&{U-`wog#_C_pIn0@jSevwDIpbAGc%qL;RBna?1;2~k!(Sa9vy7}zB&ZP2C& zrEmt~b880l!}};o(4$6UL9rATD4fAfjo}ELJsz`-``j9i>E|K7>_aTgKq9J>;f?UW z21^^^xmvFYI}9nVR_{CJSi;ciJ~`6PRTd^;HCI_#Ija>N>t(vl8rD1t?tj6!_6&Qn z&;2kz);9Vti!fmHHL#fbm5pZL(KfmC9jDd=G`PQ3T?mtZvVQjBZ{NOws0+~hdHV#*@;PORVAujTi;Ai$^b7<(nf_$#OHS3#%$655DJi|~sqLEpYb z@LMq|OMo=F%5Y=SoO_Ij!~$~U>ZGPQSZqo{-uH*?Tl;n_WC_?lWX)VlOiR)M!qzjX z0}2T`a@!x11F<9^o7_7v0eDdOKG$I#bg3T*60Ji20>%?ON2Q4Ny&b4OJ;#42N7Pq_ zNpT>v9M%oS=o-)oWDO{dy#SPHjQS4D`bN|sx9$wv-$C~?`G5;1N3B5nD$XfTKy)sr zhH%U#{x`h=!v;p&P*Z`nI|YwakSw^HVicKaXcorvPkek#ki^5gIZb>tPaA_5-8kgh32+(9>NX9$BrMe4p<8D zE7;D%2BsVM9FDmEm;*tkFQw@zc)#W@jm7Y`xcc46-=!;QPx z?8wc@`5GSB3&!aw(hb?nLyma<0}%9$=PefN4-D4X7cRYo0bFvPtW)pc*8Uy)faY~1 ziq&rQxSWt6Ob$Na*VYV#F^bISYPrY?WH7+r?u31qHY2E#H1!(PL%G=?!HGx2^qEpq z6tKiz0WZt~pxvgVjNQ7Y#OI_}Gng`r7n7PC?c9)c{5;}+eDwzyqYp5Dv#Lpcf?5`c zy%#*k9Y_JP#mi30{ZYI);%TH;t!Rkj*9N`2mDf_nPW~8|5F4Fv3lusUZD3SVq-z4X zLrsHwPFefUInuI0qW5DjtX?yE^c&0ZeZHr5bjq}hs$f(kDZIK6%u@1^sqg1o!S3I$ zQH8G;^IBlIx(u=}8Vkyj@9zQNl{Ne#E6yrvdPkr2Y};0T{kI&?sfG5r^0jKa(Enf_ zA-hHj+Ra6z(qkZcsquz{1uzI5@3iKh1-8EfYi4PjE+SFViYB<934a`Jz)1-s4?S|L zVMHMsPbbTAIaz|QNaq?XhSY>bZ`5j0&{HtB7*!rLhk5so6y7VIk2Do8G@|~UgRvb0 zDUHobq!jEg0mX@aS5N#pU-(Jc54Iu&(l;&fi zdH5a27ez%!aiYOrn~u!}t27I*N1c8YQ{dtc$1R3)E|JH!58P>~upljs)({oI=wZzB z4Dv8S?*rO$s8b0QEuy1y(9wrmQZvDj6Fp9o4KP{aD!0^PNy$V9Lk!e8-B|@~zJrdR9DpoxoG@S0jg;Gw<{6Fz45|5Ay!xem~3C%V@Fh z(iNrOxc}WhzUuXjO_#kQZijO{t+}_bZBl+$-I3jAmJRmETwUiM^`yLf*&^=RT_IKV z2?BOPH8Q7QCPHywwVh2Yu5Gl{v;A_5W>a#NqAsOjf2Ip^TzL z_^YYxZ>|qOoHHIn>c~t6hqW9u;2a36G?5GU=OTI@R$wgu%M8O4D5+iz_v3}~HnS2` zke3n2HbNdDT?p=~9JHiCwMsbb2J)4$9iB-0`;Ac4Xy2ng!`UfPq?4fi7}6-}4HM7= z=rWt#E1qx{qB^6ImlN&M+reFs_ukZtIu$Xl;3|6Asj&>;5k7SZw82j$r1TS!15W@I zZFn&yHAIa)P%HJq_utyO0ZMh^EB&31=mkZplQ978x_TeFAfoXHuU8C zwiC8ro8h4|K}H?@JqXw<{$*cZ-%ji}sE3Pa3^TG15Jq)E_ebxsGh>oV96PH^f4Hhe zN&6&)*dD`Fq#s=tvlSdu&b_?+7vpi=mXMPpPrcR{Wp%&$eTATyv6cQ@UfFCVpURZi zF`6c(xjo+&^jg>dX;x*hGs0Ek;ENYrO&$|lMxD6N#y&qD|{K)(`=uh(vE|E_gsvr#&eIZM>K+Grk z^ttbd65NHqPmcGiQHSCC4ATXq@N;nZ@4PblY#9$vvg9MMeTy&$*orc7S)0C2$SlB;G)wAR*R6;G;N83+3^lc&(!%*3U2>B+r+Z` zGLe^8_Ynbz&D)X_1%(o`NVzaKbT)$!hS$_2bpe543$_{adE!5VQ5hxOdgdde*{zMg z#(I2tBz6Mh5bN;Pn$Cul^`29v5~cMY68A5<5qR@z$(Mp%74_BWosVT(U(UCw&)z*W zc>c$S7|r3yxT)_E!;!h(i#~+JxRg0_=>7a&J7vU0J(Kr5;s{JUsj_zji2R84zMj+jyF3dVNoKJdcaX_b&l&&7Lf zh+W;cm%)Hrg~lW)d$7;Ad1uRl6JiErc-qfsV4x}QLF+6a<6H%oLM)D@M~Lv@xr|A)k3~)+f5VaW^PRuW z3$3fK=NA!Cv2j5!n|DLgyVd+L`9M`yO3mjn-n9uu!U7JhC(L>*5rhna^BkN}fpm#-^(27`DX|>));ar8N>=E20ys_513F}bI^un2k4VW zLGO@<7cJP)hlqOHupt%D@#tQN#SrBfw-1?`Wo4+~v?8FV*x1;o z?|vGdOLW*X;24}P?58g^bL(If%&R7A?#LHU_Y?;P;r4w(*BL_Vc2Zr?dSoyVy;n&{ zQ^a{?f8Z2M*K88+tpYZHF_^fzwSr_BNAN-MA)jBURHB-o%ZgHS;bQ-Y%A zDjVUL!YeGx5Nm#G{^+!H$-z%sBqS~-Sk$PMSa3i2=0Q|jMn_Yvlg%FL%_6=Q{tOCSh!@^6z5?aG0f=`r3aN`!3{3%2vTM=B1>EQ9Rss_O~ z6d^CtRNxb#4Ho)?ANQV5{cO}OGS_$ML&LYGMZHsh&OLOvnlBPm=q0#RqNC5+hw|Ux z-|`zS`eMEP>e-)59Cn{Y7k3PtSth0 zjFXpN@bkkXjeJxDesAlwhXlRRfVFz##y=p;=#8Cnb}j}qhtugY*69kJ{3~$^TLl-B zJsfU@XCSnwr?j;X`4`);IkQTkY`_&`5oa*S#No(?j_L2;XL+6r#_@x}@_*SE?w{=|by>TA9d|tw68Q;{)Pv?nXMs-MEJAwzp8#qEZ#IVrQ z(BuOJf9X>*@leSxRpUs#PMF4wp^57tMfiZWItBrVYK&jUR?wtI%nTZz4_V~*bvkcE-~zy+takOH25d;=fU4TT6VlqJF9bNa zG~~A(`QvC}nseH&-DmFC_s#X|xb4KNootvs=V%pwV3}0%#Ddivt4!X#Z)gg5cxvFI zlZ0KhP_4(b+xc6}`)h%8ToReqY({VuspA+fyy>K~cGdBrzCIkXp-ioZJbmvW&x2i~ zK?S4+YBeI0Q&K_D&#k{S9=Q52-=&^i2F~cmd~xs(I>wI%^VHYtbi|GTyfc8B0iS1+V56=aqOJ7?q1&?f6w|4MJl6=UOE1m zRIfAC!yeK*fDmNdcDer?3^&pJ#NSZn<=M7_Pa$}07JGnF*6bw=BGmEJYGP+IzG7-G z#ywv{85e`nPp@-;@BJ*RU^^xILz~OXEBe-0`(5%I+VwH?^Rnc z?w~nqfa5CPh=lC@E}w_rv;G+BTfTFZ{F9%$UtZf8`x~V8zdHM;`I~ay<%*|D3l~kB zl;7Nbj}ZC({cQwz-5Lwbx_Z$HXiQ)jg9gG)-5~m60Kg|mE;;tCzk}UbeL>yR!f@HY zHqfetdem3kYf4hIIdZd9E>JV<&OvRmv4Xceofv`!zVu|k|C8t;_4C^wd`b)>-XJIx z0d7&>byg!H2~|%x%V0k7y*g%qej-PjvDoJVWbH=$uI{*X>(((C-|!}fbtl0k9mT{! zsLMN1yp}#h+GMxLERaGWR3bGiFs5DgX2LFpq(M5oJV!k%w#5Ls>Z zs>>Y%$Z?AhU|o-gi6z9xq3>=H!sp$=mGYVJeceHeJV@Lsd&5~iKVKx8a-SM*`tz&Y z_I`~k=PwN^f!tD$Y4IA8Y?~aHP|*#3r{mSu^^YZo%9;BXm&zs^bH4Che*XNpj?Q=p zE?r5)ayKshS+~kEt|)H6X~6v7P^5`*6`6$Hqe>% zzLZ-=^c3S%uwJlSUvSzJblCGX3ez5T1tm&Kw-r=oGFtDu>Ux|_5yC_hn;E-99&vn{ z+zwaEX)e>9wXV9t`VW%(`cG@6hWe#Qyg0yf$+R$4xvId>*gW1-v?9f5e@^w`N42m1 z^-xXmcwn6};^3((7bf;WGwJ&|%WA{>!B$qCTB&Or)v9y`!#EqbYj*flM(T~RZBP8A z8^F4pois5ve!Mzl0oxGMD7ub&b4sNh)%XUYf3;#iB^ zFfP_~YK}vXaqLBeCV2Khv=^kK14%YgYdcmMu9#HpB=+|jysbFv524QBKC z0Ltn`P-k|}zjb#wd=o;2iB0L**r(ZUYcs$*{SF?x?WIbl{WG9h=N$ulW_V-vrIlDV zj%J1B9PW0wPv4aW1OzPM;aSMgi))|0EBvE6|B(^^aP+9L8@R8xWL*>;bEMZPTuNP@ zT7wARYhVZgX1N#;aQJMI%Jf3h;Q60_dm4JcJ%^Sft4B!FGZ^HXgfmY#gsM5V(vNQL z2aGjfHR(qr^8f1s-_3UyBY$|LPcneH;1)g;>ObNIR2Vh!uc5Old40vSVT)E{01%3Er=#0C7CG2N> z`Etg8#*3k)Lr61TeYJKsCLf1~hxeh;WctE5{*1Hx&wEe*6?CE}v9m=aVLP~h6tsMA z6DI7?p`PmM>Suqd?bgl1U}Q2hLRqBa#q>J&0CgcS!?RYYDZ~Ei2M-QOw+L58JW4+N z(arf>vqOh_WDRFu?{Agi@;-kSyZ66S)54UM3nu`L7VTnxNWfQUOUhivbA< zVi+m#Gzk^~i|fO1(?g%H@qfLq`w(;Bk)CC1LEMJh#}k&uH$7gjW^=#f<;%zMs}H|U zNLr_N-!j~0VtuFcm*Gu(!-~BX55A9G_~S3R&sGNV&dUcrT0MLIZg#hvOG^%)VL1c< zGBu*Y2r8pKM<~@m{E5|yp5xNALQwZ^7irz1$-cuyya7wE$b{~jyKbEsWVo*qi|6_D z&0pYlD~X!}b|)zO4_|B;#xzXyDDNqOi@W*r=X3q-Q*K;vtzo5tJ)eAxyF*#i^keyl zA=rcnh+?P~@nHa>^Wb&x>y!b4B9|_M&SXRHz4~K)rC7cuk0D-$F)UQ5wQuN+z;j50 z>ETEtsUAyM8V9{pQbi4rE{lsS{=M}N-jzwU(p_aidxkS#^53|bcD!UM5pa+?sDB=X zC8o2|JSX`3nlb$I@YqiGkEJQnr<0+^Zj%r_1YvHeUtR+D_0(8=yDqWU0wA(uw+Yb;hqYJm<}FqqhZznmfSu$SL-rO z^}z%T>!t!;EYhNI83&D037aM|u-domrS3>rYvQ%lCgu`JODGULAygDZH3b#u9k83y zxoW+rT5!XFGPsB`Iq>Ul&B9av`ch|KugHRg$i5jZn~ht)zL)>VCo~^P_I&gwuc3Xo zPn)3o;y%$}ayled#RCJ6A!LlFpONsK9FU8sxe+FL%i3U!Q`XUHmT1x;9Hgj?N@W%! zQDY+Zac$6H_z}L=q;L}l_e)OqVB8GsJ-hKb7s{)>O-GkKgtZYa%-2OZUX>AvW!(B{G^1V(3n8ov!)9P^2Y7Pvcf3LB|W)Zc=N`|CT!*FK+-Zm3}!$_X95LH-(Q zCk6Mj5jC5285;>GtUWaA@niLvxGZ1yZ-ST;iO~y}_O0CorkJ=4F*)oWDD>)uo*qGx zP}6qfP%yNN+_afO%|$@~TJ1%gr524^QKS`y-M==*N~P)U%p;tqFPR>jyo zqk@adtD|rB_t)xmHWUsq(l>2TzpE!0C~6mF>XmwA-QT?jHo}E^aNup}Ng=a1F*%Q? z95b@NgUE}_o$s(`QR7&c8UTW~TiL=laylpQBeZlqofIuk6V3i+lPpGX;thP%P|WDIb8eeXNFj^{;1 z(TJc#n-f=r+uttai6lp+KU%$bhOcvec=hkGg@Ol<@n0K_8Ab)}|G+*&{!&Y|lYe-8 zWd#HT72avluE zOa=ze|AuLN*lLD%yJ3I-b^2mNjbM<;^+gZY z7k0igv#mewG}&c}CdGGF>iQ{S-&;l9Wlqb^9G`XYL&fq_r&3?1D7!6s#IUgVc2lt( z4vGv;%_PlB(2j?j3VgnAlWT52Qh5HKe>P2ibYAG(GkX?;{%}d@Qm=LG%xWu*qAbzu zlRp|X8QYay=SIqVUi$h~$6d-<1loyOZe)I!FFhXgEg<=#2%Ae(YP^7;(DzPk1lBLh zL=g`)Nx~|0IINMTLV!sv%>dTw)R+WibqV(JV~@ROlpb6{#%Su>AZrK=y=)SUt(AMB zVdVumWC37iF^J3n+e}COYIFZSNuUOYjMLz2w1eMAMd_nIaR9v!>Fy);2B0HC10lMm z!8f>&TXG?ISu%^@8GdSQEsC8z`ZW-9QyJ7k&}z;P4FslJpTfdI`;I?tT7MxUkyWHA zQL7ID47geu`D5<+0BSRJqdspLMmDQvEln;9e34V~Ed9)+y+NM+f<;`_6%PsVtc%-Zz1?5v1~tx@3u zhXUQJql+4h569SFd!QNsivANoDUB|XjfevXL|ZNY=0?w=5NMZ(qTHTKflh@jr-g`3|?bv zus_$9`HNe-(u}|J$qt2vN&a4CV)Qieeo%eiI{(P0?eFGX9k#zszq;0IKoG>EW{1+A z7sj=NUK_2Cdl6@?k)pM7m(tH?gX`UH+`2XB)%DT&j8^M}Vz2Yuqidec5Nh7O=e1=@ zy`}H0gAFlt!ud?s$H9*qL}NMK8PNVvXj|W5dqb>ppv55xV_%X(Q9qYj};# zt&IZm7p~WKKD0M1*;Y9c-tzB&Ox0L!)lMtn4I8TR>^a?(rKheJBxKJyl=I`Es=pEde+k4Dyh;cQ9HtSkjl7Nx0eC}DLk_XO;! zMgwh=q$K`$!L4o6{~oBd3+A1=6Ja(P)mr*@QG`fb8!D{YAqJWoj;21$2depN>^$H2 z-<>JhMz%ShE@7&kq_)ASJt_WYbZ)xnS7n0(P3*3>O0BqMbuVgwY7 zwm_5x$9;I6;_C&E9-Y8I&5Uk1M)sSUnm%ZB?Y!$$>5^Kkf8YF|l~-n4h5Z|g;+ERt z$<-W+y~myEJjV2js~s!lGuz9Voi*#N?icnCjxkw&l6ANB+|W)UIb7-LGEps+RNwg5 z_yHWYa;QFb>-kk)3PC|Z3CYR(SmoiHM7Nlgm%sG#FMZhQ7-h$+CO>NJ>Gmc!Azr9T zcH6@^9LpmtGTOGPq&>=Czok=m=`Bf{TcA1CyZ_B#r0x4t<@OWTmK(}_K8oY3={{YP zX}g&?IFHYL+Kj#@1BVI;%@4&J5{H_m+S;Dt?nek9lkHpO z(sBd!OI%0pm455|A1`uSgZ1s7%;Aj11vYiar-`{u4AeS6ZCuBc{9v6qK9K6QA9r0- z=xCJ3AQK)Edpzb7=m(Vfdi%Wn&)3S@6Jv%tiuhvIVM7n|9&r3Io{cl)6Klt&YzDh7 zdW}wbP4smBsPB5zW1IUuHM{kxgLu_qIh)MA#fITB%r^xdAKYaoleoJ3@1?l(-x|UY zvG_(wlNVBj#kEGBTlhSBd$_(hzqf#+`OWbH33rK4o#TD+hu=EK*=%l3bU2KIfdoP& z9bIemJ<2Q##{@^RZ$@EO|M3XN8r@^XvlxsHAN84wY8GtBzmA~6<2n4Px*p#`&!kJX zGdTdkUy-@lrLQIY)Qgq5!=HWMA_{aBjD7bt+yzeRUVdLGx*oQYtqkzk;ftS?)+2Ke zZ>6OTjJn!JzPl);w? zv$qve_axf`ZbWI0e%f+Uax52zpeWd?&$cF%6;hON$}lI!X?0c$hL!5@XBoJDyDQTgc-X#deRhlYti>je)-3JipVKr?_tf1%CTr83sV(>Ly=n~|t^EGjVK}vd z!Yyu3`vb7fN$Cd7-*LL(Wy+&DkTWxk=hcI?v+yfq;L99wF6Y;IBa*B&g9I^AdcX*PqS2g zjz)YDuti?1;}~SH5sGD1m<0_oJAvy?jemc#n%yy+vHOYexdT zwt5}YSoffjkx}14XR?+-+l5C>beVkvkKBlvT=IkO zOi#UDAws2ywE5E-|29diTvTrNq=~WuJbq4`xRl9c0TXp7d90(dJVWowG zQXbIf)a5XLd1?Bs7Z zr+Ja(S|7&$_E-3+k(KV-pFM0xcE(3TjbVzDKm5O;*24bJC=<`q7*HH;K5qdOh}QnR zy@iN^#&v>$Y2iYtI>YPwL#iEm?!P^NTlIElaW{fWo$>iaiJnFlcZ2$93tSPs_YzOz z4JLuo^H;^XGs4D!V6};9|9TInDWYsH`e^!_&vSrLla%?M^CbiSjRX_S#CRB1S9s$ZYAtTUAIH=vEMi2iDpaQZvt~=S%7=;v)Ls&S?KP%vJ?q1Y?y#!OgY8 zR9YpT*LA4dB^3Nt)hf5H^v9eH*!pFv8y*E_H@d0{oi$Z)adFY=te1)MU#Z=6_nmHW zrD%KRL6g~kG8pD3MM?vXziyH~(${X}&TXBxqY)90y_T&~9|hxBIjN?J@6P>jaB2I| zg&MiE22o;iLEgyx_HS5QrCd)r0;%?wtT6zLU1TIANEy^b%!xoK7KqJm7Ltb(G@!(mz}`X_okoGZ3DE8H|u zlaHydoK1UQe*7J8?w1<3{)V?3?iP=JbQq}cDxGjvWW`L?S)`BETOEI;*i$lVXj|ly z)Pf-ACQrpAr*DB#Jo(zLc1=l^4Yxk#nfgRLU7^C-LADm6dU(QUk4;d%CmzBMwdgwr1Bj`%wm84w^pHQw(61NvLYVTtNH_w3n2 zn}x^qYqUjU{>V-*XM!;)@qHh5=L=s33%8aDKb+p3)R-84KKI@otmDYR>zL}mN>oZPrK%H4Y97KIRhpbStTqgTDIYyp-j=DVXxougha3Lgutp% zn{#7sf4M#mvP`pAQQ;gHYcSOvX(%=Dyg5*LPPb?4+8Ve03tS(Sva~TMsch%ooYiyP zMMXq5AAfm0Ip5X7PkEV`O_V=xNJ?R$oA{&8;a&OqQLZU>B>{|9O4-{=)*TNxSFoaR zphn-Jd{gmpD5|=6-0gCe6KC!0Tt=oI-hJYwDCu4^7L(b?tQ$7U@uC_W(C?ratjh)4fm z-z5x3$2t`aF?6b%IqNqDWBaq!foVKK!ook!@ga6`$qda4=9stYO<{p*yfdfa>4P32 zlvH*MZUr=8|NOpF2x*S}BQ!sH_OCzQ{`B^E8U)figr&85(pfn3XozG>K{fv zgeOu*%jgj^_{LCOh;C3&NzM(2zDGSCsRgP`L?C=V1*EMo;ue(Q#{B2K&f=ajVZnF{ zcdG^($vjG|cciPm3PQBR<|dv@Qj2q_Z*l zLhE)c-$#mNsX00C9mG0~apvAPtW3{=q$92GP5ceN4wJG+i9N!(aU`Wm0*yrD-<=D~ zV9MNrTWJm8a3jRj z@`dw`FXQAig9%wUH9SIN)I)r)i>WEyOrTW!ORcjER?VzW)w^bC8yQ7bKXf_(xckU6 zr6$V(RaEC#?F|84o2=?XFeZXG2jsqsJV;l)SoZ5}Dp9gLuN-d-zjIACQK<y2zVNlY`lzvW}KdEV0JuCwP^d;MS8^i6idiIpd$}gAN<0aWB^h zT&K+=BaXi=dv5z1Ndg?6z31h}x-yvsBeN&*34w0kbJrm{JAYR{!D)XehZ4P*3}ssRkzmkP&;^~tO-tDJh+c`vVz+5m2!U#!<<;?51srv|%u zaA|Hq3JeD)u;WWmwQ4rqo&H3Is<-;gfoUk_Z1~{)G2i60r@!kuR%<#uo-wbbX=>7$ zbsWQXgx9aHSR~b;Zek1XD*cmKUoL30fRD~i+ngI+%oM<@T6%ykF2m#bKo_Z*Cni?@ zxqRdZhkWmLts@=-$+>N}Bt|iN@)i)LB${+pK&xe-u5AbUz1fpa;ME#+0hI@o$8e}% zE6{@~D<83uJ|7@YbHD?|=6C$HK-0K*dlq;-F{|WsCH9Jjf}1GoX*jt8GfS8%s)cpo z?6Oa`7U|W{#sKs7x4>}^Z>#%^LW$|Ge|z;GDz_VXvCd?G$lexuP36NWwo?AgNv97s z*#`C{g3J};LKj^8)rTbkR>H8&<%b#^Bu1|Rh9<-IGE!mZ8GygtBio4YYBx3Bu1FcW z#yk~8FQid>EQ6Mgq}0mGOV&`?*M51;wfKQs{tBaC!%sf_O{OdD*=G0Q-h3FhEf94c zG}xEn0_C}F?jNLuWRv~zLB|;K`-!S;L;F~O=<_sfz^;`c`-fm6!e%T@ph2yOo=E$^LCZz?R;Y`&|@xVS|BA{hGAn?u104&t7vO{bpreu%n z{pTRLOo9KV&=1?B~_bWMaiAT@wsKx}|LmC=DIRED!<*M+%Qk_vh@ucqUNDW7v?xzY3AvVBRGvb2C4;^7)Z6^+nMRBeVl>f zz%{Ev@N)(3ukHDtYUQY*h%OxOkD4kE(l*jH;XG^epl+?tDpzwhvY^TtWo70zt~FWVyD%rY2s75nOSJhBMJT83LwgJ=y^))@L`c%pzUZ zVsGol*3&QFH-G(E->@n>A`=tq?!w|B;Up-8su#c~5XeaeOuHIcjfgfa6;0iTRO9!> zz-wY4w}qu@4;Jf$pG3`G#Hqe7d{fduh9#&Z^B!{%4J5g`9?+5)OY=Zy&!oL z=j@ss8!f14`s`)sIns)t<3xKnFgW-GsN8!UAmKD2lEYtf**(NUuSaKz{JQXyJ1Omkqn`@pPf~{!2vW)l z^rfrYDBVa@8fs_+uDK@7uZ)^_X4Mduj>d%NXB!w*0?Gcc+?jj~xAlrC^JU@g(umXt> zV;%yZgw#-(jUy#e^zOn--_Z^{P0pVqZh#hjs~ zXK{C5q`VBJY&@WzQE{KU$5Hhr#K-T&H(RM{heS{ws^_MoopE^QEZ`+yjKP+mlojO~cj^paPJf)@gfM1qdS(s2Js`I}DdmA4!Bn(em|_1O zu=qAU#XREHowX9>9C7+PLk!gM&B}^Qgr?v6_NG5n%#HI`e;6)zTlQ>9!RaW1dN~{g z_7t2zSntIh?DO{;UxR#L>t}-AIdSKPQ_7yeTN|YO?S3toV2_=@z1kjueWlq{6(J`D z9CYNG^yka@y-ycZzt|rxy@@gwj+bHMAl1Bmzdn6Q{|yVB#fT7X_#zgRKP>%+pu4%# zOTk4cW;fXT8cI&T+WW4G&lH17K4C;_-0p=mcs{-HNr{7WkwW-tI%@=&U zIT03MOpKXwAA=B7L_}=Xpd$=INSI<87Sk8N@QuwWSn%u5j9@h<74Mm%8#bunX?uhm zGy|bZB3aMoUza->bfMy?(nMOt4TeFI+9J=gZk^@H=P4RLO+lBt$L#`pptJx zjrkJrg6aVtJpFITJ1Hi7XPVCvU)8^@2FpJPrO~6B0zwy0=JAzd9L!CW&!%tpOH}~O zwlC6K5syU=dNAY&BfNCyLgJeTL+yB#|-H$iP%SNrKUODTw6HlNKic=JwLhK4?fr<5p8I^`0HIC+b815 z>KKGsXD~JYj#j6V5~xLp&HieBSQHIVF7qumfF(d1b4TZ|U-P~c|Kxk1I;UO!4<(de zcaW{t)1rIu!8QyPX6lO^qQodZl3sw-o~JYgqBXo}yiJNI$PlHvU+?tl_SRDMr=U!< z@7Sb18Z4z^3^JBx)&$1q9AG7b_0L)r2Z&A}31g~Ai1-)1Q*#pjbLY8YlsHv%0Q@pZ z*MjI)g1g_3VN>n2C5SV}6fSwH(XRLwx+E5%8ej>CCa3L8DF*kYii)1(4|KzsMn_)& z?O9W=fZ4;zSv7jTUI%fNkB^0c`|S*+Bjs zHfWk7wjT+8H(1fP1Z8BtT-{=iMuT?3odIeJp%}o@a%5^$`#bVJS3fSI-K`w?K!C`v zCmob(g7ax-yxPUFA41_9qzuwYshEeoFAC~P;frCUt9D?=GJvaa+ULZ8WPZLP+D2Fy z3SP&j-|cbn1wYZ*{ zQ7PL=YA)?E2NeIrJ(Pi&$a!VB0&jRb-9kf7Q0yfbmN0>8g}PVp-UyKI(K4bRGZ;vu z6HU#}NZ|r{xP6xJIel&1k$=&wLWBaH%!0KbfxT5l4;UMzus)o`9LQDrlpjGiZ6(qg zC1wgxupD8w#xr01AoASy6eo%(_~_2xrkdbF2ofNYIH_K;-uw>I>}DD+(TK;!<**7t zNT6hezIFQPD}+Kix~MQjOUPR+0C_BXY^(i=qgQ-=GutEGPT6>wO^koLH8lUyuEjC};A%$sD+n+{VnuHBE1>NU zFH9AT|KVT@#G^5yiD5jV4s0ZL4^7FfRbG=0)m21ju?&Ewa}l!w+%wNV$ya}Z5dsw_I2!QwHIE;}iHy__iRgys6sbbztJY`L}D(rip!{BG*>%)PwX!1LuFU+d5;>kqX zB)x_nT0RsfUmp6>-Y;%HxwI5DZ2*;J67}ex8%*r+8qAE+oHJL;(6O4zDHjYh*kbuE z-qL|q0$5QI0>T#zneX(nEo}0Ny-DRxG*O*QZQqmF5@ksN~N3J8KR%o8{BR zuNQorgZF&A`5rEs=4kkeKQupUB8l&m*dqrMiSjBJ0b-!QfNa6QRH}c`msbbP`Yq<7 zkF08+r#QTWCDoAAp7X(c(aN!ATCk=RLKWEp==ap|X`lrU{8r z3<(bLn(U8avM7HgJ~~q9$z&041}><&EgVcOlZ7KuJ#VaP!do!>{$b;)SMfadFIVeA zVeH5AG6Q8<4)Xd2u{~t8AySfB+}UXaX23)lyuzR!6JEhL$&kEF2!(Ki50Cv`swbB% zU*0-3*)!GgBN3lsT&ICeOrIyVA3RCfEDrhWyiV`ox|%#g+=}G{EpjaID6~mvWTo7H zqe7dT;!_)fbSd$^)!J~ZkbeoKwT>mbj9hea`r^M5*+AAPDqIO)hf1G1MytU(vqF;W zHkKX>pAmlu%AgIAt^{CyGcXgSY2sfqP^)g3@WthM&z)9)%mcM_j$RBu^Z6f{AJ6~f zbUA=p-X5WfGmQ|mPQkKTXWDl3^SF=sf2f1etN-_9B>gLX{v(>vpa1*C|9v+Y{|iC& z|L3(4bH89gyf0{3HWd9+P5uTM5>R93j`x*i0-IFJdID3D2Gxf_zikL-l^~PfLj5G2 zu?}&FB0iS`Dvw0)s@wWgj$(IKB7OG@I_;;c$4^?mIifcZ(`KSOQ5b2ac%X*OVk1gx(NkFB*UZC?+kI z>$*-l94e1G_hO9CkRXBqPIp9w53xvDFc7#Hf#)JHF#@kh1|nDsXh850OurH0NXXU3 z)s19n7*orRU%2ox-?`i>(rq_Q`tq74?seQ)Pzz_q%wm)Q^+e(#D(a_D*sa4i&n#M@ ze;m{)pqh3D2>P0vg%B~(GpnGQ3>zE!N2hT)qKR)3Y8K727xv(g@iMNn7hDR49`%^Q zDz=XWg29#<_)EKC_!*u5D14Ne2xtPl%o2cSI^MCI;xAorsHBaatn>G9!o9fX2apJZ zPhcD}`{$MMd_e(X3IR2u$On4UPUzF%2fNh4ek$!DA-4d?R*{6bx1E&Qlx`Y-108&z z`w9X46=kIU01}@mm;iorB{@=Wj95h|LgZ}>m2qRC<@;W) zLF=xn-zN402JW!50UR`DGv27)Jkb&m3M-wU;#ghP3fBVU4AfSIyvy)r-cRdLE!cmJ z_Ng(SY8qoO>5{u}zsMj;g)-^Yq{`qM+{3tkp?N$fx6}OqC2wpIxR|a->CPfS`GI$b zzhPz?$b&u{oOS|vKf%$!Mv9+B%Mz|u6Sl_{IOsD!nZmZpm7e?^t(FYQE z)i!bw1ZGxakzI++s*qfbo2f;YurqieedK{t9-QiY6r$t3i`b<6_*^oiSd_EnhhRl)8lP(lPbNja(>cvV`m*js2D+X)_2+)=})Ji*3wSf zE739vw7Pgs^SMMdCRRe>ZAYRfRU4^_1*FN)(?=5l>9RzS-LvGzy`-x^E8GbfNfQQi z&@onpiefX+?Zrz=ai4Xq#!xsC?)NB5!h3m}>YraEh=v9+gbeHIDyqI%Er8^zP~%ww zh=TeAw*{KIEKkBHD9D79KWL4=d`8WeRWSL~p{QU9p#7d_f+Y@p2VU81O}kc zO(sg!BLusi8VKO?)c=4mOKNfk_D5WDvKX;yX~kkp}%bwyT#mb7P!lioVwx)`dp9A3lE$S5hX>TNF>v&oQ(IGr^P z7O{N4)W<*puxv~g&~W+CO~knUtiWLLaji0JY%80nrp2^LUtK4b3*^TU6W^k#EFyY6 zrr?Y>j{Xv$;qf~^la0oJU1dNI^}_>dhIwjaDo3+c27KG%osp`h>p-$Ozfy$B}mlbd_nvl!>{~qh|w)D1~ukh z3DdQkYY>&*POKokBZ>rR+^GtYq+}M@ zcWFqgH+%BYtkjNfj0%#@(VDacE2{zlC=Bj(Ghr>5h8c@|e^m#lMec%%!|_aG*^wWn zpKJm{6Cd@sulNf$UP?dz4K@ESK+XSC3n4wA|97e(9n0tow7+9wj#)%d($_hF6uS*P z#Agj>`obdqyIJBWW zCA;w!`L=-R9k-DY>XkRn9O8#1{(uu2ua|LZ7w-81z+nn61_cQCkgZpYRtQaRG4ZaJ9s5-kuTG zwC$pR47g@oPdJee1i<@-zxc;tq#uqZ+Bu29w!ZricByxf25uL9I-Dc0wIu0{QBeXK z#1u?!Ihef=Vt2_6NCKO{5*2)cw-VtURhD&DwN`a~;b;dt*if1!uI2<0?o=OuP!Kq! zMp1{0C3|rxy6Vd4QBk%E^scYDF^W8#y_fs12-OhdO?5*lGO6&Bept@cBOnhx8iHO z0Rq73iDe1&W#`&@4lQqUR*MF+WPvG75Q7M1-aX950qY!p7bKTDb2Hj7aRdYy9`Mc( z=a^7P--400bRbWS0HUgm;ekc?=&dLFzl&bwk;%{-Bk(Rz2Wm1Y3u?6)Fb}0>@M6JA z|KOF|d2x|xxW)@LbvJvRU<#HBu-J{;*aIRae6mRX~HR^9sixkJt8CUTo(lY2iphvkU7 zGgO;?3Xtk_ozRHZfMS~XDO3dKd4L6Vf#$8Q+MpMLjbDk5_fcW5`h)H2J6ZT*0w7$E zUqjt8=NiUZXxgIW5j;g}+*l?Z61Qnwgp=Oa9EVC7Ghq#aD`L!|1$-ISR3U<*?FX;D z3E#~~4BJ(WMm{5F!Wvhiwn7$$3%mjSY&UtZ+ezgwIVWrJhhJkz z)TCQYqCO!8v|nm~(+qwtY$;7T5t44TImQTA&TI#-cT~d3Y8b?BAB-T|=vIL6Z~+5&a)~Jf-5o?Nx{sz!54Q(GFx8YQRD#e}VX;pKq}KH-mL|BI_}Zla)g4 zs{=m%a65sT$4@KpSG02v$bO5ZuR1~(4~eP;>_r8+SqMK481heF*>|u!&V3If{kKPtX@eD~SMlUT~$wK5(#LhH|{EZ+3z(q+rr`p`0RF-JnwJb{@4Y%8VTkwDUX zU%f{Rp<|#-p(Im-4_1SZr3a)qFILNW4u$wK;ozI>a<4C<$9)}qILWUIL5!el13(gW zpZFjT-p2TKd7fEsp}$uHQ1xNhk4O*7lA2UQa~y>~vKdp%Z#qTGi-cn8HUzb-O6Fvs z<2<%cr2n8^NC~jxeip!dg3=KfE|dHc?4}Krn0=5n=-ly67?ZQem_?dbGMHyy39}vl zrQ}=RahL9f{ZRyFjMWIH`U0V}66yLa#G`XHILf#g65Zd_Wd$o#o80%STPk0Mnni}T zgz@s=SdA@^X2a1j=JM@sh&>XlM`r{t9-Fa?cAR!8k}$KtC2)t|tf?u^PY@^^Wr~(= zful~Ksp0+2C`?*d$HdBb@SeRY_GF4RXk+lZEu}h=d#PIwp9p09l9Vl^NQ2X__Ye<6 z_F>{UD2tFNNItPSDa-L)xMF0x12>Wp!jE@J+oR|s`bStW`e;0gF5F5tlDQ(M`wQ?1 zYkZw~nzX};<2djx>p|5o^-&rKC z@6wDbmaM0w@}E^&FJ{*6p)3wCVmp^hQfP!-%DrA<55qULIteG!r{o-2d1BO zKaS|ajkHg13l6U7{wI)}bW}qTsj7SQ;_?;JPH@#SS+HGJ4Q=|9ybLI-lU|6w9AEWC z`j;PIK?J)RVn#Tt2(4dln4F3vE_iS%yRYoXn8*->-B;pG6K4hcR5PKrQRiwD@mvEUALzDHZBO> zS-iiKrIjtdCfOb!lt-T`^+h<4LNk-x#3vBHlbj-pLfQE& zkKz)^X=wb@iT!~BaBpAK$(=TB7nij88D9UN8M@o6MfVCF#i*qbD12maplAWQW6(WY z%D@0$a=J}e?bghn%Wz88O?sR#i&cWvx&>Aw`vtu2gyYlB zkkf?1D`jxKY}*JdT54-i?HcA#96w2%8?~?y#a8XZ!_wFmV~;dLFD6m^J%cC|wTyP~=!Ly9JqsYz~%GAmMuA_QHq<&A~_R_8|RC&K@<2BZ2+Esry*=IxoWB~Wz~EkM*9n}qer$PxhZn%KYakcC5Yr}Bgx zdRS9tfJ4t9M|2*~q5BONdt|?5W9Lt2lab-we`0G|a#~lo4XS#Y5RVE{=pV95Y;`0$ z%l-W(R8M38!k)YV*V1xoCtBYg>^p0Tk|{k&6ei?UB83(9GvqX)WXK29{)F=Ig&tHe z6~uZW4pYxOW39uoBlVO>J<4*LYU_pJfpdO$S%lgOmS1w=6>*%CoW zO%l=0B)8&ma>&p{tr<-E$Iq1@KmY}-7h&60vPG)9uHz!OO2U9ssRRE{erc9?+#?O5 z^xc>tI^s-lb}I=nozun@JGa*{YrvQH61=8BtUj*88ogNgi!@#SIX zkVoa&1p9FUiOFfCvTv_R4j`i}VYrMC0eqlCc{I=gK7%%h(-m-DwoJO#V0?8sLy{M} zb#Ie6jD1u(RXo-ksi~%p$g?)8{y>;kVj>Xw`_>Zzrp8h7+E0oi&P^brG6Q~CXInCe z&{81H$hvnmk1$9IXzrI3F&D#>QcNL{9x(5R!Dm8GR$J7lD- zZIJ5#ntO^|@Z+_R^vLLfX{*9y{P|TN?WZb-cwR=)L6NiyhKW5zWiUR|>PI_K4lPMo zJ@z~!&l=a&m`CwbFiFl)O}00m++oBf^uc|x95KS0#1V3=VbDQo?nwUwvctfy&jxm; zAsr{bEx}8ywnaE%wkrbcMwDJgRMIhHTVSLmHw*e-)>!8ZngO_6gLzzx&5ZoOaXtM8 z3?CW^21Gfh)(oLUx0K&J*ateL5XmobEmT>ds8|9m={-kgIo|F$P2O8UN+xyg)B125 zEHScJ@CuUOkB{nZ&njHMCEYVvgdg2CSC!?r3mb_%%;XxP5gKj4S7q3$)U-hzf!I8* z4vo(3p8%*~-UnPF1}K1+jK$QB$JjDK&G*56YKnw#7}X>3xm21`lLwk{b~W9jd)ZXU z!!2A&x(me#x{>W0%~0JCV8tYXm>E(rpC_dWNp94}7`cm99MwQc_Cym)(=#PBo{_ThF*$2r2`+UM1Hd&No z`*hllRm(4+FhN!B-CGuf#eU!lUFg(1*k=GQrk|T1t41?Ezgf;OdanXmGiGaFb{o5j zZksGRXs)hF#a8{j1sWiAQ}tt9W(YJEaEzjb;-V^1zvpm zi%$m(DnNPYO7=bV*!1bk_R9)BbaMvioK;1lCAE39dXYYG3*$RYG50Nvs0SQ!l-&)a?5H>inBI}`u}3B-7AA^awX z_S*X>u91#oLyg0(#X1V`c4pgl4<+d7>$y!3$+|c z({xj)`;C9Y@k3@nh<8T-QdM$bt-cpq zLMaZWLjtDwG4`C7d1GyJ>3z!rpic)!YKN6i+S;0-iy$tLCpM6`ADrH!uhY&6-v%{H zpPGTB_$GB{kWuy^@@k9##dKyG5MHPAp+@A!&Rd1~Cm7Oa1e;c3J1L@AtLuo$fpiv3 z%?$AC$96O!?|->!^kcDgcE1kgQ^>jkWZjh0`SWTIb)n&7asc&Asrw34sVxi)B5Y7U zc3>CYlTbv&)QR}tAC*>4zzVF|lFQ`22RB=R8xurbMD%%xYib9KFsWvA=@r>uH3%+| z8b#x}Y{4Jk{FekLB0AASCQVIJST7+Itw4>cqL0{@1=q(;EI9^p0-Tb%$f5cL<$S;X zhEtX`8$B=gGeyG>QA3_7puN=M_MTgzjGj^$G_qJ~GZi`L@FIQ>P&d+o4mD<`i1m@8 zYGFs!1%pAwPBOC_B9~4o#*#aj*f=p9Y9gNt#m~jP>3IdBBh-Vv11H#6O8J zZ$n4$K>+?#>L9|XlpRb6YN5#|v_+kPouqMcvW1tU`vqg_v%&I{gnfZ*2?T72groNF zkEl=S30qVM2+ADWPt+~>d`7NZC@{@_8K~Fe`3iCCOI$mJIx79gMKcdGT=Wo3_E*pdKU-3)xsP7WzC>%lpEf%ze`vExEPGIVyc zSB8#;;SWhWA%=9k?!@yZV{iNtJFAPLCRd5*Orwco`VksdwBMd(Q5L823Bj5UN2FAi zydL!gY9NC;CYu9kD8Fv!0X+<(lj+&%Yn9P}O3rNR6eItSZT_t71UdN;_lZdT@nO4y z3PK3^*zuXVi{bNsH6(dUiNYSDSihmtE#%RN9M#*W6B-Zih;^h(vsmKCl@LrrXJEDT zC;O@M=+A+dpvS-|EU3lwCJu`r3SKas7z?CNbuvMbIIjw7uru9Oz?YPPkCn?2S>k&% zZZ|W5km39XA>-&>qe*RuE;p^+AP>-GOB7tWodJCb#)l1V^8E#VeCytf{uv}^ZsuzL zqNd+)a*52p;?928j^n9^P(PQEy$K}Ail`I&FAr0`r71zk_~g~WJ@G{VT8mrAn2X!- z{c?EPLI~#&*vCyVs?eSw(WG)kRa2ufJf}qOi12&6&diAde=+aXUN3+tQAD=^WWRD0 z#l2BH4HVGKo4R@;Inl`SM4b*q1P~!cbTp+HPV3y<@7oA-8lZcXA5C6F^W#p7P!%Gl zt7h_!zZCdEi401=@x#*wUEVp5NmZe4727BeBDiLaNsjpgJvS(q^?kG-KOUU}FSPlQU|P5~aVb z{D2&8hlu~alJB@b-9KT+Vme@9!a>8Q1Qe2;hd;UnjLsR%~Tuq7p-Ac#RvF)i#>UE{C@U+#! z!aF-MJqmO*G4p|Abvm4`NSLoz@?U4LpA2FN}fStd>;SKLn`TObhbzS|)AyI|Rssl=;- zbLC8stn@9L(8fhmf>_0;r!3U#3?%z2%=;^}oX1|BE8)%I)nVr*QD+C^Ekxs_gFL?H zu4D{b80u(8X}~NRrh_Lt_`=K858zDVSR>Hb4Jzh=L}<&=a7Dj>i~@Sz$-P(B7eg%Z zWvz$xajwi!95YEN8M{B>`QN0N<|TfFoK$8F8H@Vu)(jDcKnxC*X`VRI6#(lzDK)Q0B0_24Y*$oA>9TpnYI$VN$BQ1L)C@OqGbM-n`g{64(jeJD#~ z1Rn5`A>~G*=!h8$=%5{%ml%)k#YlB(*};4!qA}4B6<#9+EAKxvvjCk@Vg&g~4+L%y z!C*O6WGCV|U^zVUaSp*si4;Lp<;kuxw-FY|XUd@xzj_F1H-%M9&k_bs4w*#o3~V^2 zIEscyxUgra1l=b`?e6hA20f9uVj}dZ*y#);*1xXe7n5rXOlO7{D_K<`>kD~u4IzWg z73%==_SCnho~5;w@Re9OES{Yl!Vu9G55T7q-_&~3aLx$h4aBkb@?;+%?^!JRGgKq? z;};Kc3D_@ti(U+dQ_P72UHS+z2 z<|yDDEoRc?umrOUZIHwg!Na$~O7{%wg0asw=dP~X^#sY$kEBOPtQMri7i=g*KH;27 z%BG0$JON2^kMJ(3-o8|sk(Y97MPdSBs-qo^1;ND}gsroQ| zoGC=$yzccRDqZ8c`sz1P6vSucSNV?3Ok+uyal-xj-hEWakTsv!{5?!pHltEuX5&;X zLPQCVXhscP5TwdL%(XlS+K^-lva4ew9|XNlqRmUpWf!nhz-nRjekU&CyP&7E-)C|L z;f~Oy=H%JXbR_Zy(z1Mz^+km$0T7Hd!3Y7SoLV|jE=t^^;=z5hC73KM3mW%Hu^KED zIsDf)9Dm7-KB4A1nnG3={JT4om}I!&t2E!!g&(o-3p)qzfs;=WZTLa-)Eebk;5+X3 za0b;T0=l5Qwc=dxBx!gn=AinJrGf^9b<{(^?nCFpD(s%4-*8K!4S7Ky-XalzO_)3a z6{>2_X1pB6&iOHO^ltao0YKA@IP$Py4z-r!v(ZF6@~)4$xXeRc9Fo^lzb=Wb)M7x* zb`n+=#FeOYfxYx zuSE-?B=p^oBz+kA(v#;F9{bb(JrVeSq^@n|GFEhJ1}dIOh(k$KwX`EM9t7d zwK$VdlyanGlX2_1t$%{49*w^wrbErdI6G;zU(HxwoP>g!!q2Z5M6ifld04>L(L}lo z1Mr^F%mny?wLEXaCk47Yl-w;eqNeD|fnH{q=Ks9f7}cAQAB?m>PM+!tAPgG{O|(&g zr=u)JVX(e6VTOzlNTql$=>wO?%2N<@L_V1x{`{SYPp*K#(VVa#CO&~RdI^VobP5Sq z$g~y|uJ+mfjY+zaxl4vzTbJQ3nz;FwBl7xnYOx1RK?nzFE(f zDO~A2iFOp_+sG`=gnYyrQK|ru>4K0K6Qh4(-<3e^_a^DJUvGhg{(rnJALkYW!}abE zcR_NCKDHuF9*KDLRU!2@4ZfoRA2i<9mxT!_Di~2j+X{CiIYjJ2!5DM#`=Nx5ZIXn1 z>Jz{XD<*D<8BK0u)9<-~3=V!cJn<(P_=$)o3FD!k`1y<^rpo`^l!jS@ zvd_k*K{d?CZV$OhjoV~(OYX&ur73O}kG5>t&oCLkE~LCth@q47cz}f3TOr~^zsP|B zyn4e3HFYO<9SB?W@IT)&!I&PLo-r|dgN=EyJHZ%C)0@s0z$`U-MD*IKXBpx@&z1&v zFib_Ev^5e-zu|aQ_x_wiu>n2s85F#r1620S}o@PE8`$wnp| zyT9K7NTh@bt^?)WH!3-Ih9u2{)|xnMTf3p<<&fmr4GE2NZ}m5R*6+Dme!}{D?7%kj z>~+7!wJ!bj3R;KRyKkgLr7PO~a1HpoX84JIPsz+qm$$gSi~TvrYKFf&Jz*r%nSDU) z{}l=3|Bu1!UcX^pk_~19R$oJCFZqIMz(^wB2(Fa07CmFo#dvk?heX`)emrKq#KgF zscO)!2X6}sZ-D%VCFFN_o`L&+dl4FWxEB;*6Ww*DhKL&pqAfvuy3k~`HdK5Sn5OXskpDh@oQip>zQLEo)CCi9 zO~9H)myi>Y0ymAjAh}W$1Kb4Rd?fCv|M)h%aKa}Gzm3B4F}>2hFYwhm0OcohNR#7xL;H@!Obn?b2?Q%sgiU>E0(L|aP_>V!i}K(@r6^Ge z0+*EfzhA(|jZ@kSN4i}hzj-T?Co@d|sehxw&c7gV{CrYFhkZL`MzE=72g~ z@%2Yvrqm03?rqWJB3c$qW1+~?N}YE!K`6>X5Tz2Oa8zL^-tO0=gApr3okzB<^C9N0 z1LLSJz;#*nUICNleo*Ufo#=s0)Mv^n2065+x(1FAR!~pNY$_U17SNGJAQZ$;`QR-T zH=ROerxs>-{sPd|dLM71?b0Xj{TxkEgXPwEM$8!j=a(}dH&7{A^U0agVb)W(9;eS_ zM8*IuVis4ypHIwUT`)==ZBNVt-*bYYV^mAA$eH037&<~}oTkOX8&{7ugRZ7x)0zXq9##BnV7+HE79Q;`VnS$q5efuCN!4v+SQsn# zQNBy=s;1AhX!=4~83N?wIwN=TN1STwDaAM|TS>=*By1^H4wA~31#)DW!9)DkbFMWv zZt?t{BucaGiE{k6xhm9tNb_cyfg>0@Wzk1;r!2e!q$HAlj<$&vG$RqFE*~r$8qPu# zROiQNyuN!uKe_+l_)+FCQ)wV9!U}=?M6q%ZBs|UfHv{r{P@@3oLZb-T9GaPvunn6_ zal%XRa5NB!Oi0vz41#0L&^j#DH?Zuj37JU;v@nN1!0lB+e@EWNm?4_CLib?WsrxKk zM#3j5mVOsb#Q%Lkedl(%1x&#(87R^bfGyAQ5UqlKi;W~`&`WF%AY1=*wY%KHvDEpE zWw{VT93dl$^RKrx{e?2gX@P4e$6uVC*M^Qrvegi|_XQ*eEv;Jog)OGz(6m*(XS%d+ zs|sk!Ck=oha|(63pkQ_iv!l~)=z<18(cfTD5)cdQJ*1U*0g5?VuW(z~D64(x zG?rR??A-o0RE?sI!)#nMy+HttTy2MyN}WTW2#+?#{tY$5AQ;^5XeYv)qKPd;M>;#G zl*Vbpi9~3gjYu}?l|6jzLGQ?@!ia3`V&zY}PNrD+FV?P~l3X2`z4uC;O7fn#6rI`k zo<{DDv3MY`ZMN_?l^1s+v$rhXT8NY8+|qN@eCNGwWqnC&I-1_ldLs8hXO`L*2iy1o z=Y2gfH^9xm`#i)XcjcoRTEAv%gdchF;zdoabz@c(4mvBW?ZGkC4^uq(S1HldLiFk~ z@|y;bF!(=_=?gY!9X+}u2_F`5&?tC0lcnAHmOx$D09(Me{`AM9(_MXO-z+qZA?ube(- zlEv%+E+J9ld*oS>^*xHc1gg*y)|#o@+%G_N`YE`%E>Lha;1FHtzijtraozYTlyqM) z9Vr?I;bz#hT<{g2vQV&v6k2?-XNwDCD?AC_=%@ypQL??T6*}v+GL9dP$;LRm49A`j zRulw8^z2XH8LqD%4+>?Zi5@AwH5{Gq3@D*$)1G-f zW1mPlS`R(QtT?@gn?XvX#Lw@kymz%h`>cq1h!*Tj$+$5GO9OXG@!SzQP{RDbM3Nwd_5Tp47 z1fn6}ux!ugy?p94ru->tJXPW5(#S$bGYu#a!0D^k7Hqr^0q`(rv>U6;udzg>r8R(J zbP(?N`1rzM0y#wAQgF+m#}j(-;>Fp@-r-2{otP7#^Z3l9TY9sb^+G~I0L(Q&q|C-I z!OfK4k#y|s=IQfgc43?#2d6IbMo$cxQAi{ozBmFpDqP<04pxL1;!=`x+`5uqX+1RF zW`(md0pKcOKeB(9&yA^uJ6&B}Pr&|Jg~L@`kg!hx&IjwQ_dbTy_W~WG_TN7rz~!m| zs@XfR-JfE4)eJd;P^@}jr0^5UWK9aZY@0w2RU)iw<2}s%{*Ybc!MF^#xL9jq5m^VM z3J>84)IBu0h!ZS?*)&OFD&I081qFo|BlM;{avOCO(dL#-$7o!lWjkIy2nAw|KR<2m zhD=s-)G+*6vjqe!QkUaYf}_da5YBbL%v6J%8g!piEbt#|I&V}rZqFL4#w>kgZULEN zw>xU$#Mv|q?Uaj4*U-@5Ao*#f3;!`oS|7Hjx!6zAa8G5f_*oqPYym~Wq9!)TH)ecD zJd~g)@Vi(EemD;1HiWJZP<-|znR&7fW9{HoZ{j*1yf3+KIr!y0Cmbh~>bp&FZ#~mX z5)XEI_!VT4A8r^PpyT9}HeTdW&dkA~vwmJ)Gfz)sZA5mY zWuX^-{i2D=p%-N9$Axd+oRhr$Sr z_pEX4?(V*9DE2x3f3^ftRo-a(+qa8wQffA^oyE)ho;vXcQQ&;_aQx>hFln*i=0#Ce zbHcGTqV*mctHsZAIt9~{4n#3~oGgnonKoUVDOiKDHWGs39v2vNqi}dqCt4K0eEz(v zzoS|^=ajPxh$yw#`O1Ni`64k<)+WVYR8t8S9d_##uA&3~3Sx`>+?~kx2I>4ILI6cB z_>pySBU9^eerY6JsRrAx&EE|)QI&9DOMiF6UP;({Vg;N|ftX}sAC5&XuBf1K-!D=2 zYe%RpU~Awi{OLvrtX2lMrm<_6HY}BfR_EI zAygttcm|K^1JM1IFTqR3-8*;h-ZCuY#R2hFdKmzn>27gTK6%za4{?_QO7P_=XTUBOk$-ggDG za8>-4h26d_WSHw}2m#dxjl@@Gi{GbBhi7q)$NN*b>UA}I=LJBSsfSOQ+12U@WuL`$ zIWC#FCHm7Rsy+38q%4X*+v80aEDX>*8voG9*Z1nifjio#KJc4gKh3)Kd- z<}!b)vL}4&(fGCLA}uoyFX7m9DWCcsQDkHO5N-~G=2(^Ea&q>HeFhJ5M>1Hn8gTfq zF3#<)1}tnyv)Kar_62Ie_Erm@%46?kG^WOfPhvJE{#86HY@#fIMl6XKSX-&}tY+%e zA}>Ma6>!_p6QGXjUjs8%Rv&sUd(F>m!nYc*e_2-$XnAq3b+Nr+|BJ^3 zSmFZc;Z>&`da)QPnLdJePO=pud`Dw8C`pz;{QO@g5nKjmEj=kcy+9n69S$y?C|K-> z6w8WGg5 zt39z@IXO8sx!XMj?xFPS!~#-*afj-XOS7p6`SrzN)){swUOD69H#>WKZ3vRQVT*tN z04hD(Ez92WgK@t1;lm;G>%ChSFJ2sM_;0V#%sorRsSuDzRQKAE4UTUUlT`op(YUz6 zkDcQQkwW!A-L_{u{E=6^H%BjUpR*@=JSfV=KnHo z(_e&OwbOrpeYtN5Ppe;t^VVsCK8lQRY7N8u^C*pbD$mfF7yR>8tp44+ygVl_#NPEl zZzY?4Q;_r$-ly&b?s|e7TNSotiq)I6-8jA4!D{Wzod|PPxNUyWOEOb%j>REZiF+5O zf+G9%{>t|eFp}-s)e!F05%{u@L)K=Kx>FRo=tzccScl34Fpge|{`LR$;muc6C3vz5Kd;xa#fK$MRCh9jwz% z)WS8$iPT~Tf}G8{FjFud(5oP`37@wr)!GR1<4)vk->*IAX8-`yfX3h!VJ_pZi@1z` z0+!!|0q-tC-?N5>hIlJTz#Y%S&eCvuc!jrYyTW%l*^Pn_K7YR0*xi@;2gfS+qX9>m zgXo3{3-_+hT_Jxw7p_>!p^3Fe#5C`FWMySNGAWutL=)~bVKo1xI|a61CsK<`G4b8I zPvd!1fqb>`>UyUqvMri&vZv0Klwgw)!uUEP8gcRL4v^4ajV1}gIJn>1z|#veDsX#~ zake!8q-AGK903v#k?VefzkKE^;&lxJfyiYtE1KD(Le6X{abejVJM7-S&MY7@BHi zvz?*xhM1V(UuJePx}Vty-odj}ydT+j>VUpcrp<}Am-*(QumHSv>3R3=4#0{fn#&~0 zXgU#B-l}dqX5Vlb>n=S)(ciy+fah=s&1L>8jB@fNnVaO!5-+&OBWaL!8^26Q89>|r zs_LhWd%}0#)bSV7)&Q;qpo#{lAz(HHM&mdpWyKlgj`+l%S*8)L99*v+Ec5u>Sj1#k zLC?mrl5li%?EL06aJ4oEXHU~z3vOXblVhQL3sllh4JN|i6ozilLjbB`MmeWq<#GF~ z!D~udTjNL;Kq_q*-jZ!vl$IN2XyG(-mDogP4=XOF0)