[hrx/gfx1151] DSV4.0 Flash optimizations

ggml/src/ggml-hrx/CMakeLists.txt

@@ -220,6 +220,8 @@ ggml_add_backend_library(ggml-hrx
     dispatch_registration/common/dispatch-get-rows.h
     dispatch_registration/common/dispatch-glu.cpp
     dispatch_registration/common/dispatch-glu.h
+    dispatch_registration/common/dispatch-hyper-connection.cpp
+    dispatch_registration/common/dispatch-hyper-connection.h
     dispatch_registration/common/dispatch-mul-mat-id.cpp
     dispatch_registration/common/dispatch-mul-mat-id-common.h
     dispatch_registration/common/dispatch-mul-mat-id.h
@@ -235,6 +237,8 @@ ggml_add_backend_library(ggml-hrx
     dispatch_registration/common/dispatch-scale.cpp
     dispatch_registration/common/dispatch-scale.h
     dispatch_registration/common/dispatch-unary.cpp
+    dispatch_registration/common/dispatch-reduction.cpp
+    dispatch_registration/common/dispatch-reduction.h
     dispatch_registration/common/dispatch-unary.h
     dispatch_registration/common/iq-codebook-resource.cpp
     dispatch_registration/common/iq-codebook-resource.h

ggml/src/ggml-hrx/README.md

@@ -52,6 +52,8 @@ jq -c 'select(.event == "compile_request") | {cache_id, execution_id, kernel, in
 
 `hrx-minimum-commit.txt` records the minimum required commit in ROCm/hrx-system. It contains one full Git commit SHA. The backend's C++ APIs, Loom syntax, and numerical behavior can all depend on that revision. When `GGML_HRX=ON` and `HRX_SOURCE_DIR` is set, CMake requires the HRX checkout's HEAD to equal or descend from the recorded commit.
 
+The corpus uses typed `check.expect.approximate` operations introduced in [HRX d03575e84](https://github.com/ROCm/hrx-system/commit/d03575e84f602d3e39b87895a30fae214f308558).
+
 For builds using installed packages, check the source revision used to produce both HRX and Loom against this floor before validating a repair. From the staging checkout:
 
 ```sh

ggml/src/ggml-hrx/dispatch/dispatch-scheduler.cpp

@@ -135,7 +135,8 @@ static bool can_elide_zero_output_node(const Graph &             graph,
 }
 
 static bool can_elide_node(const Graph & graph, const GraphNode & node, const std::vector<bool> & covered_nodes) {
-    return can_elide_layout_alias_node(graph, node, covered_nodes) ||
+    return (node.op == GGML_OP_NONE && node.inputs.empty()) ||
+           can_elide_layout_alias_node(graph, node, covered_nodes) ||
            can_elide_zero_output_node(graph, node, covered_nodes);
 }
 
@@ -204,10 +205,10 @@ bool DispatchScheduler::schedule_graph(Graph &                       graph,
                 covered_nodes[i] = true;
                 continue;
             }
-            plan_.status.append(pending_diagnostics);
-            plan_.status.append(match.status);
             const std::string message = unsupported_node_message(graph, i, *node);
             plan_.status.log("%s", message.c_str());
+            plan_.status.append(match.status);
+            plan_.status.append(pending_diagnostics);
             if (diagnostics != nullptr) {
                 diagnostics->unsupported_node_index = i;
                 diagnostics->unsupported_node       = node;

ggml/src/ggml-hrx/dispatch_registration/common/dispatch-binary.cpp

@@ -193,18 +193,14 @@ static void add_binary_strided_parameters(Dispatch & dispatch,
     dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.ne0", std::to_string(output.ne[0]));
     dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.ne1", std::to_string(output.ne[1]));
     dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.ne2", std::to_string(output.ne[2]));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src0_stride1",
-                                               std::to_string(lhs.nb[1] / sizeof(float)));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src0_stride2",
-                                               std::to_string(lhs.nb[2] / sizeof(float)));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src0_stride3",
-                                               std::to_string(lhs.nb[3] / sizeof(float)));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src1_stride1",
-                                               std::to_string(rhs.nb[1] / sizeof(float)));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src1_stride2",
-                                               std::to_string(rhs.nb[2] / sizeof(float)));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src1_stride3",
-                                               std::to_string(rhs.nb[3] / sizeof(float)));
+    for (int dim = 1; dim < GGML_MAX_DIMS; ++dim) {
+        dispatch.kernel.compile_parameters.emplace(
+            "ggml.binary_f32.src0_stride" + std::to_string(dim),
+            std::to_string(broadcast_dim_flag(lhs, output, dim) ? 0 : lhs.nb[dim] / sizeof(float)));
+        dispatch.kernel.compile_parameters.emplace(
+            "ggml.binary_f32.src1_stride" + std::to_string(dim),
+            std::to_string(broadcast_dim_flag(rhs, output, dim) ? 0 : rhs.nb[dim] / sizeof(float)));
+    }
     dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src0_span",
                                                std::to_string(lhs_byte_count / sizeof(float)));
     dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src1_span",
@@ -298,9 +294,14 @@ static bool match_binary_f32_dispatch(const DispatchMatchContext & context, Disp
     }
 
     const bool same_shape_inputs = same_shape(*lhs, *output) && same_shape(*rhs, *output);
+    const bool strided_broadcast = !same_shape_inputs && (!lhs->contiguous || !rhs->contiguous ||
+                                                          !packed_f32_layout(*lhs) || !packed_f32_layout(*rhs));
+    if (!binary_kind_allows_broadcast(params->op, *lhs, *rhs, *output)) {
+        return false;
+    }
     const CommonActivationPublicationCapabilities capabilities = {
-        binary_q8_publication_enabled(params->op, *output) ? DispatchActivationInputQ8_1X4 :
-                                                             DispatchActivationInputNone,
+        !strided_broadcast && binary_q8_publication_enabled(params->op, *output) ? DispatchActivationInputQ8_1X4 :
+                                                                                   DispatchActivationInputNone,
         DispatchActivationInputQ8_1X4,
     };
     const CommonActivationPublicationPlan publication_plan =
@@ -310,22 +311,21 @@ static bool match_binary_f32_dispatch(const DispatchMatchContext & context, Disp
     const bool publish_q8 = q8_demand != nullptr;
 
     Dispatch dispatch;
-    if (same_shape_inputs) {
+    if (same_shape_inputs || strided_broadcast) {
         dispatch.kernel = make_kernel_specialization(publish_q8 ? kBinaryF32PublishQ8_1X4Kernel : kBinaryF32Kernel);
         add_binary_strided_parameters(dispatch, *lhs, *rhs, *output, lhs_byte_count, rhs_byte_count);
         if (publish_q8) {
             dispatch.kernel.integer_parameters.erase("element_count");
             dispatch.kernel.integer_parameters.emplace("token_count", output->ne[1]);
+        } else {
+            dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src0_stride0",
+                                                       std::to_string(broadcast_dim_flag(*lhs, *output, 0) ? 0 : 1));
+            dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src1_stride0",
+                                                       std::to_string(broadcast_dim_flag(*rhs, *output, 0) ? 0 : 1));
         }
         dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.op",
                                                    std::to_string(binary_kind_config_value(params->op)));
     } else {
-        if (!lhs->contiguous || !rhs->contiguous || !packed_f32_layout(*lhs) || !packed_f32_layout(*rhs)) {
-            return false;
-        }
-        if (!binary_kind_allows_broadcast(params->op, *lhs, *rhs, *output)) {
-            return false;
-        }
         dispatch.kernel = make_kernel_specialization(publish_q8 ? kBinaryBcF32PublishQ8_1X4Kernel : kBinaryBcF32Kernel);
         if (publish_q8) {
             dispatch.kernel.integer_parameters.emplace("token_count", output->ne[1]);
@@ -356,6 +356,33 @@ static bool match_binary_f32_dispatch(const DispatchMatchContext & context, Disp
     return match.status.success();
 }
 
+static bool match_add_f16(const DispatchMatchContext & context, DispatchMatch & match) {
+    const GraphNode * node = context.root_node;
+    if (node == nullptr || node->op != GGML_OP_ADD || node->inputs.size() != 2) {
+        return false;
+    }
+    const Value * lhs    = graph_value(context.graph, node->inputs[0]);
+    const Value * rhs    = graph_value(context.graph, node->inputs[1]);
+    const Value * output = graph_value(context.graph, node->output);
+    if (lhs == nullptr || rhs == nullptr || output == nullptr || lhs->type != GGML_TYPE_F16 ||
+        rhs->type != GGML_TYPE_F16 || output->type != GGML_TYPE_F16 || !lhs->contiguous || !rhs->contiguous ||
+        !output->contiguous || lhs->ne != output->ne || rhs->ne != output->ne || output->element_count < 1 ||
+        output->element_count > 134217728 || !binary_output_storage_is_safe(*lhs, *rhs, *output)) {
+        return false;
+    }
+    Dispatch dispatch;
+    dispatch.kernel = make_kernel_specialization(GGML_HRX_KERNEL_REF("loom_libs", "ggml_add_f16"));
+    dispatch.kernel.integer_parameters.emplace("element_count", output->element_count);
+    dispatch.bindings = {
+        { lhs->id,    0, lhs->byte_count    },
+        { rhs->id,    0, rhs->byte_count    },
+        { output->id, 0, output->byte_count }
+    };
+    match.dispatches.push_back(std::move(dispatch));
+    match.covered_nodes.push_back(context.root_index);
+    return true;
+}
+
 static void register_binary_dispatch_for(DispatchRegistryBuilder & registry, ggml_op root_op) {
     registry.add({
         "common.binary_f32",
@@ -370,6 +397,8 @@ static void register_binary_dispatch_for(DispatchRegistryBuilder & registry, ggm
 }  // namespace
 
 void register_binary_dispatch(DispatchRegistryBuilder & registry) {
+    registry.add(
+        { "common.add.f16", GGML_OP_ADD, DispatchMatchKind::SingleOp, 0, DispatchSource::Common, match_add_f16 });
     registry.add({
         "common.binary_swiglu_symmetric_i4_k32",
         GGML_OP_GLU,

ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp

@@ -8,15 +8,27 @@
 #include "dispatch-gather-add.h"
 #include "dispatch-get-rows.h"
 #include "dispatch-glu.h"
+#include "dispatch-hyper-connection.h"
 #include "dispatch-mul-mat-id.h"
 #include "dispatch-mul-mat.h"
+#include "dispatch-reduction.h"
 #include "dispatch-rmsnorm.h"
 #include "dispatch-rope-set-rows.h"
 #include "dispatch-scale.h"
 #include "dispatch-unary.h"
 
+#include <limits>
+#include <sstream>
+
 namespace ggml::hrx {
 
+std::string float_config_value(float value) {
+    std::ostringstream out;
+    out.precision(std::numeric_limits<float>::max_digits10);
+    out << value;
+    return out.str();
+}
+
 void register_common_dispatches(DispatchRegistryBuilder & registry) {
     register_binary_dispatch(registry);
     register_copy_dispatch(registry);
@@ -26,8 +38,10 @@ void register_common_dispatches(DispatchRegistryBuilder & registry) {
     register_gather_add_dispatch(registry);
     register_get_rows_dispatches(registry);
     register_glu_dispatches(registry);
+    register_hyper_connection_dispatches(registry);
     register_mul_mat_id_dispatches(registry);
     register_mul_mat_dispatches(registry);
+    register_reduction_dispatches(registry);
     register_rope_set_rows_dispatches(registry);
     register_scale_dispatch(registry);
     register_unary_dispatch(registry);

ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.h

@@ -2,8 +2,12 @@
 
 #include "../dispatch-registry.h"
 
+#include <string>
+
 namespace ggml::hrx {
 
+std::string float_config_value(float value);
+
 void register_common_dispatches(DispatchRegistryBuilder & registry);
 
 }  // namespace ggml::hrx

ggml/src/ggml-hrx/dispatch_registration/common/dispatch-copy.cpp

@@ -61,7 +61,9 @@ static void add_concat_strided_source_parameters(KernelSpecialization & kernel,
     }
 }
 
-static bool make_copy_f32_dispatch(const Value & source, const Value & output, Dispatch & dispatch) {
+}  // namespace
+
+bool make_copy_f32_dispatch(const Value & source, const Value & output, Dispatch & dispatch) {
     if (source.type != GGML_TYPE_F32 || output.type != GGML_TYPE_F32 || !output.contiguous ||
         !packed_f32_layout(output) || !f32_element_strides(source) || source.element_count <= 0 ||
         source.element_count != output.element_count ||
@@ -74,7 +76,7 @@ static bool make_copy_f32_dispatch(const Value & source, const Value & output, D
         dispatch.kernel = make_kernel_specialization(kCopyF32Kernel);
         dispatch.kernel.workload_specialization = WorkloadSpecialization::Dynamic;
     } else {
-        if (!strided_f32_storage_span_bytes(source, source_span)) {
+        if (!strided_f32_storage_span_bytes(source, source_span, true)) {
             return false;
         }
         const size_t source_span_elements = source_span / sizeof(float);
@@ -97,6 +99,69 @@ static bool make_copy_f32_dispatch(const Value & source, const Value & output, D
     return true;
 }
 
+namespace {
+
+static bool make_copy_bits_dispatch(const Value & source,
+                                    const Value & output,
+                                    Dispatch &    dispatch,
+                                    bool          repeat = false) {
+    if (source.type != output.type ||
+        (source.type != GGML_TYPE_F16 && source.type != GGML_TYPE_I32 && source.type != GGML_TYPE_F32) ||
+        !output.contiguous || source.element_count < 1 || (!repeat && source.element_count != output.element_count) ||
+        output.element_count < 1 || output.element_count > static_cast<int64_t>(kMaximumCopyElements) ||
+        source.storage == output.storage) {
+        return false;
+    }
+    const size_t element_size = ggml_type_size(source.type);
+    if (source.byte_count / element_size > kMaximumCopyElements) {
+        return false;
+    }
+    for (int dim = 0; dim < GGML_MAX_DIMS; ++dim) {
+        if (source.ne[dim] < 1 || source.nb[dim] == 0 || source.nb[dim] % element_size != 0 ||
+            source.nb[dim] / element_size > kMaximumCopyElements) {
+            return false;
+        }
+    }
+    dispatch.kernel = make_kernel_specialization(GGML_HRX_KERNEL_REF("loom_libs", "ggml_copy_bits"));
+    dispatch.kernel.integer_parameters.emplace("element_count", output.element_count);
+    dispatch.kernel.integer_parameters.emplace("source_span", source.byte_count / element_size);
+    auto & config = dispatch.kernel.compile_parameters;
+    config.emplace("ggml.copy_bits.element_bits", std::to_string(element_size * 8));
+    for (int dim = 0; dim < GGML_MAX_DIMS; ++dim) {
+        if (repeat && (output.ne[dim] < 1 || output.ne[dim] % source.ne[dim] != 0)) {
+            return false;
+        }
+        if (dim < 3) {
+            config.emplace("ggml.copy_bits.ne" + std::to_string(dim),
+                           std::to_string(repeat ? output.ne[dim] : source.ne[dim]));
+        }
+        config.emplace("ggml.copy_bits.source_ne" + std::to_string(dim), std::to_string(source.ne[dim]));
+        config.emplace("ggml.copy_bits.stride" + std::to_string(dim), std::to_string(source.nb[dim] / element_size));
+    }
+    dispatch.bindings = {
+        { source.id, 0, source.byte_count },
+        { output.id, 0, output.byte_count }
+    };
+    return true;
+}
+
+static bool match_cont_bits_dispatch(const DispatchMatchContext & context, DispatchMatch & match) {
+    const GraphNode * node = context.root_node;
+    if (node == nullptr || (node->op != GGML_OP_CONT && node->op != GGML_OP_REPEAT) || node->inputs.size() != 1) {
+        return false;
+    }
+    const Value * source = graph_value(context.graph, node->inputs[0]);
+    const Value * output = graph_value(context.graph, node->output);
+    Dispatch      dispatch;
+    if (source == nullptr || output == nullptr ||
+        !make_copy_bits_dispatch(*source, *output, dispatch, node->op == GGML_OP_REPEAT)) {
+        return false;
+    }
+    match.dispatches.push_back(std::move(dispatch));
+    match.covered_nodes.push_back(context.root_index);
+    return true;
+}
+
 static bool match_copy_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & match) {
     const GraphNode * node = context.root_node;
     if (node == nullptr || node->op != GGML_OP_CPY || node->inputs.size() != 2) {
@@ -209,7 +274,69 @@ static bool match_concat_dim0_f32_dispatch(const DispatchMatchContext & context,
 
 }  // namespace
 
+static bool match_concat_packed(const DispatchMatchContext & context, DispatchMatch & match) {
+    const GraphNode * node = context.root_node;
+    if (node == nullptr || node->op != GGML_OP_CONCAT || node->inputs.size() != 2) {
+        return false;
+    }
+    const Value * lhs    = graph_value(context.graph, node->inputs[0]);
+    const Value * rhs    = graph_value(context.graph, node->inputs[1]);
+    const Value * output = graph_value(context.graph, node->output);
+    if (lhs == nullptr || rhs == nullptr || output == nullptr ||
+        (lhs->type != GGML_TYPE_F16 && lhs->type != GGML_TYPE_F32 && lhs->type != GGML_TYPE_I32) ||
+        rhs->type != lhs->type || output->type != lhs->type || !lhs->contiguous || !rhs->contiguous ||
+        !output->contiguous || output->element_count < 1 || output->element_count > 134217728 ||
+        lhs->storage == output->storage || rhs->storage == output->storage) {
+        return false;
+    }
+    int axis = -1;
+    for (int dim = 0; dim < GGML_MAX_DIMS; ++dim) {
+        if (lhs->ne[dim] < 1 || rhs->ne[dim] < 1) {
+            return false;
+        }
+        if (output->ne[dim] == lhs->ne[dim] + rhs->ne[dim]) {
+            if (axis >= 0) {
+                return false;
+            }
+            axis = dim;
+        } else if (lhs->ne[dim] != output->ne[dim] || rhs->ne[dim] != output->ne[dim]) {
+            return false;
+        }
+    }
+    if (axis < 0 || (axis == 0 && lhs->type == GGML_TYPE_F32)) {
+        return false;
+    }
+    int64_t lhs_chunk = 1, rhs_chunk = 1;
+    for (int dim = 0; dim <= axis; ++dim) {
+        lhs_chunk *= lhs->ne[dim];
+        rhs_chunk *= rhs->ne[dim];
+    }
+    Dispatch dispatch;
+    dispatch.kernel = make_kernel_specialization(GGML_HRX_KERNEL_REF("loom_libs", "ggml_concat_packed"));
+    dispatch.kernel.integer_parameters.emplace("element_count", output->element_count);
+    dispatch.kernel.compile_parameters.emplace("ggml.concat_packed.element_bits",
+                                               std::to_string(8 * ggml_type_size(lhs->type)));
+    dispatch.kernel.compile_parameters.emplace("ggml.concat_packed.lhs_chunk", std::to_string(lhs_chunk));
+    dispatch.kernel.compile_parameters.emplace("ggml.concat_packed.rhs_chunk", std::to_string(rhs_chunk));
+    dispatch.kernel.compile_parameters.emplace("ggml.concat_packed.lhs_span", std::to_string(lhs->element_count));
+    dispatch.kernel.compile_parameters.emplace("ggml.concat_packed.rhs_span", std::to_string(rhs->element_count));
+    dispatch.bindings = {
+        { lhs->id,    0, lhs->byte_count    },
+        { rhs->id,    0, rhs->byte_count    },
+        { output->id, 0, output->byte_count }
+    };
+    match.dispatches.push_back(std::move(dispatch));
+    match.covered_nodes.push_back(context.root_index);
+    return true;
+}
+
 void register_copy_dispatch(DispatchRegistryBuilder & registry) {
+    registry.add({ "common.repeat_bits", GGML_OP_REPEAT, DispatchMatchKind::SingleOp, 0, DispatchSource::Common,
+                   match_cont_bits_dispatch });
+    registry.add({ "common.concat_packed", GGML_OP_CONCAT, DispatchMatchKind::SingleOp, 10, DispatchSource::Common,
+                   match_concat_packed });
+    registry.add({ "common.cont_bits", GGML_OP_CONT, DispatchMatchKind::SingleOp, -10, DispatchSource::Common,
+                   match_cont_bits_dispatch });
     registry.add({
         "common.copy_f32",
         GGML_OP_CPY,

ggml/src/ggml-hrx/dispatch_registration/common/dispatch-copy.h

@@ -4,6 +4,8 @@
 
 namespace ggml::hrx {
 
+bool make_copy_f32_dispatch(const Value & source, const Value & output, Dispatch & dispatch);
+
 void register_copy_dispatch(DispatchRegistryBuilder & registry);
 
 }  // namespace ggml::hrx