Add ONNX Gelu operator support

src/SOFIE_core/CMakeLists.txt

@@ -47,6 +47,7 @@ set(source_headers
    SOFIE/ROperator_Erf.hxx
    SOFIE/ROperator_Swish.hxx
    SOFIE/ROperator_Elu.hxx
+   SOFIE/ROperator_Gelu.hxx
    SOFIE/ROperator_Comparision.hxx
    SOFIE/ROperator_EyeLike.hxx
    SOFIE/ROperator_Range.hxx

src/SOFIE_core/inc/SOFIE/ROperator_Gelu.hxx

@@ -0,0 +1,74 @@
+#ifndef SOFIE_ROPERATOR_GELU
+#define SOFIE_ROPERATOR_GELU
+
+#include "SOFIE_common.hxx"
+#include "ROperator.hxx"
+#include "RModel.hxx"
+
+#include <sstream>
+
+namespace SOFIE{
+
+template <typename T>
+class ROperator_GELU final : public ROperator
+{
+
+private:
+
+   std::string fNX;
+   std::string fNY;
+   std::vector<size_t> fShape;
+
+public:
+   ROperator_GELU(){}
+   ROperator_GELU(std::string nameX, std::string nameY):
+      fNX(UTILITY::Clean_name(nameX)), fNY(UTILITY::Clean_name(nameY)){
+         fInputTensorNames = { fNX };
+         fOutputTensorNames = { fNY };
+      }
+
+   std::vector<ETensorType> TypeInference(std::vector<ETensorType> input) override {
+      return input;
+   }
+
+   std::vector<std::vector<size_t>> ShapeInference(std::vector<std::vector<size_t>> input) override {
+      return input;
+   }
+
+   void Initialize(RModel& model) override {
+      //input must be a graph input, or already initialized intermediate tensor
+      if (!model.CheckIfTensorAlreadyExist(fNX)){
+         throw std::runtime_error("TMVA SOFIE GELU Op Input Tensor " + fNX + " is not found in model");
+      }
+      fShape = model.GetTensorShape(fNX);
+      model.AddIntermediateTensor(fNY, model.GetTensorType(fNX), fShape);
+   }
+
+   std::string Generate(std::string OpName) override {
+      OpName = "op_" + OpName;
+      if (fShape.empty()){
+         throw std::runtime_error("TMVA SOFIE GELU operator called to Generate without being initialized first");
+      }
+      std::stringstream out;
+      size_t length = ConvertShapeToLength(fShape);
+
+      // GELU exact formula: y = 0.5 * x * (1 + erf(x / sqrt(2)))
+      // Using hexfloat for compile-time precision:
+      //   0x1.6a09e667f3bcdp-1 = 1/sqrt(2) = 0.7071067811865476 (exact to be precise) 
+      //   0x1.0000000000000p-1 = 0.5
+
+      out << "\n//------ GELU\n";
+      out << SP << "for (int id = 0; id < " << length << " ; id++){\n";
+      out << SP << SP << "tensor_" << fNY << "[id] = 0x1.0000000000000p-1 * tensor_" << fNX
+          << "[id] * (1.0 + std::erf(tensor_" << fNX << "[id] * 0x1.6a09e667f3bcdp-1));\n";
+      out << SP << "}\n";
+      return out.str();
+   }
+
+   std::vector<std::string> GetStdLibs() override { return { std::string("cmath") };}
+};
+
+}// namespace SOFIE
+
+
+#endif //SOFIE_ROPERATOR_GELU

src/SOFIE_parsers/CMakeLists.txt

@@ -62,6 +62,7 @@ set(sources_cxx
     src/ParseExpand.cxx
     src/ParseGather.cxx
     src/ParseElu.cxx
+    src/ParseGelu.cxx
     src/ParseFuseConvAdd.cxx
     src/ParseFuseConvTransposeAdd.cxx
     src/ParseFuseGemmRelu.cxx

src/SOFIE_parsers/src/ParseGelu.cxx

@@ -0,0 +1,45 @@
+#include "SOFIE/RModelParser_ONNX.hxx"
+#include "SOFIE/ROperator_Gelu.hxx"
+#include "onnx_proto3.pb.h"
+
+namespace SOFIE {
+
+ParserFuncSignature ParseGelu = [](RModelParser_ONNX &parser, const onnx::NodeProto &nodeproto) {
+   ETensorType input_type;
+
+   // Check for unsupported tanh approximation attribute
+   for (int_t i = 0; i < nodeproto.attribute_size(); i++) {
+      std::string attribute_name = nodeproto.attribute(i).name();
+      if (attribute_name == "approximate") {
+         if (nodeproto.attribute(i).s() == "tanh") {
+            throw std::runtime_error("TMVA::SOFIE ONNX Parser Gelu tanh approximation not implemented");
+         }
+      }
+   }
+
+   auto input_name = nodeproto.input(0);
+   if (parser.IsRegisteredTensorType(input_name)) {
+      input_type = parser.GetTensorType(input_name);
+   } else {
+      throw std::runtime_error("TMVA::SOFIE ONNX Parser Gelu op has input tensor " + input_name +
+                               " but its type is not yet registered");
+   }
+
+   std::unique_ptr<ROperator> op;
+   std::string output_name = nodeproto.output(0);
+
+   switch (input_type) {
+   case ETensorType::FLOAT: op.reset(new ROperator_GELU<float>(input_name, output_name)); break;
+   default:
+      throw std::runtime_error("TMVA::SOFIE - Unsupported - Operator Gelu does not yet support input type " +
+                               std::to_string(static_cast<int>(input_type)));
+   }
+
+   if (!parser.IsRegisteredTensorType(output_name)) {
+      parser.RegisterTensorType(output_name, input_type);
+   }
+
+   return op;
+};
+
+} // namespace SOFIE

src/SOFIE_parsers/src/RModelParser_ONNX.cxx

@@ -75,6 +75,7 @@ extern ParserFuncSignature ParseLayerNormalization;
 extern ParserFuncSignature ParseGather;
 extern ParserFuncSignature ParseErf;
 extern ParserFuncSignature ParseElu;
+extern ParserFuncSignature ParseGelu;
 extern ParserFuncSignature ParseEyeLike;
 extern ParserFuncSignature ParseRange;
 extern ParserFuncSignature ParseTopK;
@@ -219,6 +220,7 @@ RModelParser_ONNX::RModelParser_ONNX() noexcept : fOperatorsMapImpl(std::make_un
    RegisterOperator("Gather", ParseGather);
    RegisterOperator("Erf", ParseErf);
    RegisterOperator("Elu", ParseElu);
+   RegisterOperator("Gelu", ParseGelu);
    RegisterOperator("EyeLike", ParseEyeLike);
    RegisterOperator("Range", ParseRange);
    RegisterOperator("TopK", ParseTopK);