Merge branch 'Swish' into 'master'

Swish

See merge request ai/esp-dl!189

Author: sun-xiangyu

esp-dl/dl/module/include/dl_module_creator.hpp

@@ -39,6 +39,7 @@
 #include "dl_module_sqrt.hpp"
 #include "dl_module_squeeze.hpp"
 #include "dl_module_sub.hpp"
+#include "dl_module_swish.hpp"
 #include "dl_module_tanh.hpp"
 #include "dl_module_transpose.hpp"
 #include "dl_module_unsqueeze.hpp"
@@ -152,6 +153,7 @@ class ModuleCreator {
             this->register_module("LessOrEqual", LessOrEqual::deserialize);
             this->register_module("ReverseSequence", ReverseSequence::deserialize);
             this->register_module("Identity", Identity::deserialize);
+            this->register_module("Swish", Swish::deserialize);
         }
     }
 

esp-dl/dl/module/include/dl_module_swish.hpp

@@ -0,0 +1,98 @@
+#pragma once
+
+#include "dl_math.hpp"
+#include "dl_module_base.hpp"
+#include "dl_module_lut.hpp"
+
+namespace dl {
+namespace module {
+/**
+ * NOTE:
+ *
+ * @tparam feature_t supports int16_t and int8_t,
+ *         - int16_t: stands for operation in int16_t, implemented by LUT
+ *         - int8_t: stands for operation in int16_t, implemented by LUT
+ *         y = x * sigmoid(x)
+ */
+class Swish : public Module {
+public:
+    /**
+     * @brief Construct a new Swish object.
+     *
+     * @param name            name of module
+     * @param inplace         inplace type.
+     */
+    Swish(const char *name = NULL,
+          module_inplace_t inplace = MODULE_NON_INPLACE,
+          quant_type_t quant_type = QUANT_TYPE_NONE) :
+        Module(name, inplace, quant_type)
+    {
+    }
+
+    /**
+     * @brief Destroy the Swish object.
+     */
+    ~Swish() {}
+
+    std::vector<std::vector<int>> get_output_shape(std::vector<std::vector<int>> &input_shapes)
+    {
+        std::vector<std::vector<int>> output_shapes(1, input_shapes[0]);
+        return output_shapes;
+    }
+
+    void forward(ModelContext *context, runtime_mode_t mode = RUNTIME_MODE_AUTO)
+    {
+        if (quant_type == QUANT_TYPE_SYMM_8BIT) {
+            forward_template<int8_t>(context, mode);
+        } else if (quant_type == QUANT_TYPE_SYMM_16BIT) {
+            forward_template<int16_t>(context, mode);
+        }
+    }
+
+    template <typename T>
+    void forward_template(ModelContext *context, runtime_mode_t mode)
+    {
+        TensorBase *input = context->get_tensor(m_inputs_index[0]);
+        TensorBase *output = context->get_tensor(m_outputs_index[0]);
+        T *input_ptr = (T *)input->get_element_ptr();
+        T *output_ptr = (T *)output->get_element_ptr();
+
+        float input_scale = DL_SCALE(input->exponent);
+        float output_scale = DL_RESCALE(output->exponent);
+        for (size_t i = 0; i < input->size; i++) {
+            float temp = input_ptr[i] * input_scale;
+            temp = dl::math::sigmoid(temp) * temp;
+            tool::truncate(output_ptr[i], tool::round(temp * output_scale));
+        }
+    }
+
+    void forward_args(void *args) {}
+
+    /**
+     * @brief deserialize Swish module instance by node serialization information
+     */
+    static Module *deserialize(fbs::FbsModel *fbs_model, std::string node_name)
+    {
+        Module *op = nullptr;
+        quant_type_t quant_type;
+        fbs_model->get_operation_attribute(node_name, "quant_type", quant_type);
+
+        // Create module
+        if (quant_type == QUANT_TYPE_SYMM_8BIT) {
+            TensorBase *table = fbs_model->get_operation_lut(node_name);
+            if (table) {
+                op = new LUT(node_name.c_str(), table, MODULE_INPLACE_CHANGED_BUFFER, quant_type);
+            } else {
+                op = new Swish(node_name.c_str(), MODULE_INPLACE_CHANGED_BUFFER, quant_type);
+            }
+        } else {
+            op = new Swish(node_name.c_str(), MODULE_INPLACE_CHANGED_BUFFER, quant_type);
+        }
+
+        return op;
+    }
+
+    void print() { ESP_LOGI("Swish", "quant_type: %s.", quant_type_to_string(quant_type)); }
+};
+} // namespace module
+} // namespace dl

tools/ops_test/config/op_cfg.toml

@@ -2445,7 +2445,13 @@
         time_axis = 1
         export_name_prefix = "ReverseSequence_ishape_1_5_20"
 
-
+    [ops_test.Swish]
+    test_func = "SWISH_TEST"
+    quant_bits = ["int8", "int16"]
+    package = "torch_ops_test"
+        [[ops_test.Swish.cfg]]
+        input_shape = [1, 96, 20, 20]
+        export_name_prefix = "swish_ishap_1_96_20_20"
 
 
 [models_test]

tools/ops_test/torch_ops_test.py

@@ -858,6 +858,16 @@ def forward(self, input):
         return output
 
 
+class SWISH_TEST(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.op = nn.SiLU()
+
+    def forward(self, input):
+        return self.op(input)
+
+
 if __name__ == "__main__":
     print(f"Test {os.path.basename(sys.argv[0])} Module Start...")