Adaptation in Diff & Mult to allow Yolo like models

include/eddl/layers/operators/layer_operators.h

@@ -30,7 +30,9 @@ class OperatorLayer : public Layer {
 public:
 
     int binary;
+    int in_tensor;
     float val;
+    Tensor *val_tensor;
 
     OperatorLayer(string name, int dev, int mem);
 
@@ -71,6 +73,7 @@ class LDiff : public OperatorLayer {
     int left;
     vector<Tensor *> tin;
 
+	LDiff(Tensor *t, Layer *l, string name, int dev, int mem);
     LDiff(Layer *l1, Layer *l2, string name, int dev, int mem);
     LDiff(Layer *l, float k, string name, int dev, int mem);
     LDiff(float k, Layer *l, string name, int dev, int mem);
@@ -212,6 +215,7 @@ class LMult : public OperatorLayer {
 public:
     static int total_layers;
 
+    LMult(Layer *l, Tensor *t, string name, int dev, int mem);
     LMult(Layer *l1, Layer *l2, string name, int dev, int mem);
     LMult(Layer *l, float k, string name, int dev, int mem);
 

include/eddl/serialization/onnx/layers/operators/diff_onnx.h

@@ -11,6 +11,7 @@
 // OPSET: 13, 7
 Layer* build_diff_layer(onnx::NodeProto *node,
                         map<string, vector<float>> &map_init_values,
+                        map<string, vector<int>> &map_init_dims,
                         map<string, Layer *> &output_node_map,
                         int dev,
                         int mem);

src/layers/operators/layer_diff.cpp

@@ -33,7 +33,6 @@ int LDiff::total_layers = 0;
 LDiff::LDiff(Layer *l1, Layer *l2, string name, int dev, int mem) : OperatorLayer(name, dev, mem) {
     if(name.empty()) this->name = "diff_" + to_string(++total_layers);
     binary=1;
-
     input=l1->output;
     tin.push_back(l1->output);
     tin.push_back(l2->output);
@@ -71,17 +70,42 @@ LDiff::LDiff(Layer *l, float k, string name, int dev, int mem) : OperatorLayer(n
     addparent(l);
 }
 
+/**
+  @brief Computes the subtraction operation between a constant Tensor and a Layer
+
+  @param t a Tensor.
+  @param l a Layer.
+  @param name a name for the operation (predefined as 'diff+TotalDiffLayers')
+  @param dev which computing service utilize
+
+  @returns the result of t-l element-wise over l
+
+  */
+LDiff::LDiff(Tensor *t, Layer *l, string name, int dev, int mem) : OperatorLayer(name, dev, mem) {
+  std::cout << "Nuevo Constructor LDiff" << std::endl;
+  if(name.empty()) this->name = "diff_" + to_string(++total_layers);
+  binary=1;
+  input= l->output;
+  val_tensor = t;
+  val_tensor->toDevice(dev);
+
+  tin.push_back(val_tensor);
+  tin.push_back(l->output);
+
+  output = new Tensor(l->output->shape, dev);
+
+  l->addchild(this);
+  addparent(l);
+}
+
 LDiff::LDiff(float k, Layer *l, string name, int dev, int mem) : OperatorLayer(name, dev, mem) {
     if(name.empty()) this->name = "diff" + to_string(++total_layers);
     val=k;
     left=0;
 
     input=l->output;
-
-
     output = new Tensor(l->output->shape, dev);
 
-
     l->addchild(this);
     addparent(l);
 }
@@ -138,14 +162,20 @@ Layer *LDiff::share(int c, int bs, vector<Layer *> p) {
 
 Layer *LDiff::clone(int c, int bs, vector<Layer *> p, int todev) {
     LDiff *n;
-    if (binary)
-        n = new LDiff(p[0], p[1], "clone_" + to_string(c) + name, todev, this->mem_level);
+    if (binary) {
+        if (p.size() == 1) {
+          n = new LDiff(val_tensor, p[0], "clone_" + to_string(c) + name, todev, this->mem_level);
+        }
         else {
-          if (left)
-            n = new LDiff(p[0], val, "clone_" + to_string(c) + name, todev, this->mem_level);
-          else
-            n = new LDiff(val, p[0], "clone_" + to_string(c) + name, todev, this->mem_level);
+          n = new LDiff(p[0], p[1], "clone_" + to_string(c) + name, todev, this->mem_level);
         }
+    }
+    else {
+      if (left)
+        n = new LDiff(p[0], val, "clone_" + to_string(c) + name, todev, this->mem_level);
+      else
+        n = new LDiff(val, p[0], "clone_" + to_string(c) + name, todev, this->mem_level);
+    }
     n->orig = this;
     return n;
 }

src/layers/operators/layer_mult.cpp

@@ -55,8 +55,34 @@ LMult::LMult(Layer *l, float k, string name, int dev, int mem) : OperatorLayer(n
     addparent(l);
 }
 
+/**
+  @brief Computes the sum operation between a layer and a float
+
+  @param l a Layer.
+  @param t a Tensor.
+  @param name a name for the operation (predefined as 'sum+TotaLMultLayers')
+  @param dev which computing service utilize
+
+  @returns the result of l+k element-wise over l
+  */
+LMult::LMult(Layer *l, Tensor *t, string name, int dev, int mem) : OperatorLayer(name, dev, mem) {
+    if(name.empty()) this->name = "mult_" + to_string(++total_layers);
+    in_tensor = 1;
+    val_tensor = t;
+    val_tensor->toDevice(dev);
+
+    input=l->output;
+    output = new Tensor(l->output->shape, dev);
+
+    l->addchild(this);
+    addparent(l);
+}
+
 void LMult::forward() {
     if (binary) Tensor::el_mult(parent[0]->output, parent[1]->output, output, 0);
+    else if (in_tensor) {
+        Tensor::el_mult(parent[0]->output, val_tensor, output, 0);
+    }    
     else {
         Tensor::mult(parent[0]->output, output, val);
     }
@@ -67,6 +93,10 @@ void LMult::backward() {
         Tensor::el_mult(delta,parent[0]->output,parent[1]->delta,1);
         Tensor::el_mult(delta,parent[1]->output,parent[0]->delta,1);
     }
+    else if (in_tensor) {
+        delta->mult_(input);
+        Tensor::inc(delta,parent[0]->delta);
+    }
     else {
         delta->mult_(val);
         Tensor::inc(delta,parent[0]->delta);
@@ -79,8 +109,10 @@ Layer *LMult::share(int c, int bs, vector<Layer *> p) {
 
 Layer *LMult::clone(int c, int bs, vector<Layer *> p, int todev) {
     LMult *n;
-    if (binary)
+    if (binary) 
         n = new LMult(p[0], p[1],  name, todev, this->mem_level);
+    else if (in_tensor)
+        n = new LMult(p[0], val_tensor,  name, todev, this->mem_level);
     else
         n = new LMult(p[0], val,  name, todev, this->mem_level);
     n->orig = this;

src/serialization/onnx/net/layers/layers_onnx.cpp

@@ -272,7 +272,7 @@ Layer* build_layer_from_node(onnx::NodeProto *node,
       new_layer = build_sqrt_layer(node, output_node_map, dev, mem);
       break;
     case ONNX_LAYERS::SUB:
-      new_layer = build_diff_layer(node, map_init_values, output_node_map, dev, mem);
+      new_layer = build_diff_layer(node, map_init_values, map_init_dims, output_node_map, dev, mem);
       break;
     case ONNX_LAYERS::RMAX:
       new_layer = build_rmax_layer(node, output_node_map, dev, mem);

src/serialization/onnx/net/layers/operators/diff_onnx.cpp

@@ -1,46 +1,115 @@
 #if defined(cPROTO)
 #include "eddl/serialization/onnx/layers/operators/diff_onnx.h"
-
 // ONNX import
 Layer* build_diff_layer(onnx::NodeProto *node,
                         map<string, vector<float>> &map_init_values,
+                        map<string, vector<int>> &map_init_dims,
                         map<string, Layer *> &output_node_map,
                         int dev,
                         int mem)
 {
   string first_operator_name = node->input(0);
   string second_operator_name = node->input(1);
+  vector<float> first_operator_scalars;
+  vector<float> second_operator_scalars;
 
-  if(map_init_values.count(first_operator_name)) // k - T
+  if (map_init_values.count(first_operator_name)) 
+  {
+    // los valores que nos interesan están aquí
+    first_operator_scalars = map_init_values[first_operator_name];
+  }
+  else if (map_init_values.count(second_operator_name)) {
+    second_operator_scalars = map_init_values[second_operator_name];
+  }
+
+  if (!first_operator_scalars.empty() && first_operator_scalars.size() == 1)         // k - T
   {
-    // Read the input scalar
-    vector<float> scalars = map_init_values[first_operator_name];
-    if (scalars.size() != 1)
-    {
-      msg("Error: The first input operand of the Diff layer " + node->name() + " is not valid", "ONNX::ImportNet");
-      return nullptr;
-    }
     Layer *second_operator = output_node_map[second_operator_name];
-    return new LDiff(scalars[0], second_operator, node->name(), dev, mem);
+    return new LDiff(first_operator_scalars[0], second_operator, node->name(), dev, mem);
+  } 
+  else if (!second_operator_scalars.empty() && second_operator_scalars.size() == 1)  // T - k
+  {
+    Layer *first_operator = output_node_map[first_operator_name];
+    return new LDiff(first_operator, second_operator_scalars[0], node->name(), dev, mem);
   }
-  else if(map_init_values.count(second_operator_name)) // T - k
+  else if (!first_operator_scalars.empty() && first_operator_scalars.size() != 1)
   {
-    // Read the input scalar
-    vector<float> scalars = map_init_values[second_operator_name];
-    if (scalars.size() != 1)
-    {
+    // Este caso se da cuando el primer operador es un Tensor constante (no escalar)
+    Layer *first_operator  = output_node_map[first_operator_name];
+    Layer *second_operator = output_node_map[second_operator_name];   
+    vector<int> shape_first_op  = map_init_dims[first_operator_name];
+    vector<int> shape_second_op = second_operator->getShape();
+    Tensor* constant_tensor;
+
+    // Same shape and shame batch size
+    if (shape_first_op.size() == shape_second_op.size() && shape_first_op[0] == shape_second_op[0]) {
+      constant_tensor = new Tensor(first_operator_scalars, shape_first_op);
+    } 
+    else if (shape_first_op.size() == shape_second_op.size() && shape_first_op[0] != shape_second_op[0]) {
+      // Assuming that the batch size is the one different, but check the other dimensions anyway
+      for (size_t i = 1; i < shape_first_op.size(); ++i) {
+        if (shape_first_op[i] != shape_second_op[i])
+          msg("Error: The second input operand of the Diff layer " + node->name() + " is not valid", "ONNX::ImportNet");
+      }
+
+      constant_tensor = new Tensor(first_operator_scalars, shape_first_op);
+      constant_tensor = Tensor::repeat(constant_tensor, shape_second_op[0], 0);
+    }
+    else {
       msg("Error: The second input operand of the Diff layer " + node->name() + " is not valid", "ONNX::ImportNet");
-      return nullptr;
     }
-    Layer *first_operator = output_node_map[first_operator_name];
-    return new LDiff(first_operator, scalars[0], node->name(), dev, mem);
+
+    return new LDiff(constant_tensor, second_operator, node->name(), dev, mem);
   }
+  // else if (!second_operator_scalars.empty() && second_operator_scalars.size() != 1)
+  // {
+  //   std::cout << "LDiff build_diff_layer 4ª cuando el segundo operador es un Tensor constante (no escalar) //TODO" << std::endl;
+  //   // Este caso se da cuando el primer operador es un Tensor constante (no escalar)
+  //   // Layer *second_operator = output_node_map[second_operator_name];
+  //   // vector<int> shape = second_operator->getShape();
+  //   // Tensor* constant_tensor = new Tensor(first_operator_scalars, shape);
+  //   // return new LDiff(constant_tensor, second_operator, node->name(), dev, mem);
+  // }
   else // T - T
   {
+    std::cout << "LDiff build_diff_layer 5ª condición" << std::endl;
     Layer *first_operator = output_node_map[first_operator_name];
     Layer *second_operator = output_node_map[second_operator_name];
+    // first_operator->info();
+    // second_operator->info();
     return new LDiff(first_operator, second_operator, node->name(), dev, mem);
-  }
+  } 
+
+  // if(map_init_values.count(first_operator_name)) // k - T
+  // {
+  //   // Read the input scalar
+  //   vector<float> scalars = map_init_values[first_operator_name];
+  //   if (scalars.size() != 1)
+  //   {
+  //     msg("Error: The first input operand of the Diff layer " + node->name() + " is not valid", "ONNX::ImportNet");
+  //     return nullptr;
+  //   }
+  //   Layer *second_operator = output_node_map[second_operator_name];
+  //   return new LDiff(scalars[0], second_operator, node->name(), dev, mem);
+  // }
+  // else if(map_init_values.count(second_operator_name)) // T - k
+  // {
+  //   // Read the input scalar
+  //   vector<float> scalars = map_init_values[second_operator_name];
+  //   if (scalars.size() != 1)
+  //   {
+  //     msg("Error: The second input operand of the Diff layer " + node->name() + " is not valid", "ONNX::ImportNet");
+  //     return nullptr;
+  //   }
+  //   Layer *first_operator = output_node_map[first_operator_name];
+  //   return new LDiff(first_operator, scalars[0], node->name(), dev, mem);
+  // }
+  // else // T - T
+  // {
+  //   Layer *first_operator = output_node_map[first_operator_name];
+  //   Layer *second_operator = output_node_map[second_operator_name];
+  //   return new LDiff(first_operator, second_operator, node->name(), dev, mem);
+  // }
 }
 
 // ONNX export

src/serialization/onnx/net/layers/operators/mult_onnx.cpp

@@ -11,15 +11,62 @@ Layer* build_mul_layer(onnx::NodeProto *node,
                        int dev,
                        int mem)
 {
+  vector<float> second_operator_scalars;
   string first_operator_name = node->input(0);
+  string second_operator_name = node->input(1);
   Layer *first_operator = output_node_map[first_operator_name];
+  Layer *second_operator = output_node_map[second_operator_name];
+
 
-  string second_operator_name = node->input(1);
   if(map_init_values.count(second_operator_name))
   {
+    second_operator_scalars = map_init_values[second_operator_name];
+    if (second_operator_scalars.size() > 1) {
+      std::cout << "Segundo operador constante" << std::endl;
+
+      // vector<int> shape_second_op = second_operator->getShape();
+      vector<int> shape_first_op  = first_operator->getShape();
+      vector<int> shape_second_op = map_init_dims[second_operator_name];
+
+      Tensor *aux_first  = Tensor::zeros(shape_first_op);
+      Tensor *aux_second = new Tensor(second_operator_scalars, shape_second_op);
+      Tensor *constant;
+
+      int first_op_ndim  = aux_first->ndim;
+      int second_op_ndim = aux_second->ndim;
+
+      // Assume the second operand as the one to be broadcasted
+      if (second_op_ndim + 1 == first_op_ndim) {
+        aux_second = aux_second->unsqueeze();
+
+        // Find which is the different dimension
+        int dim = -1;
+        for (size_t i = 1; i < shape_first_op.size(); ++i) {
+          if (shape_first_op[i] != shape_second_op[i]) {
+            dim = i;
+            std::cout << "La dimensión diferente es la " << i << std::endl;
+            break;
+          }
+        }
+        if (dim < 0) 
+          msg("Error: Could not find the different dim of the Mult layer " + node->name() + " operands", "ONNX:ImportNet");
+        int times = shape_first_op[dim];
+        constant = Tensor::repeat(aux_second, times, dim);
+      }
+      else if (second_op_ndim == first_op_ndim) {
+        constant = new Tensor(second_operator_scalars, shape_first_op);
+      }
+      else {
+        msg("Error: The second input operand of the Mult layer " + node->name() + " is not valid", "ONNX::ImportNet");
+      }
+
+      delete(aux_first);
+      delete(aux_second);
+      return new LMult(first_operator, constant, node->name(), dev, mem);
+    }
     // Detect pattern for applying scale and bias of batchnorm using Mult
     // and Add operators
-    if (LBatchNorm *l = dynamic_cast<LBatchNorm*>(first_operator))
+    else if (LBatchNorm *l = dynamic_cast<LBatchNorm*>(first_operator))
     {
       // Set the scale value of the input batchnorm layer
       vector<float> *scale_weights = &(map_init_values[second_operator_name]);
@@ -48,8 +95,6 @@ Layer* build_mul_layer(onnx::NodeProto *node,
     }
   }
 
-  Layer *second_operator = output_node_map[second_operator_name];
-
   vector<Layer *> operators = expand_broadcast({first_operator, second_operator});
 
   return new LMult(operators[0], operators[1], node->name(), dev, mem);