add matmul to proto

Author: PINTO0309

convert_script.txt

@@ -63,6 +63,20 @@ sor4onnx \
 --output_onnx_file_path nms_yolact_edge.onnx \
 --mode inputs
 
+sor4onnx \
+--input_onnx_file_path nms_yolact_edge.onnx \
+--old_new "input_5" "nms_proto" \
+--output_onnx_file_path nms_yolact_edge.onnx \
+--mode inputs
+
+
+
+sor4onnx \
+--input_onnx_file_path nms_yolact_edge.onnx \
+--old_new "Identity_1" "proto_out" \
+--output_onnx_file_path nms_yolact_edge.onnx \
+--mode outputs
+
 sor4onnx \
 --input_onnx_file_path nms_yolact_edge.onnx \
 --old_new "Identity" "x1y1x2y2_scores_classes_masks_4x1x1x32" \
@@ -73,7 +87,7 @@ sor4onnx \
 snc4onnx \
 --input_onnx_file_paths yolact_edge_mobilenetv2_54_800000.onnx nms_yolact_edge.onnx \
 --op_prefixes_after_merging main post \
---srcop_destop boxes nms_boxes scores nms_scores classes nms_classes masks nms_masks \
+--srcop_destop boxes nms_boxes scores nms_scores classes nms_classes masks nms_masks proto nms_proto \
 --output_onnx_file_path yolact_edge_mobilenetv2_550x550.onnx
 
 
@@ -92,10 +106,13 @@ sor4onnx \
 
 sor4onnx \
 --input_onnx_file_path yolact_edge_mobilenetv2_550x550.onnx \
---old_new "main_proto" "proto" \
+--old_new "post_proto_out" "proto_out" \
 --output_onnx_file_path yolact_edge_mobilenetv2_550x550.onnx \
 --mode outputs
 
+onnxsim yolact_edge_mobilenetv2_550x550.onnx yolact_edge_mobilenetv2_550x550.onnx
+onnxsim yolact_edge_mobilenetv2_550x550.onnx yolact_edge_mobilenetv2_550x550.onnx
+
 
 
 $INTEL_OPENVINO_DIR/deployment_tools/model_optimizer/mo.py \

nms_yolact_edge.py

@@ -43,6 +43,15 @@
     dtype=tf.float32,
 )
 
+protos = tf.keras.layers.Input(
+    shape=[
+        138,
+        138,
+        BOXES,
+    ],
+    batch_size=1,
+    dtype=tf.float32,
+)
 
 boxes_non_batch = tf.squeeze(boxes)
 x1 = boxes_non_batch[:,0][:,np.newaxis]
@@ -85,10 +94,17 @@
     selected_indices
 )
 
+selected_protos = tf.gather(
+    protos[0],
+    selected_indices,
+    axis=2,
+)
+selected_protos_trans = tf.transpose(selected_protos, perm=[2,0,1])
+
 
 outputs = tf.concat([selected_boxes, selected_scores, selected_classes, selected_masks], axis=1)
 
-model = tf.keras.models.Model(inputs=[boxes,scores,classes,masks], outputs=[outputs])
+model = tf.keras.models.Model(inputs=[boxes,scores,classes,masks,protos], outputs=[outputs, selected_protos_trans])
 model.summary()
 output_path = 'saved_model_postprocess'
 tf.saved_model.save(model, output_path)

yolact_edge/yolact.py

@@ -958,6 +958,11 @@ def forward(self, x, extras=None):
         priors = torch.cat(priors, 0)
         boxes, scores, classes, masks = self.detect(loc, conf, mask, priors)
 
+        proto_out = proto_out[0] @ masks[0].t()
+        proto_out = torch.sigmoid(proto_out)
+        # masks = crop(masks, boxes)
+        proto_out = proto_out.permute(2, 0, 1).contiguous()[np.newaxis, ...]
+
         return boxes, scores, classes, masks, proto_out