Add Conv3DTranspose, UpSampling3D, and stride-1 fast path for Conv3D

README.md

@@ -43,7 +43,7 @@ Would you like to build/train a model using Keras/Python? And would you like to
 * `AveragePooling1D/2D/3D`, `GlobalAveragePooling1D/2D/3D`
 * `TimeDistributed`
 * `Conv1D/2D/3D`, `SeparableConv2D`, `DepthwiseConv2D`
-* `Conv1DTranspose`, `Conv2DTranspose`
+* `Conv1DTranspose`, `Conv2DTranspose`, `Conv3DTranspose`
 * `Cropping1D/2D/3D`, `ZeroPadding1D/2D/3D`, `CenterCrop`
 * `BatchNormalization`, `Dense`, `Flatten`, `Normalization`
 * `Dropout`, `AlphaDropout`, `GaussianDropout`, `GaussianNoise`
@@ -52,7 +52,7 @@ Would you like to build/train a model using Keras/Python? And would you like to
 * `RandomContrast`, `RandomFlip`, `RandomHeight`
 * `RandomRotation`, `RandomTranslation`, `RandomWidth`, `RandomZoom`
 * `MaxPooling1D/2D/3D`, `GlobalMaxPooling1D/2D/3D`
-* `UpSampling1D/2D`, `Resizing`, `Rescaling`
+* `UpSampling1D/2D/3D`, `Resizing`, `Rescaling`
 * `Reshape`, `Permute`, `RepeatVector`
 * `Embedding`, `CategoryEncoding`
 * `Attention`, `AdditiveAttention`, `MultiHeadAttention`
@@ -79,7 +79,7 @@ Would you like to build/train a model using Keras/Python? And would you like to
 `RepeatVector`, `RNN`, `SimpleRNN`,
 `SimpleRNNCell`, `StackedRNNCells`, `StringLookup`, `TextVectorization`,
 `Bidirectional`, `GRU`, `LSTM`, `CuDNNGRU`, `CuDNNLSTM`,
-`ThresholdedReLU`, `Upsampling3D`, `temporal` models
+`ThresholdedReLU`, `temporal` models
 
 Usage
 -----

include/fdeep/convolution3d.hpp

@@ -38,7 +38,8 @@ namespace internal {
         padding pad_type,
         std::size_t input_shape_size_d4,
         std::size_t input_shape_height,
-        std::size_t input_shape_width)
+        std::size_t input_shape_width,
+        bool transposed = false)
     {
         const int filter_size_d4 = static_cast<int>(filter_shape.size_dim_4_);
         const int filter_height = static_cast<int>(filter_shape.height_);
@@ -59,9 +60,15 @@ namespace internal {
             out_height = fplus::ceil(static_cast<float>(in_height) / static_cast<float>(strides_y) - 0.001);
             out_width = fplus::ceil(static_cast<float>(in_width) / static_cast<float>(strides_x) - 0.001);
         } else {
-            out_size_d4 = fplus::ceil(static_cast<float>(in_size_d4 - filter_size_d4 + 1) / static_cast<float>(strides_d4) - 0.001);
-            out_height = fplus::ceil(static_cast<float>(in_height - filter_height + 1) / static_cast<float>(strides_y) - 0.001);
-            out_width = fplus::ceil(static_cast<float>(in_width - filter_width + 1) / static_cast<float>(strides_x) - 0.001);
+            if (transposed) {
+                out_size_d4 = fplus::ceil(static_cast<float>(in_size_d4 + filter_size_d4 - 1) / static_cast<float>(strides_d4) - 0.001);
+                out_height = fplus::ceil(static_cast<float>(in_height + filter_height - 1) / static_cast<float>(strides_y) - 0.001);
+                out_width = fplus::ceil(static_cast<float>(in_width + filter_width - 1) / static_cast<float>(strides_x) - 0.001);
+            } else {
+                out_size_d4 = fplus::ceil(static_cast<float>(in_size_d4 - filter_size_d4 + 1) / static_cast<float>(strides_d4) - 0.001);
+                out_height = fplus::ceil(static_cast<float>(in_height - filter_height + 1) / static_cast<float>(strides_y) - 0.001);
+                out_width = fplus::ceil(static_cast<float>(in_width - filter_width + 1) / static_cast<float>(strides_x) - 0.001);
+            }
         }
 
         int pad_front = 0;
@@ -71,6 +78,15 @@ namespace internal {
         int pad_left = 0;
         int pad_right = 0;
 
+        if (transposed) {
+            pad_front = filter_size_d4 - 1;
+            pad_back = filter_size_d4 - 1;
+            pad_top = filter_height - 1;
+            pad_bottom = filter_height - 1;
+            pad_left = filter_width - 1;
+            pad_right = filter_width - 1;
+        }
+
         if (pad_type == padding::same) {
             int pad_along_d4 = 0;
             int pad_along_height = 0;
@@ -125,28 +141,35 @@ namespace internal {
         tensor filter_mats_;
     };
 
-    inline tensor dilate_tensor_3d(const shape3& dilation_rate, const tensor& in)
+    inline tensor dilate_tensor_3d(const shape3& dilation_rate, const tensor& in,
+        bool trailing_zeros = false)
     {
         if (dilation_rate == shape3(1, 1, 1)) {
             return in;
         }
         assertion(in.shape().rank() == 4, "Invalid rank for 3d dilation");
 
         const auto in_shape = in.shape();
+        const std::size_t expansion_d4 = trailing_zeros ? (dilation_rate.size_dim_4_ - 1) : 0;
+        const std::size_t expansion_y = trailing_zeros ? (dilation_rate.height_ - 1) : 0;
+        const std::size_t expansion_x = trailing_zeros ? (dilation_rate.width_ - 1) : 0;
         const tensor_shape dilated_shape(
-            (in_shape.size_dim_4_ - 1) * dilation_rate.size_dim_4_ + 1,
-            (in_shape.height_ - 1) * dilation_rate.height_ + 1,
-            (in_shape.width_ - 1) * dilation_rate.width_ + 1,
+            (in_shape.size_dim_4_ - 1) * dilation_rate.size_dim_4_ + 1 + expansion_d4,
+            (in_shape.height_ - 1) * dilation_rate.height_ + 1 + expansion_y,
+            (in_shape.width_ - 1) * dilation_rate.width_ + 1 + expansion_x,
             in_shape.depth_);
+        const std::size_t offset_d4 = expansion_d4 - expansion_d4 / 2;
+        const std::size_t offset_y = expansion_y - expansion_y / 2;
+        const std::size_t offset_x = expansion_x - expansion_x / 2;
         tensor result(dilated_shape, static_cast<float_type>(0));
         for (std::size_t d4 = 0; d4 < in_shape.size_dim_4_; ++d4) {
             for (std::size_t y = 0; y < in_shape.height_; ++y) {
                 for (std::size_t x = 0; x < in_shape.width_; ++x) {
                     for (std::size_t z = 0; z < in_shape.depth_; ++z) {
                         result.set_ignore_rank(tensor_pos(
-                                                   d4 * dilation_rate.size_dim_4_,
-                                                   y * dilation_rate.height_,
-                                                   x * dilation_rate.width_,
+                                                   d4 * dilation_rate.size_dim_4_ + offset_d4,
+                                                   y * dilation_rate.height_ + offset_y,
+                                                   x * dilation_rate.width_ + offset_x,
                                                    z),
                             in.get_ignore_rank(tensor_pos(d4, y, x, z)));
                     }
@@ -156,6 +179,16 @@ namespace internal {
         return result;
     }
 
+    inline tensor reverse_size_dim_4_dimension(const tensor& in)
+    {
+        tensor out = tensor(in.shape(), static_cast<float_type>(0));
+        loop_over_all_dims(in.shape(), [&in, &out](std::size_t dim5, std::size_t dim4, std::size_t y, std::size_t x, std::size_t z) {
+            out.set_ignore_rank(tensor_pos(dim5, in.shape().size_dim_4_ - dim4 - 1, y, x, z),
+                in.get_ignore_rank(tensor_pos(dim5, dim4, y, x, z)));
+        });
+        return out;
+    }
+
     inline filter dilate_filter_3d(const shape3& dilation_rate, const filter& undilated)
     {
         return filter(dilate_tensor_3d(dilation_rate, undilated.get_tensor()),
@@ -165,7 +198,8 @@ namespace internal {
     inline filter_vec generate_filters_3d(
         const shape3& dilation_rate,
         const tensor_shape& filter_shape, std::size_t k,
-        const float_vec& weights, const float_vec& bias)
+        const float_vec& weights, const float_vec& bias,
+        bool transpose = false)
     {
         filter_vec filters(k, filter(tensor(filter_shape, 0), 0));
 
@@ -186,6 +220,11 @@ namespace internal {
         for (auto& filt : filters) {
             filt.set_params(*it_filter_val, *it_filter_bias);
             filt = dilate_filter_3d(dilation_rate, filt);
+            if (transpose) {
+                filt = filter(reverse_size_dim_4_dimension(filt.get_tensor()), filt.get_bias());
+                filt = filter(reverse_height_dimension(filt.get_tensor()), filt.get_bias());
+                filt = filter(reverse_width_dimension(filt.get_tensor()), filt.get_bias());
+            }
             ++it_filter_val;
             ++it_filter_bias;
         }
@@ -276,6 +315,73 @@ namespace internal {
             Eigen::OuterStride<>(static_cast<EigenIndex>(f_depth * strides_x)));
     }
 
+    // Special version for convolution with strides 1×1×1.
+    // Uses fewer but larger GEMMs by collapsing the spatial output loops into one big input mapping.
+    inline tensor convolve_accumulative_s1x1x1_3d(
+        std::size_t out_size_d4,
+        std::size_t out_height,
+        std::size_t out_width,
+        const convolution3d_filter_matrices& filter_mat,
+        const tensor& in)
+    {
+        const tensor& filter_mats = filter_mat.filter_mats_;
+        const auto f_size_d4 = filter_mat.filter_shape_.size_dim_4_;
+        const auto f_height = filter_mat.filter_shape_.height_;
+        const auto f_width = filter_mat.filter_shape_.width_;
+        const auto f_depth = filter_mat.filter_shape_.depth_;
+        const auto out_depth = filter_mat.filter_count_;
+
+        assertion(f_depth == in.shape().depth_, "filter depth does not match input");
+        assertion(out_height == (in.shape().height_ - f_height) + 1, "output height does not match");
+        assertion(out_width == (in.shape().width_ - f_width) + 1, "output width does not match");
+        assertion(out_size_d4 == (in.shape().size_dim_4_ - f_size_d4) + 1, "output d4 size does not match");
+        assertion(out_depth == filter_mat.biases_.size(), "invalid bias count");
+
+        tensor output = init_conv_output_tensor_3d(out_size_d4, out_height, out_width, out_depth, in.shape().rank(), filter_mat);
+
+        const std::size_t out_height_temp = out_height + f_height - 1;
+        const std::size_t out_width_temp = out_width + f_width - 1;
+        tensor output_temp(tensor_shape_with_changed_rank(
+                               tensor_shape(out_size_d4, out_height_temp, out_width_temp, out_depth),
+                               in.shape().rank()),
+            static_cast<float_type>(0));
+
+        const auto mapping_width = (out_size_d4 - 1) * out_height_temp * out_width_temp
+            + (out_height - 1) * out_width_temp
+            + out_width;
+
+        for (std::size_t d4_filt = 0; d4_filt < f_size_d4; ++d4_filt) {
+            for (std::size_t y_filt = 0; y_filt < f_height; ++y_filt) {
+                const Eigen::Map<ColMajorMatrixXf, Eigen::Unaligned>
+                    filter(const_cast<float_type*>(&filter_mats.get_ref_ignore_rank(tensor_pos(d4_filt, y_filt, 0, 0, 0))),
+                        static_cast<EigenIndex>(out_depth),
+                        static_cast<EigenIndex>(f_width * f_depth));
+
+                const auto input = get_im2col_mapping_3d(in, f_width, f_depth, 1, mapping_width, 0, 0, d4_filt, y_filt);
+
+                Eigen::Map<ColMajorMatrixXf, Eigen::Unaligned>
+                    output_temp_map(&output_temp.get_ref_ignore_rank(tensor_pos(0, 0, 0, 0, 0)),
+                        static_cast<EigenIndex>(out_depth),
+                        static_cast<EigenIndex>(mapping_width));
+
+                output_temp_map.noalias() += filter * input;
+            }
+        }
+
+        for (std::size_t d4_out = 0; d4_out < out_size_d4; ++d4_out) {
+            for (std::size_t y_out = 0; y_out < out_height; ++y_out) {
+                for (std::size_t x_out = 0; x_out < out_width; ++x_out) {
+                    for (std::size_t z_out = 0; z_out < out_depth; ++z_out) {
+                        output.get_ref_ignore_rank(tensor_pos(0, d4_out, y_out, x_out, z_out))
+                            += output_temp.get_ref_ignore_rank(tensor_pos(0, d4_out, y_out, x_out, z_out));
+                    }
+                }
+            }
+        }
+
+        return output;
+    }
+
     inline tensor convolve_accumulative_3d(
         std::size_t out_size_d4,
         std::size_t out_height,
@@ -299,6 +405,10 @@ namespace internal {
         assertion(out_width == (in.shape().width_ - f_width) / strides_x + 1, "output width does not match");
         assertion(out_depth == filter_mat.biases_.size(), "invalid bias count");
 
+        if (strides_d4 == 1 && strides_y == 1 && strides_x == 1) {
+            return convolve_accumulative_s1x1x1_3d(out_size_d4, out_height, out_width, filter_mat, in);
+        }
+
         tensor output = init_conv_output_tensor_3d(out_size_d4, out_height, out_width, out_depth, in.shape().rank(), filter_mat);
 
         for (std::size_t d4_filt = 0; d4_filt < f_size_d4; ++d4_filt) {
@@ -358,5 +468,42 @@ namespace internal {
             in_padded);
     }
 
+    inline tensor convolve_transposed_3d(
+        const shape3& strides,
+        const padding& pad_type,
+        const convolution3d_filter_matrices& filter_mat,
+        const tensor& input)
+    {
+        assertion(filter_mat.filter_shape_.depth_ == input.shape().depth_,
+            "invalid filter depth");
+
+        const auto input_dilated = dilate_tensor_3d(strides, input, pad_type == padding::same);
+
+        const shape3 filter_spatial_shape(
+            filter_mat.filter_shape_.size_dim_4_,
+            filter_mat.filter_shape_.height_,
+            filter_mat.filter_shape_.width_);
+
+        const auto conv_cfg = preprocess_convolution_3d(
+            filter_spatial_shape,
+            shape3(1, 1, 1), pad_type,
+            input_dilated.shape().size_dim_4_,
+            input_dilated.shape().height_,
+            input_dilated.shape().width_,
+            true);
+
+        const auto in_padded = pad_tensor(0,
+            conv_cfg.pad_front_, conv_cfg.pad_back_,
+            conv_cfg.pad_top_, conv_cfg.pad_bottom_,
+            conv_cfg.pad_left_, conv_cfg.pad_right_,
+            input_dilated);
+
+        return convolve_accumulative_3d(
+            conv_cfg.out_size_d4_, conv_cfg.out_height_, conv_cfg.out_width_,
+            1, 1, 1,
+            filter_mat,
+            in_padded);
+    }
+
 }
 }

include/fdeep/import_model.hpp

@@ -40,6 +40,7 @@
 #include "fdeep/layers/conv_2d_layer.hpp"
 #include "fdeep/layers/conv_2d_transpose_layer.hpp"
 #include "fdeep/layers/conv_3d_layer.hpp"
+#include "fdeep/layers/conv_3d_transpose_layer.hpp"
 #include "fdeep/layers/cropping_3d_layer.hpp"
 #include "fdeep/layers/dense_layer.hpp"
 #include "fdeep/layers/depthwise_conv_2d_layer.hpp"
@@ -94,6 +95,7 @@
 #include "fdeep/layers/unit_normalization_layer.hpp"
 #include "fdeep/layers/upsampling_1d_layer.hpp"
 #include "fdeep/layers/upsampling_2d_layer.hpp"
+#include "fdeep/layers/upsampling_3d_layer.hpp"
 #include "fdeep/layers/zero_padding_3d_layer.hpp"
 #include "fdeep/tensor.hpp"
 #include "fdeep/tensor_shape.hpp"
@@ -447,6 +449,36 @@ namespace internal {
             dilation_rate, weights, bias);
     }
 
+    inline layer_ptr create_conv_3d_transpose_layer(const get_param_f& get_param,
+        const nlohmann::json& data,
+        const std::string& name)
+    {
+        const std::string padding_str = data["config"]["padding"];
+        const auto pad_type = create_padding(padding_str);
+
+        const shape3 strides = create_shape3(data["config"]["strides"]);
+        const shape3 dilation_rate = create_shape3(data["config"]["dilation_rate"]);
+
+        const auto filter_count = create_size_t(data["config"]["filters"]);
+        float_vec bias(filter_count, 0);
+        const bool use_bias = data["config"]["use_bias"];
+        if (use_bias)
+            bias = decode_floats(get_param(name, "bias"));
+        assertion(bias.size() == filter_count, "size of bias does not match");
+
+        const float_vec weights = decode_floats(get_param(name, "weights"));
+        const shape3 kernel_size = create_shape3(data["config"]["kernel_size"]);
+        assertion(weights.size() % kernel_size.volume() == 0,
+            "invalid number of weights");
+        const std::size_t filter_depths = weights.size() / (kernel_size.volume() * filter_count);
+        const tensor_shape filter_shape(
+            kernel_size.size_dim_4_, kernel_size.height_, kernel_size.width_, filter_depths);
+
+        return std::make_shared<conv_3d_transpose_layer>(name,
+            filter_shape, filter_count, strides, pad_type,
+            dilation_rate, weights, bias);
+    }
+
     inline layer_ptr create_conv_2d_transpose_layer(const get_param_f& get_param,
         const nlohmann::json& data,
         const std::string& name)
@@ -660,6 +692,15 @@ namespace internal {
             name, scale_factor, interpolation);
     }
 
+    inline layer_ptr create_upsampling_3d_layer(
+        const get_param_f&, const nlohmann::json& data,
+        const std::string& name)
+    {
+        const auto scale_factor = create_shape3(data["config"]["size"]);
+        return std::make_shared<upsampling_3d_layer>(
+            name, scale_factor);
+    }
+
     inline layer_ptr create_dense_layer(const get_param_f& get_param,
         const nlohmann::json& data, const std::string& name)
     {
@@ -1301,6 +1342,7 @@ namespace internal {
             { "Conv3D", create_conv_3d_layer },
             { "Conv1DTranspose", create_conv_2d_transpose_layer },
             { "Conv2DTranspose", create_conv_2d_transpose_layer },
+            { "Conv3DTranspose", create_conv_3d_transpose_layer },
             { "SeparableConv1D", create_separable_conv_2D_layer },
             { "SeparableConv2D", create_separable_conv_2D_layer },
             { "DepthwiseConv2D", create_depthwise_conv_2D_layer },
@@ -1365,6 +1407,7 @@ namespace internal {
             { "GlobalAveragePooling3D", create_global_average_pooling_3d_layer },
             { "UpSampling1D", create_upsampling_1d_layer },
             { "UpSampling2D", create_upsampling_2d_layer },
+            { "UpSampling3D", create_upsampling_3d_layer },
             { "Dense", create_dense_layer },
             { "Add", create_add_layer },
             { "Maximum", create_maximum_layer },