diff --git a/README.md b/README.md index b2fe09af..faef39ca 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Would you like to build/train a model using Keras/Python? And would you like to * `Add`, `Concatenate`, `Subtract`, `Multiply`, `Average`, `Maximum`, `Minimum`, `Dot` * `AveragePooling1D/2D/3D`, `GlobalAveragePooling1D/2D/3D` * `TimeDistributed` -* `Conv1D/2D`, `SeparableConv2D`, `DepthwiseConv2D` +* `Conv1D/2D/3D`, `SeparableConv2D`, `DepthwiseConv2D` * `Conv1DTranspose`, `Conv2DTranspose` * `Cropping1D/2D/3D`, `ZeroPadding1D/2D/3D`, `CenterCrop` * `BatchNormalization`, `Dense`, `Flatten`, `Normalization` @@ -71,7 +71,7 @@ Would you like to build/train a model using Keras/Python? And would you like to ### Currently not supported are the following: `Lambda` ([why](FAQ.md#why-are-lambda-layers-not-supported)), -`Conv3D`, `ConvLSTM1D`, `ConvLSTM2D`, `Discretization`, +`ConvLSTM1D`, `ConvLSTM2D`, `Discretization`, `GRUCell`, `Hashing`, `IntegerLookup`, `LocallyConnected1D`, `LocallyConnected2D`, diff --git a/include/fdeep/convolution3d.hpp b/include/fdeep/convolution3d.hpp index 3c3e7d99..6f546908 100644 --- a/include/fdeep/convolution3d.hpp +++ b/include/fdeep/convolution3d.hpp @@ -8,7 +8,9 @@ #include "fdeep/common.hpp" +#include "fdeep/convolution.hpp" #include "fdeep/filter.hpp" +#include "fdeep/shape3.hpp" #include #include @@ -115,5 +117,246 @@ namespace internal { out_size_d4_size_t, out_height_size_t, out_width_size_t }; } + struct convolution3d_filter_matrices { + tensor_shape filter_shape_; + std::size_t filter_count_; + float_vec biases_; + bool use_bias_; + tensor filter_mats_; + }; + + inline tensor dilate_tensor_3d(const shape3& dilation_rate, const tensor& in) + { + if (dilation_rate == shape3(1, 1, 1)) { + return in; + } + assertion(in.shape().rank() == 4, "Invalid rank for 3d dilation"); + + const auto in_shape = in.shape(); + const tensor_shape dilated_shape( + (in_shape.size_dim_4_ - 1) * dilation_rate.size_dim_4_ + 1, + (in_shape.height_ - 1) * dilation_rate.height_ + 1, + (in_shape.width_ - 1) * dilation_rate.width_ + 1, + in_shape.depth_); + tensor result(dilated_shape, static_cast(0)); + for (std::size_t d4 = 0; d4 < in_shape.size_dim_4_; ++d4) { + for (std::size_t y = 0; y < in_shape.height_; ++y) { + for (std::size_t x = 0; x < in_shape.width_; ++x) { + for (std::size_t z = 0; z < in_shape.depth_; ++z) { + result.set_ignore_rank(tensor_pos( + d4 * dilation_rate.size_dim_4_, + y * dilation_rate.height_, + x * dilation_rate.width_, + z), + in.get_ignore_rank(tensor_pos(d4, y, x, z))); + } + } + } + } + return result; + } + + inline filter dilate_filter_3d(const shape3& dilation_rate, const filter& undilated) + { + return filter(dilate_tensor_3d(dilation_rate, undilated.get_tensor()), + undilated.get_bias()); + } + + inline filter_vec generate_filters_3d( + const shape3& dilation_rate, + const tensor_shape& filter_shape, std::size_t k, + const float_vec& weights, const float_vec& bias) + { + filter_vec filters(k, filter(tensor(filter_shape, 0), 0)); + + assertion(!filters.empty(), "at least one filter needed"); + const std::size_t param_count = fplus::sum(fplus::transform( + fplus_c_mem_fn_t(filter, volume, std::size_t), filters)); + + assertion(static_cast(weights.size()) == param_count, + "invalid weight size"); + const auto filter_param_cnt = filters.front().shape().volume(); + + auto filter_weights = fplus::split_every(filter_param_cnt, weights); + assertion(filter_weights.size() == filters.size(), + "invalid size of filter weights"); + assertion(bias.size() == filters.size(), "invalid bias size"); + auto it_filter_val = std::begin(filter_weights); + auto it_filter_bias = std::begin(bias); + for (auto& filt : filters) { + filt.set_params(*it_filter_val, *it_filter_bias); + filt = dilate_filter_3d(dilation_rate, filt); + ++it_filter_val; + ++it_filter_bias; + } + + return filters; + } + + inline convolution3d_filter_matrices generate_im2col_filter_matrix_3d( + const std::vector& filters) + { + assertion(fplus::all_the_same_on( + fplus_c_mem_fn_t(filter, shape, tensor_shape), filters), + "all filters must have the same shape"); + + const auto biases = fplus::transform_convert( + fplus_c_mem_fn_t(filter, get_bias, float_type), + filters); + + const bool use_bias = fplus::sum(biases) != static_cast(0) || !fplus::all_the_same(biases); + + const auto shape = filters.front().shape(); + + tensor filter_mats = tensor( + tensor_shape(shape.size_dim_4_, shape.height_, shape.width_, shape.depth_, filters.size()), + static_cast(0)); + + for (std::size_t d4 = 0; d4 < shape.size_dim_4_; ++d4) { + for (std::size_t y = 0; y < shape.height_; ++y) { + for (std::size_t n = 0; n < filters.size(); ++n) { + for (std::size_t x = 0; x < shape.width_; ++x) { + for (std::size_t z = 0; z < shape.depth_; ++z) { + filter_mats.set(tensor_pos(d4, y, x, z, n), + filters[n].get(tensor_pos(d4, y, x, z))); + } + } + } + } + } + + return { shape, filters.size(), biases, use_bias, filter_mats }; + } + + inline tensor init_conv_output_tensor_3d( + std::size_t out_size_d4, + std::size_t out_height, + std::size_t out_width, + std::size_t out_depth, + std::size_t rank, + const convolution3d_filter_matrices& filter_mat) + { + tensor output(tensor_shape_with_changed_rank( + tensor_shape(out_size_d4, out_height, out_width, out_depth), + rank), + static_cast(0)); + if (filter_mat.use_bias_) { + const auto bias_ptr = &filter_mat.biases_.front(); + const auto bias_ptr_end = bias_ptr + out_depth; + for (std::size_t d4_out = 0; d4_out < out_size_d4; ++d4_out) { + for (std::size_t y_out = 0; y_out < out_height; ++y_out) { + for (std::size_t x_out = 0; x_out < out_width; ++x_out) { + auto output_ptr = &output.get_ref_ignore_rank(tensor_pos(0, d4_out, y_out, x_out, 0)); + std::copy(bias_ptr, bias_ptr_end, output_ptr); + } + } + } + } + return output; + } + + inline Eigen::Map> get_im2col_mapping_3d( + const tensor& in, + std::size_t f_width, + std::size_t f_depth, + std::size_t strides_x, + std::size_t out_width, + std::size_t d4, + std::size_t y, + std::size_t d4_filt, + std::size_t y_filt) + { + // Same trick as in the 2D case: avoid materializing the im2col matrix + // by using an outer stride smaller than the row count, so adjacent + // columns share data along the receptive field. + return Eigen::Map>( + const_cast(&in.get_ref_ignore_rank(tensor_pos(0, d4 + d4_filt, y + y_filt, 0, 0))), + static_cast(f_width * f_depth), + static_cast(out_width), + Eigen::OuterStride<>(static_cast(f_depth * strides_x))); + } + + inline tensor convolve_accumulative_3d( + std::size_t out_size_d4, + std::size_t out_height, + std::size_t out_width, + std::size_t strides_d4, + std::size_t strides_y, + std::size_t strides_x, + const convolution3d_filter_matrices& filter_mat, + const tensor& in) + { + const tensor& filter_mats = filter_mat.filter_mats_; + const auto f_size_d4 = filter_mat.filter_shape_.size_dim_4_; + const auto f_height = filter_mat.filter_shape_.height_; + const auto f_width = filter_mat.filter_shape_.width_; + const auto f_depth = filter_mat.filter_shape_.depth_; + const auto out_depth = filter_mat.filter_count_; + + assertion(f_depth == in.shape().depth_, "filter depth does not match input"); + assertion(filter_mats.shape().size_dim_5_ == f_size_d4, "incorrect number of filter levels in d4 direction"); + assertion(filter_mats.shape().size_dim_4_ == f_height, "incorrect number of filter levels in y direction"); + assertion(out_width == (in.shape().width_ - f_width) / strides_x + 1, "output width does not match"); + assertion(out_depth == filter_mat.biases_.size(), "invalid bias count"); + + tensor output = init_conv_output_tensor_3d(out_size_d4, out_height, out_width, out_depth, in.shape().rank(), filter_mat); + + for (std::size_t d4_filt = 0; d4_filt < f_size_d4; ++d4_filt) { + for (std::size_t y_filt = 0; y_filt < f_height; ++y_filt) { + const Eigen::Map + filter(const_cast(&filter_mats.get_ref_ignore_rank(tensor_pos(d4_filt, y_filt, 0, 0, 0))), + static_cast(out_depth), + static_cast(f_width * f_depth)); + for (std::size_t d4 = 0, d4_out = 0; d4 < in.shape().size_dim_4_ + 1 - f_size_d4; d4 += strides_d4, ++d4_out) { + for (std::size_t y = 0, y_out = 0; y < in.shape().height_ + 1 - f_height; y += strides_y, ++y_out) { + const auto input = get_im2col_mapping_3d(in, f_width, f_depth, strides_x, out_width, d4, y, d4_filt, y_filt); + Eigen::Map + output_map(&output.get_ref_ignore_rank(tensor_pos(0, d4_out, y_out, 0, 0)), + static_cast(out_depth), + static_cast(out_width)); + + output_map.noalias() += filter * input; + } + } + } + } + + return output; + } + + inline tensor convolve_3d( + const shape3& strides, + const padding& pad_type, + const convolution3d_filter_matrices& filter_mat, + const tensor& input) + { + assertion(filter_mat.filter_shape_.depth_ == input.shape().depth_, + "invalid filter depth"); + + const shape3 filter_spatial_shape( + filter_mat.filter_shape_.size_dim_4_, + filter_mat.filter_shape_.height_, + filter_mat.filter_shape_.width_); + + const auto conv_cfg = preprocess_convolution_3d( + filter_spatial_shape, + strides, pad_type, + input.shape().size_dim_4_, + input.shape().height_, + input.shape().width_); + + const auto in_padded = pad_tensor(0, + conv_cfg.pad_front_, conv_cfg.pad_back_, + conv_cfg.pad_top_, conv_cfg.pad_bottom_, + conv_cfg.pad_left_, conv_cfg.pad_right_, + input); + + return convolve_accumulative_3d( + conv_cfg.out_size_d4_, conv_cfg.out_height_, conv_cfg.out_width_, + strides.size_dim_4_, strides.height_, strides.width_, + filter_mat, + in_padded); + } + } } diff --git a/include/fdeep/import_model.hpp b/include/fdeep/import_model.hpp index 2a2bafd0..0b46e393 100644 --- a/include/fdeep/import_model.hpp +++ b/include/fdeep/import_model.hpp @@ -39,6 +39,7 @@ #include "fdeep/layers/concatenate_layer.hpp" #include "fdeep/layers/conv_2d_layer.hpp" #include "fdeep/layers/conv_2d_transpose_layer.hpp" +#include "fdeep/layers/conv_3d_layer.hpp" #include "fdeep/layers/cropping_3d_layer.hpp" #include "fdeep/layers/dense_layer.hpp" #include "fdeep/layers/depthwise_conv_2d_layer.hpp" @@ -416,6 +417,36 @@ namespace internal { dilation_rate, weights, bias); } + inline layer_ptr create_conv_3d_layer(const get_param_f& get_param, + const nlohmann::json& data, + const std::string& name) + { + const std::string padding_str = data["config"]["padding"]; + const auto pad_type = create_padding(padding_str); + + const shape3 strides = create_shape3(data["config"]["strides"]); + const shape3 dilation_rate = create_shape3(data["config"]["dilation_rate"]); + + const auto filter_count = create_size_t(data["config"]["filters"]); + float_vec bias(filter_count, 0); + const bool use_bias = data["config"]["use_bias"]; + if (use_bias) + bias = decode_floats(get_param(name, "bias")); + assertion(bias.size() == filter_count, "size of bias does not match"); + + const float_vec weights = decode_floats(get_param(name, "weights")); + const shape3 kernel_size = create_shape3(data["config"]["kernel_size"]); + assertion(weights.size() % kernel_size.volume() == 0, + "invalid number of weights"); + const std::size_t filter_depths = weights.size() / (kernel_size.volume() * filter_count); + const tensor_shape filter_shape( + kernel_size.size_dim_4_, kernel_size.height_, kernel_size.width_, filter_depths); + + return std::make_shared(name, + filter_shape, filter_count, strides, pad_type, + dilation_rate, weights, bias); + } + inline layer_ptr create_conv_2d_transpose_layer(const get_param_f& get_param, const nlohmann::json& data, const std::string& name) @@ -1267,6 +1298,7 @@ namespace internal { { "Identity", create_identity_layer }, { "Conv1D", create_conv_2d_layer }, { "Conv2D", create_conv_2d_layer }, + { "Conv3D", create_conv_3d_layer }, { "Conv1DTranspose", create_conv_2d_transpose_layer }, { "Conv2DTranspose", create_conv_2d_transpose_layer }, { "SeparableConv1D", create_separable_conv_2D_layer }, diff --git a/include/fdeep/layers/conv_3d_layer.hpp b/include/fdeep/layers/conv_3d_layer.hpp new file mode 100644 index 00000000..d7df9bc6 --- /dev/null +++ b/include/fdeep/layers/conv_3d_layer.hpp @@ -0,0 +1,54 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/convolution3d.hpp" +#include "fdeep/filter.hpp" +#include "fdeep/layers/layer.hpp" +#include "fdeep/shape3.hpp" +#include "fdeep/tensor_shape.hpp" + +#include + +#include +#include +#include + +namespace fdeep { +namespace internal { + + class conv_3d_layer : public layer { + public: + explicit conv_3d_layer( + const std::string& name, const tensor_shape& filter_shape, + std::size_t k, const shape3& strides, padding p, + const shape3& dilation_rate, + const float_vec& weights, const float_vec& bias) + : layer(name) + , filters_(generate_im2col_filter_matrix_3d( + generate_filters_3d(dilation_rate, filter_shape, k, weights, bias))) + , strides_(strides) + , padding_(p) + { + assertion(k > 0, "needs at least one filter"); + assertion(filter_shape.volume() > 0, "filter must have volume"); + assertion(strides.volume() > 0, "invalid strides"); + } + + protected: + tensors apply_impl(const tensors& inputs) const override + { + const auto& input = single_tensor_from_tensors(inputs); + return { convolve_3d(strides_, padding_, filters_, input) }; + } + convolution3d_filter_matrices filters_; + shape3 strides_; + padding padding_; + }; + +} +} diff --git a/keras_export/convert_model.py b/keras_export/convert_model.py index b1137cc8..3aa58fff 100755 --- a/keras_export/convert_model.py +++ b/keras_export/convert_model.py @@ -222,6 +222,12 @@ def prepare_filter_weights_conv_2d_transpose(weights: NDFloat32Array) -> NDFloat return np.moveaxis(weights, [0, 1, 2, 3], [1, 2, 0, 3]).flatten() +def prepare_filter_weights_conv_3d(weights: NDFloat32Array) -> NDFloat32Array: + """Change dimension order of 3d filter weights to the one used in fdeep""" + assert len(weights.shape) == 5 + return np.moveaxis(weights, [0, 1, 2, 3, 4], [1, 2, 3, 4, 0]).flatten() + + def show_conv_1d_layer(layer: Layer) -> Mapping[str, list[str]]: """Serialize Conv1D layer to dict""" weights = layer.get_weights() @@ -260,6 +266,25 @@ def show_conv_2d_layer(layer: Layer) -> Mapping[str, list[str]]: return result +def show_conv_3d_layer(layer: Layer) -> Mapping[str, list[str]]: + """Serialize Conv3D layer to dict""" + weights = layer.get_weights() + assert len(weights) == 1 or len(weights) == 2 + assert len(weights[0].shape) == 5 + weights_flat = prepare_filter_weights_conv_3d(weights[0]) + assert layer.padding in ['valid', 'same'] + assert layer.groups == 1 + assert len(get_layer_input_shape(layer)) == 5 + assert get_layer_input_shape(layer)[0] in {None, 1} + result = { + 'weights': encode_floats(weights_flat) + } + if len(weights) == 2: + bias = weights[1] + result['bias'] = encode_floats(bias) + return result + + def show_separable_conv_2d_layer(layer: Layer) -> Mapping[str, list[str]]: """Serialize SeparableConv2D layer to dict""" weights = layer.get_weights() @@ -498,6 +523,7 @@ def get_layer_functions_dict() -> Mapping[str, Callable[[Layer], LayerConfig]]: return { 'Conv1D': show_conv_1d_layer, 'Conv2D': show_conv_2d_layer, + 'Conv3D': show_conv_3d_layer, 'Conv1DTranspose': show_conv_1d_transpose_layer, 'Conv2DTranspose': show_conv_2d_transpose_layer, 'SeparableConv2D': show_separable_conv_2d_layer, diff --git a/keras_export/generate_test_models.py b/keras_export/generate_test_models.py index dc2da305..b2b9522c 100644 --- a/keras_export/generate_test_models.py +++ b/keras_export/generate_test_models.py @@ -14,6 +14,7 @@ from keras.layers import CategoryEncoding, Embedding from keras.layers import Conv1D, ZeroPadding1D, Cropping1D from keras.layers import Conv2D, ZeroPadding2D, Cropping2D, CenterCrop +from keras.layers import Conv3D from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D from keras.layers import GlobalAveragePooling2D, GlobalMaxPooling2D from keras.layers import GlobalAveragePooling3D, GlobalMaxPooling3D @@ -230,6 +231,11 @@ def get_test_model_exhaustive() -> Model: outputs.append(Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4])) outputs.append(Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4])) + outputs.append(Conv3D(4, (2, 3, 3))(inputs[2])) + outputs.append(Conv3D(4, (2, 3, 3), use_bias=False, padding='valid')(inputs[2])) + outputs.append(Conv3D(4, (1, 2, 4), strides=(2, 2, 3), padding='same')(inputs[2])) + outputs.append(Conv3D(4, (1, 2, 4), padding='same', dilation_rate=(2, 2, 3))(inputs[2])) + outputs.append(SeparableConv2D(3, (3, 3))(inputs[4])) outputs.append(DepthwiseConv2D((3, 3))(inputs[4])) outputs.append(DepthwiseConv2D((1, 2))(inputs[4]))