diff --git a/README.md b/README.md index b72aae76..eff6a0f1 100644 --- a/README.md +++ b/README.md @@ -41,21 +41,31 @@ Would you like to build/train a model using Keras/Python? And would you like to * `Add`, `Concatenate`, `Subtract`, `Multiply`, `Average`, `Maximum`, `Minimum`, `Dot` * `AveragePooling1D/2D/3D`, `GlobalAveragePooling1D/2D/3D` +* `AdaptiveAveragePooling1D/2D/3D`, `AdaptiveMaxPooling1D/2D/3D` * `TimeDistributed` -* `Conv1D/2D/3D`, `SeparableConv2D`, `DepthwiseConv2D` +* `Conv1D/2D/3D`, `SeparableConv2D`, `DepthwiseConv1D`, `DepthwiseConv2D` * `Conv1DTranspose`, `Conv2DTranspose`, `Conv3DTranspose` * `Cropping1D/2D/3D`, `ZeroPadding1D/2D/3D`, `CenterCrop` -* `BatchNormalization`, `Dense`, `Flatten`, `Normalization` +* `BatchNormalization`, `Dense`, `EinsumDense`, `Flatten`, `Normalization` * `Dropout`, `AlphaDropout`, `GaussianDropout`, `GaussianNoise` * `SpatialDropout1D`, `SpatialDropout2D`, `SpatialDropout3D` -* `ActivityRegularization`, `LayerNormalization`, `UnitNormalization` -* `RandomContrast`, `RandomFlip`, `RandomHeight` -* `RandomRotation`, `RandomTranslation`, `RandomWidth`, `RandomZoom` +* `ActivityRegularization`, `LayerNormalization`, `RMSNormalization` +* `GroupNormalization`, `UnitNormalization` +* Training-only image augmentation layers (passed through at inference): + `RandomBrightness`, `RandomContrast`, `RandomCrop`, `RandomFlip`, `RandomHue`, + `RandomGrayscale`, `RandomRotation`, `RandomTranslation`, `RandomZoom`, + `RandomShear`, `RandomSaturation`, `RandomPerspective`, `AutoContrast`, + `AugMix`, `CutMix`, `MixUp`, `RandAugment`, `Solarization`, `Equalization` * `MaxPooling1D/2D/3D`, `GlobalMaxPooling1D/2D/3D` * `UpSampling1D/2D/3D`, `Resizing`, `Rescaling` * `Reshape`, `Permute`, `RepeatVector` * `Embedding`, `CategoryEncoding` -* `Attention`, `AdditiveAttention`, `MultiHeadAttention` +* `Discretization`, `IntegerLookup` +* `Masking` (passthrough at inference) +* `Attention`, `AdditiveAttention`, `MultiHeadAttention`, `GroupedQueryAttention` +* `LSTM`, `GRU`, `SimpleRNN`, `Bidirectional` +* `RNN` wrapping `LSTMCell`/`GRUCell`/`SimpleRNNCell`/`StackedRNNCells` +* `ConvLSTM1D`, `ConvLSTM2D`, `ConvLSTM3D` ### Also supported @@ -71,15 +81,11 @@ Would you like to build/train a model using Keras/Python? And would you like to ### Currently not supported are the following: `Lambda` ([why](FAQ.md#why-are-lambda-layers-not-supported)), -`ConvLSTM1D`, `ConvLSTM2D`, `Discretization`, -`GRUCell`, `Hashing`, -`IntegerLookup`, -`LocallyConnected1D`, `LocallyConnected2D`, -`LSTMCell`, `Masking`, -`RepeatVector`, `RNN`, `SimpleRNN`, -`SimpleRNNCell`, `StackedRNNCells`, `StringLookup`, `TextVectorization`, -`Bidirectional`, `GRU`, `LSTM`, `CuDNNGRU`, `CuDNNLSTM`, -`ThresholdedReLU`, `temporal` models +`Hashing`, `HashedCrossing`, +`MelSpectrogram`, `STFTSpectrogram`, +`StringLookup`, `TextVectorization`, +stateful recurrent layers, +`temporal` models Usage ----- diff --git a/include/fdeep/common.hpp b/include/fdeep/common.hpp index 6b6896dd..a3c36967 100644 --- a/include/fdeep/common.hpp +++ b/include/fdeep/common.hpp @@ -80,7 +80,7 @@ namespace internal { using RowMajorMatrixXf = Eigen::Matrix; using ArrayXf = Eigen::Array; using ArrayXf1D = Eigen::Array; - using MappedRowMajorMatrixXf = Eigen::Map; + using MappedRowMajorMatrixXf = Eigen::Map; inline float_type tanh_typed(float_type x) { diff --git a/include/fdeep/import_model.hpp b/include/fdeep/import_model.hpp index 063482ee..bd0b2093 100644 --- a/include/fdeep/import_model.hpp +++ b/include/fdeep/import_model.hpp @@ -27,12 +27,14 @@ #include "fdeep/common.hpp" +#include "fdeep/layers/adaptive_pooling_3d_layer.hpp" #include "fdeep/layers/add_layer.hpp" #include "fdeep/layers/additive_attention_layer.hpp" #include "fdeep/layers/attention_layer.hpp" #include "fdeep/layers/average_layer.hpp" #include "fdeep/layers/average_pooling_3d_layer.hpp" #include "fdeep/layers/batch_normalization_layer.hpp" +#include "fdeep/layers/bidirectional_layer.hpp" #include "fdeep/layers/category_encoding_layer.hpp" #include "fdeep/layers/celu_layer.hpp" #include "fdeep/layers/centercrop_layer.hpp" @@ -41,10 +43,14 @@ #include "fdeep/layers/conv_2d_transpose_layer.hpp" #include "fdeep/layers/conv_3d_layer.hpp" #include "fdeep/layers/conv_3d_transpose_layer.hpp" +#include "fdeep/layers/conv_lstm_2d_layer.hpp" +#include "fdeep/layers/conv_lstm_3d_layer.hpp" #include "fdeep/layers/cropping_3d_layer.hpp" #include "fdeep/layers/dense_layer.hpp" #include "fdeep/layers/depthwise_conv_2d_layer.hpp" +#include "fdeep/layers/discretization_layer.hpp" #include "fdeep/layers/dot_layer.hpp" +#include "fdeep/layers/einsum_dense_layer.hpp" #include "fdeep/layers/elu_layer.hpp" #include "fdeep/layers/embedding_layer.hpp" #include "fdeep/layers/exponential_layer.hpp" @@ -52,16 +58,21 @@ #include "fdeep/layers/gelu_layer.hpp" #include "fdeep/layers/global_average_pooling_3d_layer.hpp" #include "fdeep/layers/global_max_pooling_3d_layer.hpp" +#include "fdeep/layers/group_normalization_layer.hpp" +#include "fdeep/layers/group_query_attention_layer.hpp" +#include "fdeep/layers/gru_layer.hpp" #include "fdeep/layers/hard_shrink_layer.hpp" #include "fdeep/layers/hard_sigmoid_layer.hpp" #include "fdeep/layers/hard_tanh_layer.hpp" #include "fdeep/layers/input_layer.hpp" +#include "fdeep/layers/integer_lookup_layer.hpp" #include "fdeep/layers/layer.hpp" #include "fdeep/layers/layer_normalization_layer.hpp" #include "fdeep/layers/leaky_relu_layer.hpp" #include "fdeep/layers/linear_layer.hpp" #include "fdeep/layers/log_sigmoid_layer.hpp" #include "fdeep/layers/log_softmax_layer.hpp" +#include "fdeep/layers/lstm_layer.hpp" #include "fdeep/layers/max_pooling_3d_layer.hpp" #include "fdeep/layers/maximum_layer.hpp" #include "fdeep/layers/minimum_layer.hpp" @@ -77,15 +88,18 @@ #include "fdeep/layers/rescaling_layer.hpp" #include "fdeep/layers/reshape_layer.hpp" #include "fdeep/layers/resizing_layer.hpp" +#include "fdeep/layers/rms_normalization_layer.hpp" #include "fdeep/layers/selu_layer.hpp" #include "fdeep/layers/separable_conv_2d_layer.hpp" #include "fdeep/layers/sigmoid_layer.hpp" +#include "fdeep/layers/simple_rnn_layer.hpp" #include "fdeep/layers/soft_shrink_layer.hpp" #include "fdeep/layers/softmax_layer.hpp" #include "fdeep/layers/softplus_layer.hpp" #include "fdeep/layers/softsign_layer.hpp" #include "fdeep/layers/sparse_plus_layer.hpp" #include "fdeep/layers/square_plus_layer.hpp" +#include "fdeep/layers/stacked_rnn_layer.hpp" #include "fdeep/layers/subtract_layer.hpp" #include "fdeep/layers/swish_layer.hpp" #include "fdeep/layers/tanh_layer.hpp" @@ -282,6 +296,30 @@ namespace internal { return out; } + inline std::string get_activation_type(const nlohmann::json& data) + { + assertion(data.is_string(), "Layer activation must be a string."); + return data; + } + + inline std::string json_object_get_activation_with_default(const nlohmann::json& config, + const std::string& default_activation) + { + if (json_obj_has_member(config, "activation")) { + return get_activation_type(config["activation"]); + } + return default_activation; + } + + inline std::string json_object_get_named_activation_with_default(const nlohmann::json& config, + const std::string& key, const std::string& default_activation) + { + if (json_obj_has_member(config, key)) { + return get_activation_type(config[key]); + } + return default_activation; + } + inline tensor create_tensor(const nlohmann::json& data) { const tensor_shape shape = create_tensor_shape(data["shape"]); @@ -627,6 +665,206 @@ namespace internal { name, axes, beta, gamma, epsilon); } + inline layer_ptr create_rms_normalization_layer(const get_param_f& get_param, + const nlohmann::json& data, const std::string& name) + { + const auto axes = create_vector(create_int, data["config"]["axis"]); + const float_type epsilon = data["config"]["epsilon"]; + const float_vec scale = decode_floats(get_param(name, "scale")); + return std::make_shared( + name, axes, scale, epsilon); + } + + inline layer_ptr create_adaptive_avg_pooling_layer(const get_param_f&, + const nlohmann::json& data, const std::string& name) + { + const auto& sz = data["config"]["output_size"]; + std::vector dims; + if (sz.is_array()) + for (const auto& v : sz) + dims.push_back(static_cast(v)); + else + dims.push_back(static_cast(sz)); + const std::size_t d4 = dims.size() >= 3 ? dims[dims.size() - 3] : 1; + const std::size_t h = dims.size() >= 2 ? dims[dims.size() - 2] : 1; + const std::size_t w = dims.back(); + return std::make_shared(name, d4, h, w, + adaptive_pooling_kind::avg); + } + + inline layer_ptr create_adaptive_max_pooling_layer(const get_param_f&, + const nlohmann::json& data, const std::string& name) + { + const auto& sz = data["config"]["output_size"]; + std::vector dims; + if (sz.is_array()) + for (const auto& v : sz) + dims.push_back(static_cast(v)); + else + dims.push_back(static_cast(sz)); + const std::size_t d4 = dims.size() >= 3 ? dims[dims.size() - 3] : 1; + const std::size_t h = dims.size() >= 2 ? dims[dims.size() - 2] : 1; + const std::size_t w = dims.back(); + return std::make_shared(name, d4, h, w, + adaptive_pooling_kind::max); + } + + inline layer_ptr create_conv_lstm_3d_layer(const get_param_f& get_param, + const nlohmann::json& data, const std::string& name) + { + auto&& cfg = data["config"]; + const std::size_t units = cfg["filters"]; + const shape3 strides = create_shape3(cfg["strides"]); + const shape3 dilation_rate = create_shape3(cfg["dilation_rate"]); + const std::string padding_str = cfg["padding"]; + const auto pad_type = create_padding(padding_str); + const shape3 kernel_size = create_shape3(cfg["kernel_size"]); + const std::string activation = json_object_get_activation_with_default(cfg, "tanh"); + const std::string recurrent_activation = json_object_get_named_activation_with_default( + cfg, "recurrent_activation", "sigmoid"); + const bool use_bias = json_object_get(cfg, "use_bias", true); + const bool return_sequences = json_object_get(cfg, "return_sequences", false); + const bool return_state = json_object_get(cfg, "return_state", false); + + const float_vec weights = decode_floats(get_param(name, "weights")); + const float_vec recurrent_weights = decode_floats(get_param(name, "recurrent_weights")); + float_vec bias(units * 4, 0); + if (use_bias) + bias = decode_floats(get_param(name, "bias")); + + const std::size_t kernel_volume = kernel_size.size_dim_4_ + * kernel_size.height_ * kernel_size.width_; + const std::size_t in_c = weights.size() / (kernel_volume * units * 4); + const tensor_shape filter_shape(kernel_size.size_dim_4_, + kernel_size.height_, kernel_size.width_, in_c); + + return std::make_shared(name, units, + filter_shape, strides, pad_type, dilation_rate, + weights, recurrent_weights, bias, + activation, recurrent_activation, + return_sequences, return_state); + } + + inline layer_ptr create_conv_lstm_2d_layer(const get_param_f& get_param, + const nlohmann::json& data, const std::string& name) + { + auto&& cfg = data["config"]; + const std::string class_name = data["class_name"]; + const std::size_t rank = (class_name == "ConvLSTM1D") ? 1 : 2; + assertion(class_name == "ConvLSTM1D" || class_name == "ConvLSTM2D", + "create_conv_lstm_2d_layer: unsupported layer class."); + const std::size_t units = cfg["filters"]; + const shape2 strides = create_shape2(cfg["strides"]); + const shape2 dilation_rate = create_shape2(cfg["dilation_rate"]); + const std::string padding_str = cfg["padding"]; + const auto pad_type = create_padding(padding_str); + const shape2 kernel_size = create_shape2(cfg["kernel_size"]); + const std::string activation = json_object_get_activation_with_default(cfg, "tanh"); + const std::string recurrent_activation = json_object_get_named_activation_with_default( + cfg, "recurrent_activation", "sigmoid"); + const bool use_bias = json_object_get(cfg, "use_bias", true); + const bool return_sequences = json_object_get(cfg, "return_sequences", false); + const bool return_state = json_object_get(cfg, "return_state", false); + + const float_vec weights = decode_floats(get_param(name, "weights")); + const float_vec recurrent_weights = decode_floats(get_param(name, "recurrent_weights")); + float_vec bias(units * 4, 0); + if (use_bias) + bias = decode_floats(get_param(name, "bias")); + + // Determine in_channels from weight count. + // Input weights: (k_h, k_w, in_c, units*4). + const std::size_t in_c = weights.size() / (kernel_size.area() * units * 4); + const tensor_shape filter_shape(kernel_size.height_, kernel_size.width_, in_c); + + return std::make_shared(name, units, rank, + filter_shape, strides, pad_type, dilation_rate, + weights, recurrent_weights, bias, + activation, recurrent_activation, + return_sequences, return_state); + } + + inline layer_ptr create_discretization_layer(const get_param_f&, + const nlohmann::json& data, const std::string& name) + { + const std::string output_mode = data["config"]["output_mode"]; + assertion(output_mode == "int", + "Discretization only supports output_mode='int'."); + std::vector boundaries; + for (const auto& v : data["config"]["bin_boundaries"]) + boundaries.push_back(static_cast(v)); + return std::make_shared(name, boundaries); + } + + inline layer_ptr create_integer_lookup_layer(const get_param_f&, + const nlohmann::json& data, const std::string& name) + { + auto&& cfg = data["config"]; + const std::string output_mode = cfg["output_mode"]; + assertion(output_mode == "int", + "IntegerLookup only supports output_mode='int'."); + assertion(!cfg.value("invert", false), + "IntegerLookup with invert=True is not supported."); + const std::size_t num_oov_indices = cfg["num_oov_indices"]; + const bool has_mask_token = !cfg["mask_token"].is_null(); + const std::int64_t mask_token = has_mask_token + ? static_cast(cfg["mask_token"]) + : 0; + std::vector vocabulary; + for (const auto& v : cfg["vocabulary"]) + vocabulary.push_back(static_cast(v)); + return std::make_shared(name, vocabulary, + num_oov_indices, has_mask_token, mask_token); + } + + inline layer_ptr create_einsum_dense_layer(const get_param_f& get_param, + const nlohmann::json& data, const std::string& name) + { + auto&& config = data["config"]; + const std::string equation = config["equation"]; + const std::string bias_axes = config["bias_axes"].is_null() + ? std::string("") + : std::string(config["bias_axes"]); + // Keras's output_shape excludes the batch dimension. Prepend -1 for + // the batch char so the layer can index it positionally against rhs. + std::vector output_shape; + output_shape.push_back(-1); + for (const auto& dim : config["output_shape"]) + output_shape.push_back(dim.is_null() ? -1 : static_cast(dim)); + + const auto kernel_shape = create_vector(create_size_t, get_param(name, "kernel_shape")); + auto kernel_values = decode_floats(get_param(name, "kernel")); + const tensor kernel_tensor(create_tensor_shape_from_dims(kernel_shape), std::move(kernel_values)); + + tensor bias_tensor(tensor_shape(static_cast(1)), float_type(0)); + if (!bias_axes.empty()) { + const auto bias_shape = create_vector(create_size_t, get_param(name, "bias_shape")); + auto bias_values = decode_floats(get_param(name, "bias")); + bias_tensor = tensor(create_tensor_shape_from_dims(bias_shape), std::move(bias_values)); + } + + return std::make_shared(name, equation, + output_shape, bias_axes, kernel_tensor, bias_tensor); + } + + inline layer_ptr create_group_normalization_layer(const get_param_f& get_param, + const nlohmann::json& data, const std::string& name) + { + const std::size_t groups = data["config"]["groups"]; + const int axis = data["config"]["axis"]; + const float_type epsilon = data["config"]["epsilon"]; + const bool center = data["config"]["center"]; + const bool scale = data["config"]["scale"]; + float_vec gamma; + float_vec beta; + if (scale) + gamma = decode_floats(get_param(name, "gamma")); + if (center) + beta = decode_floats(get_param(name, "beta")); + return std::make_shared( + name, groups, axis, epsilon, beta, gamma); + } + inline layer_ptr create_unit_normalization_layer(const get_param_f&, const nlohmann::json& data, const std::string& name) { @@ -1203,6 +1441,30 @@ namespace internal { return std::make_shared(name, scale); } + inline layer_ptr create_group_query_attention_layer( + const get_param_f& get_param, + const nlohmann::json& data, const std::string& name) + { + const std::size_t head_dim = data["config"]["head_dim"]; + const std::size_t num_query_heads = data["config"]["num_query_heads"]; + const std::size_t num_kv_heads = data["config"]["num_key_value_heads"]; + const bool use_bias = data["config"]["use_bias"]; + const bool use_gate = json_object_get(data["config"], "use_gate", false); + + const auto weight_shapes = create_vector>(fplus::bind_1st_of_2( + create_vector, create_size_t), + get_param(name, "weight_shapes")); + const auto weight_values = create_vector(decode_floats, get_param(name, "weights")); + const auto weights = fplus::zip_with( + [](const std::vector& shape, const float_vec& values) -> tensor { + return tensor(create_tensor_shape_from_dims(shape), + fplus::convert_container(values)); + }, + weight_shapes, weight_values); + return std::make_shared(name, + head_dim, num_query_heads, num_kv_heads, use_bias, use_gate, weights); + } + inline layer_ptr create_multi_head_attention_layer( const get_param_f& get_param, const nlohmann::json& data, const std::string& name) @@ -1226,10 +1488,174 @@ namespace internal { num_heads, key_dim, value_dim, use_bias, weights_and_biases); } - inline std::string get_activation_type(const nlohmann::json& data) + inline layer_ptr create_lstm_layer(const get_param_f& get_param, + const nlohmann::json& data, + const std::string& name) { - assertion(data.is_string(), "Layer activation must be a string."); - return data; + auto&& config = data["config"]; + const std::size_t units = config["units"]; + const std::string unit_activation = json_object_get_activation_with_default(config, "tanh"); + const std::string recurrent_activation = json_object_get_named_activation_with_default(config, "recurrent_activation", "sigmoid"); + const bool use_bias = json_object_get(config, "use_bias", true); + const bool return_sequences = json_object_get(config, "return_sequences", false); + const bool return_state = json_object_get(config, "return_state", false); + + float_vec bias; + if (use_bias) + bias = decode_floats(get_param(name, "bias")); + + const float_vec weights = decode_floats(get_param(name, "weights")); + const float_vec recurrent_weights = decode_floats(get_param(name, "recurrent_weights")); + + return std::make_shared(name, units, unit_activation, + recurrent_activation, use_bias, + return_sequences, return_state, + weights, recurrent_weights, bias); + } + + inline layer_ptr create_gru_layer(const get_param_f& get_param, + const nlohmann::json& data, + const std::string& name) + { + auto&& config = data["config"]; + const std::size_t units = config["units"]; + const std::string unit_activation = json_object_get_activation_with_default(config, "tanh"); + const std::string recurrent_activation = json_object_get_named_activation_with_default(config, "recurrent_activation", "sigmoid"); + const bool use_bias = json_object_get(config, "use_bias", true); + const bool return_sequences = json_object_get(config, "return_sequences", false); + const bool return_state = json_object_get(config, "return_state", false); + const bool reset_after = json_object_get(config, "reset_after", true); + + float_vec bias; + if (use_bias) + bias = decode_floats(get_param(name, "bias")); + + const float_vec weights = decode_floats(get_param(name, "weights")); + const float_vec recurrent_weights = decode_floats(get_param(name, "recurrent_weights")); + + return std::make_shared(name, units, unit_activation, + recurrent_activation, use_bias, reset_after, + return_sequences, return_state, + weights, recurrent_weights, bias); + } + + inline layer_ptr create_simple_rnn_layer(const get_param_f& get_param, + const nlohmann::json& data, + const std::string& name) + { + auto&& config = data["config"]; + const std::size_t units = config["units"]; + const std::string unit_activation = json_object_get_activation_with_default(config, "tanh"); + const bool use_bias = json_object_get(config, "use_bias", true); + const bool return_sequences = json_object_get(config, "return_sequences", false); + const bool return_state = json_object_get(config, "return_state", false); + + float_vec bias; + if (use_bias) + bias = decode_floats(get_param(name, "bias")); + + const float_vec weights = decode_floats(get_param(name, "weights")); + const float_vec recurrent_weights = decode_floats(get_param(name, "recurrent_weights")); + + return std::make_shared(name, units, unit_activation, + use_bias, return_sequences, return_state, + weights, recurrent_weights, bias); + } + + inline layer_ptr create_rnn_from_cell(const get_param_f& get_param, + const std::string& cell_class, const nlohmann::json& cell_config, + const nlohmann::json& outer_cfg, const std::string& name, + bool override_return_sequences) + { + nlohmann::json synthetic; + synthetic["class_name"] = cell_class == "LSTMCell" ? "LSTM" + : cell_class == "GRUCell" ? "GRU" + : cell_class == "SimpleRNNCell" ? "SimpleRNN" + : std::string(""); + synthetic["config"] = cell_config; + for (const char* key : { "return_sequences", "return_state", "go_backwards", "stateful", "unroll" }) { + if (json_obj_has_member(outer_cfg, key)) + synthetic["config"][key] = outer_cfg[key]; + } + if (override_return_sequences) + synthetic["config"]["return_sequences"] = true; + + if (cell_class == "LSTMCell") + return create_lstm_layer(get_param, synthetic, name); + if (cell_class == "GRUCell") + return create_gru_layer(get_param, synthetic, name); + if (cell_class == "SimpleRNNCell") + return create_simple_rnn_layer(get_param, synthetic, name); + raise_error("RNN cell '" + cell_class + "' is not supported."); + return {}; + } + + inline layer_ptr create_rnn_layer(const get_param_f& get_param, + const nlohmann::json& data, const std::string& name) + { + auto&& cfg = data["config"]; + const auto& cell = cfg["cell"]; + const std::string cell_class = cell["class_name"]; + + if (cell_class == "StackedRNNCells") { + const auto& cells = cell["config"]["cells"]; + const std::size_t num_cells = cells.size(); + assertion(num_cells > 0, "StackedRNNCells must contain at least one cell."); + std::vector inner_layers; + inner_layers.reserve(num_cells); + for (std::size_t i = 0; i < num_cells; ++i) { + const auto& sub_cell = cells[i]; + const std::string sub_class = sub_cell["class_name"]; + const std::string sub_name = name + "_cell" + std::to_string(i); + const std::string prefix = "cell" + std::to_string(i) + "_"; + const get_param_f sub_get_param = + [&get_param, name, prefix](const std::string& layer_name, const std::string& key) { + (void)layer_name; + return get_param(name, prefix + key); + }; + const bool is_last = (i + 1 == num_cells); + inner_layers.push_back(create_rnn_from_cell(sub_get_param, + sub_class, sub_cell["config"], cfg, sub_name, !is_last)); + } + return std::make_shared(name, inner_layers); + } + + return create_rnn_from_cell(get_param, cell_class, cell["config"], + cfg, name, false); + } + + inline layer_ptr create_bidirectional_layer(const get_param_f& get_param, + const nlohmann::json& data, + const std::string& name) + { + const std::string merge_mode = data["config"]["merge_mode"]; + auto&& wrapped = data["config"]["layer"]; + auto&& wrapped_config = wrapped["config"]; + const std::string wrapped_layer_type = wrapped["class_name"]; + const std::size_t units = wrapped_config["units"]; + const std::string unit_activation = json_object_get_activation_with_default(wrapped_config, "tanh"); + const std::string recurrent_activation = json_object_get_named_activation_with_default(wrapped_config, "recurrent_activation", "sigmoid"); + const bool use_bias = json_object_get(wrapped_config, "use_bias", true); + const bool return_sequences = json_object_get(wrapped_config, "return_sequences", false); + const bool reset_after = json_object_get(wrapped_config, "reset_after", true); + + float_vec forward_bias; + float_vec backward_bias; + if (use_bias) { + forward_bias = decode_floats(get_param(name, "forward_bias")); + backward_bias = decode_floats(get_param(name, "backward_bias")); + } + + const float_vec forward_weights = decode_floats(get_param(name, "forward_weights")); + const float_vec backward_weights = decode_floats(get_param(name, "backward_weights")); + const float_vec forward_recurrent_weights = decode_floats(get_param(name, "forward_recurrent_weights")); + const float_vec backward_recurrent_weights = decode_floats(get_param(name, "backward_recurrent_weights")); + + return std::make_shared(name, merge_mode, units, + unit_activation, recurrent_activation, wrapped_layer_type, + use_bias, reset_after, return_sequences, + forward_weights, forward_recurrent_weights, forward_bias, + backward_weights, backward_recurrent_weights, backward_bias); } inline activation_layer_ptr create_activation_layer_type_name( @@ -1359,12 +1785,16 @@ namespace internal { { "Conv3DTranspose", create_conv_3d_transpose_layer }, { "SeparableConv1D", create_separable_conv_2D_layer }, { "SeparableConv2D", create_separable_conv_2D_layer }, + { "DepthwiseConv1D", create_depthwise_conv_2D_layer }, { "DepthwiseConv2D", create_depthwise_conv_2D_layer }, { "InputLayer", create_input_layer }, { "BatchNormalization", create_batch_normalization_layer }, + { "GroupNormalization", create_group_normalization_layer }, { "LayerNormalization", create_layer_normalization_layer }, + { "RMSNormalization", create_rms_normalization_layer }, { "UnitNormalization", create_unit_normalization_layer }, { "Dropout", create_identity_layer }, + { "Masking", create_identity_layer }, { "ActivityRegularization", create_identity_layer }, { "AlphaDropout", create_identity_layer }, { "FixedDropout", create_identity_layer }, @@ -1373,13 +1803,39 @@ namespace internal { { "SpatialDropout1D", create_identity_layer }, { "SpatialDropout2D", create_identity_layer }, { "SpatialDropout3D", create_identity_layer }, + { "RandomBrightness", create_identity_layer }, + { "RandomColorDegeneration", create_identity_layer }, + { "RandomColorJitter", create_identity_layer }, { "RandomContrast", create_identity_layer }, + { "RandomCrop", create_identity_layer }, + { "RandomElasticTransform", create_identity_layer }, + { "RandomErasing", create_identity_layer }, { "RandomFlip", create_identity_layer }, + { "RandomGaussianBlur", create_identity_layer }, + { "RandomGrayscale", create_identity_layer }, { "RandomHeight", create_identity_layer }, + { "RandomHue", create_identity_layer }, + { "RandomInvert", create_identity_layer }, + { "RandomPerspective", create_identity_layer }, + { "RandomPosterization", create_identity_layer }, { "RandomRotation", create_identity_layer }, + { "RandomSaturation", create_identity_layer }, + { "RandomSharpness", create_identity_layer }, + { "RandomShear", create_identity_layer }, { "RandomTranslation", create_identity_layer }, { "RandomWidth", create_identity_layer }, { "RandomZoom", create_identity_layer }, + // AutoContrast is intentionally NOT registered as a passthrough: + // it applies a deterministic per-image min/max stretch even at + // inference. Supporting it would require a real implementation. + { "AugMix", create_identity_layer }, + { "CutMix", create_identity_layer }, + { "Equalization", create_identity_layer }, + { "MaxNumBoundingBoxes", create_identity_layer }, + { "MixUp", create_identity_layer }, + { "Pipeline", create_identity_layer }, + { "RandAugment", create_identity_layer }, + { "Solarization", create_identity_layer }, { "LeakyReLU", create_leaky_relu_layer }, { "Permute", create_permute_layer }, { "PReLU", create_prelu_layer }, @@ -1413,6 +1869,12 @@ namespace internal { { "AveragePooling1D", create_average_pooling_3d_layer }, { "AveragePooling2D", create_average_pooling_3d_layer }, { "AveragePooling3D", create_average_pooling_3d_layer }, + { "AdaptiveAveragePooling1D", create_adaptive_avg_pooling_layer }, + { "AdaptiveAveragePooling2D", create_adaptive_avg_pooling_layer }, + { "AdaptiveAveragePooling3D", create_adaptive_avg_pooling_layer }, + { "AdaptiveMaxPooling1D", create_adaptive_max_pooling_layer }, + { "AdaptiveMaxPooling2D", create_adaptive_max_pooling_layer }, + { "AdaptiveMaxPooling3D", create_adaptive_max_pooling_layer }, { "GlobalMaxPooling1D", create_global_max_pooling_3d_layer }, { "GlobalMaxPooling2D", create_global_max_pooling_3d_layer }, { "GlobalMaxPooling3D", create_global_max_pooling_3d_layer }, @@ -1444,6 +1906,12 @@ namespace internal { { "Rescaling", create_rescaling_layer }, { "Reshape", create_reshape_layer }, { "Resizing", create_resizing_layer }, + { "ConvLSTM1D", create_conv_lstm_2d_layer }, + { "ConvLSTM2D", create_conv_lstm_2d_layer }, + { "ConvLSTM3D", create_conv_lstm_3d_layer }, + { "Discretization", create_discretization_layer }, + { "IntegerLookup", create_integer_lookup_layer }, + { "EinsumDense", create_einsum_dense_layer }, { "Embedding", create_embedding_layer }, { "Softmax", create_softmax_layer }, { "Normalization", create_normalization_layer }, @@ -1451,6 +1919,13 @@ namespace internal { { "Attention", create_attention_layer }, { "AdditiveAttention", create_additive_attention_layer }, { "MultiHeadAttention", create_multi_head_attention_layer }, + { "GroupQueryAttention", create_group_query_attention_layer }, + { "GroupedQueryAttention", create_group_query_attention_layer }, + { "LSTM", create_lstm_layer }, + { "GRU", create_gru_layer }, + { "SimpleRNN", create_simple_rnn_layer }, + { "RNN", create_rnn_layer }, + { "Bidirectional", create_bidirectional_layer }, }; const wrapper_layer_creators wrapper_creators = { @@ -1475,7 +1950,11 @@ namespace internal { fplus::get_from_map(creators, type))( get_param, data, name); - if (type != "Activation" && json_obj_has_member(data["config"], "activation")) { + const bool layer_consumes_activation_internally = type == "Activation" + || type == "LSTM" || type == "GRU" || type == "SimpleRNN" + || type == "Bidirectional" + || type == "ConvLSTM1D" || type == "ConvLSTM2D" || type == "ConvLSTM3D"; + if (!layer_consumes_activation_internally && json_obj_has_member(data["config"], "activation")) { const std::string activation = get_activation_type(data["config"]["activation"]); result->set_activation( create_activation_layer_type_name(get_param, data, diff --git a/include/fdeep/layers/adaptive_pooling_3d_layer.hpp b/include/fdeep/layers/adaptive_pooling_3d_layer.hpp new file mode 100644 index 00000000..7069021a --- /dev/null +++ b/include/fdeep/layers/adaptive_pooling_3d_layer.hpp @@ -0,0 +1,129 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" + +#include +#include +#include +#include + +namespace fdeep { +namespace internal { + + enum class adaptive_pooling_kind { + avg, + max + }; + + class adaptive_pooling_3d_layer : public layer { + public: + explicit adaptive_pooling_3d_layer(const std::string& name, + std::size_t out_d4, std::size_t out_h, std::size_t out_w, + adaptive_pooling_kind kind) + : layer(name) + , out_d4_(out_d4) + , out_h_(out_h) + , out_w_(out_w) + , kind_(kind) + { + } + + protected: + const std::size_t out_d4_; + const std::size_t out_h_; + const std::size_t out_w_; + const adaptive_pooling_kind kind_; + + struct range { + std::size_t start; + std::size_t end; + }; + + // Adaptive pooling maps output index i to input range + // [floor(i * in / out), ceil((i+1) * in / out)). For length-1 input + // dimensions (which can occur for promoted 1D/2D tensors) the range + // collapses to [0, 1). + static range adapt_range(std::size_t i, std::size_t in_size, std::size_t out_size) + { + if (in_size == 1) + return { 0, 1 }; + assertion(out_size > 0, "AdaptivePooling output_size must be > 0."); + const auto start = static_cast(std::floor( + static_cast(i * in_size) / static_cast(out_size))); + const auto end = static_cast(std::ceil( + static_cast((i + 1) * in_size) / static_cast(out_size))); + return { start, end }; + } + + static tensor_shape output_shape_for(const tensor_shape& in_shape, + std::size_t out_d4, std::size_t out_h, std::size_t out_w) + { + const std::size_t depth = in_shape.depth_; + switch (in_shape.rank()) { + case 2: + return tensor_shape(out_w, depth); + case 3: + return tensor_shape(out_h, out_w, depth); + default: + return tensor_shape(out_d4, out_h, out_w, depth); + } + } + + float_type pool_window(const tensor& input, + range d, range h, range w, std::size_t z) const + { + const bool is_max = kind_ == adaptive_pooling_kind::max; + float_type acc = is_max + ? std::numeric_limits::lowest() + : float_type(0); + std::size_t count = 0; + for (std::size_t di = d.start; di < d.end; ++di) { + for (std::size_t yi = h.start; yi < h.end; ++yi) { + for (std::size_t xi = w.start; xi < w.end; ++xi) { + const float_type v = input.get_ignore_rank(tensor_pos(0, di, yi, xi, z)); + acc = is_max ? std::max(acc, v) : acc + v; + ++count; + } + } + } + if (!is_max && count > 0) + acc /= static_cast(count); + return acc; + } + + tensors apply_impl(const tensors& inputs) const override + { + const auto& input = single_tensor_from_tensors(inputs); + const auto& sh = input.shape(); + const std::size_t out_d4 = out_d4_ == 0 ? sh.size_dim_4_ : out_d4_; + const std::size_t out_h = out_h_ == 0 ? sh.height_ : out_h_; + const std::size_t out_w = out_w_; + + tensor out(output_shape_for(sh, out_d4, out_h, out_w), float_type(0)); + + for (std::size_t od = 0; od < out_d4; ++od) { + const range d = adapt_range(od, sh.size_dim_4_, out_d4); + for (std::size_t oy = 0; oy < out_h; ++oy) { + const range h = adapt_range(oy, sh.height_, out_h); + for (std::size_t ox = 0; ox < out_w; ++ox) { + const range w = adapt_range(ox, sh.width_, out_w); + for (std::size_t z = 0; z < sh.depth_; ++z) { + out.set_ignore_rank(tensor_pos(0, od, oy, ox, z), + pool_window(input, d, h, w, z)); + } + } + } + } + + return { out }; + } + }; + +} +} diff --git a/include/fdeep/layers/bidirectional_layer.hpp b/include/fdeep/layers/bidirectional_layer.hpp new file mode 100644 index 00000000..dffc705b --- /dev/null +++ b/include/fdeep/layers/bidirectional_layer.hpp @@ -0,0 +1,130 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" +#include "fdeep/recurrent_ops.hpp" + +#include + +namespace fdeep { +namespace internal { + + class bidirectional_layer : public layer { + public: + explicit bidirectional_layer(const std::string& name, + const std::string& merge_mode, + std::size_t n_units, + const std::string& activation, + const std::string& recurrent_activation, + const std::string& wrapped_layer_type, + bool use_bias, + bool reset_after, + bool return_sequences, + const float_vec& forward_weights, + const float_vec& forward_recurrent_weights, + const float_vec& bias_forward, + const float_vec& backward_weights, + const float_vec& backward_recurrent_weights, + const float_vec& bias_backward) + : layer(name) + , merge_mode_(merge_mode) + , n_units_(n_units) + , activation_(activation) + , recurrent_activation_(recurrent_activation) + , wrapped_layer_type_(wrapped_layer_type) + , use_bias_(use_bias) + , reset_after_(reset_after) + , return_sequences_(return_sequences) + , forward_weights_(forward_weights) + , forward_recurrent_weights_(forward_recurrent_weights) + , bias_forward_(bias_forward) + , backward_weights_(backward_weights) + , backward_recurrent_weights_(backward_recurrent_weights) + , bias_backward_(bias_backward) + { + } + + protected: + tensors apply_impl(const tensors& inputs) const override + { + const auto input_shapes = fplus::transform(fplus_c_mem_fn_t(tensor, shape, tensor_shape), inputs); + assertion(inputs.size() == 1, "Invalid number of input tensors."); + assertion(inputs.front().shape().rank() == 2, + "input tensor must have rank 2, but shape is '" + show_tensor_shapes(input_shapes) + "'"); + + const tensor& input = inputs.front(); + const tensor input_reversed = reverse_time_series_in_tensor(input); + + tensors result_forward; + tensors result_backward; + + if (wrapped_layer_type_ == "LSTM") { + result_forward = lstm_impl(input, n_units_, use_bias_, + return_sequences_, false, forward_weights_, + forward_recurrent_weights_, bias_forward_, + activation_, recurrent_activation_); + result_backward = lstm_impl(input_reversed, n_units_, use_bias_, + return_sequences_, false, backward_weights_, + backward_recurrent_weights_, bias_backward_, + activation_, recurrent_activation_); + } else if (wrapped_layer_type_ == "GRU") { + result_forward = gru_impl(input, n_units_, use_bias_, + reset_after_, return_sequences_, false, + forward_weights_, forward_recurrent_weights_, + bias_forward_, activation_, recurrent_activation_); + result_backward = gru_impl(input_reversed, n_units_, use_bias_, + reset_after_, return_sequences_, false, + backward_weights_, backward_recurrent_weights_, + bias_backward_, activation_, recurrent_activation_); + } else if (wrapped_layer_type_ == "SimpleRNN") { + result_forward = simple_rnn_impl(input, n_units_, use_bias_, + return_sequences_, false, forward_weights_, + forward_recurrent_weights_, bias_forward_, activation_); + result_backward = simple_rnn_impl(input_reversed, n_units_, use_bias_, + return_sequences_, false, backward_weights_, + backward_recurrent_weights_, bias_backward_, activation_); + } else { + raise_error("Bidirectional wrapper around layer '" + wrapped_layer_type_ + "' not supported."); + } + + const tensor result_backward_reversed = return_sequences_ + ? reverse_time_series_in_tensor(result_backward.front()) + : result_backward.front(); + + if (merge_mode_ == "concat") { + return { concatenate_tensors_depth({ result_forward.front(), result_backward_reversed }) }; + } else if (merge_mode_ == "sum") { + return { sum_tensors({ result_forward.front(), result_backward_reversed }) }; + } else if (merge_mode_ == "mul") { + return { multiply_tensors({ result_forward.front(), result_backward_reversed }) }; + } else if (merge_mode_ == "ave") { + return { average_tensors({ result_forward.front(), result_backward_reversed }) }; + } + + raise_error("Bidirectional merge mode '" + merge_mode_ + "' not supported."); + return {}; + } + + const std::string merge_mode_; + const std::size_t n_units_; + const std::string activation_; + const std::string recurrent_activation_; + const std::string wrapped_layer_type_; + const bool use_bias_; + const bool reset_after_; + const bool return_sequences_; + const float_vec forward_weights_; + const float_vec forward_recurrent_weights_; + const float_vec bias_forward_; + const float_vec backward_weights_; + const float_vec backward_recurrent_weights_; + const float_vec bias_backward_; + }; + +} +} diff --git a/include/fdeep/layers/conv_lstm_2d_layer.hpp b/include/fdeep/layers/conv_lstm_2d_layer.hpp new file mode 100644 index 00000000..bffb26da --- /dev/null +++ b/include/fdeep/layers/conv_lstm_2d_layer.hpp @@ -0,0 +1,170 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/conv_2d_layer.hpp" +#include "fdeep/layers/layer.hpp" +#include "fdeep/recurrent_ops.hpp" + +#include +#include + +namespace fdeep { +namespace internal { + + // Handles ConvLSTM1D and ConvLSTM2D. ConvLSTM1D dispatches here with + // rank_ = 1 and the input/recurrent kernels reshaped to height = 1. + class conv_lstm_2d_layer : public layer { + public: + explicit conv_lstm_2d_layer(const std::string& name, + std::size_t units, std::size_t rank, + const tensor_shape& filter_shape, + const shape2& strides, padding pad_type, const shape2& dilation_rate, + const float_vec& weights, const float_vec& recurrent_weights, + const float_vec& bias, + const std::string& activation, const std::string& recurrent_activation, + bool return_sequences, bool return_state) + : layer(name) + , units_(units) + , rank_(rank) + , return_sequences_(return_sequences) + , return_state_(return_state) + , activation_(activation) + , recurrent_activation_(recurrent_activation) + , input_conv_(name + "_input_conv", filter_shape, units * 4, + strides, pad_type, dilation_rate, weights, bias) + , recurrent_conv_(name + "_recurrent_conv", + tensor_shape(filter_shape.height_, filter_shape.width_, units), + units * 4, shape2(1, 1), padding::same, shape2(1, 1), + recurrent_weights, float_vec(units * 4, 0)) + { + } + + protected: + const std::size_t units_; + const std::size_t rank_; + const bool return_sequences_; + const bool return_state_; + const std::string activation_; + const std::string recurrent_activation_; + const conv_2d_layer input_conv_; + const conv_2d_layer recurrent_conv_; + + static tensor extract_timestep(const tensor& input, std::size_t t, + std::size_t rank) + { + const auto& sh = input.shape(); + const auto& src = *input.as_vector(); + if (rank == 1) { + // Source shape (T, W, C) at rank-3 storage (height=T, width=W, depth=C). + // Return rank-3 (1, W, C) for inner conv_2d. + const std::size_t W = sh.width_; + const std::size_t C = sh.depth_; + float_vec slice_data(W * C); + std::memcpy(slice_data.data(), src.data() + t * W * C, + W * C * sizeof(float_type)); + return tensor(tensor_shape(static_cast(1), W, C), + std::move(slice_data)); + } else { + // Source shape (T, H, W, C) at rank-4 storage. + const std::size_t H = sh.height_; + const std::size_t W = sh.width_; + const std::size_t C = sh.depth_; + float_vec slice_data(H * W * C); + std::memcpy(slice_data.data(), src.data() + t * H * W * C, + H * W * C * sizeof(float_type)); + return tensor(tensor_shape(H, W, C), std::move(slice_data)); + } + } + + tensors apply_impl(const tensors& inputs) const override + { + const auto& input = single_tensor_from_tensors(inputs); + const auto& sh = input.shape(); + const std::size_t T = rank_ == 1 ? sh.height_ : sh.size_dim_4_; + + const auto act_func = get_activation_func(activation_); + const auto rec_act_func = get_activation_func(recurrent_activation_); + + // First timestep to discover spatial output shape. + tensor X = input_conv_.apply({ extract_timestep(input, 0, rank_) }).front(); + const std::size_t H_out = X.shape().height_; + const std::size_t W_out = X.shape().width_; + const std::size_t spatial = H_out * W_out; + const std::size_t depth4 = units_ * 4; + + float_vec h_buf(spatial * units_, 0); + float_vec c_buf(spatial * units_, 0); + + const auto step = [&](const tensor& X_t) { + // Build h tensor view to feed into recurrent_conv (it takes a tensor). + const tensor h_tensor(tensor_shape(H_out, W_out, units_), + float_vec(h_buf)); + const tensor U_tensor = recurrent_conv_.apply({ h_tensor }).front(); + const auto& Xv = *X_t.as_vector(); + const auto& Uv = *U_tensor.as_vector(); + for (std::size_t s = 0; s < spatial; ++s) { + for (std::size_t k = 0; k < units_; ++k) { + const std::size_t base = s * depth4; + const float_type i_val = rec_act_func(Xv[base + k] + Uv[base + k]); + const float_type f_val = rec_act_func(Xv[base + units_ + k] + Uv[base + units_ + k]); + const float_type c_pre = act_func(Xv[base + 2 * units_ + k] + Uv[base + 2 * units_ + k]); + const float_type o_val = rec_act_func(Xv[base + 3 * units_ + k] + Uv[base + 3 * units_ + k]); + const std::size_t hpos = s * units_ + k; + c_buf[hpos] = f_val * c_buf[hpos] + i_val * c_pre; + h_buf[hpos] = o_val * act_func(c_buf[hpos]); + } + } + }; + + float_vec all_h; + if (return_sequences_) + all_h.reserve(T * spatial * units_); + + for (std::size_t t = 0; t < T; ++t) { + if (t > 0) + X = input_conv_.apply({ extract_timestep(input, t, rank_) }).front(); + step(X); + if (return_sequences_) { + const std::size_t off = all_h.size(); + all_h.resize(off + spatial * units_); + std::memcpy(all_h.data() + off, h_buf.data(), + spatial * units_ * sizeof(float_type)); + } + } + + tensors result; + if (return_sequences_) { + if (rank_ == 1) { + result.emplace_back(tensor_shape(T, W_out, units_), std::move(all_h)); + } else { + result.emplace_back(tensor_shape(T, H_out, W_out, units_), std::move(all_h)); + } + } else { + if (rank_ == 1) { + result.emplace_back(tensor_shape(W_out, units_), float_vec(h_buf)); + } else { + result.emplace_back(tensor_shape(H_out, W_out, units_), float_vec(h_buf)); + } + } + + if (return_state_) { + if (rank_ == 1) { + result.emplace_back(tensor_shape(W_out, units_), float_vec(h_buf)); + result.emplace_back(tensor_shape(W_out, units_), std::move(c_buf)); + } else { + result.emplace_back(tensor_shape(H_out, W_out, units_), float_vec(h_buf)); + result.emplace_back(tensor_shape(H_out, W_out, units_), std::move(c_buf)); + } + } + + return result; + } + }; + +} +} diff --git a/include/fdeep/layers/conv_lstm_3d_layer.hpp b/include/fdeep/layers/conv_lstm_3d_layer.hpp new file mode 100644 index 00000000..9a3b19e4 --- /dev/null +++ b/include/fdeep/layers/conv_lstm_3d_layer.hpp @@ -0,0 +1,146 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/conv_3d_layer.hpp" +#include "fdeep/layers/layer.hpp" +#include "fdeep/recurrent_ops.hpp" + +#include +#include + +namespace fdeep { +namespace internal { + + class conv_lstm_3d_layer : public layer { + public: + explicit conv_lstm_3d_layer(const std::string& name, + std::size_t units, + const tensor_shape& filter_shape, + const shape3& strides, padding pad_type, const shape3& dilation_rate, + const float_vec& weights, const float_vec& recurrent_weights, + const float_vec& bias, + const std::string& activation, const std::string& recurrent_activation, + bool return_sequences, bool return_state) + : layer(name) + , units_(units) + , return_sequences_(return_sequences) + , return_state_(return_state) + , activation_(activation) + , recurrent_activation_(recurrent_activation) + , input_conv_(name + "_input_conv", filter_shape, units * 4, + strides, pad_type, dilation_rate, weights, bias) + , recurrent_conv_(name + "_recurrent_conv", + tensor_shape(filter_shape.size_dim_4_, + filter_shape.height_, filter_shape.width_, units), + units * 4, shape3(1, 1, 1), padding::same, shape3(1, 1, 1), + recurrent_weights, float_vec(units * 4, 0)) + { + } + + protected: + const std::size_t units_; + const bool return_sequences_; + const bool return_state_; + const std::string activation_; + const std::string recurrent_activation_; + const conv_3d_layer input_conv_; + const conv_3d_layer recurrent_conv_; + + static tensor extract_timestep(const tensor& input, std::size_t t) + { + const auto& sh = input.shape(); + const std::size_t D4 = sh.size_dim_4_; + const std::size_t H = sh.height_; + const std::size_t W = sh.width_; + const std::size_t C = sh.depth_; + const std::size_t step_size = D4 * H * W * C; + const auto& src = *input.as_vector(); + float_vec slice_data(step_size); + std::memcpy(slice_data.data(), src.data() + t * step_size, + step_size * sizeof(float_type)); + return tensor(tensor_shape(D4, H, W, C), std::move(slice_data)); + } + + tensors apply_impl(const tensors& inputs) const override + { + const auto& input = single_tensor_from_tensors(inputs); + const auto& sh = input.shape(); + const std::size_t T = sh.size_dim_5_; + + const auto act_func = get_activation_func(activation_); + const auto rec_act_func = get_activation_func(recurrent_activation_); + + tensor X = input_conv_.apply({ extract_timestep(input, 0) }).front(); + const std::size_t D4_out = X.shape().size_dim_4_; + const std::size_t H_out = X.shape().height_; + const std::size_t W_out = X.shape().width_; + const std::size_t spatial = D4_out * H_out * W_out; + const std::size_t depth4 = units_ * 4; + + float_vec h_buf(spatial * units_, 0); + float_vec c_buf(spatial * units_, 0); + + const auto step = [&](const tensor& X_t) { + const tensor h_tensor(tensor_shape(D4_out, H_out, W_out, units_), + float_vec(h_buf)); + const tensor U_tensor = recurrent_conv_.apply({ h_tensor }).front(); + const auto& Xv = *X_t.as_vector(); + const auto& Uv = *U_tensor.as_vector(); + for (std::size_t s = 0; s < spatial; ++s) { + for (std::size_t k = 0; k < units_; ++k) { + const std::size_t base = s * depth4; + const float_type i_val = rec_act_func(Xv[base + k] + Uv[base + k]); + const float_type f_val = rec_act_func(Xv[base + units_ + k] + Uv[base + units_ + k]); + const float_type c_pre = act_func(Xv[base + 2 * units_ + k] + Uv[base + 2 * units_ + k]); + const float_type o_val = rec_act_func(Xv[base + 3 * units_ + k] + Uv[base + 3 * units_ + k]); + const std::size_t hpos = s * units_ + k; + c_buf[hpos] = f_val * c_buf[hpos] + i_val * c_pre; + h_buf[hpos] = o_val * act_func(c_buf[hpos]); + } + } + }; + + float_vec all_h; + if (return_sequences_) + all_h.reserve(T * spatial * units_); + + for (std::size_t t = 0; t < T; ++t) { + if (t > 0) + X = input_conv_.apply({ extract_timestep(input, t) }).front(); + step(X); + if (return_sequences_) { + const std::size_t off = all_h.size(); + all_h.resize(off + spatial * units_); + std::memcpy(all_h.data() + off, h_buf.data(), + spatial * units_ * sizeof(float_type)); + } + } + + tensors result; + if (return_sequences_) { + result.emplace_back( + tensor_shape(T, D4_out, H_out, W_out, units_), + std::move(all_h)); + } else { + result.emplace_back(tensor_shape(D4_out, H_out, W_out, units_), + float_vec(h_buf)); + } + + if (return_state_) { + result.emplace_back(tensor_shape(D4_out, H_out, W_out, units_), + float_vec(h_buf)); + result.emplace_back(tensor_shape(D4_out, H_out, W_out, units_), + std::move(c_buf)); + } + + return result; + } + }; + +} +} diff --git a/include/fdeep/layers/discretization_layer.hpp b/include/fdeep/layers/discretization_layer.hpp new file mode 100644 index 00000000..0bcc72be --- /dev/null +++ b/include/fdeep/layers/discretization_layer.hpp @@ -0,0 +1,45 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" + +#include +#include +#include + +namespace fdeep { +namespace internal { + + class discretization_layer : public layer { + public: + explicit discretization_layer(const std::string& name, + const std::vector& boundaries) + : layer(name) + , boundaries_(boundaries) + { + } + + protected: + const std::vector boundaries_; + + tensors apply_impl(const tensors& inputs) const override + { + const auto& input = single_tensor_from_tensors(inputs); + const auto& src = *input.as_vector(); + float_vec out(src.size(), 0); + for (std::size_t i = 0; i < src.size(); ++i) { + const auto it = std::upper_bound( + boundaries_.begin(), boundaries_.end(), src[i]); + out[i] = static_cast(it - boundaries_.begin()); + } + return { tensor(input.shape(), std::move(out)) }; + } + }; + +} +} diff --git a/include/fdeep/layers/einsum_dense_layer.hpp b/include/fdeep/layers/einsum_dense_layer.hpp new file mode 100644 index 00000000..9f21de1e --- /dev/null +++ b/include/fdeep/layers/einsum_dense_layer.hpp @@ -0,0 +1,262 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" + +#include +#include +#include + +namespace fdeep { +namespace internal { + + class einsum_dense_layer : public layer { + public: + explicit einsum_dense_layer(const std::string& name, + const std::string& equation, + const std::vector& full_output_shape, + const std::string& bias_axes, + const tensor& kernel, + const tensor& bias) + : layer(name) + , equation_(equation) + , full_output_shape_(full_output_shape) + , bias_axes_(bias_axes) + , kernel_(kernel) + , bias_(bias) + , lhs_(parse_lhs(equation)) + , rhs_kernel_(parse_rhs_kernel(equation)) + , rhs_(parse_rhs(equation)) + , summed_(compute_summed(lhs_, rhs_kernel_, rhs_)) + { + } + + protected: + const std::string equation_; + const std::vector full_output_shape_; + const std::string bias_axes_; + const tensor kernel_; + const tensor bias_; + + const std::string lhs_; + const std::string rhs_kernel_; + const std::string rhs_; + const std::string summed_; + + // Parses "lhs,rhs_kernel->rhs" into its three pieces. + static std::string parse_lhs(const std::string& eq) + { + return eq.substr(0, eq.find(',')); + } + + static std::string parse_rhs_kernel(const std::string& eq) + { + const auto comma = eq.find(','); + return eq.substr(comma + 1, eq.find("->") - comma - 1); + } + + static std::string parse_rhs(const std::string& eq) + { + return eq.substr(eq.find("->") + 2); + } + + // Chars that appear on both sides of the equation but not in the + // output are summed away. + static std::string compute_summed(const std::string& lhs, + const std::string& rhs_kernel, const std::string& rhs) + { + std::string summed; + for (char c : lhs) + if (rhs.find(c) == std::string::npos + && rhs_kernel.find(c) != std::string::npos) + summed.push_back(c); + return summed; + } + + // Row-major strides for a list of dim sizes. + static std::vector compute_strides(const std::vector& sizes) + { + std::vector strides(sizes.size()); + std::size_t s = 1; + for (std::size_t i = sizes.size(); i-- > 0;) { + strides[i] = s; + s *= sizes[i]; + } + return strides; + } + + static std::size_t product(const std::vector& v) + { + std::size_t p = 1; + for (auto x : v) + p *= x; + return p; + } + + // Decodes a flat index back into per-char coordinates and writes them + // into pos. Used to enumerate output positions and summed-axis combos. + static void decode_index(std::size_t idx, const std::string& chars, + const std::vector& strides, + std::map& pos) + { + for (std::size_t i = 0; i < chars.size(); ++i) { + pos[chars[i]] = idx / strides[i]; + idx %= strides[i]; + } + } + + static std::size_t encode_offset(const std::string& chars, + const std::vector& strides, + const std::map& pos) + { + std::size_t off = 0; + for (std::size_t i = 0; i < chars.size(); ++i) + off += pos.at(chars[i]) * strides[i]; + return off; + } + + // Build a {char -> dim size} map by combining info from kernel + // dimensions, the input tensor, and (as a fallback) full_output_shape_. + std::map derive_char_sizes(const tensor& input) const + { + std::map char_size; + + // Kernel: rhs_kernel_ chars are the trailing dims of the kernel tensor. + const auto kernel_dims = kernel_.shape().dimensions(); + assertion(rhs_kernel_.size() <= kernel_dims.size(), + "EinsumDense: kernel chars exceed kernel dimensions."); + const std::size_t k_offset = kernel_dims.size() - rhs_kernel_.size(); + for (std::size_t i = 0; i < rhs_kernel_.size(); ++i) + char_size[rhs_kernel_[i]] = kernel_dims[k_offset + i]; + + // Input: lhs chars beyond the input rank are dropped (None) batch + // dims with size 1; trailing chars map onto physical dims. + const auto input_dims = input.shape().dimensions(); + const std::size_t input_rank = input.shape().rank(); + assertion(lhs_.size() >= input_rank, + "EinsumDense: input rank exceeds lhs char count."); + const std::size_t leading = lhs_.size() - input_rank; + for (std::size_t i = 0; i < lhs_.size(); ++i) { + std::size_t sz = 1; + if (i >= leading) + sz = input_dims[input_dims.size() - input_rank + (i - leading)]; + const char c = lhs_[i]; + if (char_size.count(c)) + assertion(char_size[c] == sz, "EinsumDense: inconsistent char size in equation."); + else + char_size[c] = sz; + } + + // Output-only chars (rare): take the size from full_output_shape_. + for (std::size_t i = 0; i < rhs_.size(); ++i) { + const char c = rhs_[i]; + if (char_size.count(c) == 0) { + const int dim = full_output_shape_[i]; + assertion(dim > 0, "EinsumDense: cannot infer size of output-only char."); + char_size[c] = static_cast(dim); + } + } + return char_size; + } + + static std::vector sizes_for_chars(const std::string& chars, + const std::map& char_size) + { + std::vector sizes(chars.size()); + for (std::size_t i = 0; i < chars.size(); ++i) + sizes[i] = char_size.at(chars[i]); + return sizes; + } + + // Computes the contraction value for one fixed output position. The + // output position is already encoded in `pos` (mutable scratch space + // shared across calls). + float_type contract_one(std::map& pos, + const float_vec& input_vals, const float_vec& kernel_vals, + const std::vector& summed_sizes, + const std::vector& summed_strides, + const std::vector& lhs_strides, + const std::vector& kernel_strides) const + { + const std::size_t summed_volume = product(summed_sizes); + float_type acc = 0; + for (std::size_t s_idx = 0; s_idx < summed_volume; ++s_idx) { + decode_index(s_idx, summed_, summed_strides, pos); + const std::size_t lhs_off = encode_offset(lhs_, lhs_strides, pos); + const std::size_t k_off = encode_offset(rhs_kernel_, kernel_strides, pos); + acc += input_vals[lhs_off] * kernel_vals[k_off]; + } + return acc; + } + + // Adds bias broadcast over the configured bias_axes (a subset of rhs). + // Keras stores the bias variable with its dims in rhs char order, not + // in the literal bias_axes string order (e.g. bias_axes='ed' on + // rhs='abde' still produces a (d, e)-shaped bias). Reorder the chars + // before computing strides so we read the right element. + void add_bias(float_vec& out, + const std::map& char_size, + const std::vector& rhs_strides) const + { + if (bias_axes_.empty()) + return; + std::string bias_chars; + for (char c : rhs_) + if (bias_axes_.find(c) != std::string::npos) + bias_chars.push_back(c); + + const auto& bias_vals = *bias_.as_vector(); + const auto bias_sizes = sizes_for_chars(bias_chars, char_size); + const auto bias_strides = compute_strides(bias_sizes); + + std::map pos; + for (std::size_t out_idx = 0; out_idx < out.size(); ++out_idx) { + decode_index(out_idx, rhs_, rhs_strides, pos); + out[out_idx] += bias_vals[encode_offset(bias_chars, bias_strides, pos)]; + } + } + + tensors apply_impl(const tensors& inputs) const override + { + const auto& input = single_tensor_from_tensors(inputs); + const auto char_size = derive_char_sizes(input); + + const auto lhs_sizes = sizes_for_chars(lhs_, char_size); + const auto kernel_sizes = sizes_for_chars(rhs_kernel_, char_size); + const auto rhs_sizes = sizes_for_chars(rhs_, char_size); + const auto summed_sizes = sizes_for_chars(summed_, char_size); + + const auto lhs_strides = compute_strides(lhs_sizes); + const auto kernel_strides = compute_strides(kernel_sizes); + const auto rhs_strides = compute_strides(rhs_sizes); + const auto summed_strides = compute_strides(summed_sizes); + + const std::size_t out_volume = product(rhs_sizes); + const auto& input_vals = *input.as_vector(); + const auto& kernel_vals = *kernel_.as_vector(); + + float_vec out(out_volume, 0); + std::map pos; + for (std::size_t out_idx = 0; out_idx < out_volume; ++out_idx) { + decode_index(out_idx, rhs_, rhs_strides, pos); + out[out_idx] = contract_one(pos, input_vals, kernel_vals, + summed_sizes, summed_strides, lhs_strides, kernel_strides); + } + + add_bias(out, char_size, rhs_strides); + + // Drop the leading batch char (its size in fdeep is always 1). + assertion(rhs_sizes.size() >= 2, + "EinsumDense output equation must have at least a batch char and one feature char."); + std::vector out_dims_no_batch(rhs_sizes.begin() + 1, rhs_sizes.end()); + return { tensor(create_tensor_shape_from_dims(out_dims_no_batch), std::move(out)) }; + } + }; + +} +} diff --git a/include/fdeep/layers/group_normalization_layer.hpp b/include/fdeep/layers/group_normalization_layer.hpp new file mode 100644 index 00000000..2d56451d --- /dev/null +++ b/include/fdeep/layers/group_normalization_layer.hpp @@ -0,0 +1,109 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" + +#include +#include +#include + +namespace fdeep { +namespace internal { + + class group_normalization_layer : public layer { + public: + explicit group_normalization_layer(const std::string& name, + std::size_t groups, + int axis, + float_type epsilon, + const float_vec& beta, + const float_vec& gamma) + : layer(name) + , groups_(groups) + , axis_(axis) + , epsilon_(epsilon) + , beta_(beta) + , gamma_(gamma) + { + } + + protected: + const std::size_t groups_; + const int axis_; + const float_type epsilon_; + const float_vec beta_; + const float_vec gamma_; + + tensors apply_impl(const tensors& inputs) const override + { + const auto& input = single_tensor_from_tensors(inputs); + const auto in_shape = input.shape(); + + const std::size_t absolute_axis = rank_aligned_axis_to_absolute_axis(in_shape.rank(), axis_); + assertion(absolute_axis == 5, + "GroupNormalization is currently only supported on the last (channel) axis."); + + const std::size_t channels = in_shape.depth_; + assertion(groups_ > 0, "GroupNormalization groups must be > 0."); + assertion(channels % groups_ == 0, + "Number of channels must be divisible by number of groups."); + const std::size_t channels_per_group = channels / groups_; + + const std::size_t spatial_volume = in_shape.size_dim_5_ + * in_shape.size_dim_4_ * in_shape.height_ * in_shape.width_; + const std::size_t group_volume = spatial_volume * channels_per_group; + assertion(group_volume > 0, "GroupNormalization input has zero volume."); + + const auto& src = *input.as_vector(); + float_vec means(groups_, 0); + float_vec vars(groups_, 0); + + // mean per group + for (std::size_t i = 0; i < src.size(); ++i) { + const std::size_t c = i % channels; + const std::size_t g = c / channels_per_group; + means[g] += src[i]; + } + for (std::size_t g = 0; g < groups_; ++g) + means[g] /= static_cast(group_volume); + + // variance per group + for (std::size_t i = 0; i < src.size(); ++i) { + const std::size_t c = i % channels; + const std::size_t g = c / channels_per_group; + const float_type d = src[i] - means[g]; + vars[g] += d * d; + } + for (std::size_t g = 0; g < groups_; ++g) + vars[g] /= static_cast(group_volume); + + float_vec inv_std(groups_, 0); + for (std::size_t g = 0; g < groups_; ++g) + inv_std[g] = static_cast(1) / std::sqrt(vars[g] + epsilon_); + + const bool has_gamma = !gamma_.empty(); + const bool has_beta = !beta_.empty(); + + float_vec out(src.size()); + for (std::size_t i = 0; i < src.size(); ++i) { + const std::size_t c = i % channels; + const std::size_t g = c / channels_per_group; + float_type v = (src[i] - means[g]) * inv_std[g]; + if (has_gamma) + v *= gamma_[c]; + if (has_beta) + v += beta_[c]; + out[i] = v; + } + + return { tensor(in_shape, std::move(out)) }; + } + }; + +} +} diff --git a/include/fdeep/layers/group_query_attention_layer.hpp b/include/fdeep/layers/group_query_attention_layer.hpp new file mode 100644 index 00000000..cd57b341 --- /dev/null +++ b/include/fdeep/layers/group_query_attention_layer.hpp @@ -0,0 +1,224 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" + +#include +#include +#include +#include + +namespace fdeep { +namespace internal { + + class group_query_attention_layer : public layer { + public: + explicit group_query_attention_layer(const std::string& name, + std::size_t head_dim, std::size_t num_query_heads, std::size_t num_kv_heads, + bool use_bias, bool use_gate, const std::vector& weights) + : layer(name) + , head_dim_(head_dim) + , num_query_heads_(num_query_heads) + , num_kv_heads_(num_kv_heads) + , use_bias_(use_bias) + , use_gate_(use_gate) + , weights_(weights) + { + assertion(num_kv_heads_ > 0, + "num_key_value_heads must be > 0."); + assertion(num_query_heads_ % num_kv_heads_ == 0, + "num_query_heads must be divisible by num_key_value_heads."); + } + + protected: + const std::size_t head_dim_; + const std::size_t num_query_heads_; + const std::size_t num_kv_heads_; + const bool use_bias_; + const bool use_gate_; + const std::vector weights_; + + // Weight order in Keras: Q, K, [Gate], V, Out, with bias right after each kernel. + std::size_t weight_idx(std::size_t projection_idx) const + { + return projection_idx * (use_bias_ ? 2 : 1); + } + const tensor& q_kernel() const { return weights_[weight_idx(0)]; } + const tensor& q_bias() const { return weights_[weight_idx(0) + 1]; } + const tensor& k_kernel() const { return weights_[weight_idx(1)]; } + const tensor& k_bias() const { return weights_[weight_idx(1) + 1]; } + const tensor& gate_kernel() const { return weights_[weight_idx(2)]; } + const tensor& gate_bias() const { return weights_[weight_idx(2) + 1]; } + const tensor& v_kernel() const { return weights_[weight_idx(use_gate_ ? 3 : 2)]; } + const tensor& v_bias() const { return weights_[weight_idx(use_gate_ ? 3 : 2) + 1]; } + const tensor& o_kernel() const { return weights_[weight_idx(use_gate_ ? 4 : 3)]; } + const tensor& o_bias() const { return weights_[weight_idx(use_gate_ ? 4 : 3) + 1]; } + + // Project an input of shape (T, dim) to (T, num_heads, head_dim) using + // a kernel of shape (dim, num_heads, head_dim) and an optional bias of + // shape (num_heads, head_dim). + tensor project(const tensor& input, + const tensor& kernel, const tensor* bias, + std::size_t num_heads) const + { + const std::size_t T = input.shape().width_; + const std::size_t in_dim = input.shape().depth_; + const auto& xv = *input.as_vector(); + const auto& kv = *kernel.as_vector(); + const float_vec dummy_bias; + const auto& bv = bias != nullptr ? *bias->as_vector() : dummy_bias; + const bool has_bias = bias != nullptr; + + float_vec out(T * num_heads * head_dim_, 0); + for (std::size_t t = 0; t < T; ++t) { + for (std::size_t h = 0; h < num_heads; ++h) { + for (std::size_t c = 0; c < head_dim_; ++c) { + float_type acc = 0; + for (std::size_t d = 0; d < in_dim; ++d) { + acc += xv[t * in_dim + d] + * kv[d * num_heads * head_dim_ + h * head_dim_ + c]; + } + if (has_bias) + acc += bv[h * head_dim_ + c]; + out[t * num_heads * head_dim_ + h * head_dim_ + c] = acc; + } + } + } + return tensor(tensor_shape(T, num_heads, head_dim_), std::move(out)); + } + + // Build the (T_q, T_k) softmax distribution for one query/kv-head pair. + float_vec attention_distribution( + const float_vec& Qv, const float_vec& Kv, + std::size_t tq, std::size_t h_q, + std::size_t T_k, std::size_t h_kv) const + { + const float_type inv_sqrt = static_cast(1) + / std::sqrt(static_cast(head_dim_)); + float_vec scores(T_k, 0); + float_type max_score = -std::numeric_limits::infinity(); + for (std::size_t tk = 0; tk < T_k; ++tk) { + float_type s = 0; + for (std::size_t c = 0; c < head_dim_; ++c) { + s += Qv[tq * num_query_heads_ * head_dim_ + h_q * head_dim_ + c] + * Kv[tk * num_kv_heads_ * head_dim_ + h_kv * head_dim_ + c]; + } + s *= inv_sqrt; + scores[tk] = s; + if (s > max_score) + max_score = s; + } + float_type sum_exp = 0; + for (std::size_t tk = 0; tk < T_k; ++tk) { + scores[tk] = std::exp(scores[tk] - max_score); + sum_exp += scores[tk]; + } + for (std::size_t tk = 0; tk < T_k; ++tk) + scores[tk] /= sum_exp; + return scores; + } + + // Compute the attention output of shape (T_q, num_query_heads, head_dim). + // Each query head h_q reads from kv head h_q / (num_query_heads / num_kv_heads). + float_vec compute_attention(const tensor& Q, const tensor& K, const tensor& V) const + { + const std::size_t T_q = Q.shape().height_; + const std::size_t T_k = K.shape().height_; + const std::size_t group_size = num_query_heads_ / num_kv_heads_; + const auto& Qv = *Q.as_vector(); + const auto& Kv = *K.as_vector(); + const auto& Vv = *V.as_vector(); + + float_vec attn(T_q * num_query_heads_ * head_dim_, 0); + for (std::size_t h_q = 0; h_q < num_query_heads_; ++h_q) { + const std::size_t h_kv = h_q / group_size; + for (std::size_t tq = 0; tq < T_q; ++tq) { + const auto distribution = attention_distribution( + Qv, Kv, tq, h_q, T_k, h_kv); + for (std::size_t c = 0; c < head_dim_; ++c) { + float_type acc = 0; + for (std::size_t tk = 0; tk < T_k; ++tk) { + acc += distribution[tk] + * Vv[tk * num_kv_heads_ * head_dim_ + h_kv * head_dim_ + c]; + } + attn[tq * num_query_heads_ * head_dim_ + h_q * head_dim_ + c] = acc; + } + } + } + return attn; + } + + static void apply_sigmoid_gate(float_vec& attn, const float_vec& gate) + { + for (std::size_t i = 0; i < attn.size(); ++i) { + const float_type sig = static_cast(1) + / (static_cast(1) + std::exp(-gate[i])); + attn[i] *= sig; + } + } + + // Project (T_q, num_query_heads, head_dim) attn back to (T_q, out_dim). + tensor output_projection(const float_vec& attn, std::size_t T_q) const + { + const tensor& out_kernel = o_kernel(); + const std::size_t out_dim = out_kernel.shape().depth_; + const auto& Ov = *out_kernel.as_vector(); + const float_vec dummy_bias; + const auto& obv = use_bias_ ? *o_bias().as_vector() : dummy_bias; + + float_vec result(T_q * out_dim, 0); + for (std::size_t t = 0; t < T_q; ++t) { + for (std::size_t d = 0; d < out_dim; ++d) { + float_type acc = 0; + for (std::size_t h = 0; h < num_query_heads_; ++h) { + for (std::size_t c = 0; c < head_dim_; ++c) { + acc += attn[t * num_query_heads_ * head_dim_ + h * head_dim_ + c] + * Ov[h * head_dim_ * out_dim + c * out_dim + d]; + } + } + if (use_bias_) + acc += obv[d]; + result[t * out_dim + d] = acc; + } + } + return tensor(tensor_shape(T_q, out_dim), std::move(result)); + } + + tensors apply_impl(const tensors& input) const override + { + assertion(input.size() == 2 || input.size() == 3, + "GroupQueryAttention requires 2 or 3 inputs (query, value[, key])."); + const tensor& query_raw = input[0]; + const tensor& value_raw = input[1]; + const tensor& key_raw = input.size() > 2 ? input[2] : value_raw; + assertion(query_raw.shape().rank() == 2 + && value_raw.shape().rank() == 2 && key_raw.shape().rank() == 2, + "GroupQueryAttention expects rank-2 inputs (T, dim)."); + + const tensor* qb = use_bias_ ? &q_bias() : nullptr; + const tensor* kb = use_bias_ ? &k_bias() : nullptr; + const tensor* vb = use_bias_ ? &v_bias() : nullptr; + + const tensor Q = project(query_raw, q_kernel(), qb, num_query_heads_); + const tensor K = project(key_raw, k_kernel(), kb, num_kv_heads_); + const tensor V = project(value_raw, v_kernel(), vb, num_kv_heads_); + + float_vec attn = compute_attention(Q, K, V); + + if (use_gate_) { + const tensor* gb = use_bias_ ? &gate_bias() : nullptr; + const tensor gate = project(query_raw, gate_kernel(), gb, num_query_heads_); + apply_sigmoid_gate(attn, *gate.as_vector()); + } + + return { output_projection(attn, Q.shape().height_) }; + } + }; + +} +} diff --git a/include/fdeep/layers/gru_layer.hpp b/include/fdeep/layers/gru_layer.hpp new file mode 100644 index 00000000..e6793090 --- /dev/null +++ b/include/fdeep/layers/gru_layer.hpp @@ -0,0 +1,72 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" +#include "fdeep/recurrent_ops.hpp" + +#include + +namespace fdeep { +namespace internal { + + class gru_layer : public layer { + public: + explicit gru_layer(const std::string& name, + std::size_t n_units, + const std::string& activation, + const std::string& recurrent_activation, + bool use_bias, + bool reset_after, + bool return_sequences, + bool return_state, + const float_vec& weights, + const float_vec& recurrent_weights, + const float_vec& bias) + : layer(name) + , n_units_(n_units) + , activation_(activation) + , recurrent_activation_(recurrent_activation) + , use_bias_(use_bias) + , reset_after_(reset_after) + , return_sequences_(return_sequences) + , return_state_(return_state) + , weights_(weights) + , recurrent_weights_(recurrent_weights) + , bias_(bias) + { + } + + protected: + tensors apply_impl(const tensors& inputs) const override + { + const auto input_shapes = fplus::transform(fplus_c_mem_fn_t(tensor, shape, tensor_shape), inputs); + assertion(inputs.front().shape().size_dim_5_ == 1 + && inputs.front().shape().size_dim_4_ == 1 + && inputs.front().shape().height_ == 1, + "size_dim_5, size_dim_4 and height dimension must be 1, but shape is '" + show_tensor_shapes(input_shapes) + "'"); + assertion(inputs.size() == 1, "Invalid number of input tensors."); + + return gru_impl(inputs.front(), n_units_, use_bias_, reset_after_, + return_sequences_, return_state_, weights_, + recurrent_weights_, bias_, activation_, recurrent_activation_); + } + + const std::size_t n_units_; + const std::string activation_; + const std::string recurrent_activation_; + const bool use_bias_; + const bool reset_after_; + const bool return_sequences_; + const bool return_state_; + const float_vec weights_; + const float_vec recurrent_weights_; + const float_vec bias_; + }; + +} +} diff --git a/include/fdeep/layers/integer_lookup_layer.hpp b/include/fdeep/layers/integer_lookup_layer.hpp new file mode 100644 index 00000000..aa9a81ef --- /dev/null +++ b/include/fdeep/layers/integer_lookup_layer.hpp @@ -0,0 +1,81 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" + +#include +#include +#include +#include + +namespace fdeep { +namespace internal { + + class integer_lookup_layer : public layer { + public: + explicit integer_lookup_layer(const std::string& name, + const std::vector& vocabulary, + std::size_t num_oov_indices, + bool has_mask_token, + std::int64_t mask_token) + : layer(name) + , has_mask_token_(has_mask_token) + , mask_token_(mask_token) + , num_oov_indices_(num_oov_indices) + , vocab_offset_((has_mask_token ? 1 : 0) + num_oov_indices) + , vocab_index_(build_vocab_index(vocabulary, num_oov_indices)) + { + } + + protected: + const bool has_mask_token_; + const std::int64_t mask_token_; + const std::size_t num_oov_indices_; + const std::size_t vocab_offset_; + const std::unordered_map vocab_index_; + + // num_oov_indices > 1 needs Keras's FarmHash, which fdeep does not + // currently replicate. num_oov_indices == 0 ("strict" mode) raises on + // OOV, also unsupported. Require exactly one OOV bucket. + static std::unordered_map build_vocab_index( + const std::vector& vocabulary, std::size_t num_oov_indices) + { + assertion(num_oov_indices == 1, + "IntegerLookup requires num_oov_indices == 1."); + std::unordered_map idx; + idx.reserve(vocabulary.size()); + for (std::size_t i = 0; i < vocabulary.size(); ++i) + idx[vocabulary[i]] = i; + return idx; + } + + tensors apply_impl(const tensors& inputs) const override + { + const auto& input = single_tensor_from_tensors(inputs); + const auto& src = *input.as_vector(); + float_vec out(src.size(), 0); + for (std::size_t i = 0; i < src.size(); ++i) { + const std::int64_t v = static_cast(src[i]); + if (has_mask_token_ && v == mask_token_) { + out[i] = 0; + continue; + } + const auto it = vocab_index_.find(v); + if (it != vocab_index_.end()) { + out[i] = static_cast(vocab_offset_ + it->second); + } else { + // OOV maps to 0, shifted by 1 if a mask token occupies index 0. + out[i] = static_cast(has_mask_token_ ? 1 : 0); + } + } + return { tensor(input.shape(), std::move(out)) }; + } + }; + +} +} diff --git a/include/fdeep/layers/lstm_layer.hpp b/include/fdeep/layers/lstm_layer.hpp new file mode 100644 index 00000000..6dd50e46 --- /dev/null +++ b/include/fdeep/layers/lstm_layer.hpp @@ -0,0 +1,69 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" +#include "fdeep/recurrent_ops.hpp" + +#include + +namespace fdeep { +namespace internal { + + class lstm_layer : public layer { + public: + explicit lstm_layer(const std::string& name, + std::size_t n_units, + const std::string& activation, + const std::string& recurrent_activation, + bool use_bias, + bool return_sequences, + bool return_state, + const float_vec& weights, + const float_vec& recurrent_weights, + const float_vec& bias) + : layer(name) + , n_units_(n_units) + , activation_(activation) + , recurrent_activation_(recurrent_activation) + , use_bias_(use_bias) + , return_sequences_(return_sequences) + , return_state_(return_state) + , weights_(weights) + , recurrent_weights_(recurrent_weights) + , bias_(bias) + { + } + + protected: + tensors apply_impl(const tensors& inputs) const override + { + const auto input_shapes = fplus::transform(fplus_c_mem_fn_t(tensor, shape, tensor_shape), inputs); + assertion(inputs.front().shape().size_dim_5_ == 1 + && inputs.front().shape().size_dim_4_ == 1 + && inputs.front().shape().height_ == 1, + "size_dim_5, size_dim_4 and height dimension must be 1, but shape is '" + show_tensor_shapes(input_shapes) + "'"); + assertion(inputs.size() == 1, "Invalid number of input tensors."); + + return lstm_impl(inputs.front(), n_units_, use_bias_, + return_sequences_, return_state_, weights_, + recurrent_weights_, bias_, activation_, recurrent_activation_); + } + + const std::size_t n_units_; + const std::string activation_; + const std::string recurrent_activation_; + const bool use_bias_; + const bool return_sequences_; + const bool return_state_; + const float_vec weights_; + const float_vec recurrent_weights_; + const float_vec bias_; + }; + +} +} diff --git a/include/fdeep/layers/rms_normalization_layer.hpp b/include/fdeep/layers/rms_normalization_layer.hpp new file mode 100644 index 00000000..ab2189fd --- /dev/null +++ b/include/fdeep/layers/rms_normalization_layer.hpp @@ -0,0 +1,73 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" + +#include +#include + +namespace fdeep { +namespace internal { + + class rms_normalization_layer : public layer { + public: + explicit rms_normalization_layer(const std::string& name, + std::vector axes, + const float_vec& scale, + float_type epsilon) + : layer(name) + , axes_(axes) + , scale_(fplus::make_shared_ref(scale)) + , epsilon_(epsilon) + { + } + + protected: + const std::vector axes_; + const shared_float_vec scale_; + const float_type epsilon_; + + tensors apply_impl(const tensors& inputs) const override + { + const auto& input = single_tensor_from_tensors(inputs); + + // mean(x^2) over the specified axes + const tensor squared = mult_tensors(input, input); + const tensor summed = reduce(add_tensors, squared, axes_); + const auto factor = static_cast(squared.shape().volume()) / static_cast(summed.shape().volume()); + const tensor mean_sq = transform_tensor(fplus::divide_by(factor), summed); + + // 1 / sqrt(mean_sq + eps) + const float_type eps = epsilon_; + const tensor inv_rms = transform_tensor( + [eps](float_type v) -> float_type { + return static_cast(1) / std::sqrt(v + eps); + }, + mean_sq); + const tensor normalized = mult_tensors(input, broadcast(inv_rms, input.shape())); + + // multiply by learnable scale, broadcast to match input shape + std::vector dims(5, 1); + tensor_shape input_shape = input.shape(); + input_shape.maximize_rank(); + const auto input_shape_dimensions = input_shape.dimensions(); + for (const auto axis : axes_) { + const std::size_t pos = rank_aligned_axis_to_absolute_axis(input.shape().rank(), axis) - 1; + dims[pos] = input_shape_dimensions[pos]; + } + const tensor_shape params_shape = create_tensor_shape_from_dims(dims); + const tensor scale_t = scale_->empty() + ? tensor(input.shape(), 1) + : broadcast(tensor(params_shape, scale_), input.shape()); + + return { mult_tensors(normalized, scale_t) }; + } + }; + +} +} diff --git a/include/fdeep/layers/simple_rnn_layer.hpp b/include/fdeep/layers/simple_rnn_layer.hpp new file mode 100644 index 00000000..3438f4f9 --- /dev/null +++ b/include/fdeep/layers/simple_rnn_layer.hpp @@ -0,0 +1,66 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" +#include "fdeep/recurrent_ops.hpp" + +#include + +namespace fdeep { +namespace internal { + + class simple_rnn_layer : public layer { + public: + explicit simple_rnn_layer(const std::string& name, + std::size_t n_units, + const std::string& activation, + bool use_bias, + bool return_sequences, + bool return_state, + const float_vec& weights, + const float_vec& recurrent_weights, + const float_vec& bias) + : layer(name) + , n_units_(n_units) + , activation_(activation) + , use_bias_(use_bias) + , return_sequences_(return_sequences) + , return_state_(return_state) + , weights_(weights) + , recurrent_weights_(recurrent_weights) + , bias_(bias) + { + } + + protected: + tensors apply_impl(const tensors& inputs) const override + { + const auto input_shapes = fplus::transform(fplus_c_mem_fn_t(tensor, shape, tensor_shape), inputs); + assertion(inputs.front().shape().size_dim_5_ == 1 + && inputs.front().shape().size_dim_4_ == 1 + && inputs.front().shape().height_ == 1, + "size_dim_5, size_dim_4 and height dimension must be 1, but shape is '" + show_tensor_shapes(input_shapes) + "'"); + assertion(inputs.size() == 1, "Invalid number of input tensors."); + + return simple_rnn_impl(inputs.front(), n_units_, use_bias_, + return_sequences_, return_state_, weights_, + recurrent_weights_, bias_, activation_); + } + + const std::size_t n_units_; + const std::string activation_; + const bool use_bias_; + const bool return_sequences_; + const bool return_state_; + const float_vec weights_; + const float_vec recurrent_weights_; + const float_vec bias_; + }; + +} +} diff --git a/include/fdeep/layers/stacked_rnn_layer.hpp b/include/fdeep/layers/stacked_rnn_layer.hpp new file mode 100644 index 00000000..60307d01 --- /dev/null +++ b/include/fdeep/layers/stacked_rnn_layer.hpp @@ -0,0 +1,43 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#pragma once + +#include "fdeep/layers/layer.hpp" + +#include +#include + +namespace fdeep { +namespace internal { + + // Wraps a chain of recurrent layers built from a StackedRNNCells. Each + // inner layer is a standalone LSTM / GRU / SimpleRNN where all but the + // last run with return_sequences=true so the next cell sees the full + // sequence at every timestep. + class stacked_rnn_layer : public layer { + public: + explicit stacked_rnn_layer(const std::string& name, + const std::vector& inner_layers) + : layer(name) + , inner_layers_(inner_layers) + { + } + + protected: + const std::vector inner_layers_; + + tensors apply_impl(const tensors& inputs) const override + { + tensors current = inputs; + for (const auto& inner : inner_layers_) + current = inner->apply(current); + return current; + } + }; + +} +} diff --git a/include/fdeep/recurrent_ops.hpp b/include/fdeep/recurrent_ops.hpp index 1f8945b4..a62ce33e 100644 --- a/include/fdeep/recurrent_ops.hpp +++ b/include/fdeep/recurrent_ops.hpp @@ -6,6 +6,10 @@ #pragma once +#include "fdeep/common.hpp" +#include "fdeep/tensor.hpp" + +#include #include #include @@ -17,6 +21,11 @@ namespace internal { template using RowVector = Eigen::Matrix; + inline float_type linear_activation(float_type x) + { + return x; + } + inline float_type tanh_activation(float_type x) { return std::tanh(x); @@ -32,6 +41,11 @@ namespace internal { return x / (1 + std::exp(-x)); } + inline float_type relu_activation(float_type x) + { + return std::max(x, 0); + } + inline float_type hard_sigmoid_activation(float_type x) { // https://github.com/keras-team/keras/blob/f7bc67e6c105c116a2ba7f5412137acf78174b1a/keras/ops/nn.py#L316C6-L316C74 @@ -71,5 +85,287 @@ namespace internal { return x >= 0 ? x : std::exp(x) - 1; } + inline std::function get_activation_func(const std::string& activation_func_name) + { + if (activation_func_name == "linear") + return linear_activation; + if (activation_func_name == "tanh") + return tanh_activation; + if (activation_func_name == "sigmoid") + return sigmoid_activation; + if (activation_func_name == "swish" || activation_func_name == "silu") + return swish_activation; + if (activation_func_name == "hard_sigmoid") + return hard_sigmoid_activation; + if (activation_func_name == "relu") + return relu_activation; + if (activation_func_name == "selu") + return selu_activation; + if (activation_func_name == "elu") + return elu_activation; + if (activation_func_name == "exponential") + return exponential_activation; + if (activation_func_name == "gelu") + return gelu_activation; + if (activation_func_name == "softsign") + return softsign_activation; + + raise_error("recurrent activation function '" + activation_func_name + "' not yet implemented"); + return {}; + } + + inline tensors lstm_impl(const tensor& input, + const std::size_t n_units, + const bool use_bias, + const bool return_sequences, + const bool return_state, + const float_vec& weights, + const float_vec& recurrent_weights, + const float_vec& bias, + const std::string& activation, + const std::string& recurrent_activation) + { + assertion(n_units > 0, "LSTM units must be > 0."); + const MappedRowMajorMatrixXf W = eigen_row_major_mat_from_shared_values( + weights.size() / (n_units * 4), n_units * 4, + weights.data()); + const MappedRowMajorMatrixXf U = eigen_row_major_mat_from_shared_values( + n_units, n_units * 4, recurrent_weights.data()); + + RowMajorMatrixXf h = RowMajorMatrixXf::Zero(1, static_cast(n_units)); + RowMajorMatrixXf c = RowMajorMatrixXf::Zero(1, static_cast(n_units)); + + const std::size_t n_timesteps = input.shape().width_; + const std::size_t n_features = input.shape().depth_; + + const MappedRowMajorMatrixXf in = eigen_row_major_mat_from_shared_values( + n_timesteps, n_features, input.as_vector()->data()); + + RowMajorMatrixXf X = in * W; + + if (use_bias) { + typedef Eigen::Matrix Vector_Xf; + const Vector_Xf b = eigen_row_major_mat_from_shared_values( + 1, n_units * 4, bias.data()); + X.rowwise() += b; + } + + const auto act_func = get_activation_func(activation); + const auto act_func_recurrent = get_activation_func(recurrent_activation); + + const EigenIndex n = static_cast(n_units); + + tensors result; + if (return_sequences) + result = { tensor(tensor_shape(n_timesteps, n_units), float_type(0)) }; + else + result = { tensor(tensor_shape(n_units), float_type(0)) }; + + for (EigenIndex k = 0; k < static_cast(n_timesteps); ++k) { + const RowMajorMatrixXf ifco = h * U; + + const RowMajorMatrixXf i = (X.block(k, 0, 1, n) + ifco.block(0, 0, 1, n)).unaryExpr(act_func_recurrent); + const RowMajorMatrixXf f = (X.block(k, n, 1, n) + ifco.block(0, n, 1, n)).unaryExpr(act_func_recurrent); + const RowMajorMatrixXf c_pre = (X.block(k, n * 2, 1, n) + ifco.block(0, n * 2, 1, n)).unaryExpr(act_func); + const RowMajorMatrixXf o = (X.block(k, n * 3, 1, n) + ifco.block(0, n * 3, 1, n)).unaryExpr(act_func_recurrent); + + c = f.array() * c.array() + i.array() * c_pre.array(); + h = o.array() * c.unaryExpr(act_func).array(); + + if (return_sequences) + for (EigenIndex idx = 0; idx < n; ++idx) + result.front().set_ignore_rank(tensor_pos(std::size_t(k), std::size_t(idx)), h(idx)); + else if (k == static_cast(n_timesteps) - 1) + for (EigenIndex idx = 0; idx < n; ++idx) + result.front().set_ignore_rank(tensor_pos(std::size_t(idx)), h(idx)); + } + + if (return_state) { + auto state_h = tensor(tensor_shape(n_units), float_type(0)); + auto state_c = tensor(tensor_shape(n_units), float_type(0)); + for (EigenIndex idx = 0; idx < n; ++idx) + state_h.set_ignore_rank(tensor_pos(std::size_t(idx)), h(idx)); + for (EigenIndex idx = 0; idx < n; ++idx) + state_c.set_ignore_rank(tensor_pos(std::size_t(idx)), c(idx)); + result.push_back(state_h); + result.push_back(state_c); + } + + return result; + } + + inline tensors gru_impl(const tensor& input, + const std::size_t n_units, + const bool use_bias, + const bool reset_after, + const bool return_sequences, + const bool return_state, + const float_vec& weights, + const float_vec& recurrent_weights, + const float_vec& bias, + const std::string& activation, + const std::string& recurrent_activation) + { + assertion(n_units > 0, "GRU units must be > 0."); + const std::size_t n_timesteps = input.shape().width_; + const std::size_t n_features = input.shape().depth_; + + const EigenIndex n = static_cast(n_units); + const MappedRowMajorMatrixXf W = eigen_row_major_mat_from_shared_values( + n_features, n_units * 3, weights.data()); + const MappedRowMajorMatrixXf U = eigen_row_major_mat_from_shared_values( + n_units, n_units * 3, recurrent_weights.data()); + + // Keras GRU bias layout: + // reset_after=False, use_bias=True -> shape (3*units,) + // reset_after=True, use_bias=True -> shape (2, 3*units) + if (use_bias) { + const std::size_t expected = reset_after ? 2 * n_units * 3 : n_units * 3; + assertion(bias.size() == expected, + "GRU bias size does not match reset_after setting."); + } + RowVector b_x(static_cast(n_units * 3)); + if (use_bias) + std::copy_n(bias.cbegin(), n_units * 3, b_x.data()); + else + b_x.setZero(); + + RowVector b_h(static_cast(n_units * 3)); + if (use_bias && reset_after) + std::copy_n(bias.cbegin() + static_cast(n_units * 3), + n_units * 3, b_h.data()); + else + b_h.setZero(); + + RowMajorMatrixXf h = RowMajorMatrixXf::Zero(1, n); + + const MappedRowMajorMatrixXf x = eigen_row_major_mat_from_shared_values( + n_timesteps, n_features, input.as_vector()->data()); + + RowMajorMatrixXf Wx = x * W; + Wx.rowwise() += b_x; + + const auto act_func = get_activation_func(activation); + const auto act_func_recurrent = get_activation_func(recurrent_activation); + + tensors result; + if (return_sequences) + result = { tensor(tensor_shape(n_timesteps, n_units), float_type(0)) }; + else + result = { tensor(tensor_shape(n_units), float_type(0)) }; + + for (EigenIndex k = 0; k < static_cast(n_timesteps); ++k) { + RowVector r; + RowVector z; + RowVector m; + + if (reset_after) { + RowMajorMatrixXf Uh = h * U; + Uh += b_h; + + z = (Wx.block(k, 0 * n, 1, n) + Uh.block(0, 0 * n, 1, n)).unaryExpr(act_func_recurrent); + r = (Wx.block(k, 1 * n, 1, n) + Uh.block(0, 1 * n, 1, n)).unaryExpr(act_func_recurrent); + m = (Wx.block(k, 2 * n, 1, n) + (r.array() * Uh.block(0, 2 * n, 1, n).array()).matrix()).unaryExpr(act_func); + } else { + z = (Wx.block(k, 0 * n, 1, n) + h * U.block(0, 0 * n, n, n) + b_h.block(0, 0 * n, 1, n)).unaryExpr(act_func_recurrent); + r = (Wx.block(k, 1 * n, 1, n) + h * U.block(0, 1 * n, n, n) + b_h.block(0, 1 * n, 1, n)).unaryExpr(act_func_recurrent); + m = (Wx.block(k, 2 * n, 1, n) + (r.array() * h.array()).matrix() * U.block(0, 2 * n, n, n) + b_h.block(0, 2 * n, 1, n)).unaryExpr(act_func); + } + + h = ((1 - z.array()) * m.array() + z.array() * h.array()).matrix(); + + if (return_sequences) + for (EigenIndex idx = 0; idx < n; ++idx) + result.front().set_ignore_rank(tensor_pos(std::size_t(k), std::size_t(idx)), h(idx)); + else if (k == static_cast(n_timesteps) - 1) + for (EigenIndex idx = 0; idx < n; ++idx) + result.front().set_ignore_rank(tensor_pos(std::size_t(idx)), h(idx)); + } + + if (return_state) { + auto state_h = tensor(tensor_shape(n_units), float_type(0)); + for (EigenIndex idx = 0; idx < n; ++idx) + state_h.set_ignore_rank(tensor_pos(std::size_t(idx)), h(idx)); + result.push_back(state_h); + } + + return result; + } + + inline tensors simple_rnn_impl(const tensor& input, + const std::size_t n_units, + const bool use_bias, + const bool return_sequences, + const bool return_state, + const float_vec& weights, + const float_vec& recurrent_weights, + const float_vec& bias, + const std::string& activation) + { + assertion(n_units > 0, "SimpleRNN units must be > 0."); + const std::size_t n_timesteps = input.shape().width_; + const std::size_t n_features = input.shape().depth_; + + const MappedRowMajorMatrixXf W = eigen_row_major_mat_from_shared_values( + n_features, n_units, weights.data()); + const MappedRowMajorMatrixXf U = eigen_row_major_mat_from_shared_values( + n_units, n_units, recurrent_weights.data()); + + const MappedRowMajorMatrixXf in = eigen_row_major_mat_from_shared_values( + n_timesteps, n_features, input.as_vector()->data()); + + RowMajorMatrixXf X = in * W; + if (use_bias) { + typedef Eigen::Matrix Vector_Xf; + const Vector_Xf b = eigen_row_major_mat_from_shared_values( + 1, n_units, bias.data()); + X.rowwise() += b; + } + + const auto act_func = get_activation_func(activation); + + const EigenIndex n = static_cast(n_units); + RowMajorMatrixXf h = RowMajorMatrixXf::Zero(1, n); + + tensors result; + if (return_sequences) + result = { tensor(tensor_shape(n_timesteps, n_units), float_type(0)) }; + else + result = { tensor(tensor_shape(n_units), float_type(0)) }; + + for (EigenIndex k = 0; k < static_cast(n_timesteps); ++k) { + h = (X.block(k, 0, 1, n) + h * U).unaryExpr(act_func); + + if (return_sequences) + for (EigenIndex idx = 0; idx < n; ++idx) + result.front().set_ignore_rank(tensor_pos(std::size_t(k), std::size_t(idx)), h(idx)); + else if (k == static_cast(n_timesteps) - 1) + for (EigenIndex idx = 0; idx < n; ++idx) + result.front().set_ignore_rank(tensor_pos(std::size_t(idx)), h(idx)); + } + + if (return_state) { + auto state_h = tensor(tensor_shape(n_units), float_type(0)); + for (EigenIndex idx = 0; idx < n; ++idx) + state_h.set_ignore_rank(tensor_pos(std::size_t(idx)), h(idx)); + result.push_back(state_h); + } + + return result; + } + + inline tensor reverse_time_series_in_tensor(const tensor& ts) + { + tensor reversed = tensor(ts.shape(), float_type(0.0)); + std::size_t n = 0; + for (std::size_t x = ts.shape().width_; x-- > 0;) { + for (std::size_t z = 0; z < ts.shape().depth_; ++z) + reversed.set_ignore_rank(tensor_pos(n, z), + ts.get_ignore_rank(tensor_pos(x, z))); + n++; + } + return reversed; + } + } } diff --git a/include/fdeep/tensor.hpp b/include/fdeep/tensor.hpp index 9a31d35c..524e398f 100644 --- a/include/fdeep/tensor.hpp +++ b/include/fdeep/tensor.hpp @@ -939,6 +939,23 @@ namespace internal { return m; } + inline MappedRowMajorMatrixXf eigen_row_major_mat_from_shared_values(std::size_t height, + std::size_t width, const float_type* data) + { + return MappedRowMajorMatrixXf( + data, + static_cast(height), + static_cast(width)); + } + + inline shared_float_vec eigen_row_major_mat_to_values(const RowMajorMatrixXf& m) + { + shared_float_vec result = fplus::make_shared_ref(); + result->resize(static_cast(m.rows() * m.cols())); + std::memcpy(result->data(), m.data(), result->size() * sizeof(float_type)); + return result; + } + inline tensor resize2d_nearest(const tensor& in_vol, const shape2& target_size) { tensor out_vol(tensor_shape(target_size.height_, target_size.width_, in_vol.shape().depth_), 0); diff --git a/keras_export/convert_model.py b/keras_export/convert_model.py index a389bef2..871371f1 100755 --- a/keras_export/convert_model.py +++ b/keras_export/convert_model.py @@ -57,6 +57,8 @@ def get_layer_input_shape_tensor_shape(layer: Layer) -> Shape: def show_tensor(tens: NDFloat32Array) -> TensorRepr: """Serialize 3-tensor to a dict""" + if tens.dtype != np.float32: + tens = tens.astype(np.float32) return { 'shape': tens.shape[1:], 'values': encode_floats(tens.flatten()) @@ -359,6 +361,27 @@ def show_depthwise_conv_2d_layer(layer: Layer) -> Mapping[str, list[str]]: return result +def show_depthwise_conv_1d_layer(layer: Layer) -> Mapping[str, list[str]]: + """Serialize DepthwiseConv1D layer to dict by promoting to a 2D depthwise representation.""" + weights = layer.get_weights() + assert layer.depth_multiplier == 1 + assert len(weights) in [1, 2] + assert len(weights[0].shape) == 3 + promoted = np.expand_dims(weights[0], axis=0) + slice_weights = prepare_filter_weights_slice_conv_2d(promoted) + + assert layer.padding in ['valid', 'same', 'causal'] + assert len(get_layer_input_shape(layer)) == 3 + assert get_layer_input_shape(layer)[0] in {None, 1} + result = { + 'slice_weights': encode_floats(slice_weights), + } + if len(weights) == 2: + bias = weights[1] + result['bias'] = encode_floats(bias) + return result + + def show_conv_1d_transpose_layer(layer: Layer) -> Mapping[str, list[str]]: """Serialize Conv1D transpose layer to dict""" weights = layer.get_weights() @@ -476,6 +499,22 @@ def show_embedding_layer(layer: Layer) -> Mapping[str, list[str]]: return result +def show_einsum_dense_layer(layer: Layer) -> Mapping[str, Union[list[str], list[int]]]: + """Serialize EinsumDense layer to dict""" + weights = layer.get_weights() + assert len(weights) in (1, 2) + kernel = weights[0] + result: dict[str, Union[list[str], list[int]]] = { + 'kernel': encode_floats(kernel), + 'kernel_shape': list(kernel.shape), + } + if len(weights) == 2: + bias = weights[1] + result['bias'] = encode_floats(bias) + result['bias_shape'] = list(bias.shape) + return result + + def show_input_layer(layer: Layer) -> None: """Serialize input layer to dict""" assert not layer.sparse @@ -497,6 +536,23 @@ def show_normalization_layer(layer: Layer) -> Mapping[str, list[str]]: } +def show_rms_normalization_layer(layer: Layer) -> Mapping[str, list[str]]: + """Serialize RMSNormalization layer to dict""" + return {'scale': encode_floats(layer.scale.numpy())} + + +def show_group_normalization_layer(layer: Layer) -> Mapping[str, list[str]]: + """Serialize GroupNormalization layer to dict""" + assert layer.axis in (-1, len(get_layer_input_shape(layer)) - 1), \ + 'GroupNormalization is only supported on the last (channel) axis.' + result: dict[str, list[str]] = {} + if layer.scale: + result['gamma'] = encode_floats(layer.gamma.numpy()) + if layer.center: + result['beta'] = encode_floats(layer.beta.numpy()) + return result + + def show_upsampling2d_layer(layer: Layer) -> None: """Serialize UpSampling2D layer to dict""" assert layer.interpolation in ['nearest', 'bilinear'] @@ -551,6 +607,136 @@ def show_multi_head_attention_layer(layer: Layer) -> Mapping[str, List[list[str] } +def show_group_query_attention_layer(layer: Layer) -> Mapping[str, List[list[str]]]: + """Serialize GroupedQueryAttention layer to dict""" + return { + 'weight_shapes': list(map(lambda w: list(w.shape), layer.weights)), + 'weights': list(map(lambda w: encode_floats(w.numpy()), layer.weights)), + } + + +def show_recurrent_layer_weights(layer: Layer) -> Mapping[str, list[str]]: + """Serialize LSTM/GRU/SimpleRNN weights to dict""" + assert not layer.go_backwards, 'go_backwards=True is not supported for standalone recurrent layers; use Bidirectional instead.' + assert not layer.unroll, 'unroll=True is not supported.' + assert not layer.stateful, 'stateful=True is not supported.' + weights = layer.get_weights() + assert len(weights) in (2, 3) + result: dict[str, list[str]] = { + 'weights': encode_floats(weights[0]), + 'recurrent_weights': encode_floats(weights[1]), + } + if len(weights) == 3: + result['bias'] = encode_floats(weights[2]) + return result + + +def show_lstm_layer(layer: Layer) -> Mapping[str, list[str]]: + """Serialize LSTM layer to dict""" + return show_recurrent_layer_weights(layer) + + +def _prepare_conv_lstm_kernel(kernel: NDFloat32Array) -> NDFloat32Array: + """Reshape ConvLSTM kernel to fdeep's (filters, ..., in_c) flat layout.""" + if kernel.ndim == 3: # ConvLSTM1D: (k_w, in_c, filters) + kernel = np.expand_dims(kernel, axis=0) # (1, k_w, in_c, filters) + if kernel.ndim == 4: # ConvLSTM2D: (k_h, k_w, in_c, filters) + return np.moveaxis(kernel, [0, 1, 2, 3], [1, 2, 3, 0]).flatten() + assert kernel.ndim == 5 # ConvLSTM3D: (k_d4, k_h, k_w, in_c, filters) + return np.moveaxis(kernel, [0, 1, 2, 3, 4], [1, 2, 3, 4, 0]).flatten() + + +def show_conv_lstm_layer(layer: Layer) -> Mapping[str, list[str]]: + """Serialize ConvLSTM1D/2D layer to dict""" + assert not layer.go_backwards, 'go_backwards=True is not supported for ConvLSTM.' + assert not layer.unroll, 'unroll=True is not supported.' + assert not layer.stateful, 'stateful=True is not supported.' + weights = layer.get_weights() + assert len(weights) in (2, 3) + result: dict[str, list[str]] = { + 'weights': encode_floats(_prepare_conv_lstm_kernel(weights[0])), + 'recurrent_weights': encode_floats(_prepare_conv_lstm_kernel(weights[1])), + } + if len(weights) == 3: + result['bias'] = encode_floats(weights[2]) + return result + + +def show_gru_layer(layer: Layer) -> Mapping[str, list[str]]: + """Serialize GRU layer to dict""" + return show_recurrent_layer_weights(layer) + + +def show_simple_rnn_layer(layer: Layer) -> Mapping[str, list[str]]: + """Serialize SimpleRNN layer to dict""" + return show_recurrent_layer_weights(layer) + + +def show_rnn_layer(layer: Layer) -> Mapping[str, list[str]]: + """Serialize a generic RNN layer wrapping LSTMCell/GRUCell/SimpleRNNCell or StackedRNNCells.""" + assert not layer.go_backwards, 'go_backwards=True is not supported.' + assert not layer.unroll, 'unroll=True is not supported.' + assert not layer.stateful, 'stateful=True is not supported.' + + cell_type = type(layer.cell).__name__ + if cell_type == 'StackedRNNCells': + result: dict[str, list[str]] = {} + for i, sub_cell in enumerate(layer.cell.cells): + sub_class = type(sub_cell).__name__ + assert sub_class in ('LSTMCell', 'GRUCell', 'SimpleRNNCell'), \ + f'StackedRNNCells with inner cell {sub_class} is not supported.' + sub_weights = sub_cell.get_weights() + assert len(sub_weights) in (2, 3) + prefix = f'cell{i}_' + result[prefix + 'weights'] = encode_floats(sub_weights[0]) + result[prefix + 'recurrent_weights'] = encode_floats(sub_weights[1]) + if len(sub_weights) == 3: + result[prefix + 'bias'] = encode_floats(sub_weights[2]) + return result + + assert cell_type in ('LSTMCell', 'GRUCell', 'SimpleRNNCell'), \ + f'RNN with cell {cell_type} is not supported.' + weights = layer.get_weights() + assert len(weights) in (2, 3) + res: dict[str, list[str]] = { + 'weights': encode_floats(weights[0]), + 'recurrent_weights': encode_floats(weights[1]), + } + if len(weights) == 3: + res['bias'] = encode_floats(weights[2]) + return res + + +def show_bidirectional_layer(layer: Layer) -> Mapping[str, list[str]]: + """Serialize Bidirectional wrapper around an LSTM/GRU/SimpleRNN to dict""" + assert layer.merge_mode in ('concat', 'sum', 'mul', 'ave'), \ + 'Bidirectional merge_mode=' + str(layer.merge_mode) + ' is not supported.' + forward_layer = layer.forward_layer + backward_layer = layer.backward_layer + assert type(forward_layer).__name__ in ('LSTM', 'GRU', 'SimpleRNN'), \ + 'Bidirectional wrapping ' + type(forward_layer).__name__ + ' not supported.' + assert type(backward_layer).__name__ == type(forward_layer).__name__ + assert not forward_layer.go_backwards, 'forward_layer.go_backwards=True is not supported.' + assert not forward_layer.unroll and not backward_layer.unroll, 'unroll=True is not supported.' + assert not forward_layer.stateful and not backward_layer.stateful, 'stateful=True is not supported.' + + forward_weights = forward_layer.get_weights() + backward_weights = backward_layer.get_weights() + assert len(forward_weights) in (2, 3) + assert len(forward_weights) == len(backward_weights) + + result: dict[str, list[str]] = { + 'forward_weights': encode_floats(forward_weights[0]), + 'forward_recurrent_weights': encode_floats(forward_weights[1]), + 'backward_weights': encode_floats(backward_weights[0]), + 'backward_recurrent_weights': encode_floats(backward_weights[1]), + } + if len(forward_weights) == 3: + result['forward_bias'] = encode_floats(forward_weights[2]) + result['backward_bias'] = encode_floats(backward_weights[2]) + return result + + def get_layer_functions_dict() -> Mapping[str, Callable[[Layer], LayerConfig]]: return { 'Conv1D': show_conv_1d_layer, @@ -560,17 +746,21 @@ def get_layer_functions_dict() -> Mapping[str, Callable[[Layer], LayerConfig]]: 'Conv2DTranspose': show_conv_2d_transpose_layer, 'Conv3DTranspose': show_conv_3d_transpose_layer, 'SeparableConv2D': show_separable_conv_2d_layer, + 'DepthwiseConv1D': show_depthwise_conv_1d_layer, 'DepthwiseConv2D': show_depthwise_conv_2d_layer, 'BatchNormalization': show_batch_normalization_layer, 'Dense': show_dense_layer, 'Dot': show_dot_layer, 'PReLU': show_prelu_layer, + 'EinsumDense': show_einsum_dense_layer, 'Embedding': show_embedding_layer, 'LayerNormalization': show_layer_normalization_layer, 'TimeDistributed': show_time_distributed_layer, 'Input': show_input_layer, 'Softmax': show_softmax_layer, 'Normalization': show_normalization_layer, + 'RMSNormalization': show_rms_normalization_layer, + 'GroupNormalization': show_group_normalization_layer, 'UpSampling2D': show_upsampling2d_layer, 'UpSampling3D': show_upsampling3d_layer, 'Resizing': show_resizing_layer, @@ -579,6 +769,16 @@ def get_layer_functions_dict() -> Mapping[str, Callable[[Layer], LayerConfig]]: 'Attention': show_attention_layer, 'AdditiveAttention': show_additive_attention_layer, 'MultiHeadAttention': show_multi_head_attention_layer, + 'GroupedQueryAttention': show_group_query_attention_layer, + 'GroupQueryAttention': show_group_query_attention_layer, + 'ConvLSTM1D': show_conv_lstm_layer, + 'ConvLSTM2D': show_conv_lstm_layer, + 'ConvLSTM3D': show_conv_lstm_layer, + 'LSTM': show_lstm_layer, + 'GRU': show_gru_layer, + 'SimpleRNN': show_simple_rnn_layer, + 'RNN': show_rnn_layer, + 'Bidirectional': show_bidirectional_layer, } @@ -684,7 +884,22 @@ def get_all_weights(model: Model, prefix: str) -> Mapping[str, LayerConfig]: layer_type = type(layer).__name__ for node in layer._inbound_nodes: if 'training' in node.arguments.kwargs: - is_layer_with_accidental_training_flag = layer_type in ('CenterCrop', 'Resizing') + is_layer_with_accidental_training_flag = layer_type in ( + 'CenterCrop', 'Resizing', + # Image augmentation layers below are identity at + # inference, so a stray training=True is harmless. + # AutoContrast is intentionally excluded — it transforms + # at inference and is not currently supported. + 'RandomBrightness', 'RandomColorDegeneration', 'RandomColorJitter', + 'RandomContrast', 'RandomCrop', 'RandomElasticTransform', + 'RandomErasing', 'RandomFlip', 'RandomGaussianBlur', + 'RandomGrayscale', 'RandomHeight', 'RandomHue', 'RandomInvert', + 'RandomPerspective', 'RandomPosterization', 'RandomRotation', + 'RandomSaturation', 'RandomSharpness', 'RandomShear', + 'RandomTranslation', 'RandomWidth', 'RandomZoom', + 'AugMix', 'CutMix', 'Equalization', + 'MaxNumBoundingBoxes', 'MixUp', 'Pipeline', 'RandAugment', + 'Solarization') has_training = node.arguments.kwargs['training'] is True assert not has_training or is_layer_with_accidental_training_flag, \ 'training=true is not supported, see https://github.com/Dobiasd/frugally-deep/issues/284' diff --git a/keras_export/generate_test_models.py b/keras_export/generate_test_models.py index e7d707cd..e142a649 100644 --- a/keras_export/generate_test_models.py +++ b/keras_export/generate_test_models.py @@ -31,6 +31,19 @@ from keras.layers import Permute, Reshape, RepeatVector from keras.layers import SeparableConv2D, DepthwiseConv2D from keras.layers import ZeroPadding3D, Cropping3D +from keras.layers import LSTM, GRU, SimpleRNN, Bidirectional +from keras.layers import DepthwiseConv1D, EinsumDense +from keras.layers import RMSNormalization, GroupNormalization +from keras.layers import AdaptiveAveragePooling1D, AdaptiveMaxPooling1D +from keras.layers import AdaptiveAveragePooling2D, AdaptiveMaxPooling2D +from keras.layers import AdaptiveAveragePooling3D, AdaptiveMaxPooling3D +from keras.layers import GroupQueryAttention +from keras.layers import ConvLSTM1D, ConvLSTM2D, ConvLSTM3D +from keras.layers import RNN, LSTMCell, GRUCell, SimpleRNNCell, StackedRNNCells +from keras.layers import Discretization, IntegerLookup, Masking +from keras.layers import RandomBrightness, RandomFlip, RandomCrop +from keras.layers import RandomContrast, RandomRotation, RandomTranslation, RandomZoom +from keras.layers import RandomHue, RandomSaturation, RandomSharpness from keras.models import Model, load_model, Sequential __author__ = "Tobias Hermann" @@ -528,6 +541,79 @@ def get_test_model_exhaustive() -> Model: num_heads=2, key_dim=3, value_dim=5, use_bias=True, output_shape=None, attention_axes=None)(inputs[49], inputs[50], inputs[51])) + # GroupedQueryAttention: shared Q/K/V seq, separate K/V seq, with/without gate. + outputs.append(GroupQueryAttention(head_dim=4, num_query_heads=6, + num_key_value_heads=2)(inputs[49], inputs[49], inputs[49])) + outputs.append(GroupQueryAttention(head_dim=4, num_query_heads=6, + num_key_value_heads=2, use_gate=True)(inputs[49], inputs[49], inputs[49])) + outputs.append(GroupQueryAttention(head_dim=4, num_query_heads=4, + num_key_value_heads=4)(inputs[49], inputs[49], inputs[49])) + # num_query_heads == num_key_value_heads with distinct K/V seq lengths + # exercises the MHA-equivalence path with cross-attention shapes. + outputs.append(GroupQueryAttention(head_dim=4, num_query_heads=4, + num_key_value_heads=4)(inputs[49], inputs[50], inputs[51])) + outputs.append(GroupQueryAttention(head_dim=3, num_query_heads=4, + num_key_value_heads=2)(inputs[49], inputs[50], inputs[51])) + + # DepthwiseConv1D variants on rank-2 sequence input. + outputs.append(DepthwiseConv1D(kernel_size=3, padding='same')(inputs[49])) + outputs.append(DepthwiseConv1D(kernel_size=2, padding='valid', strides=2)(inputs[49])) + outputs.append(DepthwiseConv1D(kernel_size=3, padding='same', use_bias=False)(inputs[49])) + + # RMSNormalization / GroupNormalization on a (T, F=4) input. + outputs.append(RMSNormalization()(inputs[49])) + outputs.append(RMSNormalization(epsilon=1e-3)(inputs[49])) + outputs.append(GroupNormalization(groups=2)(inputs[49])) + outputs.append(GroupNormalization(groups=2, scale=False)(inputs[49])) + outputs.append(GroupNormalization(groups=4, center=False, epsilon=1e-4)(inputs[49])) + outputs.append(GroupNormalization(groups=1)(inputs[49])) # = LayerNormalization + outputs.append(GroupNormalization(groups=4)(inputs[49])) # = InstanceNormalization + + # EinsumDense: Dense-equivalent, multi-head projection, no-bias, chained collapse. + outputs.append(EinsumDense('abc,cd->abd', output_shape=(None, 8), + bias_axes='d')(inputs[49])) + outputs.append(EinsumDense('abc,cde->abde', output_shape=(None, 3, 4), + bias_axes='de')(inputs[49])) + outputs.append(EinsumDense('abc,cd->abd', output_shape=(None, 6))(inputs[49])) + outputs.append(EinsumDense('abcd,cde->abe', output_shape=(None, 5), + bias_axes='e')(EinsumDense('abc,cde->abde', + output_shape=(None, 3, 4))(inputs[49]))) + # bias_axes deliberately not in rhs order: Keras still allocates the bias + # in rhs order ('de'), so this regression case ensures fdeep reads the + # right element when bias_axes characters are permuted. + outputs.append(EinsumDense('abc,cde->abde', output_shape=(None, 3, 4), + bias_axes='ed')(inputs[49])) + + # AdaptiveAvg/MaxPooling 1D/2D/3D variants. + outputs.append(AdaptiveAveragePooling1D(output_size=3)(inputs[49])) + outputs.append(AdaptiveMaxPooling1D(output_size=3)(inputs[49])) + outputs.append(AdaptiveAveragePooling1D(output_size=2)(inputs[49])) + outputs.append(AdaptiveAveragePooling2D(output_size=(3, 4))(inputs[22])) + outputs.append(AdaptiveMaxPooling2D(output_size=(3, 4))(inputs[22])) + outputs.append(AdaptiveMaxPooling2D(output_size=(13, 14))(inputs[22])) + outputs.append(AdaptiveAveragePooling3D(output_size=(2, 3, 3))(inputs[2])) + outputs.append(AdaptiveMaxPooling3D(output_size=(2, 3, 3))(inputs[2])) + outputs.append(AdaptiveAveragePooling3D(output_size=(7, 2, 3))(inputs[2])) + + # Discretization on a float input. + outputs.append(Discretization(bin_boundaries=[-0.5, 0.0, 0.5, 1.0])(inputs[49])) + + # IntegerLookup: scaled to int via Discretization so the input is float. + outputs.append(IntegerLookup(vocabulary=[0, 1, 2, 3])( + Discretization(bin_boundaries=[-1.0, 0.0, 1.0])(inputs[49]))) + + # Training-only Random* augmentation layers (passed through at inference). + outputs.append(RandomBrightness(factor=0.1)(inputs[22])) + outputs.append(RandomFlip(mode='horizontal')(inputs[22])) + outputs.append(RandomCrop(height=26, width=28)(inputs[22])) + outputs.append(RandomContrast(factor=0.1)(inputs[22])) + outputs.append(RandomRotation(factor=0.1)(inputs[22])) + outputs.append(RandomTranslation(0.1, 0.1)(inputs[22])) + outputs.append(RandomZoom(0.1)(inputs[22])) + outputs.append(RandomHue(factor=0.1, value_range=(0.0, 1.0))(inputs[22])) + outputs.append(RandomSaturation(factor=0.1, value_range=(0.0, 1.0))(inputs[22])) + outputs.append(RandomSharpness(factor=0.1, value_range=(0.0, 1.0))(inputs[22])) + shared_conv = Conv2D(1, (1, 1), padding='valid', name='shared_conv', activation='relu') @@ -814,6 +900,100 @@ def get_test_model_sequential() -> Model: return model +def get_test_model_recurrent() -> Model: + """Returns a test model exercising recurrent layers (LSTM, GRU, SimpleRNN, + Bidirectional, RNN with cells, ConvLSTM, Masking).""" + seq_len = 5 + n_features = 4 + + inputs = Input(shape=(seq_len, n_features)) + + lstm_seq = LSTM(6, return_sequences=True)(inputs) + lstm_last = LSTM(7)(lstm_seq) + lstm_no_bias = LSTM(5, use_bias=False)(inputs) + lstm_relu = LSTM(4, activation='relu', recurrent_activation='hard_sigmoid')(inputs) + lstm_no_unit_forget = LSTM(4, unit_forget_bias=False)(inputs) + lstm_state_out, lstm_state_h, lstm_state_c = LSTM(3, return_state=True)(inputs) + + gru_seq = GRU(5, return_sequences=True)(inputs) + gru_seq_no_reset_after = GRU(5, return_sequences=True, reset_after=False)(inputs) + gru_last = GRU(8, activation='relu', recurrent_activation='hard_sigmoid')(gru_seq) + gru_no_bias = GRU(4, use_bias=False)(gru_seq_no_reset_after) + + rnn_seq = SimpleRNN(6, return_sequences=True)(inputs) + rnn_last = SimpleRNN(5, activation='tanh')(rnn_seq) + rnn_relu_no_bias = SimpleRNN(4, activation='relu', use_bias=False)(inputs) + rnn_state_out, rnn_state_h = SimpleRNN(3, return_state=True)(inputs) + + gru_state_out, gru_state_h = GRU(4, return_state=True)(inputs) + + bidi_lstm_seq = Bidirectional(LSTM(4, return_sequences=True))(inputs) + bidi_lstm_last_concat = Bidirectional(LSTM(3))(bidi_lstm_seq) + bidi_lstm_sum = Bidirectional(LSTM(3), merge_mode='sum')(inputs) + bidi_gru_mul = Bidirectional(GRU(4), merge_mode='mul')(inputs) + bidi_gru_ave = Bidirectional(GRU(4), merge_mode='ave')(inputs) + bidi_rnn_concat = Bidirectional(SimpleRNN(3, return_sequences=True))(inputs) + + masked = Masking(mask_value=0.0)(inputs) + + rnn_lstm = RNN(LSTMCell(4))(inputs) + rnn_gru = RNN(GRUCell(5), return_sequences=True)(inputs) + rnn_simple = RNN(SimpleRNNCell(3))(inputs) + rnn_stacked = RNN(StackedRNNCells([LSTMCell(6), GRUCell(4), SimpleRNNCell(3)]))(inputs) + rnn_stacked_seq = RNN(StackedRNNCells([LSTMCell(5), GRUCell(3)]), + return_sequences=True)(inputs) + + inputs_clstm1d = Input(shape=(4, 6, 3)) + inputs_clstm2d = Input(shape=(3, 5, 5, 3)) + inputs_clstm3d = Input(shape=(2, 3, 4, 4, 3)) + clstm1d = ConvLSTM1D(filters=2, kernel_size=3, padding='same')(inputs_clstm1d) + clstm1d_valid = ConvLSTM1D(filters=2, kernel_size=2, padding='valid', + return_sequences=True)(inputs_clstm1d) + clstm1d_dilated = ConvLSTM1D(filters=2, kernel_size=2, padding='same', + dilation_rate=2)(inputs_clstm1d) + clstm2d = ConvLSTM2D(filters=2, kernel_size=(3, 3), padding='same', + return_sequences=True)(inputs_clstm2d) + clstm2d_valid_no_bias = ConvLSTM2D(filters=3, kernel_size=(2, 2), padding='valid', + use_bias=False, activation='relu')(inputs_clstm2d) + # Two ConvLSTM2Ds chained: validates state handoff between successive temporal layers. + clstm2d_chained = ConvLSTM2D(filters=2, kernel_size=(2, 2), padding='same')( + ConvLSTM2D(filters=3, kernel_size=(3, 3), padding='same', + return_sequences=True)(inputs_clstm2d)) + clstm3d = ConvLSTM3D(filters=2, kernel_size=(2, 2, 2), padding='same')(inputs_clstm3d) + clstm3d_valid = ConvLSTM3D(filters=2, kernel_size=(1, 2, 2), padding='valid', + return_sequences=True)(inputs_clstm3d) + + outputs = [ + lstm_last, lstm_no_bias, lstm_relu, lstm_no_unit_forget, + lstm_state_out, lstm_state_h, lstm_state_c, + gru_last, gru_no_bias, gru_state_out, gru_state_h, + rnn_last, rnn_relu_no_bias, rnn_state_out, rnn_state_h, + bidi_lstm_last_concat, + bidi_lstm_sum, + bidi_gru_mul, + bidi_gru_ave, + bidi_rnn_concat, + masked, + rnn_lstm, rnn_gru, rnn_simple, + rnn_stacked, rnn_stacked_seq, + clstm1d, clstm1d_valid, clstm1d_dilated, + clstm2d, clstm2d_valid_no_bias, clstm2d_chained, + clstm3d, clstm3d_valid, + ] + + model = Model(inputs=[inputs, inputs_clstm1d, inputs_clstm2d, inputs_clstm3d], + outputs=outputs, name='test_model_recurrent') + model.compile(loss='mse', optimizer='adam') + + training_data_size = 2 + data_in = generate_input_data(training_data_size, + [(seq_len, n_features), (4, 6, 3), (3, 5, 5, 3), (2, 3, 4, 4, 3)]) + initial_data_out = model.predict(data_in) + data_out = generate_output_data(training_data_size, initial_data_out) + model.fit(data_in, data_out, epochs=1) + return model + + def main() -> None: """Generate different test models and save them to the given directory.""" if len(sys.argv) != 3: @@ -829,6 +1009,7 @@ def main() -> None: 'variable': get_test_model_variable, 'autoencoder': get_test_model_autoencoder, 'sequential': get_test_model_sequential, + 'recurrent': get_test_model_recurrent, } if not model_name in get_model_functions: diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1781cbaa..cb568cf6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -24,6 +24,10 @@ add_custom_command ( OUTPUT test_model_sequential.keras COMMAND bash -c "${Python3_EXECUTABLE} ${FDEEP_TOP_DIR}/keras_export/generate_test_models.py sequential test_model_sequential.keras" WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/) +add_custom_command ( OUTPUT test_model_recurrent.keras + COMMAND bash -c "${Python3_EXECUTABLE} ${FDEEP_TOP_DIR}/keras_export/generate_test_models.py recurrent test_model_recurrent.keras" + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/) + add_custom_command ( OUTPUT readme_example_model.keras COMMAND bash -c "${Python3_EXECUTABLE} ${FDEEP_TOP_DIR}/test/readme_example_generate.py" WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/) @@ -53,6 +57,11 @@ add_custom_command ( OUTPUT test_model_sequential.json COMMAND bash -c "${Python3_EXECUTABLE} ${FDEEP_TOP_DIR}/keras_export/convert_model.py test_model_sequential.keras test_model_sequential.json" WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/) +add_custom_command ( OUTPUT test_model_recurrent.json + DEPENDS test_model_recurrent.keras + COMMAND bash -c "${Python3_EXECUTABLE} ${FDEEP_TOP_DIR}/keras_export/convert_model.py test_model_recurrent.keras test_model_recurrent.json" + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/) + add_custom_command ( OUTPUT readme_example_model.json DEPENDS readme_example_model.keras COMMAND bash -c "${Python3_EXECUTABLE} ${FDEEP_TOP_DIR}/keras_export/convert_model.py readme_example_model.keras readme_example_model.json" @@ -74,6 +83,7 @@ _add_test(test_model_embedding_test test_model_embedding.json) _add_test(test_model_variable_test test_model_variable.json) _add_test(test_model_autoencoder_test test_model_autoencoder.json) _add_test(test_model_sequential_test test_model_sequential.json) +_add_test(test_model_recurrent_test test_model_recurrent.json) _add_test(readme_example_main readme_example_model.json) add_custom_target(unittest @@ -82,6 +92,7 @@ add_custom_target(unittest COMMAND test_model_variable_test COMMAND test_model_autoencoder_test COMMAND test_model_sequential_test + COMMAND test_model_recurrent_test COMMAND readme_example_main COMMENT "Running unittests\n\n" diff --git a/test/test_model_recurrent_test.cpp b/test/test_model_recurrent_test.cpp new file mode 100644 index 00000000..4b180f65 --- /dev/null +++ b/test/test_model_recurrent_test.cpp @@ -0,0 +1,21 @@ +// Copyright 2016, Tobias Hermann. +// https://github.com/Dobiasd/frugally-deep +// Distributed under the MIT License. +// (See accompanying LICENSE file or at +// https://opensource.org/licenses/MIT) + +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include "doctest/doctest.h" +#define FDEEP_FLOAT_TYPE double +#include + +TEST_CASE("test_model_recurrent_test, load_model") +{ + const auto model = fdeep::load_model("../test_model_recurrent.json", + true, fdeep::cout_logger, static_cast(0.00001)); + const auto multi_inputs = fplus::generate>( + [&]() -> fdeep::tensors { return model.generate_dummy_inputs(); }, + 10); + model.predict_multi(multi_inputs, false); + model.predict_multi(multi_inputs, true); +}