Fixed couple of copy/move issues

Author: FFMG

src/neuralnetwork/gradientsandoutputs.h

@@ -46,6 +46,7 @@ class GradientsAndOutputs
       _gradients = src._gradients;
       _rnn_outputs = src._rnn_outputs;
       _rnn_gradients = src._rnn_gradients;
+      _rnn_gate_gradients = src._rnn_gate_gradients;
     }
     return *this;
   }
@@ -59,6 +60,7 @@ class GradientsAndOutputs
       _gradients = std::move(src._gradients);
       _rnn_outputs = std::move(src._rnn_outputs);
       _rnn_gradients = std::move(src._rnn_gradients);
+      _rnn_gate_gradients = std::move(src._rnn_gate_gradients);
     }
     return *this;
   }

src/neuralnetwork/hiddenstates.h

@@ -21,10 +21,86 @@ class HiddenStates
     _layer_views.resize(num_layers);
   }
 
-  HiddenStates(const HiddenStates& src) = default;
-  HiddenStates(HiddenStates&& src) noexcept = default;
-  HiddenStates& operator=(HiddenStates&& src) noexcept = default;
-  HiddenStates& operator=(const HiddenStates& src) = default;
+  HiddenStates(const HiddenStates& src) :
+    _topology(src._topology),
+    _pre_activation_sums(src._pre_activation_sums),
+    _hidden_state_values(src._hidden_state_values),
+    _cell_state_values(src._cell_state_values)
+  {
+    MYODDWEB_PROFILE_FUNCTION("HiddenStates");
+    rebuild_views();
+  }
+
+  HiddenStates(HiddenStates&& src) noexcept :
+    _topology(std::move(src._topology)),
+    _pre_activation_sums(std::move(src._pre_activation_sums)),
+    _hidden_state_values(std::move(src._hidden_state_values)),
+    _cell_state_values(std::move(src._cell_state_values))
+  {
+    MYODDWEB_PROFILE_FUNCTION("HiddenStates");
+    rebuild_views();
+  }
+
+  HiddenStates& operator=(const HiddenStates& src)
+  {
+    MYODDWEB_PROFILE_FUNCTION("HiddenStates");
+    if (this != &src)
+    {
+      _topology = src._topology;
+      _pre_activation_sums = src._pre_activation_sums;
+      _hidden_state_values = src._hidden_state_values;
+      _cell_state_values = src._cell_state_values;
+      rebuild_views();
+    }
+    return *this;
+  }
+
+  HiddenStates& operator=(HiddenStates&& src) noexcept
+  {
+    MYODDWEB_PROFILE_FUNCTION("HiddenStates");
+    if (this != &src)
+    {
+      _topology = std::move(src._topology);
+      _pre_activation_sums = std::move(src._pre_activation_sums);
+      _hidden_state_values = std::move(src._hidden_state_values);
+      _cell_state_values = std::move(src._cell_state_values);
+      rebuild_views();
+    }
+    return *this;
+  }
+
+  void rebuild_views()
+  {
+    MYODDWEB_PROFILE_FUNCTION("HiddenStates");
+    _layer_views.clear();
+    const size_t num_layers = _topology.size();
+    _layer_views.resize(num_layers);
+
+    for (size_t layer_number = 0; layer_number < num_layers; ++layer_number)
+    {
+      const size_t n = _topology[layer_number];
+      if (n == 0) continue;
+
+      const size_t total_hid = _hidden_state_values[layer_number].size();
+      const size_t num_time_steps = total_hid / n;
+      if (num_time_steps == 0) continue;
+
+      const size_t total_pre = _pre_activation_sums[layer_number].size();
+      const unsigned multiplier = static_cast<unsigned>(total_pre / (num_time_steps * n));
+
+      auto& views = _layer_views[layer_number];
+      views.reserve(num_time_steps);
+
+      double* p_pre = _pre_activation_sums[layer_number].data();
+      double* p_hid = _hidden_state_values[layer_number].data();
+      double* p_cel = _cell_state_values[layer_number].data();
+
+      for (size_t t = 0; t < num_time_steps; ++t)
+      {
+        views.emplace_back(&p_pre[t * n * multiplier], &p_hid[t * n], &p_cel[t * n], static_cast<unsigned>(n), static_cast<unsigned>(n * multiplier));
+      }
+    }
+  }
 
   void zero()
   {

src/neuralnetwork/layer.h

@@ -468,16 +468,21 @@ class Layer
       _b_timesteps = std::move(src._b_timesteps);
       _b_decays = std::move(src._b_decays);
 
+      _residual_layer_number = src._residual_layer_number;
       delete _residual_projector;
       _residual_projector = src._residual_projector;
 
       _task_queue_pool = std::move(src._task_queue_pool);
 
       _layer_activation_helper = std::move(src._layer_activation_helper);
+      _momentum = src._momentum;
 
       src._layer_index = 0;
+      src._layer_role = Role::Input;
       src._optimiser_type = OptimiserType::None;
+      src._residual_layer_number = 0;
       src._residual_projector = nullptr;
+      src._momentum = 0;
     }
     return *this;
   }

src/neuralnetwork/layers.cpp

@@ -75,12 +75,15 @@ Layers::Layers(const Layers& src) noexcept :
   _update_weights_pool(nullptr)
 {
   MYODDWEB_PROFILE_FUNCTION("Layers");
+  std::shared_lock<std::shared_mutex> lock(src._mutex);
   _update_weights_pool = new TaskQueuePool<void>(src._update_weights_pool->get_number_of_threads());
   _layers.reserve(src._layers.size());
   for(const auto& layer : src._layers)
   {
     _layers.emplace_back(std::unique_ptr<Layer>(layer->clone()));
   }
+  _training_gradients_buffer = src._training_gradients_buffer;
+  _training_hidden_states_buffer = src._training_hidden_states_buffer;
 }
 
 Layers::Layers(
@@ -113,6 +116,10 @@ Layers& Layers::operator=(const Layers& src) noexcept
   MYODDWEB_PROFILE_FUNCTION("Layers");
   if(this != &src)
   {
+    std::unique_lock<std::shared_mutex> lhs_lock(_mutex, std::defer_lock);
+    std::shared_lock<std::shared_mutex> rhs_lock(src._mutex, std::defer_lock);
+    std::lock(lhs_lock, rhs_lock);
+
     delete _update_weights_pool;
     _update_weights_pool = new TaskQueuePool<void>(src._update_weights_pool->get_number_of_threads());
 
@@ -122,6 +129,9 @@ Layers& Layers::operator=(const Layers& src) noexcept
     {
       _layers.emplace_back(std::unique_ptr<Layer>(layer->clone()));
     }
+
+    _training_gradients_buffer = src._training_gradients_buffer;
+    _training_hidden_states_buffer = src._training_hidden_states_buffer;
   }
   return *this;
 }
@@ -131,6 +141,10 @@ Layers& Layers::operator=(Layers&& src) noexcept
   MYODDWEB_PROFILE_FUNCTION("Layers");
   if (this != &src)
   {
+    std::unique_lock<std::shared_mutex> lhs_lock(_mutex, std::defer_lock);
+    std::unique_lock<std::shared_mutex> rhs_lock(src._mutex, std::defer_lock);
+    std::lock(lhs_lock, rhs_lock);
+
     _layers = std::move(src._layers);
 
     _training_gradients_buffer = std::move(src._training_gradients_buffer);

src/neuralnetwork/neuralnetwork.cpp

@@ -63,6 +63,15 @@ NeuralNetwork::NeuralNetwork(const NeuralNetwork& src) :
   *this = src;
 }
 
+NeuralNetwork::NeuralNetwork(NeuralNetwork&& src) noexcept :
+  _layers(std::move(src._layers)),
+  _options(std::move(src._options)),
+  _neural_network_helper(nullptr)
+{
+  MYODDWEB_PROFILE_FUNCTION("NeuralNetwork");
+  *this = std::move(src);
+}
+
 NeuralNetwork& NeuralNetwork::operator=(const NeuralNetwork& src)
 {
   MYODDWEB_PROFILE_FUNCTION("NeuralNetwork");
@@ -85,7 +94,36 @@ NeuralNetwork& NeuralNetwork::operator=(const NeuralNetwork& src)
     if (src._neural_network_helper != nullptr)
     {
       _neural_network_helper = new NeuralNetworkHelper(*src._neural_network_helper);
+      _neural_network_helper->_neural_network = this;
+    }
+  }
+  return *this;
+}
+
+NeuralNetwork& NeuralNetwork::operator=(NeuralNetwork&& src) noexcept
+{
+  MYODDWEB_PROFILE_FUNCTION("NeuralNetwork");
+  if (this != &src)
+  {
+    std::unique_lock<std::shared_mutex> lhs_lock(_mutex, std::defer_lock);
+    std::unique_lock<std::shared_mutex> rhs_lock(src._mutex, std::defer_lock);
+    std::lock(lhs_lock, rhs_lock);
+
+    _learning_rate = src._learning_rate;
+    _layers = std::move(src._layers);
+    _options = std::move(src._options);
+    _saved_errors = std::move(src._saved_errors);
+    _last_metrics = std::move(src._last_metrics);
+
+    delete _neural_network_helper;
+    _neural_network_helper = src._neural_network_helper;
+    if (_neural_network_helper != nullptr)
+    {
+      _neural_network_helper->_neural_network = this;
     }
+
+    src._neural_network_helper = nullptr;
+    src._learning_rate = 0.0;
   }
   return *this;
 }
@@ -252,6 +290,7 @@ void NeuralNetwork::create_batch_from_indexes(
 std::vector<std::vector<double>> NeuralNetwork::think(const std::vector<std::vector<double>>& inputs) const
 {
   MYODDWEB_PROFILE_FUNCTION("NeuralNetwork");
+  std::shared_lock<std::shared_mutex> read(_mutex);
   return _layers.think(_options, inputs);
 }
 
@@ -263,6 +302,7 @@ std::vector<double> NeuralNetwork::think(const std::vector<double>& inputs) cons
     Logger::error("The input size, '", inputs.size(),"' does not match the topology!");
     return {};
   }
+  std::shared_lock<std::shared_mutex> read(_mutex);
   return _layers.think(_options, inputs);
 }
 

src/neuralnetwork/neuralnetwork.h

@@ -34,7 +34,10 @@ class NeuralNetwork
   NeuralNetwork(const Layers& layers, const NeuralNetworkOptions& options, const std::vector<std::map<ErrorCalculation::type, double>>& errors);
 
   NeuralNetwork(const NeuralNetwork& src);
+  NeuralNetwork(NeuralNetwork&& src) noexcept;
+
   NeuralNetwork& operator=(const NeuralNetwork&);
+  NeuralNetwork& operator=(NeuralNetwork&&) noexcept;
 
   virtual ~NeuralNetwork();
 

src/neuralnetwork/neuralnetworkoptions.h

@@ -75,15 +75,13 @@ class NeuralNetworkOptions
   }
 
 public:
-  NeuralNetworkOptions(const NeuralNetworkOptions& nno) noexcept :
-    _output_layer_details(nno._output_layer_details)
+  NeuralNetworkOptions(const NeuralNetworkOptions& nno) noexcept
   {
     MYODDWEB_PROFILE_FUNCTION("NeuralNetworkOptions");
     *this = nno;
   }
 
-  NeuralNetworkOptions(NeuralNetworkOptions&& nno) noexcept :
-    _output_layer_details(std::move(nno._output_layer_details))
+  NeuralNetworkOptions(NeuralNetworkOptions&& nno) noexcept
   {
     MYODDWEB_PROFILE_FUNCTION("NeuralNetworkOptions");
     *this = std::move(nno);
@@ -136,7 +134,7 @@ class NeuralNetworkOptions
       _number_of_epoch = nno._number_of_epoch;
       _batch_size = nno._batch_size;
       _data_is_unique = nno._data_is_unique;
-      _progress_callback = nno._progress_callback;
+      _progress_callback = std::move(nno._progress_callback);
       _log_level = nno._log_level;
       _number_of_threads = nno._number_of_threads;
       _learning_rate_decay_rate = nno._learning_rate_decay_rate;
@@ -156,6 +154,7 @@ class NeuralNetworkOptions
       _has_bias = nno._has_bias;
       _multi_output_layer_details = std::move(nno._multi_output_layer_details);
 
+      nno._progress_callback = nullptr;
       nno._log_level = Logger::LogLevel::None;
       nno._number_of_epoch = 0;
       nno._batch_size = 0;
@@ -171,6 +170,11 @@ class NeuralNetworkOptions
       nno._bptt_max_ticks = 0;
       nno._update_training_monitor_percent = 0.0;
       nno._has_bias = true;
+      nno._learning_rate_decay_rate = 0;
+      nno._adaptive_learning_rate = false;
+      nno._learning_rate_restart_rate = 0;
+      nno._learning_rate_restart_boost = 0;
+      nno._number_of_threads = 0;
     }
     return *this;
   }

src/neuralnetwork/neuron.cpp

@@ -46,6 +46,7 @@ Neuron::Neuron(Neuron&& src) noexcept :
 
   src._index = 0;
   src._type = Neuron::Type::Normal;
+  src._dropout_rate = 0.0;
 }
 
 Neuron& Neuron::operator=(Neuron&& src) noexcept

src/neuralnetwork/outputlayer.h

@@ -30,6 +30,7 @@ class OutputLayer
     _number_output_layers(src._number_output_layers)
   {
     MYODDWEB_PROFILE_FUNCTION("OutputLayer");
+    src._number_output_layers = 0;
   }
 
   OutputLayer& operator=(const OutputLayer& src) noexcept
@@ -52,6 +53,7 @@ class OutputLayer
       _output_layer_details = std::move(src._output_layer_details);
       _bounds = std::move(src._bounds);
       _number_output_layers = src._number_output_layers;
+      src._number_output_layers = 0;
     }
     return *this;
   }

src/neuralnetwork/outputlayerdetails.h

@@ -85,7 +85,9 @@ class OutputLayerDetails
       _error_evaluation_config = std::move(src._error_evaluation_config);
       _weight_decay = src._weight_decay;
       _optimiser_type = src._optimiser_type;
+      _momentum = src._momentum;
       src._weight_decay = 0;
+      src._momentum = 0;
       src._layer_size = 0;
     }
     return *this;