FeedForwardNeuralNetwork.h

//-------------------------------------------------------------------------------------------------------
// Copyright (C) Taylor Woll and panann contributors. All rights reserved.
// Licensed under the MIT license. See LICENSE.txt file in the project root for
// full license information.
//-------------------------------------------------------------------------------------------------------
 
#ifndef FEEDFORWARDNEURALNETWORK_H__
#define FEEDFORWARDNEURALNETWORK_H__
 
#include <vector>
 
#include "Perceptron.h"
 
namespace panann {
 
class TrainingData;
 
class FeedForwardNeuralNetwork : public Perceptron {
 public:
  enum class TrainingAlgorithmType : uint8_t {
    Backpropagation = 0,
    BatchingBackpropagation,
    QuickBackpropagation,
    ResilientBackpropagation,
    SimulatedAnnealingResilientBackpropagation
  };
 
  FeedForwardNeuralNetwork() = default;
  FeedForwardNeuralNetwork(const FeedForwardNeuralNetwork&) = delete;
  FeedForwardNeuralNetwork& operator=(const FeedForwardNeuralNetwork&) = delete;
  ~FeedForwardNeuralNetwork() override = default;
 
  void SetLearningRate(double learning_rate);
  double GetLearningRate() const;
 
  void SetMomentum(double momentum);
  double GetMomentum() const;
 
  void SetQpropMu(double mu);
  double GetQpropMu() const;
 
  void SetQpropWeightDecay(double weight_decay);
  double GetQpropWeightDecay() const;
 
  void SetRpropWeightStepInitial(double weight_step);
  double GetRpropWeightStepInitial() const;
 
  void SetRpropWeightStepMin(double weight_step);
  double GetRpropWeightStepMin() const;
 
  void SetRpropWeightStepMax(double weight_step);
  double GetRpropWeightStepMax() const;
 
  void SetRpropIncreaseFactor(double factor);
  double GetRpropIncreaseFactor() const;
 
  void SetRpropDecreaseFactor(double factor);
  double GetRpropDecreaseFactor() const;
 
  void SetSarpropWeightDecayShift(double k1);
  double GetSarpropWeightDecayShift() const;
 
  void SetSarpropStepThresholdFactor(double k2);
  double GetSarpropStepThresholdFactor() const;
 
  void SetSarpropStepShift(double k3);
  double GetSarpropStepShift() const;
 
  void SetSarpropTemperature(double t);
  double GetSarpropTemperature() const;
 
  void SetTrainingAlgorithmType(TrainingAlgorithmType type);
  TrainingAlgorithmType GetTrainingAlgorithmType() const;
 
  void Train(TrainingData* training_data, size_t epoch_count);
 
 protected:
  void UpdateSlopes();
  void UpdateWeightsOnline();
  void UpdateWeightsOffline(size_t current_epoch, size_t step_count);
  void UpdateWeightsBatchingBackpropagation(size_t step_count);
  void UpdateWeightsQuickBackpropagation(size_t step_count);
  void UpdateWeightsResilientBackpropagation();
  void UpdateWeightsSimulatedAnnealingResilientBackpropagation(
      size_t current_epoch);
 
  void ResetWeightSteps();
  void ResetSlopes();
  void ResetPreviousSlopes();
 
  void TrainOffline(TrainingData* training_data, size_t epoch_count);
  void TrainOnline(TrainingData* training_data, size_t epoch_count);
 
 private:
  static constexpr double DefaultLearningRate = 0.7;
  static constexpr double DefaultMomentum = 0.1;
  static constexpr double DefaultQpropMu = 1.75;
  static constexpr double DefaultQpropWeightDecay = -0.0001;
  static constexpr double DefaultRpropWeightStepInitial = 0.0125;
  static constexpr double DefaultRpropWeightStepMin = 0.000001;
  static constexpr double DefaultRpropWeightStepMax = 50;
  static constexpr double DefaultRpropIncreaseFactor = 1.2;
  static constexpr double DefaultRpropDecreaseFactor = 0.5;
  static constexpr double DefaultSarpropWeightDecayShift = 0.01;
  static constexpr double DefaultSarpropStepThresholdFactor = 0.1;
  static constexpr double DefaultSarpropStepShift = 3;
  static constexpr double DefaultSarpropTemperature = 0.015;
 
  std::vector<double> previous_weight_steps_;
  std::vector<double> slopes_;
  std::vector<double> previous_slopes_;
 
  double learning_rate_ = DefaultLearningRate;
  double momentum_ = DefaultMomentum;
  double qprop_mu_ = DefaultQpropMu;
  double qprop_weight_decay_ = DefaultQpropWeightDecay;
  double rprop_weight_step_initial_ = DefaultRpropWeightStepInitial;
  double rprop_weight_step_min_ = DefaultRpropWeightStepMin;
  double rprop_weight_step_max_ = DefaultRpropWeightStepMax;
  double rprop_increase_factor_ = DefaultRpropIncreaseFactor;
  double rprop_decrease_factor_ = DefaultRpropDecreaseFactor;
  double sarprop_weight_decay_shift_ = DefaultSarpropWeightDecayShift;
  double sarprop_step_threshold_factor_ = DefaultSarpropStepThresholdFactor;
  double sarprop_step_shift_ = DefaultSarpropStepShift;
  double sarprop_temperature_ = DefaultSarpropTemperature;
 
  TrainingAlgorithmType training_algorithm_type_ =
      TrainingAlgorithmType::ResilientBackpropagation;
};
 
}  // namespace panann
 
#endif  // FEEDFORWARDNEURALNETWORK_H__