panann
All Classes Functions Variables Enumerations Enumerator
Public Types | Public Member Functions | Protected Member Functions | List of all members
panann::FeedForwardNeuralNetwork Class Reference

#include <FeedForwardNeuralNetwork.h>

Inheritance diagram for panann::FeedForwardNeuralNetwork:
Inheritance graph
[legend]
Collaboration diagram for panann::FeedForwardNeuralNetwork:
Collaboration graph
[legend]

Public Types

enum  TrainingAlgorithmType : uint8_t {
  TrainingAlgorithmType::Backpropagation = 0, TrainingAlgorithmType::BatchingBackpropagation, TrainingAlgorithmType::QuickBackpropagation, TrainingAlgorithmType::ResilientBackpropagation,
  TrainingAlgorithmType::SimulatedAnnealingResilientBackpropagation
}
 
- Public Types inherited from panann::Perceptron
enum  ErrorCostFunction : uint8_t { MeanSquareError = 1, MeanAbsoluteError }
 

Public Member Functions

 FeedForwardNeuralNetwork (const FeedForwardNeuralNetwork &)=delete
 
FeedForwardNeuralNetworkoperator= (const FeedForwardNeuralNetwork &)=delete
 
void SetLearningRate (double learning_rate)
 
double GetLearningRate () const
 
void SetMomentum (double momentum)
 
double GetMomentum () const
 
void SetQpropMu (double mu)
 
double GetQpropMu () const
 
void SetQpropWeightDecay (double weight_decay)
 
double GetQpropWeightDecay () const
 
void SetRpropWeightStepInitial (double weight_step)
 
double GetRpropWeightStepInitial () const
 
void SetRpropWeightStepMin (double weight_step)
 
double GetRpropWeightStepMin () const
 
void SetRpropWeightStepMax (double weight_step)
 
double GetRpropWeightStepMax () const
 
void SetRpropIncreaseFactor (double factor)
 
double GetRpropIncreaseFactor () const
 
void SetRpropDecreaseFactor (double factor)
 
double GetRpropDecreaseFactor () const
 
void SetSarpropWeightDecayShift (double k1)
 
double GetSarpropWeightDecayShift () const
 
void SetSarpropStepThresholdFactor (double k2)
 
double GetSarpropStepThresholdFactor () const
 
void SetSarpropStepShift (double k3)
 
double GetSarpropStepShift () const
 
void SetSarpropTemperature (double t)
 
double GetSarpropTemperature () const
 
void SetTrainingAlgorithmType (TrainingAlgorithmType type)
 
TrainingAlgorithmType GetTrainingAlgorithmType () const
 
void Train (TrainingData *training_data, size_t epoch_count)
 
- Public Member Functions inherited from panann::Perceptron
 Perceptron (const Perceptron &)=delete
 
Perceptronoperator= (const Perceptron &)=delete
 
void SetErrorCostFunction (ErrorCostFunction mode)
 
ErrorCostFunction GetErrorCostFunction () const
 
void SetHiddenNeuronActivationFunctionType (ActivationFunctionType type)
 
ActivationFunctionType GetHiddenNeuronActivationFunctionType () const
 
void SetOutputNeuronActivationFunctionType (ActivationFunctionType type)
 
ActivationFunctionType GetOutputNeuronActivationFunctionType () const
 
void InitializeWeightsRandom (double min=-1.0, double max=1.0)
 
void InitializeWeights (const TrainingData &training_data)
 
virtual void RunForward (const std::vector< double > &input)
 
void RunBackward (const std::vector< double > &output)
 
double GetError (const std::vector< double > &output)
 
double GetError (const TrainingData &training_data)
 
std::vector< double > & GetWeights ()
 
void SetWeights (const std::vector< double > &weights)
 
void GetOutput (std::vector< double > *output) const
 
virtual void Construct ()
 
bool IsConstructed () const
 
void EnableErrorShaping ()
 
void DisableErrorShaping ()
 
- Public Member Functions inherited from panann::MultiLayerNeuralTopology
 MultiLayerNeuralTopology (const MultiLayerNeuralTopology &)=delete
 
MultiLayerNeuralTopologyoperator= (const MultiLayerNeuralTopology &)=delete
 
void AddHiddenLayer (size_t neuron_count)
 
void EnableShortcutConnections ()
 
void DisableShortcutConnections ()
 
void ConstructTopology ()
 
bool IsTopologyConstructed () const
 
size_t GetHiddenLayerCount () const
 
const LayerGetHiddenLayer (size_t layer_index) const
 
size_t GetInputConnectionCount () const
 
size_t GetOutputConnectionCount () const
 
const InputConnectionGetInputConnection (size_t index) const
 
const OutputConnectionGetOutputConnection (size_t index) const
 
- Public Member Functions inherited from panann::NeuronContainer
 NeuronContainer (const NeuronContainer &)=delete
 
NeuronContaineroperator= (const NeuronContainer &)=delete
 
void SetInputNeuronCount (size_t input_neuron_count)
 
size_t GetInputNeuronCount () const
 
void SetOutputNeuronCount (size_t output_neuron_count)
 
size_t GetOutputNeuronCount () const
 
size_t GetNeuronCount () const
 
const NeuronGetNeuron (size_t neuron_index) const
 

Protected Member Functions

void UpdateSlopes ()
 
void UpdateWeightsOnline ()
 
void UpdateWeightsOffline (size_t current_epoch, size_t step_count)
 
void UpdateWeightsBatchingBackpropagation (size_t step_count)
 
void UpdateWeightsQuickBackpropagation (size_t step_count)
 
void UpdateWeightsResilientBackpropagation ()
 
void UpdateWeightsSimulatedAnnealingResilientBackpropagation (size_t current_epoch)
 
void ResetWeightSteps ()
 
void ResetSlopes ()
 
void ResetPreviousSlopes ()
 
void TrainOffline (TrainingData *training_data, size_t epoch_count)
 
void TrainOnline (TrainingData *training_data, size_t epoch_count)
 
- Protected Member Functions inherited from panann::Perceptron
void AllocateWeights ()
 
bool AreWeightsAllocated () const
 
void InitializeNeurons ()
 
virtual void InitializeHiddenNeurons ()
 
void ComputeNeuronValue (size_t neuron_index)
 
void ComputeNeuronValueRange (size_t neuron_start_index, size_t neuron_count)
 
void ComputeNeuronError (size_t neuron_index)
 
void ResetOutputLayerError ()
 
void CalculateOutputLayerError (const std::vector< double > &output)
 
double GetError () const
 
RandomWrapperGetRandom ()
 
size_t GetWeightCount () const
 
double & GetWeight (size_t index)
 
- Protected Member Functions inherited from panann::MultiLayerNeuralTopology
LayerGetHiddenLayer (size_t layer_index)
 
InputConnectionGetInputConnection (size_t index)
 
OutputConnectionGetOutputConnection (size_t index)
 
size_t AddInputConnections (size_t count)
 
size_t AddOutputConnections (size_t count)
 
virtual void FixNeuronConnectionIndices ()
 
void AllocateConnections ()
 
bool AreConnectionsAllocated () const
 
virtual void ConnectFully ()
 
void ConnectLayerToNeuron (size_t from_neuron_index, size_t from_neuron_count, size_t to_neuron_index)
 
void ConnectLayers (size_t from_neuron_index, size_t from_neuron_count, size_t to_neuron_index, size_t to_neuron_count)
 
void ConnectBiasNeuron (size_t bias_neuron_index, size_t to_neuron_index, size_t to_neuron_count)
 
void ConnectNeurons (size_t from_neuron_index, size_t to_neuron_index)
 
- Protected Member Functions inherited from panann::NeuronContainer
size_t GetHiddenNeuronStartIndex () const
 
size_t GetInputNeuronStartIndex () const
 
size_t GetOutputNeuronStartIndex () const
 
size_t GetBiasNeuronStartIndex () const
 
size_t GetHiddenNeuronCount () const
 
size_t GetBiasNeuronCount () const
 
size_t AddHiddenNeurons (size_t count)
 
void AddBiasNeurons (size_t count)
 
void AllocateNeurons ()
 
bool AreNeuronsAllocated () const
 
void SetNeuronActivationFunction (size_t neuron_index, ActivationFunctionType type)
 
NeuronGetNeuron (size_t neuron_index)
 
NeuronGetInputNeuron (size_t input_neuron_index)
 
NeuronGetOutputNeuron (size_t output_neuron_index)
 
NeuronGetBiasNeuron (size_t bias_neuron_index)
 
NeuronGetHiddenNeuron (size_t hidden_neuron_index)
 
const NeuronGetOutputNeuron (size_t output_neuron_index) const
 

Detailed Description

A feed-forward neural network supporting training via multiple algorithms.

Member Enumeration Documentation

◆ TrainingAlgorithmType

Enumerator
Backpropagation 

Backpropagation with momentum and learning rate parameters.
This is an online learning algorithm and does not perform batching weight updates.

See also
SetLearningRate
SetMomentum
BatchingBackpropagation 

Batching backpropagation with learning rate parameter.
This is an offline learning algorithm. It batches together weight updates and modifies the weights at the end of each epoch.

See also
SetLearningRate
QuickBackpropagation 

An implementation of quickprop.
It uses the learning rate, mu, and qprop weight decay parameters.
This is an offline learning algorithm.

See also
SetLearningRate
SetQpropMu
SetQpropWeightDecay
ResilientBackpropagation 

Resilient backprop is a very fast training algorithm designed to move quickly down the error curve when the derivative of the partial error doesn't change.
The implementation here is of iRPROP-. It is an offline learning algorithm.

See also
SetRpropWeightStepInitial
SetRpropWeightStepMin
SetRpropWeightStepMax
SetRpropIncreaseFactor
SetRpropDecreaseFactor
SimulatedAnnealingResilientBackpropagation 

Simulated annealing and weight decay are added to resilient backprop.
Attempts to avoid getting stuck in local minima on the error surface while training the network.
Uses all of the rprop parameters plus some specific to sarprop.

See also
SetRpropWeightStepInitial
SetRpropWeightStepMin
SetRpropWeightStepMax
SetRpropIncreaseFactor
SetRpropDecreaseFactor
SetSarpropWeightDecayShift
SetSarpropStepThresholdFactor
SetSarpropStepShift
SetSarpropTemperature

Member Function Documentation

◆ SetLearningRate()

void panann::FeedForwardNeuralNetwork::SetLearningRate ( double  learning_rate)

Set the learning rate parameter used by backprop, batch, and qprop.
Default: 0.7

◆ SetMomentum()

void panann::FeedForwardNeuralNetwork::SetMomentum ( double  momentum)

Set the momentum parameter used by backprop.
Default: 0.1

◆ SetQpropMu()

void panann::FeedForwardNeuralNetwork::SetQpropMu ( double  mu)

Set the Mu parameter for qprop.
Default: 1.75

◆ SetQpropWeightDecay()

void panann::FeedForwardNeuralNetwork::SetQpropWeightDecay ( double  weight_decay)

Set the weight decay parameter for qprop.
Default: -0.0001

◆ SetRpropDecreaseFactor()

void panann::FeedForwardNeuralNetwork::SetRpropDecreaseFactor ( double  factor)

Set the factor by which the weight step will be decreased when the sign of the partial derivative of the error changes.
This is sometimes called eta-.
Both rprop and sarprop use this paramter for the same purpose.
A lower value will make training slower and a higher value can make training unstable.
Default: 0.5

◆ SetRpropIncreaseFactor()

void panann::FeedForwardNeuralNetwork::SetRpropIncreaseFactor ( double  factor)

Set the factor by which the weight step will be increased each training step when the sign of the partial derivative of the error does not change.
This is sometimes referred to as eta+.
Both rprop and sarprop use this paramter for the same purpose.
A higher value can increase the speed of convergeance but can also make training unstable.
Default: 1.2

◆ SetRpropWeightStepInitial()

void panann::FeedForwardNeuralNetwork::SetRpropWeightStepInitial ( double  weight_step)

Set the initial weight step value for rprop and sarprop.
This is sometimes called delta_zero.
Default: 0.0125

◆ SetRpropWeightStepMax()

void panann::FeedForwardNeuralNetwork::SetRpropWeightStepMax ( double  weight_step)

Set the maximum weight step value for rprop and sarprop.
This is sometimes called delta_max.
If the weight change becomes too large, learning will be chaotic so we clamp it to some reasonably large number to ensure smooth training.
Default: 50

◆ SetRpropWeightStepMin()

void panann::FeedForwardNeuralNetwork::SetRpropWeightStepMin ( double  weight_step)

Set the minimum weight step value for rprop and sarprop.
This is sometimes called delta_min.
If the weight change becomes 0, learning will stop so we need to use some small value instead of zero.
Default: 0.000001

◆ SetSarpropStepShift()

void panann::FeedForwardNeuralNetwork::SetSarpropStepShift ( double  k3)

Set the step shift parameter used by sarprop.
It is called k3 in some formulations of sarprop.
Default: 3

◆ SetSarpropStepThresholdFactor()

void panann::FeedForwardNeuralNetwork::SetSarpropStepThresholdFactor ( double  k2)

Set the weight step threshold factor parameter used by sarprop.
It is called k2 in some formulations of sarprop.
Default: 0.1

◆ SetSarpropTemperature()

void panann::FeedForwardNeuralNetwork::SetSarpropTemperature ( double  t)

Set the temperature parameter used by sarprop.
This is referred to as T in most sarprop formulations.
Default: 0.015

◆ SetSarpropWeightDecayShift()

void panann::FeedForwardNeuralNetwork::SetSarpropWeightDecayShift ( double  k1)

Set the weight decay shift parameter used by sarprop.
This is a constant used as a weight decay.
It is called k1 in some formulations of sarprop.
Default: 0.01

◆ SetTrainingAlgorithmType()

void panann::FeedForwardNeuralNetwork::SetTrainingAlgorithmType ( TrainingAlgorithmType  type)

Set the training algorithm this network will use during training.
Default: ResilientBackpropagation

See also
TrainingAlgorithmType

◆ Train()

void panann::FeedForwardNeuralNetwork::Train ( TrainingData training_data,
size_t  epoch_count 
)

Use the training algorithm to train the network.
Training follows these steps:

  • For each example in the training data
  • Run the network forward on the example input
  • Calculate the total error by comparing output neuron values against the example output
  • Calculate the partial error contributed by each weight in the network
  • Update all the weights in the network to reduce the total error
    Execute the above once for each epoch.
    The actual method by which we will update the weights depends on the training algorithm chosen.
    Parameters
    training_dataExamples on which we will train the network.
    Note: Each epoch, shuffles the order of examples in training_data before performing the training operation.
    epoch_countThe number of epochs we should execute to train the network. One epoch is one full step through all of the training examples.
    See also
    SetTrainingAlgorithmType
    TrainingAlgorithmType
    TrainingData

The documentation for this class was generated from the following files: