#include <FeedForwardNeuralNetwork.h>

Inheritance diagram for panann::FeedForwardNeuralNetwork:

[legend]

Collaboration diagram for panann::FeedForwardNeuralNetwork:

[legend]

Public Types
enum	TrainingAlgorithmType : uint8_t { TrainingAlgorithmType::Backpropagation = 0, TrainingAlgorithmType::BatchingBackpropagation, TrainingAlgorithmType::QuickBackpropagation, TrainingAlgorithmType::ResilientBackpropagation, TrainingAlgorithmType::SimulatedAnnealingResilientBackpropagation }

Public Types inherited from panann::Perceptron
enum	ErrorCostFunction : uint8_t { MeanSquareError = 1, MeanAbsoluteError }

Public Member Functions
	FeedForwardNeuralNetwork (const FeedForwardNeuralNetwork &)=delete

FeedForwardNeuralNetwork &	operator= (const FeedForwardNeuralNetwork &)=delete

void	SetLearningRate (double learning_rate)

double	GetLearningRate () const

void	SetMomentum (double momentum)

double	GetMomentum () const

void	SetQpropMu (double mu)

double	GetQpropMu () const

void	SetQpropWeightDecay (double weight_decay)

double	GetQpropWeightDecay () const

void	SetRpropWeightStepInitial (double weight_step)

double	GetRpropWeightStepInitial () const

void	SetRpropWeightStepMin (double weight_step)

double	GetRpropWeightStepMin () const

void	SetRpropWeightStepMax (double weight_step)

double	GetRpropWeightStepMax () const

void	SetRpropIncreaseFactor (double factor)

double	GetRpropIncreaseFactor () const

void	SetRpropDecreaseFactor (double factor)

double	GetRpropDecreaseFactor () const

void	SetSarpropWeightDecayShift (double k1)

double	GetSarpropWeightDecayShift () const

void	SetSarpropStepThresholdFactor (double k2)

double	GetSarpropStepThresholdFactor () const

void	SetSarpropStepShift (double k3)

double	GetSarpropStepShift () const

void	SetSarpropTemperature (double t)

double	GetSarpropTemperature () const

void	SetTrainingAlgorithmType (TrainingAlgorithmType type)

TrainingAlgorithmType	GetTrainingAlgorithmType () const

void	Train (TrainingData *training_data, size_t epoch_count)

Public Member Functions inherited from panann::Perceptron
	Perceptron (const Perceptron &)=delete

Perceptron &	operator= (const Perceptron &)=delete

void	SetErrorCostFunction (ErrorCostFunction mode)

ErrorCostFunction	GetErrorCostFunction () const

void	SetHiddenNeuronActivationFunctionType (ActivationFunctionType type)

ActivationFunctionType	GetHiddenNeuronActivationFunctionType () const

void	SetOutputNeuronActivationFunctionType (ActivationFunctionType type)

ActivationFunctionType	GetOutputNeuronActivationFunctionType () const

void	InitializeWeightsRandom (double min=-1.0, double max=1.0)

void	InitializeWeights (const TrainingData &training_data)

virtual void	RunForward (const std::vector< double > &input)

void	RunBackward (const std::vector< double > &output)

double	GetError (const std::vector< double > &output)

double	GetError (const TrainingData &training_data)

std::vector< double > &	GetWeights ()

void	SetWeights (const std::vector< double > &weights)

void	GetOutput (std::vector< double > *output) const

virtual void	Construct ()

bool	IsConstructed () const

void	EnableErrorShaping ()

void	DisableErrorShaping ()

Public Member Functions inherited from panann::MultiLayerNeuralTopology
	MultiLayerNeuralTopology (const MultiLayerNeuralTopology &)=delete

MultiLayerNeuralTopology &	operator= (const MultiLayerNeuralTopology &)=delete

void	AddHiddenLayer (size_t neuron_count)

void	EnableShortcutConnections ()

void	DisableShortcutConnections ()

void	ConstructTopology ()

bool	IsTopologyConstructed () const

size_t	GetHiddenLayerCount () const

const Layer &	GetHiddenLayer (size_t layer_index) const

size_t	GetInputConnectionCount () const

size_t	GetOutputConnectionCount () const

const InputConnection &	GetInputConnection (size_t index) const

const OutputConnection &	GetOutputConnection (size_t index) const

Public Member Functions inherited from panann::NeuronContainer
	NeuronContainer (const NeuronContainer &)=delete

NeuronContainer &	operator= (const NeuronContainer &)=delete

void	SetInputNeuronCount (size_t input_neuron_count)

size_t	GetInputNeuronCount () const

void	SetOutputNeuronCount (size_t output_neuron_count)

size_t	GetOutputNeuronCount () const

size_t	GetNeuronCount () const

const Neuron &	GetNeuron (size_t neuron_index) const

Protected Member Functions
void	UpdateSlopes ()

void	UpdateWeightsOnline ()

void	UpdateWeightsOffline (size_t current_epoch, size_t step_count)

void	UpdateWeightsBatchingBackpropagation (size_t step_count)

void	UpdateWeightsQuickBackpropagation (size_t step_count)

void	UpdateWeightsResilientBackpropagation ()

void	UpdateWeightsSimulatedAnnealingResilientBackpropagation (size_t current_epoch)

void	ResetWeightSteps ()

void	ResetSlopes ()

void	ResetPreviousSlopes ()

void	TrainOffline (TrainingData *training_data, size_t epoch_count)

void	TrainOnline (TrainingData *training_data, size_t epoch_count)

Protected Member Functions inherited from panann::Perceptron
void	AllocateWeights ()

bool	AreWeightsAllocated () const

void	InitializeNeurons ()

virtual void	InitializeHiddenNeurons ()

void	ComputeNeuronValue (size_t neuron_index)

void	ComputeNeuronValueRange (size_t neuron_start_index, size_t neuron_count)

void	ComputeNeuronError (size_t neuron_index)

void	ResetOutputLayerError ()

void	CalculateOutputLayerError (const std::vector< double > &output)

double	GetError () const

RandomWrapper &	GetRandom ()

size_t	GetWeightCount () const

double &	GetWeight (size_t index)

Protected Member Functions inherited from panann::MultiLayerNeuralTopology
Layer &	GetHiddenLayer (size_t layer_index)

InputConnection &	GetInputConnection (size_t index)

OutputConnection &	GetOutputConnection (size_t index)

size_t	AddInputConnections (size_t count)

size_t	AddOutputConnections (size_t count)

virtual void	FixNeuronConnectionIndices ()

void	AllocateConnections ()

bool	AreConnectionsAllocated () const

virtual void	ConnectFully ()

void	ConnectLayerToNeuron (size_t from_neuron_index, size_t from_neuron_count, size_t to_neuron_index)

void	ConnectLayers (size_t from_neuron_index, size_t from_neuron_count, size_t to_neuron_index, size_t to_neuron_count)

void	ConnectBiasNeuron (size_t bias_neuron_index, size_t to_neuron_index, size_t to_neuron_count)

void	ConnectNeurons (size_t from_neuron_index, size_t to_neuron_index)

Protected Member Functions inherited from panann::NeuronContainer
size_t	GetHiddenNeuronStartIndex () const

size_t	GetInputNeuronStartIndex () const

size_t	GetOutputNeuronStartIndex () const

size_t	GetBiasNeuronStartIndex () const

size_t	GetHiddenNeuronCount () const

size_t	GetBiasNeuronCount () const

size_t	AddHiddenNeurons (size_t count)

void	AddBiasNeurons (size_t count)

void	AllocateNeurons ()

bool	AreNeuronsAllocated () const

void	SetNeuronActivationFunction (size_t neuron_index, ActivationFunctionType type)

Neuron &	GetNeuron (size_t neuron_index)

Neuron &	GetInputNeuron (size_t input_neuron_index)

Neuron &	GetOutputNeuron (size_t output_neuron_index)

Neuron &	GetBiasNeuron (size_t bias_neuron_index)

Neuron &	GetHiddenNeuron (size_t hidden_neuron_index)

const Neuron &	GetOutputNeuron (size_t output_neuron_index) const

Detailed Description

A feed-forward neural network supporting training via multiple algorithms.

Member Enumeration Documentation

◆ TrainingAlgorithmType

enum panann::FeedForwardNeuralNetwork::TrainingAlgorithmType : uint8_t

strong

Enumerator
Backpropagation	Backpropagation with momentum and learning rate parameters. This is an online learning algorithm and does not perform batching weight updates. See also SetLearningRate SetMomentum
BatchingBackpropagation	Batching backpropagation with learning rate parameter. This is an offline learning algorithm. It batches together weight updates and modifies the weights at the end of each epoch. See also SetLearningRate
QuickBackpropagation	An implementation of quickprop. It uses the learning rate, mu, and qprop weight decay parameters. This is an offline learning algorithm. See also SetLearningRate SetQpropMu SetQpropWeightDecay
ResilientBackpropagation	Resilient backprop is a very fast training algorithm designed to move quickly down the error curve when the derivative of the partial error doesn't change. The implementation here is of iRPROP-. It is an offline learning algorithm. See also SetRpropWeightStepInitial SetRpropWeightStepMin SetRpropWeightStepMax SetRpropIncreaseFactor SetRpropDecreaseFactor
SimulatedAnnealingResilientBackpropagation	Simulated annealing and weight decay are added to resilient backprop. Attempts to avoid getting stuck in local minima on the error surface while training the network. Uses all of the rprop parameters plus some specific to sarprop. See also SetRpropWeightStepInitial SetRpropWeightStepMin SetRpropWeightStepMax SetRpropIncreaseFactor SetRpropDecreaseFactor SetSarpropWeightDecayShift SetSarpropStepThresholdFactor SetSarpropStepShift SetSarpropTemperature

Member Function Documentation

◆ SetLearningRate()

void panann::FeedForwardNeuralNetwork::SetLearningRate ( double learning_rate )

Set the learning rate parameter used by backprop, batch, and qprop.
Default: 0.7

◆ SetMomentum()

void panann::FeedForwardNeuralNetwork::SetMomentum ( double momentum )

Set the momentum parameter used by backprop.
Default: 0.1

◆ SetQpropMu()

void panann::FeedForwardNeuralNetwork::SetQpropMu ( double mu )

Set the Mu parameter for qprop.
Default: 1.75

◆ SetQpropWeightDecay()

void panann::FeedForwardNeuralNetwork::SetQpropWeightDecay ( double weight_decay )

Set the weight decay parameter for qprop.
Default: -0.0001

◆ SetRpropDecreaseFactor()

void panann::FeedForwardNeuralNetwork::SetRpropDecreaseFactor ( double factor )

Set the factor by which the weight step will be decreased when the sign of the partial derivative of the error changes.
This is sometimes called eta-.
Both rprop and sarprop use this paramter for the same purpose.
A lower value will make training slower and a higher value can make training unstable.
Default: 0.5

◆ SetRpropIncreaseFactor()

void panann::FeedForwardNeuralNetwork::SetRpropIncreaseFactor ( double factor )

Set the factor by which the weight step will be increased each training step when the sign of the partial derivative of the error does not change.
This is sometimes referred to as eta+.
Both rprop and sarprop use this paramter for the same purpose.
A higher value can increase the speed of convergeance but can also make training unstable.
Default: 1.2

◆ SetRpropWeightStepInitial()

void panann::FeedForwardNeuralNetwork::SetRpropWeightStepInitial ( double weight_step )

Set the initial weight step value for rprop and sarprop.
This is sometimes called delta_zero.
Default: 0.0125

◆ SetRpropWeightStepMax()

void panann::FeedForwardNeuralNetwork::SetRpropWeightStepMax ( double weight_step )

Set the maximum weight step value for rprop and sarprop.
This is sometimes called delta_max.
If the weight change becomes too large, learning will be chaotic so we clamp it to some reasonably large number to ensure smooth training.
Default: 50

◆ SetRpropWeightStepMin()

void panann::FeedForwardNeuralNetwork::SetRpropWeightStepMin ( double weight_step )

Set the minimum weight step value for rprop and sarprop.
This is sometimes called delta_min.
If the weight change becomes 0, learning will stop so we need to use some small value instead of zero.
Default: 0.000001

◆ SetSarpropStepShift()

void panann::FeedForwardNeuralNetwork::SetSarpropStepShift ( double k3 )

Set the step shift parameter used by sarprop.
It is called k3 in some formulations of sarprop.
Default: 3

◆ SetSarpropStepThresholdFactor()

void panann::FeedForwardNeuralNetwork::SetSarpropStepThresholdFactor ( double k2 )

Set the weight step threshold factor parameter used by sarprop.
It is called k2 in some formulations of sarprop.
Default: 0.1

◆ SetSarpropTemperature()

void panann::FeedForwardNeuralNetwork::SetSarpropTemperature ( double t )

Set the temperature parameter used by sarprop.
This is referred to as T in most sarprop formulations.
Default: 0.015

◆ SetSarpropWeightDecayShift()

void panann::FeedForwardNeuralNetwork::SetSarpropWeightDecayShift ( double k1 )

Set the weight decay shift parameter used by sarprop.
This is a constant used as a weight decay.
It is called k1 in some formulations of sarprop.
Default: 0.01

◆ SetTrainingAlgorithmType()

void panann::FeedForwardNeuralNetwork::SetTrainingAlgorithmType ( TrainingAlgorithmType type )

Set the training algorithm this network will use during training.
Default: ResilientBackpropagation

See also: TrainingAlgorithmType

◆ Train()

void panann::FeedForwardNeuralNetwork::Train	(	TrainingData *	training_data,
		size_t	epoch_count
	)

Use the training algorithm to train the network.
Training follows these steps:

For each example in the training data
Run the network forward on the example input
Calculate the total error by comparing output neuron values against the example output
Calculate the partial error contributed by each weight in the network

Update all the weights in the network to reduce the total error
Execute the above once for each epoch.
The actual method by which we will update the weights depends on the training algorithm chosen.

Parameters

training_data	Examples on which we will train the network. Note: Each epoch, shuffles the order of examples in training_data before performing the training operation.
epoch_count	The number of epochs we should execute to train the network. One epoch is one full step through all of the training examples.

See also: SetTrainingAlgorithmType; TrainingAlgorithmType; TrainingData

The documentation for this class was generated from the following files:

src/FeedForwardNeuralNetwork.h
src/FeedForwardNeuralNetwork.cc

Public Types

Public Member Functions

Protected Member Functions