850 lines
32 KiB
C#
850 lines
32 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using UnityEngine;
|
|
#if UNITY_MATHEMATICS
|
|
using Unity.Mathematics;
|
|
using static Unity.Mathematics.math;
|
|
#endif
|
|
|
|
namespace NanoBrain {
|
|
|
|
/// <summary>
|
|
/// A neuron is a basic Nucleus
|
|
/// </summary>
|
|
/// A neuron combines the weighted input from other neurons and applies an activation function to it
|
|
/// to compute the output value:
|
|
/// \code
|
|
/// Vector3 combination = NanoBrain::Neuron::Combinator(bias, synapses);
|
|
/// Vector3 output = NanoBrain::Neuron::Activator(combination);
|
|
/// \endcode
|
|
/// The synapses are connections to other neurons.
|
|
/// Each connection has a weight which is used to multiply the output of that other neuron
|
|
/// before it is used by the combinator.
|
|
[Serializable]
|
|
public class Neuron : Nucleus {
|
|
|
|
/// <summary>
|
|
/// Create a new Neuron in a Cluster instance
|
|
/// </summary>
|
|
/// <param name="parent">The parent cluster in which the new Neuron should be created</param>
|
|
/// <param name="name">The name of the new Neuron</param>
|
|
public Neuron(Cluster parent, string name) {
|
|
this.parent = parent;
|
|
this.name = name;
|
|
if (this.parent != null) {
|
|
this.parent.nuclei ??= new();
|
|
this.parent.nuclei.Add(this);
|
|
}
|
|
this.resetStimulus = ResetStimulus;
|
|
}
|
|
|
|
#region Serialization
|
|
|
|
/// <summary>
|
|
/// The bias
|
|
/// </summary>
|
|
/// The bias which a value which is always added to the combined value of the neuron
|
|
/// It does not have a synapse and therefore no weight of source nucleus
|
|
//[HideInInspector]
|
|
public Vector3 bias = Vector3.zero;
|
|
|
|
/// <summary>
|
|
/// Indicator whether the bias can be trained
|
|
/// </summary>
|
|
public bool trainableBias = false;
|
|
|
|
#region Synapses
|
|
|
|
[SerializeField]
|
|
private List<Synapse> _synapses = new();
|
|
/// <summary>
|
|
/// The synapses of the nucleus
|
|
/// </summary>
|
|
public List<Synapse> synapses => _synapses;
|
|
|
|
/// <summary>
|
|
/// Add a new synapse to this nuclues
|
|
/// </summary>
|
|
/// <param name="sendingNucleus">The nucleus from which the signals may originate</param>
|
|
/// <param name="weight">The weight applied to the input. Default value = 1</param>
|
|
/// <returns>The created Synapse</returns>
|
|
/// This will add a new input to this nucleus with the given weight.
|
|
public Synapse AddSynapse(Neuron sendingNucleus, float weight = 1, bool trainable = false) {
|
|
Synapse synapse = new(sendingNucleus, weight) {
|
|
trainable = trainable
|
|
};
|
|
this.synapses.Add(synapse);
|
|
return synapse;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Find a synapse
|
|
/// </summary>
|
|
/// <param name="sender">The sender of the input to the Synapse</param>
|
|
/// <returns>The found Synapse or null when the sender has no synapse to this nucleus.</returns>
|
|
public Synapse GetSynapse(Nucleus sender) {
|
|
foreach (Synapse synapse in this.synapses)
|
|
if (synapse.neuron == sender)
|
|
return synapse;
|
|
return null;
|
|
}
|
|
|
|
public Synapse GetSynapse(string senderNeuronName) {
|
|
Neuron sender = this.parent.GetNeuron(senderNeuronName);
|
|
if (sender == null)
|
|
return null;
|
|
|
|
return this.GetSynapse(sender);
|
|
}
|
|
public Synapse GetSynapse(Cluster cluster, string senderNeuronName) {
|
|
Neuron sender = cluster.GetNeuron(senderNeuronName);
|
|
if (sender == null)
|
|
return null;
|
|
return this.GetSynapse(sender);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Remove a synapse from a Nucleus
|
|
/// </summary>
|
|
/// <param name="sendingNucleus">Remote the synapse connecting to this Nucleus</param>
|
|
public void RemoveSynapse(Nucleus sendingNucleus) {
|
|
this.synapses.RemoveAll(synapse => synapse.neuron == sendingNucleus);
|
|
}
|
|
|
|
#endregion Synapses
|
|
|
|
/// <summary>
|
|
/// Set the bias, recalculate the output and update all Nuclei receiving from this Nucleus
|
|
/// </summary>
|
|
/// <param name="inputValue"></param>
|
|
public virtual void SetBias(Vector3 inputValue) {
|
|
this.bias = inputValue;
|
|
this.lastUpdate = Time.time;
|
|
this.parent?.UpdateFromNucleus(this);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The type of combinators
|
|
/// </summary>
|
|
/// A combinator combines the weighted values of the synapses to a single value
|
|
public enum CombinatorType {
|
|
/// Add the weighted values together
|
|
Sum,
|
|
/// Multiply the weighted values
|
|
Product,
|
|
}
|
|
/// <summary>
|
|
/// The type of combinator used for this Neuron
|
|
/// </summary>
|
|
[HideInInspector]
|
|
public CombinatorType combinator = CombinatorType.Sum;
|
|
|
|
/// <summary>
|
|
/// The type of
|
|
/// </summary>
|
|
public enum ActivationType {
|
|
Linear,
|
|
Power,
|
|
Sqrt,
|
|
Reciprocal,
|
|
Tanh,
|
|
Binary,
|
|
Normalized,
|
|
Custom
|
|
}
|
|
/// <summary>
|
|
/// The activation function
|
|
/// </summary>
|
|
[SerializeField]
|
|
[HideInInspector]
|
|
public ActivationType _activator;
|
|
/// <summary>
|
|
/// The activation funtion
|
|
/// </summary>
|
|
public ActivationType activator {
|
|
get { return _activator; }
|
|
set {
|
|
_activator = value;
|
|
//this.curve = GenerateCurve();
|
|
}
|
|
}
|
|
|
|
#endregion Serialization
|
|
|
|
#if UNITY_MATHEMATICS
|
|
|
|
/// <summary>
|
|
/// The output value of the neuron
|
|
/// </summary>
|
|
[HideInInspector]
|
|
protected float3 _outputValue;
|
|
/// <summary>
|
|
/// The output value of the neuron
|
|
/// </summary>
|
|
public virtual float3 outputValue {
|
|
get {
|
|
this.parent.CheckActions();
|
|
return _outputValue;
|
|
}
|
|
set {
|
|
_outputValue = value;
|
|
if (this.isFiring)
|
|
WhenFiring?.Invoke();
|
|
else
|
|
WhenNotFiring?.Invoke();
|
|
}
|
|
}
|
|
public float3 activation => outputValue;
|
|
/// <summary>
|
|
/// The magnitude of the neuron output
|
|
/// </summary>
|
|
public float outputMagnitude => length(_outputValue);
|
|
/// <summary>
|
|
/// The squared magnitude of the neuron output
|
|
/// </summary>
|
|
public float outputSqrMagnitude => lengthsq(_outputValue);
|
|
|
|
#else
|
|
|
|
/// <summary>
|
|
/// The output value of the neuron
|
|
/// </summary>
|
|
protected Vector3 _outputValue;
|
|
/// <summary>
|
|
/// The output value of the neuron
|
|
/// </summary>
|
|
public virtual Vector3 outputValue {
|
|
get { return _outputValue; }
|
|
set {
|
|
_outputValue = value;
|
|
if (this.isFiring)
|
|
WhenFiring?.Invoke();
|
|
}
|
|
}
|
|
/// <summary>
|
|
/// The magnitude of the neuron output
|
|
/// </summary>
|
|
public float outputMagnitude => _outputValue.magnitude;
|
|
/// <summary>
|
|
/// The squared magnitude of the neuron output
|
|
/// </summary>
|
|
public float outputSqrMagnitude => _outputValue.sqrMagnitude;
|
|
|
|
#endif
|
|
/// <summary>
|
|
/// True if the neuron have a positive value with magnitude > 0.5
|
|
/// </summary>
|
|
public bool isFiring => this.outputMagnitude > 0.5f;
|
|
/// <summary>
|
|
/// An action which is called every time the neuron is updated and is firing
|
|
/// </summary>
|
|
public Action WhenFiring;
|
|
|
|
public Action WhenNotFiring;
|
|
|
|
/// <summary>
|
|
/// When true, the value will not be reset after timeToSleep.
|
|
/// </summary>
|
|
//public bool persistOutput = false;
|
|
/// <summary>
|
|
/// True when the neuron is not persisting and has not be updated for timeToSleep seconds
|
|
/// </summary>
|
|
public virtual bool isSleeping => false; //!persistOutput && (Time.time - this.lastUpdate > timeToSleep);
|
|
/// <summary>
|
|
/// Check if the neuron is sleeping.
|
|
/// </summary>
|
|
/// This will reset the output value if it is sleeping
|
|
public void SleepCheck() {
|
|
if (this.isSleeping && this.outputSqrMagnitude > 0) {
|
|
#if UNITY_MATHEMATICS
|
|
this._outputValue = new float3(0, 0, 0);
|
|
#else
|
|
this._outputValue = new Vector3(0,0,0);
|
|
#endif
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The time at which the last update has been done
|
|
/// </summary>
|
|
[HideInInspector]
|
|
public float lastUpdate = 0;
|
|
/// <summary>
|
|
/// Time in seconds after the last update the neuron can go to sleep
|
|
/// </summary>
|
|
public static readonly float timeToSleep = 0.5f;
|
|
|
|
/// <summary>
|
|
/// When true, Unity will pause exection when this neuron is updated
|
|
/// </summary>
|
|
/// Pausing is implemented using [Debug.Break()](https://docs.unity3d.com/ScriptReference/Debug.Break.html)
|
|
public bool breakOnUpdate = false;
|
|
|
|
/// \copydoc NanoBrain::Nucleus::ShallowCloneTo
|
|
public override Nucleus ShallowCloneTo(Cluster parent) {
|
|
Neuron clone = new(parent, this.name) {
|
|
// prefabNucleus = this
|
|
};
|
|
CloneFields(clone);
|
|
return clone;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Copy relevant fields of this neuron to the given neuron
|
|
/// </summary>
|
|
/// <param name="clone"></param>
|
|
protected virtual void CloneFields(Neuron clone) {
|
|
clone.bias = this.bias;
|
|
clone.trainableBias = this.trainableBias;
|
|
//clone.persistOutput = this.persistOutput;
|
|
clone.combinator = this.combinator;
|
|
clone.activator = this.activator;
|
|
clone.breakOnUpdate = this.breakOnUpdate;
|
|
}
|
|
|
|
public static bool EqualStructure(Neuron neuron1, Neuron neuron2) {
|
|
int synapseCount1 = neuron1.synapses.Count;
|
|
int synapseCount2 = neuron2.synapses.Count;
|
|
if (synapseCount1 != synapseCount2)
|
|
return false;
|
|
|
|
for (int i = 0; i < synapseCount1; i++) {
|
|
if (Synapse.EqualStructure(neuron1.synapses[i], neuron2.synapses[i]) == false)
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
public bool CopyWeightsFrom(Neuron source) {
|
|
int thisSynapseCount = this.synapses.Count;
|
|
int sourceSynapseCount = source.synapses.Count;
|
|
if (thisSynapseCount != sourceSynapseCount)
|
|
return false;
|
|
|
|
for (int i = 0; i < thisSynapseCount; i++) {
|
|
Synapse thisSynapse = this.synapses[i];
|
|
if (thisSynapse.trainable) {
|
|
Synapse sourceSynapse = source.synapses[i];
|
|
thisSynapse.weight = sourceSynapse.weight;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
public void ProcessWeightsFrom(Neuron source, Func<float, float, float> processor) {
|
|
int thisSynapseCount = this.synapses.Count;
|
|
int sourceSynapseCount = source.synapses.Count;
|
|
if (thisSynapseCount != sourceSynapseCount)
|
|
return;
|
|
|
|
for (int i = 0; i < thisSynapseCount; i++) {
|
|
Synapse thisSynapse = this.synapses[i];
|
|
if (thisSynapse.trainable) {
|
|
Synapse sourceSynapse = source.synapses[i];
|
|
thisSynapse.weight = processor(thisSynapse.weight, sourceSynapse.weight);
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
/// <summary>
|
|
/// Delete the give neuron
|
|
/// </summary>
|
|
/// <param name="nucleus">The neuron to delete</param>
|
|
public static void Delete(Nucleus nucleus) {
|
|
if (nucleus == null)
|
|
return;
|
|
if (nucleus is Neuron neuron) {
|
|
foreach (Synapse synapse in neuron.synapses) {
|
|
if (synapse.neuron is Neuron synapse_nucleus) {
|
|
if (synapse_nucleus.receivers.Count > 1) {
|
|
// there is another nucleus feeding into this input nucleus
|
|
synapse_nucleus.receivers.RemoveAll(r => r == nucleus);
|
|
}
|
|
else {
|
|
// No other links, delete it.
|
|
Neuron.Delete(synapse_nucleus);
|
|
}
|
|
}
|
|
}
|
|
foreach (Nucleus receiver in neuron.receivers) {
|
|
if (receiver is not Neuron receiverNeuron)
|
|
continue;
|
|
if (receiver != null && receiverNeuron.synapses != null)
|
|
receiverNeuron.synapses.RemoveAll(s => s.neuron == nucleus);
|
|
}
|
|
}
|
|
else if (nucleus is Cluster cluster) {
|
|
// remove all receivers for this cluster
|
|
foreach (Nucleus clusterNucleus in cluster.nuclei) {
|
|
if (clusterNucleus is Neuron output) {
|
|
foreach (Nucleus receiver in output.receivers) {
|
|
if (receiver is not Neuron receiverNeuron)
|
|
continue;
|
|
receiverNeuron.synapses.RemoveAll(s => s.neuron == output);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (nucleus.parent.prefab != null) {
|
|
nucleus.parent.nuclei.RemoveAll(n => n == nucleus);
|
|
nucleus.parent.RefreshOutputs();
|
|
}
|
|
}
|
|
|
|
/// \copydoc NanoBrain::Nucleus::UpdateStateIsolated
|
|
public override void UpdateStateIsolated() {
|
|
if (breakOnUpdate) {
|
|
Debug.Break();
|
|
}
|
|
this.combination = Combinator(this.bias, this.synapses);
|
|
this.outputValue = Activator(this.combination);
|
|
this.lastUpdate = Time.time;
|
|
}
|
|
|
|
#region Combinator
|
|
|
|
#if UNITY_MATHEMATICS
|
|
[NonSerialized]
|
|
public float3 combination;
|
|
|
|
/// <summary>
|
|
/// The combinator which combines the bias with the values from all synapses
|
|
/// </summary>
|
|
/// <param name="bias">The bias of the neuron</param>
|
|
/// <param name="synapses">The synapses of the neuron</param>
|
|
/// <returns></returns>
|
|
protected float3 Combinator(float3 bias, List<Synapse> synapses) {
|
|
switch (combinator) {
|
|
case CombinatorType.Sum:
|
|
return CombinatorSum(bias, synapses);
|
|
case CombinatorType.Product:
|
|
return CombinatorProduct(bias, synapses);
|
|
default:
|
|
return CombinatorSum(bias, synapses);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Sum the bias and synpase outputs together
|
|
/// </summary>
|
|
/// <param name="bias">The bias of the neuron</param>
|
|
/// <param name="synapses">The synapses of the neuron</param>
|
|
/// <returns></returns>
|
|
public static float3 CombinatorSum(float3 bias, List<Synapse> synapses) {
|
|
float3 sum = bias;
|
|
foreach (Synapse synapse in synapses) {
|
|
synapse.neuron.SleepCheck();
|
|
sum += synapse.weight * synapse.neuron.outputValue;
|
|
}
|
|
return sum;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Multiply the synapse outputs together
|
|
/// </summary>
|
|
/// <param name="bias">The bias of the neuron</param>
|
|
/// <param name="synapses">The synapses of the neuron</param>
|
|
/// <returns>The result of the multiplication</returns>
|
|
public static float3 CombinatorProduct(float3 bias, List<Synapse> synapses) {
|
|
float3 product = bias;
|
|
foreach (Synapse synapse in synapses) {
|
|
synapse.neuron.SleepCheck();
|
|
product *= synapse.weight * synapse.neuron.outputValue;
|
|
}
|
|
return product;
|
|
}
|
|
|
|
#else
|
|
public Vector3 combinationValue;
|
|
|
|
/// <summary>
|
|
/// The combinator which combines the bias with the values from all synapses
|
|
/// </summary>
|
|
/// <param name="bias">The bias of the neuron</param>
|
|
/// <param name="synapses">The synapses of the neuron</param>
|
|
/// <returns></returns>
|
|
protected Vector3 Combinator(Vector3 bias, List<Synapse> synapses) {
|
|
switch (combinator) {
|
|
case CombinatorType.Sum: return CombinatorSum(bias, synapses);
|
|
case CombinatorType.Product: return CombinatorProduct(bias, synapses);
|
|
default: return CombinatorSum(bias, synapses);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Sum the bias and synpase outputs together
|
|
/// </summary>
|
|
/// <param name="bias">The bias of the neuron</param>
|
|
/// <param name="synapses">The synapses of the neuron</param>
|
|
/// <returns></returns>
|
|
public static Vector3 CombinatorSum(Vector3 bias, List<Synapse> synapses) {
|
|
Vector3 sum = bias;
|
|
foreach (Synapse synapse in synapses) {
|
|
synapse.neuron.SleepCheck();
|
|
sum += synapse.weight * synapse.neuron.outputValue;
|
|
}
|
|
return sum;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Multiply the synapse outputs together
|
|
/// </summary>
|
|
/// <param name="bias">The bias of the neuron</param>
|
|
/// <param name="synapses">The synapses of the neuron</param>
|
|
/// <returns>The result of the multiplication</returns>
|
|
public static Vector3 CombinatorProduct(Vector3 bias, List<Synapse> synapses) {
|
|
Vector3 product = bias;
|
|
foreach (Synapse synapse in synapses) {
|
|
synapse.neuron.SleepCheck();
|
|
product = Vector3.Scale(product, synapse.weight * synapse.neuron.outputValue);
|
|
}
|
|
return product;
|
|
}
|
|
|
|
#endif
|
|
#endregion Combinator
|
|
|
|
#region Activator
|
|
|
|
#if UNITY_MATHEMATICS
|
|
|
|
/// <summary>
|
|
/// Apply the activation function to the input
|
|
/// </summary>
|
|
/// <param name="inputValue"></param>
|
|
/// <returns>The result of applying the activation function</returns>
|
|
// This does not allocate memory and seems faster than a switch expression
|
|
protected float3 Activator(float3 inputValue) {
|
|
switch (activator) {
|
|
case ActivationType.Linear:
|
|
return ActivatorLinear(inputValue);
|
|
case ActivationType.Sqrt:
|
|
return ActivatorSqrt(inputValue);
|
|
case ActivationType.Power:
|
|
return ActivatorPower(inputValue);
|
|
case ActivationType.Reciprocal:
|
|
return ActivatorReciprocal(inputValue);
|
|
case ActivationType.Tanh:
|
|
return ActivatorTanh(inputValue);
|
|
case ActivationType.Binary:
|
|
return ActivatorBinary(inputValue);
|
|
case ActivationType.Normalized:
|
|
return ActivatorNormalized(inputValue);
|
|
default:
|
|
return ActivatorLinear(inputValue);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Linear activation function
|
|
/// </summary>
|
|
/// <param name="input">Input value</param>
|
|
/// <returns>The unchanged value</returns>
|
|
protected float3 ActivatorLinear(float3 input) {
|
|
return input;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Square root activation function
|
|
/// </summary>
|
|
/// <param name="input">Input value</param>
|
|
/// <returns>The square root of the input</returns>
|
|
protected float3 ActivatorSqrt(float3 input) {
|
|
float3 result = normalize(input) * MathF.Sqrt(length(input));
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Power activation function
|
|
/// </summary>
|
|
/// <param name="input">Input value</param>
|
|
/// <returns>The input to the power of 2</returns>
|
|
protected float3 ActivatorPower(float3 input) {
|
|
float3 result = normalize(input) * MathF.Pow(length(input), 2);
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reciprocal activation function
|
|
/// </summary>
|
|
/// <param name="input">Input value</param>
|
|
/// <returns>1/input value</returns>
|
|
protected float3 ActivatorReciprocal(float3 input) {
|
|
float magnitude = length(input);
|
|
if (magnitude == 0)
|
|
return new float3(0, 0, 0);
|
|
|
|
float3 result = normalize(input) * (1 / magnitude);
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tanh activation function
|
|
/// </summary>
|
|
/// <param name="input">Input value</param>
|
|
/// <returns>Tanh(input value)</returns>
|
|
protected float3 ActivatorTanh(float3 input) {
|
|
float magnitude = length(input);
|
|
float3 result = normalize(input) * MathF.Tanh(magnitude);
|
|
return result;
|
|
}
|
|
/// <summary>
|
|
/// Binary activation function
|
|
/// </summary>
|
|
/// <param name="input">Input value</param>
|
|
/// <returns>An uniform vector with magnitude between 0 and 1</returns>
|
|
protected float3 ActivatorBinary(float3 input) {
|
|
float magnitude = length(input);
|
|
float value = Mathf.Clamp01(magnitude);
|
|
return float3(value, value, value);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Normalize activation function
|
|
/// </summary>
|
|
/// <param name="input">Input value</param>
|
|
/// <returns>The normalized vector</returns>
|
|
protected float3 ActivatorNormalized(float3 input) {
|
|
if (lengthsq(input) == 0)
|
|
return input;
|
|
float3 result = normalize(input);
|
|
return result;
|
|
}
|
|
|
|
#else
|
|
|
|
/// <summary>
|
|
/// Apply the activation function to the input
|
|
/// </summary>
|
|
/// <param name="inputValue"></param>
|
|
/// <returns>The result of applying the activation function</returns>
|
|
// This does not allocate memory and seems faster than a switch expression
|
|
protected Vector3 Activator(Vector3 inputValue) {
|
|
switch (activator) {
|
|
case ActivationType.Linear: return ActivatorLinear(inputValue);
|
|
case ActivationType.Sqrt: return ActivatorSqrt(inputValue);
|
|
case ActivationType.Power: return ActivatorPower(inputValue);
|
|
case ActivationType.Reciprocal: return ActivatorReciprocal(inputValue);
|
|
// case ActivationType.Tanh: return ActivatorTanh(inputValue);
|
|
// case ActivationType.Binary: return ActivatorBinary(inputValue);
|
|
// case ActivationType.Normalized: return ActivatorNormalized(inputValue);
|
|
default: return ActivatorLinear(inputValue);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Linear activation function
|
|
/// </summary>
|
|
/// <param name="input">Input value</param>
|
|
/// <returns>The unchanged value</returns>
|
|
protected Vector3 ActivatorLinear(Vector3 input) {
|
|
return input;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Square root activation function
|
|
/// </summary>
|
|
/// <param name="input">Input value</param>
|
|
/// <returns>The square root of the input</returns>
|
|
protected Vector3 ActivatorSqrt(Vector3 input) {
|
|
Vector3 result = input.normalized * System.MathF.Sqrt(input.magnitude);
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Power activation function
|
|
/// </summary>
|
|
/// <param name="input">Input value</param>
|
|
/// <returns>The input to the power of 2</returns>
|
|
protected Vector3 ActivatorPower(Vector3 input) {
|
|
Vector3 result = input.normalized * System.MathF.Pow(input.magnitude, 2);
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reciprocal activation function
|
|
/// </summary>
|
|
/// <param name="input">Input value</param>
|
|
/// <returns>1/input value</returns>
|
|
protected Vector3 ActivatorReciprocal(Vector3 input) {
|
|
float magnitude = input.magnitude;
|
|
if (magnitude == 0)
|
|
return new Vector3(0, 0, 0);
|
|
|
|
Vector3 result = input.normalized * (1 / magnitude);
|
|
return result;
|
|
}
|
|
|
|
#endif
|
|
|
|
#endregion Activator
|
|
|
|
#region Receivers
|
|
|
|
/// <summary>
|
|
/// The nuclei which have a synapse to this neuron
|
|
/// </summary>
|
|
[SerializeReference]
|
|
[HideInInspector]
|
|
private List<Nucleus> _receivers = new();
|
|
/// <summary>
|
|
/// The nuclei which have a synapse to this neuron
|
|
/// </summary>
|
|
public virtual List<Nucleus> receivers {
|
|
get { return _receivers; }
|
|
set { _receivers = value; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Add a new receiver to this neuron
|
|
/// </summary>
|
|
/// <param name="receiverToAdd">The receiver to add</param>
|
|
/// <param name="weight">The weight to use for the synapse to his neuron</param>
|
|
public virtual void AddReceiver(Nucleus receiverToAdd, float weight = 1, bool trainable = false) {
|
|
if (receiverToAdd is not Neuron receiverNeuron)
|
|
return;
|
|
|
|
this._receivers.Add(receiverNeuron);
|
|
receiverNeuron.AddSynapse(this, weight, trainable);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Remove a receiver to this neuron
|
|
/// </summary>
|
|
/// <param name="receiverToRemove">The receiver to remove</param>
|
|
public virtual void RemoveReceiver(Nucleus receiverToRemove) {
|
|
if (receiverToRemove is not Neuron receiverNeuron)
|
|
return;
|
|
this._receivers.RemoveAll(receiver => receiver == receiverNeuron);
|
|
receiverNeuron.synapses.RemoveAll(synapse => synapse.neuron == this);
|
|
|
|
// Nucleus prefabReceiver = receiverToRemove.prefabNucleus;
|
|
// if (this.prefabNucleus is Neuron prefabNeuron && prefabReceiver != null) {
|
|
// prefabNeuron.receivers.RemoveAll(receiver => receiver == prefabReceiver);
|
|
// prefabReceiver.synapses.RemoveAll(synapse => synapse.neuron == prefabNeuron);
|
|
// }
|
|
}
|
|
|
|
|
|
#endregion Receivers
|
|
|
|
#region Back propagation
|
|
|
|
public void BackPropagation1D(float derivative, float learningRate) {
|
|
foreach (Synapse synapse in this.synapses)
|
|
synapse.BackPropagation(this, derivative, learningRate);
|
|
|
|
// Bias
|
|
if (this.trainableBias) {
|
|
// This does not work well, because the derivative/error does not have a 3D direction
|
|
// float3 biasDerivative = derivative; // dSSR/dActivator
|
|
// switch (activator) { // dActivator/dBias
|
|
// case ActivationType.Linear:
|
|
// //deltaBias *= 1;
|
|
// break;
|
|
// default:
|
|
// break;
|
|
// }
|
|
// Vector3 deltaBias = biasDerivative * learningRate;
|
|
// this.bias -= deltaBias;
|
|
}
|
|
|
|
}
|
|
|
|
public void BackPropagation3D(Vector3 derivative, float learningRate) {
|
|
float derivativeMagnitude = derivative.magnitude;
|
|
if (derivativeMagnitude < 1e-03f)
|
|
// Small changes are not processed
|
|
// We are probably very close to the desired output
|
|
return;
|
|
|
|
foreach (Synapse synapse in this.synapses)
|
|
synapse.BackPropagation3D(this, derivative, learningRate);
|
|
// As the weight cannot change the direction of the derivative
|
|
// we can use the simpler, 1D backpropagation here
|
|
// But we still need to determine the sign of the derivative
|
|
|
|
// if (Synapse.AreOpposed(derivative, synapse.neuron.activation))
|
|
// synapse.BackPropagation(this, -derivativeMagnitude, learningRate);
|
|
// else
|
|
// synapse.BackPropagation(this, derivativeMagnitude, learningRate);
|
|
|
|
// Bias
|
|
if (this.trainableBias) {
|
|
switch (this.activator) { // dActivator/dBias
|
|
case ActivationType.Linear:
|
|
derivative *= 1;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
this.bias += derivative * learningRate;
|
|
Debug.Log($"bias {derivative} {this.bias}");
|
|
}
|
|
}
|
|
|
|
#endregion Back propagation
|
|
|
|
private CancellationTokenSource _cts;
|
|
private Action resetStimulus;
|
|
|
|
/// <summary>
|
|
/// Process an external stimulus
|
|
/// </summary>
|
|
/// <param name="inputValue">The value of the stimulus</param>
|
|
public virtual void ProcessStimulus(Vector3 inputValue, float autoResetDelay = 0) {
|
|
this.lastUpdate = Time.time;
|
|
this.bias = inputValue;
|
|
this.parent?.UpdateFromNucleus(this);
|
|
this.resetStimulus ??= ResetStimulus;
|
|
|
|
if (autoResetDelay > 0) {
|
|
long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
|
long resetTime = now + (long)(autoResetDelay * 1000.0f);
|
|
Cluster.TimedAction.AddTo(this.parent.actions, this.parent.name + "." +this.name, this.resetStimulus, resetTime);
|
|
}
|
|
}
|
|
|
|
private void ResetStimulus() {
|
|
// Debug.Log($"reset stimulus {this.parent.name + "." +this.name}");
|
|
this.parent.name = this.parent.baseName;
|
|
this.bias = Vector3.zero;
|
|
this.parent.thingId = 0;
|
|
this.parent?.UpdateFromNucleus(this);
|
|
}
|
|
}
|
|
|
|
[Serializable]
|
|
public class NeuronData {
|
|
public string name;
|
|
public Nucleus.Type type;
|
|
public Vector3 bias = Vector3.zero;
|
|
public Neuron.CombinatorType combinatorType;
|
|
public List<SynapseData> synapses = new();
|
|
public Neuron.ActivationType activationType;
|
|
|
|
public NeuronData(Neuron neuron) {
|
|
if (neuron is MemoryCell)
|
|
this.type = Nucleus.Type.MemoryCell;
|
|
else
|
|
this.type = Nucleus.Type.Neuron;
|
|
this.name = neuron.name;
|
|
this.bias = neuron.bias;
|
|
this.combinatorType = neuron.combinator;
|
|
this.activationType = neuron.activator;
|
|
|
|
foreach (Synapse synapse in neuron.synapses) {
|
|
SynapseData synapseData = new(synapse);
|
|
this.synapses.Add(synapseData);
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
}
|