diff --git a/Runtime/Scripts/Core/Neuron.cs b/Runtime/Scripts/Core/Neuron.cs index 4d61bc0..9a62534 100644 --- a/Runtime/Scripts/Core/Neuron.cs +++ b/Runtime/Scripts/Core/Neuron.cs @@ -695,15 +695,22 @@ namespace NanoBrain { } public void BackPropagation3D(Vector3 derivative, float learningRate) { + float derivativeMagnitude = derivative.magnitude; + if (derivativeMagnitude < 1e-03f) + // Small changes are not processed + // We are probably very close to the desired output + return; + foreach (Synapse synapse in this.synapses) - // synapse.BackPropagation(this, derivative, learningRate); + synapse.BackPropagation3D(this, derivative, learningRate); // As the weight cannot change the direction of the derivative // we can use the simpler, 1D backpropagation here // But we still need to determine the sign of the derivative - if (Synapse.AreOpposed(derivative, synapse.neuron.activation)) - synapse.BackPropagation(this, -derivative.magnitude, learningRate); - else - synapse.BackPropagation(this, derivative.magnitude, learningRate); + + // if (Synapse.AreOpposed(derivative, synapse.neuron.activation)) + // synapse.BackPropagation(this, -derivativeMagnitude, learningRate); + // else + // synapse.BackPropagation(this, derivativeMagnitude, learningRate); // Bias if (this.trainableBias) { diff --git a/Runtime/Scripts/Core/Synapse.cs b/Runtime/Scripts/Core/Synapse.cs index e34474c..8a5f83c 100644 --- a/Runtime/Scripts/Core/Synapse.cs +++ b/Runtime/Scripts/Core/Synapse.cs @@ -64,37 +64,46 @@ namespace NanoBrain { } } - public virtual void BackPropagation(Neuron receiver, Vector3 derivative, float learningRate) { - switch (receiver.activator) { - case Neuron.ActivationType.Linear: - derivative *= 1; - break; - case Neuron.ActivationType.Power: - // untested - derivative *= 2 * math.length(this.neuron.combination); - break; - case Neuron.ActivationType.Reciprocal: - // untested - derivative *= -1 / Mathf.Pow(math.length(this.neuron.combination), 2); - break; - default: - Debug.Log("other activator"); - break; - } + public virtual void BackPropagation3D(Neuron receiver, Vector3 derivative, float learningRate) { + // As the weight cannot change the direction of the derivative + // we can use the simpler, 1D backpropagation here + // But we still need to determine the sign of the derivative - this.neuron.BackPropagation3D(derivative * this.weight, learningRate); + if (Synapse.AreOpposed(derivative, this.neuron.activation)) + BackPropagation(receiver, -derivative.magnitude, learningRate); + else + BackPropagation(receiver, derivative.magnitude, learningRate); - derivative *= math.length(this.neuron.activation); + // switch (receiver.activator) { + // case Neuron.ActivationType.Linear: + // derivative *= 1; + // break; + // case Neuron.ActivationType.Power: + // // untested + // derivative *= 2 * math.length(this.neuron.combination); + // break; + // case Neuron.ActivationType.Reciprocal: + // // untested + // derivative *= -1 / Mathf.Pow(math.length(this.neuron.combination), 2); + // break; + // default: + // Debug.Log("other activator"); + // break; + // } - if (this.trainable) { - float deltaWeight = learningRate * derivative.magnitude; - // Compared to the 1D solution, this does not decrease the weight because magnitude is always positive - // derivative.direction and derivative.sign are different.... - if (AreOpposed(derivative, this.neuron.activation)) - this.weight -= deltaWeight; - else - this.weight += deltaWeight; - } + // this.neuron.BackPropagation3D(derivative * this.weight, learningRate); + + // derivative *= math.length(this.neuron.activation); + + // if (this.trainable) { + // float deltaWeight = learningRate * derivative.magnitude; + // // Compared to the 1D solution, this does not decrease the weight because magnitude is always positive + // // derivative.direction and derivative.sign are different.... + // if (AreOpposed(derivative, this.neuron.activation)) + // this.weight -= deltaWeight; + // else + // this.weight += deltaWeight; + // } } public static bool AreOpposed(Vector3 a, Vector3 b) { @@ -105,10 +114,10 @@ namespace NanoBrain { [Serializable] public class SynapseData { - public string clusterName; - public string neuronName; - public float weight; - public bool trainable; + public string clusterName; + public string neuronName; + public float weight; + public bool trainable; public SynapseData(Synapse synapse) { this.clusterName = synapse.neuron.parent.name;