diff --git a/Runtime/Scripts/Core/Neuron.cs b/Runtime/Scripts/Core/Neuron.cs index 3e876ec..646607c 100644 --- a/Runtime/Scripts/Core/Neuron.cs +++ b/Runtime/Scripts/Core/Neuron.cs @@ -666,95 +666,10 @@ namespace NanoBrain { #region Back propagation - // public void BackPropagation(Synapse synapse, Vector3 error, float learningRate) { - // // Loss function: - // // Mean Squared Error (MSE) 1/n * sum(errors^2) - // // We use simplified here 1/2 * (error^2) - // // For vectors, we need to use MSE component wise. - // Vector3 loss = 0.5f * Vector3.Scale(error, error); + public void BackPropagation1D(float derivative, float learningRate) { + foreach (Synapse synapse in this.synapses) + synapse.BackPropagation(this, derivative, learningRate); - // // loss is a derivative of error - // // Backpropagation = loss * d(combinator) - - // Vector3 delta2; - // switch (activator) { - // case ActivationType.Linear: - // // Derivative of this (f'()) would be 1. - // delta2 = loss * 1; - // break; - // case ActivationType.Power: - // delta2 = loss * (2 * this.combination); - // break; - // case ActivationType.Reciprocal: - // delta2 = loss * (-1 / (this.combination * this.combination)); - // break; - // default: - // delta2 = loss; - // break; - // } - - // Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue); - // float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z); - // synapse.weight += learningRate * deltaWeight; - // Debug.Log($"Updated weight: {error.magnitude} {error} {scaledOutput} {synapse.weight}"); - // } - - // public void BackPropagationWithLoss(Synapse synapse, Vector3 loss, float learningRate) { - // Vector3 delta2 = activator switch { - // ActivationType.Linear => loss * 1, - // ActivationType.Power => (Vector3)(loss * (2 * this.combination)), - // ActivationType.Reciprocal => (Vector3)(loss * (-1 / (this.combination * this.combination))), - // _ => loss, - // }; - // Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue); - // float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z); - // synapse.weight += learningRate * deltaWeight; - // Debug.Log($"Updated weight: {loss.magnitude} {loss} {scaledOutput} {synapse.weight}"); - // } - - // public void BackPropagation1(Vector3 cost, Vector3 error, float learningRate) { - // cost = Vector3.Scale(error, error); // error^2 - // float3 derivative = 2 * error; // derivative of (error^2) - // // inverted because it uses the non-convential - // // error=(actual-taget) instead of (target-actual) - // // dSSR / dPredicted - - // // Bias - // float3 deltaBias = derivative; - // // deltaBias *= 1; // because bias is always fully applied - // Vector3 stepSize = deltaBias * learningRate; - // this.bias -= stepSize; - - // foreach (Synapse synapse in this.synapses) { - // // derivative for the weight? - // float3 deltaSynapse = derivative; // dSSR/dPredicted - // // derivative for the previous activation - // deltaSynapse *= synapse.neuron.activation; // dPredicted/dWeight - - // // // derivative for the activator - // // switch (activator) { - // // case ActivationType.Linear: - // // //delta2 *= 1; - // // break; - // // default: - // // break; - // // } - // float deltaWeight = length(deltaSynapse); - // synapse.weight += learningRate * deltaWeight; - - // synapse.neuron.BackPropagation2(derivative * synapse.weight, learningRate); - // } - // } - - public void BackPropagation0(float error, float learningRate) { - float derivative = 2 * error; // derivative of (error^2) - // inverted because it uses the non-convential - // error=(actual-taget) instead of (target-actual) - // dSSR / dPredicted - BackPropagation2(derivative, learningRate); - } - - public void BackPropagation2(float derivative, float learningRate) { // Bias if (this.trainable) { // This does not work well, because the derivative/error does not have a 3D direction @@ -770,30 +685,6 @@ namespace NanoBrain { // this.bias -= deltaBias; } - foreach (Synapse synapse in this.synapses) { - synapse.BackPropagation(this, derivative, learningRate); - // // derivative for the weight? - // float3 deltaSynapse = derivative; // dSSR/dActivator - - // // derivative for the activator - // // dActivator/dCombinator - // switch (activator) { - // case ActivationType.Linear: - // //deltaSynapse *= 1; - // break; - // default: - // break; - // } - - // // derivative for the previous activation - // // dCombinator/dWeight - // deltaSynapse *= synapse.neuron.activation; - - // float deltaWeight = length(deltaSynapse); - // synapse.weight += learningRate * deltaWeight; - - //BackPropagation2(derivative * synapse.weight, learningRate); - } } public void BackPropagation3D(Vector3 derivative, float learningRate) { diff --git a/Runtime/Scripts/Core/Synapse.cs b/Runtime/Scripts/Core/Synapse.cs index 072ef45..640cf04 100644 --- a/Runtime/Scripts/Core/Synapse.cs +++ b/Runtime/Scripts/Core/Synapse.cs @@ -56,7 +56,7 @@ namespace NanoBrain { break; } - this.neuron.BackPropagation2(derivative * this.weight, learningRate); + this.neuron.BackPropagation1D(derivative * this.weight, learningRate); derivative *= math.length(this.neuron.activation);