From 66c57f803c7d136dd169d7ba9f03aceb578a7cb0 Mon Sep 17 00:00:00 2001 From: Pascal Serrarens Date: Fri, 3 Jul 2026 15:39:45 +0200 Subject: [PATCH] Support more activators and bias (incorrectly) --- Editor/ClusterEditor.cs | 9 ++- Runtime/Scripts/Core/Neuron.cs | 120 +++++++++++++++++--------------- Runtime/Scripts/Core/Synapse.cs | 23 +++++- 3 files changed, 92 insertions(+), 60 deletions(-) diff --git a/Editor/ClusterEditor.cs b/Editor/ClusterEditor.cs index d5aaf06..e3a1995 100644 --- a/Editor/ClusterEditor.cs +++ b/Editor/ClusterEditor.cs @@ -220,9 +220,16 @@ namespace NanoBrain.Unity { Vector3 newBias = EditorGUILayout.Vector3Field("Bias", neuron2.bias); if (newBias != neuron2.bias) { - anythingChanged |= newBias != neuron2.bias; + anythingChanged = true; neuron2.bias = newBias; } + + bool newTrainable = EditorGUILayout.Toggle("Trainable", neuron2.trainable); + if (newTrainable != neuron2.trainable) { + anythingChanged = true; + neuron2.trainable = newTrainable; + } + EditorGUIUtility.labelWidth = previousLabelWidth; } diff --git a/Runtime/Scripts/Core/Neuron.cs b/Runtime/Scripts/Core/Neuron.cs index 7cda0b8..1e99a80 100644 --- a/Runtime/Scripts/Core/Neuron.cs +++ b/Runtime/Scripts/Core/Neuron.cs @@ -47,6 +47,11 @@ namespace NanoBrain { //[HideInInspector] public Vector3 bias = Vector3.zero; + /// + /// Indicator whether the bias can be trained + /// + public bool trainable = false; + #region Synapses [SerializeField] @@ -268,6 +273,7 @@ namespace NanoBrain { /// protected virtual void CloneFields(Neuron clone) { clone.bias = this.bias; + clone.trainable = this.trainable; clone.persistOutput = this.persistOutput; clone.combinator = this.combinator; clone.activator = this.activator; @@ -660,51 +666,51 @@ namespace NanoBrain { #region Back propagation - public void BackPropagation(Synapse synapse, Vector3 error, float learningRate) { - // Loss function: - // Mean Squared Error (MSE) 1/n * sum(errors^2) - // We use simplified here 1/2 * (error^2) - // For vectors, we need to use MSE component wise. - Vector3 loss = 0.5f * Vector3.Scale(error, error); + // public void BackPropagation(Synapse synapse, Vector3 error, float learningRate) { + // // Loss function: + // // Mean Squared Error (MSE) 1/n * sum(errors^2) + // // We use simplified here 1/2 * (error^2) + // // For vectors, we need to use MSE component wise. + // Vector3 loss = 0.5f * Vector3.Scale(error, error); - // loss is a derivative of error - // Backpropagation = loss * d(combinator) + // // loss is a derivative of error + // // Backpropagation = loss * d(combinator) - Vector3 delta2; - switch (activator) { - case ActivationType.Linear: - // Derivative of this (f'()) would be 1. - delta2 = loss * 1; - break; - case ActivationType.Power: - delta2 = loss * (2 * this.combination); - break; - case ActivationType.Reciprocal: - delta2 = loss * (-1 / (this.combination * this.combination)); - break; - default: - delta2 = loss; - break; - } + // Vector3 delta2; + // switch (activator) { + // case ActivationType.Linear: + // // Derivative of this (f'()) would be 1. + // delta2 = loss * 1; + // break; + // case ActivationType.Power: + // delta2 = loss * (2 * this.combination); + // break; + // case ActivationType.Reciprocal: + // delta2 = loss * (-1 / (this.combination * this.combination)); + // break; + // default: + // delta2 = loss; + // break; + // } - Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue); - float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z); - synapse.weight += learningRate * deltaWeight; - Debug.Log($"Updated weight: {error.magnitude} {error} {scaledOutput} {synapse.weight}"); - } + // Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue); + // float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z); + // synapse.weight += learningRate * deltaWeight; + // Debug.Log($"Updated weight: {error.magnitude} {error} {scaledOutput} {synapse.weight}"); + // } - public void BackPropagationWithLoss(Synapse synapse, Vector3 loss, float learningRate) { - Vector3 delta2 = activator switch { - ActivationType.Linear => loss * 1, - ActivationType.Power => (Vector3)(loss * (2 * this.combination)), - ActivationType.Reciprocal => (Vector3)(loss * (-1 / (this.combination * this.combination))), - _ => loss, - }; - Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue); - float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z); - synapse.weight += learningRate * deltaWeight; - Debug.Log($"Updated weight: {loss.magnitude} {loss} {scaledOutput} {synapse.weight}"); - } + // public void BackPropagationWithLoss(Synapse synapse, Vector3 loss, float learningRate) { + // Vector3 delta2 = activator switch { + // ActivationType.Linear => loss * 1, + // ActivationType.Power => (Vector3)(loss * (2 * this.combination)), + // ActivationType.Reciprocal => (Vector3)(loss * (-1 / (this.combination * this.combination))), + // _ => loss, + // }; + // Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue); + // float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z); + // synapse.weight += learningRate * deltaWeight; + // Debug.Log($"Updated weight: {loss.magnitude} {loss} {scaledOutput} {synapse.weight}"); + // } // public void BackPropagation1(Vector3 cost, Vector3 error, float learningRate) { // cost = Vector3.Scale(error, error); // error^2 @@ -742,26 +748,28 @@ namespace NanoBrain { public void BackPropagation0(float error, float learningRate) { float derivative = 2 * error; // derivative of (error^2) - // inverted because it uses the non-convential - // error=(actual-taget) instead of (target-actual) - // dSSR / dPredicted + // inverted because it uses the non-convential + // error=(actual-taget) instead of (target-actual) + // dSSR / dPredicted BackPropagation2(derivative, learningRate); } public void BackPropagation2(float derivative, float learningRate) { // Bias - // float3 deltaBias = derivative; // dSSR/dActivator - // switch (activator) { // dActivator/dBias - // case ActivationType.Linear: - // //deltaBias *= 1; - // break; - // default: - // break; - // } - // // deltaBias *= 1; // because bias is always fully applied - // Vector3 stepSize = deltaBias * learningRate; - // this.bias -= stepSize; + if (this.trainable) { + // This does not work well, because the derivative/error does not have a 3D direction + // float3 biasDerivative = derivative; // dSSR/dActivator + // switch (activator) { // dActivator/dBias + // case ActivationType.Linear: + // //deltaBias *= 1; + // break; + // default: + // break; + // } + // Vector3 deltaBias = biasDerivative * learningRate; + // this.bias -= deltaBias; + } foreach (Synapse synapse in this.synapses) { synapse.BackPropagation(this, derivative, learningRate); @@ -785,7 +793,7 @@ namespace NanoBrain { // float deltaWeight = length(deltaSynapse); // synapse.weight += learningRate * deltaWeight; - // BackPropagation2(derivative * synapse.weight, learningRate); + //BackPropagation2(derivative * synapse.weight, learningRate); } } diff --git a/Runtime/Scripts/Core/Synapse.cs b/Runtime/Scripts/Core/Synapse.cs index 8a66fe6..cd7859e 100644 --- a/Runtime/Scripts/Core/Synapse.cs +++ b/Runtime/Scripts/Core/Synapse.cs @@ -21,6 +21,9 @@ namespace NanoBrain { /// public float weight; + /// + /// Indicator whether the weight can be trained + /// public bool trainable = false; /// @@ -40,14 +43,28 @@ namespace NanoBrain { case Neuron.ActivationType.Linear: derivative *= 1; break; + case Neuron.ActivationType.Power: + // untested + derivative *= 2 * math.length(this.neuron.combination); + break; + case Neuron.ActivationType.Reciprocal: + // untested + derivative *= -1 / Mathf.Pow(math.length(this.neuron.combination), 2); + break; default: Debug.Log("other activator"); break; } - derivative *= math.length(neuron.activation); - float deltaWeight = learningRate * derivative; - this.weight += deltaWeight; + this.neuron.BackPropagation2(derivative * this.weight, learningRate); + + derivative *= math.length(this.neuron.activation); + + if (this.trainable) { + float deltaWeight = learningRate * derivative; + this.weight += deltaWeight; + } + } }