diff --git a/Editor/ClusterEditor.cs b/Editor/ClusterEditor.cs
index d5aaf06..e3a1995 100644
--- a/Editor/ClusterEditor.cs
+++ b/Editor/ClusterEditor.cs
@@ -220,9 +220,16 @@ namespace NanoBrain.Unity {
Vector3 newBias = EditorGUILayout.Vector3Field("Bias", neuron2.bias);
if (newBias != neuron2.bias) {
- anythingChanged |= newBias != neuron2.bias;
+ anythingChanged = true;
neuron2.bias = newBias;
}
+
+ bool newTrainable = EditorGUILayout.Toggle("Trainable", neuron2.trainable);
+ if (newTrainable != neuron2.trainable) {
+ anythingChanged = true;
+ neuron2.trainable = newTrainable;
+ }
+
EditorGUIUtility.labelWidth = previousLabelWidth;
}
diff --git a/Runtime/Scripts/Core/Neuron.cs b/Runtime/Scripts/Core/Neuron.cs
index 7cda0b8..1e99a80 100644
--- a/Runtime/Scripts/Core/Neuron.cs
+++ b/Runtime/Scripts/Core/Neuron.cs
@@ -47,6 +47,11 @@ namespace NanoBrain {
//[HideInInspector]
public Vector3 bias = Vector3.zero;
+ ///
+ /// Indicator whether the bias can be trained
+ ///
+ public bool trainable = false;
+
#region Synapses
[SerializeField]
@@ -268,6 +273,7 @@ namespace NanoBrain {
///
protected virtual void CloneFields(Neuron clone) {
clone.bias = this.bias;
+ clone.trainable = this.trainable;
clone.persistOutput = this.persistOutput;
clone.combinator = this.combinator;
clone.activator = this.activator;
@@ -660,51 +666,51 @@ namespace NanoBrain {
#region Back propagation
- public void BackPropagation(Synapse synapse, Vector3 error, float learningRate) {
- // Loss function:
- // Mean Squared Error (MSE) 1/n * sum(errors^2)
- // We use simplified here 1/2 * (error^2)
- // For vectors, we need to use MSE component wise.
- Vector3 loss = 0.5f * Vector3.Scale(error, error);
+ // public void BackPropagation(Synapse synapse, Vector3 error, float learningRate) {
+ // // Loss function:
+ // // Mean Squared Error (MSE) 1/n * sum(errors^2)
+ // // We use simplified here 1/2 * (error^2)
+ // // For vectors, we need to use MSE component wise.
+ // Vector3 loss = 0.5f * Vector3.Scale(error, error);
- // loss is a derivative of error
- // Backpropagation = loss * d(combinator)
+ // // loss is a derivative of error
+ // // Backpropagation = loss * d(combinator)
- Vector3 delta2;
- switch (activator) {
- case ActivationType.Linear:
- // Derivative of this (f'()) would be 1.
- delta2 = loss * 1;
- break;
- case ActivationType.Power:
- delta2 = loss * (2 * this.combination);
- break;
- case ActivationType.Reciprocal:
- delta2 = loss * (-1 / (this.combination * this.combination));
- break;
- default:
- delta2 = loss;
- break;
- }
+ // Vector3 delta2;
+ // switch (activator) {
+ // case ActivationType.Linear:
+ // // Derivative of this (f'()) would be 1.
+ // delta2 = loss * 1;
+ // break;
+ // case ActivationType.Power:
+ // delta2 = loss * (2 * this.combination);
+ // break;
+ // case ActivationType.Reciprocal:
+ // delta2 = loss * (-1 / (this.combination * this.combination));
+ // break;
+ // default:
+ // delta2 = loss;
+ // break;
+ // }
- Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue);
- float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z);
- synapse.weight += learningRate * deltaWeight;
- Debug.Log($"Updated weight: {error.magnitude} {error} {scaledOutput} {synapse.weight}");
- }
+ // Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue);
+ // float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z);
+ // synapse.weight += learningRate * deltaWeight;
+ // Debug.Log($"Updated weight: {error.magnitude} {error} {scaledOutput} {synapse.weight}");
+ // }
- public void BackPropagationWithLoss(Synapse synapse, Vector3 loss, float learningRate) {
- Vector3 delta2 = activator switch {
- ActivationType.Linear => loss * 1,
- ActivationType.Power => (Vector3)(loss * (2 * this.combination)),
- ActivationType.Reciprocal => (Vector3)(loss * (-1 / (this.combination * this.combination))),
- _ => loss,
- };
- Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue);
- float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z);
- synapse.weight += learningRate * deltaWeight;
- Debug.Log($"Updated weight: {loss.magnitude} {loss} {scaledOutput} {synapse.weight}");
- }
+ // public void BackPropagationWithLoss(Synapse synapse, Vector3 loss, float learningRate) {
+ // Vector3 delta2 = activator switch {
+ // ActivationType.Linear => loss * 1,
+ // ActivationType.Power => (Vector3)(loss * (2 * this.combination)),
+ // ActivationType.Reciprocal => (Vector3)(loss * (-1 / (this.combination * this.combination))),
+ // _ => loss,
+ // };
+ // Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue);
+ // float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z);
+ // synapse.weight += learningRate * deltaWeight;
+ // Debug.Log($"Updated weight: {loss.magnitude} {loss} {scaledOutput} {synapse.weight}");
+ // }
// public void BackPropagation1(Vector3 cost, Vector3 error, float learningRate) {
// cost = Vector3.Scale(error, error); // error^2
@@ -742,26 +748,28 @@ namespace NanoBrain {
public void BackPropagation0(float error, float learningRate) {
float derivative = 2 * error; // derivative of (error^2)
- // inverted because it uses the non-convential
- // error=(actual-taget) instead of (target-actual)
- // dSSR / dPredicted
+ // inverted because it uses the non-convential
+ // error=(actual-taget) instead of (target-actual)
+ // dSSR / dPredicted
BackPropagation2(derivative, learningRate);
}
public void BackPropagation2(float derivative, float learningRate) {
// Bias
- // float3 deltaBias = derivative; // dSSR/dActivator
- // switch (activator) { // dActivator/dBias
- // case ActivationType.Linear:
- // //deltaBias *= 1;
- // break;
- // default:
- // break;
- // }
- // // deltaBias *= 1; // because bias is always fully applied
- // Vector3 stepSize = deltaBias * learningRate;
- // this.bias -= stepSize;
+ if (this.trainable) {
+ // This does not work well, because the derivative/error does not have a 3D direction
+ // float3 biasDerivative = derivative; // dSSR/dActivator
+ // switch (activator) { // dActivator/dBias
+ // case ActivationType.Linear:
+ // //deltaBias *= 1;
+ // break;
+ // default:
+ // break;
+ // }
+ // Vector3 deltaBias = biasDerivative * learningRate;
+ // this.bias -= deltaBias;
+ }
foreach (Synapse synapse in this.synapses) {
synapse.BackPropagation(this, derivative, learningRate);
@@ -785,7 +793,7 @@ namespace NanoBrain {
// float deltaWeight = length(deltaSynapse);
// synapse.weight += learningRate * deltaWeight;
- // BackPropagation2(derivative * synapse.weight, learningRate);
+ //BackPropagation2(derivative * synapse.weight, learningRate);
}
}
diff --git a/Runtime/Scripts/Core/Synapse.cs b/Runtime/Scripts/Core/Synapse.cs
index 8a66fe6..cd7859e 100644
--- a/Runtime/Scripts/Core/Synapse.cs
+++ b/Runtime/Scripts/Core/Synapse.cs
@@ -21,6 +21,9 @@ namespace NanoBrain {
///
public float weight;
+ ///
+ /// Indicator whether the weight can be trained
+ ///
public bool trainable = false;
///
@@ -40,14 +43,28 @@ namespace NanoBrain {
case Neuron.ActivationType.Linear:
derivative *= 1;
break;
+ case Neuron.ActivationType.Power:
+ // untested
+ derivative *= 2 * math.length(this.neuron.combination);
+ break;
+ case Neuron.ActivationType.Reciprocal:
+ // untested
+ derivative *= -1 / Mathf.Pow(math.length(this.neuron.combination), 2);
+ break;
default:
Debug.Log("other activator");
break;
}
- derivative *= math.length(neuron.activation);
- float deltaWeight = learningRate * derivative;
- this.weight += deltaWeight;
+ this.neuron.BackPropagation2(derivative * this.weight, learningRate);
+
+ derivative *= math.length(this.neuron.activation);
+
+ if (this.trainable) {
+ float deltaWeight = learningRate * derivative;
+ this.weight += deltaWeight;
+ }
+
}
}