NanoBrain/NanoBrain-unitypackage
NanoBrain/NanoBrain-unitypackage
2
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Support more activators and bias (incorrectly)

Browse Source
This commit is contained in:
Pascal Serrarens 2026-07-03 15:39:45 +02:00
parent a38e4c3c77
commit 66c57f803c
3 changed files with 92 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Editor/ClusterEditor.cs
View File

@ -220,9 +220,16 @@ namespace NanoBrain.Unity {
Vector3 newBias = EditorGUILayout.Vector3Field("Bias", neuron2.bias); Vector3 newBias = EditorGUILayout.Vector3Field("Bias", neuron2.bias);
if (newBias != neuron2.bias) { if (newBias != neuron2.bias) {
anythingChanged |= newBias != neuron2.bias; anythingChanged = true;
neuron2.bias = newBias; neuron2.bias = newBias;
} }
bool newTrainable = EditorGUILayout.Toggle("Trainable", neuron2.trainable);
if (newTrainable != neuron2.trainable) {
anythingChanged = true;
neuron2.trainable = newTrainable;
}
EditorGUIUtility.labelWidth = previousLabelWidth; EditorGUIUtility.labelWidth = previousLabelWidth;
} }

120
Runtime/Scripts/Core/Neuron.cs
View File

@ -47,6 +47,11 @@ namespace NanoBrain {
//[HideInInspector] //[HideInInspector]
public Vector3 bias = Vector3.zero; public Vector3 bias = Vector3.zero;
/// <summary>
/// Indicator whether the bias can be trained
/// </summary>
public bool trainable = false;
#region Synapses #region Synapses
[SerializeField] [SerializeField]
@ -268,6 +273,7 @@ namespace NanoBrain {
/// <param name="clone"></param> /// <param name="clone"></param>
protected virtual void CloneFields(Neuron clone) { protected virtual void CloneFields(Neuron clone) {
clone.bias = this.bias; clone.bias = this.bias;
clone.trainable = this.trainable;
clone.persistOutput = this.persistOutput; clone.persistOutput = this.persistOutput;
clone.combinator = this.combinator; clone.combinator = this.combinator;
clone.activator = this.activator; clone.activator = this.activator;
@ -660,51 +666,51 @@ namespace NanoBrain {
#region Back propagation #region Back propagation
public void BackPropagation(Synapse synapse, Vector3 error, float learningRate) { // public void BackPropagation(Synapse synapse, Vector3 error, float learningRate) {
// Loss function: // // Loss function:
// Mean Squared Error (MSE) 1/n * sum(errors^2) // // Mean Squared Error (MSE) 1/n * sum(errors^2)
// We use simplified here 1/2 * (error^2) // // We use simplified here 1/2 * (error^2)
// For vectors, we need to use MSE component wise. // // For vectors, we need to use MSE component wise.
Vector3 loss = 0.5f * Vector3.Scale(error, error); // Vector3 loss = 0.5f * Vector3.Scale(error, error);
// loss is a derivative of error // // loss is a derivative of error
// Backpropagation = loss * d(combinator) // // Backpropagation = loss * d(combinator)
Vector3 delta2; // Vector3 delta2;
switch (activator) { // switch (activator) {
case ActivationType.Linear: // case ActivationType.Linear:
// Derivative of this (f'()) would be 1. // // Derivative of this (f'()) would be 1.
delta2 = loss * 1; // delta2 = loss * 1;
break; // break;
case ActivationType.Power: // case ActivationType.Power:
delta2 = loss * (2 * this.combination); // delta2 = loss * (2 * this.combination);
break; // break;
case ActivationType.Reciprocal: // case ActivationType.Reciprocal:
delta2 = loss * (-1 / (this.combination * this.combination)); // delta2 = loss * (-1 / (this.combination * this.combination));
break; // break;
default: // default:
delta2 = loss; // delta2 = loss;
break; // break;
} // }
Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue); // Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue);
float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z); // float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z);
synapse.weight += learningRate * deltaWeight; // synapse.weight += learningRate * deltaWeight;
Debug.Log($"Updated weight: {error.magnitude} {error} {scaledOutput} {synapse.weight}"); // Debug.Log($"Updated weight: {error.magnitude} {error} {scaledOutput} {synapse.weight}");
} // }
public void BackPropagationWithLoss(Synapse synapse, Vector3 loss, float learningRate) { // public void BackPropagationWithLoss(Synapse synapse, Vector3 loss, float learningRate) {
Vector3 delta2 = activator switch { // Vector3 delta2 = activator switch {
ActivationType.Linear => loss * 1, // ActivationType.Linear => loss * 1,
ActivationType.Power => (Vector3)(loss * (2 * this.combination)), // ActivationType.Power => (Vector3)(loss * (2 * this.combination)),
ActivationType.Reciprocal => (Vector3)(loss * (-1 / (this.combination * this.combination))), // ActivationType.Reciprocal => (Vector3)(loss * (-1 / (this.combination * this.combination))),
_ => loss, // _ => loss,
}; // };
Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue); // Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue);
float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z); // float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z);
synapse.weight += learningRate * deltaWeight; // synapse.weight += learningRate * deltaWeight;
Debug.Log($"Updated weight: {loss.magnitude} {loss} {scaledOutput} {synapse.weight}"); // Debug.Log($"Updated weight: {loss.magnitude} {loss} {scaledOutput} {synapse.weight}");
} // }
// public void BackPropagation1(Vector3 cost, Vector3 error, float learningRate) { // public void BackPropagation1(Vector3 cost, Vector3 error, float learningRate) {
// cost = Vector3.Scale(error, error); // error^2 // cost = Vector3.Scale(error, error); // error^2
@ -742,26 +748,28 @@ namespace NanoBrain {
public void BackPropagation0(float error, float learningRate) { public void BackPropagation0(float error, float learningRate) {
float derivative = 2 * error; // derivative of (error^2) float derivative = 2 * error; // derivative of (error^2)
// inverted because it uses the non-convential // inverted because it uses the non-convential
// error=(actual-taget) instead of (target-actual) // error=(actual-taget) instead of (target-actual)
// dSSR / dPredicted // dSSR / dPredicted
BackPropagation2(derivative, learningRate); BackPropagation2(derivative, learningRate);
} }
public void BackPropagation2(float derivative, float learningRate) { public void BackPropagation2(float derivative, float learningRate) {
// Bias // Bias
// float3 deltaBias = derivative; // dSSR/dActivator if (this.trainable) {
// switch (activator) { // dActivator/dBias // This does not work well, because the derivative/error does not have a 3D direction
// case ActivationType.Linear: // float3 biasDerivative = derivative; // dSSR/dActivator
// //deltaBias *= 1; // switch (activator) { // dActivator/dBias
// break; // case ActivationType.Linear:
// default: // //deltaBias *= 1;
// break; // break;
// } // default:
// // deltaBias *= 1; // because bias is always fully applied // break;
// Vector3 stepSize = deltaBias * learningRate; // }
// this.bias -= stepSize; // Vector3 deltaBias = biasDerivative * learningRate;
// this.bias -= deltaBias;
}
foreach (Synapse synapse in this.synapses) { foreach (Synapse synapse in this.synapses) {
synapse.BackPropagation(this, derivative, learningRate); synapse.BackPropagation(this, derivative, learningRate);
@ -785,7 +793,7 @@ namespace NanoBrain {
// float deltaWeight = length(deltaSynapse); // float deltaWeight = length(deltaSynapse);
// synapse.weight += learningRate * deltaWeight; // synapse.weight += learningRate * deltaWeight;
// BackPropagation2(derivative * synapse.weight, learningRate); //BackPropagation2(derivative * synapse.weight, learningRate);
} }
} }

23
Runtime/Scripts/Core/Synapse.cs
View File

@ -21,6 +21,9 @@ namespace NanoBrain {
/// </summary> /// </summary>
public float weight; public float weight;
/// <summary>
/// Indicator whether the weight can be trained
/// </summary>
public bool trainable = false; public bool trainable = false;
/// <summary> /// <summary>
@ -40,14 +43,28 @@ namespace NanoBrain {
case Neuron.ActivationType.Linear: case Neuron.ActivationType.Linear:
derivative *= 1; derivative *= 1;
break; break;
case Neuron.ActivationType.Power:
// untested
derivative *= 2 * math.length(this.neuron.combination);
break;
case Neuron.ActivationType.Reciprocal:
// untested
derivative *= -1 / Mathf.Pow(math.length(this.neuron.combination), 2);
break;
default: default:
Debug.Log("other activator"); Debug.Log("other activator");
break; break;
} }
derivative *= math.length(neuron.activation);
float deltaWeight = learningRate * derivative; this.neuron.BackPropagation2(derivative * this.weight, learningRate);
this.weight += deltaWeight;
derivative *= math.length(this.neuron.activation);
if (this.trainable) {
float deltaWeight = learningRate * derivative;
this.weight += deltaWeight;
}
} }
} }