NanoBrain/NanoBrain-unitypackage
NanoBrain/NanoBrain-unitypackage
2
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Support more activators and bias (incorrectly)

Browse Source
This commit is contained in:
Pascal Serrarens 2026-07-03 15:39:45 +02:00
parent a38e4c3c77
commit 66c57f803c
3 changed files with 92 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Editor/ClusterEditor.cs
View File

@ -220,9 +220,16 @@ namespace NanoBrain.Unity {
Vector3 newBias = EditorGUILayout.Vector3Field("Bias", neuron2.bias);
if (newBias != neuron2.bias) {
anythingChanged |= newBias != neuron2.bias;
anythingChanged = true;
neuron2.bias = newBias;
}
bool newTrainable = EditorGUILayout.Toggle("Trainable", neuron2.trainable);
if (newTrainable != neuron2.trainable) {
anythingChanged = true;
neuron2.trainable = newTrainable;
}
EditorGUIUtility.labelWidth = previousLabelWidth;
}

100
Runtime/Scripts/Core/Neuron.cs
View File

@ -47,6 +47,11 @@ namespace NanoBrain {
//[HideInInspector]
public Vector3 bias = Vector3.zero;
/// <summary>
/// Indicator whether the bias can be trained
/// </summary>
public bool trainable = false;
#region Synapses
[SerializeField]
@ -268,6 +273,7 @@ namespace NanoBrain {
/// <param name="clone"></param>
protected virtual void CloneFields(Neuron clone) {
clone.bias = this.bias;
clone.trainable = this.trainable;
clone.persistOutput = this.persistOutput;
clone.combinator = this.combinator;
clone.activator = this.activator;
@ -660,51 +666,51 @@ namespace NanoBrain {
#region Back propagation
public void BackPropagation(Synapse synapse, Vector3 error, float learningRate) {
// Loss function:
// Mean Squared Error (MSE) 1/n * sum(errors^2)
// We use simplified here 1/2 * (error^2)
// For vectors, we need to use MSE component wise.
Vector3 loss = 0.5f * Vector3.Scale(error, error);
// public void BackPropagation(Synapse synapse, Vector3 error, float learningRate) {
// // Loss function:
// // Mean Squared Error (MSE) 1/n * sum(errors^2)
// // We use simplified here 1/2 * (error^2)
// // For vectors, we need to use MSE component wise.
// Vector3 loss = 0.5f * Vector3.Scale(error, error);
// loss is a derivative of error
// Backpropagation = loss * d(combinator)
// // loss is a derivative of error
// // Backpropagation = loss * d(combinator)
Vector3 delta2;
switch (activator) {
case ActivationType.Linear:
// Derivative of this (f'()) would be 1.
delta2 = loss * 1;
break;
case ActivationType.Power:
delta2 = loss * (2 * this.combination);
break;
case ActivationType.Reciprocal:
delta2 = loss * (-1 / (this.combination * this.combination));
break;
default:
delta2 = loss;
break;
}
// Vector3 delta2;
// switch (activator) {
// case ActivationType.Linear:
// // Derivative of this (f'()) would be 1.
// delta2 = loss * 1;
// break;
// case ActivationType.Power:
// delta2 = loss * (2 * this.combination);
// break;
// case ActivationType.Reciprocal:
// delta2 = loss * (-1 / (this.combination * this.combination));
// break;
// default:
// delta2 = loss;
// break;
// }
Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue);
float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z);
synapse.weight += learningRate * deltaWeight;
Debug.Log($"Updated weight: {error.magnitude} {error} {scaledOutput} {synapse.weight}");
}
// Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue);
// float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z);
// synapse.weight += learningRate * deltaWeight;
// Debug.Log($"Updated weight: {error.magnitude} {error} {scaledOutput} {synapse.weight}");
// }
public void BackPropagationWithLoss(Synapse synapse, Vector3 loss, float learningRate) {
Vector3 delta2 = activator switch {
ActivationType.Linear => loss * 1,
ActivationType.Power => (Vector3)(loss * (2 * this.combination)),
ActivationType.Reciprocal => (Vector3)(loss * (-1 / (this.combination * this.combination))),
_ => loss,
};
Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue);
float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z);
synapse.weight += learningRate * deltaWeight;
Debug.Log($"Updated weight: {loss.magnitude} {loss} {scaledOutput} {synapse.weight}");
}
// public void BackPropagationWithLoss(Synapse synapse, Vector3 loss, float learningRate) {
// Vector3 delta2 = activator switch {
// ActivationType.Linear => loss * 1,
// ActivationType.Power => (Vector3)(loss * (2 * this.combination)),
// ActivationType.Reciprocal => (Vector3)(loss * (-1 / (this.combination * this.combination))),
// _ => loss,
// };
// Vector3 scaledOutput = Vector3.Scale(delta2, synapse.neuron.outputValue);
// float deltaWeight = Mathf.Abs(scaledOutput.x) + Mathf.Abs(scaledOutput.y) + Mathf.Abs(scaledOutput.z);
// synapse.weight += learningRate * deltaWeight;
// Debug.Log($"Updated weight: {loss.magnitude} {loss} {scaledOutput} {synapse.weight}");
// }
// public void BackPropagation1(Vector3 cost, Vector3 error, float learningRate) {
// cost = Vector3.Scale(error, error); // error^2
@ -751,7 +757,9 @@ namespace NanoBrain {
public void BackPropagation2(float derivative, float learningRate) {
// Bias
// float3 deltaBias = derivative; // dSSR/dActivator
if (this.trainable) {
// This does not work well, because the derivative/error does not have a 3D direction
// float3 biasDerivative = derivative; // dSSR/dActivator
// switch (activator) { // dActivator/dBias
// case ActivationType.Linear:
// //deltaBias *= 1;
@ -759,9 +767,9 @@ namespace NanoBrain {
// default:
// break;
// }
// // deltaBias *= 1; // because bias is always fully applied
// Vector3 stepSize = deltaBias * learningRate;
// this.bias -= stepSize;
// Vector3 deltaBias = biasDerivative * learningRate;
// this.bias -= deltaBias;
}
foreach (Synapse synapse in this.synapses) {
synapse.BackPropagation(this, derivative, learningRate);
@ -785,7 +793,7 @@ namespace NanoBrain {
// float deltaWeight = length(deltaSynapse);
// synapse.weight += learningRate * deltaWeight;
// BackPropagation2(derivative * synapse.weight, learningRate);
//BackPropagation2(derivative * synapse.weight, learningRate);
}
}

19
Runtime/Scripts/Core/Synapse.cs
View File

@ -21,6 +21,9 @@ namespace NanoBrain {
/// </summary>
public float weight;
/// <summary>
/// Indicator whether the weight can be trained
/// </summary>
public bool trainable = false;
/// <summary>
@ -40,15 +43,29 @@ namespace NanoBrain {
case Neuron.ActivationType.Linear:
derivative *= 1;
break;
case Neuron.ActivationType.Power:
// untested
derivative *= 2 * math.length(this.neuron.combination);
break;
case Neuron.ActivationType.Reciprocal:
// untested
derivative *= -1 / Mathf.Pow(math.length(this.neuron.combination), 2);
break;
default:
Debug.Log("other activator");
break;
}
derivative *= math.length(neuron.activation);
this.neuron.BackPropagation2(derivative * this.weight, learningRate);
derivative *= math.length(this.neuron.activation);
if (this.trainable) {
float deltaWeight = learningRate * derivative;
this.weight += deltaWeight;
}
}
}
}