ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

BackProp Fixes

Browse source
This commit is contained in:
IlyaShurupov 2023-10-25 11:03:15 +03:00 • committed by Ilya Shurupov
parent 7e17b32a81
commit 90ce453b60
11 changed files with 189 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

21
DataAnalysis/applications/NumberRecognition.cpp
View file

@ -11,6 +11,19 @@ struct Dataset {
Buffer<Buffer<uint1>> images;
};
void displayImage(const Dataset& dataset, ualni idx) {
auto& image = dataset.images[idx];
auto label = dataset.labels[idx];
printf("Image : %i\n", int(label));
for (auto i : Range(dataset.imageSize.x)) {
for (auto j : Range(dataset.imageSize.y)) {
printf("%c", image[j * dataset.imageSize.x + i]);
}
printf("\n");
}
}
bool loadDataset(Dataset& out, const String& location) {
LocalConnection dataset;
dataset.connect(LocalConnection::Location(location), LocalConnection::Type(true));
@ -116,16 +129,16 @@ void testTraining(const Dataset& dataset, FullyConnectedNN& nn) {
return resultNumber;
};
nn.clearGrad();
displayImage(dataset, 2);
propagate(0);
nn.applyGrad();
// nn.applyGrad();
propagate(0);
nn.applyGrad();
// nn.applyGrad();
propagate(0);
nn.applyGrad();
// nn.applyGrad();
auto errorPercentage = test(dataset, nn, { 0, 1 });
printf("Percentage error Trained on first image: %f\n", errorPercentage);

BIN
DataAnalysis/applications/rsc/mnist
View file

Binary file not shown.

BIN
DataAnalysis/applications/rsc/mnist.gz Normal file
View file

Binary file not shown.

145
DataAnalysis/private/FullyConnectedNN.cpp
View file

@ -1,139 +1,142 @@
// #include "NewPlacement.hpp"
#include "FullyConnectedNN.hpp"
#include "NewPlacement.hpp"
#include "Utils.hpp"
#include <cmath>
using namespace tp;
static halnf sigmoid(halnf val) { return 0; }
static halnf linearFun(halnf val) { return val; }
static halnf linearFunDerivative(halnf val) { return 1; }
static halnf sigmoid(double val) { return 1.0f / (1.0f + (halnf) exp(-val)); }
static halnf sigmoidDerivative(double val) {
halnf sigmoid_val = sigmoid(val);
return sigmoid_val * (1.0f - sigmoid_val);
}
static halnf relu(halnf val) { return val < 0 ? 0 : val; }
static halnf reluDerivative(halnf val) { return val < 0 ? 0 : 1; }
void FullyConnectedNN::initializeRandom(Buffer<halni> description) {
static halnf activationFunction(halnf val) { return sigmoid(val); }
static halnf activationFunctionDerivative(halnf val) { return sigmoidDerivative(val); }
void FullyConnectedNN::initializeRandom(const Buffer<halni>& description) {
ASSERT(description.size() > 1);
mLayers.reserve(description.size());
for (auto i : Range<halni>(0, (halni) description.size())) {
mLayers[i].mNeurons.reserve(description[i]);
mLayers[i].neurons.reserve(description[i]);
if (i == 0) {
continue;
}
for (auto neuron : mLayers[i].mNeurons) {
neuron->mWeights.reserve(description[i - 1]);
for (auto weight : neuron->mWeights) {
weight.data() = (halnf) (randomFloat() - 0.5) * 2;
for (auto neuron : mLayers[i].neurons) {
neuron->weights.reserve(description[i - 1]);
for (auto weight : neuron->weights) {
weight->val = (halnf) (randomFloat() - 0) * 2;
}
neuron->mBias = (halnf) (randomFloat() - 0.5) * 2;
neuron->bias.val = (halnf) (randomFloat() - 0) * 2;
}
}
}
void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& output) {
ASSERT(output.size() == mLayers.last().mNeurons.size() && input.size() == mLayers.first().mNeurons.size())
ASSERT(output.size() == mLayers.last().neurons.size() && input.size() == mLayers.first().neurons.size())
for (auto idx : Range(input.size())) {
mLayers.first().mNeurons[idx].mActivationValue = input[idx];
mLayers.first().neurons[idx].activationValue = input[idx];
}
for (auto layerIdx : Range<halni>(1, (halni) mLayers.size())) {
auto& layer = mLayers[layerIdx];
auto& layerPrev = mLayers[layerIdx - 1];
for (auto neuron : layer.mNeurons) {
neuron->mActivationValue = 0;
for (auto connectionIdx : Range(neuron->mWeights.size())) {
neuron->mActivationValue += neuron->mWeights[connectionIdx] * layerPrev.mNeurons[connectionIdx].mActivationValue;
for (auto neuron : layer.neurons) {
neuron->activationValue = 0;
for (auto connectionIdx : Range(neuron->weights.size())) {
neuron->activationValue += neuron->weights[connectionIdx].val * layerPrev.neurons[connectionIdx].activationValue;
}
neuron->mActivationValue += neuron->mBias;
neuron->mActivationValueLinear = neuron->mActivationValue;
neuron->mActivationValue = relu(neuron->mActivationValue);
neuron->activationValue += neuron->bias.val;
neuron->activationValueLinear = neuron->activationValue;
neuron->activationValue = activationFunction(neuron->activationValue);
}
}
for (auto idx : Range(output.size())) {
output[idx] = mLayers.last().mNeurons[idx].mActivationValue;
output[idx] = mLayers.last().neurons[idx].activationValue;
}
}
halnf FullyConnectedNN::calcCost(const Buffer<halnf>& output) {
halnf out = 0;
for (auto neuronIdx : Range(mLayers.last().mNeurons.size())) {
out += pow(output[neuronIdx] - mLayers.last().mNeurons[neuronIdx].mActivationValue, 2);
for (auto neuronIdx : Range(mLayers.last().neurons.size())) {
out += pow(output[neuronIdx] - mLayers.last().neurons[neuronIdx].activationValue, 2);
}
return out;
}
void FullyConnectedNN::clearGrad() {
mLayersCache.reserve(mLayers.size());
for (auto layIdx : Range<halni>(0, (halni) mLayers.size())) {
mLayersCache[layIdx].mNeurons.reserve(mLayers[layIdx].mNeurons.size());
mLayersCache[layIdx].mCache = 1;
for (auto weightIdx : Range(mLayersCache[layIdx].mNeurons.size())) {
mLayersCache[layIdx].mNeurons[weightIdx].mBiasGrad = 0;
mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad.reserve(mLayers[layIdx].mNeurons[weightIdx].mWeights.size());
for (auto weight : mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad) {
weight.data() = 0;
}
}
}
mAvgCount = 0;
}
void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
ASSERT(mLayers.last().mNeurons.size() == output.size())
ASSERT(mLayers.last().neurons.size() == output.size())
auto& lastLayer = mLayers.last();
auto& lastCache = mLayersCache.last();
lastCache.mCache = 1;
for (auto neuronIdx : Range(lastLayer.mNeurons.size())) {
auto& neuronCache = lastCache.mNeurons[neuronIdx];
auto& neuron = lastLayer.mNeurons[neuronIdx];
lastCache.mCache += 2 * pow(output[neuronIdx] - neuron.mActivationValue, 2);
// calculate chaining cache value for each neuron in last layer
for (auto neuronIdx : Range(lastLayer.neurons.size())) {
auto& neuron = lastLayer.neurons[neuronIdx];
neuron.cache = 2 * (neuron.activationValue - output[neuronIdx]);
}
// calculate rest of the layers
for (auto layerIdx = mLayers.size() - 1; layerIdx > 0; layerIdx--) {
auto& layer = mLayers[layerIdx];
auto& cache = mLayersCache[layerIdx];
auto& layerNext = mLayers[layerIdx - 1];
auto& cacheNext = mLayersCache[layerIdx - 1];
auto& currentLayer = mLayers[layerIdx];
auto& inputLayer = mLayers[layerIdx - 1];
cacheNext.mCache = 1;
for (auto currentNeuronIdx : Range(currentLayer.neurons.size())) {
auto& currentNeuron = currentLayer.neurons[currentNeuronIdx];
for (auto neuronIdx : Range(layer.mNeurons.size())) {
auto& neuron = layer.mNeurons[neuronIdx];
auto& neuronCache = cache.mNeurons[neuronIdx];
// calculate cache value (chaining)
if (layerIdx != mLayers.size() - 1) {
auto& userLayer = mLayers[layerIdx + 1];
auto tmp = cache.mCache * reluDerivative(neuron.mActivationValueLinear);
neuronCache.mBiasGrad = tmp;
for (auto weightIdx : Range(neuron.mWeights.size())) {
neuronCache.mWeightsGrad[weightIdx] = tmp * layerNext.mNeurons[weightIdx].mActivationValue;
currentNeuron.cache = 0;
for (auto userNeuron : userLayer.neurons) {
currentNeuron.cache += userNeuron->weights[currentNeuronIdx].val * userNeuron->cache;
}
currentNeuron.cache *= activationFunctionDerivative(currentNeuron.activationValueLinear);
}
cacheNext.mCache += tmp;
// gradient for current neuron bias
currentNeuron.bias.grad = currentNeuron.cache;
// calculate gradient for weights of current neuron
for (auto weightIdx : Range(currentNeuron.weights.size())) {
currentNeuron.weights[weightIdx].grad = inputLayer.neurons[weightIdx].activationValue * currentNeuron.cache;
}
}
}
mAvgCount++;
}
void FullyConnectedNN::applyGrad() {
void FullyConnectedNN::applyGrad(halnf step) {
for (auto layIdx : Range<halni>(1, (halni) mLayers.size())) {
auto& layerCache = mLayersCache[layIdx];
auto& layer = mLayers[layIdx];
for (auto neuronIdx : Range(layer.mNeurons.size())) {
auto& neuron = layer.mNeurons[neuronIdx];
auto& neuronCache = layerCache.mNeurons[neuronIdx];
neuron.mBias += neuronCache.mBiasGrad / (halnf) mAvgCount;
for (auto weightIdx : Range(neuron.mWeights.size())) {
neuron.mWeights[weightIdx] += neuronCache.mWeightsGrad[weightIdx] / (halnf) mAvgCount;
for (auto neuron : layer.neurons) {
neuron->bias.val -= neuron->bias.grad / (halnf) mAvgCount * step;
for (auto weightIdx : Range(neuron->weights.size())) {
neuron->weights[weightIdx].val -= (neuron->weights[weightIdx].grad / (halnf) mAvgCount) * step;
}
}
}
mAvgCount = 0;
}

42
DataAnalysis/public/FullyConnectedNN.hpp
View file

@ -7,40 +7,42 @@ namespace tp {
class FullyConnectedNN {
struct Layer {
struct Neuron {
halnf mBias = 0;
Buffer<halnf> mWeights;
halnf mActivationValue = 0;
halnf mActivationValueLinear = 0;
struct Parameter {
Parameter() = default;
Parameter(halnf in) :
val(in) {}
halnf val = 0;
halnf grad = 0;
};
Parameter bias;
Buffer<Parameter> weights;
halnf activationValue = 0;
halnf activationValueLinear = 0;
halnf cache;
};
Buffer<Neuron> mNeurons;
};
struct LayerCache {
struct NeuronCache {
halnf mBiasGrad = 0;
Buffer<halnf> mWeightsGrad;
};
halnf mCache = 0;
Buffer<NeuronCache> mNeurons;
Buffer<Neuron> neurons;
};
public:
FullyConnectedNN() = default;
void initializeRandom(Buffer<halni> description);
void initializeRandom(const Buffer<halni>& description);
void evaluate(const Buffer<halnf>& input, Buffer<halnf>& output);
void clearGrad();
halnf calcCost(const Buffer<halnf>& output);
void calcGrad(const Buffer<halnf>& output);
void applyGrad();
void applyGrad(halnf step);
private:
public:
Buffer<Layer> mLayers;
Buffer<LayerCache> mLayersCache;
halni mAvgCount = 0;
};
};

33
DataAnalysis/tests/Tests.cpp
View file

@ -1,14 +1,45 @@
#include "NewPlacement.hpp"
#include "FullyConnectedNN.hpp"
#include "Testing.hpp"
#include "Utils.hpp"
#include <stdio.h>
static bool init(const tp::ModuleManifest* self) {
tp::gTesting.setRootName(self->getName());
return true;
}
void test() {}
void test() {
using namespace tp;
Buffer<halni> layers = { 4, 4, 3, 2 };
Buffer<halnf> input = { (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100 };
Buffer<halnf> outputExpected = { (halnf) randomFloat(), (halnf) randomFloat() };
FullyConnectedNN nn;
Buffer<halnf> output(2);
nn.initializeRandom(layers);
// nn.mLayers.last().neurons.first().weights = { 0.35, 0.35, 0.35, 0.35 };
// nn.mLayers.last().neurons.first().bias = 0.9;
// nn.mLayers.last().neurons.last().weights = { -0.35, -0.35, -0.35, -0.35 };
// nn.mLayers.last().neurons.last().bias = -3.9;
for (auto i : Range(50)) {
nn.evaluate(input, output);
auto lossBefore = nn.calcCost(outputExpected);
nn.calcGrad(outputExpected);
nn.applyGrad(0.1);
printf("Loss %f \n", lossBefore);
}
}
int main() {
tp::ModuleManifest* deps[] = { &tp::gModuleDataAnalysis, &tp::gModuleUtils, nullptr };