ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

BackProp Initial

Browse source
This commit is contained in:
IlyaShurupov 2023-10-23 13:38:00 +03:00 • committed by Ilya Shurupov
parent 91fb72bd49
commit 7e17b32a81
6 changed files with 166 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

60
DataAnalysis/applications/NumberRecognition.cpp
View file

@ -80,23 +80,79 @@ halnf test(const Dataset& dataset, FullyConnectedNN& nn, Range<ualni> range) {
return (halnf) numFailed / (halnf) range.idxDiff();
}
void testTraining(const Dataset& dataset, FullyConnectedNN& nn) {
auto propagate = [&](ualni idx) {
auto& image = dataset.images[idx];
auto label = dataset.labels[idx];
Buffer<halnf> results;
Buffer<halnf> input;
results.reserve(10);
input.reserve(image.size());
for (auto pixelIdx : Range(image.size())) {
input[pixelIdx] = (halnf) image[pixelIdx] / 255.f;
}
nn.evaluate(input, results);
ualni resultNumber = 0;
for (auto resIdx : Range(results.size())) {
if (results[resIdx] > results[resultNumber]) {
resultNumber = resIdx;
}
}
Buffer<halnf> resultsExpected;
resultsExpected.reserve(10);
for (auto resIdx : Range(results.size())) {
resultsExpected[resIdx] = (resIdx == label) ? 1 : 0;
}
nn.calcGrad(resultsExpected);
return resultNumber;
};
nn.clearGrad();
propagate(0);
nn.applyGrad();
propagate(0);
nn.applyGrad();
propagate(0);
nn.applyGrad();
auto errorPercentage = test(dataset, nn, { 0, 1 });
printf("Percentage error Trained on first image: %f\n", errorPercentage);
}
void numRec() {
Dataset dataset;
FullyConnectedNN nn;
// settings
Buffer<halni> layers;
layers = { 784, 128, 10 };
nn.initializeRandom(layers);
halnf trainBatchPercentage = 0.1f;
halnf testSizePercentage = 0.1f;
if (!loadDataset(dataset, "rsc/mnist")) {
printf("Cant Load Mnist Dataset\n");
return;
}
nn.initializeRandom(layers);
auto errorPercentage = test(dataset, nn, { 0, 100 });
printf("Percentage error : %f\n", errorPercentage);
testTraining(dataset, nn);
}
int main() {

BIN
DataAnalysis/applications/rsc/mnist Normal file
View file

Binary file not shown.

BIN
DataAnalysis/applications/rsc/mnist.gz
View file

Binary file not shown.

90
DataAnalysis/private/FullyConnectedNN.cpp
View file

@ -8,6 +8,8 @@ static halnf sigmoid(halnf val) { return 0; }
static halnf relu(halnf val) { return val < 0 ? 0 : val; }
static halnf reluDerivative(halnf val) { return val < 0 ? 0 : 1; }
void FullyConnectedNN::initializeRandom(Buffer<halni> description) {
mLayers.reserve(description.size());
@ -19,9 +21,9 @@ void FullyConnectedNN::initializeRandom(Buffer<halni> description) {
for (auto neuron : mLayers[i].mNeurons) {
neuron->mWeights.reserve(description[i - 1]);
for (auto weight : neuron->mWeights) {
weight.data() = (halnf) randomFloat();
weight.data() = (halnf) (randomFloat() - 0.5) * 2;
}
neuron->mBias = (halnf) randomFloat();
neuron->mBias = (halnf) (randomFloat() - 0.5) * 2;
}
}
}
@ -43,6 +45,7 @@ void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& outpu
neuron->mActivationValue += neuron->mWeights[connectionIdx] * layerPrev.mNeurons[connectionIdx].mActivationValue;
}
neuron->mActivationValue += neuron->mBias;
neuron->mActivationValueLinear = neuron->mActivationValue;
neuron->mActivationValue = relu(neuron->mActivationValue);
}
}
@ -51,3 +54,86 @@ void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& outpu
output[idx] = mLayers.last().mNeurons[idx].mActivationValue;
}
}
halnf FullyConnectedNN::calcCost(const Buffer<halnf>& output) {
halnf out = 0;
for (auto neuronIdx : Range(mLayers.last().mNeurons.size())) {
out += pow(output[neuronIdx] - mLayers.last().mNeurons[neuronIdx].mActivationValue, 2);
}
return out;
}
void FullyConnectedNN::clearGrad() {
mLayersCache.reserve(mLayers.size());
for (auto layIdx : Range<halni>(0, (halni) mLayers.size())) {
mLayersCache[layIdx].mNeurons.reserve(mLayers[layIdx].mNeurons.size());
mLayersCache[layIdx].mCache = 1;
for (auto weightIdx : Range(mLayersCache[layIdx].mNeurons.size())) {
mLayersCache[layIdx].mNeurons[weightIdx].mBiasGrad = 0;
mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad.reserve(mLayers[layIdx].mNeurons[weightIdx].mWeights.size());
for (auto weight : mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad) {
weight.data() = 0;
}
}
}
mAvgCount = 0;
}
void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
ASSERT(mLayers.last().mNeurons.size() == output.size())
auto& lastLayer = mLayers.last();
auto& lastCache = mLayersCache.last();
lastCache.mCache = 1;
for (auto neuronIdx : Range(lastLayer.mNeurons.size())) {
auto& neuronCache = lastCache.mNeurons[neuronIdx];
auto& neuron = lastLayer.mNeurons[neuronIdx];
lastCache.mCache += 2 * pow(output[neuronIdx] - neuron.mActivationValue, 2);
}
for (auto layerIdx = mLayers.size() - 1; layerIdx > 0; layerIdx--) {
auto& layer = mLayers[layerIdx];
auto& cache = mLayersCache[layerIdx];
auto& layerNext = mLayers[layerIdx - 1];
auto& cacheNext = mLayersCache[layerIdx - 1];
cacheNext.mCache = 1;
for (auto neuronIdx : Range(layer.mNeurons.size())) {
auto& neuron = layer.mNeurons[neuronIdx];
auto& neuronCache = cache.mNeurons[neuronIdx];
auto tmp = cache.mCache * reluDerivative(neuron.mActivationValueLinear);
neuronCache.mBiasGrad = tmp;
for (auto weightIdx : Range(neuron.mWeights.size())) {
neuronCache.mWeightsGrad[weightIdx] = tmp * layerNext.mNeurons[weightIdx].mActivationValue;
}
cacheNext.mCache += tmp;
}
}
}
void FullyConnectedNN::applyGrad() {
for (auto layIdx : Range<halni>(1, (halni) mLayers.size())) {
auto& layerCache = mLayersCache[layIdx];
auto& layer = mLayers[layIdx];
for (auto neuronIdx : Range(layer.mNeurons.size())) {
auto& neuron = layer.mNeurons[neuronIdx];
auto& neuronCache = layerCache.mNeurons[neuronIdx];
neuron.mBias += neuronCache.mBiasGrad / (halnf) mAvgCount;
for (auto weightIdx : Range(neuron.mWeights.size())) {
neuron.mWeights[weightIdx] += neuronCache.mWeightsGrad[weightIdx] / (halnf) mAvgCount;
}
}
}
}

19
DataAnalysis/public/FullyConnectedNN.hpp
View file

@ -5,23 +5,42 @@
namespace tp {
class FullyConnectedNN {
struct Layer {
struct Neuron {
halnf mBias = 0;
Buffer<halnf> mWeights;
halnf mActivationValue = 0;
halnf mActivationValueLinear = 0;
};
Buffer<Neuron> mNeurons;
};
struct LayerCache {
struct NeuronCache {
halnf mBiasGrad = 0;
Buffer<halnf> mWeightsGrad;
};
halnf mCache = 0;
Buffer<NeuronCache> mNeurons;
};
public:
FullyConnectedNN() = default;
void initializeRandom(Buffer<halni> description);
void evaluate(const Buffer<halnf>& input, Buffer<halnf>& output);
void clearGrad();
halnf calcCost(const Buffer<halnf>& output);
void calcGrad(const Buffer<halnf>& output);
void applyGrad();
private:
Buffer<Layer> mLayers;
Buffer<LayerCache> mLayersCache;
halni mAvgCount = 0;
};
};