From 4804e51ad7e3a56a65b8ad6f395c23596414b8e1 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Wed, 25 Oct 2023 20:41:48 +0300 Subject: [PATCH] NumRec Fixes --- .../applications/NumberRecognition.cpp | 229 ++++++++++-------- DataAnalysis/private/FullyConnectedNN.cpp | 40 ++- .../public/{FullyConnectedNN.hpp => FCNN.hpp} | 8 +- DataAnalysis/tests/Tests.cpp | 46 ++-- Utils/public/Utils.hpp | 6 +- 5 files changed, 181 insertions(+), 148 deletions(-) rename DataAnalysis/public/{FullyConnectedNN.hpp => FCNN.hpp} (84%) diff --git a/DataAnalysis/applications/NumberRecognition.cpp b/DataAnalysis/applications/NumberRecognition.cpp index dd8edd4..4aa69d9 100644 --- a/DataAnalysis/applications/NumberRecognition.cpp +++ b/DataAnalysis/applications/NumberRecognition.cpp @@ -1,4 +1,4 @@ -#include "FullyConnectedNN.hpp" +#include "FCNN.hpp" #include "LocalConnection.hpp" #include "NewPlacement.hpp" @@ -11,19 +11,6 @@ struct Dataset { Buffer> images; }; -void displayImage(const Dataset& dataset, ualni idx) { - auto& image = dataset.images[idx]; - auto label = dataset.labels[idx]; - - printf("Image : %i\n", int(label)); - for (auto i : Range(dataset.imageSize.x)) { - for (auto j : Range(dataset.imageSize.y)) { - printf("%c", image[j * dataset.imageSize.x + i]); - } - printf("\n"); - } -} - bool loadDataset(Dataset& out, const String& location) { LocalConnection dataset; dataset.connect(LocalConnection::Location(location), LocalConnection::Type(true)); @@ -59,114 +46,121 @@ bool loadDataset(Dataset& out, const String& location) { return true; } -halnf test(const Dataset& dataset, FullyConnectedNN& nn, Range range) { - ualni numFailed = 0; +struct NumberRec { + NumberRec() { + Buffer layers = { 784, 128, 10 }; + nn.initializeRandom(layers); - for (auto i : range) { - auto& image = dataset.images[i]; - auto label = dataset.labels[i]; + Dataset dataset; - Buffer results; - Buffer input; - - results.reserve(10); - input.reserve(image.size()); - - for (auto pixelIdx : Range(image.size())) { - input[pixelIdx] = (halnf) image[pixelIdx] / 255.f; + if (!loadDataset(dataset, "rsc/mnist")) { + printf("Cant Load Mnist Dataset\n"); + return; } - nn.evaluate(input, results); + mTestcases.reserve(dataset.images.size()); + for (auto i : Range(dataset.images.size())) { + auto& image = dataset.images[i]; + auto label = dataset.labels[i]; - ualni resultNumber = 0; - for (auto resIdx : Range(results.size())) { - if (results[resIdx] > results[resultNumber]) { - resultNumber = resIdx; + auto& testcase = mTestcases[i]; + + testcase.output.reserve(10); + + for (auto dig : Range(10)) { + testcase.output[dig] = label == dig ? 1 : 0; + } + + testcase.input.reserve(image.size()); + + for (auto pxl : Range(image.size())) { + testcase.input[pxl] = (halnf) image[pxl] / 255.f; } } - if (resultNumber != label) { - numFailed++; + output.reserve(10); + } + + halnf eval(ualni idx) { + nn.evaluate(mTestcases[idx].input, output); + return nn.calcCost(mTestcases[idx].output); + } + + void applyGrad(ualni idx) { + nn.calcGrad(mTestcases[idx].output); + nn.applyGrad(step); + } + + static halni getMaxIdx(const Buffer& in) { + halni out = 0; + for (auto i : Range(in.size())) { + if (in[i] > in[out]) { + out = i; + } + } + return out; + } + + bool testIncorrect(ualni idx) { + nn.evaluate(mTestcases[idx].input, output); + return getMaxIdx(mTestcases[idx].output) != getMaxIdx(output); + } + + void debLog(halni idx) { + printf("\n Got %i - ", getMaxIdx(output)); + for (auto val : output) { + printf("%f ", val.data()); + } + printf("\n Expected %i - ", getMaxIdx(mTestcases[idx].output)); + for (auto val : mTestcases[idx].output) { + printf("%f ", val.data()); + } + printf("\n\n"); + } + + void displayImage(ualni idx) { + auto& testcase = mTestcases[idx]; + printf("Image : %i\n", int(getMaxIdx(testcase.output))); + for (auto i : Range(28)) { + for (auto j : Range(28)) { + printf("%c", char(testcase.input[j * 28 + i] * 255)); + } + printf("\n"); } } - return (halnf) numFailed / (halnf) range.idxDiff(); -} + halnf test(const Range& range) { + halnf avgCost = 0; + for (auto i : range) { + avgCost += eval(i); + } + avgCost /= (halnf) range.idxDiff(); + return avgCost; + } -void testTraining(const Dataset& dataset, FullyConnectedNN& nn) { + void trainStep(const Range& range) { + nn.clearGrad(); + for (auto i : range) { + nn.evaluate(mTestcases[i].input, output); + nn.calcGrad(mTestcases[i].output); + } + nn.applyGrad(step); + } - auto propagate = [&](ualni idx) { - auto& image = dataset.images[idx]; - auto label = dataset.labels[idx]; - - Buffer results; +public: + struct Image { Buffer input; - - results.reserve(10); - input.reserve(image.size()); - - for (auto pixelIdx : Range(image.size())) { - input[pixelIdx] = (halnf) image[pixelIdx] / 255.f; - } - - nn.evaluate(input, results); - - ualni resultNumber = 0; - for (auto resIdx : Range(results.size())) { - if (results[resIdx] > results[resultNumber]) { - resultNumber = resIdx; - } - } - - Buffer resultsExpected; - resultsExpected.reserve(10); - for (auto resIdx : Range(results.size())) { - resultsExpected[resIdx] = (resIdx == label) ? 1 : 0; - } - - nn.calcGrad(resultsExpected); - - return resultNumber; + Buffer output; }; - displayImage(dataset, 2); +public: + Buffer mTestcases; - propagate(0); - // nn.applyGrad(); + FCNN nn; + Buffer output; - propagate(0); - // nn.applyGrad(); - - propagate(0); - // nn.applyGrad(); - - auto errorPercentage = test(dataset, nn, { 0, 1 }); - printf("Percentage error Trained on first image: %f\n", errorPercentage); -} - -void numRec() { - Dataset dataset; - FullyConnectedNN nn; - - // settings - Buffer layers; - layers = { 784, 128, 10 }; - halnf trainBatchPercentage = 0.1f; - halnf testSizePercentage = 0.1f; - - if (!loadDataset(dataset, "rsc/mnist")) { - printf("Cant Load Mnist Dataset\n"); - return; - } - - nn.initializeRandom(layers); - - auto errorPercentage = test(dataset, nn, { 0, 100 }); - - printf("Percentage error : %f\n", errorPercentage); - - testTraining(dataset, nn); -} + halnf step = 0.01f; +}; int main() { ModuleManifest* deps[] = { &gModuleDataAnalysis, &gModuleConnection, nullptr }; @@ -176,9 +170,32 @@ int main() { return 1; } - numRec(); + { + NumberRec app; + + auto trainRange = Range(0, 100); + auto testRange = Range(0, 100); + + halnf cost = 100; + while (cost > 0.1f) { + cost = app.test(trainRange); + app.trainStep(trainRange); + printf("Cost - %f\n", cost); + } + + auto errors = 0; + for (auto i : testRange) { + if (app.testIncorrect(i)) { + errors++; + } + // app.debLog(i); + // app.displayImage(i); + } + + printf("\n\nIncorrect - %i out of %i\n\n", errors, testRange.idxDiff()); + } module.deinitialize(); return 0; -} +} \ No newline at end of file diff --git a/DataAnalysis/private/FullyConnectedNN.cpp b/DataAnalysis/private/FullyConnectedNN.cpp index 1ddf61d..d4b3d17 100644 --- a/DataAnalysis/private/FullyConnectedNN.cpp +++ b/DataAnalysis/private/FullyConnectedNN.cpp @@ -1,6 +1,6 @@ // #include "NewPlacement.hpp" -#include "FullyConnectedNN.hpp" +#include "FCNN.hpp" #include "Utils.hpp" #include @@ -22,11 +22,13 @@ static halnf relu(halnf val) { return val < 0 ? 0 : val; } static halnf reluDerivative(halnf val) { return val < 0 ? 0 : 1; } -static halnf activationFunction(halnf val) { return sigmoid(val); } +static halnf activationFunction(halnf val) { return relu(val); } -static halnf activationFunctionDerivative(halnf val) { return sigmoidDerivative(val); } +static halnf activationFunctionDerivative(halnf val) { return reluDerivative(val); } -void FullyConnectedNN::initializeRandom(const Buffer& description) { +FCNN::FCNN(const Buffer& description) { initializeRandom(description); } + +void FCNN::initializeRandom(const Buffer& description) { ASSERT(description.size() > 1); mLayers.reserve(description.size()); @@ -39,14 +41,14 @@ void FullyConnectedNN::initializeRandom(const Buffer& description) { for (auto neuron : mLayers[i].neurons) { neuron->weights.reserve(description[i - 1]); for (auto weight : neuron->weights) { - weight->val = (halnf) (randomFloat() - 0) * 2; + weight->val = (halnf) (randomFloat() - 0.5) * 2; } - neuron->bias.val = (halnf) (randomFloat() - 0) * 2; + neuron->bias.val = (halnf) (randomFloat() - 0.5) * 2; } } } -void FullyConnectedNN::evaluate(const Buffer& input, Buffer& output) { +void FCNN::evaluate(const Buffer& input, Buffer& output) { ASSERT(output.size() == mLayers.last().neurons.size() && input.size() == mLayers.first().neurons.size()) for (auto idx : Range(input.size())) { @@ -73,7 +75,7 @@ void FullyConnectedNN::evaluate(const Buffer& input, Buffer& outpu } } -halnf FullyConnectedNN::calcCost(const Buffer& output) { +halnf FCNN::calcCost(const Buffer& output) { halnf out = 0; for (auto neuronIdx : Range(mLayers.last().neurons.size())) { out += pow(output[neuronIdx] - mLayers.last().neurons[neuronIdx].activationValue, 2); @@ -81,7 +83,21 @@ halnf FullyConnectedNN::calcCost(const Buffer& output) { return out; } -void FullyConnectedNN::calcGrad(const Buffer& output) { +void FCNN::clearGrad() { + for (auto layIdx : Range(1, (halni) mLayers.size())) { + auto& layer = mLayers[layIdx]; + + for (auto neuron : layer.neurons) { + neuron->bias.grad = 0; + for (auto weightIdx : Range(neuron->weights.size())) { + neuron->weights[weightIdx].grad = 0; + } + } + } + mAvgCount = 0; +} + +void FCNN::calcGrad(const Buffer& output) { ASSERT(mLayers.last().neurons.size() == output.size()) auto& lastLayer = mLayers.last(); @@ -113,11 +129,11 @@ void FullyConnectedNN::calcGrad(const Buffer& output) { } // gradient for current neuron bias - currentNeuron.bias.grad = currentNeuron.cache; + currentNeuron.bias.grad += currentNeuron.cache; // calculate gradient for weights of current neuron for (auto weightIdx : Range(currentNeuron.weights.size())) { - currentNeuron.weights[weightIdx].grad = inputLayer.neurons[weightIdx].activationValue * currentNeuron.cache; + currentNeuron.weights[weightIdx].grad += inputLayer.neurons[weightIdx].activationValue * currentNeuron.cache; } } } @@ -125,7 +141,7 @@ void FullyConnectedNN::calcGrad(const Buffer& output) { mAvgCount++; } -void FullyConnectedNN::applyGrad(halnf step) { +void FCNN::applyGrad(halnf step) { for (auto layIdx : Range(1, (halni) mLayers.size())) { auto& layer = mLayers[layIdx]; diff --git a/DataAnalysis/public/FullyConnectedNN.hpp b/DataAnalysis/public/FCNN.hpp similarity index 84% rename from DataAnalysis/public/FullyConnectedNN.hpp rename to DataAnalysis/public/FCNN.hpp index a699853..24a24dc 100644 --- a/DataAnalysis/public/FullyConnectedNN.hpp +++ b/DataAnalysis/public/FCNN.hpp @@ -4,7 +4,8 @@ #include "DataAnalysisCommon.hpp" namespace tp { - class FullyConnectedNN { + // Fully connected neural network + class FCNN { struct Layer { @@ -32,12 +33,15 @@ namespace tp { }; public: - FullyConnectedNN() = default; + FCNN() = default; + explicit FCNN(const Buffer& description); void initializeRandom(const Buffer& description); void evaluate(const Buffer& input, Buffer& output); halnf calcCost(const Buffer& output); + + void clearGrad(); void calcGrad(const Buffer& output); void applyGrad(halnf step); diff --git a/DataAnalysis/tests/Tests.cpp b/DataAnalysis/tests/Tests.cpp index 851c5ed..ec779ad 100644 --- a/DataAnalysis/tests/Tests.cpp +++ b/DataAnalysis/tests/Tests.cpp @@ -1,49 +1,45 @@ #include "NewPlacement.hpp" -#include "FullyConnectedNN.hpp" +#include "FCNN.hpp" #include "Testing.hpp" #include "Utils.hpp" -#include +#include -static bool init(const tp::ModuleManifest* self) { - tp::gTesting.setRootName(self->getName()); - return true; -} +using namespace tp; void test() { - using namespace tp; + Buffer layers = { 100, 70, 50, 30, 20 }; + Buffer input(layers.first()); + Buffer outputExpected(layers.last()); + Buffer output(layers.last()); - Buffer layers = { 4, 4, 3, 2 }; - Buffer input = { (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100 }; - Buffer outputExpected = { (halnf) randomFloat(), (halnf) randomFloat() }; + for (auto inputVal : Range(layers.first())) { + input[inputVal] = (halnf) randomFloat() * 100; + } - FullyConnectedNN nn; - Buffer output(2); + for (auto outIdx : Range(layers.last())) { + outputExpected[outIdx] = (halnf) randomFloat(); + } - nn.initializeRandom(layers); - - // nn.mLayers.last().neurons.first().weights = { 0.35, 0.35, 0.35, 0.35 }; - // nn.mLayers.last().neurons.first().bias = 0.9; - - // nn.mLayers.last().neurons.last().weights = { -0.35, -0.35, -0.35, -0.35 }; - // nn.mLayers.last().neurons.last().bias = -3.9; + FCNN nn(layers); + halnf steppingValue = 100; for (auto i : Range(50)) { + nn.evaluate(input, output); - - auto lossBefore = nn.calcCost(outputExpected); - + nn.calcGrad(outputExpected); - nn.applyGrad(0.1); - printf("Loss %f \n", lossBefore); + nn.applyGrad(steppingValue); + + printf("Loss %f \n", nn.calcCost(outputExpected)); } } int main() { tp::ModuleManifest* deps[] = { &tp::gModuleDataAnalysis, &tp::gModuleUtils, nullptr }; - tp::ModuleManifest testModule("DataAnalysisTest", init, nullptr, deps); + tp::ModuleManifest testModule("DataAnalysisTest", nullptr, nullptr, deps); if (!testModule.initialize()) { return 1; diff --git a/Utils/public/Utils.hpp b/Utils/public/Utils.hpp index 9cdc4f6..8c6bdb7 100644 --- a/Utils/public/Utils.hpp +++ b/Utils/public/Utils.hpp @@ -111,7 +111,7 @@ namespace tp { mBegin(pStartIndex), mEnd(pEndIndex) {} - bool valid() { return mBegin < mEnd; } + bool valid() const { return mBegin < mEnd; } tType idxBegin() const { return mBegin; } @@ -119,8 +119,8 @@ namespace tp { tType idxDiff() const { return mEnd - mBegin; } - Iterator begin() { return Iterator(mBegin); } + Iterator begin() const { return Iterator(mBegin); } - Iterator end() { return Iterator(mEnd); } + Iterator end() const { return Iterator(mEnd); } }; }