diff --git a/Containers/public/Buffer.hpp b/Containers/public/Buffer.hpp index dddd7ca..8744be4 100644 --- a/Containers/public/Buffer.hpp +++ b/Containers/public/Buffer.hpp @@ -320,6 +320,7 @@ namespace tp { } void reserve(ualni aSize) { + if (aSize == mSize) return; for (ualni i = 0; i < mLoad; i++) { mBuff[i].~tType(); } diff --git a/DataAnalysis/applications/NumberRecognition.cpp b/DataAnalysis/applications/NumberRecognition.cpp index 65c750f..a9e8cf0 100644 --- a/DataAnalysis/applications/NumberRecognition.cpp +++ b/DataAnalysis/applications/NumberRecognition.cpp @@ -80,23 +80,79 @@ halnf test(const Dataset& dataset, FullyConnectedNN& nn, Range range) { return (halnf) numFailed / (halnf) range.idxDiff(); } +void testTraining(const Dataset& dataset, FullyConnectedNN& nn) { + + auto propagate = [&](ualni idx) { + auto& image = dataset.images[idx]; + auto label = dataset.labels[idx]; + + Buffer results; + Buffer input; + + results.reserve(10); + input.reserve(image.size()); + + for (auto pixelIdx : Range(image.size())) { + input[pixelIdx] = (halnf) image[pixelIdx] / 255.f; + } + + nn.evaluate(input, results); + + ualni resultNumber = 0; + for (auto resIdx : Range(results.size())) { + if (results[resIdx] > results[resultNumber]) { + resultNumber = resIdx; + } + } + + Buffer resultsExpected; + resultsExpected.reserve(10); + for (auto resIdx : Range(results.size())) { + resultsExpected[resIdx] = (resIdx == label) ? 1 : 0; + } + + nn.calcGrad(resultsExpected); + + return resultNumber; + }; + + nn.clearGrad(); + + propagate(0); + nn.applyGrad(); + + propagate(0); + nn.applyGrad(); + + propagate(0); + nn.applyGrad(); + + auto errorPercentage = test(dataset, nn, { 0, 1 }); + printf("Percentage error Trained on first image: %f\n", errorPercentage); +} + void numRec() { Dataset dataset; FullyConnectedNN nn; + // settings Buffer layers; layers = { 784, 128, 10 }; - - nn.initializeRandom(layers); + halnf trainBatchPercentage = 0.1f; + halnf testSizePercentage = 0.1f; if (!loadDataset(dataset, "rsc/mnist")) { printf("Cant Load Mnist Dataset\n"); return; } + nn.initializeRandom(layers); + auto errorPercentage = test(dataset, nn, { 0, 100 }); printf("Percentage error : %f\n", errorPercentage); + + testTraining(dataset, nn); } int main() { diff --git a/DataAnalysis/applications/rsc/mnist b/DataAnalysis/applications/rsc/mnist new file mode 100644 index 0000000..e743796 Binary files /dev/null and b/DataAnalysis/applications/rsc/mnist differ diff --git a/DataAnalysis/applications/rsc/mnist.gz b/DataAnalysis/applications/rsc/mnist.gz deleted file mode 100644 index abb32ac..0000000 Binary files a/DataAnalysis/applications/rsc/mnist.gz and /dev/null differ diff --git a/DataAnalysis/private/FullyConnectedNN.cpp b/DataAnalysis/private/FullyConnectedNN.cpp index c9c0654..be49045 100644 --- a/DataAnalysis/private/FullyConnectedNN.cpp +++ b/DataAnalysis/private/FullyConnectedNN.cpp @@ -8,6 +8,8 @@ static halnf sigmoid(halnf val) { return 0; } static halnf relu(halnf val) { return val < 0 ? 0 : val; } +static halnf reluDerivative(halnf val) { return val < 0 ? 0 : 1; } + void FullyConnectedNN::initializeRandom(Buffer description) { mLayers.reserve(description.size()); @@ -19,9 +21,9 @@ void FullyConnectedNN::initializeRandom(Buffer description) { for (auto neuron : mLayers[i].mNeurons) { neuron->mWeights.reserve(description[i - 1]); for (auto weight : neuron->mWeights) { - weight.data() = (halnf) randomFloat(); + weight.data() = (halnf) (randomFloat() - 0.5) * 2; } - neuron->mBias = (halnf) randomFloat(); + neuron->mBias = (halnf) (randomFloat() - 0.5) * 2; } } } @@ -43,6 +45,7 @@ void FullyConnectedNN::evaluate(const Buffer& input, Buffer& outpu neuron->mActivationValue += neuron->mWeights[connectionIdx] * layerPrev.mNeurons[connectionIdx].mActivationValue; } neuron->mActivationValue += neuron->mBias; + neuron->mActivationValueLinear = neuron->mActivationValue; neuron->mActivationValue = relu(neuron->mActivationValue); } } @@ -51,3 +54,86 @@ void FullyConnectedNN::evaluate(const Buffer& input, Buffer& outpu output[idx] = mLayers.last().mNeurons[idx].mActivationValue; } } + +halnf FullyConnectedNN::calcCost(const Buffer& output) { + halnf out = 0; + for (auto neuronIdx : Range(mLayers.last().mNeurons.size())) { + out += pow(output[neuronIdx] - mLayers.last().mNeurons[neuronIdx].mActivationValue, 2); + } + return out; +} + +void FullyConnectedNN::clearGrad() { + mLayersCache.reserve(mLayers.size()); + for (auto layIdx : Range(0, (halni) mLayers.size())) { + mLayersCache[layIdx].mNeurons.reserve(mLayers[layIdx].mNeurons.size()); + mLayersCache[layIdx].mCache = 1; + + for (auto weightIdx : Range(mLayersCache[layIdx].mNeurons.size())) { + mLayersCache[layIdx].mNeurons[weightIdx].mBiasGrad = 0; + mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad.reserve(mLayers[layIdx].mNeurons[weightIdx].mWeights.size()); + for (auto weight : mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad) { + weight.data() = 0; + } + } + } + mAvgCount = 0; +} + +void FullyConnectedNN::calcGrad(const Buffer& output) { + ASSERT(mLayers.last().mNeurons.size() == output.size()) + + auto& lastLayer = mLayers.last(); + auto& lastCache = mLayersCache.last(); + + lastCache.mCache = 1; + + for (auto neuronIdx : Range(lastLayer.mNeurons.size())) { + auto& neuronCache = lastCache.mNeurons[neuronIdx]; + auto& neuron = lastLayer.mNeurons[neuronIdx]; + lastCache.mCache += 2 * pow(output[neuronIdx] - neuron.mActivationValue, 2); + } + + for (auto layerIdx = mLayers.size() - 1; layerIdx > 0; layerIdx--) { + auto& layer = mLayers[layerIdx]; + auto& cache = mLayersCache[layerIdx]; + + auto& layerNext = mLayers[layerIdx - 1]; + auto& cacheNext = mLayersCache[layerIdx - 1]; + + cacheNext.mCache = 1; + + for (auto neuronIdx : Range(layer.mNeurons.size())) { + auto& neuron = layer.mNeurons[neuronIdx]; + auto& neuronCache = cache.mNeurons[neuronIdx]; + + auto tmp = cache.mCache * reluDerivative(neuron.mActivationValueLinear); + + neuronCache.mBiasGrad = tmp; + + for (auto weightIdx : Range(neuron.mWeights.size())) { + neuronCache.mWeightsGrad[weightIdx] = tmp * layerNext.mNeurons[weightIdx].mActivationValue; + } + + cacheNext.mCache += tmp; + } + } +} + +void FullyConnectedNN::applyGrad() { + for (auto layIdx : Range(1, (halni) mLayers.size())) { + auto& layerCache = mLayersCache[layIdx]; + auto& layer = mLayers[layIdx]; + + for (auto neuronIdx : Range(layer.mNeurons.size())) { + auto& neuron = layer.mNeurons[neuronIdx]; + auto& neuronCache = layerCache.mNeurons[neuronIdx]; + + neuron.mBias += neuronCache.mBiasGrad / (halnf) mAvgCount; + + for (auto weightIdx : Range(neuron.mWeights.size())) { + neuron.mWeights[weightIdx] += neuronCache.mWeightsGrad[weightIdx] / (halnf) mAvgCount; + } + } + } +} \ No newline at end of file diff --git a/DataAnalysis/public/FullyConnectedNN.hpp b/DataAnalysis/public/FullyConnectedNN.hpp index 5a1ceae..580e3e8 100644 --- a/DataAnalysis/public/FullyConnectedNN.hpp +++ b/DataAnalysis/public/FullyConnectedNN.hpp @@ -5,23 +5,42 @@ namespace tp { class FullyConnectedNN { + struct Layer { struct Neuron { halnf mBias = 0; Buffer mWeights; halnf mActivationValue = 0; + halnf mActivationValueLinear = 0; }; Buffer mNeurons; }; + struct LayerCache { + struct NeuronCache { + halnf mBiasGrad = 0; + Buffer mWeightsGrad; + }; + + halnf mCache = 0; + Buffer mNeurons; + }; + public: FullyConnectedNN() = default; void initializeRandom(Buffer description); void evaluate(const Buffer& input, Buffer& output); + void clearGrad(); + halnf calcCost(const Buffer& output); + void calcGrad(const Buffer& output); + void applyGrad(); + private: Buffer mLayers; + Buffer mLayersCache; + halni mAvgCount = 0; }; };