ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NumRec Fixes

Browse source
This commit is contained in:
IlyaShurupov 2023-10-25 20:41:48 +03:00
parent 0542b7ba2e
commit 826713c212
5 changed files with 181 additions and 148 deletions
Download patch file Download diff file Expand all files Collapse all files

229
DataAnalysis/applications/NumberRecognition.cpp
View file

@ -1,4 +1,4 @@
#include "FullyConnectedNN.hpp"
#include "FCNN.hpp"
#include "LocalConnection.hpp"
#include "NewPlacement.hpp"
@ -11,19 +11,6 @@ struct Dataset {
Buffer<Buffer<uint1>> images;
};
void displayImage(const Dataset& dataset, ualni idx) {
auto& image = dataset.images[idx];
auto label = dataset.labels[idx];
printf("Image : %i\n", int(label));
for (auto i : Range(dataset.imageSize.x)) {
for (auto j : Range(dataset.imageSize.y)) {
printf("%c", image[j * dataset.imageSize.x + i]);
}
printf("\n");
}
}
bool loadDataset(Dataset& out, const String& location) {
LocalConnection dataset;
dataset.connect(LocalConnection::Location(location), LocalConnection::Type(true));
@ -59,114 +46,121 @@ bool loadDataset(Dataset& out, const String& location) {
return true;
}
halnf test(const Dataset& dataset, FullyConnectedNN& nn, Range<ualni> range) {
ualni numFailed = 0;
struct NumberRec {
NumberRec() {
Buffer<halni> layers = { 784, 128, 10 };
nn.initializeRandom(layers);
for (auto i : range) {
auto& image = dataset.images[i];
auto label = dataset.labels[i];
Dataset dataset;
Buffer<halnf> results;
Buffer<halnf> input;
results.reserve(10);
input.reserve(image.size());
for (auto pixelIdx : Range(image.size())) {
input[pixelIdx] = (halnf) image[pixelIdx] / 255.f;
if (!loadDataset(dataset, "rsc/mnist")) {
printf("Cant Load Mnist Dataset\n");
return;
}
nn.evaluate(input, results);
mTestcases.reserve(dataset.images.size());
for (auto i : Range(dataset.images.size())) {
auto& image = dataset.images[i];
auto label = dataset.labels[i];
ualni resultNumber = 0;
for (auto resIdx : Range(results.size())) {
if (results[resIdx] > results[resultNumber]) {
resultNumber = resIdx;
auto& testcase = mTestcases[i];
testcase.output.reserve(10);
for (auto dig : Range(10)) {
testcase.output[dig] = label == dig ? 1 : 0;
}
testcase.input.reserve(image.size());
for (auto pxl : Range(image.size())) {
testcase.input[pxl] = (halnf) image[pxl] / 255.f;
}
}
if (resultNumber != label) {
numFailed++;
output.reserve(10);
}
halnf eval(ualni idx) {
nn.evaluate(mTestcases[idx].input, output);
return nn.calcCost(mTestcases[idx].output);
}
void applyGrad(ualni idx) {
nn.calcGrad(mTestcases[idx].output);
nn.applyGrad(step);
}
static halni getMaxIdx(const Buffer<halnf>& in) {
halni out = 0;
for (auto i : Range(in.size())) {
if (in[i] > in[out]) {
out = i;
}
}
return out;
}
bool testIncorrect(ualni idx) {
nn.evaluate(mTestcases[idx].input, output);
return getMaxIdx(mTestcases[idx].output) != getMaxIdx(output);
}
void debLog(halni idx) {
printf("\n Got %i - ", getMaxIdx(output));
for (auto val : output) {
printf("%f ", val.data());
}
printf("\n Expected %i - ", getMaxIdx(mTestcases[idx].output));
for (auto val : mTestcases[idx].output) {
printf("%f ", val.data());
}
printf("\n\n");
}
void displayImage(ualni idx) {
auto& testcase = mTestcases[idx];
printf("Image : %i\n", int(getMaxIdx(testcase.output)));
for (auto i : Range(28)) {
for (auto j : Range(28)) {
printf("%c", char(testcase.input[j * 28 + i] * 255));
}
printf("\n");
}
}
return (halnf) numFailed / (halnf) range.idxDiff();
}
halnf test(const Range<halni>& range) {
halnf avgCost = 0;
for (auto i : range) {
avgCost += eval(i);
}
avgCost /= (halnf) range.idxDiff();
return avgCost;
}
void testTraining(const Dataset& dataset, FullyConnectedNN& nn) {
void trainStep(const Range<halni>& range) {
nn.clearGrad();
for (auto i : range) {
nn.evaluate(mTestcases[i].input, output);
nn.calcGrad(mTestcases[i].output);
}
nn.applyGrad(step);
}
auto propagate = [&](ualni idx) {
auto& image = dataset.images[idx];
auto label = dataset.labels[idx];
Buffer<halnf> results;
public:
struct Image {
Buffer<halnf> input;
results.reserve(10);
input.reserve(image.size());
for (auto pixelIdx : Range(image.size())) {
input[pixelIdx] = (halnf) image[pixelIdx] / 255.f;
}
nn.evaluate(input, results);
ualni resultNumber = 0;
for (auto resIdx : Range(results.size())) {
if (results[resIdx] > results[resultNumber]) {
resultNumber = resIdx;
}
}
Buffer<halnf> resultsExpected;
resultsExpected.reserve(10);
for (auto resIdx : Range(results.size())) {
resultsExpected[resIdx] = (resIdx == label) ? 1 : 0;
}
nn.calcGrad(resultsExpected);
return resultNumber;
Buffer<halnf> output;
};
displayImage(dataset, 2);
public:
Buffer<Image> mTestcases;
propagate(0);
// nn.applyGrad();
FCNN nn;
Buffer<halnf> output;
propagate(0);
// nn.applyGrad();
propagate(0);
// nn.applyGrad();
auto errorPercentage = test(dataset, nn, { 0, 1 });
printf("Percentage error Trained on first image: %f\n", errorPercentage);
}
void numRec() {
Dataset dataset;
FullyConnectedNN nn;
// settings
Buffer<halni> layers;
layers = { 784, 128, 10 };
halnf trainBatchPercentage = 0.1f;
halnf testSizePercentage = 0.1f;
if (!loadDataset(dataset, "rsc/mnist")) {
printf("Cant Load Mnist Dataset\n");
return;
}
nn.initializeRandom(layers);
auto errorPercentage = test(dataset, nn, { 0, 100 });
printf("Percentage error : %f\n", errorPercentage);
testTraining(dataset, nn);
}
halnf step = 0.01f;
};
int main() {
ModuleManifest* deps[] = { &gModuleDataAnalysis, &gModuleConnection, nullptr };
@ -176,9 +170,32 @@ int main() {
return 1;
}
numRec();
{
NumberRec app;
auto trainRange = Range(0, 100);
auto testRange = Range(0, 100);
halnf cost = 100;
while (cost > 0.1f) {
cost = app.test(trainRange);
app.trainStep(trainRange);
printf("Cost - %f\n", cost);
}
auto errors = 0;
for (auto i : testRange) {
if (app.testIncorrect(i)) {
errors++;
}
// app.debLog(i);
// app.displayImage(i);
}
printf("\n\nIncorrect - %i out of %i\n\n", errors, testRange.idxDiff());
}
module.deinitialize();
return 0;
}
}

40
DataAnalysis/private/FullyConnectedNN.cpp
View file

@ -1,6 +1,6 @@
// #include "NewPlacement.hpp"
#include "FullyConnectedNN.hpp"
#include "FCNN.hpp"
#include "Utils.hpp"
#include <cmath>
@ -22,11 +22,13 @@ static halnf relu(halnf val) { return val < 0 ? 0 : val; }
static halnf reluDerivative(halnf val) { return val < 0 ? 0 : 1; }
static halnf activationFunction(halnf val) { return sigmoid(val); }
static halnf activationFunction(halnf val) { return relu(val); }
static halnf activationFunctionDerivative(halnf val) { return sigmoidDerivative(val); }
static halnf activationFunctionDerivative(halnf val) { return reluDerivative(val); }
void FullyConnectedNN::initializeRandom(const Buffer<halni>& description) {
FCNN::FCNN(const Buffer<halni>& description) { initializeRandom(description); }
void FCNN::initializeRandom(const Buffer<halni>& description) {
ASSERT(description.size() > 1);
mLayers.reserve(description.size());
@ -39,14 +41,14 @@ void FullyConnectedNN::initializeRandom(const Buffer<halni>& description) {
for (auto neuron : mLayers[i].neurons) {
neuron->weights.reserve(description[i - 1]);
for (auto weight : neuron->weights) {
weight->val = (halnf) (randomFloat() - 0) * 2;
weight->val = (halnf) (randomFloat() - 0.5) * 2;
}
neuron->bias.val = (halnf) (randomFloat() - 0) * 2;
neuron->bias.val = (halnf) (randomFloat() - 0.5) * 2;
}
}
}
void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& output) {
void FCNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& output) {
ASSERT(output.size() == mLayers.last().neurons.size() && input.size() == mLayers.first().neurons.size())
for (auto idx : Range(input.size())) {
@ -73,7 +75,7 @@ void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& outpu
}
}
halnf FullyConnectedNN::calcCost(const Buffer<halnf>& output) {
halnf FCNN::calcCost(const Buffer<halnf>& output) {
halnf out = 0;
for (auto neuronIdx : Range(mLayers.last().neurons.size())) {
out += pow(output[neuronIdx] - mLayers.last().neurons[neuronIdx].activationValue, 2);
@ -81,7 +83,21 @@ halnf FullyConnectedNN::calcCost(const Buffer<halnf>& output) {
return out;
}
void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
void FCNN::clearGrad() {
for (auto layIdx : Range<halni>(1, (halni) mLayers.size())) {
auto& layer = mLayers[layIdx];
for (auto neuron : layer.neurons) {
neuron->bias.grad = 0;
for (auto weightIdx : Range(neuron->weights.size())) {
neuron->weights[weightIdx].grad = 0;
}
}
}
mAvgCount = 0;
}
void FCNN::calcGrad(const Buffer<halnf>& output) {
ASSERT(mLayers.last().neurons.size() == output.size())
auto& lastLayer = mLayers.last();
@ -113,11 +129,11 @@ void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
}
// gradient for current neuron bias
currentNeuron.bias.grad = currentNeuron.cache;
currentNeuron.bias.grad += currentNeuron.cache;
// calculate gradient for weights of current neuron
for (auto weightIdx : Range(currentNeuron.weights.size())) {
currentNeuron.weights[weightIdx].grad = inputLayer.neurons[weightIdx].activationValue * currentNeuron.cache;
currentNeuron.weights[weightIdx].grad += inputLayer.neurons[weightIdx].activationValue * currentNeuron.cache;
}
}
}
@ -125,7 +141,7 @@ void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
mAvgCount++;
}
void FullyConnectedNN::applyGrad(halnf step) {
void FCNN::applyGrad(halnf step) {
for (auto layIdx : Range<halni>(1, (halni) mLayers.size())) {
auto& layer = mLayers[layIdx];

8
DataAnalysis/public/FullyConnectedNN.hpp → DataAnalysis/public/FCNN.hpp
View file

@ -4,7 +4,8 @@
#include "DataAnalysisCommon.hpp"
namespace tp {
class FullyConnectedNN {
// Fully connected neural network
class FCNN {
struct Layer {
@ -32,12 +33,15 @@ namespace tp {
};
public:
FullyConnectedNN() = default;
FCNN() = default;
explicit FCNN(const Buffer<halni>& description);
void initializeRandom(const Buffer<halni>& description);
void evaluate(const Buffer<halnf>& input, Buffer<halnf>& output);
halnf calcCost(const Buffer<halnf>& output);
void clearGrad();
void calcGrad(const Buffer<halnf>& output);
void applyGrad(halnf step);

46
DataAnalysis/tests/Tests.cpp
View file

@ -1,49 +1,45 @@
#include "NewPlacement.hpp"
#include "FullyConnectedNN.hpp"
#include "FCNN.hpp"
#include "Testing.hpp"
#include "Utils.hpp"
#include <stdio.h>
#include <cstdio>
static bool init(const tp::ModuleManifest* self) {
tp::gTesting.setRootName(self->getName());
return true;
}
using namespace tp;
void test() {
using namespace tp;
Buffer<halni> layers = { 100, 70, 50, 30, 20 };
Buffer<halnf> input(layers.first());
Buffer<halnf> outputExpected(layers.last());
Buffer<halnf> output(layers.last());
Buffer<halni> layers = { 4, 4, 3, 2 };
Buffer<halnf> input = { (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100 };
Buffer<halnf> outputExpected = { (halnf) randomFloat(), (halnf) randomFloat() };
for (auto inputVal : Range(layers.first())) {
input[inputVal] = (halnf) randomFloat() * 100;
}
FullyConnectedNN nn;
Buffer<halnf> output(2);
for (auto outIdx : Range(layers.last())) {
outputExpected[outIdx] = (halnf) randomFloat();
}
nn.initializeRandom(layers);
// nn.mLayers.last().neurons.first().weights = { 0.35, 0.35, 0.35, 0.35 };
// nn.mLayers.last().neurons.first().bias = 0.9;
// nn.mLayers.last().neurons.last().weights = { -0.35, -0.35, -0.35, -0.35 };
// nn.mLayers.last().neurons.last().bias = -3.9;
FCNN nn(layers);
halnf steppingValue = 100;
for (auto i : Range(50)) {
nn.evaluate(input, output);
auto lossBefore = nn.calcCost(outputExpected);
nn.calcGrad(outputExpected);
nn.applyGrad(0.1);
printf("Loss %f \n", lossBefore);
nn.applyGrad(steppingValue);
printf("Loss %f \n", nn.calcCost(outputExpected));
}
}
int main() {
tp::ModuleManifest* deps[] = { &tp::gModuleDataAnalysis, &tp::gModuleUtils, nullptr };
tp::ModuleManifest testModule("DataAnalysisTest", init, nullptr, deps);
tp::ModuleManifest testModule("DataAnalysisTest", nullptr, nullptr, deps);
if (!testModule.initialize()) {
return 1;