NumRec Fixes

DataAnalysis/applications/NumberRecognition.cpp

@@ -1,4 +1,4 @@
-#include "FullyConnectedNN.hpp"
+#include "FCNN.hpp"
 #include "LocalConnection.hpp"
 #include "NewPlacement.hpp"
 
@@ -11,19 +11,6 @@ struct Dataset {
 	Buffer<Buffer<uint1>> images;
 };
 
-void displayImage(const Dataset& dataset, ualni idx) {
-	auto& image = dataset.images[idx];
-	auto label = dataset.labels[idx];
-
-	printf("Image : %i\n", int(label));
-	for (auto i : Range(dataset.imageSize.x)) {
-		for (auto j : Range(dataset.imageSize.y)) {
-			printf("%c", image[j * dataset.imageSize.x + i]);
-		}
-		printf("\n");
-	}
-}
-
 bool loadDataset(Dataset& out, const String& location) {
 	LocalConnection dataset;
 	dataset.connect(LocalConnection::Location(location), LocalConnection::Type(true));
@@ -59,114 +46,121 @@ bool loadDataset(Dataset& out, const String& location) {
 	return true;
 }
 
-halnf test(const Dataset& dataset, FullyConnectedNN& nn, Range<ualni> range) {
+struct NumberRec {
-	ualni numFailed = 0;
+	NumberRec() {
+		Buffer<halni> layers = { 784, 128, 10 };
+		nn.initializeRandom(layers);
 
-	for (auto i : range) {
+		Dataset dataset;
-		auto& image = dataset.images[i];
-		auto label = dataset.labels[i];
 
-		Buffer<halnf> results;
+		if (!loadDataset(dataset, "rsc/mnist")) {
-		Buffer<halnf> input;
+			printf("Cant Load Mnist Dataset\n");
-
+			return;
-		results.reserve(10);
-		input.reserve(image.size());
-
-		for (auto pixelIdx : Range(image.size())) {
-			input[pixelIdx] = (halnf) image[pixelIdx] / 255.f;
 		}
 
-		nn.evaluate(input, results);
+		mTestcases.reserve(dataset.images.size());
+		for (auto i : Range(dataset.images.size())) {
+			auto& image = dataset.images[i];
+			auto label = dataset.labels[i];
 
-		ualni resultNumber = 0;
+			auto& testcase = mTestcases[i];
-		for (auto resIdx : Range(results.size())) {
+
-			if (results[resIdx] > results[resultNumber]) {
+			testcase.output.reserve(10);
-				resultNumber = resIdx;
+
+			for (auto dig : Range(10)) {
+				testcase.output[dig] = label == dig ? 1 : 0;
+			}
+
+			testcase.input.reserve(image.size());
+
+			for (auto pxl : Range(image.size())) {
+				testcase.input[pxl] = (halnf) image[pxl] / 255.f;
 			}
 		}
 
-		if (resultNumber != label) {
+		output.reserve(10);
-			numFailed++;
+	}
+
+	halnf eval(ualni idx) {
+		nn.evaluate(mTestcases[idx].input, output);
+		return nn.calcCost(mTestcases[idx].output);
+	}
+
+	void applyGrad(ualni idx) {
+		nn.calcGrad(mTestcases[idx].output);
+		nn.applyGrad(step);
+	}
+
+	static halni getMaxIdx(const Buffer<halnf>& in) {
+		halni out = 0;
+		for (auto i : Range(in.size())) {
+			if (in[i] > in[out]) {
+				out = i;
+			}
+		}
+		return out;
+	}
+
+	bool testIncorrect(ualni idx) {
+		nn.evaluate(mTestcases[idx].input, output);
+		return getMaxIdx(mTestcases[idx].output) != getMaxIdx(output);
+	}
+
+	void debLog(halni idx) {
+		printf("\n Got      %i - ", getMaxIdx(output));
+		for (auto val : output) {
+			printf("%f ", val.data());
+		}
+		printf("\n Expected %i - ", getMaxIdx(mTestcases[idx].output));
+		for (auto val : mTestcases[idx].output) {
+			printf("%f ", val.data());
+		}
+		printf("\n\n");
+	}
+
+	void displayImage(ualni idx) {
+		auto& testcase = mTestcases[idx];
+		printf("Image : %i\n", int(getMaxIdx(testcase.output)));
+		for (auto i : Range(28)) {
+			for (auto j : Range(28)) {
+				printf("%c", char(testcase.input[j * 28 + i] * 255));
+			}
+			printf("\n");
 		}
 	}
 
-	return (halnf) numFailed / (halnf) range.idxDiff();
+	halnf test(const Range<halni>& range) {
-}
+		halnf avgCost = 0;
+		for (auto i : range) {
+			avgCost += eval(i);
+		}
+		avgCost /= (halnf) range.idxDiff();
+		return avgCost;
+	}
 
-void testTraining(const Dataset& dataset, FullyConnectedNN& nn) {
+	void trainStep(const Range<halni>& range) {
+		nn.clearGrad();
+		for (auto i : range) {
+			nn.evaluate(mTestcases[i].input, output);
+			nn.calcGrad(mTestcases[i].output);
+		}
+		nn.applyGrad(step);
+	}
 
-	auto propagate = [&](ualni idx) {
+public:
-		auto& image = dataset.images[idx];
+	struct Image {
-		auto label = dataset.labels[idx];
-
-		Buffer<halnf> results;
 		Buffer<halnf> input;
-
+		Buffer<halnf> output;
-		results.reserve(10);
-		input.reserve(image.size());
-
-		for (auto pixelIdx : Range(image.size())) {
-			input[pixelIdx] = (halnf) image[pixelIdx] / 255.f;
-		}
-
-		nn.evaluate(input, results);
-
-		ualni resultNumber = 0;
-		for (auto resIdx : Range(results.size())) {
-			if (results[resIdx] > results[resultNumber]) {
-				resultNumber = resIdx;
-			}
-		}
-
-		Buffer<halnf> resultsExpected;
-		resultsExpected.reserve(10);
-		for (auto resIdx : Range(results.size())) {
-			resultsExpected[resIdx] = (resIdx == label) ? 1 : 0;
-		}
-
-		nn.calcGrad(resultsExpected);
-
-		return resultNumber;
 	};
 
-	displayImage(dataset, 2);
+public:
+	Buffer<Image> mTestcases;
 
-	propagate(0);
+	FCNN nn;
-	// nn.applyGrad();
+	Buffer<halnf> output;
 
-	propagate(0);
+	halnf step = 0.01f;
-	// nn.applyGrad();
+};
-
-	propagate(0);
-	// nn.applyGrad();
-
-	auto errorPercentage = test(dataset, nn, { 0, 1 });
-	printf("Percentage error Trained on first image: %f\n", errorPercentage);
-}
-
-void numRec() {
-	Dataset dataset;
-	FullyConnectedNN nn;
-
-	// settings
-	Buffer<halni> layers;
-	layers = { 784, 128, 10 };
-	halnf trainBatchPercentage = 0.1f;
-	halnf testSizePercentage = 0.1f;
-
-	if (!loadDataset(dataset, "rsc/mnist")) {
-		printf("Cant Load Mnist Dataset\n");
-		return;
-	}
-
-	nn.initializeRandom(layers);
-
-	auto errorPercentage = test(dataset, nn, { 0, 100 });
-
-	printf("Percentage error : %f\n", errorPercentage);
-
-	testTraining(dataset, nn);
-}
 
 int main() {
 	ModuleManifest* deps[] = { &gModuleDataAnalysis, &gModuleConnection, nullptr };
@@ -176,9 +170,32 @@ int main() {
 		return 1;
 	}
 
-	numRec();
+	{
+		NumberRec app;
+
+		auto trainRange = Range(0, 100);
+		auto testRange = Range(0, 100);
+
+		halnf cost = 100;
+		while (cost > 0.1f) {
+			cost = app.test(trainRange);
+			app.trainStep(trainRange);
+			printf("Cost - %f\n", cost);
+		}
+
+		auto errors = 0;
+		for (auto i : testRange) {
+			if (app.testIncorrect(i)) {
+				errors++;
+			}
+			// app.debLog(i);
+			// app.displayImage(i);
+		}
+
+		printf("\n\nIncorrect - %i out of %i\n\n", errors, testRange.idxDiff());
+	}
 
 	module.deinitialize();
 
 	return 0;
-}
+}

DataAnalysis/private/FullyConnectedNN.cpp

@@ -1,6 +1,6 @@
 // #include "NewPlacement.hpp"
 
-#include "FullyConnectedNN.hpp"
+#include "FCNN.hpp"
 #include "Utils.hpp"
 
 #include <cmath>
@@ -22,11 +22,13 @@ static halnf relu(halnf val) { return val < 0 ? 0 : val; }
 
 static halnf reluDerivative(halnf val) { return val < 0 ? 0 : 1; }
 
-static halnf activationFunction(halnf val) { return sigmoid(val); }
+static halnf activationFunction(halnf val) { return relu(val); }
 
-static halnf activationFunctionDerivative(halnf val) { return sigmoidDerivative(val); }
+static halnf activationFunctionDerivative(halnf val) { return reluDerivative(val); }
 
-void FullyConnectedNN::initializeRandom(const Buffer<halni>& description) {
+FCNN::FCNN(const Buffer<halni>& description) { initializeRandom(description); }
+
+void FCNN::initializeRandom(const Buffer<halni>& description) {
 	ASSERT(description.size() > 1);
 
 	mLayers.reserve(description.size());
@@ -39,14 +41,14 @@ void FullyConnectedNN::initializeRandom(const Buffer<halni>& description) {
 		for (auto neuron : mLayers[i].neurons) {
 			neuron->weights.reserve(description[i - 1]);
 			for (auto weight : neuron->weights) {
-				weight->val = (halnf) (randomFloat() - 0) * 2;
+				weight->val = (halnf) (randomFloat() - 0.5) * 2;
 			}
-			neuron->bias.val = (halnf) (randomFloat() - 0) * 2;
+			neuron->bias.val = (halnf) (randomFloat() - 0.5) * 2;
 		}
 	}
 }
 
-void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& output) {
+void FCNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& output) {
 	ASSERT(output.size() == mLayers.last().neurons.size() && input.size() == mLayers.first().neurons.size())
 
 	for (auto idx : Range(input.size())) {
@@ -73,7 +75,7 @@ void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& outpu
 	}
 }
 
-halnf FullyConnectedNN::calcCost(const Buffer<halnf>& output) {
+halnf FCNN::calcCost(const Buffer<halnf>& output) {
 	halnf out = 0;
 	for (auto neuronIdx : Range(mLayers.last().neurons.size())) {
 		out += pow(output[neuronIdx] - mLayers.last().neurons[neuronIdx].activationValue, 2);
@@ -81,7 +83,21 @@ halnf FullyConnectedNN::calcCost(const Buffer<halnf>& output) {
 	return out;
 }
 
-void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
+void FCNN::clearGrad() {
+	for (auto layIdx : Range<halni>(1, (halni) mLayers.size())) {
+		auto& layer = mLayers[layIdx];
+
+		for (auto neuron : layer.neurons) {
+			neuron->bias.grad = 0;
+			for (auto weightIdx : Range(neuron->weights.size())) {
+				neuron->weights[weightIdx].grad = 0;
+			}
+		}
+	}
+	mAvgCount = 0;
+}
+
+void FCNN::calcGrad(const Buffer<halnf>& output) {
 	ASSERT(mLayers.last().neurons.size() == output.size())
 
 	auto& lastLayer = mLayers.last();
@@ -113,11 +129,11 @@ void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
 			}
 
 			// gradient for current neuron bias
-			currentNeuron.bias.grad = currentNeuron.cache;
+			currentNeuron.bias.grad += currentNeuron.cache;
 
 			// calculate gradient for weights of current neuron
 			for (auto weightIdx : Range(currentNeuron.weights.size())) {
-				currentNeuron.weights[weightIdx].grad = inputLayer.neurons[weightIdx].activationValue * currentNeuron.cache;
+				currentNeuron.weights[weightIdx].grad += inputLayer.neurons[weightIdx].activationValue * currentNeuron.cache;
 			}
 		}
 	}
@@ -125,7 +141,7 @@ void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
 	mAvgCount++;
 }
 
-void FullyConnectedNN::applyGrad(halnf step) {
+void FCNN::applyGrad(halnf step) {
 
 	for (auto layIdx : Range<halni>(1, (halni) mLayers.size())) {
 		auto& layer = mLayers[layIdx];

DataAnalysis/public/FCNN.hpp

@@ -4,7 +4,8 @@
 #include "DataAnalysisCommon.hpp"
 
 namespace tp {
-	class FullyConnectedNN {
+	// Fully connected neural network
+	class FCNN {
 
 		struct Layer {
 
@@ -32,12 +33,15 @@ namespace tp {
 		};
 
 	public:
-		FullyConnectedNN() = default;
+		FCNN() = default;
+		explicit FCNN(const Buffer<halni>& description);
 
 		void initializeRandom(const Buffer<halni>& description);
 		void evaluate(const Buffer<halnf>& input, Buffer<halnf>& output);
 
 		halnf calcCost(const Buffer<halnf>& output);
+
+		void clearGrad();
 		void calcGrad(const Buffer<halnf>& output);
 		void applyGrad(halnf step);
 

DataAnalysis/tests/Tests.cpp

@@ -1,49 +1,45 @@
 
 #include "NewPlacement.hpp"
 
-#include "FullyConnectedNN.hpp"
+#include "FCNN.hpp"
 #include "Testing.hpp"
 #include "Utils.hpp"
 
-#include <stdio.h>
+#include <cstdio>
 
-static bool init(const tp::ModuleManifest* self) {
+using namespace tp;
-	tp::gTesting.setRootName(self->getName());
-	return true;
-}
 
 void test() {
-	using namespace tp;
+	Buffer<halni> layers = { 100, 70, 50, 30, 20 };
+	Buffer<halnf> input(layers.first());
+	Buffer<halnf> outputExpected(layers.last());
+	Buffer<halnf> output(layers.last());
 
-	Buffer<halni> layers = { 4, 4, 3, 2 };
+	for (auto inputVal : Range(layers.first())) {
-	Buffer<halnf> input = { (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100 };
+		input[inputVal] = (halnf) randomFloat() * 100;
-	Buffer<halnf> outputExpected = { (halnf) randomFloat(), (halnf) randomFloat() };
+	}
 
-	FullyConnectedNN nn;
+	for (auto outIdx : Range(layers.last())) {
-	Buffer<halnf> output(2);
+		outputExpected[outIdx] = (halnf) randomFloat();
+	}
 
-	nn.initializeRandom(layers);
+	FCNN nn(layers);
-
+	halnf steppingValue = 100;
-	// nn.mLayers.last().neurons.first().weights = { 0.35, 0.35, 0.35, 0.35 };
-	// nn.mLayers.last().neurons.first().bias = 0.9;
-
-	// nn.mLayers.last().neurons.last().weights = { -0.35, -0.35, -0.35, -0.35 };
-	// nn.mLayers.last().neurons.last().bias = -3.9;
 
 	for (auto i : Range(50)) {
+
 		nn.evaluate(input, output);
-
+		
-		auto lossBefore = nn.calcCost(outputExpected);
-
 		nn.calcGrad(outputExpected);
-		nn.applyGrad(0.1);
+		nn.applyGrad(steppingValue);
-		printf("Loss %f \n", lossBefore);
+
+		printf("Loss %f \n", nn.calcCost(outputExpected));
 	}
 }
 
 int main() {
 	tp::ModuleManifest* deps[] = { &tp::gModuleDataAnalysis, &tp::gModuleUtils, nullptr };
-	tp::ModuleManifest testModule("DataAnalysisTest", init, nullptr, deps);
+	tp::ModuleManifest testModule("DataAnalysisTest", nullptr, nullptr, deps);
 
 	if (!testModule.initialize()) {
 		return 1;

Utils/public/Utils.hpp

@@ -111,7 +111,7 @@ namespace tp {
 			mBegin(pStartIndex),
 			mEnd(pEndIndex) {}
 
-		bool valid() { return mBegin < mEnd; }
+		bool valid() const { return mBegin < mEnd; }
 
 		tType idxBegin() const { return mBegin; }
 
@@ -119,8 +119,8 @@ namespace tp {
 
 		tType idxDiff() const { return mEnd - mBegin; }
 
-		Iterator begin() { return Iterator(mBegin); }
+		Iterator begin() const { return Iterator(mBegin); }
 
-		Iterator end() { return Iterator(mEnd); }
+		Iterator end() const { return Iterator(mEnd); }
 	};
 }