BackProp Initial

Containers/public/Buffer.hpp

@@ -320,6 +320,7 @@ namespace tp {
 		}
 
 		void reserve(ualni aSize) {
+			if (aSize == mSize) return;
 			for (ualni i = 0; i < mLoad; i++) {
 				mBuff[i].~tType();
 			}

DataAnalysis/applications/NumberRecognition.cpp

@@ -80,23 +80,79 @@ halnf test(const Dataset& dataset, FullyConnectedNN& nn, Range<ualni> range) {
 	return (halnf) numFailed / (halnf) range.idxDiff();
 }
 
+void testTraining(const Dataset& dataset, FullyConnectedNN& nn) {
+
+	auto propagate = [&](ualni idx) {
+		auto& image = dataset.images[idx];
+		auto label = dataset.labels[idx];
+
+		Buffer<halnf> results;
+		Buffer<halnf> input;
+
+		results.reserve(10);
+		input.reserve(image.size());
+
+		for (auto pixelIdx : Range(image.size())) {
+			input[pixelIdx] = (halnf) image[pixelIdx] / 255.f;
+		}
+
+		nn.evaluate(input, results);
+
+		ualni resultNumber = 0;
+		for (auto resIdx : Range(results.size())) {
+			if (results[resIdx] > results[resultNumber]) {
+				resultNumber = resIdx;
+			}
+		}
+
+		Buffer<halnf> resultsExpected;
+		resultsExpected.reserve(10);
+		for (auto resIdx : Range(results.size())) {
+			resultsExpected[resIdx] = (resIdx == label) ? 1 : 0;
+		}
+
+		nn.calcGrad(resultsExpected);
+
+		return resultNumber;
+	};
+
+	nn.clearGrad();
+
+	propagate(0);
+	nn.applyGrad();
+
+	propagate(0);
+	nn.applyGrad();
+
+	propagate(0);
+	nn.applyGrad();
+
+	auto errorPercentage = test(dataset, nn, { 0, 1 });
+	printf("Percentage error Trained on first image: %f\n", errorPercentage);
+}
+
 void numRec() {
 	Dataset dataset;
 	FullyConnectedNN nn;
 
+	// settings
 	Buffer<halni> layers;
 	layers = { 784, 128, 10 };
-
-	nn.initializeRandom(layers);
+	halnf trainBatchPercentage = 0.1f;
+	halnf testSizePercentage = 0.1f;
 
 	if (!loadDataset(dataset, "rsc/mnist")) {
 		printf("Cant Load Mnist Dataset\n");
 		return;
 	}
 
+	nn.initializeRandom(layers);
+
 	auto errorPercentage = test(dataset, nn, { 0, 100 });
 
 	printf("Percentage error : %f\n", errorPercentage);
+
+	testTraining(dataset, nn);
 }
 
 int main() {

DataAnalysis/private/FullyConnectedNN.cpp

@@ -8,6 +8,8 @@ static halnf sigmoid(halnf val) { return 0; }
 
 static halnf relu(halnf val) { return val < 0 ? 0 : val; }
 
+static halnf reluDerivative(halnf val) { return val < 0 ? 0 : 1; }
+
 void FullyConnectedNN::initializeRandom(Buffer<halni> description) {
 	mLayers.reserve(description.size());
 
@@ -19,9 +21,9 @@ void FullyConnectedNN::initializeRandom(Buffer<halni> description) {
 		for (auto neuron : mLayers[i].mNeurons) {
 			neuron->mWeights.reserve(description[i - 1]);
 			for (auto weight : neuron->mWeights) {
-				weight.data() = (halnf) randomFloat();
+				weight.data() = (halnf) (randomFloat() - 0.5) * 2;
 			}
-			neuron->mBias = (halnf) randomFloat();
+			neuron->mBias = (halnf) (randomFloat() - 0.5) * 2;
 		}
 	}
 }
@@ -43,6 +45,7 @@ void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& outpu
 				neuron->mActivationValue += neuron->mWeights[connectionIdx] * layerPrev.mNeurons[connectionIdx].mActivationValue;
 			}
 			neuron->mActivationValue += neuron->mBias;
+			neuron->mActivationValueLinear = neuron->mActivationValue;
 			neuron->mActivationValue = relu(neuron->mActivationValue);
 		}
 	}
@@ -51,3 +54,86 @@ void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& outpu
 		output[idx] = mLayers.last().mNeurons[idx].mActivationValue;
 	}
 }
+
+halnf FullyConnectedNN::calcCost(const Buffer<halnf>& output) {
+	halnf out = 0;
+	for (auto neuronIdx : Range(mLayers.last().mNeurons.size())) {
+		out += pow(output[neuronIdx] - mLayers.last().mNeurons[neuronIdx].mActivationValue, 2);
+	}
+	return out;
+}
+
+void FullyConnectedNN::clearGrad() {
+	mLayersCache.reserve(mLayers.size());
+	for (auto layIdx : Range<halni>(0, (halni) mLayers.size())) {
+		mLayersCache[layIdx].mNeurons.reserve(mLayers[layIdx].mNeurons.size());
+		mLayersCache[layIdx].mCache = 1;
+
+		for (auto weightIdx : Range(mLayersCache[layIdx].mNeurons.size())) {
+			mLayersCache[layIdx].mNeurons[weightIdx].mBiasGrad = 0;
+			mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad.reserve(mLayers[layIdx].mNeurons[weightIdx].mWeights.size());
+			for (auto weight : mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad) {
+				weight.data() = 0;
+			}
+		}
+	}
+	mAvgCount = 0;
+}
+
+void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
+	ASSERT(mLayers.last().mNeurons.size() == output.size())
+
+	auto& lastLayer = mLayers.last();
+	auto& lastCache = mLayersCache.last();
+
+	lastCache.mCache = 1;
+
+	for (auto neuronIdx : Range(lastLayer.mNeurons.size())) {
+		auto& neuronCache = lastCache.mNeurons[neuronIdx];
+		auto& neuron = lastLayer.mNeurons[neuronIdx];
+		lastCache.mCache += 2 * pow(output[neuronIdx] - neuron.mActivationValue, 2);
+	}
+
+	for (auto layerIdx = mLayers.size() - 1; layerIdx > 0; layerIdx--) {
+		auto& layer = mLayers[layerIdx];
+		auto& cache = mLayersCache[layerIdx];
+
+		auto& layerNext = mLayers[layerIdx - 1];
+		auto& cacheNext = mLayersCache[layerIdx - 1];
+
+		cacheNext.mCache = 1;
+
+		for (auto neuronIdx : Range(layer.mNeurons.size())) {
+			auto& neuron = layer.mNeurons[neuronIdx];
+			auto& neuronCache = cache.mNeurons[neuronIdx];
+
+			auto tmp = cache.mCache * reluDerivative(neuron.mActivationValueLinear);
+
+			neuronCache.mBiasGrad = tmp;
+
+			for (auto weightIdx : Range(neuron.mWeights.size())) {
+				neuronCache.mWeightsGrad[weightIdx] = tmp * layerNext.mNeurons[weightIdx].mActivationValue;
+			}
+
+			cacheNext.mCache += tmp;
+		}
+	}
+}
+
+void FullyConnectedNN::applyGrad() {
+	for (auto layIdx : Range<halni>(1, (halni) mLayers.size())) {
+		auto& layerCache = mLayersCache[layIdx];
+		auto& layer = mLayers[layIdx];
+
+		for (auto neuronIdx : Range(layer.mNeurons.size())) {
+			auto& neuron = layer.mNeurons[neuronIdx];
+			auto& neuronCache = layerCache.mNeurons[neuronIdx];
+
+			neuron.mBias += neuronCache.mBiasGrad / (halnf) mAvgCount;
+
+			for (auto weightIdx : Range(neuron.mWeights.size())) {
+				neuron.mWeights[weightIdx] += neuronCache.mWeightsGrad[weightIdx] / (halnf) mAvgCount;
+			}
+		}
+	}
+}

DataAnalysis/public/FullyConnectedNN.hpp

@@ -5,23 +5,42 @@
 
 namespace tp {
 	class FullyConnectedNN {
+
 		struct Layer {
 			struct Neuron {
 				halnf mBias = 0;
 				Buffer<halnf> mWeights;
 				halnf mActivationValue = 0;
+				halnf mActivationValueLinear = 0;
 			};
 
 			Buffer<Neuron> mNeurons;
 		};
 
+		struct LayerCache {
+			struct NeuronCache {
+				halnf mBiasGrad = 0;
+				Buffer<halnf> mWeightsGrad;
+			};
+
+			halnf mCache = 0;
+			Buffer<NeuronCache> mNeurons;
+		};
+
 	public:
 		FullyConnectedNN() = default;
 
 		void initializeRandom(Buffer<halni> description);
 		void evaluate(const Buffer<halnf>& input, Buffer<halnf>& output);
 
+		void clearGrad();
+		halnf calcCost(const Buffer<halnf>& output);
+		void calcGrad(const Buffer<halnf>& output);
+		void applyGrad();
+
 	private:
 		Buffer<Layer> mLayers;
+		Buffer<LayerCache> mLayersCache;
+		halni mAvgCount = 0;
 	};
 };