BackProp Fixes

Containers/public/Buffer.hpp

@@ -182,10 +182,26 @@ namespace tp {
 			mBuff = (tType*) mAllocator.allocate(sizeof(tType) * tMinSize);
 		}
 
-		explicit Buffer(ualni size) :
+		Buffer(const InitialierList<tType>& input) {
-			mSize(size),
+			mSize = 0;
-			mLoad(0) {
+			for (const auto& val : input) {
+				mSize++;
+			}
+			mBuff = (tType*) mAllocator.allocate(sizeof(tType) * mSize);
+			mLoad = 0;
+			for (const auto& val : input) {
+				mBuff[mLoad] = val;
+				mLoad++;
+			}
+		}
+
+		explicit Buffer(ualni size) {
 			mBuff = (tType*) mAllocator.allocate(sizeof(tType) * size);
+			mSize = size;
+			mLoad = size;
+			for (ualni i = 0; i < mLoad; i++) {
+				new (mBuff + i) tType();
+			}
 		}
 
 		Buffer(const Buffer& in) :
@@ -274,11 +290,25 @@ namespace tp {
 		}
 
 	public:
-		Buffer& operator=(const tp::InitialierList<tType>& init) {
+		Buffer& operator=(const tp::InitialierList<tType>& input) {
 			// TODO : optimize
-			for (auto arg : init) {
+			for (ualni i = 0; i < mLoad; i++) {
-				append(arg);
+				mBuff[i].~tType();
 			}
+			mAllocator.deallocate(mBuff);
+
+			mSize = 0;
+			for (const auto& val : input) {
+				mSize++;
+			}
+
+			mBuff = (tType*) mAllocator.allocate(sizeof(tType) * mSize);
+			mLoad = 0;
+			for (const auto& val : input) {
+				mBuff[mLoad] = val;
+				mLoad++;
+			}
+
 			return *this;
 		}
 
@@ -320,10 +350,10 @@ namespace tp {
 		}
 
 		void reserve(ualni aSize) {
-			if (aSize == mSize) return;
 			for (ualni i = 0; i < mLoad; i++) {
 				mBuff[i].~tType();
 			}
+			mAllocator.deallocate(mBuff);
 			mBuff = (tType*) mAllocator.allocate(sizeof(tType) * aSize);
 			mSize = aSize;
 			mLoad = aSize;

Containers/tests/BufferTest.cpp

@@ -22,10 +22,10 @@ TEST_DEF_STATIC(Simple1) {
 
 TEST_DEF_STATIC(Simple2) {
 	Buffer<TestClass, HeapAlloc> buff(size);
-	TEST(buff.size() == 0);
+	TEST(buff.size() == size);
 	for (auto i : Range(size * 10))
 		buff.append(TestClass(i));
-	TEST(buff.size() == size * 10);
+	TEST(buff.size() == size + size * 10);
 	while (buff.size())
 		buff.pop();
 	TEST(buff.size() == 0);

Containers/tests/Tests.hpp

@@ -8,6 +8,8 @@ class TestClass {
 	tp::ualni val1;
 
 public:
+	TestClass() { val1 = 0; }
+
 	explicit TestClass(tp::ualni val) :
 		val1(val) {}
 

DataAnalysis/applications/NumberRecognition.cpp

@@ -11,6 +11,19 @@ struct Dataset {
 	Buffer<Buffer<uint1>> images;
 };
 
+void displayImage(const Dataset& dataset, ualni idx) {
+	auto& image = dataset.images[idx];
+	auto label = dataset.labels[idx];
+
+	printf("Image : %i\n", int(label));
+	for (auto i : Range(dataset.imageSize.x)) {
+		for (auto j : Range(dataset.imageSize.y)) {
+			printf("%c", image[j * dataset.imageSize.x + i]);
+		}
+		printf("\n");
+	}
+}
+
 bool loadDataset(Dataset& out, const String& location) {
 	LocalConnection dataset;
 	dataset.connect(LocalConnection::Location(location), LocalConnection::Type(true));
@@ -116,16 +129,16 @@ void testTraining(const Dataset& dataset, FullyConnectedNN& nn) {
 		return resultNumber;
 	};
 
-	nn.clearGrad();
+	displayImage(dataset, 2);
 
 	propagate(0);
-	nn.applyGrad();
+	// nn.applyGrad();
 
 	propagate(0);
-	nn.applyGrad();
+	// nn.applyGrad();
 
 	propagate(0);
-	nn.applyGrad();
+	// nn.applyGrad();
 
 	auto errorPercentage = test(dataset, nn, { 0, 1 });
 	printf("Percentage error Trained on first image: %f\n", errorPercentage);

DataAnalysis/private/FullyConnectedNN.cpp

@@ -1,139 +1,142 @@
+// #include "NewPlacement.hpp"
+
 #include "FullyConnectedNN.hpp"
-#include "NewPlacement.hpp"
 #include "Utils.hpp"
 
+#include <cmath>
+
 using namespace tp;
 
-static halnf sigmoid(halnf val) { return 0; }
+static halnf linearFun(halnf val) { return val; }
+
+static halnf linearFunDerivative(halnf val) { return 1; }
+
+static halnf sigmoid(double val) { return 1.0f / (1.0f + (halnf) exp(-val)); }
+
+static halnf sigmoidDerivative(double val) {
+	halnf sigmoid_val = sigmoid(val);
+	return sigmoid_val * (1.0f - sigmoid_val);
+}
 
 static halnf relu(halnf val) { return val < 0 ? 0 : val; }
 
 static halnf reluDerivative(halnf val) { return val < 0 ? 0 : 1; }
 
-void FullyConnectedNN::initializeRandom(Buffer<halni> description) {
+static halnf activationFunction(halnf val) { return sigmoid(val); }
+
+static halnf activationFunctionDerivative(halnf val) { return sigmoidDerivative(val); }
+
+void FullyConnectedNN::initializeRandom(const Buffer<halni>& description) {
+	ASSERT(description.size() > 1);
+
 	mLayers.reserve(description.size());
 
 	for (auto i : Range<halni>(0, (halni) description.size())) {
-		mLayers[i].mNeurons.reserve(description[i]);
+		mLayers[i].neurons.reserve(description[i]);
 		if (i == 0) {
 			continue;
 		}
-		for (auto neuron : mLayers[i].mNeurons) {
+		for (auto neuron : mLayers[i].neurons) {
-			neuron->mWeights.reserve(description[i - 1]);
+			neuron->weights.reserve(description[i - 1]);
-			for (auto weight : neuron->mWeights) {
+			for (auto weight : neuron->weights) {
-				weight.data() = (halnf) (randomFloat() - 0.5) * 2;
+				weight->val = (halnf) (randomFloat() - 0) * 2;
 			}
-			neuron->mBias = (halnf) (randomFloat() - 0.5) * 2;
+			neuron->bias.val = (halnf) (randomFloat() - 0) * 2;
 		}
 	}
 }
 
 void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& output) {
-	ASSERT(output.size() == mLayers.last().mNeurons.size() && input.size() == mLayers.first().mNeurons.size())
+	ASSERT(output.size() == mLayers.last().neurons.size() && input.size() == mLayers.first().neurons.size())
 
 	for (auto idx : Range(input.size())) {
-		mLayers.first().mNeurons[idx].mActivationValue = input[idx];
+		mLayers.first().neurons[idx].activationValue = input[idx];
 	}
 
 	for (auto layerIdx : Range<halni>(1, (halni) mLayers.size())) {
 		auto& layer = mLayers[layerIdx];
 		auto& layerPrev = mLayers[layerIdx - 1];
 
-		for (auto neuron : layer.mNeurons) {
+		for (auto neuron : layer.neurons) {
-			neuron->mActivationValue = 0;
+			neuron->activationValue = 0;
-			for (auto connectionIdx : Range(neuron->mWeights.size())) {
+			for (auto connectionIdx : Range(neuron->weights.size())) {
-				neuron->mActivationValue += neuron->mWeights[connectionIdx] * layerPrev.mNeurons[connectionIdx].mActivationValue;
+				neuron->activationValue += neuron->weights[connectionIdx].val * layerPrev.neurons[connectionIdx].activationValue;
 			}
-			neuron->mActivationValue += neuron->mBias;
+			neuron->activationValue += neuron->bias.val;
-			neuron->mActivationValueLinear = neuron->mActivationValue;
+			neuron->activationValueLinear = neuron->activationValue;
-			neuron->mActivationValue = relu(neuron->mActivationValue);
+			neuron->activationValue = activationFunction(neuron->activationValue);
 		}
 	}
 
 	for (auto idx : Range(output.size())) {
-		output[idx] = mLayers.last().mNeurons[idx].mActivationValue;
+		output[idx] = mLayers.last().neurons[idx].activationValue;
 	}
 }
 
 halnf FullyConnectedNN::calcCost(const Buffer<halnf>& output) {
 	halnf out = 0;
-	for (auto neuronIdx : Range(mLayers.last().mNeurons.size())) {
+	for (auto neuronIdx : Range(mLayers.last().neurons.size())) {
-		out += pow(output[neuronIdx] - mLayers.last().mNeurons[neuronIdx].mActivationValue, 2);
+		out += pow(output[neuronIdx] - mLayers.last().neurons[neuronIdx].activationValue, 2);
 	}
 	return out;
 }
 
-void FullyConnectedNN::clearGrad() {
-	mLayersCache.reserve(mLayers.size());
-	for (auto layIdx : Range<halni>(0, (halni) mLayers.size())) {
-		mLayersCache[layIdx].mNeurons.reserve(mLayers[layIdx].mNeurons.size());
-		mLayersCache[layIdx].mCache = 1;
-
-		for (auto weightIdx : Range(mLayersCache[layIdx].mNeurons.size())) {
-			mLayersCache[layIdx].mNeurons[weightIdx].mBiasGrad = 0;
-			mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad.reserve(mLayers[layIdx].mNeurons[weightIdx].mWeights.size());
-			for (auto weight : mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad) {
-				weight.data() = 0;
-			}
-		}
-	}
-	mAvgCount = 0;
-}
-
 void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
-	ASSERT(mLayers.last().mNeurons.size() == output.size())
+	ASSERT(mLayers.last().neurons.size() == output.size())
 
 	auto& lastLayer = mLayers.last();
-	auto& lastCache = mLayersCache.last();
 
-	lastCache.mCache = 1;
+	// calculate chaining cache value for each neuron in last layer
-
+	for (auto neuronIdx : Range(lastLayer.neurons.size())) {
-	for (auto neuronIdx : Range(lastLayer.mNeurons.size())) {
+		auto& neuron = lastLayer.neurons[neuronIdx];
-		auto& neuronCache = lastCache.mNeurons[neuronIdx];
+		neuron.cache = 2 * (neuron.activationValue - output[neuronIdx]);
-		auto& neuron = lastLayer.mNeurons[neuronIdx];
-		lastCache.mCache += 2 * pow(output[neuronIdx] - neuron.mActivationValue, 2);
 	}
 
+	// calculate rest of the layers
 	for (auto layerIdx = mLayers.size() - 1; layerIdx > 0; layerIdx--) {
-		auto& layer = mLayers[layerIdx];
-		auto& cache = mLayersCache[layerIdx];
 
-		auto& layerNext = mLayers[layerIdx - 1];
+		auto& currentLayer = mLayers[layerIdx];
-		auto& cacheNext = mLayersCache[layerIdx - 1];
+		auto& inputLayer = mLayers[layerIdx - 1];
 
-		cacheNext.mCache = 1;
+		for (auto currentNeuronIdx : Range(currentLayer.neurons.size())) {
+			auto& currentNeuron = currentLayer.neurons[currentNeuronIdx];
 
-		for (auto neuronIdx : Range(layer.mNeurons.size())) {
+			// calculate cache value (chaining)
-			auto& neuron = layer.mNeurons[neuronIdx];
+			if (layerIdx != mLayers.size() - 1) {
-			auto& neuronCache = cache.mNeurons[neuronIdx];
+				auto& userLayer = mLayers[layerIdx + 1];
 
-			auto tmp = cache.mCache * reluDerivative(neuron.mActivationValueLinear);
+				currentNeuron.cache = 0;
-
+				for (auto userNeuron : userLayer.neurons) {
-			neuronCache.mBiasGrad = tmp;
+					currentNeuron.cache += userNeuron->weights[currentNeuronIdx].val * userNeuron->cache;
-
+				}
-			for (auto weightIdx : Range(neuron.mWeights.size())) {
+				currentNeuron.cache *= activationFunctionDerivative(currentNeuron.activationValueLinear);
-				neuronCache.mWeightsGrad[weightIdx] = tmp * layerNext.mNeurons[weightIdx].mActivationValue;
 			}
 
-			cacheNext.mCache += tmp;
+			// gradient for current neuron bias
+			currentNeuron.bias.grad = currentNeuron.cache;
+
+			// calculate gradient for weights of current neuron
+			for (auto weightIdx : Range(currentNeuron.weights.size())) {
+				currentNeuron.weights[weightIdx].grad = inputLayer.neurons[weightIdx].activationValue * currentNeuron.cache;
+			}
 		}
 	}
+
+	mAvgCount++;
 }
 
-void FullyConnectedNN::applyGrad() {
+void FullyConnectedNN::applyGrad(halnf step) {
+
 	for (auto layIdx : Range<halni>(1, (halni) mLayers.size())) {
-		auto& layerCache = mLayersCache[layIdx];
 		auto& layer = mLayers[layIdx];
 
-		for (auto neuronIdx : Range(layer.mNeurons.size())) {
+		for (auto neuron : layer.neurons) {
-			auto& neuron = layer.mNeurons[neuronIdx];
+			neuron->bias.val -= neuron->bias.grad / (halnf) mAvgCount * step;
-			auto& neuronCache = layerCache.mNeurons[neuronIdx];
+			for (auto weightIdx : Range(neuron->weights.size())) {
-
+				neuron->weights[weightIdx].val -= (neuron->weights[weightIdx].grad / (halnf) mAvgCount) * step;
-			neuron.mBias += neuronCache.mBiasGrad / (halnf) mAvgCount;
-
-			for (auto weightIdx : Range(neuron.mWeights.size())) {
-				neuron.mWeights[weightIdx] += neuronCache.mWeightsGrad[weightIdx] / (halnf) mAvgCount;
 			}
 		}
 	}
+
+	mAvgCount = 0;
 }

DataAnalysis/public/FullyConnectedNN.hpp

@@ -7,40 +7,42 @@ namespace tp {
 	class FullyConnectedNN {
 
 		struct Layer {
+
 			struct Neuron {
-				halnf mBias = 0;
+				struct Parameter {
-				Buffer<halnf> mWeights;
+					Parameter() = default;
-				halnf mActivationValue = 0;
+
-				halnf mActivationValueLinear = 0;
+					Parameter(halnf in) :
+						val(in) {}
+
+					halnf val = 0;
+					halnf grad = 0;
+				};
+
+				Parameter bias;
+				Buffer<Parameter> weights;
+
+				halnf activationValue = 0;
+				halnf activationValueLinear = 0;
+
+				halnf cache;
 			};
 
-			Buffer<Neuron> mNeurons;
+			Buffer<Neuron> neurons;
-		};
-
-		struct LayerCache {
-			struct NeuronCache {
-				halnf mBiasGrad = 0;
-				Buffer<halnf> mWeightsGrad;
-			};
-
-			halnf mCache = 0;
-			Buffer<NeuronCache> mNeurons;
 		};
 
 	public:
 		FullyConnectedNN() = default;
 
-		void initializeRandom(Buffer<halni> description);
+		void initializeRandom(const Buffer<halni>& description);
 		void evaluate(const Buffer<halnf>& input, Buffer<halnf>& output);
 
-		void clearGrad();
 		halnf calcCost(const Buffer<halnf>& output);
 		void calcGrad(const Buffer<halnf>& output);
-		void applyGrad();
+		void applyGrad(halnf step);
 
-	private:
+	public:
 		Buffer<Layer> mLayers;
-		Buffer<LayerCache> mLayersCache;
 		halni mAvgCount = 0;
 	};
 };

DataAnalysis/tests/Tests.cpp

@@ -1,14 +1,45 @@
 
+#include "NewPlacement.hpp"
+
 #include "FullyConnectedNN.hpp"
 #include "Testing.hpp"
 #include "Utils.hpp"
 
+#include <stdio.h>
+
 static bool init(const tp::ModuleManifest* self) {
 	tp::gTesting.setRootName(self->getName());
 	return true;
 }
 
-void test() {}
+void test() {
+	using namespace tp;
+
+	Buffer<halni> layers = { 4, 4, 3, 2 };
+	Buffer<halnf> input = { (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100 };
+	Buffer<halnf> outputExpected = { (halnf) randomFloat(), (halnf) randomFloat() };
+
+	FullyConnectedNN nn;
+	Buffer<halnf> output(2);
+
+	nn.initializeRandom(layers);
+
+	// nn.mLayers.last().neurons.first().weights = { 0.35, 0.35, 0.35, 0.35 };
+	// nn.mLayers.last().neurons.first().bias = 0.9;
+
+	// nn.mLayers.last().neurons.last().weights = { -0.35, -0.35, -0.35, -0.35 };
+	// nn.mLayers.last().neurons.last().bias = -3.9;
+
+	for (auto i : Range(50)) {
+		nn.evaluate(input, output);
+
+		auto lossBefore = nn.calcCost(outputExpected);
+
+		nn.calcGrad(outputExpected);
+		nn.applyGrad(0.1);
+		printf("Loss %f \n", lossBefore);
+	}
+}
 
 int main() {
 	tp::ModuleManifest* deps[] = { &tp::gModuleDataAnalysis, &tp::gModuleUtils, nullptr };

TODO

@@ -20,12 +20,12 @@ RayTracer:
 
 	Quality:
 		Render eq
-		Materials
 		Lenses
 		Exposure
 
 	Efficiency:
 		Casting Accelerators
+		MIS (DMIS CMIS)
 		Important sampling
 		Threading
 

Utils/private/Utils.cpp

@@ -5,6 +5,7 @@
 
 #include "Testing.hpp"
 
+#include <ctime>
 #include <random>
 
 void initializeCallStackCapture();
@@ -12,6 +13,7 @@ void deinitializeCallStackCapture();
 
 static bool initialize(const tp::ModuleManifest*) {
 	initializeCallStackCapture();
+	std::srand(std::time(nullptr));
 	return true;
 }