diff --git a/Containers/public/Buffer.hpp b/Containers/public/Buffer.hpp index 8744be4..b6f183c 100644 --- a/Containers/public/Buffer.hpp +++ b/Containers/public/Buffer.hpp @@ -182,10 +182,26 @@ namespace tp { mBuff = (tType*) mAllocator.allocate(sizeof(tType) * tMinSize); } - explicit Buffer(ualni size) : - mSize(size), - mLoad(0) { + Buffer(const InitialierList& input) { + mSize = 0; + for (const auto& val : input) { + mSize++; + } + mBuff = (tType*) mAllocator.allocate(sizeof(tType) * mSize); + mLoad = 0; + for (const auto& val : input) { + mBuff[mLoad] = val; + mLoad++; + } + } + + explicit Buffer(ualni size) { mBuff = (tType*) mAllocator.allocate(sizeof(tType) * size); + mSize = size; + mLoad = size; + for (ualni i = 0; i < mLoad; i++) { + new (mBuff + i) tType(); + } } Buffer(const Buffer& in) : @@ -274,11 +290,25 @@ namespace tp { } public: - Buffer& operator=(const tp::InitialierList& init) { + Buffer& operator=(const tp::InitialierList& input) { // TODO : optimize - for (auto arg : init) { - append(arg); + for (ualni i = 0; i < mLoad; i++) { + mBuff[i].~tType(); } + mAllocator.deallocate(mBuff); + + mSize = 0; + for (const auto& val : input) { + mSize++; + } + + mBuff = (tType*) mAllocator.allocate(sizeof(tType) * mSize); + mLoad = 0; + for (const auto& val : input) { + mBuff[mLoad] = val; + mLoad++; + } + return *this; } @@ -320,10 +350,10 @@ namespace tp { } void reserve(ualni aSize) { - if (aSize == mSize) return; for (ualni i = 0; i < mLoad; i++) { mBuff[i].~tType(); } + mAllocator.deallocate(mBuff); mBuff = (tType*) mAllocator.allocate(sizeof(tType) * aSize); mSize = aSize; mLoad = aSize; diff --git a/Containers/tests/BufferTest.cpp b/Containers/tests/BufferTest.cpp index e30809a..eb3494a 100644 --- a/Containers/tests/BufferTest.cpp +++ b/Containers/tests/BufferTest.cpp @@ -22,10 +22,10 @@ TEST_DEF_STATIC(Simple1) { TEST_DEF_STATIC(Simple2) { Buffer buff(size); - TEST(buff.size() == 0); + TEST(buff.size() == size); for (auto i : Range(size * 10)) buff.append(TestClass(i)); - TEST(buff.size() == size * 10); + TEST(buff.size() == size + size * 10); while (buff.size()) buff.pop(); TEST(buff.size() == 0); diff --git a/Containers/tests/Tests.hpp b/Containers/tests/Tests.hpp index 090aed9..24a93e3 100644 --- a/Containers/tests/Tests.hpp +++ b/Containers/tests/Tests.hpp @@ -8,6 +8,8 @@ class TestClass { tp::ualni val1; public: + TestClass() { val1 = 0; } + explicit TestClass(tp::ualni val) : val1(val) {} diff --git a/DataAnalysis/applications/NumberRecognition.cpp b/DataAnalysis/applications/NumberRecognition.cpp index a9e8cf0..dd8edd4 100644 --- a/DataAnalysis/applications/NumberRecognition.cpp +++ b/DataAnalysis/applications/NumberRecognition.cpp @@ -11,6 +11,19 @@ struct Dataset { Buffer> images; }; +void displayImage(const Dataset& dataset, ualni idx) { + auto& image = dataset.images[idx]; + auto label = dataset.labels[idx]; + + printf("Image : %i\n", int(label)); + for (auto i : Range(dataset.imageSize.x)) { + for (auto j : Range(dataset.imageSize.y)) { + printf("%c", image[j * dataset.imageSize.x + i]); + } + printf("\n"); + } +} + bool loadDataset(Dataset& out, const String& location) { LocalConnection dataset; dataset.connect(LocalConnection::Location(location), LocalConnection::Type(true)); @@ -116,16 +129,16 @@ void testTraining(const Dataset& dataset, FullyConnectedNN& nn) { return resultNumber; }; - nn.clearGrad(); + displayImage(dataset, 2); propagate(0); - nn.applyGrad(); + // nn.applyGrad(); propagate(0); - nn.applyGrad(); + // nn.applyGrad(); propagate(0); - nn.applyGrad(); + // nn.applyGrad(); auto errorPercentage = test(dataset, nn, { 0, 1 }); printf("Percentage error Trained on first image: %f\n", errorPercentage); diff --git a/DataAnalysis/applications/rsc/mnist b/DataAnalysis/applications/rsc/mnist deleted file mode 100644 index e743796..0000000 Binary files a/DataAnalysis/applications/rsc/mnist and /dev/null differ diff --git a/DataAnalysis/applications/rsc/mnist.gz b/DataAnalysis/applications/rsc/mnist.gz new file mode 100644 index 0000000..abb32ac Binary files /dev/null and b/DataAnalysis/applications/rsc/mnist.gz differ diff --git a/DataAnalysis/private/FullyConnectedNN.cpp b/DataAnalysis/private/FullyConnectedNN.cpp index be49045..1ddf61d 100644 --- a/DataAnalysis/private/FullyConnectedNN.cpp +++ b/DataAnalysis/private/FullyConnectedNN.cpp @@ -1,139 +1,142 @@ +// #include "NewPlacement.hpp" + #include "FullyConnectedNN.hpp" -#include "NewPlacement.hpp" #include "Utils.hpp" +#include + using namespace tp; -static halnf sigmoid(halnf val) { return 0; } +static halnf linearFun(halnf val) { return val; } + +static halnf linearFunDerivative(halnf val) { return 1; } + +static halnf sigmoid(double val) { return 1.0f / (1.0f + (halnf) exp(-val)); } + +static halnf sigmoidDerivative(double val) { + halnf sigmoid_val = sigmoid(val); + return sigmoid_val * (1.0f - sigmoid_val); +} static halnf relu(halnf val) { return val < 0 ? 0 : val; } static halnf reluDerivative(halnf val) { return val < 0 ? 0 : 1; } -void FullyConnectedNN::initializeRandom(Buffer description) { +static halnf activationFunction(halnf val) { return sigmoid(val); } + +static halnf activationFunctionDerivative(halnf val) { return sigmoidDerivative(val); } + +void FullyConnectedNN::initializeRandom(const Buffer& description) { + ASSERT(description.size() > 1); + mLayers.reserve(description.size()); for (auto i : Range(0, (halni) description.size())) { - mLayers[i].mNeurons.reserve(description[i]); + mLayers[i].neurons.reserve(description[i]); if (i == 0) { continue; } - for (auto neuron : mLayers[i].mNeurons) { - neuron->mWeights.reserve(description[i - 1]); - for (auto weight : neuron->mWeights) { - weight.data() = (halnf) (randomFloat() - 0.5) * 2; + for (auto neuron : mLayers[i].neurons) { + neuron->weights.reserve(description[i - 1]); + for (auto weight : neuron->weights) { + weight->val = (halnf) (randomFloat() - 0) * 2; } - neuron->mBias = (halnf) (randomFloat() - 0.5) * 2; + neuron->bias.val = (halnf) (randomFloat() - 0) * 2; } } } void FullyConnectedNN::evaluate(const Buffer& input, Buffer& output) { - ASSERT(output.size() == mLayers.last().mNeurons.size() && input.size() == mLayers.first().mNeurons.size()) + ASSERT(output.size() == mLayers.last().neurons.size() && input.size() == mLayers.first().neurons.size()) for (auto idx : Range(input.size())) { - mLayers.first().mNeurons[idx].mActivationValue = input[idx]; + mLayers.first().neurons[idx].activationValue = input[idx]; } for (auto layerIdx : Range(1, (halni) mLayers.size())) { auto& layer = mLayers[layerIdx]; auto& layerPrev = mLayers[layerIdx - 1]; - for (auto neuron : layer.mNeurons) { - neuron->mActivationValue = 0; - for (auto connectionIdx : Range(neuron->mWeights.size())) { - neuron->mActivationValue += neuron->mWeights[connectionIdx] * layerPrev.mNeurons[connectionIdx].mActivationValue; + for (auto neuron : layer.neurons) { + neuron->activationValue = 0; + for (auto connectionIdx : Range(neuron->weights.size())) { + neuron->activationValue += neuron->weights[connectionIdx].val * layerPrev.neurons[connectionIdx].activationValue; } - neuron->mActivationValue += neuron->mBias; - neuron->mActivationValueLinear = neuron->mActivationValue; - neuron->mActivationValue = relu(neuron->mActivationValue); + neuron->activationValue += neuron->bias.val; + neuron->activationValueLinear = neuron->activationValue; + neuron->activationValue = activationFunction(neuron->activationValue); } } for (auto idx : Range(output.size())) { - output[idx] = mLayers.last().mNeurons[idx].mActivationValue; + output[idx] = mLayers.last().neurons[idx].activationValue; } } halnf FullyConnectedNN::calcCost(const Buffer& output) { halnf out = 0; - for (auto neuronIdx : Range(mLayers.last().mNeurons.size())) { - out += pow(output[neuronIdx] - mLayers.last().mNeurons[neuronIdx].mActivationValue, 2); + for (auto neuronIdx : Range(mLayers.last().neurons.size())) { + out += pow(output[neuronIdx] - mLayers.last().neurons[neuronIdx].activationValue, 2); } return out; } -void FullyConnectedNN::clearGrad() { - mLayersCache.reserve(mLayers.size()); - for (auto layIdx : Range(0, (halni) mLayers.size())) { - mLayersCache[layIdx].mNeurons.reserve(mLayers[layIdx].mNeurons.size()); - mLayersCache[layIdx].mCache = 1; - - for (auto weightIdx : Range(mLayersCache[layIdx].mNeurons.size())) { - mLayersCache[layIdx].mNeurons[weightIdx].mBiasGrad = 0; - mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad.reserve(mLayers[layIdx].mNeurons[weightIdx].mWeights.size()); - for (auto weight : mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad) { - weight.data() = 0; - } - } - } - mAvgCount = 0; -} - void FullyConnectedNN::calcGrad(const Buffer& output) { - ASSERT(mLayers.last().mNeurons.size() == output.size()) + ASSERT(mLayers.last().neurons.size() == output.size()) auto& lastLayer = mLayers.last(); - auto& lastCache = mLayersCache.last(); - lastCache.mCache = 1; - - for (auto neuronIdx : Range(lastLayer.mNeurons.size())) { - auto& neuronCache = lastCache.mNeurons[neuronIdx]; - auto& neuron = lastLayer.mNeurons[neuronIdx]; - lastCache.mCache += 2 * pow(output[neuronIdx] - neuron.mActivationValue, 2); + // calculate chaining cache value for each neuron in last layer + for (auto neuronIdx : Range(lastLayer.neurons.size())) { + auto& neuron = lastLayer.neurons[neuronIdx]; + neuron.cache = 2 * (neuron.activationValue - output[neuronIdx]); } + // calculate rest of the layers for (auto layerIdx = mLayers.size() - 1; layerIdx > 0; layerIdx--) { - auto& layer = mLayers[layerIdx]; - auto& cache = mLayersCache[layerIdx]; - auto& layerNext = mLayers[layerIdx - 1]; - auto& cacheNext = mLayersCache[layerIdx - 1]; + auto& currentLayer = mLayers[layerIdx]; + auto& inputLayer = mLayers[layerIdx - 1]; - cacheNext.mCache = 1; + for (auto currentNeuronIdx : Range(currentLayer.neurons.size())) { + auto& currentNeuron = currentLayer.neurons[currentNeuronIdx]; - for (auto neuronIdx : Range(layer.mNeurons.size())) { - auto& neuron = layer.mNeurons[neuronIdx]; - auto& neuronCache = cache.mNeurons[neuronIdx]; + // calculate cache value (chaining) + if (layerIdx != mLayers.size() - 1) { + auto& userLayer = mLayers[layerIdx + 1]; - auto tmp = cache.mCache * reluDerivative(neuron.mActivationValueLinear); - - neuronCache.mBiasGrad = tmp; - - for (auto weightIdx : Range(neuron.mWeights.size())) { - neuronCache.mWeightsGrad[weightIdx] = tmp * layerNext.mNeurons[weightIdx].mActivationValue; + currentNeuron.cache = 0; + for (auto userNeuron : userLayer.neurons) { + currentNeuron.cache += userNeuron->weights[currentNeuronIdx].val * userNeuron->cache; + } + currentNeuron.cache *= activationFunctionDerivative(currentNeuron.activationValueLinear); } - cacheNext.mCache += tmp; + // gradient for current neuron bias + currentNeuron.bias.grad = currentNeuron.cache; + + // calculate gradient for weights of current neuron + for (auto weightIdx : Range(currentNeuron.weights.size())) { + currentNeuron.weights[weightIdx].grad = inputLayer.neurons[weightIdx].activationValue * currentNeuron.cache; + } } } + + mAvgCount++; } -void FullyConnectedNN::applyGrad() { +void FullyConnectedNN::applyGrad(halnf step) { + for (auto layIdx : Range(1, (halni) mLayers.size())) { - auto& layerCache = mLayersCache[layIdx]; auto& layer = mLayers[layIdx]; - for (auto neuronIdx : Range(layer.mNeurons.size())) { - auto& neuron = layer.mNeurons[neuronIdx]; - auto& neuronCache = layerCache.mNeurons[neuronIdx]; - - neuron.mBias += neuronCache.mBiasGrad / (halnf) mAvgCount; - - for (auto weightIdx : Range(neuron.mWeights.size())) { - neuron.mWeights[weightIdx] += neuronCache.mWeightsGrad[weightIdx] / (halnf) mAvgCount; + for (auto neuron : layer.neurons) { + neuron->bias.val -= neuron->bias.grad / (halnf) mAvgCount * step; + for (auto weightIdx : Range(neuron->weights.size())) { + neuron->weights[weightIdx].val -= (neuron->weights[weightIdx].grad / (halnf) mAvgCount) * step; } } } + + mAvgCount = 0; } \ No newline at end of file diff --git a/DataAnalysis/public/FullyConnectedNN.hpp b/DataAnalysis/public/FullyConnectedNN.hpp index 580e3e8..a699853 100644 --- a/DataAnalysis/public/FullyConnectedNN.hpp +++ b/DataAnalysis/public/FullyConnectedNN.hpp @@ -7,40 +7,42 @@ namespace tp { class FullyConnectedNN { struct Layer { + struct Neuron { - halnf mBias = 0; - Buffer mWeights; - halnf mActivationValue = 0; - halnf mActivationValueLinear = 0; + struct Parameter { + Parameter() = default; + + Parameter(halnf in) : + val(in) {} + + halnf val = 0; + halnf grad = 0; + }; + + Parameter bias; + Buffer weights; + + halnf activationValue = 0; + halnf activationValueLinear = 0; + + halnf cache; }; - Buffer mNeurons; - }; - - struct LayerCache { - struct NeuronCache { - halnf mBiasGrad = 0; - Buffer mWeightsGrad; - }; - - halnf mCache = 0; - Buffer mNeurons; + Buffer neurons; }; public: FullyConnectedNN() = default; - void initializeRandom(Buffer description); + void initializeRandom(const Buffer& description); void evaluate(const Buffer& input, Buffer& output); - void clearGrad(); halnf calcCost(const Buffer& output); void calcGrad(const Buffer& output); - void applyGrad(); + void applyGrad(halnf step); - private: + public: Buffer mLayers; - Buffer mLayersCache; halni mAvgCount = 0; }; }; diff --git a/DataAnalysis/tests/Tests.cpp b/DataAnalysis/tests/Tests.cpp index 968f9f7..851c5ed 100644 --- a/DataAnalysis/tests/Tests.cpp +++ b/DataAnalysis/tests/Tests.cpp @@ -1,14 +1,45 @@ +#include "NewPlacement.hpp" + #include "FullyConnectedNN.hpp" #include "Testing.hpp" #include "Utils.hpp" +#include + static bool init(const tp::ModuleManifest* self) { tp::gTesting.setRootName(self->getName()); return true; } -void test() {} +void test() { + using namespace tp; + + Buffer layers = { 4, 4, 3, 2 }; + Buffer input = { (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100 }; + Buffer outputExpected = { (halnf) randomFloat(), (halnf) randomFloat() }; + + FullyConnectedNN nn; + Buffer output(2); + + nn.initializeRandom(layers); + + // nn.mLayers.last().neurons.first().weights = { 0.35, 0.35, 0.35, 0.35 }; + // nn.mLayers.last().neurons.first().bias = 0.9; + + // nn.mLayers.last().neurons.last().weights = { -0.35, -0.35, -0.35, -0.35 }; + // nn.mLayers.last().neurons.last().bias = -3.9; + + for (auto i : Range(50)) { + nn.evaluate(input, output); + + auto lossBefore = nn.calcCost(outputExpected); + + nn.calcGrad(outputExpected); + nn.applyGrad(0.1); + printf("Loss %f \n", lossBefore); + } +} int main() { tp::ModuleManifest* deps[] = { &tp::gModuleDataAnalysis, &tp::gModuleUtils, nullptr }; diff --git a/TODO b/TODO index fc8f9a9..b092d80 100644 --- a/TODO +++ b/TODO @@ -20,12 +20,12 @@ RayTracer: Quality: Render eq - Materials Lenses Exposure Efficiency: Casting Accelerators + MIS (DMIS CMIS) Important sampling Threading diff --git a/Utils/private/Utils.cpp b/Utils/private/Utils.cpp index a0ce799..aec5efc 100644 --- a/Utils/private/Utils.cpp +++ b/Utils/private/Utils.cpp @@ -5,6 +5,7 @@ #include "Testing.hpp" +#include #include void initializeCallStackCapture(); @@ -12,6 +13,7 @@ void deinitializeCallStackCapture(); static bool initialize(const tp::ModuleManifest*) { initializeCallStackCapture(); + std::srand(std::time(nullptr)); return true; }