ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

BackProp Fixes

Browse source
This commit is contained in:
IlyaShurupov 2023-10-25 11:03:15 +03:00 • committed by Ilya Shurupov
parent 7e17b32a81
commit 90ce453b60
11 changed files with 189 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

44
Containers/public/Buffer.hpp
View file

@ -182,10 +182,26 @@ namespace tp {
mBuff = (tType*) mAllocator.allocate(sizeof(tType) * tMinSize);
}
explicit Buffer(ualni size) :
mSize(size),
mLoad(0) {
Buffer(const InitialierList<tType>& input) {
mSize = 0;
for (const auto& val : input) {
mSize++;
}
mBuff = (tType*) mAllocator.allocate(sizeof(tType) * mSize);
mLoad = 0;
for (const auto& val : input) {
mBuff[mLoad] = val;
mLoad++;
}
}
explicit Buffer(ualni size) {
mBuff = (tType*) mAllocator.allocate(sizeof(tType) * size);
mSize = size;
mLoad = size;
for (ualni i = 0; i < mLoad; i++) {
new (mBuff + i) tType();
}
}
Buffer(const Buffer& in) :
@ -274,11 +290,25 @@ namespace tp {
}
public:
Buffer& operator=(const tp::InitialierList<tType>& init) {
Buffer& operator=(const tp::InitialierList<tType>& input) {
// TODO : optimize
for (auto arg : init) {
append(arg);
for (ualni i = 0; i < mLoad; i++) {
mBuff[i].~tType();
}
mAllocator.deallocate(mBuff);
mSize = 0;
for (const auto& val : input) {
mSize++;
}
mBuff = (tType*) mAllocator.allocate(sizeof(tType) * mSize);
mLoad = 0;
for (const auto& val : input) {
mBuff[mLoad] = val;
mLoad++;
}
return *this;
}
@ -320,10 +350,10 @@ namespace tp {
}
void reserve(ualni aSize) {
if (aSize == mSize) return;
for (ualni i = 0; i < mLoad; i++) {
mBuff[i].~tType();
}
mAllocator.deallocate(mBuff);
mBuff = (tType*) mAllocator.allocate(sizeof(tType) * aSize);
mSize = aSize;
mLoad = aSize;

4
Containers/tests/BufferTest.cpp
View file

@ -22,10 +22,10 @@ TEST_DEF_STATIC(Simple1) {
TEST_DEF_STATIC(Simple2) {
Buffer<TestClass, HeapAlloc> buff(size);
TEST(buff.size() == 0);
TEST(buff.size() == size);
for (auto i : Range(size * 10))
buff.append(TestClass(i));
TEST(buff.size() == size * 10);
TEST(buff.size() == size + size * 10);
while (buff.size())
buff.pop();
TEST(buff.size() == 0);

2
Containers/tests/Tests.hpp
View file

@ -8,6 +8,8 @@ class TestClass {
tp::ualni val1;
public:
TestClass() { val1 = 0; }
explicit TestClass(tp::ualni val) :
val1(val) {}

21
DataAnalysis/applications/NumberRecognition.cpp
View file

@ -11,6 +11,19 @@ struct Dataset {
Buffer<Buffer<uint1>> images;
};
void displayImage(const Dataset& dataset, ualni idx) {
auto& image = dataset.images[idx];
auto label = dataset.labels[idx];
printf("Image : %i\n", int(label));
for (auto i : Range(dataset.imageSize.x)) {
for (auto j : Range(dataset.imageSize.y)) {
printf("%c", image[j * dataset.imageSize.x + i]);
}
printf("\n");
}
}
bool loadDataset(Dataset& out, const String& location) {
LocalConnection dataset;
dataset.connect(LocalConnection::Location(location), LocalConnection::Type(true));
@ -116,16 +129,16 @@ void testTraining(const Dataset& dataset, FullyConnectedNN& nn) {
return resultNumber;
};
nn.clearGrad();
displayImage(dataset, 2);
propagate(0);
nn.applyGrad();
// nn.applyGrad();
propagate(0);
nn.applyGrad();
// nn.applyGrad();
propagate(0);
nn.applyGrad();
// nn.applyGrad();
auto errorPercentage = test(dataset, nn, { 0, 1 });
printf("Percentage error Trained on first image: %f\n", errorPercentage);

BIN
DataAnalysis/applications/rsc/mnist
View file

Binary file not shown.

BIN
DataAnalysis/applications/rsc/mnist.gz Normal file
View file

Binary file not shown.

149
DataAnalysis/private/FullyConnectedNN.cpp
View file

@ -1,139 +1,142 @@
// #include "NewPlacement.hpp"
#include "FullyConnectedNN.hpp"
#include "NewPlacement.hpp"
#include "Utils.hpp"
#include <cmath>
using namespace tp;
static halnf sigmoid(halnf val) { return 0; }
static halnf linearFun(halnf val) { return val; }
static halnf linearFunDerivative(halnf val) { return 1; }
static halnf sigmoid(double val) { return 1.0f / (1.0f + (halnf) exp(-val)); }
static halnf sigmoidDerivative(double val) {
halnf sigmoid_val = sigmoid(val);
return sigmoid_val * (1.0f - sigmoid_val);
}
static halnf relu(halnf val) { return val < 0 ? 0 : val; }
static halnf reluDerivative(halnf val) { return val < 0 ? 0 : 1; }
void FullyConnectedNN::initializeRandom(Buffer<halni> description) {
static halnf activationFunction(halnf val) { return sigmoid(val); }
static halnf activationFunctionDerivative(halnf val) { return sigmoidDerivative(val); }
void FullyConnectedNN::initializeRandom(const Buffer<halni>& description) {
ASSERT(description.size() > 1);
mLayers.reserve(description.size());
for (auto i : Range<halni>(0, (halni) description.size())) {
mLayers[i].mNeurons.reserve(description[i]);
mLayers[i].neurons.reserve(description[i]);
if (i == 0) {
continue;
}
for (auto neuron : mLayers[i].mNeurons) {
neuron->mWeights.reserve(description[i - 1]);
for (auto weight : neuron->mWeights) {
weight.data() = (halnf) (randomFloat() - 0.5) * 2;
for (auto neuron : mLayers[i].neurons) {
neuron->weights.reserve(description[i - 1]);
for (auto weight : neuron->weights) {
weight->val = (halnf) (randomFloat() - 0) * 2;
}
neuron->mBias = (halnf) (randomFloat() - 0.5) * 2;
neuron->bias.val = (halnf) (randomFloat() - 0) * 2;
}
}
}
void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& output) {
ASSERT(output.size() == mLayers.last().mNeurons.size() && input.size() == mLayers.first().mNeurons.size())
ASSERT(output.size() == mLayers.last().neurons.size() && input.size() == mLayers.first().neurons.size())
for (auto idx : Range(input.size())) {
mLayers.first().mNeurons[idx].mActivationValue = input[idx];
mLayers.first().neurons[idx].activationValue = input[idx];
}
for (auto layerIdx : Range<halni>(1, (halni) mLayers.size())) {
auto& layer = mLayers[layerIdx];
auto& layerPrev = mLayers[layerIdx - 1];
for (auto neuron : layer.mNeurons) {
neuron->mActivationValue = 0;
for (auto connectionIdx : Range(neuron->mWeights.size())) {
neuron->mActivationValue += neuron->mWeights[connectionIdx] * layerPrev.mNeurons[connectionIdx].mActivationValue;
for (auto neuron : layer.neurons) {
neuron->activationValue = 0;
for (auto connectionIdx : Range(neuron->weights.size())) {
neuron->activationValue += neuron->weights[connectionIdx].val * layerPrev.neurons[connectionIdx].activationValue;
}
neuron->mActivationValue += neuron->mBias;
neuron->mActivationValueLinear = neuron->mActivationValue;
neuron->mActivationValue = relu(neuron->mActivationValue);
neuron->activationValue += neuron->bias.val;
neuron->activationValueLinear = neuron->activationValue;
neuron->activationValue = activationFunction(neuron->activationValue);
}
}
for (auto idx : Range(output.size())) {
output[idx] = mLayers.last().mNeurons[idx].mActivationValue;
output[idx] = mLayers.last().neurons[idx].activationValue;
}
}
halnf FullyConnectedNN::calcCost(const Buffer<halnf>& output) {
halnf out = 0;
for (auto neuronIdx : Range(mLayers.last().mNeurons.size())) {
out += pow(output[neuronIdx] - mLayers.last().mNeurons[neuronIdx].mActivationValue, 2);
for (auto neuronIdx : Range(mLayers.last().neurons.size())) {
out += pow(output[neuronIdx] - mLayers.last().neurons[neuronIdx].activationValue, 2);
}
return out;
}
void FullyConnectedNN::clearGrad() {
mLayersCache.reserve(mLayers.size());
for (auto layIdx : Range<halni>(0, (halni) mLayers.size())) {
mLayersCache[layIdx].mNeurons.reserve(mLayers[layIdx].mNeurons.size());
mLayersCache[layIdx].mCache = 1;
for (auto weightIdx : Range(mLayersCache[layIdx].mNeurons.size())) {
mLayersCache[layIdx].mNeurons[weightIdx].mBiasGrad = 0;
mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad.reserve(mLayers[layIdx].mNeurons[weightIdx].mWeights.size());
for (auto weight : mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad) {
weight.data() = 0;
}
}
}
mAvgCount = 0;
}
void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
ASSERT(mLayers.last().mNeurons.size() == output.size())
ASSERT(mLayers.last().neurons.size() == output.size())
auto& lastLayer = mLayers.last();
auto& lastCache = mLayersCache.last();
lastCache.mCache = 1;
for (auto neuronIdx : Range(lastLayer.mNeurons.size())) {
auto& neuronCache = lastCache.mNeurons[neuronIdx];
auto& neuron = lastLayer.mNeurons[neuronIdx];
lastCache.mCache += 2 * pow(output[neuronIdx] - neuron.mActivationValue, 2);
// calculate chaining cache value for each neuron in last layer
for (auto neuronIdx : Range(lastLayer.neurons.size())) {
auto& neuron = lastLayer.neurons[neuronIdx];
neuron.cache = 2 * (neuron.activationValue - output[neuronIdx]);
}
// calculate rest of the layers
for (auto layerIdx = mLayers.size() - 1; layerIdx > 0; layerIdx--) {
auto& layer = mLayers[layerIdx];
auto& cache = mLayersCache[layerIdx];
auto& layerNext = mLayers[layerIdx - 1];
auto& cacheNext = mLayersCache[layerIdx - 1];
auto& currentLayer = mLayers[layerIdx];
auto& inputLayer = mLayers[layerIdx - 1];
cacheNext.mCache = 1;
for (auto currentNeuronIdx : Range(currentLayer.neurons.size())) {
auto& currentNeuron = currentLayer.neurons[currentNeuronIdx];
for (auto neuronIdx : Range(layer.mNeurons.size())) {
auto& neuron = layer.mNeurons[neuronIdx];
auto& neuronCache = cache.mNeurons[neuronIdx];
// calculate cache value (chaining)
if (layerIdx != mLayers.size() - 1) {
auto& userLayer = mLayers[layerIdx + 1];
auto tmp = cache.mCache * reluDerivative(neuron.mActivationValueLinear);
neuronCache.mBiasGrad = tmp;
for (auto weightIdx : Range(neuron.mWeights.size())) {
neuronCache.mWeightsGrad[weightIdx] = tmp * layerNext.mNeurons[weightIdx].mActivationValue;
currentNeuron.cache = 0;
for (auto userNeuron : userLayer.neurons) {
currentNeuron.cache += userNeuron->weights[currentNeuronIdx].val * userNeuron->cache;
}
currentNeuron.cache *= activationFunctionDerivative(currentNeuron.activationValueLinear);
}
cacheNext.mCache += tmp;
// gradient for current neuron bias
currentNeuron.bias.grad = currentNeuron.cache;
// calculate gradient for weights of current neuron
for (auto weightIdx : Range(currentNeuron.weights.size())) {
currentNeuron.weights[weightIdx].grad = inputLayer.neurons[weightIdx].activationValue * currentNeuron.cache;
}
}
}
void FullyConnectedNN::applyGrad() {
mAvgCount++;
}
void FullyConnectedNN::applyGrad(halnf step) {
for (auto layIdx : Range<halni>(1, (halni) mLayers.size())) {
auto& layerCache = mLayersCache[layIdx];
auto& layer = mLayers[layIdx];
for (auto neuronIdx : Range(layer.mNeurons.size())) {
auto& neuron = layer.mNeurons[neuronIdx];
auto& neuronCache = layerCache.mNeurons[neuronIdx];
for (auto neuron : layer.neurons) {
neuron->bias.val -= neuron->bias.grad / (halnf) mAvgCount * step;
for (auto weightIdx : Range(neuron->weights.size())) {
neuron->weights[weightIdx].val -= (neuron->weights[weightIdx].grad / (halnf) mAvgCount) * step;
}
}
}
neuron.mBias += neuronCache.mBiasGrad / (halnf) mAvgCount;
for (auto weightIdx : Range(neuron.mWeights.size())) {
neuron.mWeights[weightIdx] += neuronCache.mWeightsGrad[weightIdx] / (halnf) mAvgCount;
}
}
}
mAvgCount = 0;
}

38
DataAnalysis/public/FullyConnectedNN.hpp
View file

@ -7,40 +7,42 @@ namespace tp {
class FullyConnectedNN {
struct Layer {
struct Neuron {
halnf mBias = 0;
Buffer<halnf> mWeights;
halnf mActivationValue = 0;
halnf mActivationValueLinear = 0;
struct Parameter {
Parameter() = default;
Parameter(halnf in) :
val(in) {}
halnf val = 0;
halnf grad = 0;
};
Buffer<Neuron> mNeurons;
Parameter bias;
Buffer<Parameter> weights;
halnf activationValue = 0;
halnf activationValueLinear = 0;
halnf cache;
};
struct LayerCache {
struct NeuronCache {
halnf mBiasGrad = 0;
Buffer<halnf> mWeightsGrad;
};
halnf mCache = 0;
Buffer<NeuronCache> mNeurons;
Buffer<Neuron> neurons;
};
public:
FullyConnectedNN() = default;
void initializeRandom(Buffer<halni> description);
void initializeRandom(const Buffer<halni>& description);
void evaluate(const Buffer<halnf>& input, Buffer<halnf>& output);
void clearGrad();
halnf calcCost(const Buffer<halnf>& output);
void calcGrad(const Buffer<halnf>& output);
void applyGrad();
void applyGrad(halnf step);
private:
public:
Buffer<Layer> mLayers;
Buffer<LayerCache> mLayersCache;
halni mAvgCount = 0;
};
};

33
DataAnalysis/tests/Tests.cpp
View file

@ -1,14 +1,45 @@
#include "NewPlacement.hpp"
#include "FullyConnectedNN.hpp"
#include "Testing.hpp"
#include "Utils.hpp"
#include <stdio.h>
static bool init(const tp::ModuleManifest* self) {
tp::gTesting.setRootName(self->getName());
return true;
}
void test() {}
void test() {
using namespace tp;
Buffer<halni> layers = { 4, 4, 3, 2 };
Buffer<halnf> input = { (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100 };
Buffer<halnf> outputExpected = { (halnf) randomFloat(), (halnf) randomFloat() };
FullyConnectedNN nn;
Buffer<halnf> output(2);
nn.initializeRandom(layers);
// nn.mLayers.last().neurons.first().weights = { 0.35, 0.35, 0.35, 0.35 };
// nn.mLayers.last().neurons.first().bias = 0.9;
// nn.mLayers.last().neurons.last().weights = { -0.35, -0.35, -0.35, -0.35 };
// nn.mLayers.last().neurons.last().bias = -3.9;
for (auto i : Range(50)) {
nn.evaluate(input, output);
auto lossBefore = nn.calcCost(outputExpected);
nn.calcGrad(outputExpected);
nn.applyGrad(0.1);
printf("Loss %f \n", lossBefore);
}
}
int main() {
tp::ModuleManifest* deps[] = { &tp::gModuleDataAnalysis, &tp::gModuleUtils, nullptr };

2
TODO
View file

@ -20,12 +20,12 @@ RayTracer:
Quality:
Render eq
Materials
Lenses
Exposure
Efficiency:
Casting Accelerators
MIS (DMIS CMIS)
Important sampling
Threading

2
Utils/private/Utils.cpp
View file

@ -5,6 +5,7 @@
#include "Testing.hpp"
#include <ctime>
#include <random>
void initializeCallStackCapture();
@ -12,6 +13,7 @@ void deinitializeCallStackCapture();
static bool initialize(const tp::ModuleManifest*) {
initializeCallStackCapture();
std::srand(std::time(nullptr));
return true;
}