ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

BackProp Fixes

Browse source
This commit is contained in:
IlyaShurupov 2023-10-25 11:03:15 +03:00
parent efbb09f8ba
commit 0542b7ba2e
11 changed files with 189 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

44
Containers/public/Buffer.hpp
View file

@ -182,10 +182,26 @@ namespace tp {
mBuff = (tType*) mAllocator.allocate(sizeof(tType) * tMinSize);
}
explicit Buffer(ualni size) :
mSize(size),
mLoad(0) {
Buffer(const InitialierList<tType>& input) {
mSize = 0;
for (const auto& val : input) {
mSize++;
}
mBuff = (tType*) mAllocator.allocate(sizeof(tType) * mSize);
mLoad = 0;
for (const auto& val : input) {
mBuff[mLoad] = val;
mLoad++;
}
}
explicit Buffer(ualni size) {
mBuff = (tType*) mAllocator.allocate(sizeof(tType) * size);
mSize = size;
mLoad = size;
for (ualni i = 0; i < mLoad; i++) {
new (mBuff + i) tType();
}
}
Buffer(const Buffer& in) :
@ -274,11 +290,25 @@ namespace tp {
}
public:
Buffer& operator=(const tp::InitialierList<tType>& init) {
Buffer& operator=(const tp::InitialierList<tType>& input) {
// TODO : optimize
for (auto arg : init) {
append(arg);
for (ualni i = 0; i < mLoad; i++) {
mBuff[i].~tType();
}
mAllocator.deallocate(mBuff);
mSize = 0;
for (const auto& val : input) {
mSize++;
}
mBuff = (tType*) mAllocator.allocate(sizeof(tType) * mSize);
mLoad = 0;
for (const auto& val : input) {
mBuff[mLoad] = val;
mLoad++;
}
return *this;
}
@ -320,10 +350,10 @@ namespace tp {
}
void reserve(ualni aSize) {
if (aSize == mSize) return;
for (ualni i = 0; i < mLoad; i++) {
mBuff[i].~tType();
}
mAllocator.deallocate(mBuff);
mBuff = (tType*) mAllocator.allocate(sizeof(tType) * aSize);
mSize = aSize;
mLoad = aSize;

4
Containers/tests/BufferTest.cpp
View file

@ -22,10 +22,10 @@ TEST_DEF_STATIC(Simple1) {
TEST_DEF_STATIC(Simple2) {
Buffer<TestClass, HeapAlloc> buff(size);
TEST(buff.size() == 0);
TEST(buff.size() == size);
for (auto i : Range(size * 10))
buff.append(TestClass(i));
TEST(buff.size() == size * 10);
TEST(buff.size() == size + size * 10);
while (buff.size())
buff.pop();
TEST(buff.size() == 0);

2
Containers/tests/Tests.hpp
View file

@ -8,6 +8,8 @@ class TestClass {
tp::ualni val1;
public:
TestClass() { val1 = 0; }
explicit TestClass(tp::ualni val) :
val1(val) {}

21
DataAnalysis/applications/NumberRecognition.cpp
View file

@ -11,6 +11,19 @@ struct Dataset {
Buffer<Buffer<uint1>> images;
};
void displayImage(const Dataset& dataset, ualni idx) {
auto& image = dataset.images[idx];
auto label = dataset.labels[idx];
printf("Image : %i\n", int(label));
for (auto i : Range(dataset.imageSize.x)) {
for (auto j : Range(dataset.imageSize.y)) {
printf("%c", image[j * dataset.imageSize.x + i]);
}
printf("\n");
}
}
bool loadDataset(Dataset& out, const String& location) {
LocalConnection dataset;
dataset.connect(LocalConnection::Location(location), LocalConnection::Type(true));
@ -116,16 +129,16 @@ void testTraining(const Dataset& dataset, FullyConnectedNN& nn) {
return resultNumber;
};
nn.clearGrad();
displayImage(dataset, 2);
propagate(0);
nn.applyGrad();
// nn.applyGrad();
propagate(0);
nn.applyGrad();
// nn.applyGrad();
propagate(0);
nn.applyGrad();
// nn.applyGrad();
auto errorPercentage = test(dataset, nn, { 0, 1 });
printf("Percentage error Trained on first image: %f\n", errorPercentage);

BIN
DataAnalysis/applications/rsc/mnist
View file

Binary file not shown.

BIN
DataAnalysis/applications/rsc/mnist.gz Normal file
View file

Binary file not shown.

145
DataAnalysis/private/FullyConnectedNN.cpp
View file

@ -1,139 +1,142 @@
// #include "NewPlacement.hpp"
#include "FullyConnectedNN.hpp"
#include "NewPlacement.hpp"
#include "Utils.hpp"
#include <cmath>
using namespace tp;
static halnf sigmoid(halnf val) { return 0; }
static halnf linearFun(halnf val) { return val; }
static halnf linearFunDerivative(halnf val) { return 1; }
static halnf sigmoid(double val) { return 1.0f / (1.0f + (halnf) exp(-val)); }
static halnf sigmoidDerivative(double val) {
halnf sigmoid_val = sigmoid(val);
return sigmoid_val * (1.0f - sigmoid_val);
}
static halnf relu(halnf val) { return val < 0 ? 0 : val; }
static halnf reluDerivative(halnf val) { return val < 0 ? 0 : 1; }
void FullyConnectedNN::initializeRandom(Buffer<halni> description) {
static halnf activationFunction(halnf val) { return sigmoid(val); }
static halnf activationFunctionDerivative(halnf val) { return sigmoidDerivative(val); }
void FullyConnectedNN::initializeRandom(const Buffer<halni>& description) {
ASSERT(description.size() > 1);
mLayers.reserve(description.size());
for (auto i : Range<halni>(0, (halni) description.size())) {
mLayers[i].mNeurons.reserve(description[i]);
mLayers[i].neurons.reserve(description[i]);
if (i == 0) {
continue;
}
for (auto neuron : mLayers[i].mNeurons) {
neuron->mWeights.reserve(description[i - 1]);
for (auto weight : neuron->mWeights) {
weight.data() = (halnf) (randomFloat() - 0.5) * 2;
for (auto neuron : mLayers[i].neurons) {
neuron->weights.reserve(description[i - 1]);
for (auto weight : neuron->weights) {
weight->val = (halnf) (randomFloat() - 0) * 2;
}
neuron->mBias = (halnf) (randomFloat() - 0.5) * 2;
neuron->bias.val = (halnf) (randomFloat() - 0) * 2;
}
}
}
void FullyConnectedNN::evaluate(const Buffer<halnf>& input, Buffer<halnf>& output) {
ASSERT(output.size() == mLayers.last().mNeurons.size() && input.size() == mLayers.first().mNeurons.size())
ASSERT(output.size() == mLayers.last().neurons.size() && input.size() == mLayers.first().neurons.size())
for (auto idx : Range(input.size())) {
mLayers.first().mNeurons[idx].mActivationValue = input[idx];
mLayers.first().neurons[idx].activationValue = input[idx];
}
for (auto layerIdx : Range<halni>(1, (halni) mLayers.size())) {
auto& layer = mLayers[layerIdx];
auto& layerPrev = mLayers[layerIdx - 1];
for (auto neuron : layer.mNeurons) {
neuron->mActivationValue = 0;
for (auto connectionIdx : Range(neuron->mWeights.size())) {
neuron->mActivationValue += neuron->mWeights[connectionIdx] * layerPrev.mNeurons[connectionIdx].mActivationValue;
for (auto neuron : layer.neurons) {
neuron->activationValue = 0;
for (auto connectionIdx : Range(neuron->weights.size())) {
neuron->activationValue += neuron->weights[connectionIdx].val * layerPrev.neurons[connectionIdx].activationValue;
}
neuron->mActivationValue += neuron->mBias;
neuron->mActivationValueLinear = neuron->mActivationValue;
neuron->mActivationValue = relu(neuron->mActivationValue);
neuron->activationValue += neuron->bias.val;
neuron->activationValueLinear = neuron->activationValue;
neuron->activationValue = activationFunction(neuron->activationValue);
}
}
for (auto idx : Range(output.size())) {
output[idx] = mLayers.last().mNeurons[idx].mActivationValue;
output[idx] = mLayers.last().neurons[idx].activationValue;
}
}
halnf FullyConnectedNN::calcCost(const Buffer<halnf>& output) {
halnf out = 0;
for (auto neuronIdx : Range(mLayers.last().mNeurons.size())) {
out += pow(output[neuronIdx] - mLayers.last().mNeurons[neuronIdx].mActivationValue, 2);
for (auto neuronIdx : Range(mLayers.last().neurons.size())) {
out += pow(output[neuronIdx] - mLayers.last().neurons[neuronIdx].activationValue, 2);
}
return out;
}
void FullyConnectedNN::clearGrad() {
mLayersCache.reserve(mLayers.size());
for (auto layIdx : Range<halni>(0, (halni) mLayers.size())) {
mLayersCache[layIdx].mNeurons.reserve(mLayers[layIdx].mNeurons.size());
mLayersCache[layIdx].mCache = 1;
for (auto weightIdx : Range(mLayersCache[layIdx].mNeurons.size())) {
mLayersCache[layIdx].mNeurons[weightIdx].mBiasGrad = 0;
mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad.reserve(mLayers[layIdx].mNeurons[weightIdx].mWeights.size());
for (auto weight : mLayersCache[layIdx].mNeurons[weightIdx].mWeightsGrad) {
weight.data() = 0;
}
}
}
mAvgCount = 0;
}
void FullyConnectedNN::calcGrad(const Buffer<halnf>& output) {
ASSERT(mLayers.last().mNeurons.size() == output.size())
ASSERT(mLayers.last().neurons.size() == output.size())
auto& lastLayer = mLayers.last();
auto& lastCache = mLayersCache.last();
lastCache.mCache = 1;
for (auto neuronIdx : Range(lastLayer.mNeurons.size())) {
auto& neuronCache = lastCache.mNeurons[neuronIdx];
auto& neuron = lastLayer.mNeurons[neuronIdx];
lastCache.mCache += 2 * pow(output[neuronIdx] - neuron.mActivationValue, 2);
// calculate chaining cache value for each neuron in last layer
for (auto neuronIdx : Range(lastLayer.neurons.size())) {
auto& neuron = lastLayer.neurons[neuronIdx];
neuron.cache = 2 * (neuron.activationValue - output[neuronIdx]);
}
// calculate rest of the layers
for (auto layerIdx = mLayers.size() - 1; layerIdx > 0; layerIdx--) {
auto& layer = mLayers[layerIdx];
auto& cache = mLayersCache[layerIdx];
auto& layerNext = mLayers[layerIdx - 1];
auto& cacheNext = mLayersCache[layerIdx - 1];
auto& currentLayer = mLayers[layerIdx];
auto& inputLayer = mLayers[layerIdx - 1];
cacheNext.mCache = 1;
for (auto currentNeuronIdx : Range(currentLayer.neurons.size())) {
auto& currentNeuron = currentLayer.neurons[currentNeuronIdx];
for (auto neuronIdx : Range(layer.mNeurons.size())) {
auto& neuron = layer.mNeurons[neuronIdx];
auto& neuronCache = cache.mNeurons[neuronIdx];
// calculate cache value (chaining)
if (layerIdx != mLayers.size() - 1) {
auto& userLayer = mLayers[layerIdx + 1];
auto tmp = cache.mCache * reluDerivative(neuron.mActivationValueLinear);
neuronCache.mBiasGrad = tmp;
for (auto weightIdx : Range(neuron.mWeights.size())) {
neuronCache.mWeightsGrad[weightIdx] = tmp * layerNext.mNeurons[weightIdx].mActivationValue;
currentNeuron.cache = 0;
for (auto userNeuron : userLayer.neurons) {
currentNeuron.cache += userNeuron->weights[currentNeuronIdx].val * userNeuron->cache;
}
currentNeuron.cache *= activationFunctionDerivative(currentNeuron.activationValueLinear);
}
cacheNext.mCache += tmp;
// gradient for current neuron bias
currentNeuron.bias.grad = currentNeuron.cache;
// calculate gradient for weights of current neuron
for (auto weightIdx : Range(currentNeuron.weights.size())) {
currentNeuron.weights[weightIdx].grad = inputLayer.neurons[weightIdx].activationValue * currentNeuron.cache;
}
}
}
mAvgCount++;
}
void FullyConnectedNN::applyGrad() {
void FullyConnectedNN::applyGrad(halnf step) {
for (auto layIdx : Range<halni>(1, (halni) mLayers.size())) {
auto& layerCache = mLayersCache[layIdx];
auto& layer = mLayers[layIdx];
for (auto neuronIdx : Range(layer.mNeurons.size())) {
auto& neuron = layer.mNeurons[neuronIdx];
auto& neuronCache = layerCache.mNeurons[neuronIdx];
neuron.mBias += neuronCache.mBiasGrad / (halnf) mAvgCount;
for (auto weightIdx : Range(neuron.mWeights.size())) {
neuron.mWeights[weightIdx] += neuronCache.mWeightsGrad[weightIdx] / (halnf) mAvgCount;
for (auto neuron : layer.neurons) {
neuron->bias.val -= neuron->bias.grad / (halnf) mAvgCount * step;
for (auto weightIdx : Range(neuron->weights.size())) {
neuron->weights[weightIdx].val -= (neuron->weights[weightIdx].grad / (halnf) mAvgCount) * step;
}
}
}
mAvgCount = 0;
}

42
DataAnalysis/public/FullyConnectedNN.hpp
View file

@ -7,40 +7,42 @@ namespace tp {
class FullyConnectedNN {
struct Layer {
struct Neuron {
halnf mBias = 0;
Buffer<halnf> mWeights;
halnf mActivationValue = 0;
halnf mActivationValueLinear = 0;
struct Parameter {
Parameter() = default;
Parameter(halnf in) :
val(in) {}
halnf val = 0;
halnf grad = 0;
};
Parameter bias;
Buffer<Parameter> weights;
halnf activationValue = 0;
halnf activationValueLinear = 0;
halnf cache;
};
Buffer<Neuron> mNeurons;
};
struct LayerCache {
struct NeuronCache {
halnf mBiasGrad = 0;
Buffer<halnf> mWeightsGrad;
};
halnf mCache = 0;
Buffer<NeuronCache> mNeurons;
Buffer<Neuron> neurons;
};
public:
FullyConnectedNN() = default;
void initializeRandom(Buffer<halni> description);
void initializeRandom(const Buffer<halni>& description);
void evaluate(const Buffer<halnf>& input, Buffer<halnf>& output);
void clearGrad();
halnf calcCost(const Buffer<halnf>& output);
void calcGrad(const Buffer<halnf>& output);
void applyGrad();
void applyGrad(halnf step);
private:
public:
Buffer<Layer> mLayers;
Buffer<LayerCache> mLayersCache;
halni mAvgCount = 0;
};
};

33
DataAnalysis/tests/Tests.cpp
View file

@ -1,14 +1,45 @@
#include "NewPlacement.hpp"
#include "FullyConnectedNN.hpp"
#include "Testing.hpp"
#include "Utils.hpp"
#include <stdio.h>
static bool init(const tp::ModuleManifest* self) {
tp::gTesting.setRootName(self->getName());
return true;
}
void test() {}
void test() {
using namespace tp;
Buffer<halni> layers = { 4, 4, 3, 2 };
Buffer<halnf> input = { (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100, (halnf) randomFloat() * 100 };
Buffer<halnf> outputExpected = { (halnf) randomFloat(), (halnf) randomFloat() };
FullyConnectedNN nn;
Buffer<halnf> output(2);
nn.initializeRandom(layers);
// nn.mLayers.last().neurons.first().weights = { 0.35, 0.35, 0.35, 0.35 };
// nn.mLayers.last().neurons.first().bias = 0.9;
// nn.mLayers.last().neurons.last().weights = { -0.35, -0.35, -0.35, -0.35 };
// nn.mLayers.last().neurons.last().bias = -3.9;
for (auto i : Range(50)) {
nn.evaluate(input, output);
auto lossBefore = nn.calcCost(outputExpected);
nn.calcGrad(outputExpected);
nn.applyGrad(0.1);
printf("Loss %f \n", lossBefore);
}
}
int main() {
tp::ModuleManifest* deps[] = { &tp::gModuleDataAnalysis, &tp::gModuleUtils, nullptr };

2
TODO
View file

@ -20,12 +20,12 @@ RayTracer:
Quality:
Render eq
Materials
Lenses
Exposure
Efficiency:
Casting Accelerators
MIS (DMIS CMIS)
Important sampling
Threading

2
Utils/private/Utils.cpp
View file

@ -5,6 +5,7 @@
#include "Testing.hpp"
#include <ctime>
#include <random>
void initializeCallStackCapture();
@ -12,6 +13,7 @@ void deinitializeCallStackCapture();
static bool initialize(const tp::ModuleManifest*) {
initializeCallStackCapture();
std::srand(std::time(nullptr));
return true;
}