diff --git a/Containers/public/Buffer.hpp b/Containers/public/Buffer.hpp index 3a62be5..e81abd4 100644 --- a/Containers/public/Buffer.hpp +++ b/Containers/public/Buffer.hpp @@ -179,7 +179,8 @@ namespace tp { for (ualni i = 0; i < mLoad; i++) mBuff[i].~tType(); mBuff = (tType*) mAllocator.allocate(sizeof(tType) * aSize); mSize = aSize; - mLoad = 0; + mLoad = aSize; + for (ualni i = 0; i < mLoad; i++) new (mBuff + i) tType(); } private: diff --git a/Containers/tests/Buffer2DTest.cpp b/Containers/tests/Buffer2DTest.cpp index 9a667e2..252468d 100644 --- a/Containers/tests/Buffer2DTest.cpp +++ b/Containers/tests/Buffer2DTest.cpp @@ -13,8 +13,8 @@ const ualni size = 1000; TEST_DEF_STATIC(Simple1) { Buffer2D buff; buff.reserve({ 4, 4 }); - buff.set( 2, 2, 5); - TEST(buff.get(2, 2) == 5); + buff.set( { 2, 2 }, 5); + TEST(buff.get( { 2, 2 } ) == 5); } TEST_DEF_STATIC(Simple2) { diff --git a/Tokenizer/private/Tokenizer.cpp b/Tokenizer/private/Tokenizer.cpp index c0b6e62..618498b 100644 --- a/Tokenizer/private/Tokenizer.cpp +++ b/Tokenizer/private/Tokenizer.cpp @@ -4,7 +4,6 @@ using namespace tp; static ModuleManifest* sModuleDependencies[] = { - &tp::gModuleMath, &tp::gModuleStrings, nullptr }; diff --git a/Tokenizer/public/AutomataGraph.h b/Tokenizer/public/AutomataGraph.h index 20980bd..cbb7ee5 100644 --- a/Tokenizer/public/AutomataGraph.h +++ b/Tokenizer/public/AutomataGraph.h @@ -32,7 +32,7 @@ namespace tp { bool mExclude = false; bool isTransition(const tAlphabetType& symbol) { - if (symbol == nullptr) return false; + if (symbol == 0) return false; if (!mConsumesSymbol || mAcceptsAll) return true; bool const in_range = (symbol >= mAcceptingRange.mBegin && symbol <= mAcceptingRange.mEnd); return in_range != mExclude; @@ -112,7 +112,7 @@ namespace tp { } } - return { start, end + 1 }; + return Range( start, end + 1 ); } // vertices that are reachable from initial set with no input consumption (E-transitions) @@ -187,7 +187,7 @@ namespace tp { struct DStateKey { const List* nStates; - bool operator==(const DStateKey& in) { + bool operator==(const DStateKey& in) const { if (nStates->length() != in.nStates->length()) { return false; } @@ -238,11 +238,11 @@ namespace tp { // includes closure of NFA start state by definition auto start_state = new DState(); - nfa.closure({ nfa.getStartVertex() }, start_state->nstates, ualni(start_state)); + nfa.closure({ nfa.getStartVertex() }, start_state->nStates, ualni(start_state)); Map dStates; - dStates.put({ &start_state->nstates }, start_state); + dStates.put({ &start_state->nStates }, start_state); List working_set = { start_state }; @@ -255,7 +255,7 @@ namespace tp { List reachableNStates; - nfa.move(currentDState->data->nstates, reachableNStates, symbol, ualni(currentDState + symbol)); + nfa.move(currentDState->data->nStates, reachableNStates, symbol, ualni(currentDState + symbol)); nfa.closure(reachableNStates, reachableNStates, ualni(currentDState + symbol)); if (!reachableNStates.length()) { @@ -278,11 +278,11 @@ namespace tp { targetDState->debug_idx = dStates.size(); #endif - targetDState->nstates = reachableNStates; + targetDState->nStates = reachableNStates; // append to working stack working_set.pushBack(targetDState); - dStates.put({ &targetDState->nstates }, targetDState); + dStates.put({ &targetDState->nStates }, targetDState); } // add transition to DFA state @@ -302,18 +302,18 @@ namespace tp { break; } } - node->val->dvertex = addVertex(state); + node->val->dVertex = addVertex(state); } // connect all vertices for (auto node : dStates) { for (auto edge : node->val->transitions) { - addTransition(node->val->dvertex, edge.data().state->dvertex, edge.data().accepting_code); + addTransition(node->val->dVertex, edge.data().state->dVertex, edge.data().accepting_code); } } // set the starting vertex - mStart = start_state->dvertex; + mStart = start_state->dVertex; // cleanup for (auto node : dStates) { @@ -411,7 +411,7 @@ namespace tp { Buffer2D mTransitions; Buffer mStates; - Range mSymbolRange = nullptr; + Range mSymbolRange = { 0, 0 }; ualni mIter = 0; ualni mIterPrev = 0; @@ -431,31 +431,35 @@ namespace tp { mTransitions.assign(dfa.nVertices()); mStates.reserve(sizeY); + ualni idx = 0; for (auto vertex : dfa.mVertices) { auto state = vertex.data().termination_state; - mStates[vertex.idx()] = state; + mStates[idx] = state; + idx++; } mStates[dfa.nVertices()] = tFailedStateVal; + idx = 0; for (auto vertex : dfa.mVertices) { if (&vertex.data() == dfa.mStart) { - mStart = mIter = mIterPrev = vertex.idx(); + mStart = mIter = mIterPrev = idx; } + idx++; } + ualni vertexIdx = 0; for (auto vertex : dfa.mVertices) { for (auto edge : vertex.data().edges) { - ualni idx = 0; - for (auto vertex : dfa.mVertices) { - if (edge.data().vertex == &vertex.data()) { - idx = vertex.idx(); - break; - } + ualni vertex2Idx = 0; + for (auto vertex2 : dfa.mVertices) { + if (edge.data().vertex == &vertex2.data()) break; + vertex2Idx++; } auto const code = edge.data().transition_code; - mTransitions.set(code - mSymbolRange.mBegin, (uhalni) vertex.idx(), idx); + mTransitions.set( { (ualni) (code - mSymbolRange.mBegin), (ualni) vertexIdx }, vertex2Idx); } + vertexIdx++; } } @@ -465,10 +469,10 @@ namespace tp { tStateType move(tAlphabetType symbol) { if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) { - mIter = mTransitions.get({ symbol - mSymbolRange.mBegin, mIter }); + mIter = mTransitions.get({ (ualni) (symbol - mSymbolRange.mBegin), (ualni) mIter }); } else { - mIter = mStates.length() - 1; + mIter = mStates.size() - 1; } if (mIterPrev == mStart) { diff --git a/Tokenizer/public/RegularExpression.h b/Tokenizer/public/RegularExpression.h index e906df5..b0e447b 100644 --- a/Tokenizer/public/RegularExpression.h +++ b/Tokenizer/public/RegularExpression.h @@ -136,6 +136,7 @@ namespace tp::RegEx { case TOK_ANY: out = parseAny(); break; case TOK_VAL: out = parseVal(); break; case TOK_NONE: { discardTok(); return nullptr; }; + default: break; } if (!out) { discardTok(); diff --git a/Tokenizer/rsc/script.txt b/Tokenizer/rsc/script.txt deleted file mode 100644 index 8c312e1..0000000 --- a/Tokenizer/rsc/script.txt +++ /dev/null @@ -1,18 +0,0 @@ - - - /* ba - asadadasdK */ - - -1.f; - - if (+1.f) { - var string = "asas - d"; - - } - - while - - return - - diff --git a/Tokenizer/tests/TestTokenizer.cpp b/Tokenizer/tests/TestTokenizer.cpp new file mode 100644 index 0000000..a25ce49 --- /dev/null +++ b/Tokenizer/tests/TestTokenizer.cpp @@ -0,0 +1,198 @@ + +#include "Testing.hpp" +#include "Tokenizer.hpp" + +#include + +#define LOG(val) std::cout << #val << " " + +using namespace tp; + +ualni outputHashPassed = 1156; + +const char* script = +R"( + +/* yea +comment */ + +-1.f; + +if (+1.f) { + var string = "string value + another line"; + +} + +while + +return + +)"; + +TEST_DEF_STATIC(Simple) { + + enum class TokType { + NONE, + FAILED, + TOK_SOURCE_END, + + VAR, + CLASS_DEF, + SELF, + SCOPE_IN, + SCOPE_OUT, + ASSIGN, + DEF_FUNC, + PRINT, + IF, + ELSE, + WHILE, + BRACKET_IN, + BRACKET_OUT, + COMMA, + NEW, + CHILD, + RETURN, + EQUAL, + NOT_EQUAL, + MORE, + LESS, + QE_OR_MORE, + QE_OR_LESS, + BOOL_NOT, + BOOL_AND, + BOOL_OR, + ADD, + MUL, + SUB, + DIV, + STM_END, + CONST_TRUE, + CONST_FALSE, + CONST_INT, + CONST_FLOAT, + SPACE, + CONST_STRING, + ID, + + //COMMENT_LINE, + + COMMENT_BLOCK, + }; + + SimpleTokenizer lexer; + + lexer.build({ + { "\n|\t| |\r", TokType::SPACE }, + { "var", TokType::VAR }, + { "class", TokType::CLASS_DEF }, + { "self", TokType::SELF }, + { "\\{", TokType::SCOPE_IN }, + { "\\}", TokType::SCOPE_OUT }, + { "=", TokType::ASSIGN }, + { "def", TokType::DEF_FUNC }, + { "<<", TokType::PRINT }, + { "if", TokType::IF }, + { "else", TokType::ELSE }, + { "while", TokType::WHILE }, + { "\\(", TokType::BRACKET_IN }, + { "\\)", TokType::BRACKET_OUT }, + { ",", TokType::COMMA }, + { "new", TokType::NEW }, + { "\\.", TokType::CHILD }, + { "return", TokType::RETURN }, + { "==", TokType::EQUAL }, + { "!=", TokType::NOT_EQUAL }, + { ">", TokType::MORE }, + { "<", TokType::LESS }, + { ">=", TokType::QE_OR_MORE }, + { "<=", TokType::QE_OR_LESS }, + { "!", TokType::BOOL_NOT }, + { "&&", TokType::BOOL_AND }, + { "\\|\\|", TokType::BOOL_OR }, + { "\\+", TokType::ADD }, + { "\\*", TokType::MUL }, + { "\\-", TokType::SUB }, + { "/", TokType::DIV }, + { ";", TokType::STM_END }, + { "true", TokType::CONST_TRUE }, + { "false", TokType::CONST_FALSE }, + { "((\\-)|(\\+))?[0-9]+i?", TokType::CONST_INT }, + { R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)", TokType::CONST_FLOAT }, + { R"((/\*){\*-\*}*(\*/))", TokType::COMMENT_BLOCK }, + { R"("{"-"}*")", TokType::CONST_STRING }, + { "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", TokType::ID }, + }); + + if (!lexer.isBuild()) { + std::cout << lexer.getBuildError().description; + return; + } + + lexer.bindSource(script); + + TokType tok; + ualni outputHash = 0; + do { + + tok = lexer.readTok(); + + outputHash += ualni(tok); + + switch (tok) { + case TokType::SPACE: { std::cout << " "; } break; + case TokType::VAR: { LOG(VAR); } break; + case TokType::CLASS_DEF: { LOG(CLASS_DEF); } break; + case TokType::SELF: { LOG(SELF); } break; + case TokType::SCOPE_IN: { LOG(SCOPE_IN); } break; + case TokType::SCOPE_OUT: { LOG(SCOPE_OUT); } break; + case TokType::ASSIGN: { LOG(ASSIGN); } break; + case TokType::DEF_FUNC: { LOG(DEF_FUNC); } break; + case TokType::PRINT: { LOG(PRINT); } break; + case TokType::IF: { LOG(IF); } break; + case TokType::ELSE: { LOG(ELSE); } break; + case TokType::WHILE: { LOG(WHILE); } break; + case TokType::BRACKET_IN: { LOG(BRACKET_IN); } break; + case TokType::BRACKET_OUT: { LOG(BRACKET_OUT); } break; + case TokType::COMMA: { LOG(COMMA); } break; + case TokType::NEW: { LOG(NEW); } break; + case TokType::CHILD: { LOG(CHILD); } break; + case TokType::RETURN: { LOG(RETURN); } break; + case TokType::EQUAL: { LOG(EQUAL); } break; + case TokType::NOT_EQUAL: { LOG(NOT_EQUAL); } break; + case TokType::MORE: { LOG(MORE); } break; + case TokType::LESS: { LOG(LESS); } break; + case TokType::QE_OR_MORE: { LOG(QE_OR_MORE); } break; + case TokType::QE_OR_LESS: { LOG(QE_OR_LESS); } break; + case TokType::BOOL_NOT: { LOG(BOOL_NOT); } break; + case TokType::BOOL_AND: { LOG(BOOL_AND); } break; + case TokType::BOOL_OR: { LOG(BOOL_OR); } break; + case TokType::ADD: { LOG(ADD); } break; + case TokType::MUL: { LOG(MUL); } break; + case TokType::SUB: { LOG(SUB); } break; + case TokType::DIV: { LOG(DIV); } break; + case TokType::STM_END: { LOG(STM_END); } break; + case TokType::CONST_TRUE: { LOG(TRUE); } break; + case TokType::CONST_FALSE: { LOG(FALSE); } break; + case TokType::CONST_INT: { LOG(INT); } break; + case TokType::CONST_FLOAT: { LOG(FLOAT); } break; + case TokType::CONST_STRING: { LOG(STRING); } break; + case TokType::ID: { LOG(ID); } break; + //case TokType::COMMENT_LINE: { LOG(COMMENT_LINE) } break; + case TokType::COMMENT_BLOCK: { LOG(COMMENT_BLOCK); } break; + case TokType::FAILED: { LOG(FAILED); } break; + case TokType::NONE: + case TokType::TOK_SOURCE_END: break; + } + + } while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED); + + std::cout << "\n\nOutputHash : " << outputHash; + + TEST(outputHash == outputHashPassed); +} + +TEST_DEF(Tokenizer) { + testSimple(); +} \ No newline at end of file diff --git a/Tokenizer/tests/tests.cpp b/Tokenizer/tests/tests.cpp index a29d780..4e721e9 100644 --- a/Tokenizer/tests/tests.cpp +++ b/Tokenizer/tests/tests.cpp @@ -1,182 +1,18 @@ #include "Tokenizer.hpp" -#include "filesystem.h" +#include "Testing.hpp" -#include - -#define LOG(val) std::cout << #val << " "; - -void test(const char* path) { - - tp::File script_file(path, tp::osfile_openflags::LOAD); - if (!script_file.opened) { - return; - } - char* script = new char[script_file.size() + 1]; - script[script_file.size()] = 0; - script_file.read_bytes(script, script_file.size()); - script_file.close(); - - enum class TokType { - NONE, - FAILED, - TOK_SOURCE_END, - - VAR, - CLASS_DEF, - SELF, - SCOPE_IN, - SCOPE_OUT, - ASSIGN, - DEF_FUNC, - PRINT, - IF, - ELSE, - WHILE, - BRACKET_IN, - BRACKET_OUT, - COMMA, - NEW, - CHILD, - RETURN, - EQUAL, - NOT_EQUAL, - MORE, - LESS, - QE_OR_MORE, - QE_OR_LESS, - BOOL_NOT, - BOOL_AND, - BOOL_OR, - ADD, - MUL, - SUB, - DIV, - STM_END, - CONST_TRUE, - CONST_FALSE, - CONST_INT, - CONST_FLOAT, - SPACE, - CONST_STRING, - ID, - //COMMENT_LINE, - COMMENT_BLOCK, - }; - - tp::SimpleTokenizer lexer; - - lexer.build({ - { "\n|\t| |\r", TokType::SPACE }, - { "var", TokType::VAR }, - { "class", TokType::CLASS_DEF }, - { "self", TokType::SELF }, - { "\\{", TokType::SCOPE_IN }, - { "\\}", TokType::SCOPE_OUT }, - { "=", TokType::ASSIGN }, - { "def", TokType::DEF_FUNC }, - { "<<", TokType::PRINT }, - { "if", TokType::IF }, - { "else", TokType::ELSE }, - { "while", TokType::WHILE }, - { "\\(", TokType::BRACKET_IN }, - { "\\)", TokType::BRACKET_OUT }, - { ",", TokType::COMMA }, - { "new", TokType::NEW }, - { "\\.", TokType::CHILD }, - { "return", TokType::RETURN }, - { "==", TokType::EQUAL }, - { "!=", TokType::NOT_EQUAL }, - { ">", TokType::MORE }, - { "<", TokType::LESS }, - { ">=", TokType::QE_OR_MORE }, - { "<=", TokType::QE_OR_LESS }, - { "!", TokType::BOOL_NOT }, - { "&&", TokType::BOOL_AND }, - { "\\|\\|", TokType::BOOL_OR }, - { "\\+", TokType::ADD }, - { "\\*", TokType::MUL }, - { "\\-", TokType::SUB }, - { "/", TokType::DIV }, - { ";", TokType::STM_END }, - { "true", TokType::CONST_TRUE }, - { "false", TokType::CONST_FALSE }, - { "((\\-)|(\\+))?[0-9]+i?", TokType::CONST_INT }, - { "((\\-)|(\\+))?([0-9]+)(\\.)([0-9]*)?f?", TokType::CONST_FLOAT }, - { "(/\\*){\\*-\\*}*(\\*/)", TokType::COMMENT_BLOCK }, - { "\"{\"-\"}*\"", TokType::CONST_STRING }, - { "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", TokType::ID }, - }); - - if (!lexer.isBuild()) { - std::cout << lexer.getBuildError().description; - return; - } - - lexer.bindSource(script); - - TokType tok; - do { - - tok = lexer.readTok(); - - switch (tok) { - case TokType::SPACE: { std::cout << " "; } break; - case TokType::VAR: { LOG(VAR); } break; - case TokType::CLASS_DEF: { LOG(CLASS_DEF); } break; - case TokType::SELF: { LOG(SELF); } break; - case TokType::SCOPE_IN: { LOG(SCOPE_IN); } break; - case TokType::SCOPE_OUT: { LOG(SCOPE_OUT); } break; - case TokType::ASSIGN: { LOG(ASSIGN); } break; - case TokType::DEF_FUNC: { LOG(DEF_FUNC); } break; - case TokType::PRINT: { LOG(PRINT); } break; - case TokType::IF: { LOG(IF); } break; - case TokType::ELSE: { LOG(ELSE); } break; - case TokType::WHILE: { LOG(WHILE); } break; - case TokType::BRACKET_IN: { LOG(BRACKET_IN); } break; - case TokType::BRACKET_OUT: { LOG(BRACKET_OUT); } break; - case TokType::COMMA: { LOG(COMMA); } break; - case TokType::NEW: { LOG(NEW); } break; - case TokType::CHILD: { LOG(CHILD); } break; - case TokType::RETURN: { LOG(RETURN); } break; - case TokType::EQUAL: { LOG(EQUAL); } break; - case TokType::NOT_EQUAL: { LOG(NOT_EQUAL); } break; - case TokType::MORE: { LOG(MORE); } break; - case TokType::LESS: { LOG(LESS); } break; - case TokType::QE_OR_MORE: { LOG(QE_OR_MORE); } break; - case TokType::QE_OR_LESS: { LOG(QE_OR_LESS); } break; - case TokType::BOOL_NOT: { LOG(BOOL_NOT); } break; - case TokType::BOOL_AND: { LOG(BOOL_AND); } break; - case TokType::BOOL_OR: { LOG(BOOL_OR); } break; - case TokType::ADD: { LOG(ADD); } break; - case TokType::MUL: { LOG(MUL); } break; - case TokType::SUB: { LOG(SUB); } break; - case TokType::DIV: { LOG(DIV); } break; - case TokType::STM_END: { LOG(STM_END); } break; - case TokType::CONST_TRUE: { LOG(TRUE); } break; - case TokType::CONST_FALSE: { LOG(FALSE); } break; - case TokType::CONST_INT: { LOG(INT); } break; - case TokType::CONST_FLOAT: { LOG(FLOAT); } break; - case TokType::CONST_STRING: { LOG(STRING); } break; - case TokType::ID: { LOG(ID); } break; - //case TokType::COMMENT_LINE: { LOG(COMMENT_LINE) } break; - case TokType::COMMENT_BLOCK: { LOG(COMMENT_BLOCK) } break; - case TokType::FAILED: { LOG(FAILED) } break; - } - - } while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED); - -} +void testTokenizer(); int main(int argc, char* argv[]) { - tp::ModuleManifest* ModuleDependencies[] = { &tp::gModuleTokenizer, NULL }; - tp::ModuleManifest TestModule("Test", NULL, NULL, ModuleDependencies); + tp::ModuleManifest* ModuleDependencies[] = { &tp::gModuleTokenizer, nullptr }; + tp::ModuleManifest TestModule("TokenizerTest", nullptr, nullptr, ModuleDependencies); if (!TestModule.initialize()) { return 1; } - test(argc > 1 ? argv[1] : "rsc/script.txt"); + testTokenizer(); TestModule.deinitialize(); } \ No newline at end of file