ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tokenizer Test

Browse source
This commit is contained in:
IlushaShurupov 2023-07-15 19:18:22 +03:00 • committed by Ilya Shurupov
parent 6248c294b8
commit 1aece0c617
8 changed files with 235 additions and 214 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Containers/public/Buffer.hpp
View file

@ -179,7 +179,8 @@ namespace tp {
for (ualni i = 0; i < mLoad; i++) mBuff[i].~tType();
mBuff = (tType*) mAllocator.allocate(sizeof(tType) * aSize);
mSize = aSize;
mLoad = 0;
mLoad = aSize;
for (ualni i = 0; i < mLoad; i++) new (mBuff + i) tType();
}
private:

4
Containers/tests/Buffer2DTest.cpp
View file

@ -13,8 +13,8 @@ const ualni size = 1000;
TEST_DEF_STATIC(Simple1) {
Buffer2D<int> buff;
buff.reserve({ 4, 4 });
buff.set( 2, 2, 5);
TEST(buff.get(2, 2) == 5);
buff.set( { 2, 2 }, 5);
TEST(buff.get( { 2, 2 } ) == 5);
}
TEST_DEF_STATIC(Simple2) {

1
Tokenizer/private/Tokenizer.cpp
View file

@ -4,7 +4,6 @@
using namespace tp;
static ModuleManifest* sModuleDependencies[] = {
&tp::gModuleMath,
&tp::gModuleStrings,
nullptr
};

50
Tokenizer/public/AutomataGraph.h
View file

@ -32,7 +32,7 @@ namespace tp {
bool mExclude = false;
bool isTransition(const tAlphabetType& symbol) {
if (symbol == nullptr) return false;
if (symbol == 0) return false;
if (!mConsumesSymbol || mAcceptsAll) return true;
bool const in_range = (symbol >= mAcceptingRange.mBegin && symbol <= mAcceptingRange.mEnd);
return in_range != mExclude;
@ -112,7 +112,7 @@ namespace tp {
}
}
return { start, end + 1 };
return Range<tAlphabetType>( start, end + 1 );
}
// vertices that are reachable from initial set with no input consumption (E-transitions)
@ -187,7 +187,7 @@ namespace tp {
struct DStateKey {
const List<NState*>* nStates;
bool operator==(const DStateKey& in) {
bool operator==(const DStateKey& in) const {
if (nStates->length() != in.nStates->length()) {
return false;
}
@ -238,11 +238,11 @@ namespace tp {
// includes closure of NFA start state by definition
auto start_state = new DState();
nfa.closure({ nfa.getStartVertex() }, start_state->nstates, ualni(start_state));
nfa.closure({ nfa.getStartVertex() }, start_state->nStates, ualni(start_state));
Map<DStateKey, DState*, DefaultAllocator, DStateKey::dStateHashFunc, 256> dStates;
dStates.put({ &start_state->nstates }, start_state);
dStates.put({ &start_state->nStates }, start_state);
List<DState*> working_set = { start_state };
@ -255,7 +255,7 @@ namespace tp {
List<NState*> reachableNStates;
nfa.move(currentDState->data->nstates, reachableNStates, symbol, ualni(currentDState + symbol));
nfa.move(currentDState->data->nStates, reachableNStates, symbol, ualni(currentDState + symbol));
nfa.closure(reachableNStates, reachableNStates, ualni(currentDState + symbol));
if (!reachableNStates.length()) {
@ -278,11 +278,11 @@ namespace tp {
targetDState->debug_idx = dStates.size();
#endif
targetDState->nstates = reachableNStates;
targetDState->nStates = reachableNStates;
// append to working stack
working_set.pushBack(targetDState);
dStates.put({ &targetDState->nstates }, targetDState);
dStates.put({ &targetDState->nStates }, targetDState);
}
// add transition to DFA state
@ -302,18 +302,18 @@ namespace tp {
break;
}
}
node->val->dvertex = addVertex(state);
node->val->dVertex = addVertex(state);
}
// connect all vertices
for (auto node : dStates) {
for (auto edge : node->val->transitions) {
addTransition(node->val->dvertex, edge.data().state->dvertex, edge.data().accepting_code);
addTransition(node->val->dVertex, edge.data().state->dVertex, edge.data().accepting_code);
}
}
// set the starting vertex
mStart = start_state->dvertex;
mStart = start_state->dVertex;
// cleanup
for (auto node : dStates) {
@ -411,7 +411,7 @@ namespace tp {
Buffer2D<ualni> mTransitions;
Buffer<tStateType> mStates;
Range<tAlphabetType> mSymbolRange = nullptr;
Range<tAlphabetType> mSymbolRange = { 0, 0 };
ualni mIter = 0;
ualni mIterPrev = 0;
@ -431,31 +431,35 @@ namespace tp {
mTransitions.assign(dfa.nVertices());
mStates.reserve(sizeY);
ualni idx = 0;
for (auto vertex : dfa.mVertices) {
auto state = vertex.data().termination_state;
mStates[vertex.idx()] = state;
mStates[idx] = state;
idx++;
}
mStates[dfa.nVertices()] = tFailedStateVal;
idx = 0;
for (auto vertex : dfa.mVertices) {
if (&vertex.data() == dfa.mStart) {
mStart = mIter = mIterPrev = vertex.idx();
mStart = mIter = mIterPrev = idx;
}
idx++;
}
ualni vertexIdx = 0;
for (auto vertex : dfa.mVertices) {
for (auto edge : vertex.data().edges) {
ualni idx = 0;
for (auto vertex : dfa.mVertices) {
if (edge.data().vertex == &vertex.data()) {
idx = vertex.idx();
break;
}
ualni vertex2Idx = 0;
for (auto vertex2 : dfa.mVertices) {
if (edge.data().vertex == &vertex2.data()) break;
vertex2Idx++;
}
auto const code = edge.data().transition_code;
mTransitions.set(code - mSymbolRange.mBegin, (uhalni) vertex.idx(), idx);
mTransitions.set( { (ualni) (code - mSymbolRange.mBegin), (ualni) vertexIdx }, vertex2Idx);
}
vertexIdx++;
}
}
@ -465,10 +469,10 @@ namespace tp {
tStateType move(tAlphabetType symbol) {
if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) {
mIter = mTransitions.get({ symbol - mSymbolRange.mBegin, mIter });
mIter = mTransitions.get({ (ualni) (symbol - mSymbolRange.mBegin), (ualni) mIter });
}
else {
mIter = mStates.length() - 1;
mIter = mStates.size() - 1;
}
if (mIterPrev == mStart) {

1
Tokenizer/public/RegularExpression.h
View file

@ -136,6 +136,7 @@ namespace tp::RegEx {
case TOK_ANY: out = parseAny(); break;
case TOK_VAL: out = parseVal(); break;
case TOK_NONE: { discardTok(); return nullptr; };
default: break;
}
if (!out) {
discardTok();

18
Tokenizer/rsc/script.txt
View file

@ -1,18 +0,0 @@
/* ba
asadadasdK */
-1.f;
if (+1.f) {
var string = "asas
d";
}
while
return

198
Tokenizer/tests/TestTokenizer.cpp Normal file
View file

@ -0,0 +1,198 @@
#include "Testing.hpp"
#include "Tokenizer.hpp"
#include <iostream>
#define LOG(val) std::cout << #val << " "
using namespace tp;
ualni outputHashPassed = 1156;
const char* script =
R"(
/* yea
comment */
-1.f;
if (+1.f) {
var string = "string value
another line";
}
while
return
)";
TEST_DEF_STATIC(Simple) {
enum class TokType {
NONE,
FAILED,
TOK_SOURCE_END,
VAR,
CLASS_DEF,
SELF,
SCOPE_IN,
SCOPE_OUT,
ASSIGN,
DEF_FUNC,
PRINT,
IF,
ELSE,
WHILE,
BRACKET_IN,
BRACKET_OUT,
COMMA,
NEW,
CHILD,
RETURN,
EQUAL,
NOT_EQUAL,
MORE,
LESS,
QE_OR_MORE,
QE_OR_LESS,
BOOL_NOT,
BOOL_AND,
BOOL_OR,
ADD,
MUL,
SUB,
DIV,
STM_END,
CONST_TRUE,
CONST_FALSE,
CONST_INT,
CONST_FLOAT,
SPACE,
CONST_STRING,
ID,
//COMMENT_LINE,
COMMENT_BLOCK,
};
SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
lexer.build({
{ "\n|\t| |\r", TokType::SPACE },
{ "var", TokType::VAR },
{ "class", TokType::CLASS_DEF },
{ "self", TokType::SELF },
{ "\\{", TokType::SCOPE_IN },
{ "\\}", TokType::SCOPE_OUT },
{ "=", TokType::ASSIGN },
{ "def", TokType::DEF_FUNC },
{ "<<", TokType::PRINT },
{ "if", TokType::IF },
{ "else", TokType::ELSE },
{ "while", TokType::WHILE },
{ "\\(", TokType::BRACKET_IN },
{ "\\)", TokType::BRACKET_OUT },
{ ",", TokType::COMMA },
{ "new", TokType::NEW },
{ "\\.", TokType::CHILD },
{ "return", TokType::RETURN },
{ "==", TokType::EQUAL },
{ "!=", TokType::NOT_EQUAL },
{ ">", TokType::MORE },
{ "<", TokType::LESS },
{ ">=", TokType::QE_OR_MORE },
{ "<=", TokType::QE_OR_LESS },
{ "!", TokType::BOOL_NOT },
{ "&&", TokType::BOOL_AND },
{ "\\|\\|", TokType::BOOL_OR },
{ "\\+", TokType::ADD },
{ "\\*", TokType::MUL },
{ "\\-", TokType::SUB },
{ "/", TokType::DIV },
{ ";", TokType::STM_END },
{ "true", TokType::CONST_TRUE },
{ "false", TokType::CONST_FALSE },
{ "((\\-)|(\\+))?[0-9]+i?", TokType::CONST_INT },
{ R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)", TokType::CONST_FLOAT },
{ R"((/\*){\*-\*}*(\*/))", TokType::COMMENT_BLOCK },
{ R"("{"-"}*")", TokType::CONST_STRING },
{ "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", TokType::ID },
});
if (!lexer.isBuild()) {
std::cout << lexer.getBuildError().description;
return;
}
lexer.bindSource(script);
TokType tok;
ualni outputHash = 0;
do {
tok = lexer.readTok();
outputHash += ualni(tok);
switch (tok) {
case TokType::SPACE: { std::cout << " "; } break;
case TokType::VAR: { LOG(VAR); } break;
case TokType::CLASS_DEF: { LOG(CLASS_DEF); } break;
case TokType::SELF: { LOG(SELF); } break;
case TokType::SCOPE_IN: { LOG(SCOPE_IN); } break;
case TokType::SCOPE_OUT: { LOG(SCOPE_OUT); } break;
case TokType::ASSIGN: { LOG(ASSIGN); } break;
case TokType::DEF_FUNC: { LOG(DEF_FUNC); } break;
case TokType::PRINT: { LOG(PRINT); } break;
case TokType::IF: { LOG(IF); } break;
case TokType::ELSE: { LOG(ELSE); } break;
case TokType::WHILE: { LOG(WHILE); } break;
case TokType::BRACKET_IN: { LOG(BRACKET_IN); } break;
case TokType::BRACKET_OUT: { LOG(BRACKET_OUT); } break;
case TokType::COMMA: { LOG(COMMA); } break;
case TokType::NEW: { LOG(NEW); } break;
case TokType::CHILD: { LOG(CHILD); } break;
case TokType::RETURN: { LOG(RETURN); } break;
case TokType::EQUAL: { LOG(EQUAL); } break;
case TokType::NOT_EQUAL: { LOG(NOT_EQUAL); } break;
case TokType::MORE: { LOG(MORE); } break;
case TokType::LESS: { LOG(LESS); } break;
case TokType::QE_OR_MORE: { LOG(QE_OR_MORE); } break;
case TokType::QE_OR_LESS: { LOG(QE_OR_LESS); } break;
case TokType::BOOL_NOT: { LOG(BOOL_NOT); } break;
case TokType::BOOL_AND: { LOG(BOOL_AND); } break;
case TokType::BOOL_OR: { LOG(BOOL_OR); } break;
case TokType::ADD: { LOG(ADD); } break;
case TokType::MUL: { LOG(MUL); } break;
case TokType::SUB: { LOG(SUB); } break;
case TokType::DIV: { LOG(DIV); } break;
case TokType::STM_END: { LOG(STM_END); } break;
case TokType::CONST_TRUE: { LOG(TRUE); } break;
case TokType::CONST_FALSE: { LOG(FALSE); } break;
case TokType::CONST_INT: { LOG(INT); } break;
case TokType::CONST_FLOAT: { LOG(FLOAT); } break;
case TokType::CONST_STRING: { LOG(STRING); } break;
case TokType::ID: { LOG(ID); } break;
//case TokType::COMMENT_LINE: { LOG(COMMENT_LINE) } break;
case TokType::COMMENT_BLOCK: { LOG(COMMENT_BLOCK); } break;
case TokType::FAILED: { LOG(FAILED); } break;
case TokType::NONE:
case TokType::TOK_SOURCE_END: break;
}
} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
std::cout << "\n\nOutputHash : " << outputHash;
TEST(outputHash == outputHashPassed);
}
TEST_DEF(Tokenizer) {
testSimple();
}

174
Tokenizer/tests/tests.cpp
View file

@ -1,182 +1,18 @@
#include "Tokenizer.hpp"
#include "filesystem.h"
#include "Testing.hpp"
#include <iostream>
#define LOG(val) std::cout << #val << " ";
void test(const char* path) {
tp::File script_file(path, tp::osfile_openflags::LOAD);
if (!script_file.opened) {
return;
}
char* script = new char[script_file.size() + 1];
script[script_file.size()] = 0;
script_file.read_bytes(script, script_file.size());
script_file.close();
enum class TokType {
NONE,
FAILED,
TOK_SOURCE_END,
VAR,
CLASS_DEF,
SELF,
SCOPE_IN,
SCOPE_OUT,
ASSIGN,
DEF_FUNC,
PRINT,
IF,
ELSE,
WHILE,
BRACKET_IN,
BRACKET_OUT,
COMMA,
NEW,
CHILD,
RETURN,
EQUAL,
NOT_EQUAL,
MORE,
LESS,
QE_OR_MORE,
QE_OR_LESS,
BOOL_NOT,
BOOL_AND,
BOOL_OR,
ADD,
MUL,
SUB,
DIV,
STM_END,
CONST_TRUE,
CONST_FALSE,
CONST_INT,
CONST_FLOAT,
SPACE,
CONST_STRING,
ID,
//COMMENT_LINE,
COMMENT_BLOCK,
};
tp::SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
lexer.build({
{ "\n|\t| |\r", TokType::SPACE },
{ "var", TokType::VAR },
{ "class", TokType::CLASS_DEF },
{ "self", TokType::SELF },
{ "\\{", TokType::SCOPE_IN },
{ "\\}", TokType::SCOPE_OUT },
{ "=", TokType::ASSIGN },
{ "def", TokType::DEF_FUNC },
{ "<<", TokType::PRINT },
{ "if", TokType::IF },
{ "else", TokType::ELSE },
{ "while", TokType::WHILE },
{ "\\(", TokType::BRACKET_IN },
{ "\\)", TokType::BRACKET_OUT },
{ ",", TokType::COMMA },
{ "new", TokType::NEW },
{ "\\.", TokType::CHILD },
{ "return", TokType::RETURN },
{ "==", TokType::EQUAL },
{ "!=", TokType::NOT_EQUAL },
{ ">", TokType::MORE },
{ "<", TokType::LESS },
{ ">=", TokType::QE_OR_MORE },
{ "<=", TokType::QE_OR_LESS },
{ "!", TokType::BOOL_NOT },
{ "&&", TokType::BOOL_AND },
{ "\\|\\|", TokType::BOOL_OR },
{ "\\+", TokType::ADD },
{ "\\*", TokType::MUL },
{ "\\-", TokType::SUB },
{ "/", TokType::DIV },
{ ";", TokType::STM_END },
{ "true", TokType::CONST_TRUE },
{ "false", TokType::CONST_FALSE },
{ "((\\-)|(\\+))?[0-9]+i?", TokType::CONST_INT },
{ "((\\-)|(\\+))?([0-9]+)(\\.)([0-9]*)?f?", TokType::CONST_FLOAT },
{ "(/\\*){\\*-\\*}*(\\*/)", TokType::COMMENT_BLOCK },
{ "\"{\"-\"}*\"", TokType::CONST_STRING },
{ "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", TokType::ID },
});
if (!lexer.isBuild()) {
std::cout << lexer.getBuildError().description;
return;
}
lexer.bindSource(script);
TokType tok;
do {
tok = lexer.readTok();
switch (tok) {
case TokType::SPACE: { std::cout << " "; } break;
case TokType::VAR: { LOG(VAR); } break;
case TokType::CLASS_DEF: { LOG(CLASS_DEF); } break;
case TokType::SELF: { LOG(SELF); } break;
case TokType::SCOPE_IN: { LOG(SCOPE_IN); } break;
case TokType::SCOPE_OUT: { LOG(SCOPE_OUT); } break;
case TokType::ASSIGN: { LOG(ASSIGN); } break;
case TokType::DEF_FUNC: { LOG(DEF_FUNC); } break;
case TokType::PRINT: { LOG(PRINT); } break;
case TokType::IF: { LOG(IF); } break;
case TokType::ELSE: { LOG(ELSE); } break;
case TokType::WHILE: { LOG(WHILE); } break;
case TokType::BRACKET_IN: { LOG(BRACKET_IN); } break;
case TokType::BRACKET_OUT: { LOG(BRACKET_OUT); } break;
case TokType::COMMA: { LOG(COMMA); } break;
case TokType::NEW: { LOG(NEW); } break;
case TokType::CHILD: { LOG(CHILD); } break;
case TokType::RETURN: { LOG(RETURN); } break;
case TokType::EQUAL: { LOG(EQUAL); } break;
case TokType::NOT_EQUAL: { LOG(NOT_EQUAL); } break;
case TokType::MORE: { LOG(MORE); } break;
case TokType::LESS: { LOG(LESS); } break;
case TokType::QE_OR_MORE: { LOG(QE_OR_MORE); } break;
case TokType::QE_OR_LESS: { LOG(QE_OR_LESS); } break;
case TokType::BOOL_NOT: { LOG(BOOL_NOT); } break;
case TokType::BOOL_AND: { LOG(BOOL_AND); } break;
case TokType::BOOL_OR: { LOG(BOOL_OR); } break;
case TokType::ADD: { LOG(ADD); } break;
case TokType::MUL: { LOG(MUL); } break;
case TokType::SUB: { LOG(SUB); } break;
case TokType::DIV: { LOG(DIV); } break;
case TokType::STM_END: { LOG(STM_END); } break;
case TokType::CONST_TRUE: { LOG(TRUE); } break;
case TokType::CONST_FALSE: { LOG(FALSE); } break;
case TokType::CONST_INT: { LOG(INT); } break;
case TokType::CONST_FLOAT: { LOG(FLOAT); } break;
case TokType::CONST_STRING: { LOG(STRING); } break;
case TokType::ID: { LOG(ID); } break;
//case TokType::COMMENT_LINE: { LOG(COMMENT_LINE) } break;
case TokType::COMMENT_BLOCK: { LOG(COMMENT_BLOCK) } break;
case TokType::FAILED: { LOG(FAILED) } break;
}
} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
}
void testTokenizer();
int main(int argc, char* argv[]) {
tp::ModuleManifest* ModuleDependencies[] = { &tp::gModuleTokenizer, NULL };
tp::ModuleManifest TestModule("Test", NULL, NULL, ModuleDependencies);
tp::ModuleManifest* ModuleDependencies[] = { &tp::gModuleTokenizer, nullptr };
tp::ModuleManifest TestModule("TokenizerTest", nullptr, nullptr, ModuleDependencies);
if (!TestModule.initialize()) {
return 1;
}
test(argc > 1 ? argv[1] : "rsc/script.txt");
testTokenizer();
TestModule.deinitialize();
}