Tokenizer Test

Containers/public/Buffer.hpp

@@ -179,7 +179,8 @@ namespace tp {
 			for (ualni i = 0; i < mLoad; i++) mBuff[i].~tType();
 			mBuff = (tType*) mAllocator.allocate(sizeof(tType) * aSize);
 			mSize = aSize;
-			mLoad = 0;
+			mLoad = aSize;
+			for (ualni i = 0; i < mLoad; i++) new (mBuff + i) tType();
 		}
 
   private:

Containers/tests/Buffer2DTest.cpp

@@ -13,8 +13,8 @@ const ualni size = 1000;
 TEST_DEF_STATIC(Simple1) {
 	Buffer2D<int> buff;
 	buff.reserve({ 4, 4 });
-	buff.set( 2, 2, 5);
+	buff.set( { 2, 2 }, 5);
-	TEST(buff.get(2, 2) == 5);
+	TEST(buff.get( { 2, 2 } ) == 5);
 }
 
 TEST_DEF_STATIC(Simple2) {

Tokenizer/private/Tokenizer.cpp

@@ -4,7 +4,6 @@
 using namespace tp;
 
 static ModuleManifest* sModuleDependencies[] = {
-	&tp::gModuleMath, 
 	&tp::gModuleStrings,
 	nullptr
 };

Tokenizer/public/AutomataGraph.h

@@ -32,7 +32,7 @@ namespace tp {
 			bool mExclude = false;
 
 			bool isTransition(const tAlphabetType& symbol) {
-				if (symbol == nullptr) return false;
+				if (symbol == 0) return false;
 				if (!mConsumesSymbol || mAcceptsAll) return true;
 				bool const in_range = (symbol >= mAcceptingRange.mBegin && symbol <= mAcceptingRange.mEnd);
 				return in_range != mExclude;
@@ -112,7 +112,7 @@ namespace tp {
 				}
 			}
 
-			return { start, end + 1 };
+			return Range<tAlphabetType>( start, end + 1 );
 		}
 
 		// vertices that are reachable from initial set with no input consumption (E-transitions)
@@ -187,7 +187,7 @@ namespace tp {
 
 			struct DStateKey {
 				const List<NState*>* nStates;
-				bool operator==(const DStateKey& in) {
+				bool operator==(const DStateKey& in) const {
 					if (nStates->length() != in.nStates->length()) {
 						return false;
 					}
@@ -238,11 +238,11 @@ namespace tp {
 
 			// includes closure of NFA start state by definition
 			auto start_state = new DState();
-			nfa.closure({ nfa.getStartVertex() }, start_state->nstates, ualni(start_state));
+			nfa.closure({ nfa.getStartVertex() }, start_state->nStates, ualni(start_state));
 
 			Map<DStateKey, DState*, DefaultAllocator, DStateKey::dStateHashFunc, 256> dStates;
 
-			dStates.put({ &start_state->nstates }, start_state);
+			dStates.put({ &start_state->nStates }, start_state);
 
 			List<DState*> working_set = { start_state };
 
@@ -255,7 +255,7 @@ namespace tp {
 
 					List<NState*> reachableNStates;
 
-					nfa.move(currentDState->data->nstates, reachableNStates, symbol, ualni(currentDState + symbol));
+					nfa.move(currentDState->data->nStates, reachableNStates, symbol, ualni(currentDState + symbol));
 					nfa.closure(reachableNStates, reachableNStates, ualni(currentDState + symbol));
 
 					if (!reachableNStates.length()) {
@@ -278,11 +278,11 @@ namespace tp {
 							targetDState->debug_idx = dStates.size();
 						#endif
 
-						targetDState->nstates = reachableNStates;
+						targetDState->nStates = reachableNStates;
 
 						// append to working stack
 						working_set.pushBack(targetDState);
-						dStates.put({ &targetDState->nstates }, targetDState);
+						dStates.put({ &targetDState->nStates }, targetDState);
 					}
 
 					// add transition to DFA state
@@ -302,18 +302,18 @@ namespace tp {
 						break;
 					}
 				}
-				node->val->dvertex = addVertex(state);
+				node->val->dVertex = addVertex(state);
 			}
 
 			// connect all vertices
 			for (auto node : dStates) {
 				for (auto edge : node->val->transitions) {
-					addTransition(node->val->dvertex, edge.data().state->dvertex, edge.data().accepting_code);
+					addTransition(node->val->dVertex, edge.data().state->dVertex, edge.data().accepting_code);
 				}
 			}
 
 			// set the starting vertex
-			mStart = start_state->dvertex;
+			mStart = start_state->dVertex;
 
 			// cleanup
 			for (auto node : dStates) {
@@ -411,7 +411,7 @@ namespace tp {
 
 		Buffer2D<ualni> mTransitions;
 		Buffer<tStateType> mStates;
-		Range<tAlphabetType> mSymbolRange = nullptr;
+		Range<tAlphabetType> mSymbolRange = { 0, 0 };
 		
 		ualni mIter = 0;
 		ualni mIterPrev = 0;
@@ -431,31 +431,35 @@ namespace tp {
 			mTransitions.assign(dfa.nVertices());
 			mStates.reserve(sizeY);
 
+			ualni idx = 0;
 			for (auto vertex : dfa.mVertices) {
 				auto state = vertex.data().termination_state;
-				mStates[vertex.idx()] = state;
+				mStates[idx] = state;
+				idx++;
 			}
 
 			mStates[dfa.nVertices()] = tFailedStateVal;
 
+			idx = 0;
 			for (auto vertex : dfa.mVertices) {
 				if (&vertex.data() == dfa.mStart) {
-					mStart = mIter = mIterPrev = vertex.idx();
+					mStart = mIter = mIterPrev = idx;
 				}
+				idx++;
 			}
 
+			ualni vertexIdx = 0;
 			for (auto vertex : dfa.mVertices) {
 				for (auto edge : vertex.data().edges) {
-					ualni idx = 0;
+					ualni vertex2Idx = 0;
-					for (auto vertex : dfa.mVertices) {
+					for (auto vertex2 : dfa.mVertices) {
-						if (edge.data().vertex == &vertex.data()) {
+						if (edge.data().vertex == &vertex2.data()) break;
-							idx = vertex.idx();
+						vertex2Idx++;
-							break;
-						}
 					}
 					auto const code = edge.data().transition_code;
-					mTransitions.set(code - mSymbolRange.mBegin, (uhalni) vertex.idx(), idx);
+					mTransitions.set( { (ualni) (code - mSymbolRange.mBegin), (ualni) vertexIdx }, vertex2Idx);
 				}
+				vertexIdx++;
 			}
 		}
 
@@ -465,10 +469,10 @@ namespace tp {
 
 		tStateType move(tAlphabetType symbol) {
 			if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) {
-				mIter = mTransitions.get({ symbol - mSymbolRange.mBegin, mIter });
+				mIter = mTransitions.get({ (ualni) (symbol - mSymbolRange.mBegin), (ualni) mIter });
 			}
 			else {
-				mIter = mStates.length() - 1;
+				mIter = mStates.size() - 1;
 			}
 			
 			if (mIterPrev == mStart) {

Tokenizer/public/RegularExpression.h

@@ -136,6 +136,7 @@ namespace tp::RegEx {
 					case TOK_ANY: out = parseAny(); break;
 					case TOK_VAL: out = parseVal(); break;
 					case TOK_NONE: { discardTok(); return nullptr; };
+					default: break;
 				}
 				if (!out) {
 					discardTok();

Tokenizer/rsc/script.txt

@@ -1,18 +0,0 @@
- 
- 
- /* ba
- asadadasdK  */
-
- -1.f;
-
- if (+1.f) {
-	 var string = "asas
-	 d";
-
- }
-
- while 
-
- return
- 
-

Tokenizer/tests/TestTokenizer.cpp

@@ -0,0 +1,198 @@
+
+#include "Testing.hpp"
+#include "Tokenizer.hpp"
+
+#include <iostream>
+
+#define LOG(val) std::cout << #val << " "
+
+using namespace tp;
+
+ualni outputHashPassed = 1156;
+
+const char* script =
+R"(
+
+/* yea
+comment  */
+
+-1.f;
+
+if (+1.f) {
+	var string = "string value
+	another line";
+
+}
+
+while
+
+return
+
+)";
+
+TEST_DEF_STATIC(Simple) {
+
+	enum class TokType {
+		NONE,
+		FAILED,
+		TOK_SOURCE_END,
+
+		VAR,
+		CLASS_DEF,
+		SELF,
+		SCOPE_IN,
+		SCOPE_OUT,
+		ASSIGN,
+		DEF_FUNC,
+		PRINT,
+		IF,
+		ELSE,
+		WHILE,
+		BRACKET_IN,
+		BRACKET_OUT,
+		COMMA,
+		NEW,
+		CHILD,
+		RETURN,
+		EQUAL,
+		NOT_EQUAL,
+		MORE,
+		LESS,
+		QE_OR_MORE,
+		QE_OR_LESS,
+		BOOL_NOT,
+		BOOL_AND,
+		BOOL_OR,
+		ADD,
+		MUL,
+		SUB,
+		DIV,
+		STM_END,
+		CONST_TRUE,
+		CONST_FALSE,
+		CONST_INT,
+		CONST_FLOAT,
+		SPACE,
+		CONST_STRING,
+		ID,
+
+		//COMMENT_LINE,
+
+		COMMENT_BLOCK,
+	};
+
+	SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
+
+	lexer.build({
+			{ "\n|\t| |\r", TokType::SPACE },
+			{ "var", TokType::VAR },
+			{ "class", TokType::CLASS_DEF },
+			{ "self",	 TokType::SELF },
+			{ "\\{", TokType::SCOPE_IN },
+			{ "\\}", TokType::SCOPE_OUT },
+			{ "=", TokType::ASSIGN },
+			{ "def", TokType::DEF_FUNC },
+			{ "<<",	 TokType::PRINT },
+			{ "if",	 TokType::IF },
+			{ "else",	 TokType::ELSE },
+			{ "while", TokType::WHILE },
+			{ "\\(", TokType::BRACKET_IN },
+			{ "\\)", TokType::BRACKET_OUT },
+			{ ",", TokType::COMMA },
+			{ "new", TokType::NEW },
+			{ "\\.",	 TokType::CHILD },
+			{ "return",	 TokType::RETURN },
+			{ "==",	 TokType::EQUAL },
+			{ "!=",	 TokType::NOT_EQUAL },
+			{ ">", TokType::MORE },
+			{ "<", TokType::LESS },
+			{ ">=",	 TokType::QE_OR_MORE },
+			{ "<=",	 TokType::QE_OR_LESS },
+			{ "!", TokType::BOOL_NOT },
+			{ "&&",	 TokType::BOOL_AND },
+			{ "\\|\\|",	 TokType::BOOL_OR },
+			{ "\\+", TokType::ADD },
+			{ "\\*", TokType::MUL },
+			{ "\\-", TokType::SUB },
+			{ "/", TokType::DIV },
+			{ ";", TokType::STM_END },
+			{ "true",	 TokType::CONST_TRUE },
+			{ "false", TokType::CONST_FALSE },
+			{ "((\\-)|(\\+))?[0-9]+i?",	 TokType::CONST_INT },
+			{ R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)",	 TokType::CONST_FLOAT },
+			{ R"((/\*){\*-\*}*(\*/))",	 TokType::COMMENT_BLOCK },
+			{ R"("{"-"}*")",	 TokType::CONST_STRING },
+			{ "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*",	 TokType::ID },
+	});
+
+	if (!lexer.isBuild()) {
+		std::cout << lexer.getBuildError().description;
+		return;
+	}
+
+	lexer.bindSource(script);
+
+	TokType tok;
+	ualni outputHash = 0;
+	do {
+
+		tok = lexer.readTok();
+
+		outputHash += ualni(tok);
+
+		switch (tok) {
+			case TokType::SPACE: { std::cout << " "; } break;
+			case TokType::VAR: { LOG(VAR); } break;
+			case TokType::CLASS_DEF: { LOG(CLASS_DEF); } break;
+			case TokType::SELF: { LOG(SELF); } break;
+			case TokType::SCOPE_IN: { LOG(SCOPE_IN); } break;
+			case TokType::SCOPE_OUT: { LOG(SCOPE_OUT); } break;
+			case TokType::ASSIGN: { LOG(ASSIGN); } break;
+			case TokType::DEF_FUNC: { LOG(DEF_FUNC); } break;
+			case TokType::PRINT: { LOG(PRINT); } break;
+			case TokType::IF: { LOG(IF); } break;
+			case TokType::ELSE: { LOG(ELSE); } break;
+			case TokType::WHILE: { LOG(WHILE); } break;
+			case TokType::BRACKET_IN: { LOG(BRACKET_IN); } break;
+			case TokType::BRACKET_OUT: { LOG(BRACKET_OUT); } break;
+			case TokType::COMMA: { LOG(COMMA); } break;
+			case TokType::NEW: { LOG(NEW); } break;
+			case TokType::CHILD: { LOG(CHILD); } break;
+			case TokType::RETURN: { LOG(RETURN); } break;
+			case TokType::EQUAL: { LOG(EQUAL); } break;
+			case TokType::NOT_EQUAL: { LOG(NOT_EQUAL); } break;
+			case TokType::MORE: { LOG(MORE); } break;
+			case TokType::LESS: { LOG(LESS); } break;
+			case TokType::QE_OR_MORE: { LOG(QE_OR_MORE); } break;
+			case TokType::QE_OR_LESS: { LOG(QE_OR_LESS); } break;
+			case TokType::BOOL_NOT: { LOG(BOOL_NOT); } break;
+			case TokType::BOOL_AND: { LOG(BOOL_AND); } break;
+			case TokType::BOOL_OR: { LOG(BOOL_OR); } break;
+			case TokType::ADD: { LOG(ADD); } break;
+			case TokType::MUL: { LOG(MUL); } break;
+			case TokType::SUB: { LOG(SUB); } break;
+			case TokType::DIV: { LOG(DIV); } break;
+			case TokType::STM_END: { LOG(STM_END); } break;
+			case TokType::CONST_TRUE: { LOG(TRUE); } break;
+			case TokType::CONST_FALSE: { LOG(FALSE); } break;
+			case TokType::CONST_INT: { LOG(INT); } break;
+			case TokType::CONST_FLOAT: { LOG(FLOAT); } break;
+			case TokType::CONST_STRING: { LOG(STRING); } break;
+			case TokType::ID: { LOG(ID); } break;
+			//case TokType::COMMENT_LINE: { LOG(COMMENT_LINE) } break;
+			case TokType::COMMENT_BLOCK: { LOG(COMMENT_BLOCK); } break;
+			case TokType::FAILED: { LOG(FAILED); } break;
+			case TokType::NONE:
+			case TokType::TOK_SOURCE_END: break;
+		}
+
+	} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
+
+	std::cout << "\n\nOutputHash : " << outputHash;
+
+	TEST(outputHash == outputHashPassed);
+}
+
+TEST_DEF(Tokenizer) {
+	testSimple();
+}

Tokenizer/tests/tests.cpp

@@ -1,182 +1,18 @@
 
 #include "Tokenizer.hpp"
-#include "filesystem.h"
+#include "Testing.hpp"
 
-#include <iostream>
+void testTokenizer();
-
-#define LOG(val) std::cout << #val << " ";
-
-void test(const char* path) {
-
-	tp::File script_file(path, tp::osfile_openflags::LOAD);
-	if (!script_file.opened) {
-		return;
-	}
-	char* script = new char[script_file.size() + 1];
-	script[script_file.size()] = 0;
-	script_file.read_bytes(script, script_file.size());
-	script_file.close();
-
-	enum class TokType {
-		NONE,
-		FAILED,
-		TOK_SOURCE_END,
-
-		VAR,
-		CLASS_DEF,
-		SELF,
-		SCOPE_IN,
-		SCOPE_OUT,
-		ASSIGN,
-		DEF_FUNC,
-		PRINT,
-		IF,
-		ELSE,
-		WHILE,
-		BRACKET_IN,
-		BRACKET_OUT,
-		COMMA,
-		NEW,
-		CHILD,
-		RETURN,
-		EQUAL,
-		NOT_EQUAL,
-		MORE,
-		LESS,
-		QE_OR_MORE,
-		QE_OR_LESS,
-		BOOL_NOT,
-		BOOL_AND,
-		BOOL_OR,
-		ADD,
-		MUL,
-		SUB,
-		DIV,
-		STM_END,
-		CONST_TRUE,
-		CONST_FALSE,
-		CONST_INT,
-		CONST_FLOAT,
-		SPACE,
-		CONST_STRING,
-		ID,
-		//COMMENT_LINE,
-		COMMENT_BLOCK,
-	};
-
-	tp::SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
-
-	lexer.build({
-		{ "\n|\t| |\r", TokType::SPACE },
-		{ "var", TokType::VAR },
-		{ "class", TokType::CLASS_DEF },
-		{ "self",	 TokType::SELF },
-		{ "\\{", TokType::SCOPE_IN },
-		{ "\\}", TokType::SCOPE_OUT },
-		{ "=", TokType::ASSIGN },
-		{ "def", TokType::DEF_FUNC },
-		{ "<<",	 TokType::PRINT },
-		{ "if",	 TokType::IF },
-		{ "else",	 TokType::ELSE },
-		{ "while", TokType::WHILE },
-		{ "\\(", TokType::BRACKET_IN },
-		{ "\\)", TokType::BRACKET_OUT },
-		{ ",", TokType::COMMA },
-		{ "new", TokType::NEW },
-		{ "\\.",	 TokType::CHILD },
-		{ "return",	 TokType::RETURN },
-		{ "==",	 TokType::EQUAL },
-		{ "!=",	 TokType::NOT_EQUAL },
-		{ ">", TokType::MORE },
-		{ "<", TokType::LESS },
-		{ ">=",	 TokType::QE_OR_MORE },
-		{ "<=",	 TokType::QE_OR_LESS },
-		{ "!", TokType::BOOL_NOT },
-		{ "&&",	 TokType::BOOL_AND },
-		{ "\\|\\|",	 TokType::BOOL_OR },
-		{ "\\+", TokType::ADD },
-		{ "\\*", TokType::MUL },
-		{ "\\-", TokType::SUB },
-		{ "/", TokType::DIV },
-		{ ";", TokType::STM_END },
-		{ "true",	 TokType::CONST_TRUE },
-		{ "false", TokType::CONST_FALSE },
-		{ "((\\-)|(\\+))?[0-9]+i?",	 TokType::CONST_INT },
-		{ "((\\-)|(\\+))?([0-9]+)(\\.)([0-9]*)?f?",	 TokType::CONST_FLOAT },
-		{ "(/\\*){\\*-\\*}*(\\*/)",	 TokType::COMMENT_BLOCK },
-		{ "\"{\"-\"}*\"",	 TokType::CONST_STRING },
-		{ "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*",	 TokType::ID },
-	});
-
-	if (!lexer.isBuild()) {
-		std::cout << lexer.getBuildError().description;
-		return;
-	}
-
-	lexer.bindSource(script);
-
-	TokType tok;
-	do {
-
-		tok = lexer.readTok();
-
-		switch (tok) {
-		case TokType::SPACE: { std::cout << " "; } break;
-			case TokType::VAR: { LOG(VAR); } break;
-			case TokType::CLASS_DEF: { LOG(CLASS_DEF); } break;
-			case TokType::SELF: { LOG(SELF); } break;
-			case TokType::SCOPE_IN: { LOG(SCOPE_IN); } break;
-			case TokType::SCOPE_OUT: { LOG(SCOPE_OUT); } break;
-			case TokType::ASSIGN: { LOG(ASSIGN); } break;
-			case TokType::DEF_FUNC: { LOG(DEF_FUNC); } break;
-			case TokType::PRINT: { LOG(PRINT); } break;
-			case TokType::IF: { LOG(IF); } break;
-			case TokType::ELSE: { LOG(ELSE); } break;
-			case TokType::WHILE: { LOG(WHILE); } break;
-			case TokType::BRACKET_IN: { LOG(BRACKET_IN); } break;
-			case TokType::BRACKET_OUT: { LOG(BRACKET_OUT); } break;
-			case TokType::COMMA: { LOG(COMMA); } break;
-			case TokType::NEW: { LOG(NEW); } break;
-			case TokType::CHILD: { LOG(CHILD); } break;
-			case TokType::RETURN: { LOG(RETURN); } break;
-			case TokType::EQUAL: { LOG(EQUAL); } break;
-			case TokType::NOT_EQUAL: { LOG(NOT_EQUAL); } break;
-			case TokType::MORE: { LOG(MORE); } break;
-			case TokType::LESS: { LOG(LESS); } break;
-			case TokType::QE_OR_MORE: { LOG(QE_OR_MORE); } break;
-			case TokType::QE_OR_LESS: { LOG(QE_OR_LESS); } break;
-			case TokType::BOOL_NOT: { LOG(BOOL_NOT); } break;
-			case TokType::BOOL_AND: { LOG(BOOL_AND); } break;
-			case TokType::BOOL_OR: { LOG(BOOL_OR); } break;
-			case TokType::ADD: { LOG(ADD); } break;
-			case TokType::MUL: { LOG(MUL); } break;
-			case TokType::SUB: { LOG(SUB); } break;
-			case TokType::DIV: { LOG(DIV); } break;
-			case TokType::STM_END: { LOG(STM_END); } break;
-			case TokType::CONST_TRUE: { LOG(TRUE); } break;
-			case TokType::CONST_FALSE: { LOG(FALSE); } break;
-			case TokType::CONST_INT: { LOG(INT); } break;
-			case TokType::CONST_FLOAT: { LOG(FLOAT); } break;
-			case TokType::CONST_STRING: { LOG(STRING); } break;
-			case TokType::ID: { LOG(ID); } break;
-			//case TokType::COMMENT_LINE: { LOG(COMMENT_LINE) } break;
-			case TokType::COMMENT_BLOCK: { LOG(COMMENT_BLOCK) } break;
-			case TokType::FAILED: { LOG(FAILED) } break;
-		}
-
-	} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
-
-}
 
 int main(int argc, char* argv[]) {
 
-	tp::ModuleManifest* ModuleDependencies[] = { &tp::gModuleTokenizer, NULL };
+	tp::ModuleManifest* ModuleDependencies[] = { &tp::gModuleTokenizer, nullptr };
-	tp::ModuleManifest TestModule("Test", NULL, NULL, ModuleDependencies);
+	tp::ModuleManifest TestModule("TokenizerTest", nullptr, nullptr, ModuleDependencies);
 	if (!TestModule.initialize()) {
 		return 1;
 	}
 
-	test(argc > 1 ? argv[1] : "rsc/script.txt");
+	testTokenizer();
 
 	TestModule.deinitialize();
 }