fix compile errors

.back/old/Acceptors.hpp

@@ -0,0 +1,79 @@
+
+#pragma once
+
+/*
+ * This file contains acceptors for each language
+ * Acceptor is managing an automation machine that can recognize whether this given sentence is in the language.
+ * Above that acceptors are responsible for defining those automations
+ * Some acceptors can supply additional information such as Abstract-Syntax-Tree or Sentence id
+ * */
+
+#include "LanguageModule.hpp"
+
+namespace tp {
+
+  template <typename tAlphabet>
+  class SymbolSupply {
+  public:
+    typedef ualni Cursor;
+
+  public:
+    SymbolSupply() = default;
+    virtual ~SymbolSupply() = 0;
+
+  public:
+    virtual Cursor getCursor() = 0;
+    virtual void setCursor(Cursor) = 0;
+
+    virtual void getPrevCursor() = 0;
+    virtual ualni getSymbolLength() = 0;
+
+    virtual const tAlphabet& getNextSymbol() = 0;
+
+  private:
+    Cursor mCursor = 0;
+  };
+
+  template <typename tAlphabet, typename tSentenceId>
+  class RegularAcceptor {
+    typedef SymbolSupply<tAlphabet> Sentence;
+
+  public:
+    RegularAcceptor() = default;
+    void build(const RegularGrammar<tSentenceId>& grammar);
+    tSentenceId recognize(Sentence& sentence);
+  };
+
+  // Subset of CFG
+  template <typename tAlphabet>
+  class LL {
+    typedef SymbolSupply<tAlphabet> Sentence;
+
+  public:
+    LL() = default;
+    void build(const ContextFreeGrammar& grammar);
+    void recognize(Sentence& sentence);
+  };
+
+  // Subset of CFG much larger than LL
+  template <typename tAlphabet>
+  class CLR {
+    typedef SymbolSupply<tAlphabet> Sentence;
+
+  public:
+    CLR() = default;
+    void build(const ContextFreeGrammar& grammar);
+    void recognize(Sentence& sentence);
+  };
+
+  // Subset of CFG CLR
+  template <typename tAlphabet>
+  class LALR {
+    typedef SymbolSupply<tAlphabet> Sentence;
+
+  public:
+    LALR() = default;
+    void build(const ContextFreeGrammar& grammar);
+    void recognize(Sentence& sentence);
+  };
+}

.back/old/Parser/CMakeLists.txt

@@ -0,0 +1,21 @@
+project(Parser)
+
+### ---------------------- Static Library  ---------------------  ###
+file(GLOB SOURCES "./private/*.cpp")
+file(GLOB HEADERS "./public/*.hpp")
+add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
+target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
+target_link_libraries(${PROJECT_NAME} PUBLIC Tokenizer)
+
+### -------------------------- Applications  --------------------------  ###
+add_executable(cfg ./applications/Cfg.cpp)
+target_link_libraries(cfg ${PROJECT_NAME} CommandLine)
+
+configure_file(rsc/grammar.txt grammar.txt COPYONLY)
+
+### -------------------------- Tests  --------------------------  ###
+enable_testing()
+file(GLOB TEST_SOURCES "./tests/*.cpp")
+add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES})
+target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils)
+add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests)

.back/old/Parser/applications/Cfg.cpp

@@ -0,0 +1,82 @@
+
+#include "CommandLine.hpp"
+#include "Parser.hpp"
+
+using namespace tp;
+
+void run(const String& source) {
+	auto state = CfGrammar::initializeCfGrammarParser();
+
+	CfGrammar grammar;
+
+	if (!grammar.parse(state, source)) {
+		printf("Parsing is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	if (!grammar.compile()) {
+		printf("Compilation is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	printf("Grammar accepted.\n");
+
+	List<CfGrammar::Sentence> sentences;
+	grammar.generateSentences(sentences);
+
+	printf("Example sentences formed from grammar: \n");
+	for (auto sentence : sentences)
+		tp::CfGrammar::printSentence(sentence.data());
+
+	CfGrammar::deinitializeCfGrammarParser(state);
+}
+
+int main(int argc, const char* argv[]) {
+	tp::ModuleManifest* deps[] = { &tp::gModuleParser, &tp::gModuleCommandLine, nullptr };
+	tp::ModuleManifest testModule("CommandLineTest", nullptr, nullptr, deps);
+
+	if (!testModule.initialize()) {
+		return 1;
+	}
+
+	{
+		printf("Arguments given: ");
+		for (auto idx = 0; idx < argc; idx++) {
+			printf("[%s]", argv[idx]);
+		}
+		printf("\n");
+
+		CommandLine cmd = {
+			{ "grammar", CommandLine::Arg::STR },
+		};
+
+		if (!cmd.parse((char) argc, argv, true, 1)) {
+			return 1;
+		}
+
+		auto location = cmd.getString("grammar");
+
+		LocalConnection file;
+		String grammar;
+
+		file.connect(LocalConnection::Location(location), LocalConnection::Type(true));
+
+		if (!file.getConnectionStatus().isOpened()) {
+			printf("Cant open file\n");
+			return 1;
+		}
+
+		auto size = file.size();
+		grammar.resize((String::Index) size);
+		file.readBytes(grammar.write(), size);
+
+		run(grammar);
+
+		file.disconnect();
+	}
+
+	testModule.deinitialize();
+	return 0;
+}

.back/old/Parser/private/CLR.cpp

@@ -0,0 +1,19 @@
+#include "CLR.hpp"
+
+using namespace tp;
+
+void CLR::setGrammar(const tp::CfGrammar& grammar) {
+	if (grammar.isLooped()) {
+		mState = ParserState::FAILED;
+		mBuildError.description = "Cant build lalr parser from looped context free grammar";
+		return;
+	}
+
+	mState = ParserState::FAILED;
+}
+
+void CLR::setTerminal(const String& name, TerminalStream::TerminalID id) {}
+
+void CLR::build() {}
+
+void CLR::parse(TerminalStream* source, List<ASTNode>& out, ASTNode** root) { *root = nullptr; }

.back/old/Parser/private/CfGrammar.cpp

@@ -0,0 +1,161 @@
+
+#include "CfGrammar.hpp"
+
+#include "Timing.hpp"
+
+using namespace tp;
+
+void CfGrammar::generateSentences(List<Sentence>& out) {
+	constexpr ualni maxTime = 1000;
+	constexpr ualni maxSentences = 200;
+	constexpr ualni maxQueue = 1000000;
+
+	List<Sentence> queue;
+	Timer timer(maxTime);
+
+	// add start production
+	Sentence start;
+	start.terms.pushBack({ startTerminal, false });
+	queue.pushBack(start);
+
+PASS:
+
+	auto const sentential = &queue.first()->data;
+	bool isSentence = true;
+	ualni termIdx = 0;
+	for (auto term : sentential->terms) {
+		if (term->terminal) {
+			termIdx++;
+			continue;
+		}
+
+		auto nonTerminal = &mNonTerminals.get(term->id);
+		for (auto production : nonTerminal->rules) {
+			queue.pushBack(*sentential);
+			auto copy = &queue.last()->data;
+
+			auto appendTerm = copy->terms.findIdx(termIdx);
+			auto deleteTerm = appendTerm;
+			for (auto arg : production->args) {
+				if (arg->isEpsilon) continue;
+				auto newTerm = copy->terms.newNode({ arg->id, arg->isTerminal });
+				copy->terms.attach(newTerm, appendTerm);
+				appendTerm = newTerm;
+			}
+			copy->terms.removeNode(deleteTerm);
+		}
+
+		isSentence = false;
+		termIdx++;
+	}
+
+	if (isSentence) {
+		auto exists = false;
+		for (auto iter : out) {
+			if (iter->terms == sentential->terms) {
+				exists = true;
+				break;
+			}
+		}
+		if (!exists) out.pushBack(*sentential);
+	}
+
+	queue.popFront();
+
+	bool stop = false;
+	stop |= !queue.length();
+	stop |= timer.isTimeout();
+	stop |= queue.length() > maxQueue;
+	stop |= out.length() > maxSentences;
+
+	if (!stop) goto PASS;
+}
+
+void CfGrammar::printSentence(Sentence& in) {
+	printf("[");
+	for (auto term : in.terms) {
+		printf(" %s", term->id.read());
+	}
+	printf(" ]\n");
+}
+
+bool CfGrammar::compile() {
+
+	if (!rules.length()) {
+		printf("Grammar must have at leas one rule\n");
+		return false;
+	}
+
+	if (startTerminal == String()) {
+		printf("Using first rule's non-terminal as grammar's root. Define explicitly - 'Start : id'\n");
+		startTerminal = rules.first()->data.id;
+	}
+
+	// find all rules
+	for (auto rule : rules) {
+
+		if (!rule->args.length()) {
+			printf("Rule must have at leas one terminal or non-terminal. See rule '%s'\n", rule->id.read());
+			return false;
+		}
+
+		if (!mNonTerminals.presents(rule->id)) {
+			mNonTerminals.put(rule->id, {});
+		}
+		auto nonTerminal = &mNonTerminals.get(rule->id);
+
+		nonTerminal->rules.pushBack(&rule.data());
+	}
+
+	for (auto nonTerminal : mNonTerminals) {
+		for (auto rule : nonTerminal->val.rules) {
+			for (auto arg : rule->args) {
+
+				if (arg->isTerminal) {
+					mTerminals.put(rule->id, {});
+				} else if (!arg->isEpsilon) {
+					auto idx = mNonTerminals.presents(arg->id);
+					if (!idx) {
+						printf("Referenced non-terminal '%s' is not defined\n", arg->id.read());
+						return false;
+					}
+					auto reference = &mNonTerminals.getSlotVal(idx);
+					reference->references.put(nonTerminal->key, &nonTerminal->val);
+					nonTerminal->val.referencing.put(arg->id, reference);
+				}
+			}
+		}
+	}
+
+	List<String> remove;
+	for (auto nonTerminal : mNonTerminals) {
+		if (!nonTerminal->val.references.size() && nonTerminal->key != startTerminal) {
+			printf("Non-terminal '%s' is defined but not used\n", nonTerminal->key.read());
+			remove.pushBack(nonTerminal->key);
+		}
+	}
+
+	for (auto rem : remove) {
+		auto nonTerminal = &mNonTerminals.get(rem.data());
+		for (auto referencing : nonTerminal->referencing) {
+			referencing->val->references.remove(nonTerminal->rules.first()->data->id);
+		}
+		for (auto rule : nonTerminal->rules) {
+			rules.removeNode(rules.find(*rule.data()));
+		}
+	}
+
+	for (auto nonTerminal : mNonTerminals) {
+		if (!nonTerminal->val.isProductive()) {
+			printf("Non-terminal '%s' is not productive\n", nonTerminal->val.rules.first()->data->id.read());
+			return false;
+		}
+	}
+
+	Map<String, ualni> processed;
+	if (mNonTerminals.get(startTerminal).isLooped(processed, startTerminal)) {
+		printf("Note that grammar is looped.\n");
+	}
+
+	return true;
+}

.back/old/Parser/private/CfGrammarParser.cpp

@@ -0,0 +1,201 @@
+
+#include "CfGrammar.hpp"
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+enum class Token {
+	NO_TOKEN,
+	START,
+	ID,
+	EQUAL,
+	TERMINAL_START,
+	TERMINAL_END,
+	ALTERNATION,
+	RULE_END,
+	IGNORED,
+	EPSILON,
+	COMMENT,
+	END,
+	FAILED,
+};
+
+typedef int1 AlphabetType;
+typedef SimpleTokenizer<AlphabetType, Token, Token::NO_TOKEN, Token::FAILED, Token::END> CFGTokenizer;
+
+struct State {
+
+	struct Transition {
+		Token tok = Token::FAILED;
+		State* target = nullptr;
+		void (*action)(CfGrammar* grammar, const String& tok) = nullptr;
+	};
+
+	String name;
+	String error;
+	Buffer<Transition> transitions;
+	bool accept = false;
+};
+
+typedef Buffer<State> Automata;
+
+void initializeTokenizer(CFGTokenizer* tok) {
+	tok->build({
+		{ CFGTokenizer::tTokenizer::etherRE, Token::IGNORED },
+		{ "Start", Token::START },
+		{ ":", Token::EQUAL },
+		{ "\\[", Token::TERMINAL_START },
+		{ "\\]", Token::TERMINAL_END },
+		{ "\\|", Token::ALTERNATION },
+		{ ";", Token::RULE_END },
+		{ "&", Token::EPSILON },
+		{ CFGTokenizer::tTokenizer::idRE, Token::ID },
+		{ CFGTokenizer::tTokenizer::commentBlockRE, Token::COMMENT },
+	});
+
+	ASSERT(tok->isBuild())
+}
+
+void initAutomata(Automata& automata) {
+	automata.reserve(7);
+	auto states = &automata.first();
+
+	auto root = states++;
+	auto end = states++;
+	auto start = states++;
+	auto rule = states++;
+	auto rhs = states++;
+	auto terminalStart = states++;
+	auto terminalEnd = states;
+
+	root->transitions = {
+		{ Token::END, end },
+		{ Token::START, start },
+		{ Token::ID,
+			rule,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.pushBack({ tok, {} });
+			} },
+	};
+
+	start->transitions = {
+		{ Token::ID, root, [](CfGrammar* grammar, const String& tok) { grammar->startTerminal = tok; } },
+	};
+
+	rule->transitions = {
+		{ Token::EQUAL, rhs },
+	};
+
+	rhs->transitions = {
+		{ Token::ALTERNATION,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				auto const& id = grammar->rules.last()->data.id;
+				grammar->rules.pushBack({ id, {} });
+			} },
+		{ Token::ID,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, false, false });
+			} },
+		{ Token::EPSILON,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, false, true });
+			} },
+		{ Token::TERMINAL_START, terminalStart },
+		{ Token::RULE_END, root },
+	};
+
+	terminalStart->transitions = {
+		{ Token::ID,
+			terminalEnd,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, true, false });
+			} },
+	};
+
+	terminalEnd->transitions = {
+		{ Token::TERMINAL_END, rhs },
+	};
+
+	auto idError = "Expected an identifier";
+	auto stmEndError = "Expected an statement end ';'";
+
+	root->error = "Expected 'Start' statement, rule or end of text";
+	start->error = idError;
+	rule->error = "Expected production equality ':'";
+	rhs->error = "Expected alternation '|', terminal id, statement end ';' or terminal start '[' ";
+	terminalStart->error = idError;
+	terminalEnd->error = "Expected terminal end '[' ";
+
+	root->name = "root";
+	start->name = "start";
+	rule->name = "production";
+	rhs->name = "rhs";
+	terminalStart->name = "terminalStart";
+	terminalEnd->name = "terminalEnd";
+
+	end->accept = true;
+}
+
+bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGrammar* grammar) {
+	tok->reset();
+	tok->bindSource(text);
+
+	State* state = &(*automata)[0];
+	while (!state->accept) {
+
+		auto token = tok->readTok();
+
+		if (token == Token::IGNORED || token == Token::COMMENT) {
+			continue;
+		}
+
+		if (token == Token::FAILED) {
+			auto errorLocation = tok->getCursorPrev().get2DLocation();
+			printf("Syntax error at (%llu, %llu)\n", errorLocation.line, errorLocation.character);
+			return false;
+		}
+
+		auto tokVal = tok->extractVal();
+
+		bool isTransition = false;
+
+		for (auto transition : state->transitions) {
+			if (transition->tok == token) {
+				if (transition->action) transition->action(grammar, tokVal);
+				state = transition->target;
+				isTransition = true;
+				break;
+			}
+		}
+
+		if (!isTransition) {
+			// if no tok matched to any transition produce error
+			auto errorLocation = tok->getCursorPrev().get2DLocation();
+			printf("Parse error at (%llu, %llu) - %s", errorLocation.line, errorLocation.character, state->error.read());
+			return false;
+		}
+	}
+
+	return true;
+}
+
+struct tp::CfGrammarParserState {
+	Automata automata;
+	CFGTokenizer tokenizer;
+};
+
+CfGrammarParserState* CfGrammar::initializeCfGrammarParser() {
+	auto state = new CfGrammarParserState();
+	initializeTokenizer(&state->tokenizer);
+	initAutomata(state->automata);
+	return state;
+}
+
+void CfGrammar::deinitializeCfGrammarParser(CfGrammarParserState* state) { delete state; }
+
+bool CfGrammar::parse(CfGrammarParserState* context, const String& source) { return ::parse(source.read(), &context->automata, &context->tokenizer, this); }
+
+bool CfGrammar::isLooped() const { return mIsLooped; }

.back/old/Parser/private/Parser.cpp

@@ -0,0 +1,7 @@
+#include "Parser.hpp"
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+static ModuleManifest* sModuleDependencies[] = { &gModuleTokenizer, nullptr };
+ModuleManifest tp::gModuleParser = ModuleManifest("CommandLine", nullptr, nullptr, sModuleDependencies);

.back/old/Parser/public/CLR.hpp

@@ -0,0 +1,62 @@
+#pragma once
+
+#include "CfGrammar.hpp"
+
+namespace tp {
+	class CLR {
+	public:
+		struct ASTNode {
+			const CfGrammar::Rule* production = nullptr;
+			Map<String, String> terminals;
+		};
+
+		class Automation {
+		public:
+			class State {};
+		};
+
+		class TerminalStream {
+		public:
+			typedef ualni TerminalID;
+
+		public:
+			TerminalStream() = default;
+			virtual TerminalID getNextTerminal() = 0;
+		};
+
+	public:
+		struct BuildError {
+			String description;
+		};
+
+		struct ParseError {
+			const Automation::State* state = nullptr;
+			String::Index location = 0;
+		};
+
+	private:
+		class SententialStack {};
+
+	public:
+		CLR() = default;
+
+		void setGrammar(const CfGrammar& grammar);
+		void setTerminal(const String& name, TerminalStream::TerminalID id);
+		void build();
+
+		void parse(TerminalStream* stream, List<ASTNode>& out, ASTNode** root);
+
+		[[nodiscard]] bool isBuild() { return mState == ParserState::BUILD; }
+		[[nodiscard]] const BuildError& getBuildError() { return mBuildError; }
+		[[nodiscard]] const ParseError& getParseError() { return mParseError; }
+
+	private:
+		enum class ParserState { NONE, BUILD, FAILED } mState = ParserState::NONE;
+
+		ParseError mParseError;
+		BuildError mBuildError;
+
+		SententialStack mStack;
+		Automation mAutomation;
+	};
+}

.back/old/Parser/public/CfGrammar.hpp

@@ -0,0 +1,89 @@
+#pragma once
+
+#include "List.hpp"
+#include "Map.hpp"
+#include "Strings.hpp"
+
+namespace tp {
+
+	struct CfGrammar {
+		struct Rule {
+			struct Arg {
+				String id;
+				bool isTerminal = false;
+				bool isEpsilon = false;
+
+				bool operator==(const Arg& in) const { return (id == in.id) && (isEpsilon == in.isEpsilon) && (isTerminal == in.isTerminal); }
+			};
+
+			String id;
+			List<Arg> args;
+
+			bool operator==(const Rule& in) const { return (id == in.id) && (args == in.args); }
+
+			[[nodiscard]] bool isProductive() const {
+				for (auto arg : args) {
+					if (arg->id == id) return false;
+				}
+				return true;
+			}
+		};
+
+		struct Sentence {
+			struct Term {
+				String id;
+				bool terminal;
+				bool operator==(const Term& in) const { return (id == in.id) && (terminal == in.terminal); }
+			};
+			List<Term> terms;
+		};
+
+		List<Rule> rules;
+		String startTerminal;
+
+	private:
+		struct NonTerminal {
+
+			List<Rule*> rules;
+			Map<String, NonTerminal*> references;
+			Map<String, NonTerminal*> referencing;
+
+			[[nodiscard]] bool isProductive() const {
+				for (auto rule : rules) {
+					if (rule->isProductive()) return true;
+				}
+				return false;
+			}
+
+			[[nodiscard]] bool isLooped(Map<String, ualni>& processed, const String& id) const {
+				for (auto ref : referencing) {
+					if (processed.presents(ref->key)) return true;
+				}
+				processed.put(id, {});
+				for (auto ref : referencing) {
+					if (ref->val->isLooped(processed, ref->key)) return true;
+				}
+				return false;
+			}
+		};
+
+		struct Terminal {};
+
+		Map<String, Terminal> mTerminals;
+		Map<String, NonTerminal> mNonTerminals;
+		bool mIsLooped = false;
+
+	public:
+		static struct CfGrammarParserState* initializeCfGrammarParser();
+		static void deinitializeCfGrammarParser(CfGrammarParserState*);
+
+	public:
+		bool parse(CfGrammarParserState* context, const String& source);
+		bool compile();
+
+		void generateSentences(List<Sentence>& out);
+		static void printSentence(Sentence& in);
+
+		[[nodiscard]] bool isLooped() const;
+	};
+}

.back/old/Parser/public/Parser.hpp

@@ -0,0 +1,7 @@
+#pragma once
+
+#include "CLR.hpp"
+
+namespace tp {
+	extern ModuleManifest gModuleParser;
+}

.back/old/Parser/rsc/grammar.txt

@@ -0,0 +1,20 @@
+
+/*
+Sample grammar.
+This grammar wil produce text in the form: TODO
+*/
+
+/* This is the starting production of the grammar */
+
+/* Simple Example
+Start A
+A : [ ID ] | &;
+*/
+
+/* Lists Example */
+
+Start List
+
+List : List Stm | Stm;
+Stm : StmBody [ END ];
+StmBody: [ Stm1 ] | [ Stm2 ] | [ Stm3 ];

.back/old/Parser/tests/CLRTests.cpp

@@ -0,0 +1,9 @@
+
+#include "Parser.hpp"
+#include "Testing.hpp"
+
+using namespace tp;
+
+TEST_DEF(CLR) {
+	// TEST(false);
+}

.back/old/Parser/tests/GrammarTests.cpp

@@ -0,0 +1,48 @@
+
+#include "Parser.hpp"
+#include "Testing.hpp"
+
+using namespace tp;
+
+TEST_DEF(Grammar) {
+
+	String source = R"(
+Start Body
+Body : [ Function ] | List | &;
+List : [ LIST ];
+)";
+
+	auto state = CfGrammar::initializeCfGrammarParser();
+
+	CfGrammar grammar;
+
+	if (!grammar.parse(state, source)) {
+		TEST(0 && "Parsing is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	if (!grammar.compile()) {
+		TEST(0 && "Compilation is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	CfGrammar::deinitializeCfGrammarParser(state);
+
+	printf("Grammar accepted.\n");
+
+	List<CfGrammar::Sentence> sentences;
+	grammar.generateSentences(sentences);
+
+	TEST_ASSERT(sentences.length() == 3);
+	TEST_ASSERT(sentences.first()->data.terms.length() == 1);
+	TEST_ASSERT(sentences.last()->data.terms.length() == 1);
+
+	TEST(sentences.first()->data.terms.first()->data.id == "Function");
+	TEST(sentences.last()->data.terms.first()->data.id == "LIST");
+
+	printf("Example sentences formed from grammar: \n");
+	for (auto sentence : sentences)
+		tp::CfGrammar::printSentence(sentence.data());
+}

.back/old/Parser/tests/Tests.cpp

@@ -0,0 +1,23 @@
+
+#include "Parser.hpp"
+#include "Testing.hpp"
+
+using namespace tp;
+
+void testGrammar();
+void testCLR();
+
+int main(int argc, const char* argv[]) {
+	tp::ModuleManifest* deps[] = { &tp::gModuleParser, nullptr };
+	tp::ModuleManifest testModule("CommandLineTest", nullptr, nullptr, deps);
+
+	if (!testModule.initialize()) {
+		return 1;
+	}
+
+	testGrammar();
+	testCLR();
+
+	testModule.deinitialize();
+	return 0;
+}

.back/old/Tokenizer/CMakeLists.txt

@@ -0,0 +1,15 @@
+project(Tokenizer)
+
+### ---------------------- Static Library  ---------------------  ###
+file(GLOB SOURCES "./private/*.cpp")
+file(GLOB HEADERS "./public/*.hpp")
+add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
+target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
+target_link_libraries(${PROJECT_NAME} PUBLIC Strings)
+
+### -------------------------- Tests  --------------------------  ###
+enable_testing()
+file(GLOB TEST_SOURCES "./tests/*.cpp")
+add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES})
+target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils)
+add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests)

.back/old/Tokenizer/private/Tokenizer.cpp

@@ -0,0 +1,10 @@
+
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+static ModuleManifest* sModuleDependencies[] = { &tp::gModuleStrings, nullptr };
+
+void deinit(const ModuleManifest*) {}
+
+ModuleManifest tp::gModuleTokenizer = ModuleManifest("Tokenizer", nullptr, deinit, sModuleDependencies);

.back/old/Tokenizer/public/AutomataGraph.h

@@ -0,0 +1,19 @@
+
+#pragma once
+
+#include "List.hpp"
+#include "Buffer.hpp"
+#include "Buffer2D.hpp"
+#include "Utils.hpp"
+#include "Map.hpp"
+
+#ifdef ENV_OS_WINDOWS
+#undef min
+#undef max
+#endif
+
+namespace tp {
+
+	extern ModuleManifest gModuleTokenizer;
+
+}

.back/old/Tokenizer/public/RegularExpression.h

@@ -0,0 +1,333 @@
+
+#pragma once
+
+#include "AutomataGraph.h"
+
+namespace tp {
+	extern ModuleManifest gModuleTokenizer;
+}
+
+namespace tp::RegEx {
+
+	struct AstNode {
+		enum Type {
+			NONE,
+			ANY,
+			OR,
+			IF,
+			CLASS,
+			COMPOUND,
+			REPEAT,
+			VAL,
+		} mType = NONE;
+
+		AstNode() = default;
+		virtual ~AstNode() = default;
+	};
+
+	template <typename tAlphabetType>
+	struct AstVal : public AstNode {
+		explicit AstVal(tAlphabetType val) :
+			mVal(val) {
+			mType = VAL;
+		}
+		~AstVal() override = default;
+		tAlphabetType mVal;
+	};
+
+	struct AstCompound : public AstNode {
+		AstCompound() { mType = COMPOUND; }
+		~AstCompound() override {
+			for (auto iter : mChilds) {
+				delete iter.data();
+			}
+			mChilds.removeAll();
+		}
+		List<AstNode*> mChilds;
+	};
+
+	struct AstAlternation : public AstNode {
+		AstAlternation() { mType = OR; }
+		~AstAlternation() override {
+			delete mFirst;
+			delete mSecond;
+		}
+		AstNode* mFirst = nullptr;
+		AstNode* mSecond = nullptr;
+	};
+
+	struct AstIf : public AstNode {
+		AstIf() { mType = IF; }
+		~AstIf() override { delete mNode; }
+		AstNode* mNode = nullptr;
+	};
+
+	struct AstAny : public AstNode {
+		AstAny() { mType = ANY; }
+		~AstAny() override = default;
+	};
+
+	struct AstRepetition : public AstNode {
+		AstRepetition() { mType = REPEAT; }
+		~AstRepetition() override { delete mNode; }
+		AstNode* mNode = nullptr;
+		bool mPlus = false;
+	};
+
+	template <typename tAlphabetType>
+	struct AstClass : public AstNode {
+		AstClass() { mType = CLASS; }
+		~AstClass() override { mRanges.removeAll(); }
+		List<Range<tAlphabetType>> mRanges;
+		bool mExclude = false;
+	};
+
+	struct ParseError {
+		const char* description = nullptr;
+		uhalni offset = 0;
+		[[nodiscard]] bool isError() const { return description != nullptr; }
+	};
+
+	template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal>
+	class Parser {
+
+		enum TokType : uint1 {
+			TOK_COMPOUND_START = 0,
+			TOK_COMPOUND_END,
+			TOK_CLASS_START,
+			TOK_CLASS_END,
+			TOK_CLASS_START_EXCLUDE,
+			TOK_CLASS_END_EXCLUDE,
+			TOK_OR,
+			TOK_IF,
+			TOK_ANY,
+			TOK_REPEAT,
+			TOK_REPEAT_PLUS,
+			TOK_HYPHEN,
+			TOK_SPECIALS_END_,
+			TOK_VAL,
+			TOK_NONE,
+		};
+
+		tAlphabetType SpecialSymbols[TOK_SPECIALS_END_] = {
+			'(', ')', '[', ']', '{', '}', '|', '?', '.', '*', '+', '-',
+		};
+
+		tAlphabetType mEscapeSymbol = '\\';
+
+		struct Token {
+			TokType type;
+			tAlphabetType val;
+		};
+
+		const tAlphabetType* mSource = nullptr;
+		uhalni mOffset = 0;
+		Token mCurToken;
+		uhalni mTokLength = 0;
+
+	public:
+		ParseError mError;
+
+		// regular expression must be a zero termination string
+		AstCompound* parse(const tAlphabetType* regex) {
+			mSource = regex;
+			return parseRegEx();
+		}
+
+	private:
+		AstCompound* parseRegEx() {
+			auto out = new AstCompound();
+			for (AstNode* node = parseElement(); node; node = parseElement()) {
+				out->mChilds.pushBack(node);
+			}
+			if (!out->mChilds.length()) {
+				genError("Expected A Expression");
+			}
+			if (mError.description) {
+				delete out;
+				return nullptr;
+			}
+			return out;
+		}
+
+		AstNode* parseElement() {
+			AstNode* out = nullptr;
+			switch (readTok().type) {
+				case TOK_COMPOUND_START: out = parseCompound(); break;
+				case TOK_CLASS_START: out = parseClass(); break;
+				case TOK_CLASS_START_EXCLUDE: out = parseClass(true); break;
+				case TOK_ANY: out = parseAny(); break;
+				case TOK_VAL: out = parseVal(); break;
+				case TOK_NONE:
+					{
+						discardTok();
+						return nullptr;
+					};
+				default: break;
+			}
+			if (!out) {
+				discardTok();
+				return nullptr;
+			}
+			switch (readTok().type) {
+				case TOK_OR: out = parseAlternation(out); break;
+				case TOK_REPEAT: out = parseRepetition(out); break;
+				case TOK_REPEAT_PLUS: out = parseRepetition(out, true); break;
+				case TOK_IF: out = parseIf(out); break;
+				case TOK_NONE: break;
+				default:
+					{
+						discardTok();
+					}
+			}
+			return out;
+		}
+
+		AstCompound* parseCompound() {
+			auto out = new AstCompound();
+			for (AstNode* node = parseElement(); node; node = parseElement()) {
+				out->mChilds.pushBack(node);
+			}
+			if (readTok().type != TOK_COMPOUND_END) {
+				genError("Expected Compound End");
+			}
+			if (mError.description) {
+				delete out;
+				return nullptr;
+			}
+			return out;
+		}
+
+		AstClass<tAlphabetType>* parseClass(bool exclude = false) {
+			auto out = new AstClass<tAlphabetType>();
+			out->mExclude = exclude;
+			auto& ranges = out->mRanges;
+
+			readTok();
+
+		READ_VAL:
+			if (mCurToken.type != TOK_VAL) {
+				delete out;
+				genError("Expected A Value");
+				return nullptr;
+			}
+			char range_start = mCurToken.val;
+
+			readTok();
+			if (mCurToken.type != TOK_HYPHEN) {
+				delete out;
+				genError("Expected A Range");
+				return nullptr;
+			}
+
+			readTok();
+			if (mCurToken.type != TOK_VAL) {
+				delete out;
+				genError("Expected A Value");
+				return nullptr;
+			}
+			char range_end = mCurToken.val;
+
+			ranges.pushBack({ range_start, range_end });
+
+			readTok();
+			if ((mCurToken.type == TOK_CLASS_END && !exclude) || (mCurToken.type == TOK_CLASS_END_EXCLUDE && exclude)) {
+				return out;
+			} else {
+				goto READ_VAL;
+			}
+		}
+
+		AstAny* parseAny() { return new AstAny(); }
+
+		AstVal<tAlphabetType>* parseVal() {
+			auto out = new AstVal<tAlphabetType>(mCurToken.val);
+			return out;
+		}
+
+		AstAlternation* parseAlternation(AstNode* left) {
+			auto right = parseElement();
+			if (!right) {
+				genError("Expected Alternation right Side");
+				delete left;
+				return nullptr;
+			}
+
+			auto out = new AstAlternation();
+
+			out->mFirst = left;
+			out->mSecond = right;
+
+			return out;
+		}
+
+		AstRepetition* parseRepetition(AstNode* left, bool plus = false) {
+			auto out = new AstRepetition();
+			out->mNode = left;
+			out->mPlus = plus;
+			return out;
+		}
+
+		AstIf* parseIf(AstNode* left) {
+			auto out = new AstIf();
+			out->mNode = left;
+			return out;
+		}
+
+		void genError(const char* desc) { mError = { desc, mOffset }; }
+
+		Token& readTok() {
+
+			const tAlphabetType* crs = mSource + mOffset;
+
+			// zero termination string
+			if (*crs == 0) {
+				mCurToken.type = TOK_NONE;
+				return mCurToken;
+			}
+
+			mTokLength = 1;
+			mCurToken.type = TOK_VAL;
+			mCurToken.val = crs[0];
+
+			if (crs[0] == mEscapeSymbol) {
+				mCurToken.val = crs[1];
+				mTokLength = 2;
+			} else {
+				for (uhalni tok = 0; tok < TOK_SPECIALS_END_; tok++) {
+					if (SpecialSymbols[tok] == mCurToken.val) {
+						mCurToken.type = TokType(tok);
+						break;
+					}
+				}
+			}
+
+			mOffset += mTokLength;
+			return mCurToken;
+		}
+
+		void discardTok() {
+			mOffset -= mTokLength;
+			mTokLength = 0;
+		}
+	};
+
+	template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
+	CompileError<tStateType> compile(
+		NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& out, const tAlphabetType* regex, tStateType state
+	) {
+		Compiler<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> compiler;
+		compiler.compile(out, regex, state);
+		return compiler.mError;
+	}
+
+	template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
+	CompileError<tStateType> compile(
+		NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& out,
+		const InitialierList<Pair<const tAlphabetType*, tStateType>>& rules
+	) {
+		Compiler<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> compiler;
+		compiler.compile(out, rules);
+		return compiler.mError;
+	}
+}

.back/old/Tokenizer/public/Tokenizer.hpp

@@ -0,0 +1,184 @@
+
+#pragma once
+
+#include "RegularExpression.h"
+#include "Strings.hpp"
+
+namespace tp {
+
+	extern ModuleManifest gModuleTokenizer;
+
+	template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal>
+	class Tokenizer {
+
+		TransitionMatrix<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> mTransitionMatrix;
+
+		RegEx::CompileError<tTokType> mError;
+
+		bool scanFailed() { return mTransitionMatrix.isTrapped(); }
+
+	public:
+		// Some useful RE to be reused
+		static constexpr const tAlphabetType* etherRE = "\n|\t| |\r";
+		static constexpr const tAlphabetType* intRE = "((\\-)|(\\+))?[0-9]+i?";
+		static constexpr const tAlphabetType* floatRE = R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)";
+		static constexpr const tAlphabetType* commentBlockRE = R"((/\*){\*-\*}*(\*/))";
+		static constexpr const tAlphabetType* stringRE = R"("{"-"}*")";
+		static constexpr const tAlphabetType* idRE = "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*";
+
+	public:
+		Tokenizer() { MODULE_SANITY_CHECK(gModuleTokenizer) }
+
+		void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) {
+			NFA<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> nfa;
+
+			mError = RegEx::compile(nfa, rules);
+			if (mError.isError()) {
+				return;
+			}
+
+			DFA<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> dfa(nfa);
+			mTransitionMatrix.construct(dfa);
+		}
+
+		[[nodiscard]] bool isBuild() const { return !mError.isError(); }
+
+		auto getMatrix() const { return &mTransitionMatrix; }
+
+		const RegEx::CompileError<tTokType>& getBuildError() { return mError; }
+
+		void resetMatrix() { mTransitionMatrix.reset(); }
+
+		tTokType advanceSymbol(tAlphabetType symbol) { return mTransitionMatrix.move(symbol); }
+
+		tTokType advanceToken(const tAlphabetType* source, ualni source_len, ualni* token_len) {
+			tTokType out = tNoTokVal;
+			*token_len = 0;
+			for (ualni idx = 0; idx < source_len; idx++) {
+				out = advanceSymbol(source[idx]);
+				if (out != tNoTokVal) {
+					*token_len = idx;
+					break;
+				}
+			}
+			return out;
+		}
+
+		~Tokenizer() = default;
+	};
+
+	template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal, tTokType tSourceEndTokVal>
+	class SimpleTokenizer {
+	public:
+		typedef Tokenizer<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> tTokenizer;
+
+	private:
+		tTokenizer mTokenizer;
+
+		const tAlphabetType* mSource = nullptr;
+		ualni mLastTokLen = 0;
+		ualni mSourceLen = 0;
+		ualni mAdvancedOffset = 0;
+
+	public:
+		struct Cursor {
+			const tAlphabetType* mSource = nullptr;
+			ualni mAdvancedOffset = 0;
+			const tAlphabetType* str() { return mSource + mAdvancedOffset; }
+
+			struct Location2D {
+				ualni line = 0;
+				ualni character = 0;
+			};
+
+			Location2D get2DLocation() {
+				Location2D out;
+				typedef typename StringLogic<tAlphabetType>::Index Idx;
+				Buffer<Idx> offsets;
+				StringLogic<tAlphabetType>::calcLineOffsets(mSource, StringLogic<tAlphabetType>::calcLength(mSource), offsets);
+				for (ualni idx = 1; idx < offsets.size(); idx++) {
+					if (offsets[idx] > mAdvancedOffset) {
+						out.line = idx - 1;
+						break;
+					}
+				}
+				out.character = mAdvancedOffset - offsets[out.line];
+				return out;
+			}
+		};
+
+		SimpleTokenizer() = default;
+		auto getTokenizer() const { return &mTokenizer; }
+
+		void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) { mTokenizer.build(rules); }
+
+		[[nodiscard]] bool isBuild() const { return mTokenizer.isBuild(); }
+
+		const RegEx::CompileError<tTokType>& getBuildError() { return mTokenizer.getBuildError(); }
+
+		void bindSource(const tAlphabetType* source) {
+			mSource = source;
+			while (mSource[mSourceLen])
+				mSourceLen++;
+			mSourceLen++;
+		}
+
+		[[nodiscard]] bool isInputLeft() const {
+			if (!mSource) {
+				return false;
+			}
+			if (mSource[mAdvancedOffset] == 0) {
+				return false;
+			}
+			return true;
+		}
+
+		Cursor getCursor() const { return { mSource, mAdvancedOffset }; }
+
+		Cursor getCursorPrev() const { return { mSource, mAdvancedOffset - mLastTokLen }; }
+
+		void setCursor(const Cursor& crs) {
+			mAdvancedOffset = crs.mAdvancedOffset;
+			mLastTokLen = 0;
+		}
+
+		tTokType readTok() {
+			if (mSourceLen == mAdvancedOffset + 1) {
+				return tSourceEndTokVal;
+			}
+
+			tTokType out = mTokenizer.advanceToken(mSource + mAdvancedOffset, mSourceLen - mAdvancedOffset, &mLastTokLen);
+			mAdvancedOffset += mLastTokLen;
+			return out;
+		}
+
+		tTokType lookupTok() {
+			auto out = readTok();
+			discardTok();
+			return out;
+		}
+
+		void discardTok() {
+			mTokenizer.resetMatrix();
+			mAdvancedOffset -= mLastTokLen;
+			mLastTokLen = 0;
+		}
+
+		void skipTok() { readTok(); }
+
+		void reset() {
+			mAdvancedOffset = mLastTokLen = 0;
+			mTokenizer.resetMatrix();
+		}
+
+		[[nodiscard]] ualni lastTokLEn() const { return mLastTokLen; }
+
+		String extractVal() {
+			auto crs = getCursorPrev();
+			String out;
+			out.resize(mLastTokLen);
+			memCopy(out.write(), crs.str(), mLastTokLen);
+			return out;
+		}
+	};
+}

.back/old/Tokenizer/tests/TestTokenizer.cpp

@@ -0,0 +1,398 @@
+// #include "NewPlacement.hpp"
+
+#include "Testing.hpp"
+#include "Tokenizer.hpp"
+#include <cstdio>
+#include <iostream>
+
+#define LOG(val) std::cout << #val << " "
+
+using namespace tp;
+
+ualni outputHashPassed = 1156;
+
+const char* script =
+	R"(
+
+/* yea
+comment  "and string inside"  with int 123 and float 0.123f */
+
+-1.f;
+
+if (+1.f) {
+	var string = "string value some &&characters inside string&& -= asdf 09a
+	another line";
+
+}
+
+while
+
+return
+
+)";
+
+TEST_DEF_STATIC(General) {
+
+	enum class TokType {
+		NONE,
+		FAILED,
+		TOK_SOURCE_END,
+
+		VAR,
+		CLASS_DEF,
+		SELF,
+		SCOPE_IN,
+		SCOPE_OUT,
+		ASSIGN,
+		DEF_FUNC,
+		PRINT,
+		IF,
+		ELSE,
+		WHILE,
+		BRACKET_IN,
+		BRACKET_OUT,
+		COMMA,
+		NEW,
+		CHILD,
+		RETURN,
+		EQUAL,
+		NOT_EQUAL,
+		MORE,
+		LESS,
+		QE_OR_MORE,
+		QE_OR_LESS,
+		BOOL_NOT,
+		BOOL_AND,
+		BOOL_OR,
+		ADD,
+		MUL,
+		SUB,
+		DIV,
+		STM_END,
+		CONST_TRUE,
+		CONST_FALSE,
+		CONST_INT,
+		CONST_FLOAT,
+		SPACE,
+		CONST_STRING,
+		ID,
+
+		// COMMENT_LINE,
+
+		COMMENT_BLOCK,
+	};
+
+	SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
+
+	lexer.build({
+		{ "\n|\t| |\r", TokType::SPACE },
+		{ "var", TokType::VAR },
+		{ "class", TokType::CLASS_DEF },
+		{ "self", TokType::SELF },
+		{ "\\{", TokType::SCOPE_IN },
+		{ "\\}", TokType::SCOPE_OUT },
+		{ "=", TokType::ASSIGN },
+		{ "def", TokType::DEF_FUNC },
+		{ "<<", TokType::PRINT },
+		{ "if", TokType::IF },
+		{ "else", TokType::ELSE },
+		{ "while", TokType::WHILE },
+		{ "\\(", TokType::BRACKET_IN },
+		{ "\\)", TokType::BRACKET_OUT },
+		{ ",", TokType::COMMA },
+		{ "new", TokType::NEW },
+		{ "\\.", TokType::CHILD },
+		{ "return", TokType::RETURN },
+		{ "==", TokType::EQUAL },
+		{ "!=", TokType::NOT_EQUAL },
+		{ ">", TokType::MORE },
+		{ "<", TokType::LESS },
+		{ ">=", TokType::QE_OR_MORE },
+		{ "<=", TokType::QE_OR_LESS },
+		{ "!", TokType::BOOL_NOT },
+		{ "&&", TokType::BOOL_AND },
+		{ "\\|\\|", TokType::BOOL_OR },
+		{ "\\+", TokType::ADD },
+		{ "\\*", TokType::MUL },
+		{ "\\-", TokType::SUB },
+		{ "/", TokType::DIV },
+		{ ";", TokType::STM_END },
+		{ "true", TokType::CONST_TRUE },
+		{ "false", TokType::CONST_FALSE },
+		{ "((\\-)|(\\+))?[0-9]+i?", TokType::CONST_INT },
+		{ R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)", TokType::CONST_FLOAT },
+		{ R"((/\*){\*-\*}*(\*/))", TokType::COMMENT_BLOCK },
+		{ R"("{"-"}*")", TokType::CONST_STRING },
+		{ "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", TokType::ID },
+	});
+
+	if (!lexer.isBuild()) {
+		printf("Error : %s", lexer.getBuildError().description);
+		return;
+	}
+
+	lexer.bindSource(script);
+
+	TokType tok;
+	ualni outputHash = 0;
+	do {
+
+		tok = lexer.readTok();
+
+		outputHash += ualni(tok);
+
+		switch (tok) {
+			case TokType::SPACE:
+				{
+					printf(" ");
+				}
+				break;
+			case TokType::VAR:
+				{
+					LOG(VAR);
+				}
+				break;
+			case TokType::CLASS_DEF:
+				{
+					LOG(CLASS_DEF);
+				}
+				break;
+			case TokType::SELF:
+				{
+					LOG(SELF);
+				}
+				break;
+			case TokType::SCOPE_IN:
+				{
+					LOG(SCOPE_IN);
+				}
+				break;
+			case TokType::SCOPE_OUT:
+				{
+					LOG(SCOPE_OUT);
+				}
+				break;
+			case TokType::ASSIGN:
+				{
+					LOG(ASSIGN);
+				}
+				break;
+			case TokType::DEF_FUNC:
+				{
+					LOG(DEF_FUNC);
+				}
+				break;
+			case TokType::PRINT:
+				{
+					LOG(PRINT);
+				}
+				break;
+			case TokType::IF:
+				{
+					LOG(IF);
+				}
+				break;
+			case TokType::ELSE:
+				{
+					LOG(ELSE);
+				}
+				break;
+			case TokType::WHILE:
+				{
+					LOG(WHILE);
+				}
+				break;
+			case TokType::BRACKET_IN:
+				{
+					LOG(BRACKET_IN);
+				}
+				break;
+			case TokType::BRACKET_OUT:
+				{
+					LOG(BRACKET_OUT);
+				}
+				break;
+			case TokType::COMMA:
+				{
+					LOG(COMMA);
+				}
+				break;
+			case TokType::NEW:
+				{
+					LOG(NEW);
+				}
+				break;
+			case TokType::CHILD:
+				{
+					LOG(CHILD);
+				}
+				break;
+			case TokType::RETURN:
+				{
+					LOG(RETURN);
+				}
+				break;
+			case TokType::EQUAL:
+				{
+					LOG(EQUAL);
+				}
+				break;
+			case TokType::NOT_EQUAL:
+				{
+					LOG(NOT_EQUAL);
+				}
+				break;
+			case TokType::MORE:
+				{
+					LOG(MORE);
+				}
+				break;
+			case TokType::LESS:
+				{
+					LOG(LESS);
+				}
+				break;
+			case TokType::QE_OR_MORE:
+				{
+					LOG(QE_OR_MORE);
+				}
+				break;
+			case TokType::QE_OR_LESS:
+				{
+					LOG(QE_OR_LESS);
+				}
+				break;
+			case TokType::BOOL_NOT:
+				{
+					LOG(BOOL_NOT);
+				}
+				break;
+			case TokType::BOOL_AND:
+				{
+					LOG(BOOL_AND);
+				}
+				break;
+			case TokType::BOOL_OR:
+				{
+					LOG(BOOL_OR);
+				}
+				break;
+			case TokType::ADD:
+				{
+					LOG(ADD);
+				}
+				break;
+			case TokType::MUL:
+				{
+					LOG(MUL);
+				}
+				break;
+			case TokType::SUB:
+				{
+					LOG(SUB);
+				}
+				break;
+			case TokType::DIV:
+				{
+					LOG(DIV);
+				}
+				break;
+			case TokType::STM_END:
+				{
+					LOG(STM_END);
+				}
+				break;
+			case TokType::CONST_TRUE:
+				{
+					LOG(TRUE);
+				}
+				break;
+			case TokType::CONST_FALSE:
+				{
+					LOG(FALSE);
+				}
+				break;
+			case TokType::CONST_INT:
+				{
+					LOG(INT);
+				}
+				break;
+			case TokType::CONST_FLOAT:
+				{
+					LOG(FLOAT);
+				}
+				break;
+			case TokType::CONST_STRING:
+				{
+					LOG(STRING);
+				}
+				break;
+			case TokType::ID:
+				{
+					LOG(ID);
+				}
+				break;
+			// case TokType::COMMENT_LINE: { LOG(COMMENT_LINE) } break;
+			case TokType::COMMENT_BLOCK:
+				{
+					LOG(COMMENT_BLOCK);
+				}
+				break;
+			case TokType::FAILED:
+				{
+					LOG(FAILED);
+				}
+				break;
+			case TokType::NONE:
+			case TokType::TOK_SOURCE_END: break;
+		}
+
+	} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
+
+	printf("\n\nOutputHash : %llu", outputHash);
+
+	TEST(outputHash == outputHashPassed);
+}
+
+TEST_DEF(Simple) {
+	enum class TokType {
+		START = 0,
+		NONE = 1,
+		FAILED = 2,
+		SPACE = 3,
+		ID = 4,
+		COMMENT_BLOCK = 5,
+		TOK_SOURCE_END = 6,
+	};
+
+	SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
+
+	lexer.build({
+		{ " ", TokType::SPACE },
+		{ "([a-c]|A)+", TokType::ID },
+		{ "A", TokType::START },
+	});
+
+	if (!lexer.isBuild()) {
+		printf("Error : %s", lexer.getBuildError().description);
+		return;
+	}
+
+	lexer.bindSource(" A bc cb cc ");
+
+	TokType tok;
+	ualni outputHash = 0;
+	printf("\n\n OutputHash : ");
+	do {
+		tok = lexer.readTok();
+		outputHash += ualni(tok);
+		printf(" %i ", int(tok));
+	} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
+	printf(" : %llu", outputHash);
+	TEST(outputHash == 33);
+}
+
+TEST_DEF(Tokenizer) {
+	testGeneral();
+	testSimple();
+}

.back/old/Tokenizer/tests/tests.cpp

@@ -0,0 +1,18 @@
+
+#include "Testing.hpp"
+#include "Tokenizer.hpp"
+
+void testTokenizer();
+
+int main(int argc, char* argv[]) {
+
+	tp::ModuleManifest* ModuleDependencies[] = { &tp::gModuleTokenizer, nullptr };
+	tp::ModuleManifest TestModule("TokenizerTest", nullptr, nullptr, ModuleDependencies);
+	if (!TestModule.initialize()) {
+		return 1;
+	}
+
+	testTokenizer();
+
+	TestModule.deinitialize();
+}