Language Initial

Language/CMakeLists.txt

@@ -0,0 +1,22 @@
+
+cmake_minimum_required(VERSION 3.2)
+
+set(CMAKE_CXX_STANDARD 23)
+
+project(Language)
+
+### ---------------------- Static Library  ---------------------  ###
+file(GLOB SOURCES "./private/*.cpp" "./private/*/*.cpp")
+file(GLOB HEADERS "./public/*.hpp" "./public/*/*.hpp")
+add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
+target_include_directories(${PROJECT_NAME} PUBLIC public/)
+target_link_libraries(${PROJECT_NAME} PUBLIC Strings Automatas)
+
+### -------------------------- Tests  --------------------------  ###
+enable_testing()
+file(GLOB TEST_SOURCES "./tests/*.cpp")
+add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES})
+target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils)
+add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests)
+
+install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/${PROJECT_NAME}/lib)

Language/old/Acceptors.hpp

@@ -0,0 +1,79 @@
+
+#pragma once
+
+/*
+ * This file contains acceptors for each language
+ * Acceptor is managing an automation machine that can recognize whether this given sentence is in the language.
+ * Above that acceptors are responsible for defining those automations
+ * Some acceptors can supply additional information such as Abstract-Syntax-Tree or Sentence id
+ * */
+
+#include "LanguageModule.hpp"
+
+namespace tp {
+
+  template <typename tAlphabet>
+  class SymbolSupply {
+  public:
+    typedef ualni Cursor;
+
+  public:
+    SymbolSupply() = default;
+    virtual ~SymbolSupply() = 0;
+
+  public:
+    virtual Cursor getCursor() = 0;
+    virtual void setCursor(Cursor) = 0;
+
+    virtual void getPrevCursor() = 0;
+    virtual ualni getSymbolLength() = 0;
+
+    virtual const tAlphabet& getNextSymbol() = 0;
+
+  private:
+    Cursor mCursor = 0;
+  };
+
+  template <typename tAlphabet, typename tSentenceId>
+  class RegularAcceptor {
+    typedef SymbolSupply<tAlphabet> Sentence;
+
+  public:
+    RegularAcceptor() = default;
+    void build(const RegularGrammar<tSentenceId>& grammar);
+    tSentenceId recognize(Sentence& sentence);
+  };
+
+  // Subset of CFG
+  template <typename tAlphabet>
+  class LL {
+    typedef SymbolSupply<tAlphabet> Sentence;
+
+  public:
+    LL() = default;
+    void build(const ContextFreeGrammar& grammar);
+    void recognize(Sentence& sentence);
+  };
+
+  // Subset of CFG much larger than LL
+  template <typename tAlphabet>
+  class CLR {
+    typedef SymbolSupply<tAlphabet> Sentence;
+
+  public:
+    CLR() = default;
+    void build(const ContextFreeGrammar& grammar);
+    void recognize(Sentence& sentence);
+  };
+
+  // Subset of CFG CLR
+  template <typename tAlphabet>
+  class LALR {
+    typedef SymbolSupply<tAlphabet> Sentence;
+
+  public:
+    LALR() = default;
+    void build(const ContextFreeGrammar& grammar);
+    void recognize(Sentence& sentence);
+  };
+}

Language/old/Parser/CMakeLists.txt

@@ -0,0 +1,21 @@
+project(Parser)
+
+### ---------------------- Static Library  ---------------------  ###
+file(GLOB SOURCES "./private/*.cpp")
+file(GLOB HEADERS "./public/*.hpp")
+add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
+target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
+target_link_libraries(${PROJECT_NAME} PUBLIC Tokenizer)
+
+### -------------------------- Applications  --------------------------  ###
+add_executable(cfg ./applications/Cfg.cpp)
+target_link_libraries(cfg ${PROJECT_NAME} CommandLine)
+
+configure_file(rsc/grammar.txt grammar.txt COPYONLY)
+
+### -------------------------- Tests  --------------------------  ###
+enable_testing()
+file(GLOB TEST_SOURCES "./tests/*.cpp")
+add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES})
+target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils)
+add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests)

Language/old/Parser/applications/Cfg.cpp

@@ -0,0 +1,82 @@
+
+#include "CommandLine.hpp"
+#include "Parser.hpp"
+
+using namespace tp;
+
+void run(const String& source) {
+	auto state = CfGrammar::initializeCfGrammarParser();
+
+	CfGrammar grammar;
+
+	if (!grammar.parse(state, source)) {
+		printf("Parsing is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	if (!grammar.compile()) {
+		printf("Compilation is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	printf("Grammar accepted.\n");
+
+	List<CfGrammar::Sentence> sentences;
+	grammar.generateSentences(sentences);
+
+	printf("Example sentences formed from grammar: \n");
+	for (auto sentence : sentences)
+		tp::CfGrammar::printSentence(sentence.data());
+
+	CfGrammar::deinitializeCfGrammarParser(state);
+}
+
+int main(int argc, const char* argv[]) {
+	tp::ModuleManifest* deps[] = { &tp::gModuleParser, &tp::gModuleCommandLine, nullptr };
+	tp::ModuleManifest testModule("CommandLineTest", nullptr, nullptr, deps);
+
+	if (!testModule.initialize()) {
+		return 1;
+	}
+
+	{
+		printf("Arguments given: ");
+		for (auto idx = 0; idx < argc; idx++) {
+			printf("[%s]", argv[idx]);
+		}
+		printf("\n");
+
+		CommandLine cmd = {
+			{ "grammar", CommandLine::Arg::STR },
+		};
+
+		if (!cmd.parse((char) argc, argv, true, 1)) {
+			return 1;
+		}
+
+		auto location = cmd.getString("grammar");
+
+		LocalConnection file;
+		String grammar;
+
+		file.connect(LocalConnection::Location(location), LocalConnection::Type(true));
+
+		if (!file.getConnectionStatus().isOpened()) {
+			printf("Cant open file\n");
+			return 1;
+		}
+
+		auto size = file.size();
+		grammar.resize((String::Index) size);
+		file.readBytes(grammar.write(), size);
+
+		run(grammar);
+
+		file.disconnect();
+	}
+
+	testModule.deinitialize();
+	return 0;
+}

Language/old/Parser/private/CLR.cpp

@@ -0,0 +1,19 @@
+#include "CLR.hpp"
+
+using namespace tp;
+
+void CLR::setGrammar(const tp::CfGrammar& grammar) {
+	if (grammar.isLooped()) {
+		mState = ParserState::FAILED;
+		mBuildError.description = "Cant build lalr parser from looped context free grammar";
+		return;
+	}
+
+	mState = ParserState::FAILED;
+}
+
+void CLR::setTerminal(const String& name, TerminalStream::TerminalID id) {}
+
+void CLR::build() {}
+
+void CLR::parse(TerminalStream* source, List<ASTNode>& out, ASTNode** root) { *root = nullptr; }

Language/old/Parser/private/CfGrammar.cpp

@@ -0,0 +1,161 @@
+
+#include "CfGrammar.hpp"
+
+#include "Timing.hpp"
+
+using namespace tp;
+
+void CfGrammar::generateSentences(List<Sentence>& out) {
+	constexpr ualni maxTime = 1000;
+	constexpr ualni maxSentences = 200;
+	constexpr ualni maxQueue = 1000000;
+
+	List<Sentence> queue;
+	Timer timer(maxTime);
+
+	// add start production
+	Sentence start;
+	start.terms.pushBack({ startTerminal, false });
+	queue.pushBack(start);
+
+PASS:
+
+	auto const sentential = &queue.first()->data;
+	bool isSentence = true;
+	ualni termIdx = 0;
+	for (auto term : sentential->terms) {
+		if (term->terminal) {
+			termIdx++;
+			continue;
+		}
+
+		auto nonTerminal = &mNonTerminals.get(term->id);
+		for (auto production : nonTerminal->rules) {
+			queue.pushBack(*sentential);
+			auto copy = &queue.last()->data;
+
+			auto appendTerm = copy->terms.findIdx(termIdx);
+			auto deleteTerm = appendTerm;
+			for (auto arg : production->args) {
+				if (arg->isEpsilon) continue;
+				auto newTerm = copy->terms.newNode({ arg->id, arg->isTerminal });
+				copy->terms.attach(newTerm, appendTerm);
+				appendTerm = newTerm;
+			}
+			copy->terms.removeNode(deleteTerm);
+		}
+
+		isSentence = false;
+		termIdx++;
+	}
+
+	if (isSentence) {
+		auto exists = false;
+		for (auto iter : out) {
+			if (iter->terms == sentential->terms) {
+				exists = true;
+				break;
+			}
+		}
+		if (!exists) out.pushBack(*sentential);
+	}
+
+	queue.popFront();
+
+	bool stop = false;
+	stop |= !queue.length();
+	stop |= timer.isTimeout();
+	stop |= queue.length() > maxQueue;
+	stop |= out.length() > maxSentences;
+
+	if (!stop) goto PASS;
+}
+
+void CfGrammar::printSentence(Sentence& in) {
+	printf("[");
+	for (auto term : in.terms) {
+		printf(" %s", term->id.read());
+	}
+	printf(" ]\n");
+}
+
+bool CfGrammar::compile() {
+
+	if (!rules.length()) {
+		printf("Grammar must have at leas one rule\n");
+		return false;
+	}
+
+	if (startTerminal == String()) {
+		printf("Using first rule's non-terminal as grammar's root. Define explicitly - 'Start : id'\n");
+		startTerminal = rules.first()->data.id;
+	}
+
+	// find all rules
+	for (auto rule : rules) {
+
+		if (!rule->args.length()) {
+			printf("Rule must have at leas one terminal or non-terminal. See rule '%s'\n", rule->id.read());
+			return false;
+		}
+
+		if (!mNonTerminals.presents(rule->id)) {
+			mNonTerminals.put(rule->id, {});
+		}
+		auto nonTerminal = &mNonTerminals.get(rule->id);
+
+		nonTerminal->rules.pushBack(&rule.data());
+	}
+
+	for (auto nonTerminal : mNonTerminals) {
+		for (auto rule : nonTerminal->val.rules) {
+			for (auto arg : rule->args) {
+
+				if (arg->isTerminal) {
+					mTerminals.put(rule->id, {});
+				} else if (!arg->isEpsilon) {
+					auto idx = mNonTerminals.presents(arg->id);
+					if (!idx) {
+						printf("Referenced non-terminal '%s' is not defined\n", arg->id.read());
+						return false;
+					}
+					auto reference = &mNonTerminals.getSlotVal(idx);
+					reference->references.put(nonTerminal->key, &nonTerminal->val);
+					nonTerminal->val.referencing.put(arg->id, reference);
+				}
+			}
+		}
+	}
+
+	List<String> remove;
+	for (auto nonTerminal : mNonTerminals) {
+		if (!nonTerminal->val.references.size() && nonTerminal->key != startTerminal) {
+			printf("Non-terminal '%s' is defined but not used\n", nonTerminal->key.read());
+			remove.pushBack(nonTerminal->key);
+		}
+	}
+
+	for (auto rem : remove) {
+		auto nonTerminal = &mNonTerminals.get(rem.data());
+		for (auto referencing : nonTerminal->referencing) {
+			referencing->val->references.remove(nonTerminal->rules.first()->data->id);
+		}
+		for (auto rule : nonTerminal->rules) {
+			rules.removeNode(rules.find(*rule.data()));
+		}
+	}
+
+	for (auto nonTerminal : mNonTerminals) {
+		if (!nonTerminal->val.isProductive()) {
+			printf("Non-terminal '%s' is not productive\n", nonTerminal->val.rules.first()->data->id.read());
+			return false;
+		}
+	}
+
+	Map<String, ualni> processed;
+	if (mNonTerminals.get(startTerminal).isLooped(processed, startTerminal)) {
+		printf("Note that grammar is looped.\n");
+	}
+
+	return true;
+}

Language/old/Parser/private/CfGrammarParser.cpp

@@ -0,0 +1,201 @@
+
+#include "CfGrammar.hpp"
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+enum class Token {
+	NO_TOKEN,
+	START,
+	ID,
+	EQUAL,
+	TERMINAL_START,
+	TERMINAL_END,
+	ALTERNATION,
+	RULE_END,
+	IGNORED,
+	EPSILON,
+	COMMENT,
+	END,
+	FAILED,
+};
+
+typedef int1 AlphabetType;
+typedef SimpleTokenizer<AlphabetType, Token, Token::NO_TOKEN, Token::FAILED, Token::END> CFGTokenizer;
+
+struct State {
+
+	struct Transition {
+		Token tok = Token::FAILED;
+		State* target = nullptr;
+		void (*action)(CfGrammar* grammar, const String& tok) = nullptr;
+	};
+
+	String name;
+	String error;
+	Buffer<Transition> transitions;
+	bool accept = false;
+};
+
+typedef Buffer<State> Automata;
+
+void initializeTokenizer(CFGTokenizer* tok) {
+	tok->build({
+		{ CFGTokenizer::tTokenizer::etherRE, Token::IGNORED },
+		{ "Start", Token::START },
+		{ ":", Token::EQUAL },
+		{ "\\[", Token::TERMINAL_START },
+		{ "\\]", Token::TERMINAL_END },
+		{ "\\|", Token::ALTERNATION },
+		{ ";", Token::RULE_END },
+		{ "&", Token::EPSILON },
+		{ CFGTokenizer::tTokenizer::idRE, Token::ID },
+		{ CFGTokenizer::tTokenizer::commentBlockRE, Token::COMMENT },
+	});
+
+	ASSERT(tok->isBuild())
+}
+
+void initAutomata(Automata& automata) {
+	automata.reserve(7);
+	auto states = &automata.first();
+
+	auto root = states++;
+	auto end = states++;
+	auto start = states++;
+	auto rule = states++;
+	auto rhs = states++;
+	auto terminalStart = states++;
+	auto terminalEnd = states;
+
+	root->transitions = {
+		{ Token::END, end },
+		{ Token::START, start },
+		{ Token::ID,
+			rule,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.pushBack({ tok, {} });
+			} },
+	};
+
+	start->transitions = {
+		{ Token::ID, root, [](CfGrammar* grammar, const String& tok) { grammar->startTerminal = tok; } },
+	};
+
+	rule->transitions = {
+		{ Token::EQUAL, rhs },
+	};
+
+	rhs->transitions = {
+		{ Token::ALTERNATION,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				auto const& id = grammar->rules.last()->data.id;
+				grammar->rules.pushBack({ id, {} });
+			} },
+		{ Token::ID,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, false, false });
+			} },
+		{ Token::EPSILON,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, false, true });
+			} },
+		{ Token::TERMINAL_START, terminalStart },
+		{ Token::RULE_END, root },
+	};
+
+	terminalStart->transitions = {
+		{ Token::ID,
+			terminalEnd,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, true, false });
+			} },
+	};
+
+	terminalEnd->transitions = {
+		{ Token::TERMINAL_END, rhs },
+	};
+
+	auto idError = "Expected an identifier";
+	auto stmEndError = "Expected an statement end ';'";
+
+	root->error = "Expected 'Start' statement, rule or end of text";
+	start->error = idError;
+	rule->error = "Expected production equality ':'";
+	rhs->error = "Expected alternation '|', terminal id, statement end ';' or terminal start '[' ";
+	terminalStart->error = idError;
+	terminalEnd->error = "Expected terminal end '[' ";
+
+	root->name = "root";
+	start->name = "start";
+	rule->name = "production";
+	rhs->name = "rhs";
+	terminalStart->name = "terminalStart";
+	terminalEnd->name = "terminalEnd";
+
+	end->accept = true;
+}
+
+bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGrammar* grammar) {
+	tok->reset();
+	tok->bindSource(text);
+
+	State* state = &(*automata)[0];
+	while (!state->accept) {
+
+		auto token = tok->readTok();
+
+		if (token == Token::IGNORED || token == Token::COMMENT) {
+			continue;
+		}
+
+		if (token == Token::FAILED) {
+			auto errorLocation = tok->getCursorPrev().get2DLocation();
+			printf("Syntax error at (%llu, %llu)\n", errorLocation.line, errorLocation.character);
+			return false;
+		}
+
+		auto tokVal = tok->extractVal();
+
+		bool isTransition = false;
+
+		for (auto transition : state->transitions) {
+			if (transition->tok == token) {
+				if (transition->action) transition->action(grammar, tokVal);
+				state = transition->target;
+				isTransition = true;
+				break;
+			}
+		}
+
+		if (!isTransition) {
+			// if no tok matched to any transition produce error
+			auto errorLocation = tok->getCursorPrev().get2DLocation();
+			printf("Parse error at (%llu, %llu) - %s", errorLocation.line, errorLocation.character, state->error.read());
+			return false;
+		}
+	}
+
+	return true;
+}
+
+struct tp::CfGrammarParserState {
+	Automata automata;
+	CFGTokenizer tokenizer;
+};
+
+CfGrammarParserState* CfGrammar::initializeCfGrammarParser() {
+	auto state = new CfGrammarParserState();
+	initializeTokenizer(&state->tokenizer);
+	initAutomata(state->automata);
+	return state;
+}
+
+void CfGrammar::deinitializeCfGrammarParser(CfGrammarParserState* state) { delete state; }
+
+bool CfGrammar::parse(CfGrammarParserState* context, const String& source) { return ::parse(source.read(), &context->automata, &context->tokenizer, this); }
+
+bool CfGrammar::isLooped() const { return mIsLooped; }

Language/old/Parser/private/Parser.cpp

@@ -0,0 +1,7 @@
+#include "Parser.hpp"
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+static ModuleManifest* sModuleDependencies[] = { &gModuleTokenizer, nullptr };
+ModuleManifest tp::gModuleParser = ModuleManifest("CommandLine", nullptr, nullptr, sModuleDependencies);

Language/old/Parser/public/CLR.hpp

@@ -0,0 +1,62 @@
+#pragma once
+
+#include "CfGrammar.hpp"
+
+namespace tp {
+	class CLR {
+	public:
+		struct ASTNode {
+			const CfGrammar::Rule* production = nullptr;
+			Map<String, String> terminals;
+		};
+
+		class Automation {
+		public:
+			class State {};
+		};
+
+		class TerminalStream {
+		public:
+			typedef ualni TerminalID;
+
+		public:
+			TerminalStream() = default;
+			virtual TerminalID getNextTerminal() = 0;
+		};
+
+	public:
+		struct BuildError {
+			String description;
+		};
+
+		struct ParseError {
+			const Automation::State* state = nullptr;
+			String::Index location = 0;
+		};
+
+	private:
+		class SententialStack {};
+
+	public:
+		CLR() = default;
+
+		void setGrammar(const CfGrammar& grammar);
+		void setTerminal(const String& name, TerminalStream::TerminalID id);
+		void build();
+
+		void parse(TerminalStream* stream, List<ASTNode>& out, ASTNode** root);
+
+		[[nodiscard]] bool isBuild() { return mState == ParserState::BUILD; }
+		[[nodiscard]] const BuildError& getBuildError() { return mBuildError; }
+		[[nodiscard]] const ParseError& getParseError() { return mParseError; }
+
+	private:
+		enum class ParserState { NONE, BUILD, FAILED } mState = ParserState::NONE;
+
+		ParseError mParseError;
+		BuildError mBuildError;
+
+		SententialStack mStack;
+		Automation mAutomation;
+	};
+}

Language/old/Parser/public/CfGrammar.hpp

@@ -0,0 +1,89 @@
+#pragma once
+
+#include "List.hpp"
+#include "Map.hpp"
+#include "Strings.hpp"
+
+namespace tp {
+
+	struct CfGrammar {
+		struct Rule {
+			struct Arg {
+				String id;
+				bool isTerminal = false;
+				bool isEpsilon = false;
+
+				bool operator==(const Arg& in) const { return (id == in.id) && (isEpsilon == in.isEpsilon) && (isTerminal == in.isTerminal); }
+			};
+
+			String id;
+			List<Arg> args;
+
+			bool operator==(const Rule& in) const { return (id == in.id) && (args == in.args); }
+
+			[[nodiscard]] bool isProductive() const {
+				for (auto arg : args) {
+					if (arg->id == id) return false;
+				}
+				return true;
+			}
+		};
+
+		struct Sentence {
+			struct Term {
+				String id;
+				bool terminal;
+				bool operator==(const Term& in) const { return (id == in.id) && (terminal == in.terminal); }
+			};
+			List<Term> terms;
+		};
+
+		List<Rule> rules;
+		String startTerminal;
+
+	private:
+		struct NonTerminal {
+
+			List<Rule*> rules;
+			Map<String, NonTerminal*> references;
+			Map<String, NonTerminal*> referencing;
+
+			[[nodiscard]] bool isProductive() const {
+				for (auto rule : rules) {
+					if (rule->isProductive()) return true;
+				}
+				return false;
+			}
+
+			[[nodiscard]] bool isLooped(Map<String, ualni>& processed, const String& id) const {
+				for (auto ref : referencing) {
+					if (processed.presents(ref->key)) return true;
+				}
+				processed.put(id, {});
+				for (auto ref : referencing) {
+					if (ref->val->isLooped(processed, ref->key)) return true;
+				}
+				return false;
+			}
+		};
+
+		struct Terminal {};
+
+		Map<String, Terminal> mTerminals;
+		Map<String, NonTerminal> mNonTerminals;
+		bool mIsLooped = false;
+
+	public:
+		static struct CfGrammarParserState* initializeCfGrammarParser();
+		static void deinitializeCfGrammarParser(CfGrammarParserState*);
+
+	public:
+		bool parse(CfGrammarParserState* context, const String& source);
+		bool compile();
+
+		void generateSentences(List<Sentence>& out);
+		static void printSentence(Sentence& in);
+
+		[[nodiscard]] bool isLooped() const;
+	};
+}

Language/old/Parser/public/Parser.hpp

@@ -0,0 +1,7 @@
+#pragma once
+
+#include "CLR.hpp"
+
+namespace tp {
+	extern ModuleManifest gModuleParser;
+}

Language/old/Parser/rsc/grammar.txt

@@ -0,0 +1,20 @@
+
+/*
+Sample grammar.
+This grammar wil produce text in the form: TODO
+*/
+
+/* This is the starting production of the grammar */
+
+/* Simple Example
+Start A
+A : [ ID ] | &;
+*/
+
+/* Lists Example */
+
+Start List
+
+List : List Stm | Stm;
+Stm : StmBody [ END ];
+StmBody: [ Stm1 ] | [ Stm2 ] | [ Stm3 ];

Language/old/Parser/tests/CLRTests.cpp

@@ -0,0 +1,9 @@
+
+#include "Parser.hpp"
+#include "Testing.hpp"
+
+using namespace tp;
+
+TEST_DEF(CLR) {
+	// TEST(false);
+}

Language/old/Parser/tests/GrammarTests.cpp

@@ -0,0 +1,48 @@
+
+#include "Parser.hpp"
+#include "Testing.hpp"
+
+using namespace tp;
+
+TEST_DEF(Grammar) {
+
+	String source = R"(
+Start Body
+Body : [ Function ] | List | &;
+List : [ LIST ];
+)";
+
+	auto state = CfGrammar::initializeCfGrammarParser();
+
+	CfGrammar grammar;
+
+	if (!grammar.parse(state, source)) {
+		TEST(0 && "Parsing is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	if (!grammar.compile()) {
+		TEST(0 && "Compilation is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	CfGrammar::deinitializeCfGrammarParser(state);
+
+	printf("Grammar accepted.\n");
+
+	List<CfGrammar::Sentence> sentences;
+	grammar.generateSentences(sentences);
+
+	TEST_ASSERT(sentences.length() == 3);
+	TEST_ASSERT(sentences.first()->data.terms.length() == 1);
+	TEST_ASSERT(sentences.last()->data.terms.length() == 1);
+
+	TEST(sentences.first()->data.terms.first()->data.id == "Function");
+	TEST(sentences.last()->data.terms.first()->data.id == "LIST");
+
+	printf("Example sentences formed from grammar: \n");
+	for (auto sentence : sentences)
+		tp::CfGrammar::printSentence(sentence.data());
+}

Language/old/Parser/tests/Tests.cpp

@@ -0,0 +1,23 @@
+
+#include "Parser.hpp"
+#include "Testing.hpp"
+
+using namespace tp;
+
+void testGrammar();
+void testCLR();
+
+int main(int argc, const char* argv[]) {
+	tp::ModuleManifest* deps[] = { &tp::gModuleParser, nullptr };
+	tp::ModuleManifest testModule("CommandLineTest", nullptr, nullptr, deps);
+
+	if (!testModule.initialize()) {
+		return 1;
+	}
+
+	testGrammar();
+	testCLR();
+
+	testModule.deinitialize();
+	return 0;
+}

Language/old/Tokenizer/CMakeLists.txt

@@ -0,0 +1,15 @@
+project(Tokenizer)
+
+### ---------------------- Static Library  ---------------------  ###
+file(GLOB SOURCES "./private/*.cpp")
+file(GLOB HEADERS "./public/*.hpp")
+add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
+target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
+target_link_libraries(${PROJECT_NAME} PUBLIC Strings)
+
+### -------------------------- Tests  --------------------------  ###
+enable_testing()
+file(GLOB TEST_SOURCES "./tests/*.cpp")
+add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES})
+target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils)
+add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests)

Language/old/Tokenizer/private/Tokenizer.cpp

@@ -0,0 +1,10 @@
+
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+static ModuleManifest* sModuleDependencies[] = { &tp::gModuleStrings, nullptr };
+
+void deinit(const ModuleManifest*) {}
+
+ModuleManifest tp::gModuleTokenizer = ModuleManifest("Tokenizer", nullptr, deinit, sModuleDependencies);

Language/old/Tokenizer/public/AutomataGraph.h

@@ -0,0 +1,525 @@
+
+#pragma once
+
+#include "List.hpp"
+#include "Buffer.hpp"
+#include "Buffer2D.hpp"
+#include "Utils.hpp"
+#include "Map.hpp"
+
+#ifdef ENV_OS_WINDOWS
+#undef min
+#undef max
+#endif
+
+namespace tp {
+
+	extern ModuleManifest gModuleTokenizer;
+
+	template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
+	class TransitionMatrix;
+
+	template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
+	class DFA;
+
+	// Non-Deterministic Finite-State Automata
+	template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
+	class NFA {
+		static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable.");
+
+	public:
+		struct Vertex;
+
+	private:
+		struct Edge {
+			Vertex* mVertex = nullptr;
+			bool mConsumesSymbol = false;
+			Range<tAlphabetType> mAcceptingRange;
+			bool mAcceptsAll = false;
+			bool mExclude = false;
+
+			bool isTransition(const tAlphabetType& symbol) {
+				if (symbol == 0) return false;
+				if (!mConsumesSymbol || mAcceptsAll) return true;
+				bool const in_range = (symbol >= mAcceptingRange.mBegin && symbol <= mAcceptingRange.mEnd);
+				return in_range != mExclude;
+			}
+		};
+
+	public:
+		struct Vertex {
+			List<Edge> edges;
+			tStateType termination_state = tNoStateVal;
+			#ifdef ENV_BUILD_DEBUG
+				ualni debug_idx = 0;
+			#endif
+			ualni flag = 0;
+		};
+
+	private:
+		friend DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>;
+
+		List<Vertex> mVertices;
+		Vertex* mStart = nullptr;
+
+	public:
+		NFA() = default;
+
+		Vertex* addVertex() {
+			auto node = mVertices.newNode();
+			#ifdef ENV_BUILD_DEBUG
+				node->data.debug_idx = mVertices.length() + 1;
+			#endif
+			mVertices.pushBack(node);
+			return &node->data;
+		}
+
+		void addTransition(Vertex* from, Vertex* to, Range<tAlphabetType> range, bool consumes, bool accepts_all, bool exclude) {
+			Edge edge; 
+			edge.mVertex = to;
+			edge.mConsumesSymbol = consumes;
+			edge.mAcceptingRange = range;
+			edge.mExclude = exclude;
+			edge.mAcceptsAll = accepts_all;
+			from->edges.pushBack(edge);
+		}
+
+		void setStartVertex(Vertex* start) {
+			mStart = start;
+		}
+
+		[[nodiscard]] Vertex* getStartVertex() const {
+			return mStart;
+		}
+
+		void setVertexState(Vertex* vertex, tStateType state) {
+			vertex->termination_state = state;
+		}
+
+		[[nodiscard]] bool isValid() const {
+			if (!mStart) {
+				return false;
+			}
+			return true;
+		}
+
+		Range<tAlphabetType> getAlphabetRange() const {
+			tAlphabetType start = 0, end = 0;
+			Range<tAlphabetType> all_range(std::numeric_limits<tAlphabetType>::min(), std::numeric_limits<tAlphabetType>::max());
+			bool first = true;
+
+			for (auto vertex : mVertices) {
+				for (auto edge : vertex.data().edges) {
+					if (!edge.data().mConsumesSymbol) continue;
+					auto const& tran_range = edge.data().mAcceptingRange;
+					if (edge.data().mAcceptsAll || edge.data().mExclude) return all_range;
+					if (tran_range.mBegin < start || first) start = tran_range.mBegin;
+					if (tran_range.mEnd > end || first) end = tran_range.mEnd;
+					first = false;
+				}
+			}
+
+			return Range<tAlphabetType>( start, end + 1 );
+		}
+
+		// vertices that are reachable from initial set with no input consumption (E-transitions)
+		// does not include initial set
+		void closure(const List<Vertex*>& set, List<Vertex*>& closure, ualni unique_call_id) {
+			List<Vertex*> marked;
+			marked = set;
+
+			while (marked.length()) {
+				auto first = marked.first()->data;
+				if (first->flag != unique_call_id) {
+					first->flag = unique_call_id;
+					closure.pushBack(first);
+				}
+				for (auto edge : first->edges) {
+					if (!edge.data().mConsumesSymbol && edge.data().mVertex->flag != unique_call_id) {
+						marked.pushBack(edge.data().mVertex);
+					}
+				}
+				marked.popFront();
+			}
+		}
+
+		// vertices that are reachable from initial set with symbol transition
+		void move(const List<Vertex*>& set, List<Vertex*>& reachable, tAlphabetType symbol, ualni unique_call_id) {
+			for (auto vertex : set) {
+				for (auto edge : vertex->edges) {
+					if (!edge.data().mConsumesSymbol) continue;
+					bool transition = edge.data().isTransition(symbol);
+					if (transition && edge.data().mVertex->flag != unique_call_id) {
+						edge.data().mVertex->flag = unique_call_id;
+						reachable.pushBack(edge.data().mVertex);
+					}
+				}
+			}
+		}
+	};
+
+	// Deterministic Finite-State Automata
+	template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
+	class DFA {
+		static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable.");
+		friend TransitionMatrix<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>;
+
+		struct Vertex {
+			struct Edge {
+				Vertex* vertex = nullptr;
+				tAlphabetType transition_code = nullptr;
+			};
+
+			List<Edge> edges;
+			tStateType termination_state = tNoStateVal;
+			bool marked = false;
+		};
+
+		List<Vertex> mVertices;
+		Vertex* mStart = nullptr;
+		const Vertex* mIter = nullptr;
+		Range<tAlphabetType> mAlphabetRange;
+		bool mTrapState = false;
+
+		typedef typename NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>::Vertex NState;
+
+	public:
+
+		explicit DFA(NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& nfa) {
+			if (!nfa.isValid()) {
+				return;
+			}
+			
+			mAlphabetRange = nfa.getAlphabetRange();
+
+			struct DStateKey {
+				const List<NState*>* nStates;
+				bool operator==(const DStateKey& in) const {
+					if (nStates->length() != in.nStates->length()) {
+						return false;
+					}
+					// FIXME : make linear time
+					for (auto state : *nStates) {
+						bool found = false;
+						for (auto in_state : *in.nStates) {
+							if (state.data() == in_state.data()) {
+								found = true;
+								break;
+							}
+						}
+
+						if (!found) {
+							return false;
+						}
+					}
+
+					return true;
+				}
+
+				static ualni dStateHashFunc(DStateKey key) {
+					alni out = 0;
+					for (auto state : *key.nStates) {
+						out += alni(state.data());
+					}
+					return out;
+				};
+			};
+
+			// all NFA states that are reachable from initial DFA State for specific symbol in alphabet
+			struct DState {
+				struct DTransition {
+					DState* state = nullptr;
+					tAlphabetType accepting_code;
+				};
+
+				List<NState*> nStates;
+				List<DTransition> transitions;
+
+				Vertex* dVertex= nullptr; // relevant DFA vertex
+
+				#ifdef ENV_BUILD_DEBUG
+				ualni debug_idx = 0;
+				#endif
+			};
+
+			// includes closure of NFA start state by definition
+			auto start_state = new DState();
+			nfa.closure({ nfa.getStartVertex() }, start_state->nStates, ualni(start_state));
+
+			Map<DStateKey, DState*, DefaultAllocator, DStateKey::dStateHashFunc, 256> dStates;
+
+			dStates.put({ &start_state->nStates }, start_state);
+
+			List<DState*> working_set = { start_state };
+
+			// while there is items to work with
+			auto currentDState = working_set.first();
+			while (currentDState) {
+
+				// check all possible transitions for any symbol
+				for (auto symbol : mAlphabetRange) {
+
+					List<NState*> reachableNStates;
+
+					nfa.move(currentDState->data->nStates, reachableNStates, symbol, ualni(currentDState + symbol));
+					nfa.closure(reachableNStates, reachableNStates, ualni(currentDState + symbol));
+
+					if (!reachableNStates.length()) {
+						continue;
+					}
+
+					DState* targetDState = nullptr;
+
+					// check if set of all reachable NFA states already forms existing DFA state
+					auto idx = dStates.presents({ &reachableNStates});
+					if (idx) {
+						targetDState = dStates.getSlotVal(idx);
+					}
+					
+					if (!targetDState) {
+						// register new DFA state
+						targetDState = new DState();
+
+						#ifdef ENV_BUILD_DEBUG
+							targetDState->debug_idx = dStates.size();
+						#endif
+
+						targetDState->nStates = reachableNStates;
+
+						// append to working stack
+						working_set.pushBack(targetDState);
+						dStates.put({ &targetDState->nStates }, targetDState);
+					}
+
+					// add transition to DFA state
+					currentDState->data->transitions.pushBack({targetDState, symbol });
+				}
+
+				working_set.popFront();
+				currentDState = working_set.first();
+			}
+
+			// create own vertices
+			for (auto node : dStates) {
+				tStateType state = tNoStateVal;
+				for (auto iter : node->val->nStates) {
+					if (iter->termination_state != tNoStateVal) {
+						state = iter->termination_state;
+						break;
+					}
+				}
+				node->val->dVertex = addVertex(state);
+			}
+
+			// connect all vertices
+			for (auto node : dStates) {
+				for (auto edge : node->val->transitions) {
+					addTransition(node->val->dVertex, edge.data().state->dVertex, edge.data().accepting_code);
+				}
+			}
+
+			// set the starting vertex
+			mStart = start_state->dVertex;
+
+			// cleanup
+			for (auto node : dStates) {
+				delete node->val;
+			}
+
+			collapseEquivalentVertices();
+			mAlphabetRange = getAlphabetRange();
+		}
+
+		tStateType move(tAlphabetType symbol) {
+			if (mTrapState || !mIter) {
+				return tNoStateVal;
+			}
+
+			for (auto edge : mIter->edges) {
+				if (edge.data().transition_code == symbol) {
+					mIter = edge.data().vertex;
+					return mIter->termination_state;
+				}
+			}
+
+			mTrapState = true;
+			return tNoStateVal;
+		}
+
+		void start() {
+			mIter = mStart;
+			mTrapState = false;
+		}
+
+		[[nodiscard]] uhalni nVertices() const {
+			return (uhalni) mVertices.length();
+		}
+		
+		[[nodiscard]] Range<tAlphabetType> getRange() const {
+			return mAlphabetRange;
+		}
+
+	private:
+		[[nodiscard]] Range<tAlphabetType> getAlphabetRange() const {
+			Range<tAlphabetType> out;
+			for (auto vertex : mVertices) {
+				vertex.data().marked = false;
+			}
+
+			bool first = true;
+			getAlphabetRangeUtil(mStart, &out, first);
+			out.mEnd++;
+			return out;
+		}
+
+		void getAlphabetRangeUtil(Vertex* vert, Range<tAlphabetType>* out, bool& first) const {
+			vert->marked = true;
+			for (auto edge : vert->edges) {
+				auto const code = edge.data().transition_code;
+
+				if (first) {
+					*out = { code, code };
+					first = false;
+				}
+
+				if (code < out->mBegin) {
+					out->mBegin = code;
+				}
+				if (code > out->mEnd) {
+					out->mEnd = code;
+				}
+
+				if (!edge.data().vertex->marked) {
+					getAlphabetRangeUtil(edge.data().vertex, out, first);
+				}
+			}
+		}
+
+		void collapseEquivalentVertices() {
+			// TODO
+		}
+
+		Vertex* addVertex(tStateType state) {
+			auto node = mVertices.addNodeBack();
+			node->data.termination_state = state;
+			return &node->data;
+		}
+
+		void addTransition(Vertex* from, Vertex* to, tAlphabetType transition_symbol) {
+			from->edges.pushBack({ to, transition_symbol });
+		}
+	};
+
+	template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
+	class TransitionMatrix {
+
+		static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable.");
+
+		Buffer2D<ualni> mTransitions;
+		Buffer<tStateType> mStates;
+		Range<tAlphabetType> mSymbolRange = { 0, 0 };
+		
+		ualni mIter = 0;
+		ualni mIterPrev = 0;
+		ualni mStart = 0;
+
+	public:
+
+		TransitionMatrix() = default;
+
+		auto getStates() const { return &mStates; }
+		auto getTransitions() const { return &mTransitions; }
+		auto getStart() const { return mStart; }
+
+		void construct(const DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& dfa) {
+			mSymbolRange = dfa.getRange();
+			auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin);
+			auto sizeX = range_len ? range_len : 1;
+			auto sizeY =  (ualni) (dfa.nVertices() + 1);
+
+			mTransitions.reserve({ sizeX, sizeY });
+			mTransitions.assign(dfa.nVertices());
+			mStates.reserve(sizeY);
+
+			ualni idx = 0;
+			for (auto vertex : dfa.mVertices) {
+				auto state = vertex.data().termination_state;
+				mStates[idx] = state;
+				idx++;
+			}
+
+			mStates[dfa.nVertices()] = tFailedStateVal;
+
+			idx = 0;
+			for (auto vertex : dfa.mVertices) {
+				if (&vertex.data() == dfa.mStart) {
+					mStart = mIter = mIterPrev = idx;
+				}
+				idx++;
+			}
+
+			ualni vertexIdx = 0;
+			for (auto vertex : dfa.mVertices) {
+				for (auto edge : vertex.data().edges) {
+					ualni vertex2Idx = 0;
+					for (auto vertex2 : dfa.mVertices) {
+						if (edge.data().vertex == &vertex2.data()) break;
+						vertex2Idx++;
+					}
+					auto const code = edge.data().transition_code;
+					mTransitions.set( { (ualni) (code - mSymbolRange.mBegin), (ualni) vertexIdx }, vertex2Idx);
+				}
+				vertexIdx++;
+			}
+		}
+
+		bool isTrapped() {
+			return mStates[mIter] == tFailedStateVal;
+		}
+
+		tStateType move(tAlphabetType symbol) {
+			if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) {
+				mIter = mTransitions.get({ (ualni) (symbol - mSymbolRange.mBegin), (ualni) mIter });
+			}
+			else {
+				mIter = mStates.size() - 1;
+			}
+			
+			if (mIterPrev == mStart) {
+				if (mStates[mIter] == tFailedStateVal) {
+					reset();
+					return tFailedStateVal;
+				}
+				else {
+					mIterPrev = mIter;
+					return tNoStateVal;
+				}
+			}
+			else {
+				if (mStates[mIter] == tFailedStateVal) {
+					if (mStates[mIterPrev] != tNoStateVal) {
+						auto out = mStates[mIterPrev];
+						reset();
+						return out;
+					}
+					else {
+						reset();
+						return tFailedStateVal;
+					}
+				}
+				else {
+					mIterPrev = mIter;
+					return tNoStateVal;
+				}
+			}
+
+			mIterPrev = mIter;
+			return mStates[mIter];
+		}
+
+		void reset() {
+			mIter = mStart;
+			mIterPrev = mStart;
+		}
+	};
+}

Language/old/Tokenizer/public/RegularExpression.h

@@ -0,0 +1,519 @@
+
+#pragma once
+
+#include "AutomataGraph.h"
+
+namespace tp {
+	extern ModuleManifest gModuleTokenizer;
+}
+
+namespace tp::RegEx {
+		
+		struct AstNode {
+			enum Type {
+				NONE,
+				ANY,
+				OR,
+				IF,
+				CLASS,
+				COMPOUND,
+				REPEAT,
+				VAL,
+			} mType = NONE;
+
+			AstNode() = default;
+			virtual ~AstNode() = default;
+		};
+
+		template <typename tAlphabetType>
+		struct AstVal : public AstNode {
+			explicit AstVal(tAlphabetType val) : mVal(val) { mType = VAL; }
+			~AstVal() override = default;
+			tAlphabetType mVal;
+		};
+
+		struct AstCompound : public AstNode {
+			AstCompound() { mType = COMPOUND; }
+			~AstCompound() override {
+				for (auto iter : mChilds) {
+					delete iter.data();
+				}
+				mChilds.removeAll();
+			}
+			List<AstNode*> mChilds;
+		};
+
+		struct AstAlternation : public AstNode {
+			AstAlternation() { mType = OR;	}
+			~AstAlternation() override {
+				delete mFirst;
+				delete mSecond;
+			}
+			AstNode* mFirst = nullptr;
+			AstNode* mSecond = nullptr;
+		};
+
+		struct AstIf : public AstNode {
+			AstIf() { mType = IF; }
+			~AstIf() override { delete mNode; }
+			AstNode* mNode = nullptr;
+		};
+
+		struct AstAny : public AstNode {
+			AstAny() { mType = ANY; }
+			~AstAny() override = default;
+		};
+
+		struct AstRepetition : public AstNode {
+			AstRepetition() { mType = REPEAT; }
+			~AstRepetition() override { delete mNode; }
+			AstNode* mNode = nullptr;
+			bool mPlus = false;
+		};
+
+		template <typename tAlphabetType>
+		struct AstClass : public AstNode {
+			AstClass() { mType = CLASS; }
+			~AstClass() override { mRanges.removeAll(); }
+			List<Range<tAlphabetType>> mRanges;
+			bool mExclude = false;
+		};
+
+		struct ParseError {
+			const char* description = nullptr;
+			uhalni offset = 0;
+			[[nodiscard]] bool isError() const { return description != nullptr; }
+		};
+
+		template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal>
+		class Parser {
+
+			enum TokType : uint1 {
+				TOK_COMPOUND_START = 0,
+				TOK_COMPOUND_END,
+				TOK_CLASS_START,
+				TOK_CLASS_END,
+				TOK_CLASS_START_EXCLUDE,
+				TOK_CLASS_END_EXCLUDE,
+				TOK_OR,
+				TOK_IF,
+				TOK_ANY,
+				TOK_REPEAT,
+				TOK_REPEAT_PLUS,
+				TOK_HYPHEN,
+				TOK_SPECIALS_END_,
+				TOK_VAL,
+				TOK_NONE,
+			};
+
+			tAlphabetType SpecialSymbols[TOK_SPECIALS_END_] = {
+				'(', ')', '[', ']', '{', '}', '|', '?', '.', '*', '+', '-',
+			};
+
+			tAlphabetType mEscapeSymbol = '\\';
+
+			struct Token {
+				TokType type;
+				tAlphabetType val;
+			};
+
+			const tAlphabetType* mSource = nullptr;
+			uhalni mOffset = 0;
+			Token mCurToken;
+			uhalni mTokLength = 0;
+
+		public:
+
+			ParseError mError;
+
+			// regular expression must be a zero termination string
+			AstCompound* parse(const tAlphabetType* regex) {
+				mSource = regex;
+				return parseRegEx();
+			}
+
+		private:
+
+			AstCompound* parseRegEx() {
+				auto out = new AstCompound();
+				for (AstNode* node = parseElement(); node; node = parseElement()) {
+					out->mChilds.pushBack(node);
+				}
+				if (!out->mChilds.length()) {
+					genError("Expected A Expression");
+				}
+				if (mError.description) {
+					delete out;
+					return nullptr;
+				}
+				return out;
+			}
+
+			AstNode* parseElement() {
+				AstNode* out = nullptr;
+				switch (readTok().type) {
+					case TOK_COMPOUND_START: out = parseCompound(); break;
+					case TOK_CLASS_START: out = parseClass(); break;
+					case TOK_CLASS_START_EXCLUDE: out = parseClass(true); break;
+					case TOK_ANY: out = parseAny(); break;
+					case TOK_VAL: out = parseVal(); break;
+					case TOK_NONE: { discardTok(); return nullptr; };
+					default: break;
+				}
+				if (!out) {
+					discardTok();
+					return nullptr;
+				}
+				switch (readTok().type) {
+					case TOK_OR: out = parseAlternation(out); break;
+					case TOK_REPEAT: out = parseRepetition(out); break;
+					case TOK_REPEAT_PLUS: out = parseRepetition(out, true); break;
+					case TOK_IF: out = parseIf(out); break;
+					case TOK_NONE: break;
+					default: { discardTok(); }
+				}
+				return out;
+			}
+
+			AstCompound* parseCompound() {
+				auto out = new AstCompound();
+				for (AstNode* node = parseElement(); node; node = parseElement()) {
+					out->mChilds.pushBack(node);
+				}
+				if (readTok().type != TOK_COMPOUND_END) {
+					genError("Expected Compound End");
+				}
+				if (mError.description) {
+					delete out;
+					return nullptr;
+				}
+				return out;
+			}
+
+			AstClass<tAlphabetType>* parseClass(bool exclude = false) {
+				auto out = new AstClass<tAlphabetType>();
+				out->mExclude = exclude;
+				auto& ranges = out->mRanges;
+
+				readTok();
+
+			READ_VAL:
+				if (mCurToken.type != TOK_VAL) {
+					delete out;
+					genError("Expected A Value");
+					return nullptr;
+				}
+				char range_start = mCurToken.val;
+
+				readTok();
+				if (mCurToken.type != TOK_HYPHEN) {
+					delete out;
+					genError("Expected A Range");
+					return nullptr;
+				}
+
+				readTok();
+				if (mCurToken.type != TOK_VAL) {
+					delete out;
+					genError("Expected A Value");
+					return nullptr;
+				}
+				char range_end = mCurToken.val;
+
+				ranges.pushBack({ range_start, range_end });
+
+				readTok();
+				if ((mCurToken.type == TOK_CLASS_END && !exclude) || (mCurToken.type == TOK_CLASS_END_EXCLUDE && exclude)) {
+					return out;
+				}
+				else {
+					goto READ_VAL;
+				}
+			}
+
+			AstAny* parseAny() {
+				return new AstAny();
+			}
+
+			AstVal<tAlphabetType>* parseVal() {
+				auto out = new AstVal<tAlphabetType>(mCurToken.val);
+				return out;
+			}
+
+			AstAlternation* parseAlternation(AstNode* left) {
+				auto right = parseElement();
+				if (!right) {
+					genError("Expected Alternation right Side");
+					delete left;
+					return nullptr;
+				}
+
+				auto out = new AstAlternation();
+
+				out->mFirst = left;
+				out->mSecond = right;
+
+				return out;
+			}
+
+			AstRepetition* parseRepetition(AstNode* left, bool plus = false) {
+				auto out = new AstRepetition();
+				out->mNode = left;
+				out->mPlus = plus;
+				return out;
+			}
+
+			AstIf* parseIf(AstNode* left) {
+				auto out = new AstIf();
+				out->mNode = left;
+				return out;
+			}
+
+			void genError(const char* desc) {
+				mError = { desc, mOffset };
+			}
+
+			Token& readTok() {
+
+				const tAlphabetType* crs = mSource + mOffset;
+
+				// zero termination string
+				if (*crs == 0) {
+					mCurToken.type = TOK_NONE;
+					return mCurToken;
+				}
+
+				mTokLength = 1;
+				mCurToken.type = TOK_VAL;
+				mCurToken.val = crs[0];
+
+				if (crs[0] == mEscapeSymbol) {
+					mCurToken.val = crs[1];
+					mTokLength = 2;
+				}
+				else {
+					for (uhalni tok = 0; tok < TOK_SPECIALS_END_; tok++) {
+						if (SpecialSymbols[tok] == mCurToken.val) {
+							mCurToken.type = TokType(tok);
+							break;
+						}
+					}
+				}
+
+				mOffset += mTokLength;
+				return mCurToken;
+			}
+
+			void discardTok() {
+				mOffset -= mTokLength;
+				mTokLength = 0;
+			}
+		};
+		
+		template <typename tStateType>
+		struct CompileError {
+			ParseError mParseError;
+			uhalni mRuleIndex = 0;
+			tStateType mRuleState;
+			const char* description = nullptr;
+			[[nodiscard]] bool isError() const { return description; }
+		};
+
+		template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
+		class Compiler {
+
+			typedef NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> Graph;
+			typedef typename Graph::Vertex Vertex;
+			typedef Parser<tAlphabetType, tStateType, tNoStateVal> Parser;
+
+			struct Node {
+				Vertex* left = nullptr;
+				Vertex* right = nullptr;
+			};
+
+		private:
+			Graph* mGraph = nullptr;
+
+		public:
+			CompileError<tStateType> mError;
+
+			Node compile(Graph& graph, const tAlphabetType* regex, tStateType state) {
+				mGraph = &graph;
+				return compileUtil(regex, state);
+			}
+
+			Node compile(Graph& aGraph, InitialierList<Pair<const tAlphabetType*, tStateType>> aRules) {
+				mGraph = &aGraph;
+
+				auto left = mGraph->addVertex();
+				auto right = mGraph->addVertex();
+
+				halni idx = 0;
+				for (auto rule: aRules) {
+
+					auto node = compileUtil(rule.head, rule.tail);
+
+					if (!(node.left && node.right)) {
+						mError.mRuleIndex = idx;
+						return {};
+					}
+
+					transitionAny(left, node.left);
+					transitionAny(node.right, right);
+
+					idx++;
+				}
+
+				mGraph->setStartVertex(left);
+
+				return { left, right };
+			}
+
+		private:
+
+			Node compileUtil(const tAlphabetType* regex, tStateType state) {
+				Parser parser;
+				auto astNode = parser.parse(regex);
+				if (parser.mError.isError()) {
+					mGraph->setStartVertex(nullptr);
+					mError.description = "Parsing Of Regular Expression Failed";
+					mError.mRuleState = state;
+					mError.mParseError = parser.mError;
+					return {};
+				}
+
+				auto node = compileNode(astNode, nullptr, nullptr);
+				delete astNode;
+
+				mGraph->setVertexState(node.right, state);
+				mGraph->setStartVertex(node.left);
+
+				return node;
+			}
+
+			Node compileVal(AstVal<tAlphabetType>* val, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+				auto left = aLeft ? aLeft : mGraph->addVertex();
+				auto right = aRight ? aRight : mGraph->addVertex();
+				transitionVal(left, right, val->mVal);
+				return { left, right };
+			}
+
+			Node compileAlternation(AstAlternation* alt, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+				auto first_node = compileNode(alt->mFirst, aLeft, aRight);
+				auto second_node = compileNode(alt->mSecond);
+				transitionAny(first_node.left, second_node.left);
+				transitionAny(second_node.right, first_node.right);
+				return first_node;
+			}
+
+			Node compileAny(AstAny*, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+				auto left = aLeft ? aLeft : mGraph->addVertex();
+				auto right = aRight ? aRight : mGraph->addVertex();
+				transitionAny(left, right, true);
+				return { left, right };
+			}
+
+			Node compileRepeat(AstRepetition* repeat, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+				if (repeat->mPlus) {
+					auto middle = mGraph->addVertex();
+
+					auto left_node = compileNode(repeat->mNode, aLeft, middle);
+
+					auto right_node = compileNode(repeat->mNode, middle, aRight);
+					transitionAny(right_node.right, right_node.left);
+					transitionAny(right_node.left, right_node.right);
+
+					return { left_node.left, right_node.right };
+				}
+				else {
+					auto node = compileNode(repeat->mNode, aLeft, aRight);
+					transitionAny(node.right, node.left);
+					transitionAny(node.left, node.right);
+					return node;
+				}
+			}
+
+			Node compileIf(AstIf* ifNode, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+				auto node = compileNode(ifNode->mNode, aLeft, aRight);
+				transitionAny(node.left, node.right);
+				return node;
+			}
+
+			Node compileClass(AstClass<tAlphabetType>* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+				auto left = aLeft ? aLeft : mGraph->addVertex();
+				auto right = aRight ? aRight : mGraph->addVertex();
+				
+				if (node->mRanges.length() == 1) {
+					auto const& range = node->mRanges.first()->data;
+					transitionRange(left, right, { range.mBegin, range.mEnd }, node->mExclude);
+					return { left, right };
+				}
+
+				for (auto range : node->mRanges) {
+					auto middle = mGraph->addVertex();
+					transitionRange(left, middle, { range.data().mBegin, range.data().mEnd }, node->mExclude);
+					transitionAny(middle, right);
+				}
+				return { left, right };
+			}
+
+			Node compileCompound(AstCompound* compound, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+				Vertex* left = nullptr;
+				Vertex* rigth = nullptr;
+
+				ualni idx = 0;
+				for (auto child : compound->mChilds) {
+					auto pass_left = idx == 0 ? aLeft : rigth;
+					auto pass_right = idx == compound->mChilds.length() - 1 ? aRight : nullptr;
+					auto node = compileNode(child.data(), pass_left, pass_right);
+					if (!left) left = node.left;
+					rigth = node.right;
+					idx++;
+				}
+
+				return { left, rigth };
+			}
+
+			Node compileNode(AstNode* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+				switch (node->mType) {
+					case AstNode::CLASS: return compileClass((AstClass<tAlphabetType>*)node, aLeft, aRight);
+					case AstNode::COMPOUND: return compileCompound((AstCompound*)node, aLeft, aRight);
+					case AstNode::IF: return compileIf((AstIf*)node, aLeft, aRight);
+					case AstNode::REPEAT: return compileRepeat((AstRepetition*)node, aLeft, aRight);
+					case AstNode::ANY: return compileAny((AstAny*)node, aLeft, aRight);
+					case AstNode::OR: return compileAlternation((AstAlternation*)node, aLeft, aRight);
+					case AstNode::VAL: return compileVal((AstVal<tAlphabetType>*)node, aLeft, aRight);
+					case AstNode::NONE:
+						break;
+				}
+				ASSERT(0)
+				return {};
+			}
+
+			void transitionAny(Vertex* from, Vertex* to, bool consumes = false) {
+				mGraph->addTransition(from, to, {}, consumes, true, false);
+			}
+
+			void transitionVal(Vertex* from, Vertex* to, tAlphabetType val) {
+				mGraph->addTransition(from, to, { val, val }, true, false, false);
+			}
+
+			void transitionRange(Vertex* from, Vertex* to, Range<tAlphabetType> range, bool exclude) {
+				mGraph->addTransition(from, to, range, true, false, exclude);
+			}
+		};
+
+		template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
+		CompileError<tStateType>	compile(NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& out, const tAlphabetType* regex, tStateType state) {
+			Compiler<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> compiler;
+			compiler.compile(out, regex, state);
+			return compiler.mError;
+		}
+
+		template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
+		CompileError<tStateType>	compile(NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& out, const InitialierList<Pair<const tAlphabetType*, tStateType>>& rules) {
+			Compiler<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> compiler;
+			compiler.compile(out, rules);
+			return compiler.mError;
+		}
+	}

Language/old/Tokenizer/public/Tokenizer.hpp

@@ -0,0 +1,184 @@
+
+#pragma once
+
+#include "RegularExpression.h"
+#include "Strings.hpp"
+
+namespace tp {
+
+	extern ModuleManifest gModuleTokenizer;
+
+	template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal>
+	class Tokenizer {
+
+		TransitionMatrix<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> mTransitionMatrix;
+
+		RegEx::CompileError<tTokType> mError;
+
+		bool scanFailed() { return mTransitionMatrix.isTrapped(); }
+
+	public:
+		// Some useful RE to be reused
+		static constexpr const tAlphabetType* etherRE = "\n|\t| |\r";
+		static constexpr const tAlphabetType* intRE = "((\\-)|(\\+))?[0-9]+i?";
+		static constexpr const tAlphabetType* floatRE = R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)";
+		static constexpr const tAlphabetType* commentBlockRE = R"((/\*){\*-\*}*(\*/))";
+		static constexpr const tAlphabetType* stringRE = R"("{"-"}*")";
+		static constexpr const tAlphabetType* idRE = "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*";
+
+	public:
+		Tokenizer() { MODULE_SANITY_CHECK(gModuleTokenizer) }
+
+		void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) {
+			NFA<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> nfa;
+
+			mError = RegEx::compile(nfa, rules);
+			if (mError.isError()) {
+				return;
+			}
+
+			DFA<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> dfa(nfa);
+			mTransitionMatrix.construct(dfa);
+		}
+
+		[[nodiscard]] bool isBuild() const { return !mError.isError(); }
+
+		auto getMatrix() const { return &mTransitionMatrix; }
+
+		const RegEx::CompileError<tTokType>& getBuildError() { return mError; }
+
+		void resetMatrix() { mTransitionMatrix.reset(); }
+
+		tTokType advanceSymbol(tAlphabetType symbol) { return mTransitionMatrix.move(symbol); }
+
+		tTokType advanceToken(const tAlphabetType* source, ualni source_len, ualni* token_len) {
+			tTokType out = tNoTokVal;
+			*token_len = 0;
+			for (ualni idx = 0; idx < source_len; idx++) {
+				out = advanceSymbol(source[idx]);
+				if (out != tNoTokVal) {
+					*token_len = idx;
+					break;
+				}
+			}
+			return out;
+		}
+
+		~Tokenizer() = default;
+	};
+
+	template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal, tTokType tSourceEndTokVal>
+	class SimpleTokenizer {
+	public:
+		typedef Tokenizer<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> tTokenizer;
+
+	private:
+		tTokenizer mTokenizer;
+
+		const tAlphabetType* mSource = nullptr;
+		ualni mLastTokLen = 0;
+		ualni mSourceLen = 0;
+		ualni mAdvancedOffset = 0;
+
+	public:
+		struct Cursor {
+			const tAlphabetType* mSource = nullptr;
+			ualni mAdvancedOffset = 0;
+			const tAlphabetType* str() { return mSource + mAdvancedOffset; }
+
+			struct Location2D {
+				ualni line = 0;
+				ualni character = 0;
+			};
+
+			Location2D get2DLocation() {
+				Location2D out;
+				typedef typename StringLogic<tAlphabetType>::Index Idx;
+				Buffer<Idx> offsets;
+				StringLogic<tAlphabetType>::calcLineOffsets(mSource, StringLogic<tAlphabetType>::calcLength(mSource), offsets);
+				for (ualni idx = 1; idx < offsets.size(); idx++) {
+					if (offsets[idx] > mAdvancedOffset) {
+						out.line = idx - 1;
+						break;
+					}
+				}
+				out.character = mAdvancedOffset - offsets[out.line];
+				return out;
+			}
+		};
+
+		SimpleTokenizer() = default;
+		auto getTokenizer() const { return &mTokenizer; }
+
+		void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) { mTokenizer.build(rules); }
+
+		[[nodiscard]] bool isBuild() const { return mTokenizer.isBuild(); }
+
+		const RegEx::CompileError<tTokType>& getBuildError() { return mTokenizer.getBuildError(); }
+
+		void bindSource(const tAlphabetType* source) {
+			mSource = source;
+			while (mSource[mSourceLen])
+				mSourceLen++;
+			mSourceLen++;
+		}
+
+		[[nodiscard]] bool isInputLeft() const {
+			if (!mSource) {
+				return false;
+			}
+			if (mSource[mAdvancedOffset] == 0) {
+				return false;
+			}
+			return true;
+		}
+
+		Cursor getCursor() const { return { mSource, mAdvancedOffset }; }
+
+		Cursor getCursorPrev() const { return { mSource, mAdvancedOffset - mLastTokLen }; }
+
+		void setCursor(const Cursor& crs) {
+			mAdvancedOffset = crs.mAdvancedOffset;
+			mLastTokLen = 0;
+		}
+
+		tTokType readTok() {
+			if (mSourceLen == mAdvancedOffset + 1) {
+				return tSourceEndTokVal;
+			}
+
+			tTokType out = mTokenizer.advanceToken(mSource + mAdvancedOffset, mSourceLen - mAdvancedOffset, &mLastTokLen);
+			mAdvancedOffset += mLastTokLen;
+			return out;
+		}
+
+		tTokType lookupTok() {
+			auto out = readTok();
+			discardTok();
+			return out;
+		}
+
+		void discardTok() {
+			mTokenizer.resetMatrix();
+			mAdvancedOffset -= mLastTokLen;
+			mLastTokLen = 0;
+		}
+
+		void skipTok() { readTok(); }
+
+		void reset() {
+			mAdvancedOffset = mLastTokLen = 0;
+			mTokenizer.resetMatrix();
+		}
+
+		[[nodiscard]] ualni lastTokLEn() const { return mLastTokLen; }
+
+		String extractVal() {
+			auto crs = getCursorPrev();
+			String out;
+			out.resize(mLastTokLen);
+			memCopy(out.write(), crs.str(), mLastTokLen);
+			return out;
+		}
+	};
+}

Language/old/Tokenizer/tests/TestTokenizer.cpp

@@ -0,0 +1,398 @@
+// #include "NewPlacement.hpp"
+
+#include "Testing.hpp"
+#include "Tokenizer.hpp"
+#include <cstdio>
+#include <iostream>
+
+#define LOG(val) std::cout << #val << " "
+
+using namespace tp;
+
+ualni outputHashPassed = 1156;
+
+const char* script =
+	R"(
+
+/* yea
+comment  "and string inside"  with int 123 and float 0.123f */
+
+-1.f;
+
+if (+1.f) {
+	var string = "string value some &&characters inside string&& -= asdf 09a
+	another line";
+
+}
+
+while
+
+return
+
+)";
+
+TEST_DEF_STATIC(General) {
+
+	enum class TokType {
+		NONE,
+		FAILED,
+		TOK_SOURCE_END,
+
+		VAR,
+		CLASS_DEF,
+		SELF,
+		SCOPE_IN,
+		SCOPE_OUT,
+		ASSIGN,
+		DEF_FUNC,
+		PRINT,
+		IF,
+		ELSE,
+		WHILE,
+		BRACKET_IN,
+		BRACKET_OUT,
+		COMMA,
+		NEW,
+		CHILD,
+		RETURN,
+		EQUAL,
+		NOT_EQUAL,
+		MORE,
+		LESS,
+		QE_OR_MORE,
+		QE_OR_LESS,
+		BOOL_NOT,
+		BOOL_AND,
+		BOOL_OR,
+		ADD,
+		MUL,
+		SUB,
+		DIV,
+		STM_END,
+		CONST_TRUE,
+		CONST_FALSE,
+		CONST_INT,
+		CONST_FLOAT,
+		SPACE,
+		CONST_STRING,
+		ID,
+
+		// COMMENT_LINE,
+
+		COMMENT_BLOCK,
+	};
+
+	SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
+
+	lexer.build({
+		{ "\n|\t| |\r", TokType::SPACE },
+		{ "var", TokType::VAR },
+		{ "class", TokType::CLASS_DEF },
+		{ "self", TokType::SELF },
+		{ "\\{", TokType::SCOPE_IN },
+		{ "\\}", TokType::SCOPE_OUT },
+		{ "=", TokType::ASSIGN },
+		{ "def", TokType::DEF_FUNC },
+		{ "<<", TokType::PRINT },
+		{ "if", TokType::IF },
+		{ "else", TokType::ELSE },
+		{ "while", TokType::WHILE },
+		{ "\\(", TokType::BRACKET_IN },
+		{ "\\)", TokType::BRACKET_OUT },
+		{ ",", TokType::COMMA },
+		{ "new", TokType::NEW },
+		{ "\\.", TokType::CHILD },
+		{ "return", TokType::RETURN },
+		{ "==", TokType::EQUAL },
+		{ "!=", TokType::NOT_EQUAL },
+		{ ">", TokType::MORE },
+		{ "<", TokType::LESS },
+		{ ">=", TokType::QE_OR_MORE },
+		{ "<=", TokType::QE_OR_LESS },
+		{ "!", TokType::BOOL_NOT },
+		{ "&&", TokType::BOOL_AND },
+		{ "\\|\\|", TokType::BOOL_OR },
+		{ "\\+", TokType::ADD },
+		{ "\\*", TokType::MUL },
+		{ "\\-", TokType::SUB },
+		{ "/", TokType::DIV },
+		{ ";", TokType::STM_END },
+		{ "true", TokType::CONST_TRUE },
+		{ "false", TokType::CONST_FALSE },
+		{ "((\\-)|(\\+))?[0-9]+i?", TokType::CONST_INT },
+		{ R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)", TokType::CONST_FLOAT },
+		{ R"((/\*){\*-\*}*(\*/))", TokType::COMMENT_BLOCK },
+		{ R"("{"-"}*")", TokType::CONST_STRING },
+		{ "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", TokType::ID },
+	});
+
+	if (!lexer.isBuild()) {
+		printf("Error : %s", lexer.getBuildError().description);
+		return;
+	}
+
+	lexer.bindSource(script);
+
+	TokType tok;
+	ualni outputHash = 0;
+	do {
+
+		tok = lexer.readTok();
+
+		outputHash += ualni(tok);
+
+		switch (tok) {
+			case TokType::SPACE:
+				{
+					printf(" ");
+				}
+				break;
+			case TokType::VAR:
+				{
+					LOG(VAR);
+				}
+				break;
+			case TokType::CLASS_DEF:
+				{
+					LOG(CLASS_DEF);
+				}
+				break;
+			case TokType::SELF:
+				{
+					LOG(SELF);
+				}
+				break;
+			case TokType::SCOPE_IN:
+				{
+					LOG(SCOPE_IN);
+				}
+				break;
+			case TokType::SCOPE_OUT:
+				{
+					LOG(SCOPE_OUT);
+				}
+				break;
+			case TokType::ASSIGN:
+				{
+					LOG(ASSIGN);
+				}
+				break;
+			case TokType::DEF_FUNC:
+				{
+					LOG(DEF_FUNC);
+				}
+				break;
+			case TokType::PRINT:
+				{
+					LOG(PRINT);
+				}
+				break;
+			case TokType::IF:
+				{
+					LOG(IF);
+				}
+				break;
+			case TokType::ELSE:
+				{
+					LOG(ELSE);
+				}
+				break;
+			case TokType::WHILE:
+				{
+					LOG(WHILE);
+				}
+				break;
+			case TokType::BRACKET_IN:
+				{
+					LOG(BRACKET_IN);
+				}
+				break;
+			case TokType::BRACKET_OUT:
+				{
+					LOG(BRACKET_OUT);
+				}
+				break;
+			case TokType::COMMA:
+				{
+					LOG(COMMA);
+				}
+				break;
+			case TokType::NEW:
+				{
+					LOG(NEW);
+				}
+				break;
+			case TokType::CHILD:
+				{
+					LOG(CHILD);
+				}
+				break;
+			case TokType::RETURN:
+				{
+					LOG(RETURN);
+				}
+				break;
+			case TokType::EQUAL:
+				{
+					LOG(EQUAL);
+				}
+				break;
+			case TokType::NOT_EQUAL:
+				{
+					LOG(NOT_EQUAL);
+				}
+				break;
+			case TokType::MORE:
+				{
+					LOG(MORE);
+				}
+				break;
+			case TokType::LESS:
+				{
+					LOG(LESS);
+				}
+				break;
+			case TokType::QE_OR_MORE:
+				{
+					LOG(QE_OR_MORE);
+				}
+				break;
+			case TokType::QE_OR_LESS:
+				{
+					LOG(QE_OR_LESS);
+				}
+				break;
+			case TokType::BOOL_NOT:
+				{
+					LOG(BOOL_NOT);
+				}
+				break;
+			case TokType::BOOL_AND:
+				{
+					LOG(BOOL_AND);
+				}
+				break;
+			case TokType::BOOL_OR:
+				{
+					LOG(BOOL_OR);
+				}
+				break;
+			case TokType::ADD:
+				{
+					LOG(ADD);
+				}
+				break;
+			case TokType::MUL:
+				{
+					LOG(MUL);
+				}
+				break;
+			case TokType::SUB:
+				{
+					LOG(SUB);
+				}
+				break;
+			case TokType::DIV:
+				{
+					LOG(DIV);
+				}
+				break;
+			case TokType::STM_END:
+				{
+					LOG(STM_END);
+				}
+				break;
+			case TokType::CONST_TRUE:
+				{
+					LOG(TRUE);
+				}
+				break;
+			case TokType::CONST_FALSE:
+				{
+					LOG(FALSE);
+				}
+				break;
+			case TokType::CONST_INT:
+				{
+					LOG(INT);
+				}
+				break;
+			case TokType::CONST_FLOAT:
+				{
+					LOG(FLOAT);
+				}
+				break;
+			case TokType::CONST_STRING:
+				{
+					LOG(STRING);
+				}
+				break;
+			case TokType::ID:
+				{
+					LOG(ID);
+				}
+				break;
+			// case TokType::COMMENT_LINE: { LOG(COMMENT_LINE) } break;
+			case TokType::COMMENT_BLOCK:
+				{
+					LOG(COMMENT_BLOCK);
+				}
+				break;
+			case TokType::FAILED:
+				{
+					LOG(FAILED);
+				}
+				break;
+			case TokType::NONE:
+			case TokType::TOK_SOURCE_END: break;
+		}
+
+	} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
+
+	printf("\n\nOutputHash : %llu", outputHash);
+
+	TEST(outputHash == outputHashPassed);
+}
+
+TEST_DEF(Simple) {
+	enum class TokType {
+		START = 0,
+		NONE = 1,
+		FAILED = 2,
+		SPACE = 3,
+		ID = 4,
+		COMMENT_BLOCK = 5,
+		TOK_SOURCE_END = 6,
+	};
+
+	SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
+
+	lexer.build({
+		{ " ", TokType::SPACE },
+		{ "([a-c]|A)+", TokType::ID },
+		{ "A", TokType::START },
+	});
+
+	if (!lexer.isBuild()) {
+		printf("Error : %s", lexer.getBuildError().description);
+		return;
+	}
+
+	lexer.bindSource(" A bc cb cc ");
+
+	TokType tok;
+	ualni outputHash = 0;
+	printf("\n\n OutputHash : ");
+	do {
+		tok = lexer.readTok();
+		outputHash += ualni(tok);
+		printf(" %i ", int(tok));
+	} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
+	printf(" : %llu", outputHash);
+	TEST(outputHash == 33);
+}
+
+TEST_DEF(Tokenizer) {
+	testGeneral();
+	testSimple();
+}

Language/old/Tokenizer/tests/tests.cpp

@@ -0,0 +1,18 @@
+
+#include "Testing.hpp"
+#include "Tokenizer.hpp"
+
+void testTokenizer();
+
+int main(int argc, char* argv[]) {
+
+	tp::ModuleManifest* ModuleDependencies[] = { &tp::gModuleTokenizer, nullptr };
+	tp::ModuleManifest TestModule("TokenizerTest", nullptr, nullptr, ModuleDependencies);
+	if (!TestModule.initialize()) {
+		return 1;
+	}
+
+	testTokenizer();
+
+	TestModule.deinitialize();
+}

Language/private/AST.cpp

@@ -0,0 +1,4 @@
+
+#include "AST.hpp"
+
+using namespace tp;

Language/private/Grammar.cpp

@@ -0,0 +1,4 @@
+
+#include "Grammar.hpp"
+
+using namespace tp;

Language/private/GrammarCompiler.cpp

@@ -0,0 +1,4 @@
+
+#include "GrammarCompiler.hpp"
+
+using namespace tp;

Language/private/LanguageCommon.cpp

@@ -0,0 +1,8 @@
+
+#include "LanguageCommon.hpp"
+#include "Strings.hpp"
+
+using namespace tp;
+
+static ModuleManifest* sModuleDependencies[] = { &gModuleStrings, nullptr };
+ModuleManifest tp::gModuleLanguage = ModuleManifest("Language", nullptr, nullptr, sModuleDependencies);

Language/private/Parser.cpp

@@ -0,0 +1,8 @@
+
+#include "Parser.hpp"
+
+using namespace tp;
+
+void Parser::compileTables(const ContextFreeGrammar& cfGrammar, const RegularGrammar& reGrammar) {}
+
+void Parser::parse(const Sentence& sentence, AST& out) {}

Language/private/Sentence.cpp

@@ -0,0 +1,4 @@
+
+#include "Sentence.hpp"
+
+using namespace tp;

Language/private/SimpleParser.cpp

@@ -0,0 +1,49 @@
+
+#include "SimpleParser.hpp"
+
+using namespace tp;
+
+SimpleParser::SimpleParser() {
+	// Grammar for unified grammar format sentence that tables compiled from
+
+	// Define Context-Free grammar
+	ContextFreeGrammar contextFreeGrammar;
+	{
+		auto term = contextFreeGrammar.createTerminal();
+
+		auto nonTerm = contextFreeGrammar.createNonTerminal();
+		auto nonTerm2 = contextFreeGrammar.createNonTerminal();
+
+		contextFreeGrammar.addRule(nonTerm, { term, nonTerm2 });
+		contextFreeGrammar.addRule();
+
+		contextFreeGrammar.setStart(nonTerm);
+	}
+
+	// Define Regular grammar
+	RegularGrammar regularGrammar;
+	{
+		regularGrammar.addRule(regularGrammar.alternate(regularGrammar.repeat(regularGrammar.symbols("asd")), regularGrammar.symbol("a")));
+		regularGrammar.addRule(regularGrammar.alternate(regularGrammar.repeat(regularGrammar.symbols("asd")), regularGrammar.symbol("a")));
+	}
+
+	mUnifiedGrammarParser.compileTables(contextFreeGrammar, regularGrammar);
+}
+
+void SimpleParser::compileTables(const Sentence& grammar) {
+	AST unifiedGrammarAst;
+	mUnifiedGrammarParser.parse(grammar, unifiedGrammarAst);
+
+	// compile each ast into RegularGrammar and ContextFree Grammar api instructions
+	ContextFreeGrammar userContextFreeGrammar;
+	RegularGrammar userRegularGrammar;
+	// ...
+	// split ast into RE and CF part
+	// generate and execute grammar api commands
+	// ...
+
+	// compile tables from user grammar
+	mUserParser.compileTables(userContextFreeGrammar, userRegularGrammar);
+}
+
+void SimpleParser::parse(const Sentence& sentence, AST& out) { mUserParser.parse(sentence, out); }

Language/public/AST.hpp

@@ -0,0 +1,8 @@
+
+#pragma once
+
+#include "LanguageCommon.hpp"
+
+namespace tp {
+	class AST {};
+}

Language/public/Grammar.hpp

@@ -0,0 +1,9 @@
+
+#pragma once
+
+#include "LanguageCommon.hpp"
+
+namespace tp {
+	class ContextFreeGrammar {};
+	class RegularGrammar {};
+}

Language/public/GrammarCompiler.hpp

@@ -0,0 +1,6 @@
+
+#pragma once
+
+#include "Grammar.hpp"
+
+namespace tp {}

Language/public/LanguageCommon.hpp

@@ -0,0 +1,7 @@
+#pragma once
+
+#include "Module.hpp"
+
+namespace tp {
+	extern ModuleManifest gModuleLanguage;
+}

Language/public/Parser.hpp

@@ -0,0 +1,20 @@
+
+#pragma once
+
+#include "AST.hpp"
+#include "GrammarCompiler.hpp"
+#include "Sentence.hpp"
+
+namespace tp {
+	class Parser {
+	public:
+		Parser() = default;
+
+	public:
+		void compileTables(const ContextFreeGrammar& cfGrammar, const RegularGrammar& reGrammar);
+		void parse(const Sentence& sentence, AST& out);
+
+	public:
+		// save load compiled tables
+	};
+}

Language/public/Sentence.hpp

@@ -0,0 +1,13 @@
+
+#pragma once
+
+#include "LanguageCommon.hpp"
+
+namespace tp {
+	struct Sentence {
+		const int1* buffer = nullptr;
+		alni symbolSize = 1;
+
+		explicit Sentence(const char* buff) { buffer = buff; }
+	};
+}

Language/public/SimpleParser.hpp

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "Parser.hpp"
+
+namespace tp {
+
+	// Gives ability to express grammar in the Unified Format as sentence
+	class SimpleParser {
+	public:
+		SimpleParser();
+
+	public:
+		void compileTables(const Sentence& grammar);
+		void parse(const Sentence& sentence, AST& out);
+
+	private:
+		Parser mUnifiedGrammarParser;
+		Parser mUserParser;
+	};
+}

Language/tests/Test.hpp

@@ -0,0 +1,43 @@
+#pragma once
+
+#include "SimpleParser.hpp"
+
+const char* gGrammar = R"(
+# Grammar in the CF-RE United Format (Defined by language module)
+
+Rules : {
+	ScopeList : ScopeList Scope | Scope ;
+	Scope : \ScopeBegin StatementList \ScopeEnd;
+	StatementList : StatementList Statement \StatementEnd;
+	StatementList : Statement \StatementEnd;
+	Statement : \StatementBody;
+}
+
+Terminals : {
+	Space : " " | "\t" | "\n" | "\r";
+	ScopeBegin : "{";
+	ScopeEnd : "}";
+  StatementEnd : ";";
+  StatementBody : "a" | "b";
+}
+
+Start : Scope;
+Ignore : Space;
+)";
+
+const char* gSentence = R"(
+{}
+
+{ }
+
+{
+a;
+ a ; a ;
+
+ }
+
+ {
+a;
+ a;
+}
+)";

Language/tests/Tests.cpp

@@ -0,0 +1,29 @@
+
+#include "Test.hpp"
+
+using namespace tp;
+
+void test() {
+	auto parser = SimpleParser();
+	auto grammar = Sentence(gGrammar);
+
+	auto sentence = Sentence(gSentence);
+	auto ast = AST();
+
+	parser.compileTables(grammar);
+	parser.parse(sentence, ast);
+}
+
+int main() {
+
+	tp::ModuleManifest* deps[] = { &tp::gModuleLanguage, nullptr };
+	tp::ModuleManifest testModule("Test", nullptr, nullptr, deps);
+
+	if (!testModule.initialize()) {
+		return 1;
+	}
+
+	test();
+
+	testModule.deinitialize();
+}