tmp

.wip/Language/CMakeLists.txt

@@ -0,0 +1,20 @@
+
+cmake_minimum_required(VERSION 3.2)
+
+set(CMAKE_CXX_STANDARD 23)
+
+project(Language)
+
+### ---------------------- Static Library  ---------------------  ###
+file(GLOB SOURCES "./private/*.cpp" "./private/*/*.cpp")
+file(GLOB HEADERS "./public/*.hpp" "./public/*/*.hpp")
+add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
+target_include_directories(${PROJECT_NAME} PUBLIC public/)
+target_link_libraries(${PROJECT_NAME} PUBLIC Strings)
+
+### -------------------------- Tests  --------------------------  ###
+enable_testing()
+file(GLOB TEST_SOURCES "./tests/*.cpp")
+add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES})
+target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} UnitTest++)
+add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests)

.wip/Language/private/Grammar.cpp

@@ -0,0 +1,36 @@
+
+#include "Grammar.hpp"
+
+using namespace tp;
+
+ContextFreeGrammar::Arg::Arg(const String& id, bool terminal, bool epsilon) {
+	mId = id;
+	mIsTerminal = terminal;
+	mIsEpsilon = epsilon;
+}
+
+const String& ContextFreeGrammar::Arg::getId() const { return mId; }
+
+bool ContextFreeGrammar::Arg::operator==(const Arg& in) const {
+	return (mId == in.mId) && (mIsEpsilon == in.mIsEpsilon) && (mIsTerminal == in.mIsTerminal);
+}
+
+ContextFreeGrammar::Rule::Rule(const String& id, const InitialierList<Arg>& args) {
+	mId = id;
+	mArgs = args;
+}
+
+bool ContextFreeGrammar::Rule::operator==(const Rule& in) const { return (mId == in.mId) && (mArgs == in.mArgs); }
+
+bool ContextFreeGrammar::Rule::isProductive() const {
+	for (auto arg : mArgs) {
+		if (arg->getId() == mId) return false;
+	}
+	return true;
+}
+
+void ContextFreeGrammar::addRule(const Rule& rule) { mRules.append(rule); }
+
+void ContextFreeGrammar::addRule(const String& id, const InitialierList<Arg>& args) { addRule(Rule(id, args)); }
+
+void ContextFreeGrammar::setStart(const String& startRule) { mStartTerminal = startRule; }

.wip/Language/private/LanguageCommon.cpp

@@ -0,0 +1,8 @@
+
+#include "LanguageCommon.hpp"
+#include "Strings.hpp"
+
+using namespace tp;
+
+static ModuleManifest* sModuleDependencies[] = { &gModuleStrings, nullptr };
+ModuleManifest tp::gModuleLanguage = ModuleManifest("Language", nullptr, nullptr, sModuleDependencies);

.wip/Language/public/Automata.hpp

@@ -0,0 +1,249 @@
+
+#pragma once
+
+#include "Utils.hpp"
+#include "List.hpp"
+#include "Map.hpp"
+#include "Tree.hpp"
+
+namespace tp {
+
+	// Non-Deterministic Finite-State Automata
+	template <typename tAlphabetType, typename tStateType>
+	class FiniteStateAutomation {
+	public:
+		struct State;
+
+	public:
+		class Transition {
+			friend FiniteStateAutomation;
+
+		public:
+			enum Type { ANY, EPSILON, SYMBOL };
+
+		public:
+			Transition(Type type, State* state, tAlphabetType symbol = tAlphabetType()) {
+				mState = state;
+				mType = type;
+				mSymbol = symbol;
+			}
+
+			[[nodiscard]] bool isTransition(const tAlphabetType& symbol) const {
+				return (mType == ANY || mType == EPSILON) || (mSymbol == symbol);
+			}
+
+			[[nodiscard]] bool doesConsumes(const tAlphabetType& symbol) const {
+				return (mType == ANY || (mType == SYMBOL && mSymbol == symbol));
+			}
+
+			[[nodiscard]] bool isEpsilon() const { return mType == EPSILON; }
+
+			const State* getState() const { return mState; }
+			const tAlphabetType& getSymbol() const { return mSymbol; }
+
+		private:
+			State* mState = nullptr;
+			Type mType;
+			tAlphabetType mSymbol;
+		};
+
+		class State {
+			friend FiniteStateAutomation;
+
+		public:
+			State() = default;
+
+		public:
+			void setValue(const tStateType& stateValue) { mStateVal = stateValue; }
+			void setAcceptance(bool isAccepting) { mIsAccepting = isAccepting; }
+			[[nodiscard]] bool isAccepting() const { return mIsAccepting; }
+			const tStateType& getStateVal() const { return mStateVal; }
+			[[nodiscard]] const Buffer<Transition>* getTransitions() const { return &mTransitions; }
+
+		private:
+			Buffer<Transition> mTransitions{};
+			tStateType mStateVal = tStateType();
+			bool mIsAccepting = false;
+		};
+
+	private:
+		List<State> mStates;
+		State* mStartState = nullptr;
+		Range<ualni> mAlphabetRange = { ENV_UALNI_MAX, ENV_UALNI_MIN };
+
+	public:
+		FiniteStateAutomation() = default;
+
+		State* addState(const tStateType& state, bool accepting) {
+			auto node = mStates.newNode();
+			node->data.mIsAccepting = accepting;
+			node->data.mStateVal = state;
+			mStates.pushBack(node);
+			return &node->data;
+		}
+
+		void addTransition(State* from, State* to, const tAlphabetType& symbol) {
+			from->mTransitions.append(Transition(Transition::SYMBOL, to, symbol));
+			if (mAlphabetRange.mBegin < ualni(symbol)) mAlphabetRange.mBegin = ualni(symbol);
+			if (mAlphabetRange.mEnd > ualni(symbol)) mAlphabetRange.mEnd = ualni(symbol);
+		}
+
+		void addEpsilonTransition(State* from, State* to) { from->mTransitions.append(Transition(Transition::SYMBOL, to)); }
+
+		void addAnyTransition(State* from, State* to) { from->mTransitions.append(Transition(Transition::ANY, to)); }
+
+		void setStartState(State* start) { mStartState = start; }
+
+		[[nodiscard]] State* getStartState() const { return mStartState; }
+
+		[[nodiscard]] bool isValid() const {
+			if (!mStartState) {
+				return false;
+			}
+			return true;
+		}
+
+		[[nodiscard]] ualni numStates() const { return mStates.length(); }
+
+		[[nodiscard]] const List<State>* getStates() const { return &mStates; }
+
+		[[nodiscard]] Range<ualni> getAlphabetRange() const { return mAlphabetRange; }
+
+	private:
+		typedef AvlTree<AvlNumericKey<State*>, bool> StatesSet;
+
+		// Expands initial set with states that are reachable from initial set with no input consumption (E-transitions)
+		static void expandSet(StatesSet& set) {
+			List<State*> workingSet;
+
+			set.forEach([&](AvlNumericKey<State*>& key, bool) { workingSet.pushBack(key.val); });
+
+			while (workingSet.length()) {
+				auto first = workingSet.first()->data;
+				set.insert(first, {});
+
+				for (auto transition : first->mTransitions) {
+					if (!transition->isEpsilon()) continue;
+					if (set.find(transition->mState)) continue;
+					workingSet.pushBack(transition->mState);
+				}
+
+				workingSet.popFront();
+			}
+		}
+
+		// States that are reachable from initial set with symbol transition
+		static void findMoveSet(StatesSet& from, StatesSet& moveSet, tAlphabetType symbol) {
+			from.forEach([&](AvlNumericKey<State*>& key, bool) {
+				for (auto transition : key.val->mTransitions) {
+					if (transition->isEpsilon()) continue;
+					if (!transition->isTransition(symbol)) continue;
+					if (moveSet.find(transition->mState)) continue;
+					moveSet.insert(transition->mState, {});
+				}
+			});
+		}
+
+	public:
+		bool makeDeterministic() {
+			if (!isValid()) return false;
+
+			struct GroupKey {
+				const StatesSet* group;
+				static ualni hash(GroupKey key) { return 0; }
+				bool operator==(const GroupKey& key) const { return false; }
+			};
+
+			struct GroupInfo {
+				StatesSet* group = nullptr;
+				AvlTree<AvlNumericKey<StatesSet*>, tAlphabetType> transitions;
+				State* newState = nullptr;
+				bool accepting = false;
+				tStateType stateVal = tStateType();
+			};
+
+			Buffer<StatesSet> groups = { {} };
+			Map<GroupKey, GroupInfo, DefaultAllocator, GroupKey::hash> groupInfos;
+
+			groups.first().insert(getStartState(), false);
+
+			expandSet(groups.first());
+
+			groupInfos.put({ &groups.first() }, { &groups.first() });
+
+			// 1) find new states
+			List<StatesSet*> workingSet;
+			workingSet.pushBack(&groups.first());
+
+			while (workingSet.length()) {
+				StatesSet* group = workingSet.first()->data;
+				GroupInfo* info = &groupInfos.get({ group });
+
+				for (auto symbol : getAlphabetRange()) {
+
+					// calculate new possible state
+					StatesSet potentialGroup;
+
+					findMoveSet(*group, potentialGroup, tAlphabetType(symbol));
+					expandSet(potentialGroup);
+
+					if (!potentialGroup.size()) continue;
+
+					// find existing or create group
+					StatesSet* targetGroup = nullptr;
+					auto iter = groupInfos.presents({ &potentialGroup });
+					if (iter) {
+						targetGroup = groupInfos.getSlotVal(iter).group;
+					} else {
+						targetGroup = &groups.append(potentialGroup);
+						groupInfos.put({ targetGroup }, { targetGroup });
+						workingSet.pushBack(targetGroup);
+					}
+
+					// assert transition is added
+					info->transitions.insert(targetGroup, tAlphabetType(symbol));
+				}
+
+				workingSet.popFront();
+			}
+
+			// 2) find new states termination values
+			for (auto group : groupInfos) {
+				GroupInfo* info = &group->val;
+				ualni accepting = 0;
+
+				info->group->forEach([&](AvlNumericKey<State*>& key, bool) {
+					if (key.val->mIsAccepting) {
+						accepting++;
+						info->accepting = true;
+						info->stateVal = key.val->mStateVal;
+					}
+				});
+
+				if (!accepting) {
+					info->accepting = false;
+					info->stateVal = info->group->head()->key.val->mStateVal;
+				}
+			}
+
+			// 3) transfer
+			mStates.removeAll();
+
+			// create states
+			for (auto group : groupInfos) {
+				group->val.newState = addState(group->val.stateVal, group->val.accepting);
+			}
+
+			// create transitions
+			for (auto group : groupInfos) {
+				auto functor = [&](AvlNumericKey<StatesSet*> targetGroupKey, tAlphabetType symbol) {
+					GroupInfo* targetGroup = &groupInfos.get({ (StatesSet*) targetGroupKey.val });
+					addTransition(group->val.newState, targetGroup->newState, symbol);
+				};
+				group->val.transitions.forEach(functor);
+			}
+
+			return true;
+		}
+	};
+}

.wip/Language/public/ContextFreeAutomata.hpp

@@ -0,0 +1,137 @@
+
+#pragma once
+
+#include "Strings.hpp"
+#include "Automata.hpp"
+#include "Buffer2D.hpp"
+
+namespace tp {
+
+	template <typename tAlphabetType, typename tStateType>
+	class ContextFreeAutomata {
+
+		struct Action {
+			enum Type { SHIFT, REDUCE, TRAP } type = TRAP;
+			ualni num = 0; // state to shift (shift action) or pop count (reduce action)
+		};
+
+	public:
+		struct StackItem {
+			ualni state = 0;
+			tAlphabetType symbol;
+			Buffer<StackItem*> leafs;
+		};
+
+		struct AcceptResult {
+			bool accepted = false;
+			ualni advancedIdx = 0;
+			const StackItem* ast = nullptr;
+		};
+
+	public:
+		ContextFreeAutomata() = default;
+
+		AcceptResult accept(const tAlphabetType* stream, ualni size) {
+			mCurrentState = mStartState;
+			mStack.append(&mItems.append({ mCurrentState, {}, {} }));
+
+			ualni advancedIdx = 0;
+			while (advancedIdx < size) {
+				const tAlphabetType& symbol = *(stream + advancedIdx);
+
+				if (!(symbol >= mRange.mBegin && symbol < mRange.mEnd)) {
+					return { false, advancedIdx, nullptr };
+				}
+
+				const Action& action = mTable.get({ ualni(symbol - mRange.mBegin), mCurrentState });
+
+				if (action.type == Action::TRAP) {
+					return { false, advancedIdx, nullptr };
+				}
+
+				if (action.type == Action::SHIFT) {
+					mStack.last()->symbol = symbol;
+					mStack.append(&mItems.append({ mCurrentState, {}, {} }));
+					mCurrentState = action.num;
+				}
+
+				if (mTable.get({ 0, mCurrentState }).type == Action::REDUCE) {
+					StackItem* newItem = &mItems.append(StackItem{});
+					for (auto iter : Range<ualni>(action.num)) {
+						newItem->leafs.append(mStack.last());
+						mCurrentState = mStack.last()->state;
+						mStack.pop();
+					}
+
+					if (!mStack.size()) {
+						if (advancedIdx == size) {
+							return { true, advancedIdx, newItem };
+						} else {
+							return { false, advancedIdx, {} };
+						}
+					}
+
+					mStack.append(&mItems.append({ mCurrentState, {}, {} }));
+				}
+
+				advancedIdx++;
+			}
+
+			return { false, advancedIdx, nullptr };
+		}
+
+	public:
+		typedef FiniteStateAutomation<tAlphabetType, tStateType> Automata;
+		typedef Automata::State AutomataState;
+
+		void construct(const Automata& automata) {
+			mRange = automata.getAlphabetRange();
+
+			const ualni numStates = automata.numStates();
+			const ualni numSymbols = mRange.idxDiff();
+
+			mTable.reserve({ numSymbols, numStates });
+			mTable.assign(Action{ Action::TRAP, 0 });
+
+			Map<const AutomataState*, ualni> states;
+			ualni stateIndex = 0;
+			for (auto state : *automata.getStates()) {
+				states.put(&state.data(), { stateIndex });
+				stateIndex++;
+			}
+
+			stateIndex = 0;
+			for (auto state : *automata.getStates()) {
+				if (&state.data() == automata.getStartState()) {
+					mStartState = stateIndex;
+				}
+
+				if (state->isAccepting()) {
+					ASSERT(state->getTransitions()->size() == 0)
+					for (auto symbolIndex : Range<ualni>(numSymbols)) {
+						mTable.set({ stateIndex, symbolIndex }, { Action::REDUCE, state->getStateVal().numArgs() });
+					}
+				} else {
+					for (auto transition : *state->getTransitions()) {
+						ualni symbolIndex = ualni(transition->getSymbol()) - mRange.mBegin;
+						ualni targetStateIndex = states.get(transition->getState());
+						mTable.set({ stateIndex, symbolIndex }, { Action::SHIFT, targetStateIndex });
+					}
+				}
+
+				stateIndex++;
+			}
+		}
+
+	private:
+		Buffer2D<Action> mTable;
+
+		Buffer<StackItem> mItems;
+		Buffer<StackItem*> mStack;
+
+		ualni mStartState = 0;
+		ualni mCurrentState = 0;
+
+		Range<ualni> mRange;
+	};
+}

.wip/Language/public/ContextFreeCompiler.hpp

@@ -0,0 +1,181 @@
+
+#pragma once
+
+#include "Automata.hpp"
+#include "Grammar.hpp"
+
+namespace tp {
+
+	class ContextFreeCompiler {
+	public:
+		struct SymbolVal {
+			SymbolVal() = default;
+
+			SymbolVal(ualni aId, ualni aStart, ualni aLen) {
+				id = aId;
+				start = aStart;
+				len = aLen;
+			};
+
+			SymbolVal(ualni val) { id = val; }
+
+			operator ualni() const { return id; }
+
+			bool operator==(const SymbolVal& in) const { return in.id == id; }
+			SymbolVal& operator=(const SymbolVal& in) = default;
+
+			ualni id = 0;
+			ualni start = 0;
+			ualni len = 0;
+		};
+
+		struct Item {
+			const ContextFreeGrammar::Rule* mRule = nullptr;
+			ualni mAdvanceIdx = 0;
+			ualni numArgs() const { return 0; }
+		};
+
+		struct Symbol {
+			String mId;
+			bool mIsTerminal = false;
+		};
+
+	private:
+		struct NonTerminal {
+			Buffer<ContextFreeGrammar::Rule*> rules;
+			Map<String, NonTerminal*> references;
+			Map<String, NonTerminal*> referencing;
+
+		public:
+			[[nodiscard]] bool isProductive() const {
+				for (auto rule : rules) {
+					if (rule->isProductive()) return true;
+				}
+				return false;
+			}
+
+			[[nodiscard]] bool isLooped(Map<String, ualni>& processed, const String& id) const {
+				for (auto ref : referencing) {
+					if (processed.presents(ref->key)) return true;
+				}
+				processed.put(id, {});
+				for (auto ref : referencing) {
+					if (ref->val->isLooped(processed, ref->key)) return true;
+				}
+				return false;
+			}
+		};
+
+	public:
+		bool compile(const ContextFreeGrammar& grammar, FiniteStateAutomation<SymbolVal, Item>& automata) {
+			if (!init(grammar)) return false;
+			return true;
+		}
+
+		[[nodiscard]] const Buffer<Symbol>* getSymbols() const { return &mSymbols; }
+		[[nodiscard]] SymbolVal getSymbolId(const String& name) const { return mSymbolLookup.get(name); }
+
+	private:
+		bool init(const ContextFreeGrammar& grammar) {
+
+			if (!grammar.getRules()->size()) {
+				return false;
+			}
+
+			for (auto rule : *grammar.getRules()) {
+				if (!rule->getArgs()->size()) {
+					return false;
+				}
+			}
+
+			findNonTerminals(grammar);
+
+			for (auto nonTerminal : mNonTerminals) {
+				for (auto rule : nonTerminal->val.rules) {
+					for (auto arg : *rule->getArgs()) {
+						if (arg->isTerminal() || arg->isEpsilon()) continue;
+
+						if (!mNonTerminals.presents(arg->getId())) {
+							printf("Referenced non-terminal '%s' is not defined\n", arg->getId().read());
+							return false;
+						}
+					}
+				}
+			}
+
+			findAllReferences(grammar);
+
+			for (auto nonTerminal : mNonTerminals) {
+				if (!nonTerminal->val.references.size() && nonTerminal->key != grammar.getStartTerminal()) {
+					printf("Non-terminal '%s' is defined but not used\n", nonTerminal->key.read());
+					return false;
+				}
+			}
+
+			for (auto nonTerminal : mNonTerminals) {
+				if (!nonTerminal->val.isProductive()) {
+					printf("Non-terminal '%s' is not productive\n", nonTerminal->val.rules.first()->getId().read());
+					return false;
+				}
+			}
+
+			Map<String, ualni> processed;
+			if (mNonTerminals.get(grammar.getStartTerminal()).isLooped(processed, grammar.getStartTerminal())) {
+				printf("Note that grammar is looped.\n");
+				return false;
+			}
+
+			initSymbols(grammar);
+
+			return true;
+		}
+
+		void findNonTerminals(const ContextFreeGrammar& grammar) {
+			for (auto rule : *grammar.getRules()) {
+				if (!mNonTerminals.presents(rule->getId())) {
+					mNonTerminals.put(rule->getId(), {});
+				}
+				auto nonTerminal = &mNonTerminals.get(rule->getId());
+				nonTerminal->rules.append(&rule.data());
+			}
+		}
+
+		void findAllReferences(const ContextFreeGrammar& grammar) {
+			for (auto nonTerminal : mNonTerminals) {
+				for (auto rule : nonTerminal->val.rules) {
+					for (auto arg : *rule->getArgs()) {
+						if (arg->isTerminal() || arg->isEpsilon()) continue;
+
+						NonTerminal* reference = &mNonTerminals.get(arg->getId());
+						nonTerminal->val.referencing.put(arg->getId(), reference);
+						reference->references.put(nonTerminal->key, &nonTerminal->val);
+					}
+				}
+			}
+		}
+
+		void initSymbols(const ContextFreeGrammar& grammar) {
+			for (auto nonTerminal : mNonTerminals) {
+				mSymbols.append({ nonTerminal->key, false });
+				mSymbolLookup.put(nonTerminal->key, SymbolVal(mSymbols.size() - 1));
+
+				for (auto rule : nonTerminal->val.rules) {
+					for (auto arg : *rule->getArgs()) {
+						if (arg->isEpsilon() || arg->isTerminal()) continue;
+						if (mTerminals.presents(arg->getId())) continue;
+						mTerminals.put(arg->getId(), {});
+
+						mSymbols.append({ nonTerminal->key, true });
+						mSymbolLookup.put(nonTerminal->key, SymbolVal(mSymbols.size() - 1));
+					}
+				}
+			}
+		}
+
+	private:
+		Map<String, NonTerminal> mNonTerminals;
+		Map<String, bool> mTerminals;
+		Buffer<Symbol> mSymbols;
+		Map<String, SymbolVal> mSymbolLookup;
+	};
+}

.wip/Language/public/Grammar.hpp

@@ -0,0 +1,224 @@
+
+#pragma once
+
+#include "Buffer.hpp"
+#include "LanguageCommon.hpp"
+#include "Map.hpp"
+#include "Strings.hpp"
+
+namespace tp {
+
+	class ContextFreeGrammar {
+	public:
+		struct Arg {
+			friend class Rule;
+
+		public:
+			Arg() = default;
+			explicit Arg(const String& id, bool terminal = true, bool epsilon = false);
+
+		public:
+			bool operator==(const Arg& in) const;
+			[[nodiscard]] const String& getId() const;
+			[[nodiscard]] bool isTerminal() const { return mIsTerminal; }
+			[[nodiscard]] bool isEpsilon() const { return mIsEpsilon; }
+
+		private:
+			String mId;
+			bool mIsTerminal = false;
+			bool mIsEpsilon = false;
+		};
+
+		class Rule {
+		public:
+			Rule() = default;
+			Rule(const String& id, const InitialierList<Arg>& args);
+
+		public:
+			bool operator==(const Rule& in) const;
+			[[nodiscard]] bool isProductive() const;
+			[[nodiscard]] const String& getId() const { return mId; }
+			[[nodiscard]] const Buffer<Arg>* getArgs() const { return &mArgs; }
+
+		private:
+			String mId;
+			Buffer<Arg> mArgs;
+		};
+
+	public:
+		ContextFreeGrammar() = default;
+
+	public:
+		void addRule(const Rule& rule);
+		void addRule(const String& id, const InitialierList<Arg>& args);
+		void setStart(const String& startRule);
+		[[nodiscard]] const Buffer<Rule>* getRules() const { return &mRules; }
+		[[nodiscard]] const String& getStartTerminal() const { return mStartTerminal; }
+
+	public:
+		Buffer<Rule> mRules;
+		String mStartTerminal;
+		bool mIsLooped = false;
+	};
+
+	template <typename tAlphabetType, typename tTokType>
+	class RegularGrammar {
+	public:
+		struct Node {
+			enum Type {
+				NONE,
+				ANY,
+				OR,
+				IF,
+				CLASS,
+				COMPOUND,
+				REPEAT,
+				VAL,
+			} mType = NONE;
+
+			explicit Node(Type type) :
+				mType(type) {}
+
+			virtual ~Node() = default;
+		};
+
+		class ValueNode : public Node {
+		public:
+			explicit ValueNode(tAlphabetType val) :
+				mVal(val),
+				Node(Node::VAL) {}
+
+			~ValueNode() override = default;
+
+		public:
+			tAlphabetType mVal;
+		};
+
+		class CompoundNode : public Node {
+		public:
+			CompoundNode() :
+				Node(Node::COMPOUND) {}
+
+			CompoundNode(const InitialierList<const Node*>& nodes) :
+				Node(Node::COMPOUND) {
+				mSequence = nodes;
+			}
+
+			~CompoundNode() override {
+				for (auto iter : mSequence) {
+					delete iter.data();
+				}
+				mSequence.clear();
+			}
+
+		public:
+			Buffer<const Node*> mSequence;
+		};
+
+		class AlternationNode : public Node {
+		public:
+			AlternationNode() :
+				Node(Node::OR) {}
+
+			AlternationNode(const Node* a, const Node* b) :
+				Node(Node::OR) {
+				mFirst = a;
+				mSecond = b;
+			}
+
+			~AlternationNode() override {
+				delete mFirst;
+				delete mSecond;
+			}
+
+		public:
+			const Node* mFirst = nullptr;
+			const Node* mSecond = nullptr;
+		};
+
+		class IfNode : public Node {
+		public:
+			IfNode() :
+				Node(Node::IF) {}
+
+			explicit IfNode(const Node* a) :
+				Node(Node::IF) {
+				mNode = a;
+			}
+
+			~IfNode() override { delete mNode; }
+
+		public:
+			const Node* mNode = nullptr;
+		};
+
+		class AnyNode : public Node {
+		public:
+			AnyNode() :
+				Node(Node::ANY) {}
+
+			~AnyNode() override = default;
+		};
+
+		class RepetitionNode : public Node {
+		public:
+			RepetitionNode() :
+				Node(Node::REPEAT) {}
+
+			explicit RepetitionNode(const Node* rep, bool plus = false) :
+				Node(Node::REPEAT) {
+				mNode = rep;
+				mPlus = plus;
+			}
+
+			~RepetitionNode() override { delete mNode; }
+
+		public:
+			Node* mNode = nullptr;
+			bool mPlus = false;
+		};
+
+		class ClassNode : public Node {
+		public:
+			ClassNode() :
+				Node(Node::CLASS) {}
+
+			explicit ClassNode(const Buffer<Range<tAlphabetType>>& ranges, bool exclude = false) :
+				Node(Node::CLASS) {
+				mExclude = exclude;
+				mRanges = ranges;
+			}
+
+			~ClassNode() override { mRanges.removeAll(); }
+
+		public:
+			Buffer<Range<tAlphabetType>> mRanges;
+			bool mExclude = false;
+		};
+
+	public:
+		RegularGrammar() = default;
+
+		~RegularGrammar() {
+			for (auto rule : mRules) {
+				delete rule->t1;
+			}
+		}
+
+		void addRule(const Node* node, tTokType id) { mRules.append({ node, id }); }
+
+	public:
+		const Node* seq(const InitialierList<const Node*>& nodes) { return new CompoundNode(nodes); }
+		const Node* val(tAlphabetType in) { return new ValueNode(in); }
+		const Node* alt(const Node* a, const Node* b) { return new AlternationNode(a, b); }
+		const Node* may(const Node* a) { return new IfNode(a); }
+		const Node* any() { return new AnyNode(); }
+		const Node* rep(const Node* rep, bool plus = false) { return new RepetitionNode(rep, plus); }
+		const Node* ranges(const Buffer<Range<tAlphabetType>>& ranges, bool exclude = false) {
+			return new ClassNode(ranges, exclude);
+		}
+
+	public:
+		Buffer<Pair<const Node*, tTokType>> mRules;
+	};
+}

.wip/Language/public/LanguageCommon.hpp

@@ -0,0 +1,7 @@
+#pragma once
+
+#include "Module.hpp"
+
+namespace tp {
+	extern ModuleManifest gModuleLanguage;
+}

.wip/Language/public/Parser.hpp

@@ -0,0 +1,107 @@
+
+#pragma once
+
+#include "RegularCompiler.hpp"
+#include "RegularAutomata.hpp"
+
+#include "ContextFreeCompiler.hpp"
+#include "ContextFreeAutomata.hpp"
+
+namespace tp {
+
+	template <typename tAlphabetType, typename tTokenType, tTokenType tInTransition, ualni MinSymbol, ualni MaxSymbol>
+	class Parser {
+
+		typedef RegularGrammar<tAlphabetType, tTokenType> RegularGrammar;
+		typedef RegularCompiler<tAlphabetType, tTokenType, tInTransition, MinSymbol, MaxSymbol> RegularCompiler;
+		typedef FiniteStateAutomation<tAlphabetType, tTokenType> RegularGraph;
+		typedef RegularAutomata<tAlphabetType, tTokenType> RegularAutomata;
+
+		// ContextFreeGrammar;
+		// ContextFreeCompiler;
+		typedef FiniteStateAutomation<ContextFreeCompiler::SymbolVal, ContextFreeCompiler::Item> ContextFreeGraph;
+		typedef ContextFreeAutomata<ContextFreeCompiler::SymbolVal, ContextFreeCompiler::Item> ContextFreeAutomata;
+
+	public:
+		struct ParseResult {
+			bool accepted = false;
+			const ContextFreeAutomata::StackItem* ast = nullptr;
+		};
+
+	public:
+		Parser() = default;
+
+	public:
+		bool compileTables(
+			const ContextFreeGrammar& cfGrammar,
+			const RegularGrammar& reGrammar,
+			const Map<String, tTokenType>& contextFreeToRegular
+		) {
+			// Compile Regular Grammar
+			{
+				RegularGraph graph;
+				RegularCompiler compiler;
+				compiler.compile(graph, reGrammar);
+				graph.makeDeterministic();
+				mRegularAutomata.construct(graph);
+			}
+
+			// compile context free grammar
+			{
+				ContextFreeGraph graph;
+				ContextFreeCompiler compiler;
+				compiler.compile(cfGrammar, graph);
+				graph.makeDeterministic();
+				mContextFreeAutomata.construct(graph);
+
+				// make glue
+				for (auto symbol : *compiler.getSymbols()) {
+					auto symbolId = compiler.getSymbolId(symbol->mId);
+					if (symbol->mIsTerminal) {
+						auto iter = contextFreeToRegular.presents(symbol->mId);
+						if (!iter) return false;
+						mGrammarGlue.put(contextFreeToRegular.getSlotVal(iter), symbolId);
+					} else {
+						mAstNames.put(symbolId, symbol->mId);
+					}
+				}
+			}
+
+			return true;
+		}
+
+		ParseResult parse(const tAlphabetType* sentence, ualni sentenceLength) {
+			// get tokens stream
+			Buffer<ContextFreeCompiler::SymbolVal> tokens;
+
+			const tAlphabetType* sentenceIter = sentence;
+			ualni lengthIter = sentenceLength;
+
+			while (lengthIter) {
+				auto result = mRegularAutomata.accept(sentenceIter, lengthIter);
+
+				if (!result.accepted) {
+					return { false, nullptr };
+				}
+
+				tokens.append(ContextFreeCompiler::SymbolVal(
+					mGrammarGlue.get(result.state), ualni(sentenceIter - sentence), result.advancedIdx
+				));
+
+				sentenceIter += result.advancedIdx;
+				lengthIter -= result.advancedIdx;
+			}
+
+			ContextFreeAutomata::AcceptResult result = mContextFreeAutomata.accept(tokens.getBuff(), tokens.size());
+			return { result.accepted, result.ast };
+		}
+
+	public:
+		// save load compiled tables
+		RegularAutomata mRegularAutomata;
+		ContextFreeAutomata mContextFreeAutomata;
+
+		Map<tTokenType, ContextFreeCompiler::SymbolVal> mGrammarGlue;
+		Map<ContextFreeCompiler::SymbolVal, String> mAstNames;
+	};
+}

.wip/Language/public/RegularAutomata.hpp

@@ -0,0 +1,101 @@
+
+#pragma once
+
+#include "Strings.hpp"
+#include "Automata.hpp"
+#include "Buffer2D.hpp"
+
+namespace tp {
+
+	template <typename tAlphabetType, typename tStateType>
+	class RegularAutomata {
+	public:
+		struct AcceptResult {
+			bool accepted = false;
+			ualni advancedIdx = 0;
+			tStateType state = tStateType();
+		};
+
+	public:
+		RegularAutomata() = default;
+
+		AcceptResult accept(const tAlphabetType* stream, ualni size) {
+			mCurrentState = mStartState;
+
+			ualni advancedIdx = 0;
+
+			while (advancedIdx < size) {
+				const tAlphabetType& symbol = *(stream + advancedIdx);
+
+				if (!(symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd)) {
+					return { false, advancedIdx, {} };
+				}
+
+				mCurrentState = mTable.get({ (ualni) (symbol - mSymbolRange.mBegin), mCurrentState });
+
+				if (mCurrentState == mStates.size()) {
+					return { false, advancedIdx, {} };
+				}
+
+				if (mStates[mCurrentState].first) {
+					return { true, advancedIdx, mStates[mCurrentState].second };
+				}
+
+				advancedIdx++;
+			}
+
+			return { false, advancedIdx, {} };
+		}
+
+	public:
+		void construct(const FiniteStateAutomation<tAlphabetType, tStateType>& automata) {
+			const auto range = automata.getAlphabetRange();
+			mSymbolRange = { tAlphabetType(range.mBegin), tAlphabetType(range.mEnd) };
+
+			auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin);
+			auto sizeX = range_len ? range_len : 1;
+			auto sizeY = (ualni) (automata.numStates());
+
+			mTable.reserve({ sizeX, sizeY });
+			mTable.assign(automata.numStates());
+			mStates.reserve(sizeY);
+
+			ualni idx = 0;
+			for (auto state : *automata.getStates()) {
+				mStates[idx] = { state->isAccepting(), state->getStateVal() };
+				idx++;
+			}
+
+			idx = 0;
+			for (auto state : *automata.getStates()) {
+				if (&state.data() == automata.getStartState()) {
+					mStartState = mCurrentState = idx;
+				}
+				idx++;
+			}
+
+			ualni stateIdx = 0;
+			for (auto state : *automata.getStates()) {
+				for (auto transition : *state->getTransitions()) {
+					ualni stateIdx2 = 0;
+					for (auto state2 : *automata.getStates()) {
+						if (transition->getState() == &state2.data()) break;
+						stateIdx2++;
+					}
+					auto const code = transition->getSymbol();
+					mTable.set({ (ualni) (code - mSymbolRange.mBegin), (ualni) stateIdx }, stateIdx2);
+				}
+				stateIdx++;
+			}
+		}
+
+	private:
+		Buffer2D<ualni> mTable;
+		Buffer<Pair<bool, tStateType>> mStates;
+
+		ualni mCurrentState = 0;
+		ualni mStartState = 0;
+
+		Range<tAlphabetType> mSymbolRange = { 0, 0 };
+	};
+}

.wip/Language/public/RegularCompiler.hpp

@@ -0,0 +1,208 @@
+
+#pragma once
+
+#include "Grammar.hpp"
+#include "Automata.hpp"
+
+namespace tp {
+
+	template <typename tAlphabetType, typename tStateType, tStateType tInTransition, ualni tMinSymbol, ualni tMaxSymbol>
+	class RegularCompiler {
+
+		typedef FiniteStateAutomation<tAlphabetType, tStateType> Graph;
+		typedef typename Graph::State Vertex;
+		typedef RegularGrammar<tAlphabetType, tStateType> Grammar;
+
+		struct Node {
+			Vertex* left = nullptr;
+			Vertex* right = nullptr;
+		};
+
+	private:
+		Graph* mGraph = nullptr;
+
+	public:
+		struct CompileError {
+			uhalni mRuleIndex = 0;
+			tStateType mRuleState;
+			const char* description = nullptr;
+			[[nodiscard]] bool isError() const { return description; }
+		};
+
+		CompileError mError;
+
+		void compile(Graph& graph, const tAlphabetType* regex, tStateType state) {
+			mGraph = &graph;
+			compileUtil(regex, state);
+		}
+
+		void compile(Graph& aGraph, const Grammar& grammar) {
+			mGraph = &aGraph;
+
+			auto left = addVertex();
+			auto right = addVertex();
+
+			halni idx = 0;
+			for (auto rule : grammar.mRules) {
+
+				auto node = compileUtil(rule.data().first, rule.data().second);
+
+				if (!(node.left && node.right)) {
+					mError.mRuleIndex = idx;
+					return;
+				}
+
+				transitionAny(left, node.left);
+				transitionAny(node.right, right);
+
+				idx++;
+			}
+
+			mGraph->setStartState(left);
+		}
+
+	private:
+		Node compileUtil(const Grammar::Node* astNode, tStateType state) {
+
+			auto node = compileNode(astNode, nullptr, nullptr);
+
+			node.right->setValue(state);
+			node.right->setAcceptance(true);
+
+			mGraph->setStartState(node.left);
+
+			return node;
+		}
+
+		Node compileVal(Grammar::ValueNode* val, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+			auto left = aLeft ? aLeft : addVertex();
+			auto right = aRight ? aRight : addVertex();
+			transitionVal(left, right, val->mVal);
+			return { left, right };
+		}
+
+		Node compileAlternation(const Grammar::AlternationNode* alt, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+			auto first_node = compileNode(alt->mFirst, aLeft, aRight);
+			auto second_node = compileNode(alt->mSecond);
+			transitionAny(first_node.left, second_node.left);
+			transitionAny(second_node.right, first_node.right);
+			return first_node;
+		}
+
+		Node compileAny(const Grammar::AnyNode*, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+			auto left = aLeft ? aLeft : addVertex();
+			auto right = aRight ? aRight : addVertex();
+			transitionAny(left, right, true);
+			return { left, right };
+		}
+
+		Node compileRepeat(const Grammar::RepetitionNode* repeat, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+			if (repeat->mPlus) {
+				auto middle = addVertex();
+
+				auto left_node = compileNode(repeat->mNode, aLeft, middle);
+
+				auto right_node = compileNode(repeat->mNode, middle, aRight);
+				transitionAny(right_node.right, right_node.left);
+				transitionAny(right_node.left, right_node.right);
+
+				return { left_node.left, right_node.right };
+			} else {
+				auto node = compileNode(repeat->mNode, aLeft, aRight);
+				transitionAny(node.right, node.left);
+				transitionAny(node.left, node.right);
+				return node;
+			}
+		}
+
+		Node compileIf(const Grammar::IfNode* ifNode, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+			auto node = compileNode(ifNode->mNode, aLeft, aRight);
+			transitionAny(node.left, node.right);
+			return node;
+		}
+
+		Node compileClass(const Grammar::ClassNode* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+			auto left = aLeft ? aLeft : addVertex();
+			auto right = aRight ? aRight : addVertex();
+
+			if (node->mRanges.size() == 1) {
+				auto const& range = node->mRanges.first();
+				transitionRange(left, right, { ualni(range.mBegin), ualni(range.mEnd) }, node->mExclude);
+				return { left, right };
+			}
+
+			for (auto range : node->mRanges) {
+				auto middle = addVertex();
+				transitionRange(left, middle, { ualni(range->mBegin), ualni(range->mEnd) }, node->mExclude);
+				transitionAny(middle, right);
+			}
+			return { left, right };
+		}
+
+		Node compileCompound(const Grammar::CompoundNode* compound, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+			Vertex* left = nullptr;
+			Vertex* rigth = nullptr;
+
+			ualni idx = 0;
+			for (auto child : compound->mSequence) {
+				auto pass_left = idx == 0 ? aLeft : rigth;
+				auto pass_right = idx == compound->mSequence.size() - 1 ? aRight : nullptr;
+				auto node = compileNode(child.data(), pass_left, pass_right);
+				if (!left) left = node.left;
+				rigth = node.right;
+				idx++;
+			}
+
+			return { left, rigth };
+		}
+
+		Node compileNode(const Grammar::Node* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
+			switch (node->mType) {
+				case Grammar::Node::CLASS: return compileClass((typename Grammar::ClassNode*) node, aLeft, aRight);
+				case Grammar::Node::COMPOUND: return compileCompound((typename Grammar::CompoundNode*) node, aLeft, aRight);
+				case Grammar::Node::IF: return compileIf((typename Grammar::IfNode*) node, aLeft, aRight);
+				case Grammar::Node::REPEAT: return compileRepeat((typename Grammar::RepetitionNode*) node, aLeft, aRight);
+				case Grammar::Node::ANY: return compileAny((typename Grammar::AnyNode*) node, aLeft, aRight);
+				case Grammar::Node::OR: return compileAlternation((typename Grammar::AlternationNode*) node, aLeft, aRight);
+				case Grammar::Node::VAL: return compileVal((typename Grammar::ValueNode*) node, aLeft, aRight);
+				case Grammar::Node::NONE: break;
+			}
+			ASSERT(0)
+			return {};
+		}
+
+		void transitionAny(Vertex* from, Vertex* to, bool consumes = false) {
+			for (auto symbol : Range<ualni>(tMinSymbol, tMaxSymbol)) {
+				transitionVal(from, to, symbol);
+			}
+		}
+
+		void transitionVal(Vertex* from, Vertex* to, tAlphabetType val) { mGraph->addTransition(from, to, val); }
+
+		void transitionRange(Vertex* from, Vertex* to, Range<ualni> range, bool exclude) {
+			if (exclude) {
+				Range<ualni> first = { tMinSymbol, range.mBegin - 1 };
+				Range<ualni> second = { range.mEnd + 1, tMaxSymbol };
+
+				if (first.valid()) {
+					for (auto symbol : first) {
+						transitionVal(from, to, symbol);
+					}
+				}
+
+				if (second.valid()) {
+					for (auto symbol : second) {
+						transitionVal(from, to, symbol);
+					}
+				}
+
+			} else {
+				for (auto symbol : range) {
+					transitionVal(from, to, symbol);
+				}
+			}
+		}
+
+		Vertex* addVertex() { return mGraph->addState(tInTransition, false); }
+	};
+}

.wip/Language/public/SimpleParser.hpp

@@ -0,0 +1,69 @@
+#pragma once
+
+#include "Parser.hpp"
+
+namespace tp {
+
+	// Gives ability to express grammar in the Unified Format as sentence
+	template <typename tAlphabetType>
+	class SimpleParser {
+		enum UGTokens : alni { InTransition = -1, TestSeq };
+
+		typedef Parser<tAlphabetType, UGTokens, InTransition, 0, 127> UGParser;
+		typedef Parser<tAlphabetType, alni, -1, 0, 127> UserParser;
+
+	public:
+		SimpleParser() {
+			// Grammar for unified grammar format sentence that tables compiled from
+
+			// Define Context-Free grammar
+			ContextFreeGrammar contextFreeGrammar;
+			{
+				// use existing CF grammar interface
+				contextFreeGrammar.addRule("a", { ContextFreeGrammar::Arg("") });
+				contextFreeGrammar.setStart("a");
+			}
+
+			// Define Regular grammar
+			RegularGrammar<tAlphabetType, UGTokens> regularGrammar;
+			{
+				// this is basically ast from existing tokenizer
+				regularGrammar.addRule(regularGrammar.seq({ regularGrammar.val('a'), regularGrammar.val('b') }), TestSeq);
+			}
+
+			Map<String, UGTokens> terminalsMap;
+			terminalsMap.put("TestSeq", TestSeq);
+
+			mUnifiedGrammarParser.compileTables(contextFreeGrammar, regularGrammar, terminalsMap);
+		}
+
+	public:
+		void compileTables(const tAlphabetType* grammar, ualni grammarLength) {
+			mUnifiedGrammarParser.parse(grammar, grammarLength);
+
+			// compile each ast into RegularGrammar and ContextFree Grammar api instructions
+			ContextFreeGrammar userContextFreeGrammar;
+			RegularGrammar<tAlphabetType, alni> userRegularGrammar;
+
+			// ...
+			// split ast into RE and CF part
+			// generate and execute grammar api commands
+			// use existing tokenizer code to create RE transition matrix
+			// ...
+			// compile tables from user grammar
+
+			Map<String, alni> terminalsMap;
+			terminalsMap.put("TestSeq", 0);
+
+			mUserParser.compileTables(userContextFreeGrammar, userRegularGrammar, terminalsMap);
+		}
+
+		UserParser::ParseResult parse(const tAlphabetType* grammar, ualni grammarLength) {
+			return mUserParser.parse(grammar, grammarLength);
+		}
+
+	private:
+		UGParser mUnifiedGrammarParser;
+		UserParser mUserParser;
+	};
+}

.wip/Language/tests/Test.hpp

@@ -0,0 +1,43 @@
+#pragma once
+
+// #include "SimpleParser.hpp"
+
+const char* gGrammar = R"(
+# Grammar in the CF-RE United Format (Defined by language module)
+
+Rules : {
+	ScopeList : ScopeList Scope | Scope ;
+	Scope : \ScopeBegin StatementList \ScopeEnd;
+	StatementList : StatementList Statement \StatementEnd;
+	StatementList : Statement \StatementEnd;
+	Statement : \StatementBody;
+}
+
+Terminals : {
+	Space : " " | "\t" | "\n" | "\r";
+	ScopeBegin : "{";
+	ScopeEnd : "}";
+  StatementEnd : ";";
+  StatementBody : "a" | "b";
+}
+
+Start : Scope;
+Ignore : Space;
+)";
+
+const char* gSentence = R"(
+{}
+
+{ }
+
+{
+a;
+ a ; a ;
+
+ }
+
+ {
+a;
+ a;
+}
+)";

.wip/Language/tests/Tests.cpp

@@ -0,0 +1,45 @@
+
+#include "Test.hpp"
+
+/*
+using namespace tp;
+
+void testAutomation() {
+
+	FiniteStateAutomation<char, int> automata;
+
+	auto start = automata.addState(0, false);
+	auto end = automata.addState(1, true);
+
+	automata.addTransition(start, end, 'a');
+
+	automata.setStartState(start);
+
+	automata.makeDeterministic();
+}
+
+void test() {
+	auto parser = SimpleParser<int1>();
+
+	parser.compileTables(gGrammar, String::Logic::calcLength(gGrammar));
+	auto result = parser.parse(gSentence, String::Logic::calcLength(gSentence));
+}
+
+int main() {
+
+	tp::ModuleManifest* deps[] = { &tp::gModuleLanguage, nullptr };
+	tp::ModuleManifest testModule("Test", nullptr, nullptr, deps);
+
+	if (!testModule.initialize()) {
+		return 1;
+	}
+
+	testAutomation();
+	test();
+
+	testModule.deinitialize();
+}
+
+*/
+
+int main() { return 0; }