CF Grammar Initial

CMakeLists.txt

@@ -19,6 +19,7 @@ add_subdirectory(Math)
 add_subdirectory(Allocators)
 add_subdirectory(Strings)
 add_subdirectory(Tokenizer)
+add_subdirectory(Parser)
 add_subdirectory(CommandLine)
 add_subdirectory(Externals)
 add_subdirectory(Connection)

Parser/CMakeLists.txt

@@ -10,7 +10,7 @@ file(GLOB SOURCES "./private/*.cpp")
 file(GLOB HEADERS "./public/*.hpp")
 add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
 target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
-target_link_libraries(${PROJECT_NAME} PUBLIC Strings)
+target_link_libraries(${PROJECT_NAME} PUBLIC Tokenizer)
 
 ### -------------------------- Tests  --------------------------  ###
 enable_testing()

Parser/private/CfGrammarParser.cpp

@@ -0,0 +1,190 @@
+
+#include "NewPlacement.hpp"
+
+#include "CfGrammar.hpp"
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+enum class Token {
+	NO_TOKEN,
+	START,
+	ID,
+	EQUAL,
+	TERMINAL_START,
+	TERMINAL_END,
+	ALTERNATION,
+	RULE_END,
+	IGNORED,
+	END,
+	FAILED,
+};
+
+typedef int1 AlphabetType;
+typedef SimpleTokenizer<AlphabetType, Token, Token::NO_TOKEN, Token::FAILED, Token::END> CFGTokenizer;
+
+struct State {
+
+	struct Transition {
+		Token tok = Token::FAILED;
+		State* target = nullptr;
+		void (*action)(CfGrammar* grammar, const String& tok) = nullptr;
+	};
+
+	String error;
+	Buffer<Transition> transitions;
+	bool accept = false;
+};
+
+typedef Buffer<State> Automata;
+
+void initializeTokenizer(CFGTokenizer* tok) {
+	typedef decltype(tok->getTokenizer()) TokBase;
+	tok->build({
+			{ TokBase::idRE, Token::ID },
+			{ TokBase::etherRE, Token::IGNORED },
+			{ "Start", Token::START },
+			{ ":", Token::EQUAL },
+			{ "[", Token::TERMINAL_START },
+			{ "]", Token::TERMINAL_END },
+			{ "|", Token::ALTERNATION },
+			{ ";", Token::RULE_END },
+	});
+}
+
+void initAutomata(Automata& automata) {
+	automata.reserve(8);
+	auto states = &automata.first();
+
+	auto root = states++;
+	auto end = states++;
+	auto start = states++;
+	auto startEnd = states++;
+	auto rule = states++;
+	auto rhs = states++;
+	auto terminalStart = states++;
+	auto terminalEnd = states;
+
+	root->transitions = {
+			{ Token::END, end },
+			{ Token::START, start },
+			{ Token::ID, rule, [](CfGrammar* grammar, const String& tok) {
+				 grammar->rules.pushBack({ tok, {} });
+			 }
+			},
+	};
+
+	start->transitions = {
+			{ Token::ID, startEnd},
+	};
+
+	startEnd->transitions = {
+			{ Token::RULE_END,
+			 root,
+			 [](CfGrammar* grammar, const String& tok) {
+				 grammar->startTerminal = tok;
+			 }
+			},
+	};
+
+	rule->transitions = {
+			{ Token::EQUAL, rhs },
+	};
+
+	rhs->transitions = {
+			{ Token::ALTERNATION, rhs, [](CfGrammar* grammar, const String& tok) {
+				 auto const& id = grammar->rules.last()->data.id;
+				 grammar->rules.pushBack({ id, {} });
+			 }
+			},
+			{ Token::ID, rhs, [](CfGrammar* grammar, const String& tok) {
+				 grammar->rules.last()->data.args.pushBack({ tok, false });
+			 }
+			},
+			{ Token::TERMINAL_START, terminalStart },
+			{ Token::RULE_END, root },
+	};
+
+	terminalStart->transitions = {
+			{ Token::ID, rhs, [](CfGrammar* grammar, const String& tok) {
+				 grammar->rules.last()->data.args.pushBack({ tok, true });
+			 }
+			},
+	};
+
+	terminalEnd->transitions = {
+			{ Token::TERMINAL_END, rhs },
+	};
+
+	auto idError = "Expected an identifier";
+	auto stmEndError = "Expected an statement end ';'";
+
+	root->error = "Expected 'Start' statement, rule or end of text";
+	start->error = idError;
+	startEnd->error = stmEndError;
+	rule->error = "Expected production equality ':'";
+	rhs->error = "Expected alternation '|', terminal id, statement end ';' or terminal start '[' ";
+	terminalStart->error = idError;
+	terminalEnd->error = "Expected terminal end '[' ";
+
+	end->accept = true;
+}
+
+bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGrammar* grammar) {
+	tok->reset();
+	tok->bindSource(text);
+
+	State* state = &(*automata)[0];
+	while (!state->accept) {
+
+		auto token = tok->readTok();
+
+		if (token == Token::FAILED) {
+			auto errorLocation = tok->getCursorPrev().get2DLocation();
+			printf("Syntax error at (%llu, %llu)\n", errorLocation.line, errorLocation.character);
+			return false;
+		}
+
+		auto tokVal = tok->extractVal();
+
+		bool isTransition = false;
+
+		for (auto transition : state->transitions) {
+			if (transition->tok == token) {
+				transition->action(grammar, tokVal);
+				state = transition->target;
+				isTransition = true;
+				break;
+			}
+		}
+
+		if (!isTransition) {
+			// if no tok matched to any transition produce error
+			auto errorLocation = tok->getCursorPrev().get2DLocation();
+			printf("Parse error at (%llu, %llu) - %s", errorLocation.line, errorLocation.character, state->error.read());
+			return false;
+		}
+	}
+
+	return true;
+}
+
+struct tp::CfGrammarParserState {
+	Automata automata;
+	CFGTokenizer tokenizer;
+};
+
+CfGrammarParserState* CfGrammar::initializeCfGrammarParser() {
+	auto state = new CfGrammarParserState();
+	initializeTokenizer(&state->tokenizer);
+	initAutomata(state->automata);
+	return state;
+}
+
+void CfGrammar::deinitializeCfGrammarParser(CfGrammarParserState* state) {
+	delete state;
+}
+
+void CfGrammar::parse(CfGrammarParserState* context, const int1* source) {
+	::parse(source, &context->automata, &context->tokenizer, this);
+}

Parser/public/CfGrammar.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "Strings.hpp"
+#include "List.hpp"
+
+namespace tp {
+
+	struct CfGrammar {
+		struct Rule {
+			struct Arg {
+				String id;
+				bool isTerminal;
+			};
+
+			String id;
+			List<Arg> args;
+		};
+
+		List<Rule> rules;
+		String startTerminal;
+
+	public:
+		static struct CfGrammarParserState* initializeCfGrammarParser();
+		static void deinitializeCfGrammarParser(CfGrammarParserState*);
+
+	public:
+		void parse(CfGrammarParserState* context, const int1* source);
+	};
+}

Parser/public/Parser.hpp

@@ -1,105 +0,0 @@
-#pragma once
-
-#include "Strings.hpp"
-#include "Map.hpp"
-#include "List.hpp"
-
-namespace tp {
-
-	template <typename tAlphabetType>
-	class ContextFreeGrammar {
-	public:
-		class NonTerminalSymbol;
-
-		class RHSArg {
-			tAlphabetType terminal;
-			const NonTerminalSymbol* nonTerminal = nullptr;
-		public:
-			RHSArg(const NonTerminalSymbol* aNonTerminal) : nonTerminal(aNonTerminal) {}
-			RHSArg(tAlphabetType terminal) : terminal(terminal) {}
-		};
-
-		class NonTerminalSymbol {
-			String name;
-			String description;
-
-			struct Rule {
-				List<RHSArg> mArgs;
-			};
-
-			List<Rule> mRules;
-
-			Rule* newRule() {
-				mRules.pushBack();
-				return mRules.last()->data;
-			}
-		};
-
-	private:
-		Map<String, NonTerminalSymbol> mNonTerminals;
-		NonTerminalSymbol* mStart;
-
-	public:
-		ContextFreeGrammar() = default;
-
-		void setStartNonTerminal(const NonTerminalSymbol* start) {
-			mStart = start;
-		}
-
-		const NonTerminalSymbol* addRule(const String& LHS, const InitialierList<RHSArg> RHS) {
-			NonTerminalSymbol* nonTerminal = nullptr;
-			auto idx = mNonTerminals.presents(LHS);
-			if (idx) {
-				nonTerminal = mNonTerminals.get(idx);
-			} else {
-				nonTerminal = new NonTerminalSymbol(LHS, "None");
-				mNonTerminals.put(LHS, nonTerminal);
-			}
-
-			auto* rule = nonTerminal->newRule();
-			for (auto arg : RHS) {
-				rule->pushBack(arg);
-			}
-
-			return nonTerminal;
-		}
-
-		bool checkSemantic() {
-			return false;
-		}
-
-		bool checkNonAmbiguous() {
-			return false;
-		}
-	};
-
-	template <typename tAlphabetType>
-	class Parser {
-		typedef ContextFreeGrammar<tAlphabetType> Grammar;
-
-	public:
-		class ASTNode {
-			const Grammar::NonTerminalSymbol* mNonTerminal;
-			List<const ASTNode*> mLeafs;
-		};
-
-	private:
-		class InputTape {
-		};
-
-		class StateStack {
-		};
-
-	private:
-		Grammar mGrammar;
-
-	public:
-		Parser() = default;
-
-		Grammar& getGrammar() { return mGrammar; }
-
-		void buildTables() {}
-
-		const ASTNode* parse() {}
-	};
-}

Tokenizer/public/Tokenizer.hpp

@@ -19,6 +19,15 @@ namespace tp {
 			return mTransitionMatrix.isTrapped();
 		}
 
+	public:
+		// Some useful RE to be reused
+		static constexpr const tAlphabetType* etherRE = "\n|\t| |\r";
+		static constexpr const tAlphabetType* intRE = "((\\-)|(\\+))?[0-9]+i?";
+		static constexpr const tAlphabetType* floatRE = R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)";
+		static constexpr const tAlphabetType* commentBlockRE = R"((/\*){\*-\*}*(\*/))";
+		static constexpr const tAlphabetType* stringRE = R"("{"-"}*")";
+		static constexpr const tAlphabetType* idRE =  "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*";
+
 	public:
 
 		Tokenizer() {
@@ -85,9 +94,30 @@ namespace tp {
 			const tAlphabetType* mSource = nullptr;
 			ualni mAdvancedOffset = 0;
 			const tAlphabetType* str() { return mSource + mAdvancedOffset; }
+
+			struct Location2D {
+				ualni line = 0;
+				ualni character = 0;
+			};
+
+			Location2D get2DLocation() {
+				Location2D out;
+				typedef typename StringLogic<tAlphabetType>::Index Idx;
+				Buffer<Idx> offsets;
+				StringLogic<tAlphabetType>::calcLineOffsets(mSource, StringLogic<tAlphabetType>::calcLength(mSource), offsets);
+				for (ualni idx = 1; idx < offsets.size(); idx++) {
+					if (offsets[idx] > mAdvancedOffset) {
+						out.line = idx - 1;
+						break;
+					}
+				}
+				out.character = mAdvancedOffset - offsets[out.line];
+				return out;
+			}
 		};
 
 		SimpleTokenizer() = default;
+		auto getTokenizer() const { return mTokenizer; }
 
 		void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) {
 			mTokenizer.build(rules);