CF Grammar improvements

CommandLine/private/CommandLine.cpp

@@ -337,6 +337,7 @@ void CommandLine::parseArg(Arg& arg, const char* src) {
 	auto tok = mTokenizer.readTok();
 	if (tok < TokType::INT || tok > TokType::STR) {
 		ErrInvalidArgSyntax(&arg);
+		return;
 	}
 
 	auto val = mTokenizer.extractVal();

Parser/CMakeLists.txt

@@ -12,6 +12,12 @@ add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
 target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
 target_link_libraries(${PROJECT_NAME} PUBLIC Tokenizer)
 
+### -------------------------- Applications  --------------------------  ###
+add_executable(cfg ./applications/Cfg.cpp )
+target_link_libraries(cfg ${PROJECT_NAME} CommandLine)
+
+configure_file(rsc/grammar.txt grammar.txt COPYONLY)
+
 ### -------------------------- Tests  --------------------------  ###
 enable_testing()
 file(GLOB TEST_SOURCES "./tests/*.cpp")

Parser/applications/Cfg.cpp

@@ -0,0 +1,72 @@
+
+#include "NewPlacement.hpp"
+
+#include "Parser.hpp"
+#include "CommandLine.hpp"
+
+using namespace tp;
+
+void run(const String& source) {
+	auto state = CfGrammar::initializeCfGrammarParser();
+
+	CfGrammar grammar;
+
+	if (!grammar.parse(state, source)) {
+		printf("Parsing is failed\n");
+		return;
+	}
+
+	if (!grammar.compile()) {
+		printf("Compilation is failed\n");
+		return;
+	}
+
+	printf("Grammar is accepted.\n");
+
+	printf("Example text formed from grammar : TODO\n");
+
+	CfGrammar::deinitializeCfGrammarParser(state);
+}
+
+int main(int argc, const char* argv[]) {
+	tp::ModuleManifest* deps[] = { &tp::gModuleParser, &tp::gModuleCommandLine, nullptr };
+	tp::ModuleManifest testModule("CommandLineTest", nullptr, nullptr, deps);
+
+	if (!testModule.initialize()) {
+		return 1;
+	}
+
+	{
+		CommandLine cmd = {
+				{ "grammar", CommandLine::Arg::STR },
+		};
+
+		if (!cmd.parse((char) argc, argv, true, 1)) {
+			return 1;
+		}
+
+		auto location = cmd.getString("grammar");
+
+		LocalConnection file;
+		String grammar;
+
+		file.connect(LocalConnection::Location(location), LocalConnection::Type(true));
+
+		if (!file.getConnectionStatus().isOpened()) {
+			printf("Cant open file\n");
+			return 1;
+		}
+
+		auto size = file.size();
+		grammar.resize((String::Index) size);
+		file.readBytes(grammar.write(), size);
+
+
+		run(grammar);
+
+		file.disconnect();
+	}
+
+	testModule.deinitialize();
+	return 0;
+}

Parser/private/CfGrammar.cpp

@@ -0,0 +1,62 @@
+
+#include "NewPlacement.hpp"
+#include "CfGrammar.hpp"
+
+using namespace tp;
+
+bool CfGrammar::compile() {
+
+	if (!rules.length()) {
+		printf("Grammar must have at leas one rule\n");
+		return false;
+	}
+
+	if (startTerminal == String()) {
+		printf("Using first rule's non-terminal as grammar's root. Define explicitly - 'Start : id'\n");
+		startTerminal = rules.first()->data.id;
+	}
+
+	// find all rules
+	for (auto rule : rules) {
+
+		if (!rule->args.length()) {
+			printf("Rule must have at leas one terminal or non-terminal. See rule '%s'\n", rule->id.read());
+			return false;
+		}
+
+		if (!mNonTerminals.presents(rule->id)) {
+			mNonTerminals.put(rule->id, {});
+		}
+		auto nonTerminal = &mNonTerminals.get(rule->id);
+
+		nonTerminal->rules.pushBack(&rule.data());
+	}
+
+	for (auto nonTerminal : mNonTerminals) {
+		for (auto rule : nonTerminal->val.rules) {
+			for (auto arg : rule->args) {
+
+				if (arg->isTerminal) {
+					mTerminals.put(rule->id, {});
+				} else {
+					auto idx = mNonTerminals.presents(arg->id);
+					if (!idx) {
+						printf("Referenced non-terminal '%s' is not defined\n", arg->id.read());
+						return false;
+					}
+					auto reference = &mNonTerminals.getSlotVal(idx);
+					reference->references.put(nonTerminal->key, &nonTerminal->val);
+					nonTerminal->val.referencing.put(arg->id, reference);
+				}
+			}
+		}
+	}
+
+	for (auto nonTerminal : mNonTerminals) {
+		if (!nonTerminal->val.references.size() && nonTerminal->key != startTerminal) {
+			printf("Non-terminal '%s' is defined but not used\n", nonTerminal->key.read());
+		}
+	}
+
+	return true;
+}

Parser/private/CfGrammarParser.cpp

@@ -16,6 +16,8 @@ enum class Token {
 	ALTERNATION,
 	RULE_END,
 	IGNORED,
+	EPSILON,
+	COMMENT,
 	END,
 	FAILED,
 };
@@ -31,6 +33,7 @@ struct State {
 		void (*action)(CfGrammar* grammar, const String& tok) = nullptr;
 	};
 
+	String name;
 	String error;
 	Buffer<Transition> transitions;
 	bool accept = false;
@@ -41,25 +44,28 @@ typedef Buffer<State> Automata;
 void initializeTokenizer(CFGTokenizer* tok) {
 	typedef decltype(tok->getTokenizer()) TokBase;
 	tok->build({
-			{ TokBase::idRE, Token::ID },
 			{ TokBase::etherRE, Token::IGNORED },
 			{ "Start", Token::START },
 			{ ":", Token::EQUAL },
-			{ "[", Token::TERMINAL_START },
+			{ "\\[", Token::TERMINAL_START },
-			{ "]", Token::TERMINAL_END },
+			{ "\\]", Token::TERMINAL_END },
-			{ "|", Token::ALTERNATION },
+			{ "\\|", Token::ALTERNATION },
 			{ ";", Token::RULE_END },
+			{ "&", Token::EPSILON },
+			{ TokBase::idRE, Token::ID },
+			{ TokBase::commentBlockRE, Token::COMMENT },
 	});
+
+	ASSERT(tok->isBuild())
 }
 
 void initAutomata(Automata& automata) {
-	automata.reserve(8);
+	automata.reserve(7);
 	auto states = &automata.first();
 
 	auto root = states++;
 	auto end = states++;
 	auto start = states++;
-	auto startEnd = states++;
 	auto rule = states++;
 	auto rhs = states++;
 	auto terminalStart = states++;
@@ -75,11 +81,7 @@ void initAutomata(Automata& automata) {
 	};
 
 	start->transitions = {
-			{ Token::ID, startEnd},
+			{ Token::ID,
-	};
-
-	startEnd->transitions = {
-			{ Token::RULE_END,
 			 root,
 			 [](CfGrammar* grammar, const String& tok) {
 				 grammar->startTerminal = tok;
@@ -98,7 +100,11 @@ void initAutomata(Automata& automata) {
 			 }
 			},
 			{ Token::ID, rhs, [](CfGrammar* grammar, const String& tok) {
-				 grammar->rules.last()->data.args.pushBack({ tok, false });
+				 grammar->rules.last()->data.args.pushBack({ tok, false, false });
+			 }
+			},
+			{ Token::EPSILON, rhs, [](CfGrammar* grammar, const String& tok) {
+				 grammar->rules.last()->data.args.pushBack({ tok, false, true });
 			 }
 			},
 			{ Token::TERMINAL_START, terminalStart },
@@ -106,8 +112,8 @@ void initAutomata(Automata& automata) {
 	};
 
 	terminalStart->transitions = {
-			{ Token::ID, rhs, [](CfGrammar* grammar, const String& tok) {
+			{ Token::ID, terminalEnd, [](CfGrammar* grammar, const String& tok) {
-				 grammar->rules.last()->data.args.pushBack({ tok, true });
+				 grammar->rules.last()->data.args.pushBack({ tok, true, false });
 			 }
 			},
 	};
@@ -121,12 +127,18 @@ void initAutomata(Automata& automata) {
 
 	root->error = "Expected 'Start' statement, rule or end of text";
 	start->error = idError;
-	startEnd->error = stmEndError;
 	rule->error = "Expected production equality ':'";
 	rhs->error = "Expected alternation '|', terminal id, statement end ';' or terminal start '[' ";
 	terminalStart->error = idError;
 	terminalEnd->error = "Expected terminal end '[' ";
 
+	root->name = "root";
+	start->name = "start";
+	rule->name = "production";
+	rhs->name = "rhs";
+	terminalStart->name = "terminalStart";
+	terminalEnd->name = "terminalEnd";
+
 	end->accept = true;
 }
 
@@ -139,6 +151,10 @@ bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGr
 
 		auto token = tok->readTok();
 
+		if (token == Token::IGNORED || token == Token::COMMENT) {
+			continue;
+		}
+
 		if (token == Token::FAILED) {
 			auto errorLocation = tok->getCursorPrev().get2DLocation();
 			printf("Syntax error at (%llu, %llu)\n", errorLocation.line, errorLocation.character);
@@ -151,7 +167,7 @@ bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGr
 
 		for (auto transition : state->transitions) {
 			if (transition->tok == token) {
-				transition->action(grammar, tokVal);
+				if (transition->action) transition->action(grammar, tokVal);
 				state = transition->target;
 				isTransition = true;
 				break;
@@ -185,6 +201,6 @@ void CfGrammar::deinitializeCfGrammarParser(CfGrammarParserState* state) {
 	delete state;
 }
 
-void CfGrammar::parse(CfGrammarParserState* context, const int1* source) {
+bool CfGrammar::parse(CfGrammarParserState* context, const String& source) {
-	::parse(source, &context->automata, &context->tokenizer, this);
+	return ::parse(source.read(), &context->automata, &context->tokenizer, this);
 }

Parser/private/Parser.cpp

@@ -0,0 +1,8 @@
+#include "Parser.hpp"
+
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+static ModuleManifest* sModuleDependencies[] = { &gModuleTokenizer, nullptr };
+ModuleManifest tp::gModuleParser = ModuleManifest("CommandLine", nullptr, nullptr, sModuleDependencies);

Parser/public/CfGrammar.hpp

@@ -2,6 +2,7 @@
 
 #include "Strings.hpp"
 #include "List.hpp"
+#include "Map.hpp"
 
 namespace tp {
 
@@ -9,7 +10,8 @@ namespace tp {
 		struct Rule {
 			struct Arg {
 				String id;
-				bool isTerminal;
+				bool isTerminal = false;
+				bool isEpsilon = false;
 			};
 
 			String id;
@@ -19,11 +21,27 @@ namespace tp {
 		List<Rule> rules;
 		String startTerminal;
 
+	private:
+		struct NonTerminal {
+			List<Rule*> rules;
+			Map<String, NonTerminal*> references;
+			Map<String, NonTerminal*> referencing;
+		};
+
+		struct Terminal {};
+
+		Map<String, Terminal> mTerminals;
+		Map<String, NonTerminal> mNonTerminals;
+
 	public:
 		static struct CfGrammarParserState* initializeCfGrammarParser();
 		static void deinitializeCfGrammarParser(CfGrammarParserState*);
 
 	public:
-		void parse(CfGrammarParserState* context, const int1* source);
+		bool parse(CfGrammarParserState* context, const String& source);
+		bool compile();
+
+		bool isAmbiguous();
+		void optimize();
 	};
 }

Parser/public/Parser.hpp

@@ -0,0 +1,7 @@
+#pragma once
+
+#include "CfGrammar.hpp"
+
+namespace tp {
+	extern ModuleManifest gModuleParser;
+}

Parser/rsc/grammar.txt

@@ -0,0 +1,12 @@
+
+/*
+Sample grammar.
+This grammar wil produce text in the form: TODO
+*/
+
+/* This is the starting production of the grammar */
+Start A
+
+/* Production rule */
+A : [ ID ];
+

Tokenizer/tests/TestTokenizer.cpp

@@ -193,6 +193,46 @@ TEST_DEF_STATIC(Simple) {
 	TEST(outputHash == outputHashPassed);
 }
 
+TEST_DEF(Covarage) {
+	enum class TokType {
+		START = 0,
+		NONE = 1,
+		FAILED = 2,
+		SPACE = 3,
+		ID = 4,
+		COMMENT_BLOCK = 5,
+		TOK_SOURCE_END = 6,
+	};
+
+	SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
+
+	lexer.build({
+			{ "([a-c]|A)+",	 TokType::ID },
+			{ "A", TokType::START },
+			{ " ", TokType::SPACE },
+	});
+
+	if (!lexer.isBuild()) {
+		printf("Error : %s", lexer.getBuildError().description);
+		return;
+	}
+
+	lexer.bindSource(" A bc cb cc ");
+	/* 											 3    5    3  0  3 4 3 4 3      6 */
+
+	TokType tok;
+	ualni outputHash = 0;
+	printf("\n\n OutputHash : ");
+	do {
+		tok = lexer.readTok();
+		outputHash += ualni(tok);
+		printf(" %i ", int(tok));
+	} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
+	printf(" : %llu", outputHash);
+	TEST(outputHash == 90);
+}
+
 TEST_DEF(Tokenizer) {
 	testSimple();
+	// testCovarage(); TODO : test environment for an error
 }