Language Initial

Language/old/Parser/private/CLR.cpp

@@ -0,0 +1,19 @@
+#include "CLR.hpp"
+
+using namespace tp;
+
+void CLR::setGrammar(const tp::CfGrammar& grammar) {
+	if (grammar.isLooped()) {
+		mState = ParserState::FAILED;
+		mBuildError.description = "Cant build lalr parser from looped context free grammar";
+		return;
+	}
+
+	mState = ParserState::FAILED;
+}
+
+void CLR::setTerminal(const String& name, TerminalStream::TerminalID id) {}
+
+void CLR::build() {}
+
+void CLR::parse(TerminalStream* source, List<ASTNode>& out, ASTNode** root) { *root = nullptr; }

Language/old/Parser/private/CfGrammar.cpp

@@ -0,0 +1,161 @@
+
+#include "CfGrammar.hpp"
+
+#include "Timing.hpp"
+
+using namespace tp;
+
+void CfGrammar::generateSentences(List<Sentence>& out) {
+	constexpr ualni maxTime = 1000;
+	constexpr ualni maxSentences = 200;
+	constexpr ualni maxQueue = 1000000;
+
+	List<Sentence> queue;
+	Timer timer(maxTime);
+
+	// add start production
+	Sentence start;
+	start.terms.pushBack({ startTerminal, false });
+	queue.pushBack(start);
+
+PASS:
+
+	auto const sentential = &queue.first()->data;
+	bool isSentence = true;
+	ualni termIdx = 0;
+	for (auto term : sentential->terms) {
+		if (term->terminal) {
+			termIdx++;
+			continue;
+		}
+
+		auto nonTerminal = &mNonTerminals.get(term->id);
+		for (auto production : nonTerminal->rules) {
+			queue.pushBack(*sentential);
+			auto copy = &queue.last()->data;
+
+			auto appendTerm = copy->terms.findIdx(termIdx);
+			auto deleteTerm = appendTerm;
+			for (auto arg : production->args) {
+				if (arg->isEpsilon) continue;
+				auto newTerm = copy->terms.newNode({ arg->id, arg->isTerminal });
+				copy->terms.attach(newTerm, appendTerm);
+				appendTerm = newTerm;
+			}
+			copy->terms.removeNode(deleteTerm);
+		}
+
+		isSentence = false;
+		termIdx++;
+	}
+
+	if (isSentence) {
+		auto exists = false;
+		for (auto iter : out) {
+			if (iter->terms == sentential->terms) {
+				exists = true;
+				break;
+			}
+		}
+		if (!exists) out.pushBack(*sentential);
+	}
+
+	queue.popFront();
+
+	bool stop = false;
+	stop |= !queue.length();
+	stop |= timer.isTimeout();
+	stop |= queue.length() > maxQueue;
+	stop |= out.length() > maxSentences;
+
+	if (!stop) goto PASS;
+}
+
+void CfGrammar::printSentence(Sentence& in) {
+	printf("[");
+	for (auto term : in.terms) {
+		printf(" %s", term->id.read());
+	}
+	printf(" ]\n");
+}
+
+bool CfGrammar::compile() {
+
+	if (!rules.length()) {
+		printf("Grammar must have at leas one rule\n");
+		return false;
+	}
+
+	if (startTerminal == String()) {
+		printf("Using first rule's non-terminal as grammar's root. Define explicitly - 'Start : id'\n");
+		startTerminal = rules.first()->data.id;
+	}
+
+	// find all rules
+	for (auto rule : rules) {
+
+		if (!rule->args.length()) {
+			printf("Rule must have at leas one terminal or non-terminal. See rule '%s'\n", rule->id.read());
+			return false;
+		}
+
+		if (!mNonTerminals.presents(rule->id)) {
+			mNonTerminals.put(rule->id, {});
+		}
+		auto nonTerminal = &mNonTerminals.get(rule->id);
+
+		nonTerminal->rules.pushBack(&rule.data());
+	}
+
+	for (auto nonTerminal : mNonTerminals) {
+		for (auto rule : nonTerminal->val.rules) {
+			for (auto arg : rule->args) {
+
+				if (arg->isTerminal) {
+					mTerminals.put(rule->id, {});
+				} else if (!arg->isEpsilon) {
+					auto idx = mNonTerminals.presents(arg->id);
+					if (!idx) {
+						printf("Referenced non-terminal '%s' is not defined\n", arg->id.read());
+						return false;
+					}
+					auto reference = &mNonTerminals.getSlotVal(idx);
+					reference->references.put(nonTerminal->key, &nonTerminal->val);
+					nonTerminal->val.referencing.put(arg->id, reference);
+				}
+			}
+		}
+	}
+
+	List<String> remove;
+	for (auto nonTerminal : mNonTerminals) {
+		if (!nonTerminal->val.references.size() && nonTerminal->key != startTerminal) {
+			printf("Non-terminal '%s' is defined but not used\n", nonTerminal->key.read());
+			remove.pushBack(nonTerminal->key);
+		}
+	}
+
+	for (auto rem : remove) {
+		auto nonTerminal = &mNonTerminals.get(rem.data());
+		for (auto referencing : nonTerminal->referencing) {
+			referencing->val->references.remove(nonTerminal->rules.first()->data->id);
+		}
+		for (auto rule : nonTerminal->rules) {
+			rules.removeNode(rules.find(*rule.data()));
+		}
+	}
+
+	for (auto nonTerminal : mNonTerminals) {
+		if (!nonTerminal->val.isProductive()) {
+			printf("Non-terminal '%s' is not productive\n", nonTerminal->val.rules.first()->data->id.read());
+			return false;
+		}
+	}
+
+	Map<String, ualni> processed;
+	if (mNonTerminals.get(startTerminal).isLooped(processed, startTerminal)) {
+		printf("Note that grammar is looped.\n");
+	}
+
+	return true;
+}

Language/old/Parser/private/CfGrammarParser.cpp

@@ -0,0 +1,201 @@
+
+#include "CfGrammar.hpp"
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+enum class Token {
+	NO_TOKEN,
+	START,
+	ID,
+	EQUAL,
+	TERMINAL_START,
+	TERMINAL_END,
+	ALTERNATION,
+	RULE_END,
+	IGNORED,
+	EPSILON,
+	COMMENT,
+	END,
+	FAILED,
+};
+
+typedef int1 AlphabetType;
+typedef SimpleTokenizer<AlphabetType, Token, Token::NO_TOKEN, Token::FAILED, Token::END> CFGTokenizer;
+
+struct State {
+
+	struct Transition {
+		Token tok = Token::FAILED;
+		State* target = nullptr;
+		void (*action)(CfGrammar* grammar, const String& tok) = nullptr;
+	};
+
+	String name;
+	String error;
+	Buffer<Transition> transitions;
+	bool accept = false;
+};
+
+typedef Buffer<State> Automata;
+
+void initializeTokenizer(CFGTokenizer* tok) {
+	tok->build({
+		{ CFGTokenizer::tTokenizer::etherRE, Token::IGNORED },
+		{ "Start", Token::START },
+		{ ":", Token::EQUAL },
+		{ "\\[", Token::TERMINAL_START },
+		{ "\\]", Token::TERMINAL_END },
+		{ "\\|", Token::ALTERNATION },
+		{ ";", Token::RULE_END },
+		{ "&", Token::EPSILON },
+		{ CFGTokenizer::tTokenizer::idRE, Token::ID },
+		{ CFGTokenizer::tTokenizer::commentBlockRE, Token::COMMENT },
+	});
+
+	ASSERT(tok->isBuild())
+}
+
+void initAutomata(Automata& automata) {
+	automata.reserve(7);
+	auto states = &automata.first();
+
+	auto root = states++;
+	auto end = states++;
+	auto start = states++;
+	auto rule = states++;
+	auto rhs = states++;
+	auto terminalStart = states++;
+	auto terminalEnd = states;
+
+	root->transitions = {
+		{ Token::END, end },
+		{ Token::START, start },
+		{ Token::ID,
+			rule,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.pushBack({ tok, {} });
+			} },
+	};
+
+	start->transitions = {
+		{ Token::ID, root, [](CfGrammar* grammar, const String& tok) { grammar->startTerminal = tok; } },
+	};
+
+	rule->transitions = {
+		{ Token::EQUAL, rhs },
+	};
+
+	rhs->transitions = {
+		{ Token::ALTERNATION,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				auto const& id = grammar->rules.last()->data.id;
+				grammar->rules.pushBack({ id, {} });
+			} },
+		{ Token::ID,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, false, false });
+			} },
+		{ Token::EPSILON,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, false, true });
+			} },
+		{ Token::TERMINAL_START, terminalStart },
+		{ Token::RULE_END, root },
+	};
+
+	terminalStart->transitions = {
+		{ Token::ID,
+			terminalEnd,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, true, false });
+			} },
+	};
+
+	terminalEnd->transitions = {
+		{ Token::TERMINAL_END, rhs },
+	};
+
+	auto idError = "Expected an identifier";
+	auto stmEndError = "Expected an statement end ';'";
+
+	root->error = "Expected 'Start' statement, rule or end of text";
+	start->error = idError;
+	rule->error = "Expected production equality ':'";
+	rhs->error = "Expected alternation '|', terminal id, statement end ';' or terminal start '[' ";
+	terminalStart->error = idError;
+	terminalEnd->error = "Expected terminal end '[' ";
+
+	root->name = "root";
+	start->name = "start";
+	rule->name = "production";
+	rhs->name = "rhs";
+	terminalStart->name = "terminalStart";
+	terminalEnd->name = "terminalEnd";
+
+	end->accept = true;
+}
+
+bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGrammar* grammar) {
+	tok->reset();
+	tok->bindSource(text);
+
+	State* state = &(*automata)[0];
+	while (!state->accept) {
+
+		auto token = tok->readTok();
+
+		if (token == Token::IGNORED || token == Token::COMMENT) {
+			continue;
+		}
+
+		if (token == Token::FAILED) {
+			auto errorLocation = tok->getCursorPrev().get2DLocation();
+			printf("Syntax error at (%llu, %llu)\n", errorLocation.line, errorLocation.character);
+			return false;
+		}
+
+		auto tokVal = tok->extractVal();
+
+		bool isTransition = false;
+
+		for (auto transition : state->transitions) {
+			if (transition->tok == token) {
+				if (transition->action) transition->action(grammar, tokVal);
+				state = transition->target;
+				isTransition = true;
+				break;
+			}
+		}
+
+		if (!isTransition) {
+			// if no tok matched to any transition produce error
+			auto errorLocation = tok->getCursorPrev().get2DLocation();
+			printf("Parse error at (%llu, %llu) - %s", errorLocation.line, errorLocation.character, state->error.read());
+			return false;
+		}
+	}
+
+	return true;
+}
+
+struct tp::CfGrammarParserState {
+	Automata automata;
+	CFGTokenizer tokenizer;
+};
+
+CfGrammarParserState* CfGrammar::initializeCfGrammarParser() {
+	auto state = new CfGrammarParserState();
+	initializeTokenizer(&state->tokenizer);
+	initAutomata(state->automata);
+	return state;
+}
+
+void CfGrammar::deinitializeCfGrammarParser(CfGrammarParserState* state) { delete state; }
+
+bool CfGrammar::parse(CfGrammarParserState* context, const String& source) { return ::parse(source.read(), &context->automata, &context->tokenizer, this); }
+
+bool CfGrammar::isLooped() const { return mIsLooped; }

Language/old/Parser/private/Parser.cpp

@@ -0,0 +1,7 @@
+#include "Parser.hpp"
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+static ModuleManifest* sModuleDependencies[] = { &gModuleTokenizer, nullptr };
+ModuleManifest tp::gModuleParser = ModuleManifest("CommandLine", nullptr, nullptr, sModuleDependencies);