Language Initial

Language/old/Parser/CMakeLists.txt

@@ -0,0 +1,21 @@
+project(Parser)
+
+### ---------------------- Static Library  ---------------------  ###
+file(GLOB SOURCES "./private/*.cpp")
+file(GLOB HEADERS "./public/*.hpp")
+add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
+target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
+target_link_libraries(${PROJECT_NAME} PUBLIC Tokenizer)
+
+### -------------------------- Applications  --------------------------  ###
+add_executable(cfg ./applications/Cfg.cpp)
+target_link_libraries(cfg ${PROJECT_NAME} CommandLine)
+
+configure_file(rsc/grammar.txt grammar.txt COPYONLY)
+
+### -------------------------- Tests  --------------------------  ###
+enable_testing()
+file(GLOB TEST_SOURCES "./tests/*.cpp")
+add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES})
+target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils)
+add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests)

Language/old/Parser/applications/Cfg.cpp

@@ -0,0 +1,82 @@
+
+#include "CommandLine.hpp"
+#include "Parser.hpp"
+
+using namespace tp;
+
+void run(const String& source) {
+	auto state = CfGrammar::initializeCfGrammarParser();
+
+	CfGrammar grammar;
+
+	if (!grammar.parse(state, source)) {
+		printf("Parsing is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	if (!grammar.compile()) {
+		printf("Compilation is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	printf("Grammar accepted.\n");
+
+	List<CfGrammar::Sentence> sentences;
+	grammar.generateSentences(sentences);
+
+	printf("Example sentences formed from grammar: \n");
+	for (auto sentence : sentences)
+		tp::CfGrammar::printSentence(sentence.data());
+
+	CfGrammar::deinitializeCfGrammarParser(state);
+}
+
+int main(int argc, const char* argv[]) {
+	tp::ModuleManifest* deps[] = { &tp::gModuleParser, &tp::gModuleCommandLine, nullptr };
+	tp::ModuleManifest testModule("CommandLineTest", nullptr, nullptr, deps);
+
+	if (!testModule.initialize()) {
+		return 1;
+	}
+
+	{
+		printf("Arguments given: ");
+		for (auto idx = 0; idx < argc; idx++) {
+			printf("[%s]", argv[idx]);
+		}
+		printf("\n");
+
+		CommandLine cmd = {
+			{ "grammar", CommandLine::Arg::STR },
+		};
+
+		if (!cmd.parse((char) argc, argv, true, 1)) {
+			return 1;
+		}
+
+		auto location = cmd.getString("grammar");
+
+		LocalConnection file;
+		String grammar;
+
+		file.connect(LocalConnection::Location(location), LocalConnection::Type(true));
+
+		if (!file.getConnectionStatus().isOpened()) {
+			printf("Cant open file\n");
+			return 1;
+		}
+
+		auto size = file.size();
+		grammar.resize((String::Index) size);
+		file.readBytes(grammar.write(), size);
+
+		run(grammar);
+
+		file.disconnect();
+	}
+
+	testModule.deinitialize();
+	return 0;
+}

Language/old/Parser/private/CLR.cpp

@@ -0,0 +1,19 @@
+#include "CLR.hpp"
+
+using namespace tp;
+
+void CLR::setGrammar(const tp::CfGrammar& grammar) {
+	if (grammar.isLooped()) {
+		mState = ParserState::FAILED;
+		mBuildError.description = "Cant build lalr parser from looped context free grammar";
+		return;
+	}
+
+	mState = ParserState::FAILED;
+}
+
+void CLR::setTerminal(const String& name, TerminalStream::TerminalID id) {}
+
+void CLR::build() {}
+
+void CLR::parse(TerminalStream* source, List<ASTNode>& out, ASTNode** root) { *root = nullptr; }

Language/old/Parser/private/CfGrammar.cpp

@@ -0,0 +1,161 @@
+
+#include "CfGrammar.hpp"
+
+#include "Timing.hpp"
+
+using namespace tp;
+
+void CfGrammar::generateSentences(List<Sentence>& out) {
+	constexpr ualni maxTime = 1000;
+	constexpr ualni maxSentences = 200;
+	constexpr ualni maxQueue = 1000000;
+
+	List<Sentence> queue;
+	Timer timer(maxTime);
+
+	// add start production
+	Sentence start;
+	start.terms.pushBack({ startTerminal, false });
+	queue.pushBack(start);
+
+PASS:
+
+	auto const sentential = &queue.first()->data;
+	bool isSentence = true;
+	ualni termIdx = 0;
+	for (auto term : sentential->terms) {
+		if (term->terminal) {
+			termIdx++;
+			continue;
+		}
+
+		auto nonTerminal = &mNonTerminals.get(term->id);
+		for (auto production : nonTerminal->rules) {
+			queue.pushBack(*sentential);
+			auto copy = &queue.last()->data;
+
+			auto appendTerm = copy->terms.findIdx(termIdx);
+			auto deleteTerm = appendTerm;
+			for (auto arg : production->args) {
+				if (arg->isEpsilon) continue;
+				auto newTerm = copy->terms.newNode({ arg->id, arg->isTerminal });
+				copy->terms.attach(newTerm, appendTerm);
+				appendTerm = newTerm;
+			}
+			copy->terms.removeNode(deleteTerm);
+		}
+
+		isSentence = false;
+		termIdx++;
+	}
+
+	if (isSentence) {
+		auto exists = false;
+		for (auto iter : out) {
+			if (iter->terms == sentential->terms) {
+				exists = true;
+				break;
+			}
+		}
+		if (!exists) out.pushBack(*sentential);
+	}
+
+	queue.popFront();
+
+	bool stop = false;
+	stop |= !queue.length();
+	stop |= timer.isTimeout();
+	stop |= queue.length() > maxQueue;
+	stop |= out.length() > maxSentences;
+
+	if (!stop) goto PASS;
+}
+
+void CfGrammar::printSentence(Sentence& in) {
+	printf("[");
+	for (auto term : in.terms) {
+		printf(" %s", term->id.read());
+	}
+	printf(" ]\n");
+}
+
+bool CfGrammar::compile() {
+
+	if (!rules.length()) {
+		printf("Grammar must have at leas one rule\n");
+		return false;
+	}
+
+	if (startTerminal == String()) {
+		printf("Using first rule's non-terminal as grammar's root. Define explicitly - 'Start : id'\n");
+		startTerminal = rules.first()->data.id;
+	}
+
+	// find all rules
+	for (auto rule : rules) {
+
+		if (!rule->args.length()) {
+			printf("Rule must have at leas one terminal or non-terminal. See rule '%s'\n", rule->id.read());
+			return false;
+		}
+
+		if (!mNonTerminals.presents(rule->id)) {
+			mNonTerminals.put(rule->id, {});
+		}
+		auto nonTerminal = &mNonTerminals.get(rule->id);
+
+		nonTerminal->rules.pushBack(&rule.data());
+	}
+
+	for (auto nonTerminal : mNonTerminals) {
+		for (auto rule : nonTerminal->val.rules) {
+			for (auto arg : rule->args) {
+
+				if (arg->isTerminal) {
+					mTerminals.put(rule->id, {});
+				} else if (!arg->isEpsilon) {
+					auto idx = mNonTerminals.presents(arg->id);
+					if (!idx) {
+						printf("Referenced non-terminal '%s' is not defined\n", arg->id.read());
+						return false;
+					}
+					auto reference = &mNonTerminals.getSlotVal(idx);
+					reference->references.put(nonTerminal->key, &nonTerminal->val);
+					nonTerminal->val.referencing.put(arg->id, reference);
+				}
+			}
+		}
+	}
+
+	List<String> remove;
+	for (auto nonTerminal : mNonTerminals) {
+		if (!nonTerminal->val.references.size() && nonTerminal->key != startTerminal) {
+			printf("Non-terminal '%s' is defined but not used\n", nonTerminal->key.read());
+			remove.pushBack(nonTerminal->key);
+		}
+	}
+
+	for (auto rem : remove) {
+		auto nonTerminal = &mNonTerminals.get(rem.data());
+		for (auto referencing : nonTerminal->referencing) {
+			referencing->val->references.remove(nonTerminal->rules.first()->data->id);
+		}
+		for (auto rule : nonTerminal->rules) {
+			rules.removeNode(rules.find(*rule.data()));
+		}
+	}
+
+	for (auto nonTerminal : mNonTerminals) {
+		if (!nonTerminal->val.isProductive()) {
+			printf("Non-terminal '%s' is not productive\n", nonTerminal->val.rules.first()->data->id.read());
+			return false;
+		}
+	}
+
+	Map<String, ualni> processed;
+	if (mNonTerminals.get(startTerminal).isLooped(processed, startTerminal)) {
+		printf("Note that grammar is looped.\n");
+	}
+
+	return true;
+}

Language/old/Parser/private/CfGrammarParser.cpp

@@ -0,0 +1,201 @@
+
+#include "CfGrammar.hpp"
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+enum class Token {
+	NO_TOKEN,
+	START,
+	ID,
+	EQUAL,
+	TERMINAL_START,
+	TERMINAL_END,
+	ALTERNATION,
+	RULE_END,
+	IGNORED,
+	EPSILON,
+	COMMENT,
+	END,
+	FAILED,
+};
+
+typedef int1 AlphabetType;
+typedef SimpleTokenizer<AlphabetType, Token, Token::NO_TOKEN, Token::FAILED, Token::END> CFGTokenizer;
+
+struct State {
+
+	struct Transition {
+		Token tok = Token::FAILED;
+		State* target = nullptr;
+		void (*action)(CfGrammar* grammar, const String& tok) = nullptr;
+	};
+
+	String name;
+	String error;
+	Buffer<Transition> transitions;
+	bool accept = false;
+};
+
+typedef Buffer<State> Automata;
+
+void initializeTokenizer(CFGTokenizer* tok) {
+	tok->build({
+		{ CFGTokenizer::tTokenizer::etherRE, Token::IGNORED },
+		{ "Start", Token::START },
+		{ ":", Token::EQUAL },
+		{ "\\[", Token::TERMINAL_START },
+		{ "\\]", Token::TERMINAL_END },
+		{ "\\|", Token::ALTERNATION },
+		{ ";", Token::RULE_END },
+		{ "&", Token::EPSILON },
+		{ CFGTokenizer::tTokenizer::idRE, Token::ID },
+		{ CFGTokenizer::tTokenizer::commentBlockRE, Token::COMMENT },
+	});
+
+	ASSERT(tok->isBuild())
+}
+
+void initAutomata(Automata& automata) {
+	automata.reserve(7);
+	auto states = &automata.first();
+
+	auto root = states++;
+	auto end = states++;
+	auto start = states++;
+	auto rule = states++;
+	auto rhs = states++;
+	auto terminalStart = states++;
+	auto terminalEnd = states;
+
+	root->transitions = {
+		{ Token::END, end },
+		{ Token::START, start },
+		{ Token::ID,
+			rule,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.pushBack({ tok, {} });
+			} },
+	};
+
+	start->transitions = {
+		{ Token::ID, root, [](CfGrammar* grammar, const String& tok) { grammar->startTerminal = tok; } },
+	};
+
+	rule->transitions = {
+		{ Token::EQUAL, rhs },
+	};
+
+	rhs->transitions = {
+		{ Token::ALTERNATION,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				auto const& id = grammar->rules.last()->data.id;
+				grammar->rules.pushBack({ id, {} });
+			} },
+		{ Token::ID,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, false, false });
+			} },
+		{ Token::EPSILON,
+			rhs,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, false, true });
+			} },
+		{ Token::TERMINAL_START, terminalStart },
+		{ Token::RULE_END, root },
+	};
+
+	terminalStart->transitions = {
+		{ Token::ID,
+			terminalEnd,
+			[](CfGrammar* grammar, const String& tok) {
+				grammar->rules.last()->data.args.pushBack({ tok, true, false });
+			} },
+	};
+
+	terminalEnd->transitions = {
+		{ Token::TERMINAL_END, rhs },
+	};
+
+	auto idError = "Expected an identifier";
+	auto stmEndError = "Expected an statement end ';'";
+
+	root->error = "Expected 'Start' statement, rule or end of text";
+	start->error = idError;
+	rule->error = "Expected production equality ':'";
+	rhs->error = "Expected alternation '|', terminal id, statement end ';' or terminal start '[' ";
+	terminalStart->error = idError;
+	terminalEnd->error = "Expected terminal end '[' ";
+
+	root->name = "root";
+	start->name = "start";
+	rule->name = "production";
+	rhs->name = "rhs";
+	terminalStart->name = "terminalStart";
+	terminalEnd->name = "terminalEnd";
+
+	end->accept = true;
+}
+
+bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGrammar* grammar) {
+	tok->reset();
+	tok->bindSource(text);
+
+	State* state = &(*automata)[0];
+	while (!state->accept) {
+
+		auto token = tok->readTok();
+
+		if (token == Token::IGNORED || token == Token::COMMENT) {
+			continue;
+		}
+
+		if (token == Token::FAILED) {
+			auto errorLocation = tok->getCursorPrev().get2DLocation();
+			printf("Syntax error at (%llu, %llu)\n", errorLocation.line, errorLocation.character);
+			return false;
+		}
+
+		auto tokVal = tok->extractVal();
+
+		bool isTransition = false;
+
+		for (auto transition : state->transitions) {
+			if (transition->tok == token) {
+				if (transition->action) transition->action(grammar, tokVal);
+				state = transition->target;
+				isTransition = true;
+				break;
+			}
+		}
+
+		if (!isTransition) {
+			// if no tok matched to any transition produce error
+			auto errorLocation = tok->getCursorPrev().get2DLocation();
+			printf("Parse error at (%llu, %llu) - %s", errorLocation.line, errorLocation.character, state->error.read());
+			return false;
+		}
+	}
+
+	return true;
+}
+
+struct tp::CfGrammarParserState {
+	Automata automata;
+	CFGTokenizer tokenizer;
+};
+
+CfGrammarParserState* CfGrammar::initializeCfGrammarParser() {
+	auto state = new CfGrammarParserState();
+	initializeTokenizer(&state->tokenizer);
+	initAutomata(state->automata);
+	return state;
+}
+
+void CfGrammar::deinitializeCfGrammarParser(CfGrammarParserState* state) { delete state; }
+
+bool CfGrammar::parse(CfGrammarParserState* context, const String& source) { return ::parse(source.read(), &context->automata, &context->tokenizer, this); }
+
+bool CfGrammar::isLooped() const { return mIsLooped; }

Language/old/Parser/private/Parser.cpp

@@ -0,0 +1,7 @@
+#include "Parser.hpp"
+#include "Tokenizer.hpp"
+
+using namespace tp;
+
+static ModuleManifest* sModuleDependencies[] = { &gModuleTokenizer, nullptr };
+ModuleManifest tp::gModuleParser = ModuleManifest("CommandLine", nullptr, nullptr, sModuleDependencies);

Language/old/Parser/public/CLR.hpp

@@ -0,0 +1,62 @@
+#pragma once
+
+#include "CfGrammar.hpp"
+
+namespace tp {
+	class CLR {
+	public:
+		struct ASTNode {
+			const CfGrammar::Rule* production = nullptr;
+			Map<String, String> terminals;
+		};
+
+		class Automation {
+		public:
+			class State {};
+		};
+
+		class TerminalStream {
+		public:
+			typedef ualni TerminalID;
+
+		public:
+			TerminalStream() = default;
+			virtual TerminalID getNextTerminal() = 0;
+		};
+
+	public:
+		struct BuildError {
+			String description;
+		};
+
+		struct ParseError {
+			const Automation::State* state = nullptr;
+			String::Index location = 0;
+		};
+
+	private:
+		class SententialStack {};
+
+	public:
+		CLR() = default;
+
+		void setGrammar(const CfGrammar& grammar);
+		void setTerminal(const String& name, TerminalStream::TerminalID id);
+		void build();
+
+		void parse(TerminalStream* stream, List<ASTNode>& out, ASTNode** root);
+
+		[[nodiscard]] bool isBuild() { return mState == ParserState::BUILD; }
+		[[nodiscard]] const BuildError& getBuildError() { return mBuildError; }
+		[[nodiscard]] const ParseError& getParseError() { return mParseError; }
+
+	private:
+		enum class ParserState { NONE, BUILD, FAILED } mState = ParserState::NONE;
+
+		ParseError mParseError;
+		BuildError mBuildError;
+
+		SententialStack mStack;
+		Automation mAutomation;
+	};
+}

Language/old/Parser/public/CfGrammar.hpp

@@ -0,0 +1,89 @@
+#pragma once
+
+#include "List.hpp"
+#include "Map.hpp"
+#include "Strings.hpp"
+
+namespace tp {
+
+	struct CfGrammar {
+		struct Rule {
+			struct Arg {
+				String id;
+				bool isTerminal = false;
+				bool isEpsilon = false;
+
+				bool operator==(const Arg& in) const { return (id == in.id) && (isEpsilon == in.isEpsilon) && (isTerminal == in.isTerminal); }
+			};
+
+			String id;
+			List<Arg> args;
+
+			bool operator==(const Rule& in) const { return (id == in.id) && (args == in.args); }
+
+			[[nodiscard]] bool isProductive() const {
+				for (auto arg : args) {
+					if (arg->id == id) return false;
+				}
+				return true;
+			}
+		};
+
+		struct Sentence {
+			struct Term {
+				String id;
+				bool terminal;
+				bool operator==(const Term& in) const { return (id == in.id) && (terminal == in.terminal); }
+			};
+			List<Term> terms;
+		};
+
+		List<Rule> rules;
+		String startTerminal;
+
+	private:
+		struct NonTerminal {
+
+			List<Rule*> rules;
+			Map<String, NonTerminal*> references;
+			Map<String, NonTerminal*> referencing;
+
+			[[nodiscard]] bool isProductive() const {
+				for (auto rule : rules) {
+					if (rule->isProductive()) return true;
+				}
+				return false;
+			}
+
+			[[nodiscard]] bool isLooped(Map<String, ualni>& processed, const String& id) const {
+				for (auto ref : referencing) {
+					if (processed.presents(ref->key)) return true;
+				}
+				processed.put(id, {});
+				for (auto ref : referencing) {
+					if (ref->val->isLooped(processed, ref->key)) return true;
+				}
+				return false;
+			}
+		};
+
+		struct Terminal {};
+
+		Map<String, Terminal> mTerminals;
+		Map<String, NonTerminal> mNonTerminals;
+		bool mIsLooped = false;
+
+	public:
+		static struct CfGrammarParserState* initializeCfGrammarParser();
+		static void deinitializeCfGrammarParser(CfGrammarParserState*);
+
+	public:
+		bool parse(CfGrammarParserState* context, const String& source);
+		bool compile();
+
+		void generateSentences(List<Sentence>& out);
+		static void printSentence(Sentence& in);
+
+		[[nodiscard]] bool isLooped() const;
+	};
+}

Language/old/Parser/public/Parser.hpp

@@ -0,0 +1,7 @@
+#pragma once
+
+#include "CLR.hpp"
+
+namespace tp {
+	extern ModuleManifest gModuleParser;
+}

Language/old/Parser/rsc/grammar.txt

@@ -0,0 +1,20 @@
+
+/*
+Sample grammar.
+This grammar wil produce text in the form: TODO
+*/
+
+/* This is the starting production of the grammar */
+
+/* Simple Example
+Start A
+A : [ ID ] | &;
+*/
+
+/* Lists Example */
+
+Start List
+
+List : List Stm | Stm;
+Stm : StmBody [ END ];
+StmBody: [ Stm1 ] | [ Stm2 ] | [ Stm3 ];

Language/old/Parser/tests/CLRTests.cpp

@@ -0,0 +1,9 @@
+
+#include "Parser.hpp"
+#include "Testing.hpp"
+
+using namespace tp;
+
+TEST_DEF(CLR) {
+	// TEST(false);
+}

Language/old/Parser/tests/GrammarTests.cpp

@@ -0,0 +1,48 @@
+
+#include "Parser.hpp"
+#include "Testing.hpp"
+
+using namespace tp;
+
+TEST_DEF(Grammar) {
+
+	String source = R"(
+Start Body
+Body : [ Function ] | List | &;
+List : [ LIST ];
+)";
+
+	auto state = CfGrammar::initializeCfGrammarParser();
+
+	CfGrammar grammar;
+
+	if (!grammar.parse(state, source)) {
+		TEST(0 && "Parsing is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	if (!grammar.compile()) {
+		TEST(0 && "Compilation is failed\n");
+		CfGrammar::deinitializeCfGrammarParser(state);
+		return;
+	}
+
+	CfGrammar::deinitializeCfGrammarParser(state);
+
+	printf("Grammar accepted.\n");
+
+	List<CfGrammar::Sentence> sentences;
+	grammar.generateSentences(sentences);
+
+	TEST_ASSERT(sentences.length() == 3);
+	TEST_ASSERT(sentences.first()->data.terms.length() == 1);
+	TEST_ASSERT(sentences.last()->data.terms.length() == 1);
+
+	TEST(sentences.first()->data.terms.first()->data.id == "Function");
+	TEST(sentences.last()->data.terms.first()->data.id == "LIST");
+
+	printf("Example sentences formed from grammar: \n");
+	for (auto sentence : sentences)
+		tp::CfGrammar::printSentence(sentence.data());
+}

Language/old/Parser/tests/Tests.cpp

@@ -0,0 +1,23 @@
+
+#include "Parser.hpp"
+#include "Testing.hpp"
+
+using namespace tp;
+
+void testGrammar();
+void testCLR();
+
+int main(int argc, const char* argv[]) {
+	tp::ModuleManifest* deps[] = { &tp::gModuleParser, nullptr };
+	tp::ModuleManifest testModule("CommandLineTest", nullptr, nullptr, deps);
+
+	if (!testModule.initialize()) {
+		return 1;
+	}
+
+	testGrammar();
+	testCLR();
+
+	testModule.deinitialize();
+	return 0;
+}