parsing loop closed

Language/public/AST.hpp

@@ -1,8 +0,0 @@
-
-#pragma once
-
-#include "LanguageCommon.hpp"
-
-namespace tp {
-	class AST {};
-}

Language/public/ContextFreeAutomata.hpp

@@ -37,7 +37,7 @@ namespace tp {
 
 			ualni advancedIdx = 0;
 			while (advancedIdx < size) {
-				tAlphabetType& symbol = *(stream + advancedIdx);
+				const tAlphabetType& symbol = *(stream + advancedIdx);
 
 				if (!(symbol >= mRange.mBegin && symbol < mRange.mEnd)) {
 					return { false, advancedIdx, nullptr };
@@ -56,7 +56,7 @@ namespace tp {
 				}
 
 				if (mTable.get({ 0, mCurrentState }).type == Action::REDUCE) {
-					StackItem* newItem = &mItems.append({});
+					StackItem* newItem = &mItems.append(StackItem{});
 					for (auto iter : Range<ualni>(action.num)) {
 						newItem->leafs.append(mStack.last());
 						mCurrentState = mStack.last()->state;

Language/public/ContextFreeCompiler.hpp

@@ -8,7 +8,7 @@ namespace tp {
 
 	class ContextFreeCompiler {
 	public:
-		typedef ualni SymbolID;
+		typedef ualni SymbolId;
 
 		struct Item {
 			const ContextFreeGrammar::Rule* mRule = nullptr;
@@ -16,12 +16,12 @@ namespace tp {
 			ualni numArgs() const { return 0; }
 		};
 
-	private:
 		struct Symbol {
 			String mId;
 			bool mIsTerminal = false;
 		};
 
+	private:
 		struct NonTerminal {
 			Buffer<ContextFreeGrammar::Rule*> rules;
 			Map<String, NonTerminal*> references;
@@ -48,11 +48,14 @@ namespace tp {
 		};
 
 	public:
-		bool compile(const ContextFreeGrammar& grammar, FiniteStateAutomation<SymbolID, Item>& automata) {
+		bool compile(const ContextFreeGrammar& grammar, FiniteStateAutomation<SymbolId, Item>& automata) {
 			if (!init(grammar)) return false;
 			return true;
 		}
 
+		[[nodiscard]] const Buffer<Symbol>* getSymbols() const { return &mSymbols; }
+		[[nodiscard]] SymbolId getSymbolId(const String& name) const { return mSymbolLookup.get(name); }
+
 	private:
 		bool init(const ContextFreeGrammar& grammar) {
 
@@ -135,7 +138,7 @@ namespace tp {
 		void initSymbols(const ContextFreeGrammar& grammar) {
 			for (auto nonTerminal : mNonTerminals) {
 				mSymbols.append({ nonTerminal->key, false });
-				mSymbolLookup.put(nonTerminal->key, SymbolID(mSymbols.size() - 1));
+				mSymbolLookup.put(nonTerminal->key, SymbolId(mSymbols.size() - 1));
 
 				for (auto rule : nonTerminal->val.rules) {
 					for (auto arg : *rule->getArgs()) {
@@ -144,7 +147,7 @@ namespace tp {
 						mTerminals.put(arg->getId(), {});
 
 						mSymbols.append({ nonTerminal->key, true });
-						mSymbolLookup.put(nonTerminal->key, SymbolID(mSymbols.size() - 1));
+						mSymbolLookup.put(nonTerminal->key, SymbolId(mSymbols.size() - 1));
 					}
 				}
 			}
@@ -154,6 +157,6 @@ namespace tp {
 		Map<String, NonTerminal> mNonTerminals;
 		Map<String, bool> mTerminals;
 		Buffer<Symbol> mSymbols;
-		Map<String, SymbolID> mSymbolLookup;
+		Map<String, SymbolId> mSymbolLookup;
 	};
 }

Language/public/Parser.hpp

@@ -1,8 +1,6 @@
 
 #pragma once
 
-#include "AST.hpp"
-
 #include "RegularCompiler.hpp"
 #include "RegularAutomata.hpp"
 
@@ -11,24 +9,34 @@
 
 namespace tp {
 
-	template <typename tAlphabetType, typename TokenType, ualni MinSymbol, ualni MaxSymbol>
+	template <typename tAlphabetType, typename tTokenType, tTokenType tInTransition, ualni MinSymbol, ualni MaxSymbol>
 	class Parser {
 
-		typedef RegularGrammar<tAlphabetType, TokenType> RegularGrammar;
+		typedef RegularGrammar<tAlphabetType, tTokenType> RegularGrammar;
-		typedef RegularCompiler<tAlphabetType, TokenType, MinSymbol, MaxSymbol> RegularCompiler;
+		typedef RegularCompiler<tAlphabetType, tTokenType, tInTransition, MinSymbol, MaxSymbol> RegularCompiler;
-		typedef FiniteStateAutomation<tAlphabetType, TokenType> RegularGraph;
+		typedef FiniteStateAutomation<tAlphabetType, tTokenType> RegularGraph;
-		typedef RegularAutomata<tAlphabetType, TokenType> RegularAutomata;
+		typedef RegularAutomata<tAlphabetType, tTokenType> RegularAutomata;
 
 		// ContextFreeGrammar;
 		// ContextFreeCompiler;
-		typedef FiniteStateAutomation<ContextFreeCompiler::SymbolID, ContextFreeCompiler::Item> ContextFreeGraph;
+		typedef FiniteStateAutomation<ContextFreeCompiler::SymbolId, ContextFreeCompiler::Item> ContextFreeGraph;
-		typedef ContextFreeAutomata<ContextFreeCompiler::SymbolID, ContextFreeCompiler::Item> ContextFreeAutomata;
+		typedef ContextFreeAutomata<ContextFreeCompiler::SymbolId, ContextFreeCompiler::Item> ContextFreeAutomata;
+
+	public:
+		struct ParseResult {
+			bool accepted = false;
+			const ContextFreeAutomata::StackItem* ast = nullptr;
+		};
 
 	public:
 		Parser() = default;
 
 	public:
-		bool compileTables(const ContextFreeGrammar& cfGrammar, const RegularGrammar& reGrammar) {
+		bool compileTables(
+			const ContextFreeGrammar& cfGrammar,
+			const RegularGrammar& reGrammar,
+			const Map<String, tTokenType>& contextFreeToRegular
+		) {
 			// Compile Regular Grammar
 			{
 				RegularGraph graph;
@@ -45,16 +53,53 @@ namespace tp {
 				compiler.compile(cfGrammar, graph);
 				graph.makeDeterministic();
 				mContextFreeAutomata.construct(graph);
+
+				// make glue
+				for (auto symbol : *compiler.getSymbols()) {
+					auto symbolId = compiler.getSymbolId(symbol->mId);
+					if (symbol->mIsTerminal) {
+						auto iter = contextFreeToRegular.presents(symbol->mId);
+						if (!iter) return false;
+						mGrammarGlue.put(contextFreeToRegular.getSlotVal(iter), symbolId);
+					} else {
+						mAstNames.put(symbolId, symbol->mId);
+					}
+				}
 			}
 
 			return true;
 		}
 
-		void parse(const tAlphabetType* sentence, ualni sentenceLength, AST& out) {}
+		ParseResult parse(const tAlphabetType* sentence, ualni sentenceLength) {
+			// get tokens stream
+			Buffer<ContextFreeCompiler::SymbolId> tokens;
+
+			const tAlphabetType* sentenceIter = sentence;
+			ualni lengthIter = sentenceLength;
+
+			while (lengthIter) {
+				auto result = mRegularAutomata.accept(sentenceIter, lengthIter);
+
+				if (!result.accepted) {
+					return { false, nullptr };
+				}
+
+				sentenceIter += result.advancedIdx;
+				lengthIter -= result.advancedIdx;
+
+				tokens.append(mGrammarGlue.get(result.state));
+			}
+
+			ContextFreeAutomata::AcceptResult result = mContextFreeAutomata.accept(tokens.getBuff(), tokens.size());
+			return { result.accepted, result.ast };
+		}
 
 	public:
 		// save load compiled tables
 		RegularAutomata mRegularAutomata;
 		ContextFreeAutomata mContextFreeAutomata;
+
+		Map<tTokenType, ContextFreeCompiler::SymbolId> mGrammarGlue;
+		Map<ContextFreeCompiler::SymbolId, String> mAstNames;
 	};
 }

Language/public/RegularAutomata.hpp

@@ -25,7 +25,7 @@ namespace tp {
 			ualni advancedIdx = 0;
 
 			while (advancedIdx < size) {
-				tAlphabetType& symbol = *(stream + advancedIdx);
+				const tAlphabetType& symbol = *(stream + advancedIdx);
 
 				if (!(symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd)) {
 					return { false, advancedIdx, {} };
@@ -34,7 +34,7 @@ namespace tp {
 				mCurrentState = mTable.get({ (ualni) (symbol - mSymbolRange.mBegin), mCurrentState });
 
 				if (mCurrentState == mStates.size()) {
-					return false;
+					return { false, advancedIdx, {} };
 				}
 
 				if (mStates[mCurrentState].first) {

Language/public/RegularCompiler.hpp

@@ -6,7 +6,7 @@
 
 namespace tp {
 
-	template <typename tAlphabetType, typename tStateType, ualni tMinSymbol, ualni tMaxSymbol>
+	template <typename tAlphabetType, typename tStateType, tStateType tInTransition, ualni tMinSymbol, ualni tMaxSymbol>
 	class RegularCompiler {
 
 		typedef FiniteStateAutomation<tAlphabetType, tStateType> Graph;
@@ -203,6 +203,6 @@ namespace tp {
 			}
 		}
 
-		Vertex* addVertex() { return mGraph->addState(tStateType::InTransition, false); }
+		Vertex* addVertex() { return mGraph->addState(tInTransition, false); }
 	};
 }

Language/public/SimpleParser.hpp

@@ -7,7 +7,10 @@ namespace tp {
 	// Gives ability to express grammar in the Unified Format as sentence
 	template <typename tAlphabetType>
 	class SimpleParser {
-		enum UGTokens : ualni { InTransition = 0, Failed, TestSeq };
+		enum UGTokens : alni { InTransition = -1, TestSeq };
+
+		typedef Parser<tAlphabetType, UGTokens, InTransition, 0, 127> UGParser;
+		typedef Parser<tAlphabetType, alni, -1, 0, 127> UserParser;
 
 	public:
 		SimpleParser() {
@@ -22,23 +25,25 @@ namespace tp {
 			}
 
 			// Define Regular grammar
-			RegularGrammar<tAlphabetType, UGTokens> rg;
+			RegularGrammar<tAlphabetType, UGTokens> regularGrammar;
 			{
 				// this is basically ast from existing tokenizer
-				rg.addRule(rg.seq({ rg.val('a'), rg.val('b') }), TestSeq);
+				regularGrammar.addRule(regularGrammar.seq({ regularGrammar.val('a'), regularGrammar.val('b') }), TestSeq);
 			}
 
-			mUnifiedGrammarParser.compileTables(contextFreeGrammar, rg);
+			Map<String, UGTokens> terminalsMap;
+			terminalsMap.put("TestSeq", TestSeq);
+
+			mUnifiedGrammarParser.compileTables(contextFreeGrammar, regularGrammar, terminalsMap);
 		}
 
 	public:
 		void compileTables(const tAlphabetType* grammar, ualni grammarLength) {
-			AST unifiedGrammarAst;
+			mUnifiedGrammarParser.parse(grammar, grammarLength);
-			mUnifiedGrammarParser.parse(grammar, grammarLength, unifiedGrammarAst);
 
 			// compile each ast into RegularGrammar and ContextFree Grammar api instructions
 			ContextFreeGrammar userContextFreeGrammar;
-			RegularGrammar<tAlphabetType, UGTokens> userRegularGrammar;
+			RegularGrammar<tAlphabetType, alni> userRegularGrammar;
 
 			// ...
 			// split ast into RE and CF part
@@ -47,15 +52,18 @@ namespace tp {
 			// ...
 			// compile tables from user grammar
 
-			mUserParser.compileTables(userContextFreeGrammar, userRegularGrammar);
+			Map<String, alni> terminalsMap;
+			terminalsMap.put("TestSeq", 0);
+
+			mUserParser.compileTables(userContextFreeGrammar, userRegularGrammar, terminalsMap);
 		}
 
-		void parse(const tAlphabetType* grammar, ualni grammarLength, AST& out) {
+		UserParser::ParseResult parse(const tAlphabetType* grammar, ualni grammarLength) {
-			mUserParser.parse(grammar, grammarLength, out);
+			return mUserParser.parse(grammar, grammarLength);
 		}
 
 	private:
-		Parser<tAlphabetType, UGTokens, 0, 127> mUnifiedGrammarParser;
+		UGParser mUnifiedGrammarParser;
-		Parser<tAlphabetType, UGTokens, 0, 127> mUserParser;
+		UserParser mUserParser;
 	};
 }

Language/tests/Tests.cpp

@@ -19,10 +19,9 @@ void testAutomation() {
 
 void test() {
 	auto parser = SimpleParser<int1>();
-	auto ast = AST();
 
 	parser.compileTables(gGrammar, String::Logic::calcLength(gGrammar));
-	parser.parse(gSentence, String::Logic::calcLength(gSentence), ast);
+	auto result = parser.parse(gSentence, String::Logic::calcLength(gSentence));
 }
 
 int main() {