From 30d4f194baccb7092bdd4cdce72ed0041f340929 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Thu, 8 Feb 2024 21:52:00 +0300 Subject: [PATCH] parsing loop closed --- Language/public/AST.hpp | 8 --- Language/public/ContextFreeAutomata.hpp | 4 +- Language/public/ContextFreeCompiler.hpp | 15 +++--- Language/public/Parser.hpp | 67 +++++++++++++++++++++---- Language/public/RegularAutomata.hpp | 4 +- Language/public/RegularCompiler.hpp | 4 +- Language/public/SimpleParser.hpp | 32 +++++++----- Language/tests/Tests.cpp | 3 +- 8 files changed, 92 insertions(+), 45 deletions(-) delete mode 100644 Language/public/AST.hpp diff --git a/Language/public/AST.hpp b/Language/public/AST.hpp deleted file mode 100644 index 5faffba..0000000 --- a/Language/public/AST.hpp +++ /dev/null @@ -1,8 +0,0 @@ - -#pragma once - -#include "LanguageCommon.hpp" - -namespace tp { - class AST {}; -} diff --git a/Language/public/ContextFreeAutomata.hpp b/Language/public/ContextFreeAutomata.hpp index 1295141..42c783f 100644 --- a/Language/public/ContextFreeAutomata.hpp +++ b/Language/public/ContextFreeAutomata.hpp @@ -37,7 +37,7 @@ namespace tp { ualni advancedIdx = 0; while (advancedIdx < size) { - tAlphabetType& symbol = *(stream + advancedIdx); + const tAlphabetType& symbol = *(stream + advancedIdx); if (!(symbol >= mRange.mBegin && symbol < mRange.mEnd)) { return { false, advancedIdx, nullptr }; @@ -56,7 +56,7 @@ namespace tp { } if (mTable.get({ 0, mCurrentState }).type == Action::REDUCE) { - StackItem* newItem = &mItems.append({}); + StackItem* newItem = &mItems.append(StackItem{}); for (auto iter : Range(action.num)) { newItem->leafs.append(mStack.last()); mCurrentState = mStack.last()->state; diff --git a/Language/public/ContextFreeCompiler.hpp b/Language/public/ContextFreeCompiler.hpp index 2e56f53..8434e0d 100644 --- a/Language/public/ContextFreeCompiler.hpp +++ b/Language/public/ContextFreeCompiler.hpp @@ -8,7 +8,7 @@ namespace tp { class ContextFreeCompiler { public: - typedef ualni SymbolID; + typedef ualni SymbolId; struct Item { const ContextFreeGrammar::Rule* mRule = nullptr; @@ -16,12 +16,12 @@ namespace tp { ualni numArgs() const { return 0; } }; - private: struct Symbol { String mId; bool mIsTerminal = false; }; + private: struct NonTerminal { Buffer rules; Map references; @@ -48,11 +48,14 @@ namespace tp { }; public: - bool compile(const ContextFreeGrammar& grammar, FiniteStateAutomation& automata) { + bool compile(const ContextFreeGrammar& grammar, FiniteStateAutomation& automata) { if (!init(grammar)) return false; return true; } + [[nodiscard]] const Buffer* getSymbols() const { return &mSymbols; } + [[nodiscard]] SymbolId getSymbolId(const String& name) const { return mSymbolLookup.get(name); } + private: bool init(const ContextFreeGrammar& grammar) { @@ -135,7 +138,7 @@ namespace tp { void initSymbols(const ContextFreeGrammar& grammar) { for (auto nonTerminal : mNonTerminals) { mSymbols.append({ nonTerminal->key, false }); - mSymbolLookup.put(nonTerminal->key, SymbolID(mSymbols.size() - 1)); + mSymbolLookup.put(nonTerminal->key, SymbolId(mSymbols.size() - 1)); for (auto rule : nonTerminal->val.rules) { for (auto arg : *rule->getArgs()) { @@ -144,7 +147,7 @@ namespace tp { mTerminals.put(arg->getId(), {}); mSymbols.append({ nonTerminal->key, true }); - mSymbolLookup.put(nonTerminal->key, SymbolID(mSymbols.size() - 1)); + mSymbolLookup.put(nonTerminal->key, SymbolId(mSymbols.size() - 1)); } } } @@ -154,6 +157,6 @@ namespace tp { Map mNonTerminals; Map mTerminals; Buffer mSymbols; - Map mSymbolLookup; + Map mSymbolLookup; }; } diff --git a/Language/public/Parser.hpp b/Language/public/Parser.hpp index 15855d7..59d0767 100644 --- a/Language/public/Parser.hpp +++ b/Language/public/Parser.hpp @@ -1,8 +1,6 @@ #pragma once -#include "AST.hpp" - #include "RegularCompiler.hpp" #include "RegularAutomata.hpp" @@ -11,24 +9,34 @@ namespace tp { - template + template class Parser { - typedef RegularGrammar RegularGrammar; - typedef RegularCompiler RegularCompiler; - typedef FiniteStateAutomation RegularGraph; - typedef RegularAutomata RegularAutomata; + typedef RegularGrammar RegularGrammar; + typedef RegularCompiler RegularCompiler; + typedef FiniteStateAutomation RegularGraph; + typedef RegularAutomata RegularAutomata; // ContextFreeGrammar; // ContextFreeCompiler; - typedef FiniteStateAutomation ContextFreeGraph; - typedef ContextFreeAutomata ContextFreeAutomata; + typedef FiniteStateAutomation ContextFreeGraph; + typedef ContextFreeAutomata ContextFreeAutomata; + + public: + struct ParseResult { + bool accepted = false; + const ContextFreeAutomata::StackItem* ast = nullptr; + }; public: Parser() = default; public: - bool compileTables(const ContextFreeGrammar& cfGrammar, const RegularGrammar& reGrammar) { + bool compileTables( + const ContextFreeGrammar& cfGrammar, + const RegularGrammar& reGrammar, + const Map& contextFreeToRegular + ) { // Compile Regular Grammar { RegularGraph graph; @@ -45,16 +53,53 @@ namespace tp { compiler.compile(cfGrammar, graph); graph.makeDeterministic(); mContextFreeAutomata.construct(graph); + + // make glue + for (auto symbol : *compiler.getSymbols()) { + auto symbolId = compiler.getSymbolId(symbol->mId); + if (symbol->mIsTerminal) { + auto iter = contextFreeToRegular.presents(symbol->mId); + if (!iter) return false; + mGrammarGlue.put(contextFreeToRegular.getSlotVal(iter), symbolId); + } else { + mAstNames.put(symbolId, symbol->mId); + } + } } return true; } - void parse(const tAlphabetType* sentence, ualni sentenceLength, AST& out) {} + ParseResult parse(const tAlphabetType* sentence, ualni sentenceLength) { + // get tokens stream + Buffer tokens; + + const tAlphabetType* sentenceIter = sentence; + ualni lengthIter = sentenceLength; + + while (lengthIter) { + auto result = mRegularAutomata.accept(sentenceIter, lengthIter); + + if (!result.accepted) { + return { false, nullptr }; + } + + sentenceIter += result.advancedIdx; + lengthIter -= result.advancedIdx; + + tokens.append(mGrammarGlue.get(result.state)); + } + + ContextFreeAutomata::AcceptResult result = mContextFreeAutomata.accept(tokens.getBuff(), tokens.size()); + return { result.accepted, result.ast }; + } public: // save load compiled tables RegularAutomata mRegularAutomata; ContextFreeAutomata mContextFreeAutomata; + + Map mGrammarGlue; + Map mAstNames; }; } diff --git a/Language/public/RegularAutomata.hpp b/Language/public/RegularAutomata.hpp index 7b7e94f..501dada 100644 --- a/Language/public/RegularAutomata.hpp +++ b/Language/public/RegularAutomata.hpp @@ -25,7 +25,7 @@ namespace tp { ualni advancedIdx = 0; while (advancedIdx < size) { - tAlphabetType& symbol = *(stream + advancedIdx); + const tAlphabetType& symbol = *(stream + advancedIdx); if (!(symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd)) { return { false, advancedIdx, {} }; @@ -34,7 +34,7 @@ namespace tp { mCurrentState = mTable.get({ (ualni) (symbol - mSymbolRange.mBegin), mCurrentState }); if (mCurrentState == mStates.size()) { - return false; + return { false, advancedIdx, {} }; } if (mStates[mCurrentState].first) { diff --git a/Language/public/RegularCompiler.hpp b/Language/public/RegularCompiler.hpp index 25571fa..a27f464 100644 --- a/Language/public/RegularCompiler.hpp +++ b/Language/public/RegularCompiler.hpp @@ -6,7 +6,7 @@ namespace tp { - template + template class RegularCompiler { typedef FiniteStateAutomation Graph; @@ -203,6 +203,6 @@ namespace tp { } } - Vertex* addVertex() { return mGraph->addState(tStateType::InTransition, false); } + Vertex* addVertex() { return mGraph->addState(tInTransition, false); } }; } diff --git a/Language/public/SimpleParser.hpp b/Language/public/SimpleParser.hpp index c860c36..ad1dc6e 100644 --- a/Language/public/SimpleParser.hpp +++ b/Language/public/SimpleParser.hpp @@ -7,7 +7,10 @@ namespace tp { // Gives ability to express grammar in the Unified Format as sentence template class SimpleParser { - enum UGTokens : ualni { InTransition = 0, Failed, TestSeq }; + enum UGTokens : alni { InTransition = -1, TestSeq }; + + typedef Parser UGParser; + typedef Parser UserParser; public: SimpleParser() { @@ -22,23 +25,25 @@ namespace tp { } // Define Regular grammar - RegularGrammar rg; + RegularGrammar regularGrammar; { // this is basically ast from existing tokenizer - rg.addRule(rg.seq({ rg.val('a'), rg.val('b') }), TestSeq); + regularGrammar.addRule(regularGrammar.seq({ regularGrammar.val('a'), regularGrammar.val('b') }), TestSeq); } - mUnifiedGrammarParser.compileTables(contextFreeGrammar, rg); + Map terminalsMap; + terminalsMap.put("TestSeq", TestSeq); + + mUnifiedGrammarParser.compileTables(contextFreeGrammar, regularGrammar, terminalsMap); } public: void compileTables(const tAlphabetType* grammar, ualni grammarLength) { - AST unifiedGrammarAst; - mUnifiedGrammarParser.parse(grammar, grammarLength, unifiedGrammarAst); + mUnifiedGrammarParser.parse(grammar, grammarLength); // compile each ast into RegularGrammar and ContextFree Grammar api instructions ContextFreeGrammar userContextFreeGrammar; - RegularGrammar userRegularGrammar; + RegularGrammar userRegularGrammar; // ... // split ast into RE and CF part @@ -47,15 +52,18 @@ namespace tp { // ... // compile tables from user grammar - mUserParser.compileTables(userContextFreeGrammar, userRegularGrammar); + Map terminalsMap; + terminalsMap.put("TestSeq", 0); + + mUserParser.compileTables(userContextFreeGrammar, userRegularGrammar, terminalsMap); } - void parse(const tAlphabetType* grammar, ualni grammarLength, AST& out) { - mUserParser.parse(grammar, grammarLength, out); + UserParser::ParseResult parse(const tAlphabetType* grammar, ualni grammarLength) { + return mUserParser.parse(grammar, grammarLength); } private: - Parser mUnifiedGrammarParser; - Parser mUserParser; + UGParser mUnifiedGrammarParser; + UserParser mUserParser; }; } diff --git a/Language/tests/Tests.cpp b/Language/tests/Tests.cpp index 3f7a42e..09ef6a6 100644 --- a/Language/tests/Tests.cpp +++ b/Language/tests/Tests.cpp @@ -19,10 +19,9 @@ void testAutomation() { void test() { auto parser = SimpleParser(); - auto ast = AST(); parser.compileTables(gGrammar, String::Logic::calcLength(gGrammar)); - parser.parse(gSentence, String::Logic::calcLength(gSentence), ast); + auto result = parser.parse(gSentence, String::Logic::calcLength(gSentence)); } int main() {