From 92e772cde16479cd92774d07da55d30dbf8b75a3 Mon Sep 17 00:00:00 2001 From: IlushaShurupov Date: Thu, 10 Aug 2023 18:33:17 +0300 Subject: [PATCH] Fixing tokenizer bug --- Parser/applications/Cfg.cpp | 2 +- Parser/private/CfGrammarParser.cpp | 7 +++---- TODO | 3 --- Tokenizer/public/AutomataGraph.h | 4 ++++ Tokenizer/public/RegularExpression.h | 12 +++++------- Tokenizer/public/Tokenizer.hpp | 11 ++++++++--- Tokenizer/tests/TestTokenizer.cpp | 13 ++++++------- 7 files changed, 27 insertions(+), 25 deletions(-) diff --git a/Parser/applications/Cfg.cpp b/Parser/applications/Cfg.cpp index bfedbb3..6b22bb5 100644 --- a/Parser/applications/Cfg.cpp +++ b/Parser/applications/Cfg.cpp @@ -21,7 +21,7 @@ void run(const String& source) { return; } - printf("Grammar is accepted.\n"); + printf("Grammar accepted.\n"); printf("Example text formed from grammar : TODO\n"); diff --git a/Parser/private/CfGrammarParser.cpp b/Parser/private/CfGrammarParser.cpp index 16645c6..d20b27a 100644 --- a/Parser/private/CfGrammarParser.cpp +++ b/Parser/private/CfGrammarParser.cpp @@ -42,9 +42,8 @@ struct State { typedef Buffer Automata; void initializeTokenizer(CFGTokenizer* tok) { - typedef decltype(tok->getTokenizer()) TokBase; tok->build({ - { TokBase::etherRE, Token::IGNORED }, + { CFGTokenizer::tTokenizer::etherRE, Token::IGNORED }, { "Start", Token::START }, { ":", Token::EQUAL }, { "\\[", Token::TERMINAL_START }, @@ -52,8 +51,8 @@ void initializeTokenizer(CFGTokenizer* tok) { { "\\|", Token::ALTERNATION }, { ";", Token::RULE_END }, { "&", Token::EPSILON }, - { TokBase::idRE, Token::ID }, - { TokBase::commentBlockRE, Token::COMMENT }, + { CFGTokenizer::tTokenizer::idRE, Token::ID }, + { CFGTokenizer::tTokenizer::commentBlockRE, Token::COMMENT }, }); ASSERT(tok->isBuild()) diff --git a/TODO b/TODO index 1d83f18..a8767fb 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,6 @@ Testing !! Objects: - tokenizing: - test all - parsing: make cf grammar make lalr parser diff --git a/Tokenizer/public/AutomataGraph.h b/Tokenizer/public/AutomataGraph.h index 3913535..5a03edb 100644 --- a/Tokenizer/public/AutomataGraph.h +++ b/Tokenizer/public/AutomataGraph.h @@ -422,6 +422,10 @@ namespace tp { TransitionMatrix() = default; + auto getStates() const { return &mStates; } + auto getTransitions() const { return &mTransitions; } + auto getStart() const { return mStart; } + void construct(const DFA& dfa) { mSymbolRange = dfa.getRange(); auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin); diff --git a/Tokenizer/public/RegularExpression.h b/Tokenizer/public/RegularExpression.h index 322cd6a..7fa668f 100644 --- a/Tokenizer/public/RegularExpression.h +++ b/Tokenizer/public/RegularExpression.h @@ -351,17 +351,15 @@ namespace tp::RegEx { halni idx = 0; for (auto rule: aRules) { - auto node = idx ? compileUtil(rule.head, rule.tail) : compileUtil(rule.head, rule.tail, left, right); + auto node = compileUtil(rule.head, rule.tail); if (!(node.left && node.right)) { mError.mRuleIndex = idx; return {}; } - if (idx) { - transitionAny(left, node.left); - transitionAny(node.right, right); - } + transitionAny(left, node.left); + transitionAny(node.right, right); idx++; } @@ -373,7 +371,7 @@ namespace tp::RegEx { private: - Node compileUtil(const tAlphabetType* regex, tStateType state, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + Node compileUtil(const tAlphabetType* regex, tStateType state) { Parser parser; auto astNode = parser.parse(regex); if (parser.mError.isError()) { @@ -384,7 +382,7 @@ namespace tp::RegEx { return {}; } - auto node = compileNode(astNode, aLeft, aRight); + auto node = compileNode(astNode, nullptr, nullptr); delete astNode; mGraph->setVertexState(node.right, state); diff --git a/Tokenizer/public/Tokenizer.hpp b/Tokenizer/public/Tokenizer.hpp index dad09ea..869beba 100644 --- a/Tokenizer/public/Tokenizer.hpp +++ b/Tokenizer/public/Tokenizer.hpp @@ -50,6 +50,8 @@ namespace tp { return !mError.isError(); } + auto getMatrix() const { return &mTransitionMatrix; } + const RegEx::CompileError& getBuildError() { return mError; } @@ -80,8 +82,11 @@ namespace tp { template class SimpleTokenizer { - - Tokenizer mTokenizer; + public: + typedef Tokenizer tTokenizer; + + private: + tTokenizer mTokenizer; const tAlphabetType* mSource = nullptr; ualni mLastTokLen = 0; @@ -117,7 +122,7 @@ namespace tp { }; SimpleTokenizer() = default; - auto getTokenizer() const { return mTokenizer; } + auto getTokenizer() const { return &mTokenizer; } void build(const InitialierList>& rules) { mTokenizer.build(rules); diff --git a/Tokenizer/tests/TestTokenizer.cpp b/Tokenizer/tests/TestTokenizer.cpp index 9ca3a38..350e51e 100644 --- a/Tokenizer/tests/TestTokenizer.cpp +++ b/Tokenizer/tests/TestTokenizer.cpp @@ -30,7 +30,7 @@ return )"; -TEST_DEF_STATIC(Simple) { +TEST_DEF_STATIC(General) { enum class TokType { NONE, @@ -193,7 +193,7 @@ TEST_DEF_STATIC(Simple) { TEST(outputHash == outputHashPassed); } -TEST_DEF(Covarage) { +TEST_DEF(Simple) { enum class TokType { START = 0, NONE = 1, @@ -207,9 +207,9 @@ TEST_DEF(Covarage) { SimpleTokenizer lexer; lexer.build({ + { " ", TokType::SPACE }, { "([a-c]|A)+", TokType::ID }, { "A", TokType::START }, - { " ", TokType::SPACE }, }); if (!lexer.isBuild()) { @@ -218,7 +218,6 @@ TEST_DEF(Covarage) { } lexer.bindSource(" A bc cb cc "); - /* 3 5 3 0 3 4 3 4 3 6 */ TokType tok; ualni outputHash = 0; @@ -229,10 +228,10 @@ TEST_DEF(Covarage) { printf(" %i ", int(tok)); } while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED); printf(" : %llu", outputHash); - TEST(outputHash == 90); + TEST(outputHash == 33); } TEST_DEF(Tokenizer) { + testGeneral(); testSimple(); - // testCovarage(); TODO : test environment for an error -} \ No newline at end of file +}