ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixing tokenizer bug

Browse source
This commit is contained in:
IlushaShurupov 2023-08-10 18:33:17 +03:00
parent 92f34e2a99
commit d888aeb2b4
7 changed files with 27 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Parser/applications/Cfg.cpp
View file

@ -21,7 +21,7 @@ void run(const String& source) {
return; return;
} }
printf("Grammar is accepted.\n"); printf("Grammar accepted.\n");
printf("Example text formed from grammar : TODO\n"); printf("Example text formed from grammar : TODO\n");

7
Parser/private/CfGrammarParser.cpp
View file

@ -42,9 +42,8 @@ struct State {
typedef Buffer<State> Automata; typedef Buffer<State> Automata;
void initializeTokenizer(CFGTokenizer* tok) { void initializeTokenizer(CFGTokenizer* tok) {
typedef decltype(tok->getTokenizer()) TokBase;
tok->build({ tok->build({
{ TokBase::etherRE, Token::IGNORED }, { CFGTokenizer::tTokenizer::etherRE, Token::IGNORED },
{ "Start", Token::START }, { "Start", Token::START },
{ ":", Token::EQUAL }, { ":", Token::EQUAL },
{ "\\[", Token::TERMINAL_START }, { "\\[", Token::TERMINAL_START },
@ -52,8 +51,8 @@ void initializeTokenizer(CFGTokenizer* tok) {
{ "\\|", Token::ALTERNATION }, { "\\|", Token::ALTERNATION },
{ ";", Token::RULE_END }, { ";", Token::RULE_END },
{ "&", Token::EPSILON }, { "&", Token::EPSILON },
{ TokBase::idRE, Token::ID }, { CFGTokenizer::tTokenizer::idRE, Token::ID },
{ TokBase::commentBlockRE, Token::COMMENT }, { CFGTokenizer::tTokenizer::commentBlockRE, Token::COMMENT },
}); });
ASSERT(tok->isBuild()) ASSERT(tok->isBuild())

3
TODO
View file

@ -1,9 +1,6 @@
Testing !! Testing !!
Objects: Objects:
tokenizing:
test all
parsing: parsing:
make cf grammar make cf grammar
make lalr parser make lalr parser

4
Tokenizer/public/AutomataGraph.h
View file

@ -422,6 +422,10 @@ namespace tp {
TransitionMatrix() = default; TransitionMatrix() = default;
auto getStates() const { return &mStates; }
auto getTransitions() const { return &mTransitions; }
auto getStart() const { return mStart; }
void construct(const DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& dfa) { void construct(const DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& dfa) {
mSymbolRange = dfa.getRange(); mSymbolRange = dfa.getRange();
auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin); auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin);

12
Tokenizer/public/RegularExpression.h
View file

@ -351,17 +351,15 @@ namespace tp::RegEx {
halni idx = 0; halni idx = 0;
for (auto rule: aRules) { for (auto rule: aRules) {
auto node = idx ? compileUtil(rule.head, rule.tail) : compileUtil(rule.head, rule.tail, left, right); auto node = compileUtil(rule.head, rule.tail);
if (!(node.left && node.right)) { if (!(node.left && node.right)) {
mError.mRuleIndex = idx; mError.mRuleIndex = idx;
return {}; return {};
} }
if (idx) { transitionAny(left, node.left);
transitionAny(left, node.left); transitionAny(node.right, right);
transitionAny(node.right, right);
}
idx++; idx++;
} }
@ -373,7 +371,7 @@ namespace tp::RegEx {
private: private:
Node compileUtil(const tAlphabetType* regex, tStateType state, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { Node compileUtil(const tAlphabetType* regex, tStateType state) {
Parser parser; Parser parser;
auto astNode = parser.parse(regex); auto astNode = parser.parse(regex);
if (parser.mError.isError()) { if (parser.mError.isError()) {
@ -384,7 +382,7 @@ namespace tp::RegEx {
return {}; return {};
} }
auto node = compileNode(astNode, aLeft, aRight); auto node = compileNode(astNode, nullptr, nullptr);
delete astNode; delete astNode;
mGraph->setVertexState(node.right, state); mGraph->setVertexState(node.right, state);

9
Tokenizer/public/Tokenizer.hpp
View file

@ -50,6 +50,8 @@ namespace tp {
return !mError.isError(); return !mError.isError();
} }
auto getMatrix() const { return &mTransitionMatrix; }
const RegEx::CompileError<tTokType>& getBuildError() { const RegEx::CompileError<tTokType>& getBuildError() {
return mError; return mError;
} }
@ -80,8 +82,11 @@ namespace tp {
template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal, tTokType tSourceEndTokVal> template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal, tTokType tSourceEndTokVal>
class SimpleTokenizer { class SimpleTokenizer {
public:
typedef Tokenizer<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> tTokenizer;
Tokenizer<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> mTokenizer; private:
tTokenizer mTokenizer;
const tAlphabetType* mSource = nullptr; const tAlphabetType* mSource = nullptr;
ualni mLastTokLen = 0; ualni mLastTokLen = 0;
@ -117,7 +122,7 @@ namespace tp {
}; };
SimpleTokenizer() = default; SimpleTokenizer() = default;
auto getTokenizer() const { return mTokenizer; } auto getTokenizer() const { return &mTokenizer; }
void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) { void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) {
mTokenizer.build(rules); mTokenizer.build(rules);

11
Tokenizer/tests/TestTokenizer.cpp
View file

@ -30,7 +30,7 @@ return
)"; )";
TEST_DEF_STATIC(Simple) { TEST_DEF_STATIC(General) {
enum class TokType { enum class TokType {
NONE, NONE,
@ -193,7 +193,7 @@ TEST_DEF_STATIC(Simple) {
TEST(outputHash == outputHashPassed); TEST(outputHash == outputHashPassed);
} }
TEST_DEF(Covarage) { TEST_DEF(Simple) {
enum class TokType { enum class TokType {
START = 0, START = 0,
NONE = 1, NONE = 1,
@ -207,9 +207,9 @@ TEST_DEF(Covarage) {
SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer; SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
lexer.build({ lexer.build({
{ " ", TokType::SPACE },
{ "([a-c]|A)+", TokType::ID }, { "([a-c]|A)+", TokType::ID },
{ "A", TokType::START }, { "A", TokType::START },
{ " ", TokType::SPACE },
}); });
if (!lexer.isBuild()) { if (!lexer.isBuild()) {
@ -218,7 +218,6 @@ TEST_DEF(Covarage) {
} }
lexer.bindSource(" A bc cb cc "); lexer.bindSource(" A bc cb cc ");
/* 3 5 3 0 3 4 3 4 3 6 */
TokType tok; TokType tok;
ualni outputHash = 0; ualni outputHash = 0;
@ -229,10 +228,10 @@ TEST_DEF(Covarage) {
printf(" %i ", int(tok)); printf(" %i ", int(tok));
} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED); } while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
printf(" : %llu", outputHash); printf(" : %llu", outputHash);
TEST(outputHash == 90); TEST(outputHash == 33);
} }
TEST_DEF(Tokenizer) { TEST_DEF(Tokenizer) {
testGeneral();
testSimple(); testSimple();
// testCovarage(); TODO : test environment for an error
} }