ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixing tokenizer bug

Browse source
This commit is contained in:
IlushaShurupov 2023-08-10 18:33:17 +03:00 • committed by Ilya Shurupov
parent a938bf9bb1
commit 92e772cde1
7 changed files with 27 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Parser/private/CfGrammarParser.cpp
View file

@ -42,9 +42,8 @@ struct State {
typedef Buffer<State> Automata;
void initializeTokenizer(CFGTokenizer* tok) {
typedef decltype(tok->getTokenizer()) TokBase;
tok->build({
{ TokBase::etherRE, Token::IGNORED },
{ CFGTokenizer::tTokenizer::etherRE, Token::IGNORED },
{ "Start", Token::START },
{ ":", Token::EQUAL },
{ "\\[", Token::TERMINAL_START },
@ -52,8 +51,8 @@ void initializeTokenizer(CFGTokenizer* tok) {
{ "\\|", Token::ALTERNATION },
{ ";", Token::RULE_END },
{ "&", Token::EPSILON },
{ TokBase::idRE, Token::ID },
{ TokBase::commentBlockRE, Token::COMMENT },
{ CFGTokenizer::tTokenizer::idRE, Token::ID },
{ CFGTokenizer::tTokenizer::commentBlockRE, Token::COMMENT },
});
ASSERT(tok->isBuild())