ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adding some tokinizer checks for strings and comments

Browse source
This commit is contained in:
IlushaShurupov 2023-08-07 19:02:32 +03:00
parent 9c8771a378
commit 5fab9ee507
2 changed files with 6 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Objects/private/parser/parser.cpp
View file

@ -82,9 +82,10 @@ Parser::Parser() {
{ "true", Token::CONST_TRUE }, { "true", Token::CONST_TRUE },
{ "false", Token::CONST_FALSE }, { "false", Token::CONST_FALSE },
{ "((\\-)|(\\+))?[0-9]+i?", Token::CONST_INT }, { "((\\-)|(\\+))?[0-9]+i?", Token::CONST_INT },
{ "((\\-)|(\\+))?([0-9]+)(\\.)([0-9]*)?f?", Token::CONST_FLOAT }, { R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)", Token::CONST_FLOAT },
{ "(/\\*){\\*-\\*}*(\\*/)", Token::COMMENT_BLOCK }, { R"((/\*){\*-\*}*(\*/))", Token::COMMENT_BLOCK },
{ "\"{\"-\"}*\"", Token::CONST_STRING }, { R"("{"-"}*")", Token::CONST_STRING },
{ "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", Token::ID }, { "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", Token::ID },
}); });
} }

4
Tokenizer/tests/TestTokenizer.cpp
View file

@ -14,12 +14,12 @@ const char* script =
R"( R"(
/* yea /* yea
comment */ comment "and string inside" with int 123 and float 0.123f */
-1.f; -1.f;
if (+1.f) { if (+1.f) {
var string = "string value var string = "string value some &&characters inside string&& -= asdf 09a
another line"; another line";
} }