ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adding some tokinizer checks for strings and comments

Browse source
This commit is contained in:
IlushaShurupov 2023-08-07 19:02:32 +03:00 • committed by Ilya Shurupov
parent 4825d61486
commit cef0db6fb1
2 changed files with 6 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Objects/private/parser/parser.cpp
View file

@ -82,9 +82,10 @@ Parser::Parser() {
{ "true", Token::CONST_TRUE },
{ "false", Token::CONST_FALSE },
{ "((\\-)|(\\+))?[0-9]+i?", Token::CONST_INT },
{ "((\\-)|(\\+))?([0-9]+)(\\.)([0-9]*)?f?", Token::CONST_FLOAT },
{ "(/\\*){\\*-\\*}*(\\*/)", Token::COMMENT_BLOCK },
{ "\"{\"-\"}*\"", Token::CONST_STRING },
{ R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)", Token::CONST_FLOAT },
{ R"((/\*){\*-\*}*(\*/))", Token::COMMENT_BLOCK },
{ R"("{"-"}*")", Token::CONST_STRING },
{ "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", Token::ID },
});
}

4
Tokenizer/tests/TestTokenizer.cpp
View file

@ -14,12 +14,12 @@ const char* script =
R"(
/* yea
comment */
comment "and string inside" with int 123 and float 0.123f */
-1.f;
if (+1.f) {
var string = "string value
var string = "string value some &&characters inside string&& -= asdf 09a
another line";
}