From cef0db6fb13a7c362e8d432ef40bc4911d3035fc Mon Sep 17 00:00:00 2001
From: IlushaShurupov <ilusha.main@gmail.com>
Date: Mon, 7 Aug 2023 19:02:32 +0300
Subject: [PATCH] Adding some tokinizer checks for strings and comments

---
 Objects/private/parser/parser.cpp | 7 ++++---
 Tokenizer/tests/TestTokenizer.cpp | 4 ++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Objects/private/parser/parser.cpp b/Objects/private/parser/parser.cpp
index 7139dab..8c159d7 100644
--- a/Objects/private/parser/parser.cpp
+++ b/Objects/private/parser/parser.cpp
@@ -82,9 +82,10 @@ Parser::Parser() {
 		{ "true",	 Token::CONST_TRUE },
 		{ "false", Token::CONST_FALSE },
 		{ "((\\-)|(\\+))?[0-9]+i?",	 Token::CONST_INT },
-		{ "((\\-)|(\\+))?([0-9]+)(\\.)([0-9]*)?f?",	 Token::CONST_FLOAT },
-		{ "(/\\*){\\*-\\*}*(\\*/)",	 Token::COMMENT_BLOCK },
-		{ "\"{\"-\"}*\"",	 Token::CONST_STRING },
+		{ R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)",	 Token::CONST_FLOAT },
+		{ R"((/\*){\*-\*}*(\*/))",	 Token::COMMENT_BLOCK },
+		{ R"("{"-"}*")",	 Token::CONST_STRING },
+
 		{ "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*",	 Token::ID },
 	});
 }
diff --git a/Tokenizer/tests/TestTokenizer.cpp b/Tokenizer/tests/TestTokenizer.cpp
index fd53e5d..0a991e3 100644
--- a/Tokenizer/tests/TestTokenizer.cpp
+++ b/Tokenizer/tests/TestTokenizer.cpp
@@ -14,12 +14,12 @@ const char* script =
 R"(
 
 /* yea
-comment  */
+comment  "and string inside"  with int 123 and float 0.123f */
 
 -1.f;
 
 if (+1.f) {
-	var string = "string value
+	var string = "string value some &&characters inside string&& -= asdf 09a
 	another line";
 
 }