diff --git a/CMakeLists.txt b/CMakeLists.txt index 94a5c9a..1937060 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,7 @@ add_subdirectory(Math) add_subdirectory(Allocators) add_subdirectory(Strings) add_subdirectory(Tokenizer) +add_subdirectory(Parser) add_subdirectory(CommandLine) add_subdirectory(Externals) add_subdirectory(Connection) diff --git a/Parser/CMakeLists.txt b/Parser/CMakeLists.txt index 8af7064..947b1c6 100644 --- a/Parser/CMakeLists.txt +++ b/Parser/CMakeLists.txt @@ -10,7 +10,7 @@ file(GLOB SOURCES "./private/*.cpp") file(GLOB HEADERS "./public/*.hpp") add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS}) target_include_directories(${PROJECT_NAME} PUBLIC ./public/) -target_link_libraries(${PROJECT_NAME} PUBLIC Strings) +target_link_libraries(${PROJECT_NAME} PUBLIC Tokenizer) ### -------------------------- Tests -------------------------- ### enable_testing() diff --git a/Parser/private/CfGrammarParser.cpp b/Parser/private/CfGrammarParser.cpp new file mode 100644 index 0000000..6fc4dbe --- /dev/null +++ b/Parser/private/CfGrammarParser.cpp @@ -0,0 +1,190 @@ + +#include "NewPlacement.hpp" + +#include "CfGrammar.hpp" +#include "Tokenizer.hpp" + +using namespace tp; + +enum class Token { + NO_TOKEN, + START, + ID, + EQUAL, + TERMINAL_START, + TERMINAL_END, + ALTERNATION, + RULE_END, + IGNORED, + END, + FAILED, +}; + +typedef int1 AlphabetType; +typedef SimpleTokenizer CFGTokenizer; + +struct State { + + struct Transition { + Token tok = Token::FAILED; + State* target = nullptr; + void (*action)(CfGrammar* grammar, const String& tok) = nullptr; + }; + + String error; + Buffer transitions; + bool accept = false; +}; + +typedef Buffer Automata; + +void initializeTokenizer(CFGTokenizer* tok) { + typedef decltype(tok->getTokenizer()) TokBase; + tok->build({ + { TokBase::idRE, Token::ID }, + { TokBase::etherRE, Token::IGNORED }, + { "Start", Token::START }, + { ":", Token::EQUAL }, + { "[", Token::TERMINAL_START }, + { "]", Token::TERMINAL_END }, + { "|", Token::ALTERNATION }, + { ";", Token::RULE_END }, + }); +} + +void initAutomata(Automata& automata) { + automata.reserve(8); + auto states = &automata.first(); + + auto root = states++; + auto end = states++; + auto start = states++; + auto startEnd = states++; + auto rule = states++; + auto rhs = states++; + auto terminalStart = states++; + auto terminalEnd = states; + + root->transitions = { + { Token::END, end }, + { Token::START, start }, + { Token::ID, rule, [](CfGrammar* grammar, const String& tok) { + grammar->rules.pushBack({ tok, {} }); + } + }, + }; + + start->transitions = { + { Token::ID, startEnd}, + }; + + startEnd->transitions = { + { Token::RULE_END, + root, + [](CfGrammar* grammar, const String& tok) { + grammar->startTerminal = tok; + } + }, + }; + + rule->transitions = { + { Token::EQUAL, rhs }, + }; + + rhs->transitions = { + { Token::ALTERNATION, rhs, [](CfGrammar* grammar, const String& tok) { + auto const& id = grammar->rules.last()->data.id; + grammar->rules.pushBack({ id, {} }); + } + }, + { Token::ID, rhs, [](CfGrammar* grammar, const String& tok) { + grammar->rules.last()->data.args.pushBack({ tok, false }); + } + }, + { Token::TERMINAL_START, terminalStart }, + { Token::RULE_END, root }, + }; + + terminalStart->transitions = { + { Token::ID, rhs, [](CfGrammar* grammar, const String& tok) { + grammar->rules.last()->data.args.pushBack({ tok, true }); + } + }, + }; + + terminalEnd->transitions = { + { Token::TERMINAL_END, rhs }, + }; + + auto idError = "Expected an identifier"; + auto stmEndError = "Expected an statement end ';'"; + + root->error = "Expected 'Start' statement, rule or end of text"; + start->error = idError; + startEnd->error = stmEndError; + rule->error = "Expected production equality ':'"; + rhs->error = "Expected alternation '|', terminal id, statement end ';' or terminal start '[' "; + terminalStart->error = idError; + terminalEnd->error = "Expected terminal end '[' "; + + end->accept = true; +} + +bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGrammar* grammar) { + tok->reset(); + tok->bindSource(text); + + State* state = &(*automata)[0]; + while (!state->accept) { + + auto token = tok->readTok(); + + if (token == Token::FAILED) { + auto errorLocation = tok->getCursorPrev().get2DLocation(); + printf("Syntax error at (%llu, %llu)\n", errorLocation.line, errorLocation.character); + return false; + } + + auto tokVal = tok->extractVal(); + + bool isTransition = false; + + for (auto transition : state->transitions) { + if (transition->tok == token) { + transition->action(grammar, tokVal); + state = transition->target; + isTransition = true; + break; + } + } + + if (!isTransition) { + // if no tok matched to any transition produce error + auto errorLocation = tok->getCursorPrev().get2DLocation(); + printf("Parse error at (%llu, %llu) - %s", errorLocation.line, errorLocation.character, state->error.read()); + return false; + } + } + + return true; +} + +struct tp::CfGrammarParserState { + Automata automata; + CFGTokenizer tokenizer; +}; + +CfGrammarParserState* CfGrammar::initializeCfGrammarParser() { + auto state = new CfGrammarParserState(); + initializeTokenizer(&state->tokenizer); + initAutomata(state->automata); + return state; +} + +void CfGrammar::deinitializeCfGrammarParser(CfGrammarParserState* state) { + delete state; +} + +void CfGrammar::parse(CfGrammarParserState* context, const int1* source) { + ::parse(source, &context->automata, &context->tokenizer, this); +} \ No newline at end of file diff --git a/Parser/private/Parser.cpp b/Parser/private/Parser.cpp deleted file mode 100644 index e69de29..0000000 diff --git a/Parser/public/CfGrammar.hpp b/Parser/public/CfGrammar.hpp new file mode 100644 index 0000000..0cff23f --- /dev/null +++ b/Parser/public/CfGrammar.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include "Strings.hpp" +#include "List.hpp" + +namespace tp { + + struct CfGrammar { + struct Rule { + struct Arg { + String id; + bool isTerminal; + }; + + String id; + List args; + }; + + List rules; + String startTerminal; + + public: + static struct CfGrammarParserState* initializeCfGrammarParser(); + static void deinitializeCfGrammarParser(CfGrammarParserState*); + + public: + void parse(CfGrammarParserState* context, const int1* source); + }; +} \ No newline at end of file diff --git a/Parser/public/Parser.hpp b/Parser/public/Parser.hpp deleted file mode 100644 index 4c64fe0..0000000 --- a/Parser/public/Parser.hpp +++ /dev/null @@ -1,105 +0,0 @@ -#pragma once - -#include "Strings.hpp" -#include "Map.hpp" -#include "List.hpp" - -namespace tp { - - template - class ContextFreeGrammar { - public: - class NonTerminalSymbol; - - class RHSArg { - tAlphabetType terminal; - const NonTerminalSymbol* nonTerminal = nullptr; - public: - RHSArg(const NonTerminalSymbol* aNonTerminal) : nonTerminal(aNonTerminal) {} - RHSArg(tAlphabetType terminal) : terminal(terminal) {} - }; - - class NonTerminalSymbol { - String name; - String description; - - struct Rule { - List mArgs; - }; - - List mRules; - - Rule* newRule() { - mRules.pushBack(); - return mRules.last()->data; - } - }; - - private: - Map mNonTerminals; - NonTerminalSymbol* mStart; - - public: - ContextFreeGrammar() = default; - - void setStartNonTerminal(const NonTerminalSymbol* start) { - mStart = start; - } - - const NonTerminalSymbol* addRule(const String& LHS, const InitialierList RHS) { - NonTerminalSymbol* nonTerminal = nullptr; - auto idx = mNonTerminals.presents(LHS); - if (idx) { - nonTerminal = mNonTerminals.get(idx); - } else { - nonTerminal = new NonTerminalSymbol(LHS, "None"); - mNonTerminals.put(LHS, nonTerminal); - } - - auto* rule = nonTerminal->newRule(); - for (auto arg : RHS) { - rule->pushBack(arg); - } - - return nonTerminal; - } - - bool checkSemantic() { - return false; - } - - bool checkNonAmbiguous() { - return false; - } - }; - - template - class Parser { - typedef ContextFreeGrammar Grammar; - - public: - class ASTNode { - const Grammar::NonTerminalSymbol* mNonTerminal; - List mLeafs; - }; - - private: - class InputTape { - }; - - class StateStack { - }; - - private: - Grammar mGrammar; - - public: - Parser() = default; - - Grammar& getGrammar() { return mGrammar; } - - void buildTables() {} - - const ASTNode* parse() {} - }; -} \ No newline at end of file diff --git a/Tokenizer/public/Tokenizer.hpp b/Tokenizer/public/Tokenizer.hpp index bf32739..dad09ea 100644 --- a/Tokenizer/public/Tokenizer.hpp +++ b/Tokenizer/public/Tokenizer.hpp @@ -19,6 +19,15 @@ namespace tp { return mTransitionMatrix.isTrapped(); } + public: + // Some useful RE to be reused + static constexpr const tAlphabetType* etherRE = "\n|\t| |\r"; + static constexpr const tAlphabetType* intRE = "((\\-)|(\\+))?[0-9]+i?"; + static constexpr const tAlphabetType* floatRE = R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)"; + static constexpr const tAlphabetType* commentBlockRE = R"((/\*){\*-\*}*(\*/))"; + static constexpr const tAlphabetType* stringRE = R"("{"-"}*")"; + static constexpr const tAlphabetType* idRE = "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*"; + public: Tokenizer() { @@ -85,9 +94,30 @@ namespace tp { const tAlphabetType* mSource = nullptr; ualni mAdvancedOffset = 0; const tAlphabetType* str() { return mSource + mAdvancedOffset; } + + struct Location2D { + ualni line = 0; + ualni character = 0; + }; + + Location2D get2DLocation() { + Location2D out; + typedef typename StringLogic::Index Idx; + Buffer offsets; + StringLogic::calcLineOffsets(mSource, StringLogic::calcLength(mSource), offsets); + for (ualni idx = 1; idx < offsets.size(); idx++) { + if (offsets[idx] > mAdvancedOffset) { + out.line = idx - 1; + break; + } + } + out.character = mAdvancedOffset - offsets[out.line]; + return out; + } }; SimpleTokenizer() = default; + auto getTokenizer() const { return mTokenizer; } void build(const InitialierList>& rules) { mTokenizer.build(rules);