From a938bf9bb10a12726e5c5369f9a35a3278ed2ccd Mon Sep 17 00:00:00 2001 From: IlushaShurupov Date: Thu, 10 Aug 2023 17:29:28 +0300 Subject: [PATCH] CF Grammar improvements --- CommandLine/private/CommandLine.cpp | 1 + Parser/CMakeLists.txt | 6 +++ Parser/applications/Cfg.cpp | 72 +++++++++++++++++++++++++++++ Parser/private/CfGrammar.cpp | 62 +++++++++++++++++++++++++ Parser/private/CfGrammarParser.cpp | 52 +++++++++++++-------- Parser/private/Parser.cpp | 8 ++++ Parser/public/CfGrammar.hpp | 22 ++++++++- Parser/public/Parser.hpp | 7 +++ Parser/rsc/grammar.txt | 12 +++++ Tokenizer/tests/TestTokenizer.cpp | 40 ++++++++++++++++ 10 files changed, 262 insertions(+), 20 deletions(-) create mode 100644 Parser/applications/Cfg.cpp create mode 100644 Parser/private/CfGrammar.cpp create mode 100644 Parser/private/Parser.cpp create mode 100644 Parser/public/Parser.hpp create mode 100644 Parser/rsc/grammar.txt diff --git a/CommandLine/private/CommandLine.cpp b/CommandLine/private/CommandLine.cpp index 35bfb35..a7172b7 100644 --- a/CommandLine/private/CommandLine.cpp +++ b/CommandLine/private/CommandLine.cpp @@ -337,6 +337,7 @@ void CommandLine::parseArg(Arg& arg, const char* src) { auto tok = mTokenizer.readTok(); if (tok < TokType::INT || tok > TokType::STR) { ErrInvalidArgSyntax(&arg); + return; } auto val = mTokenizer.extractVal(); diff --git a/Parser/CMakeLists.txt b/Parser/CMakeLists.txt index 947b1c6..eda4fcc 100644 --- a/Parser/CMakeLists.txt +++ b/Parser/CMakeLists.txt @@ -12,6 +12,12 @@ add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS}) target_include_directories(${PROJECT_NAME} PUBLIC ./public/) target_link_libraries(${PROJECT_NAME} PUBLIC Tokenizer) +### -------------------------- Applications -------------------------- ### +add_executable(cfg ./applications/Cfg.cpp ) +target_link_libraries(cfg ${PROJECT_NAME} CommandLine) + +configure_file(rsc/grammar.txt grammar.txt COPYONLY) + ### -------------------------- Tests -------------------------- ### enable_testing() file(GLOB TEST_SOURCES "./tests/*.cpp") diff --git a/Parser/applications/Cfg.cpp b/Parser/applications/Cfg.cpp new file mode 100644 index 0000000..bfedbb3 --- /dev/null +++ b/Parser/applications/Cfg.cpp @@ -0,0 +1,72 @@ + +#include "NewPlacement.hpp" + +#include "Parser.hpp" +#include "CommandLine.hpp" + +using namespace tp; + +void run(const String& source) { + auto state = CfGrammar::initializeCfGrammarParser(); + + CfGrammar grammar; + + if (!grammar.parse(state, source)) { + printf("Parsing is failed\n"); + return; + } + + if (!grammar.compile()) { + printf("Compilation is failed\n"); + return; + } + + printf("Grammar is accepted.\n"); + + printf("Example text formed from grammar : TODO\n"); + + CfGrammar::deinitializeCfGrammarParser(state); +} + +int main(int argc, const char* argv[]) { + tp::ModuleManifest* deps[] = { &tp::gModuleParser, &tp::gModuleCommandLine, nullptr }; + tp::ModuleManifest testModule("CommandLineTest", nullptr, nullptr, deps); + + if (!testModule.initialize()) { + return 1; + } + + { + CommandLine cmd = { + { "grammar", CommandLine::Arg::STR }, + }; + + if (!cmd.parse((char) argc, argv, true, 1)) { + return 1; + } + + auto location = cmd.getString("grammar"); + + LocalConnection file; + String grammar; + + file.connect(LocalConnection::Location(location), LocalConnection::Type(true)); + + if (!file.getConnectionStatus().isOpened()) { + printf("Cant open file\n"); + return 1; + } + + auto size = file.size(); + grammar.resize((String::Index) size); + file.readBytes(grammar.write(), size); + + + run(grammar); + + file.disconnect(); + } + + testModule.deinitialize(); + return 0; +} \ No newline at end of file diff --git a/Parser/private/CfGrammar.cpp b/Parser/private/CfGrammar.cpp new file mode 100644 index 0000000..a265f51 --- /dev/null +++ b/Parser/private/CfGrammar.cpp @@ -0,0 +1,62 @@ + +#include "NewPlacement.hpp" +#include "CfGrammar.hpp" + +using namespace tp; + +bool CfGrammar::compile() { + + if (!rules.length()) { + printf("Grammar must have at leas one rule\n"); + return false; + } + + if (startTerminal == String()) { + printf("Using first rule's non-terminal as grammar's root. Define explicitly - 'Start : id'\n"); + startTerminal = rules.first()->data.id; + } + + // find all rules + for (auto rule : rules) { + + if (!rule->args.length()) { + printf("Rule must have at leas one terminal or non-terminal. See rule '%s'\n", rule->id.read()); + return false; + } + + if (!mNonTerminals.presents(rule->id)) { + mNonTerminals.put(rule->id, {}); + } + auto nonTerminal = &mNonTerminals.get(rule->id); + + nonTerminal->rules.pushBack(&rule.data()); + } + + for (auto nonTerminal : mNonTerminals) { + for (auto rule : nonTerminal->val.rules) { + for (auto arg : rule->args) { + + if (arg->isTerminal) { + mTerminals.put(rule->id, {}); + } else { + auto idx = mNonTerminals.presents(arg->id); + if (!idx) { + printf("Referenced non-terminal '%s' is not defined\n", arg->id.read()); + return false; + } + auto reference = &mNonTerminals.getSlotVal(idx); + reference->references.put(nonTerminal->key, &nonTerminal->val); + nonTerminal->val.referencing.put(arg->id, reference); + } + } + } + } + + for (auto nonTerminal : mNonTerminals) { + if (!nonTerminal->val.references.size() && nonTerminal->key != startTerminal) { + printf("Non-terminal '%s' is defined but not used\n", nonTerminal->key.read()); + } + } + + return true; +} \ No newline at end of file diff --git a/Parser/private/CfGrammarParser.cpp b/Parser/private/CfGrammarParser.cpp index 6fc4dbe..16645c6 100644 --- a/Parser/private/CfGrammarParser.cpp +++ b/Parser/private/CfGrammarParser.cpp @@ -16,6 +16,8 @@ enum class Token { ALTERNATION, RULE_END, IGNORED, + EPSILON, + COMMENT, END, FAILED, }; @@ -31,6 +33,7 @@ struct State { void (*action)(CfGrammar* grammar, const String& tok) = nullptr; }; + String name; String error; Buffer transitions; bool accept = false; @@ -41,25 +44,28 @@ typedef Buffer Automata; void initializeTokenizer(CFGTokenizer* tok) { typedef decltype(tok->getTokenizer()) TokBase; tok->build({ - { TokBase::idRE, Token::ID }, { TokBase::etherRE, Token::IGNORED }, { "Start", Token::START }, { ":", Token::EQUAL }, - { "[", Token::TERMINAL_START }, - { "]", Token::TERMINAL_END }, - { "|", Token::ALTERNATION }, + { "\\[", Token::TERMINAL_START }, + { "\\]", Token::TERMINAL_END }, + { "\\|", Token::ALTERNATION }, { ";", Token::RULE_END }, + { "&", Token::EPSILON }, + { TokBase::idRE, Token::ID }, + { TokBase::commentBlockRE, Token::COMMENT }, }); + + ASSERT(tok->isBuild()) } void initAutomata(Automata& automata) { - automata.reserve(8); + automata.reserve(7); auto states = &automata.first(); auto root = states++; auto end = states++; auto start = states++; - auto startEnd = states++; auto rule = states++; auto rhs = states++; auto terminalStart = states++; @@ -75,11 +81,7 @@ void initAutomata(Automata& automata) { }; start->transitions = { - { Token::ID, startEnd}, - }; - - startEnd->transitions = { - { Token::RULE_END, + { Token::ID, root, [](CfGrammar* grammar, const String& tok) { grammar->startTerminal = tok; @@ -98,7 +100,11 @@ void initAutomata(Automata& automata) { } }, { Token::ID, rhs, [](CfGrammar* grammar, const String& tok) { - grammar->rules.last()->data.args.pushBack({ tok, false }); + grammar->rules.last()->data.args.pushBack({ tok, false, false }); + } + }, + { Token::EPSILON, rhs, [](CfGrammar* grammar, const String& tok) { + grammar->rules.last()->data.args.pushBack({ tok, false, true }); } }, { Token::TERMINAL_START, terminalStart }, @@ -106,8 +112,8 @@ void initAutomata(Automata& automata) { }; terminalStart->transitions = { - { Token::ID, rhs, [](CfGrammar* grammar, const String& tok) { - grammar->rules.last()->data.args.pushBack({ tok, true }); + { Token::ID, terminalEnd, [](CfGrammar* grammar, const String& tok) { + grammar->rules.last()->data.args.pushBack({ tok, true, false }); } }, }; @@ -121,12 +127,18 @@ void initAutomata(Automata& automata) { root->error = "Expected 'Start' statement, rule or end of text"; start->error = idError; - startEnd->error = stmEndError; rule->error = "Expected production equality ':'"; rhs->error = "Expected alternation '|', terminal id, statement end ';' or terminal start '[' "; terminalStart->error = idError; terminalEnd->error = "Expected terminal end '[' "; + root->name = "root"; + start->name = "start"; + rule->name = "production"; + rhs->name = "rhs"; + terminalStart->name = "terminalStart"; + terminalEnd->name = "terminalEnd"; + end->accept = true; } @@ -139,6 +151,10 @@ bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGr auto token = tok->readTok(); + if (token == Token::IGNORED || token == Token::COMMENT) { + continue; + } + if (token == Token::FAILED) { auto errorLocation = tok->getCursorPrev().get2DLocation(); printf("Syntax error at (%llu, %llu)\n", errorLocation.line, errorLocation.character); @@ -151,7 +167,7 @@ bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGr for (auto transition : state->transitions) { if (transition->tok == token) { - transition->action(grammar, tokVal); + if (transition->action) transition->action(grammar, tokVal); state = transition->target; isTransition = true; break; @@ -185,6 +201,6 @@ void CfGrammar::deinitializeCfGrammarParser(CfGrammarParserState* state) { delete state; } -void CfGrammar::parse(CfGrammarParserState* context, const int1* source) { - ::parse(source, &context->automata, &context->tokenizer, this); +bool CfGrammar::parse(CfGrammarParserState* context, const String& source) { + return ::parse(source.read(), &context->automata, &context->tokenizer, this); } \ No newline at end of file diff --git a/Parser/private/Parser.cpp b/Parser/private/Parser.cpp new file mode 100644 index 0000000..3f8ad9e --- /dev/null +++ b/Parser/private/Parser.cpp @@ -0,0 +1,8 @@ +#include "Parser.hpp" + +#include "Tokenizer.hpp" + +using namespace tp; + +static ModuleManifest* sModuleDependencies[] = { &gModuleTokenizer, nullptr }; +ModuleManifest tp::gModuleParser = ModuleManifest("CommandLine", nullptr, nullptr, sModuleDependencies); diff --git a/Parser/public/CfGrammar.hpp b/Parser/public/CfGrammar.hpp index 0cff23f..8e4a4f7 100644 --- a/Parser/public/CfGrammar.hpp +++ b/Parser/public/CfGrammar.hpp @@ -2,6 +2,7 @@ #include "Strings.hpp" #include "List.hpp" +#include "Map.hpp" namespace tp { @@ -9,7 +10,8 @@ namespace tp { struct Rule { struct Arg { String id; - bool isTerminal; + bool isTerminal = false; + bool isEpsilon = false; }; String id; @@ -19,11 +21,27 @@ namespace tp { List rules; String startTerminal; + private: + struct NonTerminal { + List rules; + Map references; + Map referencing; + }; + + struct Terminal {}; + + Map mTerminals; + Map mNonTerminals; + public: static struct CfGrammarParserState* initializeCfGrammarParser(); static void deinitializeCfGrammarParser(CfGrammarParserState*); public: - void parse(CfGrammarParserState* context, const int1* source); + bool parse(CfGrammarParserState* context, const String& source); + bool compile(); + + bool isAmbiguous(); + void optimize(); }; } \ No newline at end of file diff --git a/Parser/public/Parser.hpp b/Parser/public/Parser.hpp new file mode 100644 index 0000000..5f3f488 --- /dev/null +++ b/Parser/public/Parser.hpp @@ -0,0 +1,7 @@ +#pragma once + +#include "CfGrammar.hpp" + +namespace tp { + extern ModuleManifest gModuleParser; +} \ No newline at end of file diff --git a/Parser/rsc/grammar.txt b/Parser/rsc/grammar.txt new file mode 100644 index 0000000..93d36e3 --- /dev/null +++ b/Parser/rsc/grammar.txt @@ -0,0 +1,12 @@ + +/* +Sample grammar. +This grammar wil produce text in the form: TODO +*/ + +/* This is the starting production of the grammar */ +Start A + +/* Production rule */ +A : [ ID ]; + diff --git a/Tokenizer/tests/TestTokenizer.cpp b/Tokenizer/tests/TestTokenizer.cpp index 0a991e3..9ca3a38 100644 --- a/Tokenizer/tests/TestTokenizer.cpp +++ b/Tokenizer/tests/TestTokenizer.cpp @@ -193,6 +193,46 @@ TEST_DEF_STATIC(Simple) { TEST(outputHash == outputHashPassed); } +TEST_DEF(Covarage) { + enum class TokType { + START = 0, + NONE = 1, + FAILED = 2, + SPACE = 3, + ID = 4, + COMMENT_BLOCK = 5, + TOK_SOURCE_END = 6, + }; + + SimpleTokenizer lexer; + + lexer.build({ + { "([a-c]|A)+", TokType::ID }, + { "A", TokType::START }, + { " ", TokType::SPACE }, + }); + + if (!lexer.isBuild()) { + printf("Error : %s", lexer.getBuildError().description); + return; + } + + lexer.bindSource(" A bc cb cc "); + /* 3 5 3 0 3 4 3 4 3 6 */ + + TokType tok; + ualni outputHash = 0; + printf("\n\n OutputHash : "); + do { + tok = lexer.readTok(); + outputHash += ualni(tok); + printf(" %i ", int(tok)); + } while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED); + printf(" : %llu", outputHash); + TEST(outputHash == 90); +} + TEST_DEF(Tokenizer) { testSimple(); + // testCovarage(); TODO : test environment for an error } \ No newline at end of file