diff --git a/.docs/Designs/Language.drawio b/.docs/Designs/Language.drawio new file mode 100644 index 0000000..88238fd --- /dev/null +++ b/.docs/Designs/Language.drawio @@ -0,0 +1,428 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore index 8232fdf..5512493 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ lib install .vscode out -.vs \ No newline at end of file +.vs +*.bkp +*.$* \ No newline at end of file diff --git a/Automatas/CMakeLists.txt b/Automatas/CMakeLists.txt new file mode 100644 index 0000000..30269a6 --- /dev/null +++ b/Automatas/CMakeLists.txt @@ -0,0 +1,15 @@ +project(Automatas) + +### ---------------------- Static Library --------------------- ### +file(GLOB SOURCES "./private/*.cpp" "./private/*/*.cpp") +file(GLOB HEADERS "./public/*.hpp" "./public/*/*.hpp") +add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS}) +target_include_directories(${PROJECT_NAME} PUBLIC public/) +target_link_libraries(${PROJECT_NAME} PUBLIC Utils) + +### -------------------------- Tests -------------------------- ### +enable_testing() +file(GLOB TEST_SOURCES "./tests/*.cpp") +add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES}) +target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils) +add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests) diff --git a/Automatas/private/AutomatasCommon.cpp b/Automatas/private/AutomatasCommon.cpp new file mode 100644 index 0000000..d1ecaf6 --- /dev/null +++ b/Automatas/private/AutomatasCommon.cpp @@ -0,0 +1,8 @@ + +#include "AutomatasCommon.hpp" +#include "Utils.hpp" + +using namespace tp; + +static ModuleManifest* sModuleDependencies[] = { &gModuleUtils, nullptr }; +ModuleManifest tp::gModuleAutomatas = ModuleManifest("Automatas", nullptr, nullptr, sModuleDependencies); diff --git a/Automatas/public/AutomatasCommon.hpp b/Automatas/public/AutomatasCommon.hpp new file mode 100644 index 0000000..764c77d --- /dev/null +++ b/Automatas/public/AutomatasCommon.hpp @@ -0,0 +1,7 @@ +#pragma once + +#include "Module.hpp" + +namespace tp { + extern ModuleManifest gModuleAutomatas; +} diff --git a/Automatas/tests/Tests.cpp b/Automatas/tests/Tests.cpp new file mode 100644 index 0000000..c6a1e25 --- /dev/null +++ b/Automatas/tests/Tests.cpp @@ -0,0 +1,14 @@ +#include "AutomatasCommon.hpp" +#include "Utils.hpp" + +int main() { + + tp::ModuleManifest* deps[] = { &tp::gModuleAutomatas, &tp::gModuleUtils, nullptr }; + tp::ModuleManifest testModule("Test", nullptr, nullptr, deps); + + if (!testModule.initialize()) { + return 1; + } + + testModule.deinitialize(); +} diff --git a/CMakeLists.txt b/CMakeLists.txt index c41b819..2e8c967 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,9 +16,9 @@ add_subdirectory(Utils) add_subdirectory(Containers) add_subdirectory(Math) add_subdirectory(Allocators) +add_subdirectory(Automatas) add_subdirectory(Strings) -add_subdirectory(Tokenizer) -add_subdirectory(Parser) +add_subdirectory(Language) add_subdirectory(CommandLine) add_subdirectory(Externals) add_subdirectory(Connection) diff --git a/CommandLine/CMakeLists.txt b/CommandLine/CMakeLists.txt index eab6246..8627579 100644 --- a/CommandLine/CMakeLists.txt +++ b/CommandLine/CMakeLists.txt @@ -5,11 +5,11 @@ file(GLOB SOURCES "./private/*.cpp") file(GLOB HEADERS "./public/*.hpp") add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS}) target_include_directories(${PROJECT_NAME} PUBLIC ./public/) -target_link_libraries(${PROJECT_NAME} PUBLIC Tokenizer Connection) +target_link_libraries(${PROJECT_NAME} PUBLIC Language Connection) ### -------------------------- Tests -------------------------- ### enable_testing() file(GLOB TEST_SOURCES "./tests/*.cpp") add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES}) target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils) -add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests) \ No newline at end of file +add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests) diff --git a/Language/CMakeLists.txt b/Language/CMakeLists.txt new file mode 100644 index 0000000..0266561 --- /dev/null +++ b/Language/CMakeLists.txt @@ -0,0 +1,22 @@ + +cmake_minimum_required(VERSION 3.2) + +set(CMAKE_CXX_STANDARD 23) + +project(Language) + +### ---------------------- Static Library --------------------- ### +file(GLOB SOURCES "./private/*.cpp" "./private/*/*.cpp") +file(GLOB HEADERS "./public/*.hpp" "./public/*/*.hpp") +add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS}) +target_include_directories(${PROJECT_NAME} PUBLIC public/) +target_link_libraries(${PROJECT_NAME} PUBLIC Strings Automatas) + +### -------------------------- Tests -------------------------- ### +enable_testing() +file(GLOB TEST_SOURCES "./tests/*.cpp") +add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES}) +target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils) +add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests) + +install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/${PROJECT_NAME}/lib) \ No newline at end of file diff --git a/Language/old/Acceptors.hpp b/Language/old/Acceptors.hpp new file mode 100644 index 0000000..1299575 --- /dev/null +++ b/Language/old/Acceptors.hpp @@ -0,0 +1,79 @@ + +#pragma once + +/* + * This file contains acceptors for each language + * Acceptor is managing an automation machine that can recognize whether this given sentence is in the language. + * Above that acceptors are responsible for defining those automations + * Some acceptors can supply additional information such as Abstract-Syntax-Tree or Sentence id + * */ + +#include "LanguageModule.hpp" + +namespace tp { + + template + class SymbolSupply { + public: + typedef ualni Cursor; + + public: + SymbolSupply() = default; + virtual ~SymbolSupply() = 0; + + public: + virtual Cursor getCursor() = 0; + virtual void setCursor(Cursor) = 0; + + virtual void getPrevCursor() = 0; + virtual ualni getSymbolLength() = 0; + + virtual const tAlphabet& getNextSymbol() = 0; + + private: + Cursor mCursor = 0; + }; + + template + class RegularAcceptor { + typedef SymbolSupply Sentence; + + public: + RegularAcceptor() = default; + void build(const RegularGrammar& grammar); + tSentenceId recognize(Sentence& sentence); + }; + + // Subset of CFG + template + class LL { + typedef SymbolSupply Sentence; + + public: + LL() = default; + void build(const ContextFreeGrammar& grammar); + void recognize(Sentence& sentence); + }; + + // Subset of CFG much larger than LL + template + class CLR { + typedef SymbolSupply Sentence; + + public: + CLR() = default; + void build(const ContextFreeGrammar& grammar); + void recognize(Sentence& sentence); + }; + + // Subset of CFG CLR + template + class LALR { + typedef SymbolSupply Sentence; + + public: + LALR() = default; + void build(const ContextFreeGrammar& grammar); + void recognize(Sentence& sentence); + }; +} diff --git a/Parser/CMakeLists.txt b/Language/old/Parser/CMakeLists.txt similarity index 100% rename from Parser/CMakeLists.txt rename to Language/old/Parser/CMakeLists.txt diff --git a/Parser/applications/Cfg.cpp b/Language/old/Parser/applications/Cfg.cpp similarity index 100% rename from Parser/applications/Cfg.cpp rename to Language/old/Parser/applications/Cfg.cpp diff --git a/Parser/private/CLR.cpp b/Language/old/Parser/private/CLR.cpp similarity index 100% rename from Parser/private/CLR.cpp rename to Language/old/Parser/private/CLR.cpp diff --git a/Parser/private/CfGrammar.cpp b/Language/old/Parser/private/CfGrammar.cpp similarity index 100% rename from Parser/private/CfGrammar.cpp rename to Language/old/Parser/private/CfGrammar.cpp diff --git a/Parser/private/CfGrammarParser.cpp b/Language/old/Parser/private/CfGrammarParser.cpp similarity index 100% rename from Parser/private/CfGrammarParser.cpp rename to Language/old/Parser/private/CfGrammarParser.cpp diff --git a/Parser/private/Parser.cpp b/Language/old/Parser/private/Parser.cpp similarity index 100% rename from Parser/private/Parser.cpp rename to Language/old/Parser/private/Parser.cpp diff --git a/Parser/public/CLR.hpp b/Language/old/Parser/public/CLR.hpp similarity index 100% rename from Parser/public/CLR.hpp rename to Language/old/Parser/public/CLR.hpp diff --git a/Parser/public/CfGrammar.hpp b/Language/old/Parser/public/CfGrammar.hpp similarity index 100% rename from Parser/public/CfGrammar.hpp rename to Language/old/Parser/public/CfGrammar.hpp diff --git a/Parser/public/Parser.hpp b/Language/old/Parser/public/Parser.hpp similarity index 100% rename from Parser/public/Parser.hpp rename to Language/old/Parser/public/Parser.hpp diff --git a/Parser/rsc/grammar.txt b/Language/old/Parser/rsc/grammar.txt similarity index 100% rename from Parser/rsc/grammar.txt rename to Language/old/Parser/rsc/grammar.txt diff --git a/Parser/tests/CLRTests.cpp b/Language/old/Parser/tests/CLRTests.cpp similarity index 100% rename from Parser/tests/CLRTests.cpp rename to Language/old/Parser/tests/CLRTests.cpp diff --git a/Parser/tests/GrammarTests.cpp b/Language/old/Parser/tests/GrammarTests.cpp similarity index 100% rename from Parser/tests/GrammarTests.cpp rename to Language/old/Parser/tests/GrammarTests.cpp diff --git a/Parser/tests/Tests.cpp b/Language/old/Parser/tests/Tests.cpp similarity index 100% rename from Parser/tests/Tests.cpp rename to Language/old/Parser/tests/Tests.cpp diff --git a/Tokenizer/CMakeLists.txt b/Language/old/Tokenizer/CMakeLists.txt similarity index 100% rename from Tokenizer/CMakeLists.txt rename to Language/old/Tokenizer/CMakeLists.txt diff --git a/Tokenizer/private/Tokenizer.cpp b/Language/old/Tokenizer/private/Tokenizer.cpp similarity index 100% rename from Tokenizer/private/Tokenizer.cpp rename to Language/old/Tokenizer/private/Tokenizer.cpp diff --git a/Tokenizer/public/AutomataGraph.h b/Language/old/Tokenizer/public/AutomataGraph.h similarity index 100% rename from Tokenizer/public/AutomataGraph.h rename to Language/old/Tokenizer/public/AutomataGraph.h diff --git a/Tokenizer/public/RegularExpression.h b/Language/old/Tokenizer/public/RegularExpression.h similarity index 100% rename from Tokenizer/public/RegularExpression.h rename to Language/old/Tokenizer/public/RegularExpression.h diff --git a/Tokenizer/public/Tokenizer.hpp b/Language/old/Tokenizer/public/Tokenizer.hpp similarity index 100% rename from Tokenizer/public/Tokenizer.hpp rename to Language/old/Tokenizer/public/Tokenizer.hpp diff --git a/Tokenizer/tests/TestTokenizer.cpp b/Language/old/Tokenizer/tests/TestTokenizer.cpp similarity index 100% rename from Tokenizer/tests/TestTokenizer.cpp rename to Language/old/Tokenizer/tests/TestTokenizer.cpp diff --git a/Tokenizer/tests/tests.cpp b/Language/old/Tokenizer/tests/tests.cpp similarity index 100% rename from Tokenizer/tests/tests.cpp rename to Language/old/Tokenizer/tests/tests.cpp diff --git a/Language/private/AST.cpp b/Language/private/AST.cpp new file mode 100644 index 0000000..755eb75 --- /dev/null +++ b/Language/private/AST.cpp @@ -0,0 +1,4 @@ + +#include "AST.hpp" + +using namespace tp; diff --git a/Language/private/Grammar.cpp b/Language/private/Grammar.cpp new file mode 100644 index 0000000..0dac427 --- /dev/null +++ b/Language/private/Grammar.cpp @@ -0,0 +1,4 @@ + +#include "Grammar.hpp" + +using namespace tp; diff --git a/Language/private/GrammarCompiler.cpp b/Language/private/GrammarCompiler.cpp new file mode 100644 index 0000000..6501d1d --- /dev/null +++ b/Language/private/GrammarCompiler.cpp @@ -0,0 +1,4 @@ + +#include "GrammarCompiler.hpp" + +using namespace tp; \ No newline at end of file diff --git a/Language/private/LanguageCommon.cpp b/Language/private/LanguageCommon.cpp new file mode 100644 index 0000000..147536d --- /dev/null +++ b/Language/private/LanguageCommon.cpp @@ -0,0 +1,8 @@ + +#include "LanguageCommon.hpp" +#include "Strings.hpp" + +using namespace tp; + +static ModuleManifest* sModuleDependencies[] = { &gModuleStrings, nullptr }; +ModuleManifest tp::gModuleLanguage = ModuleManifest("Language", nullptr, nullptr, sModuleDependencies); diff --git a/Language/private/Parser.cpp b/Language/private/Parser.cpp new file mode 100644 index 0000000..113f363 --- /dev/null +++ b/Language/private/Parser.cpp @@ -0,0 +1,8 @@ + +#include "Parser.hpp" + +using namespace tp; + +void Parser::compileTables(const ContextFreeGrammar& cfGrammar, const RegularGrammar& reGrammar) {} + +void Parser::parse(const Sentence& sentence, AST& out) {} \ No newline at end of file diff --git a/Language/private/Sentence.cpp b/Language/private/Sentence.cpp new file mode 100644 index 0000000..92a4b7b --- /dev/null +++ b/Language/private/Sentence.cpp @@ -0,0 +1,4 @@ + +#include "Sentence.hpp" + +using namespace tp; \ No newline at end of file diff --git a/Language/private/SimpleParser.cpp b/Language/private/SimpleParser.cpp new file mode 100644 index 0000000..b4fa940 --- /dev/null +++ b/Language/private/SimpleParser.cpp @@ -0,0 +1,49 @@ + +#include "SimpleParser.hpp" + +using namespace tp; + +SimpleParser::SimpleParser() { + // Grammar for unified grammar format sentence that tables compiled from + + // Define Context-Free grammar + ContextFreeGrammar contextFreeGrammar; + { + auto term = contextFreeGrammar.createTerminal(); + + auto nonTerm = contextFreeGrammar.createNonTerminal(); + auto nonTerm2 = contextFreeGrammar.createNonTerminal(); + + contextFreeGrammar.addRule(nonTerm, { term, nonTerm2 }); + contextFreeGrammar.addRule(); + + contextFreeGrammar.setStart(nonTerm); + } + + // Define Regular grammar + RegularGrammar regularGrammar; + { + regularGrammar.addRule(regularGrammar.alternate(regularGrammar.repeat(regularGrammar.symbols("asd")), regularGrammar.symbol("a"))); + regularGrammar.addRule(regularGrammar.alternate(regularGrammar.repeat(regularGrammar.symbols("asd")), regularGrammar.symbol("a"))); + } + + mUnifiedGrammarParser.compileTables(contextFreeGrammar, regularGrammar); +} + +void SimpleParser::compileTables(const Sentence& grammar) { + AST unifiedGrammarAst; + mUnifiedGrammarParser.parse(grammar, unifiedGrammarAst); + + // compile each ast into RegularGrammar and ContextFree Grammar api instructions + ContextFreeGrammar userContextFreeGrammar; + RegularGrammar userRegularGrammar; + // ... + // split ast into RE and CF part + // generate and execute grammar api commands + // ... + + // compile tables from user grammar + mUserParser.compileTables(userContextFreeGrammar, userRegularGrammar); +} + +void SimpleParser::parse(const Sentence& sentence, AST& out) { mUserParser.parse(sentence, out); } \ No newline at end of file diff --git a/Language/public/AST.hpp b/Language/public/AST.hpp new file mode 100644 index 0000000..5faffba --- /dev/null +++ b/Language/public/AST.hpp @@ -0,0 +1,8 @@ + +#pragma once + +#include "LanguageCommon.hpp" + +namespace tp { + class AST {}; +} diff --git a/Language/public/Grammar.hpp b/Language/public/Grammar.hpp new file mode 100644 index 0000000..e8bd32f --- /dev/null +++ b/Language/public/Grammar.hpp @@ -0,0 +1,9 @@ + +#pragma once + +#include "LanguageCommon.hpp" + +namespace tp { + class ContextFreeGrammar {}; + class RegularGrammar {}; +} diff --git a/Language/public/GrammarCompiler.hpp b/Language/public/GrammarCompiler.hpp new file mode 100644 index 0000000..9fb4c03 --- /dev/null +++ b/Language/public/GrammarCompiler.hpp @@ -0,0 +1,6 @@ + +#pragma once + +#include "Grammar.hpp" + +namespace tp {} diff --git a/Language/public/LanguageCommon.hpp b/Language/public/LanguageCommon.hpp new file mode 100644 index 0000000..0429431 --- /dev/null +++ b/Language/public/LanguageCommon.hpp @@ -0,0 +1,7 @@ +#pragma once + +#include "Module.hpp" + +namespace tp { + extern ModuleManifest gModuleLanguage; +} diff --git a/Language/public/Parser.hpp b/Language/public/Parser.hpp new file mode 100644 index 0000000..17ed6df --- /dev/null +++ b/Language/public/Parser.hpp @@ -0,0 +1,20 @@ + +#pragma once + +#include "AST.hpp" +#include "GrammarCompiler.hpp" +#include "Sentence.hpp" + +namespace tp { + class Parser { + public: + Parser() = default; + + public: + void compileTables(const ContextFreeGrammar& cfGrammar, const RegularGrammar& reGrammar); + void parse(const Sentence& sentence, AST& out); + + public: + // save load compiled tables + }; +} diff --git a/Language/public/Sentence.hpp b/Language/public/Sentence.hpp new file mode 100644 index 0000000..82bdf2b --- /dev/null +++ b/Language/public/Sentence.hpp @@ -0,0 +1,13 @@ + +#pragma once + +#include "LanguageCommon.hpp" + +namespace tp { + struct Sentence { + const int1* buffer = nullptr; + alni symbolSize = 1; + + explicit Sentence(const char* buff) { buffer = buff; } + }; +} diff --git a/Language/public/SimpleParser.hpp b/Language/public/SimpleParser.hpp new file mode 100644 index 0000000..39c7b26 --- /dev/null +++ b/Language/public/SimpleParser.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include "Parser.hpp" + +namespace tp { + + // Gives ability to express grammar in the Unified Format as sentence + class SimpleParser { + public: + SimpleParser(); + + public: + void compileTables(const Sentence& grammar); + void parse(const Sentence& sentence, AST& out); + + private: + Parser mUnifiedGrammarParser; + Parser mUserParser; + }; +} diff --git a/Language/tests/Test.hpp b/Language/tests/Test.hpp new file mode 100644 index 0000000..98ece66 --- /dev/null +++ b/Language/tests/Test.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include "SimpleParser.hpp" + +const char* gGrammar = R"( +# Grammar in the CF-RE United Format (Defined by language module) + +Rules : { + ScopeList : ScopeList Scope | Scope ; + Scope : \ScopeBegin StatementList \ScopeEnd; + StatementList : StatementList Statement \StatementEnd; + StatementList : Statement \StatementEnd; + Statement : \StatementBody; +} + +Terminals : { + Space : " " | "\t" | "\n" | "\r"; + ScopeBegin : "{"; + ScopeEnd : "}"; + StatementEnd : ";"; + StatementBody : "a" | "b"; +} + +Start : Scope; +Ignore : Space; +)"; + +const char* gSentence = R"( +{} + +{ } + +{ +a; + a ; a ; + + } + + { +a; + a; +} +)"; \ No newline at end of file diff --git a/Language/tests/Tests.cpp b/Language/tests/Tests.cpp new file mode 100644 index 0000000..718f1cd --- /dev/null +++ b/Language/tests/Tests.cpp @@ -0,0 +1,29 @@ + +#include "Test.hpp" + +using namespace tp; + +void test() { + auto parser = SimpleParser(); + auto grammar = Sentence(gGrammar); + + auto sentence = Sentence(gSentence); + auto ast = AST(); + + parser.compileTables(grammar); + parser.parse(sentence, ast); +} + +int main() { + + tp::ModuleManifest* deps[] = { &tp::gModuleLanguage, nullptr }; + tp::ModuleManifest testModule("Test", nullptr, nullptr, deps); + + if (!testModule.initialize()) { + return 1; + } + + test(); + + testModule.deinitialize(); +} diff --git a/TODO b/TODO index b092d80..458a572 100644 --- a/TODO +++ b/TODO @@ -1,18 +1,24 @@ -Testing !! +All: + Testing + make more general structure for feature renders -Change clang format and apply to all files!!! -rename Language module to FormalLanguage -make more general structure for feature renders +Language: + rename Language module to FormalLanguage + Merge parser and tokenizer module into grammar module -Merge parser and tokenizer module into grammar module -Make two grammars Regular and Context-free -For each grammar make grammar rules parser, example sentence generation and sentence parsing -Make automations itself a universal tool - NFA DFA conversions + Define grammars + Implement automations + Implement LL parsing using hardcoded rules in the code + Implement save and loading of automatas + Using LL parsing define grammar for + + Make two grammars Regular and Context-free + For each grammar make grammar rules parser, example sentence generation and sentence parsing + Make automations itself a universal tool - NFA DFA conversions Math: FFT - RayTracer: Features: Normals flag