From 7112002e30b528d3dc58ed8d9b7caf0563067bca Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Sat, 3 Feb 2024 17:10:29 +0300 Subject: [PATCH] Reuse Regular Automata functionality --- Automatas/CMakeLists.txt | 15 - Automatas/private/AutomatasCommon.cpp | 8 - Automatas/public/AutomatasCommon.hpp | 7 - Automatas/tests/Tests.cpp | 14 - CMakeLists.txt | 1 - Containers/public/Buffer.hpp | 9 +- Language/CMakeLists.txt | 2 +- Language/old/Tokenizer/public/AutomataGraph.h | 506 ------------ .../old/Tokenizer/public/RegularExpression.h | 780 +++++++----------- Language/private/Grammar.cpp | 5 +- Language/public/Grammar.hpp | 20 +- Language/public/GrammarCompiler.hpp | 182 +++- Language/public/Parser.hpp | 22 +- Language/public/RegularAutomata.hpp | 501 +++++++++++ Language/public/SimpleParser.hpp | 16 +- Language/tests/Tests.cpp | 1 - Utils/public/Utils.hpp | 2 + 17 files changed, 1033 insertions(+), 1058 deletions(-) delete mode 100644 Automatas/CMakeLists.txt delete mode 100644 Automatas/private/AutomatasCommon.cpp delete mode 100644 Automatas/public/AutomatasCommon.hpp delete mode 100644 Automatas/tests/Tests.cpp create mode 100644 Language/public/RegularAutomata.hpp diff --git a/Automatas/CMakeLists.txt b/Automatas/CMakeLists.txt deleted file mode 100644 index 30269a6..0000000 --- a/Automatas/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -project(Automatas) - -### ---------------------- Static Library --------------------- ### -file(GLOB SOURCES "./private/*.cpp" "./private/*/*.cpp") -file(GLOB HEADERS "./public/*.hpp" "./public/*/*.hpp") -add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS}) -target_include_directories(${PROJECT_NAME} PUBLIC public/) -target_link_libraries(${PROJECT_NAME} PUBLIC Utils) - -### -------------------------- Tests -------------------------- ### -enable_testing() -file(GLOB TEST_SOURCES "./tests/*.cpp") -add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES}) -target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils) -add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests) diff --git a/Automatas/private/AutomatasCommon.cpp b/Automatas/private/AutomatasCommon.cpp deleted file mode 100644 index d1ecaf6..0000000 --- a/Automatas/private/AutomatasCommon.cpp +++ /dev/null @@ -1,8 +0,0 @@ - -#include "AutomatasCommon.hpp" -#include "Utils.hpp" - -using namespace tp; - -static ModuleManifest* sModuleDependencies[] = { &gModuleUtils, nullptr }; -ModuleManifest tp::gModuleAutomatas = ModuleManifest("Automatas", nullptr, nullptr, sModuleDependencies); diff --git a/Automatas/public/AutomatasCommon.hpp b/Automatas/public/AutomatasCommon.hpp deleted file mode 100644 index 764c77d..0000000 --- a/Automatas/public/AutomatasCommon.hpp +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -#include "Module.hpp" - -namespace tp { - extern ModuleManifest gModuleAutomatas; -} diff --git a/Automatas/tests/Tests.cpp b/Automatas/tests/Tests.cpp deleted file mode 100644 index c6a1e25..0000000 --- a/Automatas/tests/Tests.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "AutomatasCommon.hpp" -#include "Utils.hpp" - -int main() { - - tp::ModuleManifest* deps[] = { &tp::gModuleAutomatas, &tp::gModuleUtils, nullptr }; - tp::ModuleManifest testModule("Test", nullptr, nullptr, deps); - - if (!testModule.initialize()) { - return 1; - } - - testModule.deinitialize(); -} diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e8c967..0901985 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,6 @@ add_subdirectory(Utils) add_subdirectory(Containers) add_subdirectory(Math) add_subdirectory(Allocators) -add_subdirectory(Automatas) add_subdirectory(Strings) add_subdirectory(Language) add_subdirectory(CommandLine) diff --git a/Containers/public/Buffer.hpp b/Containers/public/Buffer.hpp index ec52868..be21103 100644 --- a/Containers/public/Buffer.hpp +++ b/Containers/public/Buffer.hpp @@ -165,7 +165,12 @@ namespace tp { } }; - template , ualni(tResizePolicyDown)(ualni) = BufferResizeScalingDown<2>, ualni tMinSize = 4> + template < + typename tType, + class tAllocator = DefaultAllocator, + ualni(tResizePolicy)(ualni) = BufferResizeScaling<2>, + ualni(tResizePolicyDown)(ualni) = BufferResizeScalingDown<2>, + ualni tMinSize = 4> class Buffer { typedef SelectValueOrReference Arg; @@ -305,7 +310,7 @@ namespace tp { mBuff = (tType*) mAllocator.allocate(sizeof(tType) * mSize); mLoad = 0; for (const auto& val : input) { - mBuff[mLoad] = val; + new (&mBuff[mLoad]) tType(val); mLoad++; } diff --git a/Language/CMakeLists.txt b/Language/CMakeLists.txt index 0266561..f6df55f 100644 --- a/Language/CMakeLists.txt +++ b/Language/CMakeLists.txt @@ -10,7 +10,7 @@ file(GLOB SOURCES "./private/*.cpp" "./private/*/*.cpp") file(GLOB HEADERS "./public/*.hpp" "./public/*/*.hpp") add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS}) target_include_directories(${PROJECT_NAME} PUBLIC public/) -target_link_libraries(${PROJECT_NAME} PUBLIC Strings Automatas) +target_link_libraries(${PROJECT_NAME} PUBLIC Strings) ### -------------------------- Tests -------------------------- ### enable_testing() diff --git a/Language/old/Tokenizer/public/AutomataGraph.h b/Language/old/Tokenizer/public/AutomataGraph.h index 5a91f3a..f185a00 100644 --- a/Language/old/Tokenizer/public/AutomataGraph.h +++ b/Language/old/Tokenizer/public/AutomataGraph.h @@ -16,510 +16,4 @@ namespace tp { extern ModuleManifest gModuleTokenizer; - template - class TransitionMatrix; - - template - class DFA; - - // Non-Deterministic Finite-State Automata - template - class NFA { - static_assert(TypeTraits::isIntegral, "tAlphabetType must be enumerable."); - - public: - struct Vertex; - - private: - struct Edge { - Vertex* mVertex = nullptr; - bool mConsumesSymbol = false; - Range mAcceptingRange; - bool mAcceptsAll = false; - bool mExclude = false; - - bool isTransition(const tAlphabetType& symbol) { - if (symbol == 0) return false; - if (!mConsumesSymbol || mAcceptsAll) return true; - bool const in_range = (symbol >= mAcceptingRange.mBegin && symbol <= mAcceptingRange.mEnd); - return in_range != mExclude; - } - }; - - public: - struct Vertex { - List edges; - tStateType termination_state = tNoStateVal; - #ifdef ENV_BUILD_DEBUG - ualni debug_idx = 0; - #endif - ualni flag = 0; - }; - - private: - friend DFA; - - List mVertices; - Vertex* mStart = nullptr; - - public: - NFA() = default; - - Vertex* addVertex() { - auto node = mVertices.newNode(); - #ifdef ENV_BUILD_DEBUG - node->data.debug_idx = mVertices.length() + 1; - #endif - mVertices.pushBack(node); - return &node->data; - } - - void addTransition(Vertex* from, Vertex* to, Range range, bool consumes, bool accepts_all, bool exclude) { - Edge edge; - edge.mVertex = to; - edge.mConsumesSymbol = consumes; - edge.mAcceptingRange = range; - edge.mExclude = exclude; - edge.mAcceptsAll = accepts_all; - from->edges.pushBack(edge); - } - - void setStartVertex(Vertex* start) { - mStart = start; - } - - [[nodiscard]] Vertex* getStartVertex() const { - return mStart; - } - - void setVertexState(Vertex* vertex, tStateType state) { - vertex->termination_state = state; - } - - [[nodiscard]] bool isValid() const { - if (!mStart) { - return false; - } - return true; - } - - Range getAlphabetRange() const { - tAlphabetType start = 0, end = 0; - Range all_range(std::numeric_limits::min(), std::numeric_limits::max()); - bool first = true; - - for (auto vertex : mVertices) { - for (auto edge : vertex.data().edges) { - if (!edge.data().mConsumesSymbol) continue; - auto const& tran_range = edge.data().mAcceptingRange; - if (edge.data().mAcceptsAll || edge.data().mExclude) return all_range; - if (tran_range.mBegin < start || first) start = tran_range.mBegin; - if (tran_range.mEnd > end || first) end = tran_range.mEnd; - first = false; - } - } - - return Range( start, end + 1 ); - } - - // vertices that are reachable from initial set with no input consumption (E-transitions) - // does not include initial set - void closure(const List& set, List& closure, ualni unique_call_id) { - List marked; - marked = set; - - while (marked.length()) { - auto first = marked.first()->data; - if (first->flag != unique_call_id) { - first->flag = unique_call_id; - closure.pushBack(first); - } - for (auto edge : first->edges) { - if (!edge.data().mConsumesSymbol && edge.data().mVertex->flag != unique_call_id) { - marked.pushBack(edge.data().mVertex); - } - } - marked.popFront(); - } - } - - // vertices that are reachable from initial set with symbol transition - void move(const List& set, List& reachable, tAlphabetType symbol, ualni unique_call_id) { - for (auto vertex : set) { - for (auto edge : vertex->edges) { - if (!edge.data().mConsumesSymbol) continue; - bool transition = edge.data().isTransition(symbol); - if (transition && edge.data().mVertex->flag != unique_call_id) { - edge.data().mVertex->flag = unique_call_id; - reachable.pushBack(edge.data().mVertex); - } - } - } - } - }; - - // Deterministic Finite-State Automata - template - class DFA { - static_assert(TypeTraits::isIntegral, "tAlphabetType must be enumerable."); - friend TransitionMatrix; - - struct Vertex { - struct Edge { - Vertex* vertex = nullptr; - tAlphabetType transition_code = nullptr; - }; - - List edges; - tStateType termination_state = tNoStateVal; - bool marked = false; - }; - - List mVertices; - Vertex* mStart = nullptr; - const Vertex* mIter = nullptr; - Range mAlphabetRange; - bool mTrapState = false; - - typedef typename NFA::Vertex NState; - - public: - - explicit DFA(NFA& nfa) { - if (!nfa.isValid()) { - return; - } - - mAlphabetRange = nfa.getAlphabetRange(); - - struct DStateKey { - const List* nStates; - bool operator==(const DStateKey& in) const { - if (nStates->length() != in.nStates->length()) { - return false; - } - // FIXME : make linear time - for (auto state : *nStates) { - bool found = false; - for (auto in_state : *in.nStates) { - if (state.data() == in_state.data()) { - found = true; - break; - } - } - - if (!found) { - return false; - } - } - - return true; - } - - static ualni dStateHashFunc(DStateKey key) { - alni out = 0; - for (auto state : *key.nStates) { - out += alni(state.data()); - } - return out; - }; - }; - - // all NFA states that are reachable from initial DFA State for specific symbol in alphabet - struct DState { - struct DTransition { - DState* state = nullptr; - tAlphabetType accepting_code; - }; - - List nStates; - List transitions; - - Vertex* dVertex= nullptr; // relevant DFA vertex - - #ifdef ENV_BUILD_DEBUG - ualni debug_idx = 0; - #endif - }; - - // includes closure of NFA start state by definition - auto start_state = new DState(); - nfa.closure({ nfa.getStartVertex() }, start_state->nStates, ualni(start_state)); - - Map dStates; - - dStates.put({ &start_state->nStates }, start_state); - - List working_set = { start_state }; - - // while there is items to work with - auto currentDState = working_set.first(); - while (currentDState) { - - // check all possible transitions for any symbol - for (auto symbol : mAlphabetRange) { - - List reachableNStates; - - nfa.move(currentDState->data->nStates, reachableNStates, symbol, ualni(currentDState + symbol)); - nfa.closure(reachableNStates, reachableNStates, ualni(currentDState + symbol)); - - if (!reachableNStates.length()) { - continue; - } - - DState* targetDState = nullptr; - - // check if set of all reachable NFA states already forms existing DFA state - auto idx = dStates.presents({ &reachableNStates}); - if (idx) { - targetDState = dStates.getSlotVal(idx); - } - - if (!targetDState) { - // register new DFA state - targetDState = new DState(); - - #ifdef ENV_BUILD_DEBUG - targetDState->debug_idx = dStates.size(); - #endif - - targetDState->nStates = reachableNStates; - - // append to working stack - working_set.pushBack(targetDState); - dStates.put({ &targetDState->nStates }, targetDState); - } - - // add transition to DFA state - currentDState->data->transitions.pushBack({targetDState, symbol }); - } - - working_set.popFront(); - currentDState = working_set.first(); - } - - // create own vertices - for (auto node : dStates) { - tStateType state = tNoStateVal; - for (auto iter : node->val->nStates) { - if (iter->termination_state != tNoStateVal) { - state = iter->termination_state; - break; - } - } - node->val->dVertex = addVertex(state); - } - - // connect all vertices - for (auto node : dStates) { - for (auto edge : node->val->transitions) { - addTransition(node->val->dVertex, edge.data().state->dVertex, edge.data().accepting_code); - } - } - - // set the starting vertex - mStart = start_state->dVertex; - - // cleanup - for (auto node : dStates) { - delete node->val; - } - - collapseEquivalentVertices(); - mAlphabetRange = getAlphabetRange(); - } - - tStateType move(tAlphabetType symbol) { - if (mTrapState || !mIter) { - return tNoStateVal; - } - - for (auto edge : mIter->edges) { - if (edge.data().transition_code == symbol) { - mIter = edge.data().vertex; - return mIter->termination_state; - } - } - - mTrapState = true; - return tNoStateVal; - } - - void start() { - mIter = mStart; - mTrapState = false; - } - - [[nodiscard]] uhalni nVertices() const { - return (uhalni) mVertices.length(); - } - - [[nodiscard]] Range getRange() const { - return mAlphabetRange; - } - - private: - [[nodiscard]] Range getAlphabetRange() const { - Range out; - for (auto vertex : mVertices) { - vertex.data().marked = false; - } - - bool first = true; - getAlphabetRangeUtil(mStart, &out, first); - out.mEnd++; - return out; - } - - void getAlphabetRangeUtil(Vertex* vert, Range* out, bool& first) const { - vert->marked = true; - for (auto edge : vert->edges) { - auto const code = edge.data().transition_code; - - if (first) { - *out = { code, code }; - first = false; - } - - if (code < out->mBegin) { - out->mBegin = code; - } - if (code > out->mEnd) { - out->mEnd = code; - } - - if (!edge.data().vertex->marked) { - getAlphabetRangeUtil(edge.data().vertex, out, first); - } - } - } - - void collapseEquivalentVertices() { - // TODO - } - - Vertex* addVertex(tStateType state) { - auto node = mVertices.addNodeBack(); - node->data.termination_state = state; - return &node->data; - } - - void addTransition(Vertex* from, Vertex* to, tAlphabetType transition_symbol) { - from->edges.pushBack({ to, transition_symbol }); - } - }; - - template - class TransitionMatrix { - - static_assert(TypeTraits::isIntegral, "tAlphabetType must be enumerable."); - - Buffer2D mTransitions; - Buffer mStates; - Range mSymbolRange = { 0, 0 }; - - ualni mIter = 0; - ualni mIterPrev = 0; - ualni mStart = 0; - - public: - - TransitionMatrix() = default; - - auto getStates() const { return &mStates; } - auto getTransitions() const { return &mTransitions; } - auto getStart() const { return mStart; } - - void construct(const DFA& dfa) { - mSymbolRange = dfa.getRange(); - auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin); - auto sizeX = range_len ? range_len : 1; - auto sizeY = (ualni) (dfa.nVertices() + 1); - - mTransitions.reserve({ sizeX, sizeY }); - mTransitions.assign(dfa.nVertices()); - mStates.reserve(sizeY); - - ualni idx = 0; - for (auto vertex : dfa.mVertices) { - auto state = vertex.data().termination_state; - mStates[idx] = state; - idx++; - } - - mStates[dfa.nVertices()] = tFailedStateVal; - - idx = 0; - for (auto vertex : dfa.mVertices) { - if (&vertex.data() == dfa.mStart) { - mStart = mIter = mIterPrev = idx; - } - idx++; - } - - ualni vertexIdx = 0; - for (auto vertex : dfa.mVertices) { - for (auto edge : vertex.data().edges) { - ualni vertex2Idx = 0; - for (auto vertex2 : dfa.mVertices) { - if (edge.data().vertex == &vertex2.data()) break; - vertex2Idx++; - } - auto const code = edge.data().transition_code; - mTransitions.set( { (ualni) (code - mSymbolRange.mBegin), (ualni) vertexIdx }, vertex2Idx); - } - vertexIdx++; - } - } - - bool isTrapped() { - return mStates[mIter] == tFailedStateVal; - } - - tStateType move(tAlphabetType symbol) { - if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) { - mIter = mTransitions.get({ (ualni) (symbol - mSymbolRange.mBegin), (ualni) mIter }); - } - else { - mIter = mStates.size() - 1; - } - - if (mIterPrev == mStart) { - if (mStates[mIter] == tFailedStateVal) { - reset(); - return tFailedStateVal; - } - else { - mIterPrev = mIter; - return tNoStateVal; - } - } - else { - if (mStates[mIter] == tFailedStateVal) { - if (mStates[mIterPrev] != tNoStateVal) { - auto out = mStates[mIterPrev]; - reset(); - return out; - } - else { - reset(); - return tFailedStateVal; - } - } - else { - mIterPrev = mIter; - return tNoStateVal; - } - } - - mIterPrev = mIter; - return mStates[mIter]; - } - - void reset() { - mIter = mStart; - mIterPrev = mStart; - } - }; } \ No newline at end of file diff --git a/Language/old/Tokenizer/public/RegularExpression.h b/Language/old/Tokenizer/public/RegularExpression.h index 7fa668f..f267be2 100644 --- a/Language/old/Tokenizer/public/RegularExpression.h +++ b/Language/old/Tokenizer/public/RegularExpression.h @@ -8,512 +8,326 @@ namespace tp { } namespace tp::RegEx { - - struct AstNode { - enum Type { - NONE, - ANY, - OR, - IF, - CLASS, - COMPOUND, - REPEAT, - VAL, - } mType = NONE; - AstNode() = default; - virtual ~AstNode() = default; + struct AstNode { + enum Type { + NONE, + ANY, + OR, + IF, + CLASS, + COMPOUND, + REPEAT, + VAL, + } mType = NONE; + + AstNode() = default; + virtual ~AstNode() = default; + }; + + template + struct AstVal : public AstNode { + explicit AstVal(tAlphabetType val) : + mVal(val) { + mType = VAL; + } + ~AstVal() override = default; + tAlphabetType mVal; + }; + + struct AstCompound : public AstNode { + AstCompound() { mType = COMPOUND; } + ~AstCompound() override { + for (auto iter : mChilds) { + delete iter.data(); + } + mChilds.removeAll(); + } + List mChilds; + }; + + struct AstAlternation : public AstNode { + AstAlternation() { mType = OR; } + ~AstAlternation() override { + delete mFirst; + delete mSecond; + } + AstNode* mFirst = nullptr; + AstNode* mSecond = nullptr; + }; + + struct AstIf : public AstNode { + AstIf() { mType = IF; } + ~AstIf() override { delete mNode; } + AstNode* mNode = nullptr; + }; + + struct AstAny : public AstNode { + AstAny() { mType = ANY; } + ~AstAny() override = default; + }; + + struct AstRepetition : public AstNode { + AstRepetition() { mType = REPEAT; } + ~AstRepetition() override { delete mNode; } + AstNode* mNode = nullptr; + bool mPlus = false; + }; + + template + struct AstClass : public AstNode { + AstClass() { mType = CLASS; } + ~AstClass() override { mRanges.removeAll(); } + List> mRanges; + bool mExclude = false; + }; + + struct ParseError { + const char* description = nullptr; + uhalni offset = 0; + [[nodiscard]] bool isError() const { return description != nullptr; } + }; + + template + class Parser { + + enum TokType : uint1 { + TOK_COMPOUND_START = 0, + TOK_COMPOUND_END, + TOK_CLASS_START, + TOK_CLASS_END, + TOK_CLASS_START_EXCLUDE, + TOK_CLASS_END_EXCLUDE, + TOK_OR, + TOK_IF, + TOK_ANY, + TOK_REPEAT, + TOK_REPEAT_PLUS, + TOK_HYPHEN, + TOK_SPECIALS_END_, + TOK_VAL, + TOK_NONE, }; - template - struct AstVal : public AstNode { - explicit AstVal(tAlphabetType val) : mVal(val) { mType = VAL; } - ~AstVal() override = default; - tAlphabetType mVal; + tAlphabetType SpecialSymbols[TOK_SPECIALS_END_] = { + '(', ')', '[', ']', '{', '}', '|', '?', '.', '*', '+', '-', }; - struct AstCompound : public AstNode { - AstCompound() { mType = COMPOUND; } - ~AstCompound() override { - for (auto iter : mChilds) { - delete iter.data(); - } - mChilds.removeAll(); - } - List mChilds; + tAlphabetType mEscapeSymbol = '\\'; + + struct Token { + TokType type; + tAlphabetType val; }; - struct AstAlternation : public AstNode { - AstAlternation() { mType = OR; } - ~AstAlternation() override { - delete mFirst; - delete mSecond; + const tAlphabetType* mSource = nullptr; + uhalni mOffset = 0; + Token mCurToken; + uhalni mTokLength = 0; + + public: + ParseError mError; + + // regular expression must be a zero termination string + AstCompound* parse(const tAlphabetType* regex) { + mSource = regex; + return parseRegEx(); + } + + private: + AstCompound* parseRegEx() { + auto out = new AstCompound(); + for (AstNode* node = parseElement(); node; node = parseElement()) { + out->mChilds.pushBack(node); } - AstNode* mFirst = nullptr; - AstNode* mSecond = nullptr; - }; - - struct AstIf : public AstNode { - AstIf() { mType = IF; } - ~AstIf() override { delete mNode; } - AstNode* mNode = nullptr; - }; - - struct AstAny : public AstNode { - AstAny() { mType = ANY; } - ~AstAny() override = default; - }; - - struct AstRepetition : public AstNode { - AstRepetition() { mType = REPEAT; } - ~AstRepetition() override { delete mNode; } - AstNode* mNode = nullptr; - bool mPlus = false; - }; - - template - struct AstClass : public AstNode { - AstClass() { mType = CLASS; } - ~AstClass() override { mRanges.removeAll(); } - List> mRanges; - bool mExclude = false; - }; - - struct ParseError { - const char* description = nullptr; - uhalni offset = 0; - [[nodiscard]] bool isError() const { return description != nullptr; } - }; - - template - class Parser { - - enum TokType : uint1 { - TOK_COMPOUND_START = 0, - TOK_COMPOUND_END, - TOK_CLASS_START, - TOK_CLASS_END, - TOK_CLASS_START_EXCLUDE, - TOK_CLASS_END_EXCLUDE, - TOK_OR, - TOK_IF, - TOK_ANY, - TOK_REPEAT, - TOK_REPEAT_PLUS, - TOK_HYPHEN, - TOK_SPECIALS_END_, - TOK_VAL, - TOK_NONE, - }; - - tAlphabetType SpecialSymbols[TOK_SPECIALS_END_] = { - '(', ')', '[', ']', '{', '}', '|', '?', '.', '*', '+', '-', - }; - - tAlphabetType mEscapeSymbol = '\\'; - - struct Token { - TokType type; - tAlphabetType val; - }; - - const tAlphabetType* mSource = nullptr; - uhalni mOffset = 0; - Token mCurToken; - uhalni mTokLength = 0; - - public: - - ParseError mError; - - // regular expression must be a zero termination string - AstCompound* parse(const tAlphabetType* regex) { - mSource = regex; - return parseRegEx(); + if (!out->mChilds.length()) { + genError("Expected A Expression"); } - - private: - - AstCompound* parseRegEx() { - auto out = new AstCompound(); - for (AstNode* node = parseElement(); node; node = parseElement()) { - out->mChilds.pushBack(node); - } - if (!out->mChilds.length()) { - genError("Expected A Expression"); - } - if (mError.description) { - delete out; - return nullptr; - } - return out; + if (mError.description) { + delete out; + return nullptr; } + return out; + } - AstNode* parseElement() { - AstNode* out = nullptr; - switch (readTok().type) { - case TOK_COMPOUND_START: out = parseCompound(); break; - case TOK_CLASS_START: out = parseClass(); break; - case TOK_CLASS_START_EXCLUDE: out = parseClass(true); break; - case TOK_ANY: out = parseAny(); break; - case TOK_VAL: out = parseVal(); break; - case TOK_NONE: { discardTok(); return nullptr; }; - default: break; - } - if (!out) { - discardTok(); - return nullptr; - } - switch (readTok().type) { - case TOK_OR: out = parseAlternation(out); break; - case TOK_REPEAT: out = parseRepetition(out); break; - case TOK_REPEAT_PLUS: out = parseRepetition(out, true); break; - case TOK_IF: out = parseIf(out); break; - case TOK_NONE: break; - default: { discardTok(); } - } - return out; + AstNode* parseElement() { + AstNode* out = nullptr; + switch (readTok().type) { + case TOK_COMPOUND_START: out = parseCompound(); break; + case TOK_CLASS_START: out = parseClass(); break; + case TOK_CLASS_START_EXCLUDE: out = parseClass(true); break; + case TOK_ANY: out = parseAny(); break; + case TOK_VAL: out = parseVal(); break; + case TOK_NONE: + { + discardTok(); + return nullptr; + }; + default: break; } - - AstCompound* parseCompound() { - auto out = new AstCompound(); - for (AstNode* node = parseElement(); node; node = parseElement()) { - out->mChilds.pushBack(node); - } - if (readTok().type != TOK_COMPOUND_END) { - genError("Expected Compound End"); - } - if (mError.description) { - delete out; - return nullptr; - } - return out; + if (!out) { + discardTok(); + return nullptr; } - - AstClass* parseClass(bool exclude = false) { - auto out = new AstClass(); - out->mExclude = exclude; - auto& ranges = out->mRanges; - - readTok(); - - READ_VAL: - if (mCurToken.type != TOK_VAL) { - delete out; - genError("Expected A Value"); - return nullptr; - } - char range_start = mCurToken.val; - - readTok(); - if (mCurToken.type != TOK_HYPHEN) { - delete out; - genError("Expected A Range"); - return nullptr; - } - - readTok(); - if (mCurToken.type != TOK_VAL) { - delete out; - genError("Expected A Value"); - return nullptr; - } - char range_end = mCurToken.val; - - ranges.pushBack({ range_start, range_end }); - - readTok(); - if ((mCurToken.type == TOK_CLASS_END && !exclude) || (mCurToken.type == TOK_CLASS_END_EXCLUDE && exclude)) { - return out; - } - else { - goto READ_VAL; - } - } - - AstAny* parseAny() { - return new AstAny(); - } - - AstVal* parseVal() { - auto out = new AstVal(mCurToken.val); - return out; - } - - AstAlternation* parseAlternation(AstNode* left) { - auto right = parseElement(); - if (!right) { - genError("Expected Alternation right Side"); - delete left; - return nullptr; - } - - auto out = new AstAlternation(); - - out->mFirst = left; - out->mSecond = right; - - return out; - } - - AstRepetition* parseRepetition(AstNode* left, bool plus = false) { - auto out = new AstRepetition(); - out->mNode = left; - out->mPlus = plus; - return out; - } - - AstIf* parseIf(AstNode* left) { - auto out = new AstIf(); - out->mNode = left; - return out; - } - - void genError(const char* desc) { - mError = { desc, mOffset }; - } - - Token& readTok() { - - const tAlphabetType* crs = mSource + mOffset; - - // zero termination string - if (*crs == 0) { - mCurToken.type = TOK_NONE; - return mCurToken; - } - - mTokLength = 1; - mCurToken.type = TOK_VAL; - mCurToken.val = crs[0]; - - if (crs[0] == mEscapeSymbol) { - mCurToken.val = crs[1]; - mTokLength = 2; - } - else { - for (uhalni tok = 0; tok < TOK_SPECIALS_END_; tok++) { - if (SpecialSymbols[tok] == mCurToken.val) { - mCurToken.type = TokType(tok); - break; - } + switch (readTok().type) { + case TOK_OR: out = parseAlternation(out); break; + case TOK_REPEAT: out = parseRepetition(out); break; + case TOK_REPEAT_PLUS: out = parseRepetition(out, true); break; + case TOK_IF: out = parseIf(out); break; + case TOK_NONE: break; + default: + { + discardTok(); } - } + } + return out; + } - mOffset += mTokLength; + AstCompound* parseCompound() { + auto out = new AstCompound(); + for (AstNode* node = parseElement(); node; node = parseElement()) { + out->mChilds.pushBack(node); + } + if (readTok().type != TOK_COMPOUND_END) { + genError("Expected Compound End"); + } + if (mError.description) { + delete out; + return nullptr; + } + return out; + } + + AstClass* parseClass(bool exclude = false) { + auto out = new AstClass(); + out->mExclude = exclude; + auto& ranges = out->mRanges; + + readTok(); + + READ_VAL: + if (mCurToken.type != TOK_VAL) { + delete out; + genError("Expected A Value"); + return nullptr; + } + char range_start = mCurToken.val; + + readTok(); + if (mCurToken.type != TOK_HYPHEN) { + delete out; + genError("Expected A Range"); + return nullptr; + } + + readTok(); + if (mCurToken.type != TOK_VAL) { + delete out; + genError("Expected A Value"); + return nullptr; + } + char range_end = mCurToken.val; + + ranges.pushBack({ range_start, range_end }); + + readTok(); + if ((mCurToken.type == TOK_CLASS_END && !exclude) || (mCurToken.type == TOK_CLASS_END_EXCLUDE && exclude)) { + return out; + } else { + goto READ_VAL; + } + } + + AstAny* parseAny() { return new AstAny(); } + + AstVal* parseVal() { + auto out = new AstVal(mCurToken.val); + return out; + } + + AstAlternation* parseAlternation(AstNode* left) { + auto right = parseElement(); + if (!right) { + genError("Expected Alternation right Side"); + delete left; + return nullptr; + } + + auto out = new AstAlternation(); + + out->mFirst = left; + out->mSecond = right; + + return out; + } + + AstRepetition* parseRepetition(AstNode* left, bool plus = false) { + auto out = new AstRepetition(); + out->mNode = left; + out->mPlus = plus; + return out; + } + + AstIf* parseIf(AstNode* left) { + auto out = new AstIf(); + out->mNode = left; + return out; + } + + void genError(const char* desc) { mError = { desc, mOffset }; } + + Token& readTok() { + + const tAlphabetType* crs = mSource + mOffset; + + // zero termination string + if (*crs == 0) { + mCurToken.type = TOK_NONE; return mCurToken; } - void discardTok() { - mOffset -= mTokLength; - mTokLength = 0; - } - }; - - template - struct CompileError { - ParseError mParseError; - uhalni mRuleIndex = 0; - tStateType mRuleState; - const char* description = nullptr; - [[nodiscard]] bool isError() const { return description; } - }; + mTokLength = 1; + mCurToken.type = TOK_VAL; + mCurToken.val = crs[0]; - template - class Compiler { - - typedef NFA Graph; - typedef typename Graph::Vertex Vertex; - typedef Parser Parser; - - struct Node { - Vertex* left = nullptr; - Vertex* right = nullptr; - }; - - private: - Graph* mGraph = nullptr; - - public: - CompileError mError; - - Node compile(Graph& graph, const tAlphabetType* regex, tStateType state) { - mGraph = &graph; - return compileUtil(regex, state); - } - - Node compile(Graph& aGraph, InitialierList> aRules) { - mGraph = &aGraph; - - auto left = mGraph->addVertex(); - auto right = mGraph->addVertex(); - - halni idx = 0; - for (auto rule: aRules) { - - auto node = compileUtil(rule.head, rule.tail); - - if (!(node.left && node.right)) { - mError.mRuleIndex = idx; - return {}; - } - - transitionAny(left, node.left); - transitionAny(node.right, right); - - idx++; - } - - mGraph->setStartVertex(left); - - return { left, right }; - } - - private: - - Node compileUtil(const tAlphabetType* regex, tStateType state) { - Parser parser; - auto astNode = parser.parse(regex); - if (parser.mError.isError()) { - mGraph->setStartVertex(nullptr); - mError.description = "Parsing Of Regular Expression Failed"; - mError.mRuleState = state; - mError.mParseError = parser.mError; - return {}; - } - - auto node = compileNode(astNode, nullptr, nullptr); - delete astNode; - - mGraph->setVertexState(node.right, state); - mGraph->setStartVertex(node.left); - - return node; - } - - Node compileVal(AstVal* val, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { - auto left = aLeft ? aLeft : mGraph->addVertex(); - auto right = aRight ? aRight : mGraph->addVertex(); - transitionVal(left, right, val->mVal); - return { left, right }; - } - - Node compileAlternation(AstAlternation* alt, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { - auto first_node = compileNode(alt->mFirst, aLeft, aRight); - auto second_node = compileNode(alt->mSecond); - transitionAny(first_node.left, second_node.left); - transitionAny(second_node.right, first_node.right); - return first_node; - } - - Node compileAny(AstAny*, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { - auto left = aLeft ? aLeft : mGraph->addVertex(); - auto right = aRight ? aRight : mGraph->addVertex(); - transitionAny(left, right, true); - return { left, right }; - } - - Node compileRepeat(AstRepetition* repeat, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { - if (repeat->mPlus) { - auto middle = mGraph->addVertex(); - - auto left_node = compileNode(repeat->mNode, aLeft, middle); - - auto right_node = compileNode(repeat->mNode, middle, aRight); - transitionAny(right_node.right, right_node.left); - transitionAny(right_node.left, right_node.right); - - return { left_node.left, right_node.right }; - } - else { - auto node = compileNode(repeat->mNode, aLeft, aRight); - transitionAny(node.right, node.left); - transitionAny(node.left, node.right); - return node; - } - } - - Node compileIf(AstIf* ifNode, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { - auto node = compileNode(ifNode->mNode, aLeft, aRight); - transitionAny(node.left, node.right); - return node; - } - - Node compileClass(AstClass* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { - auto left = aLeft ? aLeft : mGraph->addVertex(); - auto right = aRight ? aRight : mGraph->addVertex(); - - if (node->mRanges.length() == 1) { - auto const& range = node->mRanges.first()->data; - transitionRange(left, right, { range.mBegin, range.mEnd }, node->mExclude); - return { left, right }; - } - - for (auto range : node->mRanges) { - auto middle = mGraph->addVertex(); - transitionRange(left, middle, { range.data().mBegin, range.data().mEnd }, node->mExclude); - transitionAny(middle, right); - } - return { left, right }; - } - - Node compileCompound(AstCompound* compound, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { - Vertex* left = nullptr; - Vertex* rigth = nullptr; - - ualni idx = 0; - for (auto child : compound->mChilds) { - auto pass_left = idx == 0 ? aLeft : rigth; - auto pass_right = idx == compound->mChilds.length() - 1 ? aRight : nullptr; - auto node = compileNode(child.data(), pass_left, pass_right); - if (!left) left = node.left; - rigth = node.right; - idx++; - } - - return { left, rigth }; - } - - Node compileNode(AstNode* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { - switch (node->mType) { - case AstNode::CLASS: return compileClass((AstClass*)node, aLeft, aRight); - case AstNode::COMPOUND: return compileCompound((AstCompound*)node, aLeft, aRight); - case AstNode::IF: return compileIf((AstIf*)node, aLeft, aRight); - case AstNode::REPEAT: return compileRepeat((AstRepetition*)node, aLeft, aRight); - case AstNode::ANY: return compileAny((AstAny*)node, aLeft, aRight); - case AstNode::OR: return compileAlternation((AstAlternation*)node, aLeft, aRight); - case AstNode::VAL: return compileVal((AstVal*)node, aLeft, aRight); - case AstNode::NONE: + if (crs[0] == mEscapeSymbol) { + mCurToken.val = crs[1]; + mTokLength = 2; + } else { + for (uhalni tok = 0; tok < TOK_SPECIALS_END_; tok++) { + if (SpecialSymbols[tok] == mCurToken.val) { + mCurToken.type = TokType(tok); break; + } } - ASSERT(0) - return {}; } - void transitionAny(Vertex* from, Vertex* to, bool consumes = false) { - mGraph->addTransition(from, to, {}, consumes, true, false); - } - - void transitionVal(Vertex* from, Vertex* to, tAlphabetType val) { - mGraph->addTransition(from, to, { val, val }, true, false, false); - } - - void transitionRange(Vertex* from, Vertex* to, Range range, bool exclude) { - mGraph->addTransition(from, to, range, true, false, exclude); - } - }; - - template - CompileError compile(NFA& out, const tAlphabetType* regex, tStateType state) { - Compiler compiler; - compiler.compile(out, regex, state); - return compiler.mError; + mOffset += mTokLength; + return mCurToken; } - template - CompileError compile(NFA& out, const InitialierList>& rules) { - Compiler compiler; - compiler.compile(out, rules); - return compiler.mError; + void discardTok() { + mOffset -= mTokLength; + mTokLength = 0; } - } \ No newline at end of file + }; + + template + CompileError compile( + NFA& out, const tAlphabetType* regex, tStateType state + ) { + Compiler compiler; + compiler.compile(out, regex, state); + return compiler.mError; + } + + template + CompileError compile( + NFA& out, + const InitialierList>& rules + ) { + Compiler compiler; + compiler.compile(out, rules); + return compiler.mError; + } +} \ No newline at end of file diff --git a/Language/private/Grammar.cpp b/Language/private/Grammar.cpp index 799d917..c89f60c 100644 --- a/Language/private/Grammar.cpp +++ b/Language/private/Grammar.cpp @@ -1,6 +1,5 @@ #include "Grammar.hpp" -#include "NewPlacement.hpp" using namespace tp; @@ -12,7 +11,9 @@ ContextFreeGrammar::Arg::Arg(const String& id, bool terminal, bool epsilon) { const String& ContextFreeGrammar::Arg::getId() const { return mId; } -bool ContextFreeGrammar::Arg::operator==(const Arg& in) const { return (mId == in.mId) && (mIsEpsilon == in.mIsEpsilon) && (mIsTerminal == in.mIsTerminal); } +bool ContextFreeGrammar::Arg::operator==(const Arg& in) const { + return (mId == in.mId) && (mIsEpsilon == in.mIsEpsilon) && (mIsTerminal == in.mIsTerminal); +} ContextFreeGrammar::Rule::Rule(const String& id, const InitialierList& args) { mId = id; diff --git a/Language/public/Grammar.hpp b/Language/public/Grammar.hpp index c1f5fa1..a19fdf6 100644 --- a/Language/public/Grammar.hpp +++ b/Language/public/Grammar.hpp @@ -70,7 +70,7 @@ namespace tp { template class RegularGrammar { - + public: struct Node { enum Type { NONE, @@ -97,7 +97,7 @@ namespace tp { ~ValueNode() override = default; - private: + public: tAlphabetType mVal; }; @@ -118,7 +118,7 @@ namespace tp { mSequence.clear(); } - private: + public: Buffer mSequence; }; @@ -138,7 +138,7 @@ namespace tp { delete mSecond; } - private: + public: const Node* mFirst = nullptr; const Node* mSecond = nullptr; }; @@ -155,7 +155,7 @@ namespace tp { ~IfNode() override { delete mNode; } - private: + public: const Node* mNode = nullptr; }; @@ -180,7 +180,7 @@ namespace tp { ~RepetitionNode() override { delete mNode; } - private: + public: Node* mNode = nullptr; bool mPlus = false; }; @@ -197,7 +197,7 @@ namespace tp { ~ClassNode() override { mRanges.removeAll(); } - private: + public: Buffer> mRanges; bool mExclude = false; }; @@ -220,9 +220,11 @@ namespace tp { const Node* may(const Node* a) { return new IfNode(a); } const Node* any() { return new AnyNode(); } const Node* rep(const Node* rep, bool plus = false) { return new RepetitionNode(rep, plus); } - const Node* ranges(const Buffer>& ranges, bool exclude = false) { return new ClassNode(ranges, exclude); } + const Node* ranges(const Buffer>& ranges, bool exclude = false) { + return new ClassNode(ranges, exclude); + } - private: + public: Buffer> mRules; }; } diff --git a/Language/public/GrammarCompiler.hpp b/Language/public/GrammarCompiler.hpp index 9fb4c03..e99625f 100644 --- a/Language/public/GrammarCompiler.hpp +++ b/Language/public/GrammarCompiler.hpp @@ -2,5 +2,185 @@ #pragma once #include "Grammar.hpp" +#include "RegularAutomata.hpp" -namespace tp {} +namespace tp { + + template + class RegularCompiler { + + typedef NFA Graph; + typedef typename Graph::Vertex Vertex; + typedef RegularGrammar Grammar; + + struct Node { + Vertex* left = nullptr; + Vertex* right = nullptr; + }; + + private: + Graph* mGraph = nullptr; + + public: + struct CompileError { + uhalni mRuleIndex = 0; + tStateType mRuleState; + const char* description = nullptr; + [[nodiscard]] bool isError() const { return description; } + }; + + CompileError mError; + + Node compile(Graph& graph, const tAlphabetType* regex, tStateType state) { + mGraph = &graph; + return compileUtil(regex, state); + } + + Node compile(Graph& aGraph, const Grammar& grammar) { + mGraph = &aGraph; + + auto left = mGraph->addVertex(); + auto right = mGraph->addVertex(); + + halni idx = 0; + for (auto rule : grammar.mRules) { + + auto node = compileUtil(rule.data().first, rule.data().second); + + if (!(node.left && node.right)) { + mError.mRuleIndex = idx; + return {}; + } + + transitionAny(left, node.left); + transitionAny(node.right, right); + + idx++; + } + + mGraph->setStartVertex(left); + + return { left, right }; + } + + private: + Node compileUtil(const Grammar::Node* astNode, tStateType state) { + + auto node = compileNode(astNode, nullptr, nullptr); + + mGraph->setVertexState(node.right, state); + mGraph->setStartVertex(node.left); + + return node; + } + + Node compileVal(Grammar::ValueNode* val, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + auto left = aLeft ? aLeft : mGraph->addVertex(); + auto right = aRight ? aRight : mGraph->addVertex(); + transitionVal(left, right, val->mVal); + return { left, right }; + } + + Node compileAlternation(const Grammar::AlternationNode* alt, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + auto first_node = compileNode(alt->mFirst, aLeft, aRight); + auto second_node = compileNode(alt->mSecond); + transitionAny(first_node.left, second_node.left); + transitionAny(second_node.right, first_node.right); + return first_node; + } + + Node compileAny(const Grammar::AnyNode*, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + auto left = aLeft ? aLeft : mGraph->addVertex(); + auto right = aRight ? aRight : mGraph->addVertex(); + transitionAny(left, right, true); + return { left, right }; + } + + Node compileRepeat(const Grammar::RepetitionNode* repeat, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + if (repeat->mPlus) { + auto middle = mGraph->addVertex(); + + auto left_node = compileNode(repeat->mNode, aLeft, middle); + + auto right_node = compileNode(repeat->mNode, middle, aRight); + transitionAny(right_node.right, right_node.left); + transitionAny(right_node.left, right_node.right); + + return { left_node.left, right_node.right }; + } else { + auto node = compileNode(repeat->mNode, aLeft, aRight); + transitionAny(node.right, node.left); + transitionAny(node.left, node.right); + return node; + } + } + + Node compileIf(const Grammar::IfNode* ifNode, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + auto node = compileNode(ifNode->mNode, aLeft, aRight); + transitionAny(node.left, node.right); + return node; + } + + Node compileClass(const Grammar::ClassNode* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + auto left = aLeft ? aLeft : mGraph->addVertex(); + auto right = aRight ? aRight : mGraph->addVertex(); + + if (node->mRanges.size() == 1) { + auto const& range = node->mRanges.first(); + transitionRange(left, right, { range.mBegin, range.mEnd }, node->mExclude); + return { left, right }; + } + + for (auto range : node->mRanges) { + auto middle = mGraph->addVertex(); + transitionRange(left, middle, { range.data().mBegin, range.data().mEnd }, node->mExclude); + transitionAny(middle, right); + } + return { left, right }; + } + + Node compileCompound(const Grammar::CompoundNode* compound, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + Vertex* left = nullptr; + Vertex* rigth = nullptr; + + ualni idx = 0; + for (auto child : compound->mSequence) { + auto pass_left = idx == 0 ? aLeft : rigth; + auto pass_right = idx == compound->mSequence.size() - 1 ? aRight : nullptr; + auto node = compileNode(child.data(), pass_left, pass_right); + if (!left) left = node.left; + rigth = node.right; + idx++; + } + + return { left, rigth }; + } + + Node compileNode(const Grammar::Node* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + switch (node->mType) { + case Grammar::Node::CLASS: return compileClass((typename Grammar::ClassNode*) node, aLeft, aRight); + case Grammar::Node::COMPOUND: return compileCompound((typename Grammar::CompoundNode*) node, aLeft, aRight); + case Grammar::Node::IF: return compileIf((typename Grammar::IfNode*) node, aLeft, aRight); + case Grammar::Node::REPEAT: return compileRepeat((typename Grammar::RepetitionNode*) node, aLeft, aRight); + case Grammar::Node::ANY: return compileAny((typename Grammar::AnyNode*) node, aLeft, aRight); + case Grammar::Node::OR: return compileAlternation((typename Grammar::AlternationNode*) node, aLeft, aRight); + case Grammar::Node::VAL: return compileVal((typename Grammar::ValueNode*) node, aLeft, aRight); + case Grammar::Node::NONE: break; + } + ASSERT(0) + return {}; + } + + void transitionAny(Vertex* from, Vertex* to, bool consumes = false) { + mGraph->addTransition(from, to, {}, consumes, true, false); + } + + void transitionVal(Vertex* from, Vertex* to, tAlphabetType val) { + mGraph->addTransition(from, to, { val, val }, true, false, false); + } + + void transitionRange(Vertex* from, Vertex* to, Range range, bool exclude) { + mGraph->addTransition(from, to, range, true, false, exclude); + } + }; +} diff --git a/Language/public/Parser.hpp b/Language/public/Parser.hpp index 519ee4a..81be880 100644 --- a/Language/public/Parser.hpp +++ b/Language/public/Parser.hpp @@ -6,16 +6,34 @@ namespace tp { - template + template class Parser { + + typedef TransitionMatrix RegularTable; + typedef DFA RegularGraph; + typedef RegularCompiler RegularCompiler; + typedef NFA RegularNonDetGraph; + public: Parser() = default; public: - void compileTables(const ContextFreeGrammar& cfGrammar, const RegularGrammar& reGrammar) {} + void compileTables(const ContextFreeGrammar& cfGrammar, const RegularGrammar& reGrammar) { + // Compile Regular Grammar + { + RegularNonDetGraph nfa; + RegularCompiler compiler; + compiler.compile(nfa, reGrammar); + + RegularGraph dfa(nfa); + mRegularTable.construct(dfa); + } + } + void parse(const tAlphabetType* sentence, ualni sentenceLength, AST& out) {} public: // save load compiled tables + RegularTable mRegularTable; }; } diff --git a/Language/public/RegularAutomata.hpp b/Language/public/RegularAutomata.hpp new file mode 100644 index 0000000..3199829 --- /dev/null +++ b/Language/public/RegularAutomata.hpp @@ -0,0 +1,501 @@ + +#pragma once + +#include "Strings.hpp" +#include "List.hpp" +#include "Map.hpp" +#include "Buffer2D.hpp" + +namespace tp { + + template + class TransitionMatrix; + + template + class DFA; + + // Non-Deterministic Finite-State Automata + template + class NFA { + static_assert(TypeTraits::isIntegral, "tAlphabetType must be enumerable."); + + public: + struct Vertex; + + private: + struct Edge { + Vertex* mVertex = nullptr; + bool mConsumesSymbol = false; + Range mAcceptingRange; + bool mAcceptsAll = false; + bool mExclude = false; + + bool isTransition(const tAlphabetType& symbol) { + if (symbol == 0) return false; + if (!mConsumesSymbol || mAcceptsAll) return true; + bool const in_range = (symbol >= mAcceptingRange.mBegin && symbol <= mAcceptingRange.mEnd); + return in_range != mExclude; + } + }; + + public: + struct Vertex { + List edges; + tStateType termination_state = tNoStateVal; +#ifdef ENV_BUILD_DEBUG + ualni debug_idx = 0; +#endif + ualni flag = 0; + }; + + private: + friend DFA; + + List mVertices; + Vertex* mStart = nullptr; + + public: + NFA() = default; + + Vertex* addVertex() { + auto node = mVertices.newNode(); +#ifdef ENV_BUILD_DEBUG + node->data.debug_idx = mVertices.length() + 1; +#endif + mVertices.pushBack(node); + return &node->data; + } + + void + addTransition(Vertex* from, Vertex* to, Range range, bool consumes, bool accepts_all, bool exclude) { + Edge edge; + edge.mVertex = to; + edge.mConsumesSymbol = consumes; + edge.mAcceptingRange = range; + edge.mExclude = exclude; + edge.mAcceptsAll = accepts_all; + from->edges.pushBack(edge); + } + + void setStartVertex(Vertex* start) { mStart = start; } + + [[nodiscard]] Vertex* getStartVertex() const { return mStart; } + + void setVertexState(Vertex* vertex, tStateType state) { vertex->termination_state = state; } + + [[nodiscard]] bool isValid() const { + if (!mStart) { + return false; + } + return true; + } + + Range getAlphabetRange() const { + tAlphabetType start = 0, end = 0; + Range all_range( + std::numeric_limits::min(), std::numeric_limits::max() + ); + bool first = true; + + for (auto vertex : mVertices) { + for (auto edge : vertex.data().edges) { + if (!edge.data().mConsumesSymbol) continue; + auto const& tran_range = edge.data().mAcceptingRange; + if (edge.data().mAcceptsAll || edge.data().mExclude) return all_range; + if (tran_range.mBegin < start || first) start = tran_range.mBegin; + if (tran_range.mEnd > end || first) end = tran_range.mEnd; + first = false; + } + } + + return Range(start, end + 1); + } + + // vertices that are reachable from initial set with no input consumption (E-transitions) + // does not include initial set + void closure(const List& set, List& closure, ualni unique_call_id) { + List marked; + marked = set; + + while (marked.length()) { + auto first = marked.first()->data; + if (first->flag != unique_call_id) { + first->flag = unique_call_id; + closure.pushBack(first); + } + for (auto edge : first->edges) { + if (!edge.data().mConsumesSymbol && edge.data().mVertex->flag != unique_call_id) { + marked.pushBack(edge.data().mVertex); + } + } + marked.popFront(); + } + } + + // vertices that are reachable from initial set with symbol transition + void move(const List& set, List& reachable, tAlphabetType symbol, ualni unique_call_id) { + for (auto vertex : set) { + for (auto edge : vertex->edges) { + if (!edge.data().mConsumesSymbol) continue; + bool transition = edge.data().isTransition(symbol); + if (transition && edge.data().mVertex->flag != unique_call_id) { + edge.data().mVertex->flag = unique_call_id; + reachable.pushBack(edge.data().mVertex); + } + } + } + } + }; + + // Deterministic Finite-State Automata + template + class DFA { + static_assert(TypeTraits::isIntegral, "tAlphabetType must be enumerable."); + friend TransitionMatrix; + + struct Vertex { + struct Edge { + Vertex* vertex = nullptr; + tAlphabetType transition_code = nullptr; + }; + + List edges; + tStateType termination_state = tNoStateVal; + bool marked = false; + }; + + List mVertices; + Vertex* mStart = nullptr; + const Vertex* mIter = nullptr; + Range mAlphabetRange; + bool mTrapState = false; + + typedef typename NFA::Vertex NState; + + public: + explicit DFA(NFA& nfa) { + if (!nfa.isValid()) { + return; + } + + mAlphabetRange = nfa.getAlphabetRange(); + + struct DStateKey { + const List* nStates; + bool operator==(const DStateKey& in) const { + if (nStates->length() != in.nStates->length()) { + return false; + } + // FIXME : make linear time + for (auto state : *nStates) { + bool found = false; + for (auto in_state : *in.nStates) { + if (state.data() == in_state.data()) { + found = true; + break; + } + } + + if (!found) { + return false; + } + } + + return true; + } + + static ualni dStateHashFunc(DStateKey key) { + alni out = 0; + for (auto state : *key.nStates) { + out += alni(state.data()); + } + return out; + }; + }; + + // all NFA states that are reachable from initial DFA State for specific symbol in alphabet + struct DState { + struct DTransition { + DState* state = nullptr; + tAlphabetType accepting_code; + }; + + List nStates; + List transitions; + + Vertex* dVertex = nullptr; // relevant DFA vertex + +#ifdef ENV_BUILD_DEBUG + ualni debug_idx = 0; +#endif + }; + + // includes closure of NFA start state by definition + auto start_state = new DState(); + nfa.closure({ nfa.getStartVertex() }, start_state->nStates, ualni(start_state)); + + Map dStates; + + dStates.put({ &start_state->nStates }, start_state); + + List working_set = { start_state }; + + // while there is items to work with + auto currentDState = working_set.first(); + while (currentDState) { + + // check all possible transitions for any symbol + for (auto symbol : mAlphabetRange) { + + List reachableNStates; + + nfa.move(currentDState->data->nStates, reachableNStates, symbol, ualni(currentDState + symbol)); + nfa.closure(reachableNStates, reachableNStates, ualni(currentDState + symbol)); + + if (!reachableNStates.length()) { + continue; + } + + DState* targetDState = nullptr; + + // check if set of all reachable NFA states already forms existing DFA state + auto idx = dStates.presents({ &reachableNStates }); + if (idx) { + targetDState = dStates.getSlotVal(idx); + } + + if (!targetDState) { + // register new DFA state + targetDState = new DState(); + +#ifdef ENV_BUILD_DEBUG + targetDState->debug_idx = dStates.size(); +#endif + + targetDState->nStates = reachableNStates; + + // append to working stack + working_set.pushBack(targetDState); + dStates.put({ &targetDState->nStates }, targetDState); + } + + // add transition to DFA state + currentDState->data->transitions.pushBack({ targetDState, symbol }); + } + + working_set.popFront(); + currentDState = working_set.first(); + } + + // create own vertices + for (auto node : dStates) { + tStateType state = tNoStateVal; + for (auto iter : node->val->nStates) { + if (iter->termination_state != tNoStateVal) { + state = iter->termination_state; + break; + } + } + node->val->dVertex = addVertex(state); + } + + // connect all vertices + for (auto node : dStates) { + for (auto edge : node->val->transitions) { + addTransition(node->val->dVertex, edge.data().state->dVertex, edge.data().accepting_code); + } + } + + // set the starting vertex + mStart = start_state->dVertex; + + // cleanup + for (auto node : dStates) { + delete node->val; + } + + collapseEquivalentVertices(); + mAlphabetRange = getAlphabetRange(); + } + + tStateType move(tAlphabetType symbol) { + if (mTrapState || !mIter) { + return tNoStateVal; + } + + for (auto edge : mIter->edges) { + if (edge.data().transition_code == symbol) { + mIter = edge.data().vertex; + return mIter->termination_state; + } + } + + mTrapState = true; + return tNoStateVal; + } + + void start() { + mIter = mStart; + mTrapState = false; + } + + [[nodiscard]] uhalni nVertices() const { return (uhalni) mVertices.length(); } + + [[nodiscard]] Range getRange() const { return mAlphabetRange; } + + private: + [[nodiscard]] Range getAlphabetRange() const { + Range out; + for (auto vertex : mVertices) { + vertex.data().marked = false; + } + + bool first = true; + getAlphabetRangeUtil(mStart, &out, first); + out.mEnd++; + return out; + } + + void getAlphabetRangeUtil(Vertex* vert, Range* out, bool& first) const { + vert->marked = true; + for (auto edge : vert->edges) { + auto const code = edge.data().transition_code; + + if (first) { + *out = { code, code }; + first = false; + } + + if (code < out->mBegin) { + out->mBegin = code; + } + if (code > out->mEnd) { + out->mEnd = code; + } + + if (!edge.data().vertex->marked) { + getAlphabetRangeUtil(edge.data().vertex, out, first); + } + } + } + + void collapseEquivalentVertices() { + // TODO + } + + Vertex* addVertex(tStateType state) { + auto node = mVertices.addNodeBack(); + node->data.termination_state = state; + return &node->data; + } + + void addTransition(Vertex* from, Vertex* to, tAlphabetType transition_symbol) { + from->edges.pushBack({ to, transition_symbol }); + } + }; + + template + class TransitionMatrix { + + static_assert(TypeTraits::isIntegral, "tAlphabetType must be enumerable."); + + Buffer2D mTransitions; + Buffer mStates; + Range mSymbolRange = { 0, 0 }; + + ualni mIter = 0; + ualni mIterPrev = 0; + ualni mStart = 0; + + public: + TransitionMatrix() = default; + + auto getStates() const { return &mStates; } + auto getTransitions() const { return &mTransitions; } + auto getStart() const { return mStart; } + + void construct(const DFA& dfa) { + mSymbolRange = dfa.getRange(); + auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin); + auto sizeX = range_len ? range_len : 1; + auto sizeY = (ualni) (dfa.nVertices() + 1); + + mTransitions.reserve({ sizeX, sizeY }); + mTransitions.assign(dfa.nVertices()); + mStates.reserve(sizeY); + + ualni idx = 0; + for (auto vertex : dfa.mVertices) { + auto state = vertex.data().termination_state; + mStates[idx] = state; + idx++; + } + + mStates[dfa.nVertices()] = tFailedStateVal; + + idx = 0; + for (auto vertex : dfa.mVertices) { + if (&vertex.data() == dfa.mStart) { + mStart = mIter = mIterPrev = idx; + } + idx++; + } + + ualni vertexIdx = 0; + for (auto vertex : dfa.mVertices) { + for (auto edge : vertex.data().edges) { + ualni vertex2Idx = 0; + for (auto vertex2 : dfa.mVertices) { + if (edge.data().vertex == &vertex2.data()) break; + vertex2Idx++; + } + auto const code = edge.data().transition_code; + mTransitions.set({ (ualni) (code - mSymbolRange.mBegin), (ualni) vertexIdx }, vertex2Idx); + } + vertexIdx++; + } + } + + bool isTrapped() { return mStates[mIter] == tFailedStateVal; } + + tStateType move(tAlphabetType symbol) { + if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) { + mIter = mTransitions.get({ (ualni) (symbol - mSymbolRange.mBegin), (ualni) mIter }); + } else { + mIter = mStates.size() - 1; + } + + if (mIterPrev == mStart) { + if (mStates[mIter] == tFailedStateVal) { + reset(); + return tFailedStateVal; + } else { + mIterPrev = mIter; + return tNoStateVal; + } + } else { + if (mStates[mIter] == tFailedStateVal) { + if (mStates[mIterPrev] != tNoStateVal) { + auto out = mStates[mIterPrev]; + reset(); + return out; + } else { + reset(); + return tFailedStateVal; + } + } else { + mIterPrev = mIter; + return tNoStateVal; + } + } + + mIterPrev = mIter; + return mStates[mIter]; + } + + void reset() { + mIter = mStart; + mIterPrev = mStart; + } + }; +} diff --git a/Language/public/SimpleParser.hpp b/Language/public/SimpleParser.hpp index 940ca32..071f761 100644 --- a/Language/public/SimpleParser.hpp +++ b/Language/public/SimpleParser.hpp @@ -7,6 +7,8 @@ namespace tp { // Gives ability to express grammar in the Unified Format as sentence template class SimpleParser { + enum UGTokens : ualni { InTransition = 0, Failed, TestSeq }; + public: SimpleParser() { // Grammar for unified grammar format sentence that tables compiled from @@ -20,10 +22,10 @@ namespace tp { } // Define Regular grammar - RegularGrammar rg; + RegularGrammar rg; { // this is basically ast from existing tokenizer - rg.addRule(rg.seq({ rg.val('a'), rg.val('b') }), 0); + rg.addRule(rg.seq({ rg.val('a'), rg.val('b') }), TestSeq); } mUnifiedGrammarParser.compileTables(contextFreeGrammar, rg); @@ -36,7 +38,7 @@ namespace tp { // compile each ast into RegularGrammar and ContextFree Grammar api instructions ContextFreeGrammar userContextFreeGrammar; - RegularGrammar userRegularGrammar; + RegularGrammar userRegularGrammar; // ... // split ast into RE and CF part @@ -48,10 +50,12 @@ namespace tp { mUserParser.compileTables(userContextFreeGrammar, userRegularGrammar); } - void parse(const tAlphabetType* grammar, ualni grammarLength, AST& out) { mUserParser.parse(grammar, grammarLength, out); } + void parse(const tAlphabetType* grammar, ualni grammarLength, AST& out) { + mUserParser.parse(grammar, grammarLength, out); + } private: - Parser mUnifiedGrammarParser; - Parser mUserParser; + Parser mUnifiedGrammarParser; + Parser mUserParser; }; } diff --git a/Language/tests/Tests.cpp b/Language/tests/Tests.cpp index cbcb6b3..7619a24 100644 --- a/Language/tests/Tests.cpp +++ b/Language/tests/Tests.cpp @@ -1,5 +1,4 @@ -#include "NewPlacement.hpp" #include "Test.hpp" using namespace tp; diff --git a/Utils/public/Utils.hpp b/Utils/public/Utils.hpp index 8c6bdb7..d883b67 100644 --- a/Utils/public/Utils.hpp +++ b/Utils/public/Utils.hpp @@ -36,12 +36,14 @@ namespace tp { T1 t1; T1 head; T1 x; + T1 first; }; union { T2 t2; T2 tail; T2 y; + T2 second; }; bool operator==(const Pair& in) const { return in.t1 == t1 && in.t2 == t2; }