From 692994c2d39263c5f4cfbbbc319ee7dc8e41b100 Mon Sep 17 00:00:00 2001 From: IlushaShurupov Date: Sat, 15 Jul 2023 11:25:07 +0300 Subject: [PATCH] Tokenizer Initial --- CMakeLists.txt | 3 +- Strings/public/Strings.hpp | 4 +- Tokenizer/CMakeLists.txt | 22 ++ Tokenizer/private/CmdArgParser.cpp | 326 +++++++++++++++++ Tokenizer/private/Tokenizer.cpp | 15 + Tokenizer/public/AutomataGraph.h | 510 +++++++++++++++++++++++++++ Tokenizer/public/CmdArgParser.h | 97 +++++ Tokenizer/public/RegularExpression.h | 498 ++++++++++++++++++++++++++ Tokenizer/public/Tokenizer.hpp | 176 +++++++++ Tokenizer/rsc/script.txt | 18 + Tokenizer/tests/tests.cpp | 182 ++++++++++ 11 files changed, 1848 insertions(+), 3 deletions(-) create mode 100644 Tokenizer/CMakeLists.txt create mode 100644 Tokenizer/private/CmdArgParser.cpp create mode 100644 Tokenizer/private/Tokenizer.cpp create mode 100644 Tokenizer/public/AutomataGraph.h create mode 100644 Tokenizer/public/CmdArgParser.h create mode 100644 Tokenizer/public/RegularExpression.h create mode 100644 Tokenizer/public/Tokenizer.hpp create mode 100644 Tokenizer/rsc/script.txt create mode 100644 Tokenizer/tests/tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 489a229..4845590 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,4 +15,5 @@ add_subdirectory(Utils) add_subdirectory(Containers) add_subdirectory(Math) add_subdirectory(Allocators) -add_subdirectory(Strings) \ No newline at end of file +add_subdirectory(Strings) +add_subdirectory(Tokenizer) \ No newline at end of file diff --git a/Strings/public/Strings.hpp b/Strings/public/Strings.hpp index daba7ba..b1038bb 100644 --- a/Strings/public/Strings.hpp +++ b/Strings/public/Strings.hpp @@ -136,8 +136,8 @@ namespace tp { } // output will be null if in TextEditing mode - [[nodiscard]] tChar* resize() { - if (!mData->getEditor()) mData->resize(); + tChar* resize(uint1 size) { + if (!mData->getEditor()) mData->resize(size); return mData->getBuffer(); } diff --git a/Tokenizer/CMakeLists.txt b/Tokenizer/CMakeLists.txt new file mode 100644 index 0000000..20982ac --- /dev/null +++ b/Tokenizer/CMakeLists.txt @@ -0,0 +1,22 @@ + +cmake_minimum_required(VERSION 3.2) + +set(CMAKE_CXX_STANDARD 23) + +project(Tokenizer) + +### ---------------------- Static Library --------------------- ### +file(GLOB SOURCES "./private/*.cpp") +file(GLOB HEADERS "./public/*.hpp") +add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS}) +target_include_directories(${PROJECT_NAME} PUBLIC ./public/) +target_link_libraries(${PROJECT_NAME} PUBLIC Strings Math) + +### -------------------------- Tests -------------------------- ### +enable_testing() +file(GLOB TEST_SOURCES "./tests/*.cpp") +add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES}) +target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils) +add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests) + +install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/${PROJECT_NAME}/lib) \ No newline at end of file diff --git a/Tokenizer/private/CmdArgParser.cpp b/Tokenizer/private/CmdArgParser.cpp new file mode 100644 index 0000000..f52446d --- /dev/null +++ b/Tokenizer/private/CmdArgParser.cpp @@ -0,0 +1,326 @@ + +#include "allocators.h" + +#include "CmdArgParser.h" + +#include "log.h" + +tp::CmdArgParser::Arg::Arg(const Arg& arg) { + mId = arg.mType; + switch (arg.mType) { + case tp::CmdArgParser::Arg::INT: mType = INT; mInt = arg.mInt; break; + case tp::CmdArgParser::Arg::FLOAT: mType = FLOAT; mFloat = arg.mFloat; break; + case tp::CmdArgParser::Arg::BOOL: mType = BOOL; mBool = arg.mBool; break; + case tp::CmdArgParser::Arg::STR: mType = STR; new (&mStr) StringArg(); mStr = arg.mStr; break; + case tp::CmdArgParser::Arg::FILE_IN: mType = FILE_IN; new (&mFile) FileInputArg(); mFile = arg.mFile; break; + } + mOptional = arg.mOptional; +} + +tp::CmdArgParser::Arg::Arg(tp::string id, Type type) { + mId = id; + switch (type) { + case tp::CmdArgParser::Arg::INT: mType = INT; break; + case tp::CmdArgParser::Arg::FLOAT: mType = FLOAT; break; + case tp::CmdArgParser::Arg::BOOL: mType = BOOL; break; + case tp::CmdArgParser::Arg::STR: mType = STR; new (&mStr) StringArg(); break; + case tp::CmdArgParser::Arg::FILE_IN: mType = FILE_IN; new (&mFile) FileInputArg(); break; + } + mOptional = false; +} + +tp::CmdArgParser::Arg::Arg(tp::string id, Range aAcceptingRange) { + mId = id; + mType = FLOAT; + mInt = { 0, 0, aAcceptingRange }; + mOptional = false; +} + +tp::CmdArgParser::Arg::Arg(tp::string id, Range aAcceptingRange) { + mId = id; + mType = FLOAT; + mFloat = { 0, 0, aAcceptingRange }; + mOptional = false; +} + +tp::CmdArgParser::Arg::Arg(tp::string id, Range aAcceptingRange, alni aDefault) { + mId = id; + mType = INT; + mInt = { 0, aDefault, aAcceptingRange }; +} + +tp::CmdArgParser::Arg::Arg(tp::string id, Range aAcceptingRange, alnf aDefault) { + mId = id; + mType = FLOAT; + mFloat = { 0, aDefault, aAcceptingRange }; +} + +tp::CmdArgParser::Arg::Arg(tp::string id, bool aDefault) { + mId = id; + mType = BOOL; + mBool.mDefault = aDefault; +} + +tp::CmdArgParser::Arg::Arg(tp::string id, const char* aDefault) { + mId = id; + mType = STR; + new (&mStr) StringArg(); + mStr.mDefault = aDefault; +} + +tp::CmdArgParser::Arg::~Arg() { + switch (mType) { + case tp::CmdArgParser::Arg::STR: + mStr.~StringArg(); + break; + case tp::CmdArgParser::Arg::FILE_IN: + mFile.~FileInputArg(); + break; + } +} + +tp::CmdArgParser::CmdArgParser(tp::init_list args) { + bool optional_start = false; + + for (auto& arg : args) { + assert(!mArgs.presents(arg.mId) && "Argument Redefinition"); + + auto copy_arg = new Arg(arg); + + mArgs.put(arg.mId, copy_arg); + mArgsOrder.pushBack(mArgs.get(arg.mId)); + + if (arg.mOptional) { + mOptionals++; + } + + if (optional_start) { + assert(arg.mOptional && "Not Optional Argument After Optionals"); + } + else if (arg.mOptional) { + optional_start = true; + } + } + + mTokenizer.build({ + { "\n|\t| |\r", TokType::SPACE }, + { "N|n|(False)|(false)", TokType::BOOL_FALSE }, + { "Y|y|(True)|(true)", TokType::BOOL_TRUE }, + { "((\\-)|(\\+))?[0-9]+", TokType::INT }, + { "((\\-)|(\\+))?([0-9]+)(\\.)([0-9]*)?", TokType::FLOAT }, + { "'{'-'}*'", TokType::STR }, + }); + + assert(mTokenizer.isBuild() && "Internal Error"); +} + +tp::CmdArgParser::~CmdArgParser() { + for (auto arg : mArgsOrder) { + delete arg.data(); + } +} + +bool tp::CmdArgParser::parse(char argc, const char* argv[], bool logError) { + // discard windows working directory argument + argc--; + + if (argc < mArgs.size() - mOptionals) { + ErrInvalidArgCount(); + if (logError) { + ErrLog(); + } + return false; + } + + for (auto arg : mArgsOrder) { + if (arg.idx() < argc) { + parseArg(*arg.data(), argv[arg.idx() + 1]); + } + else { + initDefault(*arg.data()); + } + if (mError) { + if (logError) { + ErrLog(); + } + return false; + } + } + return true; + + //tp::set_working_dir(); +} + +tp::alni tp::CmdArgParser::getInt(string id) { auto& arg = getArg(id, Arg::INT); return arg.mInt.mVal; } +tp::alnf tp::CmdArgParser::getFloat(string id) { auto& arg = getArg(id, Arg::FLOAT); return arg.mFloat.mVal; } +bool tp::CmdArgParser::getBool(string id) { auto& arg = getArg(id, Arg::BOOL); return arg.mBool.mFlag; } +tp::string tp::CmdArgParser::getString(string id) { auto& arg = getArg(id, Arg::STR); return arg.mStr.mStr; } +tp::File& tp::CmdArgParser::getFile(string id) { auto& arg = getArg(id, Arg::FILE_IN); return arg.mFile.mFile; } + + +tp::CmdArgParser::Arg& tp::CmdArgParser::getArg(string id, Arg::Type type) { + auto idx = mArgs.presents(id); + assert(idx && "Invalid Id"); + auto& arg = mArgs.getSlotVal(idx); + assert(arg->mType == type && "Invalid Type Requested"); + return *arg; +} + +void tp::CmdArgParser::ErrInvalidArgCount() { mError = { "Invalid Number Of Arguments Passed", nullptr }; } +void tp::CmdArgParser::ErrInvalidArgSyntax(Arg* arg) { mError = { "Invalid Syntax Of Argument", arg }; } +void tp::CmdArgParser::ErrInvalidArgType(Arg* arg) { mError = { "Invalid Type Of Argument", arg }; } +void tp::CmdArgParser::ErrFileNotExists(Arg* arg) { mError = { "File Not Exists", arg }; } +void tp::CmdArgParser::ErrFileCouldNotOpen(Arg* arg) { mError = { "Could Not Open File", arg }; } +void tp::CmdArgParser::ErrValNotinRange(Arg* arg) { mError = { "Value Not In Range", arg }; } + +void tp::CmdArgParser::ErrLog() { + printf("Command Line Error: \n"); + + if (mError.mArg) { + alni idx = 0; + for (auto arg : mArgsOrder) { + if (mError.mArg == arg.data()) { + idx = arg.idx(); + break; + } + } + + printf("\nArgument : %lli\n", idx); + ArgLog(*mError.mArg); + } + + printf("Error Description : %s. \n", mError.mDescr); + + printf("\nExpected Arguments were : \n"); + + for (auto arg : mArgsOrder) { + printf(" -- %lli -- \n", arg.idx()); + ArgLog(*arg.data()); + } +} + +void tp::CmdArgParser::ArgLog(Arg& arg) { + switch (arg.mType) { + case Arg::INT: { + printf("Type : Int\n"); + printf("Range : %lli - %lli\n", arg.mInt.mAcceptingRange.mBegin, arg.mInt.mAcceptingRange.mEnd); + if (arg.mOptional) { + printf("Default : %lli\n", arg.mInt.mDefault); + } + printf("Given : %lli\n", arg.mInt.mVal); + } break; + case Arg::FLOAT: { + printf("Type : Float\n"); + printf("Range : %f - %f\n", arg.mFloat.mAcceptingRange.mBegin, arg.mFloat.mAcceptingRange.mEnd); + if (arg.mOptional) { + printf("Default : %f\n", arg.mFloat.mDefault); + } + printf("Given : %f\n", arg.mFloat.mVal); + } break; + case Arg::BOOL: { + printf("Type : Bool\n"); + if (arg.mOptional) { + printf("Default : %s\n", arg.mBool.mDefault ? "True" : "False"); + } + printf("Given : %s\n", arg.mBool.mFlag ? "True" : "False"); + } break; + case Arg::STR: { + printf("Type : String\n"); + if (arg.mOptional) { + printf("Default : %s\n", arg.mStr.mDefault.cstr()); + } + printf("Given : %s\n", arg.mStr.mStr.cstr()); + } break; + case Arg::FILE_IN: { + printf("Type : File Input\n"); + printf("Given Path : %s\n", arg.mFile.mFilepath.cstr()); + } break; + } +} + +void tp::CmdArgParser::initDefault(Arg& arg) { + switch (arg.mType) { + case Arg::INT: arg.mInt.mVal = arg.mInt.mDefault; break; + case Arg::FLOAT: arg.mFloat.mVal = arg.mFloat.mDefault; break; + case Arg::BOOL: arg.mBool.mFlag = arg.mBool.mDefault; break; + case Arg::STR: arg.mStr.mStr = arg.mStr.mDefault; break; + } +} + +void tp::CmdArgParser::parseArg(Arg& arg, const char* src) { + mTokenizer.reset(); + mTokenizer.bindSource(src); + + auto tok = mTokenizer.readTok(); + if (tok < TokType::INT || tok > TokType::STR) { + ErrInvalidArgSyntax(&arg); + } + auto val = mTokenizer.extractVal(); + + switch (arg.mType) { + case Arg::INT: { + if (tok != TokType::INT) { + ErrInvalidArgType(&arg); + return; + } + arg.mInt.mVal = val.operator tp::alni(); + if (arg.mInt.mVal < arg.mInt.mAcceptingRange.mBegin || arg.mInt.mVal > arg.mInt.mAcceptingRange.mEnd) { + ErrValNotinRange(&arg); + return; + } + } break; + + case Arg::FLOAT: { + if (tok != TokType::FLOAT) { + ErrInvalidArgType(&arg); + return; + } + arg.mFloat.mVal = val.operator tp::alnf(); + if (arg.mFloat.mVal < arg.mInt.mAcceptingRange.mBegin || arg.mFloat.mVal > arg.mInt.mAcceptingRange.mEnd) { + ErrValNotinRange(&arg); + return; + } + } break; + + case Arg::STR: { + if (tok != TokType::STR) { + ErrInvalidArgType(&arg); + return; + } + auto len = val.size(); + arg.mStr.mStr.reserve(len - 1); + tp::memcp(arg.mStr.mStr.get_writable(), val.cstr() + 1, len - 2); + arg.mStr.mStr.get_writable()[len - 2] = '\0'; + } break; + + case Arg::BOOL: { + if (tok != TokType::BOOL_FALSE && tok != TokType::BOOL_TRUE) { + ErrInvalidArgType(&arg); + return; + } + arg.mBool.mFlag = val.operator bool(); + } break; + + case Arg::FILE_IN: { + if (tok != TokType::STR) { + ErrInvalidArgType(&arg); + return; + } + + auto len = val.size(); + arg.mFile.mFilepath.reserve(len - 1); + tp::memcp(arg.mFile.mFilepath.get_writable(), val.cstr() + 1, len - 2); + arg.mFile.mFilepath.get_writable()[len - 2] = '\0'; + + if (!File::exists(arg.mFile.mFilepath.cstr())) { + ErrFileNotExists(&arg); + return; + } + arg.mFile.mFile.open(arg.mFile.mFilepath.cstr(), osfile_openflags::LOAD); + if (!arg.mFile.mFile.opened) { + ErrFileCouldNotOpen(&arg); + return; + } + } break; + } +} \ No newline at end of file diff --git a/Tokenizer/private/Tokenizer.cpp b/Tokenizer/private/Tokenizer.cpp new file mode 100644 index 0000000..c0b6e62 --- /dev/null +++ b/Tokenizer/private/Tokenizer.cpp @@ -0,0 +1,15 @@ + +#include "Tokenizer.hpp" + +using namespace tp; + +static ModuleManifest* sModuleDependencies[] = { + &tp::gModuleMath, + &tp::gModuleStrings, + nullptr +}; + +void deinit(const ModuleManifest*) {} + +ModuleManifest tp::gModuleTokenizer = ModuleManifest("Tokenizer", nullptr, deinit, sModuleDependencies); + diff --git a/Tokenizer/public/AutomataGraph.h b/Tokenizer/public/AutomataGraph.h new file mode 100644 index 0000000..67bda5b --- /dev/null +++ b/Tokenizer/public/AutomataGraph.h @@ -0,0 +1,510 @@ + +#pragma once + +#include "List.hpp" +#include "Buffer.hpp" +#include "Buffer2D.hpp" +#include "Mat.hpp" +#include "Map.hpp" + +namespace tp { + + template + class TransitionMatrix; + + template + class DFA; + + // Non-Deterministic Finite-State Automata + template + class NFA { + static_assert(TypeTraits::isIntegral, "tAlphabetType must be enumerable."); + + public: + struct Vertex; + + private: + struct Edge { + Vertex* mVertex = nullptr; + bool mConsumesSymbol = false; + Range mAcceptingRange; + bool mAcceptsAll = false; + bool mExclude = false; + + bool isTransition(const tAlphabetType& symbol) { + if (symbol == nullptr) return false; + if (!mConsumesSymbol || mAcceptsAll) return true; + bool const in_range = (symbol >= mAcceptingRange.mBegin && symbol <= mAcceptingRange.mEnd); + return in_range != mExclude; + } + }; + + public: + struct Vertex { + List edges; + tStateType termination_state = tNoStateVal; + #ifdef ENV_BUILD_DEBUG + ualni debug_idx = 0; + #endif + ualni flag = 0; + }; + + private: + friend DFA; + + List mVertices; + Vertex* mStart = nullptr; + + public: + NFA() = default; + + Vertex* addVertex() { + auto node = mVertices.newNode(); + #ifdef ENV_BUILD_DEBUG + node->data.debug_idx = mVertices.length() + 1; + #endif + mVertices.pushBack(node); + return &node->data; + } + + void addTransition(Vertex* from, Vertex* to, Range range, bool consumes, bool accepts_all, bool exclude) { + Edge edge; + edge.mVertex = to; + edge.mConsumesSymbol = consumes; + edge.mAcceptingRange = range; + edge.mExclude = exclude; + edge.mAcceptsAll = accepts_all; + from->edges.pushBack(edge); + } + + void setStartVertex(Vertex* start) { + mStart = start; + } + + [[nodiscard]] Vertex* getStartVertex() const { + return mStart; + } + + void setVertexState(Vertex* vertex, tStateType state) { + vertex->termination_state = state; + } + + [[nodiscard]] bool isValid() const { + if (!mStart) { + return false; + } + return true; + } + + Range getAlphabetRange() const { + tAlphabetType start = 0, end = 0; + Range all_range = { std::numeric_limits::min(), std::numeric_limits::max() }; + bool first = true; + + for (auto vertex : mVertices) { + for (auto edge : vertex.data().edges) { + if (!edge.data().mConsumesSymbol) continue; + auto const& tran_range = edge.data().mAcceptingRange; + if (edge.data().mAcceptsAll || edge.data().mExclude) return all_range; + if (tran_range.mBegin < start || first) start = tran_range.mBegin; + if (tran_range.mEnd > end || first) end = tran_range.mEnd; + first = false; + } + } + + return { start, end + 1 }; + } + + // vertices that are reachable from initial set with no input consumption (E-transitions) + // does not include initial set + void closure(const List& set, List& closure, ualni unique_call_id) { + List marked; + marked = set; + + while (marked.length()) { + auto first = marked.first()->data; + if (first->flag != unique_call_id) { + first->flag = unique_call_id; + closure.pushBack(first); + } + for (auto edge : first->edges) { + if (!edge.data().mConsumesSymbol && edge.data().mVertex->flag != unique_call_id) { + marked.pushBack(edge.data().mVertex); + } + } + marked.popFront(); + } + } + + // vertices that are reachable from initial set with symbol transition + void move(const List& set, List& reachable, tAlphabetType symbol, ualni unique_call_id) { + for (auto vertex : set) { + for (auto edge : vertex->edges) { + if (!edge.data().mConsumesSymbol) continue; + bool transition = edge.data().isTransition(symbol); + if (transition && edge.data().mVertex->flag != unique_call_id) { + edge.data().mVertex->flag = unique_call_id; + reachable.pushBack(edge.data().mVertex); + } + } + } + } + }; + + // Deterministic Finite-State Automata + template + class DFA { + static_assert(TypeTraits::isIntegral, "tAlphabetType must be enumerable."); + friend TransitionMatrix; + + struct Vertex { + struct Edge { + Vertex* vertex = nullptr; + tAlphabetType transition_code = nullptr; + }; + + List edges; + tStateType termination_state = tNoStateVal; + bool marked = false; + }; + + List mVertices; + Vertex* mStart = nullptr; + const Vertex* mIter = nullptr; + Range mAlphabetRange; + bool mTrapState = false; + + typedef typename NFA::Vertex NState; + + public: + + explicit DFA(NFA& nfa) { + if (!nfa.isValid()) { + return; + } + + mAlphabetRange = nfa.getAlphabetRange(); + + struct DStateKey { + const List* nStates; + bool operator==(const DStateKey& in) { + if (nStates->length() != in.nStates->length()) { + return false; + } + // FIXME : make linear time + for (auto state : *nStates) { + bool found = false; + for (auto in_state : *in.nStates) { + if (state.data() == in_state.data()) { + found = true; + break; + } + } + + if (!found) { + return false; + } + } + + return true; + } + + static ualni dStateHashFunc(DStateKey key) { + alni out = 0; + for (auto state : *key.nStates) { + out += alni(state.data()); + } + return out; + }; + }; + + // all NFA states that are reachable from initial DFA State for specific symbol in alphabet + struct DState { + + struct DTransition { + DState* state = nullptr; + tAlphabetType accepting_code; + }; + + List nStates; + List transitions; + + Vertex* dVertex = nullptr; // relevant DFA vertex + + #ifdef ENV_BUILD_DEBUG + ualni debug_idx = 0; + #endif + }; + + // includes closure of NFA start state by definition + auto start_state = new DState(); + nfa.closure({ nfa.getStartVertex() }, start_state->nstates, ualni(start_state)); + + Map dStates; + + dStates.put({ &start_state->nstates }, start_state); + + List working_set = { start_state }; + + // while there is items to work with + auto currentDState = working_set.first(); + while (currentDState) { + + // check all possible transitions for any symbol + for (auto symbol : mAlphabetRange) { + + List reachableNStates; + + nfa.move(currentDState->data->nstates, reachableNStates, symbol, ualni(currentDState + symbol)); + nfa.closure(reachableNStates, reachableNStates, ualni(currentDState + symbol)); + + if (!reachableNStates.length()) { + continue; + } + + DState* targetDState = nullptr; + + // check if set of all reachable NFA states already forms existing DFA state + auto idx = dStates.presents({ &reachableNStates}); + if (idx) { + targetDState = dStates.getSlotVal(idx); + } + + if (!targetDState) { + // register new DFA state + targetDState = new DState(); + + #ifdef ENV_BUILD_DEBUG + targetDState->debug_idx = dStates.size(); + #endif + + targetDState->nstates = reachableNStates; + + // append to working stack + working_set.pushBack(targetDState); + dStates.put({ &targetDState->nstates }, targetDState); + } + + // add transition to DFA state + currentDState->data->transitions.pushBack({targetDState, symbol }); + } + + working_set.popFront(); + currentDState = working_set.first(); + } + + // create own vertices + for (auto node : dStates) { + tStateType state = tNoStateVal; + for (auto iter : node->val->nStates) { + if (iter->termination_state != tNoStateVal) { + state = iter->termination_state; + break; + } + } + node->val->dvertex = addVertex(state); + } + + // connect all vertices + for (auto node : dStates) { + for (auto edge : node->val->transitions) { + addTransition(node->val->dvertex, edge.data().state->dvertex, edge.data().accepting_code); + } + } + + // set the starting vertex + mStart = start_state->dvertex; + + // cleanup + for (auto node : dStates) { + delete node->val; + } + + collapseEquivalentVertices(); + mAlphabetRange = getAlphabetRange(); + } + + tStateType move(tAlphabetType symbol) { + if (mTrapState || !mIter) { + return tNoStateVal; + } + + for (auto edge : mIter->edges) { + if (edge.data().transition_code == symbol) { + mIter = edge.data().vertex; + return mIter->termination_state; + } + } + + mTrapState = true; + return tNoStateVal; + } + + void start() { + mIter = mStart; + mTrapState = false; + } + + [[nodiscard]] uhalni nVertices() const { + return (uhalni) mVertices.length(); + } + + [[nodiscard]] Range getRange() const { + return mAlphabetRange; + } + + private: + [[nodiscard]] Range getAlphabetRange() const { + Range out; + for (auto vertex : mVertices) { + vertex.data().marked = false; + } + + bool first = true; + getAlphabetRangeUtil(mStart, &out, first); + out.mEnd++; + return out; + } + + void getAlphabetRangeUtil(Vertex* vert, Range* out, bool& first) const { + vert->marked = true; + for (auto edge : vert->edges) { + auto const code = edge.data().transition_code; + + if (first) { + *out = { code, code }; + first = false; + } + + if (code < out->mBegin) { + out->mBegin = code; + } + if (code > out->mEnd) { + out->mEnd = code; + } + + if (!edge.data().vertex->marked) { + getAlphabetRangeUtil(edge.data().vertex, out, first); + } + } + } + + void collapseEquivalentVertices() { + // TODO + } + + Vertex* addVertex(tStateType state) { + auto node = mVertices.addNodeBack(); + node->data.termination_state = state; + return &node->data; + } + + void addTransition(Vertex* from, Vertex* to, tAlphabetType transition_symbol) { + from->edges.pushBack({ to, transition_symbol }); + } + }; + + template + class TransitionMatrix { + + static_assert(TypeTraits::isIntegral, "tAlphabetType must be enumerable."); + + Buffer2D mTransitions{}; + Buffer mStates{}; + Range mSymbolRange = nullptr; + + ualni mIter = 0; + ualni mIterPrev = 0; + ualni mStart = 0; + + public: + + TransitionMatrix() = default; + + void construct(const DFA& dfa) { + mSymbolRange = dfa.getRange(); + auto range_len = uhalni(mSymbolRange.mEnd - mSymbolRange.mBegin); + Vec2 dim = { range_len ? range_len : 1, (uhalni) (dfa.nVertices() + 1) }; + + mTransitions.reserve(dim); + mTransitions.assign(dfa.nVertices()); + mStates.reserve(dim.y); + + for (auto vertex : dfa.mVertices) { + auto state = vertex.data().termination_state; + mStates[vertex.idx()] = state; + } + + mStates[dfa.nVertices()] = tFailedStateVal; + + for (auto vertex : dfa.mVertices) { + if (&vertex.data() == dfa.mStart) { + mStart = mIter = mIterPrev = vertex.idx(); + } + } + + for (auto vertex : dfa.mVertices) { + for (auto edge : vertex.data().edges) { + ualni idx = 0; + for (auto vertex : dfa.mVertices) { + if (edge.data().vertex == &vertex.data()) { + idx = vertex.idx(); + break; + } + } + auto const code = edge.data().transition_code; + mTransitions.set(code - mSymbolRange.mBegin, (uhalni) vertex.idx(), idx); + } + } + } + + bool isTrapped() { + return mStates[mIter] == tFailedStateVal; + } + + tStateType move(tAlphabetType symbol) { + if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) { + mIter = mTransitions.get((uhalni) (symbol - mSymbolRange.mBegin), (uhalni) mIter); + } + else { + mIter = mStates.length() - 1; + } + + if (mIterPrev == mStart) { + if (mStates[mIter] == tFailedStateVal) { + reset(); + return tFailedStateVal; + } + else { + mIterPrev = mIter; + return tNoStateVal; + } + } + else { + if (mStates[mIter] == tFailedStateVal) { + if (mStates[mIterPrev] != tNoStateVal) { + auto out = mStates[mIterPrev]; + reset(); + return out; + } + else { + reset(); + return tFailedStateVal; + } + } + else { + mIterPrev = mIter; + return tNoStateVal; + } + } + + mIterPrev = mIter; + return mStates[mIter]; + } + + void reset() { + mIter = mStart; + mIterPrev = mStart; + } + }; +} \ No newline at end of file diff --git a/Tokenizer/public/CmdArgParser.h b/Tokenizer/public/CmdArgParser.h new file mode 100644 index 0000000..ac5823b --- /dev/null +++ b/Tokenizer/public/CmdArgParser.h @@ -0,0 +1,97 @@ +#pragma once + +#include "Tokenizer.hpp" + +namespace tp { + struct CmdArgParser { + + struct IntArg { + alni mVal = 0; + alni mDefault = 0; + Range mAcceptingRange = { ENV_ALNI_MIN, ENV_ALNI_MAX }; + }; + + struct FloatArg { + alnf mVal = 0.f; + alnf mDefault = 0.f; + Range mAcceptingRange = { ENV_ALNF_MIN, ENV_ALNF_MAX }; + }; + + struct BoolArg { + bool mFlag = false; + bool mDefault = false; + }; + + struct StringArg { + String mStr; + String mDefault; + }; + + struct FileInputArg { + String mFilepath; + File mFile; + }; + + struct Arg { + string mId; + enum Type { INT, FLOAT, BOOL, STR, FILE_IN } mType; + union { + IntArg mInt; + FloatArg mFloat; + BoolArg mBool; + StringArg mStr; + FileInputArg mFile; + }; + + bool mOptional = true; + + Arg(const Arg& arg); + Arg(string id, Type type); + Arg(string id, Range aAcceptingRange); + Arg(string id, Range aAcceptingRange); + Arg(string id, Range aAcceptingRange, alni aDefault); + Arg(string id, Range aAcceptingRange, alnf aDefault); + Arg(string id, bool aDefault); + Arg(string id, const char* aDefault); + ~Arg(); + }; + + struct Error { + const char* mDescr = nullptr; + Arg* mArg = nullptr; + operator bool() { return mDescr != nullptr; } + } mError; + + CmdArgParser(init_list args); + ~CmdArgParser(); + + bool parse(char argc, const char* argv[], bool logError = false); + + alni getInt(string id); + alnf getFloat(string id); + bool getBool(string id); + string getString(string id); + File& getFile(string id); + + private: + enum class TokType { SPACE, INT, FLOAT, BOOL_FALSE, BOOL_TRUE, STR, NONE, FAILURE, END, } mType; + typedef SimpleTokenizer Tokenizer; + + Tokenizer mTokenizer; + HashMap mArgs; + List mArgsOrder; + ualni mOptionals = 0; + + Arg& getArg(string id, Arg::Type type); + void ErrInvalidArgCount(); + void ErrInvalidArgSyntax(Arg* arg); + void ErrInvalidArgType(Arg* arg); + void ErrFileNotExists(Arg* arg); + void ErrFileCouldNotOpen(Arg* arg); + void ErrValNotinRange(Arg* arg); + void ErrLog(); + void ArgLog(Arg& arg); + void initDefault(Arg& arg); + void parseArg(Arg& arg, const char* src); + }; +}; \ No newline at end of file diff --git a/Tokenizer/public/RegularExpression.h b/Tokenizer/public/RegularExpression.h new file mode 100644 index 0000000..e906df5 --- /dev/null +++ b/Tokenizer/public/RegularExpression.h @@ -0,0 +1,498 @@ + +#pragma once + +#include "AutomataGraph.h" + +namespace tp::RegEx { + + struct AstNode { + enum Type { + NONE, + ANY, + OR, + IF, + CLASS, + COMPOUND, + REPEAT, + VAL, + } mType = NONE; + }; + + template + struct AstVal : public AstNode { + explicit AstVal(tAlphabetType val) : mVal(val) { mType = VAL; } + tAlphabetType mVal; + }; + + struct AstCompound : public AstNode { + AstCompound() { mType = COMPOUND; } + List mChilds; + }; + + struct AstAlternation : public AstNode { + AstAlternation() { mType = OR; } + AstNode* mFirst = nullptr; + AstNode* mSecond = nullptr; + }; + + struct AstIf : public AstNode { + AstIf() { mType = IF; } + AstNode* mNode = nullptr; + }; + + struct AstAny : public AstNode { + AstAny() { mType = ANY; } + }; + + struct AstRepetition : public AstNode { + AstRepetition() { mType = REPEAT; } + AstNode* mNode = nullptr; + bool mPlus = false; + }; + + template + struct AstClass : public AstNode { + AstClass() { mType = CLASS; } + List> mRanges; + bool mExclude{}; + }; + + struct ParseError { + const char* description = nullptr; + uhalni offset = 0; + [[nodiscard]] bool isError() const { return description != nullptr; } + }; + + template + class Parser { + + enum TokType : uint1 { + TOK_COMPOUND_START = 0, + TOK_COMPOUND_END, + TOK_CLASS_START, + TOK_CLASS_END, + TOK_CLASS_START_EXCLUDE, + TOK_CLASS_END_EXCLUDE, + TOK_OR, + TOK_IF, + TOK_ANY, + TOK_REPEAT, + TOK_REPEAT_PLUS, + TOK_HYPHEN, + TOK_SPECIALS_END_, + TOK_VAL, + TOK_NONE, + }; + + tAlphabetType SpecialSymbols[TOK_SPECIALS_END_] = { + '(', ')', '[', ']', '{', '}', '|', '?', '.', '*', '+', '-', + }; + + tAlphabetType mEscapeSymbol = '\\'; + + struct Token { + TokType type; + tAlphabetType val; + }; + + const tAlphabetType* mSource = nullptr; + uhalni mOffset = 0; + Token mCurToken; + uhalni mTokLength = 0; + + public: + + ParseError mError; + + // regular expression must be a zero termination string + AstCompound* parse(const tAlphabetType* regex) { + mSource = regex; + return parseRegEx(); + } + + private: + + AstCompound* parseRegEx() { + auto out = new AstCompound(); + for (AstNode* node = parseElement(); node; node = parseElement()) { + out->mChilds.pushBack(node); + } + if (!out->mChilds.length()) { + genError("Expected A Expression"); + } + if (mError.description) { + delete out; + return nullptr; + } + return out; + } + + AstNode* parseElement() { + AstNode* out = nullptr; + switch (readTok().type) { + case TOK_COMPOUND_START: out = parseCompound(); break; + case TOK_CLASS_START: out = parseClass(); break; + case TOK_CLASS_START_EXCLUDE: out = parseClass(true); break; + case TOK_ANY: out = parseAny(); break; + case TOK_VAL: out = parseVal(); break; + case TOK_NONE: { discardTok(); return nullptr; }; + } + if (!out) { + discardTok(); + return nullptr; + } + switch (readTok().type) { + case TOK_OR: out = parseAlternation(out); break; + case TOK_REPEAT: out = parseRepetition(out); break; + case TOK_REPEAT_PLUS: out = parseRepetition(out, true); break; + case TOK_IF: out = parseIf(out); break; + case TOK_NONE: break; + default: { discardTok(); } + } + return out; + } + + AstCompound* parseCompound() { + auto out = new AstCompound(); + for (AstNode* node = parseElement(); node; node = parseElement()) { + out->mChilds.pushBack(node); + } + if (readTok().type != TOK_COMPOUND_END) { + genError("Expected Compound End"); + } + if (mError.description) { + delete out; + return nullptr; + } + return out; + } + + AstClass* parseClass(bool exclude = false) { + auto out = new AstClass(); + out->mExclude = exclude; + auto& ranges = out->mRanges; + + readTok(); + + READ_VAL: + if (mCurToken.type != TOK_VAL) { + delete out; + genError("Expected A Value"); + return nullptr; + } + char range_start = mCurToken.val; + + readTok(); + if (mCurToken.type != TOK_HYPHEN) { + delete out; + genError("Expected A Range"); + return nullptr; + } + + readTok(); + if (mCurToken.type != TOK_VAL) { + delete out; + genError("Expected A Value"); + return nullptr; + } + char range_end = mCurToken.val; + + ranges.pushBack({ range_start, range_end }); + + readTok(); + if ((mCurToken.type == TOK_CLASS_END && !exclude) || (mCurToken.type == TOK_CLASS_END_EXCLUDE && exclude)) { + return out; + } + else { + goto READ_VAL; + } + } + + AstAny* parseAny() { + return new AstAny(); + } + + AstVal* parseVal() { + auto out = new AstVal(mCurToken.val); + return out; + } + + AstAlternation* parseAlternation(AstNode* left) { + auto right = parseElement(); + if (!right) { + genError("Expected Alternation right Side"); + delete left; + return nullptr; + } + + auto out = new AstAlternation(); + + out->mFirst = left; + out->mSecond = right; + + return out; + } + + AstRepetition* parseRepetition(AstNode* left, bool plus = false) { + auto out = new AstRepetition(); + out->mNode = left; + out->mPlus = plus; + return out; + } + + AstIf* parseIf(AstNode* left) { + auto out = new AstIf(); + out->mNode = left; + return out; + } + + void genError(const char* desc) { + mError = { desc, mOffset }; + } + + Token& readTok() { + + const tAlphabetType* crs = mSource + mOffset; + + // zero termination string + if (*crs == 0) { + mCurToken.type = TOK_NONE; + return mCurToken; + } + + mTokLength = 1; + mCurToken.type = TOK_VAL; + mCurToken.val = crs[0]; + + if (crs[0] == mEscapeSymbol) { + mCurToken.val = crs[1]; + mTokLength = 2; + } + else { + for (uhalni tok = 0; tok < TOK_SPECIALS_END_; tok++) { + if (SpecialSymbols[tok] == mCurToken.val) { + mCurToken.type = TokType(tok); + break; + } + } + } + + mOffset += mTokLength; + return mCurToken; + } + + void discardTok() { + mOffset -= mTokLength; + mTokLength = 0; + } + }; + + template + struct CompileError { + ParseError mParseError; + uhalni mRuleIndex = 0; + tStateType mRuleState; + const char* description = nullptr; + [[nodiscard]] bool isError() const { return description; } + }; + + template + class Compiler { + + typedef NFA Graph; + typedef typename Graph::Vertex Vertex; + typedef Parser Parser; + + struct Node { + Vertex* left = nullptr; + Vertex* right = nullptr; + }; + + private: + Graph* mGraph = nullptr; + + public: + CompileError mError; + + Node compile(Graph& graph, const tAlphabetType* regex, tStateType state) { + mGraph = &graph; + return compileUtil(regex, state); + } + + Node compile(Graph& aGraph, init_list> aRules) { + mGraph = &aGraph; + + auto left = mGraph->addVertex(); + auto right = mGraph->addVertex(); + + halni idx = 0; + for (auto rule: aRules) { + + auto node = idx ? compileUtil(rule.head, rule.tail) : compileUtil(rule.head, rule.tail, left, right); + + if (!(node.left && node.right)) { + mError.mRuleIndex = idx; + return {}; + } + + if (idx) { + transitionAny(left, node.left); + transitionAny(node.right, right); + } + + idx++; + } + + mGraph->setStartVertex(left); + + return { left, right }; + } + + private: + + Node compileUtil(const tAlphabetType* regex, tStateType state, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + Parser parser; + auto astNode = parser.parse(regex); + if (parser.mError.isError()) { + mGraph->setStartVertex(nullptr); + mError.description = "Parsing Of Regular Expression Failed"; + mError.mRuleState = state; + mError.mParseError = parser.mError; + return {}; + } + + auto node = compileNode(astNode, aLeft, aRight); + delete astNode; + + mGraph->setVertexState(node.right, state); + mGraph->setStartVertex(node.left); + + return node; + } + + Node compileVal(AstVal* val, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + auto left = aLeft ? aLeft : mGraph->addVertex(); + auto right = aRight ? aRight : mGraph->addVertex(); + transitionVal(left, right, val->mVal); + return { left, right }; + } + + Node compileAlternation(AstAlternation* alt, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + auto first_node = compileNode(alt->mFirst, aLeft, aRight); + auto second_node = compileNode(alt->mSecond); + transitionAny(first_node.left, second_node.left); + transitionAny(second_node.right, first_node.right); + return first_node; + } + + Node compileAny(AstAny*, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + auto left = aLeft ? aLeft : mGraph->addVertex(); + auto right = aRight ? aRight : mGraph->addVertex(); + transitionAny(left, right, true); + return { left, right }; + } + + Node compileRepeat(AstRepetition* repeat, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + if (repeat->mPlus) { + auto middle = mGraph->addVertex(); + + auto left_node = compileNode(repeat->mNode, aLeft, middle); + + auto right_node = compileNode(repeat->mNode, middle, aRight); + transitionAny(right_node.right, right_node.left); + transitionAny(right_node.left, right_node.right); + + return { left_node.left, right_node.right }; + } + else { + auto node = compileNode(repeat->mNode, aLeft, aRight); + transitionAny(node.right, node.left); + transitionAny(node.left, node.right); + return node; + } + } + + Node compileIf(AstIf* ifNode, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + auto node = compileNode(ifNode->mNode, aLeft, aRight); + transitionAny(node.left, node.right); + return node; + } + + Node compileClass(AstClass* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + auto left = aLeft ? aLeft : mGraph->addVertex(); + auto right = aRight ? aRight : mGraph->addVertex(); + + if (node->mRanges.length() == 1) { + auto const& range = node->mRanges.first()->data; + transitionRange(left, right, { range.mBegin, range.mEnd }, node->mExclude); + return { left, right }; + } + + for (auto range : node->mRanges) { + auto middle = mGraph->addVertex(); + transitionRange(left, middle, { range.data().mBegin, range.data().mEnd }, node->mExclude); + transitionAny(middle, right); + } + return { left, right }; + } + + Node compileCompound(AstCompound* compound, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + Vertex* left = nullptr; + Vertex* rigth = nullptr; + + ualni idx = 0; + for (auto child : compound->mChilds) { + auto pass_left = idx == 0 ? aLeft : rigth; + auto pass_right = idx == compound->mChilds.length() - 1 ? aRight : nullptr; + auto node = compileNode(child.data(), pass_left, pass_right); + if (!left) left = node.left; + rigth = node.right; + idx++; + } + + return { left, rigth }; + } + + Node compileNode(AstNode* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) { + switch (node->mType) { + case AstNode::CLASS: return compileClass((AstClass*)node, aLeft, aRight); + case AstNode::COMPOUND: return compileCompound((AstCompound*)node, aLeft, aRight); + case AstNode::IF: return compileIf((AstIf*)node, aLeft, aRight); + case AstNode::REPEAT: return compileRepeat((AstRepetition*)node, aLeft, aRight); + case AstNode::ANY: return compileAny((AstAny*)node, aLeft, aRight); + case AstNode::OR: return compileAlternation((AstAlternation*)node, aLeft, aRight); + case AstNode::VAL: return compileVal((AstVal*)node, aLeft, aRight); + case AstNode::NONE: + break; + } + ASSERT(0) + return {}; + } + + void transitionAny(Vertex* from, Vertex* to, bool consumes = false) { + mGraph->addTransition(from, to, {}, consumes, true, false); + } + + void transitionVal(Vertex* from, Vertex* to, tAlphabetType val) { + mGraph->addTransition(from, to, { val, val }, true, false, false); + } + + void transitionRange(Vertex* from, Vertex* to, Range range, bool exclude) { + mGraph->addTransition(from, to, range, true, false, exclude); + } + }; + + template + CompileError compile(NFA& out, const tAlphabetType* regex, tStateType state) { + Compiler compiler; + compiler.compile(out, regex, state); + return compiler.mError; + } + + template + CompileError compile(NFA& out, const init_list>& rules) { + Compiler compiler; + compiler.compile(out, rules); + return compiler.mError; + } + } \ No newline at end of file diff --git a/Tokenizer/public/Tokenizer.hpp b/Tokenizer/public/Tokenizer.hpp new file mode 100644 index 0000000..c48552a --- /dev/null +++ b/Tokenizer/public/Tokenizer.hpp @@ -0,0 +1,176 @@ + +#pragma once + +#include "RegularExpression.h" +#include "Strings.hpp" + +namespace tp { + + extern ModuleManifest gModuleTokenizer; + + template + class Tokenizer { + + TransitionMatrix mTransitionMatrix; + + RegEx::CompileError mError; + + bool scanFailed() { + return mTransitionMatrix.isTrapped(); + } + + public: + + Tokenizer() { + MODULE_SANITY_CHECK(gModuleTokenizer) + } + + void build(const init_list>& rules) { + NFA nfa; + + mError = RegEx::compile(nfa, rules); + if (mError.isError()) { + return; + } + + DFA dfa(nfa); + mTransitionMatrix.construct(dfa); + } + + [[nodiscard]] bool isBuild() const { + return !mError.isError(); + } + + const RegEx::CompileError& getBuildError() { + return mError; + } + + void resetMatrix() { + mTransitionMatrix.reset(); + } + + tTokType advanceSymbol(tAlphabetType symbol) { + return mTransitionMatrix.move(symbol); + } + + tTokType advanceToken(const tAlphabetType* source, ualni source_len, ualni* token_len) { + tTokType out = tNoTokVal; + *token_len = 0; + for (ualni idx = 0; idx < source_len; idx++) { + out = advanceSymbol(source[idx]); + if (out != tNoTokVal) { + *token_len = idx; + break; + } + } + return out; + } + + ~Tokenizer() = default; + }; + + template + class SimpleTokenizer { + + Tokenizer mTokenizer; + + const tAlphabetType* mSource = nullptr; + ualni mLastTokLen = 0; + ualni mSourceLen = 0; + ualni mAdvancedOffset = 0; + + public: + + struct Cursor { + const tAlphabetType* mSource = nullptr; + ualni mAdvancedOffset = 0; + const tAlphabetType* str() { return mSource + mAdvancedOffset; } + }; + + SimpleTokenizer() = default; + + void build(const init_list>& rules) { + mTokenizer.build(rules); + } + + [[nodiscard]] bool isBuild() const { + return mTokenizer.isBuild(); + } + + const RegEx::CompileError& getBuildError() { + return mTokenizer.getBuildError(); + } + + void bindSource(const tAlphabetType* source) { + mSource = source; + while (mSource[mSourceLen]) mSourceLen++; + mSourceLen++; + } + + [[nodiscard]] bool isInputLeft() const { + if (!mSource) { + return false; + } + if (mSource[mAdvancedOffset] == nullptr) { + return false; + } + return true; + } + + Cursor getCursor() const { + return { mSource, mAdvancedOffset }; + } + + Cursor getCursorPrev() const { + return { mSource, mAdvancedOffset - mLastTokLen }; + } + + void setCursor(const Cursor& crs) { + mAdvancedOffset = crs.mAdvancedOffset; + mLastTokLen = 0; + } + + tTokType readTok() { + if (mSourceLen == mAdvancedOffset + 1) { + return tSourceEndTokVal; + } + + tTokType out = mTokenizer.advanceToken(mSource + mAdvancedOffset, mSourceLen - mAdvancedOffset, &mLastTokLen); + mAdvancedOffset += mLastTokLen; + return out; + } + + tTokType lookupTok() { + auto out = readTok(); + discardTok(); + return out; + } + + void discardTok() { + mTokenizer.resetMatrix(); + mAdvancedOffset -= mLastTokLen; + mLastTokLen = 0; + } + + void skipTok() { + readTok(); + } + + void reset() { + mAdvancedOffset = mLastTokLen = 0; + mTokenizer.resetMatrix(); + } + + [[nodiscard]] ualni lastTokLEn() const { + return mLastTokLen; + } + + String extractVal() { + auto crs = getCursorPrev(); + String out; + out.resize(mLastTokLen + 1); + memCopy(out.write(), crs.str(), mLastTokLen); + return out; + } + }; +} \ No newline at end of file diff --git a/Tokenizer/rsc/script.txt b/Tokenizer/rsc/script.txt new file mode 100644 index 0000000..8c312e1 --- /dev/null +++ b/Tokenizer/rsc/script.txt @@ -0,0 +1,18 @@ + + + /* ba + asadadasdK */ + + -1.f; + + if (+1.f) { + var string = "asas + d"; + + } + + while + + return + + diff --git a/Tokenizer/tests/tests.cpp b/Tokenizer/tests/tests.cpp new file mode 100644 index 0000000..a29d780 --- /dev/null +++ b/Tokenizer/tests/tests.cpp @@ -0,0 +1,182 @@ + +#include "Tokenizer.hpp" +#include "filesystem.h" + +#include + +#define LOG(val) std::cout << #val << " "; + +void test(const char* path) { + + tp::File script_file(path, tp::osfile_openflags::LOAD); + if (!script_file.opened) { + return; + } + char* script = new char[script_file.size() + 1]; + script[script_file.size()] = 0; + script_file.read_bytes(script, script_file.size()); + script_file.close(); + + enum class TokType { + NONE, + FAILED, + TOK_SOURCE_END, + + VAR, + CLASS_DEF, + SELF, + SCOPE_IN, + SCOPE_OUT, + ASSIGN, + DEF_FUNC, + PRINT, + IF, + ELSE, + WHILE, + BRACKET_IN, + BRACKET_OUT, + COMMA, + NEW, + CHILD, + RETURN, + EQUAL, + NOT_EQUAL, + MORE, + LESS, + QE_OR_MORE, + QE_OR_LESS, + BOOL_NOT, + BOOL_AND, + BOOL_OR, + ADD, + MUL, + SUB, + DIV, + STM_END, + CONST_TRUE, + CONST_FALSE, + CONST_INT, + CONST_FLOAT, + SPACE, + CONST_STRING, + ID, + //COMMENT_LINE, + COMMENT_BLOCK, + }; + + tp::SimpleTokenizer lexer; + + lexer.build({ + { "\n|\t| |\r", TokType::SPACE }, + { "var", TokType::VAR }, + { "class", TokType::CLASS_DEF }, + { "self", TokType::SELF }, + { "\\{", TokType::SCOPE_IN }, + { "\\}", TokType::SCOPE_OUT }, + { "=", TokType::ASSIGN }, + { "def", TokType::DEF_FUNC }, + { "<<", TokType::PRINT }, + { "if", TokType::IF }, + { "else", TokType::ELSE }, + { "while", TokType::WHILE }, + { "\\(", TokType::BRACKET_IN }, + { "\\)", TokType::BRACKET_OUT }, + { ",", TokType::COMMA }, + { "new", TokType::NEW }, + { "\\.", TokType::CHILD }, + { "return", TokType::RETURN }, + { "==", TokType::EQUAL }, + { "!=", TokType::NOT_EQUAL }, + { ">", TokType::MORE }, + { "<", TokType::LESS }, + { ">=", TokType::QE_OR_MORE }, + { "<=", TokType::QE_OR_LESS }, + { "!", TokType::BOOL_NOT }, + { "&&", TokType::BOOL_AND }, + { "\\|\\|", TokType::BOOL_OR }, + { "\\+", TokType::ADD }, + { "\\*", TokType::MUL }, + { "\\-", TokType::SUB }, + { "/", TokType::DIV }, + { ";", TokType::STM_END }, + { "true", TokType::CONST_TRUE }, + { "false", TokType::CONST_FALSE }, + { "((\\-)|(\\+))?[0-9]+i?", TokType::CONST_INT }, + { "((\\-)|(\\+))?([0-9]+)(\\.)([0-9]*)?f?", TokType::CONST_FLOAT }, + { "(/\\*){\\*-\\*}*(\\*/)", TokType::COMMENT_BLOCK }, + { "\"{\"-\"}*\"", TokType::CONST_STRING }, + { "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", TokType::ID }, + }); + + if (!lexer.isBuild()) { + std::cout << lexer.getBuildError().description; + return; + } + + lexer.bindSource(script); + + TokType tok; + do { + + tok = lexer.readTok(); + + switch (tok) { + case TokType::SPACE: { std::cout << " "; } break; + case TokType::VAR: { LOG(VAR); } break; + case TokType::CLASS_DEF: { LOG(CLASS_DEF); } break; + case TokType::SELF: { LOG(SELF); } break; + case TokType::SCOPE_IN: { LOG(SCOPE_IN); } break; + case TokType::SCOPE_OUT: { LOG(SCOPE_OUT); } break; + case TokType::ASSIGN: { LOG(ASSIGN); } break; + case TokType::DEF_FUNC: { LOG(DEF_FUNC); } break; + case TokType::PRINT: { LOG(PRINT); } break; + case TokType::IF: { LOG(IF); } break; + case TokType::ELSE: { LOG(ELSE); } break; + case TokType::WHILE: { LOG(WHILE); } break; + case TokType::BRACKET_IN: { LOG(BRACKET_IN); } break; + case TokType::BRACKET_OUT: { LOG(BRACKET_OUT); } break; + case TokType::COMMA: { LOG(COMMA); } break; + case TokType::NEW: { LOG(NEW); } break; + case TokType::CHILD: { LOG(CHILD); } break; + case TokType::RETURN: { LOG(RETURN); } break; + case TokType::EQUAL: { LOG(EQUAL); } break; + case TokType::NOT_EQUAL: { LOG(NOT_EQUAL); } break; + case TokType::MORE: { LOG(MORE); } break; + case TokType::LESS: { LOG(LESS); } break; + case TokType::QE_OR_MORE: { LOG(QE_OR_MORE); } break; + case TokType::QE_OR_LESS: { LOG(QE_OR_LESS); } break; + case TokType::BOOL_NOT: { LOG(BOOL_NOT); } break; + case TokType::BOOL_AND: { LOG(BOOL_AND); } break; + case TokType::BOOL_OR: { LOG(BOOL_OR); } break; + case TokType::ADD: { LOG(ADD); } break; + case TokType::MUL: { LOG(MUL); } break; + case TokType::SUB: { LOG(SUB); } break; + case TokType::DIV: { LOG(DIV); } break; + case TokType::STM_END: { LOG(STM_END); } break; + case TokType::CONST_TRUE: { LOG(TRUE); } break; + case TokType::CONST_FALSE: { LOG(FALSE); } break; + case TokType::CONST_INT: { LOG(INT); } break; + case TokType::CONST_FLOAT: { LOG(FLOAT); } break; + case TokType::CONST_STRING: { LOG(STRING); } break; + case TokType::ID: { LOG(ID); } break; + //case TokType::COMMENT_LINE: { LOG(COMMENT_LINE) } break; + case TokType::COMMENT_BLOCK: { LOG(COMMENT_BLOCK) } break; + case TokType::FAILED: { LOG(FAILED) } break; + } + + } while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED); + +} + +int main(int argc, char* argv[]) { + + tp::ModuleManifest* ModuleDependencies[] = { &tp::gModuleTokenizer, NULL }; + tp::ModuleManifest TestModule("Test", NULL, NULL, ModuleDependencies); + if (!TestModule.initialize()) { + return 1; + } + + test(argc > 1 ? argv[1] : "rsc/script.txt"); + + TestModule.deinitialize(); +} \ No newline at end of file