Tokenizer Initial
This commit is contained in:
IlushaShurupov
Ilya Shurupov
parent
0ec44ee2d1
commit
1310bb78ad
11 changed files with 1848 additions and 3 deletions
22
Tokenizer/CMakeLists.txt
Normal file
22
Tokenizer/CMakeLists.txt
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
|
||||
cmake_minimum_required(VERSION 3.2)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 23)
|
||||
|
||||
project(Tokenizer)
|
||||
|
||||
### ---------------------- Static Library --------------------- ###
|
||||
file(GLOB SOURCES "./private/*.cpp")
|
||||
file(GLOB HEADERS "./public/*.hpp")
|
||||
add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
|
||||
target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
|
||||
target_link_libraries(${PROJECT_NAME} PUBLIC Strings Math)
|
||||
|
||||
### -------------------------- Tests -------------------------- ###
|
||||
enable_testing()
|
||||
file(GLOB TEST_SOURCES "./tests/*.cpp")
|
||||
add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES})
|
||||
target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils)
|
||||
add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests)
|
||||
|
||||
install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/${PROJECT_NAME}/lib)
|
||||
326
Tokenizer/private/CmdArgParser.cpp
Normal file
326
Tokenizer/private/CmdArgParser.cpp
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
|
||||
#include "allocators.h"
|
||||
|
||||
#include "CmdArgParser.h"
|
||||
|
||||
#include "log.h"
|
||||
|
||||
tp::CmdArgParser::Arg::Arg(const Arg& arg) {
|
||||
mId = arg.mType;
|
||||
switch (arg.mType) {
|
||||
case tp::CmdArgParser::Arg::INT: mType = INT; mInt = arg.mInt; break;
|
||||
case tp::CmdArgParser::Arg::FLOAT: mType = FLOAT; mFloat = arg.mFloat; break;
|
||||
case tp::CmdArgParser::Arg::BOOL: mType = BOOL; mBool = arg.mBool; break;
|
||||
case tp::CmdArgParser::Arg::STR: mType = STR; new (&mStr) StringArg(); mStr = arg.mStr; break;
|
||||
case tp::CmdArgParser::Arg::FILE_IN: mType = FILE_IN; new (&mFile) FileInputArg(); mFile = arg.mFile; break;
|
||||
}
|
||||
mOptional = arg.mOptional;
|
||||
}
|
||||
|
||||
tp::CmdArgParser::Arg::Arg(tp::string id, Type type) {
|
||||
mId = id;
|
||||
switch (type) {
|
||||
case tp::CmdArgParser::Arg::INT: mType = INT; break;
|
||||
case tp::CmdArgParser::Arg::FLOAT: mType = FLOAT; break;
|
||||
case tp::CmdArgParser::Arg::BOOL: mType = BOOL; break;
|
||||
case tp::CmdArgParser::Arg::STR: mType = STR; new (&mStr) StringArg(); break;
|
||||
case tp::CmdArgParser::Arg::FILE_IN: mType = FILE_IN; new (&mFile) FileInputArg(); break;
|
||||
}
|
||||
mOptional = false;
|
||||
}
|
||||
|
||||
tp::CmdArgParser::Arg::Arg(tp::string id, Range<alni> aAcceptingRange) {
|
||||
mId = id;
|
||||
mType = FLOAT;
|
||||
mInt = { 0, 0, aAcceptingRange };
|
||||
mOptional = false;
|
||||
}
|
||||
|
||||
tp::CmdArgParser::Arg::Arg(tp::string id, Range<alnf> aAcceptingRange) {
|
||||
mId = id;
|
||||
mType = FLOAT;
|
||||
mFloat = { 0, 0, aAcceptingRange };
|
||||
mOptional = false;
|
||||
}
|
||||
|
||||
tp::CmdArgParser::Arg::Arg(tp::string id, Range<alni> aAcceptingRange, alni aDefault) {
|
||||
mId = id;
|
||||
mType = INT;
|
||||
mInt = { 0, aDefault, aAcceptingRange };
|
||||
}
|
||||
|
||||
tp::CmdArgParser::Arg::Arg(tp::string id, Range<alnf> aAcceptingRange, alnf aDefault) {
|
||||
mId = id;
|
||||
mType = FLOAT;
|
||||
mFloat = { 0, aDefault, aAcceptingRange };
|
||||
}
|
||||
|
||||
tp::CmdArgParser::Arg::Arg(tp::string id, bool aDefault) {
|
||||
mId = id;
|
||||
mType = BOOL;
|
||||
mBool.mDefault = aDefault;
|
||||
}
|
||||
|
||||
tp::CmdArgParser::Arg::Arg(tp::string id, const char* aDefault) {
|
||||
mId = id;
|
||||
mType = STR;
|
||||
new (&mStr) StringArg();
|
||||
mStr.mDefault = aDefault;
|
||||
}
|
||||
|
||||
tp::CmdArgParser::Arg::~Arg() {
|
||||
switch (mType) {
|
||||
case tp::CmdArgParser::Arg::STR:
|
||||
mStr.~StringArg();
|
||||
break;
|
||||
case tp::CmdArgParser::Arg::FILE_IN:
|
||||
mFile.~FileInputArg();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
tp::CmdArgParser::CmdArgParser(tp::init_list<Arg> args) {
|
||||
bool optional_start = false;
|
||||
|
||||
for (auto& arg : args) {
|
||||
assert(!mArgs.presents(arg.mId) && "Argument Redefinition");
|
||||
|
||||
auto copy_arg = new Arg(arg);
|
||||
|
||||
mArgs.put(arg.mId, copy_arg);
|
||||
mArgsOrder.pushBack(mArgs.get(arg.mId));
|
||||
|
||||
if (arg.mOptional) {
|
||||
mOptionals++;
|
||||
}
|
||||
|
||||
if (optional_start) {
|
||||
assert(arg.mOptional && "Not Optional Argument After Optionals");
|
||||
}
|
||||
else if (arg.mOptional) {
|
||||
optional_start = true;
|
||||
}
|
||||
}
|
||||
|
||||
mTokenizer.build({
|
||||
{ "\n|\t| |\r", TokType::SPACE },
|
||||
{ "N|n|(False)|(false)", TokType::BOOL_FALSE },
|
||||
{ "Y|y|(True)|(true)", TokType::BOOL_TRUE },
|
||||
{ "((\\-)|(\\+))?[0-9]+", TokType::INT },
|
||||
{ "((\\-)|(\\+))?([0-9]+)(\\.)([0-9]*)?", TokType::FLOAT },
|
||||
{ "'{'-'}*'", TokType::STR },
|
||||
});
|
||||
|
||||
assert(mTokenizer.isBuild() && "Internal Error");
|
||||
}
|
||||
|
||||
tp::CmdArgParser::~CmdArgParser() {
|
||||
for (auto arg : mArgsOrder) {
|
||||
delete arg.data();
|
||||
}
|
||||
}
|
||||
|
||||
bool tp::CmdArgParser::parse(char argc, const char* argv[], bool logError) {
|
||||
// discard windows working directory argument
|
||||
argc--;
|
||||
|
||||
if (argc < mArgs.size() - mOptionals) {
|
||||
ErrInvalidArgCount();
|
||||
if (logError) {
|
||||
ErrLog();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
for (auto arg : mArgsOrder) {
|
||||
if (arg.idx() < argc) {
|
||||
parseArg(*arg.data(), argv[arg.idx() + 1]);
|
||||
}
|
||||
else {
|
||||
initDefault(*arg.data());
|
||||
}
|
||||
if (mError) {
|
||||
if (logError) {
|
||||
ErrLog();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
|
||||
//tp::set_working_dir();
|
||||
}
|
||||
|
||||
tp::alni tp::CmdArgParser::getInt(string id) { auto& arg = getArg(id, Arg::INT); return arg.mInt.mVal; }
|
||||
tp::alnf tp::CmdArgParser::getFloat(string id) { auto& arg = getArg(id, Arg::FLOAT); return arg.mFloat.mVal; }
|
||||
bool tp::CmdArgParser::getBool(string id) { auto& arg = getArg(id, Arg::BOOL); return arg.mBool.mFlag; }
|
||||
tp::string tp::CmdArgParser::getString(string id) { auto& arg = getArg(id, Arg::STR); return arg.mStr.mStr; }
|
||||
tp::File& tp::CmdArgParser::getFile(string id) { auto& arg = getArg(id, Arg::FILE_IN); return arg.mFile.mFile; }
|
||||
|
||||
|
||||
tp::CmdArgParser::Arg& tp::CmdArgParser::getArg(string id, Arg::Type type) {
|
||||
auto idx = mArgs.presents(id);
|
||||
assert(idx && "Invalid Id");
|
||||
auto& arg = mArgs.getSlotVal(idx);
|
||||
assert(arg->mType == type && "Invalid Type Requested");
|
||||
return *arg;
|
||||
}
|
||||
|
||||
void tp::CmdArgParser::ErrInvalidArgCount() { mError = { "Invalid Number Of Arguments Passed", nullptr }; }
|
||||
void tp::CmdArgParser::ErrInvalidArgSyntax(Arg* arg) { mError = { "Invalid Syntax Of Argument", arg }; }
|
||||
void tp::CmdArgParser::ErrInvalidArgType(Arg* arg) { mError = { "Invalid Type Of Argument", arg }; }
|
||||
void tp::CmdArgParser::ErrFileNotExists(Arg* arg) { mError = { "File Not Exists", arg }; }
|
||||
void tp::CmdArgParser::ErrFileCouldNotOpen(Arg* arg) { mError = { "Could Not Open File", arg }; }
|
||||
void tp::CmdArgParser::ErrValNotinRange(Arg* arg) { mError = { "Value Not In Range", arg }; }
|
||||
|
||||
void tp::CmdArgParser::ErrLog() {
|
||||
printf("Command Line Error: \n");
|
||||
|
||||
if (mError.mArg) {
|
||||
alni idx = 0;
|
||||
for (auto arg : mArgsOrder) {
|
||||
if (mError.mArg == arg.data()) {
|
||||
idx = arg.idx();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
printf("\nArgument : %lli\n", idx);
|
||||
ArgLog(*mError.mArg);
|
||||
}
|
||||
|
||||
printf("Error Description : %s. \n", mError.mDescr);
|
||||
|
||||
printf("\nExpected Arguments were : \n");
|
||||
|
||||
for (auto arg : mArgsOrder) {
|
||||
printf(" -- %lli -- \n", arg.idx());
|
||||
ArgLog(*arg.data());
|
||||
}
|
||||
}
|
||||
|
||||
void tp::CmdArgParser::ArgLog(Arg& arg) {
|
||||
switch (arg.mType) {
|
||||
case Arg::INT: {
|
||||
printf("Type : Int\n");
|
||||
printf("Range : %lli - %lli\n", arg.mInt.mAcceptingRange.mBegin, arg.mInt.mAcceptingRange.mEnd);
|
||||
if (arg.mOptional) {
|
||||
printf("Default : %lli\n", arg.mInt.mDefault);
|
||||
}
|
||||
printf("Given : %lli\n", arg.mInt.mVal);
|
||||
} break;
|
||||
case Arg::FLOAT: {
|
||||
printf("Type : Float\n");
|
||||
printf("Range : %f - %f\n", arg.mFloat.mAcceptingRange.mBegin, arg.mFloat.mAcceptingRange.mEnd);
|
||||
if (arg.mOptional) {
|
||||
printf("Default : %f\n", arg.mFloat.mDefault);
|
||||
}
|
||||
printf("Given : %f\n", arg.mFloat.mVal);
|
||||
} break;
|
||||
case Arg::BOOL: {
|
||||
printf("Type : Bool\n");
|
||||
if (arg.mOptional) {
|
||||
printf("Default : %s\n", arg.mBool.mDefault ? "True" : "False");
|
||||
}
|
||||
printf("Given : %s\n", arg.mBool.mFlag ? "True" : "False");
|
||||
} break;
|
||||
case Arg::STR: {
|
||||
printf("Type : String\n");
|
||||
if (arg.mOptional) {
|
||||
printf("Default : %s\n", arg.mStr.mDefault.cstr());
|
||||
}
|
||||
printf("Given : %s\n", arg.mStr.mStr.cstr());
|
||||
} break;
|
||||
case Arg::FILE_IN: {
|
||||
printf("Type : File Input\n");
|
||||
printf("Given Path : %s\n", arg.mFile.mFilepath.cstr());
|
||||
} break;
|
||||
}
|
||||
}
|
||||
|
||||
void tp::CmdArgParser::initDefault(Arg& arg) {
|
||||
switch (arg.mType) {
|
||||
case Arg::INT: arg.mInt.mVal = arg.mInt.mDefault; break;
|
||||
case Arg::FLOAT: arg.mFloat.mVal = arg.mFloat.mDefault; break;
|
||||
case Arg::BOOL: arg.mBool.mFlag = arg.mBool.mDefault; break;
|
||||
case Arg::STR: arg.mStr.mStr = arg.mStr.mDefault; break;
|
||||
}
|
||||
}
|
||||
|
||||
void tp::CmdArgParser::parseArg(Arg& arg, const char* src) {
|
||||
mTokenizer.reset();
|
||||
mTokenizer.bindSource(src);
|
||||
|
||||
auto tok = mTokenizer.readTok();
|
||||
if (tok < TokType::INT || tok > TokType::STR) {
|
||||
ErrInvalidArgSyntax(&arg);
|
||||
}
|
||||
auto val = mTokenizer.extractVal();
|
||||
|
||||
switch (arg.mType) {
|
||||
case Arg::INT: {
|
||||
if (tok != TokType::INT) {
|
||||
ErrInvalidArgType(&arg);
|
||||
return;
|
||||
}
|
||||
arg.mInt.mVal = val.operator tp::alni();
|
||||
if (arg.mInt.mVal < arg.mInt.mAcceptingRange.mBegin || arg.mInt.mVal > arg.mInt.mAcceptingRange.mEnd) {
|
||||
ErrValNotinRange(&arg);
|
||||
return;
|
||||
}
|
||||
} break;
|
||||
|
||||
case Arg::FLOAT: {
|
||||
if (tok != TokType::FLOAT) {
|
||||
ErrInvalidArgType(&arg);
|
||||
return;
|
||||
}
|
||||
arg.mFloat.mVal = val.operator tp::alnf();
|
||||
if (arg.mFloat.mVal < arg.mInt.mAcceptingRange.mBegin || arg.mFloat.mVal > arg.mInt.mAcceptingRange.mEnd) {
|
||||
ErrValNotinRange(&arg);
|
||||
return;
|
||||
}
|
||||
} break;
|
||||
|
||||
case Arg::STR: {
|
||||
if (tok != TokType::STR) {
|
||||
ErrInvalidArgType(&arg);
|
||||
return;
|
||||
}
|
||||
auto len = val.size();
|
||||
arg.mStr.mStr.reserve(len - 1);
|
||||
tp::memcp(arg.mStr.mStr.get_writable(), val.cstr() + 1, len - 2);
|
||||
arg.mStr.mStr.get_writable()[len - 2] = '\0';
|
||||
} break;
|
||||
|
||||
case Arg::BOOL: {
|
||||
if (tok != TokType::BOOL_FALSE && tok != TokType::BOOL_TRUE) {
|
||||
ErrInvalidArgType(&arg);
|
||||
return;
|
||||
}
|
||||
arg.mBool.mFlag = val.operator bool();
|
||||
} break;
|
||||
|
||||
case Arg::FILE_IN: {
|
||||
if (tok != TokType::STR) {
|
||||
ErrInvalidArgType(&arg);
|
||||
return;
|
||||
}
|
||||
|
||||
auto len = val.size();
|
||||
arg.mFile.mFilepath.reserve(len - 1);
|
||||
tp::memcp(arg.mFile.mFilepath.get_writable(), val.cstr() + 1, len - 2);
|
||||
arg.mFile.mFilepath.get_writable()[len - 2] = '\0';
|
||||
|
||||
if (!File::exists(arg.mFile.mFilepath.cstr())) {
|
||||
ErrFileNotExists(&arg);
|
||||
return;
|
||||
}
|
||||
arg.mFile.mFile.open(arg.mFile.mFilepath.cstr(), osfile_openflags::LOAD);
|
||||
if (!arg.mFile.mFile.opened) {
|
||||
ErrFileCouldNotOpen(&arg);
|
||||
return;
|
||||
}
|
||||
} break;
|
||||
}
|
||||
}
|
||||
15
Tokenizer/private/Tokenizer.cpp
Normal file
15
Tokenizer/private/Tokenizer.cpp
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
|
||||
#include "Tokenizer.hpp"
|
||||
|
||||
using namespace tp;
|
||||
|
||||
static ModuleManifest* sModuleDependencies[] = {
|
||||
&tp::gModuleMath,
|
||||
&tp::gModuleStrings,
|
||||
nullptr
|
||||
};
|
||||
|
||||
void deinit(const ModuleManifest*) {}
|
||||
|
||||
ModuleManifest tp::gModuleTokenizer = ModuleManifest("Tokenizer", nullptr, deinit, sModuleDependencies);
|
||||
|
||||
510
Tokenizer/public/AutomataGraph.h
Normal file
510
Tokenizer/public/AutomataGraph.h
Normal file
|
|
@ -0,0 +1,510 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "List.hpp"
|
||||
#include "Buffer.hpp"
|
||||
#include "Buffer2D.hpp"
|
||||
#include "Mat.hpp"
|
||||
#include "Map.hpp"
|
||||
|
||||
namespace tp {
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
class TransitionMatrix;
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
class DFA;
|
||||
|
||||
// Non-Deterministic Finite-State Automata
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
class NFA {
|
||||
static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable.");
|
||||
|
||||
public:
|
||||
struct Vertex;
|
||||
|
||||
private:
|
||||
struct Edge {
|
||||
Vertex* mVertex = nullptr;
|
||||
bool mConsumesSymbol = false;
|
||||
Range<tAlphabetType> mAcceptingRange;
|
||||
bool mAcceptsAll = false;
|
||||
bool mExclude = false;
|
||||
|
||||
bool isTransition(const tAlphabetType& symbol) {
|
||||
if (symbol == nullptr) return false;
|
||||
if (!mConsumesSymbol || mAcceptsAll) return true;
|
||||
bool const in_range = (symbol >= mAcceptingRange.mBegin && symbol <= mAcceptingRange.mEnd);
|
||||
return in_range != mExclude;
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
struct Vertex {
|
||||
List<Edge> edges;
|
||||
tStateType termination_state = tNoStateVal;
|
||||
#ifdef ENV_BUILD_DEBUG
|
||||
ualni debug_idx = 0;
|
||||
#endif
|
||||
ualni flag = 0;
|
||||
};
|
||||
|
||||
private:
|
||||
friend DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>;
|
||||
|
||||
List<Vertex> mVertices;
|
||||
Vertex* mStart = nullptr;
|
||||
|
||||
public:
|
||||
NFA() = default;
|
||||
|
||||
Vertex* addVertex() {
|
||||
auto node = mVertices.newNode();
|
||||
#ifdef ENV_BUILD_DEBUG
|
||||
node->data.debug_idx = mVertices.length() + 1;
|
||||
#endif
|
||||
mVertices.pushBack(node);
|
||||
return &node->data;
|
||||
}
|
||||
|
||||
void addTransition(Vertex* from, Vertex* to, Range<tAlphabetType> range, bool consumes, bool accepts_all, bool exclude) {
|
||||
Edge edge;
|
||||
edge.mVertex = to;
|
||||
edge.mConsumesSymbol = consumes;
|
||||
edge.mAcceptingRange = range;
|
||||
edge.mExclude = exclude;
|
||||
edge.mAcceptsAll = accepts_all;
|
||||
from->edges.pushBack(edge);
|
||||
}
|
||||
|
||||
void setStartVertex(Vertex* start) {
|
||||
mStart = start;
|
||||
}
|
||||
|
||||
[[nodiscard]] Vertex* getStartVertex() const {
|
||||
return mStart;
|
||||
}
|
||||
|
||||
void setVertexState(Vertex* vertex, tStateType state) {
|
||||
vertex->termination_state = state;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isValid() const {
|
||||
if (!mStart) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
Range<tAlphabetType> getAlphabetRange() const {
|
||||
tAlphabetType start = 0, end = 0;
|
||||
Range<tAlphabetType> all_range = { std::numeric_limits<tAlphabetType>::min(), std::numeric_limits<tAlphabetType>::max() };
|
||||
bool first = true;
|
||||
|
||||
for (auto vertex : mVertices) {
|
||||
for (auto edge : vertex.data().edges) {
|
||||
if (!edge.data().mConsumesSymbol) continue;
|
||||
auto const& tran_range = edge.data().mAcceptingRange;
|
||||
if (edge.data().mAcceptsAll || edge.data().mExclude) return all_range;
|
||||
if (tran_range.mBegin < start || first) start = tran_range.mBegin;
|
||||
if (tran_range.mEnd > end || first) end = tran_range.mEnd;
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
|
||||
return { start, end + 1 };
|
||||
}
|
||||
|
||||
// vertices that are reachable from initial set with no input consumption (E-transitions)
|
||||
// does not include initial set
|
||||
void closure(const List<Vertex*>& set, List<Vertex*>& closure, ualni unique_call_id) {
|
||||
List<Vertex*> marked;
|
||||
marked = set;
|
||||
|
||||
while (marked.length()) {
|
||||
auto first = marked.first()->data;
|
||||
if (first->flag != unique_call_id) {
|
||||
first->flag = unique_call_id;
|
||||
closure.pushBack(first);
|
||||
}
|
||||
for (auto edge : first->edges) {
|
||||
if (!edge.data().mConsumesSymbol && edge.data().mVertex->flag != unique_call_id) {
|
||||
marked.pushBack(edge.data().mVertex);
|
||||
}
|
||||
}
|
||||
marked.popFront();
|
||||
}
|
||||
}
|
||||
|
||||
// vertices that are reachable from initial set with symbol transition
|
||||
void move(const List<Vertex*>& set, List<Vertex*>& reachable, tAlphabetType symbol, ualni unique_call_id) {
|
||||
for (auto vertex : set) {
|
||||
for (auto edge : vertex->edges) {
|
||||
if (!edge.data().mConsumesSymbol) continue;
|
||||
bool transition = edge.data().isTransition(symbol);
|
||||
if (transition && edge.data().mVertex->flag != unique_call_id) {
|
||||
edge.data().mVertex->flag = unique_call_id;
|
||||
reachable.pushBack(edge.data().mVertex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Deterministic Finite-State Automata
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
class DFA {
|
||||
static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable.");
|
||||
friend TransitionMatrix<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>;
|
||||
|
||||
struct Vertex {
|
||||
struct Edge {
|
||||
Vertex* vertex = nullptr;
|
||||
tAlphabetType transition_code = nullptr;
|
||||
};
|
||||
|
||||
List<Edge> edges;
|
||||
tStateType termination_state = tNoStateVal;
|
||||
bool marked = false;
|
||||
};
|
||||
|
||||
List<Vertex> mVertices;
|
||||
Vertex* mStart = nullptr;
|
||||
const Vertex* mIter = nullptr;
|
||||
Range<tAlphabetType> mAlphabetRange;
|
||||
bool mTrapState = false;
|
||||
|
||||
typedef typename NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>::Vertex NState;
|
||||
|
||||
public:
|
||||
|
||||
explicit DFA(NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& nfa) {
|
||||
if (!nfa.isValid()) {
|
||||
return;
|
||||
}
|
||||
|
||||
mAlphabetRange = nfa.getAlphabetRange();
|
||||
|
||||
struct DStateKey {
|
||||
const List<NState*>* nStates;
|
||||
bool operator==(const DStateKey& in) {
|
||||
if (nStates->length() != in.nStates->length()) {
|
||||
return false;
|
||||
}
|
||||
// FIXME : make linear time
|
||||
for (auto state : *nStates) {
|
||||
bool found = false;
|
||||
for (auto in_state : *in.nStates) {
|
||||
if (state.data() == in_state.data()) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static ualni dStateHashFunc(DStateKey key) {
|
||||
alni out = 0;
|
||||
for (auto state : *key.nStates) {
|
||||
out += alni(state.data());
|
||||
}
|
||||
return out;
|
||||
};
|
||||
};
|
||||
|
||||
// all NFA states that are reachable from initial DFA State for specific symbol in alphabet
|
||||
struct DState {
|
||||
|
||||
struct DTransition {
|
||||
DState* state = nullptr;
|
||||
tAlphabetType accepting_code;
|
||||
};
|
||||
|
||||
List<NState*> nStates;
|
||||
List<DTransition> transitions;
|
||||
|
||||
Vertex* dVertex = nullptr; // relevant DFA vertex
|
||||
|
||||
#ifdef ENV_BUILD_DEBUG
|
||||
ualni debug_idx = 0;
|
||||
#endif
|
||||
};
|
||||
|
||||
// includes closure of NFA start state by definition
|
||||
auto start_state = new DState();
|
||||
nfa.closure({ nfa.getStartVertex() }, start_state->nstates, ualni(start_state));
|
||||
|
||||
Map<DStateKey, DState*, DefaultAllocator, DStateKey::dStateHashFunc, 256> dStates;
|
||||
|
||||
dStates.put({ &start_state->nstates }, start_state);
|
||||
|
||||
List<DState*> working_set = { start_state };
|
||||
|
||||
// while there is items to work with
|
||||
auto currentDState = working_set.first();
|
||||
while (currentDState) {
|
||||
|
||||
// check all possible transitions for any symbol
|
||||
for (auto symbol : mAlphabetRange) {
|
||||
|
||||
List<NState*> reachableNStates;
|
||||
|
||||
nfa.move(currentDState->data->nstates, reachableNStates, symbol, ualni(currentDState + symbol));
|
||||
nfa.closure(reachableNStates, reachableNStates, ualni(currentDState + symbol));
|
||||
|
||||
if (!reachableNStates.length()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
DState* targetDState = nullptr;
|
||||
|
||||
// check if set of all reachable NFA states already forms existing DFA state
|
||||
auto idx = dStates.presents({ &reachableNStates});
|
||||
if (idx) {
|
||||
targetDState = dStates.getSlotVal(idx);
|
||||
}
|
||||
|
||||
if (!targetDState) {
|
||||
// register new DFA state
|
||||
targetDState = new DState();
|
||||
|
||||
#ifdef ENV_BUILD_DEBUG
|
||||
targetDState->debug_idx = dStates.size();
|
||||
#endif
|
||||
|
||||
targetDState->nstates = reachableNStates;
|
||||
|
||||
// append to working stack
|
||||
working_set.pushBack(targetDState);
|
||||
dStates.put({ &targetDState->nstates }, targetDState);
|
||||
}
|
||||
|
||||
// add transition to DFA state
|
||||
currentDState->data->transitions.pushBack({targetDState, symbol });
|
||||
}
|
||||
|
||||
working_set.popFront();
|
||||
currentDState = working_set.first();
|
||||
}
|
||||
|
||||
// create own vertices
|
||||
for (auto node : dStates) {
|
||||
tStateType state = tNoStateVal;
|
||||
for (auto iter : node->val->nStates) {
|
||||
if (iter->termination_state != tNoStateVal) {
|
||||
state = iter->termination_state;
|
||||
break;
|
||||
}
|
||||
}
|
||||
node->val->dvertex = addVertex(state);
|
||||
}
|
||||
|
||||
// connect all vertices
|
||||
for (auto node : dStates) {
|
||||
for (auto edge : node->val->transitions) {
|
||||
addTransition(node->val->dvertex, edge.data().state->dvertex, edge.data().accepting_code);
|
||||
}
|
||||
}
|
||||
|
||||
// set the starting vertex
|
||||
mStart = start_state->dvertex;
|
||||
|
||||
// cleanup
|
||||
for (auto node : dStates) {
|
||||
delete node->val;
|
||||
}
|
||||
|
||||
collapseEquivalentVertices();
|
||||
mAlphabetRange = getAlphabetRange();
|
||||
}
|
||||
|
||||
tStateType move(tAlphabetType symbol) {
|
||||
if (mTrapState || !mIter) {
|
||||
return tNoStateVal;
|
||||
}
|
||||
|
||||
for (auto edge : mIter->edges) {
|
||||
if (edge.data().transition_code == symbol) {
|
||||
mIter = edge.data().vertex;
|
||||
return mIter->termination_state;
|
||||
}
|
||||
}
|
||||
|
||||
mTrapState = true;
|
||||
return tNoStateVal;
|
||||
}
|
||||
|
||||
void start() {
|
||||
mIter = mStart;
|
||||
mTrapState = false;
|
||||
}
|
||||
|
||||
[[nodiscard]] uhalni nVertices() const {
|
||||
return (uhalni) mVertices.length();
|
||||
}
|
||||
|
||||
[[nodiscard]] Range<tAlphabetType> getRange() const {
|
||||
return mAlphabetRange;
|
||||
}
|
||||
|
||||
private:
|
||||
[[nodiscard]] Range<tAlphabetType> getAlphabetRange() const {
|
||||
Range<tAlphabetType> out;
|
||||
for (auto vertex : mVertices) {
|
||||
vertex.data().marked = false;
|
||||
}
|
||||
|
||||
bool first = true;
|
||||
getAlphabetRangeUtil(mStart, &out, first);
|
||||
out.mEnd++;
|
||||
return out;
|
||||
}
|
||||
|
||||
void getAlphabetRangeUtil(Vertex* vert, Range<tAlphabetType>* out, bool& first) const {
|
||||
vert->marked = true;
|
||||
for (auto edge : vert->edges) {
|
||||
auto const code = edge.data().transition_code;
|
||||
|
||||
if (first) {
|
||||
*out = { code, code };
|
||||
first = false;
|
||||
}
|
||||
|
||||
if (code < out->mBegin) {
|
||||
out->mBegin = code;
|
||||
}
|
||||
if (code > out->mEnd) {
|
||||
out->mEnd = code;
|
||||
}
|
||||
|
||||
if (!edge.data().vertex->marked) {
|
||||
getAlphabetRangeUtil(edge.data().vertex, out, first);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void collapseEquivalentVertices() {
|
||||
// TODO
|
||||
}
|
||||
|
||||
Vertex* addVertex(tStateType state) {
|
||||
auto node = mVertices.addNodeBack();
|
||||
node->data.termination_state = state;
|
||||
return &node->data;
|
||||
}
|
||||
|
||||
void addTransition(Vertex* from, Vertex* to, tAlphabetType transition_symbol) {
|
||||
from->edges.pushBack({ to, transition_symbol });
|
||||
}
|
||||
};
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
class TransitionMatrix {
|
||||
|
||||
static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable.");
|
||||
|
||||
Buffer2D<ualni> mTransitions{};
|
||||
Buffer<tStateType> mStates{};
|
||||
Range<tAlphabetType> mSymbolRange = nullptr;
|
||||
|
||||
ualni mIter = 0;
|
||||
ualni mIterPrev = 0;
|
||||
ualni mStart = 0;
|
||||
|
||||
public:
|
||||
|
||||
TransitionMatrix() = default;
|
||||
|
||||
void construct(const DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& dfa) {
|
||||
mSymbolRange = dfa.getRange();
|
||||
auto range_len = uhalni(mSymbolRange.mEnd - mSymbolRange.mBegin);
|
||||
Vec2<uhalni> dim = { range_len ? range_len : 1, (uhalni) (dfa.nVertices() + 1) };
|
||||
|
||||
mTransitions.reserve(dim);
|
||||
mTransitions.assign(dfa.nVertices());
|
||||
mStates.reserve(dim.y);
|
||||
|
||||
for (auto vertex : dfa.mVertices) {
|
||||
auto state = vertex.data().termination_state;
|
||||
mStates[vertex.idx()] = state;
|
||||
}
|
||||
|
||||
mStates[dfa.nVertices()] = tFailedStateVal;
|
||||
|
||||
for (auto vertex : dfa.mVertices) {
|
||||
if (&vertex.data() == dfa.mStart) {
|
||||
mStart = mIter = mIterPrev = vertex.idx();
|
||||
}
|
||||
}
|
||||
|
||||
for (auto vertex : dfa.mVertices) {
|
||||
for (auto edge : vertex.data().edges) {
|
||||
ualni idx = 0;
|
||||
for (auto vertex : dfa.mVertices) {
|
||||
if (edge.data().vertex == &vertex.data()) {
|
||||
idx = vertex.idx();
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto const code = edge.data().transition_code;
|
||||
mTransitions.set(code - mSymbolRange.mBegin, (uhalni) vertex.idx(), idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool isTrapped() {
|
||||
return mStates[mIter] == tFailedStateVal;
|
||||
}
|
||||
|
||||
tStateType move(tAlphabetType symbol) {
|
||||
if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) {
|
||||
mIter = mTransitions.get((uhalni) (symbol - mSymbolRange.mBegin), (uhalni) mIter);
|
||||
}
|
||||
else {
|
||||
mIter = mStates.length() - 1;
|
||||
}
|
||||
|
||||
if (mIterPrev == mStart) {
|
||||
if (mStates[mIter] == tFailedStateVal) {
|
||||
reset();
|
||||
return tFailedStateVal;
|
||||
}
|
||||
else {
|
||||
mIterPrev = mIter;
|
||||
return tNoStateVal;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (mStates[mIter] == tFailedStateVal) {
|
||||
if (mStates[mIterPrev] != tNoStateVal) {
|
||||
auto out = mStates[mIterPrev];
|
||||
reset();
|
||||
return out;
|
||||
}
|
||||
else {
|
||||
reset();
|
||||
return tFailedStateVal;
|
||||
}
|
||||
}
|
||||
else {
|
||||
mIterPrev = mIter;
|
||||
return tNoStateVal;
|
||||
}
|
||||
}
|
||||
|
||||
mIterPrev = mIter;
|
||||
return mStates[mIter];
|
||||
}
|
||||
|
||||
void reset() {
|
||||
mIter = mStart;
|
||||
mIterPrev = mStart;
|
||||
}
|
||||
};
|
||||
}
|
||||
97
Tokenizer/public/CmdArgParser.h
Normal file
97
Tokenizer/public/CmdArgParser.h
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
#pragma once
|
||||
|
||||
#include "Tokenizer.hpp"
|
||||
|
||||
namespace tp {
|
||||
struct CmdArgParser {
|
||||
|
||||
struct IntArg {
|
||||
alni mVal = 0;
|
||||
alni mDefault = 0;
|
||||
Range<alni> mAcceptingRange = { ENV_ALNI_MIN, ENV_ALNI_MAX };
|
||||
};
|
||||
|
||||
struct FloatArg {
|
||||
alnf mVal = 0.f;
|
||||
alnf mDefault = 0.f;
|
||||
Range<alnf> mAcceptingRange = { ENV_ALNF_MIN, ENV_ALNF_MAX };
|
||||
};
|
||||
|
||||
struct BoolArg {
|
||||
bool mFlag = false;
|
||||
bool mDefault = false;
|
||||
};
|
||||
|
||||
struct StringArg {
|
||||
String mStr;
|
||||
String mDefault;
|
||||
};
|
||||
|
||||
struct FileInputArg {
|
||||
String mFilepath;
|
||||
File mFile;
|
||||
};
|
||||
|
||||
struct Arg {
|
||||
string mId;
|
||||
enum Type { INT, FLOAT, BOOL, STR, FILE_IN } mType;
|
||||
union {
|
||||
IntArg mInt;
|
||||
FloatArg mFloat;
|
||||
BoolArg mBool;
|
||||
StringArg mStr;
|
||||
FileInputArg mFile;
|
||||
};
|
||||
|
||||
bool mOptional = true;
|
||||
|
||||
Arg(const Arg& arg);
|
||||
Arg(string id, Type type);
|
||||
Arg(string id, Range<alni> aAcceptingRange);
|
||||
Arg(string id, Range<alnf> aAcceptingRange);
|
||||
Arg(string id, Range<alni> aAcceptingRange, alni aDefault);
|
||||
Arg(string id, Range<alnf> aAcceptingRange, alnf aDefault);
|
||||
Arg(string id, bool aDefault);
|
||||
Arg(string id, const char* aDefault);
|
||||
~Arg();
|
||||
};
|
||||
|
||||
struct Error {
|
||||
const char* mDescr = nullptr;
|
||||
Arg* mArg = nullptr;
|
||||
operator bool() { return mDescr != nullptr; }
|
||||
} mError;
|
||||
|
||||
CmdArgParser(init_list<Arg> args);
|
||||
~CmdArgParser();
|
||||
|
||||
bool parse(char argc, const char* argv[], bool logError = false);
|
||||
|
||||
alni getInt(string id);
|
||||
alnf getFloat(string id);
|
||||
bool getBool(string id);
|
||||
string getString(string id);
|
||||
File& getFile(string id);
|
||||
|
||||
private:
|
||||
enum class TokType { SPACE, INT, FLOAT, BOOL_FALSE, BOOL_TRUE, STR, NONE, FAILURE, END, } mType;
|
||||
typedef SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILURE, TokType::END> Tokenizer;
|
||||
|
||||
Tokenizer mTokenizer;
|
||||
HashMap<Arg*, string> mArgs;
|
||||
List<Arg*> mArgsOrder;
|
||||
ualni mOptionals = 0;
|
||||
|
||||
Arg& getArg(string id, Arg::Type type);
|
||||
void ErrInvalidArgCount();
|
||||
void ErrInvalidArgSyntax(Arg* arg);
|
||||
void ErrInvalidArgType(Arg* arg);
|
||||
void ErrFileNotExists(Arg* arg);
|
||||
void ErrFileCouldNotOpen(Arg* arg);
|
||||
void ErrValNotinRange(Arg* arg);
|
||||
void ErrLog();
|
||||
void ArgLog(Arg& arg);
|
||||
void initDefault(Arg& arg);
|
||||
void parseArg(Arg& arg, const char* src);
|
||||
};
|
||||
};
|
||||
498
Tokenizer/public/RegularExpression.h
Normal file
498
Tokenizer/public/RegularExpression.h
Normal file
|
|
@ -0,0 +1,498 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "AutomataGraph.h"
|
||||
|
||||
namespace tp::RegEx {
|
||||
|
||||
struct AstNode {
|
||||
enum Type {
|
||||
NONE,
|
||||
ANY,
|
||||
OR,
|
||||
IF,
|
||||
CLASS,
|
||||
COMPOUND,
|
||||
REPEAT,
|
||||
VAL,
|
||||
} mType = NONE;
|
||||
};
|
||||
|
||||
template <typename tAlphabetType>
|
||||
struct AstVal : public AstNode {
|
||||
explicit AstVal(tAlphabetType val) : mVal(val) { mType = VAL; }
|
||||
tAlphabetType mVal;
|
||||
};
|
||||
|
||||
struct AstCompound : public AstNode {
|
||||
AstCompound() { mType = COMPOUND; }
|
||||
List<AstNode*> mChilds;
|
||||
};
|
||||
|
||||
struct AstAlternation : public AstNode {
|
||||
AstAlternation() { mType = OR; }
|
||||
AstNode* mFirst = nullptr;
|
||||
AstNode* mSecond = nullptr;
|
||||
};
|
||||
|
||||
struct AstIf : public AstNode {
|
||||
AstIf() { mType = IF; }
|
||||
AstNode* mNode = nullptr;
|
||||
};
|
||||
|
||||
struct AstAny : public AstNode {
|
||||
AstAny() { mType = ANY; }
|
||||
};
|
||||
|
||||
struct AstRepetition : public AstNode {
|
||||
AstRepetition() { mType = REPEAT; }
|
||||
AstNode* mNode = nullptr;
|
||||
bool mPlus = false;
|
||||
};
|
||||
|
||||
template <typename tAlphabetType>
|
||||
struct AstClass : public AstNode {
|
||||
AstClass() { mType = CLASS; }
|
||||
List<Range<tAlphabetType>> mRanges;
|
||||
bool mExclude{};
|
||||
};
|
||||
|
||||
struct ParseError {
|
||||
const char* description = nullptr;
|
||||
uhalni offset = 0;
|
||||
[[nodiscard]] bool isError() const { return description != nullptr; }
|
||||
};
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal>
|
||||
class Parser {
|
||||
|
||||
enum TokType : uint1 {
|
||||
TOK_COMPOUND_START = 0,
|
||||
TOK_COMPOUND_END,
|
||||
TOK_CLASS_START,
|
||||
TOK_CLASS_END,
|
||||
TOK_CLASS_START_EXCLUDE,
|
||||
TOK_CLASS_END_EXCLUDE,
|
||||
TOK_OR,
|
||||
TOK_IF,
|
||||
TOK_ANY,
|
||||
TOK_REPEAT,
|
||||
TOK_REPEAT_PLUS,
|
||||
TOK_HYPHEN,
|
||||
TOK_SPECIALS_END_,
|
||||
TOK_VAL,
|
||||
TOK_NONE,
|
||||
};
|
||||
|
||||
tAlphabetType SpecialSymbols[TOK_SPECIALS_END_] = {
|
||||
'(', ')', '[', ']', '{', '}', '|', '?', '.', '*', '+', '-',
|
||||
};
|
||||
|
||||
tAlphabetType mEscapeSymbol = '\\';
|
||||
|
||||
struct Token {
|
||||
TokType type;
|
||||
tAlphabetType val;
|
||||
};
|
||||
|
||||
const tAlphabetType* mSource = nullptr;
|
||||
uhalni mOffset = 0;
|
||||
Token mCurToken;
|
||||
uhalni mTokLength = 0;
|
||||
|
||||
public:
|
||||
|
||||
ParseError mError;
|
||||
|
||||
// regular expression must be a zero termination string
|
||||
AstCompound* parse(const tAlphabetType* regex) {
|
||||
mSource = regex;
|
||||
return parseRegEx();
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
AstCompound* parseRegEx() {
|
||||
auto out = new AstCompound();
|
||||
for (AstNode* node = parseElement(); node; node = parseElement()) {
|
||||
out->mChilds.pushBack(node);
|
||||
}
|
||||
if (!out->mChilds.length()) {
|
||||
genError("Expected A Expression");
|
||||
}
|
||||
if (mError.description) {
|
||||
delete out;
|
||||
return nullptr;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
AstNode* parseElement() {
|
||||
AstNode* out = nullptr;
|
||||
switch (readTok().type) {
|
||||
case TOK_COMPOUND_START: out = parseCompound(); break;
|
||||
case TOK_CLASS_START: out = parseClass(); break;
|
||||
case TOK_CLASS_START_EXCLUDE: out = parseClass(true); break;
|
||||
case TOK_ANY: out = parseAny(); break;
|
||||
case TOK_VAL: out = parseVal(); break;
|
||||
case TOK_NONE: { discardTok(); return nullptr; };
|
||||
}
|
||||
if (!out) {
|
||||
discardTok();
|
||||
return nullptr;
|
||||
}
|
||||
switch (readTok().type) {
|
||||
case TOK_OR: out = parseAlternation(out); break;
|
||||
case TOK_REPEAT: out = parseRepetition(out); break;
|
||||
case TOK_REPEAT_PLUS: out = parseRepetition(out, true); break;
|
||||
case TOK_IF: out = parseIf(out); break;
|
||||
case TOK_NONE: break;
|
||||
default: { discardTok(); }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
AstCompound* parseCompound() {
|
||||
auto out = new AstCompound();
|
||||
for (AstNode* node = parseElement(); node; node = parseElement()) {
|
||||
out->mChilds.pushBack(node);
|
||||
}
|
||||
if (readTok().type != TOK_COMPOUND_END) {
|
||||
genError("Expected Compound End");
|
||||
}
|
||||
if (mError.description) {
|
||||
delete out;
|
||||
return nullptr;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
AstClass<tAlphabetType>* parseClass(bool exclude = false) {
|
||||
auto out = new AstClass<tAlphabetType>();
|
||||
out->mExclude = exclude;
|
||||
auto& ranges = out->mRanges;
|
||||
|
||||
readTok();
|
||||
|
||||
READ_VAL:
|
||||
if (mCurToken.type != TOK_VAL) {
|
||||
delete out;
|
||||
genError("Expected A Value");
|
||||
return nullptr;
|
||||
}
|
||||
char range_start = mCurToken.val;
|
||||
|
||||
readTok();
|
||||
if (mCurToken.type != TOK_HYPHEN) {
|
||||
delete out;
|
||||
genError("Expected A Range");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
readTok();
|
||||
if (mCurToken.type != TOK_VAL) {
|
||||
delete out;
|
||||
genError("Expected A Value");
|
||||
return nullptr;
|
||||
}
|
||||
char range_end = mCurToken.val;
|
||||
|
||||
ranges.pushBack({ range_start, range_end });
|
||||
|
||||
readTok();
|
||||
if ((mCurToken.type == TOK_CLASS_END && !exclude) || (mCurToken.type == TOK_CLASS_END_EXCLUDE && exclude)) {
|
||||
return out;
|
||||
}
|
||||
else {
|
||||
goto READ_VAL;
|
||||
}
|
||||
}
|
||||
|
||||
AstAny* parseAny() {
|
||||
return new AstAny();
|
||||
}
|
||||
|
||||
AstVal<tAlphabetType>* parseVal() {
|
||||
auto out = new AstVal<tAlphabetType>(mCurToken.val);
|
||||
return out;
|
||||
}
|
||||
|
||||
AstAlternation* parseAlternation(AstNode* left) {
|
||||
auto right = parseElement();
|
||||
if (!right) {
|
||||
genError("Expected Alternation right Side");
|
||||
delete left;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto out = new AstAlternation();
|
||||
|
||||
out->mFirst = left;
|
||||
out->mSecond = right;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
AstRepetition* parseRepetition(AstNode* left, bool plus = false) {
|
||||
auto out = new AstRepetition();
|
||||
out->mNode = left;
|
||||
out->mPlus = plus;
|
||||
return out;
|
||||
}
|
||||
|
||||
AstIf* parseIf(AstNode* left) {
|
||||
auto out = new AstIf();
|
||||
out->mNode = left;
|
||||
return out;
|
||||
}
|
||||
|
||||
void genError(const char* desc) {
|
||||
mError = { desc, mOffset };
|
||||
}
|
||||
|
||||
Token& readTok() {
|
||||
|
||||
const tAlphabetType* crs = mSource + mOffset;
|
||||
|
||||
// zero termination string
|
||||
if (*crs == 0) {
|
||||
mCurToken.type = TOK_NONE;
|
||||
return mCurToken;
|
||||
}
|
||||
|
||||
mTokLength = 1;
|
||||
mCurToken.type = TOK_VAL;
|
||||
mCurToken.val = crs[0];
|
||||
|
||||
if (crs[0] == mEscapeSymbol) {
|
||||
mCurToken.val = crs[1];
|
||||
mTokLength = 2;
|
||||
}
|
||||
else {
|
||||
for (uhalni tok = 0; tok < TOK_SPECIALS_END_; tok++) {
|
||||
if (SpecialSymbols[tok] == mCurToken.val) {
|
||||
mCurToken.type = TokType(tok);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mOffset += mTokLength;
|
||||
return mCurToken;
|
||||
}
|
||||
|
||||
void discardTok() {
|
||||
mOffset -= mTokLength;
|
||||
mTokLength = 0;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename tStateType>
|
||||
struct CompileError {
|
||||
ParseError mParseError;
|
||||
uhalni mRuleIndex = 0;
|
||||
tStateType mRuleState;
|
||||
const char* description = nullptr;
|
||||
[[nodiscard]] bool isError() const { return description; }
|
||||
};
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
class Compiler {
|
||||
|
||||
typedef NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> Graph;
|
||||
typedef typename Graph::Vertex Vertex;
|
||||
typedef Parser<tAlphabetType, tStateType, tNoStateVal> Parser;
|
||||
|
||||
struct Node {
|
||||
Vertex* left = nullptr;
|
||||
Vertex* right = nullptr;
|
||||
};
|
||||
|
||||
private:
|
||||
Graph* mGraph = nullptr;
|
||||
|
||||
public:
|
||||
CompileError<tStateType> mError;
|
||||
|
||||
Node compile(Graph& graph, const tAlphabetType* regex, tStateType state) {
|
||||
mGraph = &graph;
|
||||
return compileUtil(regex, state);
|
||||
}
|
||||
|
||||
Node compile(Graph& aGraph, init_list<Pair<const tAlphabetType*, tStateType>> aRules) {
|
||||
mGraph = &aGraph;
|
||||
|
||||
auto left = mGraph->addVertex();
|
||||
auto right = mGraph->addVertex();
|
||||
|
||||
halni idx = 0;
|
||||
for (auto rule: aRules) {
|
||||
|
||||
auto node = idx ? compileUtil(rule.head, rule.tail) : compileUtil(rule.head, rule.tail, left, right);
|
||||
|
||||
if (!(node.left && node.right)) {
|
||||
mError.mRuleIndex = idx;
|
||||
return {};
|
||||
}
|
||||
|
||||
if (idx) {
|
||||
transitionAny(left, node.left);
|
||||
transitionAny(node.right, right);
|
||||
}
|
||||
|
||||
idx++;
|
||||
}
|
||||
|
||||
mGraph->setStartVertex(left);
|
||||
|
||||
return { left, right };
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
Node compileUtil(const tAlphabetType* regex, tStateType state, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
|
||||
Parser parser;
|
||||
auto astNode = parser.parse(regex);
|
||||
if (parser.mError.isError()) {
|
||||
mGraph->setStartVertex(nullptr);
|
||||
mError.description = "Parsing Of Regular Expression Failed";
|
||||
mError.mRuleState = state;
|
||||
mError.mParseError = parser.mError;
|
||||
return {};
|
||||
}
|
||||
|
||||
auto node = compileNode(astNode, aLeft, aRight);
|
||||
delete astNode;
|
||||
|
||||
mGraph->setVertexState(node.right, state);
|
||||
mGraph->setStartVertex(node.left);
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
Node compileVal(AstVal<tAlphabetType>* val, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
|
||||
auto left = aLeft ? aLeft : mGraph->addVertex();
|
||||
auto right = aRight ? aRight : mGraph->addVertex();
|
||||
transitionVal(left, right, val->mVal);
|
||||
return { left, right };
|
||||
}
|
||||
|
||||
Node compileAlternation(AstAlternation* alt, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
|
||||
auto first_node = compileNode(alt->mFirst, aLeft, aRight);
|
||||
auto second_node = compileNode(alt->mSecond);
|
||||
transitionAny(first_node.left, second_node.left);
|
||||
transitionAny(second_node.right, first_node.right);
|
||||
return first_node;
|
||||
}
|
||||
|
||||
Node compileAny(AstAny*, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
|
||||
auto left = aLeft ? aLeft : mGraph->addVertex();
|
||||
auto right = aRight ? aRight : mGraph->addVertex();
|
||||
transitionAny(left, right, true);
|
||||
return { left, right };
|
||||
}
|
||||
|
||||
Node compileRepeat(AstRepetition* repeat, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
|
||||
if (repeat->mPlus) {
|
||||
auto middle = mGraph->addVertex();
|
||||
|
||||
auto left_node = compileNode(repeat->mNode, aLeft, middle);
|
||||
|
||||
auto right_node = compileNode(repeat->mNode, middle, aRight);
|
||||
transitionAny(right_node.right, right_node.left);
|
||||
transitionAny(right_node.left, right_node.right);
|
||||
|
||||
return { left_node.left, right_node.right };
|
||||
}
|
||||
else {
|
||||
auto node = compileNode(repeat->mNode, aLeft, aRight);
|
||||
transitionAny(node.right, node.left);
|
||||
transitionAny(node.left, node.right);
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
||||
Node compileIf(AstIf* ifNode, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
|
||||
auto node = compileNode(ifNode->mNode, aLeft, aRight);
|
||||
transitionAny(node.left, node.right);
|
||||
return node;
|
||||
}
|
||||
|
||||
Node compileClass(AstClass<tAlphabetType>* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
|
||||
auto left = aLeft ? aLeft : mGraph->addVertex();
|
||||
auto right = aRight ? aRight : mGraph->addVertex();
|
||||
|
||||
if (node->mRanges.length() == 1) {
|
||||
auto const& range = node->mRanges.first()->data;
|
||||
transitionRange(left, right, { range.mBegin, range.mEnd }, node->mExclude);
|
||||
return { left, right };
|
||||
}
|
||||
|
||||
for (auto range : node->mRanges) {
|
||||
auto middle = mGraph->addVertex();
|
||||
transitionRange(left, middle, { range.data().mBegin, range.data().mEnd }, node->mExclude);
|
||||
transitionAny(middle, right);
|
||||
}
|
||||
return { left, right };
|
||||
}
|
||||
|
||||
Node compileCompound(AstCompound* compound, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
|
||||
Vertex* left = nullptr;
|
||||
Vertex* rigth = nullptr;
|
||||
|
||||
ualni idx = 0;
|
||||
for (auto child : compound->mChilds) {
|
||||
auto pass_left = idx == 0 ? aLeft : rigth;
|
||||
auto pass_right = idx == compound->mChilds.length() - 1 ? aRight : nullptr;
|
||||
auto node = compileNode(child.data(), pass_left, pass_right);
|
||||
if (!left) left = node.left;
|
||||
rigth = node.right;
|
||||
idx++;
|
||||
}
|
||||
|
||||
return { left, rigth };
|
||||
}
|
||||
|
||||
Node compileNode(AstNode* node, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
|
||||
switch (node->mType) {
|
||||
case AstNode::CLASS: return compileClass((AstClass<tAlphabetType>*)node, aLeft, aRight);
|
||||
case AstNode::COMPOUND: return compileCompound((AstCompound*)node, aLeft, aRight);
|
||||
case AstNode::IF: return compileIf((AstIf*)node, aLeft, aRight);
|
||||
case AstNode::REPEAT: return compileRepeat((AstRepetition*)node, aLeft, aRight);
|
||||
case AstNode::ANY: return compileAny((AstAny*)node, aLeft, aRight);
|
||||
case AstNode::OR: return compileAlternation((AstAlternation*)node, aLeft, aRight);
|
||||
case AstNode::VAL: return compileVal((AstVal<tAlphabetType>*)node, aLeft, aRight);
|
||||
case AstNode::NONE:
|
||||
break;
|
||||
}
|
||||
ASSERT(0)
|
||||
return {};
|
||||
}
|
||||
|
||||
void transitionAny(Vertex* from, Vertex* to, bool consumes = false) {
|
||||
mGraph->addTransition(from, to, {}, consumes, true, false);
|
||||
}
|
||||
|
||||
void transitionVal(Vertex* from, Vertex* to, tAlphabetType val) {
|
||||
mGraph->addTransition(from, to, { val, val }, true, false, false);
|
||||
}
|
||||
|
||||
void transitionRange(Vertex* from, Vertex* to, Range<tAlphabetType> range, bool exclude) {
|
||||
mGraph->addTransition(from, to, range, true, false, exclude);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
CompileError<tStateType> compile(NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& out, const tAlphabetType* regex, tStateType state) {
|
||||
Compiler<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> compiler;
|
||||
compiler.compile(out, regex, state);
|
||||
return compiler.mError;
|
||||
}
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
CompileError<tStateType> compile(NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& out, const init_list<Pair<const tAlphabetType*, tStateType>>& rules) {
|
||||
Compiler<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> compiler;
|
||||
compiler.compile(out, rules);
|
||||
return compiler.mError;
|
||||
}
|
||||
}
|
||||
176
Tokenizer/public/Tokenizer.hpp
Normal file
176
Tokenizer/public/Tokenizer.hpp
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "RegularExpression.h"
|
||||
#include "Strings.hpp"
|
||||
|
||||
namespace tp {
|
||||
|
||||
extern ModuleManifest gModuleTokenizer;
|
||||
|
||||
template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal>
|
||||
class Tokenizer {
|
||||
|
||||
TransitionMatrix<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> mTransitionMatrix;
|
||||
|
||||
RegEx::CompileError<tTokType> mError;
|
||||
|
||||
bool scanFailed() {
|
||||
return mTransitionMatrix.isTrapped();
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
Tokenizer() {
|
||||
MODULE_SANITY_CHECK(gModuleTokenizer)
|
||||
}
|
||||
|
||||
void build(const init_list<Pair<const tAlphabetType*, tTokType>>& rules) {
|
||||
NFA<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> nfa;
|
||||
|
||||
mError = RegEx::compile(nfa, rules);
|
||||
if (mError.isError()) {
|
||||
return;
|
||||
}
|
||||
|
||||
DFA<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> dfa(nfa);
|
||||
mTransitionMatrix.construct(dfa);
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isBuild() const {
|
||||
return !mError.isError();
|
||||
}
|
||||
|
||||
const RegEx::CompileError<tTokType>& getBuildError() {
|
||||
return mError;
|
||||
}
|
||||
|
||||
void resetMatrix() {
|
||||
mTransitionMatrix.reset();
|
||||
}
|
||||
|
||||
tTokType advanceSymbol(tAlphabetType symbol) {
|
||||
return mTransitionMatrix.move(symbol);
|
||||
}
|
||||
|
||||
tTokType advanceToken(const tAlphabetType* source, ualni source_len, ualni* token_len) {
|
||||
tTokType out = tNoTokVal;
|
||||
*token_len = 0;
|
||||
for (ualni idx = 0; idx < source_len; idx++) {
|
||||
out = advanceSymbol(source[idx]);
|
||||
if (out != tNoTokVal) {
|
||||
*token_len = idx;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
~Tokenizer() = default;
|
||||
};
|
||||
|
||||
template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal, tTokType tSourceEndTokVal>
|
||||
class SimpleTokenizer {
|
||||
|
||||
Tokenizer<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> mTokenizer;
|
||||
|
||||
const tAlphabetType* mSource = nullptr;
|
||||
ualni mLastTokLen = 0;
|
||||
ualni mSourceLen = 0;
|
||||
ualni mAdvancedOffset = 0;
|
||||
|
||||
public:
|
||||
|
||||
struct Cursor {
|
||||
const tAlphabetType* mSource = nullptr;
|
||||
ualni mAdvancedOffset = 0;
|
||||
const tAlphabetType* str() { return mSource + mAdvancedOffset; }
|
||||
};
|
||||
|
||||
SimpleTokenizer() = default;
|
||||
|
||||
void build(const init_list<Pair<const tAlphabetType*, tTokType>>& rules) {
|
||||
mTokenizer.build(rules);
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isBuild() const {
|
||||
return mTokenizer.isBuild();
|
||||
}
|
||||
|
||||
const RegEx::CompileError<tTokType>& getBuildError() {
|
||||
return mTokenizer.getBuildError();
|
||||
}
|
||||
|
||||
void bindSource(const tAlphabetType* source) {
|
||||
mSource = source;
|
||||
while (mSource[mSourceLen]) mSourceLen++;
|
||||
mSourceLen++;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isInputLeft() const {
|
||||
if (!mSource) {
|
||||
return false;
|
||||
}
|
||||
if (mSource[mAdvancedOffset] == nullptr) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
Cursor getCursor() const {
|
||||
return { mSource, mAdvancedOffset };
|
||||
}
|
||||
|
||||
Cursor getCursorPrev() const {
|
||||
return { mSource, mAdvancedOffset - mLastTokLen };
|
||||
}
|
||||
|
||||
void setCursor(const Cursor& crs) {
|
||||
mAdvancedOffset = crs.mAdvancedOffset;
|
||||
mLastTokLen = 0;
|
||||
}
|
||||
|
||||
tTokType readTok() {
|
||||
if (mSourceLen == mAdvancedOffset + 1) {
|
||||
return tSourceEndTokVal;
|
||||
}
|
||||
|
||||
tTokType out = mTokenizer.advanceToken(mSource + mAdvancedOffset, mSourceLen - mAdvancedOffset, &mLastTokLen);
|
||||
mAdvancedOffset += mLastTokLen;
|
||||
return out;
|
||||
}
|
||||
|
||||
tTokType lookupTok() {
|
||||
auto out = readTok();
|
||||
discardTok();
|
||||
return out;
|
||||
}
|
||||
|
||||
void discardTok() {
|
||||
mTokenizer.resetMatrix();
|
||||
mAdvancedOffset -= mLastTokLen;
|
||||
mLastTokLen = 0;
|
||||
}
|
||||
|
||||
void skipTok() {
|
||||
readTok();
|
||||
}
|
||||
|
||||
void reset() {
|
||||
mAdvancedOffset = mLastTokLen = 0;
|
||||
mTokenizer.resetMatrix();
|
||||
}
|
||||
|
||||
[[nodiscard]] ualni lastTokLEn() const {
|
||||
return mLastTokLen;
|
||||
}
|
||||
|
||||
String extractVal() {
|
||||
auto crs = getCursorPrev();
|
||||
String out;
|
||||
out.resize(mLastTokLen + 1);
|
||||
memCopy(out.write(), crs.str(), mLastTokLen);
|
||||
return out;
|
||||
}
|
||||
};
|
||||
}
|
||||
18
Tokenizer/rsc/script.txt
Normal file
18
Tokenizer/rsc/script.txt
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
|
||||
|
||||
/* ba
|
||||
asadadasdK */
|
||||
|
||||
-1.f;
|
||||
|
||||
if (+1.f) {
|
||||
var string = "asas
|
||||
d";
|
||||
|
||||
}
|
||||
|
||||
while
|
||||
|
||||
return
|
||||
|
||||
|
||||
182
Tokenizer/tests/tests.cpp
Normal file
182
Tokenizer/tests/tests.cpp
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
|
||||
#include "Tokenizer.hpp"
|
||||
#include "filesystem.h"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#define LOG(val) std::cout << #val << " ";
|
||||
|
||||
void test(const char* path) {
|
||||
|
||||
tp::File script_file(path, tp::osfile_openflags::LOAD);
|
||||
if (!script_file.opened) {
|
||||
return;
|
||||
}
|
||||
char* script = new char[script_file.size() + 1];
|
||||
script[script_file.size()] = 0;
|
||||
script_file.read_bytes(script, script_file.size());
|
||||
script_file.close();
|
||||
|
||||
enum class TokType {
|
||||
NONE,
|
||||
FAILED,
|
||||
TOK_SOURCE_END,
|
||||
|
||||
VAR,
|
||||
CLASS_DEF,
|
||||
SELF,
|
||||
SCOPE_IN,
|
||||
SCOPE_OUT,
|
||||
ASSIGN,
|
||||
DEF_FUNC,
|
||||
PRINT,
|
||||
IF,
|
||||
ELSE,
|
||||
WHILE,
|
||||
BRACKET_IN,
|
||||
BRACKET_OUT,
|
||||
COMMA,
|
||||
NEW,
|
||||
CHILD,
|
||||
RETURN,
|
||||
EQUAL,
|
||||
NOT_EQUAL,
|
||||
MORE,
|
||||
LESS,
|
||||
QE_OR_MORE,
|
||||
QE_OR_LESS,
|
||||
BOOL_NOT,
|
||||
BOOL_AND,
|
||||
BOOL_OR,
|
||||
ADD,
|
||||
MUL,
|
||||
SUB,
|
||||
DIV,
|
||||
STM_END,
|
||||
CONST_TRUE,
|
||||
CONST_FALSE,
|
||||
CONST_INT,
|
||||
CONST_FLOAT,
|
||||
SPACE,
|
||||
CONST_STRING,
|
||||
ID,
|
||||
//COMMENT_LINE,
|
||||
COMMENT_BLOCK,
|
||||
};
|
||||
|
||||
tp::SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
|
||||
|
||||
lexer.build({
|
||||
{ "\n|\t| |\r", TokType::SPACE },
|
||||
{ "var", TokType::VAR },
|
||||
{ "class", TokType::CLASS_DEF },
|
||||
{ "self", TokType::SELF },
|
||||
{ "\\{", TokType::SCOPE_IN },
|
||||
{ "\\}", TokType::SCOPE_OUT },
|
||||
{ "=", TokType::ASSIGN },
|
||||
{ "def", TokType::DEF_FUNC },
|
||||
{ "<<", TokType::PRINT },
|
||||
{ "if", TokType::IF },
|
||||
{ "else", TokType::ELSE },
|
||||
{ "while", TokType::WHILE },
|
||||
{ "\\(", TokType::BRACKET_IN },
|
||||
{ "\\)", TokType::BRACKET_OUT },
|
||||
{ ",", TokType::COMMA },
|
||||
{ "new", TokType::NEW },
|
||||
{ "\\.", TokType::CHILD },
|
||||
{ "return", TokType::RETURN },
|
||||
{ "==", TokType::EQUAL },
|
||||
{ "!=", TokType::NOT_EQUAL },
|
||||
{ ">", TokType::MORE },
|
||||
{ "<", TokType::LESS },
|
||||
{ ">=", TokType::QE_OR_MORE },
|
||||
{ "<=", TokType::QE_OR_LESS },
|
||||
{ "!", TokType::BOOL_NOT },
|
||||
{ "&&", TokType::BOOL_AND },
|
||||
{ "\\|\\|", TokType::BOOL_OR },
|
||||
{ "\\+", TokType::ADD },
|
||||
{ "\\*", TokType::MUL },
|
||||
{ "\\-", TokType::SUB },
|
||||
{ "/", TokType::DIV },
|
||||
{ ";", TokType::STM_END },
|
||||
{ "true", TokType::CONST_TRUE },
|
||||
{ "false", TokType::CONST_FALSE },
|
||||
{ "((\\-)|(\\+))?[0-9]+i?", TokType::CONST_INT },
|
||||
{ "((\\-)|(\\+))?([0-9]+)(\\.)([0-9]*)?f?", TokType::CONST_FLOAT },
|
||||
{ "(/\\*){\\*-\\*}*(\\*/)", TokType::COMMENT_BLOCK },
|
||||
{ "\"{\"-\"}*\"", TokType::CONST_STRING },
|
||||
{ "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", TokType::ID },
|
||||
});
|
||||
|
||||
if (!lexer.isBuild()) {
|
||||
std::cout << lexer.getBuildError().description;
|
||||
return;
|
||||
}
|
||||
|
||||
lexer.bindSource(script);
|
||||
|
||||
TokType tok;
|
||||
do {
|
||||
|
||||
tok = lexer.readTok();
|
||||
|
||||
switch (tok) {
|
||||
case TokType::SPACE: { std::cout << " "; } break;
|
||||
case TokType::VAR: { LOG(VAR); } break;
|
||||
case TokType::CLASS_DEF: { LOG(CLASS_DEF); } break;
|
||||
case TokType::SELF: { LOG(SELF); } break;
|
||||
case TokType::SCOPE_IN: { LOG(SCOPE_IN); } break;
|
||||
case TokType::SCOPE_OUT: { LOG(SCOPE_OUT); } break;
|
||||
case TokType::ASSIGN: { LOG(ASSIGN); } break;
|
||||
case TokType::DEF_FUNC: { LOG(DEF_FUNC); } break;
|
||||
case TokType::PRINT: { LOG(PRINT); } break;
|
||||
case TokType::IF: { LOG(IF); } break;
|
||||
case TokType::ELSE: { LOG(ELSE); } break;
|
||||
case TokType::WHILE: { LOG(WHILE); } break;
|
||||
case TokType::BRACKET_IN: { LOG(BRACKET_IN); } break;
|
||||
case TokType::BRACKET_OUT: { LOG(BRACKET_OUT); } break;
|
||||
case TokType::COMMA: { LOG(COMMA); } break;
|
||||
case TokType::NEW: { LOG(NEW); } break;
|
||||
case TokType::CHILD: { LOG(CHILD); } break;
|
||||
case TokType::RETURN: { LOG(RETURN); } break;
|
||||
case TokType::EQUAL: { LOG(EQUAL); } break;
|
||||
case TokType::NOT_EQUAL: { LOG(NOT_EQUAL); } break;
|
||||
case TokType::MORE: { LOG(MORE); } break;
|
||||
case TokType::LESS: { LOG(LESS); } break;
|
||||
case TokType::QE_OR_MORE: { LOG(QE_OR_MORE); } break;
|
||||
case TokType::QE_OR_LESS: { LOG(QE_OR_LESS); } break;
|
||||
case TokType::BOOL_NOT: { LOG(BOOL_NOT); } break;
|
||||
case TokType::BOOL_AND: { LOG(BOOL_AND); } break;
|
||||
case TokType::BOOL_OR: { LOG(BOOL_OR); } break;
|
||||
case TokType::ADD: { LOG(ADD); } break;
|
||||
case TokType::MUL: { LOG(MUL); } break;
|
||||
case TokType::SUB: { LOG(SUB); } break;
|
||||
case TokType::DIV: { LOG(DIV); } break;
|
||||
case TokType::STM_END: { LOG(STM_END); } break;
|
||||
case TokType::CONST_TRUE: { LOG(TRUE); } break;
|
||||
case TokType::CONST_FALSE: { LOG(FALSE); } break;
|
||||
case TokType::CONST_INT: { LOG(INT); } break;
|
||||
case TokType::CONST_FLOAT: { LOG(FLOAT); } break;
|
||||
case TokType::CONST_STRING: { LOG(STRING); } break;
|
||||
case TokType::ID: { LOG(ID); } break;
|
||||
//case TokType::COMMENT_LINE: { LOG(COMMENT_LINE) } break;
|
||||
case TokType::COMMENT_BLOCK: { LOG(COMMENT_BLOCK) } break;
|
||||
case TokType::FAILED: { LOG(FAILED) } break;
|
||||
}
|
||||
|
||||
} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
|
||||
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
|
||||
tp::ModuleManifest* ModuleDependencies[] = { &tp::gModuleTokenizer, NULL };
|
||||
tp::ModuleManifest TestModule("Test", NULL, NULL, ModuleDependencies);
|
||||
if (!TestModule.initialize()) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
test(argc > 1 ? argv[1] : "rsc/script.txt");
|
||||
|
||||
TestModule.deinitialize();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue