Tokenizer (No tests)

This commit is contained in:
IlushaShurupov 2023-07-15 18:58:31 +03:00
parent e63843c6bb
commit 44bad77e93
4 changed files with 31 additions and 26 deletions

View file

@ -1,5 +1,6 @@
#include "Buffer.hpp" #include "Buffer.hpp"
#include "Utils.hpp"
namespace tp { namespace tp {
@ -8,11 +9,14 @@ namespace tp {
template <typename tType> template <typename tType>
class Buffer2D { class Buffer2D {
public:
typedef ualni Index;
typedef Pair<Index, Index> Index2D;
typedef SelCopyArg<tType> tTypeArg; typedef SelCopyArg<tType> tTypeArg;
typedef Pair<uhalni, uhalni> Size;
Size mSize; private:
tType* mBuff = nullptr; tType* mBuff = nullptr;
Index2D mSize = { 0, 0 };
public: public:
@ -22,11 +26,11 @@ namespace tp {
delete mBuff; delete mBuff;
} }
explicit Buffer2D(Pair<ualni, ualni> aSize) { explicit Buffer2D(Index2D aSize) {
reserve(aSize); reserve(aSize);
} }
[[nodiscard]] Pair<ualni, ualni> size() const { [[nodiscard]] Index2D size() const {
return { mSize.x, mSize.y }; return { mSize.x, mSize.y };
} }
@ -34,22 +38,22 @@ namespace tp {
return mBuff; return mBuff;
} }
inline tType& get(uhalni x, uhalni y) { inline tType& get(const Index2D& at) {
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0) DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
return *(mBuff + mSize.x * y + x); return *(mBuff + mSize.x * at.y + at.x);
} }
inline const tType& get(uhalni x, uhalni y) const { inline const tType& get(const Index2D& at) const {
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0) DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
return *(mBuff + mSize.x * y + x); return *(mBuff + mSize.x * at.y + at.x);
} }
inline void set(uhalni x, uhalni y, tTypeArg value) { inline void set(const Index2D& at, tTypeArg value) {
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0) DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
*(mBuff + mSize.x * y + x) = value; *(mBuff + mSize.x * at.y + at.x) = value;
} }
void reserve(Size newSize) { void reserve(const Index2D& newSize) {
if (mSize.x != newSize.x || mSize.y != newSize.y) { if (mSize.x != newSize.x || mSize.y != newSize.y) {
delete mBuff; delete mBuff;
mBuff = new tType[newSize.x * newSize.y]; mBuff = new tType[newSize.x * newSize.y];
@ -58,8 +62,8 @@ namespace tp {
} }
void assign(tType value) { void assign(tType value) {
uhalni len = mSize.x * mSize.y; Index len = mSize.x * mSize.y;
for (uhalni i = 0; i < len; i++) { for (Index i = 0; i < len; i++) {
mBuff[i] = value; mBuff[i] = value;
} }
} }

View file

@ -10,7 +10,7 @@ file(GLOB SOURCES "./private/*.cpp")
file(GLOB HEADERS "./public/*.hpp") file(GLOB HEADERS "./public/*.hpp")
add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS}) add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
target_include_directories(${PROJECT_NAME} PUBLIC ./public/) target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
target_link_libraries(${PROJECT_NAME} PUBLIC Strings Math) target_link_libraries(${PROJECT_NAME} PUBLIC Strings)
### -------------------------- Tests -------------------------- ### ### -------------------------- Tests -------------------------- ###
enable_testing() enable_testing()

View file

@ -4,7 +4,7 @@
#include "List.hpp" #include "List.hpp"
#include "Buffer.hpp" #include "Buffer.hpp"
#include "Buffer2D.hpp" #include "Buffer2D.hpp"
#include "Mat.hpp" #include "Utils.hpp"
#include "Map.hpp" #include "Map.hpp"
namespace tp { namespace tp {
@ -409,8 +409,8 @@ namespace tp {
static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable."); static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable.");
Buffer2D<ualni> mTransitions{}; Buffer2D<ualni> mTransitions;
Buffer<tStateType> mStates{}; Buffer<tStateType> mStates;
Range<tAlphabetType> mSymbolRange = nullptr; Range<tAlphabetType> mSymbolRange = nullptr;
ualni mIter = 0; ualni mIter = 0;
@ -423,12 +423,13 @@ namespace tp {
void construct(const DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& dfa) { void construct(const DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& dfa) {
mSymbolRange = dfa.getRange(); mSymbolRange = dfa.getRange();
auto range_len = uhalni(mSymbolRange.mEnd - mSymbolRange.mBegin); auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin);
Vec2<uhalni> dim = { range_len ? range_len : 1, (uhalni) (dfa.nVertices() + 1) }; auto sizeX = range_len ? range_len : 1;
auto sizeY = (ualni) (dfa.nVertices() + 1);
mTransitions.reserve(dim); mTransitions.reserve({ sizeX, sizeY });
mTransitions.assign(dfa.nVertices()); mTransitions.assign(dfa.nVertices());
mStates.reserve(dim.y); mStates.reserve(sizeY);
for (auto vertex : dfa.mVertices) { for (auto vertex : dfa.mVertices) {
auto state = vertex.data().termination_state; auto state = vertex.data().termination_state;
@ -464,7 +465,7 @@ namespace tp {
tStateType move(tAlphabetType symbol) { tStateType move(tAlphabetType symbol) {
if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) { if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) {
mIter = mTransitions.get((uhalni) (symbol - mSymbolRange.mBegin), (uhalni) mIter); mIter = mTransitions.get({ symbol - mSymbolRange.mBegin, mIter });
} }
else { else {
mIter = mStates.length() - 1; mIter = mStates.length() - 1;

View file

@ -24,7 +24,7 @@ namespace tp {
template <typename T1, typename T2> template <typename T1, typename T2>
class Pair { class Pair {
public: public:
Pair() {} Pair() = default;
Pair(T1 t1, T2 t2) : head(t1), tail(t2) {} Pair(T1 t1, T2 t2) : head(t1), tail(t2) {}
union { T1 t1; T1 head; T1 x; }; union { T1 t1; T1 head; T1 x; };
union { T2 t2; T2 tail; T2 y; }; union { T2 t2; T2 tail; T2 y; };