Tokenizer (No tests)
This commit is contained in:
parent
f676e84521
commit
6248c294b8
4 changed files with 31 additions and 26 deletions
|
|
@ -1,5 +1,6 @@
|
||||||
|
|
||||||
#include "Buffer.hpp"
|
#include "Buffer.hpp"
|
||||||
|
#include "Utils.hpp"
|
||||||
|
|
||||||
namespace tp {
|
namespace tp {
|
||||||
|
|
||||||
|
|
@ -8,11 +9,14 @@ namespace tp {
|
||||||
|
|
||||||
template <typename tType>
|
template <typename tType>
|
||||||
class Buffer2D {
|
class Buffer2D {
|
||||||
|
public:
|
||||||
|
typedef ualni Index;
|
||||||
|
typedef Pair<Index, Index> Index2D;
|
||||||
typedef SelCopyArg<tType> tTypeArg;
|
typedef SelCopyArg<tType> tTypeArg;
|
||||||
typedef Pair<uhalni, uhalni> Size;
|
|
||||||
|
|
||||||
Size mSize;
|
private:
|
||||||
tType* mBuff = nullptr;
|
tType* mBuff = nullptr;
|
||||||
|
Index2D mSize = { 0, 0 };
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
|
|
@ -22,11 +26,11 @@ namespace tp {
|
||||||
delete mBuff;
|
delete mBuff;
|
||||||
}
|
}
|
||||||
|
|
||||||
explicit Buffer2D(Pair<ualni, ualni> aSize) {
|
explicit Buffer2D(Index2D aSize) {
|
||||||
reserve(aSize);
|
reserve(aSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
[[nodiscard]] Pair<ualni, ualni> size() const {
|
[[nodiscard]] Index2D size() const {
|
||||||
return { mSize.x, mSize.y };
|
return { mSize.x, mSize.y };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -34,22 +38,22 @@ namespace tp {
|
||||||
return mBuff;
|
return mBuff;
|
||||||
}
|
}
|
||||||
|
|
||||||
inline tType& get(uhalni x, uhalni y) {
|
inline tType& get(const Index2D& at) {
|
||||||
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0)
|
DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
|
||||||
return *(mBuff + mSize.x * y + x);
|
return *(mBuff + mSize.x * at.y + at.x);
|
||||||
}
|
}
|
||||||
|
|
||||||
inline const tType& get(uhalni x, uhalni y) const {
|
inline const tType& get(const Index2D& at) const {
|
||||||
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0)
|
DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
|
||||||
return *(mBuff + mSize.x * y + x);
|
return *(mBuff + mSize.x * at.y + at.x);
|
||||||
}
|
}
|
||||||
|
|
||||||
inline void set(uhalni x, uhalni y, tTypeArg value) {
|
inline void set(const Index2D& at, tTypeArg value) {
|
||||||
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0)
|
DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
|
||||||
*(mBuff + mSize.x * y + x) = value;
|
*(mBuff + mSize.x * at.y + at.x) = value;
|
||||||
}
|
}
|
||||||
|
|
||||||
void reserve(Size newSize) {
|
void reserve(const Index2D& newSize) {
|
||||||
if (mSize.x != newSize.x || mSize.y != newSize.y) {
|
if (mSize.x != newSize.x || mSize.y != newSize.y) {
|
||||||
delete mBuff;
|
delete mBuff;
|
||||||
mBuff = new tType[newSize.x * newSize.y];
|
mBuff = new tType[newSize.x * newSize.y];
|
||||||
|
|
@ -58,8 +62,8 @@ namespace tp {
|
||||||
}
|
}
|
||||||
|
|
||||||
void assign(tType value) {
|
void assign(tType value) {
|
||||||
uhalni len = mSize.x * mSize.y;
|
Index len = mSize.x * mSize.y;
|
||||||
for (uhalni i = 0; i < len; i++) {
|
for (Index i = 0; i < len; i++) {
|
||||||
mBuff[i] = value;
|
mBuff[i] = value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ file(GLOB SOURCES "./private/*.cpp")
|
||||||
file(GLOB HEADERS "./public/*.hpp")
|
file(GLOB HEADERS "./public/*.hpp")
|
||||||
add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
|
add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
|
||||||
target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
|
target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
|
||||||
target_link_libraries(${PROJECT_NAME} PUBLIC Strings Math)
|
target_link_libraries(${PROJECT_NAME} PUBLIC Strings)
|
||||||
|
|
||||||
### -------------------------- Tests -------------------------- ###
|
### -------------------------- Tests -------------------------- ###
|
||||||
enable_testing()
|
enable_testing()
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@
|
||||||
#include "List.hpp"
|
#include "List.hpp"
|
||||||
#include "Buffer.hpp"
|
#include "Buffer.hpp"
|
||||||
#include "Buffer2D.hpp"
|
#include "Buffer2D.hpp"
|
||||||
#include "Mat.hpp"
|
#include "Utils.hpp"
|
||||||
#include "Map.hpp"
|
#include "Map.hpp"
|
||||||
|
|
||||||
namespace tp {
|
namespace tp {
|
||||||
|
|
@ -409,8 +409,8 @@ namespace tp {
|
||||||
|
|
||||||
static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable.");
|
static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable.");
|
||||||
|
|
||||||
Buffer2D<ualni> mTransitions{};
|
Buffer2D<ualni> mTransitions;
|
||||||
Buffer<tStateType> mStates{};
|
Buffer<tStateType> mStates;
|
||||||
Range<tAlphabetType> mSymbolRange = nullptr;
|
Range<tAlphabetType> mSymbolRange = nullptr;
|
||||||
|
|
||||||
ualni mIter = 0;
|
ualni mIter = 0;
|
||||||
|
|
@ -423,12 +423,13 @@ namespace tp {
|
||||||
|
|
||||||
void construct(const DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& dfa) {
|
void construct(const DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& dfa) {
|
||||||
mSymbolRange = dfa.getRange();
|
mSymbolRange = dfa.getRange();
|
||||||
auto range_len = uhalni(mSymbolRange.mEnd - mSymbolRange.mBegin);
|
auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin);
|
||||||
Vec2<uhalni> dim = { range_len ? range_len : 1, (uhalni) (dfa.nVertices() + 1) };
|
auto sizeX = range_len ? range_len : 1;
|
||||||
|
auto sizeY = (ualni) (dfa.nVertices() + 1);
|
||||||
|
|
||||||
mTransitions.reserve(dim);
|
mTransitions.reserve({ sizeX, sizeY });
|
||||||
mTransitions.assign(dfa.nVertices());
|
mTransitions.assign(dfa.nVertices());
|
||||||
mStates.reserve(dim.y);
|
mStates.reserve(sizeY);
|
||||||
|
|
||||||
for (auto vertex : dfa.mVertices) {
|
for (auto vertex : dfa.mVertices) {
|
||||||
auto state = vertex.data().termination_state;
|
auto state = vertex.data().termination_state;
|
||||||
|
|
@ -464,7 +465,7 @@ namespace tp {
|
||||||
|
|
||||||
tStateType move(tAlphabetType symbol) {
|
tStateType move(tAlphabetType symbol) {
|
||||||
if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) {
|
if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) {
|
||||||
mIter = mTransitions.get((uhalni) (symbol - mSymbolRange.mBegin), (uhalni) mIter);
|
mIter = mTransitions.get({ symbol - mSymbolRange.mBegin, mIter });
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
mIter = mStates.length() - 1;
|
mIter = mStates.length() - 1;
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,7 @@ namespace tp {
|
||||||
template <typename T1, typename T2>
|
template <typename T1, typename T2>
|
||||||
class Pair {
|
class Pair {
|
||||||
public:
|
public:
|
||||||
Pair() {}
|
Pair() = default;
|
||||||
Pair(T1 t1, T2 t2) : head(t1), tail(t2) {}
|
Pair(T1 t1, T2 t2) : head(t1), tail(t2) {}
|
||||||
union { T1 t1; T1 head; T1 x; };
|
union { T1 t1; T1 head; T1 x; };
|
||||||
union { T2 t2; T2 tail; T2 y; };
|
union { T2 t2; T2 tail; T2 y; };
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue