Tokenizer (No tests)
This commit is contained in:
parent
f676e84521
commit
6248c294b8
4 changed files with 31 additions and 26 deletions
|
|
@ -1,5 +1,6 @@
|
|||
|
||||
#include "Buffer.hpp"
|
||||
#include "Utils.hpp"
|
||||
|
||||
namespace tp {
|
||||
|
||||
|
|
@ -8,11 +9,14 @@ namespace tp {
|
|||
|
||||
template <typename tType>
|
||||
class Buffer2D {
|
||||
public:
|
||||
typedef ualni Index;
|
||||
typedef Pair<Index, Index> Index2D;
|
||||
typedef SelCopyArg<tType> tTypeArg;
|
||||
typedef Pair<uhalni, uhalni> Size;
|
||||
|
||||
Size mSize;
|
||||
private:
|
||||
tType* mBuff = nullptr;
|
||||
Index2D mSize = { 0, 0 };
|
||||
|
||||
public:
|
||||
|
||||
|
|
@ -22,11 +26,11 @@ namespace tp {
|
|||
delete mBuff;
|
||||
}
|
||||
|
||||
explicit Buffer2D(Pair<ualni, ualni> aSize) {
|
||||
explicit Buffer2D(Index2D aSize) {
|
||||
reserve(aSize);
|
||||
}
|
||||
|
||||
[[nodiscard]] Pair<ualni, ualni> size() const {
|
||||
[[nodiscard]] Index2D size() const {
|
||||
return { mSize.x, mSize.y };
|
||||
}
|
||||
|
||||
|
|
@ -34,22 +38,22 @@ namespace tp {
|
|||
return mBuff;
|
||||
}
|
||||
|
||||
inline tType& get(uhalni x, uhalni y) {
|
||||
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0)
|
||||
return *(mBuff + mSize.x * y + x);
|
||||
inline tType& get(const Index2D& at) {
|
||||
DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
|
||||
return *(mBuff + mSize.x * at.y + at.x);
|
||||
}
|
||||
|
||||
inline const tType& get(uhalni x, uhalni y) const {
|
||||
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0)
|
||||
return *(mBuff + mSize.x * y + x);
|
||||
inline const tType& get(const Index2D& at) const {
|
||||
DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
|
||||
return *(mBuff + mSize.x * at.y + at.x);
|
||||
}
|
||||
|
||||
inline void set(uhalni x, uhalni y, tTypeArg value) {
|
||||
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0)
|
||||
*(mBuff + mSize.x * y + x) = value;
|
||||
inline void set(const Index2D& at, tTypeArg value) {
|
||||
DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
|
||||
*(mBuff + mSize.x * at.y + at.x) = value;
|
||||
}
|
||||
|
||||
void reserve(Size newSize) {
|
||||
void reserve(const Index2D& newSize) {
|
||||
if (mSize.x != newSize.x || mSize.y != newSize.y) {
|
||||
delete mBuff;
|
||||
mBuff = new tType[newSize.x * newSize.y];
|
||||
|
|
@ -58,8 +62,8 @@ namespace tp {
|
|||
}
|
||||
|
||||
void assign(tType value) {
|
||||
uhalni len = mSize.x * mSize.y;
|
||||
for (uhalni i = 0; i < len; i++) {
|
||||
Index len = mSize.x * mSize.y;
|
||||
for (Index i = 0; i < len; i++) {
|
||||
mBuff[i] = value;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ file(GLOB SOURCES "./private/*.cpp")
|
|||
file(GLOB HEADERS "./public/*.hpp")
|
||||
add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
|
||||
target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
|
||||
target_link_libraries(${PROJECT_NAME} PUBLIC Strings Math)
|
||||
target_link_libraries(${PROJECT_NAME} PUBLIC Strings)
|
||||
|
||||
### -------------------------- Tests -------------------------- ###
|
||||
enable_testing()
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
#include "List.hpp"
|
||||
#include "Buffer.hpp"
|
||||
#include "Buffer2D.hpp"
|
||||
#include "Mat.hpp"
|
||||
#include "Utils.hpp"
|
||||
#include "Map.hpp"
|
||||
|
||||
namespace tp {
|
||||
|
|
@ -409,8 +409,8 @@ namespace tp {
|
|||
|
||||
static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable.");
|
||||
|
||||
Buffer2D<ualni> mTransitions{};
|
||||
Buffer<tStateType> mStates{};
|
||||
Buffer2D<ualni> mTransitions;
|
||||
Buffer<tStateType> mStates;
|
||||
Range<tAlphabetType> mSymbolRange = nullptr;
|
||||
|
||||
ualni mIter = 0;
|
||||
|
|
@ -423,12 +423,13 @@ namespace tp {
|
|||
|
||||
void construct(const DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& dfa) {
|
||||
mSymbolRange = dfa.getRange();
|
||||
auto range_len = uhalni(mSymbolRange.mEnd - mSymbolRange.mBegin);
|
||||
Vec2<uhalni> dim = { range_len ? range_len : 1, (uhalni) (dfa.nVertices() + 1) };
|
||||
auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin);
|
||||
auto sizeX = range_len ? range_len : 1;
|
||||
auto sizeY = (ualni) (dfa.nVertices() + 1);
|
||||
|
||||
mTransitions.reserve(dim);
|
||||
mTransitions.reserve({ sizeX, sizeY });
|
||||
mTransitions.assign(dfa.nVertices());
|
||||
mStates.reserve(dim.y);
|
||||
mStates.reserve(sizeY);
|
||||
|
||||
for (auto vertex : dfa.mVertices) {
|
||||
auto state = vertex.data().termination_state;
|
||||
|
|
@ -464,7 +465,7 @@ namespace tp {
|
|||
|
||||
tStateType move(tAlphabetType symbol) {
|
||||
if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) {
|
||||
mIter = mTransitions.get((uhalni) (symbol - mSymbolRange.mBegin), (uhalni) mIter);
|
||||
mIter = mTransitions.get({ symbol - mSymbolRange.mBegin, mIter });
|
||||
}
|
||||
else {
|
||||
mIter = mStates.length() - 1;
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ namespace tp {
|
|||
template <typename T1, typename T2>
|
||||
class Pair {
|
||||
public:
|
||||
Pair() {}
|
||||
Pair() = default;
|
||||
Pair(T1 t1, T2 t2) : head(t1), tail(t2) {}
|
||||
union { T1 t1; T1 head; T1 x; };
|
||||
union { T2 t2; T2 tail; T2 y; };
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue