ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tokenizer (No tests)

Browse source
This commit is contained in:
IlushaShurupov 2023-07-15 18:58:31 +03:00
parent e63843c6bb
commit 44bad77e93
4 changed files with 31 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

36
Containers/public/Buffer2D.hpp
View file

@ -1,5 +1,6 @@
#include "Buffer.hpp"
#include "Utils.hpp"
namespace tp {
@ -8,11 +9,14 @@ namespace tp {
template <typename tType>
class Buffer2D {
public:
typedef ualni Index;
typedef Pair<Index, Index> Index2D;
typedef SelCopyArg<tType> tTypeArg;
typedef Pair<uhalni, uhalni> Size;
Size mSize;
private:
tType* mBuff = nullptr;
Index2D mSize = { 0, 0 };
public:
@ -22,11 +26,11 @@ namespace tp {
delete mBuff;
}
explicit Buffer2D(Pair<ualni, ualni> aSize) {
explicit Buffer2D(Index2D aSize) {
reserve(aSize);
}
[[nodiscard]] Pair<ualni, ualni> size() const {
[[nodiscard]] Index2D size() const {
return { mSize.x, mSize.y };
}
@ -34,22 +38,22 @@ namespace tp {
return mBuff;
}
inline tType& get(uhalni x, uhalni y) {
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0)
return *(mBuff + mSize.x * y + x);
inline tType& get(const Index2D& at) {
DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
return *(mBuff + mSize.x * at.y + at.x);
}
inline const tType& get(uhalni x, uhalni y) const {
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0)
return *(mBuff + mSize.x * y + x);
inline const tType& get(const Index2D& at) const {
DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
return *(mBuff + mSize.x * at.y + at.x);
}
inline void set(uhalni x, uhalni y, tTypeArg value) {
DEBUG_ASSERT(x < mSize.x && y < mSize.y && x >= 0 && y >= 0)
*(mBuff + mSize.x * y + x) = value;
inline void set(const Index2D& at, tTypeArg value) {
DEBUG_ASSERT(at.x < mSize.x && at.y < mSize.y && at.x >= 0 && at.y >= 0)
*(mBuff + mSize.x * at.y + at.x) = value;
}
void reserve(Size newSize) {
void reserve(const Index2D& newSize) {
if (mSize.x != newSize.x || mSize.y != newSize.y) {
delete mBuff;
mBuff = new tType[newSize.x * newSize.y];
@ -58,8 +62,8 @@ namespace tp {
}
void assign(tType value) {
uhalni len = mSize.x * mSize.y;
for (uhalni i = 0; i < len; i++) {
Index len = mSize.x * mSize.y;
for (Index i = 0; i < len; i++) {
mBuff[i] = value;
}
}

2
Tokenizer/CMakeLists.txt
View file

@ -10,7 +10,7 @@ file(GLOB SOURCES "./private/*.cpp")
file(GLOB HEADERS "./public/*.hpp")
add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
target_link_libraries(${PROJECT_NAME} PUBLIC Strings Math)
target_link_libraries(${PROJECT_NAME} PUBLIC Strings)
### -------------------------- Tests -------------------------- ###
enable_testing()

17
Tokenizer/public/AutomataGraph.h
View file

@ -4,7 +4,7 @@
#include "List.hpp"
#include "Buffer.hpp"
#include "Buffer2D.hpp"
#include "Mat.hpp"
#include "Utils.hpp"
#include "Map.hpp"
namespace tp {
@ -409,8 +409,8 @@ namespace tp {
static_assert(TypeTraits<tAlphabetType>::isIntegral, "tAlphabetType must be enumerable.");
Buffer2D<ualni> mTransitions{};
Buffer<tStateType> mStates{};
Buffer2D<ualni> mTransitions;
Buffer<tStateType> mStates;
Range<tAlphabetType> mSymbolRange = nullptr;
ualni mIter = 0;
@ -423,12 +423,13 @@ namespace tp {
void construct(const DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& dfa) {
mSymbolRange = dfa.getRange();
auto range_len = uhalni(mSymbolRange.mEnd - mSymbolRange.mBegin);
Vec2<uhalni> dim = { range_len ? range_len : 1, (uhalni) (dfa.nVertices() + 1) };
auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin);
auto sizeX = range_len ? range_len : 1;
auto sizeY = (ualni) (dfa.nVertices() + 1);
mTransitions.reserve(dim);
mTransitions.reserve({ sizeX, sizeY });
mTransitions.assign(dfa.nVertices());
mStates.reserve(dim.y);
mStates.reserve(sizeY);
for (auto vertex : dfa.mVertices) {
auto state = vertex.data().termination_state;
@ -464,7 +465,7 @@ namespace tp {
tStateType move(tAlphabetType symbol) {
if (symbol >= mSymbolRange.mBegin && symbol < mSymbolRange.mEnd) {
mIter = mTransitions.get((uhalni) (symbol - mSymbolRange.mBegin), (uhalni) mIter);
mIter = mTransitions.get({ symbol - mSymbolRange.mBegin, mIter });
}
else {
mIter = mStates.length() - 1;

2
Utils/public/Utils.hpp
View file

@ -24,7 +24,7 @@ namespace tp {
template <typename T1, typename T2>
class Pair {
public:
Pair() {}
Pair() = default;
Pair(T1 t1, T2 t2) : head(t1), tail(t2) {}
union { T1 t1; T1 head; T1 x; };
union { T2 t2; T2 tail; T2 y; };