ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixing tokenizer bug

Browse source
This commit is contained in:
IlushaShurupov 2023-08-10 18:33:17 +03:00 • committed by Ilya Shurupov
parent a938bf9bb1
commit 92e772cde1
7 changed files with 27 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

11
Tokenizer/public/Tokenizer.hpp
View file

@ -50,6 +50,8 @@ namespace tp {
return !mError.isError();
}
auto getMatrix() const { return &mTransitionMatrix; }
const RegEx::CompileError<tTokType>& getBuildError() {
return mError;
}
@ -80,8 +82,11 @@ namespace tp {
template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal, tTokType tSourceEndTokVal>
class SimpleTokenizer {
Tokenizer<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> mTokenizer;
public:
typedef Tokenizer<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> tTokenizer;
private:
tTokenizer mTokenizer;
const tAlphabetType* mSource = nullptr;
ualni mLastTokLen = 0;
@ -117,7 +122,7 @@ namespace tp {
};
SimpleTokenizer() = default;
auto getTokenizer() const { return mTokenizer; }
auto getTokenizer() const { return &mTokenizer; }
void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) {
mTokenizer.build(rules);