ilusha/Modules
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixing tokenizer bug

Browse source
This commit is contained in:
IlushaShurupov 2023-08-10 18:33:17 +03:00
parent 92f34e2a99
commit d888aeb2b4
7 changed files with 27 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Tokenizer/public/AutomataGraph.h
View file

@ -422,6 +422,10 @@ namespace tp {
TransitionMatrix() = default;
auto getStates() const { return &mStates; }
auto getTransitions() const { return &mTransitions; }
auto getStart() const { return mStart; }
void construct(const DFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& dfa) {
mSymbolRange = dfa.getRange();
auto range_len = ualni(mSymbolRange.mEnd - mSymbolRange.mBegin);

12
Tokenizer/public/RegularExpression.h
View file

@ -351,17 +351,15 @@ namespace tp::RegEx {
halni idx = 0;
for (auto rule: aRules) {
auto node = idx ? compileUtil(rule.head, rule.tail) : compileUtil(rule.head, rule.tail, left, right);
auto node = compileUtil(rule.head, rule.tail);
if (!(node.left && node.right)) {
mError.mRuleIndex = idx;
return {};
}
if (idx) {
transitionAny(left, node.left);
transitionAny(node.right, right);
}
transitionAny(left, node.left);
transitionAny(node.right, right);
idx++;
}
@ -373,7 +371,7 @@ namespace tp::RegEx {
private:
Node compileUtil(const tAlphabetType* regex, tStateType state, Vertex* aLeft = nullptr, Vertex* aRight = nullptr) {
Node compileUtil(const tAlphabetType* regex, tStateType state) {
Parser parser;
auto astNode = parser.parse(regex);
if (parser.mError.isError()) {
@ -384,7 +382,7 @@ namespace tp::RegEx {
return {};
}
auto node = compileNode(astNode, aLeft, aRight);
auto node = compileNode(astNode, nullptr, nullptr);
delete astNode;
mGraph->setVertexState(node.right, state);

11
Tokenizer/public/Tokenizer.hpp
View file

@ -50,6 +50,8 @@ namespace tp {
return !mError.isError();
}
auto getMatrix() const { return &mTransitionMatrix; }
const RegEx::CompileError<tTokType>& getBuildError() {
return mError;
}
@ -80,8 +82,11 @@ namespace tp {
template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal, tTokType tSourceEndTokVal>
class SimpleTokenizer {
Tokenizer<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> mTokenizer;
public:
typedef Tokenizer<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> tTokenizer;
private:
tTokenizer mTokenizer;
const tAlphabetType* mSource = nullptr;
ualni mLastTokLen = 0;
@ -117,7 +122,7 @@ namespace tp {
};
SimpleTokenizer() = default;
auto getTokenizer() const { return mTokenizer; }
auto getTokenizer() const { return &mTokenizer; }
void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) {
mTokenizer.build(rules);