fix compile errors
This commit is contained in:
IlyaShurupov
Ilya Shurupov
parent
b2a1883369
commit
aaeb6438a1
27 changed files with 19 additions and 21 deletions
19
.back/old/Tokenizer/public/AutomataGraph.h
Normal file
19
.back/old/Tokenizer/public/AutomataGraph.h
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "List.hpp"
|
||||
#include "Buffer.hpp"
|
||||
#include "Buffer2D.hpp"
|
||||
#include "Utils.hpp"
|
||||
#include "Map.hpp"
|
||||
|
||||
#ifdef ENV_OS_WINDOWS
|
||||
#undef min
|
||||
#undef max
|
||||
#endif
|
||||
|
||||
namespace tp {
|
||||
|
||||
extern ModuleManifest gModuleTokenizer;
|
||||
|
||||
}
|
||||
333
.back/old/Tokenizer/public/RegularExpression.h
Normal file
333
.back/old/Tokenizer/public/RegularExpression.h
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "AutomataGraph.h"
|
||||
|
||||
namespace tp {
|
||||
extern ModuleManifest gModuleTokenizer;
|
||||
}
|
||||
|
||||
namespace tp::RegEx {
|
||||
|
||||
struct AstNode {
|
||||
enum Type {
|
||||
NONE,
|
||||
ANY,
|
||||
OR,
|
||||
IF,
|
||||
CLASS,
|
||||
COMPOUND,
|
||||
REPEAT,
|
||||
VAL,
|
||||
} mType = NONE;
|
||||
|
||||
AstNode() = default;
|
||||
virtual ~AstNode() = default;
|
||||
};
|
||||
|
||||
template <typename tAlphabetType>
|
||||
struct AstVal : public AstNode {
|
||||
explicit AstVal(tAlphabetType val) :
|
||||
mVal(val) {
|
||||
mType = VAL;
|
||||
}
|
||||
~AstVal() override = default;
|
||||
tAlphabetType mVal;
|
||||
};
|
||||
|
||||
struct AstCompound : public AstNode {
|
||||
AstCompound() { mType = COMPOUND; }
|
||||
~AstCompound() override {
|
||||
for (auto iter : mChilds) {
|
||||
delete iter.data();
|
||||
}
|
||||
mChilds.removeAll();
|
||||
}
|
||||
List<AstNode*> mChilds;
|
||||
};
|
||||
|
||||
struct AstAlternation : public AstNode {
|
||||
AstAlternation() { mType = OR; }
|
||||
~AstAlternation() override {
|
||||
delete mFirst;
|
||||
delete mSecond;
|
||||
}
|
||||
AstNode* mFirst = nullptr;
|
||||
AstNode* mSecond = nullptr;
|
||||
};
|
||||
|
||||
struct AstIf : public AstNode {
|
||||
AstIf() { mType = IF; }
|
||||
~AstIf() override { delete mNode; }
|
||||
AstNode* mNode = nullptr;
|
||||
};
|
||||
|
||||
struct AstAny : public AstNode {
|
||||
AstAny() { mType = ANY; }
|
||||
~AstAny() override = default;
|
||||
};
|
||||
|
||||
struct AstRepetition : public AstNode {
|
||||
AstRepetition() { mType = REPEAT; }
|
||||
~AstRepetition() override { delete mNode; }
|
||||
AstNode* mNode = nullptr;
|
||||
bool mPlus = false;
|
||||
};
|
||||
|
||||
template <typename tAlphabetType>
|
||||
struct AstClass : public AstNode {
|
||||
AstClass() { mType = CLASS; }
|
||||
~AstClass() override { mRanges.removeAll(); }
|
||||
List<Range<tAlphabetType>> mRanges;
|
||||
bool mExclude = false;
|
||||
};
|
||||
|
||||
struct ParseError {
|
||||
const char* description = nullptr;
|
||||
uhalni offset = 0;
|
||||
[[nodiscard]] bool isError() const { return description != nullptr; }
|
||||
};
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal>
|
||||
class Parser {
|
||||
|
||||
enum TokType : uint1 {
|
||||
TOK_COMPOUND_START = 0,
|
||||
TOK_COMPOUND_END,
|
||||
TOK_CLASS_START,
|
||||
TOK_CLASS_END,
|
||||
TOK_CLASS_START_EXCLUDE,
|
||||
TOK_CLASS_END_EXCLUDE,
|
||||
TOK_OR,
|
||||
TOK_IF,
|
||||
TOK_ANY,
|
||||
TOK_REPEAT,
|
||||
TOK_REPEAT_PLUS,
|
||||
TOK_HYPHEN,
|
||||
TOK_SPECIALS_END_,
|
||||
TOK_VAL,
|
||||
TOK_NONE,
|
||||
};
|
||||
|
||||
tAlphabetType SpecialSymbols[TOK_SPECIALS_END_] = {
|
||||
'(', ')', '[', ']', '{', '}', '|', '?', '.', '*', '+', '-',
|
||||
};
|
||||
|
||||
tAlphabetType mEscapeSymbol = '\\';
|
||||
|
||||
struct Token {
|
||||
TokType type;
|
||||
tAlphabetType val;
|
||||
};
|
||||
|
||||
const tAlphabetType* mSource = nullptr;
|
||||
uhalni mOffset = 0;
|
||||
Token mCurToken;
|
||||
uhalni mTokLength = 0;
|
||||
|
||||
public:
|
||||
ParseError mError;
|
||||
|
||||
// regular expression must be a zero termination string
|
||||
AstCompound* parse(const tAlphabetType* regex) {
|
||||
mSource = regex;
|
||||
return parseRegEx();
|
||||
}
|
||||
|
||||
private:
|
||||
AstCompound* parseRegEx() {
|
||||
auto out = new AstCompound();
|
||||
for (AstNode* node = parseElement(); node; node = parseElement()) {
|
||||
out->mChilds.pushBack(node);
|
||||
}
|
||||
if (!out->mChilds.length()) {
|
||||
genError("Expected A Expression");
|
||||
}
|
||||
if (mError.description) {
|
||||
delete out;
|
||||
return nullptr;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
AstNode* parseElement() {
|
||||
AstNode* out = nullptr;
|
||||
switch (readTok().type) {
|
||||
case TOK_COMPOUND_START: out = parseCompound(); break;
|
||||
case TOK_CLASS_START: out = parseClass(); break;
|
||||
case TOK_CLASS_START_EXCLUDE: out = parseClass(true); break;
|
||||
case TOK_ANY: out = parseAny(); break;
|
||||
case TOK_VAL: out = parseVal(); break;
|
||||
case TOK_NONE:
|
||||
{
|
||||
discardTok();
|
||||
return nullptr;
|
||||
};
|
||||
default: break;
|
||||
}
|
||||
if (!out) {
|
||||
discardTok();
|
||||
return nullptr;
|
||||
}
|
||||
switch (readTok().type) {
|
||||
case TOK_OR: out = parseAlternation(out); break;
|
||||
case TOK_REPEAT: out = parseRepetition(out); break;
|
||||
case TOK_REPEAT_PLUS: out = parseRepetition(out, true); break;
|
||||
case TOK_IF: out = parseIf(out); break;
|
||||
case TOK_NONE: break;
|
||||
default:
|
||||
{
|
||||
discardTok();
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
AstCompound* parseCompound() {
|
||||
auto out = new AstCompound();
|
||||
for (AstNode* node = parseElement(); node; node = parseElement()) {
|
||||
out->mChilds.pushBack(node);
|
||||
}
|
||||
if (readTok().type != TOK_COMPOUND_END) {
|
||||
genError("Expected Compound End");
|
||||
}
|
||||
if (mError.description) {
|
||||
delete out;
|
||||
return nullptr;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
AstClass<tAlphabetType>* parseClass(bool exclude = false) {
|
||||
auto out = new AstClass<tAlphabetType>();
|
||||
out->mExclude = exclude;
|
||||
auto& ranges = out->mRanges;
|
||||
|
||||
readTok();
|
||||
|
||||
READ_VAL:
|
||||
if (mCurToken.type != TOK_VAL) {
|
||||
delete out;
|
||||
genError("Expected A Value");
|
||||
return nullptr;
|
||||
}
|
||||
char range_start = mCurToken.val;
|
||||
|
||||
readTok();
|
||||
if (mCurToken.type != TOK_HYPHEN) {
|
||||
delete out;
|
||||
genError("Expected A Range");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
readTok();
|
||||
if (mCurToken.type != TOK_VAL) {
|
||||
delete out;
|
||||
genError("Expected A Value");
|
||||
return nullptr;
|
||||
}
|
||||
char range_end = mCurToken.val;
|
||||
|
||||
ranges.pushBack({ range_start, range_end });
|
||||
|
||||
readTok();
|
||||
if ((mCurToken.type == TOK_CLASS_END && !exclude) || (mCurToken.type == TOK_CLASS_END_EXCLUDE && exclude)) {
|
||||
return out;
|
||||
} else {
|
||||
goto READ_VAL;
|
||||
}
|
||||
}
|
||||
|
||||
AstAny* parseAny() { return new AstAny(); }
|
||||
|
||||
AstVal<tAlphabetType>* parseVal() {
|
||||
auto out = new AstVal<tAlphabetType>(mCurToken.val);
|
||||
return out;
|
||||
}
|
||||
|
||||
AstAlternation* parseAlternation(AstNode* left) {
|
||||
auto right = parseElement();
|
||||
if (!right) {
|
||||
genError("Expected Alternation right Side");
|
||||
delete left;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto out = new AstAlternation();
|
||||
|
||||
out->mFirst = left;
|
||||
out->mSecond = right;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
AstRepetition* parseRepetition(AstNode* left, bool plus = false) {
|
||||
auto out = new AstRepetition();
|
||||
out->mNode = left;
|
||||
out->mPlus = plus;
|
||||
return out;
|
||||
}
|
||||
|
||||
AstIf* parseIf(AstNode* left) {
|
||||
auto out = new AstIf();
|
||||
out->mNode = left;
|
||||
return out;
|
||||
}
|
||||
|
||||
void genError(const char* desc) { mError = { desc, mOffset }; }
|
||||
|
||||
Token& readTok() {
|
||||
|
||||
const tAlphabetType* crs = mSource + mOffset;
|
||||
|
||||
// zero termination string
|
||||
if (*crs == 0) {
|
||||
mCurToken.type = TOK_NONE;
|
||||
return mCurToken;
|
||||
}
|
||||
|
||||
mTokLength = 1;
|
||||
mCurToken.type = TOK_VAL;
|
||||
mCurToken.val = crs[0];
|
||||
|
||||
if (crs[0] == mEscapeSymbol) {
|
||||
mCurToken.val = crs[1];
|
||||
mTokLength = 2;
|
||||
} else {
|
||||
for (uhalni tok = 0; tok < TOK_SPECIALS_END_; tok++) {
|
||||
if (SpecialSymbols[tok] == mCurToken.val) {
|
||||
mCurToken.type = TokType(tok);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mOffset += mTokLength;
|
||||
return mCurToken;
|
||||
}
|
||||
|
||||
void discardTok() {
|
||||
mOffset -= mTokLength;
|
||||
mTokLength = 0;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
CompileError<tStateType> compile(
|
||||
NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& out, const tAlphabetType* regex, tStateType state
|
||||
) {
|
||||
Compiler<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> compiler;
|
||||
compiler.compile(out, regex, state);
|
||||
return compiler.mError;
|
||||
}
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
CompileError<tStateType> compile(
|
||||
NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& out,
|
||||
const InitialierList<Pair<const tAlphabetType*, tStateType>>& rules
|
||||
) {
|
||||
Compiler<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> compiler;
|
||||
compiler.compile(out, rules);
|
||||
return compiler.mError;
|
||||
}
|
||||
}
|
||||
184
.back/old/Tokenizer/public/Tokenizer.hpp
Normal file
184
.back/old/Tokenizer/public/Tokenizer.hpp
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "RegularExpression.h"
|
||||
#include "Strings.hpp"
|
||||
|
||||
namespace tp {
|
||||
|
||||
extern ModuleManifest gModuleTokenizer;
|
||||
|
||||
template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal>
|
||||
class Tokenizer {
|
||||
|
||||
TransitionMatrix<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> mTransitionMatrix;
|
||||
|
||||
RegEx::CompileError<tTokType> mError;
|
||||
|
||||
bool scanFailed() { return mTransitionMatrix.isTrapped(); }
|
||||
|
||||
public:
|
||||
// Some useful RE to be reused
|
||||
static constexpr const tAlphabetType* etherRE = "\n|\t| |\r";
|
||||
static constexpr const tAlphabetType* intRE = "((\\-)|(\\+))?[0-9]+i?";
|
||||
static constexpr const tAlphabetType* floatRE = R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)";
|
||||
static constexpr const tAlphabetType* commentBlockRE = R"((/\*){\*-\*}*(\*/))";
|
||||
static constexpr const tAlphabetType* stringRE = R"("{"-"}*")";
|
||||
static constexpr const tAlphabetType* idRE = "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*";
|
||||
|
||||
public:
|
||||
Tokenizer() { MODULE_SANITY_CHECK(gModuleTokenizer) }
|
||||
|
||||
void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) {
|
||||
NFA<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> nfa;
|
||||
|
||||
mError = RegEx::compile(nfa, rules);
|
||||
if (mError.isError()) {
|
||||
return;
|
||||
}
|
||||
|
||||
DFA<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> dfa(nfa);
|
||||
mTransitionMatrix.construct(dfa);
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isBuild() const { return !mError.isError(); }
|
||||
|
||||
auto getMatrix() const { return &mTransitionMatrix; }
|
||||
|
||||
const RegEx::CompileError<tTokType>& getBuildError() { return mError; }
|
||||
|
||||
void resetMatrix() { mTransitionMatrix.reset(); }
|
||||
|
||||
tTokType advanceSymbol(tAlphabetType symbol) { return mTransitionMatrix.move(symbol); }
|
||||
|
||||
tTokType advanceToken(const tAlphabetType* source, ualni source_len, ualni* token_len) {
|
||||
tTokType out = tNoTokVal;
|
||||
*token_len = 0;
|
||||
for (ualni idx = 0; idx < source_len; idx++) {
|
||||
out = advanceSymbol(source[idx]);
|
||||
if (out != tNoTokVal) {
|
||||
*token_len = idx;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
~Tokenizer() = default;
|
||||
};
|
||||
|
||||
template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal, tTokType tSourceEndTokVal>
|
||||
class SimpleTokenizer {
|
||||
public:
|
||||
typedef Tokenizer<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> tTokenizer;
|
||||
|
||||
private:
|
||||
tTokenizer mTokenizer;
|
||||
|
||||
const tAlphabetType* mSource = nullptr;
|
||||
ualni mLastTokLen = 0;
|
||||
ualni mSourceLen = 0;
|
||||
ualni mAdvancedOffset = 0;
|
||||
|
||||
public:
|
||||
struct Cursor {
|
||||
const tAlphabetType* mSource = nullptr;
|
||||
ualni mAdvancedOffset = 0;
|
||||
const tAlphabetType* str() { return mSource + mAdvancedOffset; }
|
||||
|
||||
struct Location2D {
|
||||
ualni line = 0;
|
||||
ualni character = 0;
|
||||
};
|
||||
|
||||
Location2D get2DLocation() {
|
||||
Location2D out;
|
||||
typedef typename StringLogic<tAlphabetType>::Index Idx;
|
||||
Buffer<Idx> offsets;
|
||||
StringLogic<tAlphabetType>::calcLineOffsets(mSource, StringLogic<tAlphabetType>::calcLength(mSource), offsets);
|
||||
for (ualni idx = 1; idx < offsets.size(); idx++) {
|
||||
if (offsets[idx] > mAdvancedOffset) {
|
||||
out.line = idx - 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
out.character = mAdvancedOffset - offsets[out.line];
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
SimpleTokenizer() = default;
|
||||
auto getTokenizer() const { return &mTokenizer; }
|
||||
|
||||
void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) { mTokenizer.build(rules); }
|
||||
|
||||
[[nodiscard]] bool isBuild() const { return mTokenizer.isBuild(); }
|
||||
|
||||
const RegEx::CompileError<tTokType>& getBuildError() { return mTokenizer.getBuildError(); }
|
||||
|
||||
void bindSource(const tAlphabetType* source) {
|
||||
mSource = source;
|
||||
while (mSource[mSourceLen])
|
||||
mSourceLen++;
|
||||
mSourceLen++;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isInputLeft() const {
|
||||
if (!mSource) {
|
||||
return false;
|
||||
}
|
||||
if (mSource[mAdvancedOffset] == 0) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
Cursor getCursor() const { return { mSource, mAdvancedOffset }; }
|
||||
|
||||
Cursor getCursorPrev() const { return { mSource, mAdvancedOffset - mLastTokLen }; }
|
||||
|
||||
void setCursor(const Cursor& crs) {
|
||||
mAdvancedOffset = crs.mAdvancedOffset;
|
||||
mLastTokLen = 0;
|
||||
}
|
||||
|
||||
tTokType readTok() {
|
||||
if (mSourceLen == mAdvancedOffset + 1) {
|
||||
return tSourceEndTokVal;
|
||||
}
|
||||
|
||||
tTokType out = mTokenizer.advanceToken(mSource + mAdvancedOffset, mSourceLen - mAdvancedOffset, &mLastTokLen);
|
||||
mAdvancedOffset += mLastTokLen;
|
||||
return out;
|
||||
}
|
||||
|
||||
tTokType lookupTok() {
|
||||
auto out = readTok();
|
||||
discardTok();
|
||||
return out;
|
||||
}
|
||||
|
||||
void discardTok() {
|
||||
mTokenizer.resetMatrix();
|
||||
mAdvancedOffset -= mLastTokLen;
|
||||
mLastTokLen = 0;
|
||||
}
|
||||
|
||||
void skipTok() { readTok(); }
|
||||
|
||||
void reset() {
|
||||
mAdvancedOffset = mLastTokLen = 0;
|
||||
mTokenizer.resetMatrix();
|
||||
}
|
||||
|
||||
[[nodiscard]] ualni lastTokLEn() const { return mLastTokLen; }
|
||||
|
||||
String extractVal() {
|
||||
auto crs = getCursorPrev();
|
||||
String out;
|
||||
out.resize(mLastTokLen);
|
||||
memCopy(out.write(), crs.str(), mLastTokLen);
|
||||
return out;
|
||||
}
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue