CF Grammar Initial
This commit is contained in:
IlushaShurupov
Ilya Shurupov
parent
42ee676bc9
commit
8feeb3d1c1
7 changed files with 251 additions and 106 deletions
|
|
@ -19,6 +19,7 @@ add_subdirectory(Math)
|
|||
add_subdirectory(Allocators)
|
||||
add_subdirectory(Strings)
|
||||
add_subdirectory(Tokenizer)
|
||||
add_subdirectory(Parser)
|
||||
add_subdirectory(CommandLine)
|
||||
add_subdirectory(Externals)
|
||||
add_subdirectory(Connection)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ file(GLOB SOURCES "./private/*.cpp")
|
|||
file(GLOB HEADERS "./public/*.hpp")
|
||||
add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
|
||||
target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
|
||||
target_link_libraries(${PROJECT_NAME} PUBLIC Strings)
|
||||
target_link_libraries(${PROJECT_NAME} PUBLIC Tokenizer)
|
||||
|
||||
### -------------------------- Tests -------------------------- ###
|
||||
enable_testing()
|
||||
|
|
|
|||
190
Parser/private/CfGrammarParser.cpp
Normal file
190
Parser/private/CfGrammarParser.cpp
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
|
||||
#include "NewPlacement.hpp"
|
||||
|
||||
#include "CfGrammar.hpp"
|
||||
#include "Tokenizer.hpp"
|
||||
|
||||
using namespace tp;
|
||||
|
||||
enum class Token {
|
||||
NO_TOKEN,
|
||||
START,
|
||||
ID,
|
||||
EQUAL,
|
||||
TERMINAL_START,
|
||||
TERMINAL_END,
|
||||
ALTERNATION,
|
||||
RULE_END,
|
||||
IGNORED,
|
||||
END,
|
||||
FAILED,
|
||||
};
|
||||
|
||||
typedef int1 AlphabetType;
|
||||
typedef SimpleTokenizer<AlphabetType, Token, Token::NO_TOKEN, Token::FAILED, Token::END> CFGTokenizer;
|
||||
|
||||
struct State {
|
||||
|
||||
struct Transition {
|
||||
Token tok = Token::FAILED;
|
||||
State* target = nullptr;
|
||||
void (*action)(CfGrammar* grammar, const String& tok) = nullptr;
|
||||
};
|
||||
|
||||
String error;
|
||||
Buffer<Transition> transitions;
|
||||
bool accept = false;
|
||||
};
|
||||
|
||||
typedef Buffer<State> Automata;
|
||||
|
||||
void initializeTokenizer(CFGTokenizer* tok) {
|
||||
typedef decltype(tok->getTokenizer()) TokBase;
|
||||
tok->build({
|
||||
{ TokBase::idRE, Token::ID },
|
||||
{ TokBase::etherRE, Token::IGNORED },
|
||||
{ "Start", Token::START },
|
||||
{ ":", Token::EQUAL },
|
||||
{ "[", Token::TERMINAL_START },
|
||||
{ "]", Token::TERMINAL_END },
|
||||
{ "|", Token::ALTERNATION },
|
||||
{ ";", Token::RULE_END },
|
||||
});
|
||||
}
|
||||
|
||||
void initAutomata(Automata& automata) {
|
||||
automata.reserve(8);
|
||||
auto states = &automata.first();
|
||||
|
||||
auto root = states++;
|
||||
auto end = states++;
|
||||
auto start = states++;
|
||||
auto startEnd = states++;
|
||||
auto rule = states++;
|
||||
auto rhs = states++;
|
||||
auto terminalStart = states++;
|
||||
auto terminalEnd = states;
|
||||
|
||||
root->transitions = {
|
||||
{ Token::END, end },
|
||||
{ Token::START, start },
|
||||
{ Token::ID, rule, [](CfGrammar* grammar, const String& tok) {
|
||||
grammar->rules.pushBack({ tok, {} });
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
start->transitions = {
|
||||
{ Token::ID, startEnd},
|
||||
};
|
||||
|
||||
startEnd->transitions = {
|
||||
{ Token::RULE_END,
|
||||
root,
|
||||
[](CfGrammar* grammar, const String& tok) {
|
||||
grammar->startTerminal = tok;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
rule->transitions = {
|
||||
{ Token::EQUAL, rhs },
|
||||
};
|
||||
|
||||
rhs->transitions = {
|
||||
{ Token::ALTERNATION, rhs, [](CfGrammar* grammar, const String& tok) {
|
||||
auto const& id = grammar->rules.last()->data.id;
|
||||
grammar->rules.pushBack({ id, {} });
|
||||
}
|
||||
},
|
||||
{ Token::ID, rhs, [](CfGrammar* grammar, const String& tok) {
|
||||
grammar->rules.last()->data.args.pushBack({ tok, false });
|
||||
}
|
||||
},
|
||||
{ Token::TERMINAL_START, terminalStart },
|
||||
{ Token::RULE_END, root },
|
||||
};
|
||||
|
||||
terminalStart->transitions = {
|
||||
{ Token::ID, rhs, [](CfGrammar* grammar, const String& tok) {
|
||||
grammar->rules.last()->data.args.pushBack({ tok, true });
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
terminalEnd->transitions = {
|
||||
{ Token::TERMINAL_END, rhs },
|
||||
};
|
||||
|
||||
auto idError = "Expected an identifier";
|
||||
auto stmEndError = "Expected an statement end ';'";
|
||||
|
||||
root->error = "Expected 'Start' statement, rule or end of text";
|
||||
start->error = idError;
|
||||
startEnd->error = stmEndError;
|
||||
rule->error = "Expected production equality ':'";
|
||||
rhs->error = "Expected alternation '|', terminal id, statement end ';' or terminal start '[' ";
|
||||
terminalStart->error = idError;
|
||||
terminalEnd->error = "Expected terminal end '[' ";
|
||||
|
||||
end->accept = true;
|
||||
}
|
||||
|
||||
bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGrammar* grammar) {
|
||||
tok->reset();
|
||||
tok->bindSource(text);
|
||||
|
||||
State* state = &(*automata)[0];
|
||||
while (!state->accept) {
|
||||
|
||||
auto token = tok->readTok();
|
||||
|
||||
if (token == Token::FAILED) {
|
||||
auto errorLocation = tok->getCursorPrev().get2DLocation();
|
||||
printf("Syntax error at (%llu, %llu)\n", errorLocation.line, errorLocation.character);
|
||||
return false;
|
||||
}
|
||||
|
||||
auto tokVal = tok->extractVal();
|
||||
|
||||
bool isTransition = false;
|
||||
|
||||
for (auto transition : state->transitions) {
|
||||
if (transition->tok == token) {
|
||||
transition->action(grammar, tokVal);
|
||||
state = transition->target;
|
||||
isTransition = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!isTransition) {
|
||||
// if no tok matched to any transition produce error
|
||||
auto errorLocation = tok->getCursorPrev().get2DLocation();
|
||||
printf("Parse error at (%llu, %llu) - %s", errorLocation.line, errorLocation.character, state->error.read());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
struct tp::CfGrammarParserState {
|
||||
Automata automata;
|
||||
CFGTokenizer tokenizer;
|
||||
};
|
||||
|
||||
CfGrammarParserState* CfGrammar::initializeCfGrammarParser() {
|
||||
auto state = new CfGrammarParserState();
|
||||
initializeTokenizer(&state->tokenizer);
|
||||
initAutomata(state->automata);
|
||||
return state;
|
||||
}
|
||||
|
||||
void CfGrammar::deinitializeCfGrammarParser(CfGrammarParserState* state) {
|
||||
delete state;
|
||||
}
|
||||
|
||||
void CfGrammar::parse(CfGrammarParserState* context, const int1* source) {
|
||||
::parse(source, &context->automata, &context->tokenizer, this);
|
||||
}
|
||||
29
Parser/public/CfGrammar.hpp
Normal file
29
Parser/public/CfGrammar.hpp
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
#pragma once
|
||||
|
||||
#include "Strings.hpp"
|
||||
#include "List.hpp"
|
||||
|
||||
namespace tp {
|
||||
|
||||
struct CfGrammar {
|
||||
struct Rule {
|
||||
struct Arg {
|
||||
String id;
|
||||
bool isTerminal;
|
||||
};
|
||||
|
||||
String id;
|
||||
List<Arg> args;
|
||||
};
|
||||
|
||||
List<Rule> rules;
|
||||
String startTerminal;
|
||||
|
||||
public:
|
||||
static struct CfGrammarParserState* initializeCfGrammarParser();
|
||||
static void deinitializeCfGrammarParser(CfGrammarParserState*);
|
||||
|
||||
public:
|
||||
void parse(CfGrammarParserState* context, const int1* source);
|
||||
};
|
||||
}
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "Strings.hpp"
|
||||
#include "Map.hpp"
|
||||
#include "List.hpp"
|
||||
|
||||
namespace tp {
|
||||
|
||||
template <typename tAlphabetType>
|
||||
class ContextFreeGrammar {
|
||||
public:
|
||||
class NonTerminalSymbol;
|
||||
|
||||
class RHSArg {
|
||||
tAlphabetType terminal;
|
||||
const NonTerminalSymbol* nonTerminal = nullptr;
|
||||
public:
|
||||
RHSArg(const NonTerminalSymbol* aNonTerminal) : nonTerminal(aNonTerminal) {}
|
||||
RHSArg(tAlphabetType terminal) : terminal(terminal) {}
|
||||
};
|
||||
|
||||
class NonTerminalSymbol {
|
||||
String name;
|
||||
String description;
|
||||
|
||||
struct Rule {
|
||||
List<RHSArg> mArgs;
|
||||
};
|
||||
|
||||
List<Rule> mRules;
|
||||
|
||||
Rule* newRule() {
|
||||
mRules.pushBack();
|
||||
return mRules.last()->data;
|
||||
}
|
||||
};
|
||||
|
||||
private:
|
||||
Map<String, NonTerminalSymbol> mNonTerminals;
|
||||
NonTerminalSymbol* mStart;
|
||||
|
||||
public:
|
||||
ContextFreeGrammar() = default;
|
||||
|
||||
void setStartNonTerminal(const NonTerminalSymbol* start) {
|
||||
mStart = start;
|
||||
}
|
||||
|
||||
const NonTerminalSymbol* addRule(const String& LHS, const InitialierList<RHSArg> RHS) {
|
||||
NonTerminalSymbol* nonTerminal = nullptr;
|
||||
auto idx = mNonTerminals.presents(LHS);
|
||||
if (idx) {
|
||||
nonTerminal = mNonTerminals.get(idx);
|
||||
} else {
|
||||
nonTerminal = new NonTerminalSymbol(LHS, "None");
|
||||
mNonTerminals.put(LHS, nonTerminal);
|
||||
}
|
||||
|
||||
auto* rule = nonTerminal->newRule();
|
||||
for (auto arg : RHS) {
|
||||
rule->pushBack(arg);
|
||||
}
|
||||
|
||||
return nonTerminal;
|
||||
}
|
||||
|
||||
bool checkSemantic() {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool checkNonAmbiguous() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename tAlphabetType>
|
||||
class Parser {
|
||||
typedef ContextFreeGrammar<tAlphabetType> Grammar;
|
||||
|
||||
public:
|
||||
class ASTNode {
|
||||
const Grammar::NonTerminalSymbol* mNonTerminal;
|
||||
List<const ASTNode*> mLeafs;
|
||||
};
|
||||
|
||||
private:
|
||||
class InputTape {
|
||||
};
|
||||
|
||||
class StateStack {
|
||||
};
|
||||
|
||||
private:
|
||||
Grammar mGrammar;
|
||||
|
||||
public:
|
||||
Parser() = default;
|
||||
|
||||
Grammar& getGrammar() { return mGrammar; }
|
||||
|
||||
void buildTables() {}
|
||||
|
||||
const ASTNode* parse() {}
|
||||
};
|
||||
}
|
||||
|
|
@ -19,6 +19,15 @@ namespace tp {
|
|||
return mTransitionMatrix.isTrapped();
|
||||
}
|
||||
|
||||
public:
|
||||
// Some useful RE to be reused
|
||||
static constexpr const tAlphabetType* etherRE = "\n|\t| |\r";
|
||||
static constexpr const tAlphabetType* intRE = "((\\-)|(\\+))?[0-9]+i?";
|
||||
static constexpr const tAlphabetType* floatRE = R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)";
|
||||
static constexpr const tAlphabetType* commentBlockRE = R"((/\*){\*-\*}*(\*/))";
|
||||
static constexpr const tAlphabetType* stringRE = R"("{"-"}*")";
|
||||
static constexpr const tAlphabetType* idRE = "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*";
|
||||
|
||||
public:
|
||||
|
||||
Tokenizer() {
|
||||
|
|
@ -85,9 +94,30 @@ namespace tp {
|
|||
const tAlphabetType* mSource = nullptr;
|
||||
ualni mAdvancedOffset = 0;
|
||||
const tAlphabetType* str() { return mSource + mAdvancedOffset; }
|
||||
|
||||
struct Location2D {
|
||||
ualni line = 0;
|
||||
ualni character = 0;
|
||||
};
|
||||
|
||||
Location2D get2DLocation() {
|
||||
Location2D out;
|
||||
typedef typename StringLogic<tAlphabetType>::Index Idx;
|
||||
Buffer<Idx> offsets;
|
||||
StringLogic<tAlphabetType>::calcLineOffsets(mSource, StringLogic<tAlphabetType>::calcLength(mSource), offsets);
|
||||
for (ualni idx = 1; idx < offsets.size(); idx++) {
|
||||
if (offsets[idx] > mAdvancedOffset) {
|
||||
out.line = idx - 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
out.character = mAdvancedOffset - offsets[out.line];
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
SimpleTokenizer() = default;
|
||||
auto getTokenizer() const { return mTokenizer; }
|
||||
|
||||
void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) {
|
||||
mTokenizer.build(rules);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue