tmp
This commit is contained in:
Ilusha
parent
aa2d8f1200
commit
c2ad9ce501
32 changed files with 232 additions and 2023 deletions
|
|
@ -1,79 +0,0 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
/*
|
||||
* This file contains acceptors for each language
|
||||
* Acceptor is managing an automation machine that can recognize whether this given sentence is in the language.
|
||||
* Above that acceptors are responsible for defining those automations
|
||||
* Some acceptors can supply additional information such as Abstract-Syntax-Tree or Sentence id
|
||||
* */
|
||||
|
||||
#include "LanguageModule.hpp"
|
||||
|
||||
namespace tp {
|
||||
|
||||
template <typename tAlphabet>
|
||||
class SymbolSupply {
|
||||
public:
|
||||
typedef ualni Cursor;
|
||||
|
||||
public:
|
||||
SymbolSupply() = default;
|
||||
virtual ~SymbolSupply() = 0;
|
||||
|
||||
public:
|
||||
virtual Cursor getCursor() = 0;
|
||||
virtual void setCursor(Cursor) = 0;
|
||||
|
||||
virtual void getPrevCursor() = 0;
|
||||
virtual ualni getSymbolLength() = 0;
|
||||
|
||||
virtual const tAlphabet& getNextSymbol() = 0;
|
||||
|
||||
private:
|
||||
Cursor mCursor = 0;
|
||||
};
|
||||
|
||||
template <typename tAlphabet, typename tSentenceId>
|
||||
class RegularAcceptor {
|
||||
typedef SymbolSupply<tAlphabet> Sentence;
|
||||
|
||||
public:
|
||||
RegularAcceptor() = default;
|
||||
void build(const RegularGrammar<tSentenceId>& grammar);
|
||||
tSentenceId recognize(Sentence& sentence);
|
||||
};
|
||||
|
||||
// Subset of CFG
|
||||
template <typename tAlphabet>
|
||||
class LL {
|
||||
typedef SymbolSupply<tAlphabet> Sentence;
|
||||
|
||||
public:
|
||||
LL() = default;
|
||||
void build(const ContextFreeGrammar& grammar);
|
||||
void recognize(Sentence& sentence);
|
||||
};
|
||||
|
||||
// Subset of CFG much larger than LL
|
||||
template <typename tAlphabet>
|
||||
class CLR {
|
||||
typedef SymbolSupply<tAlphabet> Sentence;
|
||||
|
||||
public:
|
||||
CLR() = default;
|
||||
void build(const ContextFreeGrammar& grammar);
|
||||
void recognize(Sentence& sentence);
|
||||
};
|
||||
|
||||
// Subset of CFG CLR
|
||||
template <typename tAlphabet>
|
||||
class LALR {
|
||||
typedef SymbolSupply<tAlphabet> Sentence;
|
||||
|
||||
public:
|
||||
LALR() = default;
|
||||
void build(const ContextFreeGrammar& grammar);
|
||||
void recognize(Sentence& sentence);
|
||||
};
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
project(Parser)
|
||||
|
||||
### ---------------------- Static Library --------------------- ###
|
||||
file(GLOB SOURCES "./private/*.cpp")
|
||||
file(GLOB HEADERS "./public/*.hpp")
|
||||
add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
|
||||
target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
|
||||
target_link_libraries(${PROJECT_NAME} PUBLIC Tokenizer)
|
||||
|
||||
### -------------------------- Applications -------------------------- ###
|
||||
add_executable(cfg ./applications/Cfg.cpp)
|
||||
target_link_libraries(cfg ${PROJECT_NAME} CommandLine)
|
||||
|
||||
configure_file(rsc/grammar.txt grammar.txt COPYONLY)
|
||||
|
||||
### -------------------------- Tests -------------------------- ###
|
||||
enable_testing()
|
||||
file(GLOB TEST_SOURCES "./tests/*.cpp")
|
||||
add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES})
|
||||
target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils)
|
||||
add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests)
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
|
||||
#include "CommandLine.hpp"
|
||||
#include "Parser.hpp"
|
||||
|
||||
using namespace tp;
|
||||
|
||||
void run(const String& source) {
|
||||
auto state = CfGrammar::initializeCfGrammarParser();
|
||||
|
||||
CfGrammar grammar;
|
||||
|
||||
if (!grammar.parse(state, source)) {
|
||||
printf("Parsing is failed\n");
|
||||
CfGrammar::deinitializeCfGrammarParser(state);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!grammar.compile()) {
|
||||
printf("Compilation is failed\n");
|
||||
CfGrammar::deinitializeCfGrammarParser(state);
|
||||
return;
|
||||
}
|
||||
|
||||
printf("Grammar accepted.\n");
|
||||
|
||||
List<CfGrammar::Sentence> sentences;
|
||||
grammar.generateSentences(sentences);
|
||||
|
||||
printf("Example sentences formed from grammar: \n");
|
||||
for (auto sentence : sentences)
|
||||
tp::CfGrammar::printSentence(sentence.data());
|
||||
|
||||
CfGrammar::deinitializeCfGrammarParser(state);
|
||||
}
|
||||
|
||||
int main(int argc, const char* argv[]) {
|
||||
tp::ModuleManifest* deps[] = { &tp::gModuleParser, &tp::gModuleCommandLine, nullptr };
|
||||
tp::ModuleManifest testModule("CommandLineTest", nullptr, nullptr, deps);
|
||||
|
||||
if (!testModule.initialize()) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
{
|
||||
printf("Arguments given: ");
|
||||
for (auto idx = 0; idx < argc; idx++) {
|
||||
printf("[%s]", argv[idx]);
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
CommandLine cmd = {
|
||||
{ "grammar", CommandLine::Arg::STR },
|
||||
};
|
||||
|
||||
if (!cmd.parse((char) argc, argv, true, 1)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
auto location = cmd.getString("grammar");
|
||||
|
||||
LocalConnection file;
|
||||
String grammar;
|
||||
|
||||
file.connect(LocalConnection::Location(location), LocalConnection::Type(true));
|
||||
|
||||
if (!file.getConnectionStatus().isOpened()) {
|
||||
printf("Cant open file\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
auto size = file.size();
|
||||
grammar.resize((String::Index) size);
|
||||
file.readBytes(grammar.write(), size);
|
||||
|
||||
run(grammar);
|
||||
|
||||
file.disconnect();
|
||||
}
|
||||
|
||||
testModule.deinitialize();
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
#include "CLR.hpp"
|
||||
|
||||
using namespace tp;
|
||||
|
||||
void CLR::setGrammar(const tp::CfGrammar& grammar) {
|
||||
if (grammar.isLooped()) {
|
||||
mState = ParserState::FAILED;
|
||||
mBuildError.description = "Cant build lalr parser from looped context free grammar";
|
||||
return;
|
||||
}
|
||||
|
||||
mState = ParserState::FAILED;
|
||||
}
|
||||
|
||||
void CLR::setTerminal(const String& name, TerminalStream::TerminalID id) {}
|
||||
|
||||
void CLR::build() {}
|
||||
|
||||
void CLR::parse(TerminalStream* source, List<ASTNode>& out, ASTNode** root) { *root = nullptr; }
|
||||
|
|
@ -1,161 +0,0 @@
|
|||
|
||||
#include "CfGrammar.hpp"
|
||||
|
||||
#include "Timing.hpp"
|
||||
|
||||
using namespace tp;
|
||||
|
||||
void CfGrammar::generateSentences(List<Sentence>& out) {
|
||||
constexpr ualni maxTime = 1000;
|
||||
constexpr ualni maxSentences = 200;
|
||||
constexpr ualni maxQueue = 1000000;
|
||||
|
||||
List<Sentence> queue;
|
||||
Timer timer(maxTime);
|
||||
|
||||
// add start production
|
||||
Sentence start;
|
||||
start.terms.pushBack({ startTerminal, false });
|
||||
queue.pushBack(start);
|
||||
|
||||
PASS:
|
||||
|
||||
auto const sentential = &queue.first()->data;
|
||||
bool isSentence = true;
|
||||
ualni termIdx = 0;
|
||||
for (auto term : sentential->terms) {
|
||||
if (term->terminal) {
|
||||
termIdx++;
|
||||
continue;
|
||||
}
|
||||
|
||||
auto nonTerminal = &mNonTerminals.get(term->id);
|
||||
for (auto production : nonTerminal->rules) {
|
||||
queue.pushBack(*sentential);
|
||||
auto copy = &queue.last()->data;
|
||||
|
||||
auto appendTerm = copy->terms.findIdx(termIdx);
|
||||
auto deleteTerm = appendTerm;
|
||||
for (auto arg : production->args) {
|
||||
if (arg->isEpsilon) continue;
|
||||
auto newTerm = copy->terms.newNode({ arg->id, arg->isTerminal });
|
||||
copy->terms.attach(newTerm, appendTerm);
|
||||
appendTerm = newTerm;
|
||||
}
|
||||
copy->terms.removeNode(deleteTerm);
|
||||
}
|
||||
|
||||
isSentence = false;
|
||||
termIdx++;
|
||||
}
|
||||
|
||||
if (isSentence) {
|
||||
auto exists = false;
|
||||
for (auto iter : out) {
|
||||
if (iter->terms == sentential->terms) {
|
||||
exists = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!exists) out.pushBack(*sentential);
|
||||
}
|
||||
|
||||
queue.popFront();
|
||||
|
||||
bool stop = false;
|
||||
stop |= !queue.length();
|
||||
stop |= timer.isTimeout();
|
||||
stop |= queue.length() > maxQueue;
|
||||
stop |= out.length() > maxSentences;
|
||||
|
||||
if (!stop) goto PASS;
|
||||
}
|
||||
|
||||
void CfGrammar::printSentence(Sentence& in) {
|
||||
printf("[");
|
||||
for (auto term : in.terms) {
|
||||
printf(" %s", term->id.read());
|
||||
}
|
||||
printf(" ]\n");
|
||||
}
|
||||
|
||||
bool CfGrammar::compile() {
|
||||
|
||||
if (!rules.length()) {
|
||||
printf("Grammar must have at leas one rule\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (startTerminal == String()) {
|
||||
printf("Using first rule's non-terminal as grammar's root. Define explicitly - 'Start : id'\n");
|
||||
startTerminal = rules.first()->data.id;
|
||||
}
|
||||
|
||||
// find all rules
|
||||
for (auto rule : rules) {
|
||||
|
||||
if (!rule->args.length()) {
|
||||
printf("Rule must have at leas one terminal or non-terminal. See rule '%s'\n", rule->id.read());
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!mNonTerminals.presents(rule->id)) {
|
||||
mNonTerminals.put(rule->id, {});
|
||||
}
|
||||
auto nonTerminal = &mNonTerminals.get(rule->id);
|
||||
|
||||
nonTerminal->rules.pushBack(&rule.data());
|
||||
}
|
||||
|
||||
for (auto nonTerminal : mNonTerminals) {
|
||||
for (auto rule : nonTerminal->val.rules) {
|
||||
for (auto arg : rule->args) {
|
||||
|
||||
if (arg->isTerminal) {
|
||||
mTerminals.put(rule->id, {});
|
||||
} else if (!arg->isEpsilon) {
|
||||
auto idx = mNonTerminals.presents(arg->id);
|
||||
if (!idx) {
|
||||
printf("Referenced non-terminal '%s' is not defined\n", arg->id.read());
|
||||
return false;
|
||||
}
|
||||
auto reference = &mNonTerminals.getSlotVal(idx);
|
||||
reference->references.put(nonTerminal->key, &nonTerminal->val);
|
||||
nonTerminal->val.referencing.put(arg->id, reference);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<String> remove;
|
||||
for (auto nonTerminal : mNonTerminals) {
|
||||
if (!nonTerminal->val.references.size() && nonTerminal->key != startTerminal) {
|
||||
printf("Non-terminal '%s' is defined but not used\n", nonTerminal->key.read());
|
||||
remove.pushBack(nonTerminal->key);
|
||||
}
|
||||
}
|
||||
|
||||
for (auto rem : remove) {
|
||||
auto nonTerminal = &mNonTerminals.get(rem.data());
|
||||
for (auto referencing : nonTerminal->referencing) {
|
||||
referencing->val->references.remove(nonTerminal->rules.first()->data->id);
|
||||
}
|
||||
for (auto rule : nonTerminal->rules) {
|
||||
rules.removeNode(rules.find(*rule.data()));
|
||||
}
|
||||
}
|
||||
|
||||
for (auto nonTerminal : mNonTerminals) {
|
||||
if (!nonTerminal->val.isProductive()) {
|
||||
printf("Non-terminal '%s' is not productive\n", nonTerminal->val.rules.first()->data->id.read());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, ualni> processed;
|
||||
if (mNonTerminals.get(startTerminal).isLooped(processed, startTerminal)) {
|
||||
printf("Note that grammar is looped.\n");
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
@ -1,201 +0,0 @@
|
|||
|
||||
#include "CfGrammar.hpp"
|
||||
#include "Tokenizer.hpp"
|
||||
|
||||
using namespace tp;
|
||||
|
||||
enum class Token {
|
||||
NO_TOKEN,
|
||||
START,
|
||||
ID,
|
||||
EQUAL,
|
||||
TERMINAL_START,
|
||||
TERMINAL_END,
|
||||
ALTERNATION,
|
||||
RULE_END,
|
||||
IGNORED,
|
||||
EPSILON,
|
||||
COMMENT,
|
||||
END,
|
||||
FAILED,
|
||||
};
|
||||
|
||||
typedef int1 AlphabetType;
|
||||
typedef SimpleTokenizer<AlphabetType, Token, Token::NO_TOKEN, Token::FAILED, Token::END> CFGTokenizer;
|
||||
|
||||
struct State {
|
||||
|
||||
struct Transition {
|
||||
Token tok = Token::FAILED;
|
||||
State* target = nullptr;
|
||||
void (*action)(CfGrammar* grammar, const String& tok) = nullptr;
|
||||
};
|
||||
|
||||
String name;
|
||||
String error;
|
||||
Buffer<Transition> transitions;
|
||||
bool accept = false;
|
||||
};
|
||||
|
||||
typedef Buffer<State> Automata;
|
||||
|
||||
void initializeTokenizer(CFGTokenizer* tok) {
|
||||
tok->build({
|
||||
{ CFGTokenizer::tTokenizer::etherRE, Token::IGNORED },
|
||||
{ "Start", Token::START },
|
||||
{ ":", Token::EQUAL },
|
||||
{ "\\[", Token::TERMINAL_START },
|
||||
{ "\\]", Token::TERMINAL_END },
|
||||
{ "\\|", Token::ALTERNATION },
|
||||
{ ";", Token::RULE_END },
|
||||
{ "&", Token::EPSILON },
|
||||
{ CFGTokenizer::tTokenizer::idRE, Token::ID },
|
||||
{ CFGTokenizer::tTokenizer::commentBlockRE, Token::COMMENT },
|
||||
});
|
||||
|
||||
ASSERT(tok->isBuild())
|
||||
}
|
||||
|
||||
void initAutomata(Automata& automata) {
|
||||
automata.reserve(7);
|
||||
auto states = &automata.first();
|
||||
|
||||
auto root = states++;
|
||||
auto end = states++;
|
||||
auto start = states++;
|
||||
auto rule = states++;
|
||||
auto rhs = states++;
|
||||
auto terminalStart = states++;
|
||||
auto terminalEnd = states;
|
||||
|
||||
root->transitions = {
|
||||
{ Token::END, end },
|
||||
{ Token::START, start },
|
||||
{ Token::ID,
|
||||
rule,
|
||||
[](CfGrammar* grammar, const String& tok) {
|
||||
grammar->rules.pushBack({ tok, {} });
|
||||
} },
|
||||
};
|
||||
|
||||
start->transitions = {
|
||||
{ Token::ID, root, [](CfGrammar* grammar, const String& tok) { grammar->startTerminal = tok; } },
|
||||
};
|
||||
|
||||
rule->transitions = {
|
||||
{ Token::EQUAL, rhs },
|
||||
};
|
||||
|
||||
rhs->transitions = {
|
||||
{ Token::ALTERNATION,
|
||||
rhs,
|
||||
[](CfGrammar* grammar, const String& tok) {
|
||||
auto const& id = grammar->rules.last()->data.id;
|
||||
grammar->rules.pushBack({ id, {} });
|
||||
} },
|
||||
{ Token::ID,
|
||||
rhs,
|
||||
[](CfGrammar* grammar, const String& tok) {
|
||||
grammar->rules.last()->data.args.pushBack({ tok, false, false });
|
||||
} },
|
||||
{ Token::EPSILON,
|
||||
rhs,
|
||||
[](CfGrammar* grammar, const String& tok) {
|
||||
grammar->rules.last()->data.args.pushBack({ tok, false, true });
|
||||
} },
|
||||
{ Token::TERMINAL_START, terminalStart },
|
||||
{ Token::RULE_END, root },
|
||||
};
|
||||
|
||||
terminalStart->transitions = {
|
||||
{ Token::ID,
|
||||
terminalEnd,
|
||||
[](CfGrammar* grammar, const String& tok) {
|
||||
grammar->rules.last()->data.args.pushBack({ tok, true, false });
|
||||
} },
|
||||
};
|
||||
|
||||
terminalEnd->transitions = {
|
||||
{ Token::TERMINAL_END, rhs },
|
||||
};
|
||||
|
||||
auto idError = "Expected an identifier";
|
||||
auto stmEndError = "Expected an statement end ';'";
|
||||
|
||||
root->error = "Expected 'Start' statement, rule or end of text";
|
||||
start->error = idError;
|
||||
rule->error = "Expected production equality ':'";
|
||||
rhs->error = "Expected alternation '|', terminal id, statement end ';' or terminal start '[' ";
|
||||
terminalStart->error = idError;
|
||||
terminalEnd->error = "Expected terminal end '[' ";
|
||||
|
||||
root->name = "root";
|
||||
start->name = "start";
|
||||
rule->name = "production";
|
||||
rhs->name = "rhs";
|
||||
terminalStart->name = "terminalStart";
|
||||
terminalEnd->name = "terminalEnd";
|
||||
|
||||
end->accept = true;
|
||||
}
|
||||
|
||||
bool parse(const AlphabetType* text, Automata* automata, CFGTokenizer* tok, CfGrammar* grammar) {
|
||||
tok->reset();
|
||||
tok->bindSource(text);
|
||||
|
||||
State* state = &(*automata)[0];
|
||||
while (!state->accept) {
|
||||
|
||||
auto token = tok->readTok();
|
||||
|
||||
if (token == Token::IGNORED || token == Token::COMMENT) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (token == Token::FAILED) {
|
||||
auto errorLocation = tok->getCursorPrev().get2DLocation();
|
||||
printf("Syntax error at (%llu, %llu)\n", errorLocation.line, errorLocation.character);
|
||||
return false;
|
||||
}
|
||||
|
||||
auto tokVal = tok->extractVal();
|
||||
|
||||
bool isTransition = false;
|
||||
|
||||
for (auto transition : state->transitions) {
|
||||
if (transition->tok == token) {
|
||||
if (transition->action) transition->action(grammar, tokVal);
|
||||
state = transition->target;
|
||||
isTransition = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!isTransition) {
|
||||
// if no tok matched to any transition produce error
|
||||
auto errorLocation = tok->getCursorPrev().get2DLocation();
|
||||
printf("Parse error at (%llu, %llu) - %s", errorLocation.line, errorLocation.character, state->error.read());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
struct tp::CfGrammarParserState {
|
||||
Automata automata;
|
||||
CFGTokenizer tokenizer;
|
||||
};
|
||||
|
||||
CfGrammarParserState* CfGrammar::initializeCfGrammarParser() {
|
||||
auto state = new CfGrammarParserState();
|
||||
initializeTokenizer(&state->tokenizer);
|
||||
initAutomata(state->automata);
|
||||
return state;
|
||||
}
|
||||
|
||||
void CfGrammar::deinitializeCfGrammarParser(CfGrammarParserState* state) { delete state; }
|
||||
|
||||
bool CfGrammar::parse(CfGrammarParserState* context, const String& source) { return ::parse(source.read(), &context->automata, &context->tokenizer, this); }
|
||||
|
||||
bool CfGrammar::isLooped() const { return mIsLooped; }
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
#include "Parser.hpp"
|
||||
#include "Tokenizer.hpp"
|
||||
|
||||
using namespace tp;
|
||||
|
||||
static ModuleManifest* sModuleDependencies[] = { &gModuleTokenizer, nullptr };
|
||||
ModuleManifest tp::gModuleParser = ModuleManifest("CommandLine", nullptr, nullptr, sModuleDependencies);
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "CfGrammar.hpp"
|
||||
|
||||
namespace tp {
|
||||
class CLR {
|
||||
public:
|
||||
struct ASTNode {
|
||||
const CfGrammar::Rule* production = nullptr;
|
||||
Map<String, String> terminals;
|
||||
};
|
||||
|
||||
class Automation {
|
||||
public:
|
||||
class State {};
|
||||
};
|
||||
|
||||
class TerminalStream {
|
||||
public:
|
||||
typedef ualni TerminalID;
|
||||
|
||||
public:
|
||||
TerminalStream() = default;
|
||||
virtual TerminalID getNextTerminal() = 0;
|
||||
};
|
||||
|
||||
public:
|
||||
struct BuildError {
|
||||
String description;
|
||||
};
|
||||
|
||||
struct ParseError {
|
||||
const Automation::State* state = nullptr;
|
||||
String::Index location = 0;
|
||||
};
|
||||
|
||||
private:
|
||||
class SententialStack {};
|
||||
|
||||
public:
|
||||
CLR() = default;
|
||||
|
||||
void setGrammar(const CfGrammar& grammar);
|
||||
void setTerminal(const String& name, TerminalStream::TerminalID id);
|
||||
void build();
|
||||
|
||||
void parse(TerminalStream* stream, List<ASTNode>& out, ASTNode** root);
|
||||
|
||||
[[nodiscard]] bool isBuild() { return mState == ParserState::BUILD; }
|
||||
[[nodiscard]] const BuildError& getBuildError() { return mBuildError; }
|
||||
[[nodiscard]] const ParseError& getParseError() { return mParseError; }
|
||||
|
||||
private:
|
||||
enum class ParserState { NONE, BUILD, FAILED } mState = ParserState::NONE;
|
||||
|
||||
ParseError mParseError;
|
||||
BuildError mBuildError;
|
||||
|
||||
SententialStack mStack;
|
||||
Automation mAutomation;
|
||||
};
|
||||
}
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "List.hpp"
|
||||
#include "Map.hpp"
|
||||
#include "Strings.hpp"
|
||||
|
||||
namespace tp {
|
||||
|
||||
struct CfGrammar {
|
||||
struct Rule {
|
||||
struct Arg {
|
||||
String id;
|
||||
bool isTerminal = false;
|
||||
bool isEpsilon = false;
|
||||
|
||||
bool operator==(const Arg& in) const { return (id == in.id) && (isEpsilon == in.isEpsilon) && (isTerminal == in.isTerminal); }
|
||||
};
|
||||
|
||||
String id;
|
||||
List<Arg> args;
|
||||
|
||||
bool operator==(const Rule& in) const { return (id == in.id) && (args == in.args); }
|
||||
|
||||
[[nodiscard]] bool isProductive() const {
|
||||
for (auto arg : args) {
|
||||
if (arg->id == id) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
struct Sentence {
|
||||
struct Term {
|
||||
String id;
|
||||
bool terminal;
|
||||
bool operator==(const Term& in) const { return (id == in.id) && (terminal == in.terminal); }
|
||||
};
|
||||
List<Term> terms;
|
||||
};
|
||||
|
||||
List<Rule> rules;
|
||||
String startTerminal;
|
||||
|
||||
private:
|
||||
struct NonTerminal {
|
||||
|
||||
List<Rule*> rules;
|
||||
Map<String, NonTerminal*> references;
|
||||
Map<String, NonTerminal*> referencing;
|
||||
|
||||
[[nodiscard]] bool isProductive() const {
|
||||
for (auto rule : rules) {
|
||||
if (rule->isProductive()) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isLooped(Map<String, ualni>& processed, const String& id) const {
|
||||
for (auto ref : referencing) {
|
||||
if (processed.presents(ref->key)) return true;
|
||||
}
|
||||
processed.put(id, {});
|
||||
for (auto ref : referencing) {
|
||||
if (ref->val->isLooped(processed, ref->key)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
struct Terminal {};
|
||||
|
||||
Map<String, Terminal> mTerminals;
|
||||
Map<String, NonTerminal> mNonTerminals;
|
||||
bool mIsLooped = false;
|
||||
|
||||
public:
|
||||
static struct CfGrammarParserState* initializeCfGrammarParser();
|
||||
static void deinitializeCfGrammarParser(CfGrammarParserState*);
|
||||
|
||||
public:
|
||||
bool parse(CfGrammarParserState* context, const String& source);
|
||||
bool compile();
|
||||
|
||||
void generateSentences(List<Sentence>& out);
|
||||
static void printSentence(Sentence& in);
|
||||
|
||||
[[nodiscard]] bool isLooped() const;
|
||||
};
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "CLR.hpp"
|
||||
|
||||
namespace tp {
|
||||
extern ModuleManifest gModuleParser;
|
||||
}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
|
||||
/*
|
||||
Sample grammar.
|
||||
This grammar wil produce text in the form: TODO
|
||||
*/
|
||||
|
||||
/* This is the starting production of the grammar */
|
||||
|
||||
/* Simple Example
|
||||
Start A
|
||||
A : [ ID ] | &;
|
||||
*/
|
||||
|
||||
/* Lists Example */
|
||||
|
||||
Start List
|
||||
|
||||
List : List Stm | Stm;
|
||||
Stm : StmBody [ END ];
|
||||
StmBody: [ Stm1 ] | [ Stm2 ] | [ Stm3 ];
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
|
||||
#include "Parser.hpp"
|
||||
#include "Testing.hpp"
|
||||
|
||||
using namespace tp;
|
||||
|
||||
TEST_DEF(CLR) {
|
||||
// TEST(false);
|
||||
}
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
|
||||
#include "Parser.hpp"
|
||||
#include "Testing.hpp"
|
||||
|
||||
using namespace tp;
|
||||
|
||||
TEST_DEF(Grammar) {
|
||||
|
||||
String source = R"(
|
||||
Start Body
|
||||
Body : [ Function ] | List | &;
|
||||
List : [ LIST ];
|
||||
)";
|
||||
|
||||
auto state = CfGrammar::initializeCfGrammarParser();
|
||||
|
||||
CfGrammar grammar;
|
||||
|
||||
if (!grammar.parse(state, source)) {
|
||||
TEST(0 && "Parsing is failed\n");
|
||||
CfGrammar::deinitializeCfGrammarParser(state);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!grammar.compile()) {
|
||||
TEST(0 && "Compilation is failed\n");
|
||||
CfGrammar::deinitializeCfGrammarParser(state);
|
||||
return;
|
||||
}
|
||||
|
||||
CfGrammar::deinitializeCfGrammarParser(state);
|
||||
|
||||
printf("Grammar accepted.\n");
|
||||
|
||||
List<CfGrammar::Sentence> sentences;
|
||||
grammar.generateSentences(sentences);
|
||||
|
||||
TEST_ASSERT(sentences.length() == 3);
|
||||
TEST_ASSERT(sentences.first()->data.terms.length() == 1);
|
||||
TEST_ASSERT(sentences.last()->data.terms.length() == 1);
|
||||
|
||||
TEST(sentences.first()->data.terms.first()->data.id == "Function");
|
||||
TEST(sentences.last()->data.terms.first()->data.id == "LIST");
|
||||
|
||||
printf("Example sentences formed from grammar: \n");
|
||||
for (auto sentence : sentences)
|
||||
tp::CfGrammar::printSentence(sentence.data());
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
|
||||
#include "Parser.hpp"
|
||||
#include "Testing.hpp"
|
||||
|
||||
using namespace tp;
|
||||
|
||||
void testGrammar();
|
||||
void testCLR();
|
||||
|
||||
int main(int argc, const char* argv[]) {
|
||||
tp::ModuleManifest* deps[] = { &tp::gModuleParser, nullptr };
|
||||
tp::ModuleManifest testModule("CommandLineTest", nullptr, nullptr, deps);
|
||||
|
||||
if (!testModule.initialize()) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
testGrammar();
|
||||
testCLR();
|
||||
|
||||
testModule.deinitialize();
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
project(Tokenizer)
|
||||
|
||||
### ---------------------- Static Library --------------------- ###
|
||||
file(GLOB SOURCES "./private/*.cpp")
|
||||
file(GLOB HEADERS "./public/*.hpp")
|
||||
add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
|
||||
target_include_directories(${PROJECT_NAME} PUBLIC ./public/)
|
||||
target_link_libraries(${PROJECT_NAME} PUBLIC Strings)
|
||||
|
||||
### -------------------------- Tests -------------------------- ###
|
||||
enable_testing()
|
||||
file(GLOB TEST_SOURCES "./tests/*.cpp")
|
||||
add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES})
|
||||
target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils)
|
||||
add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests)
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
|
||||
#include "Tokenizer.hpp"
|
||||
|
||||
using namespace tp;
|
||||
|
||||
static ModuleManifest* sModuleDependencies[] = { &tp::gModuleStrings, nullptr };
|
||||
|
||||
void deinit(const ModuleManifest*) {}
|
||||
|
||||
ModuleManifest tp::gModuleTokenizer = ModuleManifest("Tokenizer", nullptr, deinit, sModuleDependencies);
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "List.hpp"
|
||||
#include "Buffer.hpp"
|
||||
#include "Buffer2D.hpp"
|
||||
#include "Utils.hpp"
|
||||
#include "Map.hpp"
|
||||
|
||||
#ifdef ENV_OS_WINDOWS
|
||||
#undef min
|
||||
#undef max
|
||||
#endif
|
||||
|
||||
namespace tp {
|
||||
|
||||
extern ModuleManifest gModuleTokenizer;
|
||||
|
||||
}
|
||||
|
|
@ -1,333 +0,0 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "AutomataGraph.h"
|
||||
|
||||
namespace tp {
|
||||
extern ModuleManifest gModuleTokenizer;
|
||||
}
|
||||
|
||||
namespace tp::RegEx {
|
||||
|
||||
struct AstNode {
|
||||
enum Type {
|
||||
NONE,
|
||||
ANY,
|
||||
OR,
|
||||
IF,
|
||||
CLASS,
|
||||
COMPOUND,
|
||||
REPEAT,
|
||||
VAL,
|
||||
} mType = NONE;
|
||||
|
||||
AstNode() = default;
|
||||
virtual ~AstNode() = default;
|
||||
};
|
||||
|
||||
template <typename tAlphabetType>
|
||||
struct AstVal : public AstNode {
|
||||
explicit AstVal(tAlphabetType val) :
|
||||
mVal(val) {
|
||||
mType = VAL;
|
||||
}
|
||||
~AstVal() override = default;
|
||||
tAlphabetType mVal;
|
||||
};
|
||||
|
||||
struct AstCompound : public AstNode {
|
||||
AstCompound() { mType = COMPOUND; }
|
||||
~AstCompound() override {
|
||||
for (auto iter : mChilds) {
|
||||
delete iter.data();
|
||||
}
|
||||
mChilds.removeAll();
|
||||
}
|
||||
List<AstNode*> mChilds;
|
||||
};
|
||||
|
||||
struct AstAlternation : public AstNode {
|
||||
AstAlternation() { mType = OR; }
|
||||
~AstAlternation() override {
|
||||
delete mFirst;
|
||||
delete mSecond;
|
||||
}
|
||||
AstNode* mFirst = nullptr;
|
||||
AstNode* mSecond = nullptr;
|
||||
};
|
||||
|
||||
struct AstIf : public AstNode {
|
||||
AstIf() { mType = IF; }
|
||||
~AstIf() override { delete mNode; }
|
||||
AstNode* mNode = nullptr;
|
||||
};
|
||||
|
||||
struct AstAny : public AstNode {
|
||||
AstAny() { mType = ANY; }
|
||||
~AstAny() override = default;
|
||||
};
|
||||
|
||||
struct AstRepetition : public AstNode {
|
||||
AstRepetition() { mType = REPEAT; }
|
||||
~AstRepetition() override { delete mNode; }
|
||||
AstNode* mNode = nullptr;
|
||||
bool mPlus = false;
|
||||
};
|
||||
|
||||
template <typename tAlphabetType>
|
||||
struct AstClass : public AstNode {
|
||||
AstClass() { mType = CLASS; }
|
||||
~AstClass() override { mRanges.removeAll(); }
|
||||
List<Range<tAlphabetType>> mRanges;
|
||||
bool mExclude = false;
|
||||
};
|
||||
|
||||
struct ParseError {
|
||||
const char* description = nullptr;
|
||||
uhalni offset = 0;
|
||||
[[nodiscard]] bool isError() const { return description != nullptr; }
|
||||
};
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal>
|
||||
class Parser {
|
||||
|
||||
enum TokType : uint1 {
|
||||
TOK_COMPOUND_START = 0,
|
||||
TOK_COMPOUND_END,
|
||||
TOK_CLASS_START,
|
||||
TOK_CLASS_END,
|
||||
TOK_CLASS_START_EXCLUDE,
|
||||
TOK_CLASS_END_EXCLUDE,
|
||||
TOK_OR,
|
||||
TOK_IF,
|
||||
TOK_ANY,
|
||||
TOK_REPEAT,
|
||||
TOK_REPEAT_PLUS,
|
||||
TOK_HYPHEN,
|
||||
TOK_SPECIALS_END_,
|
||||
TOK_VAL,
|
||||
TOK_NONE,
|
||||
};
|
||||
|
||||
tAlphabetType SpecialSymbols[TOK_SPECIALS_END_] = {
|
||||
'(', ')', '[', ']', '{', '}', '|', '?', '.', '*', '+', '-',
|
||||
};
|
||||
|
||||
tAlphabetType mEscapeSymbol = '\\';
|
||||
|
||||
struct Token {
|
||||
TokType type;
|
||||
tAlphabetType val;
|
||||
};
|
||||
|
||||
const tAlphabetType* mSource = nullptr;
|
||||
uhalni mOffset = 0;
|
||||
Token mCurToken;
|
||||
uhalni mTokLength = 0;
|
||||
|
||||
public:
|
||||
ParseError mError;
|
||||
|
||||
// regular expression must be a zero termination string
|
||||
AstCompound* parse(const tAlphabetType* regex) {
|
||||
mSource = regex;
|
||||
return parseRegEx();
|
||||
}
|
||||
|
||||
private:
|
||||
AstCompound* parseRegEx() {
|
||||
auto out = new AstCompound();
|
||||
for (AstNode* node = parseElement(); node; node = parseElement()) {
|
||||
out->mChilds.pushBack(node);
|
||||
}
|
||||
if (!out->mChilds.length()) {
|
||||
genError("Expected A Expression");
|
||||
}
|
||||
if (mError.description) {
|
||||
delete out;
|
||||
return nullptr;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
AstNode* parseElement() {
|
||||
AstNode* out = nullptr;
|
||||
switch (readTok().type) {
|
||||
case TOK_COMPOUND_START: out = parseCompound(); break;
|
||||
case TOK_CLASS_START: out = parseClass(); break;
|
||||
case TOK_CLASS_START_EXCLUDE: out = parseClass(true); break;
|
||||
case TOK_ANY: out = parseAny(); break;
|
||||
case TOK_VAL: out = parseVal(); break;
|
||||
case TOK_NONE:
|
||||
{
|
||||
discardTok();
|
||||
return nullptr;
|
||||
};
|
||||
default: break;
|
||||
}
|
||||
if (!out) {
|
||||
discardTok();
|
||||
return nullptr;
|
||||
}
|
||||
switch (readTok().type) {
|
||||
case TOK_OR: out = parseAlternation(out); break;
|
||||
case TOK_REPEAT: out = parseRepetition(out); break;
|
||||
case TOK_REPEAT_PLUS: out = parseRepetition(out, true); break;
|
||||
case TOK_IF: out = parseIf(out); break;
|
||||
case TOK_NONE: break;
|
||||
default:
|
||||
{
|
||||
discardTok();
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
AstCompound* parseCompound() {
|
||||
auto out = new AstCompound();
|
||||
for (AstNode* node = parseElement(); node; node = parseElement()) {
|
||||
out->mChilds.pushBack(node);
|
||||
}
|
||||
if (readTok().type != TOK_COMPOUND_END) {
|
||||
genError("Expected Compound End");
|
||||
}
|
||||
if (mError.description) {
|
||||
delete out;
|
||||
return nullptr;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
AstClass<tAlphabetType>* parseClass(bool exclude = false) {
|
||||
auto out = new AstClass<tAlphabetType>();
|
||||
out->mExclude = exclude;
|
||||
auto& ranges = out->mRanges;
|
||||
|
||||
readTok();
|
||||
|
||||
READ_VAL:
|
||||
if (mCurToken.type != TOK_VAL) {
|
||||
delete out;
|
||||
genError("Expected A Value");
|
||||
return nullptr;
|
||||
}
|
||||
char range_start = mCurToken.val;
|
||||
|
||||
readTok();
|
||||
if (mCurToken.type != TOK_HYPHEN) {
|
||||
delete out;
|
||||
genError("Expected A Range");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
readTok();
|
||||
if (mCurToken.type != TOK_VAL) {
|
||||
delete out;
|
||||
genError("Expected A Value");
|
||||
return nullptr;
|
||||
}
|
||||
char range_end = mCurToken.val;
|
||||
|
||||
ranges.pushBack({ range_start, range_end });
|
||||
|
||||
readTok();
|
||||
if ((mCurToken.type == TOK_CLASS_END && !exclude) || (mCurToken.type == TOK_CLASS_END_EXCLUDE && exclude)) {
|
||||
return out;
|
||||
} else {
|
||||
goto READ_VAL;
|
||||
}
|
||||
}
|
||||
|
||||
AstAny* parseAny() { return new AstAny(); }
|
||||
|
||||
AstVal<tAlphabetType>* parseVal() {
|
||||
auto out = new AstVal<tAlphabetType>(mCurToken.val);
|
||||
return out;
|
||||
}
|
||||
|
||||
AstAlternation* parseAlternation(AstNode* left) {
|
||||
auto right = parseElement();
|
||||
if (!right) {
|
||||
genError("Expected Alternation right Side");
|
||||
delete left;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto out = new AstAlternation();
|
||||
|
||||
out->mFirst = left;
|
||||
out->mSecond = right;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
AstRepetition* parseRepetition(AstNode* left, bool plus = false) {
|
||||
auto out = new AstRepetition();
|
||||
out->mNode = left;
|
||||
out->mPlus = plus;
|
||||
return out;
|
||||
}
|
||||
|
||||
AstIf* parseIf(AstNode* left) {
|
||||
auto out = new AstIf();
|
||||
out->mNode = left;
|
||||
return out;
|
||||
}
|
||||
|
||||
void genError(const char* desc) { mError = { desc, mOffset }; }
|
||||
|
||||
Token& readTok() {
|
||||
|
||||
const tAlphabetType* crs = mSource + mOffset;
|
||||
|
||||
// zero termination string
|
||||
if (*crs == 0) {
|
||||
mCurToken.type = TOK_NONE;
|
||||
return mCurToken;
|
||||
}
|
||||
|
||||
mTokLength = 1;
|
||||
mCurToken.type = TOK_VAL;
|
||||
mCurToken.val = crs[0];
|
||||
|
||||
if (crs[0] == mEscapeSymbol) {
|
||||
mCurToken.val = crs[1];
|
||||
mTokLength = 2;
|
||||
} else {
|
||||
for (uhalni tok = 0; tok < TOK_SPECIALS_END_; tok++) {
|
||||
if (SpecialSymbols[tok] == mCurToken.val) {
|
||||
mCurToken.type = TokType(tok);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mOffset += mTokLength;
|
||||
return mCurToken;
|
||||
}
|
||||
|
||||
void discardTok() {
|
||||
mOffset -= mTokLength;
|
||||
mTokLength = 0;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
CompileError<tStateType> compile(
|
||||
NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& out, const tAlphabetType* regex, tStateType state
|
||||
) {
|
||||
Compiler<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> compiler;
|
||||
compiler.compile(out, regex, state);
|
||||
return compiler.mError;
|
||||
}
|
||||
|
||||
template <typename tAlphabetType, typename tStateType, tStateType tNoStateVal, tStateType tFailedStateVal>
|
||||
CompileError<tStateType> compile(
|
||||
NFA<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal>& out,
|
||||
const InitialierList<Pair<const tAlphabetType*, tStateType>>& rules
|
||||
) {
|
||||
Compiler<tAlphabetType, tStateType, tNoStateVal, tFailedStateVal> compiler;
|
||||
compiler.compile(out, rules);
|
||||
return compiler.mError;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,184 +0,0 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "RegularExpression.h"
|
||||
#include "Strings.hpp"
|
||||
|
||||
namespace tp {
|
||||
|
||||
extern ModuleManifest gModuleTokenizer;
|
||||
|
||||
template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal>
|
||||
class Tokenizer {
|
||||
|
||||
TransitionMatrix<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> mTransitionMatrix;
|
||||
|
||||
RegEx::CompileError<tTokType> mError;
|
||||
|
||||
bool scanFailed() { return mTransitionMatrix.isTrapped(); }
|
||||
|
||||
public:
|
||||
// Some useful RE to be reused
|
||||
static constexpr const tAlphabetType* etherRE = "\n|\t| |\r";
|
||||
static constexpr const tAlphabetType* intRE = "((\\-)|(\\+))?[0-9]+i?";
|
||||
static constexpr const tAlphabetType* floatRE = R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)";
|
||||
static constexpr const tAlphabetType* commentBlockRE = R"((/\*){\*-\*}*(\*/))";
|
||||
static constexpr const tAlphabetType* stringRE = R"("{"-"}*")";
|
||||
static constexpr const tAlphabetType* idRE = "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*";
|
||||
|
||||
public:
|
||||
Tokenizer() { MODULE_SANITY_CHECK(gModuleTokenizer) }
|
||||
|
||||
void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) {
|
||||
NFA<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> nfa;
|
||||
|
||||
mError = RegEx::compile(nfa, rules);
|
||||
if (mError.isError()) {
|
||||
return;
|
||||
}
|
||||
|
||||
DFA<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> dfa(nfa);
|
||||
mTransitionMatrix.construct(dfa);
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isBuild() const { return !mError.isError(); }
|
||||
|
||||
auto getMatrix() const { return &mTransitionMatrix; }
|
||||
|
||||
const RegEx::CompileError<tTokType>& getBuildError() { return mError; }
|
||||
|
||||
void resetMatrix() { mTransitionMatrix.reset(); }
|
||||
|
||||
tTokType advanceSymbol(tAlphabetType symbol) { return mTransitionMatrix.move(symbol); }
|
||||
|
||||
tTokType advanceToken(const tAlphabetType* source, ualni source_len, ualni* token_len) {
|
||||
tTokType out = tNoTokVal;
|
||||
*token_len = 0;
|
||||
for (ualni idx = 0; idx < source_len; idx++) {
|
||||
out = advanceSymbol(source[idx]);
|
||||
if (out != tNoTokVal) {
|
||||
*token_len = idx;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
~Tokenizer() = default;
|
||||
};
|
||||
|
||||
template <typename tAlphabetType, typename tTokType, tTokType tNoTokVal, tTokType tFailedTokVal, tTokType tSourceEndTokVal>
|
||||
class SimpleTokenizer {
|
||||
public:
|
||||
typedef Tokenizer<tAlphabetType, tTokType, tNoTokVal, tFailedTokVal> tTokenizer;
|
||||
|
||||
private:
|
||||
tTokenizer mTokenizer;
|
||||
|
||||
const tAlphabetType* mSource = nullptr;
|
||||
ualni mLastTokLen = 0;
|
||||
ualni mSourceLen = 0;
|
||||
ualni mAdvancedOffset = 0;
|
||||
|
||||
public:
|
||||
struct Cursor {
|
||||
const tAlphabetType* mSource = nullptr;
|
||||
ualni mAdvancedOffset = 0;
|
||||
const tAlphabetType* str() { return mSource + mAdvancedOffset; }
|
||||
|
||||
struct Location2D {
|
||||
ualni line = 0;
|
||||
ualni character = 0;
|
||||
};
|
||||
|
||||
Location2D get2DLocation() {
|
||||
Location2D out;
|
||||
typedef typename StringLogic<tAlphabetType>::Index Idx;
|
||||
Buffer<Idx> offsets;
|
||||
StringLogic<tAlphabetType>::calcLineOffsets(mSource, StringLogic<tAlphabetType>::calcLength(mSource), offsets);
|
||||
for (ualni idx = 1; idx < offsets.size(); idx++) {
|
||||
if (offsets[idx] > mAdvancedOffset) {
|
||||
out.line = idx - 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
out.character = mAdvancedOffset - offsets[out.line];
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
SimpleTokenizer() = default;
|
||||
auto getTokenizer() const { return &mTokenizer; }
|
||||
|
||||
void build(const InitialierList<Pair<const tAlphabetType*, tTokType>>& rules) { mTokenizer.build(rules); }
|
||||
|
||||
[[nodiscard]] bool isBuild() const { return mTokenizer.isBuild(); }
|
||||
|
||||
const RegEx::CompileError<tTokType>& getBuildError() { return mTokenizer.getBuildError(); }
|
||||
|
||||
void bindSource(const tAlphabetType* source) {
|
||||
mSource = source;
|
||||
while (mSource[mSourceLen])
|
||||
mSourceLen++;
|
||||
mSourceLen++;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isInputLeft() const {
|
||||
if (!mSource) {
|
||||
return false;
|
||||
}
|
||||
if (mSource[mAdvancedOffset] == 0) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
Cursor getCursor() const { return { mSource, mAdvancedOffset }; }
|
||||
|
||||
Cursor getCursorPrev() const { return { mSource, mAdvancedOffset - mLastTokLen }; }
|
||||
|
||||
void setCursor(const Cursor& crs) {
|
||||
mAdvancedOffset = crs.mAdvancedOffset;
|
||||
mLastTokLen = 0;
|
||||
}
|
||||
|
||||
tTokType readTok() {
|
||||
if (mSourceLen == mAdvancedOffset + 1) {
|
||||
return tSourceEndTokVal;
|
||||
}
|
||||
|
||||
tTokType out = mTokenizer.advanceToken(mSource + mAdvancedOffset, mSourceLen - mAdvancedOffset, &mLastTokLen);
|
||||
mAdvancedOffset += mLastTokLen;
|
||||
return out;
|
||||
}
|
||||
|
||||
tTokType lookupTok() {
|
||||
auto out = readTok();
|
||||
discardTok();
|
||||
return out;
|
||||
}
|
||||
|
||||
void discardTok() {
|
||||
mTokenizer.resetMatrix();
|
||||
mAdvancedOffset -= mLastTokLen;
|
||||
mLastTokLen = 0;
|
||||
}
|
||||
|
||||
void skipTok() { readTok(); }
|
||||
|
||||
void reset() {
|
||||
mAdvancedOffset = mLastTokLen = 0;
|
||||
mTokenizer.resetMatrix();
|
||||
}
|
||||
|
||||
[[nodiscard]] ualni lastTokLEn() const { return mLastTokLen; }
|
||||
|
||||
String extractVal() {
|
||||
auto crs = getCursorPrev();
|
||||
String out;
|
||||
out.resize(mLastTokLen);
|
||||
memCopy(out.write(), crs.str(), mLastTokLen);
|
||||
return out;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -1,398 +0,0 @@
|
|||
// #include "NewPlacement.hpp"
|
||||
|
||||
#include "Testing.hpp"
|
||||
#include "Tokenizer.hpp"
|
||||
#include <cstdio>
|
||||
#include <iostream>
|
||||
|
||||
#define LOG(val) std::cout << #val << " "
|
||||
|
||||
using namespace tp;
|
||||
|
||||
ualni outputHashPassed = 1156;
|
||||
|
||||
const char* script =
|
||||
R"(
|
||||
|
||||
/* yea
|
||||
comment "and string inside" with int 123 and float 0.123f */
|
||||
|
||||
-1.f;
|
||||
|
||||
if (+1.f) {
|
||||
var string = "string value some &&characters inside string&& -= asdf 09a
|
||||
another line";
|
||||
|
||||
}
|
||||
|
||||
while
|
||||
|
||||
return
|
||||
|
||||
)";
|
||||
|
||||
TEST_DEF_STATIC(General) {
|
||||
|
||||
enum class TokType {
|
||||
NONE,
|
||||
FAILED,
|
||||
TOK_SOURCE_END,
|
||||
|
||||
VAR,
|
||||
CLASS_DEF,
|
||||
SELF,
|
||||
SCOPE_IN,
|
||||
SCOPE_OUT,
|
||||
ASSIGN,
|
||||
DEF_FUNC,
|
||||
PRINT,
|
||||
IF,
|
||||
ELSE,
|
||||
WHILE,
|
||||
BRACKET_IN,
|
||||
BRACKET_OUT,
|
||||
COMMA,
|
||||
NEW,
|
||||
CHILD,
|
||||
RETURN,
|
||||
EQUAL,
|
||||
NOT_EQUAL,
|
||||
MORE,
|
||||
LESS,
|
||||
QE_OR_MORE,
|
||||
QE_OR_LESS,
|
||||
BOOL_NOT,
|
||||
BOOL_AND,
|
||||
BOOL_OR,
|
||||
ADD,
|
||||
MUL,
|
||||
SUB,
|
||||
DIV,
|
||||
STM_END,
|
||||
CONST_TRUE,
|
||||
CONST_FALSE,
|
||||
CONST_INT,
|
||||
CONST_FLOAT,
|
||||
SPACE,
|
||||
CONST_STRING,
|
||||
ID,
|
||||
|
||||
// COMMENT_LINE,
|
||||
|
||||
COMMENT_BLOCK,
|
||||
};
|
||||
|
||||
SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
|
||||
|
||||
lexer.build({
|
||||
{ "\n|\t| |\r", TokType::SPACE },
|
||||
{ "var", TokType::VAR },
|
||||
{ "class", TokType::CLASS_DEF },
|
||||
{ "self", TokType::SELF },
|
||||
{ "\\{", TokType::SCOPE_IN },
|
||||
{ "\\}", TokType::SCOPE_OUT },
|
||||
{ "=", TokType::ASSIGN },
|
||||
{ "def", TokType::DEF_FUNC },
|
||||
{ "<<", TokType::PRINT },
|
||||
{ "if", TokType::IF },
|
||||
{ "else", TokType::ELSE },
|
||||
{ "while", TokType::WHILE },
|
||||
{ "\\(", TokType::BRACKET_IN },
|
||||
{ "\\)", TokType::BRACKET_OUT },
|
||||
{ ",", TokType::COMMA },
|
||||
{ "new", TokType::NEW },
|
||||
{ "\\.", TokType::CHILD },
|
||||
{ "return", TokType::RETURN },
|
||||
{ "==", TokType::EQUAL },
|
||||
{ "!=", TokType::NOT_EQUAL },
|
||||
{ ">", TokType::MORE },
|
||||
{ "<", TokType::LESS },
|
||||
{ ">=", TokType::QE_OR_MORE },
|
||||
{ "<=", TokType::QE_OR_LESS },
|
||||
{ "!", TokType::BOOL_NOT },
|
||||
{ "&&", TokType::BOOL_AND },
|
||||
{ "\\|\\|", TokType::BOOL_OR },
|
||||
{ "\\+", TokType::ADD },
|
||||
{ "\\*", TokType::MUL },
|
||||
{ "\\-", TokType::SUB },
|
||||
{ "/", TokType::DIV },
|
||||
{ ";", TokType::STM_END },
|
||||
{ "true", TokType::CONST_TRUE },
|
||||
{ "false", TokType::CONST_FALSE },
|
||||
{ "((\\-)|(\\+))?[0-9]+i?", TokType::CONST_INT },
|
||||
{ R"(((\-)|(\+))?([0-9]+)(\.)([0-9]*)?f?)", TokType::CONST_FLOAT },
|
||||
{ R"((/\*){\*-\*}*(\*/))", TokType::COMMENT_BLOCK },
|
||||
{ R"("{"-"}*")", TokType::CONST_STRING },
|
||||
{ "([a-z]|[A-Z]|_)+([a-z]|[A-Z]|[0-9]|_)*", TokType::ID },
|
||||
});
|
||||
|
||||
if (!lexer.isBuild()) {
|
||||
printf("Error : %s", lexer.getBuildError().description);
|
||||
return;
|
||||
}
|
||||
|
||||
lexer.bindSource(script);
|
||||
|
||||
TokType tok;
|
||||
ualni outputHash = 0;
|
||||
do {
|
||||
|
||||
tok = lexer.readTok();
|
||||
|
||||
outputHash += ualni(tok);
|
||||
|
||||
switch (tok) {
|
||||
case TokType::SPACE:
|
||||
{
|
||||
printf(" ");
|
||||
}
|
||||
break;
|
||||
case TokType::VAR:
|
||||
{
|
||||
LOG(VAR);
|
||||
}
|
||||
break;
|
||||
case TokType::CLASS_DEF:
|
||||
{
|
||||
LOG(CLASS_DEF);
|
||||
}
|
||||
break;
|
||||
case TokType::SELF:
|
||||
{
|
||||
LOG(SELF);
|
||||
}
|
||||
break;
|
||||
case TokType::SCOPE_IN:
|
||||
{
|
||||
LOG(SCOPE_IN);
|
||||
}
|
||||
break;
|
||||
case TokType::SCOPE_OUT:
|
||||
{
|
||||
LOG(SCOPE_OUT);
|
||||
}
|
||||
break;
|
||||
case TokType::ASSIGN:
|
||||
{
|
||||
LOG(ASSIGN);
|
||||
}
|
||||
break;
|
||||
case TokType::DEF_FUNC:
|
||||
{
|
||||
LOG(DEF_FUNC);
|
||||
}
|
||||
break;
|
||||
case TokType::PRINT:
|
||||
{
|
||||
LOG(PRINT);
|
||||
}
|
||||
break;
|
||||
case TokType::IF:
|
||||
{
|
||||
LOG(IF);
|
||||
}
|
||||
break;
|
||||
case TokType::ELSE:
|
||||
{
|
||||
LOG(ELSE);
|
||||
}
|
||||
break;
|
||||
case TokType::WHILE:
|
||||
{
|
||||
LOG(WHILE);
|
||||
}
|
||||
break;
|
||||
case TokType::BRACKET_IN:
|
||||
{
|
||||
LOG(BRACKET_IN);
|
||||
}
|
||||
break;
|
||||
case TokType::BRACKET_OUT:
|
||||
{
|
||||
LOG(BRACKET_OUT);
|
||||
}
|
||||
break;
|
||||
case TokType::COMMA:
|
||||
{
|
||||
LOG(COMMA);
|
||||
}
|
||||
break;
|
||||
case TokType::NEW:
|
||||
{
|
||||
LOG(NEW);
|
||||
}
|
||||
break;
|
||||
case TokType::CHILD:
|
||||
{
|
||||
LOG(CHILD);
|
||||
}
|
||||
break;
|
||||
case TokType::RETURN:
|
||||
{
|
||||
LOG(RETURN);
|
||||
}
|
||||
break;
|
||||
case TokType::EQUAL:
|
||||
{
|
||||
LOG(EQUAL);
|
||||
}
|
||||
break;
|
||||
case TokType::NOT_EQUAL:
|
||||
{
|
||||
LOG(NOT_EQUAL);
|
||||
}
|
||||
break;
|
||||
case TokType::MORE:
|
||||
{
|
||||
LOG(MORE);
|
||||
}
|
||||
break;
|
||||
case TokType::LESS:
|
||||
{
|
||||
LOG(LESS);
|
||||
}
|
||||
break;
|
||||
case TokType::QE_OR_MORE:
|
||||
{
|
||||
LOG(QE_OR_MORE);
|
||||
}
|
||||
break;
|
||||
case TokType::QE_OR_LESS:
|
||||
{
|
||||
LOG(QE_OR_LESS);
|
||||
}
|
||||
break;
|
||||
case TokType::BOOL_NOT:
|
||||
{
|
||||
LOG(BOOL_NOT);
|
||||
}
|
||||
break;
|
||||
case TokType::BOOL_AND:
|
||||
{
|
||||
LOG(BOOL_AND);
|
||||
}
|
||||
break;
|
||||
case TokType::BOOL_OR:
|
||||
{
|
||||
LOG(BOOL_OR);
|
||||
}
|
||||
break;
|
||||
case TokType::ADD:
|
||||
{
|
||||
LOG(ADD);
|
||||
}
|
||||
break;
|
||||
case TokType::MUL:
|
||||
{
|
||||
LOG(MUL);
|
||||
}
|
||||
break;
|
||||
case TokType::SUB:
|
||||
{
|
||||
LOG(SUB);
|
||||
}
|
||||
break;
|
||||
case TokType::DIV:
|
||||
{
|
||||
LOG(DIV);
|
||||
}
|
||||
break;
|
||||
case TokType::STM_END:
|
||||
{
|
||||
LOG(STM_END);
|
||||
}
|
||||
break;
|
||||
case TokType::CONST_TRUE:
|
||||
{
|
||||
LOG(TRUE);
|
||||
}
|
||||
break;
|
||||
case TokType::CONST_FALSE:
|
||||
{
|
||||
LOG(FALSE);
|
||||
}
|
||||
break;
|
||||
case TokType::CONST_INT:
|
||||
{
|
||||
LOG(INT);
|
||||
}
|
||||
break;
|
||||
case TokType::CONST_FLOAT:
|
||||
{
|
||||
LOG(FLOAT);
|
||||
}
|
||||
break;
|
||||
case TokType::CONST_STRING:
|
||||
{
|
||||
LOG(STRING);
|
||||
}
|
||||
break;
|
||||
case TokType::ID:
|
||||
{
|
||||
LOG(ID);
|
||||
}
|
||||
break;
|
||||
// case TokType::COMMENT_LINE: { LOG(COMMENT_LINE) } break;
|
||||
case TokType::COMMENT_BLOCK:
|
||||
{
|
||||
LOG(COMMENT_BLOCK);
|
||||
}
|
||||
break;
|
||||
case TokType::FAILED:
|
||||
{
|
||||
LOG(FAILED);
|
||||
}
|
||||
break;
|
||||
case TokType::NONE:
|
||||
case TokType::TOK_SOURCE_END: break;
|
||||
}
|
||||
|
||||
} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
|
||||
|
||||
printf("\n\nOutputHash : %llu", outputHash);
|
||||
|
||||
TEST(outputHash == outputHashPassed);
|
||||
}
|
||||
|
||||
TEST_DEF(Simple) {
|
||||
enum class TokType {
|
||||
START = 0,
|
||||
NONE = 1,
|
||||
FAILED = 2,
|
||||
SPACE = 3,
|
||||
ID = 4,
|
||||
COMMENT_BLOCK = 5,
|
||||
TOK_SOURCE_END = 6,
|
||||
};
|
||||
|
||||
SimpleTokenizer<char, TokType, TokType::NONE, TokType::FAILED, TokType::TOK_SOURCE_END> lexer;
|
||||
|
||||
lexer.build({
|
||||
{ " ", TokType::SPACE },
|
||||
{ "([a-c]|A)+", TokType::ID },
|
||||
{ "A", TokType::START },
|
||||
});
|
||||
|
||||
if (!lexer.isBuild()) {
|
||||
printf("Error : %s", lexer.getBuildError().description);
|
||||
return;
|
||||
}
|
||||
|
||||
lexer.bindSource(" A bc cb cc ");
|
||||
|
||||
TokType tok;
|
||||
ualni outputHash = 0;
|
||||
printf("\n\n OutputHash : ");
|
||||
do {
|
||||
tok = lexer.readTok();
|
||||
outputHash += ualni(tok);
|
||||
printf(" %i ", int(tok));
|
||||
} while (tok != TokType::TOK_SOURCE_END && tok != TokType::FAILED);
|
||||
printf(" : %llu", outputHash);
|
||||
TEST(outputHash == 33);
|
||||
}
|
||||
|
||||
TEST_DEF(Tokenizer) {
|
||||
testGeneral();
|
||||
testSimple();
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
|
||||
#include "Testing.hpp"
|
||||
#include "Tokenizer.hpp"
|
||||
|
||||
void testTokenizer();
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
|
||||
tp::ModuleManifest* ModuleDependencies[] = { &tp::gModuleTokenizer, nullptr };
|
||||
tp::ModuleManifest TestModule("TokenizerTest", nullptr, nullptr, ModuleDependencies);
|
||||
if (!TestModule.initialize()) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
testTokenizer();
|
||||
|
||||
TestModule.deinitialize();
|
||||
}
|
||||
1
Externals/CMakeLists.txt
vendored
1
Externals/CMakeLists.txt
vendored
|
|
@ -29,6 +29,7 @@ endif()
|
|||
#add_subdirectory(glew/build/cmake/)
|
||||
#target_compile_definitions(glew_s PUBLIC GLEW_NO_GLU)
|
||||
|
||||
add_subdirectory(unittest-cpp)
|
||||
|
||||
add_subdirectory(lalr)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
project(Objects)
|
||||
|
||||
### ---------------------- Externals --------------------- ###
|
||||
|
||||
### ---------------------- Static Library --------------------- ###
|
||||
file(GLOB SOURCES "./private/*.cpp" "./private/*/*.cpp")
|
||||
file(GLOB HEADERS "./public/*.hpp" "./public/*/*.hpp" "./applications/*.hpp")
|
||||
|
|
@ -26,5 +24,5 @@ file(COPY "rsc/Font.ttf" DESTINATION "${CMAKE_BINARY_DIR}/${PROJECT_NAME}/")
|
|||
### -------------------------- Tests -------------------------- ###
|
||||
file(GLOB TEST_SOURCES "./tests/*.cpp" "./tests/*/*.cpp")
|
||||
add_executable(${PROJECT_NAME}Tests ${TEST_SOURCES})
|
||||
target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} Utils)
|
||||
target_link_libraries(${PROJECT_NAME}Tests ${PROJECT_NAME} UnitTest++)
|
||||
add_test(NAME ${PROJECT_NAME}Tests COMMAND ${PROJECT_NAME}Tests)
|
||||
|
|
@ -11,6 +11,7 @@ bool uninit_flag = false;
|
|||
void NullObject::uninit() {
|
||||
uninit_flag = true;
|
||||
NDO->destroy(NdoNull_globalInstance);
|
||||
NdoNull_globalInstance = nullptr;
|
||||
}
|
||||
|
||||
void NullObject::destructor(Object* self) { DEBUG_ASSERT(uninit_flag && "Only one the instance of NullObject exists and thus it can't be destroyed"); }
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
|
||||
#include "Testing.hpp"
|
||||
#include "Module.hpp"
|
||||
|
||||
#include "primitives/primitives.h"
|
||||
#include <UnitTest++/UnitTest++.h>
|
||||
#include <UnitTest++/TestReporterStdout.h>
|
||||
|
||||
extern tp::ModuleManifest objTestModule;
|
||||
|
||||
#define TEST_OFF(name) static void name()
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
#include "ObjectTests.hpp"
|
||||
|
||||
#include "compiler/function.h"
|
||||
#include "parser/parser.h"
|
||||
#include "core/object.h"
|
||||
#include "interpreter/interpreter.h"
|
||||
#include "primitives/interpreterobject.h"
|
||||
|
|
@ -11,7 +12,10 @@
|
|||
using namespace tp;
|
||||
using namespace obj;
|
||||
|
||||
TEST_DEF_STATIC(Basic) {
|
||||
SUITE(Compiler) {
|
||||
TEST(Basic) {
|
||||
objTestModule.initialize();
|
||||
|
||||
{
|
||||
// no errors
|
||||
auto method = NDO_CAST(MethodObject, NDO->create("method"));
|
||||
|
|
@ -35,23 +39,24 @@ TEST_DEF_STATIC(Basic) {
|
|||
|
||||
assertNoLeaks();
|
||||
}
|
||||
|
||||
objTestModule.deinitialize();
|
||||
}
|
||||
|
||||
TEST_DEF_STATIC(ErrorHandling) {
|
||||
Parser parser;
|
||||
/*
|
||||
TEST(ErrorHandling) {
|
||||
objTestModule.initialize();
|
||||
|
||||
obj::Parser parser;
|
||||
|
||||
String stream = "var i = true; print (i + 1) * 10; invalidCharacter ";
|
||||
auto res = parser.parse(stream);
|
||||
|
||||
TEST(res.isError);
|
||||
CHECK(res.isError);
|
||||
|
||||
delete res.scope;
|
||||
}
|
||||
|
||||
TEST_DEF(Compiler) {
|
||||
if (objTestModule.initialize()) {
|
||||
testBasic();
|
||||
testErrorHandling();
|
||||
objTestModule.deinitialize();
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
|
@ -8,7 +8,11 @@
|
|||
using namespace tp;
|
||||
using namespace obj;
|
||||
|
||||
TEST_DEF_STATIC(BasicAPI) {
|
||||
SUITE(Core) {
|
||||
TEST(BasicAPI) {
|
||||
objTestModule.initialize();
|
||||
|
||||
{
|
||||
auto integer = NDO_CAST(IntObject, NDO->create("int"));
|
||||
|
||||
integer->val = 10;
|
||||
|
|
@ -20,17 +24,14 @@ TEST_DEF_STATIC(BasicAPI) {
|
|||
|
||||
printf("%s\n", NDO->toString(savedInt).read());
|
||||
|
||||
TEST(NDO->compare(integer, savedInt));
|
||||
TEST(NDO_CAST(IntObject, savedInt));
|
||||
TEST(integer->val == NDO_CAST(IntObject, savedInt)->val);
|
||||
CHECK(NDO->compare(integer, savedInt));
|
||||
CHECK(NDO_CAST(IntObject, savedInt));
|
||||
CHECK(integer->val == NDO_CAST(IntObject, savedInt)->val);
|
||||
|
||||
NDO->destroy(integer);
|
||||
NDO->destroy(savedInt);
|
||||
}
|
||||
|
||||
TEST_DEF(Core) {
|
||||
if (objTestModule.initialize()) {
|
||||
testBasicAPI();
|
||||
objTestModule.deinitialize();
|
||||
}
|
||||
}
|
||||
|
|
@ -44,7 +44,11 @@ auto script = R"(
|
|||
print false;
|
||||
)";
|
||||
|
||||
TEST_DEF_STATIC(Entry) {
|
||||
SUITE(Interpreter) {
|
||||
TEST(Entry) {
|
||||
objTestModule.initialize();
|
||||
|
||||
{
|
||||
auto method = NDO_CAST(MethodObject, NDO->create("method"));
|
||||
auto interpreter = NDO_CAST(InterpreterObject, NDO->create("interpreter"));
|
||||
|
||||
|
|
@ -60,7 +64,13 @@ TEST_DEF_STATIC(Entry) {
|
|||
printf("\n");
|
||||
}
|
||||
|
||||
TEST_DEF_STATIC(SimpleSave) {
|
||||
objTestModule.deinitialize();
|
||||
}
|
||||
|
||||
TEST(SimpleSave) {
|
||||
objTestModule.initialize();
|
||||
|
||||
{
|
||||
auto method = NDO_CAST(MethodObject, NDO->create("method"));
|
||||
auto interpreter = NDO_CAST(InterpreterObject, NDO->create("interpreter"));
|
||||
|
||||
|
|
@ -83,7 +93,13 @@ TEST_DEF_STATIC(SimpleSave) {
|
|||
printf("\n");
|
||||
}
|
||||
|
||||
TEST_DEF_STATIC(Simple) {
|
||||
objTestModule.deinitialize();
|
||||
}
|
||||
|
||||
TEST_OFF(Simple) {
|
||||
objTestModule.initialize();
|
||||
|
||||
{
|
||||
auto compileStartCount = getObjCount();
|
||||
|
||||
auto method = NDO_CAST(MethodObject, NDO->create("method"));
|
||||
|
|
@ -99,7 +115,7 @@ TEST_DEF_STATIC(Simple) {
|
|||
interpreter->exec();
|
||||
|
||||
if (getObjCount() != startCount) {
|
||||
TEST(false && "Mem leaks in interpreter");
|
||||
CHECK(false && "Mem leaks in interpreter");
|
||||
}
|
||||
|
||||
printf("\n");
|
||||
|
|
@ -116,11 +132,17 @@ TEST_DEF_STATIC(Simple) {
|
|||
NDO->destroy(interpreter);
|
||||
|
||||
if (getObjCount() != compileStartCount) {
|
||||
TEST(false && "Mem leaks in compiler and interpreter");
|
||||
CHECK(false && "Mem leaks in compiler and interpreter");
|
||||
}
|
||||
}
|
||||
|
||||
TEST_DEF_STATIC(Complex) {
|
||||
objTestModule.deinitialize();
|
||||
}
|
||||
|
||||
TEST(Complex) {
|
||||
objTestModule.initialize();
|
||||
|
||||
{
|
||||
auto method = NDO_CAST(MethodObject, NDO->create("method"));
|
||||
auto interpreter = NDO_CAST(InterpreterObject, NDO->create("interpreter"));
|
||||
|
||||
|
|
@ -134,14 +156,6 @@ TEST_DEF_STATIC(Complex) {
|
|||
NDO->destroy(interpreter);
|
||||
}
|
||||
|
||||
TEST_DEF(Interpreter) {
|
||||
if (objTestModule.initialize()) {
|
||||
|
||||
testEntry();
|
||||
testSimple();
|
||||
testSimpleSave();
|
||||
// testComplex();
|
||||
|
||||
objTestModule.deinitialize();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,14 +6,17 @@
|
|||
using namespace tp;
|
||||
using namespace obj;
|
||||
|
||||
TEST_DEF_STATIC(Basic) {
|
||||
SUITE(Parser) {
|
||||
TEST(Basic) {
|
||||
objTestModule.initialize();
|
||||
|
||||
{
|
||||
Parser parser;
|
||||
|
||||
String stream = "";
|
||||
auto res = parser.parse(stream);
|
||||
|
||||
TEST(!res.isError);
|
||||
CHECK(!res.isError);
|
||||
|
||||
delete res.scope;
|
||||
}
|
||||
|
|
@ -24,7 +27,7 @@ TEST_DEF_STATIC(Basic) {
|
|||
String stream = "var i = true;";
|
||||
auto res = parser.parse(stream);
|
||||
|
||||
TEST(!res.isError);
|
||||
CHECK(!res.isError);
|
||||
|
||||
delete res.scope;
|
||||
}
|
||||
|
|
@ -35,29 +38,27 @@ TEST_DEF_STATIC(Basic) {
|
|||
String stream = "var i = true; print (i + 1) * 10;";
|
||||
auto res = parser.parse(stream);
|
||||
|
||||
TEST(!res.isError);
|
||||
CHECK(!res.isError);
|
||||
|
||||
delete res.scope;
|
||||
}
|
||||
|
||||
objTestModule.deinitialize();
|
||||
}
|
||||
|
||||
TEST_DEF_STATIC(ErrorHandling) {
|
||||
Parser parser;
|
||||
TEST(ErrorHandling) {
|
||||
objTestModule.initialize();
|
||||
|
||||
{
|
||||
Parser parser;
|
||||
String stream = "var i = true; print (i + 1) * 10; invalidCharacter ";
|
||||
auto res = parser.parse(stream);
|
||||
|
||||
TEST(res.isError);
|
||||
CHECK(res.isError);
|
||||
|
||||
delete res.scope;
|
||||
}
|
||||
|
||||
TEST_DEF(Parser) {
|
||||
if (objTestModule.initialize()) {
|
||||
|
||||
testBasic();
|
||||
testErrorHandling();
|
||||
|
||||
objTestModule.deinitialize();
|
||||
}
|
||||
}
|
||||
|
|
@ -11,7 +11,11 @@
|
|||
using namespace tp;
|
||||
using namespace obj;
|
||||
|
||||
TEST_DEF_STATIC(Dict) {
|
||||
SUITE(PrimitiveObjects) {
|
||||
TEST(Dict) {
|
||||
objTestModule.initialize();
|
||||
|
||||
{
|
||||
auto integer = NDO_CAST(IntObject, NDO->create("int"));
|
||||
integer->val = 10;
|
||||
|
||||
|
|
@ -23,16 +27,13 @@ TEST_DEF_STATIC(Dict) {
|
|||
|
||||
auto dictLoaded = NDO_CAST(DictObject, NDO->load("dict.o"));
|
||||
|
||||
TEST(dictLoaded->presents("val").isValid());
|
||||
TEST(NDO_CAST(IntObject, dictLoaded->get("val"))->val == 10);
|
||||
CHECK(dictLoaded->presents("val").isValid());
|
||||
CHECK(NDO_CAST(IntObject, dictLoaded->get("val"))->val == 10);
|
||||
|
||||
NDO->destroy(dict);
|
||||
NDO->destroy(dictLoaded);
|
||||
}
|
||||
|
||||
TEST_DEF(Primitives) {
|
||||
if (objTestModule.initialize()) {
|
||||
testDict();
|
||||
objTestModule.deinitialize();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,23 +0,0 @@
|
|||
|
||||
#include "ObjectTests.hpp"
|
||||
|
||||
using namespace tp;
|
||||
using namespace obj;
|
||||
|
||||
void testParser();
|
||||
void testCore();
|
||||
void testPrimitives();
|
||||
void testInterpreter();
|
||||
|
||||
tp::ModuleManifest* objDeps[] = { &gModuleObjects, nullptr };
|
||||
tp::ModuleManifest objTestModule("ObjectsTests", nullptr, nullptr, objDeps);
|
||||
|
||||
int main() {
|
||||
|
||||
testParser();
|
||||
testCore();
|
||||
testPrimitives();
|
||||
testInterpreter();
|
||||
|
||||
return 0;
|
||||
}
|
||||
12
Objects/tests/main.cpp
Normal file
12
Objects/tests/main.cpp
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
|
||||
#include "ObjectTests.hpp"
|
||||
|
||||
#include "primitives/primitives.h"
|
||||
|
||||
tp::ModuleManifest* objDeps[] = { &obj::gModuleObjects, nullptr };
|
||||
tp::ModuleManifest objTestModule("ObjectsTests", nullptr, nullptr, objDeps);
|
||||
|
||||
#include <UnitTest++/UnitTest++.h>
|
||||
#include <UnitTest++/TestReporterStdout.h>
|
||||
|
||||
int main(int /*argc*/, char** /*argv*/) { return UnitTest::RunAllTests(); }
|
||||
Loading…
Add table
Add a link
Reference in a new issue