ilusha/lalr
1
0
Fork 0
mirror of https://github.com/cwbaker/lalr.git synced 2026-09-26 16:33:18 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Generate errors for missing lexical action handlers

Browse source
This commit is contained in:
Charles Baker 2023-05-21 21:00:37 +12:00
commit c5a7c1728d
6 changed files with 303 additions and 217 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/lalr/ErrorCode.hpp
View file

@ -12,6 +12,7 @@ enum ErrorCode
PARSER_ERROR_NONE, ///< No %error.
LALR_ERROR_SYNTAX, ///< Syntax %error occured while parsing input.
LALR_ERROR_UNTERMINATED_LITERAL, ///< Unterminated literal in an lalr grammar.
LEXER_ERROR_MISSING_ACTION_HANDLER, ///< A lexer action hasn't been bound to a function.
LEXER_ERROR_SYNTAX, ///< Syntax %error occured while parsing some input.
LEXER_ERROR_SYMBOL_CONFLICT, ///< A lexer state matches more than one symbol.
LEXER_ERROR_LEXICAL_ERROR, ///< A lexical error occured while scanning an input sequence.

3
src/lalr/Lexer.hpp
View file

@ -48,13 +48,14 @@ class Lexer
public:
Lexer( const LexerStateMachine* state_machine, const LexerStateMachine* whitespace_state_machine = nullptr, const void* end_symbol = nullptr, ErrorPolicy* error_policy = nullptr );
void set_action_handler( const char* identifier, LexerActionFunction function );
const std::basic_string<Char, Traits, Allocator>& lexeme() const;
int line() const;
int column() const;
const void* symbol() const;
const Iterator& position() const;
bool full() const;
bool valid() const;
void set_action_handler( const char* identifier, LexerActionFunction function );
void reset( Iterator start, Iterator finish );
void advance();

113
src/lalr/Lexer.ipp
View file

@ -88,43 +88,6 @@ Lexer<Iterator, Char, Traits, Allocator>::Lexer( const LexerStateMachine* state_
}
}
/**
// Set the action handler for \e identifier to \e function.
//
// @param identifier
// The identifier of the action to set a handler for.
//
// @param function
// The function to set as the handler.
*/
template <class Iterator, class Char, class Traits, class Allocator>
void Lexer<Iterator, Char, Traits, Allocator>::set_action_handler( const char* identifier, LexerActionFunction function )
{
LALR_ASSERT( identifier );
typename std::vector<LexerActionHandler>::iterator action_handler = action_handlers_.begin();
while ( action_handler != action_handlers_.end() && strcmp(action_handler->action_->identifier, identifier) != 0 )
{
++action_handler;
}
if ( action_handler != action_handlers_.end() )
{
action_handler->function_ = function;
}
action_handler = whitespace_action_handlers_.begin();
while ( action_handler != whitespace_action_handlers_.end() && strcmp(action_handler->action_->identifier, identifier) != 0 )
{
++action_handler;
}
if ( action_handler != whitespace_action_handlers_.end() )
{
action_handler->function_ = function;
}
}
/**
// Get the most recently scanned lexeme.
//
@ -197,6 +160,82 @@ bool Lexer<Iterator, Char, Traits, Allocator>::full() const
return full_;
}
/**
// Is this Lexer valid?
//
// Reports errors for any action handlers that haven't been set.
//
// @return
// True if this Lexer is valid and can be used to split input into tokens
// otherwise false.
*/
template <class Iterator, class Char, class Traits, class Allocator>
bool Lexer<Iterator, Char, Traits, Allocator>::valid() const
{
bool valid = true;
for ( const LexerActionHandler& handler : action_handlers_ )
{
if ( !handler.function_ )
{
const LexerAction* action = handler.action_;
LALR_ASSERT( action );
fire_error( -1, -1, LEXER_ERROR_MISSING_ACTION_HANDLER, "Lexical action '%s' has no handler", action->identifier );
valid = false;
}
}
for ( const LexerActionHandler& handler : whitespace_action_handlers_ )
{
if ( !handler.function_ )
{
const LexerAction* action = handler.action_;
LALR_ASSERT( action );
fire_error( -1, -1, LEXER_ERROR_MISSING_ACTION_HANDLER, "Lexical action '%s' has no handler", action->identifier );
valid = false;
}
}
return valid;
}
/**
// Set the action handler for \e identifier to \e function.
//
// @param identifier
// The identifier of the action to set a handler for.
//
// @param function
// The function to set as the handler.
*/
template <class Iterator, class Char, class Traits, class Allocator>
void Lexer<Iterator, Char, Traits, Allocator>::set_action_handler( const char* identifier, LexerActionFunction function )
{
LALR_ASSERT( identifier );
typename std::vector<LexerActionHandler>::iterator action_handler = action_handlers_.begin();
while ( action_handler != action_handlers_.end() && strcmp(action_handler->action_->identifier, identifier) != 0 )
{
++action_handler;
}
if ( action_handler != action_handlers_.end() )
{
action_handler->function_ = function;
}
action_handler = whitespace_action_handlers_.begin();
while ( action_handler != whitespace_action_handlers_.end() && strcmp(action_handler->action_->identifier, identifier) != 0 )
{
++action_handler;
}
if ( action_handler != whitespace_action_handlers_.end() )
{
action_handler->function_ = function;
}
}
/**
// Reset this %Lexer to scan [\e start, \e finish) starting its line count
// from \e line.

21
src/lalr/Parser.hpp
View file

@ -41,7 +41,7 @@ public:
private:
struct ParserActionHandler
{
const ParserAction* action_;
const ParserAction* action_;
ParserActionFunction function_;
ParserActionHandler( const ParserAction* action, ParserActionFunction function );
};
@ -60,26 +60,25 @@ private:
public:
Parser( const ParserStateMachine* state_machine, ErrorPolicy* error_policy = nullptr );
void reset();
void parse( Iterator start, Iterator finish );
bool parse( const void* symbol, const std::basic_string<Char, Traits, Allocator>& lexeme, int line, int column );
bool parse( const ParserSymbol* symbol, const std::basic_string<Char, Traits, Allocator>& lexeme, int line, int column );
bool accepted() const;
bool full() const;
bool valid() const;
const UserData& user_data() const;
const Lexer<Iterator, Char, Traits, Allocator>& lexer() const;
void fire_error(int line, int column, int error, const char* format, ... ) const;
void fire_printf( const char* format, ... ) const;
bool is_debug_enabled() const;
AddParserActionHandler<Iterator, UserData, Char, Traits, Allocator> parser_action_handlers();
AddLexerActionHandler<Iterator, Char, Traits, Allocator> lexer_action_handlers();
void set_default_action_handler( ParserActionFunction function );
void set_action_handler( const char* identifier, ParserActionFunction function );
void set_lexer_action_handler( const char* identifier, LexerActionFunction function );
void fire_error(int line, int column, int error, const char* format, ... ) const;
void fire_printf( const char* format, ... ) const;
void set_lexer_action_handler( const char* identifier, LexerActionFunction function );
void set_debug_enabled( bool debug_enabled );
bool is_debug_enabled() const;
void reset();
void parse( Iterator start, Iterator finish );
bool parse( const void* symbol, const std::basic_string<Char, Traits, Allocator>& lexeme, int line, int column );
bool parse( const ParserSymbol* symbol, const std::basic_string<Char, Traits, Allocator>& lexeme, int line, int column );
private:
const ParserTransition* find_transition( const ParserSymbol* symbol, const ParserState* state ) const;

354
src/lalr/Parser.ipp
View file

@ -88,117 +88,6 @@ Parser<Iterator, UserData, Char, Traits, Allocator>::Parser( const ParserStateMa
user_data_.push_back( UserData() );
}
/**
// Reset this Parser so that it can parse another sequence of input.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
void Parser<Iterator, UserData, Char, Traits, Allocator>::reset()
{
accepted_ = false;
full_ = false;
nodes_.clear();
user_data_.clear();
nodes_.push_back( ParserNode(state_machine_->start_state, nullptr, 0, 1) );
user_data_.push_back( UserData() );
}
/**
// Parse [\e start, \e finish).
//
// After the parse the Parser::full() and Parser::accepted() functions can
// be used to determine whether or not the parse was successful and whether
// or not it consumed all of the available input.
//
// In the case of a successful parse the Parser::user_data() function can
// be used to retrieve the user data that resulted from the parse.
//
// @param start
// The first character in the sequence to parse.
//
// @param finish
// One past the last character in the sequence to parse.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
void Parser<Iterator, UserData, Char, Traits, Allocator>::parse( Iterator start, Iterator finish )
{
LALR_ASSERT( state_machine_ );
reset();
lexer_.reset( start, finish );
lexer_.advance();
const ParserSymbol* symbol = reinterpret_cast<const ParserSymbol*>( lexer_.symbol() );
while ( parse(symbol, lexer_.lexeme(), lexer_.line(), lexer_.column()) )
{
lexer_.advance();
symbol = reinterpret_cast<const ParserSymbol*>( lexer_.symbol() );
}
full_ = lexer_.full();
}
/**
// Continue a parse by accepting \e symbol as the next token.
//
// @param symbol
// The next token from the lexical analyzer in the current parse (assumed to
// be a ParserSymbol).
//
// @param lexeme
// The lexeme of the next token from the lexical analyzer.
//
// @param line
// The line number at the start of the next token.
//
// @return
// True until parsing is complete or an error occurs.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
bool Parser<Iterator, UserData, Char, Traits, Allocator>::parse( const void* symbol, const std::basic_string<Char, Traits, Allocator>& lexeme, int line, int column )
{
return parse( reinterpret_cast<const ParserSymbol*>(symbol), lexeme, line, column );
}
/**
// Continue a parse by accepting \e symbol as the next token.
//
// @param symbol
// The next token from the lexical analyzer in the current parse.
//
// @param lexeme
// The lexeme of the next token from the lexical analyzer.
//
// @param line
// The line number at the start of the next token.
//
// @return
// True until parsing is complete or an error occurs.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
bool Parser<Iterator, UserData, Char, Traits, Allocator>::parse( const ParserSymbol* symbol, const std::basic_string<Char, Traits, Allocator>& lexeme, int line, int column )
{
bool accepted = false;
bool rejected = false;
const ParserTransition* transition = find_transition( symbol, nodes_.back().state() );
while ( !accepted && !rejected && transition && transition->reduced_symbol )
{
reduce( transition, &accepted, &rejected );
transition = find_transition( symbol, nodes_.back().state() );
}
if ( transition && transition->state )
{
shift( transition, lexeme, line, column );
}
else
{
error( &accepted, &rejected, line, column);
}
accepted_ = accepted;
return !accepted_ && !rejected;
}
/**
// Did the most recent parse accept input successfully?
//
@ -223,6 +112,22 @@ bool Parser<Iterator, UserData, Char, Traits, Allocator>::full() const
return full_;
}
/**
// Is this Parser valid?
//
// Reports errors for and returns false if the lexer is missing any action
// handlers.
//
// @return
// True if this Parser is valid or false if there are any missing
// action handlers.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
bool Parser<Iterator, UserData, Char, Traits, Allocator>::valid() const
{
return lexer_.valid();
}
/**
// Get the user data that resulted from the most recent call to
// Parser::parser() on this %Parser.
@ -254,6 +159,69 @@ const Lexer<Iterator, Char, Traits, Allocator>& Parser<Iterator, UserData, Char,
return lexer_;
}
/**
// Fire an %error event.
//
// @param line
// The line number to associate with the %error (or 0 if there is no line
// to associate with the %error).
//
// @param error
// The %Error that describes the %error that has occured.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
void Parser<Iterator, UserData, Char, Traits, Allocator>::fire_error( int line, int column, int error, const char* format, ... ) const
{
if ( error_policy_ )
{
va_list args;
va_start( args, format );
error_policy_->lalr_error( line, column, error, format, args );
va_end( args );
}
}
/**
// Fire a printf event.
//
// @param format
// A printf style format string that describes the message to print.
//
// @param ...
// Parameters to fill in the message as specified by \e format.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
void Parser<Iterator, UserData, Char, Traits, Allocator>::fire_printf( const char* format, ... ) const
{
if ( error_policy_ )
{
LALR_ASSERT( format );
va_list args;
va_start( args, format );
error_policy_->lalr_vprintf( format, args );
va_end( args );
}
else
{
va_list args;
va_start( args, format );
vfprintf( stdout, format, args );
va_end( args );
}
}
/**
// Are shift and reduce operations printed?
//
// @return
// True if shift and reduce operations are printed otherwise false.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
bool Parser<Iterator, UserData, Char, Traits, Allocator>::is_debug_enabled() const
{
return debug_enabled_;
}
/**
// Add action handlers to this %Parser.
//
@ -337,57 +305,6 @@ void Parser<Iterator, UserData, Char, Traits, Allocator>::set_lexer_action_handl
lexer_.set_action_handler( identifier, function );
}
/**
// Fire an %error event.
//
// @param line
// The line number to associate with the %error (or 0 if there is no line
// to associate with the %error).
//
// @param error
// The %Error that describes the %error that has occured.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
void Parser<Iterator, UserData, Char, Traits, Allocator>::fire_error( int line, int column, int error, const char* format, ... ) const
{
if ( error_policy_ )
{
va_list args;
va_start( args, format );
error_policy_->lalr_error( line, column, error, format, args );
va_end( args );
}
}
/**
// Fire a printf event.
//
// @param format
// A printf style format string that describes the message to print.
//
// @param ...
// Parameters to fill in the message as specified by \e format.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
void Parser<Iterator, UserData, Char, Traits, Allocator>::fire_printf( const char* format, ... ) const
{
if ( error_policy_ )
{
LALR_ASSERT( format );
va_list args;
va_start( args, format );
error_policy_->lalr_vprintf( format, args );
va_end( args );
}
else
{
va_list args;
va_start( args, format );
vfprintf( stdout, format, args );
va_end( args );
}
}
/**
// Set whether or not shift operations are printed.
//
@ -402,15 +319,116 @@ void Parser<Iterator, UserData, Char, Traits, Allocator>::set_debug_enabled( boo
}
/**
// Are shift and reduce operations printed?
//
// @return
// True if shift and reduce operations are printed otherwise false.
// Reset this Parser so that it can parse another sequence of input.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
bool Parser<Iterator, UserData, Char, Traits, Allocator>::is_debug_enabled() const
void Parser<Iterator, UserData, Char, Traits, Allocator>::reset()
{
return debug_enabled_;
accepted_ = false;
full_ = false;
nodes_.clear();
user_data_.clear();
nodes_.push_back( ParserNode(state_machine_->start_state, nullptr, 0, 1) );
user_data_.push_back( UserData() );
}
/**
// Parse [\e start, \e finish).
//
// After the parse the Parser::full() and Parser::accepted() functions can
// be used to determine whether or not the parse was successful and whether
// or not it consumed all of the available input.
//
// In the case of a successful parse the Parser::user_data() function can
// be used to retrieve the user data that resulted from the parse.
//
// @param start
// The first character in the sequence to parse.
//
// @param finish
// One past the last character in the sequence to parse.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
void Parser<Iterator, UserData, Char, Traits, Allocator>::parse( Iterator start, Iterator finish )
{
LALR_ASSERT( state_machine_ );
if ( valid() )
{
reset();
lexer_.reset( start, finish );
lexer_.advance();
const ParserSymbol* symbol = reinterpret_cast<const ParserSymbol*>( lexer_.symbol() );
while ( parse(symbol, lexer_.lexeme(), lexer_.line(), lexer_.column()) )
{
lexer_.advance();
symbol = reinterpret_cast<const ParserSymbol*>( lexer_.symbol() );
}
full_ = lexer_.full();
}
}
/**
// Continue a parse by accepting \e symbol as the next token.
//
// @param symbol
// The next token from the lexical analyzer in the current parse (assumed to
// be a ParserSymbol).
//
// @param lexeme
// The lexeme of the next token from the lexical analyzer.
//
// @param line
// The line number at the start of the next token.
//
// @return
// True until parsing is complete or an error occurs.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
bool Parser<Iterator, UserData, Char, Traits, Allocator>::parse( const void* symbol, const std::basic_string<Char, Traits, Allocator>& lexeme, int line, int column )
{
return parse( reinterpret_cast<const ParserSymbol*>(symbol), lexeme, line, column );
}
/**
// Continue a parse by accepting \e symbol as the next token.
//
// @param symbol
// The next token from the lexical analyzer in the current parse.
//
// @param lexeme
// The lexeme of the next token from the lexical analyzer.
//
// @param line
// The line number at the start of the next token.
//
// @return
// True until parsing is complete or an error occurs.
*/
template <class Iterator, class UserData, class Char, class Traits, class Allocator>
bool Parser<Iterator, UserData, Char, Traits, Allocator>::parse( const ParserSymbol* symbol, const std::basic_string<Char, Traits, Allocator>& lexeme, int line, int column )
{
bool accepted = false;
bool rejected = false;
const ParserTransition* transition = find_transition( symbol, nodes_.back().state() );
while ( !accepted && !rejected && transition && transition->reduced_symbol )
{
reduce( transition, &accepted, &rejected );
transition = find_transition( symbol, nodes_.back().state() );
}
if ( transition && transition->state )
{
shift( transition, lexeme, line, column );
}
else
{
error( &accepted, &rejected, line, column);
}
accepted_ = accepted;
return !accepted_ && !rejected;
}
/**

28
src/lalr/lalr_test/TestParsers.cpp
View file

@ -1255,4 +1255,32 @@ SUITE( Parsers )
CHECK( parser.accepted() );
CHECK( parser.full() );
}
TEST( MissingLexicalActionHandler )
{
const char* line_comment_grammar =
"MissingLexicalActionHandler {\n"
" %whitespace \"([ \\t\\r\\n]|\\/\\/:line_comment:)*\";\n"
" unit: '1';\n"
"}"
;
GrammarCompiler compiler;
compiler.compile( line_comment_grammar, line_comment_grammar + strlen(line_comment_grammar) );
Parser<const char*> parser( compiler.parser_state_machine() );
CHECK( !parser.valid() );
const char* input = "1";
parser.parse( input, input + strlen(input) );
CHECK( !parser.accepted() );
CHECK( !parser.full() );
parser.lexer_action_handlers()
( "line_comment", &line_comment<const char*> )
;
parser.parse( input, input + strlen(input) );
CHECK( parser.accepted() );
CHECK( parser.full() );
}
}