From b7b7c4d61e8b9c196f4c0cb578cd5799caad72d6 Mon Sep 17 00:00:00 2001 From: Charles Baker Date: Sun, 21 May 2023 19:53:46 +1200 Subject: [PATCH 1/5] Move Lexer::set_action_handler() to non-const member section --- src/lalr/Lexer.hpp | 2 +- src/lalr/Lexer.ipp | 74 +++++++++++++++++++++++----------------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/lalr/Lexer.hpp b/src/lalr/Lexer.hpp index e8060fa..6847e33 100644 --- a/src/lalr/Lexer.hpp +++ b/src/lalr/Lexer.hpp @@ -48,13 +48,13 @@ class Lexer public: Lexer( const LexerStateMachine* state_machine, const LexerStateMachine* whitespace_state_machine = nullptr, const void* end_symbol = nullptr, ErrorPolicy* error_policy = nullptr ); - void set_action_handler( const char* identifier, LexerActionFunction function ); const std::basic_string& lexeme() const; int line() const; int column() const; const void* symbol() const; const Iterator& position() const; bool full() const; + void set_action_handler( const char* identifier, LexerActionFunction function ); void reset( Iterator start, Iterator finish ); void advance(); diff --git a/src/lalr/Lexer.ipp b/src/lalr/Lexer.ipp index 50973c0..d0702a6 100644 --- a/src/lalr/Lexer.ipp +++ b/src/lalr/Lexer.ipp @@ -88,43 +88,6 @@ Lexer::Lexer( const LexerStateMachine* state_ } } -/** -// Set the action handler for \e identifier to \e function. -// -// @param identifier -// The identifier of the action to set a handler for. -// -// @param function -// The function to set as the handler. -*/ -template -void Lexer::set_action_handler( const char* identifier, LexerActionFunction function ) -{ - LALR_ASSERT( identifier ); - - typename std::vector::iterator action_handler = action_handlers_.begin(); - while ( action_handler != action_handlers_.end() && strcmp(action_handler->action_->identifier, identifier) != 0 ) - { - ++action_handler; - } - - if ( action_handler != action_handlers_.end() ) - { - action_handler->function_ = function; - } - - action_handler = whitespace_action_handlers_.begin(); - while ( action_handler != whitespace_action_handlers_.end() && strcmp(action_handler->action_->identifier, identifier) != 0 ) - { - ++action_handler; - } - - if ( action_handler != whitespace_action_handlers_.end() ) - { - action_handler->function_ = function; - } -} - /** // Get the most recently scanned lexeme. // @@ -197,6 +160,43 @@ bool Lexer::full() const return full_; } +/** +// Set the action handler for \e identifier to \e function. +// +// @param identifier +// The identifier of the action to set a handler for. +// +// @param function +// The function to set as the handler. +*/ +template +void Lexer::set_action_handler( const char* identifier, LexerActionFunction function ) +{ + LALR_ASSERT( identifier ); + + typename std::vector::iterator action_handler = action_handlers_.begin(); + while ( action_handler != action_handlers_.end() && strcmp(action_handler->action_->identifier, identifier) != 0 ) + { + ++action_handler; + } + + if ( action_handler != action_handlers_.end() ) + { + action_handler->function_ = function; + } + + action_handler = whitespace_action_handlers_.begin(); + while ( action_handler != whitespace_action_handlers_.end() && strcmp(action_handler->action_->identifier, identifier) != 0 ) + { + ++action_handler; + } + + if ( action_handler != whitespace_action_handlers_.end() ) + { + action_handler->function_ = function; + } +} + /** // Reset this %Lexer to scan [\e start, \e finish) starting its line count // from \e line. From e56556f390219a2e063bef2cae294113dcc73c61 Mon Sep 17 00:00:00 2001 From: Charles Baker Date: Sun, 21 May 2023 20:03:36 +1200 Subject: [PATCH 2/5] Move Parser::fire_{error,print}() and Parser::is_debug_enabled() to const member section --- src/lalr/Parser.hpp | 10 ++-- src/lalr/Parser.ipp | 126 ++++++++++++++++++++++---------------------- 2 files changed, 67 insertions(+), 69 deletions(-) diff --git a/src/lalr/Parser.hpp b/src/lalr/Parser.hpp index b46ccae..425ab6d 100644 --- a/src/lalr/Parser.hpp +++ b/src/lalr/Parser.hpp @@ -68,18 +68,16 @@ public: bool full() const; const UserData& user_data() const; const Lexer& lexer() const; + void fire_error(int line, int column, int error, const char* format, ... ) const; + void fire_printf( const char* format, ... ) const; + bool is_debug_enabled() const; AddParserActionHandler parser_action_handlers(); AddLexerActionHandler lexer_action_handlers(); void set_default_action_handler( ParserActionFunction function ); void set_action_handler( const char* identifier, ParserActionFunction function ); - void set_lexer_action_handler( const char* identifier, LexerActionFunction function ); - - void fire_error(int line, int column, int error, const char* format, ... ) const; - void fire_printf( const char* format, ... ) const; - + void set_lexer_action_handler( const char* identifier, LexerActionFunction function ); void set_debug_enabled( bool debug_enabled ); - bool is_debug_enabled() const; private: const ParserTransition* find_transition( const ParserSymbol* symbol, const ParserState* state ) const; diff --git a/src/lalr/Parser.ipp b/src/lalr/Parser.ipp index 4848108..2fa1995 100644 --- a/src/lalr/Parser.ipp +++ b/src/lalr/Parser.ipp @@ -254,6 +254,69 @@ const Lexer& Parser +void Parser::fire_error( int line, int column, int error, const char* format, ... ) const +{ + if ( error_policy_ ) + { + va_list args; + va_start( args, format ); + error_policy_->lalr_error( line, column, error, format, args ); + va_end( args ); + } +} + +/** +// Fire a printf event. +// +// @param format +// A printf style format string that describes the message to print. +// +// @param ... +// Parameters to fill in the message as specified by \e format. +*/ +template +void Parser::fire_printf( const char* format, ... ) const +{ + if ( error_policy_ ) + { + LALR_ASSERT( format ); + va_list args; + va_start( args, format ); + error_policy_->lalr_vprintf( format, args ); + va_end( args ); + } + else + { + va_list args; + va_start( args, format ); + vfprintf( stdout, format, args ); + va_end( args ); + } +} + +/** +// Are shift and reduce operations printed? +// +// @return +// True if shift and reduce operations are printed otherwise false. +*/ +template +bool Parser::is_debug_enabled() const +{ + return debug_enabled_; +} + /** // Add action handlers to this %Parser. // @@ -337,57 +400,6 @@ void Parser::set_lexer_action_handl lexer_.set_action_handler( identifier, function ); } -/** -// Fire an %error event. -// -// @param line -// The line number to associate with the %error (or 0 if there is no line -// to associate with the %error). -// -// @param error -// The %Error that describes the %error that has occured. -*/ -template -void Parser::fire_error( int line, int column, int error, const char* format, ... ) const -{ - if ( error_policy_ ) - { - va_list args; - va_start( args, format ); - error_policy_->lalr_error( line, column, error, format, args ); - va_end( args ); - } -} - -/** -// Fire a printf event. -// -// @param format -// A printf style format string that describes the message to print. -// -// @param ... -// Parameters to fill in the message as specified by \e format. -*/ -template -void Parser::fire_printf( const char* format, ... ) const -{ - if ( error_policy_ ) - { - LALR_ASSERT( format ); - va_list args; - va_start( args, format ); - error_policy_->lalr_vprintf( format, args ); - va_end( args ); - } - else - { - va_list args; - va_start( args, format ); - vfprintf( stdout, format, args ); - va_end( args ); - } -} - /** // Set whether or not shift operations are printed. // @@ -401,18 +413,6 @@ void Parser::set_debug_enabled( boo debug_enabled_ = debug_enabled; } -/** -// Are shift and reduce operations printed? -// -// @return -// True if shift and reduce operations are printed otherwise false. -*/ -template -bool Parser::is_debug_enabled() const -{ - return debug_enabled_; -} - /** // Find the Transition for \e symbol in \e state. // From 184a8b34c03e30b52d90eb32abbcaa853d2e74ed Mon Sep 17 00:00:00 2001 From: Charles Baker Date: Sun, 21 May 2023 20:04:14 +1200 Subject: [PATCH 3/5] Report missing lexical action handlers from Lexer::valid() --- src/lalr/ErrorCode.hpp | 1 + src/lalr/Lexer.hpp | 1 + src/lalr/Lexer.ipp | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/src/lalr/ErrorCode.hpp b/src/lalr/ErrorCode.hpp index 2212a75..4ed44c3 100644 --- a/src/lalr/ErrorCode.hpp +++ b/src/lalr/ErrorCode.hpp @@ -12,6 +12,7 @@ enum ErrorCode PARSER_ERROR_NONE, ///< No %error. LALR_ERROR_SYNTAX, ///< Syntax %error occured while parsing input. LALR_ERROR_UNTERMINATED_LITERAL, ///< Unterminated literal in an lalr grammar. + LEXER_ERROR_MISSING_ACTION_HANDLER, ///< A lexer action hasn't been bound to a function. LEXER_ERROR_SYNTAX, ///< Syntax %error occured while parsing some input. LEXER_ERROR_SYMBOL_CONFLICT, ///< A lexer state matches more than one symbol. LEXER_ERROR_LEXICAL_ERROR, ///< A lexical error occured while scanning an input sequence. diff --git a/src/lalr/Lexer.hpp b/src/lalr/Lexer.hpp index 6847e33..c4de8c8 100644 --- a/src/lalr/Lexer.hpp +++ b/src/lalr/Lexer.hpp @@ -54,6 +54,7 @@ public: const void* symbol() const; const Iterator& position() const; bool full() const; + bool valid() const; void set_action_handler( const char* identifier, LexerActionFunction function ); void reset( Iterator start, Iterator finish ); void advance(); diff --git a/src/lalr/Lexer.ipp b/src/lalr/Lexer.ipp index d0702a6..8b1a951 100644 --- a/src/lalr/Lexer.ipp +++ b/src/lalr/Lexer.ipp @@ -160,6 +160,45 @@ bool Lexer::full() const return full_; } +/** +// Is this Lexer valid? +// +// Reports errors for any action handlers that haven't been set. +// +// @return +// True if this Lexer is valid and can be used to split input into tokens +// otherwise false. +*/ +template +bool Lexer::valid() const +{ + bool valid = true; + + for ( const LexerActionHandler& handler : action_handlers_ ) + { + if ( !handler.function_ ) + { + const LexerAction* action = handler.action_; + LALR_ASSERT( action ); + fire_error( -1, -1, LEXER_ERROR_MISSING_ACTION_HANDLER, "Lexical action '%s' has no handler", action->identifier ); + valid = false; + } + } + + for ( const LexerActionHandler& handler : whitespace_action_handlers_ ) + { + if ( !handler.function_ ) + { + const LexerAction* action = handler.action_; + LALR_ASSERT( action ); + fire_error( -1, -1, LEXER_ERROR_MISSING_ACTION_HANDLER, "Lexical action '%s' has no handler", action->identifier ); + valid = false; + } + } + + return valid; +} + /** // Set the action handler for \e identifier to \e function. // From d8948dd9361d197f60f02ad59ccf00ce6ba8182f Mon Sep 17 00:00:00 2001 From: Charles Baker Date: Sun, 21 May 2023 20:09:41 +1200 Subject: [PATCH 4/5] Report missing action handlers from Parser::valid() --- src/lalr/Parser.hpp | 11 +- src/lalr/Parser.ipp | 240 ++++++++++++++++++++++++-------------------- 2 files changed, 135 insertions(+), 116 deletions(-) diff --git a/src/lalr/Parser.hpp b/src/lalr/Parser.hpp index 425ab6d..a9e2a69 100644 --- a/src/lalr/Parser.hpp +++ b/src/lalr/Parser.hpp @@ -41,7 +41,7 @@ public: private: struct ParserActionHandler { - const ParserAction* action_; + const ParserAction* action_; ParserActionFunction function_; ParserActionHandler( const ParserAction* action, ParserActionFunction function ); }; @@ -60,12 +60,9 @@ private: public: Parser( const ParserStateMachine* state_machine, ErrorPolicy* error_policy = nullptr ); - void reset(); - void parse( Iterator start, Iterator finish ); - bool parse( const void* symbol, const std::basic_string& lexeme, int line, int column ); - bool parse( const ParserSymbol* symbol, const std::basic_string& lexeme, int line, int column ); bool accepted() const; bool full() const; + bool valid() const; const UserData& user_data() const; const Lexer& lexer() const; void fire_error(int line, int column, int error, const char* format, ... ) const; @@ -78,6 +75,10 @@ public: void set_action_handler( const char* identifier, ParserActionFunction function ); void set_lexer_action_handler( const char* identifier, LexerActionFunction function ); void set_debug_enabled( bool debug_enabled ); + void reset(); + void parse( Iterator start, Iterator finish ); + bool parse( const void* symbol, const std::basic_string& lexeme, int line, int column ); + bool parse( const ParserSymbol* symbol, const std::basic_string& lexeme, int line, int column ); private: const ParserTransition* find_transition( const ParserSymbol* symbol, const ParserState* state ) const; diff --git a/src/lalr/Parser.ipp b/src/lalr/Parser.ipp index 2fa1995..30f5670 100644 --- a/src/lalr/Parser.ipp +++ b/src/lalr/Parser.ipp @@ -88,117 +88,6 @@ Parser::Parser( const ParserStateMa user_data_.push_back( UserData() ); } -/** -// Reset this Parser so that it can parse another sequence of input. -*/ -template -void Parser::reset() -{ - accepted_ = false; - full_ = false; - nodes_.clear(); - user_data_.clear(); - nodes_.push_back( ParserNode(state_machine_->start_state, nullptr, 0, 1) ); - user_data_.push_back( UserData() ); -} - -/** -// Parse [\e start, \e finish). -// -// After the parse the Parser::full() and Parser::accepted() functions can -// be used to determine whether or not the parse was successful and whether -// or not it consumed all of the available input. -// -// In the case of a successful parse the Parser::user_data() function can -// be used to retrieve the user data that resulted from the parse. -// -// @param start -// The first character in the sequence to parse. -// -// @param finish -// One past the last character in the sequence to parse. -*/ -template -void Parser::parse( Iterator start, Iterator finish ) -{ - LALR_ASSERT( state_machine_ ); - - reset(); - lexer_.reset( start, finish ); - lexer_.advance(); - const ParserSymbol* symbol = reinterpret_cast( lexer_.symbol() ); - while ( parse(symbol, lexer_.lexeme(), lexer_.line(), lexer_.column()) ) - { - lexer_.advance(); - symbol = reinterpret_cast( lexer_.symbol() ); - } - - full_ = lexer_.full(); -} - -/** -// Continue a parse by accepting \e symbol as the next token. -// -// @param symbol -// The next token from the lexical analyzer in the current parse (assumed to -// be a ParserSymbol). -// -// @param lexeme -// The lexeme of the next token from the lexical analyzer. -// -// @param line -// The line number at the start of the next token. -// -// @return -// True until parsing is complete or an error occurs. -*/ -template -bool Parser::parse( const void* symbol, const std::basic_string& lexeme, int line, int column ) -{ - return parse( reinterpret_cast(symbol), lexeme, line, column ); -} - -/** -// Continue a parse by accepting \e symbol as the next token. -// -// @param symbol -// The next token from the lexical analyzer in the current parse. -// -// @param lexeme -// The lexeme of the next token from the lexical analyzer. -// -// @param line -// The line number at the start of the next token. -// -// @return -// True until parsing is complete or an error occurs. -*/ -template -bool Parser::parse( const ParserSymbol* symbol, const std::basic_string& lexeme, int line, int column ) -{ - bool accepted = false; - bool rejected = false; - - const ParserTransition* transition = find_transition( symbol, nodes_.back().state() ); - while ( !accepted && !rejected && transition && transition->reduced_symbol ) - { - reduce( transition, &accepted, &rejected ); - transition = find_transition( symbol, nodes_.back().state() ); - } - - if ( transition && transition->state ) - { - shift( transition, lexeme, line, column ); - } - else - { - error( &accepted, &rejected, line, column); - } - - accepted_ = accepted; - return !accepted_ && !rejected; -} - /** // Did the most recent parse accept input successfully? // @@ -223,6 +112,22 @@ bool Parser::full() const return full_; } +/** +// Is this Parser valid? +// +// Reports errors for and returns false if the lexer is missing any action +// handlers. +// +// @return +// True if this Parser is valid or false if there are any missing +// action handlers. +*/ +template +bool Parser::valid() const +{ + return lexer_.valid(); +} + /** // Get the user data that resulted from the most recent call to // Parser::parser() on this %Parser. @@ -413,6 +318,119 @@ void Parser::set_debug_enabled( boo debug_enabled_ = debug_enabled; } +/** +// Reset this Parser so that it can parse another sequence of input. +*/ +template +void Parser::reset() +{ + accepted_ = false; + full_ = false; + nodes_.clear(); + user_data_.clear(); + nodes_.push_back( ParserNode(state_machine_->start_state, nullptr, 0, 1) ); + user_data_.push_back( UserData() ); +} + +/** +// Parse [\e start, \e finish). +// +// After the parse the Parser::full() and Parser::accepted() functions can +// be used to determine whether or not the parse was successful and whether +// or not it consumed all of the available input. +// +// In the case of a successful parse the Parser::user_data() function can +// be used to retrieve the user data that resulted from the parse. +// +// @param start +// The first character in the sequence to parse. +// +// @param finish +// One past the last character in the sequence to parse. +*/ +template +void Parser::parse( Iterator start, Iterator finish ) +{ + LALR_ASSERT( state_machine_ ); + if ( valid() ) + { + reset(); + lexer_.reset( start, finish ); + lexer_.advance(); + const ParserSymbol* symbol = reinterpret_cast( lexer_.symbol() ); + while ( parse(symbol, lexer_.lexeme(), lexer_.line(), lexer_.column()) ) + { + lexer_.advance(); + symbol = reinterpret_cast( lexer_.symbol() ); + } + + full_ = lexer_.full(); + } +} + +/** +// Continue a parse by accepting \e symbol as the next token. +// +// @param symbol +// The next token from the lexical analyzer in the current parse (assumed to +// be a ParserSymbol). +// +// @param lexeme +// The lexeme of the next token from the lexical analyzer. +// +// @param line +// The line number at the start of the next token. +// +// @return +// True until parsing is complete or an error occurs. +*/ +template +bool Parser::parse( const void* symbol, const std::basic_string& lexeme, int line, int column ) +{ + return parse( reinterpret_cast(symbol), lexeme, line, column ); +} + +/** +// Continue a parse by accepting \e symbol as the next token. +// +// @param symbol +// The next token from the lexical analyzer in the current parse. +// +// @param lexeme +// The lexeme of the next token from the lexical analyzer. +// +// @param line +// The line number at the start of the next token. +// +// @return +// True until parsing is complete or an error occurs. +*/ +template +bool Parser::parse( const ParserSymbol* symbol, const std::basic_string& lexeme, int line, int column ) +{ + bool accepted = false; + bool rejected = false; + + const ParserTransition* transition = find_transition( symbol, nodes_.back().state() ); + while ( !accepted && !rejected && transition && transition->reduced_symbol ) + { + reduce( transition, &accepted, &rejected ); + transition = find_transition( symbol, nodes_.back().state() ); + } + + if ( transition && transition->state ) + { + shift( transition, lexeme, line, column ); + } + else + { + error( &accepted, &rejected, line, column); + } + + accepted_ = accepted; + return !accepted_ && !rejected; +} + /** // Find the Transition for \e symbol in \e state. // From 77602f9052b3099b109e5cd24fdb01e53017966a Mon Sep 17 00:00:00 2001 From: Charles Baker Date: Sun, 21 May 2023 20:15:29 +1200 Subject: [PATCH 5/5] Test that parsers with missing lexical action handlers generate errors --- src/lalr/lalr_test/TestParsers.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/lalr/lalr_test/TestParsers.cpp b/src/lalr/lalr_test/TestParsers.cpp index e08231b..e65cfaf 100644 --- a/src/lalr/lalr_test/TestParsers.cpp +++ b/src/lalr/lalr_test/TestParsers.cpp @@ -1255,4 +1255,32 @@ SUITE( Parsers ) CHECK( parser.accepted() ); CHECK( parser.full() ); } + + TEST( MissingLexicalActionHandler ) + { + const char* line_comment_grammar = + "MissingLexicalActionHandler {\n" + " %whitespace \"([ \\t\\r\\n]|\\/\\/:line_comment:)*\";\n" + " unit: '1';\n" + "}" + ; + + GrammarCompiler compiler; + compiler.compile( line_comment_grammar, line_comment_grammar + strlen(line_comment_grammar) ); + Parser parser( compiler.parser_state_machine() ); + CHECK( !parser.valid() ); + + const char* input = "1"; + parser.parse( input, input + strlen(input) ); + CHECK( !parser.accepted() ); + CHECK( !parser.full() ); + + parser.lexer_action_handlers() + ( "line_comment", &line_comment ) + ; + + parser.parse( input, input + strlen(input) ); + CHECK( parser.accepted() ); + CHECK( parser.full() ); + } }