From d03a02d3e5ade46c73a40084c7fc6d9631d2b988 Mon Sep 17 00:00:00 2001 From: Charles Baker Date: Mon, 29 May 2023 21:19:24 +1200 Subject: [PATCH 1/4] Use correct string for lexeme when debugging parsers This works for UTF-8 encoded strings but debugging will fail when using UTF-16 or UTF-32 as the printf() formatted string won't deal with null characters embedded within the wider character encodings. Committing this for now because it doesn't break any existing functionality but does enable parsing of wider character inputs. --- src/lalr/Parser.ipp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lalr/Parser.ipp b/src/lalr/Parser.ipp index 30f5670..dad237c 100644 --- a/src/lalr/Parser.ipp +++ b/src/lalr/Parser.ipp @@ -494,7 +494,7 @@ void Parser::debug_shift( const Par if ( debug_enabled_ ) { const ParserSymbol* symbol = node.symbol(); - const std::string& lexeme = node.lexeme(); + const std::basic_string& lexeme = node.lexeme(); int line = node.line(); int column = node.column(); fire_printf( "SHIFT: (%s %s %d:%d)\n", symbol ? symbol->identifier : "", lexeme.c_str(), line, column ); @@ -521,14 +521,14 @@ void Parser::debug_reduce( const Pa if ( debug_enabled_ ) { - fire_printf( "REDUCE: %s <- ", reduced_symbol->identifier ); + fire_printf( "REDUCE: %s <- ", reduced_symbol->identifier ); const ParserNode* node = nodes_.data() + start; const ParserNode* node_end = nodes_.data() + finish; if ( node != node_end ) { const ParserSymbol* symbol = node->symbol(); - const std::string& lexeme = node->lexeme(); + const std::basic_string& lexeme = node->lexeme(); int line = node->line(); int column = node->column(); fire_printf( "(%s %s %d:%d)", symbol ? symbol->identifier : "", lexeme.c_str(), line, column ); @@ -538,7 +538,7 @@ void Parser::debug_reduce( const Pa while ( node != node_end ) { const ParserSymbol* symbol = node->symbol(); - const std::string& lexeme = node->lexeme(); + const std::basic_string& lexeme = node->lexeme(); int line = node->line(); int column = node->column(); fire_printf( " (%s %s %d:%d)", symbol ? symbol->identifier : "", lexeme.c_str(), line, column ); From 0dc846f6eb7f81678a2aaa7e60309116d809f5d4 Mon Sep 17 00:00:00 2001 From: Charles Baker Date: Mon, 29 May 2023 21:20:09 +1200 Subject: [PATCH 2/4] Update JSON example to parse from a UTF-8 encoded JSON file This is an example of using Lalr to parse UTF-8 encoded input. --- src/lalr/lalr_examples/lalr_examples.forge | 3 + src/lalr/lalr_examples/lalr_json_example.cpp | 88 +++++++++++-------- src/lalr/lalr_examples/lalr_json_example.json | 11 +++ 3 files changed, 67 insertions(+), 35 deletions(-) create mode 100644 src/lalr/lalr_examples/lalr_json_example.json diff --git a/src/lalr/lalr_examples/lalr_examples.forge b/src/lalr/lalr_examples/lalr_examples.forge index 203f948..d6c0ab4 100644 --- a/src/lalr/lalr_examples/lalr_examples.forge +++ b/src/lalr/lalr_examples/lalr_examples.forge @@ -15,6 +15,9 @@ for _, cc in toolsets('^cc.*') do libraries = libraries; '${lib}/lalr_${platform}_${architecture}'; cc:Cxx '${obj}/%1' { + defines = { + ([[LALR_EXAMPLES=\"%s/\"]]):format( pwd() ); + }; "lalr_examples.cpp", "lalr_error_handling_calculator_example.cpp", "lalr_hello_world_example.cpp", diff --git a/src/lalr/lalr_examples/lalr_json_example.cpp b/src/lalr/lalr_examples/lalr_json_example.cpp index cdc17c4..cfb611c 100644 --- a/src/lalr/lalr_examples/lalr_json_example.cpp +++ b/src/lalr/lalr_examples/lalr_json_example.cpp @@ -4,7 +4,16 @@ #include #include #include +#include #include +#include +#include +#include +#ifdef __APPLE__ +#include +#else +#include +#endif using namespace std; using namespace lalr; @@ -12,11 +21,13 @@ using namespace lalr; namespace { +typedef std::basic_string String; + struct Attribute; struct Value { - std::string value_; + String value_; std::vector> attributes_; std::vector> elements_; @@ -24,10 +35,10 @@ struct Value : value_() , attributes_() , elements_() - { + { } - Value( const std::string& value ) + Value( const String& value ) : value_( value ) , attributes_() , elements_() @@ -37,7 +48,7 @@ struct Value struct Attribute { - std::string name_; + String name_; shared_ptr value_; Attribute() @@ -46,7 +57,7 @@ struct Attribute { } - Attribute( const std::string& name, const shared_ptr& value ) + Attribute( const String& name, const shared_ptr& value ) : name_( name ) , value_( value ) { @@ -55,7 +66,7 @@ struct Attribute struct JsonUserData { - std::string name_; + String name_; shared_ptr value_; JsonUserData() @@ -70,42 +81,42 @@ struct JsonUserData { } - JsonUserData( const std::string& name, shared_ptr value ) + JsonUserData( const String& name, shared_ptr value ) : name_( name ) , value_( value ) { } }; -static JsonUserData document( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData document( const JsonUserData* start, const ParserNode* nodes, size_t length ) { return start[1]; } -static JsonUserData attribute( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData attribute( const JsonUserData* start, const ParserNode* nodes, size_t length ) { const shared_ptr& attribute = start[2].value_; return JsonUserData( nodes[0].lexeme(), attribute ); } -static JsonUserData null( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData null( const JsonUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr null_value = make_shared(); return JsonUserData( null_value ); } -static JsonUserData value( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData value( const JsonUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr value = make_shared( nodes[0].lexeme() ); return JsonUserData( value ); } -static JsonUserData object( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData object( const JsonUserData* start, const ParserNode* nodes, size_t length ) { return start[1]; } -static JsonUserData add_to_object( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData add_to_object( const JsonUserData* start, const ParserNode* nodes, size_t length ) { const shared_ptr& object = start[0].value_; shared_ptr attribute = make_shared( start[2].name_, start[2].value_ ); @@ -113,7 +124,7 @@ static JsonUserData add_to_object( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData create_object( const JsonUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr object = make_shared(); shared_ptr attribute = make_shared( start[0].name_, start[0].value_ ); @@ -121,7 +132,7 @@ static JsonUserData create_object( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData add_to_array( const JsonUserData* start, const ParserNode* nodes, size_t length ) { const shared_ptr& array = start[0].value_; const shared_ptr& element = start[2].value_; @@ -129,7 +140,7 @@ static JsonUserData add_to_array( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData create_array( const JsonUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr array = make_shared(); const shared_ptr& element = start[0].value_; @@ -147,20 +158,28 @@ static void indent( int level ) static void print( const Value& value, int level ) { + std::setlocale( LC_ALL, "en_US.UTF-8" ); + std::wstring_convert, char32_t> utf8; + for ( const shared_ptr& attribute : value.attributes_ ) { LALR_ASSERT( attribute ); - const string& name = attribute->name_; + const String& name = attribute->name_; const Value& value = *attribute->value_; if ( value.attributes_.empty() && value.elements_.empty() ) { indent( level + 1 ); - printf( "%s='%s'\n", attribute->name_.c_str(), attribute->value_->value_.c_str() ); + printf( "%s='%s'\n", + utf8.to_bytes(attribute->name_).c_str(), + utf8.to_bytes(attribute->value_->value_).c_str() + ); } else { indent( level + 1 ); - printf( "%s:\n", attribute->name_.c_str() ); + printf( "%s:\n", + utf8.to_bytes(attribute->name_).c_str() + ); print( value, level + 1 ); } } @@ -173,7 +192,10 @@ static void print( const Value& value, int level ) if ( value.attributes_.empty() && value.elements_.empty() ) { indent( level + 1 ); - printf( "%d: '%s'\n", index, value.value_.c_str() ); + printf( "%d: '%s'\n", + index, + utf8.to_bytes(value.value_).c_str() + ); } else { @@ -190,8 +212,8 @@ static void print( const Value& value, int level ) void lalr_json_example() { extern const lalr::ParserStateMachine* json_parser_state_machine; - Parser parser( json_parser_state_machine ); - parser.set_lexer_action_handler( "string", &string_literal ); + Parser, JsonUserData> parser( json_parser_state_machine ); + parser.set_lexer_action_handler( "string", &string_literal> ); parser.parser_action_handlers() ( "document", &document ) ( "add_to_object", &add_to_object ) @@ -205,20 +227,16 @@ void lalr_json_example() ( "array", &object ) ; - const char* input = - "{\n" - " \"model\": {\n" - " \"format\": \"Model\",\n" - " \"version\": 1,\n" - " \"address\": \"0017FAB0\",\n" - " \"items\": {\n" - " \"name\": \"Albert\"\n" - " },\n" - " \"more_items\": ['one', 2, 3]\n" - " }\n" - "}\n"; + using std::locale; + using std::codecvt; + using std::basic_ifstream; + using std::istreambuf_iterator; + std::basic_ifstream file( LALR_EXAMPLES "lalr_json_example.json", std::ios_base::binary ); + file.imbue( locale(file.getloc(), new codecvt) ); + istreambuf_iterator input( file ); + istreambuf_iterator input_end; - parser.parse( input, input + strlen(input) ); + parser.parse( input, input_end ); LALR_ASSERT( parser.accepted() ); LALR_ASSERT( parser.full() ); print( *parser.user_data().value_, 0 ); diff --git a/src/lalr/lalr_examples/lalr_json_example.json b/src/lalr/lalr_examples/lalr_json_example.json new file mode 100644 index 0000000..65f178c --- /dev/null +++ b/src/lalr/lalr_examples/lalr_json_example.json @@ -0,0 +1,11 @@ +{ + "model": { + "format": "Model", + "version": 1, + "address": "0017FAB0", + "items": { + "name": "Albert" + }, + "more_items": ["one", 2, 3, "to prove that it's really UTF-8... 😁 😄!"] + } +} From 071273032ec490f1a6ac25c5062cbc72e48105f4 Mon Sep 17 00:00:00 2001 From: Charles Baker Date: Sat, 3 Jun 2023 11:37:13 +1200 Subject: [PATCH 3/4] Update XML example to parser from UTF-8 to UTF-8 in memory --- src/lalr/lalr_examples/lalr_xml_example.cpp | 37 +++++++++++---------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/lalr/lalr_examples/lalr_xml_example.cpp b/src/lalr/lalr_examples/lalr_xml_example.cpp index 5def603..6512426 100644 --- a/src/lalr/lalr_examples/lalr_xml_example.cpp +++ b/src/lalr/lalr_examples/lalr_xml_example.cpp @@ -4,6 +4,7 @@ #include #include #include +#include using namespace std; using namespace lalr; @@ -11,21 +12,23 @@ using namespace lalr; namespace { +typedef std::basic_string String; + struct Attribute { - std::string name_; - std::string value_; + String name_; + String value_; - Attribute( const std::string& name, const std::string& value ) - : name_( name ), - value_( value ) + Attribute( const String& name, const String& value ) + : name_( name ) + , value_( value ) { } }; struct Element { - std::string name_; + String name_; std::list > attributes_; std::list > elements_; @@ -61,7 +64,7 @@ struct XmlUserData } }; -static XmlUserData document( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData document( const XmlUserData* start, const ParserNode* nodes, size_t length ) { const XmlUserData* end = start + length; while ( start != end && !start[0].element_ ) @@ -71,28 +74,28 @@ static XmlUserData document( const XmlUserData* start, const ParserNode* n return start != end ? start[0] : XmlUserData(); } -static XmlUserData add_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData add_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr element = start[0].element_; element->elements_.push_back( start[1].element_ ); return XmlUserData( element ); } -static XmlUserData create_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData create_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr element( new Element() ); element->elements_.push_back( start[0].element_ ); return XmlUserData( element ); } -static XmlUserData short_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData short_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr element = start[2].element_; element->name_ = nodes[1].lexeme(); return XmlUserData( element ); } -static XmlUserData long_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData long_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr element = start[2].element_; if ( !element ) @@ -108,7 +111,7 @@ static XmlUserData long_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData add_attribute( const XmlUserData* start, const ParserNode* nodes, size_t length ) { LALR_ASSERT( start[0].element_ ); shared_ptr element = start[0].element_; @@ -117,7 +120,7 @@ static XmlUserData add_attribute( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData create_attribute( const XmlUserData* start, const ParserNode* nodes, size_t length ) { LALR_ASSERT( start[0].attribute_ ); shared_ptr element( new Element() ); @@ -125,7 +128,7 @@ static XmlUserData create_attribute( const XmlUserData* start, const ParserNode< return XmlUserData( element ); } -static XmlUserData attribute( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData attribute( const XmlUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr attribute( new Attribute(nodes[0].lexeme(), nodes[2].lexeme()) ); return XmlUserData( attribute ); @@ -166,9 +169,9 @@ static void print( const Element* element, int level ) void lalr_xml_example() { extern const lalr::ParserStateMachine* xml_parser_state_machine; - Parser parser( xml_parser_state_machine ); + Parser parser( xml_parser_state_machine ); parser.lexer_action_handlers() - ( "string", &string_literal ) + ( "string", &string_literal ) ; parser.parser_action_handlers() ( "document", &document ) @@ -190,7 +193,7 @@ void lalr_xml_example() " " ; - parser.parse( input, input + strlen(input) ); + parser.parse( (const uint8_t*) input, (const uint8_t*) input + strlen(input) ); LALR_ASSERT( parser.accepted() ); LALR_ASSERT( parser.full() ); print( parser.user_data().element_.get(), 0 ); From e346baf911a0c4d3348e51afb9005eabb5da38bb Mon Sep 17 00:00:00 2001 From: Charles Baker Date: Sat, 3 Jun 2023 11:37:30 +1200 Subject: [PATCH 4/4] Update README.md to describe parsing with different encodings --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0a62350..4f3c346 100644 --- a/README.md +++ b/README.md @@ -183,9 +183,9 @@ Parser parser( xml_parser_state_machine ); Reference a parse table as an `extern` variable for offline generated parse tables. See [lalr_calculator_example.cpp](#lalr_calculator_example.cpp) for an example of compiling a grammar to parse tables at runtime. -Create a `Parser` object with the parse table as the sole argument to the constructor. The `Parser` class template requires an iterator type template argument and optionally allows for user data, character type, character traits, and allocator to be overridden. +Create a `Parser` object with the parse table as the sole argument to the constructor. The `Parser` class template requires an iterator type template argument and optionally allows for user data; and character type, traits, and allocator to be overridden. In the above example the iterator type is `const char*`, user data is the custom `XmlUserData` type, and the character parameters default to those implied by the iterator. -In the above example the iterator type is `const char*` and the user data is the custom `XmlUserData` type. +Change the `Iterator` template parameter to read input from different sources and convert character encodings, e.g. from UTF-8 in a file to UTF-32 in memory. See [lalr_json_example.cpp](lalr/lalr_examples/lalr_json_example.cpp) for an example of reading a UTF-8 encoded file to UTF-32, `char32_t` in memory. To parse UTF-8 input to UTF-8 encoding in memory it is usually sufficient to use a iterator templated to `unsigned char` or `uint8_t`, see [lalr_xml_example.cpp](lalr/lalr_examples/lalr_xml_example.cpp) for an example of doing so in practice. **3. Bind lexer action handlers**