diff --git a/README.md b/README.md index 0a62350..4f3c346 100644 --- a/README.md +++ b/README.md @@ -183,9 +183,9 @@ Parser parser( xml_parser_state_machine ); Reference a parse table as an `extern` variable for offline generated parse tables. See [lalr_calculator_example.cpp](#lalr_calculator_example.cpp) for an example of compiling a grammar to parse tables at runtime. -Create a `Parser` object with the parse table as the sole argument to the constructor. The `Parser` class template requires an iterator type template argument and optionally allows for user data, character type, character traits, and allocator to be overridden. +Create a `Parser` object with the parse table as the sole argument to the constructor. The `Parser` class template requires an iterator type template argument and optionally allows for user data; and character type, traits, and allocator to be overridden. In the above example the iterator type is `const char*`, user data is the custom `XmlUserData` type, and the character parameters default to those implied by the iterator. -In the above example the iterator type is `const char*` and the user data is the custom `XmlUserData` type. +Change the `Iterator` template parameter to read input from different sources and convert character encodings, e.g. from UTF-8 in a file to UTF-32 in memory. See [lalr_json_example.cpp](lalr/lalr_examples/lalr_json_example.cpp) for an example of reading a UTF-8 encoded file to UTF-32, `char32_t` in memory. To parse UTF-8 input to UTF-8 encoding in memory it is usually sufficient to use a iterator templated to `unsigned char` or `uint8_t`, see [lalr_xml_example.cpp](lalr/lalr_examples/lalr_xml_example.cpp) for an example of doing so in practice. **3. Bind lexer action handlers** diff --git a/src/lalr/Parser.ipp b/src/lalr/Parser.ipp index 30f5670..dad237c 100644 --- a/src/lalr/Parser.ipp +++ b/src/lalr/Parser.ipp @@ -494,7 +494,7 @@ void Parser::debug_shift( const Par if ( debug_enabled_ ) { const ParserSymbol* symbol = node.symbol(); - const std::string& lexeme = node.lexeme(); + const std::basic_string& lexeme = node.lexeme(); int line = node.line(); int column = node.column(); fire_printf( "SHIFT: (%s %s %d:%d)\n", symbol ? symbol->identifier : "", lexeme.c_str(), line, column ); @@ -521,14 +521,14 @@ void Parser::debug_reduce( const Pa if ( debug_enabled_ ) { - fire_printf( "REDUCE: %s <- ", reduced_symbol->identifier ); + fire_printf( "REDUCE: %s <- ", reduced_symbol->identifier ); const ParserNode* node = nodes_.data() + start; const ParserNode* node_end = nodes_.data() + finish; if ( node != node_end ) { const ParserSymbol* symbol = node->symbol(); - const std::string& lexeme = node->lexeme(); + const std::basic_string& lexeme = node->lexeme(); int line = node->line(); int column = node->column(); fire_printf( "(%s %s %d:%d)", symbol ? symbol->identifier : "", lexeme.c_str(), line, column ); @@ -538,7 +538,7 @@ void Parser::debug_reduce( const Pa while ( node != node_end ) { const ParserSymbol* symbol = node->symbol(); - const std::string& lexeme = node->lexeme(); + const std::basic_string& lexeme = node->lexeme(); int line = node->line(); int column = node->column(); fire_printf( " (%s %s %d:%d)", symbol ? symbol->identifier : "", lexeme.c_str(), line, column ); diff --git a/src/lalr/lalr_examples/lalr_examples.forge b/src/lalr/lalr_examples/lalr_examples.forge index 203f948..d6c0ab4 100644 --- a/src/lalr/lalr_examples/lalr_examples.forge +++ b/src/lalr/lalr_examples/lalr_examples.forge @@ -15,6 +15,9 @@ for _, cc in toolsets('^cc.*') do libraries = libraries; '${lib}/lalr_${platform}_${architecture}'; cc:Cxx '${obj}/%1' { + defines = { + ([[LALR_EXAMPLES=\"%s/\"]]):format( pwd() ); + }; "lalr_examples.cpp", "lalr_error_handling_calculator_example.cpp", "lalr_hello_world_example.cpp", diff --git a/src/lalr/lalr_examples/lalr_json_example.cpp b/src/lalr/lalr_examples/lalr_json_example.cpp index cdc17c4..cfb611c 100644 --- a/src/lalr/lalr_examples/lalr_json_example.cpp +++ b/src/lalr/lalr_examples/lalr_json_example.cpp @@ -4,7 +4,16 @@ #include #include #include +#include #include +#include +#include +#include +#ifdef __APPLE__ +#include +#else +#include +#endif using namespace std; using namespace lalr; @@ -12,11 +21,13 @@ using namespace lalr; namespace { +typedef std::basic_string String; + struct Attribute; struct Value { - std::string value_; + String value_; std::vector> attributes_; std::vector> elements_; @@ -24,10 +35,10 @@ struct Value : value_() , attributes_() , elements_() - { + { } - Value( const std::string& value ) + Value( const String& value ) : value_( value ) , attributes_() , elements_() @@ -37,7 +48,7 @@ struct Value struct Attribute { - std::string name_; + String name_; shared_ptr value_; Attribute() @@ -46,7 +57,7 @@ struct Attribute { } - Attribute( const std::string& name, const shared_ptr& value ) + Attribute( const String& name, const shared_ptr& value ) : name_( name ) , value_( value ) { @@ -55,7 +66,7 @@ struct Attribute struct JsonUserData { - std::string name_; + String name_; shared_ptr value_; JsonUserData() @@ -70,42 +81,42 @@ struct JsonUserData { } - JsonUserData( const std::string& name, shared_ptr value ) + JsonUserData( const String& name, shared_ptr value ) : name_( name ) , value_( value ) { } }; -static JsonUserData document( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData document( const JsonUserData* start, const ParserNode* nodes, size_t length ) { return start[1]; } -static JsonUserData attribute( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData attribute( const JsonUserData* start, const ParserNode* nodes, size_t length ) { const shared_ptr& attribute = start[2].value_; return JsonUserData( nodes[0].lexeme(), attribute ); } -static JsonUserData null( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData null( const JsonUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr null_value = make_shared(); return JsonUserData( null_value ); } -static JsonUserData value( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData value( const JsonUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr value = make_shared( nodes[0].lexeme() ); return JsonUserData( value ); } -static JsonUserData object( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData object( const JsonUserData* start, const ParserNode* nodes, size_t length ) { return start[1]; } -static JsonUserData add_to_object( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData add_to_object( const JsonUserData* start, const ParserNode* nodes, size_t length ) { const shared_ptr& object = start[0].value_; shared_ptr attribute = make_shared( start[2].name_, start[2].value_ ); @@ -113,7 +124,7 @@ static JsonUserData add_to_object( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData create_object( const JsonUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr object = make_shared(); shared_ptr attribute = make_shared( start[0].name_, start[0].value_ ); @@ -121,7 +132,7 @@ static JsonUserData create_object( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData add_to_array( const JsonUserData* start, const ParserNode* nodes, size_t length ) { const shared_ptr& array = start[0].value_; const shared_ptr& element = start[2].value_; @@ -129,7 +140,7 @@ static JsonUserData add_to_array( const JsonUserData* start, const ParserNode* nodes, size_t length ) +static JsonUserData create_array( const JsonUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr array = make_shared(); const shared_ptr& element = start[0].value_; @@ -147,20 +158,28 @@ static void indent( int level ) static void print( const Value& value, int level ) { + std::setlocale( LC_ALL, "en_US.UTF-8" ); + std::wstring_convert, char32_t> utf8; + for ( const shared_ptr& attribute : value.attributes_ ) { LALR_ASSERT( attribute ); - const string& name = attribute->name_; + const String& name = attribute->name_; const Value& value = *attribute->value_; if ( value.attributes_.empty() && value.elements_.empty() ) { indent( level + 1 ); - printf( "%s='%s'\n", attribute->name_.c_str(), attribute->value_->value_.c_str() ); + printf( "%s='%s'\n", + utf8.to_bytes(attribute->name_).c_str(), + utf8.to_bytes(attribute->value_->value_).c_str() + ); } else { indent( level + 1 ); - printf( "%s:\n", attribute->name_.c_str() ); + printf( "%s:\n", + utf8.to_bytes(attribute->name_).c_str() + ); print( value, level + 1 ); } } @@ -173,7 +192,10 @@ static void print( const Value& value, int level ) if ( value.attributes_.empty() && value.elements_.empty() ) { indent( level + 1 ); - printf( "%d: '%s'\n", index, value.value_.c_str() ); + printf( "%d: '%s'\n", + index, + utf8.to_bytes(value.value_).c_str() + ); } else { @@ -190,8 +212,8 @@ static void print( const Value& value, int level ) void lalr_json_example() { extern const lalr::ParserStateMachine* json_parser_state_machine; - Parser parser( json_parser_state_machine ); - parser.set_lexer_action_handler( "string", &string_literal ); + Parser, JsonUserData> parser( json_parser_state_machine ); + parser.set_lexer_action_handler( "string", &string_literal> ); parser.parser_action_handlers() ( "document", &document ) ( "add_to_object", &add_to_object ) @@ -205,20 +227,16 @@ void lalr_json_example() ( "array", &object ) ; - const char* input = - "{\n" - " \"model\": {\n" - " \"format\": \"Model\",\n" - " \"version\": 1,\n" - " \"address\": \"0017FAB0\",\n" - " \"items\": {\n" - " \"name\": \"Albert\"\n" - " },\n" - " \"more_items\": ['one', 2, 3]\n" - " }\n" - "}\n"; + using std::locale; + using std::codecvt; + using std::basic_ifstream; + using std::istreambuf_iterator; + std::basic_ifstream file( LALR_EXAMPLES "lalr_json_example.json", std::ios_base::binary ); + file.imbue( locale(file.getloc(), new codecvt) ); + istreambuf_iterator input( file ); + istreambuf_iterator input_end; - parser.parse( input, input + strlen(input) ); + parser.parse( input, input_end ); LALR_ASSERT( parser.accepted() ); LALR_ASSERT( parser.full() ); print( *parser.user_data().value_, 0 ); diff --git a/src/lalr/lalr_examples/lalr_json_example.json b/src/lalr/lalr_examples/lalr_json_example.json new file mode 100644 index 0000000..65f178c --- /dev/null +++ b/src/lalr/lalr_examples/lalr_json_example.json @@ -0,0 +1,11 @@ +{ + "model": { + "format": "Model", + "version": 1, + "address": "0017FAB0", + "items": { + "name": "Albert" + }, + "more_items": ["one", 2, 3, "to prove that it's really UTF-8... 😁 😄!"] + } +} diff --git a/src/lalr/lalr_examples/lalr_xml_example.cpp b/src/lalr/lalr_examples/lalr_xml_example.cpp index 5def603..6512426 100644 --- a/src/lalr/lalr_examples/lalr_xml_example.cpp +++ b/src/lalr/lalr_examples/lalr_xml_example.cpp @@ -4,6 +4,7 @@ #include #include #include +#include using namespace std; using namespace lalr; @@ -11,21 +12,23 @@ using namespace lalr; namespace { +typedef std::basic_string String; + struct Attribute { - std::string name_; - std::string value_; + String name_; + String value_; - Attribute( const std::string& name, const std::string& value ) - : name_( name ), - value_( value ) + Attribute( const String& name, const String& value ) + : name_( name ) + , value_( value ) { } }; struct Element { - std::string name_; + String name_; std::list > attributes_; std::list > elements_; @@ -61,7 +64,7 @@ struct XmlUserData } }; -static XmlUserData document( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData document( const XmlUserData* start, const ParserNode* nodes, size_t length ) { const XmlUserData* end = start + length; while ( start != end && !start[0].element_ ) @@ -71,28 +74,28 @@ static XmlUserData document( const XmlUserData* start, const ParserNode* n return start != end ? start[0] : XmlUserData(); } -static XmlUserData add_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData add_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr element = start[0].element_; element->elements_.push_back( start[1].element_ ); return XmlUserData( element ); } -static XmlUserData create_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData create_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr element( new Element() ); element->elements_.push_back( start[0].element_ ); return XmlUserData( element ); } -static XmlUserData short_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData short_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr element = start[2].element_; element->name_ = nodes[1].lexeme(); return XmlUserData( element ); } -static XmlUserData long_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData long_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr element = start[2].element_; if ( !element ) @@ -108,7 +111,7 @@ static XmlUserData long_element( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData add_attribute( const XmlUserData* start, const ParserNode* nodes, size_t length ) { LALR_ASSERT( start[0].element_ ); shared_ptr element = start[0].element_; @@ -117,7 +120,7 @@ static XmlUserData add_attribute( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData create_attribute( const XmlUserData* start, const ParserNode* nodes, size_t length ) { LALR_ASSERT( start[0].attribute_ ); shared_ptr element( new Element() ); @@ -125,7 +128,7 @@ static XmlUserData create_attribute( const XmlUserData* start, const ParserNode< return XmlUserData( element ); } -static XmlUserData attribute( const XmlUserData* start, const ParserNode* nodes, size_t length ) +static XmlUserData attribute( const XmlUserData* start, const ParserNode* nodes, size_t length ) { shared_ptr attribute( new Attribute(nodes[0].lexeme(), nodes[2].lexeme()) ); return XmlUserData( attribute ); @@ -166,9 +169,9 @@ static void print( const Element* element, int level ) void lalr_xml_example() { extern const lalr::ParserStateMachine* xml_parser_state_machine; - Parser parser( xml_parser_state_machine ); + Parser parser( xml_parser_state_machine ); parser.lexer_action_handlers() - ( "string", &string_literal ) + ( "string", &string_literal ) ; parser.parser_action_handlers() ( "document", &document ) @@ -190,7 +193,7 @@ void lalr_xml_example() " " ; - parser.parse( input, input + strlen(input) ); + parser.parse( (const uint8_t*) input, (const uint8_t*) input + strlen(input) ); LALR_ASSERT( parser.accepted() ); LALR_ASSERT( parser.full() ); print( parser.user_data().element_.get(), 0 );