Generate state machine from kernel items only

src/lalr/GrammarGenerator.cpp

@@ -11,6 +11,7 @@
 #include "GrammarSymbol.hpp"   
 #include "GrammarAction.hpp"
 #include "GrammarCompiler.hpp"
+#include "GrammarProductionLess.hpp"
 #include "ParserState.hpp"
 #include "ParserAction.hpp"
 #include "ParserSymbol.hpp"
@@ -24,8 +25,10 @@
 
 using std::set;
 using std::vector;
+using std::multimap;
 using std::unique_ptr;
 using std::shared_ptr;
+using std::make_shared;
 using namespace lalr;
 
 /**
@@ -107,6 +110,7 @@ int GrammarGenerator::generate( Grammar& grammar, ErrorPolicy* error_policy )
         calculate_first();
         calculate_follow();
         calculate_precedence_of_productions();
+        calculate_reachable_productions();
         generate_states( start_symbol_, end_symbol_, symbols_ );
     }
 
@@ -240,13 +244,26 @@ std::shared_ptr<GrammarState> GrammarGenerator::goto_( const std::shared_ptr<Gra
     const set<GrammarItem>& items = state->items();
     for ( set<GrammarItem>::const_iterator item = items.begin(); item != items.end(); ++item )
     {
-        if ( item->next_node(symbol) )
+        const GrammarSymbol* next_symbol = item->next_symbol();
+        if ( next_symbol )
         {
-            goto_state->add_item( item->production(), item->position() + 1 );
+            if ( next_symbol == &symbol )
+            {
+                goto_state->add_item( item->production(), item->position() + 1 );
+            }
+    
+            const multimap<const GrammarSymbol*, GrammarProduction*>& reachable_productions = next_symbol->reachable_productions_by_first_symbol();
+            auto i = reachable_productions.find( &symbol );
+            while ( i != reachable_productions.end() && i->first == &symbol )
+            {
+                GrammarProduction* production = i->second;
+                const int position = 1;
+                goto_state->add_item( production, position );
+                ++i;
+            }
         }
     }
 
-    closure( goto_state );
     return goto_state;
 }
 
@@ -620,6 +637,49 @@ void GrammarGenerator::calculate_follow()
     }
 }
 
+/**
+// Calculate the productions reachable by right-most derivation from each
+// non-terminal.
+//
+// This pre-calculation is used to add items to goto states that would have
+// been added for transitions from non-kernel items.  This is needed because
+// non-kernel items are excluded in an efficient construction of LALR(1)
+// states.  The pre-calculation provides a more efficient and convenient
+// means of discovering items added for transitions from non-kernel items.
+*/
+void GrammarGenerator::calculate_reachable_productions()
+{
+    for ( const auto& symbol : symbols_ )
+    {
+        set<GrammarProduction*, GrammarProductionLess> productions;
+        calculate_reachable_productions_for_symbol( *symbol, &productions );
+        for ( const auto& production : productions )
+        {
+            symbol->append_reachable_production( production );
+        }
+    }
+}
+
+/**
+// Recursively calculate productions reachable by derivation from symbol.
+*/
+void GrammarGenerator::calculate_reachable_productions_for_symbol( const GrammarSymbol& symbol, std::set<GrammarProduction*, GrammarProductionLess>* productions )
+{
+    LALR_ASSERT( productions );
+    for ( const auto& production : symbol.productions() )
+    {
+        const GrammarSymbol* first_symbol = production->symbol_by_position( 0 );
+        if ( first_symbol )
+        {
+            bool inserted = productions->insert( production ).second;
+            if ( inserted )
+            {
+                calculate_reachable_productions_for_symbol( *first_symbol, productions );
+            }
+        }
+    }
+}
+
 /**
 // Generate the states for a grammar with \e symbols starting with 
 // \e start_symbol and ending when \e end_symbol is accepted.
@@ -641,7 +701,7 @@ void GrammarGenerator::generate_states( const GrammarSymbol* start_symbol, const
 
     if ( !start_symbol->productions().empty() )
     {
-        std::shared_ptr<GrammarState> start_state( new GrammarState() );
+        shared_ptr<GrammarState> start_state = make_shared<GrammarState>();
         start_state->add_item( start_symbol->productions().front(), 0 );
         closure( start_state );
         states_.insert( start_state );
@@ -681,6 +741,20 @@ void GrammarGenerator::generate_states( const GrammarSymbol* start_symbol, const
                 }
             }
         }
+
+        vector<shared_ptr<GrammarState>> states;
+        states.reserve( states_.size() );
+        for ( const auto& state : states_ )
+        {
+            states.push_back( state );
+        }
+
+        states_.clear();
+        for ( const auto& state : states )
+        {
+            closure( state );
+            states_.insert( state );
+        }
         
         generate_indices_for_states();
 

src/lalr/GrammarGenerator.hpp

@@ -4,6 +4,7 @@
 #include "RegexToken.hpp"
 #include "GrammarSymbolLess.hpp"
 #include "GrammarStateLess.hpp"
+#include "GrammarProductionLess.hpp"
 #include <memory>
 #include <set>
 #include <vector>
@@ -64,12 +65,14 @@ private:
     void check_for_error_symbol_on_left_hand_side_errors();
     void check_for_implicit_terminal_duplicate_associativity();
     void calculate_identifiers();
-    void calculate_terminal_and_non_terminal_symbols();
+    void calculate_terminal_and_non_terminal_symbols();    
     void calculate_implicit_terminal_symbols();
     void calculate_precedence_of_productions();
     void calculate_symbol_indices();
     void calculate_first();
     void calculate_follow();
+    void calculate_reachable_productions();
+    void calculate_reachable_productions_for_symbol( const GrammarSymbol& symbol, std::set<GrammarProduction*, GrammarProductionLess>* productions );
     void generate_states( const GrammarSymbol* start_symbol, const GrammarSymbol* end_symbol, const std::vector<std::unique_ptr<GrammarSymbol>>& symbols );
     void generate_indices_for_states();
     void generate_reduce_transitions();

src/lalr/GrammarItem.cpp

@@ -94,7 +94,19 @@ bool GrammarItem::dot_at_end() const
 */
 bool GrammarItem::next_node( const GrammarSymbol& symbol ) const
 {
-    return production_->symbol_by_position(position_) == &symbol;
+    return production_->symbol_by_position( position_ ) == &symbol;
+}
+
+/**
+// Get the symbol after the dot in this item.
+//
+// @return
+//  The next symbol, i.e. following the dot, in this item or null if there
+//  is no such symbol.
+*/
+const GrammarSymbol* GrammarItem::next_symbol() const
+{
+    return production_->symbol_by_position( position_ );
 }
 
 /**

src/lalr/GrammarItem.hpp

@@ -29,6 +29,7 @@ public:
     bool dot_at_beginning() const;
     bool dot_at_end() const;
     bool next_node( const GrammarSymbol& symbol ) const;
+    const GrammarSymbol* next_symbol() const;
     const std::set<const GrammarSymbol*, GrammarSymbolLess>& lookahead_symbols() const;
     bool operator<( const GrammarItem& item ) const;
     int add_lookahead_symbols( const std::set<const GrammarSymbol*, GrammarSymbolLess>& lookahead_symbols ) const;

src/lalr/GrammarProductionLess.cpp

@@ -0,0 +1,17 @@
+//
+// GrammarProductionLess.cpp
+// Copyright (c) Charles Baker. All rights reserved.
+//
+
+#include "GrammarProductionLess.hpp"
+#include "GrammarProduction.hpp"
+#include "assert.hpp"
+
+using namespace lalr;
+
+bool GrammarProductionLess::operator()( const GrammarProduction* lhs, const GrammarProduction* rhs ) const
+{
+    LALR_ASSERT( lhs->index() >= 0 );
+    LALR_ASSERT( rhs->index() >= 0 );
+    return lhs->index() < rhs->index();
+}

src/lalr/GrammarProductionLess.hpp

@@ -0,0 +1,17 @@
+#pragma once
+
+namespace lalr
+{
+
+class GrammarProduction;
+
+/**
+// Indirectly compare productions through pointers.
+*/
+class GrammarProductionLess
+{
+public:
+    bool operator()( const GrammarProduction* lhs, const GrammarProduction* rhs ) const;
+};
+
+}

src/lalr/GrammarState.cpp

@@ -23,6 +23,14 @@ GrammarState::GrammarState()
 {
 }
 
+GrammarState::GrammarState( const GrammarState& state )
+: items_( state.items_ )
+, transitions_( state.transitions_ )
+, processed_( state.processed_ )
+, index_( state.index_ )
+{
+}
+
 /**
 // Get the items in this state.
 //

src/lalr/GrammarState.hpp

@@ -24,6 +24,7 @@ class GrammarState
 
 public:
     GrammarState();
+    GrammarState( const GrammarState& state );
 
     const std::set<GrammarItem>& items() const;
     const GrammarTransition* find_transition_by_symbol( const GrammarSymbol* symbol ) const;

src/lalr/GrammarSymbol.cpp

@@ -11,6 +11,7 @@
 using std::set;
 using std::vector;
 using std::shared_ptr;
+using std::make_pair;
 using namespace lalr;
 
 GrammarSymbol::GrammarSymbol( const std::string& lexeme )
@@ -95,6 +96,16 @@ const std::vector<GrammarProduction*>& GrammarSymbol::productions() const
     return productions_;
 }
 
+const std::multimap<const GrammarSymbol*, GrammarProduction*>& GrammarSymbol::reachable_productions_by_first_symbol() const
+{
+    return reachable_productions_by_first_symbol_;
+}
+
+std::multimap<const GrammarSymbol*, GrammarProduction*>::const_iterator GrammarSymbol::find_reachable_productions( const GrammarSymbol& first_symbol ) const
+{
+    return reachable_productions_by_first_symbol_.find( &first_symbol );
+}
+
 /**
 // Get the implicit terminal for this symbol.
 //
@@ -197,6 +208,14 @@ void GrammarSymbol::append_production( GrammarProduction* production )
     productions_.push_back( production );
 }
 
+void GrammarSymbol::append_reachable_production( GrammarProduction* production )
+{
+    LALR_ASSERT( production );
+    const GrammarSymbol* first_symbol = production->symbol_by_position( 0 );
+    LALR_ASSERT( first_symbol );
+    reachable_productions_by_first_symbol_.insert( make_pair(first_symbol, production) );
+}
+
 /**
 // Calculate the identifier for this symbol.
 */

src/lalr/GrammarSymbol.hpp

@@ -8,6 +8,7 @@
 #include <string>
 #include <vector>
 #include <set>
+#include <map>
 
 namespace lalr
 {
@@ -29,6 +30,7 @@ class GrammarSymbol
     std::set<const GrammarSymbol*, GrammarSymbolLess> first_; ///< The symbols that can start this symbol in a production or regular expression.
     std::set<const GrammarSymbol*, GrammarSymbolLess> follow_; ///< The symbols that can follow this symbol in a production or regular expression.
     std::vector<GrammarProduction*> productions_; ///< The productions that reduce to this symbol.
+    std::multimap<const GrammarSymbol*, GrammarProduction*> reachable_productions_by_first_symbol_; ///< The productions reachable by right-most derivation from this symbol by their first symbol.
 
 public:
     GrammarSymbol( const std::string& lexeme );
@@ -46,6 +48,8 @@ public:
     const std::set<const GrammarSymbol*, GrammarSymbolLess>& first() const;
     const std::set<const GrammarSymbol*, GrammarSymbolLess>& follow() const;
     const std::vector<GrammarProduction*>& productions() const;
+    const std::multimap<const GrammarSymbol*, GrammarProduction*>& reachable_productions_by_first_symbol() const;
+    std::multimap<const GrammarSymbol*, GrammarProduction*>::const_iterator find_reachable_productions( const GrammarSymbol& first_symbol ) const;
     GrammarSymbol* implicit_terminal() const;
     bool matches( const std::string& lexeme, SymbolType symbol_type ) const;
 
@@ -60,6 +64,7 @@ public:
     void set_nullable( bool nullable );
     void set_referenced_in_precedence_directive( bool referenced_in_precedence_directive );
     void append_production( GrammarProduction* production );
+    void append_reachable_production( GrammarProduction* production );
     void calculate_identifier();
     void replace_by_non_terminal( const GrammarSymbol* non_terminal_symbol );    
     int add_symbol_to_first( const GrammarSymbol* symbol );

src/lalr/lalr.forge

@@ -18,6 +18,7 @@ for _, cc in toolsets('^cc.*') do
                 'GrammarItem.cpp',
                 'GrammarParser.cpp',
                 'GrammarProduction.cpp',
+                'GrammarProductionLess.cpp',
                 'GrammarState.cpp',
                 'GrammarStateLess.cpp',
                 'GrammarSymbol.cpp',