Add indexes for market transactions, improve performance for blockchain_...

libraries/blockchain/chain_database.cpp

@@ -181,6 +181,8 @@ namespace bts { namespace blockchain {
           _feed_index_to_record.open( data_dir / "index/feed_index_to_record" );
 
           _market_transactions_db.open( data_dir / "index/market_transactions_db" );
+          _market_transactions_index.open( data_dir / "index/market_transactions_index" );
+          _market_transactions_owner_index.open( data_dir / "index/market_transactions_owner_index" );
 
           _pending_transaction_db.open( data_dir / "index/pending_transaction_db" );
 
@@ -1464,6 +1466,8 @@ namespace bts { namespace blockchain {
                   my->_collateral_db.set_write_through( write_through );
 
                   my->_market_transactions_db.set_write_through( write_through );
+                  my->_market_transactions_index.set_write_through( write_through );
+                  //my->_market_transactions_owner_index.set_write_through( write_through );
                   my->_market_history_db.set_write_through( write_through );
               };
 
@@ -1648,6 +1652,8 @@ namespace bts { namespace blockchain {
 
       my->_market_history_db.close();
       my->_market_transactions_db.close();
+      my->_market_transactions_index.close();
+      my->_market_transactions_owner_index.close();
 
       my->_slot_index_to_record.close();
       my->_slot_timestamp_to_delegate.close();
@@ -3000,14 +3006,38 @@ namespace bts { namespace blockchain {
 
    void chain_database::set_market_transactions( vector<market_transaction> trxs )
    {
-      if( trxs.size() == 0 )
+      // cleanup old index
+      auto new_head_num = get_head_block_num() + 1;
+      const auto& old_transactions = get_market_transactions( new_head_num );
+      for( uint32_t i = 0; i < old_transactions.size(); i++ )
       {
-         my->_market_transactions_db.remove( get_head_block_num()+1 );
+         auto& transaction = old_transactions[i];
+         auto& base_id = transaction.ask_index.order_price.base_asset_id;
+         auto& quote_id = transaction.ask_index.order_price.quote_asset_id;
+         auto& bid_owner = transaction.bid_index.owner;
+         auto& ask_owner = transaction.ask_index.owner;
+         my->_market_transactions_index.remove( { base_id, quote_id, new_head_num, i } );
+         my->_market_transactions_owner_index.remove( { bid_owner, new_head_num, i } );
+         my->_market_transactions_owner_index.remove( { ask_owner, new_head_num, i } );
       }
-      else
+      // update index
+      int32_t dummy_value = 0;
+      for( uint32_t i = 0; i < trxs.size(); i++ )
       {
-         my->_market_transactions_db.store( get_head_block_num()+1, trxs );
+         auto& transaction = trxs[i];
+         auto& base_id = transaction.ask_index.order_price.base_asset_id;
+         auto& quote_id = transaction.ask_index.order_price.quote_asset_id;
+         auto& bid_owner = transaction.bid_index.owner;
+         auto& ask_owner = transaction.ask_index.owner;
+         my->_market_transactions_index.store( { base_id, quote_id, new_head_num, i }, dummy_value );
+         my->_market_transactions_owner_index.store( { bid_owner, new_head_num, i }, dummy_value );
+         my->_market_transactions_owner_index.store( { ask_owner, new_head_num, i }, dummy_value );
       }
+      // update data
+      if( trxs.size() == 0 )
+         my->_market_transactions_db.remove( get_head_block_num()+1 );
+      else
+         my->_market_transactions_db.store( get_head_block_num()+1, trxs );
    }
 
    vector<market_transaction> chain_database::get_market_transactions( const uint32_t block_num )const
@@ -3022,6 +3052,85 @@ namespace bts { namespace blockchain {
                                                                      uint32_t skip_count,
                                                                      uint32_t limit,
                                                                      const address& owner)
+   {
+      FC_ASSERT( skip_count >= 0, "Skip_count must be at least 0!" );
+      FC_ASSERT( skip_count <= 1000, "Skip_count must be at most 1000!" );
+      FC_ASSERT( limit > 0, "Limit must be at least 1!" );
+      FC_ASSERT( limit <= 1000, "Limit must be at most 1000!" );
+
+      uint32_t current_block_num = get_head_block_num();
+      FC_ASSERT( current_block_num > 0, "No blocks have been created yet!" );
+
+      // if owner is null, fetch data with market_transaction_index
+      // else fetch data with market_transaction_owner_index
+      if( owner == address() )
+      {
+        std::vector<order_history_record> results;
+        auto itr = my->_market_transactions_index.lower_bound( { base, quote + 1, 0, 0 } );
+        uint32_t skipped_count = 0;
+        uint32_t count = 0;
+        for( --itr;
+             count < limit && itr.valid() && itr.key().base_id == base && itr.key().quote_id == quote;
+             --itr, ++skipped_count )
+        {
+          if( skipped_count < skip_count ) continue;
+          auto block_num = itr.key().block_num;
+          auto trx_index = itr.key().transaction_index;
+          const auto& transactions = get_market_transactions( block_num );
+          FC_ASSERT( transactions.size() > trx_index,
+              "Invalid _market_transaction_index: block_num='${block_num}', trx_index='${trx_index}'!",
+              ("block_num", block_num)
+              ("trx_index", trx_index) );
+          auto& trx = transactions.at(trx_index);
+          const auto& stamp = get_block_header(block_num).timestamp;
+          results.emplace_back(order_history_record(trx, stamp));
+          ++count;
+        }
+        return results;
+      }
+      else
+      {
+        // TODO best if we can start from the end and travel backwards.
+        // if not, poor performance here:
+        //    search the index, fetch the ones we want into an vector, erase the last "skip_count" records, reverse
+        std::vector<order_history_record> results;
+        auto itr = my->_market_transactions_owner_index.lower_bound( { owner, 0, 0 } );
+        for( ; itr.valid() && itr.key().owner == owner; ++itr )
+        {
+          auto block_num = itr.key().block_num;
+          auto trx_index = itr.key().transaction_index;
+          const auto& transactions = get_market_transactions( block_num );
+          FC_ASSERT( transactions.size() > trx_index,
+              "Invalid _market_transaction_index: block_num='${block_num}', trx_index='${trx_index}'!",
+              ("block_num", block_num)
+              ("trx_index", trx_index) );
+          auto& trx = transactions.at(trx_index);
+          if( trx.ask_index.order_price.quote_asset_id != quote
+              || trx.ask_index.order_price.base_asset_id != base )
+            continue;
+          const auto& stamp = get_block_header(block_num).timestamp;
+          results.emplace_back(order_history_record(trx, stamp));
+        }
+        if( skip_count >= results.size() )
+          return std::vector<order_history_record>();
+        // copy the old vector to a new vector
+        uint32_t new_end = results.size() - skip_count;
+        uint32_t new_size = new_end;
+        if( new_size > limit ) new_size = limit;
+        std::vector<order_history_record> new_results( new_size );
+        for( uint32_t i = 0; i < new_size; i++ )
+        {
+          new_results[i] = results[new_end - i - 1];
+        }
+        return new_results;
+      }
+   }
+
+   vector<order_history_record> chain_database::market_order_history_old(asset_id_type quote,
+                                                                     asset_id_type base,
+                                                                     uint32_t skip_count,
+                                                                     uint32_t limit,
+                                                                     const address& owner)
    {
       FC_ASSERT(limit <= 10000, "Limit must be at most 10000!");
 

libraries/blockchain/include/bts/blockchain/chain_database.hpp

@@ -260,6 +260,11 @@ namespace bts { namespace blockchain {
                                                                   uint32_t skip_count,
                                                                   uint32_t limit,
                                                                   const address& owner );
+         vector<order_history_record>       market_order_history_old(asset_id_type quote,
+                                                                  asset_id_type base,
+                                                                  uint32_t skip_count,
+                                                                  uint32_t limit,
+                                                                  const address& owner );
 
          void                               generate_snapshot( const fc::path& filename )const;
          void                               generate_issuance_map( const string& symbol, const fc::path& filename )const;

libraries/blockchain/include/bts/blockchain/chain_database_impl.hpp

@@ -160,6 +160,9 @@ namespace bts { namespace blockchain {
             set<expiration_index>                                                       _collateral_expiration_index;
 
             bts::db::cached_level_map<uint32_t, vector<market_transaction>>             _market_transactions_db;
+            bts::db::cached_level_map<market_transaction_index, int32_t>                _market_transactions_index; //fc::int32_t is unused, this is a set
+            bts::db::level_map<market_transaction_owner_index, int32_t>                 _market_transactions_owner_index; //fc::int32_t is unused, this is a set
+
             bts::db::cached_level_map<market_history_key, market_history_record>        _market_history_db;
 
             bts::db::level_map<slot_index, slot_record>                                 _slot_index_to_record;

libraries/blockchain/include/bts/blockchain/config.hpp

@@ -7,7 +7,7 @@
 
 #define BTS_TEST_NETWORK_VERSION                            84 // autogenerated
 
-#define BTS_BLOCKCHAIN_DATABASE_VERSION                     uint64_t( 210 )
+#define BTS_BLOCKCHAIN_DATABASE_VERSION                     uint64_t( 211 )
 
 #define BTS_ADDRESS_PREFIX                                  "XTS"
 #define BTS_BLOCKCHAIN_SYMBOL                               "XTS"

libraries/blockchain/include/bts/blockchain/market_records.hpp

@@ -216,6 +216,45 @@ namespace bts { namespace blockchain {
    };
    typedef optional<market_order> omarket_order;
 
+   struct market_transaction_index
+   {
+      asset_id_type      base_id;
+      asset_id_type      quote_id;
+      uint32_t           block_num;
+      uint32_t           transaction_index; // index in the transactions vector of one block
+
+      friend bool operator < ( const market_transaction_index& a, const market_transaction_index& b )
+      {
+         return std::tie( a.base_id, a.quote_id, a.block_num, a.transaction_index )
+              < std::tie( b.base_id, b.quote_id, b.block_num, b.transaction_index );
+      }
+      friend bool operator == ( const market_transaction_index& a, const market_transaction_index& b )
+      {
+         return std::tie( a.base_id, a.quote_id, a.block_num, a.transaction_index )
+             == std::tie( b.base_id, b.quote_id, b.block_num, b.transaction_index );
+      }
+
+   };
+
+   struct market_transaction_owner_index
+   {
+      address            owner;
+      uint32_t           block_num;
+      uint32_t           transaction_index; // index in the transactions vector of one block
+
+      friend bool operator < ( const market_transaction_owner_index& a, const market_transaction_owner_index& b )
+      {
+         return std::tie( a.owner, a.block_num, a.transaction_index )
+              < std::tie( b.owner, b.block_num, b.transaction_index );
+      }
+      friend bool operator == ( const market_transaction_owner_index& a, const market_transaction_owner_index& b )
+      {
+         return std::tie( a.owner, a.block_num, a.transaction_index )
+             == std::tie( b.owner, b.block_num, b.transaction_index );
+      }
+
+   };
+
    struct order_history_record : public market_transaction
    {
       order_history_record( const market_transaction& market_trans = market_transaction(),
@@ -278,3 +317,5 @@ FC_REFLECT( bts::blockchain::market_transaction,
             (base_fees)
           )
 FC_REFLECT_DERIVED( bts::blockchain::order_history_record, (bts::blockchain::market_transaction), (timestamp) )
+FC_REFLECT( bts::blockchain::market_transaction_index, (base_id)(quote_id)(block_num)(transaction_index) )
+FC_REFLECT( bts::blockchain::market_transaction_owner_index, (owner)(block_num)(transaction_index) )