Merge #32

Add support to BlockchainDB and BlockchainLMDB for batch transactions.

Add profiling to block and tx processing and DB operations.

Improve block and tx processing efficiency by less repeat hashing.

Move LMDB storage to "lmdb" subfolder.
 - Upon startup, if old LMDB files are detected, abort with a message for the user to move them to subfolder or delete them.

Update and fix log statements and formatting.

src/blockchain_converter/blockchain_converter.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
 //
 // All rights reserved.
 //

src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp

@@ -33,6 +33,7 @@
 
 #include "cryptonote_core/cryptonote_format_utils.h"
 #include "crypto/crypto.h"
+#include "profile_tools.h"
 
 using epee::string_tools::pod_to_hex;
 
@@ -152,12 +153,13 @@ void BlockchainLMDB::add_block( const block& blk
               , const size_t& block_size
               , const difficulty_type& cumulative_difficulty
               , const uint64_t& coins_generated
+              , const crypto::hash& blk_hash
               )
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  MDB_val_copy<crypto::hash> val_h(get_block_hash(blk));
+  MDB_val_copy<crypto::hash> val_h(blk_hash);
   MDB_val unused;
   if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0)
     throw1(BLOCK_EXISTS("Attempting to add block that's already in the db"));
@@ -167,8 +169,11 @@ void BlockchainLMDB::add_block( const block& blk
     MDB_val_copy<crypto::hash> parent_key(blk.prev_id);
     MDB_val parent_h;
     if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h))
+    {
+      LOG_PRINT_L3("m_height: " << m_height);
+      LOG_PRINT_L3("parent_key: " << blk.prev_id);
       throw0(DB_ERROR("Failed to get top block hash to check for new block's parent"));
-
+    }
     uint64_t parent_height = *(const uint64_t *)parent_h.mv_data;
     if (parent_height != m_height - 1)
       throw0(BLOCK_PARENT_DNE("Top block is not new block's parent"));
@@ -177,8 +182,9 @@ void BlockchainLMDB::add_block( const block& blk
   MDB_val_copy<uint64_t> key(m_height);
 
   MDB_val_copy<blobdata> blob(block_to_blob(blk));
-  if (mdb_put(*m_write_txn, m_blocks, &key, &blob, 0))
+  auto res = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0);
-    throw0(DB_ERROR("Failed to add block blob to db transaction"));
+  if (res)
+    throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(res)).c_str()));
 
   MDB_val_copy<size_t> sz(block_size);
   if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0))
@@ -239,12 +245,12 @@ void BlockchainLMDB::remove_block()
       throw1(DB_ERROR("Failed to add removal of block hash to db transaction"));
 }
 
-void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx)
+void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  MDB_val_copy<crypto::hash> val_h(get_transaction_hash(tx));
+  MDB_val_copy<crypto::hash> val_h(tx_hash);
   MDB_val unused;
   if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0)
       throw1(TX_EXISTS("Attempting to add transaction that's already in the db"));
@@ -382,7 +388,6 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo
   check_open();
 
   MDB_val_copy<uint64_t> k(out_index);
-  MDB_val v;
 
 /****** Uncomment if ever outputs actually need to be stored in this manner
   blobdata b;
@@ -505,8 +510,8 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image)
   char anything = '\0';
   unused.mv_size = sizeof(char);
   unused.mv_data = &anything;
-  if (mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0))
+  if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0))
-      throw1(DB_ERROR("Error adding spent key image to db transaction"));
+    throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str()));
 }
 
 void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image)
@@ -596,15 +601,23 @@ void BlockchainLMDB::check_open() const
 BlockchainLMDB::~BlockchainLMDB()
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+
+  // batch transaction shouldn't be active at this point. If it is, consider it aborted.
+  if (m_batch_active)
+    batch_abort();
 }
 
-BlockchainLMDB::BlockchainLMDB()
+BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   // initialize folder to something "safe" just in case
   // someone accidentally misuses this class...
   m_folder = "thishsouldnotexistbecauseitisgibberish";
   m_open = false;
+
+  m_batch_transactions = batch_transactions;
+  m_write_txn = nullptr;
+  m_batch_active = false;
   m_height = 0;
 }
 
@@ -627,6 +640,16 @@ void BlockchainLMDB::open(const std::string& filename)
       throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str()));
   }
 
+  // check for existing LMDB files in base directory
+  boost::filesystem::path old_files = direc.parent_path();
+  if (boost::filesystem::exists(old_files / "data.mdb") ||
+      boost::filesystem::exists(old_files / "lock.mdb"))
+  {
+    LOG_PRINT_L0("Found existing LMDB files in " << old_files.c_str());
+    LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart");
+    throw DB_ERROR("Database could not be opened");
+  }
+
   m_folder = filename;
 
   // set up lmdb environment
@@ -672,7 +695,7 @@ void BlockchainLMDB::open(const std::string& filename)
   lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices");
 *************************************************/
 
-  lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_outputs");
+  lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys");
 
   mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
   mdb_set_dupsort(txn, m_tx_outputs, compare_uint64);
@@ -706,6 +729,13 @@ void BlockchainLMDB::create(const std::string& filename)
 void BlockchainLMDB::close()
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  if (m_batch_active)
+  {
+    LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction");
+    batch_abort();
+  }
+  this->sync();
+
   // FIXME: not yet thread safe!!!  Use with care.
   mdb_env_close(m_env);
 }
@@ -713,7 +743,14 @@ void BlockchainLMDB::close()
 void BlockchainLMDB::sync()
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
-  // LMDB documentation leads me to believe this is unnecessary
+  check_open();
+
+  // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part
+  // MDB_NOMETASYNC. Force flush to be synchronous.
+  if (auto result = mdb_env_sync(m_env, true))
+  {
+    throw0(DB_ERROR(std::string("Failed to sync database").append(mdb_strerror(result)).c_str()));
+  }
 }
 
 void BlockchainLMDB::reset()
@@ -753,7 +790,6 @@ void BlockchainLMDB::unlock()
   check_open();
 }
 
-
 bool BlockchainLMDB::block_exists(const crypto::hash& h) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -854,12 +890,17 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const
   check_open();
 
   txn_safe txn;
+  txn_safe* txn_ptr = &txn;
+  if (m_batch_active)
+    txn_ptr = m_write_txn;
+  else
+  {
     if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
-
+  }
   MDB_val_copy<uint64_t> key(height);
   MDB_val result;
-  auto get_result = mdb_get(txn, m_block_timestamps, &key, &result);
+  auto get_result = mdb_get(*txn_ptr, m_block_timestamps, &key, &result);
   if (get_result == MDB_NOTFOUND)
   {
     throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str()));
@@ -867,6 +908,7 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const
   else if (get_result)
     throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db"));
 
+  if (! m_batch_active)
     txn.commit();
   return *(const uint64_t*)result.mv_data;
 }
@@ -891,12 +933,18 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
   check_open();
 
   txn_safe txn;
+  txn_safe* txn_ptr = &txn;
+  if (m_batch_active)
+    txn_ptr = m_write_txn;
+  else
+  {
     if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
+  }
 
   MDB_val_copy<uint64_t> key(height);
   MDB_val result;
-  auto get_result = mdb_get(txn, m_block_sizes, &key, &result);
+  auto get_result = mdb_get(*txn_ptr, m_block_sizes, &key, &result);
   if (get_result == MDB_NOTFOUND)
   {
     throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str()));
@@ -904,22 +952,28 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
   else if (get_result)
     throw0(DB_ERROR("Error attempting to retrieve a block size from the db"));
 
+  if (! m_batch_active)
     txn.commit();
   return *(const size_t*)result.mv_data;
 }
 
 difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const
 {
-  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__ << "  height: " << height);
   check_open();
 
   txn_safe txn;
+  txn_safe* txn_ptr = &txn;
+  if (m_batch_active)
+    txn_ptr = m_write_txn;
+  else
+  {
     if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
-
+  }
   MDB_val_copy<uint64_t> key(height);
   MDB_val result;
-  auto get_result = mdb_get(txn, m_block_diffs, &key, &result);
+  auto get_result = mdb_get(*txn_ptr, m_block_diffs, &key, &result);
   if (get_result == MDB_NOTFOUND)
   {
     throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str()));
@@ -927,6 +981,7 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t&
   else if (get_result)
     throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db"));
 
+  if (! m_batch_active)
     txn.commit();
   return *(difficulty_type*)result.mv_data;
 }
@@ -954,12 +1009,18 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh
   check_open();
 
   txn_safe txn;
+  txn_safe* txn_ptr = &txn;
+  if (m_batch_active)
+    txn_ptr = m_write_txn;
+  else
+  {
     if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
+  }
 
   MDB_val_copy<uint64_t> key(height);
   MDB_val result;
-  auto get_result = mdb_get(txn, m_block_coins, &key, &result);
+  auto get_result = mdb_get(*txn_ptr, m_block_coins, &key, &result);
   if (get_result == MDB_NOTFOUND)
   {
     throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str()));
@@ -967,6 +1028,7 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh
   else if (get_result)
     throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db"));
 
+  if (! m_batch_active)
     txn.commit();
   return *(const uint64_t*)result.mv_data;
 }
@@ -977,19 +1039,27 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height)
   check_open();
 
   txn_safe txn;
+  txn_safe* txn_ptr = &txn;
+  if (m_batch_active)
+    txn_ptr = m_write_txn;
+  else
+  {
     if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
+  }
 
   MDB_val_copy<uint64_t> key(height);
   MDB_val result;
-  auto get_result = mdb_get(txn, m_block_hashes, &key, &result);
+  auto get_result = mdb_get(*txn_ptr, m_block_hashes, &key, &result);
   if (get_result == MDB_NOTFOUND)
   {
     throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str()));
   }
   else if (get_result)
-    throw0(DB_ERROR("Error attempting to retrieve a block hash from the db"));
+    throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: ").
+          append(mdb_strerror(get_result)).c_str()));
 
+  if (! m_batch_active)
     txn.commit();
   return *(crypto::hash*)result.mv_data;
 }
@@ -1056,21 +1126,31 @@ uint64_t BlockchainLMDB::height() const
   return m_height;
 }
 
-
 bool BlockchainLMDB::tx_exists(const crypto::hash& h) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
   txn_safe txn;
+  txn_safe* txn_ptr = &txn;
+  if (m_batch_active)
+    txn_ptr = m_write_txn;
+  else
+  {
     if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
+  }
 
   MDB_val_copy<crypto::hash> key(h);
   MDB_val result;
-  auto get_result = mdb_get(txn, m_txs, &key, &result);
+
+  TIME_MEASURE_START(time1);
+  auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result);
+  TIME_MEASURE_FINISH(time1);
+  time_tx_exists += time1;
   if (get_result == MDB_NOTFOUND)
   {
+    if (! m_batch_active)
       txn.commit();
     LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
     return false;
@@ -1107,12 +1187,18 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const
   check_open();
 
   txn_safe txn;
+  txn_safe* txn_ptr = &txn;
+  if (m_batch_active)
+    txn_ptr = m_write_txn;
+  else
+  {
     if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
+  }
 
   MDB_val_copy<crypto::hash> key(h);
   MDB_val result;
-  auto get_result = mdb_get(txn, m_txs, &key, &result);
+  auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result);
   if (get_result == MDB_NOTFOUND)
     throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
   else if (get_result)
@@ -1124,6 +1210,8 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const
   transaction tx;
   if (!parse_and_validate_tx_from_blob(bd, tx))
     throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
+  if (! m_batch_active)
+    txn.commit();
 
   return tx;
 }
@@ -1165,13 +1253,27 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
+  // If m_batch_active is set, a batch transaction exists beyond this class,
+  // such as a batch import with verification enabled, or possibly (later) a
+  // batch network sync.
+  //
+  // A regular network sync without batching would be expected to open a new
+  // read transaction here, as validation is done prior to the write for block
+  // and tx data.
+
   txn_safe txn;
+  txn_safe* txn_ptr = &txn;
+  if (m_batch_active)
+    txn_ptr = m_write_txn;
+  else
+  {
     if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
+  }
 
   MDB_val_copy<crypto::hash> key(h);
   MDB_val result;
-  auto get_result = mdb_get(txn, m_tx_heights, &key, &result);
+  auto get_result = mdb_get(*txn_ptr, m_tx_heights, &key, &result);
   if (get_result == MDB_NOTFOUND)
   {
     throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
@@ -1179,6 +1281,9 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
   else if (get_result)
     throw0(DB_ERROR("DB error attempting to fetch tx height from hash"));
 
+  if (! m_batch_active)
+    txn.commit();
+
   return *(const uint64_t*)result.mv_data;
 }
 
@@ -1321,13 +1426,18 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t&
   check_open();
 
   txn_safe txn;
+  txn_safe* txn_ptr = &txn;
+  if (m_batch_active)
+    txn_ptr = m_write_txn;
+  else
+  {
     if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
-
+  }
   MDB_val_copy<uint64_t> k(index);
   MDB_val v;
 
-  auto get_result = mdb_get(txn, m_output_txs, &k, &v);
+  auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v);
   if (get_result == MDB_NOTFOUND)
     throw1(OUTPUT_DNE("output with given index not in db"));
   else if (get_result)
@@ -1335,11 +1445,13 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t&
 
   crypto::hash tx_hash = *(crypto::hash*)v.mv_data;
 
-  get_result = mdb_get(txn, m_output_indices, &k, &v);
+  get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v);
   if (get_result == MDB_NOTFOUND)
     throw1(OUTPUT_DNE("output with given index not in db"));
   else if (get_result)
     throw0(DB_ERROR("DB error attempting to fetch output tx index"));
+  if (! m_batch_active)
+    txn.commit();
 
   return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data);
 }
@@ -1350,10 +1462,15 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con
   check_open();
 
   txn_safe txn;
+  txn_safe* txn_ptr = &txn;
+  if (m_batch_active)
+    txn_ptr = m_write_txn;
+  else
+  {
     if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
-
+  }
-  lmdb_cur cur(txn, m_output_amounts);
+  lmdb_cur cur(*txn_ptr, m_output_amounts);
 
   MDB_val_copy<uint64_t> k(amount);
   MDB_val v;
@@ -1382,6 +1499,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con
 
   cur.close();
 
+  if (! m_batch_active)
     txn.commit();
 
   return get_output_tx_and_index_from_global(glob_index);
@@ -1525,6 +1643,91 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const
   return false;
 }
 
+void BlockchainLMDB::batch_start()
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  if (! m_batch_transactions)
+    throw0(DB_ERROR("batch transactions not enabled"));
+  if (m_batch_active)
+    throw0(DB_ERROR("batch transaction already in progress"));
+  if (m_write_txn)
+    throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use"));
+  check_open();
+  // NOTE: need to make sure it's destroyed properly when done
+  if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn))
+    throw0(DB_ERROR("Failed to create a transaction for the db"));
+  // indicates this transaction is for batch transactions, but not whether it's
+  // active
+  m_write_batch_txn.m_batch_txn = true;
+  m_write_txn = &m_write_batch_txn;
+  m_batch_active = true;
+  LOG_PRINT_L3("batch transaction: begin");
+}
+
+void BlockchainLMDB::batch_commit()
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  if (! m_batch_transactions)
+    throw0(DB_ERROR("batch transactions not enabled"));
+  if (! m_batch_active)
+    throw0(DB_ERROR("batch transaction not in progress"));
+  check_open();
+  LOG_PRINT_L3("batch transaction: committing...");
+  TIME_MEASURE_START(time1);
+  m_write_txn->commit();
+  TIME_MEASURE_FINISH(time1);
+  time_commit1 += time1;
+  LOG_PRINT_L3("batch transaction: committed");
+
+  if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn))
+    throw0(DB_ERROR("Failed to create a transaction for the db"));
+  if (! m_write_batch_txn.m_batch_txn)
+    throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction"));
+  m_write_txn = &m_write_batch_txn;
+}
+
+void BlockchainLMDB::batch_stop()
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  if (! m_batch_transactions)
+    throw0(DB_ERROR("batch transactions not enabled"));
+  if (! m_batch_active)
+    throw0(DB_ERROR("batch transaction not in progress"));
+  check_open();
+  LOG_PRINT_L3("batch transaction: committing...");
+  TIME_MEASURE_START(time1);
+  m_write_txn->commit();
+  TIME_MEASURE_FINISH(time1);
+  time_commit1 += time1;
+  // for destruction of batch transaction
+  m_write_txn = nullptr;
+  m_batch_active = false;
+  LOG_PRINT_L3("batch transaction: end");
+}
+
+void BlockchainLMDB::batch_abort()
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  if (! m_batch_transactions)
+    throw0(DB_ERROR("batch transactions not enabled"));
+  if (! m_batch_active)
+    throw0(DB_ERROR("batch transaction not in progress"));
+  check_open();
+  // for destruction of batch transaction
+  m_write_txn = nullptr;
+  // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called.
+  m_write_batch_txn.abort();
+  m_batch_active = false;
+  LOG_PRINT_L3("batch transaction: aborted");
+}
+
+void BlockchainLMDB::set_batch_transactions(bool batch_transactions)
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  m_batch_transactions = batch_transactions;
+  LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled"));
+}
+
 uint64_t BlockchainLMDB::add_block( const block& blk
                                   , const size_t& block_size
                                   , const difficulty_type& cumulative_difficulty
@@ -1534,22 +1737,33 @@ uint64_t BlockchainLMDB::add_block( const block& blk
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
+
   txn_safe txn;
+  if (! m_batch_active)
+  {
     if (mdb_txn_begin(m_env, NULL, 0, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
     m_write_txn = &txn;
+  }
 
   uint64_t num_outputs = m_num_outputs;
   try
   {
     BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
+    if (! m_batch_active)
+    {
       m_write_txn = NULL;
 
+      TIME_MEASURE_START(time1);
       txn.commit();
+      TIME_MEASURE_FINISH(time1);
+      time_commit1 += time1;
+    }
   }
   catch (...)
   {
     m_num_outputs = num_outputs;
+    if (! m_batch_active)
       m_write_txn = NULL;
     throw;
   }
@@ -1559,19 +1773,28 @@ uint64_t BlockchainLMDB::add_block( const block& blk
 
 void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
 {
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
   txn_safe txn;
+  if (! m_batch_active)
+  {
     if (mdb_txn_begin(m_env, NULL, 0, txn))
       throw0(DB_ERROR("Failed to create a transaction for the db"));
     m_write_txn = &txn;
+  }
 
   uint64_t num_outputs = m_num_outputs;
   try
   {
     BlockchainDB::pop_block(blk, txs);
+    if (! m_batch_active)
+    {
       m_write_txn = NULL;
 
       txn.commit();
     }
+  }
   catch (...)
   {
     m_num_outputs = num_outputs;

src/cryptonote_core/BlockchainDB_impl/db_lmdb.h

@@ -38,8 +38,23 @@ struct txn_safe
   txn_safe() : m_txn(NULL) { }
   ~txn_safe()
   {
+    LOG_PRINT_L3("txn_safe: destructor");
     if (m_txn != NULL)
     {
+      if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
+      {
+        LOG_PRINT_L0("WARNING: txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
+      }
+      else
+      {
+        // Example of when this occurs: a lookup fails, so a read-only txn is
+        // aborted through this destructor. However, successful read-only txns
+        // ideally should have been committed when done and not end up here.
+        //
+        // NOTE: not sure if this is ever reached for a non-batch write
+        // transaction, but it's probably not ideal if it did.
+        LOG_PRINT_L3("txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
+      }
       mdb_txn_abort(m_txn);
     }
   }
@@ -60,6 +75,24 @@ struct txn_safe
     m_txn = NULL;
   }
 
+  // This should only be needed for batch transaction which must be ensured to
+  // be aborted before mdb_env_close, not after. So we can't rely on
+  // BlockchainLMDB destructor to call txn_safe destructor, as that's too late
+  // to properly abort, since mdb_env_close would have been called earlier.
+  void abort()
+  {
+    LOG_PRINT_L3("txn_safe: abort()");
+    if(m_txn != NULL)
+    {
+      mdb_txn_abort(m_txn);
+      m_txn = NULL;
+    }
+    else
+    {
+      LOG_PRINT_L0("WARNING: txn_safe: abort() called, but m_txn is NULL");
+    }
+  }
+
   operator MDB_txn*()
   {
     return m_txn;
@@ -71,13 +104,14 @@ struct txn_safe
   }
 
   MDB_txn* m_txn;
+  bool m_batch_txn = false;
 };
 
 
 class BlockchainLMDB : public BlockchainDB
 {
 public:
-  BlockchainLMDB();
+  BlockchainLMDB(bool batch_transactions=false);
   ~BlockchainLMDB();
 
   virtual void open(const std::string& filename);
@@ -177,6 +211,12 @@ public:
                             , const std::vector<transaction>& txs
                             );
 
+  virtual void set_batch_transactions(bool batch_transactions);
+  virtual void batch_start();
+  virtual void batch_commit();
+  virtual void batch_stop();
+  virtual void batch_abort();
+
   virtual void pop_block(block& blk, std::vector<transaction>& txs);
 
 private:
@@ -184,11 +224,12 @@ private:
                 , const size_t& block_size
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
+                , const crypto::hash& block_hash
                 );
 
   virtual void remove_block();
 
-  virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx);
+  virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash);
 
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
 
@@ -263,7 +304,11 @@ private:
   uint64_t m_height;
   uint64_t m_num_outputs;
   std::string m_folder;
-  txn_safe* m_write_txn;
+  txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
+  txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
+
+  bool m_batch_transactions; // support for batch transactions
+  bool m_batch_active; // whether batch transaction is in progress
 };
 
 }  // namespace cryptonote

src/cryptonote_core/blockchain.cpp

@@ -237,8 +237,9 @@ bool Blockchain::init(const std::string& config_folder, bool testnet)
   m_testnet = testnet;
 
   boost::filesystem::path folder(m_config_folder);
+  folder /= "lmdb";
 
-  LOG_PRINT_L0("Loading blockchain...");
+  LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ...");
 
   //FIXME: update filename for BlockchainDB
   const std::string filename = folder.string();
@@ -2288,7 +2289,7 @@ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc
   return handle_block_to_main_chain(bl, id, bvc);
 }
 //------------------------------------------------------------------
-void Blockchain::check_against_checkpoints(checkpoints& points, bool enforce)
+void Blockchain::check_against_checkpoints(const checkpoints& points, bool enforce)
 {
   const auto& pts = points.get_points();
 

src/cryptonote_core/blockchain.h

@@ -141,7 +141,7 @@ namespace cryptonote
     void print_blockchain_index();
     void print_blockchain_outs(const std::string& file);
 
-    void check_against_checkpoints(checkpoints& points, bool enforce);
+    void check_against_checkpoints(const checkpoints& points, bool enforce);
     void set_enforce_dns_checkpoints(bool enforce);
     bool update_checkpoints(const std::string& file_path, bool check_dns);
 

src/cryptonote_core/blockchain_db.cpp

@@ -28,6 +28,7 @@
 
 #include "cryptonote_core/blockchain_db.h"
 #include "cryptonote_format_utils.h"
+#include "profile_tools.h"
 
 using epee::string_tools::pod_to_hex;
 
@@ -41,11 +42,21 @@ void BlockchainDB::pop_block()
   pop_block(blk, txs);
 }
 
-void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx)
+void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr)
 {
-  crypto::hash tx_hash = get_transaction_hash(tx);
+  crypto::hash tx_hash;
+  if (!tx_hash_ptr)
+  {
+    // should only need to compute hash for miner transactions
+    tx_hash = get_transaction_hash(tx);
+    LOG_PRINT_L3("null tx_hash_ptr - needed to compute: " << tx_hash);
+  }
+  else
+  {
+    tx_hash = *tx_hash_ptr;
+  }
 
-  add_transaction_data(blk_hash, tx);
+  add_transaction_data(blk_hash, tx, tx_hash);
 
   // iterate tx.vout using indices instead of C++11 foreach syntax because
   // we need the index
@@ -73,18 +84,33 @@ uint64_t BlockchainDB::add_block( const block& blk
                                 , const std::vector<transaction>& txs
                                 )
 {
+  TIME_MEASURE_START(time1);
   crypto::hash blk_hash = get_block_hash(blk);
+  TIME_MEASURE_FINISH(time1);
+  time_blk_hash += time1;
 
   // call out to subclass implementation to add the block & metadata
-  add_block(blk, block_size, cumulative_difficulty, coins_generated);
+  time1 = epee::misc_utils::get_tick_count();
+  add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash);
+  TIME_MEASURE_FINISH(time1);
+  time_add_block1 += time1;
 
   // call out to add the transactions
 
+  time1 = epee::misc_utils::get_tick_count();
   add_transaction(blk_hash, blk.miner_tx);
+  int tx_i = 0;
+  crypto::hash tx_hash = null_hash;
   for (const transaction& tx : txs)
   {
-    add_transaction(blk_hash, tx);
+    tx_hash = blk.tx_hashes[tx_i];
+    add_transaction(blk_hash, tx, &tx_hash);
+    ++tx_i;
   }
+  TIME_MEASURE_FINISH(time1);
+  time_add_transaction += time1;
+
+  ++num_calls;
 
   return height();
 }
@@ -119,4 +145,36 @@ void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
   remove_transaction_data(tx_hash, tx);
 }
 
+void BlockchainDB::reset_stats()
+{
+  num_calls = 0;
+  time_blk_hash = 0;
+  time_tx_exists = 0;
+  time_add_block1 = 0;
+  time_add_transaction = 0;
+  time_commit1 = 0;
+}
+
+void BlockchainDB::show_stats()
+{
+  LOG_PRINT_L1(ENDL
+    << "*********************************"
+    << ENDL
+    << "num_calls: " << num_calls
+    << ENDL
+    << "time_blk_hash: " << time_blk_hash << "ms"
+    << ENDL
+    << "time_tx_exists: " << time_tx_exists << "ms"
+    << ENDL
+    << "time_add_block1: " << time_add_block1 << "ms"
+    << ENDL
+    << "time_add_transaction: " << time_add_transaction << "ms"
+    << ENDL
+    << "time_commit1: " << time_commit1 << "ms"
+    << ENDL
+    << "*********************************"
+    << ENDL
+  );
+}
+
 }  // namespace cryptonote

src/cryptonote_core/blockchain_db.h

@@ -265,13 +265,14 @@ private:
                 , const size_t& block_size
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
+                , const crypto::hash& blk_hash
                 ) = 0;
 
   // tells the subclass to remove data about the top block
   virtual void remove_block() = 0;
 
   // tells the subclass to store the transaction and its metadata
-  virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) = 0;
+  virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0;
 
   // tells the subclass to remove data about a transaction
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0;
@@ -296,17 +297,33 @@ private:
   void pop_block();
 
   // helper function for add_transactions, to add each individual tx
-  void add_transaction(const crypto::hash& blk_hash, const transaction& tx);
+  void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL);
 
   // helper function to remove transaction from blockchain
   void remove_transaction(const crypto::hash& tx_hash);
 
+  uint64_t num_calls = 0;
+  uint64_t time_blk_hash = 0;
+  uint64_t time_add_block1 = 0;
+  uint64_t time_add_transaction = 0;
+
+
+protected:
+
+  mutable uint64_t time_tx_exists = 0;
+  uint64_t time_commit1 = 0;
+
 
 public:
 
   // virtual dtor
   virtual ~BlockchainDB() { };
 
+  // reset profiling stats
+  void reset_stats();
+
+  // show profiling stats
+  void show_stats();
 
   // open the db at location <filename>, or create it if there isn't one.
   virtual void open(const std::string& filename) = 0;
@@ -336,6 +353,9 @@ public:
   // release db lock
   virtual void unlock() = 0;
 
+  virtual void batch_start() = 0;
+  virtual void batch_stop() = 0;
+  virtual void set_batch_transactions(bool) = 0;
 
   // adds a block with the given metadata to the top of the blockchain, returns the new height
   // NOTE: subclass implementations of this (or the functions it calls) need