core: cache tx and block hashes in the respective classes

An idea from smooth

src/cryptonote_basic/cryptonote_basic.h

@@ -190,11 +190,24 @@ namespace cryptonote
     std::vector<std::vector<crypto::signature> > signatures; //count signatures  always the same as inputs count
     rct::rctSig rct_signatures;
 
+    // hash cash
+    mutable crypto::hash hash;
+    mutable size_t blob_size;
+    mutable bool hash_valid;
+    mutable bool blob_size_valid;
+
     transaction();
     virtual ~transaction();
     void set_null();
+    void invalidate_hashes();
 
     BEGIN_SERIALIZE_OBJECT()
+      if (!typename Archive<W>::is_saving())
+      {
+        hash_valid = false;
+        blob_size_valid = false;
+      }
+
       FIELDS(*static_cast<transaction_prefix *>(this))
 
       if (version == 1)
@@ -299,6 +312,15 @@ namespace cryptonote
     extra.clear();
     signatures.clear();
     rct_signatures.type = rct::RCTTypeNull;
+    hash_valid = false;
+    blob_size_valid = false;
+  }
+
+  inline
+  void transaction::invalidate_hashes()
+  {
+    hash_valid = false;
+    blob_size_valid = false;
   }
 
   inline
@@ -339,10 +361,20 @@ namespace cryptonote
 
   struct block: public block_header
   {
+    block(): block_header(), hash_valid(false) {}
+    void invalidate_hashes() { hash_valid = false; }
+
     transaction miner_tx;
     std::vector<crypto::hash> tx_hashes;
 
+    // hash cash
+    mutable crypto::hash hash;
+    mutable bool hash_valid;
+
     BEGIN_SERIALIZE_OBJECT()
+      if (!typename Archive<W>::is_saving())
+        hash_valid = false;
+
       FIELDS(*static_cast<block_header *>(this))
       FIELD(miner_tx)
       FIELD(tx_hashes)

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -43,6 +43,8 @@ using namespace epee;
 
 #define ENCRYPTED_PAYMENT_ID_TAIL 0x8d
 
+// #define ENABLE_HASH_CASH_INTEGRITY_CHECK
+
 static const uint64_t valid_decomposed_outputs[] = {
   (uint64_t)1, (uint64_t)2, (uint64_t)3, (uint64_t)4, (uint64_t)5, (uint64_t)6, (uint64_t)7, (uint64_t)8, (uint64_t)9, // 1 piconero
   (uint64_t)10, (uint64_t)20, (uint64_t)30, (uint64_t)40, (uint64_t)50, (uint64_t)60, (uint64_t)70, (uint64_t)80, (uint64_t)90,
@@ -68,6 +70,11 @@ static const uint64_t valid_decomposed_outputs[] = {
 
 static std::atomic<unsigned int> default_decimal_point(CRYPTONOTE_DISPLAY_DECIMAL_POINT);
 
+static std::atomic<uint64_t> tx_hashes_calculated_count(0);
+static std::atomic<uint64_t> tx_hashes_cached_count(0);
+static std::atomic<uint64_t> block_hashes_calculated_count(0);
+static std::atomic<uint64_t> block_hashes_cached_count(0);
+
 namespace cryptonote
 {
   //---------------------------------------------------------------
@@ -93,6 +100,8 @@ namespace cryptonote
     binary_archive<false> ba(ss);
     bool r = ::serialization::serialize(ba, tx);
     CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob");
+    tx.hash_valid = false;
+    tx.blob_size_valid = false;
     return true;
   }
   //---------------------------------------------------------------
@@ -113,6 +122,8 @@ namespace cryptonote
     binary_archive<false> ba(ss);
     bool r = ::serialization::serialize(ba, tx);
     CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob");
+    tx.hash_valid = false;
+    tx.blob_size_valid = false;
     //TODO: validate tx
 
     get_transaction_hash(tx, tx_hash);
@@ -592,7 +603,7 @@ namespace cryptonote
     return get_transaction_hash(t, res, NULL);
   }
   //---------------------------------------------------------------
-  bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size)
+  bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size)
   {
     // v1 transactions hash the entire blob
     if (t.version == 1)
@@ -647,6 +658,40 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------
+  bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size)
+  {
+    if (t.hash_valid)
+    {
+#ifdef ENABLE_HASH_CASH_INTEGRITY_CHECK
+      CHECK_AND_ASSERT_THROW_MES(!calculate_transaction_hash(t, res, blob_size) || t.hash == res, "tx hash cash integrity failure");
+#endif
+      res = t.hash;
+      if (blob_size)
+      {
+        if (!t.blob_size_valid)
+        {
+          t.blob_size = get_object_blobsize(t);
+          t.blob_size_valid = true;
+        }
+        *blob_size = t.blob_size;
+      }
+      ++tx_hashes_cached_count;
+      return true;
+    }
+    ++tx_hashes_calculated_count;
+    bool ret = calculate_transaction_hash(t, res, blob_size);
+    if (!ret)
+      return false;
+    t.hash = res;
+    t.hash_valid = true;
+    if (blob_size)
+    {
+      t.blob_size = *blob_size;
+      t.blob_size_valid = true;
+    }
+    return true;
+  }
+  //---------------------------------------------------------------
   bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size)
   {
     return get_transaction_hash(t, res, &blob_size);
@@ -661,7 +706,7 @@ namespace cryptonote
     return blob;
   }
   //---------------------------------------------------------------
-  bool get_block_hash(const block& b, crypto::hash& res)
+  bool calculate_block_hash(const block& b, crypto::hash& res)
   {
     // EXCEPTION FOR BLOCK 202612
     const std::string correct_blob_hash_202612 = "3a8a2b3a29b50fc86ff73dd087ea43c6f0d6b8f936c849194d5c84c737903966";
@@ -688,6 +733,26 @@ namespace cryptonote
     return hash_result;
   }
   //---------------------------------------------------------------
+  bool get_block_hash(const block& b, crypto::hash& res)
+  {
+    if (b.hash_valid)
+    {
+#ifdef ENABLE_HASH_CASH_INTEGRITY_CHECK
+      CHECK_AND_ASSERT_THROW_MES(!calculate_block_hash(b, res) || b.hash == res, "block hash cash integrity failure");
+#endif
+      res = b.hash;
+      ++block_hashes_cached_count;
+      return true;
+    }
+    ++block_hashes_calculated_count;
+    bool ret = calculate_block_hash(b, res);
+    if (!ret)
+      return false;
+    b.hash = res;
+    b.hash_valid = true;
+    return true;
+  }
+  //---------------------------------------------------------------
   crypto::hash get_block_hash(const block& b)
   {
     crypto::hash p = null_hash;
@@ -744,6 +809,9 @@ namespace cryptonote
     binary_archive<false> ba(ss);
     bool r = ::serialization::serialize(ba, b);
     CHECK_AND_ASSERT_MES(r, false, "Failed to parse block from blob");
+    b.hash_valid = false;
+    b.miner_tx.hash_valid = false;
+    b.miner_tx.blob_size_valid = false;
     return true;
   }
   //---------------------------------------------------------------
@@ -798,4 +866,11 @@ namespace cryptonote
     return std::binary_search(begin, end, amount);
   }
   //---------------------------------------------------------------
+  void get_hash_stats(uint64_t &tx_hashes_calculated, uint64_t &tx_hashes_cached, uint64_t &block_hashes_calculated, uint64_t & block_hashes_cached)
+  {
+    tx_hashes_calculated = tx_hashes_calculated_count;
+    tx_hashes_cached = tx_hashes_cached_count;
+    block_hashes_calculated = block_hashes_calculated_count;
+    block_hashes_cached = block_hashes_cached_count;
+  }
 }

src/cryptonote_basic/cryptonote_format_utils.h

@@ -85,7 +85,9 @@ namespace cryptonote
   bool get_transaction_hash(const transaction& t, crypto::hash& res);
   bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size);
   bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size);
+  bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size);
   blobdata get_block_hashing_blob(const block& b);
+  bool calculate_block_hash(const block& b, crypto::hash& res);
   bool get_block_hash(const block& b, crypto::hash& res);
   crypto::hash get_block_hash(const block& b);
   bool get_block_longhash(const block& b, crypto::hash& res, uint64_t height);
@@ -209,6 +211,7 @@ namespace cryptonote
   crypto::hash get_tx_tree_hash(const std::vector<crypto::hash>& tx_hashes);
   crypto::hash get_tx_tree_hash(const block& b);
   bool is_valid_decomposed_amount(uint64_t amount);
+  void get_hash_stats(uint64_t &tx_hashes_calculated, uint64_t &tx_hashes_cached, uint64_t &block_hashes_calculated, uint64_t & block_hashes_cached);
 
 #define CHECKED_GET_SPECIFIC_VARIANT(variant_var, specific_type, variable_name, fail_return_val) \
   CHECK_AND_ASSERT_MES(variant_var.type() == typeid(specific_type), fail_return_val, "wrong variant type: " << variant_var.type().name() << ", expected " << typeid(specific_type).name()); \

src/cryptonote_basic/miner.cpp

@@ -355,9 +355,11 @@ namespace cryptonote
 
       if(check_hash(h, diffic))
       {
+        bl.hash_valid = false;
         return true;
       }
     }
+    bl.hash_valid = false;
     return false;
   }
   //-----------------------------------------------------------------------------------------------------

src/cryptonote_core/cryptonote_tx_utils.cpp

@@ -132,6 +132,9 @@ namespace cryptonote
     //lock
     tx.unlock_time = height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW;
     tx.vin.push_back(in);
+
+    tx.hash_valid = tx.blob_size_valid = false;
+
     //LOG_PRINT("MINER_TX generated ok, block_reward=" << print_money(block_reward) << "("  << print_money(block_reward - fee) << "+" << print_money(fee)
     //  << "), current_block_size=" << current_block_size << ", already_generated_coins=" << already_generated_coins << ", tx_id=" << get_transaction_hash(tx), LOG_LEVEL_2);
     return true;
@@ -451,6 +454,8 @@ namespace cryptonote
       MCINFO("construct_tx", "transaction_created: " << get_transaction_hash(tx) << ENDL << obj_to_json_str(tx) << ENDL);
     }
 
+    tx.hash_valid = tx.blob_size_valid = false;
+
     return true;
   }
   //---------------------------------------------------------------
@@ -487,6 +492,7 @@ namespace cryptonote
     bl.timestamp = 0;
     bl.nonce = nonce;
     miner::find_nonce_for_given_block(bl, 1, 0);
+    bl.hash_valid = false;
     return true;
   }
   //---------------------------------------------------------------

tests/unit_tests/serialization.cpp

@@ -331,6 +331,7 @@ TEST(Serialization, serializes_transacion_signatures_correctly)
 
   // Miner tx with empty signatures 2nd vector
   tx.signatures.resize(1);
+  tx.invalidate_hashes();
   ASSERT_TRUE(serialization::dump_binary(tx, blob));
   ASSERT_EQ(7, blob.size()); // 5 bytes + 2 bytes vin[0] + 0 bytes extra + 0 bytes signatures
   ASSERT_TRUE(serialization::parse_binary(blob, tx1));
@@ -345,16 +346,19 @@ TEST(Serialization, serializes_transacion_signatures_correctly)
   tx.signatures.resize(2);
   tx.signatures[0].resize(0);
   tx.signatures[1].resize(0);
+  tx.invalidate_hashes();
   ASSERT_FALSE(serialization::dump_binary(tx, blob));
 
   // Miner tx with 2 signatures
   tx.signatures[0].resize(1);
   tx.signatures[1].resize(1);
+  tx.invalidate_hashes();
   ASSERT_FALSE(serialization::dump_binary(tx, blob));
 
   // Two txin_gen, no signatures
   tx.vin.push_back(txin_gen1);
   tx.signatures.resize(0);
+  tx.invalidate_hashes();
   ASSERT_TRUE(serialization::dump_binary(tx, blob));
   ASSERT_EQ(9, blob.size()); // 5 bytes + 2 * 2 bytes vins + 0 bytes extra + 0 bytes signatures
   ASSERT_TRUE(serialization::parse_binary(blob, tx1));
@@ -363,10 +367,12 @@ TEST(Serialization, serializes_transacion_signatures_correctly)
 
   // Two txin_gen, signatures vector contains only one empty element
   tx.signatures.resize(1);
+  tx.invalidate_hashes();
   ASSERT_FALSE(serialization::dump_binary(tx, blob));
 
   // Two txin_gen, signatures vector contains two empty elements
   tx.signatures.resize(2);
+  tx.invalidate_hashes();
   ASSERT_TRUE(serialization::dump_binary(tx, blob));
   ASSERT_EQ(9, blob.size()); // 5 bytes + 2 * 2 bytes vins + 0 bytes extra + 0 bytes signatures
   ASSERT_TRUE(serialization::parse_binary(blob, tx1));
@@ -375,18 +381,21 @@ TEST(Serialization, serializes_transacion_signatures_correctly)
 
   // Two txin_gen, signatures vector contains three empty elements
   tx.signatures.resize(3);
+  tx.invalidate_hashes();
   ASSERT_FALSE(serialization::dump_binary(tx, blob));
 
   // Two txin_gen, signatures vector contains two non empty elements
   tx.signatures.resize(2);
   tx.signatures[0].resize(1);
   tx.signatures[1].resize(1);
+  tx.invalidate_hashes();
   ASSERT_FALSE(serialization::dump_binary(tx, blob));
 
   // A few bytes instead of signature
   tx.vin.clear();
   tx.vin.push_back(txin_gen1);
   tx.signatures.clear();
+  tx.invalidate_hashes();
   ASSERT_TRUE(serialization::dump_binary(tx, blob));
   blob.append(std::string(sizeof(crypto::signature) / 2, 'x'));
   ASSERT_FALSE(serialization::parse_binary(blob, tx1));
@@ -406,6 +415,7 @@ TEST(Serialization, serializes_transacion_signatures_correctly)
   tx.vin.push_back(txin_to_key1);
   tx.signatures.resize(1);
   tx.signatures[0].resize(2);
+  tx.invalidate_hashes();
   ASSERT_FALSE(serialization::dump_binary(tx, blob));
 
   // Too much signatures for two inputs
@@ -413,24 +423,28 @@ TEST(Serialization, serializes_transacion_signatures_correctly)
   tx.signatures[0].resize(2);
   tx.signatures[1].resize(2);
   tx.signatures[2].resize(2);
+  tx.invalidate_hashes();
   ASSERT_FALSE(serialization::dump_binary(tx, blob));
 
   // First signatures vector contains too little elements
   tx.signatures.resize(2);
   tx.signatures[0].resize(1);
   tx.signatures[1].resize(2);
+  tx.invalidate_hashes();
   ASSERT_FALSE(serialization::dump_binary(tx, blob));
 
   // First signatures vector contains too much elements
   tx.signatures.resize(2);
   tx.signatures[0].resize(3);
   tx.signatures[1].resize(2);
+  tx.invalidate_hashes();
   ASSERT_FALSE(serialization::dump_binary(tx, blob));
 
   // There are signatures for each input
   tx.signatures.resize(2);
   tx.signatures[0].resize(2);
   tx.signatures[1].resize(2);
+  tx.invalidate_hashes();
   ASSERT_TRUE(serialization::dump_binary(tx, blob));
   ASSERT_TRUE(serialization::parse_binary(blob, tx1));
   ASSERT_EQ(tx, tx1);