Merge pull request #4260

a54dbaee blockchain_blackball: add --force-chain-reaction-pass flag (moneromooo-monero)
44439c32 record blackballs as amount/offset, and add export ability (moneromooo-monero)
4bce935b blockchain_blackball: more optimizations (moneromooo-monero)
b66ba783 blockchain_blackball: do not process duplicate blockchains parts (moneromooo-monero)
639a3c01 blockchain_blackball: make it clear secondary passes are not incremental (moneromooo-monero)
eb8a51be blockchain_blackball: detect spent outputs by partial ring reuse (moneromooo-monero)
d6d276c6 blockchain_blackball: fix chain reaction phase in incremental mode (moneromooo-monero)
2b2a681b blockchain_blackball: avoid false positives for different amounts (moneromooo-monero)
80e4fef3 blockchain_blackball: set transaction looping txn to read only (moneromooo-monero)
4801d6b5 blockchain_blackball: add stats (moneromooo-monero)
846190fd blockchain_blackball: support pre-v2 databases (moneromooo-monero)
daa6cc7d blockchain_blackball: use LMDB for the cache (moneromooo-monero)
50cb370d ringdb: allow blackballing many outputs at once (moneromooo-monero)

src/simplewallet/simplewallet.cpp

@@ -1627,23 +1627,23 @@ bool simple_wallet::set_ring(const std::vector<std::string> &args)
 
 bool simple_wallet::blackball(const std::vector<std::string> &args)
 {
-  crypto::public_key output;
+  uint64_t amount = std::numeric_limits<uint64_t>::max(), offset, num_offsets;
   if (args.size() == 0)
   {
-    fail_msg_writer() << tr("usage: blackball <output_public_key> | <filename> [add]");
+    fail_msg_writer() << tr("usage: blackball <amount>/<offset> | <filename> [add]");
     return true;
   }
 
   try
   {
-    if (epee::string_tools::hex_to_pod(args[0], output))
+    if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &amount, &offset) == 2)
     {
-      m_wallet->blackball_output(output);
+      m_wallet->blackball_output(std::make_pair(amount, offset));
     }
     else if (epee::file_io_utils::is_file_exist(args[0]))
     {
-      std::vector<crypto::public_key> outputs;
+      std::vector<std::pair<uint64_t, uint64_t>> outputs;
-      char str[65];
+      char str[256];
 
       std::unique_ptr<FILE, tools::close_file> f(fopen(args[0].c_str(), "r"));
       if (f)
@@ -1657,10 +1657,27 @@ bool simple_wallet::blackball(const std::vector<std::string> &args)
             str[len - 1] = 0;
           if (!str[0])
             continue;
-          outputs.push_back(crypto::public_key());
+          if (sscanf(str, "@%" PRIu64, &amount) == 1)
-          if (!epee::string_tools::hex_to_pod(str, outputs.back()))
           {
-            fail_msg_writer() << tr("Invalid public key: ") << str;
+            continue;
+          }
+          if (amount == std::numeric_limits<uint64_t>::max())
+          {
+            fail_msg_writer() << tr("First line is not an amount");
+            return true;
+          }
+          if (sscanf(str, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets <= std::numeric_limits<uint64_t>::max() - offset)
+          {
+            while (num_offsets--)
+              outputs.push_back(std::make_pair(amount, offset++));
+          }
+          else if (sscanf(str, "%" PRIu64, &offset) == 1)
+          {
+            outputs.push_back(std::make_pair(amount, offset));
+          }
+          else
+          {
+            fail_msg_writer() << tr("Invalid output: ") << str;
             return true;
           }
         }
@@ -1685,7 +1702,7 @@ bool simple_wallet::blackball(const std::vector<std::string> &args)
     }
     else
     {
-      fail_msg_writer() << tr("Invalid public key, and file doesn't exist");
+      fail_msg_writer() << tr("Invalid output key, and file doesn't exist");
       return true;
     }
   }
@@ -1699,16 +1716,16 @@ bool simple_wallet::blackball(const std::vector<std::string> &args)
 
 bool simple_wallet::unblackball(const std::vector<std::string> &args)
 {
-  crypto::public_key output;
+  std::pair<uint64_t, uint64_t> output;
   if (args.size() != 1)
   {
-    fail_msg_writer() << tr("usage: unblackball <output_public_key>");
+    fail_msg_writer() << tr("usage: unblackball <amount>/<offset>");
     return true;
   }
 
-  if (!epee::string_tools::hex_to_pod(args[0], output))
+  if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2)
   {
-    fail_msg_writer() << tr("Invalid public key");
+    fail_msg_writer() << tr("Invalid output");
     return true;
   }
 
@@ -1726,25 +1743,25 @@ bool simple_wallet::unblackball(const std::vector<std::string> &args)
 
 bool simple_wallet::blackballed(const std::vector<std::string> &args)
 {
-  crypto::public_key output;
+  std::pair<uint64_t, uint64_t> output;
   if (args.size() != 1)
   {
-    fail_msg_writer() << tr("usage: blackballed <output_public_key>");
+    fail_msg_writer() << tr("usage: blackballed <amount>/<offset>");
     return true;
   }
 
-  if (!epee::string_tools::hex_to_pod(args[0], output))
+  if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2)
   {
-    fail_msg_writer() << tr("Invalid public key");
+    fail_msg_writer() << tr("Invalid output");
     return true;
   }
 
   try
   {
     if (m_wallet->is_output_blackballed(output))
-      message_writer() << tr("Blackballed: ") << output;
+      message_writer() << tr("Blackballed: ") << output.first << "/" << output.second;
     else
-      message_writer() << tr("not blackballed: ") << output;
+      message_writer() << tr("not blackballed: ") << output.first << "/" << output.second;
   }
   catch (const std::exception &e)
   {
@@ -2559,15 +2576,15 @@ simple_wallet::simple_wallet()
                            tr("Save known rings to the shared rings database"));
   m_cmd_binder.set_handler("blackball",
                            boost::bind(&simple_wallet::blackball, this, _1),
-                           tr("blackball <output public key> | <filename> [add]"),
+                           tr("blackball <amount>/<offset> | <filename> [add]"),
                            tr("Blackball output(s) so they never get selected as fake outputs in a ring"));
   m_cmd_binder.set_handler("unblackball",
                            boost::bind(&simple_wallet::unblackball, this, _1),
-                           tr("unblackball <output public key>"),
+                           tr("unblackball <amount>/<offset>"),
                            tr("Unblackballs an output so it may get selected as a fake output in a ring"));
   m_cmd_binder.set_handler("blackballed",
                            boost::bind(&simple_wallet::blackballed, this, _1),
-                           tr("blackballed <output public key>"),
+                           tr("blackballed <amount>/<offset>"),
                            tr("Checks whether an output is blackballed"));
   m_cmd_binder.set_handler("version",
                            boost::bind(&simple_wallet::version, this, _1),

src/wallet/ringdb.cpp

@@ -55,6 +55,13 @@ static int compare_hash32(const MDB_val *a, const MDB_val *b)
   return 0;
 }
 
+static int compare_uint64(const MDB_val *a, const MDB_val *b)
+{
+  const uint64_t va = *(const uint64_t*) a->mv_data;
+  const uint64_t vb = *(const uint64_t*) b->mv_data;
+  return va < vb ? -1 : va > vb;
+}
+
 static std::string compress_ring(const std::vector<uint64_t> &ring)
 {
   std::string s;
@@ -146,7 +153,7 @@ static int resize_env(MDB_env *env, const char *db_path, size_t needed)
   MDB_stat mst;
   int ret;
 
-  needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB
+  needed = std::max(needed, (size_t)(100ul * 1024 * 1024)); // at least 100 MB
 
   ret = mdb_env_info(env, &mei);
   if (ret)
@@ -217,9 +224,9 @@ ringdb::ringdb(std::string filename, const std::string &genesis):
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
   mdb_set_compare(txn, dbi_rings, compare_hash32);
 
-  dbr = mdb_dbi_open(txn, ("blackballs-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs);
+  dbr = mdb_dbi_open(txn, ("blackballs2-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs);
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
-  mdb_set_dupsort(txn, dbi_blackballs, compare_hash32);
+  mdb_set_dupsort(txn, dbi_blackballs, compare_uint64);
 
   dbr = mdb_txn_commit(txn);
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr)));
@@ -374,7 +381,7 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im
   return true;
 }
 
-bool ringdb::blackball_worker(const crypto::public_key &output, int op)
+bool ringdb::blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op)
 {
   MDB_txn *txn;
   MDB_cursor *cursor;
@@ -382,49 +389,61 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op)
   bool tx_active = false;
   bool ret = true;
 
-  dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack
+  THROW_WALLET_EXCEPTION_IF(outputs.size() > 1 && op == BLACKBALL_QUERY, tools::error::wallet_internal_error, "Blackball query only makes sense for a single output");
+
+  dbr = resize_env(env, filename.c_str(), 32 * 2 * outputs.size()); // a pubkey, and some slack
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr)));
   dbr = mdb_txn_begin(env, NULL, 0, &txn);
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
   epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
   tx_active = true;
 
-  MDB_val key = zerokeyval;
-  MDB_val data;
-  data.mv_data = (void*)&output;
-  data.mv_size = sizeof(output);
 
-  switch (op)
+  MDB_val key, data;
+  for (const std::pair<uint64_t, uint64_t> &output: outputs)
   {
-    case BLACKBALL_BLACKBALL:
+    key.mv_data = (void*)&output.first;
-      MDEBUG("Blackballing output " << output);
+    key.mv_size = sizeof(output.first);
-      dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA);
+    data.mv_data = (void*)&output.second;
-      if (dbr == MDB_KEYEXIST)
+    data.mv_size = sizeof(output.second);
-        dbr = 0;
+
-      break;
+    switch (op)
-    case BLACKBALL_UNBLACKBALL:
+    {
-      MDEBUG("Unblackballing output " << output);
+      case BLACKBALL_BLACKBALL:
-      dbr = mdb_del(txn, dbi_blackballs, &key, &data);
+        MDEBUG("Blackballing output " << output.first << "/" << output.second);
-      if (dbr == MDB_NOTFOUND)
+        dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_APPENDDUP);
-        dbr = 0;
+        if (dbr == MDB_KEYEXIST)
-      break;
+          dbr = 0;
-    case BLACKBALL_QUERY:
+        break;
-      dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor);
+      case BLACKBALL_UNBLACKBALL:
-      THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr)));
+        MDEBUG("Unblackballing output " << output.first << "/" << output.second);
-      dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH);
+        dbr = mdb_del(txn, dbi_blackballs, &key, &data);
-      THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr)));
+        if (dbr == MDB_NOTFOUND)
-      ret = dbr != MDB_NOTFOUND;
+          dbr = 0;
-      if (dbr == MDB_NOTFOUND)
+        break;
-        dbr = 0;
+      case BLACKBALL_QUERY:
-      mdb_cursor_close(cursor);
+        dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor);
-      break;
+        THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr)));
-    case BLACKBALL_CLEAR:
+        dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH);
-      dbr = mdb_drop(txn, dbi_blackballs, 0);
+        THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr)));
-      break;
+        ret = dbr != MDB_NOTFOUND;
-    default:
+        if (dbr == MDB_NOTFOUND)
-      THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op");
+          dbr = 0;
+        mdb_cursor_close(cursor);
+        break;
+      case BLACKBALL_CLEAR:
+        break;
+      default:
+        THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op");
+    }
+    THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr)));
+  }
+
+  if (op == BLACKBALL_CLEAR)
+  {
+    dbr = mdb_drop(txn, dbi_blackballs, 0);
+    THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to clear blackballs table: " + std::string(mdb_strerror(dbr)));
   }
-  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr)));
 
   dbr = mdb_txn_commit(txn);
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr)));
@@ -432,24 +451,32 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op)
   return ret;
 }
 
-bool ringdb::blackball(const crypto::public_key &output)
+bool ringdb::blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs)
 {
-  return blackball_worker(output, BLACKBALL_BLACKBALL);
+  return blackball_worker(outputs, BLACKBALL_BLACKBALL);
 }
 
-bool ringdb::unblackball(const crypto::public_key &output)
+bool ringdb::blackball(const std::pair<uint64_t, uint64_t> &output)
 {
-  return blackball_worker(output, BLACKBALL_UNBLACKBALL);
+  std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output);
+  return blackball_worker(outputs, BLACKBALL_BLACKBALL);
 }
 
-bool ringdb::blackballed(const crypto::public_key &output)
+bool ringdb::unblackball(const std::pair<uint64_t, uint64_t> &output)
 {
-  return blackball_worker(output, BLACKBALL_QUERY);
+  std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output);
+  return blackball_worker(outputs, BLACKBALL_UNBLACKBALL);
+}
+
+bool ringdb::blackballed(const std::pair<uint64_t, uint64_t> &output)
+{
+  std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output);
+  return blackball_worker(outputs, BLACKBALL_QUERY);
 }
 
 bool ringdb::clear_blackballs()
 {
-  return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR);
+  return blackball_worker(std::vector<std::pair<uint64_t, uint64_t>>(), BLACKBALL_CLEAR);
 }
 
 }

src/wallet/ringdb.h

@@ -49,13 +49,14 @@ namespace tools
     bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs);
     bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative);
 
-    bool blackball(const crypto::public_key &output);
+    bool blackball(const std::pair<uint64_t, uint64_t> &output);
-    bool unblackball(const crypto::public_key &output);
+    bool blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs);
-    bool blackballed(const crypto::public_key &output);
+    bool unblackball(const std::pair<uint64_t, uint64_t> &output);
+    bool blackballed(const std::pair<uint64_t, uint64_t> &output);
     bool clear_blackballs();
 
   private:
-    bool blackball_worker(const crypto::public_key &output, int op);
+    bool blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op);
 
   private:
     std::string filename;

src/wallet/wallet2.cpp

@@ -6341,7 +6341,7 @@ bool wallet2::find_and_save_rings(bool force)
   return true;
 }
 
-bool wallet2::blackball_output(const crypto::public_key &output)
+bool wallet2::blackball_output(const std::pair<uint64_t, uint64_t> &output)
 {
   if (!m_ringdb)
     return false;
@@ -6349,7 +6349,7 @@ bool wallet2::blackball_output(const crypto::public_key &output)
   catch (const std::exception &e) { return false; }
 }
 
-bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add)
+bool wallet2::set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add)
 {
   if (!m_ringdb)
     return false;
@@ -6358,14 +6358,13 @@ bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &out
     bool ret = true;
     if (!add)
       ret &= m_ringdb->clear_blackballs();
-    for (const auto &output: outputs)
+    ret &= m_ringdb->blackball(outputs);
-      ret &= m_ringdb->blackball(output);
     return ret;
   }
   catch (const std::exception &e) { return false; }
 }
 
-bool wallet2::unblackball_output(const crypto::public_key &output)
+bool wallet2::unblackball_output(const std::pair<uint64_t, uint64_t> &output)
 {
   if (!m_ringdb)
     return false;
@@ -6373,7 +6372,7 @@ bool wallet2::unblackball_output(const crypto::public_key &output)
   catch (const std::exception &e) { return false; }
 }
 
-bool wallet2::is_output_blackballed(const crypto::public_key &output) const
+bool wallet2::is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const
 {
   if (!m_ringdb)
     return false;
@@ -6418,8 +6417,8 @@ bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_out
   CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty");
   if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates
     return false;
-  if (is_output_blackballed(output_public_key)) // don't add blackballed outputs
+//  if (is_output_blackballed(output_public_key)) // don't add blackballed outputs
-    return false;
+//    return false;
   outs.back().push_back(item);
   return true;
 }
@@ -6916,6 +6915,8 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
 
           if (seen_indices.count(i))
             continue;
+          if (is_output_blackballed(std::make_pair(amount, i))) // don't add blackballed outputs
+            continue;
           seen_indices.emplace(i);
 
           LOG_PRINT_L2("picking " << i << " as " << type);

src/wallet/wallet2.h

@@ -1159,10 +1159,10 @@ namespace tools
     bool set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative);
     bool find_and_save_rings(bool force = true);
 
-    bool blackball_output(const crypto::public_key &output);
+    bool blackball_output(const std::pair<uint64_t, uint64_t> &output);
-    bool set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add = false);
+    bool set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add = false);
-    bool unblackball_output(const crypto::public_key &output);
+    bool unblackball_output(const std::pair<uint64_t, uint64_t> &output);
-    bool is_output_blackballed(const crypto::public_key &output) const;
+    bool is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const;
 
     bool lock_keys_file();
     bool unlock_keys_file();

tests/unit_tests/ringdb.cpp

@@ -59,17 +59,17 @@ static crypto::key_image generate_key_image()
   return key_image;
 }
 
-static crypto::public_key generate_output()
+static std::pair<uint64_t, uint64_t> generate_output()
 {
-  return rct::rct2pk(rct::scalarmultBase(rct::skGen()));
+  return std::make_pair(rand(), rand());
 }
 
 
 static const crypto::chacha_key KEY_1 = generate_chacha_key();
 static const crypto::chacha_key KEY_2 = generate_chacha_key();
 static const crypto::key_image KEY_IMAGE_1 = generate_key_image();
-static const crypto::public_key OUTPUT_1 = generate_output();
+static const std::pair<uint64_t, uint64_t> OUTPUT_1 = generate_output();
-static const crypto::public_key OUTPUT_2 = generate_output();
+static const std::pair<uint64_t, uint64_t> OUTPUT_2 = generate_output();
 
 class RingDB: public tools::ringdb
 {