Merge pull request #3322

eac3a11e wallet: more user friendly print_ring (moneromooo-monero)
79853514 wallet2_api: add key reuse mitigations API (moneromooo-monero)
b057a21d wallet2_api: add ring api (moneromooo-monero)
d32ef7b0 ringdb: factor ring addition code (moneromooo-monero)
a7da8208 wallet2_api: add blackball api (moneromooo-monero)
2ab66ff1 liblmdb: install lmdb library for wallet2_api usage (stoffu)
504428ab ringdb: use the genesis block as a db name (moneromooo-monero)
b09e5181 wallet: add a set_ring command (moneromooo-monero)
0590f62a new blockchain_usage tool, reports on output usage (moneromooo-monero)
db10dd6d wallet: make ringdb an object with database state (moneromooo-monero)
df6fad4c blockchain_utilities: new blockchain_blackball tool (moneromooo-monero)
d29ea045 wallet: add an output blackball list to avoid using those in rings (moneromooo-monero)
18eaf194 wallet: key reuse mitigation options (moneromooo-monero)
5f146873 wallet: add shared ring database (moneromooo-monero)
41f727ce add RPC to get a histogram of outputs of a given amount (moneromooo-monero)

external/db_drivers/liblmdb/CMakeLists.txt

@@ -50,4 +50,16 @@ if(${ARCH_WIDTH} EQUAL 32)
   target_compile_definitions(lmdb
     PUBLIC -DMDB_VL32)
 endif()
+
+# GUI/libwallet install target
+if (BUILD_GUI_DEPS)
+    if(IOS)
+        set(lib_folder lib-${ARCH})
+    else()
+        set(lib_folder lib)
+    endif()
+    install(TARGETS lmdb
+        ARCHIVE DESTINATION ${lib_folder}
+        LIBRARY DESTINATION ${lib_folder})
+endif()
 set_property(TARGET lmdb APPEND PROPERTY COMPILE_FLAGS "-fPIC")

src/blockchain_db/blockchain_db.h

@@ -1448,7 +1448,9 @@ public:
    *
    * @return false if the function returns false for any output, otherwise true
    */
-  virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const = 0;
+  virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const = 0;
+  virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const = 0;
+
 
 
   //

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -2564,7 +2564,7 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&
   return fret;
 }
 
-bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const
+bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -2588,7 +2588,47 @@ bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const c
     uint64_t amount = *(const uint64_t*)k.mv_data;
     outkey *ok = (outkey *)v.mv_data;
     tx_out_index toi = get_output_tx_and_index_from_global(ok->output_id);
-    if (!f(amount, toi.first, toi.second)) {
+    if (!f(amount, toi.first, ok->data.height, toi.second)) {
+      fret = false;
+      break;
+    }
+  }
+
+  TXN_POSTFIX_RDONLY();
+
+  return fret;
+}
+
+bool BlockchainLMDB::for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(output_amounts);
+
+  MDB_val_set(k, amount);
+  MDB_val v;
+  bool fret = true;
+
+  MDB_cursor_op op = MDB_SET;
+  while (1)
+  {
+    int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op);
+    op = MDB_NEXT_DUP;
+    if (ret == MDB_NOTFOUND)
+      break;
+    if (ret)
+      throw0(DB_ERROR("Failed to enumerate outputs"));
+    uint64_t out_amount = *(const uint64_t*)k.mv_data;
+    if (amount != out_amount)
+    {
+      MERROR("Amount is not the expected amount");
+      fret = false;
+      break;
+    }
+    const outkey *ok = (const outkey *)v.mv_data;
+    if (!f(ok->data.height)) {
       fret = false;
       break;
     }

src/blockchain_db/lmdb/db_lmdb.h

@@ -255,7 +255,8 @@ public:
   virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const;
   virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const;
   virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const;
-  virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const;
+  virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const;
+  virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const;
 
   virtual uint64_t add_block( const block& blk
                             , const size_t& block_size

src/blockchain_utilities/CMakeLists.txt

@@ -67,6 +67,30 @@ monero_private_headers(blockchain_export
 	  ${blockchain_export_private_headers})
 
 
+set(blockchain_blackball_sources
+  blockchain_blackball.cpp
+  )
+
+set(blockchain_blackball_private_headers
+  bootstrap_file.h
+  blocksdat_file.h
+  bootstrap_serialization.h
+  )
+
+monero_private_headers(blockchain_blackball
+	  ${blockchain_blackball_private_headers})
+
+set(blockchain_usage_sources
+  blockchain_usage.cpp
+  )
+
+set(blockchain_usage_private_headers)
+
+monero_private_headers(blockchain_usage
+	  ${blockchain_usage_private_headers})
+
+
+
 monero_add_executable(blockchain_import
   ${blockchain_import_sources}
   ${blockchain_import_private_headers}
@@ -117,3 +141,49 @@ set_property(TARGET blockchain_export
 	OUTPUT_NAME "monero-blockchain-export")
 install(TARGETS blockchain_export DESTINATION bin)
 
+monero_add_executable(blockchain_blackball
+  ${blockchain_blackball_sources}
+  ${blockchain_blackball_private_headers})
+
+target_link_libraries(blockchain_blackball
+  PRIVATE
+    wallet
+    cryptonote_core
+    blockchain_db
+    p2p
+    version
+    epee
+    ${Boost_FILESYSTEM_LIBRARY}
+    ${Boost_SYSTEM_LIBRARY}
+    ${Boost_THREAD_LIBRARY}
+    ${CMAKE_THREAD_LIBS_INIT}
+    ${EXTRA_LIBRARIES})
+
+set_property(TARGET blockchain_blackball
+	PROPERTY
+	OUTPUT_NAME "monero-blockchain-blackball")
+install(TARGETS blockchain_blackball DESTINATION bin)
+
+
+monero_add_executable(blockchain_usage
+  ${blockchain_usage_sources}
+  ${blockchain_usage_private_headers})
+
+target_link_libraries(blockchain_usage
+  PRIVATE
+    cryptonote_core
+    blockchain_db
+    p2p
+    version
+    epee
+    ${Boost_FILESYSTEM_LIBRARY}
+    ${Boost_SYSTEM_LIBRARY}
+    ${Boost_THREAD_LIBRARY}
+    ${CMAKE_THREAD_LIBS_INIT}
+    ${EXTRA_LIBRARIES})
+
+set_property(TARGET blockchain_usage
+	PROPERTY
+	OUTPUT_NAME "monero-blockchain-usage")
+install(TARGETS blockchain_usage DESTINATION bin)
+

src/blockchain_utilities/blockchain_blackball.cpp

@@ -0,0 +1,427 @@
+// Copyright (c) 2014-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <boost/range/adaptor/transformed.hpp>
+#include <boost/algorithm/string.hpp>
+#include "common/command_line.h"
+#include "common/varint.h"
+#include "cryptonote_core/tx_pool.h"
+#include "cryptonote_core/cryptonote_core.h"
+#include "cryptonote_core/blockchain.h"
+#include "blockchain_db/blockchain_db.h"
+#include "blockchain_db/db_types.h"
+#include "wallet/ringdb.h"
+#include "version.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
+
+namespace po = boost::program_options;
+using namespace epee;
+using namespace cryptonote;
+
+struct output_data
+{
+  uint64_t amount;
+  uint64_t index;
+  output_data(uint64_t a, uint64_t i): amount(a), index(i) {}
+  bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; }
+};
+namespace std
+{
+  template<> struct hash<output_data>
+  {
+    size_t operator()(const output_data &od) const
+    {
+      const uint64_t data[2] = {od.amount, od.index};
+      crypto::hash h;
+      crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h);
+      return reinterpret_cast<const std::size_t &>(h);
+    }
+  };
+}
+
+static std::string get_default_db_path()
+{
+  boost::filesystem::path dir = tools::get_default_data_dir();
+  // remove .bitmonero, replace with .shared-ringdb
+  dir = dir.remove_filename();
+  dir /= ".shared-ringdb";
+  return dir.string();
+}
+
+static bool for_all_transactions(const std::string &filename, const std::function<bool(const cryptonote::transaction_prefix&)> &f)
+{
+  MDB_env *env;
+  MDB_dbi dbi;
+  MDB_txn *txn;
+  MDB_cursor *cur;
+  int dbr;
+  bool tx_active = false;
+
+  dbr = mdb_env_create(&env);
+  if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_env_set_maxdbs(env, 2);
+  if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
+  const std::string actual_filename = filename;
+  dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664);
+  if (dbr) throw std::runtime_error("Failed to open rings database file '"
+      + actual_filename + "': " + std::string(mdb_strerror(dbr)));
+
+  dbr = mdb_txn_begin(env, NULL, 0, &txn);
+  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
+  tx_active = true;
+
+  dbr = mdb_dbi_open(txn, "txs_pruned", MDB_INTEGERKEY, &dbi);
+  if (dbr)
+    dbr = mdb_dbi_open(txn, "txs", MDB_INTEGERKEY, &dbi);
+  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_cursor_open(txn, dbi, &cur);
+  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+
+  MDB_val k;
+  MDB_val v;
+  bool fret = true;
+
+  MDB_cursor_op op = MDB_FIRST;
+  while (1)
+  {
+    int ret = mdb_cursor_get(cur, &k, &v, op);
+    op = MDB_NEXT;
+    if (ret == MDB_NOTFOUND)
+      break;
+    if (ret)
+      throw std::runtime_error("Failed to enumerate transactions: " + std::string(mdb_strerror(ret)));
+
+    cryptonote::transaction_prefix tx;
+    blobdata bd;
+    bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
+    std::stringstream ss;
+    ss << bd;
+    binary_archive<false> ba(ss);
+    bool r = do_serialize(ba, tx);
+    CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob");
+
+    if (!f(tx)) {
+      fret = false;
+      break;
+    }
+  }
+
+  mdb_cursor_close(cur);
+  mdb_txn_commit(txn);
+  tx_active = false;
+  mdb_dbi_close(env, dbi);
+  mdb_env_close(env);
+  return fret;
+}
+
+int main(int argc, char* argv[])
+{
+  TRY_ENTRY();
+
+  epee::string_tools::set_module_name_and_folder(argv[0]);
+
+  std::string default_db_type = "lmdb";
+
+  std::string available_dbs = cryptonote::blockchain_db_types(", ");
+  available_dbs = "available: " + available_dbs;
+
+  uint32_t log_level = 0;
+
+  tools::on_startup();
+
+  boost::filesystem::path output_file_path;
+
+  po::options_description desc_cmd_only("Command line options");
+  po::options_description desc_cmd_sett("Command line options and settings options");
+  const command_line::arg_descriptor<std::string, false, true, 2> arg_blackball_db_dir = {
+      "blackball-db-dir", "Specify blackball database directory",
+      get_default_db_path(),
+      {{ &arg_testnet_on, &arg_stagenet_on }},
+      [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val) {
+        if (testnet_stagenet[0])
+          return (boost::filesystem::path(val) / "testnet").string();
+        else if (testnet_stagenet[1])
+          return (boost::filesystem::path(val) / "stagenet").string();
+        return val;
+      }
+  };
+  const command_line::arg_descriptor<std::string> arg_log_level  = {"log-level",  "0-4 or categories", ""};
+  const command_line::arg_descriptor<std::string> arg_database = {
+    "database", available_dbs.c_str(), default_db_type
+  };
+  const command_line::arg_descriptor<bool> arg_rct_only  = {"rct-only", "Only work on ringCT outputs", false};
+  const command_line::arg_descriptor<std::vector<std::string> > arg_inputs = {"inputs", "Path to Monero DB, and path to any fork DBs"};
+
+  command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir);
+  command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
+  command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on);
+  command_line::add_arg(desc_cmd_sett, arg_log_level);
+  command_line::add_arg(desc_cmd_sett, arg_database);
+  command_line::add_arg(desc_cmd_sett, arg_rct_only);
+  command_line::add_arg(desc_cmd_sett, arg_inputs);
+  command_line::add_arg(desc_cmd_only, command_line::arg_help);
+
+  po::options_description desc_options("Allowed options");
+  desc_options.add(desc_cmd_only).add(desc_cmd_sett);
+
+  po::positional_options_description positional_options;
+  positional_options.add(arg_inputs.name, -1);
+
+  po::variables_map vm;
+  bool r = command_line::handle_error_helper(desc_options, [&]()
+  {
+    auto parser = po::command_line_parser(argc, argv).options(desc_options).positional(positional_options);
+    po::store(parser.run(), vm);
+    po::notify(vm);
+    return true;
+  });
+  if (! r)
+    return 1;
+
+  if (command_line::get_arg(vm, command_line::arg_help))
+  {
+    std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL;
+    std::cout << desc_options << std::endl;
+    return 1;
+  }
+
+  mlog_configure(mlog_get_default_log_path("monero-blockchain-blackball.log"), true);
+  if (!command_line::is_arg_defaulted(vm, arg_log_level))
+    mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
+  else
+    mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
+
+  LOG_PRINT_L0("Starting...");
+
+  bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
+  bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on);
+  network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET;
+  output_file_path = command_line::get_arg(vm, arg_blackball_db_dir);
+  bool opt_rct_only = command_line::get_arg(vm, arg_rct_only);
+
+  std::string db_type = command_line::get_arg(vm, arg_database);
+  if (!cryptonote::blockchain_valid_db_type(db_type))
+  {
+    std::cerr << "Invalid database type: " << db_type << std::endl;
+    return 1;
+  }
+
+  // If we wanted to use the memory pool, we would set up a fake_core.
+
+  // Use Blockchain instead of lower-level BlockchainDB for two reasons:
+  // 1. Blockchain has the init() method for easy setup
+  // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash()
+  //
+  // cannot match blockchain_storage setup above with just one line,
+  // e.g.
+  //   Blockchain* core_storage = new Blockchain(NULL);
+  // because unlike blockchain_storage constructor, which takes a pointer to
+  // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object.
+  LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
+  const std::vector<std::string> inputs = command_line::get_arg(vm, arg_inputs);
+  if (inputs.empty())
+  {
+    LOG_PRINT_L0("No inputs given");
+    return 1;
+  }
+  std::vector<std::unique_ptr<Blockchain>> core_storage(inputs.size());
+  tx_memory_pool m_mempool(*(Blockchain*)NULL);
+  for (size_t n = 0; n < inputs.size(); ++n)
+  {
+    core_storage[n].reset(new Blockchain(m_mempool));
+
+    BlockchainDB* db = new_db(db_type);
+    if (db == NULL)
+    {
+      LOG_ERROR("Attempted to use non-existent database type: " << db_type);
+      throw std::runtime_error("Attempting to use non-existent database type");
+    }
+    LOG_PRINT_L0("database: " << db_type);
+
+    std::string filename = inputs[n];
+    while (boost::ends_with(filename, "/") || boost::ends_with(filename, "\\"))
+      filename.pop_back();
+    LOG_PRINT_L0("Loading blockchain from folder " << filename << " ...");
+
+    try
+    {
+      db->open(filename, DBF_RDONLY);
+    }
+    catch (const std::exception& e)
+    {
+      LOG_PRINT_L0("Error opening database: " << e.what());
+      return 1;
+    }
+    r = core_storage[n]->init(db, net_type);
+
+    CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
+    LOG_PRINT_L0("Source blockchain storage initialized OK");
+  }
+
+  boost::filesystem::path direc(output_file_path.string());
+  if (boost::filesystem::exists(direc))
+  {
+    if (!boost::filesystem::is_directory(direc))
+    {
+      MERROR("LMDB needs a directory path, but a file was passed: " << output_file_path.string());
+      return 1;
+    }
+  }
+  else
+  {
+    if (!boost::filesystem::create_directories(direc))
+    {
+      MERROR("Failed to create directory: " << output_file_path.string());
+      return 1;
+    }
+  }
+
+  LOG_PRINT_L0("Scanning for blackballable outputs...");
+
+  size_t done = 0;
+  std::unordered_map<crypto::key_image, std::vector<uint64_t>> relative_rings;
+  std::unordered_map<output_data, std::unordered_set<crypto::key_image>> outputs;
+  std::unordered_set<output_data> spent, newly_spent;
+
+  cryptonote::block b = core_storage[0]->get_db().get_block_from_height(0);
+  tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_block_hash(b)));
+
+  for (size_t n = 0; n < inputs.size(); ++n)
+  {
+    LOG_PRINT_L0("Reading blockchain from " << inputs[n]);
+    for_all_transactions(inputs[n], [&](const cryptonote::transaction_prefix &tx)->bool
+    {
+      for (const auto &in: tx.vin)
+      {
+        if (in.type() != typeid(txin_to_key))
+          continue;
+        const auto &txin = boost::get<txin_to_key>(in);
+        if (opt_rct_only && txin.amount != 0)
+          continue;
+
+        const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets);
+        if (n == 0)
+          for (uint64_t out: absolute)
+            outputs[output_data(txin.amount, out)].insert(txin.k_image);
+
+        std::vector<uint64_t> new_ring = txin.key_offsets;
+        const uint32_t ring_size = txin.key_offsets.size();
+        if (ring_size == 1)
+        {
+          const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, txin.key_offsets[0]);
+          MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring");
+          ringdb.blackball(pkey);
+          newly_spent.insert(output_data(txin.amount, txin.key_offsets[0]));
+          spent.insert(output_data(txin.amount, txin.key_offsets[0]));
+        }
+        else if (relative_rings.find(txin.k_image) != relative_rings.end())
+        {
+          MINFO("Key image " << txin.k_image << " already seen: rings " <<
+              boost::join(relative_rings[txin.k_image] | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") <<
+              ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " "));
+          if (relative_rings[txin.k_image] != txin.key_offsets)
+          {
+            MINFO("Rings are different");
+            const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(relative_rings[txin.k_image]);
+            const std::vector<uint64_t> r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets);
+            std::vector<uint64_t> common;
+            for (uint64_t out: r0)
+            {
+              if (std::find(r1.begin(), r1.end(), out) != r1.end())
+                common.push_back(out);
+            }
+            if (common.empty())
+            {
+              MERROR("Rings for the same key image are disjoint");
+            }
+            else if (common.size() == 1)
+            {
+              const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]);
+              MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element");
+              ringdb.blackball(pkey);
+              newly_spent.insert(output_data(txin.amount, common[0]));
+              spent.insert(output_data(txin.amount, common[0]));
+            }
+            else
+            {
+              MINFO("The intersection has more than one element, it's still ok");
+              for (const auto &out: r0)
+                if (std::find(common.begin(), common.end(), out) != common.end())
+                  new_ring.push_back(out);
+              new_ring = cryptonote::absolute_output_offsets_to_relative(new_ring);
+            }
+          }
+        }
+        relative_rings[txin.k_image] = new_ring;
+      }
+      return true;
+    });
+  }
+
+  while (!newly_spent.empty())
+  {
+    LOG_PRINT_L0("Secondary pass due to " << newly_spent.size() << " newly found spent outputs");
+    std::unordered_set<output_data> work_spent = std::move(newly_spent);
+    newly_spent.clear();
+
+    for (const output_data &od: work_spent)
+    {
+      for (const crypto::key_image &ki: outputs[od])
+      {
+        std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(relative_rings[ki]);
+        size_t known = 0;
+        uint64_t last_unknown = 0;
+        for (uint64_t out: absolute)
+        {
+          output_data new_od(od.amount, out);
+          if (spent.find(new_od) != spent.end())
+            ++known;
+          else
+            last_unknown = out;
+        }
+        if (known == absolute.size() - 1)
+        {
+          const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown);
+          MINFO("Blackballing output " << pkey << ", due to being used in a " <<
+              absolute.size() << "-ring where all other outputs are known to be spent");
+          ringdb.blackball(pkey);
+          newly_spent.insert(output_data(od.amount, last_unknown));
+          spent.insert(output_data(od.amount, last_unknown));
+        }
+      }
+    }
+  }
+
+  LOG_PRINT_L0("Blockchain blackball data exported OK");
+  return 0;
+
+  CATCH_ENTRY("Export error", 1);
+}

src/blockchain_utilities/blockchain_usage.cpp

@@ -0,0 +1,256 @@
+// Copyright (c) 2014-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <boost/range/adaptor/transformed.hpp>
+#include <boost/algorithm/string.hpp>
+#include "common/command_line.h"
+#include "common/varint.h"
+#include "cryptonote_core/tx_pool.h"
+#include "cryptonote_core/cryptonote_core.h"
+#include "cryptonote_core/blockchain.h"
+#include "blockchain_db/blockchain_db.h"
+#include "blockchain_db/db_types.h"
+#include "version.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
+
+namespace po = boost::program_options;
+using namespace epee;
+using namespace cryptonote;
+
+struct output_data
+{
+  uint64_t amount;
+  uint64_t index;
+  mutable bool coinbase;
+  mutable uint64_t height;
+  output_data(uint64_t a, uint64_t i, bool cb, uint64_t h): amount(a), index(i), coinbase(cb), height(h) {}
+  bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; }
+  void info(bool c, uint64_t h) const { coinbase = c; height =h; }
+};
+namespace std
+{
+  template<> struct hash<output_data>
+  {
+    size_t operator()(const output_data &od) const
+    {
+      const uint64_t data[2] = {od.amount, od.index};
+      crypto::hash h;
+      crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h);
+      return reinterpret_cast<const std::size_t &>(h);
+    }
+  };
+}
+
+struct reference
+{
+  uint64_t height;
+  uint64_t ring_size;
+  uint64_t position;
+  reference(uint64_t h, uint64_t rs, uint64_t p): height(h), ring_size(rs), position(p) {}
+};
+
+int main(int argc, char* argv[])
+{
+  TRY_ENTRY();
+
+  epee::string_tools::set_module_name_and_folder(argv[0]);
+
+  std::string default_db_type = "lmdb";
+
+  std::string available_dbs = cryptonote::blockchain_db_types(", ");
+  available_dbs = "available: " + available_dbs;
+
+  uint32_t log_level = 0;
+
+  tools::on_startup();
+
+  boost::filesystem::path output_file_path;
+
+  po::options_description desc_cmd_only("Command line options");
+  po::options_description desc_cmd_sett("Command line options and settings options");
+  const command_line::arg_descriptor<std::string> arg_log_level  = {"log-level",  "0-4 or categories", ""};
+  const command_line::arg_descriptor<std::string> arg_database = {
+    "database", available_dbs.c_str(), default_db_type
+  };
+  const command_line::arg_descriptor<bool> arg_rct_only  = {"rct-only", "Only work on ringCT outputs", false};
+  const command_line::arg_descriptor<std::string> arg_input = {"input", ""};
+
+  command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
+  command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on);
+  command_line::add_arg(desc_cmd_sett, arg_log_level);
+  command_line::add_arg(desc_cmd_sett, arg_database);
+  command_line::add_arg(desc_cmd_sett, arg_rct_only);
+  command_line::add_arg(desc_cmd_sett, arg_input);
+  command_line::add_arg(desc_cmd_only, command_line::arg_help);
+
+  po::options_description desc_options("Allowed options");
+  desc_options.add(desc_cmd_only).add(desc_cmd_sett);
+
+  po::positional_options_description positional_options;
+  positional_options.add(arg_input.name, -1);
+
+  po::variables_map vm;
+  bool r = command_line::handle_error_helper(desc_options, [&]()
+  {
+    auto parser = po::command_line_parser(argc, argv).options(desc_options).positional(positional_options);
+    po::store(parser.run(), vm);
+    po::notify(vm);
+    return true;
+  });
+  if (! r)
+    return 1;
+
+  if (command_line::get_arg(vm, command_line::arg_help))
+  {
+    std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL;
+    std::cout << desc_options << std::endl;
+    return 1;
+  }
+
+  mlog_configure(mlog_get_default_log_path("monero-blockchain-usage.log"), true);
+  if (!command_line::is_arg_defaulted(vm, arg_log_level))
+    mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
+  else
+    mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
+
+  LOG_PRINT_L0("Starting...");
+
+  bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
+  bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on);
+  network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET;
+  bool opt_rct_only = command_line::get_arg(vm, arg_rct_only);
+
+  std::string db_type = command_line::get_arg(vm, arg_database);
+  if (!cryptonote::blockchain_valid_db_type(db_type))
+  {
+    std::cerr << "Invalid database type: " << db_type << std::endl;
+    return 1;
+  }
+
+  // If we wanted to use the memory pool, we would set up a fake_core.
+
+  // Use Blockchain instead of lower-level BlockchainDB for two reasons:
+  // 1. Blockchain has the init() method for easy setup
+  // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash()
+  //
+  // cannot match blockchain_storage setup above with just one line,
+  // e.g.
+  //   Blockchain* core_storage = new Blockchain(NULL);
+  // because unlike blockchain_storage constructor, which takes a pointer to
+  // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object.
+  LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
+  const std::string input = command_line::get_arg(vm, arg_input);
+  std::unique_ptr<Blockchain> core_storage;
+  tx_memory_pool m_mempool(*core_storage);
+  core_storage.reset(new Blockchain(m_mempool));
+  BlockchainDB* db = new_db(db_type);
+  if (db == NULL)
+  {
+    LOG_ERROR("Attempted to use non-existent database type: " << db_type);
+    throw std::runtime_error("Attempting to use non-existent database type");
+  }
+  LOG_PRINT_L0("database: " << db_type);
+
+  const std::string filename = input;
+  LOG_PRINT_L0("Loading blockchain from folder " << filename << " ...");
+
+  try
+  {
+    db->open(filename, DBF_RDONLY);
+  }
+  catch (const std::exception& e)
+  {
+    LOG_PRINT_L0("Error opening database: " << e.what());
+    return 1;
+  }
+  r = core_storage->init(db, net_type);
+
+  CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
+  LOG_PRINT_L0("Source blockchain storage initialized OK");
+
+  LOG_PRINT_L0("Building usage patterns...");
+
+  size_t done = 0;
+  std::unordered_map<output_data, std::list<reference>> outputs;
+  std::unordered_map<uint64_t,uint64_t> indices;
+
+  LOG_PRINT_L0("Reading blockchain from " << input);
+  core_storage->for_all_transactions([&](const crypto::hash &hash, const cryptonote::transaction &tx)->bool
+  {
+    const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen);
+    const uint64_t height = core_storage->get_db().get_tx_block_height(hash);
+
+    // create new outputs
+    for (const auto &out: tx.vout)
+    {
+      if (opt_rct_only && out.amount)
+        continue;
+      uint64_t index = indices[out.amount]++;
+      output_data od(out.amount, indices[out.amount], coinbase, height);
+      auto itb = outputs.emplace(od, std::list<reference>());
+      itb.first->first.info(coinbase, height);
+    }
+
+    for (const auto &in: tx.vin)
+    {
+      if (in.type() != typeid(txin_to_key))
+        continue;
+      const auto &txin = boost::get<txin_to_key>(in);
+      if (opt_rct_only && txin.amount != 0)
+        continue;
+
+      const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets);
+      for (size_t n = 0; n < txin.key_offsets.size(); ++n)
+      {
+        output_data od(txin.amount, absolute[n], coinbase, height);
+        outputs[od].push_back(reference(height, txin.key_offsets.size(), n));
+      }
+    }
+    return true;
+  });
+
+  std::unordered_map<uint64_t, uint64_t> counts;
+  size_t total = 0;
+  for (const auto &out: outputs)
+  {
+    counts[out.second.size()]++;
+    total++;
+  }
+  for (const auto &c: counts)
+  {
+    float percent = 100.f * c.second / total;
+    MINFO(std::to_string(c.second) << " outputs used " << c.first << " times (" << percent << "%)");
+  }
+
+  LOG_PRINT_L0("Blockchain usage exported OK");
+  return 0;
+
+  CATCH_ENTRY("Export error", 1);
+}

src/cryptonote_core/blockchain.cpp

@@ -1903,6 +1903,45 @@ void Blockchain::get_output_key_mask_unlocked(const uint64_t& amount, const uint
   unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first));
 }
 //------------------------------------------------------------------
+bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const
+{
+  // rct outputs don't exist before v3
+  if (amount == 0)
+  {
+    switch (m_nettype)
+    {
+      case STAGENET: start_height = stagenet_hard_forks[2].height; break;
+      case TESTNET: start_height = testnet_hard_forks[2].height; break;
+      case MAINNET: start_height = mainnet_hard_forks[2].height; break;
+      default: return false;
+    }
+  }
+  else
+    start_height = 0;
+  base = 0;
+
+  const uint64_t real_start_height = start_height;
+  if (from_height > start_height)
+    start_height = from_height;
+
+  distribution.clear();
+  uint64_t db_height = m_db->height();
+  if (start_height >= db_height)
+    return false;
+  distribution.resize(db_height - start_height, 0);
+  bool r = for_all_outputs(amount, [&](uint64_t height) {
+    CHECK_AND_ASSERT_MES(height >= real_start_height && height <= db_height, false, "Height not in expected range");
+    if (height >= start_height)
+      distribution[height - start_height]++;
+    else
+      base++;
+    return true;
+  });
+  if (!r)
+    return false;
+  return true;
+}
+//------------------------------------------------------------------
 // This function takes a list of block hashes from another node
 // on the network to find where the split point is between us and them.
 // This is used to see what to send another node that needs to sync.
@@ -4441,11 +4480,16 @@ bool Blockchain::for_all_transactions(std::function<bool(const crypto::hash&, co
   return m_db->for_all_transactions(f);
 }
 
-bool Blockchain::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const
+bool Blockchain::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const
 {
   return m_db->for_all_outputs(f);;
 }
 
+bool Blockchain::for_all_outputs(uint64_t amount, std::function<bool(uint64_t height)> f) const
+{
+  return m_db->for_all_outputs(amount, f);;
+}
+
 namespace cryptonote {
 template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::list<transaction>&, std::list<crypto::hash>&) const;
 }

src/cryptonote_core/blockchain.h

@@ -523,6 +523,17 @@ namespace cryptonote
      */
     bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const;
 
+    /**
+     * @brief gets per block distribution of outputs of a given amount
+     *
+     * @param amount the amount to get a ditribution for
+     * @param return-by-reference from_height the height before which we do not care about the data
+     * @param return-by-reference start_height the height of the first rct output
+     * @param return-by-reference distribution the start offset of the first rct output in this block (same as previous if none)
+     * @param return-by-reference base how many outputs of that amount are before the stated distribution
+     */
+    bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const;
+
     /**
      * @brief gets the global indices for outputs from a given transaction
      *
@@ -857,7 +868,17 @@ namespace cryptonote
      *
      * @return false if any output fails the check, otherwise true
      */
-    bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)>) const;
+    bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)>) const;
+
+    /**
+     * @brief perform a check on all outputs of a given amount in the blockchain
+     *
+     * @param amount the amount to iterate through
+     * @param std::function the check to perform, pass/fail
+     *
+     * @return false if any output fails the check, otherwise true
+     */
+    bool for_all_outputs(uint64_t amount, std::function<bool(uint64_t height)>) const;
 
     /**
      * @brief get a reference to the BlockchainDB in use by Blockchain

src/cryptonote_core/cryptonote_core.cpp

@@ -1105,6 +1105,11 @@ namespace cryptonote
     return m_blockchain_storage.get_random_rct_outs(req, res);
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const
+  {
+    return m_blockchain_storage.get_output_distribution(amount, from_height, start_height, distribution, base);
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const
   {
     return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, indexs);

src/cryptonote_core/cryptonote_core.h

@@ -571,6 +571,12 @@ namespace cryptonote
       */
      bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const;
 
+     /**
+      * @copydoc Blockchain::get_output_distribution
+      *
+      * @brief get per block distribution of outputs of a given amount
+      */
+     bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const;
 
      /**
       * @copydoc miner::pause

src/rpc/core_rpc_server.cpp

@@ -2076,6 +2076,41 @@ namespace cryptonote
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool core_rpc_server::on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp)
+  {
+    try
+    {
+      for (uint64_t amount: req.amounts)
+      {
+        std::vector<uint64_t> distribution;
+        uint64_t start_height, base;
+        if (!m_core.get_output_distribution(amount, req.from_height, start_height, distribution, base))
+        {
+          error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
+          error_resp.message = "Failed to get rct distribution";
+          return false;
+        }
+        if (req.cumulative)
+        {
+          distribution[0] += base;
+          for (size_t n = 1; n < distribution.size(); ++n)
+            distribution[n] += distribution[n-1];
+        }
+        res.distributions.push_back({amount, start_height, std::move(distribution), base});
+      }
+    }
+    catch (const std::exception &e)
+    {
+      error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
+      error_resp.message = "Failed to get output distribution";
+      return false;
+    }
+
+    res.status = CORE_RPC_STATUS_OK;
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
+
 
   const command_line::arg_descriptor<std::string, false, true, 2> core_rpc_server::arg_rpc_bind_port = {
       "rpc-bind-port"

src/rpc/core_rpc_server.h

@@ -153,6 +153,7 @@ namespace cryptonote
         MAP_JON_RPC_WE_IF("relay_tx",            on_relay_tx,                   COMMAND_RPC_RELAY_TX, !m_restricted)
         MAP_JON_RPC_WE_IF("sync_info",           on_sync_info,                  COMMAND_RPC_SYNC_INFO, !m_restricted)
         MAP_JON_RPC_WE("get_txpool_backlog",     on_get_txpool_backlog,         COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG)
+        MAP_JON_RPC_WE_IF("get_output_distribution", on_get_output_distribution, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION, !m_restricted)
       END_JSON_RPC_MAP()
     END_URI_MAP2()
 
@@ -214,6 +215,7 @@ namespace cryptonote
     bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp);
     bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp);
     bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp);
+    bool on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp);
     //-----------------------
 
 private:

src/rpc/core_rpc_server_commands_defs.h

@@ -49,7 +49,7 @@ namespace cryptonote
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define CORE_RPC_VERSION_MAJOR 1
-#define CORE_RPC_VERSION_MINOR 18
+#define CORE_RPC_VERSION_MINOR 19
 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)
 
@@ -2203,4 +2203,47 @@ namespace cryptonote
       END_KV_SERIALIZE_MAP()
     };
   };
+
+  struct COMMAND_RPC_GET_OUTPUT_DISTRIBUTION
+  {
+    struct request
+    {
+      std::vector<uint64_t> amounts;
+      uint64_t from_height;
+      bool cumulative;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(amounts)
+        KV_SERIALIZE_OPT(from_height, (uint64_t)0)
+        KV_SERIALIZE_OPT(cumulative, false)
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct distribution
+    {
+      uint64_t amount;
+      uint64_t start_height;
+      std::vector<uint64_t> distribution;
+      uint64_t base;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(amount)
+        KV_SERIALIZE(start_height)
+        KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution)
+        KV_SERIALIZE(base)
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct response
+    {
+      std::string status;
+      std::vector<distribution> distributions;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(status)
+        KV_SERIALIZE(distributions)
+      END_KV_SERIALIZE_MAP()
+    };
+  };
+
 }

src/simplewallet/simplewallet.cpp

@@ -1294,6 +1294,275 @@ bool simple_wallet::export_raw_multisig(const std::vector<std::string> &args)
   return true;
 }
 
+bool simple_wallet::print_ring(const std::vector<std::string> &args)
+{
+  crypto::key_image key_image;
+  crypto::hash txid;
+  if (args.size() != 1)
+  {
+    fail_msg_writer() << tr("usage: print_ring <key_image|txid>");
+    return true;
+  }
+
+  if (!epee::string_tools::hex_to_pod(args[0], key_image))
+  {
+    fail_msg_writer() << tr("Invalid key image");
+    return true;
+  }
+  // this one will always work, they're all 32 byte hex
+  if (!epee::string_tools::hex_to_pod(args[0], txid))
+  {
+    fail_msg_writer() << tr("Invalid txid");
+    return true;
+  }
+
+  std::vector<uint64_t> ring;
+  std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> rings;
+  try
+  {
+    if (m_wallet->get_ring(key_image, ring))
+      rings.push_back({key_image, ring});
+    else if (!m_wallet->get_rings(txid, rings))
+    {
+      fail_msg_writer() << tr("Key image either not spent, or spent with mixin 0");
+      return true;
+    }
+
+    for (const auto &ring: rings)
+    {
+      std::stringstream str;
+      for (const auto &x: ring.second)
+        str << x<< " ";
+      // do NOT translate this "absolute" below, the lin can be used as input to set_ring
+      success_msg_writer() << epee::string_tools::pod_to_hex(ring.first) <<  " absolute " << str.str();
+    }
+  }
+  catch (const std::exception &e)
+  {
+    fail_msg_writer() << tr("Failed to get key image ring: ") << e.what();
+  }
+
+  return true;
+}
+
+bool simple_wallet::set_ring(const std::vector<std::string> &args)
+{
+  crypto::key_image key_image;
+  if (args.size() < 3)
+  {
+    fail_msg_writer() << tr("usage: set_ring <key_image> absolute|relative <index> [<index>...]");
+    return true;
+  }
+
+  if (!epee::string_tools::hex_to_pod(args[0], key_image))
+  {
+    fail_msg_writer() << tr("Invalid key image");
+    return true;
+  }
+
+  bool relative;
+  if (args[1] == "absolute")
+  {
+    relative = false;
+  }
+  else if (args[1] == "relative")
+  {
+    relative = true;
+  }
+  else
+  {
+    fail_msg_writer() << tr("Missing absolute or relative keyword");
+    return true;
+  }
+
+  std::vector<uint64_t> ring;
+  for (size_t n = 2; n < args.size(); ++n)
+  {
+    ring.resize(ring.size() + 1);
+    if (!string_tools::get_xtype_from_string(ring.back(), args[n]))
+    {
+      fail_msg_writer() << tr("invalid index: must be a strictly positive unsigned integer");
+      return true;
+    }
+    if (relative)
+    {
+      if (ring.size() > 1 && !ring.back())
+      {
+        fail_msg_writer() << tr("invalid index: must be a strictly positive unsigned integer");
+        return true;
+      }
+      uint64_t sum = 0;
+      for (uint64_t out: ring)
+      {
+        if (out > std::numeric_limits<uint64_t>::max() - sum)
+        {
+          fail_msg_writer() << tr("invalid index: indices wrap");
+          return true;
+        }
+        sum += out;
+      }
+    }
+    else
+    {
+      if (ring.size() > 1 && ring[ring.size() - 2] >= ring[ring.size() - 1])
+      {
+        fail_msg_writer() << tr("invalid index: indices should be in strictly ascending order");
+        return true;
+      }
+    }
+  }
+  if (!m_wallet->set_ring(key_image, ring, relative))
+  {
+    fail_msg_writer() << tr("failed to set ring");
+    return true;
+  }
+
+  return true;
+}
+
+bool simple_wallet::blackball(const std::vector<std::string> &args)
+{
+  crypto::public_key output;
+  if (args.size() == 0)
+  {
+    fail_msg_writer() << tr("usage: blackball <output_public_key> | <filename> [add]");
+    return true;
+  }
+
+  try
+  {
+    if (epee::string_tools::hex_to_pod(args[0], output))
+    {
+      m_wallet->blackball_output(output);
+    }
+    else if (epee::file_io_utils::is_file_exist(args[0]))
+    {
+      std::vector<crypto::public_key> outputs;
+      char str[65];
+
+      std::unique_ptr<FILE, tools::close_file> f(fopen(args[0].c_str(), "r"));
+      if (f)
+      {
+        while (!feof(f.get()))
+        {
+          if (!fgets(str, sizeof(str), f.get()))
+            break;
+          const size_t len = strlen(str);
+          if (len > 0 && str[len - 1] == '\n')
+            str[len - 1] = 0;
+          if (!str[0])
+            continue;
+          outputs.push_back(crypto::public_key());
+          if (!epee::string_tools::hex_to_pod(str, outputs.back()))
+          {
+            fail_msg_writer() << tr("Invalid public key: ") << str;
+            return true;
+          }
+        }
+        f.reset();
+        bool add = false;
+        if (args.size() > 1)
+        {
+          if (args[1] != "add")
+          {
+            fail_msg_writer() << tr("Bad argument: ") + args[1] + ": " + tr("should be \"add\"");
+            return true;
+          }
+          add = true;
+        }
+        m_wallet->set_blackballed_outputs(outputs, add);
+      }
+      else
+      {
+        fail_msg_writer() << tr("Failed to open file");
+        return true;
+      }
+    }
+    else
+    {
+      fail_msg_writer() << tr("Invalid public key, and file doesn't exist");
+      return true;
+    }
+  }
+  catch (const std::exception &e)
+  {
+    fail_msg_writer() << tr("Failed to blackball output: ") << e.what();
+  }
+
+  return true;
+}
+
+bool simple_wallet::unblackball(const std::vector<std::string> &args)
+{
+  crypto::public_key output;
+  if (args.size() != 1)
+  {
+    fail_msg_writer() << tr("usage: unblackball <output_public_key>");
+    return true;
+  }
+
+  if (!epee::string_tools::hex_to_pod(args[0], output))
+  {
+    fail_msg_writer() << tr("Invalid public key");
+    return true;
+  }
+
+  try
+  {
+    m_wallet->unblackball_output(output);
+  }
+  catch (const std::exception &e)
+  {
+    fail_msg_writer() << tr("Failed to unblackball output: ") << e.what();
+  }
+
+  return true;
+}
+
+bool simple_wallet::blackballed(const std::vector<std::string> &args)
+{
+  crypto::public_key output;
+  if (args.size() != 1)
+  {
+    fail_msg_writer() << tr("usage: blackballed <output_public_key>");
+    return true;
+  }
+
+  if (!epee::string_tools::hex_to_pod(args[0], output))
+  {
+    fail_msg_writer() << tr("Invalid public key");
+    return true;
+  }
+
+  try
+  {
+    if (m_wallet->is_output_blackballed(output))
+      message_writer() << tr("Blackballed: ") << output;
+    else
+      message_writer() << tr("not blackballed: ") << output;
+  }
+  catch (const std::exception &e)
+  {
+    fail_msg_writer() << tr("Failed to unblackball output: ") << e.what();
+  }
+
+  return true;
+}
+
+bool simple_wallet::save_known_rings(const std::vector<std::string> &args)
+{
+  try
+  {
+    LOCK_IDLE_SCOPE();
+    m_wallet->find_and_save_rings();
+  }
+  catch (const std::exception &e)
+  {
+    fail_msg_writer() << tr("Failed to save known rings: ") << e.what();
+  }
+  return true;
+}
+
 bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
   const auto pwd_container = get_and_verify_password();
@@ -1638,6 +1907,32 @@ bool simple_wallet::set_auto_low_priority(const std::vector<std::string> &args/*
   return true;
 }
 
+bool simple_wallet::set_segregate_pre_fork_outputs(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
+{
+  const auto pwd_container = get_and_verify_password();
+  if (pwd_container)
+  {
+    parse_bool_and_use(args[1], [&](bool r) {
+      m_wallet->segregate_pre_fork_outputs(r);
+      m_wallet->rewrite(m_wallet_file, pwd_container->password());
+    });
+  }
+  return true;
+}
+
+bool simple_wallet::set_key_reuse_mitigation2(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
+{
+  const auto pwd_container = get_and_verify_password();
+  if (pwd_container)
+  {
+    parse_bool_and_use(args[1], [&](bool r) {
+      m_wallet->key_reuse_mitigation2(r);
+      m_wallet->rewrite(m_wallet_file, pwd_container->password());
+    });
+  }
+  return true;
+}
+
 bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
   if(args.empty())
@@ -1812,7 +2107,11 @@ simple_wallet::simple_wallet()
                                   "refresh-from-block-height [n]\n "
                                   "  Set the height before which to ignore blocks.\n "
                                   "auto-low-priority <1|0>\n "
-                                  "  Whether to automatically use the low priority fee level when it's safe to do so."));
+                                  "  Whether to automatically use the low priority fee level when it's safe to do so.\n "
+                                  "segregate-pre-fork-outputs <1|0>\n "
+                                  "  Set this if you intend to spend outputs on both Monero AND a key reusing fork.\n "
+                                  "key-reuse-mitigation2 <1|0>\n "
+                                  "  Set this if you are not sure whether you will spend on a key reusing Monero fork later."));
   m_cmd_binder.set_handler("encrypted_seed",
                            boost::bind(&simple_wallet::encrypted_seed, this, _1),
                            tr("Display the encrypted Electrum-style mnemonic seed."));
@@ -1951,6 +2250,30 @@ simple_wallet::simple_wallet()
                            boost::bind(&simple_wallet::export_raw_multisig, this, _1),
                            tr("export_raw_multisig_tx <filename>"),
                            tr("Export a signed multisig transaction to a file"));
+  m_cmd_binder.set_handler("print_ring",
+                           boost::bind(&simple_wallet::print_ring, this, _1),
+                           tr("print_ring <key_image> | <txid>"),
+                           tr("Print the ring(s) used to spend a given key image or transaction (if the ring size is > 1)"));
+  m_cmd_binder.set_handler("set_ring",
+                           boost::bind(&simple_wallet::set_ring, this, _1),
+                           tr("set_ring <key_image> absolute|relative <index> [<index>...]"),
+                           tr("Set the ring used for a given key image, so it can be reused in a fork"));
+  m_cmd_binder.set_handler("save_known_rings",
+                           boost::bind(&simple_wallet::save_known_rings, this, _1),
+                           tr("save_known_rings"),
+                           tr("Save known rings to the shared rings database"));
+  m_cmd_binder.set_handler("blackball",
+                           boost::bind(&simple_wallet::blackball, this, _1),
+                           tr("blackball <output public key> | <filename> [add]"),
+                           tr("Blackball output(s) so they never get selected as fake outputs in a ring"));
+  m_cmd_binder.set_handler("unblackball",
+                           boost::bind(&simple_wallet::unblackball, this, _1),
+                           tr("unblackball <output public key>"),
+                           tr("Unblackballs an output so it may get selected as a fake output in a ring"));
+  m_cmd_binder.set_handler("blackballed",
+                           boost::bind(&simple_wallet::blackballed, this, _1),
+                           tr("blackballed <output public key>"),
+                           tr("Checks whether an output is blackballed"));
   m_cmd_binder.set_handler("help",
                            boost::bind(&simple_wallet::help, this, _1),
                            tr("help [<command>]"),
@@ -1980,6 +2303,8 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
     success_msg_writer() << "confirm-export-overwrite = " << m_wallet->confirm_export_overwrite();
     success_msg_writer() << "refresh-from-block-height = " << m_wallet->get_refresh_from_block_height();
     success_msg_writer() << "auto-low-priority = " << m_wallet->auto_low_priority();
+    success_msg_writer() << "segregate-pre-fork-outputs = " << m_wallet->segregate_pre_fork_outputs();
+    success_msg_writer() << "key-reuse-mitigation2 = " << m_wallet->key_reuse_mitigation2();
     return true;
   }
   else
@@ -2030,6 +2355,8 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
     CHECK_SIMPLE_VARIABLE("confirm-export-overwrite", set_confirm_export_overwrite, tr("0 or 1"));
     CHECK_SIMPLE_VARIABLE("refresh-from-block-height", set_refresh_from_block_height, tr("block height"));
     CHECK_SIMPLE_VARIABLE("auto-low-priority", set_auto_low_priority, tr("0 or 1"));
+    CHECK_SIMPLE_VARIABLE("segregate-pre-fork-outputs", set_segregate_pre_fork_outputs, tr("0 or 1"));
+    CHECK_SIMPLE_VARIABLE("key-reuse-mitigation2", set_key_reuse_mitigation2, tr("0 or 1"));
   }
   fail_msg_writer() << tr("set: unrecognized argument(s)");
   return true;

src/simplewallet/simplewallet.h

@@ -133,6 +133,8 @@ namespace cryptonote
     bool set_confirm_export_overwrite(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_refresh_from_block_height(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_auto_low_priority(const std::vector<std::string> &args = std::vector<std::string>());
+    bool set_segregate_pre_fork_outputs(const std::vector<std::string> &args = std::vector<std::string>());
+    bool set_key_reuse_mitigation2(const std::vector<std::string> &args = std::vector<std::string>());
     bool help(const std::vector<std::string> &args = std::vector<std::string>());
     bool start_mining(const std::vector<std::string> &args);
     bool stop_mining(const std::vector<std::string> &args);
@@ -208,6 +210,12 @@ namespace cryptonote
     bool sign_multisig(const std::vector<std::string>& args);
     bool submit_multisig(const std::vector<std::string>& args);
     bool export_raw_multisig(const std::vector<std::string>& args);
+    bool print_ring(const std::vector<std::string>& args);
+    bool set_ring(const std::vector<std::string>& args);
+    bool save_known_rings(const std::vector<std::string>& args);
+    bool blackball(const std::vector<std::string>& args);
+    bool unblackball(const std::vector<std::string>& args);
+    bool blackballed(const std::vector<std::string>& args);
 
     uint64_t get_daemon_blockchain_height(std::string& err);
     bool try_connect_to_daemon(bool silent = false, uint32_t* version = nullptr);

src/wallet/CMakeLists.txt

@@ -33,6 +33,7 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 set(wallet_sources
   wallet2.cpp
   wallet_args.cpp
+  ringdb.cpp
   node_rpc_proxy.cpp)
 
 set(wallet_private_headers
@@ -42,6 +43,7 @@ set(wallet_private_headers
   wallet_rpc_server.h
   wallet_rpc_server_commands_defs.h
   wallet_rpc_server_error_codes.h
+  ringdb.h
   node_rpc_proxy.h)
 
 monero_private_headers(wallet
@@ -55,6 +57,7 @@ target_link_libraries(wallet
     common
     cryptonote_core
     mnemonics
+    ${LMDB_LIBRARY}
     ${Boost_CHRONO_LIBRARY}
     ${Boost_SERIALIZATION_LIBRARY}
     ${Boost_FILESYSTEM_LIBRARY}

src/wallet/api/CMakeLists.txt

@@ -69,6 +69,7 @@ target_link_libraries(wallet_api
     common
     cryptonote_core
     mnemonics
+    ${LMDB_LIBRARY}
     ${Boost_CHRONO_LIBRARY}
     ${Boost_SERIALIZATION_LIBRARY}
     ${Boost_FILESYSTEM_LIBRARY}

src/wallet/api/wallet.cpp

@@ -68,6 +68,15 @@ namespace {
     static const int    DEFAULT_REMOTE_NODE_REFRESH_INTERVAL_MILLIS = 1000 * 10;
     // Connection timeout 30 sec
     static const int    DEFAULT_CONNECTION_TIMEOUT_MILLIS = 1000 * 30;
+
+    std::string get_default_ringdb_path()
+    {
+      boost::filesystem::path dir = tools::get_default_data_dir();
+      // remove .bitmonero, replace with .shared-ringdb
+      dir = dir.remove_filename();
+      dir /= ".shared-ringdb";
+      return dir.string();
+    }
 }
 
 struct Wallet2CallbackImpl : public tools::i_wallet2_callback
@@ -590,6 +599,7 @@ bool WalletImpl::open(const std::string &path, const std::string &password)
             // Rebuilding wallet cache, using refresh height from .keys file
             m_rebuildWalletCache = true;
         }
+        m_wallet->set_ring_database(get_default_ringdb_path());
         m_wallet->load(path, password);
 
         m_password = password;
@@ -1777,6 +1787,7 @@ void WalletImpl::doRefresh()
             if (m_history->count() == 0) {
                 m_history->refresh();
             }
+            m_wallet->find_and_save_rings(false);
         } else {
            LOG_PRINT_L3(__FUNCTION__ << ": skipping refresh - daemon is not synced");
         }
@@ -1897,6 +1908,127 @@ bool WalletImpl::useForkRules(uint8_t version, int64_t early_blocks) const
     return m_wallet->use_fork_rules(version,early_blocks);
 }
 
+bool WalletImpl::blackballOutputs(const std::vector<std::string> &pubkeys, bool add)
+{
+    std::vector<crypto::public_key> raw_pubkeys;
+    raw_pubkeys.reserve(pubkeys.size());
+    for (const std::string &str: pubkeys)
+    {
+        crypto::public_key pkey;
+        if (!epee::string_tools::hex_to_pod(str, pkey))
+        {
+            m_status = Status_Error;
+            m_errorString = tr("Failed to parse output public key");
+            return false;
+        }
+        raw_pubkeys.push_back(pkey);
+    }
+    bool ret = m_wallet->set_blackballed_outputs(raw_pubkeys, add);
+    if (!ret)
+    {
+        m_status = Status_Error;
+        m_errorString = tr("Failed to set blackballed outputs");
+        return false;
+    }
+    return true;
+}
+
+bool WalletImpl::unblackballOutput(const std::string &pubkey)
+{
+    crypto::public_key raw_pubkey;
+    if (!epee::string_tools::hex_to_pod(pubkey, raw_pubkey))
+    {
+        m_status = Status_Error;
+        m_errorString = tr("Failed to parse output public key");
+        return false;
+    }
+    bool ret = m_wallet->unblackball_output(raw_pubkey);
+    if (!ret)
+    {
+        m_status = Status_Error;
+        m_errorString = tr("Failed to unblackball output");
+        return false;
+    }
+    return true;
+}
+
+bool WalletImpl::getRing(const std::string &key_image, std::vector<uint64_t> &ring) const
+{
+    crypto::key_image raw_key_image;
+    if (!epee::string_tools::hex_to_pod(key_image, raw_key_image))
+    {
+        m_status = Status_Error;
+        m_errorString = tr("Failed to parse key image");
+        return false;
+    }
+    bool ret = m_wallet->get_ring(raw_key_image, ring);
+    if (!ret)
+    {
+        m_status = Status_Error;
+        m_errorString = tr("Failed to get ring");
+        return false;
+    }
+    return true;
+}
+
+bool WalletImpl::getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const
+{
+    crypto::hash raw_txid;
+    if (!epee::string_tools::hex_to_pod(txid, raw_txid))
+    {
+        m_status = Status_Error;
+        m_errorString = tr("Failed to parse txid");
+        return false;
+    }
+    std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> raw_rings;
+    bool ret = m_wallet->get_rings(raw_txid, raw_rings);
+    if (!ret)
+    {
+        m_status = Status_Error;
+        m_errorString = tr("Failed to get rings");
+        return false;
+    }
+    for (const auto &r: raw_rings)
+    {
+      rings.push_back(std::make_pair(epee::string_tools::pod_to_hex(r.first), r.second));
+    }
+    return true;
+}
+
+bool WalletImpl::setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative)
+{
+    crypto::key_image raw_key_image;
+    if (!epee::string_tools::hex_to_pod(key_image, raw_key_image))
+    {
+        m_status = Status_Error;
+        m_errorString = tr("Failed to parse key image");
+        return false;
+    }
+    bool ret = m_wallet->set_ring(raw_key_image, ring, relative);
+    if (!ret)
+    {
+        m_status = Status_Error;
+        m_errorString = tr("Failed to set ring");
+        return false;
+    }
+    return true;
+}
+
+void WalletImpl::segregatePreForkOutputs(bool segregate)
+{
+    m_wallet->segregate_pre_fork_outputs(segregate);
+}
+
+void WalletImpl::segregationHeight(uint64_t height)
+{
+    m_wallet->segregation_height(height);
+}
+
+void WalletImpl::keyReuseMitigation2(bool mitigation)
+{
+    m_wallet->key_reuse_mitigation2(mitigation);
+}
+
 } // namespace
 
 namespace Bitmonero = Monero;

src/wallet/api/wallet.h

@@ -163,6 +163,14 @@ public:
     virtual std::string getDefaultDataDir() const;
     virtual bool lightWalletLogin(bool &isNewWallet) const;
     virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status);
+    virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add);
+    virtual bool unblackballOutput(const std::string &pubkey);
+    virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const;
+    virtual bool getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const;
+    virtual bool setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative);
+    virtual void segregatePreForkOutputs(bool segregate);
+    virtual void segregationHeight(uint64_t height);
+    virtual void keyReuseMitigation2(bool mitigation);
 
 private:
     void clearStatus() const;

src/wallet/api/wallet2_api.h

@@ -33,6 +33,7 @@
 
 #include <string>
 #include <vector>
+#include <list>
 #include <set>
 #include <ctime>
 #include <iostream>
@@ -756,6 +757,30 @@ struct Wallet
     */
     virtual bool rescanSpent() = 0;
     
+    //! blackballs a set of outputs
+    virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add) = 0;
+
+    //! unblackballs an output
+    virtual bool unblackballOutput(const std::string &pubkey) = 0;
+
+    //! gets the ring used for a key image, if any
+    virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const = 0;
+
+    //! gets the rings used for a txid, if any
+    virtual bool getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const = 0;
+
+    //! sets the ring used for a key image
+    virtual bool setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative) = 0;
+
+    //! sets whether pre-fork outs are to be segregated
+    virtual void segregatePreForkOutputs(bool segregate) = 0;
+
+    //! sets the height where segregation should occur
+    virtual void segregationHeight(uint64_t height) = 0;
+
+    //! secondary key reuse mitigation
+    virtual void keyReuseMitigation2(bool mitigation) = 0;
+
     //! Light wallet authenticate and login
     virtual bool lightWalletLogin(bool &isNewWallet) const = 0;
     

src/wallet/ringdb.cpp

@@ -0,0 +1,445 @@
+// Copyright (c) 2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <lmdb.h>
+#include <boost/algorithm/string.hpp>
+#include <boost/range/adaptor/transformed.hpp>
+#include <boost/filesystem.hpp>
+#include "misc_log_ex.h"
+#include "misc_language.h"
+#include "wallet_errors.h"
+#include "ringdb.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "wallet.ringdb"
+
+static const char zerokey[8] = {0};
+static const MDB_val zerokeyval = { sizeof(zerokey), (void *)zerokey };
+
+static int compare_hash32(const MDB_val *a, const MDB_val *b)
+{
+  uint32_t *va = (uint32_t*) a->mv_data;
+  uint32_t *vb = (uint32_t*) b->mv_data;
+  for (int n = 7; n >= 0; n--)
+  {
+    if (va[n] == vb[n])
+      continue;
+    return va[n] < vb[n] ? -1 : 1;
+  }
+
+  return 0;
+}
+
+static std::string compress_ring(const std::vector<uint64_t> &ring)
+{
+  std::string s;
+  for (uint64_t out: ring)
+    s += tools::get_varint_data(out);
+  return s;
+}
+
+static std::vector<uint64_t> decompress_ring(const std::string &s)
+{
+  std::vector<uint64_t> ring;
+  int read = 0;
+  for (std::string::const_iterator i = s.begin(); i != s.cend(); std::advance(i, read))
+  {
+    uint64_t out;
+    std::string tmp(i, s.cend());
+    read = tools::read_varint(tmp.begin(), tmp.end(), out);
+    THROW_WALLET_EXCEPTION_IF(read <= 0 || read > 256, tools::error::wallet_internal_error, "Internal error decompressing ring");
+    ring.push_back(out);
+  }
+  return ring;
+}
+
+std::string get_rings_filename(boost::filesystem::path filename)
+{
+  if (!boost::filesystem::is_directory(filename))
+    filename.remove_filename();
+  return filename.string();
+}
+
+static crypto::chacha_iv make_iv(const crypto::key_image &key_image, const crypto::chacha_key &key)
+{
+  static const char salt[] = "ringdsb";
+
+  uint8_t buffer[sizeof(key_image) + sizeof(key) + sizeof(salt)];
+  memcpy(buffer, &key_image, sizeof(key_image));
+  memcpy(buffer + sizeof(key_image), &key, sizeof(key));
+  memcpy(buffer + sizeof(key_image) + sizeof(key), salt, sizeof(salt));
+  crypto::hash hash;
+  crypto::cn_fast_hash(buffer, sizeof(buffer), hash.data);
+  static_assert(sizeof(hash) >= CHACHA_IV_SIZE, "Incompatible hash and chacha IV sizes");
+  crypto::chacha_iv iv;
+  memcpy(&iv, &hash, CHACHA_IV_SIZE);
+  return iv;
+}
+
+static std::string encrypt(const std::string &plaintext, const crypto::key_image &key_image, const crypto::chacha_key &key)
+{
+  const crypto::chacha_iv iv = make_iv(key_image, key);
+  std::string ciphertext;
+  ciphertext.resize(plaintext.size() + sizeof(iv));
+  crypto::chacha20(plaintext.data(), plaintext.size(), key, iv, &ciphertext[sizeof(iv)]);
+  memcpy(&ciphertext[0], &iv, sizeof(iv));
+  return ciphertext;
+}
+
+static std::string encrypt(const crypto::key_image &key_image, const crypto::chacha_key &key)
+{
+  return encrypt(std::string((const char*)&key_image, sizeof(key_image)), key_image, key);
+}
+
+static std::string decrypt(const std::string &ciphertext, const crypto::key_image &key_image, const crypto::chacha_key &key)
+{
+  const crypto::chacha_iv iv = make_iv(key_image, key);
+  std::string plaintext;
+  THROW_WALLET_EXCEPTION_IF(ciphertext.size() < sizeof(iv), tools::error::wallet_internal_error, "Bad ciphertext text");
+  plaintext.resize(ciphertext.size() - sizeof(iv));
+  crypto::chacha20(ciphertext.data() + sizeof(iv), ciphertext.size() - sizeof(iv), key, iv, &plaintext[0]);
+  return plaintext;
+}
+
+static void store_relative_ring(MDB_txn *txn, MDB_dbi &dbi, const crypto::key_image &key_image, const std::vector<uint64_t> &relative_ring, const crypto::chacha_key &chacha_key)
+{
+  MDB_val key, data;
+  std::string key_ciphertext = encrypt(key_image, chacha_key);
+  key.mv_data = (void*)key_ciphertext.data();
+  key.mv_size = key_ciphertext.size();
+  std::string compressed_ring = compress_ring(relative_ring);
+  std::string data_ciphertext = encrypt(compressed_ring, key_image, chacha_key);
+  data.mv_size = data_ciphertext.size();
+  data.mv_data = (void*)data_ciphertext.c_str();
+  int dbr = mdb_put(txn, dbi, &key, &data, 0);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set ring for key image in LMDB table: " + std::string(mdb_strerror(dbr)));
+}
+
+static int resize_env(MDB_env *env, const char *db_path, size_t needed)
+{
+  MDB_envinfo mei;
+  MDB_stat mst;
+  int ret;
+
+  needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB
+
+  ret = mdb_env_info(env, &mei);
+  if (ret)
+    return ret;
+  ret = mdb_env_stat(env, &mst);
+  if (ret)
+    return ret;
+  uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
+  uint64_t mapsize = mei.me_mapsize;
+  if (size_used + needed > mei.me_mapsize)
+  {
+    try
+    {
+      boost::filesystem::path path(db_path);
+      boost::filesystem::space_info si = boost::filesystem::space(path);
+      if(si.available < needed)
+      {
+        MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available >> 20L) << " MB available");
+        return ENOSPC;
+      }
+    }
+    catch(...)
+    {
+      // print something but proceed.
+      MWARNING("Unable to query free disk space.");
+    }
+
+    mapsize += needed;
+  }
+  return mdb_env_set_mapsize(env, mapsize);
+}
+
+static size_t get_ring_data_size(size_t n_entries)
+{
+  return n_entries * (32 + 1024); // highball 1kB for the ring data to make sure
+}
+
+enum { BLACKBALL_BLACKBALL, BLACKBALL_UNBLACKBALL, BLACKBALL_QUERY, BLACKBALL_CLEAR};
+
+namespace tools
+{
+
+ringdb::ringdb(std::string filename, const std::string &genesis):
+  filename(filename)
+{
+  MDB_txn *txn;
+  bool tx_active = false;
+  int dbr;
+
+  tools::create_directories_if_necessary(filename);
+
+  dbr = mdb_env_create(&env);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_env_set_maxdbs(env, 2);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
+  const std::string actual_filename = get_rings_filename(filename); 
+  dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open rings database file '"
+      + actual_filename + "': " + std::string(mdb_strerror(dbr)));
+
+  dbr = mdb_txn_begin(env, NULL, 0, &txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
+  tx_active = true;
+
+  dbr = mdb_dbi_open(txn, ("rings-" + genesis).c_str(), MDB_CREATE, &dbi_rings);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  mdb_set_compare(txn, dbi_rings, compare_hash32);
+
+  dbr = mdb_dbi_open(txn, ("blackballs-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  mdb_set_dupsort(txn, dbi_blackballs, compare_hash32);
+
+  dbr = mdb_txn_commit(txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr)));
+  tx_active = false;
+}
+
+ringdb::~ringdb()
+{
+  mdb_dbi_close(env, dbi_rings);
+  mdb_dbi_close(env, dbi_blackballs);
+  mdb_env_close(env);
+}
+
+bool ringdb::add_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx)
+{
+  MDB_txn *txn;
+  int dbr;
+  bool tx_active = false;
+
+  dbr = resize_env(env, filename.c_str(), get_ring_data_size(tx.vin.size()));
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size");
+  dbr = mdb_txn_begin(env, NULL, 0, &txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
+  tx_active = true;
+
+  for (const auto &in: tx.vin)
+  {
+    if (in.type() != typeid(cryptonote::txin_to_key))
+      continue;
+    const auto &txin = boost::get<cryptonote::txin_to_key>(in);
+    const uint32_t ring_size = txin.key_offsets.size();
+    if (ring_size == 1)
+      continue;
+
+    store_relative_ring(txn, dbi_rings, txin.k_image, txin.key_offsets, chacha_key);
+  }
+
+  dbr = mdb_txn_commit(txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn adding ring to database: " + std::string(mdb_strerror(dbr)));
+  tx_active = false;
+  return true;
+}
+
+bool ringdb::remove_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx)
+{
+  MDB_txn *txn;
+  int dbr;
+  bool tx_active = false;
+
+  dbr = resize_env(env, filename.c_str(), 0);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size");
+  dbr = mdb_txn_begin(env, NULL, 0, &txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
+  tx_active = true;
+
+  for (const auto &in: tx.vin)
+  {
+    if (in.type() != typeid(cryptonote::txin_to_key))
+      continue;
+    const auto &txin = boost::get<cryptonote::txin_to_key>(in);
+    const uint32_t ring_size = txin.key_offsets.size();
+    if (ring_size == 1)
+      continue;
+
+    MDB_val key, data;
+    std::string key_ciphertext = encrypt(txin.k_image, chacha_key);
+    key.mv_data = (void*)key_ciphertext.data();
+    key.mv_size = key_ciphertext.size();
+
+    dbr = mdb_get(txn, dbi_rings, &key, &data);
+    THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to look for key image in LMDB table: " + std::string(mdb_strerror(dbr)));
+    if (dbr == MDB_NOTFOUND)
+      continue;
+    THROW_WALLET_EXCEPTION_IF(data.mv_size <= 0, tools::error::wallet_internal_error, "Invalid ring data size");
+
+    MDEBUG("Removing ring data for key image " << txin.k_image);
+    dbr = mdb_del(txn, dbi_rings, &key, NULL);
+    THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to remove ring to database: " + std::string(mdb_strerror(dbr)));
+  }
+
+  dbr = mdb_txn_commit(txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn removing ring to database: " + std::string(mdb_strerror(dbr)));
+  tx_active = false;
+  return true;
+}
+
+bool ringdb::get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs)
+{
+  MDB_txn *txn;
+  int dbr;
+  bool tx_active = false;
+
+  dbr = resize_env(env, filename.c_str(), 0);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_txn_begin(env, NULL, 0, &txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
+  tx_active = true;
+
+  MDB_val key, data;
+  std::string key_ciphertext = encrypt(key_image, chacha_key);
+  key.mv_data = (void*)key_ciphertext.data();
+  key.mv_size = key_ciphertext.size();
+  dbr = mdb_get(txn, dbi_rings, &key, &data);
+  THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to look for key image in LMDB table: " + std::string(mdb_strerror(dbr)));
+  if (dbr == MDB_NOTFOUND)
+    return false;
+  THROW_WALLET_EXCEPTION_IF(data.mv_size <= 0, tools::error::wallet_internal_error, "Invalid ring data size");
+
+  std::string data_plaintext = decrypt(std::string((const char*)data.mv_data, data.mv_size), key_image, chacha_key);
+  outs = decompress_ring(data_plaintext);
+  MDEBUG("Found ring for key image " << key_image << ":");
+  MDEBUG("Relative: " << boost::join(outs | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " "));
+  outs = cryptonote::relative_output_offsets_to_absolute(outs);
+  MDEBUG("Absolute: " << boost::join(outs | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " "));
+
+  dbr = mdb_txn_commit(txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn getting ring from database: " + std::string(mdb_strerror(dbr)));
+  tx_active = false;
+  return true;
+}
+
+bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative)
+{
+  MDB_txn *txn;
+  int dbr;
+  bool tx_active = false;
+
+  dbr = resize_env(env, filename.c_str(), outs.size() * 64);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_txn_begin(env, NULL, 0, &txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
+  tx_active = true;
+
+  store_relative_ring(txn, dbi_rings, key_image, relative ? outs : cryptonote::absolute_output_offsets_to_relative(outs), chacha_key);
+
+  dbr = mdb_txn_commit(txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn setting ring to database: " + std::string(mdb_strerror(dbr)));
+  tx_active = false;
+  return true;
+}
+
+bool ringdb::blackball_worker(const crypto::public_key &output, int op)
+{
+  MDB_txn *txn;
+  MDB_cursor *cursor;
+  int dbr;
+  bool tx_active = false;
+  bool ret = true;
+
+  dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_txn_begin(env, NULL, 0, &txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
+  tx_active = true;
+
+  MDB_val key = zerokeyval;
+  MDB_val data;
+  data.mv_data = (void*)&output;
+  data.mv_size = sizeof(output);
+
+  switch (op)
+  {
+    case BLACKBALL_BLACKBALL:
+      MDEBUG("Blackballing output " << output);
+      dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA);
+      if (dbr == MDB_KEYEXIST)
+        dbr = 0;
+      break;
+    case BLACKBALL_UNBLACKBALL:
+      MDEBUG("Unblackballing output " << output);
+      dbr = mdb_del(txn, dbi_blackballs, &key, &data);
+      if (dbr == MDB_NOTFOUND)
+        dbr = 0;
+      break;
+    case BLACKBALL_QUERY:
+      dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor);
+      THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr)));
+      dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH);
+      THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr)));
+      ret = dbr != MDB_NOTFOUND;
+      if (dbr == MDB_NOTFOUND)
+        dbr = 0;
+      mdb_cursor_close(cursor);
+      break;
+    case BLACKBALL_CLEAR:
+      dbr = mdb_drop(txn, dbi_blackballs, 0);
+      break;
+    default:
+      THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op");
+  }
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr)));
+
+  dbr = mdb_txn_commit(txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr)));
+  tx_active = false;
+  return ret;
+}
+
+bool ringdb::blackball(const crypto::public_key &output)
+{
+  return blackball_worker(output, BLACKBALL_BLACKBALL);
+}
+
+bool ringdb::unblackball(const crypto::public_key &output)
+{
+  return blackball_worker(output, BLACKBALL_UNBLACKBALL);
+}
+
+bool ringdb::blackballed(const crypto::public_key &output)
+{
+  return blackball_worker(output, BLACKBALL_QUERY);
+}
+
+bool ringdb::clear_blackballs()
+{
+  return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR);
+}
+
+}

src/wallet/ringdb.h

@@ -0,0 +1,65 @@
+// Copyright (c) 2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <lmdb.h>
+#include "wipeable_string.h"
+#include "crypto/crypto.h"
+#include "cryptonote_basic/cryptonote_basic.h"
+
+namespace tools
+{
+  class ringdb
+  {
+  public:
+    ringdb(std::string filename, const std::string &genesis);
+    ~ringdb();
+
+    bool add_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx);
+    bool remove_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx);
+    bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs);
+    bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative);
+
+    bool blackball(const crypto::public_key &output);
+    bool unblackball(const crypto::public_key &output);
+    bool blackballed(const crypto::public_key &output);
+    bool clear_blackballs();
+
+  private:
+    bool blackball_worker(const crypto::public_key &output, int op);
+
+  private:
+    std::string filename;
+    MDB_env *env;
+    MDB_dbi dbi_rings;
+    MDB_dbi dbi_blackballs;
+  };
+}

src/wallet/wallet2.cpp

@@ -66,6 +66,7 @@ using namespace epee;
 #include "memwipe.h"
 #include "common/base58.h"
 #include "ringct/rctSigs.h"
+#include "ringdb.h"
 
 extern "C"
 {
@@ -93,7 +94,9 @@ using namespace cryptonote;
 #define MULTISIG_UNSIGNED_TX_PREFIX "Monero multisig unsigned tx set\001"
 
 #define RECENT_OUTPUT_RATIO (0.5) // 50% of outputs are from the recent zone
-#define RECENT_OUTPUT_ZONE ((time_t)(1.8 * 86400)) // last 1.8 day makes up the recent zone (taken from monerolink.pdf, Miller et al)
+#define RECENT_OUTPUT_DAYS (1.8) // last 1.8 day makes up the recent zone (taken from monerolink.pdf, Miller et al)
+#define RECENT_OUTPUT_ZONE ((time_t)(RECENT_OUTPUT_DAYS * 86400))
+#define RECENT_OUTPUT_BLOCKS (RECENT_OUTPUT_DAYS * 720)
 
 #define FEE_ESTIMATE_GRACE_BLOCKS 10 // estimate fee valid for that many blocks
 
@@ -106,6 +109,24 @@ using namespace cryptonote;
 
 #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001"
 
+#define SEGREGATION_FORK_HEIGHT 1564965
+#define TESTNET_SEGREGATION_FORK_HEIGHT 1000000
+#define STAGENET_SEGREGATION_FORK_HEIGHT 1000000
+#define SEGREGATION_FORK_VICINITY 1500 /* blocks */
+
+
+namespace
+{
+  std::string get_default_ringdb_path()
+  {
+    boost::filesystem::path dir = tools::get_default_data_dir();
+    // remove .bitmonero, replace with .shared-ringdb
+    dir = dir.remove_filename();
+    dir /= ".shared-ringdb";
+    return dir.string();
+  }
+}
+
 namespace
 {
 // Create on-demand to prevent static initialization order fiasco issues.
@@ -119,6 +140,16 @@ struct options {
   const command_line::arg_descriptor<bool> testnet = {"testnet", tools::wallet2::tr("For testnet. Daemon must also be launched with --testnet flag"), false};
   const command_line::arg_descriptor<bool> stagenet = {"stagenet", tools::wallet2::tr("For stagenet. Daemon must also be launched with --stagenet flag"), false};
   const command_line::arg_descriptor<bool> restricted = {"restricted-rpc", tools::wallet2::tr("Restricts to view-only commands"), false};
+  const command_line::arg_descriptor<std::string, false, true> shared_ringdb_dir = {
+    "shared-ringdb-dir", tools::wallet2::tr("Set shared ring database path"),
+    get_default_ringdb_path(),
+    testnet,
+    [](bool testnet, bool defaulted, std::string val) {
+      if (testnet)
+        return (boost::filesystem::path(val) / "testnet").string();
+      return val;
+    }
+  };
 };
 
 void do_prepare_file_names(const std::string& file_path, std::string& keys_file, std::string& wallet_file)
@@ -196,6 +227,8 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl
 
   std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(testnet ? TESTNET : stagenet ? STAGENET : MAINNET, restricted));
   wallet->init(std::move(daemon_address), std::move(login));
+  boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir);
+  wallet->set_ring_database(ringdb_path.string());
   return wallet;
 }
 
@@ -627,6 +660,8 @@ wallet2::wallet2(network_type nettype, bool restricted):
   m_confirm_backlog_threshold(0),
   m_confirm_export_overwrite(true),
   m_auto_low_priority(true),
+  m_segregate_pre_fork_outputs(true),
+  m_key_reuse_mitigation2(true),
   m_is_initialized(false),
   m_restricted(restricted),
   is_old_file_format(false),
@@ -639,7 +674,13 @@ wallet2::wallet2(network_type nettype, bool restricted):
   m_light_wallet_connected(false),
   m_light_wallet_balance(0),
   m_light_wallet_unlocked_balance(0),
-  m_key_on_device(false)
+  m_key_on_device(false),
+  m_ring_history_saved(false),
+  m_ringdb()
+{
+}
+
+wallet2::~wallet2()
 {
 }
 
@@ -665,6 +706,7 @@ void wallet2::init_options(boost::program_options::options_description& desc_par
   command_line::add_arg(desc_params, opts.testnet);
   command_line::add_arg(desc_params, opts.stagenet);
   command_line::add_arg(desc_params, opts.restricted);
+  command_line::add_arg(desc_params, opts.shared_ringdb_dir);
 }
 
 std::unique_ptr<wallet2> wallet2::make_from_json(const boost::program_options::variables_map& vm, const std::string& json_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter)
@@ -1479,9 +1521,19 @@ void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::trans
     entry.first->second.m_subaddr_account = subaddr_account;
     entry.first->second.m_subaddr_indices = subaddr_indices;
   }
+
+  for (const auto &in: tx.vin)
+  {
+    if (in.type() != typeid(cryptonote::txin_to_key))
+      continue;
+    const auto &txin = boost::get<cryptonote::txin_to_key>(in);
+    entry.first->second.m_rings.push_back(std::make_pair(txin.k_image, txin.key_offsets));
+  }
   entry.first->second.m_block_height = height;
   entry.first->second.m_timestamp = ts;
   entry.first->second.m_unlock_time = tx.unlock_time;
+
+  add_rings(tx);
 }
 //----------------------------------------------------------------------------------------------------
 void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices)
@@ -1852,6 +1904,7 @@ void wallet2::update_pool_state(bool refreshed)
         pit->second.m_state = wallet2::unconfirmed_transfer_details::failed;
 
         // the inputs aren't spent anymore, since the tx failed
+        remove_rings(pit->second.m_tx);
         for (size_t vini = 0; vini < pit->second.m_tx.vin.size(); ++vini)
         {
           if (pit->second.m_tx.vin[vini].type() == typeid(txin_to_key))
@@ -2266,6 +2319,73 @@ bool wallet2::refresh(uint64_t & blocks_fetched, bool& received_money, bool& ok)
   return ok;
 }
 //----------------------------------------------------------------------------------------------------
+bool wallet2::get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution)
+{
+  uint32_t rpc_version;
+  boost::optional<std::string> result = m_node_rpc_proxy.get_rpc_version(rpc_version);
+  // no error
+  if (!!result)
+  {
+    // empty string -> not connection
+    THROW_WALLET_EXCEPTION_IF(result->empty(), tools::error::no_connection_to_daemon, "getversion");
+    THROW_WALLET_EXCEPTION_IF(*result == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "getversion");
+    if (*result != CORE_RPC_STATUS_OK)
+    {
+      MDEBUG("Cannot determine daemon RPC version, not requesting rct distribution");
+      return false;
+    }
+  }
+  else
+  {
+    if (rpc_version >= MAKE_CORE_RPC_VERSION(1, 19))
+    {
+      MDEBUG("Daemon is recent enough, requesting rct distribution");
+    }
+    else
+    {
+      MDEBUG("Daemon is too old, not requesting rct distribution");
+      return false;
+    }
+  }
+
+  cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request req = AUTO_VAL_INIT(req);
+  cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response res = AUTO_VAL_INIT(res);
+  req.amounts.push_back(0);
+  req.from_height = 0;
+  req.cumulative = true;
+  m_daemon_rpc_mutex.lock();
+  bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req, res, m_http_client, rpc_timeout);
+  m_daemon_rpc_mutex.unlock();
+  if (!r)
+  {
+    MWARNING("Failed to request output distribution: no connection to daemon");
+    return false;
+  }
+  if (res.status == CORE_RPC_STATUS_BUSY)
+  {
+    MWARNING("Failed to request output distribution: daemon is busy");
+    return false;
+  }
+  if (res.status != CORE_RPC_STATUS_OK)
+  {
+    MWARNING("Failed to request output distribution: " << res.status);
+    return false;
+  }
+  if (res.distributions.size() != 1)
+  {
+    MWARNING("Failed to request output distribution: not the expected single result");
+    return false;
+  }
+  if (res.distributions[0].amount != 0)
+  {
+    MWARNING("Failed to request output distribution: results are not for amount 0");
+    return false;
+  }
+  start_height = res.distributions[0].start_height;
+  distribution = std::move(res.distributions[0].distribution);
+  return true;
+}
+//----------------------------------------------------------------------------------------------------
 void wallet2::detach_blockchain(uint64_t height)
 {
   LOG_PRINT_L0("Detaching blockchain on height " << height);
@@ -2468,6 +2588,12 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable
   value2.SetUint(m_nettype);
   json.AddMember("nettype", value2, json.GetAllocator());
 
+  value2.SetInt(m_segregate_pre_fork_outputs ? 1 : 0);
+  json.AddMember("segregate_pre_fork_outputs", value2, json.GetAllocator());
+
+  value2.SetInt(m_key_reuse_mitigation2 ? 1 : 0);
+  json.AddMember("key_reuse_mitigation2", value2, json.GetAllocator());
+
   // Serialize the JSON object
   rapidjson::StringBuffer buffer;
   rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
@@ -3677,6 +3803,15 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass
     add_subaddress_account(tr("Primary account"));
 
   m_local_bc_height = m_blockchain.size();
+
+  try
+  {
+    find_and_save_rings(false);
+  }
+  catch (const std::exception &e)
+  {
+    MERROR("Failed to save rings, will try again next time");
+  }
 }
 //----------------------------------------------------------------------------------------------------
 void wallet2::trim_hashchain()
@@ -4263,6 +4398,13 @@ void wallet2::add_unconfirmed_tx(const cryptonote::transaction& tx, uint64_t amo
   utd.m_timestamp = time(NULL);
   utd.m_subaddr_account = subaddr_account;
   utd.m_subaddr_indices = subaddr_indices;
+  for (const auto &in: tx.vin)
+  {
+    if (in.type() != typeid(cryptonote::txin_to_key))
+      continue;
+    const auto &txin = boost::get<cryptonote::txin_to_key>(in);
+    utd.m_rings.push_back(std::make_pair(txin.k_image, txin.key_offsets));
+  }
 }
 
 //----------------------------------------------------------------------------------------------------
@@ -5320,17 +5462,195 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<crypto
   }
 }
 
+void wallet2::set_ring_database(const std::string &filename)
+{
+  m_ring_database = filename;
+  MINFO("ringdb path set to " << filename);
+  m_ringdb.reset();
+  cryptonote::block b;
+  generate_genesis(b);
+  if (!m_ring_database.empty())
+    m_ringdb.reset(new tools::ringdb(m_ring_database, epee::string_tools::pod_to_hex(get_block_hash(b))));
+}
 
-bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const
+bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx)
+{
+  if (!m_ringdb)
+    return true;
+  return m_ringdb->add_rings(key, tx);
+}
+
+bool wallet2::add_rings(const cryptonote::transaction_prefix &tx)
+{
+  crypto::chacha_key key;
+  generate_chacha_key_from_secret_keys(key);
+  return add_rings(key, tx);
+}
+
+bool wallet2::remove_rings(const cryptonote::transaction_prefix &tx)
+{
+  if (!m_ringdb)
+    return true;
+  crypto::chacha_key key;
+  generate_chacha_key_from_secret_keys(key);
+  return m_ringdb->remove_rings(key, tx);
+}
+
+bool wallet2::get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs)
+{
+  if (!m_ringdb)
+    return true;
+  return m_ringdb->get_ring(key, key_image, outs);
+}
+
+bool wallet2::get_rings(const crypto::hash &txid, std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> &outs)
+{
+  for (auto i: m_confirmed_txs)
+  {
+    if (txid == i.first)
+    {
+      for (const auto &x: i.second.m_rings)
+        outs.push_back({x.first, cryptonote::relative_output_offsets_to_absolute(x.second)});
+      return true;
+    }
+  }
+  for (auto i: m_unconfirmed_txs)
+  {
+    if (txid == i.first)
+    {
+      for (const auto &x: i.second.m_rings)
+        outs.push_back({x.first, cryptonote::relative_output_offsets_to_absolute(x.second)});
+      return true;
+    }
+  }
+  return false;
+}
+
+bool wallet2::get_ring(const crypto::key_image &key_image, std::vector<uint64_t> &outs)
+{
+  crypto::chacha_key key;
+  generate_chacha_key_from_secret_keys(key);
+
+  return get_ring(key, key_image, outs);
+}
+
+bool wallet2::set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative)
+{
+  if (!m_ringdb)
+    return true;
+
+  crypto::chacha_key key;
+  generate_chacha_key_from_secret_keys(key);
+
+  return m_ringdb->set_ring(key, key_image, outs, relative);
+}
+
+bool wallet2::find_and_save_rings(bool force)
+{
+  if (!force && m_ring_history_saved)
+    return true;
+  if (!m_ringdb)
+    return true;
+
+  COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req);
+  COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res);
+
+  MDEBUG("Finding and saving rings...");
+
+  // get payments we made
+  std::list<std::pair<crypto::hash,wallet2::confirmed_transfer_details>> payments;
+  get_payments_out(payments, 0, std::numeric_limits<uint64_t>::max(), boost::none, std::set<uint32_t>());
+  for (const std::pair<crypto::hash,wallet2::confirmed_transfer_details> &entry: payments)
+  {
+    const crypto::hash &txid = entry.first;
+    req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
+  }
+
+  MDEBUG("Found " << std::to_string(req.txs_hashes.size()) << " transactions");
+
+  // get those transactions from the daemon
+  req.decode_as_json = false;
+  bool r;
+  {
+    const boost::lock_guard<boost::mutex> lock{m_daemon_rpc_mutex};
+    r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
+  }
+  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gettransactions");
+  THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions");
+  THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::wallet_internal_error, "gettransactions");
+  THROW_WALLET_EXCEPTION_IF(res.txs.size() != req.txs_hashes.size(), error::wallet_internal_error,
+    "daemon returned wrong response for gettransactions, wrong txs count = " +
+    std::to_string(res.txs.size()) + ", expected " + std::to_string(req.txs_hashes.size()));
+
+  MDEBUG("Scanning " << res.txs.size() << " transactions");
+
+  crypto::chacha_key key;
+  generate_chacha_key_from_secret_keys(key);
+
+  auto it = req.txs_hashes.begin();
+  for (size_t i = 0; i < res.txs.size(); ++i, ++it)
+  {
+    const auto &tx_info = res.txs[i];
+    THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != *it, error::wallet_internal_error, "Wrong txid received");
+    cryptonote::blobdata bd;
+    THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(tx_info.as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr");
+    cryptonote::transaction tx;
+    crypto::hash tx_hash, tx_prefix_hash;
+    THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob");
+    THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch");
+    THROW_WALLET_EXCEPTION_IF(!add_rings(key, tx), error::wallet_internal_error, "Failed to save ring");
+  }
+
+  MINFO("Found and saved rings for " << res.txs.size() << " transactions");
+  m_ring_history_saved = true;
+  return true;
+}
+
+bool wallet2::blackball_output(const crypto::public_key &output)
+{
+  if (!m_ringdb)
+    return true;
+  return m_ringdb->blackball(output);
+}
+
+bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add)
+{
+  if (!m_ringdb)
+    return true;
+  bool ret = true;
+  if (!add)
+    ret &= m_ringdb->clear_blackballs();
+  for (const auto &output: outputs)
+    ret &= m_ringdb->blackball(output);
+  return ret;
+}
+
+bool wallet2::unblackball_output(const crypto::public_key &output)
+{
+  if (!m_ringdb)
+    return true;
+  return m_ringdb->unblackball(output);
+}
+
+bool wallet2::is_output_blackballed(const crypto::public_key &output) const
+{
+  if (!m_ringdb)
+    return true;
+  return m_ringdb->blackballed(output);
+}
+
+bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& output_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const
 {
   if (!unlocked) // don't add locked outs
     return false;
   if (global_index == real_index) // don't re-add real one
     return false;
-  auto item = std::make_tuple(global_index, tx_public_key, mask);
+  auto item = std::make_tuple(global_index, output_public_key, mask);
   CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty");
   if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates
     return false;
+  if (is_output_blackballed(output_public_key)) // don't add blackballed outputs
+    return false;
   outs.back().push_back(item);
   return true;
 }
@@ -5449,8 +5769,25 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
     return;
   }
 
+  crypto::chacha_key key;
+  generate_chacha_key_from_secret_keys(key);
+
   if (fake_outputs_count > 0)
   {
+    uint64_t segregation_fork_height;
+    switch (m_nettype)
+    {
+      case TESTNET: segregation_fork_height = TESTNET_SEGREGATION_FORK_HEIGHT; break;
+      case STAGENET: segregation_fork_height = STAGENET_SEGREGATION_FORK_HEIGHT; break;
+      case MAINNET: segregation_fork_height = SEGREGATION_FORK_HEIGHT; break;
+      default: THROW_WALLET_EXCEPTION(error::wallet_internal_error, "Invalid network type");
+    }
+    // check whether we're shortly after the fork
+    uint64_t height;
+    boost::optional<std::string> result = m_node_rpc_proxy.get_height(height);
+    throw_on_rpc_response_error(result, "get_info");
+    bool is_shortly_after_segregation_fork = height >= segregation_fork_height && height < segregation_fork_height + SEGREGATION_FORK_VICINITY;
+
     // get histogram for the amounts we need
     cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t);
     cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t);
@@ -5468,6 +5805,50 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
     THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram");
     THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status);
 
+    // if we want to segregate fake outs pre or post fork, get distribution
+    std::unordered_map<uint64_t, std::pair<uint64_t, uint64_t>> segregation_limit;
+    if (m_segregate_pre_fork_outputs || m_key_reuse_mitigation2)
+    {
+      cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request req_t = AUTO_VAL_INIT(req_t);
+      cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response resp_t = AUTO_VAL_INIT(resp_t);
+      for(size_t idx: selected_transfers)
+        req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount());
+      std::sort(req_t.amounts.begin(), req_t.amounts.end());
+      auto end = std::unique(req_t.amounts.begin(), req_t.amounts.end());
+      req_t.amounts.resize(std::distance(req_t.amounts.begin(), end));
+      req_t.from_height = segregation_fork_height >= RECENT_OUTPUT_ZONE ? height >= (segregation_fork_height ? segregation_fork_height : height) - RECENT_OUTPUT_BLOCKS : 0;
+      req_t.cumulative = true;
+      m_daemon_rpc_mutex.lock();
+      bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, m_http_client, rpc_timeout);
+      m_daemon_rpc_mutex.unlock();
+      THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected");
+      THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_distribution");
+      THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_output_distribution, resp_t.status);
+
+      // check we got all data
+      for(size_t idx: selected_transfers)
+      {
+        const uint64_t amount = m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount();
+        bool found = false;
+        for (const auto &d: resp_t.distributions)
+        {
+          if (d.amount == amount)
+          {
+            THROW_WALLET_EXCEPTION_IF(d.start_height > segregation_fork_height, error::get_output_distribution, "Distribution start_height too high");
+            THROW_WALLET_EXCEPTION_IF(segregation_fork_height - d.start_height >= d.distribution.size(), error::get_output_distribution, "Distribution size too small");
+            THROW_WALLET_EXCEPTION_IF(segregation_fork_height <= RECENT_OUTPUT_BLOCKS, error::wallet_internal_error, "Fork height too low");
+            THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS < d.start_height, error::get_output_distribution, "Bad start height");
+            uint64_t till_fork = d.distribution[segregation_fork_height - d.start_height];
+            uint64_t recent = till_fork - d.distribution[segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.start_height];
+            segregation_limit[amount] = std::make_pair(till_fork, recent);
+            found = true;
+            break;
+          }
+        }
+        THROW_WALLET_EXCEPTION_IF(!found, error::get_output_distribution, "Requested amount not found in response");
+      }
+    }
+
     // we ask for more, to have spares if some outputs are still locked
     size_t base_requested_outputs_count = (size_t)((fake_outputs_count + 1) * 1.5 + 1);
     LOG_PRINT_L2("base_requested_outputs_count: " << base_requested_outputs_count);
@@ -5487,9 +5868,21 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
       size_t requested_outputs_count = base_requested_outputs_count + (td.is_rct() ? CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE : 0);
       size_t start = req.outputs.size();
 
+      const bool output_is_pre_fork = td.m_block_height < segregation_fork_height;
+      uint64_t num_outs = 0, num_recent_outs = 0;
+      uint64_t num_post_fork_outs = 0;
+      float pre_fork_num_out_ratio = 0.0f;
+      float post_fork_num_out_ratio = 0.0f;
+
+      if (m_segregate_pre_fork_outputs && output_is_pre_fork)
+      {
+        num_outs = segregation_limit[amount].first;
+        num_recent_outs = segregation_limit[amount].second;
+      }
+      else
+      {
         // if there are just enough outputs to mix with, use all of them.
         // Eventually this should become impossible.
-      uint64_t num_outs = 0, num_recent_outs = 0;
         for (const auto &he: resp_t.histogram)
         {
           if (he.amount == amount)
@@ -5501,23 +5894,100 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
             break;
           }
         }
+        if (m_key_reuse_mitigation2)
+        {
+          if (output_is_pre_fork)
+          {
+            if (is_shortly_after_segregation_fork)
+            {
+              pre_fork_num_out_ratio = 33.4/100.0f * (1.0f - RECENT_OUTPUT_RATIO);
+            }
+            else
+            {
+              pre_fork_num_out_ratio = 33.4/100.0f * (1.0f - RECENT_OUTPUT_RATIO);
+              post_fork_num_out_ratio = 33.4/100.0f * (1.0f - RECENT_OUTPUT_RATIO);
+            }
+          }
+          else
+          {
+            if (is_shortly_after_segregation_fork)
+            {
+            }
+            else
+            {
+              post_fork_num_out_ratio = 67.8/100.0f * (1.0f - RECENT_OUTPUT_RATIO);
+            }
+          }
+        }
+        num_post_fork_outs = num_outs - segregation_limit[amount].first;
+      }
+
       LOG_PRINT_L1("" << num_outs << " unlocked outputs of size " << print_money(amount));
       THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error,
           "histogram reports no unlocked outputs for " + boost::lexical_cast<std::string>(amount) + ", not even ours");
       THROW_WALLET_EXCEPTION_IF(num_recent_outs > num_outs, error::wallet_internal_error,
           "histogram reports more recent outs than outs for " + boost::lexical_cast<std::string>(amount));
 
+      // how many fake outs to draw on a pre-fork triangular distribution
+      size_t pre_fork_outputs_count = requested_outputs_count * pre_fork_num_out_ratio;
+      size_t post_fork_outputs_count = requested_outputs_count * post_fork_num_out_ratio;
+      // how many fake outs to draw otherwise
+      size_t normal_output_count = requested_outputs_count - pre_fork_outputs_count - post_fork_outputs_count;
+
       // X% of those outs are to be taken from recent outputs
-      size_t recent_outputs_count = requested_outputs_count * RECENT_OUTPUT_RATIO;
+      size_t recent_outputs_count = normal_output_count * RECENT_OUTPUT_RATIO;
       if (recent_outputs_count == 0)
         recent_outputs_count = 1; // ensure we have at least one, if possible
       if (recent_outputs_count > num_recent_outs)
         recent_outputs_count = num_recent_outs;
       if (td.m_global_output_index >= num_outs - num_recent_outs && recent_outputs_count > 0)
         --recent_outputs_count; // if the real out is recent, pick one less recent fake out
-      LOG_PRINT_L1("Using " << recent_outputs_count << " recent outputs");
+      LOG_PRINT_L1("Fake output makeup: " << requested_outputs_count << " requested: " << recent_outputs_count << " recent, " <<
+          pre_fork_outputs_count << " pre-fork, " << post_fork_outputs_count << " post-fork, " <<
+          (requested_outputs_count - recent_outputs_count - pre_fork_outputs_count - post_fork_outputs_count) << " full-chain");
 
-      if (num_outs <= requested_outputs_count)
+      uint64_t num_found = 0;
+
+      // if we have a known ring, use it
+      bool existing_ring_found = false;
+      if (td.m_key_image_known && !td.m_key_image_partial)
+      {
+        std::vector<uint64_t> ring;
+        if (get_ring(key, td.m_key_image, ring))
+        {
+          MINFO("This output has a known ring, reusing (size " << ring.size() << ")");
+          THROW_WALLET_EXCEPTION_IF(ring.size() > fake_outputs_count + 1, error::wallet_internal_error,
+              "An output in this transaction was previously spent on another chain with ring size " +
+              std::to_string(ring.size()) + ", it cannot be spent now with ring size " +
+              std::to_string(fake_outputs_count + 1) + " as it is smaller: use a higher ring size");
+          bool own_found = false;
+          existing_ring_found = true;
+          for (const auto &out: ring)
+          {
+            MINFO("Ring has output " << out);
+            if (out < num_outs)
+            {
+              MINFO("Using it");
+              req.outputs.push_back({amount, out});
+              ++num_found;
+              seen_indices.emplace(out);
+              if (out == td.m_global_output_index)
+              {
+                MINFO("This is the real output");
+                own_found = true;
+              }
+            }
+            else
+            {
+              MINFO("Ignoring output " << out << ", too recent");
+            }
+          }
+          THROW_WALLET_EXCEPTION_IF(!own_found, error::wallet_internal_error,
+              "Known ring does not include the spent output: " + std::to_string(td.m_global_output_index));
+        }
+      }
+
+      if (num_outs <= requested_outputs_count && !existing_ring_found)
       {
         for (uint64_t i = 0; i < num_outs; i++)
           req.outputs.push_back({amount, i});
@@ -5530,10 +6000,13 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
       else
       {
         // start with real one
-        uint64_t num_found = 1;
+        if (num_found == 0)
+        {
+          num_found = 1;
           seen_indices.emplace(td.m_global_output_index);
           req.outputs.push_back({amount, td.m_global_output_index});
           LOG_PRINT_L1("Selecting real output: " << td.m_global_output_index << " for " << print_money(amount));
+        }
 
         // while we still need more mixins
         while (num_found < requested_outputs_count)
@@ -5547,6 +6020,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
           // list of output indices we've seen.
 
           uint64_t i;
+          const char *type = "";
           if (num_found - 1 < recent_outputs_count) // -1 to account for the real one we seeded with
           {
             // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit
@@ -5556,7 +6030,29 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
             // just in case rounding up to 1 occurs after calc
             if (i == num_outs)
               --i;
-            LOG_PRINT_L2("picking " << i << " as recent");
+            type = "recent";
+          }
+          else if (num_found -1 < recent_outputs_count + pre_fork_outputs_count)
+          {
+            // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit
+            uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53);
+            double frac = std::sqrt((double)r / ((uint64_t)1 << 53));
+            i = (uint64_t)(frac*segregation_limit[amount].first);
+            // just in case rounding up to 1 occurs after calc
+            if (i == num_outs)
+              --i;
+            type = " pre-fork";
+          }
+          else if (num_found -1 < recent_outputs_count + pre_fork_outputs_count + post_fork_outputs_count)
+          {
+            // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit
+            uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53);
+            double frac = std::sqrt((double)r / ((uint64_t)1 << 53));
+            i = (uint64_t)(frac*num_post_fork_outs) + segregation_limit[amount].first;
+            // just in case rounding up to 1 occurs after calc
+            if (i == num_post_fork_outs+segregation_limit[amount].first)
+              --i;
+            type = "post-fork";
           }
           else
           {
@@ -5567,13 +6063,14 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
             // just in case rounding up to 1 occurs after calc
             if (i == num_outs)
               --i;
-            LOG_PRINT_L2("picking " << i << " as triangular");
+            type = "triangular";
           }
 
           if (seen_indices.count(i))
             continue;
           seen_indices.emplace(i);
 
+          LOG_PRINT_L2("picking " << i << " as " << type);
           req.outputs.push_back({amount, i});
           ++num_found;
         }
@@ -5609,6 +6106,20 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
       outs.back().reserve(fake_outputs_count + 1);
       const rct::key mask = td.is_rct() ? rct::commit(td.amount(), td.m_mask) : rct::zeroCommit(td.amount());
 
+      uint64_t num_outs = 0;
+      const uint64_t amount = td.is_rct() ? 0 : td.amount();
+      const bool output_is_pre_fork = td.m_block_height < segregation_fork_height;
+      if (m_segregate_pre_fork_outputs && output_is_pre_fork)
+        num_outs = segregation_limit[amount].first;
+      else for (const auto &he: resp_t.histogram)
+      {
+        if (he.amount == amount)
+        {
+          num_outs = he.unlocked_instances;
+          break;
+        }
+      }
+
       // make sure the real outputs we asked for are really included, along
       // with the correct key and mask: this guards against an active attack
       // where the node sends dummy data for all outputs, and we then send
@@ -5629,6 +6140,38 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
       // pick real out first (it will be sorted when done)
       outs.back().push_back(std::make_tuple(td.m_global_output_index, boost::get<txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key, mask));
 
+      // then pick outs from an existing ring, if any
+      bool existing_ring_found = false;
+      if (td.m_key_image_known && !td.m_key_image_partial)
+      {
+        std::vector<uint64_t> ring;
+        if (get_ring(key, td.m_key_image, ring))
+        {
+          for (uint64_t out: ring)
+          {
+            if (out < num_outs)
+            {
+              if (out != td.m_global_output_index)
+              {
+                bool found = false;
+                for (size_t o = 0; o < requested_outputs_count; ++o)
+                {
+                  size_t i = base + o;
+                  if (req.outputs[i].index == out)
+                  {
+                    LOG_PRINT_L2("Index " << i << "/" << requested_outputs_count << ": idx " << req.outputs[i].index << " (real " << td.m_global_output_index << "), unlocked " << daemon_resp.outs[i].unlocked << ", key " << daemon_resp.outs[i].key << " (from existing ring)");
+                    tx_add_fake_output(outs, req.outputs[i].index, daemon_resp.outs[i].key, daemon_resp.outs[i].mask, td.m_global_output_index, daemon_resp.outs[i].unlocked);
+                    found = true;
+                    break;
+                  }
+                }
+                THROW_WALLET_EXCEPTION_IF(!found, error::wallet_internal_error, "Falied to find existing ring output in daemon out data");
+              }
+            }
+          }
+        }
+      }
+
       // then pick others in random order till we reach the required number
       // since we use an equiprobable pick here, we don't upset the triangular distribution
       std::vector<size_t> order;

src/wallet/wallet2.h

@@ -66,6 +66,8 @@ class Serialization_portability_wallet_Test;
 
 namespace tools
 {
+  class ringdb;
+
   class i_wallet2_callback
   {
   public:
@@ -166,6 +168,7 @@ namespace tools
     static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev);
 
     wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, bool restricted = false);
+    ~wallet2();
 
     struct multisig_info
     {
@@ -286,6 +289,7 @@ namespace tools
       uint64_t m_timestamp;
       uint32_t m_subaddr_account;   // subaddress account of your wallet to be used in this transfer
       std::set<uint32_t> m_subaddr_indices;  // set of address indices used as inputs in this transfer
+      std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> m_rings; // relative
     };
 
     struct confirmed_transfer_details
@@ -300,10 +304,11 @@ namespace tools
       uint64_t m_unlock_time;
       uint32_t m_subaddr_account;   // subaddress account of your wallet to be used in this transfer
       std::set<uint32_t> m_subaddr_indices;  // set of address indices used as inputs in this transfer
+      std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> m_rings; // relative
 
       confirmed_transfer_details(): m_amount_in(0), m_amount_out(0), m_change((uint64_t)-1), m_block_height(0), m_payment_id(crypto::null_hash), m_timestamp(0), m_unlock_time(0), m_subaddr_account((uint32_t)-1) {}
       confirmed_transfer_details(const unconfirmed_transfer_details &utd, uint64_t height):
-        m_amount_in(utd.m_amount_in), m_amount_out(utd.m_amount_out), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id), m_timestamp(utd.m_timestamp), m_unlock_time(utd.m_tx.unlock_time), m_subaddr_account(utd.m_subaddr_account), m_subaddr_indices(utd.m_subaddr_indices) {}
+        m_amount_in(utd.m_amount_in), m_amount_out(utd.m_amount_out), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id), m_timestamp(utd.m_timestamp), m_unlock_time(utd.m_tx.unlock_time), m_subaddr_account(utd.m_subaddr_account), m_subaddr_indices(utd.m_subaddr_indices), m_rings(utd.m_rings) {}
     };
 
     struct tx_construction_data
@@ -810,6 +815,9 @@ namespace tools
       if(ver < 23)
         return;
       a & m_account_tags;
+      if(ver < 24)
+        return;
+      a & m_ring_history_saved;
     }
 
     /*!
@@ -859,6 +867,10 @@ namespace tools
     void confirm_export_overwrite(bool always) { m_confirm_export_overwrite = always; }
     bool auto_low_priority() const { return m_auto_low_priority; }
     void auto_low_priority(bool value) { m_auto_low_priority = value; }
+    bool segregate_pre_fork_outputs() const { return m_segregate_pre_fork_outputs; }
+    void segregate_pre_fork_outputs(bool value) { m_segregate_pre_fork_outputs = value; }
+    bool key_reuse_mitigation2() const { return m_key_reuse_mitigation2; }
+    void key_reuse_mitigation2(bool value) { m_key_reuse_mitigation2 = value; }
 
     bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const;
     void check_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations);
@@ -1046,6 +1058,18 @@ namespace tools
       return epee::net_utils::invoke_http_json_rpc(uri, method_name, req, res, m_http_client, timeout, http_method, req_id);
     }
 
+    void set_ring_database(const std::string &filename);
+    const std::string get_ring_database() const { return m_ring_database; }
+    bool get_ring(const crypto::key_image &key_image, std::vector<uint64_t> &outs);
+    bool get_rings(const crypto::hash &txid, std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> &outs);
+    bool set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative);
+    bool find_and_save_rings(bool force = true);
+
+    bool blackball_output(const crypto::public_key &output);
+    bool set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add = false);
+    bool unblackball_output(const crypto::public_key &output);
+    bool is_output_blackballed(const crypto::public_key &output) const;
+
   private:
     /*!
      * \brief  Stores wallet information to wallet file.
@@ -1102,6 +1126,12 @@ namespace tools
     rct::multisig_kLRki get_multisig_kLRki(size_t n, const rct::key &k) const;
     rct::key get_multisig_k(size_t idx, const std::unordered_set<rct::key> &used_L) const;
     void update_multisig_rescan_info(const std::vector<std::vector<rct::key>> &multisig_k, const std::vector<std::vector<tools::wallet2::multisig_info>> &info, size_t n);
+    bool add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx);
+    bool add_rings(const cryptonote::transaction_prefix &tx);
+    bool remove_rings(const cryptonote::transaction_prefix &tx);
+    bool get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs);
+
+    bool get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution);
 
     cryptonote::account_base m_account;
     boost::optional<epee::net_utils::http::login> m_daemon_login;
@@ -1167,6 +1197,8 @@ namespace tools
     uint32_t m_confirm_backlog_threshold;
     bool m_confirm_export_overwrite;
     bool m_auto_low_priority;
+    bool m_segregate_pre_fork_outputs;
+    bool m_key_reuse_mitigation2;
     bool m_is_initialized;
     NodeRPCProxy m_node_rpc_proxy;
     std::unordered_set<crypto::hash> m_scanned_pool_txs[2];
@@ -1185,17 +1217,21 @@ namespace tools
     std::unordered_map<crypto::hash, address_tx> m_light_wallet_address_txs;
     // store calculated key image for faster lookup
     std::unordered_map<crypto::public_key, std::map<uint64_t, crypto::key_image> > m_key_image_cache;
+
+    std::string m_ring_database;
+    bool m_ring_history_saved;
+    std::unique_ptr<ringdb> m_ringdb;
   };
 }
-BOOST_CLASS_VERSION(tools::wallet2, 23)
+BOOST_CLASS_VERSION(tools::wallet2, 24)
 BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_tx_set, 1)
 BOOST_CLASS_VERSION(tools::wallet2::payment_details, 3)
 BOOST_CLASS_VERSION(tools::wallet2::pool_payment_details, 1)
-BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 7)
+BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 8)
-BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 5)
+BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 6)
 BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17)
 BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0)
 BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0)
@@ -1395,6 +1431,9 @@ namespace boost
       }
       a & x.m_subaddr_account;
       a & x.m_subaddr_indices;
+      if (ver < 8)
+        return;
+      a & x.m_rings;
     }
 
     template <class Archive>
@@ -1439,6 +1478,9 @@ namespace boost
       }
       a & x.m_subaddr_account;
       a & x.m_subaddr_indices;
+      if (ver < 6)
+        return;
+      a & x.m_rings;
     }
 
     template <class Archive>

src/wallet/wallet_errors.h

@@ -36,6 +36,7 @@
 #include <vector>
 
 #include "cryptonote_basic/cryptonote_format_utils.h"
+#include "cryptonote_core/cryptonote_tx_utils.h"
 #include "rpc/core_rpc_server_commands_defs.h"
 #include "include_base_utils.h"
 
@@ -85,6 +86,7 @@ namespace tools
     //         no_connection_to_daemon
     //         is_key_image_spent_error
     //         get_histogram_error
+    //         get_output_distribution
     //       wallet_files_doesnt_correspond
     //
     // * - class with protected ctor
@@ -757,6 +759,14 @@ namespace tools
       }
     };
     //----------------------------------------------------------------------------------------------------
+    struct get_output_distribution : public wallet_rpc_error
+    {
+      explicit get_output_distribution(std::string&& loc, const std::string& request)
+        : wallet_rpc_error(std::move(loc), "failed to get output distribution", request)
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
     struct wallet_files_doesnt_correspond : public wallet_logic_error
     {
       explicit wallet_files_doesnt_correspond(std::string&& loc, const std::string& keys_file, const std::string& wallet_file)

tests/unit_tests/hardfork.cpp

@@ -109,7 +109,8 @@ public:
   virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const { return true; }
   virtual bool for_blocks_range(const uint64_t&, const uint64_t&, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const { return true; }
   virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const { return true; }
-  virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const { return true; }
+  virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const { return true; }
+  virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const { return true; }
   virtual bool is_read_only() const { return false; }
   virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff) const { return std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>(); }