2015-01-02 18:52:46 +02:00
|
|
|
// Copyright (c) 2014-2015, The Monero Project
|
2014-07-23 16:03:52 +03:00
|
|
|
//
|
|
|
|
// All rights reserved.
|
|
|
|
//
|
|
|
|
// Redistribution and use in source and binary forms, with or without modification, are
|
|
|
|
// permitted provided that the following conditions are met:
|
|
|
|
//
|
|
|
|
// 1. Redistributions of source code must retain the above copyright notice, this list of
|
|
|
|
// conditions and the following disclaimer.
|
|
|
|
//
|
|
|
|
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
|
|
|
|
// of conditions and the following disclaimer in the documentation and/or other
|
|
|
|
// materials provided with the distribution.
|
|
|
|
//
|
|
|
|
// 3. Neither the name of the copyright holder nor the names of its contributors may be
|
|
|
|
// used to endorse or promote products derived from this software without specific
|
|
|
|
// prior written permission.
|
|
|
|
//
|
|
|
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
|
|
|
|
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
|
|
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
|
|
|
|
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
|
|
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
|
|
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
|
|
|
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
|
|
|
|
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
//
|
|
|
|
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
|
2014-03-04 00:07:58 +02:00
|
|
|
|
|
|
|
#pragma once
|
|
|
|
|
2014-09-25 08:15:28 +03:00
|
|
|
#include <ctime>
|
|
|
|
|
2014-03-04 00:07:58 +02:00
|
|
|
#include <boost/program_options/options_description.hpp>
|
|
|
|
#include <boost/program_options/variables_map.hpp>
|
|
|
|
|
|
|
|
#include "p2p/net_node_common.h"
|
|
|
|
#include "cryptonote_protocol/cryptonote_protocol_handler_common.h"
|
|
|
|
#include "storages/portable_storage_template_helper.h"
|
|
|
|
#include "tx_pool.h"
|
2015-01-26 07:36:09 +02:00
|
|
|
#if BLOCKCHAIN_DB == DB_LMDB
|
2014-10-28 05:44:45 +02:00
|
|
|
#include "blockchain.h"
|
2015-01-26 07:36:09 +02:00
|
|
|
#else
|
2014-03-04 00:07:58 +02:00
|
|
|
#include "blockchain_storage.h"
|
2015-01-26 07:36:09 +02:00
|
|
|
#endif
|
2014-03-04 00:07:58 +02:00
|
|
|
#include "miner.h"
|
|
|
|
#include "connection_context.h"
|
|
|
|
#include "cryptonote_core/cryptonote_stat_info.h"
|
|
|
|
#include "warnings.h"
|
|
|
|
#include "crypto/hash.h"
|
|
|
|
|
|
|
|
PUSH_WARNINGS
|
|
|
|
DISABLE_VS_WARNINGS(4355)
|
|
|
|
|
|
|
|
namespace cryptonote
|
|
|
|
{
|
|
|
|
/************************************************************************/
|
|
|
|
/* */
|
|
|
|
/************************************************************************/
|
|
|
|
class core: public i_miner_handler
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
core(i_cryptonote_protocol* pprotocol);
|
|
|
|
bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote_connection_context& context);
|
|
|
|
bool on_idle();
|
|
|
|
bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block);
|
|
|
|
bool handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate = true);
|
** CHANGES ARE EXPERIMENTAL (FOR TESTING ONLY)
Bockchain:
1. Optim: Multi-thread long-hash computation when encountering groups of blocks.
2. Optim: Cache verified txs and return result from cache instead of re-checking whenever possible.
3. Optim: Preload output-keys when encoutering groups of blocks. Sort by amount and global-index before bulk querying database and multi-thread when possible.
4. Optim: Disable double spend check on block verification, double spend is already detected when trying to add blocks.
5. Optim: Multi-thread signature computation whenever possible.
6. Patch: Disable locking (recursive mutex) on called functions from check_tx_inputs which causes slowdowns (only seems to happen on ubuntu/VMs??? Reason: TBD)
7. Optim: Removed looped full-tx hash computation when retrieving transactions from pool (???).
8. Optim: Cache difficulty/timestamps (735 blocks) for next-difficulty calculations so that only 2 db reads per new block is needed when a new block arrives (instead of 1470 reads).
Berkeley-DB:
1. Fix: 32-bit data errors causing wrong output global indices and failure to send blocks to peers (etc).
2. Fix: Unable to pop blocks on reorganize due to transaction errors.
3. Patch: Large number of transaction aborts when running multi-threaded bulk queries.
4. Patch: Insufficient locks error when running full sync.
5. Patch: Incorrect db stats when returning from an immediate exit from "pop block" operation.
6. Optim: Add bulk queries to get output global indices.
7. Optim: Modified output_keys table to store public_key+unlock_time+height for single transaction lookup (vs 3)
8. Optim: Used output_keys table retrieve public_keys instead of going through output_amounts->output_txs+output_indices->txs->output:public_key
9. Optim: Added thread-safe buffers used when multi-threading bulk queries.
10. Optim: Added support for nosync/write_nosync options for improved performance (*see --db-sync-mode option for details)
11. Mod: Added checkpoint thread and auto-remove-logs option.
12. *Now usable on 32-bit systems like RPI2.
LMDB:
1. Optim: Added custom comparison for 256-bit key tables (minor speed-up, TBD: get actual effect)
2. Optim: Modified output_keys table to store public_key+unlock_time+height for single transaction lookup (vs 3)
3. Optim: Used output_keys table retrieve public_keys instead of going through output_amounts->output_txs+output_indices->txs->output:public_key
4. Optim: Added support for sync/writemap options for improved performance (*see --db-sync-mode option for details)
5. Mod: Auto resize to +1GB instead of multiplier x1.5
ETC:
1. Minor optimizations for slow-hash for ARM (RPI2). Incomplete.
2. Fix: 32-bit saturation bug when computing next difficulty on large blocks.
[PENDING ISSUES]
1. Berkely db has a very slow "pop-block" operation. This is very noticeable on the RPI2 as it sometimes takes > 10 MINUTES to pop a block during reorganization.
This does not happen very often however, most reorgs seem to take a few seconds but it possibly depends on the number of outputs present. TBD.
2. Berkeley db, possible bug "unable to allocate memory". TBD.
[NEW OPTIONS] (*Currently all enabled for testing purposes)
1. --fast-block-sync arg=[0:1] (default: 1)
a. 0 = Compute long hash per block (may take a while depending on CPU)
b. 1 = Skip long-hash and verify blocks based on embedded known good block hashes (faster, minimal CPU dependence)
2. --db-sync-mode arg=[[safe|fast|fastest]:[sync|async]:[nblocks_per_sync]] (default: fastest:async:1000)
a. safe = fdatasync/fsync (or equivalent) per stored block. Very slow, but safest option to protect against power-out/crash conditions.
b. fast/fastest = Enables asynchronous fdatasync/fsync (or equivalent). Useful for battery operated devices or STABLE systems with UPS and/or systems with battery backed write cache/solid state cache.
Fast - Write meta-data but defer data flush.
Fastest - Defer meta-data and data flush.
Sync - Flush data after nblocks_per_sync and wait.
Async - Flush data after nblocks_per_sync but do not wait for the operation to finish.
3. --prep-blocks-threads arg=[n] (default: 4 or system max threads, whichever is lower)
Max number of threads to use when computing long-hash in groups.
4. --show-time-stats arg=[0:1] (default: 1)
Show benchmark related time stats.
5. --db-auto-remove-logs arg=[0:1] (default: 1)
For berkeley-db only. Auto remove logs if enabled.
**Note: lmdb and berkeley-db have changes to the tables and are not compatible with official git head version.
At the moment, you need a full resync to use this optimized version.
[PERFORMANCE COMPARISON]
**Some figures are approximations only.
Using a baseline machine of an i7-2600K+SSD+(with full pow computation):
1. The optimized lmdb/blockhain core can process blocks up to 585K for ~1.25 hours + download time, so it usually takes 2.5 hours to sync the full chain.
2. The current head with memory can process blocks up to 585K for ~4.2 hours + download time, so it usually takes 5.5 hours to sync the full chain.
3. The current head with lmdb can process blocks up to 585K for ~32 hours + download time and usually takes 36 hours to sync the full chain.
Averate procesing times (with full pow computation):
lmdb-optimized:
1. tx_ave = 2.5 ms / tx
2. block_ave = 5.87 ms / block
memory-official-repo:
1. tx_ave = 8.85 ms / tx
2. block_ave = 19.68 ms / block
lmdb-official-repo (0f4a036437fd41a5498ee5e74e2422ea6177aa3e)
1. tx_ave = 47.8 ms / tx
2. block_ave = 64.2 ms / block
**Note: The following data denotes processing times only (does not include p2p download time)
lmdb-optimized processing times (with full pow computation):
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 1.25 hours processing time (--db-sync-mode=fastest:async:1000).
2. Laptop, Dual-core / 4-threads U4200 (3Mb) - 4.90 hours processing time (--db-sync-mode=fastest:async:1000).
3. Embedded, Quad-core / 4-threads Z3735F (2x1Mb) - 12.0 hours processing time (--db-sync-mode=fastest:async:1000).
lmdb-optimized processing times (with per-block-checkpoint)
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 10 minutes processing time (--db-sync-mode=fastest:async:1000).
berkeley-db optimized processing times (with full pow computation)
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 1.8 hours processing time (--db-sync-mode=fastest:async:1000).
2. RPI2. Improved from estimated 3 months(???) into 2.5 days (*Need 2AMP supply + Clock:1Ghz + [usb+ssd] to achieve this speed) (--db-sync-mode=fastest:async:1000).
berkeley-db optimized processing times (with per-block-checkpoint)
1. RPI2. 12-15 hours (*Need 2AMP supply + Clock:1Ghz + [usb+ssd] to achieve this speed) (--db-sync-mode=fastest:async:1000).
2015-07-10 23:09:32 +03:00
|
|
|
bool prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks);
|
|
|
|
bool cleanup_handle_incoming_blocks(bool force_sync = false);
|
|
|
|
|
2014-06-12 00:32:53 +03:00
|
|
|
bool check_incoming_block_size(const blobdata& block_blob);
|
2014-03-04 00:07:58 +02:00
|
|
|
i_cryptonote_protocol* get_protocol(){return m_pprotocol;}
|
|
|
|
|
|
|
|
//-------------------- i_miner_handler -----------------------
|
|
|
|
virtual bool handle_block_found( block& b);
|
|
|
|
virtual bool get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce);
|
|
|
|
|
|
|
|
|
|
|
|
miner& get_miner(){return m_miner;}
|
|
|
|
static void init_options(boost::program_options::options_description& desc);
|
2015-01-30 00:10:53 +02:00
|
|
|
bool init(const boost::program_options::variables_map& vm);
|
2014-03-04 00:07:58 +02:00
|
|
|
bool set_genesis_block(const block& b);
|
|
|
|
bool deinit();
|
2015-02-12 21:59:39 +02:00
|
|
|
static void set_fast_exit();
|
|
|
|
static bool get_fast_exit();
|
2015-02-20 23:28:03 +02:00
|
|
|
void test_drop_download();
|
|
|
|
void test_drop_download_height(uint64_t height);
|
|
|
|
bool get_test_drop_download();
|
|
|
|
bool get_test_drop_download_height();
|
2014-03-04 00:07:58 +02:00
|
|
|
uint64_t get_current_blockchain_height();
|
|
|
|
bool get_blockchain_top(uint64_t& heeight, crypto::hash& top_id);
|
|
|
|
bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs);
|
|
|
|
bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks);
|
|
|
|
template<class t_ids_container, class t_blocks_container, class t_missed_container>
|
|
|
|
bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs)
|
|
|
|
{
|
|
|
|
return m_blockchain_storage.get_blocks(block_ids, blocks, missed_bs);
|
|
|
|
}
|
|
|
|
crypto::hash get_block_id_by_height(uint64_t height);
|
|
|
|
bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs);
|
|
|
|
bool get_block_by_hash(const crypto::hash &h, block &blk);
|
2014-06-28 01:06:21 +03:00
|
|
|
//void get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid);
|
2014-03-04 00:07:58 +02:00
|
|
|
|
|
|
|
bool get_alternative_blocks(std::list<block>& blocks);
|
|
|
|
size_t get_alternative_blocks_count();
|
|
|
|
|
|
|
|
void set_cryptonote_protocol(i_cryptonote_protocol* pprotocol);
|
|
|
|
void set_checkpoints(checkpoints&& chk_pts);
|
2014-09-25 08:15:28 +03:00
|
|
|
void set_checkpoints_file_path(const std::string& path);
|
2014-09-26 08:01:58 +03:00
|
|
|
void set_enforce_dns_checkpoints(bool enforce_dns);
|
2014-03-04 00:07:58 +02:00
|
|
|
|
|
|
|
bool get_pool_transactions(std::list<transaction>& txs);
|
2015-04-23 15:13:07 +03:00
|
|
|
bool get_pool_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const;
|
2014-03-04 00:07:58 +02:00
|
|
|
size_t get_pool_transactions_count();
|
|
|
|
size_t get_blockchain_total_transactions();
|
2014-06-28 01:06:21 +03:00
|
|
|
//bool get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys);
|
2014-03-04 00:07:58 +02:00
|
|
|
bool have_block(const crypto::hash& id);
|
|
|
|
bool get_short_chain_history(std::list<crypto::hash>& ids);
|
|
|
|
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp);
|
2014-08-01 11:17:50 +03:00
|
|
|
bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count);
|
2014-03-04 00:07:58 +02:00
|
|
|
bool get_stat_info(core_stat_info& st_inf);
|
2014-06-28 01:06:21 +03:00
|
|
|
//bool get_backward_blocks_sizes(uint64_t from_height, std::vector<size_t>& sizes, size_t count);
|
2014-03-04 00:07:58 +02:00
|
|
|
bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs);
|
|
|
|
crypto::hash get_tail_id();
|
|
|
|
bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res);
|
|
|
|
void pause_mine();
|
|
|
|
void resume_mine();
|
2015-01-26 07:36:09 +02:00
|
|
|
#if BLOCKCHAIN_DB == DB_LMDB
|
2014-10-28 05:44:45 +02:00
|
|
|
Blockchain& get_blockchain_storage(){return m_blockchain_storage;}
|
2015-01-26 07:36:09 +02:00
|
|
|
#else
|
2014-03-04 00:07:58 +02:00
|
|
|
blockchain_storage& get_blockchain_storage(){return m_blockchain_storage;}
|
2015-01-26 07:36:09 +02:00
|
|
|
#endif
|
2014-03-04 00:07:58 +02:00
|
|
|
//debug functions
|
|
|
|
void print_blockchain(uint64_t start_index, uint64_t end_index);
|
|
|
|
void print_blockchain_index();
|
|
|
|
std::string print_pool(bool short_format);
|
|
|
|
void print_blockchain_outs(const std::string& file);
|
|
|
|
void on_synchronized();
|
|
|
|
|
2014-06-04 23:50:13 +03:00
|
|
|
void set_target_blockchain_height(uint64_t target_blockchain_height);
|
|
|
|
uint64_t get_target_blockchain_height() const;
|
|
|
|
|
2014-09-26 08:01:58 +03:00
|
|
|
bool update_checkpoints();
|
2014-09-25 08:15:28 +03:00
|
|
|
|
2015-02-05 11:11:20 +02:00
|
|
|
void stop();
|
|
|
|
|
2015-08-11 12:49:15 +03:00
|
|
|
bool is_key_image_spent(const crypto::key_image& key_im);
|
|
|
|
bool are_key_images_spent(const std::vector<crypto::key_image>& key_im, std::vector<bool> &spent);
|
|
|
|
|
2014-09-26 08:01:58 +03:00
|
|
|
private:
|
2014-09-25 09:24:42 +03:00
|
|
|
bool add_new_tx(const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block);
|
2014-03-04 00:07:58 +02:00
|
|
|
bool add_new_tx(const transaction& tx, tx_verification_context& tvc, bool keeped_by_block);
|
|
|
|
bool add_new_block(const block& b, block_verification_context& bvc);
|
|
|
|
bool load_state_data();
|
|
|
|
bool parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash, const blobdata& blob);
|
|
|
|
|
|
|
|
bool check_tx_syntax(const transaction& tx);
|
|
|
|
//check correct values, amounts and all lightweight checks not related with database
|
|
|
|
bool check_tx_semantic(const transaction& tx, bool keeped_by_block);
|
|
|
|
//check if tx already in memory pool or in main blockchain
|
|
|
|
|
|
|
|
bool check_tx_ring_signature(const txin_to_key& tx, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig);
|
|
|
|
bool is_tx_spendtime_unlocked(uint64_t unlock_time);
|
|
|
|
bool update_miner_block_template();
|
2015-01-30 00:10:53 +02:00
|
|
|
bool handle_command_line(const boost::program_options::variables_map& vm);
|
2014-03-04 00:07:58 +02:00
|
|
|
bool on_update_blocktemplate_interval();
|
|
|
|
bool check_tx_inputs_keyimages_diff(const transaction& tx);
|
2014-09-30 22:24:43 +03:00
|
|
|
void graceful_exit();
|
2015-02-12 21:59:39 +02:00
|
|
|
static std::atomic<bool> m_fast_exit;
|
2015-02-20 23:28:03 +02:00
|
|
|
bool m_test_drop_download = true;
|
|
|
|
uint64_t m_test_drop_download_height = 0;
|
2014-03-04 00:07:58 +02:00
|
|
|
|
|
|
|
tx_memory_pool m_mempool;
|
2015-01-26 07:36:09 +02:00
|
|
|
#if BLOCKCHAIN_DB == DB_LMDB
|
2014-10-28 05:44:45 +02:00
|
|
|
Blockchain m_blockchain_storage;
|
2015-01-26 07:36:09 +02:00
|
|
|
#else
|
2014-03-04 00:07:58 +02:00
|
|
|
blockchain_storage m_blockchain_storage;
|
2015-01-26 07:36:09 +02:00
|
|
|
#endif
|
2014-03-04 00:07:58 +02:00
|
|
|
i_cryptonote_protocol* m_pprotocol;
|
2014-05-25 20:06:40 +03:00
|
|
|
epee::critical_section m_incoming_tx_lock;
|
2014-03-04 00:07:58 +02:00
|
|
|
//m_miner and m_miner_addres are probably temporary here
|
|
|
|
miner m_miner;
|
|
|
|
account_public_address m_miner_address;
|
|
|
|
std::string m_config_folder;
|
|
|
|
cryptonote_protocol_stub m_protocol_stub;
|
2014-05-25 20:06:40 +03:00
|
|
|
epee::math_helper::once_a_time_seconds<60*60*12, false> m_store_blockchain_interval;
|
2014-03-04 00:07:58 +02:00
|
|
|
friend class tx_validate_inputs;
|
|
|
|
std::atomic<bool> m_starter_message_showed;
|
2014-06-04 23:50:13 +03:00
|
|
|
|
|
|
|
uint64_t m_target_blockchain_height;
|
2014-09-25 08:15:28 +03:00
|
|
|
|
2015-01-30 00:10:53 +02:00
|
|
|
bool m_testnet;
|
2014-09-25 08:15:28 +03:00
|
|
|
std::string m_checkpoints_path;
|
2014-09-29 23:30:47 +03:00
|
|
|
time_t m_last_dns_checkpoints_update;
|
|
|
|
time_t m_last_json_checkpoints_update;
|
2015-04-22 11:36:39 +03:00
|
|
|
|
|
|
|
std::atomic_flag m_checkpoints_updating;
|
2014-03-04 00:07:58 +02:00
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
POP_WARNINGS
|