diff --git a/src/daemon/monero_daemon_model.cpp b/src/daemon/monero_daemon_model.cpp index 698553a4..caf6f052 100644 --- a/src/daemon/monero_daemon_model.cpp +++ b/src/daemon/monero_daemon_model.cpp @@ -717,9 +717,8 @@ namespace monero { std::vector> monero_key_image::deserialize_key_images(const std::string& key_images_json) { // deserialize json to property node - std::istringstream iss = key_images_json.empty() ? std::istringstream() : std::istringstream(key_images_json); boost::property_tree::ptree node; - boost::property_tree::read_json(iss, node); + gen_utils::deserialize(key_images_json, node); // convert property tree to key images std::vector> key_images; diff --git a/src/daemon/monero_daemon_rpc.cpp b/src/daemon/monero_daemon_rpc.cpp index dc5e4f22..5de5622e 100644 --- a/src/daemon/monero_daemon_rpc.cpp +++ b/src/daemon/monero_daemon_rpc.cpp @@ -58,6 +58,10 @@ #include "utils/monero_utils.h" static const uint64_t MAX_REQ_SIZE = 3000000; + +// the number of heights that get_blocks_by_range() asks for in one request: the daemon's restricted +// rpc refuses a request for more than a thousand blocks, which the chunk size matches +static const uint64_t MAX_HEIGHTS_PER_REQUEST = 1000; static const uint64_t NUM_HEADERS_PER_REQ = 750; namespace monero { @@ -391,15 +395,29 @@ namespace monero { std::vector> monero_daemon_rpc::get_blocks_by_range(boost::optional start_height, boost::optional end_height) { if (start_height == boost::none) start_height = 0; - if (end_height == boost::none) { + + // the end can not be past the top of the chain: an end beyond it, UINT64_MAX among them, would + // make the range below never end. An explicit end that fits one request is passed through as it + // is, the way it was before the chunking, so that the common small range pays no extra call + if (end_height == boost::none || end_height.get() - start_height.get() >= MAX_HEIGHTS_PER_REQUEST) { uint64_t block_height = get_height(); - end_height = block_height == 0 ? 0 : block_height - 1; + uint64_t chain_end = block_height == 0 ? 0 : block_height - 1; + if (end_height == boost::none || end_height.get() > chain_end) end_height = chain_end; } - std::vector heights; - for (uint64_t height = start_height.get(); height <= end_height.get(); height++) heights.push_back(height); + std::vector> blocks; + if (start_height.get() > end_height.get()) return blocks; + + // fetch the heights in chunks, so that no vector of every height of the range is built before + // the first request, the way get_blocks_by_range_chunked() chunks its fetch + for (uint64_t height = start_height.get(); height <= end_height.get();) { + std::vector heights; + while (heights.size() < MAX_HEIGHTS_PER_REQUEST && height <= end_height.get()) heights.push_back(height++); + auto chunk = get_blocks_by_height(heights); + blocks.insert(blocks.end(), chunk.begin(), chunk.end()); + } - return get_blocks_by_height(heights); + return blocks; } std::vector> monero_daemon_rpc::get_blocks_by_range_chunked(boost::optional start_height, boost::optional end_height, boost::optional max_chunk_size) { diff --git a/src/daemon/monero_daemon_rpc_model.cpp b/src/daemon/monero_daemon_rpc_model.cpp index a3a283ed..8b003b79 100644 --- a/src/daemon/monero_daemon_rpc_model.cpp +++ b/src/daemon/monero_daemon_rpc_model.cpp @@ -977,6 +977,8 @@ namespace monero { for (boost::property_tree::ptree::const_iterator it = node.begin(); it != node.end(); ++it) { std::string key = it->first; if (key == std::string("blob")) block->m_hex = it->second.data(); + // the json of a block repeats the tx hashes that this field of the response already took, so the parse of + // the json, which is nested, skips them instead of taking them twice else if (key == std::string("tx_hashes") && !is_nested) { for(const auto &hex : it->second) block->m_tx_hashes.push_back(hex.second.data()); } @@ -1002,9 +1004,8 @@ namespace monero { } else if (key == std::string("json")) { auto json = it->second.data(); - std::istringstream iss = json.empty() ? std::istringstream() : std::istringstream(json); boost::property_tree::ptree json_node; - boost::property_tree::read_json(iss, json_node); + gen_utils::deserialize(json, json_node); deserialize_block(json_node, block, true); } } diff --git a/src/utils/gen_utils.cpp b/src/utils/gen_utils.cpp index a3368f2f..2635332d 100644 --- a/src/utils/gen_utils.cpp +++ b/src/utils/gen_utils.cpp @@ -73,6 +73,25 @@ std::string gen_utils::serialize(const boost::property_tree::ptree& node) { } void gen_utils::deserialize(const std::string& json, boost::property_tree::ptree& root) { + + // scan the nesting before parsing: boost's json parser recurses per level, so a document that + // nests deeply enough, which a malicious daemon can send as "[[[[...", overflows the stack of + // the thread that parses it. the limit matches the one monero-c checks at its boundary and is + // far above the depth of any model the daemon and the wallet exchange + constexpr int MAX_JSON_DEPTH = 64; + int depth = 0; + bool in_string = false; + for (size_t i = 0; i < json.size(); i++) { + char c = json[i]; + if (in_string) { + if (c == '\\') i++; + else if (c == '"') in_string = false; + } else if (c == '"') in_string = true; + else if (c == '[' || c == '{') { + if (++depth > MAX_JSON_DEPTH) throw std::runtime_error("Invalid JSON"); + } else if ((c == ']' || c == '}') && depth > 0) depth--; + } + std::istringstream iss = json.empty() ? std::istringstream() : std::istringstream(json); try { boost::property_tree::read_json(iss, root); diff --git a/src/utils/monero_utils.cpp b/src/utils/monero_utils.cpp index 17743a56..bb9035c8 100644 --- a/src/utils/monero_utils.cpp +++ b/src/utils/monero_utils.cpp @@ -121,8 +121,7 @@ namespace { std::string response_json; if (is_fast) monero_utils::binary_blocks_fast_to_json(bin, response_json); else monero_utils::binary_blocks_to_json(bin, response_json); - std::istringstream iss(response_json); - boost::property_tree::read_json(iss, node); + gen_utils::deserialize(response_json, node); auto blocks = node.get_child("blocks"); auto block_hashes = node.get_child("block_hashes"); // parallel to "blocks", see add_blocks_and_txs_to_ptree() diff --git a/src/utils/monero_utils.h b/src/utils/monero_utils.h index 407d3e56..9eb963e4 100644 --- a/src/utils/monero_utils.h +++ b/src/utils/monero_utils.h @@ -288,33 +288,43 @@ namespace monero_utils // ------------------------------ FREE MEMORY ------------------------------- - static void free(std::shared_ptr block) { - if (block == nullptr) return; - for (std::shared_ptr& tx : block->m_txs) { - tx->m_block.reset(); - monero_tx_wallet* tx_wallet = dynamic_cast(tx.get()); - if (tx_wallet != nullptr) { - if (tx_wallet->m_tx_set != nullptr) tx_wallet->m_tx_set.reset(); - if (tx_wallet->m_outgoing_transfer != nullptr) tx_wallet->m_outgoing_transfer->m_tx.reset(); - for (std::shared_ptr transfer : tx_wallet->m_incoming_transfers) transfer->m_tx.reset(); - for (std::shared_ptr output : tx_wallet->m_outputs) output->m_tx.reset(); - for (std::shared_ptr input : tx_wallet->m_inputs) { - input->m_key_image.reset(); - input->m_tx.reset(); - } + // releases the links of the models of a tx that point back at them, so dropping the shared + // pointers frees the models: a wallet tx owns transfers that point back at it, every tx owns + // inputs and outputs that do, and a tx query owns its transfer, input and output queries + static void free_tx_links(const std::shared_ptr& tx) { + tx->m_block.reset(); + monero_tx_wallet* tx_wallet = dynamic_cast(tx.get()); + if (tx_wallet != nullptr) { + if (tx_wallet->m_tx_set != nullptr) tx_wallet->m_tx_set.reset(); + if (tx_wallet->m_outgoing_transfer != nullptr) tx_wallet->m_outgoing_transfer->m_tx.reset(); + for (std::shared_ptr transfer : tx_wallet->m_incoming_transfers) transfer->m_tx.reset(); + } + for (std::shared_ptr output : tx->m_outputs) output->m_tx.reset(); + for (std::shared_ptr input : tx->m_inputs) { + input->m_key_image.reset(); + input->m_tx.reset(); + } + monero_tx_query* tx_query = dynamic_cast(tx.get()); + if (tx_query != nullptr) { + if (tx_query->m_transfer_query != nullptr) { + tx_query->m_transfer_query->m_tx_query.reset(); + tx_query->m_transfer_query.reset(); + } + if (tx_query->m_input_query != nullptr) { + tx_query->m_input_query->m_tx_query.reset(); + tx_query->m_input_query.reset(); } - monero_tx_query* tx_query = dynamic_cast(tx.get()); - if (tx_query != nullptr) { - if (tx_query->m_transfer_query != nullptr) { - tx_query->m_transfer_query->m_tx_query.reset(); - tx_query->m_transfer_query.reset(); - } - if (tx_query->m_output_query != nullptr) { - tx_query->m_output_query->m_tx_query.reset(); - tx_query->m_output_query.reset(); - } + if (tx_query->m_output_query != nullptr) { + tx_query->m_output_query->m_tx_query.reset(); + tx_query->m_output_query.reset(); } } + } + + static void free(std::shared_ptr block) { + if (block == nullptr) return; + for (std::shared_ptr& tx : block->m_txs) free_tx_links(tx); + if (block->m_miner_tx != nullptr) free_tx_links(block->m_miner_tx); block.reset(); } diff --git a/src/wallet/monero_wallet_full.cpp b/src/wallet/monero_wallet_full.cpp index 1ea97ba5..c6267e94 100644 --- a/src/wallet/monero_wallet_full.cpp +++ b/src/wallet/monero_wallet_full.cpp @@ -971,7 +971,7 @@ namespace monero { monero_wallet_full* monero_wallet_full::open_wallet(const std::string& path, const std::string& password, const monero_network_type network_type, bool regtest) { MTRACE("open_wallet(" << path << ", ***, " << network_type << ")"); - monero_wallet_full* wallet = new monero_wallet_full(); + std::unique_ptr wallet(new monero_wallet_full()); wallet->m_w2 = std::unique_ptr(new tools::wallet2(static_cast(network_type), 1, true)); wallet->m_w2->load(path, password); wallet->m_w2->init(""); @@ -982,12 +982,12 @@ namespace monero { } if (network_type == monero_network_type::TESTNET) wallet->m_w2->allow_mismatched_daemon_version(true); // testnet may use custom hard fork heights wallet->init_common(); - return wallet; + return wallet.release(); } monero_wallet_full* monero_wallet_full::open_wallet_data(const std::string& password, const monero_network_type network_type, const std::string& keys_data, const std::string& cache_data, const std::shared_ptr& daemon_connection, std::unique_ptr http_client_factory, bool regtest) { MTRACE("open_wallet_data(...)"); - monero_wallet_full* wallet = new monero_wallet_full(); + std::unique_ptr wallet(new monero_wallet_full()); if (http_client_factory == nullptr) wallet->m_w2 = std::unique_ptr(new tools::wallet2(static_cast(network_type), 1, true)); else wallet->m_w2 = std::unique_ptr(new tools::wallet2(static_cast(network_type), 1, true, std::move(http_client_factory))); wallet->m_w2->load("", password, keys_data, cache_data); @@ -1000,7 +1000,7 @@ namespace monero { if (network_type == monero_network_type::TESTNET) wallet->m_w2->allow_mismatched_daemon_version(true); // testnet may use custom hard fork heights wallet->set_daemon_connection(daemon_connection); wallet->init_common(); - return wallet; + return wallet.release(); } monero_wallet_full* monero_wallet_full::create_wallet(const monero_wallet_config& config, std::unique_ptr http_client_factory) { @@ -1061,7 +1061,7 @@ namespace monero { if (!config.m_seed_offset.get().empty()) recovery_key = cryptonote::decrypt_key(recovery_key, config.m_seed_offset.get()); // initialize wallet - monero_wallet_full* wallet = new monero_wallet_full(); + std::unique_ptr wallet(new monero_wallet_full()); if (http_client_factory == nullptr) wallet->m_w2 = std::unique_ptr(new tools::wallet2(static_cast(config.m_network_type.get()), 1, true)); else wallet->m_w2 = std::unique_ptr(new tools::wallet2(static_cast(config.m_network_type.get()), 1, true, std::move(http_client_factory))); wallet->m_w2->init(""); @@ -1096,7 +1096,7 @@ namespace monero { if (!crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, language)) throw std::runtime_error("Failed to encode seed"); } wallet->init_common(); - return wallet; + return wallet.release(); } monero_wallet_full* monero_wallet_full::create_wallet_from_keys(monero_wallet_config& config, std::unique_ptr http_client_factory) { @@ -1156,8 +1156,7 @@ namespace monero { if (!monero_utils::is_valid_language(config.m_language.get())) throw std::runtime_error("Unknown language: " + config.m_language.get()); // initialize wallet - // TODO: delete wallet pointer if exception - monero_wallet_full* wallet = new monero_wallet_full(); + std::unique_ptr wallet(new monero_wallet_full()); if (http_client_factory == nullptr) wallet->m_w2 = std::unique_ptr(new tools::wallet2(static_cast(config.m_network_type.get()), 1, true)); else wallet->m_w2 = std::unique_ptr(new tools::wallet2(static_cast(config.m_network_type.get()), 1, true, std::move(http_client_factory))); if (config.m_account_lookahead != boost::none) wallet->m_w2->set_subaddress_lookahead(config.m_account_lookahead.get(), config.m_subaddress_lookahead.get()); @@ -1174,7 +1173,7 @@ namespace monero { wallet->set_daemon_connection(config.m_server, config.m_is_trusted_daemon); wallet->m_w2->set_seed_language(config.m_language.get()); wallet->init_common(); - return wallet; + return wallet.release(); } monero_wallet_full* monero_wallet_full::create_wallet_random(monero_wallet_config& config, std::unique_ptr http_client_factory) { @@ -1185,7 +1184,7 @@ namespace monero { if (config.m_restore_height != boost::none) throw std::runtime_error("Cannot specify restore height when creating random wallet"); // initialize wallet - monero_wallet_full* wallet = new monero_wallet_full(); + std::unique_ptr wallet(new monero_wallet_full()); if (http_client_factory == nullptr) wallet->m_w2 = std::unique_ptr(new tools::wallet2(static_cast(config.m_network_type.get()), 1, true)); else wallet->m_w2 = std::unique_ptr(new tools::wallet2(static_cast(config.m_network_type.get()), 1, true, std::move(http_client_factory))); wallet->m_w2->init(""); @@ -1201,7 +1200,7 @@ namespace monero { if (wallet->is_connected_to_daemon()) wallet->m_w2->set_refresh_from_block_height(wallet->get_daemon_height()); // must be set before generate so the height is written to the keys file wallet->m_w2->generate(config.m_path.get(), config.m_password.get(), secret_key, false, false); wallet->init_common(); - return wallet; + return wallet.release(); } std::vector monero_wallet_full::get_seed_languages() { @@ -1846,6 +1845,8 @@ namespace monero { tx_map.erase(tx->m_hash.get()); tx_iter = txs.erase(tx_iter); if (tx->m_block != nullptr) tx->m_block->m_txs.erase(std::remove(tx->m_block->m_txs.begin(), tx->m_block->m_txs.end(), tx), tx->m_block->m_txs.end()); // TODO, no way to use tx_iter? + tx->m_block = nullptr; // free this tx alone, not its block, which can hold txs that stay in the result + monero_utils::free(tx); } } txs = queried_txs; @@ -3884,6 +3885,13 @@ namespace monero { if (_query->m_tx_query == nullptr) _query->m_tx_query = std::make_shared(); std::shared_ptr tx_query = _query->m_tx_query; + // the copied query is local, so release its links when the call ends, on the path that + // returns and the path that throws, or the copy of an input query points back at the copy + // of the tx query and keeps the whole query alive + auto free_copied_query = epee::misc_utils::create_scope_leave_handler([&]() { + monero_utils::free(tx_query); + }); + // get output data from wallet2 tools::wallet2::transfer_container outputs_w2; m_w2->get_transfers(outputs_w2); @@ -3918,8 +3926,7 @@ namespace monero { if (tx->m_outputs.empty() && tx->m_block != nullptr) tx->m_block->m_txs.erase(std::remove(tx->m_block->m_txs.begin(), tx->m_block->m_txs.end(), tx), tx->m_block->m_txs.end()); // TODO, no way to use const_iterator? } - // free query and return outputs - monero_utils::free(tx_query); + // return outputs return outputs; } diff --git a/src/wallet/monero_wallet_keys.cpp b/src/wallet/monero_wallet_keys.cpp index 0e0895e5..458e248e 100644 --- a/src/wallet/monero_wallet_keys.cpp +++ b/src/wallet/monero_wallet_keys.cpp @@ -116,7 +116,7 @@ namespace monero { if (!monero_utils::is_valid_language(config_normalized.m_language.get())) throw std::runtime_error("Unknown language: " + config_normalized.m_language.get()); // initialize random wallet account - monero_wallet_keys* wallet = new monero_wallet_keys(); + std::unique_ptr wallet(new monero_wallet_keys()); crypto::secret_key spend_key_sk = wallet->m_account.generate(); // initialize remaining wallet @@ -129,7 +129,7 @@ namespace monero { wallet->m_seed = std::string(wipeable_mnemonic.data(), wipeable_mnemonic.size()); wallet->init_common(); - return wallet; + return wallet.release(); } monero_wallet_keys* monero_wallet_keys::create_wallet_from_seed(const monero_wallet_config& config) { @@ -150,7 +150,7 @@ namespace monero { if (config.m_seed_offset != boost::none && !config.m_seed_offset.get().empty()) spend_key_sk = cryptonote::decrypt_key(spend_key_sk, config.m_seed_offset.get()); // initialize wallet account - monero_wallet_keys* wallet = new monero_wallet_keys(); + std::unique_ptr wallet(new monero_wallet_keys()); wallet->m_account = cryptonote::account_base{}; wallet->m_account.generate(spend_key_sk, true, false); @@ -164,7 +164,7 @@ namespace monero { wallet->m_seed = std::string(wipeable_mnemonic.data(), wipeable_mnemonic.size()); wallet->init_common(); - return wallet; + return wallet.release(); } monero_wallet_keys* monero_wallet_keys::create_wallet_from_keys(const monero_wallet_config& config) { @@ -225,7 +225,7 @@ namespace monero { } // initialize wallet account - monero_wallet_keys* wallet = new monero_wallet_keys(); + std::unique_ptr wallet(new monero_wallet_keys()); if (has_spend_key && has_view_key) { wallet->m_account.create_from_keys(address_info.address, spend_key_sk, view_key_sk); } else if (has_spend_key) { @@ -247,7 +247,7 @@ namespace monero { } wallet->init_common(); - return wallet; + return wallet.release(); } std::vector monero_wallet_keys::get_seed_languages() { diff --git a/src/wallet/monero_wallet_model.cpp b/src/wallet/monero_wallet_model.cpp index 2d8a8526..9ff6aae6 100644 --- a/src/wallet/monero_wallet_model.cpp +++ b/src/wallet/monero_wallet_model.cpp @@ -653,9 +653,8 @@ namespace monero { std::shared_ptr monero_tx_query::deserialize_from_block(const std::string& tx_query_json) { // deserialize tx query std::string to property rooted at block - std::istringstream iss = tx_query_json.empty() ? std::istringstream() : std::istringstream(tx_query_json); boost::property_tree::ptree block_node; - boost::property_tree::read_json(iss, block_node); + gen_utils::deserialize(tx_query_json, block_node); // convert query property tree to block std::shared_ptr block = node_to_block_query(block_node); @@ -1138,9 +1137,8 @@ namespace monero { std::shared_ptr monero_transfer_query::deserialize_from_block(const std::string& transfer_query_json) { // deserialize transfer query std::string to property rooted at block - std::istringstream iss = transfer_query_json.empty() ? std::istringstream() : std::istringstream(transfer_query_json); boost::property_tree::ptree blockNode; - boost::property_tree::read_json(iss, blockNode); + gen_utils::deserialize(transfer_query_json, blockNode); // convert query property tree to block std::shared_ptr block = node_to_block_query(blockNode); @@ -1383,9 +1381,8 @@ namespace monero { std::shared_ptr monero_output_query::deserialize_from_block(const std::string& output_query_json) { // deserialize output query string to property rooted at block - std::istringstream iss = output_query_json.empty() ? std::istringstream() : std::istringstream(output_query_json); boost::property_tree::ptree blockNode; - boost::property_tree::read_json(iss, blockNode); + gen_utils::deserialize(output_query_json, blockNode); // convert query property tree to block std::shared_ptr block = node_to_block_query(blockNode);