Add the batch of release hardware-wallets-alpha-20190214 commits

.gitignore

@@ -0,0 +1,22 @@
+# IDEs/editors
+.idea/
+.vs/
+xcuserdata
+/bytecoin.cbp
+*swp
+WorkspaceSettings.xcsettings
+
+# cmake specific
+CMakeLists.txt.user*
+cmake-build-*
+
+# Build
+bin/
+libs/
+build/
+
+# Tests
+tests/scratchpad/
+
+# Mac specific
+.DS_Store

CMakeLists.txt

@@ -7,6 +7,8 @@ set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_C_STANDARD 11)
 set(CRYPTONOTE_NAME bytecoin)
 add_definitions(-DCRYPTONOTE_NAME=\"${CRYPTONOTE_NAME}\")
+option(WITH_TREZOR "Builds with support of Trezor hardware wallets" ON)
+option(WITH_LEDGER "Builds with support of Ledger hardware wallets" OFF)
 option(USE_INSTRUMENTATION "For testing - builds with address sanitizer instrument" OFF)
 option(WITH_THREAD_SANITIZER "For testing - builds with thread sanitizer instrument, USE_INSTRUMENTATION must be also set" OFF)
 option(BETTER_DEBUG "Disables optimizations. We do not use standard debug/realease configurations because they change too much" OFF)
@@ -86,18 +88,24 @@ file(GLOB SRC_COMMON src/common/*.cpp src/common/*.hpp)
 file(GLOB SRC_SERIA src/seria/*.cpp src/seria/*.hpp)
 file(GLOB SRC_LOGGING src/logging/*.cpp src/logging/*.hpp)
 file(GLOB SRC_P2P src/p2p/*.cpp src/p2p/*.hpp)
-file(GLOB SRC_CORE src/Core/*.cpp src/Core/*.hpp src/Core/hw/*.cpp src/Core/hw/*.hpp  src/CryptoNote.hpp src/CryptoNote.cpp src/rpc_api.hpp src/rpc_api.cpp)
+file(GLOB SRC_CORE src/Core/*.cpp src/Core/*.hpp
+        src/Core/hardware/*.cpp src/Core/hardware/*.hpp
+        src/CryptoNote.hpp src/CryptoNote.cpp
+        src/rpc_api.hpp src/rpc_api.cpp)
 file(GLOB SRC_HTTP src/http/*.cpp src/http/*.hpp)
 file(GLOB SRC_PLATFORM
-    src/platform/ExclusiveLock.cpp src/platform/ExclusiveLock.hpp
-    src/platform/Files.cpp src/platform/Files.hpp
-    src/platform/Ledger.cpp src/platform/Ledger.hpp
-    src/platform/Time.cpp src/platform/Time.hpp
-    src/platform/Network.cpp src/platform/Network.hpp
-    src/platform/PathTools.cpp src/platform/PathTools.hpp
-    src/platform/PreventSleep.cpp src/platform/PreventSleep.hpp
-    src/platform/Windows.hpp src/platform/DB.hpp
-)
+        src/platform/ExclusiveLock.cpp src/platform/ExclusiveLock.hpp
+        src/platform/Files.cpp src/platform/Files.hpp
+        src/platform/Ledger.cpp src/platform/Ledger.hpp
+        src/platform/Time.cpp src/platform/Time.hpp
+        src/platform/Network.cpp src/platform/Network.hpp
+        src/platform/PathTools.cpp src/platform/PathTools.hpp
+        src/platform/PreventSleep.cpp src/platform/PreventSleep.hpp
+        src/platform/Windows.hpp src/platform/DB.hpp
+        )
+#file(GLOB SRC_HARDWARE
+#        src/Core/hw/trezor/protob/*.cpp src/Core/hw/trezor/protob/*.hpp
+#        )
 # We compile those folders with full optimization even in debug mode, otherwise binaries will run much slower in debug
 if(WIN32)
     set_property(SOURCE ${SRC_CRYPTO} PROPERTY COMPILE_FLAGS -Ot)
@@ -129,18 +137,49 @@ set(SOURCE_FILES
     src/rpc_api.hpp
     src/version.hpp
 )
+
+
+if(WITH_TREZOR)
+    include_directories(${PARENT_DIR}/trezor-core/vendor/trezor-common)
+    add_definitions(-Dcn_WITH_TREZOR=1)
+    # howto not working, best attempt at https://stackoverflow.com/questions/20824194/cmake-with-google-protocol-buffers
+    # file(GLOB ProtoFiles "../trezor-core/vendor/trezor-common/protob/*.proto")
+    #protobuf_generate(LANGUAGE cpp TARGET bytecoin-core PROTOS ${ProtoFiles})
+    set(Protobuf_USE_STATIC_LIBS ON)
+    find_package(Protobuf REQUIRED)
+    file(GLOB SRC_PROTOB ../trezor-core/vendor/trezor-common/protob/*.h ../trezor-core/vendor/trezor-common/protob/*.cc)
+    include_directories(${PROTOBUF_INCLUDE_DIR})
+    message(STATUS "Protobuf version " ${Protobuf_VERSION} " protoc is " ${Protobuf_PROTOC_EXECUTABLE} " headers are in  " ${PROTOBUF_INCLUDE_DIR} " and static libs are in " ${PROTOBUF_LIBRARY})
+    message(STATUS "Proto files found " ${ProtoFiles})
+else()
+endif()
+if(WITH_LEDGER)
+    # on Linux - apt-get install libusb-1.0-0-dev
+    # permission problem - https://raw.githubusercontent.com/LedgerHQ/udev-rules/master/add_udev_rules.sh
+    add_definitions(-Dcn_WITH_LEDGER=1)
+if(WIN32)
+    set(USBLIB_LIBRARY ${PARENT_DIR}/libusb/x64/Release/lib/libusb-1.0.lib)
+else()
+    set(USBLIB_LIBRARY "usb-1.0")
+endif()
+    message(STATUS "With ledger " ${USBLIB_LIBRARY})
+    include_directories(${PARENT_DIR}/libusb)
+else()
+endif()
+
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bin")
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/libs")
 add_library(bytecoin-crypto ${SRC_CRYPTO})
-add_library(bytecoin-core ${SOURCE_FILES})
-target_link_libraries(bytecoin-core bytecoin-crypto)
+add_library(bytecoin-core ${SOURCE_FILES} ${SRC_PROTOB})
+target_link_libraries(bytecoin-core bytecoin-crypto  ${PROTOBUF_LIBRARY} ${USBLIB_LIBRARY})
 if(WIN32)
     add_executable(walletd src/main_walletd.cpp src/bytecoin.rc) # .rc works only if referenced directly in add_executable
     add_executable(${CRYPTONOTE_NAME}d src/main_bytecoind.cpp src/bytecoin.rc) # .rc works only if referenced directly in add_executable
 else()
     add_executable(walletd src/main_walletd.cpp)
     add_executable(${CRYPTONOTE_NAME}d src/main_bytecoind.cpp)
 endif()
+
 add_executable(tests src/main_tests.cpp tests/io.hpp tests/Random.hpp
         tests/blockchain/test_blockchain.cpp tests/blockchain/test_blockchain.hpp
         tests/crypto/test_crypto.cpp tests/crypto/test_crypto.hpp
@@ -165,9 +204,9 @@ else()
 endif()
 include_directories(${Boost_INCLUDE_DIRS})
 
-target_link_libraries(walletd bytecoin-crypto bytecoin-core)
-target_link_libraries(${CRYPTONOTE_NAME}d bytecoin-crypto bytecoin-core)
-target_link_libraries(tests bytecoin-crypto bytecoin-core)
+target_link_libraries(walletd bytecoin-crypto bytecoin-core ${PROTOBUF_LIBRARY})
+target_link_libraries(${CRYPTONOTE_NAME}d bytecoin-crypto bytecoin-core ${PROTOBUF_LIBRARY})
+target_link_libraries(tests bytecoin-crypto bytecoin-core ${PROTOBUF_LIBRARY})
 
 if(NOT WIN32)
     if(APPLE)

README.md

@@ -77,6 +77,12 @@ To go futher you have to have a number of packages and utilities. You need at le
     Please note, we use LMDB only when building 64-bit daemons. For 32-bit daemons SQLite is used instead.
     ```
     $bcndev> git clone https://github.com/bcndev/lmdb.git
+
+    ```
+
+* Protobuf (if building walletd with Trezor support)
+    ```
+    $bcndev> sudo apt-get install libprotobuf-dev
     ```
 
 Git-clone (or git-pull) Bytecoin source code in that folder:
@@ -238,3 +244,13 @@ Currently bytecoin does not work out of the box on any Big-Endian platform, due
 ## Building with parameters
 
 If you want to use tools like `clang-tidy`, run `cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..` instead of `cmake ..`
+
+## Building daemons with hardware wallet support on Linux 64-bit
+
+1. Clone `trezor-core` repository into the same folder where `bytecoin` resides.
+2. Install all Google protobuf stuff:
+```
+sudo apt install protobuf-compiler libprotobuf-dev
+```
+3. If your version of proto buffers library is not `3.0.0`, you should run `protoc` on proto files in `trezor-core/vendor/trezor-common/protob` overwriting `bytecoin/src/Core/hardware/trezor/protob`.
+4. Clean your `bytecoin/build` folder if you have built the Bytecoin source code before.

ReleaseNotes.md

@@ -1,5 +1,15 @@
 ## Release Notes
 
+### hardware-wallets-alpha-20190214
+
+- Added an early support for hardware wallets.
+- Fully working Trezor Model T prototype.
+- Partial support for Ledger Nano.
+
+*Current Limitations*
+- If you disconnect a hardware wallet while `walletd` is running, it will immediately crash.
+- Works in the stagenet only.
+
 ### v3.4.0 (Amethyst)
 
 - Sendproofs are now in base58 format, which eases copying and sharing.

src/Core/BlockChainState.cpp

@@ -229,8 +229,8 @@ BlockChainState::BlockChainState(logging::ILogger &log, const Config &config, co
 	}
 	// Upgrades from 5 should restart internal import if m_internal_import_chain is not empty
 	if (version != version_current)
-		throw std::runtime_error("Blockchain database format unknown (version=" + version + "), please delete " +
-		                         config.get_data_folder() + "/blockchain");
+		throw Exception("Blockchain database format too new (version=" + version + "), please delete " +
+		                config.get_data_folder() + "/blockchain");
 	if (get_tip_height() == (Height)-1) {
 		//		Block genesis_block;
 		//		genesis_block.header = currency.genesis_block_template;
@@ -522,8 +522,11 @@ void BlockChainState::create_mining_block_template(const Hash &parent_bid, const
 
 	b->previous_block_hash                = parent_bid;
 	const Timestamp next_median_timestamp = calculate_next_median_timestamp(parent_info);
-	b->root_block.timestamp               = std::max(platform::now_unix_timestamp(), next_median_timestamp);
-	b->timestamp                          = b->root_block.timestamp;
+	auto now                              = platform::now_unix_timestamp();
+	if (*height < 100)  // Tweak for testnet so first 100 blocks are mined quickly
+		now -= (100 - *height) * m_currency.difficulty_target;
+	b->root_block.timestamp = std::max(now, next_median_timestamp);
+	b->timestamp            = b->root_block.timestamp;
 
 	size_t max_txs_size          = 0;
 	size_t effective_size_median = 0;

src/Core/BlockChainState.hpp

@@ -37,6 +37,10 @@ class IBlockChainState {
 
 class BlockChainState : public BlockChain, private IBlockChainState {
 public:
+	class Exception : public std::runtime_error {
+	public:
+		using std::runtime_error::runtime_error;
+	};
 	BlockChainState(logging::ILogger &, const Config &, const Currency &, bool read_only);
 
 	std::vector<api::Output> get_random_outputs(uint8_t block_major_version, Amount, size_t output_count, Height,

src/Core/Currency.cpp

@@ -314,9 +314,9 @@ Transaction Currency::construct_miner_tx(
 
 	Amount summary_amounts = 0;
 	for (size_t out_index = 0; out_index < out_amounts.size(); out_index++) {
-		const KeyPair output_det_keys = crypto::random_keypair();
-		OutputKey tk                  = TransactionBuilder::create_output(
-            is_tx_amethyst, miner_address, txkey.secret_key, tx_inputs_hash, out_index, output_det_keys.public_key);
+		const KeyPair output_seed_keys = crypto::random_keypair();
+		OutputKey tk                   = TransactionBuilder::create_output(
+            is_tx_amethyst, miner_address, txkey.secret_key, tx_inputs_hash, out_index, output_seed_keys.public_key);
 		tk.amount = out_amounts.at(out_index);
 		summary_amounts += tk.amount;
 		tx.outputs.push_back(tk);
@@ -575,9 +575,9 @@ Hash cn::get_transaction_hash(const Transaction &tx) {
 		BinaryArray binary_sigs = seria::to_binary(tx.signatures, static_cast<const TransactionPrefix &>(tx));
 		ha.second               = crypto::cn_fast_hash(binary_sigs.data(), binary_sigs.size());
 		BinaryArray ba          = seria::to_binary(ha);
-//		BinaryArray tx_body = seria::to_binary(static_cast<const TransactionPrefix&>(tx));
-//		common::append(tx_body, binary_sigs);
-//		invariant(tx_body == seria::to_binary(tx), "");
+		//		BinaryArray tx_body = seria::to_binary(static_cast<const TransactionPrefix&>(tx));
+		//		common::append(tx_body, binary_sigs);
+		//		invariant(tx_body == seria::to_binary(tx), "");
 		return crypto::cn_fast_hash(ba.data(), ba.size());
 	}
 	BinaryArray ba = seria::to_binary(tx);

src/Core/Multicore.cpp

@@ -250,38 +250,42 @@ WalletPreparatorMulticore::~WalletPreparatorMulticore() {
 		th.join();
 }
 
-PreparedWalletTransaction::PreparedWalletTransaction(TransactionPrefix &&ttx, const Wallet::OutputHandler &o_handler)
+PreparedWalletTransaction::PreparedWalletTransaction(
+    TransactionPrefix &&ttx, const Wallet::OutputHandler &o_handler, const SecretKey &view_secret_key)
     : tx(std::move(ttx)) {
 	// We ignore results of most crypto calls here and absence of tx_public_key
-	// All errors will lead to spend_key not found in our wallet
+	// All errors will lead to spend_key not found in our wallet for legacy crypto
 	PublicKey tx_public_key = extra_get_transaction_public_key(tx.extra);
-	prefix_hash             = get_transaction_prefix_hash(tx);
-	inputs_hash             = get_transaction_inputs_hash(tx);
+	derivation              = generate_key_derivation(tx_public_key, view_secret_key);
+
+	prefix_hash = get_transaction_prefix_hash(tx);
+	inputs_hash = get_transaction_inputs_hash(tx);
 
 	KeyPair tx_keys;
 	address_public_keys.resize(tx.outputs.size());
-	output_spend_scalars.resize(tx.outputs.size());
+	output_secret_hashes.resize(tx.outputs.size());
 	for (size_t out_index = 0; out_index != tx.outputs.size(); ++out_index) {
 		const auto &output = tx.outputs.at(out_index);
 		if (output.type() != typeid(OutputKey))
 			continue;
 		const auto &key_output = boost::get<OutputKey>(output);
-		o_handler(tx.version, tx_public_key, &derivation, inputs_hash, out_index, key_output,
-		    &address_public_keys.at(out_index), &output_spend_scalars.at(out_index));
+		o_handler(tx.version, derivation, inputs_hash, out_index, key_output, &address_public_keys.at(out_index),
+		    &output_secret_hashes.at(out_index));
 	}
 }
 
-PreparedWalletTransaction::PreparedWalletTransaction(Transaction &&tx, const Wallet::OutputHandler &o_handler)
-    : PreparedWalletTransaction(std::move(static_cast<TransactionPrefix &&>(tx)), o_handler) {}
+PreparedWalletTransaction::PreparedWalletTransaction(
+    Transaction &&tx, const Wallet::OutputHandler &o_handler, const SecretKey &view_secret_key)
+    : PreparedWalletTransaction(std::move(static_cast<TransactionPrefix &&>(tx)), o_handler, view_secret_key) {}
 
 PreparedWalletBlock::PreparedWalletBlock(BlockTemplate &&bc_header, std::vector<TransactionPrefix> &&raw_transactions,
-    Hash base_transaction_hash, const Wallet::OutputHandler &o_handler)
+    Hash base_transaction_hash, const Wallet::OutputHandler &o_handler, const SecretKey &view_secret_key)
     : base_transaction_hash(base_transaction_hash) {
 	header           = bc_header;
-	base_transaction = PreparedWalletTransaction(std::move(bc_header.base_transaction), o_handler);
+	base_transaction = PreparedWalletTransaction(std::move(bc_header.base_transaction), o_handler, view_secret_key);
 	transactions.reserve(raw_transactions.size());
 	for (size_t tx_index = 0; tx_index != raw_transactions.size(); ++tx_index) {
-		transactions.emplace_back(std::move(raw_transactions.at(tx_index)), o_handler);
+		transactions.emplace_back(std::move(raw_transactions.at(tx_index)), o_handler, view_secret_key);
 	}
 }
 
@@ -308,7 +312,7 @@ void WalletPreparatorMulticore::thread_run() {
 			work.blocks.erase(work.blocks.begin());
 		}
 		PreparedWalletBlock result(std::move(sync_block.raw_header), std::move(sync_block.raw_transactions),
-		    sync_block.transactions.at(0).hash, o_handler);
+		    sync_block.transactions.at(0).hash, o_handler, m_view_secret_key);
 		{
 			std::unique_lock<std::mutex> lock(mu);
 			if (local_work_counter == work_counter) {
@@ -326,11 +330,12 @@ void WalletPreparatorMulticore::cancel_work() {
 	work_counter += 1;
 }
 
-void WalletPreparatorMulticore::start_work(
-    const api::cnd::SyncBlocks::Response &new_work, Wallet::OutputHandler &&o_handler) {
+void WalletPreparatorMulticore::start_work(const api::cnd::SyncBlocks::Response &new_work,
+    Wallet::OutputHandler &&o_handler, const SecretKey &view_secret_key) {
 	std::unique_lock<std::mutex> lock(mu);
-	work        = new_work;
-	m_o_handler = std::move(o_handler);
+	work              = new_work;
+	m_o_handler       = std::move(o_handler);
+	m_view_secret_key = view_secret_key;
 	work_counter += 1;
 	have_work.notify_all();
 }

src/Core/Multicore.hpp

@@ -93,13 +93,15 @@ struct PreparedWalletTransaction {
 	TransactionPrefix tx;
 	Hash prefix_hash;
 	Hash inputs_hash;
-	boost::optional<KeyDerivation> derivation;  // Will be assigned on first actual use
+	KeyDerivation derivation;  // Will be KeyDerivation{} if invalid or no tx_public_key
 	std::vector<PublicKey> address_public_keys;
-	std::vector<SecretKey> output_spend_scalars;
+	std::vector<SecretKey> output_secret_hashes;
 
 	PreparedWalletTransaction() = default;
-	PreparedWalletTransaction(TransactionPrefix &&tx, const Wallet::OutputHandler &o_handler);
-	PreparedWalletTransaction(Transaction &&tx, const Wallet::OutputHandler &o_handler);
+	PreparedWalletTransaction(
+	    TransactionPrefix &&tx, const Wallet::OutputHandler &o_handler, const SecretKey &view_secret_key);
+	PreparedWalletTransaction(
+	    Transaction &&tx, const Wallet::OutputHandler &o_handler, const SecretKey &view_secret_key);
 };
 
 struct PreparedWalletBlock {
@@ -109,7 +111,7 @@ struct PreparedWalletBlock {
 	std::vector<PreparedWalletTransaction> transactions;
 	PreparedWalletBlock() = default;
 	PreparedWalletBlock(BlockTemplate &&bc_header, std::vector<TransactionPrefix> &&raw_transactions,
-	    Hash base_transaction_hash, const Wallet::OutputHandler &o_handler);
+	    Hash base_transaction_hash, const Wallet::OutputHandler &o_handler, const SecretKey &view_secret_key);
 };
 
 class WalletPreparatorMulticore {
@@ -123,13 +125,15 @@ class WalletPreparatorMulticore {
 	api::cnd::SyncBlocks::Response work;
 	int work_counter = 0;
 	Wallet::OutputHandler m_o_handler;
+	SecretKey m_view_secret_key;
 	void thread_run();
 
 public:
 	WalletPreparatorMulticore();
 	~WalletPreparatorMulticore();
 	void cancel_work();
-	void start_work(const api::cnd::SyncBlocks::Response &new_work, Wallet::OutputHandler &&o_handler);
+	void start_work(const api::cnd::SyncBlocks::Response &new_work, Wallet::OutputHandler &&o_handler,
+	    const SecretKey &view_secret_key);
 	PreparedWalletBlock get_ready_work(Height height);
 };
 }  // namespace cn

src/Core/Node.cpp

@@ -786,9 +786,9 @@ void Node::check_sendproof(const BinaryArray &data_inside_base58, api::cnd::Chec
 		    api::cnd::CheckSendproof::FAILED_TO_PARSE, "Failed to parse proof object - too many bytes");
 
 	const auto proof_body = seria::to_binary(sp);
-//	std::cout << "Proof body: " << common::to_hex(proof_body) << std::endl;
+	//	std::cout << "Proof body: " << common::to_hex(proof_body) << std::endl;
 	const auto proof_prefix_hash = crypto::cn_fast_hash(proof_body);
-//	std::cout << "Proof hash: " << proof_prefix_hash << std::endl;
+	//	std::cout << "Proof hash: " << proof_prefix_hash << std::endl;
 
 	std::vector<KeyImage> all_keyimages{in.key_image};
 	std::vector<std::vector<PublicKey>> all_output_keys{m_block_chain.get_mixed_public_keys(in)};