Add the batch of release v3.4.3 commits

CMakeLists.txt

@@ -7,7 +7,6 @@ set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_C_STANDARD 11)
 set(CRYPTONOTE_NAME bytecoin)
 add_definitions(-DCRYPTONOTE_NAME=\"${CRYPTONOTE_NAME}\" -DNDEBUG=1)
-option(WITH_LEDGER "Builds with support of Ledger hardware wallets" OFF)
 option(USE_INSTRUMENTATION "For testing - builds with address sanitizer instrument" OFF)
 option(WITH_THREAD_SANITIZER "For testing - builds with thread sanitizer instrument, USE_INSTRUMENTATION must be also set" OFF)
 option(BETTER_DEBUG "Disables optimizations. We do not use standard debug/realease configurations because they change too much" OFF)
@@ -26,6 +25,7 @@ if(APPLE)
 endif()
 if(WIN32)
     add_definitions(-D_SCL_SECURE_NO_WARNINGS=1 -D_CRT_SECURE_NO_WARNINGS=1 -D_WIN32_WINNT=0x0501)
+    add_compile_options(/we4715)
 else()
     # We can build on old Ubuntu 14 with clang by adding next line (does not work with add_compile_options)
     # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
@@ -48,10 +48,9 @@ else()
         endif()
     endif()
 endif()
-include_directories(vendor/sqlite)
-include_directories(vendor/hmac)
-set(SRC_WARNINGS_DB vendor/sqlite/sqlite3.c)
-set(SRC_DB src/platform/DBsqlite3.cpp src/platform/DBsqlite3.hpp)
+include_directories(vendor)
+set(SRC_NO_WARNINGS vendor/sqlite/sqlite3.c)
+set(SRC_DB src/platform/DBsqlite3.cpp src/platform/DBsqlite3.hpp src/platform/DBmemory.cpp src/platform/DBmemory.hpp)
 if(USE_SQLITE)
     # Requires dl on Linux, we add it unconditionally for simplicity.
     message(STATUS "Database selected: SQLite 3")
@@ -62,7 +61,7 @@ else()
     if(NOT EXISTS "${PARENT_DIR}/lmdb/libraries/liblmdb/mdb.c")
         message(FATAL_ERROR "${PARENT_DIR}/lmdb/libraries/liblmdb/mdb.c not found. Make sure lmdb is cloned into ../lmdb/")
     endif()
-    set(SRC_WARNINGS_DB ${SRC_WARNINGS_DB} ../lmdb/libraries/liblmdb/mdb.c ../lmdb/libraries/liblmdb/midl.c)
+    set(SRC_NO_WARNINGS ${SRC_NO_WARNINGS} ../lmdb/libraries/liblmdb/mdb.c ../lmdb/libraries/liblmdb/midl.c)
     set(SRC_DB ${SRC_DB} src/platform/DBlmdb.cpp src/platform/DBlmdb.hpp)
 endif()
 
@@ -104,25 +103,49 @@ file(GLOB SRC_PLATFORM
         src/platform/PreventSleep.cpp src/platform/PreventSleep.hpp
         src/platform/Windows.hpp src/platform/DB.hpp
         )
+if(APPLE)
+    set(SRC_PLATFORM ${SRC_PLATFORM} src/platform/PathTools.mm)
+endif()
+
+include_directories(vendor/hidapi) # hid.c incorrectly reference hidapi.h
+if(WIN32)
+    set(HID_LIBRARY setupapi.lib)
+    set(SRC_NO_WARNINGS
+        ${SRC_NO_WARNINGS}
+        vendor/hidapi/windows/hid.c
+    )
+elseif(APPLE)
+    set(SRC_NO_WARNINGS
+        ${SRC_NO_WARNINGS}
+        vendor/hidapi/mac/hid.c
+    )
+else()
+    set(SRC_NO_WARNINGS
+        ${SRC_NO_WARNINGS}
+        vendor/hidapi/linux/hid.c
+    )
+    set(HID_LIBRARY "udev")
+endif()
+
 # We compile those folders with full optimization even in debug mode, otherwise binaries will run much slower in debug
 if(WIN32)
     set_property(SOURCE ${SRC_CRYPTO} PROPERTY COMPILE_FLAGS -Ot)
     set_property(SOURCE ${SRC_DB} PROPERTY COMPILE_FLAGS -Ot)
-    set_property(SOURCE ${SRC_WARNINGS_DB} PROPERTY COMPILE_FLAGS "-Ot -w")
+    set_property(SOURCE ${SRC_NO_WARNINGS} PROPERTY COMPILE_FLAGS "-Ot -w")
     set_property(SOURCE ${SRC_COMMON} PROPERTY COMPILE_FLAGS -Ot)
     set_property(SOURCE ${SRC_SERIA} PROPERTY COMPILE_FLAGS -Ot)
 else()
     set_property(SOURCE ${SRC_CRYPTO} PROPERTY COMPILE_FLAGS -O3)
     set_property(SOURCE ${SRC_DB} PROPERTY COMPILE_FLAGS -O3)
-    set_property(SOURCE ${SRC_WARNINGS_DB} PROPERTY COMPILE_FLAGS "-O3 -w")
+    set_property(SOURCE ${SRC_NO_WARNINGS} PROPERTY COMPILE_FLAGS "-O3 -w")
     set_property(SOURCE ${SRC_COMMON} PROPERTY COMPILE_FLAGS -O3)
     set_property(SOURCE ${SRC_SERIA} PROPERTY COMPILE_FLAGS -O3)
 endif()
 include_directories(src)
 include_directories(${CMAKE_BINARY_DIR})
 set(SOURCE_FILES
     ${SRC_DB}
-    ${SRC_WARNINGS_DB}
+    ${SRC_NO_WARNINGS}
     ${SRC_COMMON}
     ${SRC_HTTP}
     ${SRC_CORE}
@@ -136,40 +159,16 @@ set(SOURCE_FILES
     src/version.hpp
 )
 
-
-if(WITH_LEDGER)
-    # on Linux - apt-get install libusb-1.0-0-dev
-    # permission problem - https://raw.githubusercontent.com/LedgerHQ/udev-rules/master/add_udev_rules.sh
-    add_definitions(-Dcn_WITH_LEDGER=1)
-    set(LIBUSB_ROOT ${PARENT_DIR}/libusb)
-if(WIN32)
-    # Link libusb dynamically.
-    if(CMAKE_SIZEOF_VOID_P EQUAL 8)
-        set(USBLIB_LIBRARY ${LIBUSB_ROOT}/MS64/dll/libusb-1.0.lib)
-    else()
-        set(USBLIB_LIBRARY ${LIBUSB_ROOT}/MS32/dll/libusb-1.0.lib)
-    endif()
-elseif(APPLE)
-    # Link libusb statically.
-    set(USBLIB_LIBRARY ${LIBUSB_ROOT}/lib/libusb-1.0.a "-framework Foundation" "-framework IOKit")
-else()
-    set(USBLIB_LIBRARY "usb-1.0")
-endif()
-    message(STATUS "With ledger " ${USBLIB_LIBRARY})
-    include_directories(${LIBUSB_ROOT}/include)
-else()
-endif()
-
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bin")
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/libs")
 add_library(bytecoin-crypto ${SRC_CRYPTO})
+if(APPLE)
+    link_libraries(-dead_strip  "-framework Foundation")
+endif()
 add_library(bytecoin-core ${SOURCE_FILES})
-target_link_libraries(bytecoin-core bytecoin-crypto ${USBLIB_LIBRARY})
+target_link_libraries(bytecoin-core bytecoin-crypto ${HID_LIBRARY})
 
 link_libraries(bytecoin-crypto bytecoin-core)
-if(APPLE)
-    link_libraries(-dead_strip)
-endif()
 if(NOT WIN32)
     link_libraries(${LINK_OPENSSL} dl pthread)
 endif()
@@ -207,10 +206,10 @@ else()
 endif()
 include_directories(${Boost_INCLUDE_DIRS})
 
-target_link_libraries(walletd ${USBLIB_LIBRARY})
+target_link_libraries(walletd ${HID_LIBRARY})
 
 if(APPLE)
-    target_link_libraries(walletd "-framework Foundation" "-framework IOKit") # -dead_strip_dylibs
-    target_link_libraries(${CRYPTONOTE_NAME}d "-framework Foundation" "-framework IOKit") # -dead_strip_dylibs
+    target_link_libraries(walletd "-framework IOKit") # -dead_strip_dylibs
+    target_link_libraries(${CRYPTONOTE_NAME}d "-framework IOKit") # -dead_strip_dylibs
 endif()
 

README.md

@@ -1,6 +1,6 @@
 # Bytecoin
 
-[![Build Status](https://dev.azure.com/bcndev/bytecoin/_apis/build/status/bytecoin-daemons?branchName=releases/3.4.2)](https://dev.azure.com/bcndev/bytecoin/_build/latest?definitionId=1&branchName=releases/3.4.2)
+[![Build Status](https://dev.azure.com/bcndev/bytecoin/_apis/build/status/bytecoin-daemons?branchName=releases/3.4.3)](https://dev.azure.com/bcndev/bytecoin/_build/latest?definitionId=1&branchName=releases/3.4.3)
 
 ## About
 
@@ -31,6 +31,11 @@ To go futher you have to have a number of packages and utilities. You need at le
     $bcndev> sudo apt-get install build-essential
     ```
 
+* `libudev`:
+    ```
+    sudo apt-get install libudev-dev
+    ```
+    
 * CMake (3.0 or newer):
     ```
     $bcndev> sudo apt-get install cmake
@@ -40,12 +45,12 @@ To go futher you have to have a number of packages and utilities. You need at le
 
 * Boost (1.65 or newer):
     We use boost as a header-only library via find_boost package. So, if your system has boost installed and set up, it will be used automatically.
-    
+
     Note - there is a bug in `boost::asio` 1.66 that affects `bytecoind`. Please use either version 1.65 or 1.67+.
     ```
     $bcndev> sudo apt-get install libboost-dev
     ```
-    If the latest boost installed is too old (e.g. for Ubuntu 16.*), then you need to download and unpack boost into the `bcndev/boost` folder. 
+    If the latest boost installed is too old (e.g. for Ubuntu 16.*), then you need to download and unpack boost into the `bcndev/boost` folder.
 
     ```
     $bcndev> wget -c 'https://dl.bintray.com/boostorg/release/1.69.0/source/boost_1_69_0.tar.gz'
@@ -57,10 +62,10 @@ To go futher you have to have a number of packages and utilities. You need at le
 * OpenSSL (1.1.1 or newer):
     Install OpenSSL to `bcndev/openssl` folder. (In below commands use switch `linux-x86_64-clang` instead of `linux-x86_64` if using clang.)
     ```
-    $bcndev> git clone https://github.com/openssl/openssl.git
+    $bcndev> git clone --single-branch --branch OpenSSL_1_1_1b --depth 1 https://github.com/openssl/openssl.git
     $bcndev> cd openssl
     $bcndev/openssl> ./Configure linux-x86_64 no-shared
-    $bcndev/openssl> make -j4
+    $bcndev/openssl> make -j8
     $bcndev/openssl> cd ..
     ```
 
@@ -84,7 +89,7 @@ Create build directory inside bytecoin, go there and run CMake and Make:
 $bcndev> mkdir -p bytecoin/build
 $bcndev> cd bytecoin/build
 $bcndev/bytecoin/build> cmake ..
-$bcndev/bytecoin/build> make -j4
+$bcndev/bytecoin/build> make -j8
 ```
 
 Check built binaries by running them from `../bin` folder
@@ -121,39 +126,19 @@ $~/Downloads/bcndev> git clone https://github.com/bcndev/lmdb.git
 
 Install OpenSSL to `bcndev/openssl` folder:
 ```
-$~/Downloads/bcndev> git clone https://github.com/openssl/openssl.git
+$~/Downloads/bcndev> git clone --single-branch --branch OpenSSL_1_1_1b --depth 1 https://github.com/openssl/openssl.git
 $~/Downloads/bcndev> cd openssl
-```
-
-If you need binaries to run on all versions of OS X starting from El Capitan, you need to build OpenSSL targeting El Capitan SDK.
-```
-$~/Downloads/bcndev/openssl> ./Configure darwin64-x86_64-cc no-shared -mmacosx-version-min=10.11 -isysroot/Users/user/Downloads/MacOSX10.11.sdk
-```
-Otherwise just use
-```
-$~/Downloads/bcndev/openssl> ./Configure darwin64-x86_64-cc no-shared
-```
-
-```
-$~/Downloads/bcndev/openssl> make -j4
+$~/Downloads/bcndev/openssl> ./Configure darwin64-x86_64-cc no-shared -mmacosx-version-min=10.11
+$~/Downloads/bcndev/openssl> make -j8
 $~/Downloads/bcndev/openssl> cd ..
 ```
 
-Download amalgamated [SQLite 3](https://www.sqlite.org/download.html) and unpack it into `bcndev/sqlite` folder (source files are referenced via relative paths, so you do not need to separately build it).
-Please, note the direct download link is periodically updated with old versions removed, so you might need to tweak instructions below
-```
-$~/Downloads/bcndev> wget -c https://www.sqlite.org/2018/sqlite-amalgamation-3260000.zip
-$~/Downloads/bcndev> unzip sqlite-amalgamation-3260000.zip
-$~/Downloads/bcndev> rm sqlite-amalgamation-3260000.zip
-$~/Downloads/bcndev> mv sqlite-amalgamation-3260000 sqlite
-```
-
 Create build directory inside bytecoin, go there and run CMake and Make:
 ```
 $~/Downloads/bcndev> mkdir bytecoin/build
 $~/Downloads/bcndev> cd bytecoin/build
 $~/Downloads/bcndev/bytecoin/build> cmake ..
-$~/Downloads/bcndev/bytecoin/build> make -j4
+$~/Downloads/bcndev/bytecoin/build> make -j8
 ```
 
 Check built binaries by running them from `../bin` folder:
@@ -189,15 +174,15 @@ Download amalgamated [SQLite 3](https://www.sqlite.org/download.html) and unpack
 
 You need to build openssl, first install ActivePerl (select "add to PATH" option, then restart console):
 ```
-$C:\bcndev> git clone https://github.com/openssl/openssl.git
+$C:\bcndev> git clone --single-branch --branch OpenSSL_1_1_1b --depth 1 https://github.com/openssl/openssl.git
 $C:\bcndev> cd openssl
 $C:\bcndev\openssl> perl Configure VC-WIN64A no-shared no-asm
 $C:\bcndev\openssl> nmake
 $C:\bcndev\openssl> cd ..
 ```
 If you want to build 32-bit binaries, you will also need 32-bit build of openssl in separate folder (configuring openssl changes header files, so there is no way to have both 32-bit and 64-bit versions in the same folder):
 ```
-$C:\bcndev> git clone https://github.com/openssl/openssl.git openssl32
+$C:\bcndev> git clone --single-branch --branch OpenSSL_1_1_1b --depth 1 https://github.com/openssl/openssl.git openssl32
 $C:\bcndev> cd openssl32
 $C:\bcndev\openssl> perl Configure VC-WIN32 no-shared no-asm
 $C:\bcndev\openssl> nmake
@@ -233,13 +218,3 @@ Currently bytecoin does not work out of the box on any Big-Endian platform, due
 ## Building with parameters
 
 If you want to use tools like `clang-tidy`, run `cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..` instead of `cmake ..`
-
-## Building daemons with hardware wallet support on Linux 64-bit
-
-1. Clone `trezor-core` repository into the same folder where `bytecoin` resides.
-2. Install all Google protobuf stuff:
-```
-sudo apt install protobuf-compiler libprotobuf-dev
-```
-3. If your version of proto buffers library is not `3.0.0`, you should run `protoc` on proto files in `trezor-core/vendor/trezor-common/protob` overwriting `bytecoin/src/Core/hardware/trezor/protob`.
-4. Clean your `bytecoin/build` folder if you have built the Bytecoin source code before.

ReleaseNotes.md

@@ -1,19 +1,40 @@
 ## Release Notes
 
+### v3.4.3 (Amethyst)
+
+- In `get_transfers` `walletd`'s method error is returned if `from_height` is larger than `to_height` or top block height.
+- `WALLET_FILE_EXISTS (209)` is correctly returned when `walletd` is instructed to overwrite an existing wallet.
+- Fixed a bug when wrong error message was displayed when passing some invalid addresses to `walletd`'s `create_transaction` RPC method.
+- LMDB virtual memory usage by `bytecoind` and `walletd` reduced from fixed 512Gb to approx. actual blockchain database/wallet cache size.
+- `walletd` will not undo sync progress, if connected node is behind wallet state and behind latest hard checkpoint. It will wait for a node to advance to a checkpoint.
+- Wallet cache now takes much less space, especially for wallets with small number of transactions.
+- For all TCP sockets `keep_alive` option is now set after they are created (with default timeouts for each system, common value is 2 hours), solving very rare bug when there would be no reply to long poll ever for external services (like block explorers) calling from remote machines.
+- All paths now subject to substitution of `~` (Mac & Linux) and `%appdata%` (Windows). Also, on Linux and Max backslash is no more considered path separator.
+
+*API tweaks*
+- JSON numbers are interpreted according to spec throughout API, for example 10.01E2 is now good value for height (1001). This helps integration with some weird languages.
+- Better error messages throughout API when required parameters are not specified.
+- More strict JSON RPC - excess fields on top level are no more allowed in requests.
+- `bytecoind`'s binary version of `sync_blocks` now uses separate zero overhead response to save traffic during sync.
+
+*Incompatible API changes*
+- `bytecoind`'s `getblocktemplate`, `submitblock` and JSON versions of `sync_blocks` and `sync_mempool` now require private authorization. This helps with preventing excess load on public nodes.
+- deprecated field `outs_count` in `bytecoind.get_random_outputs` removed.
+
 ### v3.4.2 (Amethyst)
 
 - Fixed merge mining related bug affecting blocks version 4 (amethyst). All miners not upgraded to 3.4.2 at the moment of consensus update will produce broken blocks.
 - Fixed crash when disconnecting Ledger while scanning blockchain.
-- Fixed behavior of the get_transfers method when transactions from a memory pool are wrongly returned for some `from_height` and `to_height` values. 
+- Fixed behavior of the `get_transfers` `walletd`'s method when transactions from a memory pool are wrongly returned for some `from_height` and `to_height` values.
 
 *Incompatible API changes*
 - In response to the `get_wallet_info` `walletd`'s method, boolean field `amethyst` changed to string `wallet_type`
 
-### v3.4.2-beta-20190412 (Amethyst)
+### v3.4.2-beta-20190412
 
 - Fixed a stagenet voting bug.
 
-### v3.4.2-beta-20190411 (Amethyst)
+### v3.4.2-beta-20190411
 
 - Fixed problem when `bytecoind` stops responding via JSON RPC API.
 - Tweaked random output distribution for mixins.