Pre-taproot elements updates

.gitlab-ci.yml

@@ -1,87 +1,67 @@
 variables:
   GIT_SUBMODULE_STRATEGY: recursive
 
-build_wally_release_files:
-  image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c
-  artifacts:
-    expire_in: 7 days
-    name: wallycore-bindings
-    when: on_success
-    paths:
-    - dist/*
-  tags:
-    - ga
-  script:
-    - python3 -m build
-    - virtualenv -p python3 .smoketest
-    - source .smoketest/bin/activate
-    - pip install --find-links=./dist wallycore
-    - python -c "import wallycore as w; assert w.hex_from_bytes(w.hex_to_bytes('ff')) == 'ff'"
-    - deactivate
-    - rm -rf .smoketest dist/*.whl
-    - mv dist wally_dist
-    - ./tools/build_android_libraries.sh
-    - mv release wallycore-android-jni
-    - tar czf wally_dist/wallycore-android-jni.tar.gz --remove-files wallycore-android-jni
-    - source /opt/emsdk/emsdk_env.sh
-    - tools/build_wasm.sh
-    - cd dist
-    - tar czf wallycore-wasm.tar.gz --remove-files wallycore.html wallycore.js wallycore.wasm
-    - cd ..
-    - sphinx-build -b html -a -c docs/source docs/source docs/build/html
-    - cd docs/build
-    - tar czf ../../wally_dist/apidocs.tar.gz html/
-    - cd ../..
-    - mv wally_dist/* dist/
-    - rmdir wally_dist
+stages:
+  - test
+  - release
 
-build_mingw_static:
-  image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c
-  tags:
-    - ga
-  script:
-    - ./tools/cleanup.sh && ./tools/autogen.sh
-    - CC=x86_64-w64-mingw32-gcc ./configure --host=x86_64-w64-mingw32 --disable-swig-python --disable-swig-java --disable-shared --enable-static
-    - make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2))
-
-run_tests:
+test_with_valgrind:
+  stage: test
   image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c
+  parallel:
+    matrix:
+      - CONFIGURE_ARGS: [--enable-elements=yes,--enable-elements=no,--enable-minimal=yes]
   tags:
     - ga
   artifacts:
     reports:
       codequality: valgrind.json
   script:
     - ./tools/cleanup.sh && ./tools/autogen.sh
-    - ./configure --enable-export-all --enable-swig-python --enable-swig-java --enable-shared --disable-static
+    - CFLAGS='-Werror' ./configure --enable-export-all --enable-swig-python --enable-swig-java $CONFIGURE_ARGS --enable-shared --disable-static
     - make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2))
     - make check -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2))
     - for t in $(ls src/.libs/test_* | egrep -v '_clear|xml|json' | tr '\n' ' '); do LD_LIBRARY_PATH=./src/.libs/ valgrind --tool=memcheck --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose --xml=yes --xml-file=$t.xml $t; done
     - for t in $(ls src/.libs/test_* | egrep -v '_clear|xml|json' | tr '\n' ' '); do valgrind-codequality --input-file $t.xml --output-file $t.json; done
     - jq '[.[]|.[]]' -s ./src/.libs/test_*.json > valgrind.json
 
-run_asan_ubsan_gcc:
+test_asan_ubsan_gcc:
+  stage: test
   image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c
+  parallel:
+    matrix:
+      - CONFIGURE_ARGS: [--enable-elements=yes, --enable-elements=no,--enable-minimal=yes]
   tags:
     - ga
   script:
     - ./tools/cleanup.sh && ./tools/autogen.sh
-    - CC=gcc CFLAGS="-O1 -fsanitize=address -fsanitize=undefined -fsanitize=alignment -fsanitize-address-use-after-scope -fno-sanitize-recover=all" ./configure --enable-export-all --enable-swig-python --enable-swig-java --enable-shared --disable-static --disable-clear-tests --disable-asm
+    - CC=gcc CFLAGS="-O2 -fsanitize=address -fsanitize=undefined -fsanitize=alignment -fsanitize-address-use-after-scope -fno-sanitize-recover=all" ./configure --enable-export-all --enable-swig-python --enable-swig-java $CONFIGURE_ARGS --enable-shared --disable-static --disable-clear-tests --disable-asm
     - sed -i 's/^PYTHON = /PYTHON = LD_PRELOAD=\/usr\/lib\/gcc\/x86_64-linux-gnu\/10\/libasan.so /g' src/Makefile
     - sed -i 's/^JAVA = /JAVA = LD_PRELOAD=\/usr\/lib\/gcc\/x86_64-linux-gnu\/10\/libasan.so /g' src/Makefile
     - make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2))
     - ASAN_OPTIONS=abort_on_error=1:fast_unwind_on_malloc=0:detect_leaks=0 UBSAN_OPTIONS=print_stacktrace=1 make check V=1
 
-run_scan_build_clang:
+test_scan_build_clang:
+  stage: test
   image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c
+  parallel:
+    matrix:
+      - CONFIGURE_ARGS: [--enable-elements=yes, --enable-elements=no,--enable-minimal=yes]
   tags:
     - ga
   script:
     - ./tools/cleanup.sh && ./tools/autogen.sh
-    - CC=clang CFLAGS="-O0" scan-build-11 ./configure --enable-export-all --enable-swig-python --enable-swig-java --disable-clear-tests --disable-asm
-    - scan-build-11 --keep-cc --exclude src/secp256k1/ make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2))
+    - CC=clang scan-build-11 ./configure --enable-export-all --enable-swig-python --enable-swig-java --disable-clear-tests --disable-asm
+    - scan-build-11 --keep-cc --exclude src/secp256k1/ --status-bugs --keep-empty -o scan-build$CONFIGURE_ARGS make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2))
+  artifacts:
+    expire_in: 3 days
+    name: scan-build$CONFIGURE_ARGS
+    when: on_success
+    paths:
+    - scan-build$CONFIGURE_ARGS/*
 
-cmake-test:
+test_cmake:
+  stage: test
   image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c
   tags:
     - ga
@@ -110,7 +90,8 @@ cmake-test:
         coverage_format: cobertura
         path: coverage.xml
 
-amalgamation-test:
+test_amalgamation:
+  stage: test
   image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c
   tags:
     - ga
@@ -120,3 +101,49 @@ amalgamation-test:
     - gcc -DBUILD_ELEMENTS -Wall -W -Wextra -Werror -I. -I./src -I./src/ccan -I./src/secp256k1/include src/ctest/amalgamation_compile_test.c
     - clang -Wall -W -Wextra -Werror -I. -I./src -I./src/ccan -I./src/secp256k1/include src/ctest/amalgamation_compile_test.c
     - clang -DBUILD_ELEMENTS -Wall -W -Wextra -Werror -I. -I./src -I./src/ccan -I./src/secp256k1/include src/ctest/amalgamation_compile_test.c
+
+test_mingw_static_build:
+  stage: test
+  image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c
+  tags:
+    - ga
+  script:
+    - ./tools/cleanup.sh && ./tools/autogen.sh
+    - CC=x86_64-w64-mingw32-gcc ./configure --host=x86_64-w64-mingw32 --disable-swig-python --disable-swig-java --disable-shared --enable-static
+    - make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2))
+
+build_wally_release_files:
+  stage: release
+  needs: [test_mingw_static_build,test_with_valgrind,test_asan_ubsan_gcc,test_scan_build_clang,test_cmake,test_amalgamation]
+  image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c
+  artifacts:
+    expire_in: 7 days
+    name: wallycore-bindings
+    when: on_success
+    paths:
+    - dist/*
+  tags:
+    - ga
+  script:
+    - python3 -m build
+    - virtualenv -p python3 .smoketest
+    - source .smoketest/bin/activate
+    - pip install --find-links=./dist wallycore
+    - python -c "import wallycore as w; assert w.hex_from_bytes(w.hex_to_bytes('ff')) == 'ff'"
+    - deactivate
+    - rm -rf .smoketest dist/*.whl
+    - mv dist wally_dist
+    - ./tools/build_android_libraries.sh
+    - mv release wallycore-android-jni
+    - tar czf wally_dist/wallycore-android-jni.tar.gz --remove-files wallycore-android-jni
+    - source /opt/emsdk/emsdk_env.sh
+    - tools/build_wasm.sh
+    - cd dist
+    - tar czf wallycore-wasm.tar.gz --remove-files wallycore.html wallycore.js wallycore.wasm
+    - cd ..
+    - sphinx-build -b html -a -c docs/source docs/source docs/build/html
+    - cd docs/build
+    - tar czf ../../wally_dist/apidocs.tar.gz html/
+    - cd ../..
+    - mv wally_dist/* dist/
+    - rmdir wally_dist

configure.ac

@@ -195,6 +195,7 @@ AC_SUBST([NOBUILTIN_CFLAGS])
 SWIG_WARN_CFLAGS="-fno-strict-aliasing"
 AX_CHECK_COMPILE_FLAG([-Wno-unused-parameter], [SWIG_WARN_CFLAGS="$SWIG_WARN_CFLAGS -Wno-unused-parameter"])
 AX_CHECK_COMPILE_FLAG([-Wno-shadow], [SWIG_WARN_CFLAGS="$SWIG_WARN_CFLAGS -Wno-shadow"])
+AX_CHECK_COMPILE_FLAG([-Wno-pedantic], [SWIG_WARN_CFLAGS="$SWIG_WARN_CFLAGS -Wno-pedantic"])
 AX_CHECK_COMPILE_FLAG([-Wno-missing-field-initializers], [SWIG_WARN_CFLAGS="$SWIG_WARN_CFLAGS -Wno-missing-field-initializers"])
 if echo | "$CC" -dM -E - | grep __clang__ >/dev/null; then
     AX_CHECK_COMPILE_FLAG([-Wno-self-assign], [SWIG_WARN_CFLAGS="$SWIG_WARN_CFLAGS -Wno-self-assign"])

docs/source/conf.py

@@ -177,7 +177,7 @@ def extract_docs(infile, outfile):
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.

include/wally.hpp

@@ -202,17 +202,29 @@ inline int bip32_key_unserialize_alloc(const BYTES& bytes, struct ext_key** outp
 }
 
 template <class PATH_STR>
-inline int bip32_path_from_str(const PATH_STR& path_str, uint32_t child_num, uint32_t multi_index, uint32_t flags, uint32_t* child_path_out, uint32_t child_path_out_len, size_t* written) {
+inline int bip32_path_from_str(const PATH_STR& path_str, uint32_t child_num, uint32_t multi_index, uint32_t flags, uint32_t* child_path_out, size_t child_path_out_len, size_t* written) {
     int ret = ::bip32_path_from_str(detail::get_p(path_str), child_num, multi_index, flags, child_path_out, child_path_out_len, written);
     return detail::check_ret(__FUNCTION__, ret);
 }
 
 template <class PATH_STR>
-inline int bip32_path_from_str_n(const PATH_STR& path_str, size_t path_str_len, uint32_t child_num, uint32_t multi_index, uint32_t flags, uint32_t* child_path_out, uint32_t child_path_out_len, size_t* written) {
+inline int bip32_path_from_str_len(const PATH_STR& path_str, uint32_t child_num, uint32_t multi_index, uint32_t flags, size_t* written) {
+    int ret = ::bip32_path_from_str_len(detail::get_p(path_str), child_num, multi_index, flags, written);
+    return detail::check_ret(__FUNCTION__, ret);
+}
+
+template <class PATH_STR>
+inline int bip32_path_from_str_n(const PATH_STR& path_str, size_t path_str_len, uint32_t child_num, uint32_t multi_index, uint32_t flags, uint32_t* child_path_out, size_t child_path_out_len, size_t* written) {
     int ret = ::bip32_path_from_str_n(detail::get_p(path_str), path_str_len, child_num, multi_index, flags, child_path_out, child_path_out_len, written);
     return detail::check_ret(__FUNCTION__, ret);
 }
 
+template <class PATH_STR>
+inline int bip32_path_from_str_n_len(const PATH_STR& path_str, size_t path_str_len, uint32_t child_num, uint32_t multi_index, uint32_t flags, size_t* written) {
+    int ret = ::bip32_path_from_str_n_len(detail::get_p(path_str), path_str_len, child_num, multi_index, flags, written);
+    return detail::check_ret(__FUNCTION__, ret);
+}
+
 template <class PATH_STR>
 inline int bip32_path_str_get_features(const PATH_STR& path_str, uint32_t* value_out) {
     int ret = ::bip32_path_str_get_features(detail::get_p(path_str), value_out);
@@ -1144,12 +1156,24 @@ inline int pbkdf2_hmac_sha512(const PASS& pass, const SALT& salt, uint32_t flags
     return detail::check_ret(__FUNCTION__, ret);
 }
 
+template <class PSBT, class PUB_KEY, class FINGERPRINT, class CHILD_PATH>
+inline int psbt_add_input_keypath(const PSBT& psbt, uint32_t index, const PUB_KEY& pub_key, const FINGERPRINT& fingerprint, const CHILD_PATH& child_path) {
+    int ret = ::wally_psbt_add_input_keypath(detail::get_p(psbt), index, pub_key.data(), pub_key.size(), fingerprint.data(), fingerprint.size(), child_path.data(), child_path.size());
+    return detail::check_ret(__FUNCTION__, ret);
+}
+
 template <class PSBT, class PUB_KEY, class TAPLEAF_HASHES, class FINGERPRINT, class CHILD_PATH>
 inline int psbt_add_input_taproot_keypath(const PSBT& psbt, uint32_t index, uint32_t flags, const PUB_KEY& pub_key, const TAPLEAF_HASHES& tapleaf_hashes, const FINGERPRINT& fingerprint, const CHILD_PATH& child_path) {
     int ret = ::wally_psbt_add_input_taproot_keypath(detail::get_p(psbt), index, flags, pub_key.data(), pub_key.size(), tapleaf_hashes.data(), tapleaf_hashes.size(), fingerprint.data(), fingerprint.size(), child_path.data(), child_path.size());
     return detail::check_ret(__FUNCTION__, ret);
 }
 
+template <class PSBT, class PUB_KEY, class FINGERPRINT, class CHILD_PATH>
+inline int psbt_add_output_keypath(const PSBT& psbt, uint32_t index, const PUB_KEY& pub_key, const FINGERPRINT& fingerprint, const CHILD_PATH& child_path) {
+    int ret = ::wally_psbt_add_output_keypath(detail::get_p(psbt), index, pub_key.data(), pub_key.size(), fingerprint.data(), fingerprint.size(), child_path.data(), child_path.size());
+    return detail::check_ret(__FUNCTION__, ret);
+}
+
 template <class PSBT, class PUB_KEY, class TAPLEAF_HASHES, class FINGERPRINT, class CHILD_PATH>
 inline int psbt_add_output_taproot_keypath(const PSBT& psbt, uint32_t index, uint32_t flags, const PUB_KEY& pub_key, const TAPLEAF_HASHES& tapleaf_hashes, const FINGERPRINT& fingerprint, const CHILD_PATH& child_path) {
     int ret = ::wally_psbt_add_output_taproot_keypath(detail::get_p(psbt), index, flags, pub_key.data(), pub_key.size(), tapleaf_hashes.data(), tapleaf_hashes.size(), fingerprint.data(), fingerprint.size(), child_path.data(), child_path.size());
@@ -2415,6 +2439,18 @@ inline int psbt_find_global_scalar(const PSBT& psbt, const SCALAR& scalar, size_
     return detail::check_ret(__FUNCTION__, ret);
 }
 
+template <class PSBT, class BYTES_OUT>
+inline int psbt_get_global_genesis_blockhash(const PSBT& psbt, BYTES_OUT& bytes_out, size_t* written) {
+    int ret = ::wally_psbt_get_global_genesis_blockhash(detail::get_p(psbt), bytes_out.data(), bytes_out.size(), written);
+    return detail::check_ret(__FUNCTION__, ret);
+}
+
+template <class PSBT>
+inline int psbt_has_global_genesis_blockhash(const PSBT& psbt, size_t* written) {
+    int ret = ::wally_psbt_has_global_genesis_blockhash(detail::get_p(psbt), written);
+    return detail::check_ret(__FUNCTION__, ret);
+}
+
 inline int psbt_input_clear_amount_rangeproof(struct wally_psbt_input* input) {
     int ret = ::wally_psbt_input_clear_amount_rangeproof(input);
     return detail::check_ret(__FUNCTION__, ret);
@@ -3026,6 +3062,12 @@ inline int psbt_output_set_value_rangeproof(const OUTPUT& output, const RANGEPRO
     return detail::check_ret(__FUNCTION__, ret);
 }
 
+template <class PSBT, class GENESIS_BLOCKHASH>
+inline int psbt_set_global_genesis_blockhash(const PSBT& psbt, const GENESIS_BLOCKHASH& genesis_blockhash) {
+    int ret = ::wally_psbt_set_global_genesis_blockhash(detail::get_p(psbt), genesis_blockhash.data(), genesis_blockhash.size());
+    return detail::check_ret(__FUNCTION__, ret);
+}
+
 template <class PSBT>
 inline int psbt_set_global_scalars(const PSBT& psbt, const struct wally_map* map_in) {
     int ret = ::wally_psbt_set_global_scalars(detail::get_p(psbt), map_in);

include/wally_bip32.h

@@ -465,6 +465,22 @@ WALLY_CORE_API int bip32_key_get_fingerprint(
     unsigned char *bytes_out,
     size_t len);
 
+/**
+ * Get the number of child path elements in a BIP32 path string.
+ *
+ * :param path_str: The BIP32 path string of child numbers to convert from.
+ * :param child_num: The child number to use if ``path_str`` contains a ``*`` wildcard.
+ * :param multi_index: The multi-path item to use if ``path_str`` contains a ``<>`` multi-path.
+ * :param flags: :ref:`bip32-flags` controlling path parsing behaviour.
+ * :param written: Destination for the number of path elements in the path string.
+ */
+WALLY_CORE_API int bip32_path_from_str_len(
+    const char *path_str,
+    uint32_t child_num,
+    uint32_t multi_index,
+    uint32_t flags,
+    size_t *written);
+
 /**
  * Convert a BIP32 path string to a path.
  *
@@ -482,7 +498,20 @@ WALLY_CORE_API int bip32_path_from_str(
     uint32_t multi_index,
     uint32_t flags,
     uint32_t *child_path_out,
-    uint32_t child_path_out_len,
+    size_t child_path_out_len,
+    size_t *written);
+
+/**
+ * Get the number of child path elements in a known-length BIP32 path string.
+ *
+ * See `bip32_path_from_str_len`.
+ */
+WALLY_CORE_API int bip32_path_from_str_n_len(
+    const char *path_str,
+    size_t path_str_len,
+    uint32_t child_num,
+    uint32_t multi_index,
+    uint32_t flags,
     size_t *written);
 
 /**
@@ -497,7 +526,7 @@ WALLY_CORE_API int bip32_path_from_str_n(
     uint32_t multi_index,
     uint32_t flags,
     uint32_t *child_path_out,
-    uint32_t child_path_out_len,
+    size_t child_path_out_len,
     size_t *written);
 
 /**

include/wally_crypto.h

@@ -379,7 +379,7 @@ WALLY_CORE_API int wally_pbkdf2_hmac_sha512(
 #define EC_FLAG_ELEMENTS 0x10
 
 /* All defined flags */
-#define EC_FLAGS_ALL (0x1 | 0x2 | 0x4 | 0x8)
+#define EC_FLAGS_ALL (0x1 | 0x2 | 0x4 | 0x8 | 0x10)
 
 /**
  * Verify that a private key is valid.

include/wally_descriptor.h

@@ -17,18 +17,20 @@ struct wally_descriptor;
 #define WALLY_MINISCRIPT_REQUIRE_CHECKSUM 0x04 /** Require a checksum to be present */
 #define WALLY_MINISCRIPT_POLICY_TEMPLATE  0x08 /** Only allow policy templates with @n BIP32 keys */
 #define WALLY_MINISCRIPT_UNIQUE_KEYPATHS  0x10 /** For policy templates, ensure BIP32 derivation paths differ for identical keys */
+#define WALLY_MINISCRIPT_AS_ELEMENTS      0x20 /** Treat non-elements expressions as elements, e.g. tr() as eltr() */
 #define WALLY_MINISCRIPT_DEPTH_MASK       0xffff0000 /** Mask for limiting maximum depth */
 #define WALLY_MINISCRIPT_DEPTH_SHIFT      16 /** Shift to convert maximum depth to flags */
 
 /*** miniscript-features Miniscript/Descriptor feature flags */
-#define WALLY_MS_IS_RANGED       0x01 /** Allows key ranges via ``*`` */
-#define WALLY_MS_IS_MULTIPATH    0x02 /** Allows multiple paths via ``<a;b;c>`` */
-#define WALLY_MS_IS_PRIVATE      0x04 /** Contains at least one private key */
-#define WALLY_MS_IS_UNCOMPRESSED 0x08 /** Contains at least one uncompressed key */
-#define WALLY_MS_IS_RAW          0x10 /** Contains at least one raw key */
-#define WALLY_MS_IS_DESCRIPTOR   0x20 /** Contains only descriptor expressions (no miniscript) */
-#define WALLY_MS_IS_X_ONLY       0x40 /** Contains at least one x-only key */
-#define WALLY_MS_IS_PARENTED     0x80 /** Contains at least one key key with a parent key origin */
+#define WALLY_MS_IS_RANGED       0x001 /** Allows key ranges via ``*`` */
+#define WALLY_MS_IS_MULTIPATH    0x002 /** Allows multiple paths via ``<a;b;c>`` */
+#define WALLY_MS_IS_PRIVATE      0x004 /** Contains at least one private key */
+#define WALLY_MS_IS_UNCOMPRESSED 0x008 /** Contains at least one uncompressed key */
+#define WALLY_MS_IS_RAW          0x010 /** Contains at least one raw key */
+#define WALLY_MS_IS_DESCRIPTOR   0x020 /** Contains only descriptor expressions (no miniscript) */
+#define WALLY_MS_IS_X_ONLY       0x040 /** Contains at least one x-only key */
+#define WALLY_MS_IS_PARENTED     0x080 /** Contains at least one key key with a parent key origin */
+#define WALLY_MS_IS_ELEMENTS     0x100 /** Contains Elements expressions or was parsed as Elements */
 
 /*** ms-canonicalization-flags Miniscript/Descriptor canonicalization flags */
 #define WALLY_MS_CANONICAL_NO_CHECKSUM 0x01 /** Do not include a checksum */