diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 95ea2079f9c..8050658c933 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -24,7 +24,7 @@ jobs: id: cache-go-deps with: path: ~/go/bin - key: go-deps-${{ runner.os }}-v4.0.2-protoc-gen-go-v1.34.1-gopls-latest + key: go-deps-${{ runner.os }}-v4.0.2-protoc-gen-go-v1.34.1 restore-keys: | go-deps-${{ runner.os }}- @@ -32,8 +32,6 @@ jobs: if: steps.cache-go-deps.outputs.cache-hit != 'true' run: | go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.1 - go install golang.org/x/tools/gopls@latest - which gopls - name: Setup Protocol Buffer Compiler uses: arduino/setup-protoc@v3 @@ -58,7 +56,7 @@ jobs: uses: golangci/golangci-lint-action@v8 with: version: v2.3.1 - args: --build-tags=gowaku_no_rln + args: --build-tags=gowaku_no_rln,lint - name: lint-panics run: | diff --git a/Dockerfile b/Dockerfile index e113223fabb..e96aa4302eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,6 +52,9 @@ COPY --from=builder /go/src/github.com/status-im/status-go/build/bin/push-notifi COPY --from=builder /go/src/github.com/status-im/status-go/tests-functional/scripts/scan_waku_fleet.py /usr/local/bin COPY --from=builder /go/src/github.com/status-im/status-go/static/keys/* /static/keys/ COPY --from=builder /go/src/github.com/status-im/status-go/tests-functional/waku_configs/* /static/configs/ +COPY --from=builder /go/src/github.com/status-im/nim-sds/build/libsds.so /usr/local/lib/ + +ENV LD_LIBRARY_PATH=/usr/local/lib/ COPY _assets/scripts/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh diff --git a/Makefile b/Makefile index a02fd0dbbbe..58f3ec40a95 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ .PHONY: statusgo all test clean help .PHONY: statusgo-ios-library statusgo-android-library .PHONY: build-libwaku test-libwaku clean-libwaku rebuild-libwaku +.PHONY: build-libsds clean-libsds rebuild-libsds # Clear any GOROOT set outside of the Nix shell export GOROOT= @@ -75,14 +76,14 @@ endif ifeq ($(detected_OS),Darwin) GOBIN_SHARED_LIB_EXT := dylib - LIBWAKU_EXT := so + LIB_EXT := dylib GOBIN_SHARED_LIB_CFLAGS := CGO_ENABLED=1 GOOS=darwin else ifeq ($(detected_OS),Windows) GOBIN_SHARED_LIB_EXT := dll - LIBWAKU_EXT := dll + LIB_EXT := dll else ifeq ($(detected_OS),Linux) GOBIN_SHARED_LIB_EXT := so - LIBWAKU_EXT := so + LIB_EXT := so CGO_LDFLAGS += "-Wl,-soname,libstatus.so.0" endif @@ -95,15 +96,46 @@ GIT_AUTHOR ?= $(shell git config user.email || echo $$USER) BUILD_TAGS ?= gowaku_no_rln +# `nwaku` variables + ifeq ($(USE_NWAKU), true) BUILD_TAGS += use_nwaku NWAKU_VERSION ?= v0.37.0-rc.3 NWAKU_SOURCE_DIR ?= $(GIT_ROOT)/../nwaku - LIBWAKU := $(NWAKU_SOURCE_DIR)/build/libwaku.$(LIBWAKU_EXT) + LIBWAKU := $(NWAKU_SOURCE_DIR)/build/libwaku.$(LIB_EXT) CGO_CFLAGS+=-I$(NWAKU_SOURCE_DIR)/library CGO_LDFLAGS+=-L$(NWAKU_SOURCE_DIR)/build -lwaku -Wl,-rpath,$(NWAKU_SOURCE_DIR)/build endif + +# `nim-sds` variables + +# Option 1: Provide NIM_SDS_SOURCE_DIR. Make clones it if missing. +NIM_SDS_SOURCE_DIR ?= $(GIT_ROOT)/../nim-sds + +# Option 2: Provide NIM_SDS_LIB_DIR and NIM_SDS_INC_DIR + +# Determine which approach to use +ifdef NIM_SDS_LIB_DIR +ifdef NIM_SDS_INC_DIR + # External lib/include approach (e.g. used in Nix) + NIM_SDS_BUILD_FROM_SOURCE := false +else + $(error NIM_SDS_INC_DIR must be provided when NIM_SDS_LIB_DIR is set) +endif +else + # Source directory approach + NIM_SDS_LIB_DIR := $(NIM_SDS_SOURCE_DIR)/build + NIM_SDS_INC_DIR := $(NIM_SDS_SOURCE_DIR)/library + NIM_SDS_BUILD_FROM_SOURCE := true +endif + +LIBSDS := $(NIM_SDS_LIB_DIR)/libsds.$(LIB_EXT) +CGO_CFLAGS+=-I$(NIM_SDS_INC_DIR) +CGO_LDFLAGS+=-L$(NIM_SDS_LIB_DIR) -lsds + +# Common flags + BUILD_FLAGS ?= -ldflags="" BUILD_FLAGS_MOBILE ?= @@ -154,7 +186,6 @@ nix-gc-protected: @echo -e "$(YELLOW)The following paths are protected:$(RESET)" && \ ls -1 $(_NIX_GCROOTS) | sed 's/^/ - /' - nix-upgrade: SHELL := /bin/sh nix-upgrade: ##@nix Upgrade Nix interpreter to current version. nix/scripts/upgrade.sh @@ -175,7 +206,7 @@ nix-purge: ##@nix Completely remove Nix setup, including /nix directory all: $(GO_CMD_NAMES) .PHONY: $(GO_CMD_NAMES) $(GO_CMD_PATHS) $(GO_CMD_BUILDS) -$(GO_CMD_BUILDS): generate $(LIBWAKU) +$(GO_CMD_BUILDS): generate $(LIBWAKU) $(LIBSDS) $(GO_CMD_BUILDS): ##@build Build any Go project from cmd folder CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" \ go build -v \ @@ -184,6 +215,13 @@ $(GO_CMD_BUILDS): ##@build Build any Go project from cmd folder @echo "Compilation done." @echo "Run \"build/bin/$(notdir $@) -h\" to view available commands." +# Flag needed by nim-based dependencies (e.g., nwaku and nim-sds) that also use nimbus-build-system. +# When USE_SYSTEM_NIM=1 skips compiling Nim compiler locally and instead, +# enforces to use system-installed Nim. +# This is not needed if nim dependencies use nimble. +USE_SYSTEM_NIM ?= 0 + +# Libwaku targets $(NWAKU_SOURCE_DIR): ##@build Clone nwaku ifeq ($(USE_NWAKU),true) @@ -196,7 +234,7 @@ clone-nwaku: $(NWAKU_SOURCE_DIR) $(LIBWAKU): clone-nwaku ifeq ($(USE_NWAKU),true) @echo "Building libwaku" $(LIBWAKU) - $(MAKE) -C $(NWAKU_SOURCE_DIR) libwaku SHELL=/bin/bash + $(MAKE) -C $(NWAKU_SOURCE_DIR) libwaku USE_SYSTEM_NIM=$(USE_SYSTEM_NIM) SHELL=/bin/bash endif build-libwaku: $(LIBWAKU) @@ -210,6 +248,39 @@ clean-libwaku: rebuild-libwaku: | clean-libwaku $(LIBWAKU) +# libsds targets + +$(NIM_SDS_SOURCE_DIR): ##@build Clone nim-sds +ifeq ($(NIM_SDS_BUILD_FROM_SOURCE),true) + @echo "Cloning nim-sds ..." + git clone --branch v0.1.0 https://github.com/waku-org/nim-sds.git $(NIM_SDS_SOURCE_DIR) +endif + +clone-nim-sds: $(NIM_SDS_SOURCE_DIR) + +$(LIBSDS): clone-nim-sds +ifeq ($(NIM_SDS_BUILD_FROM_SOURCE),true) + @echo "Building nim-sds: $(LIBSDS)" + $(MAKE) -C $(NIM_SDS_SOURCE_DIR) update + $(MAKE) -C $(NIM_SDS_SOURCE_DIR) libsds USE_SYSTEM_NIM=$(USE_SYSTEM_NIM) SHELL=/bin/bash +else + @test -f $(LIBSDS) || (echo "Error: libsds not found at $(LIBSDS)" && exit 1) +endif + +build-libsds: $(LIBSDS) + +build-libsds-android: clone-nim-sds + @echo "Building nim-sds for Android" $(LIBSDS) + $(MAKE) -C $(NIM_SDS_SOURCE_DIR) libsds-android USE_SYSTEM_NIM=$(USE_SYSTEM_NIM) SHELL=/bin/bash + +clean-libsds: + @echo "Removing libsds" + rm $(LIBSDS) + +rebuild-libsds: | clean-libsds $(LIBSDS) + +# Status-go targets + statusgo: ##@build Build status-go as status-backend server statusgo: build/bin/status-backend @@ -227,34 +298,33 @@ push-notification-server: build/bin/push-notification-server cmd: ##@build Build all public apps in ./cmd cmd: status-backend push-notification-server - status-go-deps: go clean -cache || true go clean -modcache || true go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.1 - - +statusgo-c-bindings: STATUS_GO_BINDINGS_PATH ?= build/bin/statusgo-lib statusgo-c-bindings: - ## cmd/library/README.md explains the magic incantation behind this - mkdir -p build/bin/statusgo-lib - go run -mod=mod cmd/library/*.go > build/bin/statusgo-lib/main.go + @## cmd/library/README.md explains the magic incantation behind this + mkdir -p $(STATUS_GO_BINDINGS_PATH) + go run -mod=mod cmd/library/*.go > $(STATUS_GO_BINDINGS_PATH)/main.go +statusgo-library: STATUS_GO_BINDINGS_PATH ?= build/bin/statusgo-lib +statusgo-library: STATUS_GO_LIBRARY_OUT ?= build/bin statusgo-library: generate -statusgo-library: statusgo-c-bindings $(LIBWAKU) ##@cross-compile Build status-go as static library for current platform +statusgo-library: statusgo-c-bindings $(LIBWAKU) $(LIBSDS) ##@cross-compile Build status-go as static library for current platform @echo "Building static library..." CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" \ go build \ -tags '$(BUILD_TAGS)' \ $(BUILD_FLAGS) \ -buildmode=c-archive \ - -o build/bin/libstatus.a \ - ./build/bin/statusgo-lib - @echo "Static library built:" - @ls -la build/bin/libstatus.* + -o $(STATUS_GO_LIBRARY_OUT)/libstatus.a \ + "$(STATUS_GO_BINDINGS_PATH)/main.go" + @echo "Static library built: $(STATUS_GO_LIBRARY_OUT)/libstatus.a" statusgo-shared-library: generate -statusgo-shared-library: statusgo-c-bindings $(LIBWAKU) ##@cross-compile Build status-go as shared library for current platform +statusgo-shared-library: statusgo-c-bindings $(LIBWAKU) $(LIBSDS) ##@cross-compile Build status-go as shared library for current platform @echo "Building shared library..." @echo "Tags: $(BUILD_TAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" \ @@ -340,7 +410,7 @@ clean-release: rm -rf $(RELEASE_DIR) lint-fix: - golangci-lint --build-tags '$(BUILD_TAGS)' run --fix ./... + golangci-lint --build-tags '$(BUILD_TAGS) lint' run --fix ./... docker-test: ##@tests Run tests in a docker container with golang. docker run --privileged --rm -it -v "$(PWD):$(DOCKER_TEST_WORKDIR)" -w "$(DOCKER_TEST_WORKDIR)" $(DOCKER_TEST_IMAGE) go test ${ARGS} @@ -363,6 +433,7 @@ test-unit: export UNIT_TEST_PACKAGES ?= $(call sh, go list ./... | \ grep -v /transactions/fake | \ grep -v /tests-unit-network) test-unit: ##@tests Run unit and integration tests + LD_LIBRARY_PATH="$(NIM_SDS_LIB_DIR)" CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CFLAGS="$(CGO_CFLAGS)" \ ./_assets/scripts/run_unit_tests.sh test-unit-network: test-unit-prep @@ -384,12 +455,13 @@ benchmark: export FUNCTIONAL_TESTS_DOCKER_UID ?= $(call sh, id -u) benchmark: @./_assets/scripts/run_benchmark.sh -lint-panics: export GOFLAGS ?= -tags='$(BUILD_TAGS)' lint-panics: generate + GOFLAGS=-tags='$(BUILD_TAGS),lint' \ go tool goroutine-defer-guard -skip=./cmd -test=false ./... lint: generate lint-panics - golangci-lint --build-tags '$(BUILD_TAGS)' run ./... +lint: + golangci-lint --build-tags '$(BUILD_TAGS) lint' run ./... clean: ##@other Cleanup rm -fr build/bin/* diff --git a/_assets/scripts/run_functional_tests.sh b/_assets/scripts/run_functional_tests.sh index 72260ae857d..ec866f764cc 100755 --- a/_assets/scripts/run_functional_tests.sh +++ b/_assets/scripts/run_functional_tests.sh @@ -76,7 +76,8 @@ pytest --reruns 2 -m rpc -c "${root_path}/pytest.ini" -n 12 \ --docker-image=${image_name} \ --codecov_dir="${binary_coverage_reports_path}" \ --logs-dir="${logs_path}" \ - --junitxml="${test_results_path}/report.xml" + --junitxml="${test_results_path}/report.xml" \ + ${root_path} exit_code=$? # Stop containers diff --git a/flake.lock b/flake.lock index 0762f62dadd..862793f9d35 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,25 @@ { "nodes": { + "nim-sds": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1762510817, + "narHash": "sha256-VwTp+2JB8ZM8Yx9RPOZ95GBrw5hFdDGMKhS5w9uQ2CQ=", + "rev": "b6431260110fe62956ff4d7bab6767556c19d040", + "revCount": 67, + "submodules": true, + "type": "git", + "url": "https://github.com/waku-org/nim-sds" + }, + "original": { + "rev": "b6431260110fe62956ff4d7bab6767556c19d040", + "submodules": true, + "type": "git", + "url": "https://github.com/waku-org/nim-sds" + } + }, "nixpkgs": { "locked": { "lastModified": 1757590060, @@ -17,6 +37,22 @@ } }, "nixpkgs_2": { + "locked": { + "lastModified": 1757590060, + "narHash": "sha256-EWwwdKLMZALkgHFyKW7rmyhxECO74+N+ZO5xTDnY/5c=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "0ef228213045d2cdb5a169a95d63ded38670b293", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "0ef228213045d2cdb5a169a95d63ded38670b293", + "type": "github" + } + }, + "nixpkgs_3": { "locked": { "lastModified": 1740603184, "narHash": "sha256-t+VaahjQAWyA+Ctn2idyo1yxRIYpaDxMgHkgCNiMJa4=", @@ -34,7 +70,7 @@ }, "nwaku": { "inputs": { - "nixpkgs": "nixpkgs_2", + "nixpkgs": "nixpkgs_3", "zerokit": "zerokit" }, "locked": { @@ -56,7 +92,8 @@ }, "root": { "inputs": { - "nixpkgs": "nixpkgs", + "nim-sds": "nim-sds", + "nixpkgs": "nixpkgs_2", "nwaku": "nwaku" } }, diff --git a/flake.nix b/flake.nix index 21a65ade7a2..358d80d180e 100644 --- a/flake.nix +++ b/flake.nix @@ -18,9 +18,10 @@ nixpkgs.url = "github:NixOS/nixpkgs/0ef228213045d2cdb5a169a95d63ded38670b293"; # We cannot do follows since the nim-unwrapped-2_0 doesn't exist in this nixpkgs version above nwaku.url = "git+https://github.com/waku-org/nwaku?submodules=1&rev=e755fd834f5f3d6fba216b09469316f0328b3b6f"; + nim-sds.url = "git+https://github.com/waku-org/nim-sds?submodules=1&rev=b6431260110fe62956ff4d7bab6767556c19d040"; }; - outputs = { self, nixpkgs, nwaku }: + outputs = { self, nixpkgs, nwaku, nim-sds }: let stableSystems = [ "x86_64-linux" "aarch64-linux" @@ -38,7 +39,10 @@ }; overlays = [ pkgsOverlay - (final: prev: { nwaku = nwaku.packages.${system}; }) + (final: prev: { + nwaku = nwaku.packages.${system}; + lib-sds-pkg = nim-sds.packages.${system}.libsds; + }) ]; } ); diff --git a/go.mod b/go.mod index e00fe81cb5e..04f104b16c6 100644 --- a/go.mod +++ b/go.mod @@ -103,6 +103,7 @@ require ( github.com/status-im/extkeys v1.4.0 github.com/status-im/go-wallet-sdk v0.0.0-20251027141302-43edbd6abc92 github.com/waku-org/go-waku v0.10.1-0.20251003225121-06c9af60f35b + github.com/waku-org/sds-go-bindings v0.0.0-20251119184907-e78f76307965 github.com/waku-org/waku-go-bindings v0.0.0-20251030105913-aa64c47d2bf3 github.com/wk8/go-ordered-map/v2 v2.1.7 go.uber.org/atomic v1.11.0 diff --git a/go.sum b/go.sum index 64111c3554f..0f7d9a8f8b6 100644 --- a/go.sum +++ b/go.sum @@ -2414,6 +2414,8 @@ github.com/waku-org/go-zerokit-rln-arm v0.0.0-20230916171929-1dd9494ff065 h1:Sd7 github.com/waku-org/go-zerokit-rln-arm v0.0.0-20230916171929-1dd9494ff065/go.mod h1:7cSGUoGVIla1IpnChrLbkVjkYgdOcr7rcifEfh4ReR4= github.com/waku-org/go-zerokit-rln-x86_64 v0.0.0-20230916171518-2a77c3734dd1 h1:4HSdWMFMufpRo3ECTX6BrvA+VzKhXZf7mS0rTa5cCWU= github.com/waku-org/go-zerokit-rln-x86_64 v0.0.0-20230916171518-2a77c3734dd1/go.mod h1:+LeEYoW5/uBUTVjtBGLEVCUe9mOYAlu5ZPkIxLOSr5Y= +github.com/waku-org/sds-go-bindings v0.0.0-20251119184907-e78f76307965 h1:1xx0AQeLfxSLxuGvu7em1LrNM5lVXTU6PiuaPfTkbpA= +github.com/waku-org/sds-go-bindings v0.0.0-20251119184907-e78f76307965/go.mod h1:rC5yaoM7ee8h/zEyCGfdnYTOEhyW3jpdck7maoiRQQE= github.com/waku-org/waku-go-bindings v0.0.0-20251030105913-aa64c47d2bf3 h1:VimYvS377VZh/b/kipnv2rp0a2r8Dd7D+4ihVd3hipI= github.com/waku-org/waku-go-bindings v0.0.0-20251030105913-aa64c47d2bf3/go.mod h1:v6ogkMCyQRUTazRyQR5LIouukY4/2bkjjHpnBKBxcAY= github.com/wealdtech/go-ens/v3 v3.6.0 h1:EAByZlHRQ3vxqzzwNi0GvEq1AjVozfWO4DMldHcoVg8= diff --git a/messaging/layers/reliability/reliability.go b/messaging/layers/reliability/reliability.go index fae3b75f03d..cc1af5e9c54 100644 --- a/messaging/layers/reliability/reliability.go +++ b/messaging/layers/reliability/reliability.go @@ -7,6 +7,8 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" + _ "github.com/waku-org/sds-go-bindings/sds" + mvdsnode "github.com/status-im/mvds/node" mvdsproto "github.com/status-im/mvds/protobuf" mvdsstate "github.com/status-im/mvds/state" diff --git a/nix/pkgs/status-go/library/default.nix b/nix/pkgs/status-go/library/default.nix index 4d7d122a4c2..d46cbdc1f54 100644 --- a/nix/pkgs/status-go/library/default.nix +++ b/nix/pkgs/status-go/library/default.nix @@ -11,7 +11,7 @@ let in pkgs.buildGoModule { pname = "status-go"; src = builtins.path { path = ./../../../..; name = "status-go-library"; }; - vendorHash = "sha256-pxkFTHFLrTOdCh9V/Mzjt5/NHCzjmIuN5fIt5EptHSE="; + vendorHash = "sha256-t6dtjRx3xBg15WhNe7oenkPPHTZrsajvc60pWnj1OxY="; inherit meta version; @@ -39,8 +39,10 @@ in pkgs.buildGoModule { allowGoReference = true; preBuild = '' - go run cmd/library/*.go > $NIX_BUILD_TOP/main.go - make generate SHELL=$SHELL GO111MODULE=on GO_GENERATE_CMD='go generate' + export NIM_SDS_INC_DIR="${pkgs.lib-sds-pkg}/include" + export NIM_SDS_LIB_DIR="${pkgs.lib-sds-pkg}/lib" + export GO_GENERATE_CMD='go generate' + make generate ''; # Build the Go library @@ -48,12 +50,9 @@ in pkgs.buildGoModule { # https://github.com/status-im/status-mobile/issues/20135 buildPhase = '' runHook preBuild - go build \ - -buildmode='c-archive' \ - ${optionalString stdenv.isDarwin "-ldflags=-extldflags=-lresolv"} \ - -tags='gowaku_skip_migrations gowaku_no_rln ${optionalString stdenv.isDarwin "netgo"}' \ - -o "$out/libstatus.a" \ - $NIX_BUILD_TOP/main.go + make statusgo-library \ + STATUS_GO_BINDINGS_PATH="$NIX_BUILD_TOP" \ + STATUS_GO_LIBRARY_OUT="$out" runHook postBuild ''; } diff --git a/nix/shell.nix b/nix/shell.nix index 3e936e37dd2..17e671e4a4e 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -21,14 +21,20 @@ in mkShell { go golangci-lint go-junit-report gopls codecov-cli protobuf3_24 protoc-gen-go gotestsum openjdk openssl rustc cargo + nim + lib-sds-pkg ]; - shellHook = lib.optionalString (!isMacM1) '' - ANDROID_HOME=${pkgs.androidPkgs.androidsdk}/libexec/android-sdk/ - ANDROID_NDK=$ANDROID_HOME/ndk-bundle - ANDROID_SDK_ROOT=$ANDROID_HOME - ANDROID_NDK_HOME=$ANDROID_NDK - '' + lib.optionalString (stdenv.isDarwin) '' + shellHook = '' + export USE_SYSTEM_NIM=1 + '' + + lib.optionalString (!isMacM1) '' + export ANDROID_HOME=${pkgs.androidPkgs.androidsdk}/libexec/android-sdk/ + export ANDROID_NDK=\$ANDROID_HOME/ndk-bundle + export ANDROID_SDK_ROOT=\$ANDROID_HOME + export ANDROID_NDK_HOME=\$ANDROID_NDK + '' + + lib.optionalString (stdenv.isDarwin) '' export PATH="/usr/bin:$PATH" ''; # Sandbox causes Xcode issues on MacOS. Requires sandbox=relaxed.