poc/status-go-codex-integraion

.gitignore

@@ -65,7 +65,6 @@ Session.vim
 .undodir/*
 .idea/*
 !.idea/codeStyles/
-/.vscode/
 /cmd/*/.ethereum/
 *.iml
 

.vscode/settings.json

@@ -1,14 +1,16 @@
 {
-  "go.testTags": "gowaku_skip_migrations,gowaku_no_rln",
-  "cSpell.words": [
-    "unmarshalling"
-  ],
+  "go.testTags": "gowaku_skip_migrations,gowaku_no_rln,codex_integration",
+  "cSpell.words": ["unmarshalling"],
   "gopls": {
-    "formatting.local": "github.com/status-im/status-go"
+    "formatting.local": "github.com/status-im/status-go",
+    "build.buildFlags": [
+      "-tags=gowaku_skip_migrations,gowaku_no_rln,codex_integration"
+    ]
   },
   // format all files on save if a formatter is available
   "editor.formatOnSave": true,
   // I use "goimports" instead of "gofmt"
   // because it does the same thing but also formats imports
   "go.formatTool": "goimports",
+  "idf.pythonInstallPath": "/usr/bin/python"
 }

_assets/scripts/build_status_go_docker.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -o nounset
+
+GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
+source "${GIT_ROOT}/_assets/scripts/colors.sh"
+source "${GIT_ROOT}/_assets/scripts/codecov.sh"
+
+echo -e "${GRN}Building status-go docker image...${RST}"
+
+identifier=${BUILD_ID:-$(git rev-parse --short HEAD)}
+image_name="statusgo-${identifier}"
+
+# Build statusgo image
+echo -e "${GRN}Building status-go${RST}"
+docker build . \
+  --build-arg "build_tags='gowaku_no_rln'" \
+  --build-arg "enable_go_cache=false" \
+  --tag "${image_name}"
+
+echo -e "${GRN}Building status-go docker image DONE!${RST}"

_assets/scripts/functional_tests_commons.sh

@@ -0,0 +1,153 @@
+remove_old_logs() {
+  # Cleanup any previous logs
+  rm -rf "${logs_path}"
+
+  # Create directories
+  mkdir -p "${logs_path}"
+}
+
+set_pyenv() {
+  venv_path="${root_path}/.venv"
+
+  if [[ -d "${venv_path}" ]]; then
+      echo -e "${GRN}Using existing virtual environment${RST}"
+  else
+      echo -e "${GRN}Creating new virtual environment${RST}"
+      python3 -m venv "${venv_path}"
+  fi
+
+  source "${venv_path}/bin/activate"
+
+  # Upgrade pip and install requirements
+  echo -e "${GRN}Installing dependencies${RST}"
+  pip install --upgrade pip &>/dev/null
+  pip install -r "${root_path}/requirements.txt" &>/dev/null
+}
+
+discover_tests() {
+  pytest --collect-only -q -m rpc -c "${root_path}/pytest.ini" "$1"
+}
+
+start_services() {
+  # Run docker
+  echo -e "${GRN}Running status-go external dependencies${RST}"
+  docker compose -p ${project_name} ${all_compose_files} up -d --build --remove-orphans
+}
+
+remove_old_containers() {
+  # Remove any remaining containers if any
+  echo -e "${GRN}Removing any remaining containers (if any relevant left)${RST}"
+  docker ps -a --filter "name=status-go-func-tests-${identifier}" -q | xargs -r docker rm -f
+
+  # Remove networks
+  docker network rm "status-go-func-tests-${identifier}_default" 2>/dev/null || true
+}
+
+clean_all_containers() {
+  # Stop containers
+  echo -e "${GRN}Stopping docker containers${RST}"
+  docker compose -p ${project_name} ${all_compose_files} stop
+
+  # Cleanup containers
+  echo -e "${GRN}Removing docker containers${RST}"
+  docker compose -p ${project_name} ${all_compose_files} down
+
+  remove_old_containers 
+}
+
+wait_for_services() {
+  local timeout="$1"
+  local project="status-go-func-tests-$(git rev-parse --short HEAD)"
+
+  echo "Waiting up to ${timeout}s for boot-1 and store to be healthy..."
+
+  for ((i=1; i<=timeout; i++)); do
+    printf "\r%02d" "$i"
+    local healthy=$(
+      docker compose -p "$project" ps --format json \
+      | jq -s -r '
+          [ .[]
+            | select(.Service=="boot-1" or .Service=="store")
+            | (.State=="running" and .Health=="healthy")
+          ]
+          | (length==2 and all)
+        '
+    )
+
+    if [[ $healthy == "true" ]]; then
+      echo -e "\r✅ Services are healthy (after ${i}s)"
+      return 0
+    fi
+
+    sleep 1
+  done
+
+  echo "❌ Services did not become healthy within ${timeout}s"
+  exit 1
+}
+
+list_tests_and_confirm() {
+  local selected_test="${1:+-k $1}"
+  echo -e "${GRN}Discovering tests to be run...${RST}"
+  collected_output=$(discover_tests "$selected_test")
+  test_count=$(echo "$collected_output" | grep -c "^\s*<Function test_.*>$")
+  if [ -z "$selected_test" ]; then
+    if [ "$test_count" -eq "0" ]; then
+      echo -e "${RED}No tests found!${RST}"
+      exit 1
+    fi
+    echo -e "${RED}No test pattern provided. This will run all ${test_count} tests!${RST}"
+  else
+    # Early exit if no tests found
+    if [ "$test_count" -eq "0" ]; then
+      echo -e "${RED}No tests found matching: $1${RST}"
+      exit 1
+    fi
+
+    echo -e "${GRN}Found ${test_count} tests matching:${RST} $1"
+
+    # Show the tests that will run
+    echo -e "${GRN}Tests to execute:${RST}"
+    echo "$collected_output" \
+    | grep -oP "^\s*<Function \Ktest_[^>]*(?=>$)" \
+    | nl -w2 -s') '
+  fi
+
+  read -p "Continue with execution? (y/n): " -n 1 -r
+  echo
+  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+      exit 0
+  fi
+}
+
+run_tests() {
+  # Decide parallelization strategy based on number of tests matched
+  if [ "$test_count" -eq 1 ]; then
+      echo -e "${GRN}Running single test without parallelization${RST}"
+      parallel_opts=""
+  elif [ "$test_count" -le 4 ]; then
+      echo -e "${GRN}Running with limited parallelization (-n $test_count)${RST}"
+      parallel_opts="-n $test_count --dist loadgroup"
+  else
+      echo -e "${GRN}Running with full parallelization (-n 12)${RST}"
+      parallel_opts="-n 12 --dist loadgroup"
+  fi
+
+  local selected_test="${1:+-k $1}"
+
+  # Run with dynamic parallelization
+  pytest --reruns 2 -m rpc -c "${root_path}/pytest.ini" $parallel_opts \
+    --log-cli-level="${FUNCTIONAL_TESTS_LOG_LEVEL}" \
+    --docker_project_name="${project_name}" \
+    --docker-image=${image_name} \
+    --logs-dir="${logs_path}" ${selected_test}
+  exit_code=$?
+}
+
+# Set up cleanup trap to ensure containers are always cleaned up
+cleanup() {
+    echo -e "${YLW}Running cleanup...${RST}"
+    clean_all_containers
+}
+
+trap cleanup EXIT

_assets/scripts/run_functional_tests_dev.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
+source "${GIT_ROOT}/_assets/scripts/colors.sh"
+source "${GIT_ROOT}/_assets/scripts/codecov.sh"
+
+: "${FUNCTIONAL_TESTS_LOG_LEVEL:=INFO}"
+
+root_path="${GIT_ROOT}/tests-functional"
+logs_path="${root_path}/logs"
+
+all_compose_files="-f ${root_path}/docker-compose.anvil.yml -f ${root_path}/docker-compose.waku.yml"
+identifier=${BUILD_ID:-$(git rev-parse --short HEAD)}
+project_name="status-go-func-tests-${identifier}"
+image_name="statusgo-${identifier}"
+
+source "${GIT_ROOT}/_assets/scripts/functional_tests_commons.sh"
+
+test_pattern="${1-}"
+
+set_pyenv
+
+list_tests_and_confirm "${test_pattern}"
+
+remove_old_logs
+
+remove_old_containers
+
+start_services
+
+wait_for_services 60
+
+run_tests "${test_pattern}"
+
+# Cleanup will be handled automatically by the trap
+echo -e "${GRN}Testing finished${RST}"
+exit $exit_code

_assets/scripts/run_unit_tests.sh

@@ -47,6 +47,14 @@ run_test_for_packages() {
   local exit_code_file="exit_code_${iteration}.txt"
   local timeout="$(( single_timeout * count))m"
 
+  rm $output_file 2>/dev/null || true
+  rm $report_file 2>/dev/null || true
+  rm $rerun_report_file 2>/dev/null || true
+  rm $exit_code_file 2>/dev/null || true
+  rm $coverage_file 2>/dev/null || true
+  rm coverage_merged.out 2>/dev/null || true
+  rm test-coverage.html 2>/dev/null || true
+
   if [[ "${UNIT_TEST_DRY_RUN}" == 'true' ]]; then
     echo -e "${GRN}Dry run ${iteration}. message:${RST} ${log_message}\n"\
     "${YLW}Dry run ${iteration}. packages:${RST} ${packages}\n"\
@@ -70,6 +78,13 @@ run_test_for_packages() {
   # Cleanup previous coverage reports
   rm -f "${TEST_WITH_COVERAGE_REPORTS_DIR}/coverage.out.rerun.*"
 
+  echo "gotestsum --packages=\"${packages}\" ${gotestsum_flags} --raw-command -- \
+    ./_assets/scripts/test-with-coverage.sh \
+    ${GOTEST_EXTRAFLAGS} \
+    -timeout \"${timeout}\" \
+    -tags \"${BUILD_TAGS}\" | \
+    redirect_stdout \"${output_file}\""
+
   # Run tests
   gotestsum --packages="${packages}" ${gotestsum_flags} --raw-command -- \
     ./_assets/scripts/test-with-coverage.sh \
@@ -115,14 +130,21 @@ if [[ $HAS_PROTOCOL_PACKAGE == 'false' ]]; then
 else
   # Spawn a process to test all packages except `protocol`
   UNIT_TEST_PACKAGES_FILTERED=$(echo "${UNIT_TEST_PACKAGES}" | tr ' ' '\n' | grep -v '/protocol$' | tr '\n' ' ')
+  # echo "UNIT_TEST_PACKAGES=${UNIT_TEST_PACKAGES}"
+  # echo "------------------------------------------------------"
+  # echo "UNIT_TEST_PACKAGES_FILTERED=${UNIT_TEST_PACKAGES_FILTERED}"
+  # run_test_for_packages github.com/status-im/status-go/protocol/communities "0" "${UNIT_TEST_COUNT}" "${DEFAULT_TIMEOUT_MINUTES}" "ONLY 'protocol/communities'" &
+  # bg_pids+=("$!")
+  # run_test_for_packages github.com/status-im/status-go/wakuv2 "0" "${UNIT_TEST_COUNT}" "${DEFAULT_TIMEOUT_MINUTES}" "ONLY 'wakuv2'" &
+  # bg_pids+=("$!")
   run_test_for_packages "${UNIT_TEST_PACKAGES_FILTERED}" "0" "${UNIT_TEST_COUNT}" "${DEFAULT_TIMEOUT_MINUTES}" "All packages except 'protocol'" &
   bg_pids+=("$!")
 
   # Spawn separate processes to run `protocol` package
-  for ((i=1; i<=UNIT_TEST_COUNT; i++)); do
-    run_test_for_packages github.com/status-im/status-go/protocol "${i}" 1 "${PROTOCOL_TIMEOUT_MINUTES}" "Only 'protocol' package" &
-    bg_pids+=("$!")
-  done
+  # for ((i=1; i<=UNIT_TEST_COUNT; i++)); do
+  #   run_test_for_packages github.com/status-im/status-go/protocol "${i}" 1 "${PROTOCOL_TIMEOUT_MINUTES}" "Only 'protocol' package" &
+  #   bg_pids+=("$!")
+  # done
 fi
 
 # Wait for jobs and handle failfast

_assets/scripts/run_waku.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
+source "${GIT_ROOT}/_assets/scripts/colors.sh"
+source "${GIT_ROOT}/_assets/scripts/codecov.sh"
+
+root_path="${GIT_ROOT}/tests-functional"
+
+identifier=${BUILD_ID:-$(git rev-parse --short HEAD)}
+container_name="waku-${identifier}"
+
+echo -e "${YLW}Starting waku node...${RST}"
+
+IP_ADDRESS=$(ip -o -4 addr show up primary scope global | awk '{print $4}' | cut -d/ -f1 | head -n1);
+docker run -d --name ${container_name} \
+  -p 60000:60000/tcp -p 9000:9000/udp -p 8645:8645/tcp \
+  harbor.status.im/wakuorg/nwaku:v0.36.0 \
+  --tcp-port=60000 --discv5-discovery=true \
+  --cluster-id=16 --shard=32 --shard=64 \
+  --nat=extip:${IP_ADDRESS} --discv5-udp-port=9000 \
+  --rest-address=0.0.0.0 --store
+
+echo -e "${GRN}Waku node started.${RST}"
+
+read -p "Press any button to exit..." -n 1 -r
+echo
+
+cleanup() {
+  echo -e "${YLW}Removing containers...${RST}"
+  docker ps -a --filter "name=${container_name}" -q | xargs -r docker rm -f
+  echo -e "${GRN}DONE!${RST}"
+}
+
+trap cleanup EXIT

appdatabase/migrations/sql/1761795530_add_index_cid_columns_to_communities_archive_info.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE communities_archive_info ADD COLUMN last_index_cid TEXT DEFAULT "";
+ALTER TABLE communities_archive_info ADD COLUMN index_cid_clock INTEGER DEFAULT 0;

appdatabase/migrations/sql/1761795811_add_archive_distribution_preference.up.sql

@@ -0,0 +1 @@
+ALTER TABLE communities_archive_info ADD COLUMN preferred_distribution_method TEXT DEFAULT 'torrent';

messaging/waku/waku_test.go

@@ -160,12 +160,14 @@ func parseNodes(rec []string) []*enode.Node {
 // In order to run these tests, you must run an nwaku node
 //
 // Using Docker:
-//
-//	IP_ADDRESS=$(hostname -I | awk '{print $1}');
+//  IP_ADDRESS=$(hostname -I | awk '{print $1}');
+// or (if -I on hostname is not available):
+//	IP_ADDRESS=$(ip -o -4 addr show up primary scope global | awk '{print $4}' | cut -d/ -f1 | head -n1);
+// Then run:
 //	docker run \
 //	 -p 60000:60000/tcp -p 9000:9000/udp -p 8645:8645/tcp harbor.status.im/wakuorg/nwaku:v0.36.0 \
-//	 --tcp-port=60000 --discv5-discovery=true --cluster-id=16 --pubsub-topic=/waku/2/rs/16/32 --pubsub-topic=/waku/2/rs/16/64 \
-//	 --nat=extip:${IP_ADDRESS} --discv5-discovery --discv5-udp-port=9000 --rest-address=0.0.0.0 --store
+//	 --tcp-port=60000 --discv5-discovery=true --cluster-id=16 --shard=32 --shard=64 \
+//	 --nat=extip:${IP_ADDRESS} --discv5-udp-port=9000 --rest-address=0.0.0.0 --store
 
 func TestBasicWakuV2(t *testing.T) {
 	nwakuInfo, err := GetNwakuInfo(nil, nil)

protocol/communities/archive_processor_nop.go

@@ -0,0 +1,31 @@
+//go:build disable_torrent
+// +build disable_torrent
+
+package communities
+
+import (
+	"github.com/status-im/status-go/crypto/types"
+)
+
+// CodexArchiveProcessor interface for builds without torrent support
+type CodexArchiveProcessor interface {
+	ProcessArchiveData(communityID types.HexBytes, archiveHash string, archiveData []byte, from, to uint64) error
+}
+
+// CodexArchiveMessageProcessorNop is a no-op implementation for builds without torrent support
+type CodexArchiveMessageProcessorNop struct{}
+
+// NewCodexArchiveMessageProcessor creates a no-op processor for builds without torrent support
+func NewCodexArchiveMessageProcessor(identity interface{}, messaging interface{}, persistence interface{}, logger interface{}, messageHandler interface{}) *CodexArchiveMessageProcessorNop {
+	return &CodexArchiveMessageProcessorNop{}
+}
+
+// SetOnArchiveProcessed is a no-op
+func (p *CodexArchiveMessageProcessorNop) SetOnArchiveProcessed(callback func(hash string, from, to uint64)) {
+	// no-op
+}
+
+// ProcessArchiveData is a no-op that returns an error
+func (p *CodexArchiveMessageProcessorNop) ProcessArchiveData(communityID types.HexBytes, archiveHash string, archiveData []byte, from, to uint64) error {
+	return ErrArchiveNotSupported
+}