From 583892b47955c08a91e248a4552985f2f8d7bafa Mon Sep 17 00:00:00 2001 From: Dario Gabriel Lipicar Date: Fri, 26 Sep 2025 15:24:45 -0300 Subject: [PATCH 1/2] feat: improve blockchainstate --- .../wallet/blockchainstate/blockchainstate.go | 164 ++++++++++++++---- .../blockchainstate/blockchainstate_test.go | 128 +++++++++++--- services/wallet/common/const.go | 5 + 3 files changed, 238 insertions(+), 59 deletions(-) diff --git a/services/wallet/blockchainstate/blockchainstate.go b/services/wallet/blockchainstate/blockchainstate.go index d21f0b78607..793a64ba7ca 100644 --- a/services/wallet/blockchainstate/blockchainstate.go +++ b/services/wallet/blockchainstate/blockchainstate.go @@ -1,63 +1,163 @@ package blockchainstate +//go:generate go tool mockgen -package=mock_blockchainstate -source=blockchainstate.go -destination=mock/blockchainstate.go + import ( "context" + "errors" + "math" + "strconv" "sync" "time" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + + "github.com/status-im/go-wallet-sdk/pkg/contracts/multicall3" + + "github.com/status-im/status-go/rpc/chain" "github.com/status-im/status-go/services/wallet/common" ) +type EthClientGetter interface { + EthClient(chainID uint64) (chain.ClientInterface, error) +} + type LatestBlockData struct { - blockNumber uint64 - timestamp time.Time - blockDuration time.Duration + blockNumber uint64 + timestamp time.Time } type BlockChainState struct { - blkMu sync.RWMutex - latestBlockNumbers map[uint64]LatestBlockData - sinceFn func(time.Time) time.Duration + ethClientGetter EthClientGetter + latestBlockData map[uint64]LatestBlockData // map[chainID]*LatestBlockData + sinceFn func(time.Time) time.Duration + blockDurationFn func(chainID uint64) time.Duration + mu sync.RWMutex } -func NewBlockChainState() *BlockChainState { +func NewBlockChainState(ethClientGetter EthClientGetter) *BlockChainState { return &BlockChainState{ - blkMu: sync.RWMutex{}, - latestBlockNumbers: make(map[uint64]LatestBlockData), - sinceFn: time.Since, + ethClientGetter: ethClientGetter, + latestBlockData: make(map[uint64]LatestBlockData), + sinceFn: time.Since, + blockDurationFn: func(chainID uint64) time.Duration { + blockDuration, found := common.AverageBlockDurationForChain[common.ChainID(chainID)] + if !found { + blockDuration = common.AverageBlockDurationForChain[common.ChainID(common.UnknownChainID)] + } + return blockDuration + }, } } +// Estimate the latest block number based on the last real value and the average block duration. func (s *BlockChainState) GetEstimatedLatestBlockNumber(ctx context.Context, chainID uint64) (uint64, error) { - blockNumber, _ := s.estimateLatestBlockNumber(chainID) - return blockNumber, nil + blockData, err := s.getOrInitializeLatestBlockData(ctx, chainID) + if err != nil { + return 0, err + } + + timeDiff := s.sinceFn(blockData.timestamp) + blockDuration := s.blockDurationFn(chainID) + blockDiff := uint64(math.Floor(float64(timeDiff) / float64(blockDuration))) + return blockData.blockNumber + blockDiff, nil +} + +// Estimate the timestamp for a given block number based on the last real value and the average block duration. +func (s *BlockChainState) GetEstimatedBlockTime(ctx context.Context, chainID uint64, blockNumber uint64) (time.Time, error) { + blockData, err := s.getOrInitializeLatestBlockData(ctx, chainID) + if err != nil { + return time.Time{}, err + } + blockDiff := int64(blockNumber) - int64(blockData.blockNumber) + blockDuration := s.blockDurationFn(chainID) + timeDiff := time.Duration(blockDiff * int64(blockDuration)) + blockTime := blockData.timestamp.Add(timeDiff) + return blockTime, nil +} + +// Set the latest known block number for a chain +func (s *BlockChainState) SetLatestBlockNumber(chainID uint64, blockNumber uint64) { + s.setLatestBlockNumber(chainID, blockNumber) +} + +func (s *BlockChainState) getOrInitializeLatestBlockData(ctx context.Context, chainID uint64) (LatestBlockData, error) { + // Try read lock first for fast path + s.mu.RLock() + blockData, exists := s.latestBlockData[chainID] + s.mu.RUnlock() + if exists { + return blockData, nil + } + + // Not found, need to initialize + s.mu.Lock() + defer s.mu.Unlock() + + // Double check after acquiring write lock + blockData, exists = s.latestBlockData[chainID] + if exists { + return blockData, nil + } + + blockNumber, err := s.getLatestBlockNumber(ctx, chainID) + if err != nil { + return LatestBlockData{}, err + } + s.latestBlockData[chainID] = buildLatestBlockData(blockNumber) + return s.latestBlockData[chainID], nil } -func (s *BlockChainState) SetLastBlockNumber(chainID uint64, blockNumber uint64) { - blockDuration, found := common.AverageBlockDurationForChain[common.ChainID(chainID)] - if !found { - blockDuration = common.AverageBlockDurationForChain[common.ChainID(common.UnknownChainID)] +// Get the latest block number for a chain +func (s *BlockChainState) getLatestBlockNumber(ctx context.Context, chainID uint64) (uint64, error) { + ethClient, err := s.ethClientGetter.EthClient(chainID) + if err != nil { + return 0, err + } + + // Get multicall3 contract address + multicallAddr, exists := multicall3.GetMulticall3Address(int64(chainID)) + if !exists { + return 0, errors.New("Multicall3 not supported on chain ID " + strconv.Itoa(int(chainID))) + } + + // Create multicall3 contract instance for the caller interface + multicallContract, err := multicall3.NewMulticall3(multicallAddr, ethClient) + if err != nil { + return 0, err } - s.setLatestBlockDataForChain(chainID, LatestBlockData{ - blockNumber: blockNumber, - timestamp: time.Now(), - blockDuration: blockDuration, + + blockNumber, err := multicallContract.GetBlockNumber(&bind.CallOpts{ + Context: ctx, }) + if err != nil { + return 0, err + } + + return blockNumber.Uint64(), nil } -func (s *BlockChainState) setLatestBlockDataForChain(chainID uint64, latestBlockData LatestBlockData) { - s.blkMu.Lock() - defer s.blkMu.Unlock() - s.latestBlockNumbers[chainID] = latestBlockData +func (s *BlockChainState) setLatestBlockNumber(chainID uint64, blockNumber uint64) { + s.mu.Lock() + defer s.mu.Unlock() + + oldData, exists := s.latestBlockData[chainID] + if !exists { + // No old value, initialized with newData + s.latestBlockData[chainID] = buildLatestBlockData(blockNumber) + return + } + if oldData.blockNumber >= blockNumber { + // Only update if the new block number is greater than the current one + return + } + // Update the existing value + s.latestBlockData[chainID] = buildLatestBlockData(blockNumber) } -func (s *BlockChainState) estimateLatestBlockNumber(chainID uint64) (uint64, bool) { - s.blkMu.RLock() - defer s.blkMu.RUnlock() - blockData, ok := s.latestBlockNumbers[chainID] - if !ok { - return 0, false +func buildLatestBlockData(blockNumber uint64) LatestBlockData { + return LatestBlockData{ + blockNumber: blockNumber, + timestamp: time.Now(), } - timeDiff := s.sinceFn(blockData.timestamp) - return blockData.blockNumber + uint64((timeDiff / blockData.blockDuration)), true } diff --git a/services/wallet/blockchainstate/blockchainstate_test.go b/services/wallet/blockchainstate/blockchainstate_test.go index eebc5c8f4f3..3e5395ca89f 100644 --- a/services/wallet/blockchainstate/blockchainstate_test.go +++ b/services/wallet/blockchainstate/blockchainstate_test.go @@ -1,10 +1,16 @@ package blockchainstate import ( + "context" + "errors" "testing" "time" + mock_blockchainstate "github.com/status-im/status-go/services/wallet/blockchainstate/mock" + "github.com/stretchr/testify/require" + + "go.uber.org/mock/gomock" ) var mockupTime = time.Unix(946724400, 0) // 2000-01-01 12:00:00 @@ -13,34 +19,102 @@ func mockupSince(t time.Time) time.Duration { return mockupTime.Sub(t) } -func setupTestState(t *testing.T) (s *BlockChainState) { - state := NewBlockChainState() +var mockupBlockDuration = 10 * time.Second + +func mockupBlockDurationFn(chainID uint64) time.Duration { + return mockupBlockDuration +} + +func setupTestState(t *testing.T) (*BlockChainState, *mock_blockchainstate.MockEthClientGetter) { + ctrl := gomock.NewController(t) + ethClientGetter := mock_blockchainstate.NewMockEthClientGetter(ctrl) + + state := NewBlockChainState(ethClientGetter) state.sinceFn = mockupSince - return state + state.blockDurationFn = mockupBlockDurationFn + return state, ethClientGetter } -func TestEstimateLatestBlockNumber(t *testing.T) { - state := setupTestState(t) - - state.setLatestBlockDataForChain(1, LatestBlockData{ - blockNumber: uint64(100), - timestamp: mockupTime.Add(-31 * time.Second), - blockDuration: 10 * time.Second, - }) - - state.setLatestBlockDataForChain(2, LatestBlockData{ - blockNumber: uint64(200), - timestamp: mockupTime.Add(-5 * time.Second), - blockDuration: 12 * time.Second, - }) - - val, ok := state.estimateLatestBlockNumber(1) - require.True(t, ok) - require.Equal(t, uint64(103), val) - val, ok = state.estimateLatestBlockNumber(2) - require.True(t, ok) - require.Equal(t, uint64(200), val) - val, ok = state.estimateLatestBlockNumber(3) - require.False(t, ok) - require.Equal(t, uint64(0), val) +func TestGetEstimatedLatestBlockNumber_WithExistingData(t *testing.T) { + state, _ := setupTestState(t) + + // Manually set block data (simulating what would happen after initialization) + state.latestBlockData[1] = LatestBlockData{ + blockNumber: 100, + timestamp: mockupTime.Add(-31 * time.Second), + } + + state.latestBlockData[2] = LatestBlockData{ + blockNumber: 200, + timestamp: mockupTime.Add(-5 * time.Second), + } + + // Test chain 1: should estimate 3 blocks ahead (31 seconds / 10 seconds per block) + mockupBlockDuration = 10 * time.Second + blockNumber, err := state.GetEstimatedLatestBlockNumber(context.Background(), 1) + require.NoError(t, err) + require.Equal(t, uint64(103), blockNumber) + + // Test chain 2: should not estimate ahead (5 seconds < 12 seconds per block) + mockupBlockDuration = 12 * time.Second + blockNumber, err = state.GetEstimatedLatestBlockNumber(context.Background(), 2) + require.NoError(t, err) + require.Equal(t, uint64(200), blockNumber) +} + +func TestGetEstimatedLatestBlockNumber_WithInitialization(t *testing.T) { + state, ethClientGetter := setupTestState(t) + + // Mock eth client to return an error (simulating network failure) + ethClientGetter.EXPECT().EthClient(uint64(1)).Return(nil, errors.New("network error")).AnyTimes() + + // This should trigger initialization but fail + blockNumber, err := state.GetEstimatedLatestBlockNumber(context.Background(), 1) + require.Error(t, err) + require.Equal(t, uint64(0), blockNumber) + require.Contains(t, err.Error(), "network error") +} + +func TestGetEstimatedBlockTime(t *testing.T) { + state, _ := setupTestState(t) + + // Set up test data + state.latestBlockData[1] = LatestBlockData{ + blockNumber: 100, + timestamp: mockupTime.Add(-10 * time.Second), + } + + // Test block time estimation + mockupBlockDuration = 2 * time.Second + blockTime, err := state.GetEstimatedBlockTime(context.Background(), 1, 105) + require.NoError(t, err) + + // Block 105 is 5 blocks ahead of 100, so 5 * 2 seconds = 10 seconds ahead + expectedTime := mockupTime + require.Equal(t, expectedTime, blockTime) +} + +func TestSetLatestBlockNumber(t *testing.T) { + state, _ := setupTestState(t) + + // Test setting block number + state.SetLatestBlockNumber(1, 100) + + // Verify it was set + blockData, exists := state.latestBlockData[1] + require.True(t, exists) + require.Equal(t, uint64(100), blockData.blockNumber) + require.True(t, blockData.timestamp.After(mockupTime.Add(-time.Minute))) // Should be recent + + // Test setting a smaller block number (should not update) + state.SetLatestBlockNumber(1, 50) + blockData, exists = state.latestBlockData[1] + require.True(t, exists) + require.Equal(t, uint64(100), blockData.blockNumber) // Should still be 100 + + // Test setting a larger block number (should update) + state.SetLatestBlockNumber(1, 150) + blockData, exists = state.latestBlockData[1] + require.True(t, exists) + require.Equal(t, uint64(150), blockData.blockNumber) // Should be updated to 150 } diff --git a/services/wallet/common/const.go b/services/wallet/common/const.go index f38b91e38d8..0dd2598f3bd 100644 --- a/services/wallet/common/const.go +++ b/services/wallet/common/const.go @@ -129,10 +129,15 @@ func IsSupportedChainID(chainID uint64) bool { var AverageBlockDurationForChain = map[ChainID]time.Duration{ ChainID(UnknownChainID): time.Duration(12000) * time.Millisecond, ChainID(EthereumMainnet): time.Duration(12000) * time.Millisecond, + ChainID(EthereumSepolia): time.Duration(12000) * time.Millisecond, ChainID(OptimismMainnet): time.Duration(2000) * time.Millisecond, + ChainID(OptimismSepolia): time.Duration(2000) * time.Millisecond, ChainID(ArbitrumMainnet): time.Duration(250) * time.Millisecond, + ChainID(ArbitrumSepolia): time.Duration(250) * time.Millisecond, ChainID(BaseMainnet): time.Duration(2000) * time.Millisecond, + ChainID(BaseSepolia): time.Duration(2000) * time.Millisecond, ChainID(BSCMainnet): time.Duration(3000) * time.Millisecond, + ChainID(BSCTestnet): time.Duration(3000) * time.Millisecond, ChainID(StatusNetworkSepolia): time.Duration(2000) * time.Millisecond, } From c9ab1fe6df3567606853bc4e1bd9a64c4e738341 Mon Sep 17 00:00:00 2001 From: Dario Gabriel Lipicar Date: Fri, 26 Sep 2025 15:26:24 -0300 Subject: [PATCH 2/2] feat: replace old transfer detection algorithm with multistandardbalance and transferdetector --- services/wallet/activity/service_test.go | 8 - services/wallet/activity/session_service.go | 7 - .../wallet/collectibles/contract_type_db.go | 22 + .../collectibles/ownership/controller.go | 179 +- .../collectibles/ownership/controller_test.go | 181 +- services/wallet/collectibles/service.go | 141 +- .../wallet/multistandardbalance_adaptors.go | 78 + services/wallet/reader.go | 222 +- services/wallet/reader_test.go | 9 +- services/wallet/routeexecution/manager.go | 4 +- services/wallet/service.go | 191 +- services/wallet/token/token.go | 3 + services/wallet/transfer/block_dao.go | 178 -- .../transfer/block_ranges_sequential_dao.go | 291 --- .../block_ranges_sequential_dao_test.go | 141 -- services/wallet/transfer/block_test.go | 208 -- services/wallet/transfer/commands.go | 565 ----- .../wallet/transfer/commands_sequential.go | 1587 -------------- .../transfer/commands_sequential_test.go | 1894 ----------------- services/wallet/transfer/concurrent.go | 298 --- services/wallet/transfer/concurrent_test.go | 151 -- services/wallet/transfer/controller.go | 241 --- services/wallet/transfer/controller_test.go | 248 --- services/wallet/transfer/database.go | 621 ------ services/wallet/transfer/database_test.go | 217 -- services/wallet/transfer/downloader.go | 652 ------ services/wallet/transfer/downloader_test.go | 47 - services/wallet/transfer/iterative.go | 97 - services/wallet/transfer/iterative_test.go | 118 - services/wallet/transfer/query.go | 272 --- services/wallet/transfer/reactor.go | 123 -- .../transfer/sequential_fetch_strategy.go | 129 -- services/wallet/transfer/testutils.go | 345 --- 33 files changed, 674 insertions(+), 8794 deletions(-) create mode 100644 services/wallet/multistandardbalance_adaptors.go delete mode 100644 services/wallet/transfer/block_dao.go delete mode 100644 services/wallet/transfer/block_ranges_sequential_dao.go delete mode 100644 services/wallet/transfer/block_ranges_sequential_dao_test.go delete mode 100644 services/wallet/transfer/block_test.go delete mode 100644 services/wallet/transfer/commands.go delete mode 100644 services/wallet/transfer/commands_sequential.go delete mode 100644 services/wallet/transfer/commands_sequential_test.go delete mode 100644 services/wallet/transfer/concurrent.go delete mode 100644 services/wallet/transfer/concurrent_test.go delete mode 100644 services/wallet/transfer/controller.go delete mode 100644 services/wallet/transfer/controller_test.go delete mode 100644 services/wallet/transfer/database.go delete mode 100644 services/wallet/transfer/database_test.go delete mode 100644 services/wallet/transfer/downloader.go delete mode 100644 services/wallet/transfer/downloader_test.go delete mode 100644 services/wallet/transfer/iterative.go delete mode 100644 services/wallet/transfer/iterative_test.go delete mode 100644 services/wallet/transfer/query.go delete mode 100644 services/wallet/transfer/reactor.go delete mode 100644 services/wallet/transfer/sequential_fetch_strategy.go delete mode 100644 services/wallet/transfer/testutils.go diff --git a/services/wallet/activity/service_test.go b/services/wallet/activity/service_test.go index c6a7937e352..6d3f7c2d748 100644 --- a/services/wallet/activity/service_test.go +++ b/services/wallet/activity/service_test.go @@ -18,7 +18,6 @@ import ( "github.com/status-im/status-go/services/wallet/thirdparty" mock_token "github.com/status-im/status-go/services/wallet/token/mock/token" tokenTypes "github.com/status-im/status-go/services/wallet/token/types" - "github.com/status-im/status-go/services/wallet/transfer" "github.com/status-im/status-go/services/wallet/walletevent" "github.com/status-im/status-go/t/helpers" "github.com/status-im/status-go/transactions" @@ -108,13 +107,6 @@ func setupTransactions(t *testing.T, state testState, txCount int, testTxs []tra allAddresses = append(allAddresses, p.From, p.To) } - txs, fromTrs, toTrs := transfer.GenerateTestTransfers(t, state.service.db, len(pendings), txCount) - for i := range txs { - transfer.InsertTestTransfer(t, state.service.db, txs[i].To, &txs[i]) - } - - allAddresses = append(append(allAddresses, fromTrs...), toTrs...) - state.tokenMock.EXPECT().LookupTokenIdentity(gomock.Any(), gomock.Any(), gomock.Any()).Return( &tokenTypes.Token{ ChainID: 5, diff --git a/services/wallet/activity/session_service.go b/services/wallet/activity/session_service.go index c2eb9432441..cb979c29ee9 100644 --- a/services/wallet/activity/session_service.go +++ b/services/wallet/activity/session_service.go @@ -19,7 +19,6 @@ import ( "github.com/status-im/status-go/services/wallet/common" "github.com/status-im/status-go/services/wallet/responses" "github.com/status-im/status-go/services/wallet/routeexecution" - "github.com/status-im/status-go/services/wallet/transfer" "github.com/status-im/status-go/services/wallet/walletevent" "github.com/status-im/status-go/transactions" @@ -317,12 +316,6 @@ func (s *Service) processEvents(ctx context.Context) { Hash: payload.Hash, }) debounceProcessChangesFn() - case transfer.EventNewTransfers: - eventCount++ - // No updates here, these are detected with their final state, just trigger - // the detection of new entries - newTxs = true - debounceProcessChangesFn() case routeexecution.EventRouteExecutionTransactionSent: sentTxs, ok := event.EventParams.(*responses.RouterSentTransactions) if ok && sentTxs != nil { diff --git a/services/wallet/collectibles/contract_type_db.go b/services/wallet/collectibles/contract_type_db.go index 6516c00097e..585165e5a9e 100644 --- a/services/wallet/collectibles/contract_type_db.go +++ b/services/wallet/collectibles/contract_type_db.go @@ -10,6 +10,28 @@ import ( "github.com/status-im/status-go/sqlite" ) +type ContractTypeDB struct { + db *sql.DB +} + +func NewContractTypeDB(sqlDb *sql.DB) *ContractTypeDB { + return &ContractTypeDB{ + db: sqlDb, + } +} + +func (o *ContractTypeDB) GetContractTypes(ids []thirdparty.ContractID) (map[thirdparty.ContractID]w_common.ContractType, error) { + ret := make(map[thirdparty.ContractID]w_common.ContractType) + var err error + for _, id := range ids { + ret[id], err = readContractType(o.db, id) + if err != nil { + return nil, err + } + } + return ret, nil +} + func upsertContractType(creator sqlite.StatementCreator, id thirdparty.ContractID, contractType w_common.ContractType) error { if contractType == w_common.ContractTypeUnknown { return nil diff --git a/services/wallet/collectibles/ownership/controller.go b/services/wallet/collectibles/ownership/controller.go index 7b8bff076b4..df4a5ecf05b 100644 --- a/services/wallet/collectibles/ownership/controller.go +++ b/services/wallet/collectibles/ownership/controller.go @@ -4,12 +4,18 @@ package ownership import ( "context" + "slices" "sync" + "time" "go.uber.org/zap" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/event" + + "github.com/status-im/go-wallet-sdk/pkg/balance/multistandardfetcher" + "github.com/status-im/go-wallet-sdk/pkg/contracts/erc1155" + "github.com/status-im/go-wallet-sdk/pkg/contracts/erc721" + "github.com/status-im/go-wallet-sdk/pkg/eventlog" gocommon "github.com/status-im/status-go/common" "github.com/status-im/status-go/crypto/types" @@ -18,8 +24,9 @@ import ( "github.com/status-im/status-go/rpc/network" "github.com/status-im/status-go/services/accounts/accountsevent" walletCommon "github.com/status-im/status-go/services/wallet/common" + "github.com/status-im/status-go/services/wallet/multistandardbalance" "github.com/status-im/status-go/services/wallet/thirdparty" - "github.com/status-im/status-go/services/wallet/transfer" + "github.com/status-im/status-go/services/wallet/transferdetector" "github.com/status-im/status-go/services/wallet/walletevent" ) @@ -30,8 +37,6 @@ const ( type loaderPerChainID = map[walletCommon.ChainID]*PeriodicalLoader type loaderPerAddressAndChainID = map[common.Address]loaderPerChainID -type TransferCb func(common.Address, walletCommon.ChainID, []transfer.Transfer) - type AccountsProvider interface { GetWalletAddresses() ([]types.Address, error) } @@ -41,12 +46,18 @@ type NetworksProvider interface { GetPublisher() *pubsub.Publisher } +type BlockChainStateProvider interface { + GetEstimatedBlockTime(ctx context.Context, chainID uint64, blockNumber uint64) (time.Time, error) +} + type Controller struct { - fetcher CollectibleOwnershipFetcher - storage CollectibleOwnershipStorage - walletFeed *event.Feed - accountsProvider AccountsProvider - accountsPublisher *pubsub.Publisher + fetcher CollectibleOwnershipFetcher + storage CollectibleOwnershipStorage + accountsProvider AccountsProvider + accountsPublisher *pubsub.Publisher + multistandardBalancePublisher *pubsub.Publisher + transferDetectorPublisher *pubsub.Publisher + blockChainStateProvider BlockChainStateProvider networksProvider NetworksProvider @@ -66,24 +77,28 @@ type Controller struct { func NewController( storage CollectibleOwnershipStorage, - walletFeed *event.Feed, accountsProvider AccountsProvider, accountsPublisher *pubsub.Publisher, networksProvider NetworksProvider, + multistandardBalancePublisher *pubsub.Publisher, + transferDetectorPublisher *pubsub.Publisher, + blockChainStateProvider BlockChainStateProvider, fetcher CollectibleOwnershipFetcher, collectiblesPublisher *pubsub.Publisher, logger *zap.Logger, ) *Controller { return &Controller{ - fetcher: fetcher, - storage: storage, - walletFeed: walletFeed, - accountsProvider: accountsProvider, - accountsPublisher: accountsPublisher, - networksProvider: networksProvider, - periodicalLoaders: make(loaderPerAddressAndChainID), - collectiblesPublisher: collectiblesPublisher, - logger: logger.Named("OwnershipController"), + fetcher: fetcher, + storage: storage, + accountsProvider: accountsProvider, + accountsPublisher: accountsPublisher, + networksProvider: networksProvider, + multistandardBalancePublisher: multistandardBalancePublisher, + transferDetectorPublisher: transferDetectorPublisher, + blockChainStateProvider: blockChainStateProvider, + periodicalLoaders: make(loaderPerAddressAndChainID), + collectiblesPublisher: collectiblesPublisher, + logger: logger.Named("OwnershipController"), } } @@ -105,11 +120,14 @@ func (c *Controller) Start() { // Setup collectibles fetch when a new account gets added c.startAccountsWatcher() - // Setup collectibles fetch when relevant activity is detected - c.startWalletEventsWatcher() - // Setup collectibles fetch when active networks change c.startNetworkEventsWatcher() + + // Start balance change watcher + c.startBalanceChangeWatcher() + + // Start transfer detection watcher + c.startTransferDetectionWatcher() } func (c *Controller) Stop() { @@ -120,8 +138,6 @@ func (c *Controller) Stop() { close(c.stopCh) c.stopCh = nil - c.stopWalletEventsWatcher() - c.stopPeriodicalLoaders() } @@ -237,45 +253,63 @@ func (c *Controller) startAccountsWatcher() { }() } -func (c *Controller) startWalletEventsWatcher() { - if c.walletEventsWatcher != nil { - return - } - - if c.walletFeed == nil { +func (c *Controller) startNetworkEventsWatcher() { + if c.networksProvider == nil { return } - walletEventCb := func(event walletevent.Event) { - if event.Type != transfer.EventInternalERC721TransferDetected && - event.Type != transfer.EventInternalERC1155TransferDetected { - return - } - - chainID := walletCommon.ChainID(event.ChainID) - for _, account := range event.Accounts { - c.refetchOwnershipIfRecentTransfer(account, chainID, event.At) + ch, unsub := pubsub.Subscribe[network.EventActiveNetworksChanged](c.networksProvider.GetPublisher(), 10) + go func() { + defer gocommon.LogOnPanic() + defer unsub() + for { + select { + case <-c.stopCh: + return + case _, ok := <-ch: + if !ok { + return + } + c.checkPeriodicalLoaders() + } } - } - - c.walletEventsWatcher = walletevent.NewWatcher(c.walletFeed, walletEventCb) - - c.walletEventsWatcher.Start() + }() } -func (c *Controller) stopWalletEventsWatcher() { - if c.walletEventsWatcher != nil { - c.walletEventsWatcher.Stop() - c.walletEventsWatcher = nil +func (c *Controller) startBalanceChangeWatcher() { + if c.multistandardBalancePublisher == nil { + return } + + ch, unsub := pubsub.Subscribe[multistandardbalance.EventBalanceFetchFinished](c.multistandardBalancePublisher, 10) + go func() { + defer gocommon.LogOnPanic() + defer unsub() + for { + select { + case <-c.stopCh: + return + case event, ok := <-ch: + if !ok { + return + } + switch event.ResultType { + case multistandardfetcher.ResultTypeERC721, multistandardfetcher.ResultTypeERC1155: + if event.BalanceChanged { + c.refetchOwnershipIfRecentTx(event.Key.Account, walletCommon.ChainID(event.Key.ChainID), event.NewState.FetchedAt) + } + } + } + } + }() } -func (c *Controller) startNetworkEventsWatcher() { - if c.networksProvider == nil { +func (c *Controller) startTransferDetectionWatcher() { + if c.transferDetectorPublisher == nil { return } - ch, unsub := pubsub.Subscribe[network.EventActiveNetworksChanged](c.networksProvider.GetPublisher(), 10) + ch, unsub := pubsub.Subscribe[transferdetector.EventTransferDetectionFinished](c.transferDetectorPublisher, 10) go func() { defer gocommon.LogOnPanic() defer unsub() @@ -283,17 +317,54 @@ func (c *Controller) startNetworkEventsWatcher() { select { case <-c.stopCh: return - case _, ok := <-ch: + case msg, ok := <-ch: if !ok { return } - c.checkPeriodicalLoaders() + for _, event := range msg.Events { + switch event.EventKey { + case eventlog.ERC721Transfer: + unpackedEvent, ok := event.Unpacked.(erc721.Erc721Transfer) + if !ok { + c.logger.Error("failed to unpack ERC721Transfer event") + continue + } + c.refetchOwnershipIfRelevantEvent(msg.Accounts, unpackedEvent.From, unpackedEvent.To, msg.ChainID, unpackedEvent.Raw.BlockNumber) + case eventlog.ERC1155TransferSingle: + unpackedEvent, ok := event.Unpacked.(erc1155.Erc1155TransferSingle) + if !ok { + c.logger.Error("failed to unpack ERC1155TransferSingle event") + continue + } + c.refetchOwnershipIfRelevantEvent(msg.Accounts, unpackedEvent.From, unpackedEvent.To, msg.ChainID, unpackedEvent.Raw.BlockNumber) + case eventlog.ERC1155TransferBatch: + unpackedEvent, ok := event.Unpacked.(erc1155.Erc1155TransferBatch) + if !ok { + c.logger.Error("failed to unpack ERC1155TransferBatch event") + continue + } + c.refetchOwnershipIfRelevantEvent(msg.Accounts, unpackedEvent.From, unpackedEvent.To, msg.ChainID, unpackedEvent.Raw.BlockNumber) + } + } } } }() } -func (c *Controller) refetchOwnershipIfRecentTransfer(account common.Address, chainID walletCommon.ChainID, latestTxTimestamp int64) { +func (c *Controller) refetchOwnershipIfRelevantEvent(checkedAccounts []common.Address, eventFrom common.Address, eventTo common.Address, chainID uint64, blockNumber uint64) { + for _, address := range []common.Address{eventFrom, eventTo} { + if slices.Contains(checkedAccounts, address) { + blockTime, err := c.blockChainStateProvider.GetEstimatedBlockTime(context.TODO(), chainID, blockNumber) + if err != nil { + c.logger.Error("failed to get estimated block time", zap.Error(err)) + continue + } + c.refetchOwnershipIfRecentTx(address, walletCommon.ChainID(chainID), blockTime.Unix()) + } + } +} + +func (c *Controller) refetchOwnershipIfRecentTx(account common.Address, chainID walletCommon.ChainID, latestTxTimestamp int64) { // Check last ownership update timestamp timestamp, err := c.storage.GetOwnershipUpdateTimestamp(account, chainID) diff --git a/services/wallet/collectibles/ownership/controller_test.go b/services/wallet/collectibles/ownership/controller_test.go index e9af387cff4..cebdbc3a555 100644 --- a/services/wallet/collectibles/ownership/controller_test.go +++ b/services/wallet/collectibles/ownership/controller_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/event" "github.com/status-im/status-go/crypto/types" "github.com/status-im/status-go/params" @@ -18,13 +17,13 @@ import ( "github.com/status-im/status-go/services/wallet/collectibles/ownership" mock_ownership "github.com/status-im/status-go/services/wallet/collectibles/ownership/mock" walletCommon "github.com/status-im/status-go/services/wallet/common" - mock_common "github.com/status-im/status-go/services/wallet/common/mock" + "github.com/status-im/status-go/services/wallet/multistandardbalance" "github.com/status-im/status-go/services/wallet/thirdparty" - "github.com/status-im/status-go/services/wallet/transfer" - "github.com/status-im/status-go/services/wallet/walletevent" "github.com/status-im/status-go/t/helpers" "github.com/status-im/status-go/walletdatabase" + "github.com/status-im/go-wallet-sdk/pkg/balance/multistandardfetcher" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -70,20 +69,20 @@ func TestControllerMultipleAccountsAddedEvent(t *testing.T) { return emptyCollectiblesContainer, nil }).AnyTimes() - feed := new(event.Feed) - feedSub := mock_common.NewFeedSubscription(feed) - defer feedSub.Close() - + multistandardBalancePublisher := pubsub.NewPublisher() + transferDetectorPublisher := pubsub.NewPublisher() + blockChainStateProvider := mock_ownership.NewMockBlockChainStateProvider(mockCtrl) publisher := pubsub.NewPublisher() - logger := zaptest.NewLogger(t).WithOptions(zap.AddCallerSkip(1)) controller := ownership.NewController( ownership.NewOwnershipDB(walletDB), - feed, accountsProvider, accountsPublisher, networksProvider, + multistandardBalancePublisher, + transferDetectorPublisher, + blockChainStateProvider, ownershipFetcher, publisher, logger, @@ -121,7 +120,7 @@ func TestControllerMultipleAccountsAddedEvent(t *testing.T) { controller.Stop() } -func TestControllerWalletEventsWatcher(t *testing.T) { +func TestControllerMultiStandardBalanceEvents(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() @@ -157,19 +156,20 @@ func TestControllerWalletEventsWatcher(t *testing.T) { return emptyCollectiblesContainer, nil }).AnyTimes() - feed := new(event.Feed) - feedSub := mock_common.NewFeedSubscription(feed) - defer feedSub.Close() - + multistandardBalancePublisher := pubsub.NewPublisher() + transferDetectorPublisher := pubsub.NewPublisher() + blockChainStateProvider := mock_ownership.NewMockBlockChainStateProvider(mockCtrl) publisher := pubsub.NewPublisher() logger := zaptest.NewLogger(t).WithOptions(zap.AddCallerSkip(1)) controller := ownership.NewController( ownershipDB, - feed, accountsProvider, accountsPublisher, networksProvider, + multistandardBalancePublisher, + transferDetectorPublisher, + blockChainStateProvider, ownershipFetcher, publisher, logger, @@ -199,13 +199,21 @@ func TestControllerWalletEventsWatcher(t *testing.T) { }, 1*time.Second, 100*time.Millisecond) // Test ERC721 transfer event - erc721Event := walletevent.Event{ - Type: transfer.EventInternalERC721TransferDetected, - ChainID: 1, - Accounts: []common.Address{common.Address(fakeAddress)}, - At: time.Now().Unix(), + erc721Event := multistandardbalance.EventBalanceFetchFinished{ + Key: multistandardbalance.BalancesKey{ + ChainID: 1, + Account: common.Address(fakeAddress), + }, + ResultType: multistandardfetcher.ResultTypeERC721, + BalanceChanged: true, + OldState: multistandardbalance.State{ + FetchedAt: time.Now().Unix() - 1000, + }, + NewState: multistandardbalance.State{ + FetchedAt: time.Now().Unix(), + }, } - feed.Send(erc721Event) + pubsub.Publish(multistandardBalancePublisher, erc721Event) // Check that the controller eventually reaches the delayed state require.Eventually(t, func() bool { @@ -226,13 +234,21 @@ func TestControllerWalletEventsWatcher(t *testing.T) { }, 1*time.Second, 100*time.Millisecond) // Test ERC1155 transfer event - erc1155Event := walletevent.Event{ - Type: transfer.EventInternalERC1155TransferDetected, - ChainID: 1, - Accounts: []common.Address{common.Address(fakeAddress)}, - At: time.Now().Unix(), + erc1155Event := multistandardbalance.EventBalanceFetchFinished{ + Key: multistandardbalance.BalancesKey{ + ChainID: 1, + Account: common.Address(fakeAddress), + }, + ResultType: multistandardfetcher.ResultTypeERC1155, + BalanceChanged: true, + OldState: multistandardbalance.State{ + FetchedAt: time.Now().Unix() - 1000, + }, + NewState: multistandardbalance.State{ + FetchedAt: time.Now().Unix(), + }, } - feed.Send(erc1155Event) + pubsub.Publish(multistandardBalancePublisher, erc1155Event) // Check that the controller eventually reaches the delayed state require.Eventually(t, func() bool { @@ -253,23 +269,62 @@ func TestControllerWalletEventsWatcher(t *testing.T) { }, 1*time.Second, 100*time.Millisecond) // Test too old transfer detected event (should not trigger refetch) - tooOldEvent := walletevent.Event{ - Type: transfer.EventInternalERC721TransferDetected, - ChainID: 1, - Accounts: []common.Address{common.Address(fakeAddress)}, - At: time.Now().Unix() - 1000, + tooOldEvent := multistandardbalance.EventBalanceFetchFinished{ + Key: multistandardbalance.BalancesKey{ + ChainID: 1, + Account: common.Address(fakeAddress), + }, + ResultType: multistandardfetcher.ResultTypeERC721, + BalanceChanged: true, + OldState: multistandardbalance.State{ + FetchedAt: time.Now().Unix() - 2000 - 30*60, + }, + NewState: multistandardbalance.State{ + FetchedAt: time.Now().Unix() - 1000 - 30*60, + }, } - feed.Send(tooOldEvent) + pubsub.Publish(multistandardBalancePublisher, tooOldEvent) + + time.Sleep(100 * time.Millisecond) + require.Equal(t, ownership.LoaderStateIdle, controller.GetLoaderState(walletCommon.ChainID(1), common.Address(fakeAddress))) // Test non-relevant event (should not trigger refetch) - nonRelevantEvent := walletevent.Event{ - Type: "non-relevant-event", - ChainID: 1, - Accounts: []common.Address{common.Address(fakeAddress)}, - At: time.Now().Unix(), - } - feed.Send(nonRelevantEvent) + pubsub.Publish(multistandardBalancePublisher, multistandardbalance.EventBalanceFetchFinished{ + Key: multistandardbalance.BalancesKey{ + ChainID: 1, + Account: common.Address(fakeAddress), + }, + ResultType: multistandardfetcher.ResultTypeNative, // Native balance fetch + BalanceChanged: true, + OldState: multistandardbalance.State{ + FetchedAt: time.Now().Unix() - 1000, + }, + NewState: multistandardbalance.State{ + FetchedAt: time.Now().Unix(), + }, + }) + + time.Sleep(100 * time.Millisecond) + + require.Equal(t, ownership.LoaderStateIdle, controller.GetLoaderState(walletCommon.ChainID(1), common.Address(fakeAddress))) + + pubsub.Publish(multistandardBalancePublisher, multistandardbalance.EventBalanceFetchFinished{ + Key: multistandardbalance.BalancesKey{ + ChainID: 1, + Account: common.Address(fakeAddress), + }, + ResultType: multistandardfetcher.ResultTypeERC721, + BalanceChanged: false, // Balance unchanged + OldState: multistandardbalance.State{ + FetchedAt: time.Now().Unix() - 1000, + }, + NewState: multistandardbalance.State{ + FetchedAt: time.Now().Unix(), + }, + }) + + time.Sleep(100 * time.Millisecond) require.Equal(t, ownership.LoaderStateIdle, controller.GetLoaderState(walletCommon.ChainID(1), common.Address(fakeAddress))) @@ -308,19 +363,20 @@ func TestControllerNetworkEventsWatcher(t *testing.T) { return emptyCollectiblesContainer, nil }).AnyTimes() - feed := new(event.Feed) - feedSub := mock_common.NewFeedSubscription(feed) - defer feedSub.Close() - + multistandardBalancePublisher := pubsub.NewPublisher() + transferDetectorPublisher := pubsub.NewPublisher() + blockChainStateProvider := mock_ownership.NewMockBlockChainStateProvider(mockCtrl) publisher := pubsub.NewPublisher() logger := zaptest.NewLogger(t).WithOptions(zap.AddCallerSkip(1)) controller := ownership.NewController( ownership.NewOwnershipDB(walletDB), - feed, accountsProvider, accountsPublisher, networksProvider, + multistandardBalancePublisher, + transferDetectorPublisher, + blockChainStateProvider, ownershipFetcher, publisher, logger, @@ -480,19 +536,20 @@ func TestControllerTriggerLoad(t *testing.T) { gomock.Any(), ).Return(collectiblesContainer999, nil).Times(1) - feed := new(event.Feed) - feedSub := mock_common.NewFeedSubscription(feed) - defer feedSub.Close() - + multistandardBalancePublisher := pubsub.NewPublisher() + transferDetectorPublisher := pubsub.NewPublisher() + blockChainStateProvider := mock_ownership.NewMockBlockChainStateProvider(mockCtrl) publisher := pubsub.NewPublisher() logger := zaptest.NewLogger(t).WithOptions(zap.AddCallerSkip(1)) controller := ownership.NewController( ownership.NewOwnershipDB(walletDB), - feed, accountsProvider, accountsPublisher, networksProvider, + multistandardBalancePublisher, + transferDetectorPublisher, + blockChainStateProvider, ownershipFetcher, publisher, logger, @@ -576,19 +633,20 @@ func TestControllerAccountsEvents(t *testing.T) { return emptyCollectiblesContainer, nil }).AnyTimes() - feed := new(event.Feed) - feedSub := mock_common.NewFeedSubscription(feed) - defer feedSub.Close() - + multistandardBalancePublisher := pubsub.NewPublisher() + transferDetectorPublisher := pubsub.NewPublisher() + blockChainStateProvider := mock_ownership.NewMockBlockChainStateProvider(mockCtrl) publisher := pubsub.NewPublisher() logger := zaptest.NewLogger(t).WithOptions(zap.AddCallerSkip(1)) controller := ownership.NewController( ownership.NewOwnershipDB(walletDB), - feed, accountsProvider, accountsPublisher, networksProvider, + multistandardBalancePublisher, + transferDetectorPublisher, + blockChainStateProvider, ownershipFetcher, publisher, logger, @@ -698,19 +756,20 @@ func TestControllerPeriodicalLoads(t *testing.T) { return emptyCollectiblesContainer, nil }).AnyTimes() - feed := new(event.Feed) - feedSub := mock_common.NewFeedSubscription(feed) - defer feedSub.Close() - + multistandardBalancePublisher := pubsub.NewPublisher() + transferDetectorPublisher := pubsub.NewPublisher() + blockChainStateProvider := mock_ownership.NewMockBlockChainStateProvider(mockCtrl) publisher := pubsub.NewPublisher() logger := zaptest.NewLogger(t).WithOptions(zap.AddCallerSkip(1)) controller := ownership.NewController( ownership.NewOwnershipDB(walletDB), - feed, accountsProvider, accountsPublisher, networksProvider, + multistandardBalancePublisher, + transferDetectorPublisher, + blockChainStateProvider, ownershipFetcher, publisher, logger, diff --git a/services/wallet/collectibles/service.go b/services/wallet/collectibles/service.go index cacedb89867..8e5fe197c4b 100644 --- a/services/wallet/collectibles/service.go +++ b/services/wallet/collectibles/service.go @@ -17,17 +17,13 @@ import ( gocommon "github.com/status-im/status-go/common" "github.com/status-im/status-go/logutils" - "github.com/status-im/status-go/multiaccounts/accounts" "github.com/status-im/status-go/pkg/pubsub" - "github.com/status-im/status-go/rpc/network" "github.com/status-im/status-go/services/wallet/async" - "github.com/status-im/status-go/services/wallet/bigint" "github.com/status-im/status-go/services/wallet/collectibles/ownership" walletCommon "github.com/status-im/status-go/services/wallet/common" "github.com/status-im/status-go/services/wallet/community" "github.com/status-im/status-go/services/wallet/thirdparty" - "github.com/status-im/status-go/services/wallet/transfer" "github.com/status-im/status-go/services/wallet/walletevent" ) @@ -100,7 +96,6 @@ type Service struct { ownershipController *ownership.Controller db *sql.DB ownershipDB ownership.OwnershipStorage - transferDB *transfer.Database communityManager *community.Manager walletFeed *event.Feed scheduler *async.MultiClientScheduler @@ -117,38 +112,26 @@ type Service struct { func NewService( db *sql.DB, walletFeed *event.Feed, - accountsDB *accounts.Database, - accountsPublisher *pubsub.Publisher, communityManager *community.Manager, - networkManager *network.Manager, - manager *Manager) *Service { - - publisher := pubsub.NewPublisher() + manager *Manager, + ownershipController *ownership.Controller, + publisher *pubsub.Publisher, +) *Service { ownershipDB := ownership.NewOwnershipDB(db) logger := logutils.ZapLogger().Named("Collectibles") s := &Service{ - manager: manager, - ownershipController: ownership.NewController( - ownershipDB, - walletFeed, - accountsDB, - accountsPublisher, - networkManager, - manager, - publisher, - logger, - ), - db: db, - ownershipDB: ownershipDB, - transferDB: transfer.NewDB(db), - communityManager: communityManager, - walletFeed: walletFeed, - scheduler: async.NewMultiClientScheduler(), - group: async.NewGroup(context.Background()), - publisher: publisher, - logger: logger.Named("Service"), + manager: manager, + ownershipController: ownershipController, + db: db, + ownershipDB: ownershipDB, + communityManager: communityManager, + walletFeed: walletFeed, + scheduler: async.NewMultiClientScheduler(), + group: async.NewGroup(context.Background()), + publisher: publisher, + logger: logger.Named("Service"), } return s } @@ -363,7 +346,6 @@ func (s *Service) Start(ctx context.Context) { s.closeCh = make(chan struct{}) s.ownershipController.Start() - s.startWalletEventsWatcher() s.startOwnershipLoadWatcher() } @@ -377,7 +359,6 @@ func (s *Service) Stop() { s.scheduler.Stop() s.ownershipController.Stop() - s.stopWalletEventsWatcher() } func (s *Service) sendResponseEvent(requestID *int32, eventType walletevent.EventType, payloadObj interface{}, resErr error) { @@ -468,62 +449,6 @@ func (s *Service) collectibleIDsToDataType(ctx context.Context, ids []thirdparty return nil, errors.New("unknown data type") } -func (s *Service) onOwnedCollectiblesChanged(account common.Address, chainID walletCommon.ChainID, newOrUpdated []thirdparty.CollectibleUniqueID) { - // Try to find a matching transfer for newly added/updated collectibles - if len(newOrUpdated) > 0 { - hashMap := s.lookupTransferForCollectibles(account, newOrUpdated) - s.notifyCommunityCollectiblesReceived(account, chainID, newOrUpdated, hashMap) - } -} - -func (s *Service) onCollectiblesTransfer(account common.Address, chainID walletCommon.ChainID, transfers []transfer.Transfer) { - for _, transfer := range transfers { - // If Collectible is already in the DB, update transfer ID with the latest detected transfer - id := thirdparty.CollectibleUniqueID{ - ContractID: thirdparty.ContractID{ - ChainID: chainID, - Address: transfer.Log.Address, - }, - TokenID: &bigint.BigInt{Int: transfer.TokenID}, - } - err := s.manager.SetCollectibleTransferID(account, id, transfer.ID, true) - if err != nil { - s.logger.Error("Error setting transfer ID for collectible", zap.Error(err)) - } - } -} - -func (s *Service) lookupTransferForCollectibles(account common.Address, collectibles []thirdparty.CollectibleUniqueID) map[thirdparty.CollectibleUniqueID]TxHashData { - // There are some limitations to this approach: - // - Collectibles ownership and transfers are not in sync and might represent the state at different moments. - // - We have no way of knowing if the latest collectible transfer we've detected is actually the latest one, so the timestamp we - // use might be older than the real one. - // - There might be detected transfers that are temporarily not reflected in the collectibles ownership. - // - For ERC721 tokens we should only look for incoming transfers. For ERC1155 tokens we should look for both incoming and outgoing transfers. - // We need to get the contract standard for each collectible to know which approach to take. - - result := make(map[thirdparty.CollectibleUniqueID]TxHashData) - - for _, id := range collectibles { - transfer, err := s.transferDB.GetLatestCollectibleTransfer(account, id) - if err != nil { - s.logger.Error("Error fetching latest collectible transfer", zap.Error(err)) - continue - } - if transfer != nil { - result[id] = TxHashData{ - Hash: transfer.Transaction.Hash(), - TxID: transfer.ID, - } - err = s.manager.SetCollectibleTransferID(account, id, transfer.ID, false) - if err != nil { - s.logger.Error("Error setting transfer ID for collectible", zap.Error(err)) - } - } - } - return result -} - func (s *Service) notifyCommunityCollectiblesReceived(account common.Address, chainID walletCommon.ChainID, collectibles []thirdparty.CollectibleUniqueID, hashMap map[thirdparty.CollectibleUniqueID]TxHashData) { ctx := context.Background() @@ -601,39 +526,6 @@ func (s *Service) notifyCommunityCollectiblesReceived(account common.Address, ch }) } -func (s *Service) startWalletEventsWatcher() { - if s.walletEventsWatcher != nil { - return - } - - if s.walletFeed == nil { - return - } - - walletEventCb := func(event walletevent.Event) { - if event.Type != transfer.EventInternalERC721TransferDetected && - event.Type != transfer.EventInternalERC1155TransferDetected { - return - } - - chainID := walletCommon.ChainID(event.ChainID) - for _, account := range event.Accounts { - s.onCollectiblesTransfer(account, chainID, event.EventParams.([]transfer.Transfer)) - } - } - - s.walletEventsWatcher = walletevent.NewWatcher(s.walletFeed, walletEventCb) - - s.walletEventsWatcher.Start() -} - -func (s *Service) stopWalletEventsWatcher() { - if s.walletEventsWatcher != nil { - s.walletEventsWatcher.Stop() - s.walletEventsWatcher = nil - } -} - func (s *Service) startOwnershipLoadWatcher() { if s.publisher == nil { return @@ -656,7 +548,6 @@ func (s *Service) startOwnershipLoadWatcher() { // Send WalletEvent to the client s.triggerWalletEvent(EventCollectiblesOwnershipUpdateStarted, event.ChainID, event.Account, "") case event := <-loadPartialCh: - s.onOwnedCollectiblesChanged(event.Account, event.ChainID, event.Added) // Send WalletEvent to the client updateMessage := OwnershipUpdateMessage{ Added: event.Added, @@ -667,10 +558,6 @@ func (s *Service) startOwnershipLoadWatcher() { } s.triggerWalletEvent(EventCollectiblesOwnershipUpdatePartial, event.ChainID, event.Account, string(encodedMessage)) case event := <-loadFinishedCh: - addedOrUpdated := make([]thirdparty.CollectibleUniqueID, 0, len(event.Added)+len(event.Updated)) - addedOrUpdated = append(addedOrUpdated, event.Added...) - addedOrUpdated = append(addedOrUpdated, event.Updated...) - s.onOwnedCollectiblesChanged(event.Account, event.ChainID, addedOrUpdated) // Send WalletEvent to the client updateMessage := OwnershipUpdateMessage{ Added: event.Added, diff --git a/services/wallet/multistandardbalance_adaptors.go b/services/wallet/multistandardbalance_adaptors.go new file mode 100644 index 00000000000..b8ed1953ed7 --- /dev/null +++ b/services/wallet/multistandardbalance_adaptors.go @@ -0,0 +1,78 @@ +package wallet + +import ( + "github.com/ethereum/go-ethereum/common" + collectibles "github.com/status-im/status-go/services/wallet/collectibles" + collectibles_ownership "github.com/status-im/status-go/services/wallet/collectibles/ownership" + w_common "github.com/status-im/status-go/services/wallet/common" + "github.com/status-im/status-go/services/wallet/multistandardbalance" + "github.com/status-im/status-go/services/wallet/thirdparty" + "github.com/status-im/status-go/services/wallet/token" +) + +type MultistandardBalanceTokenListProvider struct { + tokenManager *token.Manager +} + +func NewMultistandardBalanceTokenListProvider(tokenManager *token.Manager) *MultistandardBalanceTokenListProvider { + return &MultistandardBalanceTokenListProvider{tokenManager: tokenManager} +} + +func (p *MultistandardBalanceTokenListProvider) GetTokenContractAddresses(chainID uint64) ([]common.Address, error) { + tokens, err := p.tokenManager.GetTokens(chainID) + if err != nil { + return nil, err + } + + addresses := make([]common.Address, 0) + for _, token := range tokens { + addresses = append(addresses, token.Address) + } + + return addresses, nil +} + +type MultistandardBalanceCollectiblesListProvider struct { + storage collectibles_ownership.OwnershipStorage + contractTypeDB *collectibles.ContractTypeDB +} + +func NewMultistandardBalanceCollectiblesListProvider(storage collectibles_ownership.OwnershipStorage, contractTypeDB *collectibles.ContractTypeDB) *MultistandardBalanceCollectiblesListProvider { + return &MultistandardBalanceCollectiblesListProvider{storage: storage, contractTypeDB: contractTypeDB} +} + +func (p *MultistandardBalanceCollectiblesListProvider) GetCollectiblesList(chainID uint64, account common.Address) (erc721 []multistandardbalance.CollectibleID, erc1155 []multistandardbalance.CollectibleID, err error) { + const offset = 0 + const noLimit = -1 + ownedCollectibles, err := p.storage.GetOwnedCollectibles([]w_common.ChainID{w_common.ChainID(chainID)}, []common.Address{account}, offset, noLimit) + if err != nil { + return nil, nil, err + } + + contracts := make([]thirdparty.ContractID, len(ownedCollectibles)) + for idx, collectible := range ownedCollectibles { + contracts[idx] = collectible.ContractID + } + + contractTypes, err := p.contractTypeDB.GetContractTypes(contracts) + if err != nil { + return nil, nil, err + } + + for _, ownedCollectible := range ownedCollectibles { + contractType := contractTypes[ownedCollectible.ContractID] + if contractType == w_common.ContractTypeERC721 { + erc721 = append(erc721, multistandardbalance.CollectibleID{ + ContractAddress: ownedCollectible.ContractID.Address, + TokenID: ownedCollectible.TokenID.Int, + }) + } else if contractType == w_common.ContractTypeERC1155 { + erc1155 = append(erc1155, multistandardbalance.CollectibleID{ + ContractAddress: ownedCollectible.ContractID.Address, + TokenID: ownedCollectible.TokenID.Int, + }) + } + } + + return erc721, erc1155, nil +} diff --git a/services/wallet/reader.go b/services/wallet/reader.go index 857b25ed3f6..19b225337be 100644 --- a/services/wallet/reader.go +++ b/services/wallet/reader.go @@ -7,6 +7,7 @@ import ( "encoding/json" "math" "math/big" + "slices" "sync" "time" @@ -16,14 +17,21 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/event" + + "github.com/status-im/go-wallet-sdk/pkg/balance/multistandardfetcher" + "github.com/status-im/go-wallet-sdk/pkg/contracts/erc20" + "github.com/status-im/go-wallet-sdk/pkg/eventlog" + gocommon "github.com/status-im/status-go/common" "github.com/status-im/status-go/healthmanager/rpcstatus" "github.com/status-im/status-go/logutils" + "github.com/status-im/status-go/pkg/pubsub" "github.com/status-im/status-go/rpc/chain" "github.com/status-im/status-go/services/wallet/market" + "github.com/status-im/status-go/services/wallet/multistandardbalance" "github.com/status-im/status-go/services/wallet/token" tokenTypes "github.com/status-im/status-go/services/wallet/token/types" - "github.com/status-im/status-go/services/wallet/transfer" + "github.com/status-im/status-go/services/wallet/transferdetector" "github.com/status-im/status-go/services/wallet/walletevent" "github.com/status-im/status-go/transactions" ) @@ -58,27 +66,45 @@ type ReaderInterface interface { GetLastTokenUpdateTimestamps() map[common.Address]int64 } -func NewReader(tokenManager token.ManagerInterface, marketManager *market.Manager, persistence token.TokenBalancesStorage, walletFeed *event.Feed) *Reader { +type BlockChainStateProvider interface { + GetEstimatedBlockTime(ctx context.Context, chainID uint64, blockNumber uint64) (time.Time, error) +} + +func NewReader( + tokenManager token.ManagerInterface, + marketManager *market.Manager, + persistence token.TokenBalancesStorage, + walletFeed *event.Feed, + multistandardBalancePublisher *pubsub.Publisher, + transferDetectorPublisher *pubsub.Publisher, + blockChainStateProvider BlockChainStateProvider) *Reader { return &Reader{ - tokenManager: tokenManager, - marketManager: marketManager, - persistence: persistence, - walletFeed: walletFeed, - refreshBalanceCache: true, + tokenManager: tokenManager, + marketManager: marketManager, + multistandardBalancePublisher: multistandardBalancePublisher, + transferDetectorPublisher: transferDetectorPublisher, + blockChainStateProvider: blockChainStateProvider, + persistence: persistence, + walletFeed: walletFeed, + refreshBalanceCache: true, } } type Reader struct { tokenManager token.ManagerInterface marketManager *market.Manager + multistandardBalancePublisher *pubsub.Publisher + transferDetectorPublisher *pubsub.Publisher + blockChainStateProvider BlockChainStateProvider persistence token.TokenBalancesStorage walletFeed *event.Feed - cancel context.CancelFunc walletEventsWatcher *walletevent.Watcher lastWalletTokenUpdateTimestamp sync.Map reloadDelayTimer *time.Timer refreshBalanceCache bool rw sync.RWMutex + + stopCh chan struct{} } func splitVerifiedTokens(tokens []*tokenTypes.Token) ([]*tokenTypes.Token, []*tokenTypes.Token) { @@ -123,32 +149,34 @@ func getTokenAddresses(tokens []*tokenTypes.Token) []common.Address { } func (r *Reader) Start() error { - ctx, cancel := context.WithCancel(context.Background()) - r.cancel = cancel + if r.stopCh != nil { + return nil + } + + r.stopCh = make(chan struct{}) r.startWalletEventsWatcher() - go func() { - defer gocommon.LogOnPanic() - ticker := time.NewTicker(walletTickReloadPeriod) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - r.triggerWalletReload() - } - } - }() + // Start balance change watcher + r.startBalanceChangeWatcher() + + // Start transfer detection watcher + r.startTransferDetectionWatcher() + + // Start periodic load + r.startPeriodicLoad() + return nil } func (r *Reader) Stop() { - if r.cancel != nil { - r.cancel() + if r.stopCh == nil { + return } + close(r.stopCh) + r.stopCh = nil + r.stopWalletEventsWatcher() r.cancelDelayedWalletReload() @@ -192,8 +220,6 @@ func (r *Reader) startWalletEventsWatcher() { walletEventCb := func(event walletevent.Event) { delayed := true switch event.Type { - case transfer.EventInternalETHTransferDetected, transfer.EventInternalERC20TransferDetected: - // Delayed refresh case transactions.EventPendingTransactionUpdate: var p transactions.PendingTxUpdatePayload err := json.Unmarshal([]byte(event.Message), &p) @@ -211,21 +237,7 @@ func (r *Reader) startWalletEventsWatcher() { } for _, address := range event.Accounts { - timestamp, ok := r.lastWalletTokenUpdateTimestamp.Load(address) - timecheck := int64(0) - if ok { - timecheck = timestamp.(int64) - activityReloadMarginSeconds - } - - if !ok || event.At > timecheck { - r.invalidateBalanceCache() - if delayed { - r.triggerDelayedWalletReload() - } else { - r.triggerWalletReload() - } - break - } + r.triggerReloadIfRecent(event.At, address, delayed) } } @@ -241,6 +253,132 @@ func (r *Reader) stopWalletEventsWatcher() { } } +func (r *Reader) startBalanceChangeWatcher() { + if r.multistandardBalancePublisher == nil { + return + } + + ch, unsub := pubsub.Subscribe[multistandardbalance.EventBalanceFetchFinished](r.multistandardBalancePublisher, 10) + go func() { + defer gocommon.LogOnPanic() + defer unsub() + for { + select { + case <-r.stopCh: + return + case event, ok := <-ch: + if !ok { + return + } + switch event.ResultType { + case multistandardfetcher.ResultTypeNative, multistandardfetcher.ResultTypeERC20: + if !event.BalanceChanged { + continue + } + + r.triggerReloadIfRecent(event.NewState.FetchedAt, event.Key.Account, true) + } + } + } + }() +} + +func (r *Reader) startTransferDetectionWatcher() { + if r.transferDetectorPublisher == nil { + return + } + + ch, unsub := pubsub.Subscribe[transferdetector.EventTransferDetectionFinished](r.transferDetectorPublisher, 10) + go func() { + defer gocommon.LogOnPanic() + defer unsub() + for { + select { + case <-r.stopCh: + return + case msg, ok := <-ch: + if !ok { + return + } + for _, event := range msg.Events { + switch event.EventKey { + case eventlog.ERC20Transfer: + unpackedEvent, ok := event.Unpacked.(erc20.Erc20Transfer) + if !ok { + logutils.ZapLogger().Error("failed to unpack ERC20Transfer event") + continue + } + r.processERC20TransferEvent(msg.ChainID, unpackedEvent) + r.triggerReloadIfRelevantEvent(msg.Accounts, unpackedEvent.From, unpackedEvent.To, msg.ChainID, unpackedEvent.Raw.BlockNumber) + } + } + } + } + }() +} + +func (r *Reader) startPeriodicLoad() { + go func() { + defer gocommon.LogOnPanic() + ticker := time.NewTicker(walletTickReloadPeriod) + defer ticker.Stop() + for { + select { + case <-r.stopCh: + return + case <-ticker.C: + r.triggerWalletReload() + } + } + }() +} + +func (r *Reader) processERC20TransferEvent(chainID uint64, event erc20.Erc20Transfer) { + // Find token in db or if this is a community token, find its metadata + token := r.tokenManager.FindOrCreateTokenByAddress(context.TODO(), chainID, event.Raw.Address) + if token != nil { + isFirst := false + if token.Verified || token.CommunityData != nil { + isFirst, _ = r.tokenManager.MarkAsPreviouslyOwnedToken(token, event.To) + } + if token.CommunityData != nil { + go func() { + defer gocommon.LogOnPanic() + r.tokenManager.SignalCommunityTokenReceived(event.To, event.Raw.TxHash, event.Value, token, isFirst) + }() + } + } +} + +func (r *Reader) triggerReloadIfRelevantEvent(checkedAccounts []common.Address, eventFrom common.Address, eventTo common.Address, chainID uint64, blockNumber uint64) { + for _, address := range []common.Address{eventFrom, eventTo} { + if slices.Contains(checkedAccounts, address) { + blockTime, err := r.blockChainStateProvider.GetEstimatedBlockTime(context.TODO(), chainID, blockNumber) + if err != nil { + logutils.ZapLogger().Error("failed to get estimated block time", zap.Error(err)) + continue + } + r.triggerReloadIfRecent(blockTime.Unix(), address, true) + } + } +} + +func (r *Reader) triggerReloadIfRecent(timestamp int64, account common.Address, delayed bool) { + lastTimestamp, ok := r.lastWalletTokenUpdateTimestamp.Load(account) + timecheck := int64(0) + if ok { + timecheck = lastTimestamp.(int64) - activityReloadMarginSeconds + } + if !ok || timestamp > timecheck { + r.invalidateBalanceCache() + if delayed { + r.triggerDelayedWalletReload() + } else { + r.triggerWalletReload() + } + } +} + func (r *Reader) tokensCachedForAddresses(addresses []common.Address) bool { cachedTokens, err := r.getCachedWalletTokensWithoutMarketData() if err != nil { diff --git a/services/wallet/reader_test.go b/services/wallet/reader_test.go index 48e0b70e7e2..7c4ee3c7f97 100644 --- a/services/wallet/reader_test.go +++ b/services/wallet/reader_test.go @@ -17,8 +17,10 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/event" "github.com/status-im/status-go/healthmanager/rpcstatus" + "github.com/status-im/status-go/pkg/pubsub" "github.com/status-im/status-go/rpc/chain" mock_client "github.com/status-im/status-go/rpc/chain/mock/client" + mock_reader "github.com/status-im/status-go/services/wallet/mock/reader" "github.com/status-im/status-go/services/wallet/testutils" mock_balance_persistence "github.com/status-im/status-go/services/wallet/token/mock/balance_persistence" mock_token "github.com/status-im/status-go/services/wallet/token/mock/token" @@ -167,8 +169,11 @@ func setupReader(t *testing.T) (*Reader, *mock_token.MockManagerInterface, *mock mockTokenManager := mock_token.NewMockManagerInterface(mockCtrl) tokenBalanceStorage := mock_balance_persistence.NewMockTokenBalancesStorage(mockCtrl) eventsFeed := &event.Feed{} + multistandardBalancePublisher := pubsub.NewPublisher() + transferDetectorPublisher := pubsub.NewPublisher() + blockChainStateProvider := mock_reader.NewMockBlockChainStateProvider(mockCtrl) - return NewReader(mockTokenManager, nil, tokenBalanceStorage, eventsFeed), mockTokenManager, tokenBalanceStorage, mockCtrl + return NewReader(mockTokenManager, nil, tokenBalanceStorage, eventsFeed, multistandardBalancePublisher, transferDetectorPublisher, blockChainStateProvider), mockTokenManager, tokenBalanceStorage, mockCtrl } func TestGetCachedWalletTokensWithoutMarketData(t *testing.T) { @@ -991,7 +996,7 @@ func TestReaderRestart(t *testing.T) { err = reader.Restart() require.NoError(t, err) require.NotNil(t, reader.walletEventsWatcher) - require.NotEqual(t, previousWalletEventsWatcher, reader.walletEventsWatcher) + require.NotSame(t, previousWalletEventsWatcher, reader.walletEventsWatcher) } func TestFetchOrGetCachedWalletBalances(t *testing.T) { diff --git a/services/wallet/routeexecution/manager.go b/services/wallet/routeexecution/manager.go index 81798275326..49f82660587 100644 --- a/services/wallet/routeexecution/manager.go +++ b/services/wallet/routeexecution/manager.go @@ -32,16 +32,14 @@ const ( type Manager struct { router *router.Router transactionManager *transfer.TransactionManager - transferController *transfer.Controller db *storage.DB eventFeed *event.Feed } -func NewManager(walletDB *sql.DB, eventFeed *event.Feed, router *router.Router, transactionManager *transfer.TransactionManager, transferController *transfer.Controller) *Manager { +func NewManager(walletDB *sql.DB, eventFeed *event.Feed, router *router.Router, transactionManager *transfer.TransactionManager) *Manager { return &Manager{ router: router, transactionManager: transactionManager, - transferController: transferController, db: storage.NewDB(walletDB), eventFeed: eventFeed, } diff --git a/services/wallet/service.go b/services/wallet/service.go index a8ea648c740..212105a003c 100644 --- a/services/wallet/service.go +++ b/services/wallet/service.go @@ -10,7 +10,9 @@ import ( "github.com/golang/protobuf/proto" "github.com/status-im/status-go/services/wallet/common" + "github.com/status-im/status-go/services/wallet/multistandardbalance" "github.com/status-im/status-go/services/wallet/thirdparty/market/cryptocompare" + "github.com/status-im/status-go/services/wallet/transferdetector" "github.com/ethereum/go-ethereum/event" gethrpc "github.com/ethereum/go-ethereum/rpc" @@ -33,6 +35,8 @@ import ( alchemymanager "github.com/status-im/status-go/services/wallet/activityfetcher/alchemy" "github.com/status-im/status-go/services/wallet/blockchainstate" "github.com/status-im/status-go/services/wallet/collectibles" + "github.com/status-im/status-go/services/wallet/collectibles/ownership" + collectibles_ownership "github.com/status-im/status-go/services/wallet/collectibles/ownership" "github.com/status-im/status-go/services/wallet/community" "github.com/status-im/status-go/services/wallet/currency" "github.com/status-im/status-go/services/wallet/leaderboard" @@ -104,8 +108,7 @@ func NewService( savedAddressesManager := &SavedAddressesManager{db: db} transactionManager := transfer.NewTransactionManager(gethManager, transactor, config, accountsDB, pendingTxManager, feed) - blockChainState := blockchainstate.NewBlockChainState() - transferController := transfer.NewTransferController(db, accountsDB, rpcClient, accountsPublisher, transactionManager, blockChainState) + blockChainState := blockchainstate.NewBlockChainState(rpcClient) thirdpartyServicesEnabled := ThirdpartyServicesEnabled(accountsDB) @@ -187,9 +190,44 @@ func NewService( cryptoOnRampManager := onramp.NewManager(cryptoOnRampProviders) marketManager := market.NewManager(marketProviders, tokenManager, feed) - reader := NewReader(tokenManager, marketManager, token.NewPersistence(db), feed) currency := currency.NewService(db, feed, tokenManager, marketManager) + multistandardBalanceFetcher := multistandardbalance.NewFetcher(rpcClient, multistandardbalance.DefaultBatchSize) + multistandardBalanceStorage := multistandardbalance.NewStorageMemory() + multistandardBalanceController := multistandardbalance.NewController( + multistandardbalance.DefaultControllerConfig(), + multistandardBalanceStorage, + multistandardBalanceFetcher, + accountsDB, + accountsPublisher, + rpcClient.GetNetworkManager(), + NewMultistandardBalanceTokenListProvider(tokenManager), + NewMultistandardBalanceCollectiblesListProvider(ownership.NewOwnershipDB(db), collectibles.NewContractTypeDB(db)), + blockChainState, + logutils.ZapLogger().Named("MultistandardBalanceController"), + ) + + transferDetectorController := transferdetector.NewController( + transferdetector.DefaultControllerConfig(), + transferdetector.NewFetcher(rpcClient), + accountsDB, + accountsPublisher, + rpcClient.GetNetworkManager(), + blockChainState, + logutils.ZapLogger().Named("TransferDetectorController"), + ) + + reader := NewReader( + tokenManager, + marketManager, + token.NewPersistence(db), + feed, + multistandardBalanceController.GetPublisher(), + transferDetectorController.GetPublisher(), + blockChainState, + ) + + collectiblesPublisher := pubsub.NewPublisher() collectiblesManager := collectibles.NewManager( db, rpcClient, @@ -198,7 +236,22 @@ func NewService( mediaServer, feed, ) - collectibles := collectibles.NewService(db, feed, accountsDB, accountsPublisher, communityManager, rpcClient.GetNetworkManager(), collectiblesManager) + collectiblesOwnershipController := collectibles_ownership.NewController( + ownership.NewOwnershipDB(db), accountsDB, accountsPublisher, rpcClient.GetNetworkManager(), + multistandardBalanceController.GetPublisher(), + transferDetectorController.GetPublisher(), + blockChainState, + collectiblesManager, + collectiblesPublisher, + logutils.ZapLogger().Named("CollectiblesOwnershipController"), + ) + collectibles := collectibles.NewService( + db, + feed, + communityManager, + collectiblesManager, + collectiblesOwnershipController, + collectiblesPublisher) activity := activity.NewService(db, accountsDB, tokenManager, collectiblesManager, feed) @@ -208,7 +261,7 @@ func NewService( router.AddPathProcessor(processor) } - routeExecutionManager := routeexecution.NewManager(db, feed, router, transactionManager, transferController) + routeExecutionManager := routeexecution.NewManager(db, feed, router, transactionManager) leaderboardService := leaderboard.NewMarketDataService(leaderboardConfig, db, feed) @@ -220,36 +273,37 @@ func NewService( activityFetcherService := activityfetcher.NewService(activityFetcherManager, rpcClient.GetNetworkManager(), accountsDB, accountsPublisher, rpcClient, feed) return &Service{ - db: db, - accountsDB: accountsDB, - rpcClient: rpcClient, - tokenManager: tokenManager, - communityManager: communityManager, - savedAddressesManager: savedAddressesManager, - transactionManager: transactionManager, - pendingTxManager: pendingTxManager, - transferController: transferController, - cryptoOnRampManager: cryptoOnRampManager, - collectiblesManager: collectiblesManager, - collectibles: collectibles, - gethManager: gethManager, - marketManager: marketManager, - transactor: transactor, - feed: feed, - signals: signals, - reader: reader, - currency: currency, - activity: activity, - decoder: NewDecoder(), - blockChainState: blockChainState, - keycardPairings: NewKeycardPairings(), - config: config, - featureFlags: featureFlags, - router: router, - routeExecutionManager: routeExecutionManager, - leaderboardService: leaderboardService, - activityFetcherService: activityFetcherService, - started: false, + db: db, + accountsDB: accountsDB, + rpcClient: rpcClient, + tokenManager: tokenManager, + communityManager: communityManager, + savedAddressesManager: savedAddressesManager, + transactionManager: transactionManager, + pendingTxManager: pendingTxManager, + multistandardBalanceController: multistandardBalanceController, + transferDetectorController: transferDetectorController, + cryptoOnRampManager: cryptoOnRampManager, + collectiblesManager: collectiblesManager, + collectibles: collectibles, + gethManager: gethManager, + marketManager: marketManager, + transactor: transactor, + feed: feed, + signals: signals, + reader: reader, + currency: currency, + activity: activity, + decoder: NewDecoder(), + blockChainState: blockChainState, + keycardPairings: NewKeycardPairings(), + config: config, + featureFlags: featureFlags, + router: router, + routeExecutionManager: routeExecutionManager, + leaderboardService: leaderboardService, + activityFetcherService: activityFetcherService, + started: false, } } @@ -315,36 +369,37 @@ func buildPathProcessors( // Service is a wallet service. type Service struct { - db *sql.DB - accountsDB *accounts.Database - rpcClient *rpc.Client - tokenManager *token.Manager - communityManager *community.Manager - savedAddressesManager *SavedAddressesManager - transactionManager *transfer.TransactionManager - pendingTxManager *transactions.PendingTxTracker - transferController *transfer.Controller - cryptoOnRampManager *onramp.Manager - collectiblesManager *collectibles.Manager - collectibles *collectibles.Service - gethManager *accsmanagement.AccountsManager - marketManager *market.Manager - transactor *transactions.Transactor - feed *event.Feed - signals *walletevent.SignalsTransmitter - reader *Reader - currency *currency.Service - activity *activity.Service - decoder *Decoder - blockChainState *blockchainstate.BlockChainState - keycardPairings *KeycardPairings - config *params.NodeConfig - featureFlags *protocolCommon.FeatureFlags - router *router.Router - routeExecutionManager *routeexecution.Manager - leaderboardService *leaderboard.MarketDataService - activityFetcherService *activityfetcher.Service - started bool + db *sql.DB + accountsDB *accounts.Database + rpcClient *rpc.Client + tokenManager *token.Manager + communityManager *community.Manager + savedAddressesManager *SavedAddressesManager + transactionManager *transfer.TransactionManager + pendingTxManager *transactions.PendingTxTracker + multistandardBalanceController *multistandardbalance.Controller + transferDetectorController *transferdetector.Controller + cryptoOnRampManager *onramp.Manager + collectiblesManager *collectibles.Manager + collectibles *collectibles.Service + gethManager *accsmanagement.AccountsManager + marketManager *market.Manager + transactor *transactions.Transactor + feed *event.Feed + signals *walletevent.SignalsTransmitter + reader *Reader + currency *currency.Service + activity *activity.Service + decoder *Decoder + blockChainState *blockchainstate.BlockChainState + keycardPairings *KeycardPairings + config *params.NodeConfig + featureFlags *protocolCommon.FeatureFlags + router *router.Router + routeExecutionManager *routeexecution.Manager + leaderboardService *leaderboard.MarketDataService + activityFetcherService *activityfetcher.Service + started bool cancelWalletServiceCtx context.CancelFunc } @@ -355,7 +410,8 @@ func (s *Service) Start() error { ctx, cancel := context.WithCancel(context.Background()) s.cancelWalletServiceCtx = cancel - s.transferController.Start(ctx) + s.multistandardBalanceController.Start() + s.transferDetectorController.Start() s.currency.Start(ctx) err := s.signals.Start(ctx) s.collectibles.Start(ctx) @@ -377,7 +433,8 @@ func (s *Service) Stop() error { logutils.ZapLogger().Info("wallet will be stopped") s.router.Stop() s.signals.Stop() - s.transferController.Stop() + s.multistandardBalanceController.Stop() + s.transferDetectorController.Stop() s.reader.Stop() s.activity.Stop() s.collectibles.Stop() diff --git a/services/wallet/token/token.go b/services/wallet/token/token.go index bc00037539c..0ed785b0fba 100644 --- a/services/wallet/token/token.go +++ b/services/wallet/token/token.go @@ -74,6 +74,9 @@ type ManagerInterface interface { LookupTokenIdentity(chainID uint64, address common.Address, native bool) *tokenTypes.Token LookupToken(chainID *uint64, tokenSymbol string) (token *tokenTypes.Token, isNative bool) GetTokensByChainIDs(chainIDs []uint64) ([]*tokenTypes.Token, error) + FindOrCreateTokenByAddress(ctx context.Context, chainID uint64, address common.Address) *tokenTypes.Token + MarkAsPreviouslyOwnedToken(token *tokenTypes.Token, owner common.Address) (bool, error) + SignalCommunityTokenReceived(address common.Address, txHash common.Hash, value *big.Int, t *tokenTypes.Token, isFirst bool) } // Manager is used for accessing token store. It changes the token store based on overridden tokens diff --git a/services/wallet/transfer/block_dao.go b/services/wallet/transfer/block_dao.go deleted file mode 100644 index 0772d645230..00000000000 --- a/services/wallet/transfer/block_dao.go +++ /dev/null @@ -1,178 +0,0 @@ -package transfer - -import ( - "database/sql" - "math/big" - - "go.uber.org/zap" - - "github.com/ethereum/go-ethereum/common" - "github.com/status-im/status-go/logutils" - "github.com/status-im/status-go/services/wallet/bigint" -) - -type BlocksRange struct { - from *big.Int - to *big.Int -} - -type Block struct { - Number *big.Int - Balance *big.Int - Nonce *int64 -} - -type BlockDAO struct { - db *sql.DB -} - -func (b *BlockDAO) insertRange(chainID uint64, account common.Address, from, to, balance *big.Int, nonce uint64) error { - logutils.ZapLogger().Debug( - "insert blocks range", - zap.Stringer("account", account), - zap.Uint64("network id", chainID), - zap.Stringer("from", from), - zap.Stringer("to", to), - zap.Stringer("balance", balance), - zap.Uint64("nonce", nonce), - ) - insert, err := b.db.Prepare("INSERT INTO blocks_ranges (network_id, address, blk_from, blk_to, balance, nonce) VALUES (?, ?, ?, ?, ?, ?)") - if err != nil { - return err - } - defer insert.Close() - _, err = insert.Exec(chainID, account, (*bigint.SQLBigInt)(from), (*bigint.SQLBigInt)(to), (*bigint.SQLBigIntBytes)(balance), &nonce) - return err -} - -// GetBlocksToLoadByAddress gets unloaded blocks for a given address. -func (b *BlockDAO) GetBlocksToLoadByAddress(chainID uint64, address common.Address, limit int) (rst []*big.Int, err error) { - query := `SELECT blk_number FROM blocks - WHERE address = ? AND network_id = ? AND loaded = 0 - ORDER BY blk_number DESC - LIMIT ?` - rows, err := b.db.Query(query, address, chainID, limit) - if err != nil { - return - } - defer rows.Close() - for rows.Next() { - block := &big.Int{} - err = rows.Scan((*bigint.SQLBigInt)(block)) - if err != nil { - return nil, err - } - rst = append(rst, block) - } - return rst, nil -} - -func (b *BlockDAO) GetLastBlockByAddress(chainID uint64, address common.Address, limit int) (rst *big.Int, err error) { - query := `SELECT * FROM - (SELECT blk_number FROM blocks WHERE address = ? AND network_id = ? ORDER BY blk_number DESC LIMIT ?) - ORDER BY blk_number LIMIT 1` - rows, err := b.db.Query(query, address, chainID, limit) - if err != nil { - return - } - defer rows.Close() - - if rows.Next() { - block := &big.Int{} - err = rows.Scan((*bigint.SQLBigInt)(block)) - if err != nil { - return nil, err - } - - return block, nil - } - - return nil, nil -} - -func (b *BlockDAO) GetLastKnownBlockByAddress(chainID uint64, address common.Address) (block *Block, err error) { - query := `SELECT blk_to, balance, nonce FROM blocks_ranges - WHERE address = ? - AND network_id = ? - ORDER BY blk_to DESC - LIMIT 1` - - rows, err := b.db.Query(query, address, chainID) - if err != nil { - return - } - defer rows.Close() - - if rows.Next() { - var nonce sql.NullInt64 - block = &Block{Number: &big.Int{}, Balance: &big.Int{}} - err = rows.Scan((*bigint.SQLBigInt)(block.Number), (*bigint.SQLBigIntBytes)(block.Balance), &nonce) - if err != nil { - return nil, err - } - - if nonce.Valid { - block.Nonce = &nonce.Int64 - } - return block, nil - } - - return nil, nil -} - -func getNewRanges(ranges []*BlocksRange) ([]*BlocksRange, []*BlocksRange) { - initValue := big.NewInt(-1) - prevFrom := big.NewInt(-1) - prevTo := big.NewInt(-1) - hasMergedRanges := false - var newRanges []*BlocksRange - var deletedRanges []*BlocksRange - for idx, blocksRange := range ranges { - if prevTo.Cmp(initValue) == 0 { - prevTo = blocksRange.to - prevFrom = blocksRange.from - } else if prevTo.Cmp(blocksRange.from) >= 0 { - hasMergedRanges = true - deletedRanges = append(deletedRanges, ranges[idx-1]) - if prevTo.Cmp(blocksRange.to) <= 0 { - prevTo = blocksRange.to - } - } else { - if hasMergedRanges { - deletedRanges = append(deletedRanges, ranges[idx-1]) - newRanges = append(newRanges, &BlocksRange{ - from: prevFrom, - to: prevTo, - }) - } - logutils.ZapLogger().Info("blocks ranges gap detected", - zap.Stringer("from", prevTo), - zap.Stringer("to", blocksRange.from), - ) - hasMergedRanges = false - - prevFrom = blocksRange.from - prevTo = blocksRange.to - } - } - - if hasMergedRanges { - deletedRanges = append(deletedRanges, ranges[len(ranges)-1]) - newRanges = append(newRanges, &BlocksRange{ - from: prevFrom, - to: prevTo, - }) - } - - return newRanges, deletedRanges -} - -func deleteAllRanges(creator statementCreator, account common.Address) error { - delete, err := creator.Prepare(`DELETE FROM blocks_ranges WHERE address = ?`) - if err != nil { - return err - } - defer delete.Close() - _, err = delete.Exec(account) - return err -} diff --git a/services/wallet/transfer/block_ranges_sequential_dao.go b/services/wallet/transfer/block_ranges_sequential_dao.go deleted file mode 100644 index 90e89d54999..00000000000 --- a/services/wallet/transfer/block_ranges_sequential_dao.go +++ /dev/null @@ -1,291 +0,0 @@ -package transfer - -import ( - "database/sql" - "math/big" - - "go.uber.org/zap" - - "github.com/ethereum/go-ethereum/common" - "github.com/status-im/status-go/logutils" - "github.com/status-im/status-go/services/wallet/bigint" -) - -type BlockRangeDAOer interface { - getBlockRange(chainID uint64, address common.Address) (blockRange *ethTokensBlockRanges, exists bool, err error) - getBlockRanges(chainID uint64, addresses []common.Address) (blockRanges map[common.Address]*ethTokensBlockRanges, err error) - upsertRange(chainID uint64, account common.Address, newBlockRange *ethTokensBlockRanges) (err error) - updateTokenRange(chainID uint64, account common.Address, newBlockRange *BlockRange) (err error) - upsertEthRange(chainID uint64, account common.Address, newBlockRange *BlockRange) (err error) -} - -type BlockRangeSequentialDAO struct { - db *sql.DB -} - -type BlockRange struct { - Start *big.Int // Block of first transfer - FirstKnown *big.Int // Oldest scanned block - LastKnown *big.Int // Last scanned block -} - -func NewBlockRange() *BlockRange { - return &BlockRange{Start: nil, FirstKnown: nil, LastKnown: nil} -} - -type ethTokensBlockRanges struct { - eth *BlockRange - tokens *BlockRange - balanceCheckHash string -} - -func newEthTokensBlockRanges() *ethTokensBlockRanges { - return ðTokensBlockRanges{eth: NewBlockRange(), tokens: NewBlockRange()} -} - -func scanRanges(rows *sql.Rows) (map[common.Address]*ethTokensBlockRanges, error) { - blockRanges := make(map[common.Address]*ethTokensBlockRanges) - for rows.Next() { - efk := &bigint.NilableSQLBigInt{} - elk := &bigint.NilableSQLBigInt{} - es := &bigint.NilableSQLBigInt{} - tfk := &bigint.NilableSQLBigInt{} - tlk := &bigint.NilableSQLBigInt{} - ts := &bigint.NilableSQLBigInt{} - addressB := []byte{} - blockRange := newEthTokensBlockRanges() - err := rows.Scan(&addressB, es, efk, elk, ts, tfk, tlk, &blockRange.balanceCheckHash) - if err != nil { - return nil, err - } - address := common.BytesToAddress(addressB) - blockRanges[address] = blockRange - - if !es.IsNil() { - blockRanges[address].eth.Start = big.NewInt(es.Int64()) - } - if !efk.IsNil() { - blockRanges[address].eth.FirstKnown = big.NewInt(efk.Int64()) - } - if !elk.IsNil() { - blockRanges[address].eth.LastKnown = big.NewInt(elk.Int64()) - } - if !ts.IsNil() { - blockRanges[address].tokens.Start = big.NewInt(ts.Int64()) - } - if !tfk.IsNil() { - blockRanges[address].tokens.FirstKnown = big.NewInt(tfk.Int64()) - } - if !tlk.IsNil() { - blockRanges[address].tokens.LastKnown = big.NewInt(tlk.Int64()) - } - } - return blockRanges, nil -} - -func (b *BlockRangeSequentialDAO) getBlockRange(chainID uint64, address common.Address) (blockRange *ethTokensBlockRanges, exists bool, err error) { - query := `SELECT address, blk_start, blk_first, blk_last, token_blk_start, token_blk_first, token_blk_last, balance_check_hash FROM blocks_ranges_sequential - WHERE address = ? - AND network_id = ?` - - rows, err := b.db.Query(query, address, chainID) - if err != nil { - return - } - defer rows.Close() - - ranges, err := scanRanges(rows) - if err != nil { - return nil, false, err - } - - blockRange, exists = ranges[address] - if !exists { - blockRange = newEthTokensBlockRanges() - } - - return blockRange, exists, nil -} - -func (b *BlockRangeSequentialDAO) getBlockRanges(chainID uint64, addresses []common.Address) (blockRanges map[common.Address]*ethTokensBlockRanges, err error) { - blockRanges = make(map[common.Address]*ethTokensBlockRanges) - addressesPlaceholder := "" - for i := 0; i < len(addresses); i++ { - addressesPlaceholder += "?" - if i < len(addresses)-1 { - addressesPlaceholder += "," - } - } - - query := "SELECT address, blk_start, blk_first, blk_last, token_blk_start, token_blk_first, token_blk_last, balance_check_hash FROM blocks_ranges_sequential WHERE address IN (" + //nolint: gosec - addressesPlaceholder + ") AND network_id = ?" - - params := []interface{}{} - for _, address := range addresses { - params = append(params, address) - } - params = append(params, chainID) - - rows, err := b.db.Query(query, params...) - if err != nil { - return - } - defer rows.Close() - - return scanRanges(rows) -} - -func (b *BlockRangeSequentialDAO) deleteRange(account common.Address) error { - logutils.ZapLogger().Debug("delete blocks range", zap.Stringer("account", account)) - delete, err := b.db.Prepare(`DELETE FROM blocks_ranges_sequential WHERE address = ?`) - if err != nil { - logutils.ZapLogger().Error("Failed to prepare deletion of sequential block range", zap.Error(err)) - return err - } - defer delete.Close() - - _, err = delete.Exec(account) - return err -} - -func (b *BlockRangeSequentialDAO) upsertRange(chainID uint64, account common.Address, newBlockRange *ethTokensBlockRanges) (err error) { - ethTokensBlockRange, exists, err := b.getBlockRange(chainID, account) - if err != nil { - return err - } - - ethBlockRange := prepareUpdatedBlockRange(ethTokensBlockRange.eth, newBlockRange.eth) - tokensBlockRange := prepareUpdatedBlockRange(ethTokensBlockRange.tokens, newBlockRange.tokens) - - logutils.ZapLogger().Debug("upsert eth and tokens blocks range", - zap.Stringer("account", account), - zap.Uint64("chainID", chainID), - zap.Stringer("eth.start", ethBlockRange.Start), - zap.Stringer("eth.first", ethBlockRange.FirstKnown), - zap.Stringer("eth.last", ethBlockRange.LastKnown), - zap.Stringer("tokens.first", tokensBlockRange.FirstKnown), - zap.Stringer("tokens.last", tokensBlockRange.LastKnown), - zap.String("hash", newBlockRange.balanceCheckHash), - ) - - var query *sql.Stmt - - if exists { - query, err = b.db.Prepare(`UPDATE blocks_ranges_sequential SET - blk_start = ?, - blk_first = ?, - blk_last = ?, - token_blk_start = ?, - token_blk_first = ?, - token_blk_last = ?, - balance_check_hash = ? - WHERE network_id = ? AND address = ?`) - - } else { - query, err = b.db.Prepare(`INSERT INTO blocks_ranges_sequential - (blk_start, blk_first, blk_last, token_blk_start, token_blk_first, token_blk_last, balance_check_hash, network_id, address) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`) - } - - if err != nil { - return err - } - defer query.Close() - _, err = query.Exec((*bigint.SQLBigInt)(ethBlockRange.Start), (*bigint.SQLBigInt)(ethBlockRange.FirstKnown), (*bigint.SQLBigInt)(ethBlockRange.LastKnown), - (*bigint.SQLBigInt)(tokensBlockRange.Start), (*bigint.SQLBigInt)(tokensBlockRange.FirstKnown), (*bigint.SQLBigInt)(tokensBlockRange.LastKnown), newBlockRange.balanceCheckHash, chainID, account) - - return err -} - -func (b *BlockRangeSequentialDAO) upsertEthRange(chainID uint64, account common.Address, - newBlockRange *BlockRange) (err error) { - - ethTokensBlockRange, exists, err := b.getBlockRange(chainID, account) - if err != nil { - return err - } - - blockRange := prepareUpdatedBlockRange(ethTokensBlockRange.eth, newBlockRange) - - logutils.ZapLogger().Debug("upsert eth blocks range", - zap.Stringer("account", account), - zap.Uint64("chainID", chainID), - zap.Stringer("start", blockRange.Start), - zap.Stringer("first", blockRange.FirstKnown), - zap.Stringer("last", blockRange.LastKnown), - zap.String("old hash", ethTokensBlockRange.balanceCheckHash), - ) - - var query *sql.Stmt - - if exists { - query, err = b.db.Prepare(`UPDATE blocks_ranges_sequential SET - blk_start = ?, - blk_first = ?, - blk_last = ? - WHERE network_id = ? AND address = ?`) - } else { - query, err = b.db.Prepare(`INSERT INTO blocks_ranges_sequential - (blk_start, blk_first, blk_last, network_id, address) VALUES (?, ?, ?, ?, ?)`) - } - - if err != nil { - return err - } - defer query.Close() - _, err = query.Exec((*bigint.SQLBigInt)(blockRange.Start), (*bigint.SQLBigInt)(blockRange.FirstKnown), (*bigint.SQLBigInt)(blockRange.LastKnown), chainID, account) - - return err -} - -func (b *BlockRangeSequentialDAO) updateTokenRange(chainID uint64, account common.Address, - newBlockRange *BlockRange) (err error) { - - ethTokensBlockRange, _, err := b.getBlockRange(chainID, account) - if err != nil { - return err - } - - blockRange := prepareUpdatedBlockRange(ethTokensBlockRange.tokens, newBlockRange) - - logutils.ZapLogger().Debug("update tokens blocks range", - zap.Stringer("first", blockRange.FirstKnown), - zap.Stringer("last", blockRange.LastKnown), - ) - - update, err := b.db.Prepare(`UPDATE blocks_ranges_sequential SET token_blk_start = ?, token_blk_first = ?, token_blk_last = ? WHERE network_id = ? AND address = ?`) - if err != nil { - return err - } - defer update.Close() - - _, err = update.Exec((*bigint.SQLBigInt)(blockRange.Start), (*bigint.SQLBigInt)(blockRange.FirstKnown), - (*bigint.SQLBigInt)(blockRange.LastKnown), chainID, account) - - return err -} - -func prepareUpdatedBlockRange(blockRange, newBlockRange *BlockRange) *BlockRange { - if newBlockRange != nil { - // Ovewrite start block if there was not any or if new one is older, because it can be precised only - // to a greater value, because no history can be before some block that is considered - // as a start of history, but due to concurrent block range checks, a newer greater block - // can be found that matches criteria of a start block (nonce is zero, balances are equal) - if newBlockRange.Start != nil && (blockRange.Start == nil || blockRange.Start.Cmp(newBlockRange.Start) < 0) { - blockRange.Start = newBlockRange.Start - } - - // Overwrite first known block if there was not any or if new one is older - if (blockRange.FirstKnown == nil && newBlockRange.FirstKnown != nil) || - (blockRange.FirstKnown != nil && newBlockRange.FirstKnown != nil && blockRange.FirstKnown.Cmp(newBlockRange.FirstKnown) > 0) { - blockRange.FirstKnown = newBlockRange.FirstKnown - } - - // Overwrite last known block if there was not any or if new one is newer - if (blockRange.LastKnown == nil && newBlockRange.LastKnown != nil) || - (blockRange.LastKnown != nil && newBlockRange.LastKnown != nil && blockRange.LastKnown.Cmp(newBlockRange.LastKnown) < 0) { - blockRange.LastKnown = newBlockRange.LastKnown - } - } - - return blockRange -} diff --git a/services/wallet/transfer/block_ranges_sequential_dao_test.go b/services/wallet/transfer/block_ranges_sequential_dao_test.go deleted file mode 100644 index c18381608ee..00000000000 --- a/services/wallet/transfer/block_ranges_sequential_dao_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package transfer - -import ( - "database/sql" - "math/big" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/ethereum/go-ethereum/common" - "github.com/status-im/status-go/t/helpers" - "github.com/status-im/status-go/walletdatabase" -) - -func setupBlockRangesTestDB(t *testing.T) (*sql.DB, func()) { - db, err := helpers.SetupTestMemorySQLDB(walletdatabase.DbInitializer{}) - require.NoError(t, err) - return db, func() { - require.NoError(t, db.Close()) - } -} - -func TestBlockRangeSequentialDAO_updateTokenRange(t *testing.T) { - walletDb, stop := setupBlockRangesTestDB(t) - defer stop() - - type fields struct { - db *sql.DB - } - type args struct { - chainID uint64 - account common.Address - newBlockRange *BlockRange - } - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - { - "testTokenBlockRange", - fields{db: walletDb}, - args{ - chainID: 1, - account: common.Address{}, - newBlockRange: &BlockRange{ - LastKnown: big.NewInt(1), - }, - }, - false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &BlockRangeSequentialDAO{ - db: tt.fields.db, - } - - err := b.upsertRange(tt.args.chainID, tt.args.account, newEthTokensBlockRanges()) - require.NoError(t, err) - - if err := b.updateTokenRange(tt.args.chainID, tt.args.account, tt.args.newBlockRange); (err != nil) != tt.wantErr { - t.Errorf("BlockRangeSequentialDAO.updateTokenRange() error = %v, wantErr %v", err, tt.wantErr) - } - - ethTokensBlockRanges, _, err := b.getBlockRange(tt.args.chainID, tt.args.account) - require.NoError(t, err) - require.NotNil(t, ethTokensBlockRanges.tokens) - require.Equal(t, tt.args.newBlockRange.LastKnown, ethTokensBlockRanges.tokens.LastKnown) - }) - } -} - -func TestBlockRangeSequentialDAO_updateEthRange(t *testing.T) { - walletDb, stop := setupBlockRangesTestDB(t) - defer stop() - - type fields struct { - db *sql.DB - } - type args struct { - chainID uint64 - account common.Address - newBlockRange *BlockRange - } - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - { - "testEthBlockRange", - fields{db: walletDb}, - args{ - chainID: 1, - account: common.Address{}, - newBlockRange: &BlockRange{ - Start: big.NewInt(2), - FirstKnown: big.NewInt(1), - LastKnown: big.NewInt(3), - }, - }, - false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &BlockRangeSequentialDAO{ - db: tt.fields.db, - } - - // Initial insert - dummyBlockRange := NewBlockRange() - dummyBlockRange.FirstKnown = big.NewInt(2) // To confirm that it is updated it must be greater than newBlockRange.FirstKnown - if err := b.upsertEthRange(tt.args.chainID, tt.args.account, dummyBlockRange); (err != nil) != tt.wantErr { - t.Errorf("BlockRangeSequentialDAO.upsertEthRange() insert error = %v, wantErr %v", err, tt.wantErr) - } - - ethTokensBlockRanges, _, err := b.getBlockRange(tt.args.chainID, tt.args.account) - require.NoError(t, err) - require.NotNil(t, ethTokensBlockRanges.eth) - require.Equal(t, dummyBlockRange.Start, ethTokensBlockRanges.eth.Start) - require.Equal(t, dummyBlockRange.FirstKnown, ethTokensBlockRanges.eth.FirstKnown) - require.Equal(t, dummyBlockRange.LastKnown, ethTokensBlockRanges.eth.LastKnown) - - // Update - if err := b.upsertEthRange(tt.args.chainID, tt.args.account, tt.args.newBlockRange); (err != nil) != tt.wantErr { - t.Errorf("BlockRangeSequentialDAO.upsertEthRange() update error = %v, wantErr %v", err, tt.wantErr) - } - - ethTokensBlockRanges, _, err = b.getBlockRange(tt.args.chainID, tt.args.account) - require.NoError(t, err) - require.NotNil(t, ethTokensBlockRanges.eth) - require.Equal(t, tt.args.newBlockRange.Start, ethTokensBlockRanges.eth.Start) - require.Equal(t, tt.args.newBlockRange.LastKnown, ethTokensBlockRanges.eth.LastKnown) - require.Equal(t, tt.args.newBlockRange.FirstKnown, ethTokensBlockRanges.eth.FirstKnown) - }) - } -} diff --git a/services/wallet/transfer/block_test.go b/services/wallet/transfer/block_test.go deleted file mode 100644 index d3b703ac18e..00000000000 --- a/services/wallet/transfer/block_test.go +++ /dev/null @@ -1,208 +0,0 @@ -package transfer - -import ( - "math/big" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/status-im/status-go/t/helpers" - "github.com/status-im/status-go/walletdatabase" - - "github.com/ethereum/go-ethereum/common" -) - -func setupTestTransferDB(t *testing.T) (*BlockDAO, func()) { - db, err := helpers.SetupTestMemorySQLDB(walletdatabase.DbInitializer{}) - require.NoError(t, err) - return &BlockDAO{db}, func() { - require.NoError(t, db.Close()) - } -} - -func TestInsertRange(t *testing.T) { - b, stop := setupTestTransferDB(t) - defer stop() - - r := &BlocksRange{ - from: big.NewInt(0), - to: big.NewInt(10), - } - nonce := uint64(199) - balance := big.NewInt(7657) - account := common.Address{2} - - err := b.insertRange(777, account, r.from, r.to, balance, nonce) - require.NoError(t, err) - - block, err := b.GetLastKnownBlockByAddress(777, account) - require.NoError(t, err) - - require.Equal(t, 0, block.Number.Cmp(r.to)) - require.Equal(t, 0, block.Balance.Cmp(balance)) - require.Equal(t, nonce, uint64(*block.Nonce)) -} - -func TestGetNewRanges(t *testing.T) { - ranges := []*BlocksRange{ - &BlocksRange{ - from: big.NewInt(0), - to: big.NewInt(10), - }, - &BlocksRange{ - from: big.NewInt(10), - to: big.NewInt(20), - }, - } - - n, d := getNewRanges(ranges) - require.Equal(t, 1, len(n)) - newRange := n[0] - require.Equal(t, int64(0), newRange.from.Int64()) - require.Equal(t, int64(20), newRange.to.Int64()) - require.Equal(t, 2, len(d)) - - ranges = []*BlocksRange{ - &BlocksRange{ - from: big.NewInt(0), - to: big.NewInt(11), - }, - &BlocksRange{ - from: big.NewInt(10), - to: big.NewInt(20), - }, - } - - n, d = getNewRanges(ranges) - require.Equal(t, 1, len(n)) - newRange = n[0] - require.Equal(t, int64(0), newRange.from.Int64()) - require.Equal(t, int64(20), newRange.to.Int64()) - require.Equal(t, 2, len(d)) - - ranges = []*BlocksRange{ - &BlocksRange{ - from: big.NewInt(0), - to: big.NewInt(20), - }, - &BlocksRange{ - from: big.NewInt(5), - to: big.NewInt(15), - }, - } - - n, d = getNewRanges(ranges) - require.Equal(t, 1, len(n)) - newRange = n[0] - require.Equal(t, int64(0), newRange.from.Int64()) - require.Equal(t, int64(20), newRange.to.Int64()) - require.Equal(t, 2, len(d)) - - ranges = []*BlocksRange{ - &BlocksRange{ - from: big.NewInt(5), - to: big.NewInt(15), - }, - &BlocksRange{ - from: big.NewInt(5), - to: big.NewInt(20), - }, - } - - n, d = getNewRanges(ranges) - require.Equal(t, 1, len(n)) - newRange = n[0] - require.Equal(t, int64(5), newRange.from.Int64()) - require.Equal(t, int64(20), newRange.to.Int64()) - require.Equal(t, 2, len(d)) - - ranges = []*BlocksRange{ - &BlocksRange{ - from: big.NewInt(5), - to: big.NewInt(10), - }, - &BlocksRange{ - from: big.NewInt(15), - to: big.NewInt(20), - }, - } - - n, d = getNewRanges(ranges) - require.Equal(t, 0, len(n)) - require.Equal(t, 0, len(d)) - - ranges = []*BlocksRange{ - &BlocksRange{ - from: big.NewInt(0), - to: big.NewInt(10), - }, - &BlocksRange{ - from: big.NewInt(10), - to: big.NewInt(20), - }, - &BlocksRange{ - from: big.NewInt(30), - to: big.NewInt(40), - }, - } - - n, d = getNewRanges(ranges) - require.Equal(t, 1, len(n)) - newRange = n[0] - require.Equal(t, int64(0), newRange.from.Int64()) - require.Equal(t, int64(20), newRange.to.Int64()) - require.Equal(t, 2, len(d)) - - ranges = []*BlocksRange{ - &BlocksRange{ - from: big.NewInt(0), - to: big.NewInt(10), - }, - &BlocksRange{ - from: big.NewInt(10), - to: big.NewInt(20), - }, - &BlocksRange{ - from: big.NewInt(30), - to: big.NewInt(40), - }, - &BlocksRange{ - from: big.NewInt(40), - to: big.NewInt(50), - }, - } - - n, d = getNewRanges(ranges) - require.Equal(t, 2, len(n)) - newRange = n[0] - require.Equal(t, int64(0), newRange.from.Int64()) - require.Equal(t, int64(20), newRange.to.Int64()) - newRange = n[1] - require.Equal(t, int64(30), newRange.from.Int64()) - require.Equal(t, int64(50), newRange.to.Int64()) - require.Equal(t, 4, len(d)) -} - -func TestInsertZeroBalance(t *testing.T) { - db, _, err := helpers.SetupTestSQLDB(walletdatabase.DbInitializer{}, "zero-balance") - require.NoError(t, err) - - b := &BlockDAO{db} - r := &BlocksRange{ - from: big.NewInt(0), - to: big.NewInt(10), - } - nonce := uint64(199) - balance := big.NewInt(0) - account := common.Address{2} - - err = b.insertRange(777, account, r.from, r.to, balance, nonce) - require.NoError(t, err) - - block, err := b.GetLastKnownBlockByAddress(777, account) - require.NoError(t, err) - - require.Equal(t, 0, block.Number.Cmp(r.to)) - require.Equal(t, big.NewInt(0).Int64(), block.Balance.Int64()) - require.Equal(t, nonce, uint64(*block.Nonce)) -} diff --git a/services/wallet/transfer/commands.go b/services/wallet/transfer/commands.go deleted file mode 100644 index 0e3cc1538b0..00000000000 --- a/services/wallet/transfer/commands.go +++ /dev/null @@ -1,565 +0,0 @@ -package transfer - -import ( - "context" - "database/sql" - "math/big" - "time" - - "go.uber.org/zap" - "golang.org/x/exp/maps" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/event" - - "github.com/status-im/status-go/logutils" - "github.com/status-im/status-go/rpc/chain" - "github.com/status-im/status-go/services/wallet/async" - "github.com/status-im/status-go/services/wallet/balance" - w_common "github.com/status-im/status-go/services/wallet/common" - "github.com/status-im/status-go/services/wallet/token" - "github.com/status-im/status-go/services/wallet/walletevent" - "github.com/status-im/status-go/transactions" -) - -const ( - // EventNewTransfers emitted when new block was added to the same canonical chan. - EventNewTransfers walletevent.EventType = "new-transfers" - // EventFetchingRecentHistory emitted when fetching of lastest tx history is started - EventFetchingRecentHistory walletevent.EventType = "recent-history-fetching" - // EventRecentHistoryReady emitted when fetching of lastest tx history is started - EventRecentHistoryReady walletevent.EventType = "recent-history-ready" - // EventFetchingHistoryError emitted when fetching of tx history failed - EventFetchingHistoryError walletevent.EventType = "fetching-history-error" - // EventNonArchivalNodeDetected emitted when a connection to a non archival node is detected - EventNonArchivalNodeDetected walletevent.EventType = "non-archival-node-detected" - - // Internal events emitted when different kinds of transfers are detected - EventInternalETHTransferDetected walletevent.EventType = walletevent.InternalEventTypePrefix + "eth-transfer-detected" - EventInternalERC20TransferDetected walletevent.EventType = walletevent.InternalEventTypePrefix + "erc20-transfer-detected" - EventInternalERC721TransferDetected walletevent.EventType = walletevent.InternalEventTypePrefix + "erc721-transfer-detected" - EventInternalERC1155TransferDetected walletevent.EventType = walletevent.InternalEventTypePrefix + "erc1155-transfer-detected" - - numberOfBlocksCheckedPerIteration = 40 - noBlockLimit = 0 -) - -var ( - bscBatchSize = big.NewInt(10000) - testnetBSCBatchSize = big.NewInt(5000) - sepoliaErc20BatchSize = big.NewInt(100000) - sepoliaErc20ArbitrumBatchSize = big.NewInt(10000) - sepoliaErc20OptimismBatchSize = big.NewInt(10000) - sepoliaErc20BaseBatchSize = big.NewInt(10000) - sepoliaErc20StatusNetworkBatchSize = big.NewInt(10000) - erc20BatchSize = big.NewInt(100000) - - transfersRetryInterval = 5 * time.Second -) - -type ethHistoricalCommand struct { - address common.Address - chainClient chain.ClientInterface - balanceCacher balance.Cacher - feed *event.Feed - foundHeaders []*DBHeader - error error - noLimit bool - - from *Block - to, resultingFrom, startBlock *big.Int - threadLimit uint32 -} - -type Transaction []*Transfer - -func (c *ethHistoricalCommand) Command() async.Command { - return async.FiniteCommand{ - Interval: 5 * time.Second, - Runable: c.Run, - }.Run -} - -func (c *ethHistoricalCommand) Run(ctx context.Context) (err error) { - logutils.ZapLogger().Debug("eth historical downloader start", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("address", c.address), - zap.Stringer("from", c.from.Number), - zap.Stringer("to", c.to), - zap.Bool("noLimit", c.noLimit), - ) - - start := time.Now() - if c.from.Number != nil && c.from.Balance != nil { - c.balanceCacher.Cache().AddBalance(c.address, c.chainClient.NetworkID(), c.from.Number, c.from.Balance) - } - if c.from.Number != nil && c.from.Nonce != nil { - c.balanceCacher.Cache().AddNonce(c.address, c.chainClient.NetworkID(), c.from.Number, c.from.Nonce) - } - from, headers, startBlock, err := findBlocksWithEthTransfers(ctx, c.chainClient, - c.balanceCacher, c.address, c.from.Number, c.to, c.noLimit, c.threadLimit) - - if err != nil { - c.error = err - logutils.ZapLogger().Error("failed to find blocks with transfers", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("address", c.address), - zap.Stringer("from", c.from.Number), - zap.Stringer("to", c.to), - zap.Error(err), - ) - return nil - } - - c.foundHeaders = headers - c.resultingFrom = from - c.startBlock = startBlock - - logutils.ZapLogger().Debug("eth historical downloader finished successfully", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("address", c.address), - zap.Stringer("from", from), - zap.Stringer("to", c.to), - zap.Int("totalBlocks", len(headers)), - zap.Duration("time", time.Since(start)), - ) - - return nil -} - -type erc20HistoricalCommand struct { - erc20 BatchDownloader - chainClient chain.ClientInterface - feed *event.Feed - - iterator *IterativeDownloader - to *big.Int - from *big.Int - foundHeaders []*DBHeader -} - -func (c *erc20HistoricalCommand) Command() async.Command { - return async.FiniteCommand{ - Interval: 5 * time.Second, - Runable: c.Run, - }.Run -} - -func getErc20BatchSize(chainID uint64) *big.Int { - switch chainID { - case w_common.EthereumSepolia: - return sepoliaErc20BatchSize - case w_common.OptimismSepolia: - return sepoliaErc20OptimismBatchSize - case w_common.ArbitrumSepolia: - return sepoliaErc20ArbitrumBatchSize - case w_common.BaseSepolia: - return sepoliaErc20BaseBatchSize - case w_common.StatusNetworkSepolia: - return sepoliaErc20StatusNetworkBatchSize - case w_common.BSCMainnet: - return bscBatchSize - case w_common.BSCTestnet: - return testnetBSCBatchSize - default: - return erc20BatchSize - } -} - -func (c *erc20HistoricalCommand) Run(ctx context.Context) (err error) { - logutils.ZapLogger().Debug("wallet historical downloader for erc20 transfers start", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("from", c.from), - zap.Stringer("to", c.to), - ) - - start := time.Now() - if c.iterator == nil { - c.iterator, err = SetupIterativeDownloader( - c.chainClient, - c.erc20, getErc20BatchSize(c.chainClient.NetworkID()), c.to, c.from) - if err != nil { - logutils.ZapLogger().Error("failed to setup historical downloader for erc20") - return err - } - } - for !c.iterator.Finished() { - headers, _, _, err := c.iterator.Next(ctx) - if err != nil { - logutils.ZapLogger().Error("failed to get next batch", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Error(err), - ) // TODO: stop inifinite command in case of an error that we can't fix like missing trie node - return err - } - c.foundHeaders = append(c.foundHeaders, headers...) - } - logutils.ZapLogger().Debug("wallet historical downloader for erc20 transfers finished", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("from", c.from), - zap.Stringer("to", c.to), - zap.Duration("time", time.Since(start)), - zap.Int("headers", len(c.foundHeaders)), - ) - return nil -} - -type transfersCommand struct { - db *Database - blockDAO *BlockDAO - eth *ETHDownloader - blockNums []*big.Int - address common.Address - chainClient chain.ClientInterface - blocksLimit int - pendingTxManager *transactions.PendingTxTracker - tokenManager *token.Manager - feed *event.Feed - - // result - fetchedTransfers []Transfer -} - -func (c *transfersCommand) Runner(interval ...time.Duration) async.Runner { - intvl := transfersRetryInterval - if len(interval) > 0 { - intvl = interval[0] - } - return async.FiniteCommandWithErrorCounter{ - FiniteCommand: async.FiniteCommand{ - Interval: intvl, - Runable: c.Run, - }, - ErrorCounter: async.NewErrorCounter(5, "transfersCommand"), - } -} - -func (c *transfersCommand) Command(interval ...time.Duration) async.Command { - return c.Runner(interval...).Run -} - -func (c *transfersCommand) Run(ctx context.Context) (err error) { - // Take blocks from cache if available and disrespect the limit - // If no blocks are available in cache, take blocks from DB respecting the limit - // If no limit is set, take all blocks from DB - logutils.ZapLogger().Debug("start transfersCommand", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Stringer("address", c.address), - zap.Stringers("blockNums", c.blockNums), - ) - startTs := time.Now() - - for { - blocks := c.blockNums - if blocks == nil { - blocks, _ = c.blockDAO.GetBlocksToLoadByAddress(c.chainClient.NetworkID(), c.address, numberOfBlocksCheckedPerIteration) - } - - for _, blockNum := range blocks { - logutils.ZapLogger().Debug("transfersCommand block start", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Stringer("address", c.address), - zap.Stringer("blockNum", blockNum), - ) - - allTransfers, err := c.eth.GetTransfersByNumber(ctx, blockNum) - if err != nil { - logutils.ZapLogger().Error("getTransfersByBlocks error", zap.Error(err)) - return err - } - - c.processUnknownErc20CommunityTransactions(ctx, allTransfers) - - if len(allTransfers) > 0 { - // First, try to match to any pre-existing pending/multi-transaction - err := c.saveAndConfirmPending(allTransfers, blockNum) - if err != nil { - logutils.ZapLogger().Error("saveAndConfirmPending error", zap.Error(err)) - return err - } - } else { - // If no transfers found, that is suspecting, because downloader returned this block as containing transfers - logutils.ZapLogger().Error("no transfers found in block", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Stringer("address", c.address), - zap.Stringer("block", blockNum), - ) - - err = markBlocksAsLoaded(c.chainClient.NetworkID(), c.db.client, c.address, []*big.Int{blockNum}) - if err != nil { - logutils.ZapLogger().Error("Mark blocks loaded error", zap.Error(err)) - return err - } - } - - c.fetchedTransfers = append(c.fetchedTransfers, allTransfers...) - - c.notifyOfNewTransfers(blockNum, allTransfers) - c.notifyOfLatestTransfers(allTransfers, w_common.EthTransfer) - c.notifyOfLatestTransfers(allTransfers, w_common.Erc20Transfer) - c.notifyOfLatestTransfers(allTransfers, w_common.Erc721Transfer) - c.notifyOfLatestTransfers(allTransfers, w_common.Erc1155Transfer) - - logutils.ZapLogger().Debug("transfersCommand block end", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Stringer("address", c.address), - zap.Stringer("blockNum", blockNum), - zap.Int("transfersLen", len(allTransfers)), - zap.Int("fetchedTransfersLen", len(c.fetchedTransfers)), - ) - } - - if c.blockNums != nil || len(blocks) == 0 || - (c.blocksLimit > noBlockLimit && len(blocks) >= c.blocksLimit) { - logutils.ZapLogger().Debug("loadTransfers breaking loop on block limits reached or 0 blocks", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Stringer("address", c.address), - zap.Int("limit", c.blocksLimit), - zap.Int("blocks", len(blocks)), - ) - break - } - } - - logutils.ZapLogger().Debug("end transfersCommand", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Stringer("address", c.address), - zap.Int("blocks.len", len(c.blockNums)), - zap.Int("transfers.len", len(c.fetchedTransfers)), - zap.Duration("in", time.Since(startTs)), - ) - - return nil -} - -// saveAndConfirmPending ensures only the transaction that has owner (Address) as a sender is matched to the -// corresponding multi-transaction (by multi-transaction ID). This way we ensure that if receiver is in the list -// of accounts filter will discard the proper one -func (c *transfersCommand) saveAndConfirmPending(allTransfers []Transfer, blockNum *big.Int) error { - tx, resErr := c.db.client.Begin() - if resErr != nil { - return resErr - } - notifyFunctions := c.confirmPendingTransactions(tx, allTransfers) - defer func() { - if resErr == nil { - commitErr := tx.Commit() - if commitErr != nil { - logutils.ZapLogger().Error("failed to commit", zap.Error(commitErr)) - } - for _, notify := range notifyFunctions { - notify() - } - } else { - rollbackErr := tx.Rollback() - if rollbackErr != nil { - logutils.ZapLogger().Error("failed to rollback", zap.Error(rollbackErr)) - } - } - }() - - resErr = saveTransfersMarkBlocksLoaded(tx, c.chainClient.NetworkID(), c.address, allTransfers, []*big.Int{blockNum}) - if resErr != nil { - logutils.ZapLogger().Error("SaveTransfers error", zap.Error(resErr)) - } - - return resErr -} - -func (c *transfersCommand) confirmPendingTransactions(tx *sql.Tx, allTransfers []Transfer) (notifyFunctions []func()) { - notifyFunctions = make([]func(), 0) - - // Confirm all pending transactions that are included in this block - for _, tr := range allTransfers { - chainID := w_common.ChainID(tr.NetworkID) - txHash := tr.Receipt.TxHash - txType, err := transactions.GetOwnedPendingStatus(tx, chainID, txHash, tr.Address) - if err == sql.ErrNoRows { - // Outside transaction, already confirmed by another duplicate or not yet downloaded - continue - } else if err != nil { - logutils.ZapLogger().Warn("GetOwnedPendingStatus", zap.Error(err)) - continue - } - if txType != nil && *txType == transactions.WalletTransfer { - notify, err := c.pendingTxManager.DeleteBySQLTx(tx, chainID, txHash) - if err != nil && err != transactions.ErrStillPending { - logutils.ZapLogger().Error("DeleteBySqlTx error", zap.Error(err)) - } - notifyFunctions = append(notifyFunctions, notify) - } - } - return notifyFunctions -} - -func (c *transfersCommand) processUnknownErc20CommunityTransactions(ctx context.Context, allTransfers []Transfer) { - for _, tx := range allTransfers { - // To can be nil in case of erc20 contract creation - if tx.Type == w_common.Erc20Transfer && tx.Transaction.To() != nil { - // Find token in db or if this is a community token, find its metadata - token := c.tokenManager.FindOrCreateTokenByAddress(ctx, tx.NetworkID, *tx.Transaction.To()) - if token != nil { - isFirst := false - if token.Verified || token.CommunityData != nil { - isFirst, _ = c.tokenManager.MarkAsPreviouslyOwnedToken(token, tx.Address) - } - if token.CommunityData != nil { - go c.tokenManager.SignalCommunityTokenReceived(tx.Address, tx.ID, tx.TokenValue, token, isFirst) - } - } - } - } -} - -func (c *transfersCommand) notifyOfNewTransfers(blockNum *big.Int, transfers []Transfer) { - if c.feed != nil { - if len(transfers) > 0 { - c.feed.Send(walletevent.Event{ - Type: EventNewTransfers, - Accounts: []common.Address{c.address}, - ChainID: c.chainClient.NetworkID(), - BlockNumber: blockNum, - }) - } - } -} - -func transferTypeToEventType(transferType w_common.Type) walletevent.EventType { - switch transferType { - case w_common.EthTransfer: - return EventInternalETHTransferDetected - case w_common.Erc20Transfer: - return EventInternalERC20TransferDetected - case w_common.Erc721Transfer: - return EventInternalERC721TransferDetected - case w_common.Erc1155Transfer: - return EventInternalERC1155TransferDetected - default: - return "" - } -} - -func (c *transfersCommand) notifyOfLatestTransfers(transfers []Transfer, transferType w_common.Type) { - if c.feed != nil { - eventTransfers := make([]Transfer, 0, len(transfers)) - latestTransferTimestamp := uint64(0) - for _, transfer := range transfers { - if transfer.Type == transferType { - eventTransfers = append(eventTransfers, transfer) - if transfer.Timestamp > latestTransferTimestamp { - latestTransferTimestamp = transfer.Timestamp - } - } - } - if len(eventTransfers) > 0 { - c.feed.Send(walletevent.Event{ - Type: transferTypeToEventType(transferType), - Accounts: []common.Address{c.address}, - ChainID: c.chainClient.NetworkID(), - At: int64(latestTransferTimestamp), - EventParams: eventTransfers, - }) - } - } -} - -type loadTransfersCommand struct { - accounts []common.Address - db *Database - blockDAO *BlockDAO - chainClient chain.ClientInterface - blocksByAddress map[common.Address][]*big.Int - pendingTxManager *transactions.PendingTxTracker - blocksLimit int - tokenManager *token.Manager - feed *event.Feed -} - -func (c *loadTransfersCommand) Command() async.Command { - return async.FiniteCommand{ - Interval: 5 * time.Second, - Runable: c.Run, - }.Run -} - -// This command always returns nil, even if there is an error in one of the commands. -// `transferCommand`s retry until maxError, but this command doesn't retry. -// In case some transfer is not loaded after max retries, it will be retried only after restart of the app. -// Currently there is no implementation to keep retrying until success. I think this should be implemented -// in `transferCommand` with exponential backoff instead of `loadTransfersCommand` (issue #4608). -func (c *loadTransfersCommand) Run(parent context.Context) (err error) { - return loadTransfers(parent, c.blockDAO, c.db, c.chainClient, c.blocksLimit, c.blocksByAddress, - c.pendingTxManager, c.tokenManager, c.feed) -} - -func loadTransfers(ctx context.Context, blockDAO *BlockDAO, db *Database, - chainClient chain.ClientInterface, blocksLimitPerAccount int, blocksByAddress map[common.Address][]*big.Int, - pendingTxManager *transactions.PendingTxTracker, tokenManager *token.Manager, feed *event.Feed) error { - - logutils.ZapLogger().Debug("loadTransfers start", - zap.Uint64("chain", chainClient.NetworkID()), - zap.Int("limit", blocksLimitPerAccount), - ) - - start := time.Now() - group := async.NewGroup(ctx) - - accounts := maps.Keys(blocksByAddress) - for _, address := range accounts { - transfers := &transfersCommand{ - db: db, - blockDAO: blockDAO, - chainClient: chainClient, - address: address, - eth: ÐDownloader{ - chainClient: chainClient, - accounts: []common.Address{address}, - signer: types.LatestSignerForChainID(chainClient.ToBigInt()), - db: db, - }, - blockNums: blocksByAddress[address], - pendingTxManager: pendingTxManager, - tokenManager: tokenManager, - feed: feed, - } - group.Add(transfers.Command()) - } - - select { - case <-ctx.Done(): - logutils.ZapLogger().Debug("loadTransfers cancelled", - zap.Uint64("chain", chainClient.NetworkID()), - zap.Error(ctx.Err()), - ) - case <-group.WaitAsync(): - logutils.ZapLogger().Debug("loadTransfers finished for account", - zap.Duration("in", time.Since(start)), - zap.Uint64("chain", chainClient.NetworkID()), - ) - } - return nil -} - -// Ensure 1 DBHeader per Block Hash -func uniqueHeaderPerBlockHash(allHeaders []*DBHeader) []*DBHeader { - uniqHeadersByHash := map[common.Hash]*DBHeader{} - for _, header := range allHeaders { - uniqHeader, ok := uniqHeadersByHash[header.Hash] - if ok { - if len(header.PreloadedTransactions) > 0 { - uniqHeader.PreloadedTransactions = append(uniqHeader.PreloadedTransactions, header.PreloadedTransactions...) - } - uniqHeadersByHash[header.Hash] = uniqHeader - } else { - uniqHeadersByHash[header.Hash] = header - } - } - - uniqHeaders := []*DBHeader{} - for _, header := range uniqHeadersByHash { - uniqHeaders = append(uniqHeaders, header) - } - - return uniqHeaders -} diff --git a/services/wallet/transfer/commands_sequential.go b/services/wallet/transfer/commands_sequential.go deleted file mode 100644 index 666e32d7307..00000000000 --- a/services/wallet/transfer/commands_sequential.go +++ /dev/null @@ -1,1587 +0,0 @@ -package transfer - -import ( - "context" - "math/big" - "sync/atomic" - "time" - - "go.uber.org/zap" - - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/event" - accsmanagementtypes "github.com/status-im/status-go/accounts-management/types" - gocommon "github.com/status-im/status-go/common" - "github.com/status-im/status-go/contracts" - cryptotypes "github.com/status-im/status-go/crypto/types" - "github.com/status-im/status-go/logutils" - "github.com/status-im/status-go/multiaccounts/accounts" - "github.com/status-im/status-go/rpc/chain" - "github.com/status-im/status-go/rpc/chain/rpclimiter" - "github.com/status-im/status-go/services/wallet/async" - "github.com/status-im/status-go/services/wallet/balance" - "github.com/status-im/status-go/services/wallet/blockchainstate" - "github.com/status-im/status-go/services/wallet/token" - "github.com/status-im/status-go/services/wallet/walletevent" - "github.com/status-im/status-go/transactions" -) - -var findBlocksRetryInterval = 5 * time.Second - -const ( - transferHistoryTag = "transfer_history" - newTransferHistoryTag = "new_transfer_history" - - transferHistoryLimit = 10000 - transferHistoryLimitPerAccount = 5000 - transferHistoryLimitPeriod = 24 * time.Hour -) - -type nonceInfo struct { - nonce *int64 - blockNumber *big.Int -} - -type findNewBlocksCommand struct { - *findBlocksCommand - contractMaker *contracts.ContractMaker - iteration int - blockChainState *blockchainstate.BlockChainState - lastNonces map[common.Address]nonceInfo - nonceCheckIntervalIterations int - logsCheckIntervalIterations int -} - -func (c *findNewBlocksCommand) Command() async.Command { - return async.InfiniteCommand{ - Interval: 2 * time.Minute, - Runable: c.Run, - }.Run -} - -var requestTimeout = 20 * time.Second - -func (c *findNewBlocksCommand) detectTransfers(parent context.Context, accounts []common.Address) (*big.Int, []common.Address, error) { - bc, err := c.contractMaker.NewBalanceChecker(c.chainClient.NetworkID()) - if err != nil { - logutils.ZapLogger().Error("findNewBlocksCommand error creating balance checker", zap.Uint64("chain", c.chainClient.NetworkID()), zap.Error(err)) - return nil, nil, err - } - - tokens, err := c.tokenManager.GetTokens(c.chainClient.NetworkID()) - if err != nil { - return nil, nil, err - } - tokenAddresses := []common.Address{} - nilAddress := common.Address{} - for _, token := range tokens { - if token.Address != nilAddress { - tokenAddresses = append(tokenAddresses, token.Address) - } - } - logutils.ZapLogger().Debug("findNewBlocksCommand detectTransfers", zap.Int("cnt", len(tokenAddresses))) - - ctx, cancel := context.WithTimeout(parent, requestTimeout) - defer cancel() - blockNum, hashes, err := bc.BalancesHash(&bind.CallOpts{Context: ctx}, c.accounts, tokenAddresses) - if err != nil { - logutils.ZapLogger().Error("findNewBlocksCommand can't get balances hashes", zap.Error(err)) - return nil, nil, err - } - - addressesToCheck := []common.Address{} - for idx, account := range accounts { - blockRange, _, err := c.blockRangeDAO.getBlockRange(c.chainClient.NetworkID(), account) - if err != nil { - logutils.ZapLogger().Error("findNewBlocksCommand can't get block range", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(err), - ) - return nil, nil, err - } - - checkHash := common.BytesToHash(hashes[idx][:]) - logutils.ZapLogger().Debug("findNewBlocksCommand comparing hashes", - zap.Stringer("account", account), - zap.Uint64("network", c.chainClient.NetworkID()), - zap.String("old hash", blockRange.balanceCheckHash), - zap.Stringer("new hash", checkHash), - ) - if checkHash.String() != blockRange.balanceCheckHash { - addressesToCheck = append(addressesToCheck, account) - } - - blockRange.balanceCheckHash = checkHash.String() - - err = c.blockRangeDAO.upsertRange(c.chainClient.NetworkID(), account, blockRange) - if err != nil { - logutils.ZapLogger().Error("findNewBlocksCommand can't update balance check", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(err), - ) - return nil, nil, err - } - } - - return blockNum, addressesToCheck, nil -} - -func (c *findNewBlocksCommand) detectNonceChange(parent context.Context, to *big.Int, accounts []common.Address) (map[common.Address]*big.Int, error) { - addressesWithChange := map[common.Address]*big.Int{} - for _, account := range accounts { - var oldNonce *int64 - - blockRange, _, err := c.blockRangeDAO.getBlockRange(c.chainClient.NetworkID(), account) - if err != nil { - logutils.ZapLogger().Error("findNewBlocksCommand can't get block range", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(err), - ) - return nil, err - } - - lastNonceInfo, ok := c.lastNonces[account] - if !ok || lastNonceInfo.blockNumber.Cmp(blockRange.eth.LastKnown) != 0 { - logutils.ZapLogger().Debug("Fetching old nonce", - zap.Stringer("at", blockRange.eth.LastKnown), - zap.Stringer("acc", account), - ) - if blockRange.eth.LastKnown == nil { - blockRange.eth.LastKnown = big.NewInt(0) - oldNonce = new(int64) // At 0 block nonce is 0 - } else { - oldNonce, err = c.balanceCacher.NonceAt(parent, c.chainClient, account, blockRange.eth.LastKnown) - if err != nil { - logutils.ZapLogger().Error("findNewBlocksCommand can't get nonce", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(err), - ) - return nil, err - } - } - } else { - oldNonce = lastNonceInfo.nonce - } - - newNonce, err := c.balanceCacher.NonceAt(parent, c.chainClient, account, to) - if err != nil { - logutils.ZapLogger().Error("findNewBlocksCommand can't get nonce", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(err), - ) - return nil, err - } - - logutils.ZapLogger().Debug("Comparing nonces", - zap.Int64p("oldNonce", oldNonce), - zap.Int64p("newNonce", newNonce), - zap.Stringer("to", to), - zap.Stringer("acc", account), - ) - - if *newNonce != *oldNonce { - addressesWithChange[account] = blockRange.eth.LastKnown - } - - if c.lastNonces == nil { - c.lastNonces = map[common.Address]nonceInfo{} - } - - c.lastNonces[account] = nonceInfo{ - nonce: newNonce, - blockNumber: to, - } - } - - return addressesWithChange, nil -} - -var nonceCheckIntervalIterations = 30 -var logsCheckIntervalIterations = 5 - -func (c *findNewBlocksCommand) Run(parent context.Context) error { - mnemonicWasNotShown, err := c.accountsDB.GetMnemonicWasNotShown() - if err != nil { - return err - } - - accountsToCheck := []common.Address{} - // accounts which might have outgoing transfers initiated outside - // the application, e.g. watch only or restored from mnemonic phrase - accountsWithOutsideTransfers := []common.Address{} - - for _, account := range c.accounts { - acc, err := c.accountsDB.GetAccountByAddress(cryptotypes.Address(account)) - if err != nil { - return err - } - if mnemonicWasNotShown { - if acc.AddressWasNotShown { - logutils.ZapLogger().Info("skip findNewBlocksCommand, mnemonic has not been shown and the address has not been shared yet", zap.Stringer("address", account)) - continue - } - } - if !mnemonicWasNotShown || acc.Type != accsmanagementtypes.AccountTypeGenerated { - accountsWithOutsideTransfers = append(accountsWithOutsideTransfers, account) - } - - accountsToCheck = append(accountsToCheck, account) - } - - if len(accountsToCheck) == 0 { - return nil - } - - headNum, accountsWithDetectedChanges, err := c.detectTransfers(parent, accountsToCheck) - if err != nil { - logutils.ZapLogger().Error("findNewBlocksCommand error on transfer detection", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(err), - ) - return err - } - - c.blockChainState.SetLastBlockNumber(c.chainClient.NetworkID(), headNum.Uint64()) - - if len(accountsWithDetectedChanges) != 0 { - logutils.ZapLogger().Debug("findNewBlocksCommand detected accounts with changes", - zap.Stringers("accounts", accountsWithDetectedChanges), - zap.Stringer("from", c.fromBlockNumber), - ) - err = c.findAndSaveEthBlocks(parent, c.fromBlockNumber, headNum, accountsToCheck) - if err != nil { - return err - } - } else if c.iteration%c.nonceCheckIntervalIterations == 0 && len(accountsWithOutsideTransfers) > 0 { - logutils.ZapLogger().Debug("findNewBlocksCommand nonce check", zap.Stringers("accounts", accountsWithOutsideTransfers)) - accountsWithNonceChanges, err := c.detectNonceChange(parent, headNum, accountsWithOutsideTransfers) - if err != nil { - return err - } - - if len(accountsWithNonceChanges) > 0 { - logutils.ZapLogger().Debug("findNewBlocksCommand detected nonce diff", zap.Any("accounts", accountsWithNonceChanges)) - for account, from := range accountsWithNonceChanges { - err = c.findAndSaveEthBlocks(parent, from, headNum, []common.Address{account}) - if err != nil { - return err - } - } - } - - for _, account := range accountsToCheck { - if _, ok := accountsWithNonceChanges[account]; ok { - continue - } - err := c.markEthBlockRangeChecked(account, &BlockRange{nil, c.fromBlockNumber, headNum}) - if err != nil { - return err - } - } - } - - if len(accountsWithDetectedChanges) != 0 || c.iteration%c.logsCheckIntervalIterations == 0 { - from := c.fromBlockNumber - if c.logsCheckLastKnownBlock != nil { - from = c.logsCheckLastKnownBlock - } - err = c.findAndSaveTokenBlocks(parent, from, headNum) - if err != nil { - return err - } - c.logsCheckLastKnownBlock = headNum - } - c.fromBlockNumber = headNum - c.iteration++ - - return nil -} - -func (c *findNewBlocksCommand) findAndSaveEthBlocks(parent context.Context, fromNum, headNum *big.Int, accounts []common.Address) error { - // Check ETH transfers for each account independently - mnemonicWasNotShown, err := c.accountsDB.GetMnemonicWasNotShown() - if err != nil { - return err - } - - for _, account := range accounts { - if mnemonicWasNotShown { - acc, err := c.accountsDB.GetAccountByAddress(cryptotypes.Address(account)) - if err != nil { - return err - } - if acc.AddressWasNotShown { - logutils.ZapLogger().Info("skip findNewBlocksCommand, mnemonic has not been shown and the address has not been shared yet", zap.Stringer("address", account)) - continue - } - } - - logutils.ZapLogger().Debug("start findNewBlocksCommand", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Bool("noLimit", c.noLimit), - zap.Stringer("from", fromNum), - zap.Stringer("to", headNum), - ) - - headers, startBlockNum, err := c.findBlocksWithEthTransfers(parent, account, fromNum, headNum) - if err != nil { - return err - } - - if len(headers) > 0 { - logutils.ZapLogger().Debug("findNewBlocksCommand saving headers", - zap.Int("len", len(headers)), - zap.Stringer("lastBlockNumber", headNum), - zap.Stringer("balance", c.balanceCacher.Cache().GetBalance(account, c.chainClient.NetworkID(), headNum)), - zap.Int64p("nonce", c.balanceCacher.Cache().GetNonce(account, c.chainClient.NetworkID(), headNum)), - ) - - err := c.db.SaveBlocks(c.chainClient.NetworkID(), headers) - if err != nil { - return err - } - - c.blocksFound(headers) - } - - err = c.markEthBlockRangeChecked(account, &BlockRange{startBlockNum, fromNum, headNum}) - if err != nil { - return err - } - - logutils.ZapLogger().Debug("end findNewBlocksCommand", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Bool("noLimit", c.noLimit), - zap.Stringer("from", fromNum), - zap.Stringer("to", headNum), - ) - } - - return nil -} - -func (c *findNewBlocksCommand) findAndSaveTokenBlocks(parent context.Context, fromNum, headNum *big.Int) error { - // Check token transfers for all accounts. - // Each account's last checked block can be different, so we can get duplicated headers, - // so we need to deduplicate them - const incomingOnly = false - erc20Headers, err := c.fastIndexErc20(parent, fromNum, headNum, incomingOnly) - if err != nil { - logutils.ZapLogger().Error("findNewBlocksCommand fastIndexErc20", - zap.Stringers("account", c.accounts), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(err), - ) - return err - } - - if len(erc20Headers) > 0 { - logutils.ZapLogger().Debug("findNewBlocksCommand saving headers", - zap.Int("len", len(erc20Headers)), - zap.Stringer("from", fromNum), - zap.Stringer("to", headNum), - ) - - // get not loaded headers from DB for all accs and blocks - preLoadedTransactions, err := c.db.GetTransactionsToLoad(c.chainClient.NetworkID(), common.Address{}, nil) - if err != nil { - return err - } - - tokenBlocksFiltered := filterNewPreloadedTransactions(erc20Headers, preLoadedTransactions) - - err = c.db.SaveBlocks(c.chainClient.NetworkID(), tokenBlocksFiltered) - if err != nil { - return err - } - - c.blocksFound(tokenBlocksFiltered) - } - - return c.markTokenBlockRangeChecked(c.accounts, fromNum, headNum) -} - -func (c *findBlocksCommand) markTokenBlockRangeChecked(accounts []common.Address, from, to *big.Int) error { - logutils.ZapLogger().Debug("markTokenBlockRangeChecked", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Uint64("from", from.Uint64()), - zap.Uint64("to", to.Uint64()), - ) - - for _, account := range accounts { - err := c.blockRangeDAO.updateTokenRange(c.chainClient.NetworkID(), account, &BlockRange{FirstKnown: from, LastKnown: to}) - if err != nil { - logutils.ZapLogger().Error("findNewBlocksCommand upsertTokenRange", zap.Error(err)) - return err - } - } - - return nil -} - -func filterNewPreloadedTransactions(erc20Headers []*DBHeader, preLoadedTransfers []*PreloadedTransaction) []*DBHeader { - var uniqueErc20Headers []*DBHeader - for _, header := range erc20Headers { - loaded := false - for _, transfer := range preLoadedTransfers { - if header.PreloadedTransactions[0].ID == transfer.ID { - loaded = true - break - } - } - - if !loaded { - uniqueErc20Headers = append(uniqueErc20Headers, header) - } - } - - return uniqueErc20Headers -} - -func (c *findNewBlocksCommand) findBlocksWithEthTransfers(parent context.Context, account common.Address, fromOrig, toOrig *big.Int) (headers []*DBHeader, startBlockNum *big.Int, err error) { - logutils.ZapLogger().Debug("start findNewBlocksCommand::findBlocksWithEthTransfers", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Bool("noLimit", c.noLimit), - zap.Stringer("from", c.fromBlockNumber), - zap.Stringer("to", c.toBlockNumber), - ) - - rangeSize := big.NewInt(int64(c.defaultNodeBlockChunkSize)) - - from, to := new(big.Int).Set(fromOrig), new(big.Int).Set(toOrig) - - // Limit the range size to DefaultNodeBlockChunkSize - if new(big.Int).Sub(to, from).Cmp(rangeSize) > 0 { - from.Sub(to, rangeSize) - } - - for { - if from.Cmp(to) == 0 { - logutils.ZapLogger().Debug("findNewBlocksCommand empty range", - zap.Stringer("from", from), - zap.Stringer("to", to), - ) - break - } - - fromBlock := &Block{Number: from} - - var newFromBlock *Block - var ethHeaders []*DBHeader - newFromBlock, ethHeaders, startBlockNum, err = c.fastIndex(parent, account, c.balanceCacher, fromBlock, to) - if err != nil { - logutils.ZapLogger().Error("findNewBlocksCommand checkRange fastIndex", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(err), - ) - return nil, nil, err - } - logutils.ZapLogger().Debug("findNewBlocksCommand checkRange", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("account", account), - zap.Stringer("startBlock", startBlockNum), - zap.Stringer("newFromBlock", newFromBlock.Number), - zap.Stringer("toBlockNumber", to), - zap.Bool("noLimit", c.noLimit), - ) - - headers = append(headers, ethHeaders...) - - if startBlockNum != nil && startBlockNum.Cmp(from) >= 0 { - logutils.ZapLogger().Debug("Checked all ranges, stop execution", - zap.Stringer("startBlock", startBlockNum), - zap.Stringer("from", from), - zap.Stringer("to", to), - ) - break - } - - nextFrom, nextTo := nextRange(c.defaultNodeBlockChunkSize, newFromBlock.Number, fromOrig) - - if nextFrom.Cmp(from) == 0 && nextTo.Cmp(to) == 0 { - logutils.ZapLogger().Debug("findNewBlocksCommand empty next range", - zap.Stringer("from", from), - zap.Stringer("to", to), - ) - break - } - - from = nextFrom - to = nextTo - } - - logutils.ZapLogger().Debug("end findNewBlocksCommand::findBlocksWithEthTransfers", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Bool("noLimit", c.noLimit), - ) - - return headers, startBlockNum, nil -} - -// TODO NewFindBlocksCommand -type findBlocksCommand struct { - accounts []common.Address - db *Database - accountsDB *accounts.Database - blockRangeDAO BlockRangeDAOer - chainClient chain.ClientInterface - balanceCacher balance.Cacher - feed *event.Feed - noLimit bool - tokenManager *token.Manager - fromBlockNumber *big.Int - logsCheckLastKnownBlock *big.Int - toBlockNumber *big.Int - blocksLoadedCh chan<- []*DBHeader - defaultNodeBlockChunkSize int - - // Not to be set by the caller - resFromBlock *Block - startBlockNumber *big.Int - reachedETHHistoryStart bool -} - -func (c *findBlocksCommand) Runner(interval ...time.Duration) async.Runner { - intvl := findBlocksRetryInterval - if len(interval) > 0 { - intvl = interval[0] - } - return async.FiniteCommandWithErrorCounter{ - FiniteCommand: async.FiniteCommand{ - Interval: intvl, - Runable: c.Run, - }, - ErrorCounter: async.NewErrorCounter(3, "findBlocksCommand"), - } -} - -func (c *findBlocksCommand) Command(interval ...time.Duration) async.Command { - return c.Runner(interval...).Run -} - -type ERC20BlockRange struct { - from *big.Int - to *big.Int -} - -func (c *findBlocksCommand) ERC20ScanByBalance(parent context.Context, account common.Address, fromBlock, toBlock *big.Int, token common.Address) ([]ERC20BlockRange, error) { - var err error - batchSize := getErc20BatchSize(c.chainClient.NetworkID()) - ranges := [][]*big.Int{{fromBlock, toBlock}} - foundRanges := []ERC20BlockRange{} - cache := map[int64]*big.Int{} - for { - nextRanges := [][]*big.Int{} - for _, blockRange := range ranges { - from, to := blockRange[0], blockRange[1] - fromBalance, ok := cache[from.Int64()] - if !ok { - fromBalance, err = c.tokenManager.GetTokenBalanceAt(parent, c.chainClient, account, token, from) - if err != nil { - return nil, err - } - - if fromBalance == nil { - fromBalance = big.NewInt(0) - } - cache[from.Int64()] = fromBalance - } - - toBalance, ok := cache[to.Int64()] - if !ok { - toBalance, err = c.tokenManager.GetTokenBalanceAt(parent, c.chainClient, account, token, to) - if err != nil { - return nil, err - } - if toBalance == nil { - toBalance = big.NewInt(0) - } - cache[to.Int64()] = toBalance - } - - if fromBalance.Cmp(toBalance) != 0 { - diff := new(big.Int).Sub(to, from) - if diff.Cmp(batchSize) <= 0 { - foundRanges = append(foundRanges, ERC20BlockRange{from, to}) - continue - } - - halfOfDiff := new(big.Int).Div(diff, big.NewInt(2)) - mid := new(big.Int).Add(from, halfOfDiff) - - nextRanges = append(nextRanges, []*big.Int{from, mid}) - nextRanges = append(nextRanges, []*big.Int{mid, to}) - } - } - - if len(nextRanges) == 0 { - break - } - - ranges = nextRanges - } - - return foundRanges, nil -} - -func (c *findBlocksCommand) checkERC20Tail(parent context.Context, account common.Address) ([]*DBHeader, error) { - logutils.ZapLogger().Debug( - "checkERC20Tail", - zap.Stringer("account", account), - zap.Stringer("to block", c.startBlockNumber), - zap.Stringer("from", c.resFromBlock.Number), - ) - tokens, err := c.tokenManager.GetTokens(c.chainClient.NetworkID()) - if err != nil { - return nil, err - } - addresses := make([]common.Address, len(tokens)) - for i, token := range tokens { - addresses[i] = token.Address - } - - from := new(big.Int).Sub(c.resFromBlock.Number, big.NewInt(1)) - - clients := make(map[uint64]chain.ClientInterface, 1) - clients[c.chainClient.NetworkID()] = c.chainClient - atBlocks := make(map[uint64]*big.Int, 1) - atBlocks[c.chainClient.NetworkID()] = from - balances, err := c.tokenManager.GetBalancesAtByChain(parent, clients, []common.Address{account}, addresses, atBlocks) - if err != nil { - return nil, err - } - - foundRanges := []ERC20BlockRange{} - for token, balance := range balances[c.chainClient.NetworkID()][account] { - bigintBalance := big.NewInt(balance.ToInt().Int64()) - if bigintBalance.Cmp(big.NewInt(0)) <= 0 { - continue - } - result, err := c.ERC20ScanByBalance(parent, account, big.NewInt(0), from, token) - if err != nil { - return nil, err - } - - foundRanges = append(foundRanges, result...) - } - - uniqRanges := []ERC20BlockRange{} - rangesMap := map[string]bool{} - for _, rangeItem := range foundRanges { - key := rangeItem.from.String() + "-" + rangeItem.to.String() - if _, ok := rangesMap[key]; !ok { - rangesMap[key] = true - uniqRanges = append(uniqRanges, rangeItem) - } - } - - foundHeaders := []*DBHeader{} - for _, rangeItem := range uniqRanges { - headers, err := c.fastIndexErc20(parent, rangeItem.from, rangeItem.to, true) - if err != nil { - return nil, err - } - foundHeaders = append(foundHeaders, headers...) - } - - return foundHeaders, nil -} - -func (c *findBlocksCommand) Run(parent context.Context) (err error) { - logutils.ZapLogger().Debug("start findBlocksCommand", - zap.Any("accounts", c.accounts), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Bool("noLimit", c.noLimit), - zap.Stringer("from", c.fromBlockNumber), - zap.Stringer("to", c.toBlockNumber), - ) - - account := c.accounts[0] // For now this command supports only 1 account - mnemonicWasNotShown, err := c.accountsDB.GetMnemonicWasNotShown() - if err != nil { - return err - } - - if mnemonicWasNotShown { - account, err := c.accountsDB.GetAccountByAddress(cryptotypes.BytesToAddress(account.Bytes())) - if err != nil { - return err - } - if account.AddressWasNotShown { - logutils.ZapLogger().Info("skip findBlocksCommand, mnemonic has not been shown and the address has not been shared yet", zap.Stringer("address", account.Address)) - return nil - } - } - - rangeSize := big.NewInt(int64(c.defaultNodeBlockChunkSize)) - from, to := new(big.Int).Set(c.fromBlockNumber), new(big.Int).Set(c.toBlockNumber) - - // Limit the range size to DefaultNodeBlockChunkSize - if new(big.Int).Sub(to, from).Cmp(rangeSize) > 0 { - from.Sub(to, rangeSize) - } - - for { - if from.Cmp(to) == 0 { - logutils.ZapLogger().Debug("findBlocksCommand empty range", - zap.Stringer("from", from), - zap.Stringer("to", to)) - break - } - - var headers []*DBHeader - if c.reachedETHHistoryStart { - if c.fromBlockNumber.Cmp(zero) == 0 && c.startBlockNumber != nil && c.startBlockNumber.Cmp(zero) == 1 { - headers, err = c.checkERC20Tail(parent, account) - if err != nil { - logutils.ZapLogger().Error("findBlocksCommand checkERC20Tail", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(err), - ) - break - } - } - } else { - headers, err = c.checkRange(parent, from, to) - if err != nil { - break - } - } - - if len(headers) > 0 { - logutils.ZapLogger().Debug("findBlocksCommand saving headers", - zap.Int("len", len(headers)), - zap.Stringer("lastBlockNumber", to), - zap.Stringer("balance", c.balanceCacher.Cache().GetBalance(account, c.chainClient.NetworkID(), to)), - zap.Int64p("nonce", c.balanceCacher.Cache().GetNonce(account, c.chainClient.NetworkID(), to)), - ) - - err = c.db.SaveBlocks(c.chainClient.NetworkID(), headers) - if err != nil { - break - } - - c.blocksFound(headers) - } - - if c.reachedETHHistoryStart { - err = c.markTokenBlockRangeChecked([]common.Address{account}, big.NewInt(0), to) - if err != nil { - break - } - logutils.ZapLogger().Debug("findBlocksCommand reached first ETH transfer and checked erc20 tail", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Stringer("account", account), - ) - break - } - - err = c.markEthBlockRangeChecked(account, &BlockRange{c.startBlockNumber, c.resFromBlock.Number, to}) - if err != nil { - break - } - - err = c.markTokenBlockRangeChecked([]common.Address{account}, c.resFromBlock.Number, to) - if err != nil { - break - } - - // if we have found first ETH block and we have not reached the start of ETH history yet - if c.startBlockNumber != nil && c.fromBlockNumber.Cmp(from) == -1 { - logutils.ZapLogger().Debug("ERC20 tail should be checked", - zap.Stringer("initial from", c.fromBlockNumber), - zap.Stringer("actual from", from), - zap.Stringer("first ETH block", c.startBlockNumber), - ) - c.reachedETHHistoryStart = true - continue - } - - if c.startBlockNumber != nil && c.startBlockNumber.Cmp(from) >= 0 { - logutils.ZapLogger().Debug("Checked all ranges, stop execution", - zap.Stringer("startBlock", c.startBlockNumber), - zap.Stringer("from", from), - zap.Stringer("to", to), - ) - break - } - - nextFrom, nextTo := nextRange(c.defaultNodeBlockChunkSize, c.resFromBlock.Number, c.fromBlockNumber) - - if nextFrom.Cmp(from) == 0 && nextTo.Cmp(to) == 0 { - logutils.ZapLogger().Debug("findBlocksCommand empty next range", - zap.Stringer("from", from), - zap.Stringer("to", to), - ) - break - } - - from = nextFrom - to = nextTo - } - - logutils.ZapLogger().Debug("end findBlocksCommand", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Bool("noLimit", c.noLimit), - zap.Error(err), - ) - - return err -} - -func (c *findBlocksCommand) blocksFound(headers []*DBHeader) { - c.blocksLoadedCh <- headers -} - -func (c *findBlocksCommand) markEthBlockRangeChecked(account common.Address, blockRange *BlockRange) error { - logutils.ZapLogger().Debug("upsert block range", - zap.Stringer("Start", blockRange.Start), - zap.Stringer("FirstKnown", blockRange.FirstKnown), - zap.Stringer("LastKnown", blockRange.LastKnown), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Stringer("account", account), - ) - - err := c.blockRangeDAO.upsertEthRange(c.chainClient.NetworkID(), account, blockRange) - if err != nil { - logutils.ZapLogger().Error("findBlocksCommand upsertRange", zap.Error(err)) - return err - } - - return nil -} - -func (c *findBlocksCommand) checkRange(parent context.Context, from *big.Int, to *big.Int) ( - foundHeaders []*DBHeader, err error) { - - account := c.accounts[0] - fromBlock := &Block{Number: from} - - newFromBlock, ethHeaders, startBlock, err := c.fastIndex(parent, account, c.balanceCacher, fromBlock, to) - if err != nil { - logutils.ZapLogger().Error("findBlocksCommand checkRange fastIndex", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(err), - ) - return nil, err - } - logutils.ZapLogger().Debug("findBlocksCommand checkRange", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("account", account), - zap.Stringer("startBlock", startBlock), - zap.Stringer("newFromBlock", newFromBlock.Number), - zap.Stringer("toBlockNumber", to), - zap.Bool("noLimit", c.noLimit), - ) - - // There could be incoming ERC20 transfers which don't change the balance - // and nonce of ETH account, so we keep looking for them - erc20Headers, err := c.fastIndexErc20(parent, newFromBlock.Number, to, false) - if err != nil { - logutils.ZapLogger().Error("findBlocksCommand checkRange fastIndexErc20", - zap.Stringer("account", account), - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(err), - ) - return nil, err - } - - allHeaders := append(ethHeaders, erc20Headers...) - - if len(allHeaders) > 0 { - foundHeaders = uniqueHeaderPerBlockHash(allHeaders) - } - - c.resFromBlock = newFromBlock - c.startBlockNumber = startBlock - - logutils.ZapLogger().Debug("end findBlocksCommand checkRange", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("account", account), - zap.Stringer("c.startBlock", c.startBlockNumber), - zap.Stringer("newFromBlock", newFromBlock.Number), - zap.Stringer("toBlockNumber", to), - zap.Stringer("c.resFromBlock", c.resFromBlock.Number), - ) - - return -} - -func loadBlockRangeInfo(chainID uint64, account common.Address, blockDAO BlockRangeDAOer) ( - *ethTokensBlockRanges, error) { - - blockRange, _, err := blockDAO.getBlockRange(chainID, account) - if err != nil { - logutils.ZapLogger().Error( - "failed to load block ranges from database", - zap.Uint64("chain", chainID), - zap.Stringer("account", account), - zap.Error(err), - ) - return nil, err - } - - return blockRange, nil -} - -// Returns if all blocks are loaded, which means that start block (beginning of account history) -// has been found and all block headers saved to the DB -func areAllHistoryBlocksLoaded(blockInfo *BlockRange) bool { - if blockInfo != nil && blockInfo.FirstKnown != nil && - ((blockInfo.Start != nil && blockInfo.Start.Cmp(blockInfo.FirstKnown) >= 0) || - blockInfo.FirstKnown.Cmp(zero) == 0) { - return true - } - - return false -} - -func areAllHistoryBlocksLoadedForAddress(blockRangeDAO BlockRangeDAOer, chainID uint64, - address common.Address) (bool, error) { - - blockRange, _, err := blockRangeDAO.getBlockRange(chainID, address) - if err != nil { - logutils.ZapLogger().Error("findBlocksCommand getBlockRange", zap.Error(err)) - return false, err - } - - return areAllHistoryBlocksLoaded(blockRange.eth) && areAllHistoryBlocksLoaded(blockRange.tokens), nil -} - -// run fast indexing for every accont up to canonical chain head minus safety depth. -// every account will run it from last synced header. -func (c *findBlocksCommand) fastIndex(ctx context.Context, account common.Address, bCacher balance.Cacher, - fromBlock *Block, toBlockNumber *big.Int) (resultingFrom *Block, headers []*DBHeader, - startBlock *big.Int, err error) { - - logutils.ZapLogger().Debug("fast index started", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("account", account), - zap.Stringer("from", fromBlock.Number), - zap.Stringer("to", toBlockNumber), - ) - - start := time.Now() - group := async.NewGroup(ctx) - - command := ðHistoricalCommand{ - chainClient: c.chainClient, - balanceCacher: bCacher, - address: account, - feed: c.feed, - from: fromBlock, - to: toBlockNumber, - noLimit: c.noLimit, - threadLimit: SequentialThreadLimit, - } - group.Add(command.Command()) - - select { - case <-ctx.Done(): - err = ctx.Err() - logutils.ZapLogger().Debug("fast indexer ctx Done", zap.Error(err)) - return - case <-group.WaitAsync(): - if command.error != nil { - err = command.error - return - } - resultingFrom = &Block{Number: command.resultingFrom} - headers = command.foundHeaders - startBlock = command.startBlock - logutils.ZapLogger().Debug("fast indexer finished", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("account", account), - zap.Duration("in", time.Since(start)), - zap.Stringer("startBlock", command.startBlock), - zap.Stringer("resultingFrom", resultingFrom.Number), - zap.Int("headers", len(headers)), - ) - return - } -} - -// run fast indexing for every accont up to canonical chain head minus safety depth. -// every account will run it from last synced header. -func (c *findBlocksCommand) fastIndexErc20(ctx context.Context, fromBlockNumber *big.Int, - toBlockNumber *big.Int, incomingOnly bool) ([]*DBHeader, error) { - - start := time.Now() - group := async.NewGroup(ctx) - - erc20 := &erc20HistoricalCommand{ - erc20: NewERC20TransfersDownloader(c.chainClient, c.accounts, types.LatestSignerForChainID(c.chainClient.ToBigInt()), incomingOnly), - chainClient: c.chainClient, - feed: c.feed, - from: fromBlockNumber, - to: toBlockNumber, - foundHeaders: []*DBHeader{}, - } - group.Add(erc20.Command()) - - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-group.WaitAsync(): - headers := erc20.foundHeaders - logutils.ZapLogger().Debug("fast indexer Erc20 finished", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Duration("in", time.Since(start)), - zap.Int("headers", len(headers)), - ) - return headers, nil - } -} - -// Start transfers loop to load transfers for new blocks -func (c *loadBlocksAndTransfersCommand) startTransfersLoop(ctx context.Context) { - c.incLoops() - go func() { - defer gocommon.LogOnPanic() - defer func() { - c.decLoops() - }() - - logutils.ZapLogger().Debug("loadTransfersLoop start", zap.Uint64("chain", c.chainClient.NetworkID())) - - for { - select { - case <-ctx.Done(): - logutils.ZapLogger().Debug("startTransfersLoop done", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Error(ctx.Err()), - ) - return - case dbHeaders := <-c.blocksLoadedCh: - logutils.ZapLogger().Debug("loadTransfersOnDemand transfers received", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Int("headers", len(dbHeaders)), - ) - - blocksByAddress := map[common.Address][]*big.Int{} - // iterate over headers and group them by address - for _, dbHeader := range dbHeaders { - blocksByAddress[dbHeader.Address] = append(blocksByAddress[dbHeader.Address], dbHeader.Number) - } - - go func() { - defer gocommon.LogOnPanic() - _ = loadTransfers(ctx, c.blockDAO, c.db, c.chainClient, noBlockLimit, - blocksByAddress, c.pendingTxManager, c.tokenManager, c.feed) - }() - } - } - }() -} - -func newLoadBlocksAndTransfersCommand(accounts []common.Address, db *Database, accountsDB *accounts.Database, - blockDAO *BlockDAO, blockRangesSeqDAO BlockRangeDAOer, chainClient chain.ClientInterface, feed *event.Feed, - pendingTxManager *transactions.PendingTxTracker, - tokenManager *token.Manager, balanceCacher balance.Cacher, omitHistory bool, - blockChainState *blockchainstate.BlockChainState) *loadBlocksAndTransfersCommand { - - return &loadBlocksAndTransfersCommand{ - accounts: accounts, - db: db, - blockRangeDAO: blockRangesSeqDAO, - accountsDB: accountsDB, - blockDAO: blockDAO, - chainClient: chainClient, - feed: feed, - balanceCacher: balanceCacher, - pendingTxManager: pendingTxManager, - tokenManager: tokenManager, - blocksLoadedCh: make(chan []*DBHeader, 100), - omitHistory: omitHistory, - contractMaker: tokenManager.ContractMaker, - blockChainState: blockChainState, - } -} - -type loadBlocksAndTransfersCommand struct { - accounts []common.Address - db *Database - accountsDB *accounts.Database - blockRangeDAO BlockRangeDAOer - blockDAO *BlockDAO - chainClient chain.ClientInterface - feed *event.Feed - balanceCacher balance.Cacher - // nonArchivalRPCNode bool // TODO Make use of it - pendingTxManager *transactions.PendingTxTracker - tokenManager *token.Manager - blocksLoadedCh chan []*DBHeader - omitHistory bool - contractMaker *contracts.ContractMaker - blockChainState *blockchainstate.BlockChainState - - // Not to be set by the caller - transfersLoaded map[common.Address]bool // For event RecentHistoryReady to be sent only once per account during app lifetime - loops atomic.Int32 -} - -func (c *loadBlocksAndTransfersCommand) incLoops() { - c.loops.Add(1) -} - -func (c *loadBlocksAndTransfersCommand) decLoops() { - c.loops.Add(-1) -} - -func (c *loadBlocksAndTransfersCommand) isStarted() bool { - return c.loops.Load() > 0 -} - -func (c *loadBlocksAndTransfersCommand) Run(parent context.Context) (err error) { - logutils.ZapLogger().Debug("start load all transfers command", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Any("accounts", c.accounts), - ) - - // Finite processes (to be restarted on error, but stopped on success or context cancel): - // fetching transfers for loaded blocks - // fetching history blocks - - // Infinite processes (to be restarted on error), but stopped on context cancel: - // fetching new blocks - // fetching transfers for new blocks - - ctx := parent - finiteGroup := async.NewAtomicGroup(ctx) - finiteGroup.SetName("finiteGroup") - defer func() { - finiteGroup.Stop() - finiteGroup.Wait() - }() - - blockRanges, err := c.blockRangeDAO.getBlockRanges(c.chainClient.NetworkID(), c.accounts) - if err != nil { - return err - } - - firstScan := false - var headNum *big.Int - for _, address := range c.accounts { - blockRange, ok := blockRanges[address] - if !ok || blockRange.tokens.LastKnown == nil { - firstScan = true - break - } - - if headNum == nil || blockRange.tokens.LastKnown.Cmp(headNum) < 0 { - headNum = blockRange.tokens.LastKnown - } - } - - fromNum := big.NewInt(0) - if firstScan { - headNum, err = getHeadBlockNumber(ctx, c.chainClient) - if err != nil { - return err - } - } - - // It will start loadTransfersCommand which will run until all transfers from DB are loaded or any one failed to load - err = c.startFetchingTransfersForLoadedBlocks(finiteGroup) - if err != nil { - logutils.ZapLogger().Error("loadBlocksAndTransfersCommand fetchTransfersForLoadedBlocks", zap.Error(err)) - return err - } - - if !c.isStarted() { - c.startTransfersLoop(ctx) - c.startFetchingNewBlocks(ctx, c.accounts, headNum, c.blocksLoadedCh) - } - - // It will start findBlocksCommands which will run until success when all blocks are loaded - err = c.fetchHistoryBlocks(finiteGroup, c.accounts, fromNum, headNum, c.blocksLoadedCh) - if err != nil { - logutils.ZapLogger().Error("loadBlocksAndTransfersCommand fetchHistoryBlocks", zap.Error(err)) - return err - } - - select { - case <-ctx.Done(): - logutils.ZapLogger().Debug("loadBlocksAndTransfers command cancelled", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Stringers("accounts", c.accounts), - zap.Error(ctx.Err()), - ) - case <-finiteGroup.WaitAsync(): - err = finiteGroup.Error() // if there was an error, rerun the command - logutils.ZapLogger().Debug( - "end loadBlocksAndTransfers command", - zap.Uint64("chain", c.chainClient.NetworkID()), - zap.Stringers("accounts", c.accounts), - zap.String("group", finiteGroup.Name()), - zap.Error(err), - ) - } - - return err -} - -func (c *loadBlocksAndTransfersCommand) Runner(interval ...time.Duration) async.Runner { - // 30s - default interval for Infura's delay returned in error. That should increase chances - // for request to succeed with the next attempt for now until we have a proper retry mechanism - intvl := 30 * time.Second - if len(interval) > 0 { - intvl = interval[0] - } - - return async.FiniteCommand{ - Interval: intvl, - Runable: c.Run, - } -} - -func (c *loadBlocksAndTransfersCommand) Command(interval ...time.Duration) async.Command { - return c.Runner(interval...).Run -} - -func (c *loadBlocksAndTransfersCommand) fetchHistoryBlocks(group *async.AtomicGroup, accounts []common.Address, fromNum, toNum *big.Int, blocksLoadedCh chan []*DBHeader) (err error) { - for _, account := range accounts { - err = c.fetchHistoryBlocksForAccount(group, account, fromNum, toNum, c.blocksLoadedCh) - if err != nil { - return err - } - } - return nil -} - -func (c *loadBlocksAndTransfersCommand) fetchHistoryBlocksForAccount(group *async.AtomicGroup, account common.Address, fromNum, toNum *big.Int, blocksLoadedCh chan []*DBHeader) error { - logutils.ZapLogger().Debug("fetchHistoryBlocks start", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("account", account), - zap.Bool("omit", c.omitHistory), - ) - - if c.omitHistory { - blockRange := ðTokensBlockRanges{eth: &BlockRange{nil, big.NewInt(0), toNum}, tokens: &BlockRange{nil, big.NewInt(0), toNum}} - err := c.blockRangeDAO.upsertRange(c.chainClient.NetworkID(), account, blockRange) - logutils.ZapLogger().Error("fetchHistoryBlocks upsertRange", zap.Error(err)) - return err - } - - blockRange, err := loadBlockRangeInfo(c.chainClient.NetworkID(), account, c.blockRangeDAO) - if err != nil { - logutils.ZapLogger().Error("fetchHistoryBlocks loadBlockRangeInfo", zap.Error(err)) - return err - } - - ranges := [][]*big.Int{} - // There are 2 history intervals: - // 1) from 0 to FirstKnown - // 2) from LastKnown to `toNum`` (head) - // If we blockRange is nil, we need to load all blocks from `fromNum` to `toNum` - // As current implementation checks ETH first then tokens, tokens ranges maybe behind ETH ranges in - // cases when block searching was interrupted, so we use tokens ranges - if blockRange.tokens.LastKnown != nil || blockRange.tokens.FirstKnown != nil { - if blockRange.tokens.LastKnown != nil && toNum.Cmp(blockRange.tokens.LastKnown) > 0 { - ranges = append(ranges, []*big.Int{blockRange.tokens.LastKnown, toNum}) - } - - if blockRange.tokens.FirstKnown != nil { - if fromNum.Cmp(blockRange.tokens.FirstKnown) < 0 { - ranges = append(ranges, []*big.Int{fromNum, blockRange.tokens.FirstKnown}) - } else { - if !c.transfersLoaded[account] { - transfersLoaded, err := c.areAllTransfersLoaded(account) - if err != nil { - return err - } - - if transfersLoaded { - if c.transfersLoaded == nil { - c.transfersLoaded = make(map[common.Address]bool) - } - c.transfersLoaded[account] = true - c.notifyHistoryReady(account) - } - } - } - } - } else { - ranges = append(ranges, []*big.Int{fromNum, toNum}) - } - - if len(ranges) > 0 { - storage := rpclimiter.NewLimitsDBStorage(c.db.client) - limiter := rpclimiter.NewRequestLimiter(storage) - chainClient, _ := createChainClientWithLimiter(c.chainClient, account, limiter) - if chainClient == nil { - chainClient = c.chainClient - } - - for _, rangeItem := range ranges { - logutils.ZapLogger().Debug("range item", - zap.Stringers("r", rangeItem), - zap.Uint64("n", c.chainClient.NetworkID()), - zap.Stringer("a", account), - ) - - fbc := &findBlocksCommand{ - accounts: []common.Address{account}, - db: c.db, - accountsDB: c.accountsDB, - blockRangeDAO: c.blockRangeDAO, - chainClient: chainClient, - balanceCacher: c.balanceCacher, - feed: c.feed, - noLimit: false, - fromBlockNumber: rangeItem[0], - toBlockNumber: rangeItem[1], - tokenManager: c.tokenManager, - blocksLoadedCh: blocksLoadedCh, - defaultNodeBlockChunkSize: DefaultNodeBlockChunkSize, - } - group.Add(fbc.Command()) - } - } - - return nil -} - -func (c *loadBlocksAndTransfersCommand) startFetchingNewBlocks(ctx context.Context, addresses []common.Address, fromNum *big.Int, blocksLoadedCh chan<- []*DBHeader) { - logutils.ZapLogger().Debug("startFetchingNewBlocks start", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringers("accounts", addresses), - ) - - c.incLoops() - go func() { - defer gocommon.LogOnPanic() - defer func() { - c.decLoops() - }() - - newBlocksCmd := &findNewBlocksCommand{ - findBlocksCommand: &findBlocksCommand{ - accounts: addresses, - db: c.db, - accountsDB: c.accountsDB, - blockRangeDAO: c.blockRangeDAO, - chainClient: c.chainClient, - balanceCacher: c.balanceCacher, - feed: c.feed, - noLimit: false, - fromBlockNumber: fromNum, - tokenManager: c.tokenManager, - blocksLoadedCh: blocksLoadedCh, - defaultNodeBlockChunkSize: DefaultNodeBlockChunkSize, - }, - contractMaker: c.contractMaker, - blockChainState: c.blockChainState, - nonceCheckIntervalIterations: nonceCheckIntervalIterations, - logsCheckIntervalIterations: logsCheckIntervalIterations, - } - group := async.NewGroup(ctx) - group.Add(newBlocksCmd.Command()) - - // No need to wait for the group since it is infinite - <-ctx.Done() - - logutils.ZapLogger().Debug("startFetchingNewBlocks end", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringers("accounts", addresses), - zap.Error(ctx.Err()), - ) - }() -} - -func (c *loadBlocksAndTransfersCommand) getBlocksToLoad() (map[common.Address][]*big.Int, error) { - blocksMap := make(map[common.Address][]*big.Int) - for _, account := range c.accounts { - blocks, err := c.blockDAO.GetBlocksToLoadByAddress(c.chainClient.NetworkID(), account, numberOfBlocksCheckedPerIteration) - if err != nil { - logutils.ZapLogger().Error("loadBlocksAndTransfersCommand GetBlocksToLoadByAddress", zap.Error(err)) - return nil, err - } - - if len(blocks) == 0 { - logutils.ZapLogger().Debug("fetchTransfers no blocks to load", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringer("account", account), - ) - continue - } - - blocksMap[account] = blocks - } - - if len(blocksMap) == 0 { - logutils.ZapLogger().Debug("fetchTransfers no blocks to load", zap.Uint64("chainID", c.chainClient.NetworkID())) - } - - return blocksMap, nil -} - -func (c *loadBlocksAndTransfersCommand) startFetchingTransfersForLoadedBlocks(group *async.AtomicGroup) error { - logutils.ZapLogger().Debug("fetchTransfers start", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringers("accounts", c.accounts), - ) - - blocksMap, err := c.getBlocksToLoad() - if err != nil { - return err - } - - go func() { - defer gocommon.LogOnPanic() - txCommand := &loadTransfersCommand{ - accounts: c.accounts, - db: c.db, - blockDAO: c.blockDAO, - chainClient: c.chainClient, - pendingTxManager: c.pendingTxManager, - tokenManager: c.tokenManager, - blocksByAddress: blocksMap, - feed: c.feed, - } - - group.Add(txCommand.Command()) - logutils.ZapLogger().Debug("fetchTransfers end", - zap.Uint64("chainID", c.chainClient.NetworkID()), - zap.Stringers("accounts", c.accounts), - ) - }() - - return nil -} - -func (c *loadBlocksAndTransfersCommand) notifyHistoryReady(account common.Address) { - if c.feed != nil { - c.feed.Send(walletevent.Event{ - Type: EventRecentHistoryReady, - Accounts: []common.Address{account}, - ChainID: c.chainClient.NetworkID(), - }) - } -} - -func (c *loadBlocksAndTransfersCommand) areAllTransfersLoaded(account common.Address) (bool, error) { - allBlocksLoaded, err := areAllHistoryBlocksLoadedForAddress(c.blockRangeDAO, c.chainClient.NetworkID(), account) - if err != nil { - logutils.ZapLogger().Error("loadBlockAndTransfersCommand allHistoryBlocksLoaded", zap.Error(err)) - return false, err - } - - if allBlocksLoaded { - headers, err := c.blockDAO.GetBlocksToLoadByAddress(c.chainClient.NetworkID(), account, 1) - if err != nil { - logutils.ZapLogger().Error("loadBlocksAndTransfersCommand GetBlocksToLoadByAddress", zap.Error(err)) - return false, err - } - - if len(headers) == 0 { - return true, nil - } - } - - return false, nil -} - -// TODO - make it a common method for every service that wants head block number, that will cache the latest block -// and updates it on timeout -func getHeadBlockNumber(parent context.Context, chainClient chain.ClientInterface) (*big.Int, error) { - ctx, cancel := context.WithTimeout(parent, 3*time.Second) - head, err := chainClient.HeaderByNumber(ctx, nil) - cancel() - if err != nil { - logutils.ZapLogger().Error("getHeadBlockNumber", zap.Error(err)) - return nil, err - } - - return head.Number, err -} - -func nextRange(maxRangeSize int, prevFrom, zeroBlockNumber *big.Int) (*big.Int, *big.Int) { - logutils.ZapLogger().Debug("next range start", - zap.Stringer("from", prevFrom), - zap.Stringer("zeroBlockNumber", zeroBlockNumber), - ) - - rangeSize := big.NewInt(int64(maxRangeSize)) - - to := big.NewInt(0).Set(prevFrom) - from := big.NewInt(0).Sub(to, rangeSize) - if from.Cmp(zeroBlockNumber) < 0 { - from = new(big.Int).Set(zeroBlockNumber) - } - - logutils.ZapLogger().Debug("next range end", - zap.Stringer("from", from), - zap.Stringer("to", to), - zap.Stringer("zeroBlockNumber", zeroBlockNumber), - ) - - return from, to -} - -func accountLimiterTag(account common.Address) string { - return transferHistoryTag + "_" + account.String() -} - -func createChainClientWithLimiter(client chain.ClientInterface, account common.Address, limiter rpclimiter.RequestLimiter) (chain.ClientInterface, error) { - // Each account has its own limit and a global limit for all accounts - accountTag := accountLimiterTag(account) - chainClient := chain.ClientWithTag(client, accountTag, transferHistoryTag) - - // Check if limit is already reached, then skip the comamnd - if allow, err := limiter.Allow(accountTag); !allow { - logutils.ZapLogger().Info("fetchHistoryBlocksForAccount limit reached", - zap.Stringer("account", account), - zap.Uint64("chain", chainClient.NetworkID()), - zap.Error(err), - ) - return nil, err - } - - if allow, err := limiter.Allow(transferHistoryTag); !allow { - logutils.ZapLogger().Info("fetchHistoryBlocksForAccount common limit reached", - zap.Uint64("chain", chainClient.NetworkID()), - zap.Error(err), - ) - return nil, err - } - - limit, _ := limiter.GetLimit(accountTag) - if limit == nil { - err := limiter.SetLimit(accountTag, transferHistoryLimitPerAccount, rpclimiter.LimitInfinitely) - if err != nil { - logutils.ZapLogger().Error("fetchHistoryBlocksForAccount SetLimit", - zap.String("accountTag", accountTag), - zap.Error(err), - ) - } - } - - // Here total limit per day is overwriten on each app start, that still saves us RPC calls, but allows to proceed - // after app restart if the limit was reached. Currently there is no way to reset the limit from UI - err := limiter.SetLimit(transferHistoryTag, transferHistoryLimit, transferHistoryLimitPeriod) - if err != nil { - logutils.ZapLogger().Error("fetchHistoryBlocksForAccount SetLimit", - zap.String("groupTag", transferHistoryTag), - zap.Error(err), - ) - } - chainClient.SetLimiter(limiter) - - return chainClient, nil -} diff --git a/services/wallet/transfer/commands_sequential_test.go b/services/wallet/transfer/commands_sequential_test.go deleted file mode 100644 index c0c93c176e8..00000000000 --- a/services/wallet/transfer/commands_sequential_test.go +++ /dev/null @@ -1,1894 +0,0 @@ -package transfer - -import ( - "context" - "database/sql" - "fmt" - "math/big" - "slices" - "sort" - "strings" - "sync" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" - - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/rpc" - - accsmanagementtypes "github.com/status-im/status-go/accounts-management/types" - "github.com/status-im/status-go/appdatabase" - "github.com/status-im/status-go/contracts" - "github.com/status-im/status-go/contracts/balancechecker" - "github.com/status-im/status-go/contracts/ethscan" - "github.com/status-im/status-go/contracts/ierc20" - cryptotypes "github.com/status-im/status-go/crypto/types" - "github.com/status-im/status-go/healthmanager/rpcstatus" - "github.com/status-im/status-go/multiaccounts/accounts" - multicommon "github.com/status-im/status-go/multiaccounts/common" - "github.com/status-im/status-go/params" - statusRpc "github.com/status-im/status-go/rpc" - "github.com/status-im/status-go/rpc/chain/ethclient" - mock_client "github.com/status-im/status-go/rpc/chain/mock/client" - "github.com/status-im/status-go/rpc/chain/rpclimiter" - mock_rpcclient "github.com/status-im/status-go/rpc/mock/client" - "github.com/status-im/status-go/rpc/network" - "github.com/status-im/status-go/server" - "github.com/status-im/status-go/services/wallet/async" - "github.com/status-im/status-go/services/wallet/balance" - "github.com/status-im/status-go/services/wallet/blockchainstate" - walletcommon "github.com/status-im/status-go/services/wallet/common" - "github.com/status-im/status-go/services/wallet/community" - "github.com/status-im/status-go/services/wallet/token" - "github.com/status-im/status-go/services/wallet/token/token-lists/fetcher" - "github.com/status-im/status-go/t/helpers" - "github.com/status-im/status-go/t/utils" - "github.com/status-im/status-go/transactions" - "github.com/status-im/status-go/walletdatabase" -) - -type TestClient struct { - t *testing.T - // [][block, newBalance, nonceDiff] - balances map[common.Address][][]int - outgoingERC20Transfers map[common.Address][]testERC20Transfer - incomingERC20Transfers map[common.Address][]testERC20Transfer - outgoingERC1155SingleTransfers map[common.Address][]testERC20Transfer - incomingERC1155SingleTransfers map[common.Address][]testERC20Transfer - balanceHistory map[common.Address]map[uint64]*big.Int - tokenBalanceHistory map[common.Address]map[common.Address]map[uint64]*big.Int - nonceHistory map[common.Address]map[uint64]uint64 - traceAPICalls bool - printPreparedData bool - rw sync.RWMutex - callsCounter map[string]int - currentBlock uint64 - limiter rpclimiter.RequestLimiter - tag string - groupTag string -} - -var countAndlog = func(tc *TestClient, method string, params ...interface{}) error { - tc.incCounter(method) - if tc.traceAPICalls { - if len(params) > 0 { - tc.t.Log(method, params) - } else { - tc.t.Log(method) - } - } - - return nil -} - -func (tc *TestClient) countAndlog(method string, params ...interface{}) error { - return countAndlog(tc, method, params...) -} - -func (tc *TestClient) incCounter(method string) { - tc.rw.Lock() - defer tc.rw.Unlock() - tc.callsCounter[method] = tc.callsCounter[method] + 1 -} - -func (tc *TestClient) getCounter() int { - tc.rw.RLock() - defer tc.rw.RUnlock() - cnt := 0 - for _, v := range tc.callsCounter { - cnt += v - } - return cnt -} - -func (tc *TestClient) printCounter() { - total := tc.getCounter() - - tc.rw.RLock() - defer tc.rw.RUnlock() - - tc.t.Log("========================================= Total calls", total) - for k, v := range tc.callsCounter { - tc.t.Log(k, v) - } - tc.t.Log("=========================================") -} - -func (tc *TestClient) resetCounter() { - tc.rw.Lock() - defer tc.rw.Unlock() - tc.callsCounter = map[string]int{} -} - -func (tc *TestClient) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { - if tc.traceAPICalls { - tc.t.Log("BatchCallContext") - } - return nil -} - -func (tc *TestClient) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { - err := tc.countAndlog("HeaderByHash") - if err != nil { - return nil, err - } - return nil, nil -} - -func (tc *TestClient) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { - err := tc.countAndlog("BlockByHash") - if err != nil { - return nil, err - } - return nil, nil -} - -func (tc *TestClient) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { - err := tc.countAndlog("BlockByNumber") - if err != nil { - return nil, err - } - return nil, nil -} - -func (tc *TestClient) NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) { - nonce := tc.nonceHistory[account][blockNumber.Uint64()] - err := tc.countAndlog("NonceAt", fmt.Sprintf("result: %d", nonce)) - if err != nil { - return nonce, err - } - return nonce, nil -} - -func (tc *TestClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) { - err := tc.countAndlog("FilterLogs") - if err != nil { - return nil, err - } - - // We do not verify addresses for now - allTransfers := []testERC20Transfer{} - signatures := q.Topics[0] - erc20TransferSignature := walletcommon.GetEventSignatureHash(walletcommon.Erc20_721TransferEventSignature) - erc1155TransferSingleSignature := walletcommon.GetEventSignatureHash(walletcommon.Erc1155TransferSingleEventSignature) - - var address common.Hash - for i := 1; i < len(q.Topics); i++ { - if len(q.Topics[i]) > 0 { - address = q.Topics[i][0] - break - } - } - - if slices.Contains(signatures, erc1155TransferSingleSignature) { - from := q.Topics[2] - var to []common.Hash - if len(q.Topics) > 3 { - to = q.Topics[3] - } - - if len(to) > 0 { - for _, addressHash := range to { - address := &common.Address{} - address.SetBytes(addressHash.Bytes()) - allTransfers = append(allTransfers, tc.incomingERC1155SingleTransfers[*address]...) - } - } - if len(from) > 0 { - for _, addressHash := range from { - address := &common.Address{} - address.SetBytes(addressHash.Bytes()) - allTransfers = append(allTransfers, tc.outgoingERC1155SingleTransfers[*address]...) - } - } - } - - if slices.Contains(signatures, erc20TransferSignature) { - from := q.Topics[1] - to := q.Topics[2] - - if len(to) > 0 { - for _, addressHash := range to { - address := &common.Address{} - address.SetBytes(addressHash.Bytes()) - allTransfers = append(allTransfers, tc.incomingERC20Transfers[*address]...) - } - } - - if len(from) > 0 { - for _, addressHash := range from { - address := &common.Address{} - address.SetBytes(addressHash.Bytes()) - allTransfers = append(allTransfers, tc.outgoingERC20Transfers[*address]...) - } - } - } - - logs := []types.Log{} - for _, transfer := range allTransfers { - if transfer.block.Cmp(q.FromBlock) >= 0 && transfer.block.Cmp(q.ToBlock) <= 0 { - header := getTestHeader(transfer.block) - log := types.Log{ - BlockNumber: header.Number.Uint64(), - BlockHash: header.Hash(), - } - - // Use the address at least in one any(from/to) topic to trick the implementation - switch transfer.eventType { - case walletcommon.Erc20TransferEventType, walletcommon.Erc721TransferEventType: - // To detect properly ERC721, we need a different number of topics. For now we use only ERC20 for testing - log.Topics = []common.Hash{walletcommon.GetEventSignatureHash(walletcommon.Erc20_721TransferEventSignature), address, address} - case walletcommon.Erc1155TransferSingleEventType: - log.Topics = []common.Hash{walletcommon.GetEventSignatureHash(walletcommon.Erc1155TransferSingleEventSignature), address, address, address} - log.Data = make([]byte, 2*common.HashLength) - case walletcommon.Erc1155TransferBatchEventType: - log.Topics = []common.Hash{walletcommon.GetEventSignatureHash(walletcommon.Erc1155TransferBatchEventSignature), address, address, address} - } - - logs = append(logs, log) - } - } - - return logs, nil -} - -func (tc *TestClient) getBalance(address common.Address, blockNumber *big.Int) *big.Int { - balance := tc.balanceHistory[address][blockNumber.Uint64()] - if balance == nil { - balance = big.NewInt(0) - } - - return balance -} - -func (tc *TestClient) BalanceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (*big.Int, error) { - balance := tc.getBalance(account, blockNumber) - err := tc.countAndlog("BalanceAt", fmt.Sprintf("account: %s, result: %d", account, balance)) - if err != nil { - return nil, err - } - - return balance, nil -} - -func (tc *TestClient) tokenBalanceAt(account common.Address, token common.Address, blockNumber *big.Int) *big.Int { - balance := tc.tokenBalanceHistory[account][token][blockNumber.Uint64()] - if balance == nil { - balance = big.NewInt(0) - } - - if tc.traceAPICalls { - tc.t.Log("tokenBalanceAt", token, blockNumber, "account:", account, "result:", balance) - } - return balance -} - -func getTestHeader(number *big.Int) *types.Header { - return &types.Header{ - Number: big.NewInt(0).Set(number), - Time: 0, - Difficulty: big.NewInt(0), - ParentHash: common.Hash{}, - Nonce: types.BlockNonce{}, - MixDigest: common.Hash{}, - Extra: make([]byte, 0), - } -} - -func (tc *TestClient) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { - if number == nil { - number = big.NewInt(int64(tc.currentBlock)) - } - - err := tc.countAndlog("HeaderByNumber", fmt.Sprintf("number: %d", number)) - if err != nil { - return nil, err - } - - header := getTestHeader(number) - - return header, nil -} - -func (tc *TestClient) GetBaseFeeFromBlock(ctx context.Context, blockNumber *big.Int) (string, error) { - err := tc.countAndlog("GetBaseFeeFromBlock") - return "", err -} - -func (tc *TestClient) NetworkID() uint64 { - return walletcommon.TestnetChainID -} - -func (tc *TestClient) ToBigInt() *big.Int { - if tc.traceAPICalls { - tc.t.Log("ToBigInt") - } - return nil -} - -var ethscanAddress = common.HexToAddress("0x0000000000000000000000000000000000777333") -var balanceCheckAddress = common.HexToAddress("0x0000000000000000000000000000000010777333") - -func (tc *TestClient) CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) { - err := tc.countAndlog("CodeAt", fmt.Sprintf("contract: %s, blockNumber: %d", contract, blockNumber)) - if err != nil { - return nil, err - } - - if ethscanAddress == contract || balanceCheckAddress == contract { - return []byte{1}, nil - } - - return nil, nil -} - -func (tc *TestClient) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { - err := tc.countAndlog("CallContract", fmt.Sprintf("call: %v, blockNumber: %d, to: %s", call, blockNumber, call.To)) - if err != nil { - return nil, err - } - - if *call.To == ethscanAddress { - parsed, err := abi.JSON(strings.NewReader(ethscan.BalanceScannerABI)) - if err != nil { - return nil, err - } - method := parsed.Methods["tokensBalance"] - params := call.Data[len(method.ID):] - args, err := method.Inputs.Unpack(params) - - if err != nil { - tc.t.Log("ERROR on unpacking", err) - return nil, err - } - - account := args[0].(common.Address) - tokens := args[1].([]common.Address) - balances := []*big.Int{} - for _, token := range tokens { - balances = append(balances, tc.tokenBalanceAt(account, token, blockNumber)) - } - results := []ethscan.BalanceScannerResult{} - for _, balance := range balances { - results = append(results, ethscan.BalanceScannerResult{ - Success: true, - Data: balance.Bytes(), - }) - } - - output, err := method.Outputs.Pack(results) - if err != nil { - tc.t.Log("ERROR on packing", err) - return nil, err - } - - return output, nil - } - - if *call.To == tokenTXXAddress || *call.To == tokenTXYAddress { - parsed, err := abi.JSON(strings.NewReader(ierc20.IERC20ABI)) - if err != nil { - return nil, err - } - - method := parsed.Methods["balanceOf"] - params := call.Data[len(method.ID):] - args, err := method.Inputs.Unpack(params) - - if err != nil { - tc.t.Log("ERROR on unpacking", err) - return nil, err - } - - account := args[0].(common.Address) - - balance := tc.tokenBalanceAt(account, *call.To, blockNumber) - - output, err := method.Outputs.Pack(balance) - if err != nil { - tc.t.Log("ERROR on packing ERC20 balance", err) - return nil, err - } - - return output, nil - } - - if *call.To == balanceCheckAddress { - parsed, err := abi.JSON(strings.NewReader(balancechecker.BalanceCheckerABI)) - if err != nil { - return nil, err - } - - method := parsed.Methods["balancesHash"] - params := call.Data[len(method.ID):] - args, err := method.Inputs.Unpack(params) - - if err != nil { - tc.t.Log("ERROR on unpacking", err) - return nil, err - } - - addresses := args[0].([]common.Address) - tokens := args[1].([]common.Address) - bn := big.NewInt(int64(tc.currentBlock)) - hashes := [][32]byte{} - - for _, address := range addresses { - balance := tc.getBalance(address, big.NewInt(int64(tc.currentBlock))) - balanceBytes := balance.Bytes() - for _, token := range tokens { - balance := tc.tokenBalanceAt(address, token, bn) - balanceBytes = append(balanceBytes, balance.Bytes()...) - } - - hash := [32]byte{} - for i, b := range cryptotypes.BytesToHash(balanceBytes).Bytes() { - hash[i] = b - } - - hashes = append(hashes, hash) - } - - output, err := method.Outputs.Pack(bn, hashes) - if err != nil { - tc.t.Log("ERROR on packing", err) - return nil, err - } - - return output, nil - } - - return nil, nil -} - -func (tc *TestClient) prepareBalanceHistory(toBlock int) { - tc.balanceHistory = map[common.Address]map[uint64]*big.Int{} - tc.nonceHistory = map[common.Address]map[uint64]uint64{} - - for address, balances := range tc.balances { - var currentBlock, currentBalance, currentNonce int - - tc.balanceHistory[address] = map[uint64]*big.Int{} - tc.nonceHistory[address] = map[uint64]uint64{} - - if len(balances) == 0 { - balances = append(balances, []int{toBlock + 1, 0, 0}) - } else { - lastBlock := balances[len(balances)-1] - balances = append(balances, []int{toBlock + 1, lastBlock[1], 0}) - } - for _, change := range balances { - for blockN := currentBlock; blockN < change[0]; blockN++ { - tc.balanceHistory[address][uint64(blockN)] = big.NewInt(int64(currentBalance)) - tc.nonceHistory[address][uint64(blockN)] = uint64(currentNonce) - } - currentBlock = change[0] - currentBalance = change[1] - currentNonce += change[2] - } - } - - if tc.printPreparedData { - tc.t.Log("========================================= ETH BALANCES") - tc.t.Log(tc.balanceHistory) - tc.t.Log(tc.nonceHistory) - tc.t.Log(tc.tokenBalanceHistory) - tc.t.Log("=========================================") - } -} - -func (tc *TestClient) prepareTokenBalanceHistory(toBlock int) { - transfersPerAddress := map[common.Address]map[common.Address][]testERC20Transfer{} - for account, transfers := range tc.outgoingERC20Transfers { - if _, ok := transfersPerAddress[account]; !ok { - transfersPerAddress[account] = map[common.Address][]testERC20Transfer{} - } - for _, transfer := range transfers { - transfer.amount = new(big.Int).Neg(transfer.amount) - transfer.eventType = walletcommon.Erc20TransferEventType - transfersPerAddress[account][transfer.address] = append(transfersPerAddress[account][transfer.address], transfer) - } - } - - for account, transfers := range tc.incomingERC20Transfers { - if _, ok := transfersPerAddress[account]; !ok { - transfersPerAddress[account] = map[common.Address][]testERC20Transfer{} - } - for _, transfer := range transfers { - transfer.amount = new(big.Int).Neg(transfer.amount) - transfer.eventType = walletcommon.Erc20TransferEventType - transfersPerAddress[account][transfer.address] = append(transfersPerAddress[account][transfer.address], transfer) - } - } - - for account, transfers := range tc.outgoingERC1155SingleTransfers { - if _, ok := transfersPerAddress[account]; !ok { - transfersPerAddress[account] = map[common.Address][]testERC20Transfer{} - } - for _, transfer := range transfers { - transfer.amount = new(big.Int).Neg(transfer.amount) - transfer.eventType = walletcommon.Erc1155TransferSingleEventType - transfersPerAddress[account][transfer.address] = append(transfersPerAddress[account][transfer.address], transfer) - } - } - - for account, transfers := range tc.incomingERC1155SingleTransfers { - if _, ok := transfersPerAddress[account]; !ok { - transfersPerAddress[account] = map[common.Address][]testERC20Transfer{} - } - for _, transfer := range transfers { - transfer.amount = new(big.Int).Neg(transfer.amount) - transfer.eventType = walletcommon.Erc1155TransferSingleEventType - transfersPerAddress[account][transfer.address] = append(transfersPerAddress[account][transfer.address], transfer) - } - } - - tc.tokenBalanceHistory = map[common.Address]map[common.Address]map[uint64]*big.Int{} - - for account, transfersPerToken := range transfersPerAddress { - tc.tokenBalanceHistory[account] = map[common.Address]map[uint64]*big.Int{} - for token, transfers := range transfersPerToken { - sort.Slice(transfers, func(i, j int) bool { - return transfers[i].block.Cmp(transfers[j].block) < 0 - }) - - currentBlock := uint64(0) - currentBalance := big.NewInt(0) - - tc.tokenBalanceHistory[token] = map[common.Address]map[uint64]*big.Int{} - transfers = append(transfers, testERC20Transfer{big.NewInt(int64(toBlock + 1)), token, big.NewInt(0), walletcommon.Erc20TransferEventType}) - - tc.tokenBalanceHistory[account][token] = map[uint64]*big.Int{} - for _, transfer := range transfers { - for blockN := currentBlock; blockN < transfer.block.Uint64(); blockN++ { - tc.tokenBalanceHistory[account][token][blockN] = new(big.Int).Set(currentBalance) - } - currentBlock = transfer.block.Uint64() - currentBalance = new(big.Int).Add(currentBalance, transfer.amount) - } - } - } - if tc.printPreparedData { - tc.t.Log("========================================= ERC20 BALANCES") - tc.t.Log(tc.tokenBalanceHistory) - tc.t.Log("=========================================") - } -} - -func (tc *TestClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { - err := tc.countAndlog("CallContext") - return err -} - -func (tc *TestClient) EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { - err = tc.countAndlog("EstimateGas") - return 0, err -} - -func (tc *TestClient) PendingCodeAt(ctx context.Context, account common.Address) ([]byte, error) { - err := tc.countAndlog("PendingCodeAt") - return nil, err -} - -func (tc *TestClient) PendingCallContract(ctx context.Context, call ethereum.CallMsg) ([]byte, error) { - err := tc.countAndlog("PendingCallContract") - return nil, err -} - -func (tc *TestClient) PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) { - err := tc.countAndlog("PendingNonceAt") - return 0, err -} - -func (tc *TestClient) SuggestGasPrice(ctx context.Context) (*big.Int, error) { - err := tc.countAndlog("SuggestGasPrice") - return nil, err -} - -func (tc *TestClient) SendTransaction(ctx context.Context, tx *types.Transaction) error { - err := tc.countAndlog("SendTransaction") - return err -} - -func (tc *TestClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { - err := tc.countAndlog("SuggestGasTipCap") - return nil, err -} - -func (tc *TestClient) BatchCallContextIgnoringLocalHandlers(ctx context.Context, b []rpc.BatchElem) error { - err := tc.countAndlog("BatchCallContextIgnoringLocalHandlers") - return err -} - -func (tc *TestClient) CallContextIgnoringLocalHandlers(ctx context.Context, result interface{}, method string, args ...interface{}) error { - err := tc.countAndlog("CallContextIgnoringLocalHandlers") - return err -} - -func (tc *TestClient) CallRaw(data string) string { - _ = tc.countAndlog("CallRaw") - return "" -} - -func (tc *TestClient) GetChainID() *big.Int { - return big.NewInt(1) -} - -func (tc *TestClient) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) { - err := tc.countAndlog("SubscribeFilterLogs") - return nil, err -} - -func (tc *TestClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { - err := tc.countAndlog("TransactionReceipt") - return nil, err -} - -func (tc *TestClient) TransactionByHash(ctx context.Context, txHash common.Hash) (*types.Transaction, bool, error) { - err := tc.countAndlog("TransactionByHash") - return nil, false, err -} - -func (tc *TestClient) BlockNumber(ctx context.Context) (uint64, error) { - err := tc.countAndlog("BlockNumber") - return 0, err -} - -func (tc *TestClient) FeeHistory(ctx context.Context, blockCount uint64, lastBlock *big.Int, rewardPercentiles []float64) (*ethereum.FeeHistory, error) { - err := tc.countAndlog("FeeHistory") - if err != nil { - return nil, err - } - return nil, nil -} - -func (tc *TestClient) PendingBalanceAt(ctx context.Context, account common.Address) (*big.Int, error) { - err := tc.countAndlog("PendingBalanceAt") - return nil, err -} - -func (tc *TestClient) PendingStorageAt(ctx context.Context, account common.Address, key common.Hash) ([]byte, error) { - err := tc.countAndlog("PendingStorageAt") - return nil, err -} - -func (tc *TestClient) PendingTransactionCount(ctx context.Context) (uint, error) { - err := tc.countAndlog("PendingTransactionCount") - return 0, err -} - -func (tc *TestClient) StorageAt(ctx context.Context, account common.Address, key common.Hash, blockNumber *big.Int) ([]byte, error) { - err := tc.countAndlog("StorageAt") - return nil, err -} - -func (tc *TestClient) SyncProgress(ctx context.Context) (*ethereum.SyncProgress, error) { - err := tc.countAndlog("SyncProgress") - return nil, err -} - -func (tc *TestClient) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) { - err := tc.countAndlog("TransactionSender") - return common.Address{}, err -} - -func (tc *TestClient) SetIsConnected(value bool) { - if tc.traceAPICalls { - tc.t.Log("SetIsConnected") - } -} - -func (tc *TestClient) GetConnectionStatus() rpcstatus.StatusType { - if tc.traceAPICalls { - tc.t.Log("GetConnectionStatus") - } - - return rpcstatus.StatusUp -} - -func (tc *TestClient) GetLimiter() rpclimiter.RequestLimiter { - return tc.limiter -} - -func (tc *TestClient) SetLimiter(limiter rpclimiter.RequestLimiter) { - tc.limiter = limiter -} - -func (tc *TestClient) Close() { - if tc.traceAPICalls { - tc.t.Log("Close") - } -} - -func (tc *TestClient) GetProviderClient(provider string) ethclient.EthClientInterface { - return tc -} - -type testERC20Transfer struct { - block *big.Int - address common.Address - amount *big.Int - eventType walletcommon.EventType -} - -type findBlockCase struct { - balanceChanges [][]int - ERC20BalanceChanges [][]int - fromBlock int64 - toBlock int64 - rangeSize int - expectedBlocksFound int - outgoingERC20Transfers []testERC20Transfer - incomingERC20Transfers []testERC20Transfer - outgoingERC1155SingleTransfers []testERC20Transfer - incomingERC1155SingleTransfers []testERC20Transfer - label string - expectedCalls map[string]int -} - -func transferInEachBlock() [][]int { - res := [][]int{} - - for i := 1; i < 101; i++ { - res = append(res, []int{i, i, i}) - } - - return res -} - -func getCases() []findBlockCase { - cases := []findBlockCase{} - case1 := findBlockCase{ - balanceChanges: [][]int{ - {5, 1, 0}, - {20, 2, 0}, - {45, 1, 1}, - {46, 50, 0}, - {75, 0, 1}, - }, - outgoingERC20Transfers: []testERC20Transfer{ - {big.NewInt(6), tokenTXXAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}, - }, - toBlock: 100, - expectedBlocksFound: 6, - expectedCalls: map[string]int{ - "FilterLogs": 15, - "HeaderByNumber": 5, - }, - } - - case100transfers := findBlockCase{ - balanceChanges: transferInEachBlock(), - toBlock: 100, - expectedBlocksFound: 100, - expectedCalls: map[string]int{ - "BalanceAt": 101, - "NonceAt": 0, - "FilterLogs": 15, - "HeaderByNumber": 100, - }, - } - - case3 := findBlockCase{ - balanceChanges: [][]int{ - {1, 1, 1}, - {2, 2, 2}, - {45, 1, 1}, - {46, 50, 0}, - {75, 0, 1}, - }, - toBlock: 100, - expectedBlocksFound: 5, - } - case4 := findBlockCase{ - balanceChanges: [][]int{ - {20, 1, 0}, - }, - toBlock: 100, - fromBlock: 10, - expectedBlocksFound: 1, - label: "single block", - } - - case5 := findBlockCase{ - balanceChanges: [][]int{}, - toBlock: 100, - fromBlock: 20, - expectedBlocksFound: 0, - } - - case6 := findBlockCase{ - balanceChanges: [][]int{ - {20, 1, 0}, - {45, 1, 1}, - }, - toBlock: 100, - fromBlock: 30, - expectedBlocksFound: 1, - rangeSize: 20, - label: "single block in range", - } - - case7emptyHistoryWithOneERC20Transfer := findBlockCase{ - balanceChanges: [][]int{}, - toBlock: 100, - rangeSize: 20, - expectedBlocksFound: 1, - incomingERC20Transfers: []testERC20Transfer{ - {big.NewInt(6), tokenTXXAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}, - }, - } - - case8emptyHistoryWithERC20Transfers := findBlockCase{ - balanceChanges: [][]int{}, - toBlock: 100, - rangeSize: 20, - expectedBlocksFound: 2, - incomingERC20Transfers: []testERC20Transfer{ - // edge case when a regular scan will find transfer at 80, - // but erc20 tail scan should only find transfer at block 6 - {big.NewInt(80), tokenTXXAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}, - {big.NewInt(6), tokenTXXAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}, - }, - expectedCalls: map[string]int{ - "FilterLogs": 5, - "CallContract": 3, - }, - } - - case9emptyHistoryWithERC20Transfers := findBlockCase{ - balanceChanges: [][]int{}, - toBlock: 100, - rangeSize: 20, - // we expect only a single eth_getLogs to be executed here for both erc20 transfers, - // thus only 2 blocks found - expectedBlocksFound: 2, - incomingERC20Transfers: []testERC20Transfer{ - {big.NewInt(7), tokenTXYAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}, - {big.NewInt(6), tokenTXXAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}, - }, - expectedCalls: map[string]int{ - "FilterLogs": 5, - }, - } - - case10 := findBlockCase{ - balanceChanges: [][]int{}, - toBlock: 100, - fromBlock: 99, - expectedBlocksFound: 0, - label: "single block range, no transactions", - expectedCalls: map[string]int{ - // only two requests to check the range for incoming ERC20 - "FilterLogs": 3, - // no contract calls as ERC20 is not checked - "CallContract": 0, - }, - } - - case11IncomingERC1155SingleTransfers := findBlockCase{ - balanceChanges: [][]int{}, - toBlock: 100, - rangeSize: 20, - // we expect only a single eth_getLogs to be executed here for both erc20 transfers, - // thus only 2 blocks found - expectedBlocksFound: 2, - incomingERC1155SingleTransfers: []testERC20Transfer{ - {big.NewInt(7), tokenTXYAddress, big.NewInt(1), walletcommon.Erc1155TransferSingleEventType}, - {big.NewInt(6), tokenTXXAddress, big.NewInt(1), walletcommon.Erc1155TransferSingleEventType}, - }, - expectedCalls: map[string]int{ - "FilterLogs": 5, - "CallContract": 5, - }, - } - - case12OutgoingERC1155SingleTransfers := findBlockCase{ - balanceChanges: [][]int{ - {6, 1, 0}, - }, - toBlock: 100, - rangeSize: 20, - expectedBlocksFound: 3, - outgoingERC1155SingleTransfers: []testERC20Transfer{ - {big.NewInt(80), tokenTXYAddress, big.NewInt(1), walletcommon.Erc1155TransferSingleEventType}, - {big.NewInt(6), tokenTXXAddress, big.NewInt(1), walletcommon.Erc1155TransferSingleEventType}, - }, - expectedCalls: map[string]int{ - "FilterLogs": 15, // 3 for each range - }, - } - - case13outgoingERC20ERC1155SingleTransfers := findBlockCase{ - balanceChanges: [][]int{ - {63, 1, 0}, - }, - toBlock: 100, - rangeSize: 20, - expectedBlocksFound: 3, - outgoingERC1155SingleTransfers: []testERC20Transfer{ - {big.NewInt(80), tokenTXYAddress, big.NewInt(1), walletcommon.Erc1155TransferSingleEventType}, - }, - outgoingERC20Transfers: []testERC20Transfer{ - {big.NewInt(63), tokenTXYAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}, - }, - expectedCalls: map[string]int{ - "FilterLogs": 6, // 3 for each range, 0 for tail check becauseERC20ScanByBalance returns no ranges - }, - } - - case14outgoingERC20ERC1155SingleTransfersMoreFilterLogs := findBlockCase{ - balanceChanges: [][]int{ - {61, 1, 0}, - }, - toBlock: 100, - rangeSize: 20, - expectedBlocksFound: 3, - outgoingERC1155SingleTransfers: []testERC20Transfer{ - {big.NewInt(80), tokenTXYAddress, big.NewInt(1), walletcommon.Erc1155TransferSingleEventType}, - }, - outgoingERC20Transfers: []testERC20Transfer{ - {big.NewInt(61), tokenTXYAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}, - }, - expectedCalls: map[string]int{ - "FilterLogs": 9, // 3 for each range of [40-100], 0 for tail check because ERC20ScanByBalance returns no ranges - }, - label: "outgoing ERC20 and ERC1155 transfers but more FilterLogs calls because startFromBlock is not detected at range [60-80] as it is in the first subrange", - } - - case15incomingERC20outgoingERC1155SingleTransfers := findBlockCase{ - balanceChanges: [][]int{ - {85, 1, 0}, - }, - toBlock: 100, - rangeSize: 20, - expectedBlocksFound: 2, - outgoingERC1155SingleTransfers: []testERC20Transfer{ - {big.NewInt(85), tokenTXYAddress, big.NewInt(1), walletcommon.Erc1155TransferSingleEventType}, - }, - incomingERC20Transfers: []testERC20Transfer{ - {big.NewInt(88), tokenTXYAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}, - }, - expectedCalls: map[string]int{ - "FilterLogs": 3, // 3 for each range of [40-100], 0 for tail check because ERC20ScanByBalance returns no ranges - }, - label: "incoming ERC20 and outgoing ERC1155 transfers are fetched with same topic", - } - - case16 := findBlockCase{ - balanceChanges: [][]int{ - {75, 0, 1}, - }, - outgoingERC20Transfers: []testERC20Transfer{ - {big.NewInt(80), tokenTXXAddress, big.NewInt(4), walletcommon.Erc20TransferEventType}, - }, - toBlock: 100, - rangeSize: 20, - expectedBlocksFound: 3, // ideally we should find 2 blocks, but we will find 3 and this test shows that we are ok with that - label: `duplicate blocks detected but we wont fix it because we want to save requests on the edges of the ranges, - taking balance and nonce from cache while ETH and tokens ranges searching are tightly coupled`, - } - - cases = append(cases, case1) - cases = append(cases, case100transfers) - cases = append(cases, case3) - cases = append(cases, case4) - cases = append(cases, case5) - - cases = append(cases, case6) - cases = append(cases, case7emptyHistoryWithOneERC20Transfer) - cases = append(cases, case8emptyHistoryWithERC20Transfers) - cases = append(cases, case9emptyHistoryWithERC20Transfers) - cases = append(cases, case10) - cases = append(cases, case11IncomingERC1155SingleTransfers) - cases = append(cases, case12OutgoingERC1155SingleTransfers) - cases = append(cases, case13outgoingERC20ERC1155SingleTransfers) - cases = append(cases, case14outgoingERC20ERC1155SingleTransfersMoreFilterLogs) - cases = append(cases, case15incomingERC20outgoingERC1155SingleTransfers) - cases = append(cases, case16) - - //cases = append([]findBlockCase{}, case10) - - return cases -} - -var tokenTXXAddress = common.HexToAddress("0x53211") -var tokenTXYAddress = common.HexToAddress("0x73211") - -// #nosec G101 -const sequentialCommandsTestsListJsonData = `{ - "name": "Sequential Commands Tests List", - "timestamp": "2025-03-01T01:01:01.111Z", - "version": { - "major": 1, - "minor": 1, - "patch": 1 - }, - "tags": {}, - "logoURI": "", - "keywords": [], - "tokens": [ - { - "name": "Test Token 1", - "address": "0x0000000000000000000000000000000000053211", - "symbol": "TXX", - "decimals": 18, - "chainId": 777333 - }, - { - "name": "Test Token 2", - "address": "0x0000000000000000000000000000000000073211", - "symbol": "TXY", - "decimals": 18, - "chainId": 777333 - } - ] -}` - -func setupTestAppDB(t *testing.T) (*sql.DB, func()) { - db, cleanup, err := helpers.SetupTestSQLDB(appdatabase.DbInitializer{}, "app-tests") - require.NoError(t, err) - return db, func() { require.NoError(t, cleanup()) } -} - -func setupTestWalletDB(t *testing.T) (*sql.DB, func()) { - db, cleanup, err := helpers.SetupTestSQLDB(walletdatabase.DbInitializer{}, "wallet-tests") - require.NoError(t, err) - return db, func() { require.NoError(t, cleanup()) } -} - -func setupFindBlocksCommand(t *testing.T, accountAddress common.Address, fromBlock, toBlock *big.Int, rangeSize int, balances map[common.Address][][]int, outgoingERC20Transfers, incomingERC20Transfers, outgoingERC1155SingleTransfers, incomingERC1155SingleTransfers map[common.Address][]testERC20Transfer) (*findBlocksCommand, *TestClient, chan []*DBHeader, *BlockRangeSequentialDAO) { - var tokenManager *token.Manager - appDb, closeAppDb := setupTestAppDB(t) - walletDb, closeWalletDb := setupTestWalletDB(t) - t.Cleanup(func() { - if tokenManager != nil { - tokenManager.Stop() - } - closeAppDb() - closeWalletDb() - }) - - mediaServer, err := server.NewMediaServer(appDb, nil, nil, walletDb) - require.NoError(t, err) - - wdb := NewDB(walletDb) - tc := &TestClient{ - t: t, - balances: balances, - outgoingERC20Transfers: outgoingERC20Transfers, - incomingERC20Transfers: incomingERC20Transfers, - outgoingERC1155SingleTransfers: outgoingERC1155SingleTransfers, - incomingERC1155SingleTransfers: incomingERC1155SingleTransfers, - callsCounter: map[string]int{}, - } - // tc.traceAPICalls = true - // tc.printPreparedData = true - tc.prepareBalanceHistory(100) - tc.prepareTokenBalanceHistory(100) - blockChannel := make(chan []*DBHeader, 100) - - // Reimplement the common function that is called from every method to check for the limit - countAndlog = func(tc *TestClient, method string, params ...interface{}) error { - if tc.GetLimiter() != nil { - if allow, _ := tc.GetLimiter().Allow(tc.tag); !allow { - t.Log("ERROR: requests over limit") - return rpclimiter.ErrRequestsOverLimit - } - if allow, _ := tc.GetLimiter().Allow(tc.groupTag); !allow { - t.Log("ERROR: requests over limit for group tag") - return rpclimiter.ErrRequestsOverLimit - } - } - - tc.incCounter(method) - if tc.traceAPICalls { - if len(params) > 0 { - tc.t.Log(method, params) - } else { - tc.t.Log(method) - } - } - - return nil - } - - config := statusRpc.ClientConfig{ - UpstreamChainID: 1, - Networks: []params.Network{}, - DB: appDb, - } - client, err := statusRpc.NewClient(config) - require.NoError(t, err) - - client.SetClient(tc.NetworkID(), tc) - tokenManager = token.NewTokenManager(walletDb, client, community.NewManager(appDb, nil, nil), network.NewManager(appDb, nil), appDb, mediaServer, nil, nil, nil, token.NewPersistence(walletDb)) - - tokenListsFetcher := fetcher.NewTokenListsFetcher(walletDb) - err = tokenListsFetcher.StoreTokenList("sequential-commands-tests-list", "", "abcd", sequentialCommandsTestsListJsonData) - require.NoError(t, err) - - tokenManager.Start(context.Background(), time.Hour, time.Hour) - - accDB, err := accounts.NewDB(appDb) - require.NoError(t, err) - blockRangeDAO := &BlockRangeSequentialDAO{wdb.client} - fbc := &findBlocksCommand{ - accounts: []common.Address{accountAddress}, - db: wdb, - blockRangeDAO: blockRangeDAO, - accountsDB: accDB, - chainClient: tc, - balanceCacher: balance.NewCacherWithTTL(5 * time.Minute), - feed: &event.Feed{}, - noLimit: false, - fromBlockNumber: fromBlock, - toBlockNumber: toBlock, - blocksLoadedCh: blockChannel, - defaultNodeBlockChunkSize: rangeSize, - tokenManager: tokenManager, - } - return fbc, tc, blockChannel, blockRangeDAO -} - -func TestFindBlocksCommand(t *testing.T) { - for idx, testCase := range getCases() { - t.Log("case #", idx+1) - - accountAddress := common.HexToAddress("0x1234") - rangeSize := 20 - if testCase.rangeSize != 0 { - rangeSize = testCase.rangeSize - } - - balances := map[common.Address][][]int{accountAddress: testCase.balanceChanges} - outgoingERC20Transfers := map[common.Address][]testERC20Transfer{accountAddress: testCase.outgoingERC20Transfers} - incomingERC20Transfers := map[common.Address][]testERC20Transfer{accountAddress: testCase.incomingERC20Transfers} - outgoingERC1155SingleTransfers := map[common.Address][]testERC20Transfer{accountAddress: testCase.outgoingERC1155SingleTransfers} - incomingERC1155SingleTransfers := map[common.Address][]testERC20Transfer{accountAddress: testCase.incomingERC1155SingleTransfers} - - fbc, tc, blockChannel, blockRangeDAO := setupFindBlocksCommand(t, accountAddress, big.NewInt(testCase.fromBlock), big.NewInt(testCase.toBlock), rangeSize, balances, outgoingERC20Transfers, incomingERC20Transfers, outgoingERC1155SingleTransfers, incomingERC1155SingleTransfers) - ctx := context.Background() - group := async.NewGroup(ctx) - group.Add(fbc.Command()) - - foundBlocks := []*DBHeader{} - select { - case <-ctx.Done(): - t.Log("ERROR") - case <-group.WaitAsync(): - close(blockChannel) - for { - bloks, ok := <-blockChannel - if !ok { - break - } - foundBlocks = append(foundBlocks, bloks...) - } - - numbers := []int64{} - for _, block := range foundBlocks { - numbers = append(numbers, block.Number.Int64()) - } - - if tc.traceAPICalls { - tc.printCounter() - } - - for name, cnt := range testCase.expectedCalls { - require.Equal(t, cnt, tc.callsCounter[name], "calls to "+name) - } - - sort.Slice(numbers, func(i, j int) bool { return numbers[i] < numbers[j] }) - require.Equal(t, testCase.expectedBlocksFound, len(foundBlocks), testCase.label, "found blocks", numbers) - - blRange, _, err := blockRangeDAO.getBlockRange(tc.NetworkID(), accountAddress) - require.NoError(t, err) - require.NotNil(t, blRange.eth.FirstKnown) - require.NotNil(t, blRange.tokens.FirstKnown) - if testCase.fromBlock == 0 { - require.Equal(t, 0, blRange.tokens.FirstKnown.Cmp(zero)) - } - } - } -} - -func TestFindBlocksCommandWithLimiter(t *testing.T) { - maxRequests := 1 - rangeSize := 20 - accountAddress := common.HexToAddress("0x1234") - balances := map[common.Address][][]int{accountAddress: {{5, 1, 0}, {20, 2, 0}, {45, 1, 1}, {46, 50, 0}, {75, 0, 1}}} - fbc, tc, blockChannel, _ := setupFindBlocksCommand(t, accountAddress, big.NewInt(0), big.NewInt(20), rangeSize, balances, nil, nil, nil, nil) - - limiter := rpclimiter.NewRequestLimiter(rpclimiter.NewInMemRequestsMapStorage()) - err := limiter.SetLimit(transferHistoryTag, maxRequests, time.Hour) - require.NoError(t, err) - tc.SetLimiter(limiter) - tc.tag = transferHistoryTag - - ctx := context.Background() - group := async.NewAtomicGroup(ctx) - group.Add(fbc.Command(1 * time.Millisecond)) - - select { - case <-ctx.Done(): - t.Log("ERROR") - case <-group.WaitAsync(): - close(blockChannel) - require.Error(t, rpclimiter.ErrRequestsOverLimit, group.Error()) - require.Equal(t, maxRequests, tc.getCounter()) - } -} - -func TestFindBlocksCommandWithLimiterTagDifferentThanTransfers(t *testing.T) { - rangeSize := 20 - maxRequests := 1 - accountAddress := common.HexToAddress("0x1234") - balances := map[common.Address][][]int{accountAddress: {{5, 1, 0}, {20, 2, 0}, {45, 1, 1}, {46, 50, 0}, {75, 0, 1}}} - outgoingERC20Transfers := map[common.Address][]testERC20Transfer{accountAddress: {{big.NewInt(6), tokenTXXAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}}} - incomingERC20Transfers := map[common.Address][]testERC20Transfer{accountAddress: {{big.NewInt(6), tokenTXXAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}}} - - fbc, tc, blockChannel, _ := setupFindBlocksCommand(t, accountAddress, big.NewInt(0), big.NewInt(20), rangeSize, balances, outgoingERC20Transfers, incomingERC20Transfers, nil, nil) - limiter := rpclimiter.NewRequestLimiter(rpclimiter.NewInMemRequestsMapStorage()) - err := limiter.SetLimit("some-other-tag-than-transfer-history", maxRequests, time.Hour) - require.NoError(t, err) - tc.SetLimiter(limiter) - - ctx := context.Background() - group := async.NewAtomicGroup(ctx) - group.Add(fbc.Command(1 * time.Millisecond)) - - select { - case <-ctx.Done(): - t.Log("ERROR") - case <-group.WaitAsync(): - close(blockChannel) - require.NoError(t, group.Error()) - require.Greater(t, tc.getCounter(), maxRequests) - } -} - -func TestFindBlocksCommandWithLimiterForMultipleAccountsSameGroup(t *testing.T) { - rangeSize := 20 - maxRequestsTotal := 5 - limit1 := 3 - limit2 := 3 - account1 := common.HexToAddress("0x1234") - account2 := common.HexToAddress("0x5678") - balances := map[common.Address][][]int{account1: {{5, 1, 0}, {20, 2, 0}, {45, 1, 1}, {46, 50, 0}, {75, 0, 1}}, account2: {{5, 1, 0}, {20, 2, 0}, {45, 1, 1}, {46, 50, 0}, {75, 0, 1}}} - outgoingERC20Transfers := map[common.Address][]testERC20Transfer{account1: {{big.NewInt(6), tokenTXXAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}}} - incomingERC20Transfers := map[common.Address][]testERC20Transfer{account2: {{big.NewInt(6), tokenTXXAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}}} - - // Limiters share the same storage - storage := rpclimiter.NewInMemRequestsMapStorage() - - // Set up the first account - fbc, tc, blockChannel, _ := setupFindBlocksCommand(t, account1, big.NewInt(0), big.NewInt(20), rangeSize, balances, outgoingERC20Transfers, nil, nil, nil) - tc.tag = transferHistoryTag + account1.String() - tc.groupTag = transferHistoryTag - - limiter1 := rpclimiter.NewRequestLimiter(storage) - err := limiter1.SetLimit(transferHistoryTag, maxRequestsTotal, time.Hour) - require.NoError(t, err) - err = limiter1.SetLimit(transferHistoryTag+account1.String(), limit1, time.Hour) - require.NoError(t, err) - tc.SetLimiter(limiter1) - - // Set up the second account - fbc2, tc2, _, _ := setupFindBlocksCommand(t, account2, big.NewInt(0), big.NewInt(20), rangeSize, balances, nil, incomingERC20Transfers, nil, nil) - tc2.tag = transferHistoryTag + account2.String() - tc2.groupTag = transferHistoryTag - limiter2 := rpclimiter.NewRequestLimiter(storage) - err = limiter2.SetLimit(transferHistoryTag, maxRequestsTotal, time.Hour) - require.NoError(t, err) - err = limiter2.SetLimit(transferHistoryTag+account2.String(), limit2, time.Hour) - require.NoError(t, err) - tc2.SetLimiter(limiter2) - fbc2.blocksLoadedCh = blockChannel - - ctx := context.Background() - group := async.NewGroup(ctx) - group.Add(fbc.Command(1 * time.Millisecond)) - group.Add(fbc2.Command(1 * time.Millisecond)) - - select { - case <-ctx.Done(): - t.Log("ERROR") - case <-group.WaitAsync(): - close(blockChannel) - require.LessOrEqual(t, tc.getCounter(), maxRequestsTotal) - } -} - -type MockETHClient struct { - mock.Mock -} - -func (m *MockETHClient) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { - args := m.Called(ctx, b) - return args.Error(0) -} - -type MockChainClient struct { - mock.Mock - mock_client.MockClientInterface - - clients map[walletcommon.ChainID]*MockETHClient -} - -func newMockChainClient() *MockChainClient { - return &MockChainClient{ - clients: make(map[walletcommon.ChainID]*MockETHClient), - } -} - -func (m *MockChainClient) AbstractEthClient(chainID walletcommon.ChainID) (ethclient.BatchCallClient, error) { - if _, ok := m.clients[chainID]; !ok { - panic(fmt.Sprintf("no mock client for chainID %d", chainID)) - } - return m.clients[chainID], nil -} - -func TestFetchTransfersForLoadedBlocks(t *testing.T) { - appdb, err := helpers.SetupTestMemorySQLDB(appdatabase.DbInitializer{}) - require.NoError(t, err) - - db, err := helpers.SetupTestMemorySQLDB(walletdatabase.DbInitializer{}) - require.NoError(t, err) - - mediaServer, err := server.NewMediaServer(appdb, nil, nil, db) - require.NoError(t, err) - - wdb := NewDB(db) - blockChannel := make(chan []*DBHeader, 100) - - tc := &TestClient{ - t: t, - balances: map[common.Address][][]int{}, - outgoingERC20Transfers: map[common.Address][]testERC20Transfer{}, - incomingERC20Transfers: map[common.Address][]testERC20Transfer{}, - callsCounter: map[string]int{}, - currentBlock: 100, - } - - config := statusRpc.ClientConfig{ - UpstreamChainID: 1, - Networks: []params.Network{}, - DB: appdb, - } - client, _ := statusRpc.NewClient(config) - - client.SetClient(tc.NetworkID(), tc) - tokenManager := token.NewTokenManager(db, client, community.NewManager(appdb, nil, nil), network.NewManager(appdb, nil), appdb, mediaServer, nil, nil, nil, token.NewPersistence(db)) - - address := common.HexToAddress("0x1234") - chainClient := newMockChainClient() - ctrl := gomock.NewController(t) - defer ctrl.Finish() - rpcClient := mock_rpcclient.NewMockClientInterface(ctrl) - rpcClient.EXPECT().AbstractEthClient(tc.NetworkID()).Return(chainClient, nil).AnyTimes() - tracker := transactions.NewPendingTxTracker(db, rpcClient, &event.Feed{}, transactions.PendingCheckInterval) - accDB, err := accounts.NewDB(appdb) - require.NoError(t, err) - - cmd := &loadBlocksAndTransfersCommand{ - accounts: []common.Address{address}, - db: wdb, - blockRangeDAO: &BlockRangeSequentialDAO{wdb.client}, - blockDAO: &BlockDAO{db}, - accountsDB: accDB, - chainClient: tc, - feed: &event.Feed{}, - balanceCacher: balance.NewCacherWithTTL(5 * time.Minute), - pendingTxManager: tracker, - tokenManager: tokenManager, - blocksLoadedCh: blockChannel, - omitHistory: true, - contractMaker: tokenManager.ContractMaker, - } - - tc.prepareBalanceHistory(int(tc.currentBlock)) - tc.prepareTokenBalanceHistory(int(tc.currentBlock)) - // tc.traceAPICalls = true - - ctx := context.Background() - group := async.NewAtomicGroup(ctx) - - fromNum := big.NewInt(0) - toNum, err := getHeadBlockNumber(ctx, cmd.chainClient) - require.NoError(t, err) - err = cmd.fetchHistoryBlocksForAccount(group, address, fromNum, toNum, blockChannel) - require.NoError(t, err) - - select { - case <-ctx.Done(): - t.Log("ERROR") - case <-group.WaitAsync(): - require.Equal(t, 1, tc.getCounter()) - } -} - -func getNewBlocksCases() []findBlockCase { - cases := []findBlockCase{ - findBlockCase{ - balanceChanges: [][]int{ - {20, 1, 0}, - }, - fromBlock: 0, - toBlock: 10, - expectedBlocksFound: 0, - label: "single block, but not in range", - }, - findBlockCase{ - balanceChanges: [][]int{ - {20, 1, 0}, - }, - fromBlock: 10, - toBlock: 20, - expectedBlocksFound: 1, - label: "single block in range", - }, - } - - return cases -} - -func TestFetchNewBlocksCommand_findBlocksWithEthTransfers(t *testing.T) { - appdb, err := helpers.SetupTestMemorySQLDB(appdatabase.DbInitializer{}) - require.NoError(t, err) - - db, err := helpers.SetupTestMemorySQLDB(walletdatabase.DbInitializer{}) - require.NoError(t, err) - - mediaServer, err := server.NewMediaServer(appdb, nil, nil, db) - require.NoError(t, err) - - wdb := NewDB(db) - blockChannel := make(chan []*DBHeader, 10) - - address := common.HexToAddress("0x1234") - accDB, err := accounts.NewDB(appdb) - require.NoError(t, err) - - for idx, testCase := range getNewBlocksCases() { - t.Log("case #", idx+1) - tc := &TestClient{ - t: t, - balances: map[common.Address][][]int{address: testCase.balanceChanges}, - outgoingERC20Transfers: map[common.Address][]testERC20Transfer{}, - incomingERC20Transfers: map[common.Address][]testERC20Transfer{}, - callsCounter: map[string]int{}, - currentBlock: 100, - } - - config := statusRpc.ClientConfig{ - UpstreamChainID: 1, - Networks: []params.Network{}, - DB: appdb, - } - client, _ := statusRpc.NewClient(config) - - client.SetClient(tc.NetworkID(), tc) - tokenManager := token.NewTokenManager(db, client, community.NewManager(appdb, nil, nil), network.NewManager(appdb, nil), appdb, mediaServer, nil, nil, nil, token.NewPersistence(db)) - - cmd := &findNewBlocksCommand{ - findBlocksCommand: &findBlocksCommand{ - accounts: []common.Address{address}, - db: wdb, - accountsDB: accDB, - blockRangeDAO: &BlockRangeSequentialDAO{wdb.client}, - chainClient: tc, - balanceCacher: balance.NewCacherWithTTL(5 * time.Minute), - feed: &event.Feed{}, - noLimit: false, - tokenManager: tokenManager, - blocksLoadedCh: blockChannel, - defaultNodeBlockChunkSize: DefaultNodeBlockChunkSize, - }, - nonceCheckIntervalIterations: nonceCheckIntervalIterations, - logsCheckIntervalIterations: logsCheckIntervalIterations, - } - tc.prepareBalanceHistory(int(tc.currentBlock)) - tc.prepareTokenBalanceHistory(int(tc.currentBlock)) - - ctx := context.Background() - blocks, _, err := cmd.findBlocksWithEthTransfers(ctx, address, big.NewInt(testCase.fromBlock), big.NewInt(testCase.toBlock)) - require.NoError(t, err) - require.Equal(t, testCase.expectedBlocksFound, len(blocks), fmt.Sprintf("case %d: %s, blocks from %d to %d", idx+1, testCase.label, testCase.fromBlock, testCase.toBlock)) - } -} - -func TestFetchNewBlocksCommand_nonceDetection(t *testing.T) { - balanceChanges := [][]int{ - {5, 1, 0}, - {6, 0, 1}, - } - - scanRange := 5 - address := common.HexToAddress("0x1234") - - tc := &TestClient{ - t: t, - balances: map[common.Address][][]int{address: balanceChanges}, - outgoingERC20Transfers: map[common.Address][]testERC20Transfer{}, - incomingERC20Transfers: map[common.Address][]testERC20Transfer{}, - callsCounter: map[string]int{}, - currentBlock: 0, - } - - //tc.printPreparedData = true - tc.prepareBalanceHistory(20) - - appdb, err := helpers.SetupTestMemorySQLDB(appdatabase.DbInitializer{}) - require.NoError(t, err) - - db, err := helpers.SetupTestMemorySQLDB(walletdatabase.DbInitializer{}) - require.NoError(t, err) - - mediaServer, err := server.NewMediaServer(appdb, nil, nil, db) - require.NoError(t, err) - - config := statusRpc.ClientConfig{ - UpstreamChainID: 1, - Networks: []params.Network{}, - DB: appdb, - } - client, _ := statusRpc.NewClient(config) - - client.SetClient(tc.NetworkID(), tc) - tokenManager := token.NewTokenManager(db, client, community.NewManager(appdb, nil, nil), network.NewManager(appdb, nil), appdb, mediaServer, nil, nil, nil, token.NewPersistence(db)) - - wdb := NewDB(db) - blockChannel := make(chan []*DBHeader, 10) - - accDB, err := accounts.NewDB(appdb) - require.NoError(t, err) - - maker, _ := contracts.NewContractMaker(client) - - cmd := &findNewBlocksCommand{ - findBlocksCommand: &findBlocksCommand{ - accounts: []common.Address{address}, - db: wdb, - accountsDB: accDB, - blockRangeDAO: &BlockRangeSequentialDAO{wdb.client}, - chainClient: tc, - balanceCacher: balance.NewCacherWithTTL(5 * time.Minute), - feed: &event.Feed{}, - noLimit: false, - tokenManager: tokenManager, - blocksLoadedCh: blockChannel, - defaultNodeBlockChunkSize: scanRange, - fromBlockNumber: big.NewInt(0), - }, - blockChainState: blockchainstate.NewBlockChainState(), - contractMaker: maker, - nonceCheckIntervalIterations: 2, - logsCheckIntervalIterations: 2, - } - - acc := &accsmanagementtypes.Account{ - Address: cryptotypes.BytesToAddress(address.Bytes()), - Type: accsmanagementtypes.AccountTypeWatch, - Name: address.String(), - ColorID: multicommon.CustomizationColorPrimary, - Emoji: "emoji", - } - err = accDB.SaveOrUpdateAccounts([]*accsmanagementtypes.Account{acc}, false) - require.NoError(t, err) - - ctx := context.Background() - tc.currentBlock = 3 - for i := 0; i < 3; i++ { - err := cmd.Run(ctx) - require.NoError(t, err) - close(blockChannel) - - foundBlocks := []*DBHeader{} - for { - bloks, ok := <-blockChannel - if !ok { - break - } - foundBlocks = append(foundBlocks, bloks...) - } - - numbers := []int64{} - for _, block := range foundBlocks { - numbers = append(numbers, block.Number.Int64()) - } - if i == 2 { - require.Equal(t, 2, len(foundBlocks), "blocks", numbers) - } else { - require.Equal(t, 0, len(foundBlocks), "no blocks expected to be found") - } - blockChannel = make(chan []*DBHeader, 10) - cmd.blocksLoadedCh = blockChannel - tc.currentBlock += uint64(scanRange) - } -} - -func TestFetchNewBlocksCommand(t *testing.T) { - appdb, err := helpers.SetupTestMemorySQLDB(appdatabase.DbInitializer{}) - require.NoError(t, err) - - db, err := helpers.SetupTestMemorySQLDB(walletdatabase.DbInitializer{}) - require.NoError(t, err) - - mediaServer, err := server.NewMediaServer(appdb, nil, nil, db) - require.NoError(t, err) - - wdb := NewDB(db) - blockChannel := make(chan []*DBHeader, 10) - - address1 := common.HexToAddress("0x1234") - address2 := common.HexToAddress("0x5678") - accDB, err := accounts.NewDB(appdb) - require.NoError(t, err) - - for _, address := range []*common.Address{&address1, &address2} { - acc := &accsmanagementtypes.Account{ - Address: cryptotypes.BytesToAddress(address.Bytes()), - Type: accsmanagementtypes.AccountTypeWatch, - Name: address.String(), - ColorID: multicommon.CustomizationColorPrimary, - Emoji: "emoji", - } - err = accDB.SaveOrUpdateAccounts([]*accsmanagementtypes.Account{acc}, false) - require.NoError(t, err) - } - - tc := &TestClient{ - t: t, - balances: map[common.Address][][]int{}, - outgoingERC20Transfers: map[common.Address][]testERC20Transfer{}, - incomingERC20Transfers: map[common.Address][]testERC20Transfer{}, - callsCounter: map[string]int{}, - currentBlock: 1, - } - //tc.printPreparedData = true - - config := statusRpc.ClientConfig{ - UpstreamChainID: 1, - Networks: []params.Network{}, - DB: appdb, - } - client, _ := statusRpc.NewClient(config) - - client.SetClient(tc.NetworkID(), tc) - - tokenManager := token.NewTokenManager(db, client, community.NewManager(appdb, nil, nil), network.NewManager(appdb, nil), appdb, mediaServer, nil, nil, nil, token.NewPersistence(db)) - - cmd := &findNewBlocksCommand{ - findBlocksCommand: &findBlocksCommand{ - accounts: []common.Address{address1, address2}, - db: wdb, - accountsDB: accDB, - blockRangeDAO: &BlockRangeSequentialDAO{wdb.client}, - chainClient: tc, - balanceCacher: balance.NewCacherWithTTL(5 * time.Minute), - feed: &event.Feed{}, - noLimit: false, - fromBlockNumber: big.NewInt(int64(tc.currentBlock)), - tokenManager: tokenManager, - blocksLoadedCh: blockChannel, - defaultNodeBlockChunkSize: DefaultNodeBlockChunkSize, - }, - contractMaker: tokenManager.ContractMaker, - blockChainState: blockchainstate.NewBlockChainState(), - nonceCheckIntervalIterations: nonceCheckIntervalIterations, - logsCheckIntervalIterations: logsCheckIntervalIterations, - } - - ctx := context.Background() - - // I don't prepare lots of data and a loop, as I just need to verify a few cases - - // Verify that cmd.fromBlockNumber stays the same - tc.prepareBalanceHistory(int(tc.currentBlock)) - tc.prepareTokenBalanceHistory(int(tc.currentBlock)) - err = cmd.Run(ctx) - require.NoError(t, err) - require.Equal(t, uint64(1), cmd.fromBlockNumber.Uint64()) - - // Verify that cmd.fromBlockNumber is incremented, equal to the head block number - tc.currentBlock = 2 // this is the head block number that will be returned by the mock client - tc.prepareBalanceHistory(int(tc.currentBlock)) - tc.prepareTokenBalanceHistory(int(tc.currentBlock)) - err = cmd.Run(ctx) - require.NoError(t, err) - require.Equal(t, tc.currentBlock, cmd.fromBlockNumber.Uint64()) - - // Verify that blocks are found and cmd.fromBlockNumber is incremented - tc.resetCounter() - tc.currentBlock = 3 - tc.balances = map[common.Address][][]int{ - address1: {{3, 1, 0}}, - address2: {{3, 1, 0}}, - } - tc.incomingERC20Transfers = map[common.Address][]testERC20Transfer{ - address1: {{big.NewInt(3), tokenTXXAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}}, - address2: {{big.NewInt(3), tokenTXYAddress, big.NewInt(1), walletcommon.Erc20TransferEventType}}, - } - tc.prepareBalanceHistory(int(tc.currentBlock)) - tc.prepareTokenBalanceHistory(int(tc.currentBlock)) - - group := async.NewGroup(ctx) - group.Add(cmd.Command()) // This is an infinite command, I can't use WaitAsync() here to wait for it to finish - - expectedBlocksNumber := 3 // ETH block is found twice for each account as we don't handle addresses in MockClient. A block with ERC20 transfer is found once - blocksFound := 0 - stop := false - for stop == false { - select { - case <-ctx.Done(): - require.Fail(t, "context done") - stop = true - case <-blockChannel: - blocksFound++ - case <-time.After(100 * time.Millisecond): - stop = true - } - } - group.Stop() - group.Wait() - require.Equal(t, expectedBlocksNumber, blocksFound) - require.Equal(t, tc.currentBlock, cmd.fromBlockNumber.Uint64()) - // We must check all the logs for all accounts with a single iteration of eth_getLogs call - require.Equal(t, 3, tc.callsCounter["FilterLogs"], "calls to FilterLogs") -} - -type TestClientWithError struct { - *TestClient -} - -func (tc *TestClientWithError) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { - tc.incCounter("BlockByNumber") - if tc.traceAPICalls { - tc.t.Log("BlockByNumber", number) - } - - return nil, errors.New("Network error") -} - -type BlockRangeSequentialDAOMockError struct { - *BlockRangeSequentialDAO -} - -func (b *BlockRangeSequentialDAOMockError) getBlockRange(chainID uint64, address common.Address) (blockRange *ethTokensBlockRanges, exists bool, err error) { - return nil, true, errors.New("DB error") -} - -type BlockRangeSequentialDAOMockSuccess struct { - *BlockRangeSequentialDAO -} - -func (b *BlockRangeSequentialDAOMockSuccess) getBlockRange(chainID uint64, address common.Address) (blockRange *ethTokensBlockRanges, exists bool, err error) { - return newEthTokensBlockRanges(), true, nil -} - -func TestLoadBlocksAndTransfersCommand_FiniteFinishedInfiniteRunning(t *testing.T) { - appdb, err := helpers.SetupTestMemorySQLDB(appdatabase.DbInitializer{}) - require.NoError(t, err) - - db, err := helpers.SetupTestMemorySQLDB(walletdatabase.DbInitializer{}) - require.NoError(t, err) - - config := statusRpc.ClientConfig{ - UpstreamChainID: 1, - Networks: []params.Network{}, - DB: appdb, - } - client, _ := statusRpc.NewClient(config) - - maker, _ := contracts.NewContractMaker(client) - - wdb := NewDB(db) - tc := &TestClient{ - t: t, - callsCounter: map[string]int{}, - } - accDB, err := accounts.NewDB(appdb) - require.NoError(t, err) - - cmd := &loadBlocksAndTransfersCommand{ - accounts: []common.Address{common.HexToAddress("0x1234")}, - chainClient: tc, - blockDAO: &BlockDAO{db}, - blockRangeDAO: &BlockRangeSequentialDAOMockSuccess{ - &BlockRangeSequentialDAO{ - wdb.client, - }, - }, - accountsDB: accDB, - db: wdb, - contractMaker: maker, - } - - ctx, cancel := context.WithCancel(context.Background()) - group := async.NewGroup(ctx) - - group.Add(cmd.Command(1 * time.Millisecond)) - - select { - case <-ctx.Done(): - cancel() // linter is not happy if cancel is not called on all code paths - t.Log("Done") - case <-group.WaitAsync(): - require.True(t, cmd.isStarted()) - - // Test that it stops if canceled - cancel() - require.NoError(t, utils.Eventually(func() error { - if !cmd.isStarted() { - return nil - } - return errors.New("command is still running") - }, 100*time.Millisecond, 10*time.Millisecond)) - } -} - -func TestTransfersCommand_RetryAndQuitOnMaxError(t *testing.T) { - tc := &TestClientWithError{ - &TestClient{ - t: t, - callsCounter: map[string]int{}, - }, - } - - address := common.HexToAddress("0x1234") - cmd := &transfersCommand{ - chainClient: tc, - address: address, - eth: ÐDownloader{ - chainClient: tc, - accounts: []common.Address{address}, - }, - blockNums: []*big.Int{big.NewInt(1)}, - } - - ctx := context.Background() - group := async.NewGroup(ctx) - - runner := cmd.Runner(1 * time.Millisecond) - group.Add(runner.Run) - - select { - case <-ctx.Done(): - t.Log("Done") - case <-group.WaitAsync(): - errorCounter := runner.(async.FiniteCommandWithErrorCounter).ErrorCounter - require.Equal(t, errorCounter.MaxErrors(), tc.callsCounter["BlockByNumber"]) - - _, expectedErr := tc.BlockByNumber(context.TODO(), nil) - require.Error(t, expectedErr, errorCounter.Error()) - } -} diff --git a/services/wallet/transfer/concurrent.go b/services/wallet/transfer/concurrent.go deleted file mode 100644 index 7af2aabce2e..00000000000 --- a/services/wallet/transfer/concurrent.go +++ /dev/null @@ -1,298 +0,0 @@ -package transfer - -import ( - "context" - "math/big" - "sort" - "sync" - "time" - - "github.com/pkg/errors" - "go.uber.org/zap" - - "github.com/ethereum/go-ethereum/common" - "github.com/status-im/status-go/logutils" - "github.com/status-im/status-go/services/wallet/async" - "github.com/status-im/status-go/services/wallet/balance" -) - -const ( - NoThreadLimit uint32 = 0 - SequentialThreadLimit uint32 = 10 -) - -// NewConcurrentDownloader creates ConcurrentDownloader instance. -func NewConcurrentDownloader(ctx context.Context, limit uint32) *ConcurrentDownloader { - runner := async.NewQueuedAtomicGroup(ctx, limit) - result := &Result{} - return &ConcurrentDownloader{runner, result} -} - -type ConcurrentDownloader struct { - *async.QueuedAtomicGroup - *Result -} - -type Result struct { - mu sync.Mutex - transfers []Transfer - headers []*DBHeader - blockRanges [][]*big.Int -} - -var errDownloaderStuck = errors.New("eth downloader is stuck") - -func (r *Result) Push(transfers ...Transfer) { - r.mu.Lock() - defer r.mu.Unlock() - r.transfers = append(r.transfers, transfers...) -} - -func (r *Result) Get() []Transfer { - r.mu.Lock() - defer r.mu.Unlock() - rst := make([]Transfer, len(r.transfers)) - copy(rst, r.transfers) - return rst -} - -func (r *Result) PushHeader(block *DBHeader) { - r.mu.Lock() - defer r.mu.Unlock() - - r.headers = append(r.headers, block) -} - -func (r *Result) GetHeaders() []*DBHeader { - r.mu.Lock() - defer r.mu.Unlock() - rst := make([]*DBHeader, len(r.headers)) - copy(rst, r.headers) - return rst -} - -func (r *Result) PushRange(blockRange []*big.Int) { - r.mu.Lock() - defer r.mu.Unlock() - - r.blockRanges = append(r.blockRanges, blockRange) -} - -func (r *Result) GetRanges() [][]*big.Int { - r.mu.Lock() - defer r.mu.Unlock() - rst := make([][]*big.Int, len(r.blockRanges)) - copy(rst, r.blockRanges) - r.blockRanges = [][]*big.Int{} - - return rst -} - -// Downloader downloads transfers from single block using number. -type Downloader interface { - GetTransfersByNumber(context.Context, *big.Int) ([]Transfer, error) -} - -// Returns new block ranges that contain transfers and found block headers that contain transfers, and a block where -// beginning of trasfers history detected -func checkRangesWithStartBlock(parent context.Context, client balance.Reader, cache balance.Cacher, - account common.Address, ranges [][]*big.Int, threadLimit uint32, startBlock *big.Int) ( - resRanges [][]*big.Int, headers []*DBHeader, newStartBlock *big.Int, err error) { - - logutils.ZapLogger().Debug("start checkRanges", - zap.Stringer("account", account), - zap.Int("ranges len", len(ranges)), - zap.Stringer("startBlock", startBlock), - ) - - ctx, cancel := context.WithTimeout(parent, 30*time.Second) - defer cancel() - - c := NewConcurrentDownloader(ctx, threadLimit) - - newStartBlock = startBlock - - for _, blocksRange := range ranges { - from := blocksRange[0] - to := blocksRange[1] - - logutils.ZapLogger().Debug("check block range", - zap.Stringer("from", from), - zap.Stringer("to", to), - ) - - if startBlock != nil { - if to.Cmp(newStartBlock) <= 0 { - logutils.ZapLogger().Debug("'to' block is less than 'start' block", - zap.Stringer("to", to), - zap.Stringer("startBlock", startBlock), - ) - continue - } - } - - c.Add(func(ctx context.Context) error { - if from.Cmp(to) >= 0 { - logutils.ZapLogger().Debug("'from' block is greater than or equal to 'to' block", - zap.Stringer("from", from), - zap.Stringer("to", to), - ) - return nil - } - logutils.ZapLogger().Debug("eth transfers comparing blocks", - zap.Stringer("from", from), - zap.Stringer("to", to), - ) - - if startBlock != nil { - if to.Cmp(startBlock) <= 0 { - logutils.ZapLogger().Debug("'to' block is less than 'start' block", - zap.Stringer("to", to), - zap.Stringer("startBlock", startBlock), - ) - return nil - } - } - - lb, err := cache.BalanceAt(ctx, client, account, from) - if err != nil { - return err - } - hb, err := cache.BalanceAt(ctx, client, account, to) - if err != nil { - return err - } - if lb.Cmp(hb) == 0 { - logutils.ZapLogger().Debug("balances are equal", - zap.Stringer("from", from), - zap.Stringer("to", to), - zap.Stringer("lb", lb), - zap.Stringer("hb", hb), - ) - - hn, err := cache.NonceAt(ctx, client, account, to) - if err != nil { - return err - } - // if nonce is zero in a newer block then there is no need to check an older one - if *hn == 0 { - logutils.ZapLogger().Debug("zero nonce", zap.Stringer("to", to)) - - if hb.Cmp(big.NewInt(0)) == 0 { // balance is 0, nonce is 0, we stop checking further, that will be the start block (even though the real one can be a later one) - if startBlock != nil { - if to.Cmp(newStartBlock) > 0 { - logutils.ZapLogger().Debug("found possible start block, we should not search back", zap.Stringer("block", to)) - newStartBlock = to // increase newStartBlock if we found a new higher block - } - } else { - newStartBlock = to - } - } - - return nil - } - - ln, err := cache.NonceAt(ctx, client, account, from) - if err != nil { - return err - } - if *ln == *hn { - logutils.ZapLogger().Debug("transaction count is also equal", - zap.Stringer("from", from), - zap.Stringer("to", to), - zap.Int64p("ln", ln), - zap.Int64p("hn", hn), - ) - return nil - } - } - if new(big.Int).Sub(to, from).Cmp(one) == 0 { - header, err := client.HeaderByNumber(ctx, to) - if err != nil { - return err - } - c.PushHeader(toDBHeader(header, account)) - return nil - } - mid := new(big.Int).Add(from, to) - mid = mid.Div(mid, two) - _, err = cache.BalanceAt(ctx, client, account, mid) - if err != nil { - return err - } - logutils.ZapLogger().Debug("balances are not equal", - zap.Stringer("from", from), - zap.Stringer("mid", mid), - zap.Stringer("to", to), - ) - - c.PushRange([]*big.Int{mid, to}) - c.PushRange([]*big.Int{from, mid}) - return nil - }) - } - - select { - case <-c.WaitAsync(): - case <-ctx.Done(): - return nil, nil, nil, errDownloaderStuck - } - - if c.Error() != nil { - return nil, nil, nil, errors.Wrap(c.Error(), "failed to dowload transfers using concurrent downloader") - } - - logutils.ZapLogger().Debug("end checkRanges", - zap.Stringer("account", account), - zap.Stringer("newStartBlock", newStartBlock), - ) - return c.GetRanges(), c.GetHeaders(), newStartBlock, nil -} - -func findBlocksWithEthTransfers(parent context.Context, client balance.Reader, cache balance.Cacher, - account common.Address, low, high *big.Int, noLimit bool, threadLimit uint32) ( - from *big.Int, headers []*DBHeader, resStartBlock *big.Int, err error) { - - ranges := [][]*big.Int{{low, high}} - from = big.NewInt(low.Int64()) - headers = []*DBHeader{} - var lvl = 1 - - for len(ranges) > 0 && lvl <= 30 { - logutils.ZapLogger().Debug("check blocks ranges", - zap.Int("lvl", lvl), - zap.Int("ranges len", len(ranges)), - ) - lvl++ - // Check if there are transfers in blocks in ranges. To do that, nonce and balance is checked - // the block ranges that have transfers are returned - newRanges, newHeaders, strtBlock, err := checkRangesWithStartBlock(parent, client, cache, - account, ranges, threadLimit, resStartBlock) - resStartBlock = strtBlock - if err != nil { - return nil, nil, nil, err - } - - headers = append(headers, newHeaders...) - - if len(newRanges) > 0 { - logutils.ZapLogger().Debug("found new ranges", - zap.Stringer("account", account), - zap.Int("lvl", lvl), - zap.Int("new ranges len", len(newRanges)), - ) - } - if len(newRanges) > 60 && !noLimit { - sort.SliceStable(newRanges, func(i, j int) bool { - return newRanges[i][0].Cmp(newRanges[j][0]) == 1 - }) - - newRanges = newRanges[:60] - from = newRanges[len(newRanges)-1][0] - } - - ranges = newRanges - } - - return -} diff --git a/services/wallet/transfer/concurrent_test.go b/services/wallet/transfer/concurrent_test.go deleted file mode 100644 index 504201ec822..00000000000 --- a/services/wallet/transfer/concurrent_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package transfer - -import ( - "context" - "errors" - "math/big" - "sort" - "testing" - "time" - - "github.com/status-im/status-go/services/wallet/balance" - - "github.com/stretchr/testify/require" - - "github.com/ethereum/go-ethereum/core/types" - - "github.com/ethereum/go-ethereum/common" -) - -func TestConcurrentErrorInterrupts(t *testing.T) { - concurrent := NewConcurrentDownloader(context.Background(), NoThreadLimit) - var interrupted bool - concurrent.Add(func(ctx context.Context) error { - select { - case <-ctx.Done(): - interrupted = true - case <-time.After(10 * time.Second): - } - return nil - }) - err := errors.New("interrupt") - concurrent.Add(func(ctx context.Context) error { - return err - }) - concurrent.Wait() - require.True(t, interrupted) - require.Equal(t, err, concurrent.Error()) -} - -func TestConcurrentCollectsTransfers(t *testing.T) { - concurrent := NewConcurrentDownloader(context.Background(), NoThreadLimit) - concurrent.Add(func(context.Context) error { - concurrent.Push(Transfer{}) - return nil - }) - concurrent.Add(func(context.Context) error { - concurrent.Push(Transfer{}) - return nil - }) - concurrent.Wait() - require.Len(t, concurrent.Get(), 2) -} - -type balancesFixture []*big.Int - -func (f balancesFixture) BalanceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (*big.Int, error) { - index := int(blockNumber.Int64()) - if index > len(f)-1 { - return nil, errors.New("balance unknown") - } - return f[index], nil -} - -func (f balancesFixture) NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) { - return uint64(0), nil -} - -func (f balancesFixture) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { - return &types.Header{ - Number: number, - }, nil -} - -func (f balancesFixture) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { - return &types.Header{ - Number: big.NewInt(0), - }, nil -} - -func (f balancesFixture) NetworkID() uint64 { - return 0 -} - -type batchesFixture [][]Transfer - -func (f batchesFixture) GetTransfersByNumber(ctx context.Context, number *big.Int) (rst []Transfer, err error) { - index := int(number.Int64()) - if index > len(f)-1 { - return nil, errors.New("unknown block") - } - return f[index], nil -} - -func TestConcurrentEthDownloader(t *testing.T) { - type options struct { - balances balancesFixture - batches batchesFixture - result []DBHeader - last *big.Int - } - type testCase struct { - desc string - options options - } - for _, tc := range []testCase{ - { - desc: "NoBalances", - options: options{ - last: big.NewInt(3), - balances: balancesFixture{big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0)}, - }, - }, - { - desc: "LastBlock", - options: options{ - last: big.NewInt(3), - balances: balancesFixture{big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(10)}, - batches: batchesFixture{{}, {}, {}, {{BlockNumber: big.NewInt(3)}, {BlockNumber: big.NewInt(3)}}}, - result: []DBHeader{{Number: big.NewInt(3)}}, - }, - }, - { - desc: "ChangesInEveryBlock", - options: options{ - last: big.NewInt(3), - balances: balancesFixture{big.NewInt(0), big.NewInt(3), big.NewInt(7), big.NewInt(10)}, - batches: batchesFixture{{}, {{BlockNumber: big.NewInt(1)}}, {{BlockNumber: big.NewInt(2)}}, {{BlockNumber: big.NewInt(3)}}}, - result: []DBHeader{{Number: big.NewInt(1)}, {Number: big.NewInt(2)}, {Number: big.NewInt(3)}}, - }, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - concurrent := NewConcurrentDownloader(ctx, 0) - _, headers, _, _ := findBlocksWithEthTransfers( - ctx, tc.options.balances, balance.NewCacherWithTTL(5*time.Minute), - common.Address{}, zero, tc.options.last, false, NoThreadLimit) - concurrent.Wait() - require.NoError(t, concurrent.Error()) - rst := concurrent.Get() - require.Len(t, headers, len(tc.options.result)) - sort.Slice(rst, func(i, j int) bool { - return rst[i].BlockNumber.Cmp(rst[j].BlockNumber) < 0 - }) - /*for i := range rst { - require.Equal(t, tc.options.result[i].BlockNumber, rst[i].BlockNumber) - }*/ - }) - } -} diff --git a/services/wallet/transfer/controller.go b/services/wallet/transfer/controller.go deleted file mode 100644 index b8acb99f329..00000000000 --- a/services/wallet/transfer/controller.go +++ /dev/null @@ -1,241 +0,0 @@ -package transfer - -import ( - "context" - "database/sql" - "slices" - - "go.uber.org/zap" - - "github.com/ethereum/go-ethereum/common" - gocommon "github.com/status-im/status-go/common" - "github.com/status-im/status-go/logutils" - statusaccounts "github.com/status-im/status-go/multiaccounts/accounts" - "github.com/status-im/status-go/pkg/pubsub" - "github.com/status-im/status-go/rpc" - "github.com/status-im/status-go/rpc/chain/rpclimiter" - "github.com/status-im/status-go/rpc/network" - "github.com/status-im/status-go/services/accounts/accountsevent" - "github.com/status-im/status-go/services/wallet/blockchainstate" -) - -type Controller struct { - db *Database - accountsDB *statusaccounts.Database - rpcClient *rpc.Client - blockDAO *BlockDAO - blockRangesSeqDAO *BlockRangeSequentialDAO - reactor *Reactor - accountPublisher *pubsub.Publisher - transactionManager *TransactionManager - blockChainState *blockchainstate.BlockChainState - stopCh chan struct{} -} - -func NewTransferController(db *sql.DB, accountsDB *statusaccounts.Database, rpcClient *rpc.Client, accountPublisher *pubsub.Publisher, - transactionManager *TransactionManager, blockChainState *blockchainstate.BlockChainState) *Controller { - - blockDAO := &BlockDAO{db} - return &Controller{ - db: NewDB(db), - accountsDB: accountsDB, - blockDAO: blockDAO, - blockRangesSeqDAO: &BlockRangeSequentialDAO{db}, - rpcClient: rpcClient, - accountPublisher: accountPublisher, - transactionManager: transactionManager, - blockChainState: blockChainState, - } -} - -func (c *Controller) Start(ctx context.Context) { - c.stopCh = make(chan struct{}) - go func() { - defer gocommon.LogOnPanic() - _ = c.cleanupAccountsLeftovers() - }() - c.startAccountWatcher() - c.startNetworksWatcher() -} - -func (c *Controller) Stop() { - if c.stopCh != nil { - close(c.stopCh) - c.stopCh = nil - } - - if c.reactor != nil { - c.reactor.stop() - } -} - -func (c *Controller) startAccountWatcher() { - if c.accountPublisher == nil { - return - } - - chAdded, unsubAddedFn := pubsub.Subscribe[accountsevent.AccountsAddedEvent](c.accountPublisher, 10) - go func() { - defer gocommon.LogOnPanic() - defer unsubAddedFn() - for { - select { - case <-c.stopCh: - return - case _, ok := <-chAdded: - if !ok { - return - } - c.restartReactor() - } - } - }() - - chRemoved, unsubRemovedFn := pubsub.Subscribe[accountsevent.AccountsRemovedEvent](c.accountPublisher, 10) - go func() { - defer gocommon.LogOnPanic() - defer unsubRemovedFn() - for { - select { - case <-c.stopCh: - return - case _, ok := <-chRemoved: - if !ok { - return - } - c.restartReactor() - } - } - }() -} - -func (c *Controller) startNetworksWatcher() { - if c.rpcClient == nil { - return - } - - networksPublisher := c.rpcClient.GetNetworksPublisher() - if networksPublisher == nil { - return - } - - ch, unsubFn := pubsub.Subscribe[network.EventActiveNetworksChanged](networksPublisher, 10) - go func() { - defer gocommon.LogOnPanic() - defer unsubFn() - for { - select { - case <-c.stopCh: - return - case _, ok := <-ch: - if !ok { - return - } - c.restartReactor() - } - } - }() -} - -func (c *Controller) restartReactor() { - if c.reactor == nil { - logutils.ZapLogger().Warn("reactor is not initialized") - return - } - - currentEthAddresses, err := c.accountsDB.GetWalletAddresses() - - if err != nil { - logutils.ZapLogger().Error("failed getting wallet addresses", zap.Error(err)) - return - } - - currentAddresses := make([]common.Address, 0, len(currentEthAddresses)) - for _, ethAddress := range currentEthAddresses { - currentAddresses = append(currentAddresses, common.Address(ethAddress)) - } - - logutils.ZapLogger().Debug("list of accounts was changed from a previous version. reactor will be restarted", zap.Stringers("new", currentAddresses)) - - currentNetworks, err := c.rpcClient.GetNetworkManager().Get(false) - if err != nil { - logutils.ZapLogger().Error("failed getting active networks", zap.Error(err)) - return - } - - chainIDs := make([]uint64, 0, len(currentNetworks)) - for _, network := range currentNetworks { - chainIDs = append(chainIDs, network.ChainID) - } - - chainClients, err := c.rpcClient.EthClients(chainIDs) - if err != nil { - return - } - - err = c.reactor.restart(chainClients, currentAddresses) - if err != nil { - logutils.ZapLogger().Error("failed to restart reactor with new accounts", zap.Error(err)) - } -} - -func (c *Controller) cleanUpRemovedAccount(address common.Address) { - // Transfers will be deleted by foreign key constraint by cascade - err := deleteBlocks(c.db.client, address) - if err != nil { - logutils.ZapLogger().Error("Failed to delete blocks", zap.Error(err)) - } - err = deleteAllRanges(c.db.client, address) - if err != nil { - logutils.ZapLogger().Error("Failed to delete old blocks ranges", zap.Error(err)) - } - - err = c.blockRangesSeqDAO.deleteRange(address) - if err != nil { - logutils.ZapLogger().Error("Failed to delete blocks ranges sequential", zap.Error(err)) - } - - rpcLimitsStorage := rpclimiter.NewLimitsDBStorage(c.db.client) - err = rpcLimitsStorage.Delete(accountLimiterTag(address)) - if err != nil { - logutils.ZapLogger().Error("Failed to delete limits", zap.Error(err)) - } -} - -func (c *Controller) cleanupAccountsLeftovers() error { - // We clean up accounts that were deleted and soft removed - accounts, err := c.accountsDB.GetWalletAddresses() - if err != nil { - logutils.ZapLogger().Error("Failed to get accounts", zap.Error(err)) - return err - } - - existingAddresses := make([]common.Address, len(accounts)) - for i, account := range accounts { - existingAddresses[i] = (common.Address)(account) - } - - addressesInWalletDB, err := getAddresses(c.db.client) - if err != nil { - logutils.ZapLogger().Error("Failed to get addresses from wallet db", zap.Error(err)) - return err - } - - missing := findMissingItems(addressesInWalletDB, existingAddresses) - for _, address := range missing { - c.cleanUpRemovedAccount(address) - } - - return nil -} - -// find items from one slice that are not in another -func findMissingItems(slice1 []common.Address, slice2 []common.Address) []common.Address { - var missing []common.Address - for _, item := range slice1 { - if !slices.Contains(slice2, item) { - missing = append(missing, item) - } - } - return missing -} diff --git a/services/wallet/transfer/controller_test.go b/services/wallet/transfer/controller_test.go deleted file mode 100644 index 47328a95936..00000000000 --- a/services/wallet/transfer/controller_test.go +++ /dev/null @@ -1,248 +0,0 @@ -package transfer - -import ( - "context" - "math/big" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/ethereum/go-ethereum/common" - accsmanagementtypes "github.com/status-im/status-go/accounts-management/types" - "github.com/status-im/status-go/appdatabase" - "github.com/status-im/status-go/crypto/types" - "github.com/status-im/status-go/multiaccounts/accounts" - "github.com/status-im/status-go/pkg/pubsub" - "github.com/status-im/status-go/services/accounts/accountsevent" - "github.com/status-im/status-go/services/wallet/blockchainstate" - "github.com/status-im/status-go/t/helpers" - "github.com/status-im/status-go/walletdatabase" -) - -func TestController_watchAccountsChanges(t *testing.T) { - appDB, err := helpers.SetupTestMemorySQLDB(appdatabase.DbInitializer{}) - require.NoError(t, err) - - accountsDB, err := accounts.NewDB(appDB) - require.NoError(t, err) - - walletDB, err := helpers.SetupTestMemorySQLDB(walletdatabase.DbInitializer{}) - require.NoError(t, err) - - accountsPublisher := pubsub.NewPublisher() - - bcstate := blockchainstate.NewBlockChainState() - transactionManager := NewTransactionManager(nil, nil, nil, accountsDB, nil, nil) - c := NewTransferController( - walletDB, - accountsDB, - nil, // rpcClient - accountsPublisher, - transactionManager, // transactionManager - bcstate, - ) - - address := common.HexToAddress("0x1234") - chainID := uint64(777) - // Insert blocks - database := NewDB(walletDB) - err = database.SaveBlocks(chainID, []*DBHeader{ - { - Number: big.NewInt(1), - Hash: common.Hash{1}, - Network: chainID, - Address: address, - Loaded: false, - }, - }) - require.NoError(t, err) - - // Insert transfers - err = saveTransfersMarkBlocksLoaded(walletDB, chainID, address, []Transfer{ - { - ID: common.Hash{1}, - BlockHash: common.Hash{1}, - BlockNumber: big.NewInt(1), - Address: address, - NetworkID: chainID, - }, - }, []*big.Int{big.NewInt(1)}) - require.NoError(t, err) - - // Insert block ranges - blockRangesDAO := &BlockRangeSequentialDAO{walletDB} - err = blockRangesDAO.upsertRange(chainID, address, newEthTokensBlockRanges()) - require.NoError(t, err) - - ranges, _, err := blockRangesDAO.getBlockRange(chainID, address) - require.NoError(t, err) - require.NotNil(t, ranges) - - // Insert multitransactions - // Save address to accounts DB which transactions we want to preserve - counterparty := common.Address{0x1} - err = accountsDB.SaveOrUpdateAccounts([]*accsmanagementtypes.Account{ - {Address: types.Address(counterparty), Chat: false, Wallet: true}, - }, false) - require.NoError(t, err) - - c.Start(context.Background()) - - // Start watching accounts - wg := sync.WaitGroup{} - wg.Add(1) - - ch, unsubFn := pubsub.Subscribe[accountsevent.AccountsRemovedEvent](accountsPublisher, 10) - go func() { - _, ok := <-ch - if !ok { - return - } - defer wg.Done() - - time.Sleep(1 * time.Millisecond) - // Wait for DB to be cleaned up - - // Check that transfers, blocks and block ranges were deleted - transfers, err := database.GetTransfersByAddress(chainID, address, big.NewInt(2), 1) - require.NoError(t, err) - require.Len(t, transfers, 0) - - blocksDAO := &BlockDAO{walletDB} - block, err := blocksDAO.GetLastBlockByAddress(chainID, address, 1) - require.NoError(t, err) - require.Nil(t, block) - - ranges, _, err = blockRangesDAO.getBlockRange(chainID, address) - require.NoError(t, err) - require.Nil(t, ranges.eth.FirstKnown) - require.Nil(t, ranges.eth.LastKnown) - require.Nil(t, ranges.eth.Start) - require.Nil(t, ranges.tokens.FirstKnown) - require.Nil(t, ranges.tokens.LastKnown) - require.Nil(t, ranges.tokens.Start) - - }() - defer unsubFn() - - // Watching accounts must start before sending event. - // To avoid running goroutine immediately and let the controller subscribe first, - // use any delay. - go func() { - time.Sleep(1 * time.Millisecond) - - pubsub.Publish(accountsPublisher, accountsevent.AccountsRemovedEvent{ - Accounts: []common.Address{address}, - }) - }() - - wg.Wait() -} - -func TestController_cleanupAccountLeftovers(t *testing.T) { - appDB, err := helpers.SetupTestMemorySQLDB(appdatabase.DbInitializer{}) - require.NoError(t, err) - - walletDB, err := helpers.SetupTestMemorySQLDB(walletdatabase.DbInitializer{}) - require.NoError(t, err) - - accountsDB, err := accounts.NewDB(appDB) - require.NoError(t, err) - - removedAddr := common.HexToAddress("0x5678") - existingAddr := types.HexToAddress("0x1234") - accounts := []*accsmanagementtypes.Account{ - {Address: existingAddr, Chat: false, Wallet: true}, - } - err = accountsDB.SaveOrUpdateAccounts(accounts, false) - require.NoError(t, err) - - storedAccs, err := accountsDB.GetWalletAddresses() - require.NoError(t, err) - require.Len(t, storedAccs, 1) - - transactionManager := NewTransactionManager(nil, nil, nil, accountsDB, nil, nil) - bcstate := blockchainstate.NewBlockChainState() - c := NewTransferController( - walletDB, - accountsDB, - nil, // rpcClient - nil, - transactionManager, // transactionManager - bcstate, - ) - chainID := uint64(777) - // Insert blocks - database := NewDB(walletDB) - err = database.SaveBlocks(chainID, []*DBHeader{ - { - Number: big.NewInt(1), - Hash: common.Hash{1}, - Network: chainID, - Address: removedAddr, - Loaded: false, - }, - }) - require.NoError(t, err) - err = database.SaveBlocks(chainID, []*DBHeader{ - { - Number: big.NewInt(2), - Hash: common.Hash{2}, - Network: chainID, - Address: common.Address(existingAddr), - Loaded: false, - }, - }) - require.NoError(t, err) - - blocksDAO := &BlockDAO{walletDB} - block, err := blocksDAO.GetLastBlockByAddress(chainID, removedAddr, 1) - require.NoError(t, err) - require.NotNil(t, block) - block, err = blocksDAO.GetLastBlockByAddress(chainID, common.Address(existingAddr), 1) - require.NoError(t, err) - require.NotNil(t, block) - - // Insert transfers - err = saveTransfersMarkBlocksLoaded(walletDB, chainID, removedAddr, []Transfer{ - { - ID: common.Hash{1}, - BlockHash: common.Hash{1}, - BlockNumber: big.NewInt(1), - Address: removedAddr, - NetworkID: chainID, - }, - }, []*big.Int{big.NewInt(1)}) - require.NoError(t, err) - - err = saveTransfersMarkBlocksLoaded(walletDB, chainID, common.Address(existingAddr), []Transfer{ - { - ID: common.Hash{2}, - BlockHash: common.Hash{2}, - BlockNumber: big.NewInt(2), - Address: common.Address(existingAddr), - NetworkID: chainID, - }, - }, []*big.Int{big.NewInt(2)}) - require.NoError(t, err) - - err = c.cleanupAccountsLeftovers() - require.NoError(t, err) - - // Check that transfers and blocks of removed account were deleted - transfers, err := database.GetTransfers(chainID, big.NewInt(1), big.NewInt(2)) - require.NoError(t, err) - require.Len(t, transfers, 1) - require.Equal(t, transfers[0].Address, common.Address(existingAddr)) - - block, err = blocksDAO.GetLastBlockByAddress(chainID, removedAddr, 1) - require.NoError(t, err) - require.Nil(t, block) - - // Make sure that transfers and blocks of existing account were not deleted - existingBlock, err := blocksDAO.GetLastBlockByAddress(chainID, common.Address(existingAddr), 1) - require.NoError(t, err) - require.NotNil(t, existingBlock) -} diff --git a/services/wallet/transfer/database.go b/services/wallet/transfer/database.go deleted file mode 100644 index 9a69915b57a..00000000000 --- a/services/wallet/transfer/database.go +++ /dev/null @@ -1,621 +0,0 @@ -package transfer - -import ( - "context" - "database/sql" - "database/sql/driver" - "encoding/json" - "errors" - "math/big" - "reflect" - - "go.uber.org/zap" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - - "github.com/status-im/status-go/logutils" - ac "github.com/status-im/status-go/services/wallet/activity/common" - "github.com/status-im/status-go/services/wallet/bigint" - w_common "github.com/status-im/status-go/services/wallet/common" - "github.com/status-im/status-go/services/wallet/thirdparty" - "github.com/status-im/status-go/sqlite" -) - -// DBHeader fields from header that are stored in database. -type DBHeader struct { - Number *big.Int - Hash common.Hash - Timestamp uint64 - PreloadedTransactions []*PreloadedTransaction - Network uint64 - Address common.Address - // Head is true if the block was a head at the time it was pulled from chain. - Head bool - // Loaded is true if transfers from this block have been already fetched - Loaded bool -} - -func toDBHeader(header *types.Header, account common.Address) *DBHeader { - return &DBHeader{ - Hash: header.Hash(), - Number: header.Number, - Timestamp: header.Time, - Loaded: false, - Address: account, - } -} - -// SyncOption is used to specify that application processed transfers for that block. -type SyncOption uint - -// JSONBlob type for marshaling/unmarshaling inner type to json. -type JSONBlob struct { - data interface{} -} - -// Scan implements interface. -func (blob *JSONBlob) Scan(value interface{}) error { - if value == nil || reflect.ValueOf(blob.data).IsNil() { - return nil - } - bytes, ok := value.([]byte) - if !ok { - return errors.New("not a byte slice") - } - if len(bytes) == 0 { - return nil - } - err := json.Unmarshal(bytes, blob.data) - return err -} - -// Value implements interface. -func (blob *JSONBlob) Value() (driver.Value, error) { - if blob.data == nil || reflect.ValueOf(blob.data).IsNil() { - return nil, nil - } - return json.Marshal(blob.data) -} - -func NewDB(client *sql.DB) *Database { - return &Database{client: client} -} - -// Database sql wrapper for operations with wallet objects. -type Database struct { - client *sql.DB -} - -// Close closes database. -func (db *Database) Close() error { - return db.client.Close() -} - -func (db *Database) SaveBlocks(chainID uint64, headers []*DBHeader) (err error) { - var ( - tx *sql.Tx - ) - tx, err = db.client.Begin() - if err != nil { - return err - } - defer func() { - if err == nil { - err = tx.Commit() - return - } - _ = tx.Rollback() - }() - - err = insertBlocksWithTransactions(chainID, tx, headers) - if err != nil { - return - } - - return -} - -func saveTransfersMarkBlocksLoaded(creator statementCreator, chainID uint64, address common.Address, transfers []Transfer, blocks []*big.Int) (err error) { - err = updateOrInsertTransfers(chainID, creator, transfers) - if err != nil { - return - } - - err = markBlocksAsLoaded(chainID, creator, address, blocks) - if err != nil { - return - } - - return -} - -// GetTransfersInRange loads transfers for a given address between two blocks. -func (db *Database) GetTransfersInRange(chainID uint64, address common.Address, start, end *big.Int) (rst []Transfer, err error) { - query := newTransfersQuery().FilterNetwork(chainID).FilterAddress(address).FilterStart(start).FilterEnd(end).FilterLoaded(1) - rows, err := db.client.Query(query.String(), query.Args()...) - if err != nil { - return - } - defer rows.Close() - return query.TransferScan(rows) -} - -// GetTransfersByAddress loads transfers for a given address between two blocks. -func (db *Database) GetTransfersByAddress(chainID uint64, address common.Address, toBlock *big.Int, limit int64) (rst []Transfer, err error) { - query := newTransfersQuery(). - FilterNetwork(chainID). - FilterAddress(address). - FilterEnd(toBlock). - FilterLoaded(1). - SortByBlockNumberAndHash(). - Limit(limit) - - rows, err := db.client.Query(query.String(), query.Args()...) - if err != nil { - return - } - defer rows.Close() - return query.TransferScan(rows) -} - -// GetTransfersByAddressAndBlock loads transfers for a given address and block. -func (db *Database) GetTransfersByAddressAndBlock(chainID uint64, address common.Address, block *big.Int, limit int64) (rst []Transfer, err error) { - query := newTransfersQuery(). - FilterNetwork(chainID). - FilterAddress(address). - FilterBlockNumber(block). - FilterLoaded(1). - SortByBlockNumberAndHash(). - Limit(limit) - - rows, err := db.client.Query(query.String(), query.Args()...) - if err != nil { - return - } - defer rows.Close() - return query.TransferScan(rows) -} - -// GetTransfers load transfers transfer between two blocks. -func (db *Database) GetTransfers(chainID uint64, start, end *big.Int) (rst []Transfer, err error) { - query := newTransfersQuery().FilterNetwork(chainID).FilterStart(start).FilterEnd(end).FilterLoaded(1) - rows, err := db.client.Query(query.String(), query.Args()...) - if err != nil { - return - } - defer rows.Close() - return query.TransferScan(rows) -} - -func (db *Database) GetTransfersForIdentities(ctx context.Context, identities []ac.TransactionIdentity) (rst []Transfer, err error) { - query := newTransfersQuery() - for _, identity := range identities { - subQuery := newSubQuery() - subQuery = subQuery.FilterNetwork(uint64(identity.ChainID)).FilterTransactionID(identity.Hash).FilterAddress(identity.Address) - query.addSubQuery(subQuery, OrSeparator) - } - rows, err := db.client.QueryContext(ctx, query.String(), query.Args()...) - if err != nil { - return - } - defer rows.Close() - return query.TransferScan(rows) -} - -func (db *Database) GetTransactionsToLoad(chainID uint64, address common.Address, blockNumber *big.Int) (rst []*PreloadedTransaction, err error) { - query := newTransfersQueryForPreloadedTransactions(). - FilterNetwork(chainID). - FilterLoaded(0) - - if address != (common.Address{}) { - query.FilterAddress(address) - } - - if blockNumber != nil { - query.FilterBlockNumber(blockNumber) - } - - rows, err := db.client.Query(query.String(), query.Args()...) - if err != nil { - return - } - defer rows.Close() - return query.PreloadedTransactionScan(rows) -} - -// statementCreator allows to pass transaction or database to use in consumer. -type statementCreator interface { - Prepare(query string) (*sql.Stmt, error) -} - -type blockDBFields struct { - chainID uint64 - account common.Address - blockNumber *big.Int - blockHash common.Hash -} - -func insertBlockDBFields(creator statementCreator, block blockDBFields) error { - insert, err := creator.Prepare("INSERT OR IGNORE INTO blocks(network_id, address, blk_number, blk_hash, loaded) VALUES (?, ?, ?, ?, ?)") - if err != nil { - return err - } - defer insert.Close() - - _, err = insert.Exec(block.chainID, block.account, (*bigint.SQLBigInt)(block.blockNumber), block.blockHash, true) - return err -} - -func insertBlocksWithTransactions(chainID uint64, creator statementCreator, headers []*DBHeader) error { - insert, err := creator.Prepare("INSERT OR IGNORE INTO blocks(network_id, address, blk_number, blk_hash, loaded) VALUES (?, ?, ?, ?, ?)") - if err != nil { - return err - } - defer insert.Close() - updateTx, err := creator.Prepare(`UPDATE transfers - SET log = ?, log_index = ? - WHERE network_id = ? AND address = ? AND hash = ?`) - if err != nil { - return err - } - defer updateTx.Close() - - insertTx, err := creator.Prepare(`INSERT OR IGNORE - INTO transfers (network_id, address, sender, hash, blk_number, blk_hash, type, timestamp, log, loaded, log_index, token_id, amount_padded128hex) - VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, 0, ?, ?, ?)`) - if err != nil { - return err - } - defer insertTx.Close() - - for _, header := range headers { - _, err = insert.Exec(chainID, header.Address, (*bigint.SQLBigInt)(header.Number), header.Hash, header.Loaded) - if err != nil { - return err - } - for _, transaction := range header.PreloadedTransactions { - var logIndex *uint - if transaction.Log != nil { - logIndex = new(uint) - *logIndex = transaction.Log.Index - } - res, err := updateTx.Exec(&JSONBlob{transaction.Log}, logIndex, chainID, header.Address, transaction.ID) - if err != nil { - return err - } - affected, err := res.RowsAffected() - if err != nil { - return err - } - if affected > 0 { - continue - } - - tokenID := (*bigint.SQLBigIntBytes)(transaction.TokenID) - txValue := sqlite.BigIntToPadded128BitsStr(transaction.Value) - // Is that correct to set sender as account address? - _, err = insertTx.Exec(chainID, header.Address, header.Address, transaction.ID, (*bigint.SQLBigInt)(header.Number), header.Hash, transaction.Type, &JSONBlob{transaction.Log}, logIndex, tokenID, txValue) - if err != nil { - logutils.ZapLogger().Error("error saving token transfer", zap.Error(err)) - return err - } - } - } - return nil -} - -func updateOrInsertTransfers(chainID uint64, creator statementCreator, transfers []Transfer) error { - txsDBFields := make([]transferDBFields, 0, len(transfers)) - for _, localTransfer := range transfers { - // to satisfy gosec: C601 checks - t := localTransfer - var receiptType *uint8 - var txHash, blockHash *common.Hash - var receiptStatus, cumulativeGasUsed, gasUsed *uint64 - var contractAddress *common.Address - var transactionIndex, logIndex *uint - - if t.Receipt != nil { - receiptType = &t.Receipt.Type - receiptStatus = &t.Receipt.Status - txHash = &t.Receipt.TxHash - if t.Log != nil { - logIndex = new(uint) - *logIndex = t.Log.Index - } - blockHash = &t.Receipt.BlockHash - cumulativeGasUsed = &t.Receipt.CumulativeGasUsed - contractAddress = &t.Receipt.ContractAddress - gasUsed = &t.Receipt.GasUsed - transactionIndex = &t.Receipt.TransactionIndex - } - - var txProtected *bool - var txGas, txNonce, txSize *uint64 - var txGasPrice, txGasTipCap, txGasFeeCap *big.Int - var txType *uint8 - var txValue *big.Int - var tokenAddress *common.Address - var tokenID *big.Int - var txFrom *common.Address - var txTo *common.Address - if t.Transaction != nil { - if t.Log != nil { - _, tokenAddress, txFrom, txTo = w_common.ExtractTokenTransferData(t.Type, t.Log, t.Transaction) - tokenID = t.TokenID - // Zero tokenID can be used for ERC721 and ERC1155 transfers but when serialzed/deserialized it becomes nil - // as 0 value of big.Int bytes is nil. - if tokenID == nil && (t.Type == w_common.Erc721Transfer || t.Type == w_common.Erc1155Transfer) { - tokenID = big.NewInt(0) - } - txValue = t.TokenValue - } else { - txValue = new(big.Int).Set(t.Transaction.Value()) - txFrom = &t.From - txTo = t.Transaction.To() - } - - txType = new(uint8) - *txType = t.Transaction.Type() - txProtected = new(bool) - *txProtected = t.Transaction.Protected() - txGas = new(uint64) - *txGas = t.Transaction.Gas() - txGasPrice = t.Transaction.GasPrice() - txGasTipCap = t.Transaction.GasTipCap() - txGasFeeCap = t.Transaction.GasFeeCap() - txNonce = new(uint64) - *txNonce = t.Transaction.Nonce() - txSize = new(uint64) - *txSize = t.Transaction.Size() - } - - dbFields := transferDBFields{ - chainID: chainID, - id: t.ID, - blockHash: t.BlockHash, - blockNumber: t.BlockNumber, - timestamp: t.Timestamp, - address: t.Address, - transaction: t.Transaction, - sender: t.From, - receipt: t.Receipt, - log: t.Log, - transferType: t.Type, - baseGasFees: t.BaseGasFees, - receiptStatus: receiptStatus, - receiptType: receiptType, - txHash: txHash, - logIndex: logIndex, - receiptBlockHash: blockHash, - cumulativeGasUsed: cumulativeGasUsed, - contractAddress: contractAddress, - gasUsed: gasUsed, - transactionIndex: transactionIndex, - txType: txType, - txProtected: txProtected, - txGas: txGas, - txGasPrice: txGasPrice, - txGasTipCap: txGasTipCap, - txGasFeeCap: txGasFeeCap, - txValue: txValue, - txNonce: txNonce, - txSize: txSize, - tokenAddress: tokenAddress, - tokenID: tokenID, - txFrom: txFrom, - txTo: txTo, - } - txsDBFields = append(txsDBFields, dbFields) - } - - return updateOrInsertTransfersDBFields(creator, txsDBFields) -} - -type transferDBFields struct { - chainID uint64 - id common.Hash - blockHash common.Hash - blockNumber *big.Int - timestamp uint64 - address common.Address - transaction *types.Transaction - sender common.Address - receipt *types.Receipt - log *types.Log - transferType w_common.Type - baseGasFees string - receiptStatus *uint64 - receiptType *uint8 - txHash *common.Hash - logIndex *uint - receiptBlockHash *common.Hash - cumulativeGasUsed *uint64 - contractAddress *common.Address - gasUsed *uint64 - transactionIndex *uint - txType *uint8 - txProtected *bool - txGas *uint64 - txGasPrice *big.Int - txGasTipCap *big.Int - txGasFeeCap *big.Int - txValue *big.Int - txNonce *uint64 - txSize *uint64 - tokenAddress *common.Address - tokenID *big.Int - txFrom *common.Address - txTo *common.Address -} - -func updateOrInsertTransfersDBFields(creator statementCreator, transfers []transferDBFields) error { - insert, err := creator.Prepare(`INSERT OR REPLACE INTO transfers - (network_id, hash, blk_hash, blk_number, timestamp, address, tx, sender, receipt, log, type, loaded, base_gas_fee, - status, receipt_type, tx_hash, log_index, block_hash, cumulative_gas_used, contract_address, gas_used, tx_index, - tx_type, protected, gas_limit, gas_price_clamped64, gas_tip_cap_clamped64, gas_fee_cap_clamped64, amount_padded128hex, account_nonce, size, token_address, token_id, tx_from_address, tx_to_address) - VALUES - (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) - if err != nil { - return err - } - defer insert.Close() - for _, t := range transfers { - txGasPrice := sqlite.BigIntToClampedInt64(t.txGasPrice) - txGasTipCap := sqlite.BigIntToClampedInt64(t.txGasTipCap) - txGasFeeCap := sqlite.BigIntToClampedInt64(t.txGasFeeCap) - txValue := sqlite.BigIntToPadded128BitsStr(t.txValue) - - _, err = insert.Exec(t.chainID, t.id, t.blockHash, (*bigint.SQLBigInt)(t.blockNumber), t.timestamp, t.address, &JSONBlob{t.transaction}, t.sender, &JSONBlob{t.receipt}, &JSONBlob{t.log}, t.transferType, t.baseGasFees, - t.receiptStatus, t.receiptType, t.txHash, t.logIndex, t.receiptBlockHash, t.cumulativeGasUsed, t.contractAddress, t.gasUsed, t.transactionIndex, - t.txType, t.txProtected, t.txGas, txGasPrice, txGasTipCap, txGasFeeCap, txValue, t.txNonce, t.txSize, t.tokenAddress, (*bigint.SQLBigIntBytes)(t.tokenID), t.txFrom, t.txTo) - if err != nil { - logutils.ZapLogger().Error("can't save transfer", - zap.Stringer("b-hash", t.blockHash), - zap.Stringer("b-n", t.blockNumber), - zap.Stringer("a", t.address), - zap.Stringer("h", t.id), - zap.Error(err), - ) - return err - } - } - - for _, t := range transfers { - err = removeGasOnlyEthTransfer(creator, t) - if err != nil { - logutils.ZapLogger().Error("can't remove gas only eth transfer", - zap.Stringer("b-hash", t.blockHash), - zap.Stringer("b-n", t.blockNumber), - zap.Stringer("a", t.address), - zap.Stringer("h", t.id), - zap.Error(err), - ) - // no return err, since it's not critical - } - } - return nil -} - -func removeGasOnlyEthTransfer(creator statementCreator, t transferDBFields) error { - if t.transferType == w_common.EthTransfer { - countQuery, err := creator.Prepare(`SELECT COUNT(*) FROM transfers WHERE tx_hash = ?`) - if err != nil { - return err - } - defer countQuery.Close() - - var count int - err = countQuery.QueryRow(t.txHash).Scan(&count) - if err != nil { - return err - } - - // If there's only one (or none), return without deleting - if count <= 1 { - logutils.ZapLogger().Debug("Only one or no transfer found with the same tx_hash, skipping deletion.") - return nil - } - } - query, err := creator.Prepare(`DELETE FROM transfers WHERE tx_hash = ? AND address = ? AND network_id = ? AND account_nonce = ? AND type = 'eth' AND amount_padded128hex = '00000000000000000000000000000000'`) - if err != nil { - return err - } - defer query.Close() - - res, err := query.Exec(t.txHash, t.address, t.chainID, t.txNonce) - if err != nil { - return err - } - count, err := res.RowsAffected() - if err != nil { - return err - } - logutils.ZapLogger().Debug("removeGasOnlyEthTransfer rows deleted", zap.Int64("count", count)) - return nil -} - -// markBlocksAsLoaded(chainID, tx, address, blockNumbers) -// In case block contains both ETH and token transfers, it will be marked as loaded on ETH transfer processing. -// This is not a problem since for token transfers we have preloaded transactions and blocks 'loaded' flag is needed -// for ETH transfers only. -func markBlocksAsLoaded(chainID uint64, creator statementCreator, address common.Address, blocks []*big.Int) error { - update, err := creator.Prepare("UPDATE blocks SET loaded=? WHERE address=? AND blk_number=? AND network_id=?") - if err != nil { - return err - } - defer update.Close() - - for _, block := range blocks { - _, err := update.Exec(true, address, (*bigint.SQLBigInt)(block), chainID) - if err != nil { - return err - } - } - return nil -} - -func (db *Database) GetLatestCollectibleTransfer(address common.Address, id thirdparty.CollectibleUniqueID) (*Transfer, error) { - query := newTransfersQuery(). - FilterAddress(address). - FilterNetwork(uint64(id.ContractID.ChainID)). - FilterTokenAddress(id.ContractID.Address). - FilterTokenID(id.TokenID.Int). - FilterLoaded(1). - SortByTimestamp(false). - Limit(1) - rows, err := db.client.Query(query.String(), query.Args()...) - if err != nil { - return nil, err - } - defer rows.Close() - - transfers, err := query.TransferScan(rows) - if err == sql.ErrNoRows || len(transfers) == 0 { - return nil, nil - } else if err != nil { - return nil, err - } - - return &transfers[0], nil -} - -// Delete blocks for address and chainID -// Transfers will be deleted by cascade -func deleteBlocks(creator statementCreator, address common.Address) error { - delete, err := creator.Prepare("DELETE FROM blocks WHERE address = ?") - if err != nil { - return err - } - defer delete.Close() - - _, err = delete.Exec(address) - return err -} - -func getAddresses(creator statementCreator) (rst []common.Address, err error) { - stmt, err := creator.Prepare(`SELECT address FROM transfers UNION SELECT address FROM blocks UNION - SELECT address FROM blocks_ranges_sequential UNION SELECT address FROM blocks_ranges`) - if err != nil { - return - } - defer stmt.Close() - - rows, err := stmt.Query() - if err != nil { - return nil, err - } - defer rows.Close() - - address := common.Address{} - for rows.Next() { - err = rows.Scan(&address) - if err != nil { - return nil, err - } - rst = append(rst, address) - } - - return rst, nil -} diff --git a/services/wallet/transfer/database_test.go b/services/wallet/transfer/database_test.go deleted file mode 100644 index fb68f34da9d..00000000000 --- a/services/wallet/transfer/database_test.go +++ /dev/null @@ -1,217 +0,0 @@ -package transfer - -import ( - "context" - "math/big" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - - ac "github.com/status-im/status-go/services/wallet/activity/common" - "github.com/status-im/status-go/services/wallet/bigint" - w_common "github.com/status-im/status-go/services/wallet/common" - "github.com/status-im/status-go/services/wallet/thirdparty" - "github.com/status-im/status-go/t/helpers" - "github.com/status-im/status-go/walletdatabase" -) - -func setupTestDB(t *testing.T) (*Database, *BlockDAO, func()) { - db, err := helpers.SetupTestMemorySQLDB(walletdatabase.DbInitializer{}) - require.NoError(t, err) - return NewDB(db), &BlockDAO{db}, func() { - require.NoError(t, db.Close()) - } -} - -func TestDBSaveBlocks(t *testing.T) { - db, _, stop := setupTestDB(t) - defer stop() - address := common.Address{1} - blocks := []*DBHeader{ - { - Number: big.NewInt(1), - Hash: common.Hash{1}, - Address: address, - }, - { - Number: big.NewInt(2), - Hash: common.Hash{2}, - Address: address, - }} - require.NoError(t, db.SaveBlocks(777, blocks)) - transfers := []Transfer{ - { - ID: common.Hash{1}, - Type: w_common.EthTransfer, - BlockHash: common.Hash{2}, - BlockNumber: big.NewInt(1), - Address: address, - Timestamp: 123, - From: address, - }, - } - tx, err := db.client.BeginTx(context.Background(), nil) - require.NoError(t, err) - - require.NoError(t, saveTransfersMarkBlocksLoaded(tx, 777, address, transfers, []*big.Int{big.NewInt(1), big.NewInt(2)})) - require.NoError(t, tx.Commit()) -} - -func TestDBSaveTransfers(t *testing.T) { - db, _, stop := setupTestDB(t) - defer stop() - address := common.Address{1} - header := &DBHeader{ - Number: big.NewInt(1), - Hash: common.Hash{1}, - Address: address, - } - tx := types.NewTransaction(1, address, nil, 10, big.NewInt(10), nil) - transfers := []Transfer{ - { - ID: common.Hash{1}, - Type: w_common.EthTransfer, - BlockHash: header.Hash, - BlockNumber: header.Number, - Transaction: tx, - Receipt: types.NewReceipt(nil, false, 100), - Address: address, - }, - } - require.NoError(t, db.SaveBlocks(777, []*DBHeader{header})) - require.NoError(t, saveTransfersMarkBlocksLoaded(db.client, 777, address, transfers, []*big.Int{header.Number})) -} - -func TestDBGetTransfersFromBlock(t *testing.T) { - db, _, stop := setupTestDB(t) - defer stop() - headers := []*DBHeader{} - transfers := []Transfer{} - address := common.Address{1} - blockNumbers := []*big.Int{} - for i := 1; i < 10; i++ { - header := &DBHeader{ - Number: big.NewInt(int64(i)), - Hash: common.Hash{byte(i)}, - Address: address, - } - headers = append(headers, header) - blockNumbers = append(blockNumbers, header.Number) - tx := types.NewTransaction(uint64(i), address, nil, 10, big.NewInt(10), nil) - receipt := types.NewReceipt(nil, false, 100) - receipt.Logs = []*types.Log{} - transfer := Transfer{ - ID: tx.Hash(), - Type: w_common.EthTransfer, - BlockNumber: header.Number, - BlockHash: header.Hash, - Transaction: tx, - Receipt: receipt, - Address: address, - } - transfers = append(transfers, transfer) - } - require.NoError(t, db.SaveBlocks(777, headers)) - require.NoError(t, saveTransfersMarkBlocksLoaded(db.client, 777, address, transfers, blockNumbers)) - rst, err := db.GetTransfers(777, big.NewInt(7), nil) - require.NoError(t, err) - require.Len(t, rst, 1) -} - -func TestGetTransfersForIdentities(t *testing.T) { - db, _, stop := setupTestDB(t) - defer stop() - - trs, _, _ := GenerateTestTransfers(t, db.client, 1, 4) - for i := range trs { - InsertTestTransfer(t, db.client, trs[i].To, &trs[i]) - } - - entries, err := db.GetTransfersForIdentities(context.Background(), []ac.TransactionIdentity{ - { - ChainID: trs[1].ChainID, - Hash: trs[1].Hash, - Address: trs[1].To, - }, - { - ChainID: trs[3].ChainID, - Hash: trs[3].Hash, - Address: trs[3].To, - }}) - require.NoError(t, err) - require.Equal(t, 2, len(entries)) - require.Equal(t, trs[1].Hash, entries[0].ID) - require.Equal(t, trs[3].Hash, entries[1].ID) - require.Equal(t, trs[1].From, entries[0].From) - require.Equal(t, trs[3].From, entries[1].From) - require.Equal(t, trs[1].To, entries[0].Address) - require.Equal(t, trs[3].To, entries[1].Address) - require.Equal(t, big.NewInt(trs[1].BlkNumber), entries[0].BlockNumber) - require.Equal(t, big.NewInt(trs[3].BlkNumber), entries[1].BlockNumber) - require.Equal(t, uint64(trs[1].Timestamp), entries[0].Timestamp) - require.Equal(t, uint64(trs[3].Timestamp), entries[1].Timestamp) - require.Equal(t, uint64(trs[1].ChainID), entries[0].NetworkID) - require.Equal(t, uint64(trs[3].ChainID), entries[1].NetworkID) -} - -func TestGetLatestCollectibleTransfer(t *testing.T) { - db, _, stop := setupTestDB(t) - defer stop() - - trs, _, _ := GenerateTestTransfers(t, db.client, 1, len(TestCollectibles)) - - collectible := TestCollectibles[0] - collectibleID := thirdparty.CollectibleUniqueID{ - ContractID: thirdparty.ContractID{ - ChainID: collectible.ChainID, - Address: collectible.TokenAddress, - }, - TokenID: &bigint.BigInt{Int: collectible.TokenID}, - } - firstTr := trs[0] - lastTr := firstTr - - // ExtraTrs is a sequence of send+receive of the same collectible - extraTrs, _, _ := GenerateTestTransfers(t, db.client, len(trs)+1, 2) - for i := range extraTrs { - if i%2 == 0 { - extraTrs[i].From = firstTr.To - extraTrs[i].To = firstTr.From - } else { - extraTrs[i].From = firstTr.From - extraTrs[i].To = firstTr.To - } - extraTrs[i].ChainID = collectible.ChainID - } - - for i := range trs { - collectibleData := TestCollectibles[i] - trs[i].ChainID = collectibleData.ChainID - InsertTestTransferWithOptions(t, db.client, trs[i].To, &trs[i], &TestTransferOptions{ - TokenAddress: collectibleData.TokenAddress, - TokenID: collectibleData.TokenID, - }) - } - - foundTx, err := db.GetLatestCollectibleTransfer(lastTr.To, collectibleID) - require.NoError(t, err) - require.NotEmpty(t, foundTx) - require.Equal(t, lastTr.Hash, foundTx.ID) - - for i := range extraTrs { - InsertTestTransferWithOptions(t, db.client, firstTr.To, &extraTrs[i], &TestTransferOptions{ - TokenAddress: collectible.TokenAddress, - TokenID: collectible.TokenID, - }) - } - - lastTr = extraTrs[len(extraTrs)-1] - - foundTx, err = db.GetLatestCollectibleTransfer(lastTr.To, collectibleID) - require.NoError(t, err) - require.NotEmpty(t, foundTx) - require.Equal(t, lastTr.Hash, foundTx.ID) -} diff --git a/services/wallet/transfer/downloader.go b/services/wallet/transfer/downloader.go deleted file mode 100644 index a06142c72cd..00000000000 --- a/services/wallet/transfer/downloader.go +++ /dev/null @@ -1,652 +0,0 @@ -package transfer - -import ( - "context" - "errors" - "math/big" - "slices" - "time" - - "go.uber.org/zap" - - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/status-im/status-go/logutils" - "github.com/status-im/status-go/rpc/chain" - w_common "github.com/status-im/status-go/services/wallet/common" -) - -var ( - zero = big.NewInt(0) - one = big.NewInt(1) - two = big.NewInt(2) -) - -// Partial transaction info obtained by ERC20Downloader. -// A PreloadedTransaction represents a Transaction which contains one -// ERC20/ERC721/ERC1155 transfer event. -// To be converted into one Transfer object post-indexing. -type PreloadedTransaction struct { - Type w_common.Type `json:"type"` - ID common.Hash `json:"-"` - Address common.Address `json:"address"` - // Log that was used to generate preloaded transaction. - Log *types.Log `json:"log"` - TokenID *big.Int `json:"tokenId"` - Value *big.Int `json:"value"` -} - -// Transfer stores information about transfer. -// A Transfer represents a plain ETH transfer or some token activity inside a Transaction -// Since ERC1155 transfers can contain multiple tokens, a single Transfer represents a single token transfer, -// that means ERC1155 batch transfers will be represented by multiple Transfer objects. -type Transfer struct { - Type w_common.Type `json:"type"` - ID common.Hash `json:"-"` - Address common.Address `json:"address"` - BlockNumber *big.Int `json:"blockNumber"` - BlockHash common.Hash `json:"blockhash"` - Timestamp uint64 `json:"timestamp"` - Transaction *types.Transaction `json:"transaction"` - Loaded bool - NetworkID uint64 - // From is derived from tx signature in order to offload this computation from UI component. - From common.Address `json:"from"` - Receipt *types.Receipt `json:"receipt"` - // Log that was used to generate erc20 transfer. Nil for eth transfer. - Log *types.Log `json:"log"` - // TokenID is the id of the transferred token. Nil for eth transfer. - TokenID *big.Int `json:"tokenId"` - // TokenValue is the value of the token transfer. Nil for eth transfer. - TokenValue *big.Int `json:"tokenValue"` - BaseGasFees string -} - -// ETHDownloader downloads regular eth transfers and tokens transfers. -type ETHDownloader struct { - chainClient chain.ClientInterface - accounts []common.Address - signer types.Signer - db *Database -} - -var errLogsDownloaderStuck = errors.New("logs downloader stuck") - -func (d *ETHDownloader) GetTransfersByNumber(ctx context.Context, number *big.Int) ([]Transfer, error) { - blk, err := d.chainClient.BlockByNumber(ctx, number) - if err != nil { - return nil, err - } - rst, err := d.getTransfersInBlock(ctx, blk, d.accounts) - if err != nil { - return nil, err - } - return rst, err -} - -func (d *ETHDownloader) getTransfersInBlock(ctx context.Context, blk *types.Block, accounts []common.Address) ([]Transfer, error) { - startTs := time.Now() - - rst := make([]Transfer, 0, len(blk.Transactions())) - - receiptsByAddressAndTxHash := make(map[common.Address]map[common.Hash]*types.Receipt) - txsByAddressAndTxHash := make(map[common.Address]map[common.Hash]*types.Transaction) - - addReceiptToCache := func(address common.Address, txHash common.Hash, receipt *types.Receipt) { - if receiptsByAddressAndTxHash[address] == nil { - receiptsByAddressAndTxHash[address] = make(map[common.Hash]*types.Receipt) - } - receiptsByAddressAndTxHash[address][txHash] = receipt - } - - addTxToCache := func(address common.Address, txHash common.Hash, tx *types.Transaction) { - if txsByAddressAndTxHash[address] == nil { - txsByAddressAndTxHash[address] = make(map[common.Hash]*types.Transaction) - } - txsByAddressAndTxHash[address][txHash] = tx - } - - getReceiptFromCache := func(address common.Address, txHash common.Hash) *types.Receipt { - if receiptsByAddressAndTxHash[address] == nil { - return nil - } - return receiptsByAddressAndTxHash[address][txHash] - } - - getTxFromCache := func(address common.Address, txHash common.Hash) *types.Transaction { - if txsByAddressAndTxHash[address] == nil { - return nil - } - return txsByAddressAndTxHash[address][txHash] - } - - getReceipt := func(address common.Address, txHash common.Hash) (receipt *types.Receipt, err error) { - receipt = getReceiptFromCache(address, txHash) - if receipt == nil { - receipt, err = d.fetchTransactionReceipt(ctx, txHash) - if err != nil { - return nil, err - } - addReceiptToCache(address, txHash, receipt) - } - return receipt, nil - } - - getTx := func(address common.Address, txHash common.Hash) (tx *types.Transaction, err error) { - tx = getTxFromCache(address, txHash) - if tx == nil { - tx, err = d.fetchTransaction(ctx, txHash) - if err != nil { - return nil, err - } - addTxToCache(address, txHash, tx) - } - return tx, nil - } - - for _, address := range accounts { - // During block discovery, we should have populated the DB with 1 item per transfer log containing - // erc20/erc721/erc1155 transfers. - // ID is a hash of the tx hash and the log index. log_index is unique per ERC20/721 tx, but not per ERC1155 tx. - transactionsToLoad, err := d.db.GetTransactionsToLoad(d.chainClient.NetworkID(), address, blk.Number()) - if err != nil { - return nil, err - } - - areSubTxsCheckedForTxHash := make(map[common.Hash]bool) - - logutils.ZapLogger().Debug("getTransfersInBlock", - zap.Stringer("block", blk.Number()), - zap.Int("transactionsToLoad", len(transactionsToLoad)), - ) - - for _, t := range transactionsToLoad { - receipt, err := getReceipt(address, t.Log.TxHash) - if err != nil { - return nil, err - } - - tx, err := getTx(address, t.Log.TxHash) - if err != nil { - return nil, err - } - - subtransactions, err := d.subTransactionsFromPreloaded(t, tx, receipt, blk) - if err != nil { - logutils.ZapLogger().Error("can't fetch subTxs for erc20/erc721/erc1155 transfer", zap.Error(err)) - return nil, err - } - rst = append(rst, subtransactions...) - areSubTxsCheckedForTxHash[t.Log.TxHash] = true - } - - for _, tx := range blk.Transactions() { - // Skip dummy blob transactions, as they are not supported by us - if tx.Type() == types.BlobTxType { - continue - } - if tx.ChainId().Cmp(big.NewInt(0)) != 0 && tx.ChainId().Cmp(d.chainClient.ToBigInt()) != 0 { - logutils.ZapLogger().Info("chain id mismatch", - zap.Stringer("tx hash", tx.Hash()), - zap.Stringer("tx chain id", tx.ChainId()), - zap.Uint64("expected chain id", d.chainClient.NetworkID()), - ) - continue - } - from, err := types.Sender(d.signer, tx) - - if err != nil { - if err == core.ErrTxTypeNotSupported { - logutils.ZapLogger().Error("Tx Type not supported", - zap.Stringer("tx chain id", tx.ChainId()), - zap.Uint8("type", tx.Type()), - zap.Error(err), - ) - continue - } - return nil, err - } - - isPlainTransfer := from == address || (tx.To() != nil && *tx.To() == address) - mustCheckSubTxs := false - - if !isPlainTransfer { - // We might miss some subTransactions of interest for some transaction types. We need to check if we - // find the address in the transaction data. - switch tx.Type() { - case types.DynamicFeeTxType, types.OptimismDepositTxType, types.ArbitrumDepositTxType, types.ArbitrumRetryTxType: - mustCheckSubTxs = !areSubTxsCheckedForTxHash[tx.Hash()] && w_common.TxDataContainsAddress(tx.Type(), tx.Data(), address) - } - } - - if isPlainTransfer || mustCheckSubTxs { - receipt, err := getReceipt(address, tx.Hash()) - if err != nil { - return nil, err - } - - // Since we've already got the receipt, check for subTxs of - // interest in case we haven't already. - if !areSubTxsCheckedForTxHash[tx.Hash()] { - subtransactions, err := d.subTransactionsFromTransactionData(address, from, tx, receipt, blk) - if err != nil { - logutils.ZapLogger().Error("can't fetch subTxs for eth transfer", zap.Error(err)) - return nil, err - } - rst = append(rst, subtransactions...) - areSubTxsCheckedForTxHash[tx.Hash()] = true - } - - // If it's a plain ETH transfer, add it to the list - if isPlainTransfer { - rst = append(rst, Transfer{ - Type: w_common.EthTransfer, - NetworkID: tx.ChainId().Uint64(), - ID: tx.Hash(), - Address: address, - BlockNumber: blk.Number(), - BlockHash: receipt.BlockHash, - Timestamp: blk.Time(), - Transaction: tx, - From: from, - Receipt: receipt, - Log: nil, - BaseGasFees: blk.BaseFee().String(), - }) - } - } - } - } - logutils.ZapLogger().Debug("getTransfersInBlock found", - zap.Stringer("block", blk.Number()), - zap.Int("len", len(rst)), - zap.Duration("time", time.Since(startTs)), - ) - // TODO(dshulyak) test that balance difference was covered by transactions - return rst, nil -} - -// NewERC20TransfersDownloader returns new instance. -func NewERC20TransfersDownloader(client chain.ClientInterface, accounts []common.Address, signer types.Signer, incomingOnly bool) *ERC20TransfersDownloader { - signature := w_common.GetEventSignatureHash(w_common.Erc20_721TransferEventSignature) - - return &ERC20TransfersDownloader{ - client: client, - accounts: accounts, - signature: signature, - incomingOnly: incomingOnly, - signatureErc1155Single: w_common.GetEventSignatureHash(w_common.Erc1155TransferSingleEventSignature), - signatureErc1155Batch: w_common.GetEventSignatureHash(w_common.Erc1155TransferBatchEventSignature), - signer: signer, - } -} - -// ERC20TransfersDownloader is a downloader for erc20 and erc721 tokens transfers. -// Since both transaction types share the same signature, both will be assigned -// type Erc20Transfer. Until the downloader gets refactored and a migration of the -// database gets implemented, differentiation between erc20 and erc721 will handled -// in the controller. -type ERC20TransfersDownloader struct { - client chain.ClientInterface - accounts []common.Address - incomingOnly bool - - // hash of the Transfer event signature - signature common.Hash - signatureErc1155Single common.Hash - signatureErc1155Batch common.Hash - - // signer is used to derive tx sender from tx signature - signer types.Signer -} - -func topicFromAddressSlice(addresses []common.Address) []common.Hash { - rst := make([]common.Hash, len(addresses)) - for i, address := range addresses { - rst[i] = common.BytesToHash(address.Bytes()) - } - return rst -} - -func (d *ERC20TransfersDownloader) inboundTopics(addresses []common.Address) [][]common.Hash { - return [][]common.Hash{{d.signature}, {}, topicFromAddressSlice(addresses)} -} - -func (d *ERC20TransfersDownloader) outboundTopics(addresses []common.Address) [][]common.Hash { - return [][]common.Hash{{d.signature}, topicFromAddressSlice(addresses), {}} -} - -func (d *ERC20TransfersDownloader) inboundERC20OutboundERC1155Topics(addresses []common.Address) [][]common.Hash { - return [][]common.Hash{{d.signature, d.signatureErc1155Single, d.signatureErc1155Batch}, {}, topicFromAddressSlice(addresses)} -} - -func (d *ERC20TransfersDownloader) inboundTopicsERC1155(addresses []common.Address) [][]common.Hash { - return [][]common.Hash{{d.signatureErc1155Single, d.signatureErc1155Batch}, {}, {}, topicFromAddressSlice(addresses)} -} - -func (d *ETHDownloader) fetchTransactionReceipt(parent context.Context, txHash common.Hash) (*types.Receipt, error) { - ctx, cancel := context.WithTimeout(parent, 3*time.Second) - receipt, err := d.chainClient.TransactionReceipt(ctx, txHash) - cancel() - if err != nil { - return nil, err - } - return receipt, nil -} - -func (d *ETHDownloader) fetchTransaction(parent context.Context, txHash common.Hash) (*types.Transaction, error) { - ctx, cancel := context.WithTimeout(parent, 3*time.Second) - tx, _, err := d.chainClient.TransactionByHash(ctx, txHash) // TODO Save on requests by checking in the DB first - cancel() - if err != nil { - return nil, err - } - return tx, nil -} - -func (d *ETHDownloader) subTransactionsFromPreloaded(preloadedTx *PreloadedTransaction, tx *types.Transaction, receipt *types.Receipt, blk *types.Block) ([]Transfer, error) { - logutils.ZapLogger().Debug("subTransactionsFromPreloaded start", - zap.Stringer("txHash", tx.Hash()), - zap.Stringer("address", preloadedTx.Address), - zap.Stringer("tokenID", preloadedTx.TokenID), - zap.Stringer("value", preloadedTx.Value), - ) - address := preloadedTx.Address - txLog := preloadedTx.Log - - rst := make([]Transfer, 0, 1) - - from, err := types.Sender(d.signer, tx) - if err != nil { - if err == core.ErrTxTypeNotSupported { - return nil, nil - } - return nil, err - } - - eventType := w_common.GetEventType(preloadedTx.Log) - // Only add ERC20/ERC721/ERC1155 transfers from/to the given account - // from/to matching is already handled by getLogs filter - switch eventType { - case w_common.Erc20TransferEventType, - w_common.Erc721TransferEventType, - w_common.Erc1155TransferSingleEventType, w_common.Erc1155TransferBatchEventType: - logutils.ZapLogger().Debug("subTransactionsFromPreloaded transfer", - zap.String("eventType", string(eventType)), - zap.Uint("logIdx", txLog.Index), - zap.Stringer("txHash", tx.Hash()), - zap.Stringer("address", address), - zap.Stringer("tokenID", preloadedTx.TokenID), - zap.Stringer("value", preloadedTx.Value), - zap.Stringer("baseFee", blk.BaseFee()), - ) - - transfer := Transfer{ - Type: w_common.EventTypeToSubtransactionType(eventType), - ID: preloadedTx.ID, - Address: address, - BlockNumber: new(big.Int).SetUint64(txLog.BlockNumber), - BlockHash: txLog.BlockHash, - Loaded: true, - NetworkID: d.signer.ChainID().Uint64(), - From: from, - Log: txLog, - TokenID: preloadedTx.TokenID, - TokenValue: preloadedTx.Value, - BaseGasFees: blk.BaseFee().String(), - Transaction: tx, - Receipt: receipt, - Timestamp: blk.Time(), - } - - rst = append(rst, transfer) - } - - logutils.ZapLogger().Debug("subTransactionsFromPreloaded end", - zap.Stringer("txHash", tx.Hash()), - zap.Stringer("address", address), - zap.Stringer("tokenID", preloadedTx.TokenID), - zap.Stringer("value", preloadedTx.Value), - ) - return rst, nil -} - -func (d *ETHDownloader) subTransactionsFromTransactionData(address, from common.Address, tx *types.Transaction, receipt *types.Receipt, blk *types.Block) ([]Transfer, error) { - logutils.ZapLogger().Debug("subTransactionsFromTransactionData start", - zap.Stringer("txHash", tx.Hash()), - zap.Stringer("address", address), - ) - - rst := make([]Transfer, 0, 1) - - for _, txLog := range receipt.Logs { - eventType := w_common.GetEventType(txLog) - switch eventType { - case w_common.UniswapV2SwapEventType, w_common.UniswapV3SwapEventType, - w_common.HopBridgeTransferSentToL2EventType, w_common.HopBridgeTransferFromL1CompletedEventType, - w_common.HopBridgeWithdrawalBondedEventType, w_common.HopBridgeTransferSentEventType: - transfer := Transfer{ - Type: w_common.EventTypeToSubtransactionType(eventType), - ID: w_common.GetLogSubTxID(*txLog), - Address: address, - BlockNumber: new(big.Int).SetUint64(txLog.BlockNumber), - BlockHash: txLog.BlockHash, - Loaded: true, - NetworkID: d.signer.ChainID().Uint64(), - From: from, - Log: txLog, - BaseGasFees: blk.BaseFee().String(), - Transaction: tx, - Receipt: receipt, - Timestamp: blk.Time(), - } - - rst = append(rst, transfer) - } - } - - logutils.ZapLogger().Debug("subTransactionsFromTransactionData end", - zap.Stringer("txHash", tx.Hash()), - zap.Stringer("address", address), - ) - return rst, nil -} - -func (d *ERC20TransfersDownloader) blocksFromLogs(parent context.Context, logs []types.Log) ([]*DBHeader, error) { - concurrent := NewConcurrentDownloader(parent, NoThreadLimit) - - for i := range logs { - l := logs[i] - - if l.Removed { - continue - } - - var address common.Address - from, to, txIDs, tokenIDs, values, err := w_common.ParseTransferLog(l) - if err != nil { - logutils.ZapLogger().Error("failed to parse transfer log", - zap.Any("log", l), - zap.Stringers("address", d.accounts), - zap.Error(err), - ) - continue - } - - // Double check provider returned the correct log - if slices.Contains(d.accounts, from) { - address = from - } else if slices.Contains(d.accounts, to) { - address = to - } else { - logutils.ZapLogger().Error("from/to address mismatch", - zap.Any("log", l), - zap.Stringers("addresses", d.accounts), - ) - continue - } - - eventType := w_common.GetEventType(&l) - logType := w_common.EventTypeToSubtransactionType(eventType) - - for i, txID := range txIDs { - logutils.ZapLogger().Debug("block from logs", - zap.Uint64("block", l.BlockNumber), - zap.Any("log", l), - zap.String("logType", string(logType)), - zap.Stringer("txID", txID), - ) - - // For ERC20 there is no tokenID, so we use nil - var tokenID *big.Int - if len(tokenIDs) > i { - tokenID = tokenIDs[i] - } - - header := &DBHeader{ - Number: big.NewInt(int64(l.BlockNumber)), - Hash: l.BlockHash, - Address: address, - PreloadedTransactions: []*PreloadedTransaction{{ - ID: txID, - Type: logType, - Log: &l, - TokenID: tokenID, - Value: values[i], - }}, - Loaded: false, - } - - concurrent.Add(func(ctx context.Context) error { - concurrent.PushHeader(header) - return nil - }) - } - } - select { - case <-concurrent.WaitAsync(): - case <-parent.Done(): - return nil, errLogsDownloaderStuck - } - return concurrent.GetHeaders(), concurrent.Error() -} - -// GetHeadersInRange returns transfers between two blocks. -// time to get logs for 100000 blocks = 1.144686979s. with 249 events in the result set. -func (d *ERC20TransfersDownloader) GetHeadersInRange(parent context.Context, from, to *big.Int) ([]*DBHeader, error) { - start := time.Now() - logutils.ZapLogger().Debug("get erc20 transfers in range start", - zap.Uint64("chainID", d.client.NetworkID()), - zap.Stringer("from", from), - zap.Stringer("to", to), - zap.Stringers("accounts", d.accounts), - ) - - // TODO #16062: Figure out real root cause of invalid range - if from != nil && to != nil && from.Cmp(to) > 0 { - logutils.ZapLogger().Error("invalid range", - zap.Uint64("chainID", d.client.NetworkID()), - zap.Stringer("from", from), - zap.Stringer("to", to), - zap.Stringers("accounts", d.accounts), - ) - return nil, errors.New("invalid range") - } - - headers := []*DBHeader{} - ctx := context.Background() - var err error - outbound := []types.Log{} - var inboundOrMixed []types.Log // inbound ERC20 or outbound ERC1155 share the same signature for our purposes - if !d.incomingOnly { - outbound, err = d.client.FilterLogs(ctx, ethereum.FilterQuery{ - FromBlock: from, - ToBlock: to, - Topics: d.outboundTopics(d.accounts), - }) - if err != nil { - return nil, err - } - inboundOrMixed, err = d.client.FilterLogs(ctx, ethereum.FilterQuery{ - FromBlock: from, - ToBlock: to, - Topics: d.inboundERC20OutboundERC1155Topics(d.accounts), - }) - if err != nil { - return nil, err - } - } else { - inboundOrMixed, err = d.client.FilterLogs(ctx, ethereum.FilterQuery{ - FromBlock: from, - ToBlock: to, - Topics: d.inboundTopics(d.accounts), - }) - if err != nil { - return nil, err - } - } - - inbound1155, err := d.client.FilterLogs(ctx, ethereum.FilterQuery{ - FromBlock: from, - ToBlock: to, - Topics: d.inboundTopicsERC1155(d.accounts), - }) - if err != nil { - return nil, err - } - - logs := concatLogs(outbound, inboundOrMixed, inbound1155) - - if len(logs) == 0 { - logutils.ZapLogger().Debug("no logs found for account") - return nil, nil - } - - rst, err := d.blocksFromLogs(parent, logs) - if err != nil { - return nil, err - } - if len(rst) == 0 { - logutils.ZapLogger().Warn("no headers found in logs for account", - zap.Uint64("chainID", d.client.NetworkID()), - zap.Stringers("addresses", d.accounts), - zap.Stringer("from", from), - zap.Stringer("to", to), - ) - } else { - headers = append(headers, rst...) - logutils.ZapLogger().Debug("found erc20 transfers for account", - zap.Uint64("chainID", d.client.NetworkID()), - zap.Stringers("addresses", d.accounts), - zap.Stringer("from", from), - zap.Stringer("to", to), - zap.Int("headers", len(headers)), - ) - } - - logutils.ZapLogger().Debug("get erc20 transfers in range end", - zap.Uint64("chainID", d.client.NetworkID()), - zap.Stringer("from", from), - zap.Stringer("to", to), - zap.Int("headers", len(headers)), - zap.Stringers("accounts", d.accounts), - zap.Duration("took", time.Since(start)), - ) - return headers, nil -} - -func concatLogs(slices ...[]types.Log) []types.Log { - var totalLen int - for _, s := range slices { - totalLen += len(s) - } - tmp := make([]types.Log, totalLen) - var i int - for _, s := range slices { - i += copy(tmp[i:], s) - } - - return tmp -} diff --git a/services/wallet/transfer/downloader_test.go b/services/wallet/transfer/downloader_test.go deleted file mode 100644 index 1ee16af4f39..00000000000 --- a/services/wallet/transfer/downloader_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package transfer - -import ( - "context" - "math/big" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - - "go.uber.org/mock/gomock" - - mock_client "github.com/status-im/status-go/rpc/chain/mock/client" - walletCommon "github.com/status-im/status-go/services/wallet/common" - - "github.com/stretchr/testify/require" -) - -func TestERC20Downloader_getHeadersInRange(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockClientIface := mock_client.NewMockClientInterface(mockCtrl) - - accounts := []common.Address{ - common.HexToAddress("0x1"), - } - chainID := walletCommon.EthereumMainnet - signer := types.LatestSignerForChainID(big.NewInt(int64(chainID))) - mockClientIface.EXPECT().NetworkID().Return(chainID).AnyTimes() - - downloader := NewERC20TransfersDownloader( - mockClientIface, - accounts, - signer, - false, - ) - - ctx := context.Background() - - _, err := downloader.GetHeadersInRange(ctx, big.NewInt(10), big.NewInt(0)) - require.Error(t, err) - - mockClientIface.EXPECT().FilterLogs(ctx, gomock.Any()).Return(nil, nil).AnyTimes() - _, err = downloader.GetHeadersInRange(ctx, big.NewInt(0), big.NewInt(10)) - require.NoError(t, err) -} diff --git a/services/wallet/transfer/iterative.go b/services/wallet/transfer/iterative.go deleted file mode 100644 index 95af7b2f52e..00000000000 --- a/services/wallet/transfer/iterative.go +++ /dev/null @@ -1,97 +0,0 @@ -package transfer - -import ( - "context" - "errors" - "math/big" - - "go.uber.org/zap" - - "github.com/status-im/status-go/logutils" -) - -// SetupIterativeDownloader configures IterativeDownloader with last known synced block. -func SetupIterativeDownloader( - client HeaderReader, downloader BatchDownloader, size *big.Int, to *big.Int, from *big.Int) (*IterativeDownloader, error) { - - if to == nil || from == nil { - return nil, errors.New("to or from cannot be nil") - } - - logutils.ZapLogger().Debug("iterative downloader", - zap.Stringer("from", from), - zap.Stringer("to", to), - zap.Stringer("size", size), - ) - - d := &IterativeDownloader{ - client: client, - batchSize: size, - downloader: downloader, - from: from, - to: to, - } - return d, nil -} - -// BatchDownloader interface for loading transfers in batches in speificed range of blocks. -type BatchDownloader interface { - GetHeadersInRange(ctx context.Context, from, to *big.Int) ([]*DBHeader, error) -} - -// IterativeDownloader downloads batches of transfers in a specified size. -type IterativeDownloader struct { - client HeaderReader - - batchSize *big.Int - - downloader BatchDownloader - - from, to *big.Int - previous *big.Int -} - -// Finished true when earliest block with given sync option is zero. -func (d *IterativeDownloader) Finished() bool { - return d.from.Cmp(d.to) == 0 -} - -// Header return last synced header. -func (d *IterativeDownloader) Header() *big.Int { - return d.previous -} - -// Next moves closer to the end on every new iteration. -func (d *IterativeDownloader) Next(parent context.Context) ([]*DBHeader, *big.Int, *big.Int, error) { - to := d.to - from := new(big.Int).Sub(to, d.batchSize) - // if start < 0; start = 0 - if from.Cmp(d.from) == -1 { - from = d.from - } - headers, err := d.downloader.GetHeadersInRange(parent, from, to) - logutils.ZapLogger().Debug("load erc20 transfers in range", - zap.Stringer("from", from), - zap.Stringer("to", to), - zap.Stringer("batchSize", d.batchSize), - ) - if err != nil { - logutils.ZapLogger().Error("failed to get transfer in between two blocks", - zap.Stringer("from", from), - zap.Stringer("to", to), - zap.Error(err), - ) - return nil, nil, nil, err - } - - d.previous, d.to = d.to, from - return headers, d.from, to, nil -} - -// Revert reverts last step progress. Should be used if application failed to process transfers. -// For example failed to persist them. -func (d *IterativeDownloader) Revert() { - if d.previous != nil { - d.from = d.previous - } -} diff --git a/services/wallet/transfer/iterative_test.go b/services/wallet/transfer/iterative_test.go deleted file mode 100644 index 1125e84abd3..00000000000 --- a/services/wallet/transfer/iterative_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package transfer - -import ( - "context" - "errors" - "math/big" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" -) - -type transfersFixture []Transfer - -func (f transfersFixture) GetHeadersInRange(ctx context.Context, from, to *big.Int) ([]*DBHeader, error) { - rst := []*DBHeader{} - for _, t := range f { - if t.BlockNumber.Cmp(from) >= 0 && t.BlockNumber.Cmp(to) <= 0 { - rst = append(rst, &DBHeader{Number: t.BlockNumber}) - } - } - return rst, nil -} - -func TestIterFinished(t *testing.T) { - iterator := IterativeDownloader{ - from: big.NewInt(10), - to: big.NewInt(10), - } - require.True(t, iterator.Finished()) -} - -func TestIterNotFinished(t *testing.T) { - iterator := IterativeDownloader{ - from: big.NewInt(2), - to: big.NewInt(5), - } - require.False(t, iterator.Finished()) -} - -func TestIterRevert(t *testing.T) { - iterator := IterativeDownloader{ - from: big.NewInt(12), - to: big.NewInt(12), - previous: big.NewInt(9), - } - require.True(t, iterator.Finished()) - iterator.Revert() - require.False(t, iterator.Finished()) -} - -func TestIterProgress(t *testing.T) { - var ( - chain headers = genHeadersChain(10, 1) - transfers = make(transfersFixture, 10) - ) - for i := range transfers { - transfers[i] = Transfer{ - BlockNumber: chain[i].Number, - BlockHash: chain[i].Hash(), - } - } - iter := &IterativeDownloader{ - client: chain, - downloader: transfers, - batchSize: big.NewInt(5), - from: big.NewInt(0), - to: big.NewInt(9), - } - batch, _, _, err := iter.Next(context.TODO()) - require.NoError(t, err) - require.Len(t, batch, 6) - batch, _, _, err = iter.Next(context.TODO()) - require.NoError(t, err) - require.Len(t, batch, 5) - require.True(t, iter.Finished()) -} - -type headers []*types.Header - -func (h headers) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { - for _, item := range h { - if item.Hash() == hash { - return item, nil - } - } - return nil, errors.New("not found") -} - -func (h headers) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { - for _, item := range h { - if item.Number.Cmp(number) == 0 { - return item, nil - } - } - return nil, errors.New("not found") -} - -func (h headers) BalanceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (*big.Int, error) { - return nil, errors.New("not implemented") -} - -func genHeadersChain(size, difficulty int) []*types.Header { - rst := make([]*types.Header, size) - for i := 0; i < size; i++ { - rst[i] = &types.Header{ - Number: big.NewInt(int64(i)), - Difficulty: big.NewInt(int64(difficulty)), - Time: 1, - } - if i != 0 { - rst[i].ParentHash = rst[i-1].Hash() - } - } - return rst -} diff --git a/services/wallet/transfer/query.go b/services/wallet/transfer/query.go deleted file mode 100644 index 262048f207d..00000000000 --- a/services/wallet/transfer/query.go +++ /dev/null @@ -1,272 +0,0 @@ -package transfer - -import ( - "bytes" - "database/sql" - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/status-im/status-go/services/wallet/bigint" - w_common "github.com/status-im/status-go/services/wallet/common" -) - -const baseTransfersQuery = "SELECT hash, type, blk_hash, blk_number, timestamp, address, tx, sender, receipt, log, network_id, base_gas_fee %s FROM transfers" -const preloadedTransfersQuery = "SELECT hash, type, address, log, token_id, amount_padded128hex FROM transfers" - -type transfersQuery struct { - buf *bytes.Buffer - args []interface{} - whereAdded bool - subQuery bool -} - -func newTransfersQuery() *transfersQuery { - newQuery := newEmptyQuery() - transfersQueryString := fmt.Sprintf(baseTransfersQuery, "") - newQuery.buf.WriteString(transfersQueryString) - return newQuery -} - -func newTransfersQueryForPreloadedTransactions() *transfersQuery { - newQuery := newEmptyQuery() - newQuery.buf.WriteString(preloadedTransfersQuery) - return newQuery -} - -func newSubQuery() *transfersQuery { - newQuery := newEmptyQuery() - newQuery.subQuery = true - return newQuery -} - -func newEmptyQuery() *transfersQuery { - buf := bytes.NewBuffer(nil) - return &transfersQuery{buf: buf} -} - -func (q *transfersQuery) addWhereSeparator(separator SeparatorType) { - if !q.whereAdded { - if !q.subQuery { - q.buf.WriteString(" WHERE") - } - q.whereAdded = true - } else if separator == OrSeparator { - q.buf.WriteString(" OR") - } else if separator == AndSeparator { - q.buf.WriteString(" AND") - } else if separator != NoSeparator { - panic("Unknown separator. Need to handle current SeparatorType value") - } -} - -type SeparatorType int - -// Beware: please update addWhereSeparator if changing this enum -const ( - NoSeparator SeparatorType = iota + 1 - OrSeparator - AndSeparator -) - -// addSubQuery adds where clause formed as: WHERE/ () -func (q *transfersQuery) addSubQuery(subQuery *transfersQuery, separator SeparatorType) *transfersQuery { - q.addWhereSeparator(separator) - q.buf.WriteString(" (") - q.buf.Write(subQuery.buf.Bytes()) - q.buf.WriteString(")") - q.args = append(q.args, subQuery.args...) - return q -} - -func (q *transfersQuery) FilterStart(start *big.Int) *transfersQuery { - if start != nil { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" blk_number >= ?") - q.args = append(q.args, (*bigint.SQLBigInt)(start)) - } - return q -} - -func (q *transfersQuery) FilterEnd(end *big.Int) *transfersQuery { - if end != nil { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" blk_number <= ?") - q.args = append(q.args, (*bigint.SQLBigInt)(end)) - } - return q -} - -func (q *transfersQuery) FilterLoaded(loaded int) *transfersQuery { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" loaded = ? ") - q.args = append(q.args, loaded) - - return q -} - -func (q *transfersQuery) FilterNetwork(network uint64) *transfersQuery { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" network_id = ?") - q.args = append(q.args, network) - return q -} - -func (q *transfersQuery) FilterAddress(address common.Address) *transfersQuery { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" address = ?") - q.args = append(q.args, address) - return q -} - -func (q *transfersQuery) FilterTransactionID(hash common.Hash) *transfersQuery { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" hash = ?") - q.args = append(q.args, hash) - return q -} - -func (q *transfersQuery) FilterTransactionHash(hash common.Hash) *transfersQuery { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" tx_hash = ?") - q.args = append(q.args, hash) - return q -} - -func (q *transfersQuery) FilterBlockHash(blockHash common.Hash) *transfersQuery { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" blk_hash = ?") - q.args = append(q.args, blockHash) - return q -} - -func (q *transfersQuery) FilterBlockNumber(blockNumber *big.Int) *transfersQuery { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" blk_number = ?") - q.args = append(q.args, (*bigint.SQLBigInt)(blockNumber)) - return q -} - -func ascendingString(ascending bool) string { - if ascending { - return "ASC" - } - return "DESC" -} - -func (q *transfersQuery) SortByBlockNumberAndHash() *transfersQuery { - q.buf.WriteString(" ORDER BY blk_number DESC, hash ASC ") - return q -} - -func (q *transfersQuery) SortByTimestamp(ascending bool) *transfersQuery { - q.buf.WriteString(fmt.Sprintf(" ORDER BY timestamp %s ", ascendingString(ascending))) - return q -} - -func (q *transfersQuery) Limit(pageSize int64) *transfersQuery { - q.buf.WriteString(" LIMIT ?") - q.args = append(q.args, pageSize) - return q -} - -func (q *transfersQuery) FilterType(txType w_common.Type) *transfersQuery { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" type = ?") - q.args = append(q.args, txType) - return q -} - -func (q *transfersQuery) FilterTokenAddress(address common.Address) *transfersQuery { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" token_address = ?") - q.args = append(q.args, address) - return q -} - -func (q *transfersQuery) FilterTokenID(tokenID *big.Int) *transfersQuery { - q.addWhereSeparator(AndSeparator) - q.buf.WriteString(" token_id = ?") - q.args = append(q.args, (*bigint.SQLBigIntBytes)(tokenID)) - return q -} - -func (q *transfersQuery) String() string { - return q.buf.String() -} - -func (q *transfersQuery) Args() []interface{} { - return q.args -} - -func (q *transfersQuery) TransferScan(rows *sql.Rows) (rst []Transfer, err error) { - for rows.Next() { - transfer := Transfer{ - BlockNumber: &big.Int{}, - Transaction: &types.Transaction{}, - Receipt: &types.Receipt{}, - Log: &types.Log{}, - } - err = rows.Scan( - &transfer.ID, &transfer.Type, &transfer.BlockHash, - (*bigint.SQLBigInt)(transfer.BlockNumber), &transfer.Timestamp, &transfer.Address, - &JSONBlob{transfer.Transaction}, &transfer.From, &JSONBlob{transfer.Receipt}, &JSONBlob{transfer.Log}, &transfer.NetworkID, &transfer.BaseGasFees) - if err != nil { - return nil, err - } - rst = append(rst, transfer) - } - - return rst, nil -} - -func (q *transfersQuery) PreloadedTransactionScan(rows *sql.Rows) (rst []*PreloadedTransaction, err error) { - transfers := make([]Transfer, 0) - for rows.Next() { - transfer := Transfer{ - Log: &types.Log{}, - } - tokenValue := sql.NullString{} - tokenID := sql.RawBytes{} - err = rows.Scan( - &transfer.ID, &transfer.Type, - &transfer.Address, - &JSONBlob{transfer.Log}, - &tokenID, &tokenValue) - - if len(tokenID) > 0 { - transfer.TokenID = new(big.Int).SetBytes(tokenID) - } - - if tokenValue.Valid { - var ok bool - transfer.TokenValue, ok = new(big.Int).SetString(tokenValue.String, 16) - if !ok { - panic("failed to parse token value") - } - } - - if err != nil { - return nil, err - } - transfers = append(transfers, transfer) - } - - rst = make([]*PreloadedTransaction, 0, len(transfers)) - - for _, transfer := range transfers { - preloadedTransaction := &PreloadedTransaction{ - ID: transfer.ID, - Type: transfer.Type, - Address: transfer.Address, - Log: transfer.Log, - TokenID: transfer.TokenID, - Value: transfer.TokenValue, - } - - rst = append(rst, preloadedTransaction) - } - - return rst, nil -} diff --git a/services/wallet/transfer/reactor.go b/services/wallet/transfer/reactor.go deleted file mode 100644 index 70102f7255b..00000000000 --- a/services/wallet/transfer/reactor.go +++ /dev/null @@ -1,123 +0,0 @@ -package transfer - -import ( - "context" - "errors" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/event" - "github.com/status-im/status-go/multiaccounts/accounts" - "github.com/status-im/status-go/rpc/chain" - "github.com/status-im/status-go/services/wallet/balance" - "github.com/status-im/status-go/services/wallet/blockchainstate" - "github.com/status-im/status-go/services/wallet/token" - "github.com/status-im/status-go/transactions" -) - -const ( - ReactorNotStarted string = "reactor not started" - - NonArchivalNodeBlockChunkSize = 100 - DefaultNodeBlockChunkSize = 100000 -) - -var errAlreadyRunning = errors.New("already running") - -type FetchStrategyType int32 - -const ( - SequentialFetchStrategyType FetchStrategyType = iota -) - -// HeaderReader interface for reading headers using block number or hash. -type HeaderReader interface { - HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) - HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) -} - -type HistoryFetcher interface { - start() error - stop() - kind() FetchStrategyType - - getTransfersByAddress(ctx context.Context, chainID uint64, address common.Address, toBlock *big.Int, - limit int64) ([]Transfer, error) -} - -// Reactor listens to new blocks and stores transfers into the database. -type Reactor struct { - db *Database - blockDAO *BlockDAO - blockRangesSeqDAO *BlockRangeSequentialDAO - accountsDB *accounts.Database - feed *event.Feed - pendingTxManager *transactions.PendingTxTracker - tokenManager *token.Manager - strategy HistoryFetcher - balanceCacher balance.Cacher - omitHistory bool - blockChainState *blockchainstate.BlockChainState - chainIDs []uint64 -} - -func NewReactor(db *Database, blockDAO *BlockDAO, blockRangesSeqDAO *BlockRangeSequentialDAO, accountsDB *accounts.Database, feed *event.Feed, tm *TransactionManager, - pendingTxManager *transactions.PendingTxTracker, tokenManager *token.Manager, - balanceCacher balance.Cacher, omitHistory bool, blockChainState *blockchainstate.BlockChainState) *Reactor { - return &Reactor{ - db: db, - accountsDB: accountsDB, - blockDAO: blockDAO, - blockRangesSeqDAO: blockRangesSeqDAO, - feed: feed, - pendingTxManager: pendingTxManager, - tokenManager: tokenManager, - balanceCacher: balanceCacher, - omitHistory: omitHistory, - blockChainState: blockChainState, - } -} - -// Start runs reactor loop in background. -func (r *Reactor) start(chainClients map[uint64]chain.ClientInterface, accounts []common.Address) error { - chainIDs := []uint64{} - for _, client := range chainClients { - chainIDs = append(chainIDs, client.NetworkID()) - } - r.chainIDs = chainIDs - r.strategy = r.createFetchStrategy(chainClients, accounts) - return r.strategy.start() -} - -// Stop stops reactor loop and waits till it exits. -func (r *Reactor) stop() { - if r.strategy != nil { - r.strategy.stop() - } -} - -func (r *Reactor) restart(chainClients map[uint64]chain.ClientInterface, accounts []common.Address) error { - - r.stop() - return r.start(chainClients, accounts) -} - -func (r *Reactor) createFetchStrategy(chainClients map[uint64]chain.ClientInterface, - accounts []common.Address) HistoryFetcher { - - return NewSequentialFetchStrategy( - r.db, - r.blockDAO, - r.blockRangesSeqDAO, - r.accountsDB, - r.feed, - r.pendingTxManager, - r.tokenManager, - chainClients, - accounts, - r.balanceCacher, - r.omitHistory, - r.blockChainState, - ) -} diff --git a/services/wallet/transfer/sequential_fetch_strategy.go b/services/wallet/transfer/sequential_fetch_strategy.go deleted file mode 100644 index 3de56144db0..00000000000 --- a/services/wallet/transfer/sequential_fetch_strategy.go +++ /dev/null @@ -1,129 +0,0 @@ -package transfer - -import ( - "context" - "math/big" - "sync" - - "go.uber.org/zap" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/event" - "github.com/status-im/status-go/logutils" - "github.com/status-im/status-go/multiaccounts/accounts" - "github.com/status-im/status-go/rpc/chain" - "github.com/status-im/status-go/services/wallet/async" - "github.com/status-im/status-go/services/wallet/balance" - "github.com/status-im/status-go/services/wallet/blockchainstate" - "github.com/status-im/status-go/services/wallet/token" - "github.com/status-im/status-go/services/wallet/walletevent" - "github.com/status-im/status-go/transactions" -) - -func NewSequentialFetchStrategy(db *Database, blockDAO *BlockDAO, blockRangesSeqDAO *BlockRangeSequentialDAO, accountsDB *accounts.Database, feed *event.Feed, - pendingTxManager *transactions.PendingTxTracker, - tokenManager *token.Manager, - chainClients map[uint64]chain.ClientInterface, - accounts []common.Address, - balanceCacher balance.Cacher, - omitHistory bool, - blockChainState *blockchainstate.BlockChainState, -) *SequentialFetchStrategy { - - return &SequentialFetchStrategy{ - db: db, - blockDAO: blockDAO, - blockRangesSeqDAO: blockRangesSeqDAO, - accountsDB: accountsDB, - feed: feed, - pendingTxManager: pendingTxManager, - tokenManager: tokenManager, - chainClients: chainClients, - accounts: accounts, - balanceCacher: balanceCacher, - omitHistory: omitHistory, - blockChainState: blockChainState, - } -} - -type SequentialFetchStrategy struct { - db *Database - blockDAO *BlockDAO - blockRangesSeqDAO *BlockRangeSequentialDAO - accountsDB *accounts.Database - feed *event.Feed - mu sync.Mutex - group *async.Group - pendingTxManager *transactions.PendingTxTracker - tokenManager *token.Manager - chainClients map[uint64]chain.ClientInterface - accounts []common.Address - balanceCacher balance.Cacher - omitHistory bool - blockChainState *blockchainstate.BlockChainState -} - -func (s *SequentialFetchStrategy) newCommand(chainClient chain.ClientInterface, - accounts []common.Address) async.Commander { - - return newLoadBlocksAndTransfersCommand(accounts, s.db, s.accountsDB, s.blockDAO, s.blockRangesSeqDAO, chainClient, s.feed, - s.pendingTxManager, s.tokenManager, s.balanceCacher, s.omitHistory, s.blockChainState) -} - -func (s *SequentialFetchStrategy) start() error { - s.mu.Lock() - defer s.mu.Unlock() - - if s.group != nil { - return errAlreadyRunning - } - s.group = async.NewGroup(context.Background()) - - if s.feed != nil { - s.feed.Send(walletevent.Event{ - Type: EventFetchingRecentHistory, - Accounts: s.accounts, - }) - } - - for _, chainClient := range s.chainClients { - ctl := s.newCommand(chainClient, s.accounts) - s.group.Add(ctl.Command()) - } - - return nil -} - -// Stop stops reactor loop and waits till it exits. -func (s *SequentialFetchStrategy) stop() { - s.mu.Lock() - defer s.mu.Unlock() - if s.group == nil { - return - } - s.group.Stop() - s.group.Wait() - s.group = nil -} - -func (s *SequentialFetchStrategy) kind() FetchStrategyType { - return SequentialFetchStrategyType -} - -func (s *SequentialFetchStrategy) getTransfersByAddress(ctx context.Context, chainID uint64, address common.Address, toBlock *big.Int, - limit int64) ([]Transfer, error) { - - logutils.ZapLogger().Debug("[WalletAPI:: GetTransfersByAddress] get transfers for an address", - zap.Stringer("address", address), - zap.Uint64("chainID", chainID), - zap.Stringer("toBlock", toBlock), - zap.Int64("limit", limit)) - - rst, err := s.db.GetTransfersByAddress(chainID, address, toBlock, limit) - if err != nil { - logutils.ZapLogger().Error("[WalletAPI:: GetTransfersByAddress] can't fetch transfers", zap.Error(err)) - return nil, err - } - - return rst, nil -} diff --git a/services/wallet/transfer/testutils.go b/services/wallet/transfer/testutils.go deleted file mode 100644 index 65a4eefaec0..00000000000 --- a/services/wallet/transfer/testutils.go +++ /dev/null @@ -1,345 +0,0 @@ -package transfer - -import ( - "database/sql" - "fmt" - "math/big" - "testing" - - eth_common "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - - "github.com/status-im/status-go/services/wallet/bigint" - "github.com/status-im/status-go/services/wallet/common" - "github.com/status-im/status-go/services/wallet/testutils" - tokenTypes "github.com/status-im/status-go/services/wallet/token/types" - - "github.com/stretchr/testify/require" -) - -type TestTransaction struct { - Hash eth_common.Hash - ChainID common.ChainID - From eth_common.Address // [sender] - Timestamp int64 - BlkNumber int64 - Success bool - Nonce uint64 - Contract eth_common.Address -} - -type TestTransfer struct { - TestTransaction - To eth_common.Address // [address] - Value int64 - Token *tokenTypes.Token -} - -type TestCollectibleTransfer struct { - TestTransfer - TestCollectible -} - -func SeedToToken(seed int) *tokenTypes.Token { - tokenIndex := seed % len(TestTokens) - return TestTokens[tokenIndex] -} - -func TestTrToToken(t *testing.T, tt *TestTransaction) (token *tokenTypes.Token, isNative bool) { - // Sanity check that none of the markers changed and they should be equal to seed - require.Equal(t, tt.Timestamp, tt.BlkNumber) - - tokenIndex := int(tt.Timestamp) % len(TestTokens) - isNative = testutils.SliceContains(NativeTokenIndices, tokenIndex) - - return TestTokens[tokenIndex], isNative -} - -func generateTestTransaction(seed int) TestTransaction { - token := SeedToToken(seed) - return TestTransaction{ - Hash: eth_common.HexToHash(fmt.Sprintf("0x1%d", seed)), - ChainID: common.ChainID(token.ChainID), - From: eth_common.HexToAddress(fmt.Sprintf("0x2%d", seed)), - Timestamp: int64(seed), - BlkNumber: int64(seed), - Success: true, - Nonce: uint64(seed), - // In practice this is last20Bytes(Keccak256(RLP(From, nonce))) - Contract: eth_common.HexToAddress(fmt.Sprintf("0x4%d", seed)), - } -} - -func generateTestTransfer(seed int) TestTransfer { - tokenIndex := seed % len(TestTokens) - token := TestTokens[tokenIndex] - return TestTransfer{ - TestTransaction: generateTestTransaction(seed), - To: eth_common.HexToAddress(fmt.Sprintf("0x3%d", seed)), - Value: int64(seed), - Token: token, - } -} - -// Will be used in tests to generate a collectible transfer -// nolint:unused -func generateTestCollectibleTransfer(seed int) TestCollectibleTransfer { - collectibleIndex := seed % len(TestCollectibles) - collectible := TestCollectibles[collectibleIndex] - tr := TestCollectibleTransfer{ - TestTransfer: TestTransfer{ - TestTransaction: generateTestTransaction(seed), - To: eth_common.HexToAddress(fmt.Sprintf("0x3%d", seed)), - Value: int64(seed), - Token: &tokenTypes.Token{ - Address: collectible.TokenAddress, - Name: "Collectible", - ChainID: uint64(collectible.ChainID), - }, - }, - TestCollectible: collectible, - } - tr.TestTransaction.ChainID = collectible.ChainID - return tr -} - -// GenerateTestTransfers will generate transaction based on the TestTokens index and roll over if there are more than -// len(TestTokens) transactions -func GenerateTestTransfers(tb testing.TB, db *sql.DB, firstStartIndex int, count int) (result []TestTransfer, fromAddresses, toAddresses []eth_common.Address) { - for i := firstStartIndex; i < (firstStartIndex + count); i++ { - tr := generateTestTransfer(i) - fromAddresses = append(fromAddresses, tr.From) - toAddresses = append(toAddresses, tr.To) - result = append(result, tr) - } - return -} - -type TestCollectible struct { - TokenAddress eth_common.Address - TokenID *big.Int - ChainID common.ChainID -} - -var TestCollectibles = []TestCollectible{ - TestCollectible{ - TokenAddress: eth_common.HexToAddress("0x97a04fda4d97c6e3547d66b572e29f4a4ff40392"), - TokenID: big.NewInt(1), - ChainID: 1, - }, - TestCollectible{ // Same token ID as above but different address - TokenAddress: eth_common.HexToAddress("0x2cec8879915cdbd80c88d8b1416aa9413a24ddfa"), - TokenID: big.NewInt(1), - ChainID: 1, - }, - TestCollectible{ // TokenID (big.Int) value 0 might be problematic if not handled properly - TokenAddress: eth_common.HexToAddress("0x97a04fda4d97c6e3547d66b572e29f4a4ff4ABCD"), - TokenID: big.NewInt(0), - ChainID: 420, - }, - TestCollectible{ - TokenAddress: eth_common.HexToAddress("0x1dea7a3e04849840c0eb15fd26a55f6c40c4a69b"), - TokenID: big.NewInt(11), - ChainID: 5, - }, - TestCollectible{ // Same address as above but different token ID - TokenAddress: eth_common.HexToAddress("0x1dea7a3e04849840c0eb15fd26a55f6c40c4a69b"), - TokenID: big.NewInt(12), - ChainID: 5, - }, -} - -var EthMainnet = tokenTypes.Token{ - Address: eth_common.HexToAddress("0x"), - Name: "Ether", - Symbol: "ETH", - ChainID: 1, -} - -var EthSepolia = tokenTypes.Token{ - Address: eth_common.HexToAddress("0x"), - Name: "Ether", - Symbol: "ETH", - ChainID: 11155111, -} - -var EthOptimism = tokenTypes.Token{ - Address: eth_common.HexToAddress("0x"), - Name: "Ether", - Symbol: "ETH", - ChainID: 10, -} - -var UsdcMainnet = tokenTypes.Token{ - Address: eth_common.HexToAddress("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"), - Name: "USD Coin", - Symbol: "USDC", - ChainID: 1, -} - -var UsdcSepolia = tokenTypes.Token{ - Address: eth_common.HexToAddress("0x1c7D4B196Cb0C7B01d743Fbc6116a902379C7238"), - Name: "USD Coin", - Symbol: "USDC", - Decimals: 6, - ChainID: 11155111, -} - -var UsdcOptimism = tokenTypes.Token{ - Address: eth_common.HexToAddress("0x7f5c764cbc14f9669b88837ca1490cca17c31607"), - Name: "USD Coin", - Symbol: "USDC", - ChainID: 10, -} - -var SntMainnet = tokenTypes.Token{ - Address: eth_common.HexToAddress("0x744d70fdbe2ba4cf95131626614a1763df805b9e"), - Name: "Status Network Token", - Symbol: "SNT", - ChainID: 1, -} - -var DaiMainnet = tokenTypes.Token{ - Address: eth_common.HexToAddress("0xf2edF1c091f683E3fb452497d9a98A49cBA84666"), - Name: "DAI Stablecoin", - Symbol: "DAI", - ChainID: 5, -} - -var DaiSepolia = tokenTypes.Token{ - Address: eth_common.HexToAddress("0x3e622317f8c93f7328350cf0b56d9ed4c620c5d6"), - Name: "DAI Stablecoin", - Symbol: "DAI", - Decimals: 18, - ChainID: 11155111, -} - -// TestTokens contains ETH/Mainnet, ETH/Sepolia, ETH/Optimism, USDC/Mainnet, USDC/Sepolia, USDC/Optimism, SNT/Mainnet, DAI/Mainnet, DAI/Sepolia -var TestTokens = []*tokenTypes.Token{ - &EthMainnet, &EthSepolia, &EthOptimism, &UsdcMainnet, &UsdcSepolia, &UsdcOptimism, &SntMainnet, &DaiMainnet, &DaiSepolia, -} - -func LookupTokenIdentity(chainID uint64, address eth_common.Address, native bool) *tokenTypes.Token { - for _, token := range TestTokens { - if token.ChainID == chainID && token.Address == address && token.IsNative() == native { - return token - } - } - return nil -} - -var NativeTokenIndices = []int{0, 1, 2} - -func InsertTestTransfer(tb testing.TB, db *sql.DB, address eth_common.Address, tr *TestTransfer) { - token := TestTokens[int(tr.Timestamp)%len(TestTokens)] - InsertTestTransferWithOptions(tb, db, address, tr, &TestTransferOptions{ - TokenAddress: token.Address, - }) -} - -type TestTransferOptions struct { - TokenAddress eth_common.Address - TokenID *big.Int - NullifyAddresses []eth_common.Address - Tx *types.Transaction - Receipt *types.Receipt -} - -func GenerateTxField(data []byte) *types.Transaction { - return types.NewTx(&types.DynamicFeeTx{ - Data: data, - }) -} - -func InsertTestTransferWithOptions(tb testing.TB, db *sql.DB, address eth_common.Address, tr *TestTransfer, opt *TestTransferOptions) { - var ( - tx *sql.Tx - ) - tx, err := db.Begin() - require.NoError(tb, err) - defer func() { - if err == nil { - err = tx.Commit() - return - } - _ = tx.Rollback() - }() - - blkHash := eth_common.HexToHash("4") - - block := blockDBFields{ - chainID: uint64(tr.ChainID), - account: address, - blockNumber: big.NewInt(tr.BlkNumber), - blockHash: blkHash, - } - - // Respect `FOREIGN KEY(network_id,address,blk_hash)` of `transfers` table - err = insertBlockDBFields(tx, block) - require.NoError(tb, err) - - receiptStatus := uint64(0) - if tr.Success { - receiptStatus = 1 - } - - tokenType := "eth" - if (opt.TokenAddress != eth_common.Address{}) { - if opt.TokenID == nil { - tokenType = "erc20" - } else { - tokenType = "erc721" - } - } - - // Workaround to simulate writing of NULL values for addresses - txTo := &tr.To - txFrom := &tr.From - for i := 0; i < len(opt.NullifyAddresses); i++ { - if opt.NullifyAddresses[i] == tr.To { - txTo = nil - } - if opt.NullifyAddresses[i] == tr.From { - txFrom = nil - } - } - - transfer := transferDBFields{ - chainID: uint64(tr.ChainID), - id: tr.Hash, - txHash: &tr.Hash, - address: address, - blockHash: blkHash, - blockNumber: big.NewInt(tr.BlkNumber), - sender: tr.From, - transferType: common.Type(tokenType), - timestamp: uint64(tr.Timestamp), - baseGasFees: "0x0", - receiptStatus: &receiptStatus, - txValue: big.NewInt(tr.Value), - txFrom: txFrom, - txTo: txTo, - txNonce: &tr.Nonce, - tokenAddress: &opt.TokenAddress, - contractAddress: &tr.Contract, - tokenID: opt.TokenID, - transaction: opt.Tx, - receipt: opt.Receipt, - } - err = updateOrInsertTransfersDBFields(tx, []transferDBFields{transfer}) - require.NoError(tb, err) -} - -func InsertTestPendingTransaction(tb testing.TB, db *sql.DB, tr *TestTransfer) { - _, err := db.Exec(` - INSERT INTO pending_transactions (network_id, hash, timestamp, from_address, to_address, - symbol, gas_price, gas_limit, value, data, type, additional_data - ) VALUES (?, ?, ?, ?, ?, 'ETH', 0, 0, ?, '', 'eth', '')`, - tr.ChainID, tr.Hash, tr.Timestamp, tr.From, tr.To, (*bigint.SQLBigIntBytes)(big.NewInt(tr.Value))) - require.NoError(tb, err) -} - -// For using in tests only outside the package -func SaveTransfersMarkBlocksLoaded(database *Database, chainID uint64, address eth_common.Address, transfers []Transfer, blocks []*big.Int) error { - return saveTransfersMarkBlocksLoaded(database.client, chainID, address, transfers, blocks) -}